1988 lines
50 KiB
C
1988 lines
50 KiB
C
/*-
|
|
* Copyright (c) 2016, Vincenzo Maffione
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice unmodified, this list of conditions, and the following
|
|
* disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/* Driver for ptnet paravirtualized network device. */
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
#include <sys/types.h>
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/sockio.h>
|
|
#include <sys/mbuf.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/module.h>
|
|
#include <sys/socket.h>
|
|
#include <sys/sysctl.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/taskqueue.h>
|
|
#include <sys/smp.h>
|
|
#include <sys/time.h>
|
|
#include <machine/smp.h>
|
|
|
|
#include <vm/uma.h>
|
|
#include <vm/vm.h>
|
|
#include <vm/pmap.h>
|
|
|
|
#include <net/ethernet.h>
|
|
#include <net/if.h>
|
|
#include <net/if_var.h>
|
|
#include <net/if_arp.h>
|
|
#include <net/if_dl.h>
|
|
#include <net/if_types.h>
|
|
#include <net/if_media.h>
|
|
#include <net/if_vlan_var.h>
|
|
#include <net/bpf.h>
|
|
|
|
#include <netinet/in_systm.h>
|
|
#include <netinet/in.h>
|
|
#include <netinet/ip.h>
|
|
#include <netinet/ip6.h>
|
|
#include <netinet6/ip6_var.h>
|
|
#include <netinet/udp.h>
|
|
#include <netinet/tcp.h>
|
|
|
|
#include <machine/bus.h>
|
|
#include <machine/resource.h>
|
|
#include <sys/bus.h>
|
|
#include <sys/rman.h>
|
|
|
|
#include <dev/pci/pcivar.h>
|
|
#include <dev/pci/pcireg.h>
|
|
|
|
#include "opt_inet.h"
|
|
#include "opt_inet6.h"
|
|
|
|
#include <sys/selinfo.h>
|
|
#include <net/netmap.h>
|
|
#include <dev/netmap/netmap_kern.h>
|
|
#include <net/netmap_virt.h>
|
|
#include <dev/netmap/netmap_mem2.h>
|
|
#include <dev/virtio/network/virtio_net.h>
|
|
|
|
#ifdef WITH_PTNETMAP
|
|
|
|
#ifndef INET
|
|
#error "INET not defined, cannot support offloadings"
|
|
#endif
|
|
|
|
static uint64_t ptnet_get_counter(if_t, ift_counter);
|
|
|
|
//#define PTNETMAP_STATS
|
|
//#define DEBUG
|
|
#ifdef DEBUG
|
|
#define DBG(x) x
|
|
#else /* !DEBUG */
|
|
#define DBG(x)
|
|
#endif /* !DEBUG */
|
|
|
|
extern int ptnet_vnet_hdr; /* Tunable parameter */
|
|
|
|
struct ptnet_softc;
|
|
|
|
struct ptnet_queue_stats {
|
|
uint64_t packets; /* if_[io]packets */
|
|
uint64_t bytes; /* if_[io]bytes */
|
|
uint64_t errors; /* if_[io]errors */
|
|
uint64_t iqdrops; /* if_iqdrops */
|
|
uint64_t mcasts; /* if_[io]mcasts */
|
|
#ifdef PTNETMAP_STATS
|
|
uint64_t intrs;
|
|
uint64_t kicks;
|
|
#endif /* PTNETMAP_STATS */
|
|
};
|
|
|
|
struct ptnet_queue {
|
|
struct ptnet_softc *sc;
|
|
struct resource *irq;
|
|
void *cookie;
|
|
int kring_id;
|
|
struct nm_csb_atok *atok;
|
|
struct nm_csb_ktoa *ktoa;
|
|
unsigned int kick;
|
|
struct mtx lock;
|
|
struct buf_ring *bufring; /* for TX queues */
|
|
struct ptnet_queue_stats stats;
|
|
#ifdef PTNETMAP_STATS
|
|
struct ptnet_queue_stats last_stats;
|
|
#endif /* PTNETMAP_STATS */
|
|
struct taskqueue *taskq;
|
|
struct task task;
|
|
char lock_name[16];
|
|
};
|
|
|
|
#define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock)
|
|
#define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock)
|
|
#define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock)
|
|
|
|
struct ptnet_softc {
|
|
device_t dev;
|
|
if_t ifp;
|
|
struct ifmedia media;
|
|
struct mtx lock;
|
|
char lock_name[16];
|
|
char hwaddr[ETHER_ADDR_LEN];
|
|
|
|
/* Mirror of PTFEAT register. */
|
|
uint32_t ptfeatures;
|
|
unsigned int vnet_hdr_len;
|
|
|
|
/* PCI BARs support. */
|
|
struct resource *iomem;
|
|
struct resource *msix_mem;
|
|
|
|
unsigned int num_rings;
|
|
unsigned int num_tx_rings;
|
|
struct ptnet_queue *queues;
|
|
struct ptnet_queue *rxqueues;
|
|
struct nm_csb_atok *csb_gh;
|
|
struct nm_csb_ktoa *csb_hg;
|
|
|
|
unsigned int min_tx_space;
|
|
|
|
struct netmap_pt_guest_adapter *ptna;
|
|
|
|
struct callout tick;
|
|
#ifdef PTNETMAP_STATS
|
|
struct timeval last_ts;
|
|
#endif /* PTNETMAP_STATS */
|
|
};
|
|
|
|
#define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock)
|
|
#define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock)
|
|
|
|
static int ptnet_probe(device_t);
|
|
static int ptnet_attach(device_t);
|
|
static int ptnet_detach(device_t);
|
|
static int ptnet_suspend(device_t);
|
|
static int ptnet_resume(device_t);
|
|
static int ptnet_shutdown(device_t);
|
|
|
|
static void ptnet_init(void *opaque);
|
|
static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
|
|
static int ptnet_init_locked(struct ptnet_softc *sc);
|
|
static int ptnet_stop(struct ptnet_softc *sc);
|
|
static int ptnet_transmit(if_t ifp, struct mbuf *m);
|
|
static int ptnet_drain_transmit_queue(struct ptnet_queue *pq,
|
|
unsigned int budget,
|
|
bool may_resched);
|
|
static void ptnet_qflush(if_t ifp);
|
|
static void ptnet_tx_task(void *context, int pending);
|
|
|
|
static int ptnet_media_change(if_t ifp);
|
|
static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
|
|
#ifdef PTNETMAP_STATS
|
|
static void ptnet_tick(void *opaque);
|
|
#endif
|
|
|
|
static int ptnet_irqs_init(struct ptnet_softc *sc);
|
|
static void ptnet_irqs_fini(struct ptnet_softc *sc);
|
|
|
|
static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
|
|
static int ptnet_nm_config(struct netmap_adapter *na,
|
|
struct nm_config_info *info);
|
|
static void ptnet_update_vnet_hdr(struct ptnet_softc *sc);
|
|
static int ptnet_nm_register(struct netmap_adapter *na, int onoff);
|
|
static int ptnet_nm_txsync(struct netmap_kring *kring, int flags);
|
|
static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
|
|
static void ptnet_nm_intr(struct netmap_adapter *na, int onoff);
|
|
|
|
static void ptnet_tx_intr(void *opaque);
|
|
static void ptnet_rx_intr(void *opaque);
|
|
|
|
static unsigned ptnet_rx_discard(struct netmap_kring *kring,
|
|
unsigned int head);
|
|
static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
|
|
bool may_resched);
|
|
static void ptnet_rx_task(void *context, int pending);
|
|
|
|
#ifdef DEVICE_POLLING
|
|
static poll_handler_t ptnet_poll;
|
|
#endif
|
|
|
|
static device_method_t ptnet_methods[] = {
|
|
DEVMETHOD(device_probe, ptnet_probe),
|
|
DEVMETHOD(device_attach, ptnet_attach),
|
|
DEVMETHOD(device_detach, ptnet_detach),
|
|
DEVMETHOD(device_suspend, ptnet_suspend),
|
|
DEVMETHOD(device_resume, ptnet_resume),
|
|
DEVMETHOD(device_shutdown, ptnet_shutdown),
|
|
DEVMETHOD_END
|
|
};
|
|
|
|
static driver_t ptnet_driver = {
|
|
"ptnet",
|
|
ptnet_methods,
|
|
sizeof(struct ptnet_softc)
|
|
};
|
|
|
|
/* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
|
|
DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, NULL, NULL,
|
|
SI_ORDER_MIDDLE + 2);
|
|
|
|
static int
|
|
ptnet_probe(device_t dev)
|
|
{
|
|
if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
|
|
pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
|
|
return (ENXIO);
|
|
}
|
|
|
|
device_set_desc(dev, "ptnet network adapter");
|
|
|
|
return (BUS_PROBE_DEFAULT);
|
|
}
|
|
|
|
static inline void ptnet_kick(struct ptnet_queue *pq)
|
|
{
|
|
#ifdef PTNETMAP_STATS
|
|
pq->stats.kicks ++;
|
|
#endif /* PTNETMAP_STATS */
|
|
bus_write_4(pq->sc->iomem, pq->kick, 0);
|
|
}
|
|
|
|
#define PTNET_BUF_RING_SIZE 4096
|
|
#define PTNET_RX_BUDGET 512
|
|
#define PTNET_RX_BATCH 1
|
|
#define PTNET_TX_BUDGET 512
|
|
#define PTNET_TX_BATCH 64
|
|
#define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
|
|
#define PTNET_MAX_PKT_SIZE 65536
|
|
|
|
#define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP)
|
|
#define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
|
|
#define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\
|
|
PTNET_CSUM_OFFLOAD_IPV6)
|
|
|
|
static int
|
|
ptnet_attach(device_t dev)
|
|
{
|
|
uint32_t ptfeatures = 0;
|
|
unsigned int num_rx_rings, num_tx_rings;
|
|
struct netmap_adapter na_arg;
|
|
unsigned int nifp_offset;
|
|
struct ptnet_softc *sc;
|
|
if_t ifp;
|
|
uint32_t macreg;
|
|
int err, rid;
|
|
int i;
|
|
|
|
sc = device_get_softc(dev);
|
|
sc->dev = dev;
|
|
|
|
/* Setup PCI resources. */
|
|
pci_enable_busmaster(dev);
|
|
|
|
rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
|
|
sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
|
|
RF_ACTIVE);
|
|
if (sc->iomem == NULL) {
|
|
device_printf(dev, "Failed to map I/O BAR\n");
|
|
return (ENXIO);
|
|
}
|
|
|
|
/* Negotiate features with the hypervisor. */
|
|
if (ptnet_vnet_hdr) {
|
|
ptfeatures |= PTNETMAP_F_VNET_HDR;
|
|
}
|
|
bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
|
|
ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
|
|
sc->ptfeatures = ptfeatures;
|
|
|
|
num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
|
|
num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
|
|
sc->num_rings = num_tx_rings + num_rx_rings;
|
|
sc->num_tx_rings = num_tx_rings;
|
|
|
|
if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
|
|
device_printf(dev, "CSB cannot handle that many rings (%u)\n",
|
|
sc->num_rings);
|
|
err = ENOMEM;
|
|
goto err_path;
|
|
}
|
|
|
|
/* Allocate CSB and carry out CSB allocation protocol. */
|
|
sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
|
|
(size_t)0, -1UL, PAGE_SIZE, 0);
|
|
if (sc->csb_gh == NULL) {
|
|
device_printf(dev, "Failed to allocate CSB\n");
|
|
err = ENOMEM;
|
|
goto err_path;
|
|
}
|
|
sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
|
|
|
|
{
|
|
/*
|
|
* We use uint64_t rather than vm_paddr_t since we
|
|
* need 64 bit addresses even on 32 bit platforms.
|
|
*/
|
|
uint64_t paddr = vtophys(sc->csb_gh);
|
|
|
|
/* CSB allocation protocol: write to BAH first, then
|
|
* to BAL (for both GH and HG sections). */
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
|
|
(paddr >> 32) & 0xffffffff);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
|
|
paddr & 0xffffffff);
|
|
paddr = vtophys(sc->csb_hg);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
|
|
(paddr >> 32) & 0xffffffff);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
|
|
paddr & 0xffffffff);
|
|
}
|
|
|
|
/* Allocate and initialize per-queue data structures. */
|
|
sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
|
|
M_DEVBUF, M_NOWAIT | M_ZERO);
|
|
if (sc->queues == NULL) {
|
|
err = ENOMEM;
|
|
goto err_path;
|
|
}
|
|
sc->rxqueues = sc->queues + num_tx_rings;
|
|
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
pq->sc = sc;
|
|
pq->kring_id = i;
|
|
pq->kick = PTNET_IO_KICK_BASE + 4 * i;
|
|
pq->atok = sc->csb_gh + i;
|
|
pq->ktoa = sc->csb_hg + i;
|
|
snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
|
|
device_get_nameunit(dev), i);
|
|
mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
|
|
if (i >= num_tx_rings) {
|
|
/* RX queue: fix kring_id. */
|
|
pq->kring_id -= num_tx_rings;
|
|
} else {
|
|
/* TX queue: allocate buf_ring. */
|
|
pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
|
|
M_DEVBUF, M_NOWAIT, &pq->lock);
|
|
if (pq->bufring == NULL) {
|
|
err = ENOMEM;
|
|
goto err_path;
|
|
}
|
|
}
|
|
}
|
|
|
|
sc->min_tx_space = 64; /* Safe initial value. */
|
|
|
|
err = ptnet_irqs_init(sc);
|
|
if (err) {
|
|
goto err_path;
|
|
}
|
|
|
|
/* Setup Ethernet interface. */
|
|
sc->ifp = ifp = if_alloc(IFT_ETHER);
|
|
if (ifp == NULL) {
|
|
device_printf(dev, "Failed to allocate ifnet\n");
|
|
err = ENOMEM;
|
|
goto err_path;
|
|
}
|
|
|
|
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
|
|
if_setbaudrate(ifp, IF_Gbps(10));
|
|
if_setsoftc(ifp, sc);
|
|
if_setflags(ifp, IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX);
|
|
if_setinitfn(ifp, ptnet_init);
|
|
if_setioctlfn(ifp, ptnet_ioctl);
|
|
if_setget_counter(ifp, ptnet_get_counter);
|
|
if_settransmitfn(ifp, ptnet_transmit);
|
|
if_setqflushfn(ifp, ptnet_qflush);
|
|
|
|
ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
|
|
ptnet_media_status);
|
|
ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
|
|
ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
|
|
|
|
macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
|
|
sc->hwaddr[0] = (macreg >> 8) & 0xff;
|
|
sc->hwaddr[1] = macreg & 0xff;
|
|
macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
|
|
sc->hwaddr[2] = (macreg >> 24) & 0xff;
|
|
sc->hwaddr[3] = (macreg >> 16) & 0xff;
|
|
sc->hwaddr[4] = (macreg >> 8) & 0xff;
|
|
sc->hwaddr[5] = macreg & 0xff;
|
|
|
|
ether_ifattach(ifp, sc->hwaddr);
|
|
|
|
if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
|
|
if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU, 0);
|
|
|
|
if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
|
|
/* Similarly to what the vtnet driver does, we can emulate
|
|
* VLAN offloadings by inserting and removing the 802.1Q
|
|
* header during transmit and receive. We are then able
|
|
* to do checksum offloading of VLAN frames. */
|
|
if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
|
|
| IFCAP_VLAN_HWCSUM
|
|
| IFCAP_TSO | IFCAP_LRO
|
|
| IFCAP_VLAN_HWTSO
|
|
| IFCAP_VLAN_HWTAGGING, 0);
|
|
}
|
|
|
|
if_setcapenable(ifp, if_getcapabilities(ifp));
|
|
#ifdef DEVICE_POLLING
|
|
/* Don't enable polling by default. */
|
|
if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0);
|
|
#endif
|
|
snprintf(sc->lock_name, sizeof(sc->lock_name),
|
|
"%s", device_get_nameunit(dev));
|
|
mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
|
|
callout_init_mtx(&sc->tick, &sc->lock, 0);
|
|
|
|
/* Prepare a netmap_adapter struct instance to do netmap_attach(). */
|
|
nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
|
|
memset(&na_arg, 0, sizeof(na_arg));
|
|
na_arg.ifp = ifp;
|
|
na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
|
|
na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
|
|
na_arg.num_tx_rings = num_tx_rings;
|
|
na_arg.num_rx_rings = num_rx_rings;
|
|
na_arg.nm_config = ptnet_nm_config;
|
|
na_arg.nm_krings_create = ptnet_nm_krings_create;
|
|
na_arg.nm_krings_delete = ptnet_nm_krings_delete;
|
|
na_arg.nm_dtor = ptnet_nm_dtor;
|
|
na_arg.nm_intr = ptnet_nm_intr;
|
|
na_arg.nm_register = ptnet_nm_register;
|
|
na_arg.nm_txsync = ptnet_nm_txsync;
|
|
na_arg.nm_rxsync = ptnet_nm_rxsync;
|
|
|
|
netmap_pt_guest_attach(&na_arg, nifp_offset,
|
|
bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
|
|
|
|
/* Now a netmap adapter for this ifp has been allocated, and it
|
|
* can be accessed through NA(ifp). We also have to initialize the CSB
|
|
* pointer. */
|
|
sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
|
|
|
|
/* If virtio-net header was negotiated, set the virt_hdr_len field in
|
|
* the netmap adapter, to inform users that this netmap adapter requires
|
|
* the application to deal with the headers. */
|
|
ptnet_update_vnet_hdr(sc);
|
|
|
|
device_printf(dev, "%s() completed\n", __func__);
|
|
|
|
return (0);
|
|
|
|
err_path:
|
|
ptnet_detach(dev);
|
|
return err;
|
|
}
|
|
|
|
/* Stop host sync-kloop if it was running. */
|
|
static void
|
|
ptnet_device_shutdown(struct ptnet_softc *sc)
|
|
{
|
|
ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
|
|
bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
|
|
}
|
|
|
|
static int
|
|
ptnet_detach(device_t dev)
|
|
{
|
|
struct ptnet_softc *sc = device_get_softc(dev);
|
|
int i;
|
|
|
|
ptnet_device_shutdown(sc);
|
|
|
|
#ifdef DEVICE_POLLING
|
|
if (if_getcapenable(sc->ifp) & IFCAP_POLLING) {
|
|
ether_poll_deregister(sc->ifp);
|
|
}
|
|
#endif
|
|
callout_drain(&sc->tick);
|
|
|
|
if (sc->queues) {
|
|
/* Drain taskqueues before calling if_detach. */
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
if (pq->taskq) {
|
|
taskqueue_drain(pq->taskq, &pq->task);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (sc->ifp) {
|
|
ether_ifdetach(sc->ifp);
|
|
|
|
/* Uninitialize netmap adapters for this device. */
|
|
netmap_detach(sc->ifp);
|
|
|
|
ifmedia_removeall(&sc->media);
|
|
if_free(sc->ifp);
|
|
sc->ifp = NULL;
|
|
}
|
|
|
|
ptnet_irqs_fini(sc);
|
|
|
|
if (sc->csb_gh) {
|
|
contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
|
|
sc->csb_gh = NULL;
|
|
sc->csb_hg = NULL;
|
|
}
|
|
|
|
if (sc->queues) {
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
if (mtx_initialized(&pq->lock)) {
|
|
mtx_destroy(&pq->lock);
|
|
}
|
|
if (pq->bufring != NULL) {
|
|
buf_ring_free(pq->bufring, M_DEVBUF);
|
|
}
|
|
}
|
|
free(sc->queues, M_DEVBUF);
|
|
sc->queues = NULL;
|
|
}
|
|
|
|
if (sc->iomem) {
|
|
bus_release_resource(dev, SYS_RES_IOPORT,
|
|
PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
|
|
sc->iomem = NULL;
|
|
}
|
|
|
|
mtx_destroy(&sc->lock);
|
|
|
|
device_printf(dev, "%s() completed\n", __func__);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ptnet_suspend(device_t dev)
|
|
{
|
|
struct ptnet_softc *sc = device_get_softc(dev);
|
|
|
|
(void)sc;
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ptnet_resume(device_t dev)
|
|
{
|
|
struct ptnet_softc *sc = device_get_softc(dev);
|
|
|
|
(void)sc;
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ptnet_shutdown(device_t dev)
|
|
{
|
|
struct ptnet_softc *sc = device_get_softc(dev);
|
|
|
|
ptnet_device_shutdown(sc);
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
ptnet_irqs_init(struct ptnet_softc *sc)
|
|
{
|
|
int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
|
|
int nvecs = sc->num_rings;
|
|
device_t dev = sc->dev;
|
|
int err = ENOSPC;
|
|
int cpu_cur;
|
|
int i;
|
|
|
|
if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) {
|
|
device_printf(dev, "Could not find MSI-X capability\n");
|
|
return (ENXIO);
|
|
}
|
|
|
|
sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
|
|
&rid, RF_ACTIVE);
|
|
if (sc->msix_mem == NULL) {
|
|
device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
|
|
return (ENXIO);
|
|
}
|
|
|
|
if (pci_msix_count(dev) < nvecs) {
|
|
device_printf(dev, "Not enough MSI-X vectors\n");
|
|
goto err_path;
|
|
}
|
|
|
|
err = pci_alloc_msix(dev, &nvecs);
|
|
if (err) {
|
|
device_printf(dev, "Failed to allocate MSI-X vectors\n");
|
|
goto err_path;
|
|
}
|
|
|
|
for (i = 0; i < nvecs; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
rid = i + 1;
|
|
pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
|
|
RF_ACTIVE);
|
|
if (pq->irq == NULL) {
|
|
device_printf(dev, "Failed to allocate interrupt "
|
|
"for queue #%d\n", i);
|
|
err = ENOSPC;
|
|
goto err_path;
|
|
}
|
|
}
|
|
|
|
cpu_cur = CPU_FIRST();
|
|
for (i = 0; i < nvecs; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
void (*handler)(void *) = ptnet_tx_intr;
|
|
|
|
if (i >= sc->num_tx_rings) {
|
|
handler = ptnet_rx_intr;
|
|
}
|
|
err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
|
|
NULL /* intr_filter */, handler,
|
|
pq, &pq->cookie);
|
|
if (err) {
|
|
device_printf(dev, "Failed to register intr handler "
|
|
"for queue #%d\n", i);
|
|
goto err_path;
|
|
}
|
|
|
|
bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
|
|
#if 0
|
|
bus_bind_intr(sc->dev, pq->irq, cpu_cur);
|
|
#endif
|
|
cpu_cur = CPU_NEXT(cpu_cur);
|
|
}
|
|
|
|
device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
|
|
|
|
cpu_cur = CPU_FIRST();
|
|
for (i = 0; i < nvecs; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
if (i < sc->num_tx_rings)
|
|
TASK_INIT(&pq->task, 0, ptnet_tx_task, pq);
|
|
else
|
|
NET_TASK_INIT(&pq->task, 0, ptnet_rx_task, pq);
|
|
|
|
pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
|
|
taskqueue_thread_enqueue, &pq->taskq);
|
|
taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
|
|
device_get_nameunit(sc->dev), cpu_cur);
|
|
cpu_cur = CPU_NEXT(cpu_cur);
|
|
}
|
|
|
|
return 0;
|
|
err_path:
|
|
ptnet_irqs_fini(sc);
|
|
return err;
|
|
}
|
|
|
|
static void
|
|
ptnet_irqs_fini(struct ptnet_softc *sc)
|
|
{
|
|
device_t dev = sc->dev;
|
|
int i;
|
|
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
if (pq->taskq) {
|
|
taskqueue_free(pq->taskq);
|
|
pq->taskq = NULL;
|
|
}
|
|
|
|
if (pq->cookie) {
|
|
bus_teardown_intr(dev, pq->irq, pq->cookie);
|
|
pq->cookie = NULL;
|
|
}
|
|
|
|
if (pq->irq) {
|
|
bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
|
|
pq->irq = NULL;
|
|
}
|
|
}
|
|
|
|
if (sc->msix_mem) {
|
|
pci_release_msi(dev);
|
|
|
|
bus_release_resource(dev, SYS_RES_MEMORY,
|
|
PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
|
|
sc->msix_mem);
|
|
sc->msix_mem = NULL;
|
|
}
|
|
}
|
|
|
|
static void
|
|
ptnet_init(void *opaque)
|
|
{
|
|
struct ptnet_softc *sc = opaque;
|
|
|
|
PTNET_CORE_LOCK(sc);
|
|
ptnet_init_locked(sc);
|
|
PTNET_CORE_UNLOCK(sc);
|
|
}
|
|
|
|
static int
|
|
ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
device_t dev = sc->dev;
|
|
struct ifreq *ifr = (struct ifreq *)data;
|
|
int mask __unused, err = 0;
|
|
|
|
switch (cmd) {
|
|
case SIOCSIFFLAGS:
|
|
device_printf(dev, "SIOCSIFFLAGS %x\n", if_getflags(ifp));
|
|
PTNET_CORE_LOCK(sc);
|
|
if (if_getflags(ifp) & IFF_UP) {
|
|
/* Network stack wants the iff to be up. */
|
|
err = ptnet_init_locked(sc);
|
|
} else {
|
|
/* Network stack wants the iff to be down. */
|
|
err = ptnet_stop(sc);
|
|
}
|
|
/* We don't need to do nothing to support IFF_PROMISC,
|
|
* since that is managed by the backend port. */
|
|
PTNET_CORE_UNLOCK(sc);
|
|
break;
|
|
|
|
case SIOCSIFCAP:
|
|
device_printf(dev, "SIOCSIFCAP %x %x\n",
|
|
ifr->ifr_reqcap, if_getcapenable(ifp));
|
|
mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
|
|
#ifdef DEVICE_POLLING
|
|
if (mask & IFCAP_POLLING) {
|
|
struct ptnet_queue *pq;
|
|
int i;
|
|
|
|
if (ifr->ifr_reqcap & IFCAP_POLLING) {
|
|
err = ether_poll_register(ptnet_poll, ifp);
|
|
if (err) {
|
|
break;
|
|
}
|
|
/* Stop queues and sync with taskqueues. */
|
|
if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
pq = sc-> queues + i;
|
|
/* Make sure the worker sees the
|
|
* IFF_DRV_RUNNING down. */
|
|
PTNET_Q_LOCK(pq);
|
|
pq->atok->appl_need_kick = 0;
|
|
PTNET_Q_UNLOCK(pq);
|
|
/* Wait for rescheduling to finish. */
|
|
if (pq->taskq) {
|
|
taskqueue_drain(pq->taskq,
|
|
&pq->task);
|
|
}
|
|
}
|
|
if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
|
|
} else {
|
|
err = ether_poll_deregister(ifp);
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
pq = sc-> queues + i;
|
|
PTNET_Q_LOCK(pq);
|
|
pq->atok->appl_need_kick = 1;
|
|
PTNET_Q_UNLOCK(pq);
|
|
}
|
|
}
|
|
}
|
|
#endif /* DEVICE_POLLING */
|
|
if_setcapenable(ifp, ifr->ifr_reqcap);
|
|
break;
|
|
|
|
case SIOCSIFMTU:
|
|
/* We support any reasonable MTU. */
|
|
if (ifr->ifr_mtu < ETHERMIN ||
|
|
ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
|
|
err = EINVAL;
|
|
} else {
|
|
PTNET_CORE_LOCK(sc);
|
|
if_setmtu(ifp, ifr->ifr_mtu);
|
|
PTNET_CORE_UNLOCK(sc);
|
|
}
|
|
break;
|
|
|
|
case SIOCSIFMEDIA:
|
|
case SIOCGIFMEDIA:
|
|
err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
|
|
break;
|
|
|
|
default:
|
|
err = ether_ioctl(ifp, cmd, data);
|
|
break;
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
ptnet_init_locked(struct ptnet_softc *sc)
|
|
{
|
|
if_t ifp = sc->ifp;
|
|
struct netmap_adapter *na_dr = &sc->ptna->dr.up;
|
|
struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
|
|
unsigned int nm_buf_size;
|
|
int ret;
|
|
|
|
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
|
|
return 0; /* nothing to do */
|
|
}
|
|
|
|
device_printf(sc->dev, "%s\n", __func__);
|
|
|
|
/* Translate offload capabilities according to if_capenable. */
|
|
if_sethwassist(ifp, 0);
|
|
if (if_getcapenable(ifp) & IFCAP_TXCSUM)
|
|
if_sethwassistbits(ifp, PTNET_CSUM_OFFLOAD, 0);
|
|
if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
|
|
if_sethwassistbits(ifp, PTNET_CSUM_OFFLOAD_IPV6, 0);
|
|
if (if_getcapenable(ifp) & IFCAP_TSO4)
|
|
if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
|
|
if (if_getcapenable(ifp) & IFCAP_TSO6)
|
|
if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
|
|
|
|
/*
|
|
* Prepare the interface for netmap mode access.
|
|
*/
|
|
netmap_update_config(na_dr);
|
|
|
|
ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
|
|
if (ret) {
|
|
device_printf(sc->dev, "netmap_mem_finalize() failed\n");
|
|
return ret;
|
|
}
|
|
|
|
if (sc->ptna->backend_users == 0) {
|
|
ret = ptnet_nm_krings_create(na_nm);
|
|
if (ret) {
|
|
device_printf(sc->dev, "ptnet_nm_krings_create() "
|
|
"failed\n");
|
|
goto err_mem_finalize;
|
|
}
|
|
|
|
ret = netmap_mem_rings_create(na_dr);
|
|
if (ret) {
|
|
device_printf(sc->dev, "netmap_mem_rings_create() "
|
|
"failed\n");
|
|
goto err_rings_create;
|
|
}
|
|
|
|
ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
|
|
if (ret) {
|
|
device_printf(sc->dev, "netmap_mem_get_lut() "
|
|
"failed\n");
|
|
goto err_get_lut;
|
|
}
|
|
}
|
|
|
|
ret = ptnet_nm_register(na_dr, 1 /* on */);
|
|
if (ret) {
|
|
goto err_register;
|
|
}
|
|
|
|
nm_buf_size = NETMAP_BUF_SIZE(na_dr);
|
|
|
|
KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
|
|
sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
|
|
device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
|
|
sc->min_tx_space);
|
|
#ifdef PTNETMAP_STATS
|
|
callout_reset(&sc->tick, hz, ptnet_tick, sc);
|
|
#endif
|
|
|
|
if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
|
|
|
|
return 0;
|
|
|
|
err_register:
|
|
memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
|
|
err_get_lut:
|
|
netmap_mem_rings_delete(na_dr);
|
|
err_rings_create:
|
|
ptnet_nm_krings_delete(na_nm);
|
|
err_mem_finalize:
|
|
netmap_mem_deref(na_dr->nm_mem, na_dr);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* To be called under core lock. */
|
|
static int
|
|
ptnet_stop(struct ptnet_softc *sc)
|
|
{
|
|
if_t ifp = sc->ifp;
|
|
struct netmap_adapter *na_dr = &sc->ptna->dr.up;
|
|
struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
|
|
int i;
|
|
|
|
device_printf(sc->dev, "%s\n", __func__);
|
|
|
|
if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
|
|
return 0; /* nothing to do */
|
|
}
|
|
|
|
/* Clear the driver-ready flag, and synchronize with all the queues,
|
|
* so that after this loop we are sure nobody is working anymore with
|
|
* the device. This scheme is taken from the vtnet driver. */
|
|
if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
|
|
callout_stop(&sc->tick);
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
PTNET_Q_LOCK(sc->queues + i);
|
|
PTNET_Q_UNLOCK(sc->queues + i);
|
|
}
|
|
|
|
ptnet_nm_register(na_dr, 0 /* off */);
|
|
|
|
if (sc->ptna->backend_users == 0) {
|
|
netmap_mem_rings_delete(na_dr);
|
|
ptnet_nm_krings_delete(na_nm);
|
|
}
|
|
netmap_mem_deref(na_dr->nm_mem, na_dr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
ptnet_qflush(if_t ifp)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
int i;
|
|
|
|
/* Flush all the bufrings and do the interface flush. */
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
struct mbuf *m;
|
|
|
|
PTNET_Q_LOCK(pq);
|
|
if (pq->bufring) {
|
|
while ((m = buf_ring_dequeue_sc(pq->bufring))) {
|
|
m_freem(m);
|
|
}
|
|
}
|
|
PTNET_Q_UNLOCK(pq);
|
|
}
|
|
|
|
if_qflush(ifp);
|
|
}
|
|
|
|
static int
|
|
ptnet_media_change(if_t ifp)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
struct ifmedia *ifm = &sc->media;
|
|
|
|
if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
|
|
return EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static uint64_t
|
|
ptnet_get_counter(if_t ifp, ift_counter cnt)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
struct ptnet_queue_stats stats[2];
|
|
int i;
|
|
|
|
/* Accumulate statistics over the queues. */
|
|
memset(stats, 0, sizeof(stats));
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
int idx = (i < sc->num_tx_rings) ? 0 : 1;
|
|
|
|
stats[idx].packets += pq->stats.packets;
|
|
stats[idx].bytes += pq->stats.bytes;
|
|
stats[idx].errors += pq->stats.errors;
|
|
stats[idx].iqdrops += pq->stats.iqdrops;
|
|
stats[idx].mcasts += pq->stats.mcasts;
|
|
}
|
|
|
|
switch (cnt) {
|
|
case IFCOUNTER_IPACKETS:
|
|
return (stats[1].packets);
|
|
case IFCOUNTER_IQDROPS:
|
|
return (stats[1].iqdrops);
|
|
case IFCOUNTER_IERRORS:
|
|
return (stats[1].errors);
|
|
case IFCOUNTER_OPACKETS:
|
|
return (stats[0].packets);
|
|
case IFCOUNTER_OBYTES:
|
|
return (stats[0].bytes);
|
|
case IFCOUNTER_OMCASTS:
|
|
return (stats[0].mcasts);
|
|
default:
|
|
return (if_get_counter_default(ifp, cnt));
|
|
}
|
|
}
|
|
|
|
|
|
#ifdef PTNETMAP_STATS
|
|
/* Called under core lock. */
|
|
static void
|
|
ptnet_tick(void *opaque)
|
|
{
|
|
struct ptnet_softc *sc = opaque;
|
|
int i;
|
|
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
struct ptnet_queue_stats cur = pq->stats;
|
|
struct timeval now;
|
|
unsigned int delta;
|
|
|
|
microtime(&now);
|
|
delta = now.tv_usec - sc->last_ts.tv_usec +
|
|
(now.tv_sec - sc->last_ts.tv_sec) * 1000000;
|
|
delta /= 1000; /* in milliseconds */
|
|
|
|
if (delta == 0)
|
|
continue;
|
|
|
|
device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
|
|
"intr %lu\n", i, delta,
|
|
(cur.packets - pq->last_stats.packets),
|
|
(cur.kicks - pq->last_stats.kicks),
|
|
(cur.intrs - pq->last_stats.intrs));
|
|
pq->last_stats = cur;
|
|
}
|
|
microtime(&sc->last_ts);
|
|
callout_schedule(&sc->tick, hz);
|
|
}
|
|
#endif /* PTNETMAP_STATS */
|
|
|
|
static void
|
|
ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
|
|
{
|
|
/* We are always active, as the backend netmap port is
|
|
* always open in netmap mode. */
|
|
ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
|
|
ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
|
|
}
|
|
|
|
static uint32_t
|
|
ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
|
|
{
|
|
/*
|
|
* Write a command and read back error status,
|
|
* with zero meaning success.
|
|
*/
|
|
bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
|
|
return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
|
|
}
|
|
|
|
static int
|
|
ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(na->ifp);
|
|
|
|
info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
|
|
info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
|
|
info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
|
|
info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
|
|
info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
|
|
|
|
device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
|
|
info->num_tx_rings, info->num_rx_rings,
|
|
info->num_tx_descs, info->num_rx_descs,
|
|
info->rx_buf_maxsize);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
|
|
{
|
|
int i;
|
|
|
|
/* Sync krings from the host, reading from
|
|
* CSB. */
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct nm_csb_atok *atok = sc->queues[i].atok;
|
|
struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
|
|
struct netmap_kring *kring;
|
|
|
|
if (i < na->num_tx_rings) {
|
|
kring = na->tx_rings[i];
|
|
} else {
|
|
kring = na->rx_rings[i - na->num_tx_rings];
|
|
}
|
|
kring->rhead = kring->ring->head = atok->head;
|
|
kring->rcur = kring->ring->cur = atok->cur;
|
|
kring->nr_hwcur = ktoa->hwcur;
|
|
kring->nr_hwtail = kring->rtail =
|
|
kring->ring->tail = ktoa->hwtail;
|
|
|
|
nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
|
|
ktoa->hwcur, atok->head, atok->cur,
|
|
ktoa->hwtail);
|
|
nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
|
|
t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
|
|
kring->ring->head, kring->ring->cur, kring->nr_hwtail,
|
|
kring->rtail, kring->ring->tail);
|
|
}
|
|
}
|
|
|
|
static void
|
|
ptnet_update_vnet_hdr(struct ptnet_softc *sc)
|
|
{
|
|
unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
|
|
|
|
bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
|
|
sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
|
|
sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
|
|
}
|
|
|
|
static int
|
|
ptnet_nm_register(struct netmap_adapter *na, int onoff)
|
|
{
|
|
/* device-specific */
|
|
if_t ifp = na->ifp;
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
int native = (na == &sc->ptna->hwup.up);
|
|
struct ptnet_queue *pq;
|
|
int ret = 0;
|
|
int i;
|
|
|
|
if (!onoff) {
|
|
sc->ptna->backend_users--;
|
|
}
|
|
|
|
/* If this is the last netmap client, guest interrupt enable flags may
|
|
* be in arbitrary state. Since these flags are going to be used also
|
|
* by the netdevice driver, we have to make sure to start with
|
|
* notifications enabled. Also, schedule NAPI to flush pending packets
|
|
* in the RX rings, since we will not receive further interrupts
|
|
* until these will be processed. */
|
|
if (native && !onoff && na->active_fds == 0) {
|
|
nm_prinf("Exit netmap mode, re-enable interrupts");
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
pq = sc->queues + i;
|
|
pq->atok->appl_need_kick = 1;
|
|
}
|
|
}
|
|
|
|
if (onoff) {
|
|
if (sc->ptna->backend_users == 0) {
|
|
/* Initialize notification enable fields in the CSB. */
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
pq = sc->queues + i;
|
|
pq->ktoa->kern_need_kick = 1;
|
|
pq->atok->appl_need_kick =
|
|
(!(if_getcapenable(ifp) & IFCAP_POLLING)
|
|
&& i >= sc->num_tx_rings);
|
|
}
|
|
|
|
/* Set the virtio-net header length. */
|
|
ptnet_update_vnet_hdr(sc);
|
|
|
|
/* Make sure the host adapter passed through is ready
|
|
* for txsync/rxsync. */
|
|
ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
|
|
/* Align the guest krings and rings to the state stored
|
|
* in the CSB. */
|
|
ptnet_sync_from_csb(sc, na);
|
|
}
|
|
|
|
/* If not native, don't call nm_set_native_flags, since we don't want
|
|
* to replace if_transmit method, nor set NAF_NETMAP_ON */
|
|
if (native) {
|
|
netmap_krings_mode_commit(na, onoff);
|
|
nm_set_native_flags(na);
|
|
}
|
|
|
|
} else {
|
|
if (native) {
|
|
nm_clear_native_flags(na);
|
|
netmap_krings_mode_commit(na, onoff);
|
|
}
|
|
|
|
if (sc->ptna->backend_users == 0) {
|
|
ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
|
|
}
|
|
}
|
|
|
|
if (onoff) {
|
|
sc->ptna->backend_users++;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int
|
|
ptnet_nm_txsync(struct netmap_kring *kring, int flags)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
|
|
struct ptnet_queue *pq = sc->queues + kring->ring_id;
|
|
bool notify;
|
|
|
|
notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
|
|
if (notify) {
|
|
ptnet_kick(pq);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
|
|
struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
|
|
bool notify;
|
|
|
|
notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
|
|
if (notify) {
|
|
ptnet_kick(pq);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
ptnet_nm_intr(struct netmap_adapter *na, int onoff)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(na->ifp);
|
|
int i;
|
|
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
pq->atok->appl_need_kick = onoff;
|
|
}
|
|
}
|
|
|
|
static void
|
|
ptnet_tx_intr(void *opaque)
|
|
{
|
|
struct ptnet_queue *pq = opaque;
|
|
struct ptnet_softc *sc = pq->sc;
|
|
|
|
DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
|
|
#ifdef PTNETMAP_STATS
|
|
pq->stats.intrs ++;
|
|
#endif /* PTNETMAP_STATS */
|
|
|
|
if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
|
|
return;
|
|
}
|
|
|
|
/* Schedule the tasqueue to flush process transmissions requests.
|
|
* However, vtnet, if_em and if_igb just call ptnet_transmit() here,
|
|
* at least when using MSI-X interrupts. The if_em driver, instead
|
|
* schedule taskqueue when using legacy interrupts. */
|
|
taskqueue_enqueue(pq->taskq, &pq->task);
|
|
}
|
|
|
|
static void
|
|
ptnet_rx_intr(void *opaque)
|
|
{
|
|
struct ptnet_queue *pq = opaque;
|
|
struct ptnet_softc *sc = pq->sc;
|
|
unsigned int unused;
|
|
|
|
DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
|
|
#ifdef PTNETMAP_STATS
|
|
pq->stats.intrs ++;
|
|
#endif /* PTNETMAP_STATS */
|
|
|
|
if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
|
|
return;
|
|
}
|
|
|
|
/* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
|
|
* receive-side processing is executed directly in the interrupt
|
|
* service routine. Alternatively, we may schedule the taskqueue. */
|
|
ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
|
|
}
|
|
|
|
static void
|
|
ptnet_vlan_tag_remove(struct mbuf *m)
|
|
{
|
|
struct ether_vlan_header *evh;
|
|
|
|
evh = mtod(m, struct ether_vlan_header *);
|
|
m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
|
|
m->m_flags |= M_VLANTAG;
|
|
|
|
/* Strip the 802.1Q header. */
|
|
bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
|
|
ETHER_HDR_LEN - ETHER_TYPE_LEN);
|
|
m_adj(m, ETHER_VLAN_ENCAP_LEN);
|
|
}
|
|
|
|
static void
|
|
ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
|
|
unsigned int head, unsigned int sync_flags)
|
|
{
|
|
struct netmap_ring *ring = kring->ring;
|
|
struct nm_csb_atok *atok = pq->atok;
|
|
struct nm_csb_ktoa *ktoa = pq->ktoa;
|
|
|
|
/* Some packets have been pushed to the netmap ring. We have
|
|
* to tell the host to process the new packets, updating cur
|
|
* and head in the CSB. */
|
|
ring->head = ring->cur = head;
|
|
|
|
/* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
|
|
kring->rcur = kring->rhead = head;
|
|
|
|
nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
|
|
|
|
/* Kick the host if needed. */
|
|
if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
|
|
atok->sync_flags = sync_flags;
|
|
ptnet_kick(pq);
|
|
}
|
|
}
|
|
|
|
#define PTNET_TX_NOSPACE(_h, _k, _min) \
|
|
((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
|
|
(_k)->rtail - (_h)) < (_min)
|
|
|
|
/* This function may be called by the network stack, or by
|
|
* by the taskqueue thread. */
|
|
static int
|
|
ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
|
|
bool may_resched)
|
|
{
|
|
struct ptnet_softc *sc = pq->sc;
|
|
bool have_vnet_hdr = sc->vnet_hdr_len;
|
|
struct netmap_adapter *na = &sc->ptna->dr.up;
|
|
if_t ifp = sc->ifp;
|
|
unsigned int batch_count = 0;
|
|
struct nm_csb_atok *atok;
|
|
struct nm_csb_ktoa *ktoa;
|
|
struct netmap_kring *kring;
|
|
struct netmap_ring *ring;
|
|
struct netmap_slot *slot;
|
|
unsigned int count = 0;
|
|
unsigned int minspace;
|
|
unsigned int head;
|
|
unsigned int lim;
|
|
struct mbuf *mhead;
|
|
struct mbuf *mf;
|
|
int nmbuf_bytes;
|
|
uint8_t *nmbuf;
|
|
|
|
if (!PTNET_Q_TRYLOCK(pq)) {
|
|
/* We failed to acquire the lock, schedule the taskqueue. */
|
|
nm_prlim(1, "Deferring TX work");
|
|
if (may_resched) {
|
|
taskqueue_enqueue(pq->taskq, &pq->task);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
if (unlikely(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) {
|
|
PTNET_Q_UNLOCK(pq);
|
|
nm_prlim(1, "Interface is down");
|
|
return ENETDOWN;
|
|
}
|
|
|
|
atok = pq->atok;
|
|
ktoa = pq->ktoa;
|
|
kring = na->tx_rings[pq->kring_id];
|
|
ring = kring->ring;
|
|
lim = kring->nkr_num_slots - 1;
|
|
head = ring->head;
|
|
minspace = sc->min_tx_space;
|
|
|
|
while (count < budget) {
|
|
if (PTNET_TX_NOSPACE(head, kring, minspace)) {
|
|
/* We ran out of slot, let's see if the host has
|
|
* freed up some, by reading hwcur and hwtail from
|
|
* the CSB. */
|
|
ptnet_sync_tail(ktoa, kring);
|
|
|
|
if (PTNET_TX_NOSPACE(head, kring, minspace)) {
|
|
/* Still no slots available. Reactivate the
|
|
* interrupts so that we can be notified
|
|
* when some free slots are made available by
|
|
* the host. */
|
|
atok->appl_need_kick = 1;
|
|
|
|
/* Double check. We need a full barrier to
|
|
* prevent the store to atok->appl_need_kick
|
|
* to be reordered with the load from
|
|
* ktoa->hwcur and ktoa->hwtail (store-load
|
|
* barrier). */
|
|
nm_stld_barrier();
|
|
ptnet_sync_tail(ktoa, kring);
|
|
if (likely(PTNET_TX_NOSPACE(head, kring,
|
|
minspace))) {
|
|
break;
|
|
}
|
|
|
|
nm_prlim(1, "Found more slots by doublecheck");
|
|
/* More slots were freed before reactivating
|
|
* the interrupts. */
|
|
atok->appl_need_kick = 0;
|
|
}
|
|
}
|
|
|
|
mhead = drbr_peek(ifp, pq->bufring);
|
|
if (!mhead) {
|
|
break;
|
|
}
|
|
|
|
/* Initialize transmission state variables. */
|
|
slot = ring->slot + head;
|
|
nmbuf = NMB(na, slot);
|
|
nmbuf_bytes = 0;
|
|
|
|
/* If needed, prepare the virtio-net header at the beginning
|
|
* of the first slot. */
|
|
if (have_vnet_hdr) {
|
|
struct virtio_net_hdr *vh =
|
|
(struct virtio_net_hdr *)nmbuf;
|
|
|
|
/* For performance, we could replace this memset() with
|
|
* two 8-bytes-wide writes. */
|
|
memset(nmbuf, 0, PTNET_HDR_SIZE);
|
|
if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
|
|
mhead = virtio_net_tx_offload(ifp, mhead, false,
|
|
vh);
|
|
if (unlikely(!mhead)) {
|
|
/* Packet dropped because errors
|
|
* occurred while preparing the vnet
|
|
* header. Let's go ahead with the next
|
|
* packet. */
|
|
pq->stats.errors ++;
|
|
drbr_advance(ifp, pq->bufring);
|
|
continue;
|
|
}
|
|
}
|
|
nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
|
|
"csum_start %u csum_ofs %u hdr_len = %u "
|
|
"gso_size %u gso_type %x", __func__,
|
|
mhead->m_pkthdr.csum_flags, vh->flags,
|
|
vh->csum_start, vh->csum_offset, vh->hdr_len,
|
|
vh->gso_size, vh->gso_type);
|
|
|
|
nmbuf += PTNET_HDR_SIZE;
|
|
nmbuf_bytes += PTNET_HDR_SIZE;
|
|
}
|
|
|
|
for (mf = mhead; mf; mf = mf->m_next) {
|
|
uint8_t *mdata = mf->m_data;
|
|
int mlen = mf->m_len;
|
|
|
|
for (;;) {
|
|
int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
|
|
|
|
if (mlen < copy) {
|
|
copy = mlen;
|
|
}
|
|
memcpy(nmbuf, mdata, copy);
|
|
|
|
mdata += copy;
|
|
mlen -= copy;
|
|
nmbuf += copy;
|
|
nmbuf_bytes += copy;
|
|
|
|
if (!mlen) {
|
|
break;
|
|
}
|
|
|
|
slot->len = nmbuf_bytes;
|
|
slot->flags = NS_MOREFRAG;
|
|
|
|
head = nm_next(head, lim);
|
|
KASSERT(head != ring->tail,
|
|
("Unexpectedly run out of TX space"));
|
|
slot = ring->slot + head;
|
|
nmbuf = NMB(na, slot);
|
|
nmbuf_bytes = 0;
|
|
}
|
|
}
|
|
|
|
/* Complete last slot and update head. */
|
|
slot->len = nmbuf_bytes;
|
|
slot->flags = 0;
|
|
head = nm_next(head, lim);
|
|
|
|
/* Consume the packet just processed. */
|
|
drbr_advance(ifp, pq->bufring);
|
|
|
|
/* Copy the packet to listeners. */
|
|
ETHER_BPF_MTAP(ifp, mhead);
|
|
|
|
pq->stats.packets ++;
|
|
pq->stats.bytes += mhead->m_pkthdr.len;
|
|
if (mhead->m_flags & M_MCAST) {
|
|
pq->stats.mcasts ++;
|
|
}
|
|
|
|
m_freem(mhead);
|
|
|
|
count ++;
|
|
if (++batch_count == PTNET_TX_BATCH) {
|
|
ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
|
|
batch_count = 0;
|
|
}
|
|
}
|
|
|
|
if (batch_count) {
|
|
ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
|
|
}
|
|
|
|
if (count >= budget && may_resched) {
|
|
DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
|
|
drbr_inuse(ifp, pq->bufring)));
|
|
taskqueue_enqueue(pq->taskq, &pq->task);
|
|
}
|
|
|
|
PTNET_Q_UNLOCK(pq);
|
|
|
|
return count;
|
|
}
|
|
|
|
static int
|
|
ptnet_transmit(if_t ifp, struct mbuf *m)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
struct ptnet_queue *pq;
|
|
unsigned int queue_idx;
|
|
int err;
|
|
|
|
DBG(device_printf(sc->dev, "transmit %p\n", m));
|
|
|
|
/* Insert 802.1Q header if needed. */
|
|
if (m->m_flags & M_VLANTAG) {
|
|
m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
|
|
if (m == NULL) {
|
|
return ENOBUFS;
|
|
}
|
|
m->m_flags &= ~M_VLANTAG;
|
|
}
|
|
|
|
/* Get the flow-id if available. */
|
|
queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
|
|
m->m_pkthdr.flowid : curcpu;
|
|
|
|
if (unlikely(queue_idx >= sc->num_tx_rings)) {
|
|
queue_idx %= sc->num_tx_rings;
|
|
}
|
|
|
|
pq = sc->queues + queue_idx;
|
|
|
|
err = drbr_enqueue(ifp, pq->bufring, m);
|
|
if (err) {
|
|
/* ENOBUFS when the bufring is full */
|
|
nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
|
|
__func__, err);
|
|
pq->stats.errors ++;
|
|
return err;
|
|
}
|
|
|
|
if (if_getcapenable(ifp) & IFCAP_POLLING) {
|
|
/* If polling is on, the transmit queues will be
|
|
* drained by the poller. */
|
|
return 0;
|
|
}
|
|
|
|
err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
|
|
|
|
return (err < 0) ? err : 0;
|
|
}
|
|
|
|
static unsigned int
|
|
ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
|
|
{
|
|
struct netmap_ring *ring = kring->ring;
|
|
struct netmap_slot *slot = ring->slot + head;
|
|
|
|
for (;;) {
|
|
head = nm_next(head, kring->nkr_num_slots - 1);
|
|
if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
|
|
break;
|
|
}
|
|
slot = ring->slot + head;
|
|
}
|
|
|
|
return head;
|
|
}
|
|
|
|
static inline struct mbuf *
|
|
ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
|
|
{
|
|
uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
|
|
|
|
do {
|
|
unsigned int copy;
|
|
|
|
if (mtail->m_len == MCLBYTES) {
|
|
struct mbuf *mf;
|
|
|
|
mf = m_getcl(M_NOWAIT, MT_DATA, 0);
|
|
if (unlikely(!mf)) {
|
|
return NULL;
|
|
}
|
|
|
|
mtail->m_next = mf;
|
|
mtail = mf;
|
|
mdata = mtod(mtail, uint8_t *);
|
|
mtail->m_len = 0;
|
|
}
|
|
|
|
copy = MCLBYTES - mtail->m_len;
|
|
if (nmbuf_len < copy) {
|
|
copy = nmbuf_len;
|
|
}
|
|
|
|
memcpy(mdata, nmbuf, copy);
|
|
|
|
nmbuf += copy;
|
|
nmbuf_len -= copy;
|
|
mdata += copy;
|
|
mtail->m_len += copy;
|
|
} while (nmbuf_len);
|
|
|
|
return mtail;
|
|
}
|
|
|
|
static int
|
|
ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
|
|
{
|
|
struct ptnet_softc *sc = pq->sc;
|
|
bool have_vnet_hdr = sc->vnet_hdr_len;
|
|
struct nm_csb_atok *atok = pq->atok;
|
|
struct nm_csb_ktoa *ktoa = pq->ktoa;
|
|
struct netmap_adapter *na = &sc->ptna->dr.up;
|
|
struct netmap_kring *kring = na->rx_rings[pq->kring_id];
|
|
struct netmap_ring *ring = kring->ring;
|
|
unsigned int const lim = kring->nkr_num_slots - 1;
|
|
unsigned int batch_count = 0;
|
|
if_t ifp = sc->ifp;
|
|
unsigned int count = 0;
|
|
uint32_t head;
|
|
|
|
PTNET_Q_LOCK(pq);
|
|
|
|
if (unlikely(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) {
|
|
goto unlock;
|
|
}
|
|
|
|
kring->nr_kflags &= ~NKR_PENDINTR;
|
|
|
|
head = ring->head;
|
|
while (count < budget) {
|
|
uint32_t prev_head = head;
|
|
struct mbuf *mhead, *mtail;
|
|
struct virtio_net_hdr *vh;
|
|
struct netmap_slot *slot;
|
|
unsigned int nmbuf_len;
|
|
uint8_t *nmbuf;
|
|
int deliver = 1; /* the mbuf to the network stack. */
|
|
host_sync:
|
|
if (head == ring->tail) {
|
|
/* We ran out of slot, let's see if the host has
|
|
* added some, by reading hwcur and hwtail from
|
|
* the CSB. */
|
|
ptnet_sync_tail(ktoa, kring);
|
|
|
|
if (head == ring->tail) {
|
|
/* Still no slots available. Reactivate
|
|
* interrupts as they were disabled by the
|
|
* host thread right before issuing the
|
|
* last interrupt. */
|
|
atok->appl_need_kick = 1;
|
|
|
|
/* Double check for more completed RX slots.
|
|
* We need a full barrier to prevent the store
|
|
* to atok->appl_need_kick to be reordered with
|
|
* the load from ktoa->hwcur and ktoa->hwtail
|
|
* (store-load barrier). */
|
|
nm_stld_barrier();
|
|
ptnet_sync_tail(ktoa, kring);
|
|
if (likely(head == ring->tail)) {
|
|
break;
|
|
}
|
|
atok->appl_need_kick = 0;
|
|
}
|
|
}
|
|
|
|
/* Initialize ring state variables, possibly grabbing the
|
|
* virtio-net header. */
|
|
slot = ring->slot + head;
|
|
nmbuf = NMB(na, slot);
|
|
nmbuf_len = slot->len;
|
|
|
|
vh = (struct virtio_net_hdr *)nmbuf;
|
|
if (have_vnet_hdr) {
|
|
if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
|
|
/* There is no good reason why host should
|
|
* put the header in multiple netmap slots.
|
|
* If this is the case, discard. */
|
|
nm_prlim(1, "Fragmented vnet-hdr: dropping");
|
|
head = ptnet_rx_discard(kring, head);
|
|
pq->stats.iqdrops ++;
|
|
deliver = 0;
|
|
goto skip;
|
|
}
|
|
nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
|
|
"csum_ofs %u hdr_len = %u gso_size %u "
|
|
"gso_type %x", __func__, vh->flags,
|
|
vh->csum_start, vh->csum_offset, vh->hdr_len,
|
|
vh->gso_size, vh->gso_type);
|
|
nmbuf += PTNET_HDR_SIZE;
|
|
nmbuf_len -= PTNET_HDR_SIZE;
|
|
}
|
|
|
|
/* Allocate the head of a new mbuf chain.
|
|
* We use m_getcl() to allocate an mbuf with standard cluster
|
|
* size (MCLBYTES). In the future we could use m_getjcl()
|
|
* to choose different sizes. */
|
|
mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
|
|
if (unlikely(mhead == NULL)) {
|
|
device_printf(sc->dev, "%s: failed to allocate mbuf "
|
|
"head\n", __func__);
|
|
pq->stats.errors ++;
|
|
break;
|
|
}
|
|
|
|
/* Initialize the mbuf state variables. */
|
|
mhead->m_pkthdr.len = nmbuf_len;
|
|
mtail->m_len = 0;
|
|
|
|
/* Scan all the netmap slots containing the current packet. */
|
|
for (;;) {
|
|
DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
|
|
"len %u, flags %u\n", __func__,
|
|
head, ring->tail, slot->len,
|
|
slot->flags));
|
|
|
|
mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
|
|
if (unlikely(!mtail)) {
|
|
/* Ouch. We ran out of memory while processing
|
|
* a packet. We have to restore the previous
|
|
* head position, free the mbuf chain, and
|
|
* schedule the taskqueue to give the packet
|
|
* another chance. */
|
|
device_printf(sc->dev, "%s: failed to allocate"
|
|
" mbuf frag, reset head %u --> %u\n",
|
|
__func__, head, prev_head);
|
|
head = prev_head;
|
|
m_freem(mhead);
|
|
pq->stats.errors ++;
|
|
if (may_resched) {
|
|
taskqueue_enqueue(pq->taskq,
|
|
&pq->task);
|
|
}
|
|
goto escape;
|
|
}
|
|
|
|
/* We have to increment head irrespective of the
|
|
* NS_MOREFRAG being set or not. */
|
|
head = nm_next(head, lim);
|
|
|
|
if (!(slot->flags & NS_MOREFRAG)) {
|
|
break;
|
|
}
|
|
|
|
if (unlikely(head == ring->tail)) {
|
|
/* The very last slot prepared by the host has
|
|
* the NS_MOREFRAG set. Drop it and continue
|
|
* the outer cycle (to do the double-check). */
|
|
nm_prlim(1, "Incomplete packet: dropping");
|
|
m_freem(mhead);
|
|
pq->stats.iqdrops ++;
|
|
goto host_sync;
|
|
}
|
|
|
|
slot = ring->slot + head;
|
|
nmbuf = NMB(na, slot);
|
|
nmbuf_len = slot->len;
|
|
mhead->m_pkthdr.len += nmbuf_len;
|
|
}
|
|
|
|
mhead->m_pkthdr.rcvif = ifp;
|
|
mhead->m_pkthdr.csum_flags = 0;
|
|
|
|
/* Store the queue idx in the packet header. */
|
|
mhead->m_pkthdr.flowid = pq->kring_id;
|
|
M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
|
|
|
|
if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
|
|
struct ether_header *eh;
|
|
|
|
eh = mtod(mhead, struct ether_header *);
|
|
if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
|
|
ptnet_vlan_tag_remove(mhead);
|
|
/*
|
|
* With the 802.1Q header removed, update the
|
|
* checksum starting location accordingly.
|
|
*/
|
|
if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
|
|
vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
|
|
}
|
|
}
|
|
|
|
if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) {
|
|
m_freem(mhead);
|
|
nm_prlim(1, "Csum offload error: dropping");
|
|
pq->stats.iqdrops ++;
|
|
deliver = 0;
|
|
}
|
|
|
|
skip:
|
|
count ++;
|
|
if (++batch_count >= PTNET_RX_BATCH) {
|
|
/* Some packets have been (or will be) pushed to the network
|
|
* stack. We need to update the CSB to tell the host about
|
|
* the new ring->cur and ring->head (RX buffer refill). */
|
|
ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
|
|
batch_count = 0;
|
|
}
|
|
|
|
if (likely(deliver)) {
|
|
pq->stats.packets ++;
|
|
pq->stats.bytes += mhead->m_pkthdr.len;
|
|
|
|
PTNET_Q_UNLOCK(pq);
|
|
if_input(ifp, mhead);
|
|
PTNET_Q_LOCK(pq);
|
|
/* The ring->head index (and related indices) are
|
|
* updated under pq lock by ptnet_ring_update().
|
|
* Since we dropped the lock to call if_input(), we
|
|
* must reload ring->head and restart processing the
|
|
* ring from there. */
|
|
head = ring->head;
|
|
|
|
if (unlikely(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) {
|
|
/* The interface has gone down while we didn't
|
|
* have the lock. Stop any processing and exit. */
|
|
goto unlock;
|
|
}
|
|
}
|
|
}
|
|
escape:
|
|
if (batch_count) {
|
|
ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
|
|
|
|
}
|
|
|
|
if (count >= budget && may_resched) {
|
|
/* If we ran out of budget or the double-check found new
|
|
* slots to process, schedule the taskqueue. */
|
|
DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
|
|
head, ring->tail));
|
|
taskqueue_enqueue(pq->taskq, &pq->task);
|
|
}
|
|
unlock:
|
|
PTNET_Q_UNLOCK(pq);
|
|
|
|
return count;
|
|
}
|
|
|
|
static void
|
|
ptnet_rx_task(void *context, int pending)
|
|
{
|
|
struct ptnet_queue *pq = context;
|
|
|
|
DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
|
|
ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
|
|
}
|
|
|
|
static void
|
|
ptnet_tx_task(void *context, int pending)
|
|
{
|
|
struct ptnet_queue *pq = context;
|
|
|
|
DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
|
|
ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
|
|
}
|
|
|
|
#ifdef DEVICE_POLLING
|
|
/* We don't need to handle differently POLL_AND_CHECK_STATUS and
|
|
* POLL_ONLY, since we don't have an Interrupt Status Register. */
|
|
static int
|
|
ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
|
|
{
|
|
struct ptnet_softc *sc = if_getsoftc(ifp);
|
|
unsigned int queue_budget;
|
|
unsigned int count = 0;
|
|
bool borrow = false;
|
|
int i;
|
|
|
|
KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
|
|
queue_budget = MAX(budget / sc->num_rings, 1);
|
|
nm_prlim(1, "Per-queue budget is %d", queue_budget);
|
|
|
|
while (budget) {
|
|
unsigned int rcnt = 0;
|
|
|
|
for (i = 0; i < sc->num_rings; i++) {
|
|
struct ptnet_queue *pq = sc->queues + i;
|
|
|
|
if (borrow) {
|
|
queue_budget = MIN(queue_budget, budget);
|
|
if (queue_budget == 0) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (i < sc->num_tx_rings) {
|
|
rcnt += ptnet_drain_transmit_queue(pq,
|
|
queue_budget, false);
|
|
} else {
|
|
rcnt += ptnet_rx_eof(pq, queue_budget,
|
|
false);
|
|
}
|
|
}
|
|
|
|
if (!rcnt) {
|
|
/* A scan of the queues gave no result, we can
|
|
* stop here. */
|
|
break;
|
|
}
|
|
|
|
if (rcnt > budget) {
|
|
/* This may happen when initial budget < sc->num_rings,
|
|
* since one packet budget is given to each queue
|
|
* anyway. Just pretend we didn't eat "so much". */
|
|
rcnt = budget;
|
|
}
|
|
count += rcnt;
|
|
budget -= rcnt;
|
|
borrow = true;
|
|
}
|
|
|
|
|
|
return count;
|
|
}
|
|
#endif /* DEVICE_POLLING */
|
|
#endif /* WITH_PTNETMAP */
|