In emulated mode, the FreeBSD netmap port attempts to perform zero-copy transmission. This works as follows: the kernel ring is populated with mbuf headers to which netmap buffers are attached. When transmitting, the mbuf refcount is initialized to 2, and when the counter value has been decremented to 1 netmap infers that the driver has freed the mbuf and thus transmission is complete. This scheme does not generalize to the situation where netmap is attaching to a software interface which may transmit packets among multiple "queues", as is the case with bridge or lagg interfaces. In that case, we would be relying on backing hardware drivers to free transmitted mbufs promptly, but this isn't guaranteed; a driver may reasonably defer freeing a small number of transmitted buffers indefinitely. If such a buffer ends up at the tail of a netmap transmit ring, further transmits can end up blocked indefinitely. Fix the problem by removing the zero-copy scheme (which is also not implemented in the Linux port of netmap). Instead, the kernel ring is populated with regular mbuf clusters into which netmap buffers are copied by nm_os_generic_xmit_frame(). The refcounting scheme is preserved, and this lets us avoid allocating a fresh cluster per transmitted packet in the common case. If the transmit ring is full, a callout is used to free the "stuck" mbuf, avoiding the queue deadlock described above. Furthermore, when recycling mbuf clusters, be sure to fully reinitialize the mbuf header instead of simply re-setting M_PKTHDR. Some software interfaces, like if_vlan, may set fields in the header which should be reset before the mbuf is reused. Reviewed by: vmaffione MFC after: 1 month Sponsored by: Zenarmor Sponsored by: OPNsense Sponsored by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D38065
1173 lines
31 KiB
C
1173 lines
31 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright (C) 2013-2016 Vincenzo Maffione
|
|
* Copyright (C) 2013-2016 Luigi Rizzo
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* This module implements netmap support on top of standard,
|
|
* unmodified device drivers.
|
|
*
|
|
* A NIOCREGIF request is handled here if the device does not
|
|
* have native support. TX and RX rings are emulated as follows:
|
|
*
|
|
* NIOCREGIF
|
|
* We preallocate a block of TX mbufs (roughly as many as
|
|
* tx descriptors; the number is not critical) to speed up
|
|
* operation during transmissions. The refcount on most of
|
|
* these buffers is artificially bumped up so we can recycle
|
|
* them more easily. Also, the destructor is intercepted
|
|
* so we use it as an interrupt notification to wake up
|
|
* processes blocked on a poll().
|
|
*
|
|
* For each receive ring we allocate one "struct mbq"
|
|
* (an mbuf tailq plus a spinlock). We intercept packets
|
|
* (through if_input)
|
|
* on the receive path and put them in the mbq from which
|
|
* netmap receive routines can grab them.
|
|
*
|
|
* TX:
|
|
* in the generic_txsync() routine, netmap buffers are copied
|
|
* (or linked, in a future) to the preallocated mbufs
|
|
* and pushed to the transmit queue. Some of these mbufs
|
|
* (those with NS_REPORT, or otherwise every half ring)
|
|
* have the refcount=1, others have refcount=2.
|
|
* When the destructor is invoked, we take that as
|
|
* a notification that all mbufs up to that one in
|
|
* the specific ring have been completed, and generate
|
|
* the equivalent of a transmit interrupt.
|
|
*
|
|
* RX:
|
|
*
|
|
*/
|
|
|
|
#ifdef __FreeBSD__
|
|
|
|
#include <sys/cdefs.h> /* prerequisite */
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/types.h>
|
|
#include <sys/errno.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/lock.h> /* PROT_EXEC */
|
|
#include <sys/rwlock.h>
|
|
#include <sys/socket.h> /* sockaddrs */
|
|
#include <sys/selinfo.h>
|
|
#include <net/if.h>
|
|
#include <net/if_types.h>
|
|
#include <net/if_var.h>
|
|
#include <machine/bus.h> /* bus_dmamap_* in netmap_kern.h */
|
|
|
|
#include <net/netmap.h>
|
|
#include <dev/netmap/netmap_kern.h>
|
|
#include <dev/netmap/netmap_mem2.h>
|
|
|
|
#define MBUF_RXQ(m) ((m)->m_pkthdr.flowid)
|
|
#define smp_mb()
|
|
|
|
#elif defined _WIN32
|
|
|
|
#include "win_glue.h"
|
|
|
|
#define MBUF_TXQ(m) 0//((m)->m_pkthdr.flowid)
|
|
#define MBUF_RXQ(m) 0//((m)->m_pkthdr.flowid)
|
|
#define smp_mb() //XXX: to be correctly defined
|
|
|
|
#else /* linux */
|
|
|
|
#include "bsd_glue.h"
|
|
|
|
#include <linux/ethtool.h> /* struct ethtool_ops, get_ringparam */
|
|
#include <linux/hrtimer.h>
|
|
|
|
static inline struct mbuf *
|
|
nm_os_get_mbuf(struct ifnet *ifp, int len)
|
|
{
|
|
return alloc_skb(LL_RESERVED_SPACE(ifp) + len +
|
|
ifp->needed_tailroom, GFP_ATOMIC);
|
|
}
|
|
|
|
#endif /* linux */
|
|
|
|
|
|
/* Common headers. */
|
|
#include <net/netmap.h>
|
|
#include <dev/netmap/netmap_kern.h>
|
|
#include <dev/netmap/netmap_mem2.h>
|
|
|
|
|
|
#define for_each_kring_n(_i, _k, _karr, _n) \
|
|
for ((_k)=*(_karr), (_i) = 0; (_i) < (_n); (_i)++, (_k) = (_karr)[(_i)])
|
|
|
|
#define for_each_tx_kring(_i, _k, _na) \
|
|
for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings)
|
|
#define for_each_tx_kring_h(_i, _k, _na) \
|
|
for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings + 1)
|
|
|
|
#define for_each_rx_kring(_i, _k, _na) \
|
|
for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings)
|
|
#define for_each_rx_kring_h(_i, _k, _na) \
|
|
for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings + 1)
|
|
|
|
|
|
/* ======================== PERFORMANCE STATISTICS =========================== */
|
|
|
|
#ifdef RATE_GENERIC
|
|
#define IFRATE(x) x
|
|
struct rate_stats {
|
|
unsigned long txpkt;
|
|
unsigned long txsync;
|
|
unsigned long txirq;
|
|
unsigned long txrepl;
|
|
unsigned long txdrop;
|
|
unsigned long rxpkt;
|
|
unsigned long rxirq;
|
|
unsigned long rxsync;
|
|
};
|
|
|
|
struct rate_context {
|
|
unsigned refcount;
|
|
struct timer_list timer;
|
|
struct rate_stats new;
|
|
struct rate_stats old;
|
|
};
|
|
|
|
#define RATE_PRINTK(_NAME_) \
|
|
printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD);
|
|
#define RATE_PERIOD 2
|
|
static void rate_callback(unsigned long arg)
|
|
{
|
|
struct rate_context * ctx = (struct rate_context *)arg;
|
|
struct rate_stats cur = ctx->new;
|
|
int r;
|
|
|
|
RATE_PRINTK(txpkt);
|
|
RATE_PRINTK(txsync);
|
|
RATE_PRINTK(txirq);
|
|
RATE_PRINTK(txrepl);
|
|
RATE_PRINTK(txdrop);
|
|
RATE_PRINTK(rxpkt);
|
|
RATE_PRINTK(rxsync);
|
|
RATE_PRINTK(rxirq);
|
|
printk("\n");
|
|
|
|
ctx->old = cur;
|
|
r = mod_timer(&ctx->timer, jiffies +
|
|
msecs_to_jiffies(RATE_PERIOD * 1000));
|
|
if (unlikely(r))
|
|
nm_prerr("mod_timer() failed");
|
|
}
|
|
|
|
static struct rate_context rate_ctx;
|
|
|
|
void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi)
|
|
{
|
|
if (txp) rate_ctx.new.txpkt++;
|
|
if (txs) rate_ctx.new.txsync++;
|
|
if (txi) rate_ctx.new.txirq++;
|
|
if (rxp) rate_ctx.new.rxpkt++;
|
|
if (rxs) rate_ctx.new.rxsync++;
|
|
if (rxi) rate_ctx.new.rxirq++;
|
|
}
|
|
|
|
#else /* !RATE */
|
|
#define IFRATE(x)
|
|
#endif /* !RATE */
|
|
|
|
|
|
/* ========== GENERIC (EMULATED) NETMAP ADAPTER SUPPORT ============= */
|
|
|
|
/*
|
|
* Wrapper used by the generic adapter layer to notify
|
|
* the poller threads. Differently from netmap_rx_irq(), we check
|
|
* only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq.
|
|
*/
|
|
void
|
|
netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done)
|
|
{
|
|
if (unlikely(!nm_netmap_on(na)))
|
|
return;
|
|
|
|
netmap_common_irq(na, q, work_done);
|
|
#ifdef RATE_GENERIC
|
|
if (work_done)
|
|
rate_ctx.new.rxirq++;
|
|
else
|
|
rate_ctx.new.txirq++;
|
|
#endif /* RATE_GENERIC */
|
|
}
|
|
|
|
static int
|
|
generic_netmap_unregister(struct netmap_adapter *na)
|
|
{
|
|
struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
|
|
struct netmap_kring *kring = NULL;
|
|
int i, r;
|
|
|
|
if (na->active_fds == 0) {
|
|
na->na_flags &= ~NAF_NETMAP_ON;
|
|
|
|
/* Stop intercepting packets on the RX path. */
|
|
nm_os_catch_rx(gna, 0);
|
|
|
|
/* Release packet steering control. */
|
|
nm_os_catch_tx(gna, 0);
|
|
}
|
|
|
|
netmap_krings_mode_commit(na, /*onoff=*/0);
|
|
|
|
for_each_rx_kring(r, kring, na) {
|
|
/* Free the mbufs still pending in the RX queues,
|
|
* that did not end up into the corresponding netmap
|
|
* RX rings. */
|
|
mbq_safe_purge(&kring->rx_queue);
|
|
nm_os_mitigation_cleanup(&gna->mit[r]);
|
|
}
|
|
|
|
/* Decrement reference counter for the mbufs in the
|
|
* TX pools. These mbufs can be still pending in drivers,
|
|
* (e.g. this happens with virtio-net driver, which
|
|
* does lazy reclaiming of transmitted mbufs). */
|
|
for_each_tx_kring(r, kring, na) {
|
|
/* We must remove the destructor on the TX event,
|
|
* because the destructor invokes netmap code, and
|
|
* the netmap module may disappear before the
|
|
* TX event is consumed. */
|
|
mtx_lock_spin(&kring->tx_event_lock);
|
|
if (kring->tx_event) {
|
|
SET_MBUF_DESTRUCTOR(kring->tx_event, NULL);
|
|
}
|
|
kring->tx_event = NULL;
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
}
|
|
|
|
if (na->active_fds == 0) {
|
|
nm_os_free(gna->mit);
|
|
|
|
for_each_rx_kring(r, kring, na) {
|
|
mbq_safe_fini(&kring->rx_queue);
|
|
}
|
|
|
|
for_each_tx_kring(r, kring, na) {
|
|
callout_drain(&kring->tx_event_callout);
|
|
mtx_destroy(&kring->tx_event_lock);
|
|
if (kring->tx_pool == NULL) {
|
|
continue;
|
|
}
|
|
|
|
for (i=0; i<na->num_tx_desc; i++) {
|
|
if (kring->tx_pool[i]) {
|
|
m_freem(kring->tx_pool[i]);
|
|
}
|
|
}
|
|
nm_os_free(kring->tx_pool);
|
|
kring->tx_pool = NULL;
|
|
}
|
|
|
|
#ifdef RATE_GENERIC
|
|
if (--rate_ctx.refcount == 0) {
|
|
nm_prinf("del_timer()");
|
|
del_timer(&rate_ctx.timer);
|
|
}
|
|
#endif
|
|
nm_prinf("Emulated adapter for %s deactivated", na->name);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Enable/disable netmap mode for a generic network interface. */
|
|
static int
|
|
generic_netmap_register(struct netmap_adapter *na, int enable)
|
|
{
|
|
struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
|
|
struct netmap_kring *kring = NULL;
|
|
int error;
|
|
int i, r;
|
|
|
|
if (!na) {
|
|
return EINVAL;
|
|
}
|
|
|
|
if (!enable) {
|
|
/* This is actually an unregif. */
|
|
return generic_netmap_unregister(na);
|
|
}
|
|
|
|
if (na->active_fds == 0) {
|
|
nm_prinf("Emulated adapter for %s activated", na->name);
|
|
/* Do all memory allocations when (na->active_fds == 0), to
|
|
* simplify error management. */
|
|
|
|
/* Allocate memory for mitigation support on all the rx queues. */
|
|
gna->mit = nm_os_malloc(na->num_rx_rings * sizeof(struct nm_generic_mit));
|
|
if (!gna->mit) {
|
|
nm_prerr("mitigation allocation failed");
|
|
error = ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
for_each_rx_kring(r, kring, na) {
|
|
/* Init mitigation support. */
|
|
nm_os_mitigation_init(&gna->mit[r], r, na);
|
|
|
|
/* Initialize the rx queue, as generic_rx_handler() can
|
|
* be called as soon as nm_os_catch_rx() returns.
|
|
*/
|
|
mbq_safe_init(&kring->rx_queue);
|
|
}
|
|
|
|
/*
|
|
* Prepare mbuf pools (parallel to the tx rings), for packet
|
|
* transmission. Don't preallocate the mbufs here, it's simpler
|
|
* to leave this task to txsync.
|
|
*/
|
|
for_each_tx_kring(r, kring, na) {
|
|
kring->tx_pool = NULL;
|
|
}
|
|
for_each_tx_kring(r, kring, na) {
|
|
kring->tx_pool =
|
|
nm_os_malloc(na->num_tx_desc * sizeof(struct mbuf *));
|
|
if (!kring->tx_pool) {
|
|
nm_prerr("tx_pool allocation failed");
|
|
error = ENOMEM;
|
|
goto free_tx_pools;
|
|
}
|
|
mtx_init(&kring->tx_event_lock, "tx_event_lock",
|
|
NULL, MTX_SPIN);
|
|
callout_init_mtx(&kring->tx_event_callout,
|
|
&kring->tx_event_lock,
|
|
CALLOUT_RETURNUNLOCKED);
|
|
}
|
|
}
|
|
|
|
netmap_krings_mode_commit(na, /*onoff=*/1);
|
|
|
|
for_each_tx_kring(r, kring, na) {
|
|
/* Initialize tx_pool and tx_event. */
|
|
for (i=0; i<na->num_tx_desc; i++) {
|
|
kring->tx_pool[i] = NULL;
|
|
}
|
|
|
|
kring->tx_event = NULL;
|
|
}
|
|
|
|
if (na->active_fds == 0) {
|
|
/* Prepare to intercept incoming traffic. */
|
|
error = nm_os_catch_rx(gna, 1);
|
|
if (error) {
|
|
nm_prerr("nm_os_catch_rx(1) failed (%d)", error);
|
|
goto free_tx_pools;
|
|
}
|
|
|
|
/* Let netmap control the packet steering. */
|
|
error = nm_os_catch_tx(gna, 1);
|
|
if (error) {
|
|
nm_prerr("nm_os_catch_tx(1) failed (%d)", error);
|
|
goto catch_rx;
|
|
}
|
|
|
|
na->na_flags |= NAF_NETMAP_ON;
|
|
|
|
#ifdef RATE_GENERIC
|
|
if (rate_ctx.refcount == 0) {
|
|
nm_prinf("setup_timer()");
|
|
memset(&rate_ctx, 0, sizeof(rate_ctx));
|
|
setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx);
|
|
if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) {
|
|
nm_prerr("Error: mod_timer()");
|
|
}
|
|
}
|
|
rate_ctx.refcount++;
|
|
#endif /* RATE */
|
|
}
|
|
|
|
return 0;
|
|
|
|
/* Here (na->active_fds == 0) holds. */
|
|
catch_rx:
|
|
nm_os_catch_rx(gna, 0);
|
|
free_tx_pools:
|
|
for_each_tx_kring(r, kring, na) {
|
|
mtx_destroy(&kring->tx_event_lock);
|
|
if (kring->tx_pool == NULL) {
|
|
continue;
|
|
}
|
|
nm_os_free(kring->tx_pool);
|
|
kring->tx_pool = NULL;
|
|
}
|
|
for_each_rx_kring(r, kring, na) {
|
|
mbq_safe_fini(&kring->rx_queue);
|
|
}
|
|
nm_os_free(gna->mit);
|
|
out:
|
|
|
|
return error;
|
|
}
|
|
|
|
/*
|
|
* Callback invoked when the device driver frees an mbuf used
|
|
* by netmap to transmit a packet. This usually happens when
|
|
* the NIC notifies the driver that transmission is completed.
|
|
*/
|
|
static void
|
|
generic_mbuf_dtor(struct mbuf *m)
|
|
{
|
|
struct netmap_adapter *na = NA(GEN_TX_MBUF_IFP(m));
|
|
struct netmap_kring *kring;
|
|
unsigned int r = MBUF_TXQ(m);
|
|
unsigned int r_orig = r;
|
|
|
|
if (unlikely(!nm_netmap_on(na) || r >= na->num_tx_rings)) {
|
|
nm_prerr("Error: no netmap adapter on device %p",
|
|
GEN_TX_MBUF_IFP(m));
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* First, clear the event mbuf.
|
|
* In principle, the event 'm' should match the one stored
|
|
* on ring 'r'. However we check it explicitly to stay
|
|
* safe against lower layers (qdisc, driver, etc.) changing
|
|
* MBUF_TXQ(m) under our feet. If the match is not found
|
|
* on 'r', we try to see if it belongs to some other ring.
|
|
*/
|
|
for (;;) {
|
|
bool match = false;
|
|
|
|
kring = na->tx_rings[r];
|
|
mtx_lock_spin(&kring->tx_event_lock);
|
|
if (kring->tx_event == m) {
|
|
kring->tx_event = NULL;
|
|
match = true;
|
|
}
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
|
|
if (match) {
|
|
if (r != r_orig) {
|
|
nm_prlim(1, "event %p migrated: ring %u --> %u",
|
|
m, r_orig, r);
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (++r == na->num_tx_rings) r = 0;
|
|
|
|
if (r == r_orig) {
|
|
#ifndef __FreeBSD__
|
|
/*
|
|
* On FreeBSD this situation can arise if the tx_event
|
|
* callout handler cleared a stuck packet.
|
|
*/
|
|
nm_prlim(1, "Cannot match event %p", m);
|
|
#endif
|
|
nm_generic_mbuf_dtor(m);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* Second, wake up clients. They will reclaim the event through
|
|
* txsync. */
|
|
netmap_generic_irq(na, r, NULL);
|
|
nm_generic_mbuf_dtor(m);
|
|
}
|
|
|
|
/* Record completed transmissions and update hwtail.
|
|
*
|
|
* The oldest tx buffer not yet completed is at nr_hwtail + 1,
|
|
* nr_hwcur is the first unsent buffer.
|
|
*/
|
|
static u_int
|
|
generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc)
|
|
{
|
|
u_int const lim = kring->nkr_num_slots - 1;
|
|
u_int nm_i = nm_next(kring->nr_hwtail, lim);
|
|
u_int hwcur = kring->nr_hwcur;
|
|
u_int n = 0;
|
|
struct mbuf **tx_pool = kring->tx_pool;
|
|
|
|
nm_prdis("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail);
|
|
|
|
while (nm_i != hwcur) { /* buffers not completed */
|
|
struct mbuf *m = tx_pool[nm_i];
|
|
|
|
if (txqdisc) {
|
|
if (m == NULL) {
|
|
/* Nothing to do, this is going
|
|
* to be replenished. */
|
|
nm_prlim(3, "Is this happening?");
|
|
|
|
} else if (MBUF_QUEUED(m)) {
|
|
break; /* Not dequeued yet. */
|
|
|
|
} else if (MBUF_REFCNT(m) != 1) {
|
|
/* This mbuf has been dequeued but is still busy
|
|
* (refcount is 2).
|
|
* Leave it to the driver and replenish. */
|
|
m_freem(m);
|
|
tx_pool[nm_i] = NULL;
|
|
}
|
|
|
|
} else {
|
|
if (unlikely(m == NULL)) {
|
|
int event_consumed;
|
|
|
|
/* This slot was used to place an event. */
|
|
mtx_lock_spin(&kring->tx_event_lock);
|
|
event_consumed = (kring->tx_event == NULL);
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
if (!event_consumed) {
|
|
/* The event has not been consumed yet,
|
|
* still busy in the driver. */
|
|
break;
|
|
}
|
|
/* The event has been consumed, we can go
|
|
* ahead. */
|
|
} else if (MBUF_REFCNT(m) != 1) {
|
|
/* This mbuf is still busy: its refcnt is 2. */
|
|
break;
|
|
}
|
|
}
|
|
|
|
n++;
|
|
nm_i = nm_next(nm_i, lim);
|
|
}
|
|
kring->nr_hwtail = nm_prev(nm_i, lim);
|
|
nm_prdis("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail);
|
|
|
|
return n;
|
|
}
|
|
|
|
/* Compute a slot index in the middle between inf and sup. */
|
|
static inline u_int
|
|
ring_middle(u_int inf, u_int sup, u_int lim)
|
|
{
|
|
u_int n = lim + 1;
|
|
u_int e;
|
|
|
|
if (sup >= inf) {
|
|
e = (sup + inf) / 2;
|
|
} else { /* wrap around */
|
|
e = (sup + n + inf) / 2;
|
|
if (e >= n) {
|
|
e -= n;
|
|
}
|
|
}
|
|
|
|
if (unlikely(e >= n)) {
|
|
nm_prerr("This cannot happen");
|
|
e = 0;
|
|
}
|
|
|
|
return e;
|
|
}
|
|
|
|
#ifdef __FreeBSD__
|
|
static void
|
|
generic_tx_callout(void *arg)
|
|
{
|
|
struct netmap_kring *kring = arg;
|
|
|
|
kring->tx_event = NULL;
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
netmap_generic_irq(kring->na, kring->ring_id, NULL);
|
|
}
|
|
#endif
|
|
|
|
static void
|
|
generic_set_tx_event(struct netmap_kring *kring, u_int hwcur)
|
|
{
|
|
u_int lim = kring->nkr_num_slots - 1;
|
|
struct mbuf *m;
|
|
u_int e;
|
|
u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */
|
|
|
|
if (ntc == hwcur) {
|
|
return; /* all buffers are free */
|
|
}
|
|
|
|
/*
|
|
* We have pending packets in the driver between hwtail+1
|
|
* and hwcur, and we have to chose one of these slot to
|
|
* generate a notification.
|
|
* There is a race but this is only called within txsync which
|
|
* does a double check.
|
|
*/
|
|
#if 0
|
|
/* Choose a slot in the middle, so that we don't risk ending
|
|
* up in a situation where the client continuously wake up,
|
|
* fills one or a few TX slots and go to sleep again. */
|
|
e = ring_middle(ntc, hwcur, lim);
|
|
#else
|
|
/* Choose the first pending slot, to be safe against driver
|
|
* reordering mbuf transmissions. */
|
|
e = ntc;
|
|
#endif
|
|
|
|
m = kring->tx_pool[e];
|
|
if (m == NULL) {
|
|
/* An event is already in place. */
|
|
return;
|
|
}
|
|
|
|
mtx_lock_spin(&kring->tx_event_lock);
|
|
if (kring->tx_event) {
|
|
/* An event is already in place. */
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
return;
|
|
}
|
|
|
|
SET_MBUF_DESTRUCTOR(m, generic_mbuf_dtor);
|
|
kring->tx_event = m;
|
|
#ifdef __FreeBSD__
|
|
/*
|
|
* Handle the possibility that the transmitted buffer isn't reclaimed
|
|
* within a bounded period of time. This can arise when transmitting
|
|
* out of multiple ports via a lagg or bridge interface, since the
|
|
* member ports may legitimately only free transmitted buffers in
|
|
* batches.
|
|
*
|
|
* The callout handler clears the stuck packet from the ring, allowing
|
|
* transmission to proceed. In the common case we let
|
|
* generic_mbuf_dtor() unstick the ring, allowing mbufs to be
|
|
* reused most of the time.
|
|
*/
|
|
callout_reset_sbt_curcpu(&kring->tx_event_callout, SBT_1MS, 0,
|
|
generic_tx_callout, kring, 0);
|
|
#endif
|
|
mtx_unlock_spin(&kring->tx_event_lock);
|
|
|
|
kring->tx_pool[e] = NULL;
|
|
|
|
nm_prdis("Request Event at %d mbuf %p refcnt %d", e, m, m ? MBUF_REFCNT(m) : -2 );
|
|
|
|
/* Decrement the refcount. This will free it if we lose the race
|
|
* with the driver. */
|
|
m_freem(m);
|
|
}
|
|
|
|
/*
|
|
* generic_netmap_txsync() transforms netmap buffers into mbufs
|
|
* and passes them to the standard device driver
|
|
* (ndo_start_xmit() or ifp->if_transmit() ).
|
|
* On linux this is not done directly, but using dev_queue_xmit(),
|
|
* since it implements the TX flow control (and takes some locks).
|
|
*/
|
|
static int
|
|
generic_netmap_txsync(struct netmap_kring *kring, int flags)
|
|
{
|
|
struct netmap_adapter *na = kring->na;
|
|
struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
|
|
if_t ifp = na->ifp;
|
|
struct netmap_ring *ring = kring->ring;
|
|
u_int nm_i; /* index into the netmap ring */ // j
|
|
u_int const lim = kring->nkr_num_slots - 1;
|
|
u_int const head = kring->rhead;
|
|
u_int ring_nr = kring->ring_id;
|
|
|
|
IFRATE(rate_ctx.new.txsync++);
|
|
|
|
rmb();
|
|
|
|
/*
|
|
* First part: process new packets to send.
|
|
*/
|
|
nm_i = kring->nr_hwcur;
|
|
if (nm_i != head) { /* we have new packets to send */
|
|
struct nm_os_gen_arg a;
|
|
u_int event = -1;
|
|
#ifdef __FreeBSD__
|
|
struct epoch_tracker et;
|
|
|
|
NET_EPOCH_ENTER(et);
|
|
#endif
|
|
|
|
if (gna->txqdisc && nm_kr_txempty(kring)) {
|
|
/* In txqdisc mode, we ask for a delayed notification,
|
|
* but only when cur == hwtail, which means that the
|
|
* client is going to block. */
|
|
event = ring_middle(nm_i, head, lim);
|
|
nm_prdis("Place txqdisc event (hwcur=%u,event=%u,"
|
|
"head=%u,hwtail=%u)", nm_i, event, head,
|
|
kring->nr_hwtail);
|
|
}
|
|
|
|
a.ifp = ifp;
|
|
a.ring_nr = ring_nr;
|
|
a.head = a.tail = NULL;
|
|
|
|
while (nm_i != head) {
|
|
struct netmap_slot *slot = &ring->slot[nm_i];
|
|
u_int len = slot->len;
|
|
void *addr = NMB(na, slot);
|
|
/* device-specific */
|
|
struct mbuf *m;
|
|
int tx_ret;
|
|
|
|
NM_CHECK_ADDR_LEN(na, addr, len);
|
|
|
|
/* Tale a mbuf from the tx pool (replenishing the pool
|
|
* entry if necessary) and copy in the user packet. */
|
|
m = kring->tx_pool[nm_i];
|
|
if (unlikely(m == NULL)) {
|
|
kring->tx_pool[nm_i] = m =
|
|
nm_os_get_mbuf(ifp, NETMAP_BUF_SIZE(na));
|
|
if (m == NULL) {
|
|
nm_prlim(2, "Failed to replenish mbuf");
|
|
/* Here we could schedule a timer which
|
|
* retries to replenish after a while,
|
|
* and notifies the client when it
|
|
* manages to replenish some slots. In
|
|
* any case we break early to avoid
|
|
* crashes. */
|
|
break;
|
|
}
|
|
IFRATE(rate_ctx.new.txrepl++);
|
|
} else {
|
|
nm_os_mbuf_reinit(m);
|
|
}
|
|
|
|
a.m = m;
|
|
a.addr = addr;
|
|
a.len = len;
|
|
a.qevent = (nm_i == event);
|
|
/* When not in txqdisc mode, we should ask
|
|
* notifications when NS_REPORT is set, or roughly
|
|
* every half ring. To optimize this, we set a
|
|
* notification event when the client runs out of
|
|
* TX ring space, or when transmission fails. In
|
|
* the latter case we also break early.
|
|
*/
|
|
tx_ret = nm_os_generic_xmit_frame(&a);
|
|
if (unlikely(tx_ret)) {
|
|
if (!gna->txqdisc) {
|
|
/*
|
|
* No room for this mbuf in the device driver.
|
|
* Request a notification FOR A PREVIOUS MBUF,
|
|
* then call generic_netmap_tx_clean(kring) to do the
|
|
* double check and see if we can free more buffers.
|
|
* If there is space continue, else break;
|
|
* NOTE: the double check is necessary if the problem
|
|
* occurs in the txsync call after selrecord().
|
|
* Also, we need some way to tell the caller that not
|
|
* all buffers were queued onto the device (this was
|
|
* not a problem with native netmap driver where space
|
|
* is preallocated). The bridge has a similar problem
|
|
* and we solve it there by dropping the excess packets.
|
|
*/
|
|
generic_set_tx_event(kring, nm_i);
|
|
if (generic_netmap_tx_clean(kring, gna->txqdisc)) {
|
|
/* space now available */
|
|
continue;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* In txqdisc mode, the netmap-aware qdisc
|
|
* queue has the same length as the number of
|
|
* netmap slots (N). Since tail is advanced
|
|
* only when packets are dequeued, qdisc
|
|
* queue overrun cannot happen, so
|
|
* nm_os_generic_xmit_frame() did not fail
|
|
* because of that.
|
|
* However, packets can be dropped because
|
|
* carrier is off, or because our qdisc is
|
|
* being deactivated, or possibly for other
|
|
* reasons. In these cases, we just let the
|
|
* packet to be dropped. */
|
|
IFRATE(rate_ctx.new.txdrop++);
|
|
}
|
|
|
|
slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
|
|
nm_i = nm_next(nm_i, lim);
|
|
IFRATE(rate_ctx.new.txpkt++);
|
|
}
|
|
if (a.head != NULL) {
|
|
a.addr = NULL;
|
|
nm_os_generic_xmit_frame(&a);
|
|
}
|
|
/* Update hwcur to the next slot to transmit. Here nm_i
|
|
* is not necessarily head, we could break early. */
|
|
kring->nr_hwcur = nm_i;
|
|
|
|
#ifdef __FreeBSD__
|
|
NET_EPOCH_EXIT(et);
|
|
#endif
|
|
}
|
|
|
|
if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) {
|
|
/* No more available slots? Set a notification event
|
|
* on a netmap slot that will be cleaned in the future.
|
|
* No doublecheck is performed, since txsync() will be
|
|
* called twice by netmap_poll().
|
|
*/
|
|
generic_set_tx_event(kring, nm_i);
|
|
}
|
|
|
|
/*
|
|
* Second, reclaim completed buffers
|
|
*/
|
|
generic_netmap_tx_clean(kring, gna->txqdisc);
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/*
|
|
* This handler is registered (through nm_os_catch_rx())
|
|
* within the attached network interface
|
|
* in the RX subsystem, so that every mbuf passed up by
|
|
* the driver can be stolen to the network stack.
|
|
* Stolen packets are put in a queue where the
|
|
* generic_netmap_rxsync() callback can extract them.
|
|
* Returns 1 if the packet was stolen, 0 otherwise.
|
|
*/
|
|
int
|
|
generic_rx_handler(if_t ifp, struct mbuf *m)
|
|
{
|
|
struct netmap_adapter *na = NA(ifp);
|
|
struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
|
|
struct netmap_kring *kring;
|
|
u_int work_done;
|
|
u_int r = MBUF_RXQ(m); /* receive ring number */
|
|
|
|
if (r >= na->num_rx_rings) {
|
|
r = r % na->num_rx_rings;
|
|
}
|
|
|
|
kring = na->rx_rings[r];
|
|
|
|
if (kring->nr_mode == NKR_NETMAP_OFF) {
|
|
/* We must not intercept this mbuf. */
|
|
return 0;
|
|
}
|
|
|
|
/* limit the size of the queue */
|
|
if (unlikely(!gna->rxsg && MBUF_LEN(m) > NETMAP_BUF_SIZE(na))) {
|
|
/* This may happen when GRO/LRO features are enabled for
|
|
* the NIC driver when the generic adapter does not
|
|
* support RX scatter-gather. */
|
|
nm_prlim(2, "Warning: driver pushed up big packet "
|
|
"(size=%d)", (int)MBUF_LEN(m));
|
|
if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
|
|
m_freem(m);
|
|
} else if (unlikely(mbq_len(&kring->rx_queue) > na->num_rx_desc)) {
|
|
if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
|
|
m_freem(m);
|
|
} else {
|
|
mbq_safe_enqueue(&kring->rx_queue, m);
|
|
}
|
|
|
|
if (netmap_generic_mit < 32768) {
|
|
/* no rx mitigation, pass notification up */
|
|
netmap_generic_irq(na, r, &work_done);
|
|
} else {
|
|
/* same as send combining, filter notification if there is a
|
|
* pending timer, otherwise pass it up and start a timer.
|
|
*/
|
|
if (likely(nm_os_mitigation_active(&gna->mit[r]))) {
|
|
/* Record that there is some pending work. */
|
|
gna->mit[r].mit_pending = 1;
|
|
} else {
|
|
netmap_generic_irq(na, r, &work_done);
|
|
nm_os_mitigation_start(&gna->mit[r]);
|
|
}
|
|
}
|
|
|
|
/* We have intercepted the mbuf. */
|
|
return 1;
|
|
}
|
|
|
|
/*
|
|
* generic_netmap_rxsync() extracts mbufs from the queue filled by
|
|
* generic_netmap_rx_handler() and puts their content in the netmap
|
|
* receive ring.
|
|
* Access must be protected because the rx handler is asynchronous,
|
|
*/
|
|
static int
|
|
generic_netmap_rxsync(struct netmap_kring *kring, int flags)
|
|
{
|
|
struct netmap_ring *ring = kring->ring;
|
|
struct netmap_adapter *na = kring->na;
|
|
u_int nm_i; /* index into the netmap ring */ //j,
|
|
u_int n;
|
|
u_int const lim = kring->nkr_num_slots - 1;
|
|
u_int const head = kring->rhead;
|
|
int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
|
|
|
|
/* Adapter-specific variables. */
|
|
u_int nm_buf_len = NETMAP_BUF_SIZE(na);
|
|
struct mbq tmpq;
|
|
struct mbuf *m;
|
|
int avail; /* in bytes */
|
|
int mlen;
|
|
int copy;
|
|
|
|
if (head > lim)
|
|
return netmap_ring_reinit(kring);
|
|
|
|
IFRATE(rate_ctx.new.rxsync++);
|
|
|
|
/*
|
|
* First part: skip past packets that userspace has released.
|
|
* This can possibly make room for the second part.
|
|
*/
|
|
nm_i = kring->nr_hwcur;
|
|
if (nm_i != head) {
|
|
/* Userspace has released some packets. */
|
|
for (n = 0; nm_i != head; n++) {
|
|
struct netmap_slot *slot = &ring->slot[nm_i];
|
|
|
|
slot->flags &= ~NS_BUF_CHANGED;
|
|
nm_i = nm_next(nm_i, lim);
|
|
}
|
|
kring->nr_hwcur = head;
|
|
}
|
|
|
|
/*
|
|
* Second part: import newly received packets.
|
|
*/
|
|
if (!netmap_no_pendintr && !force_update) {
|
|
return 0;
|
|
}
|
|
|
|
nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */
|
|
|
|
/* Compute the available space (in bytes) in this netmap ring.
|
|
* The first slot that is not considered in is the one before
|
|
* nr_hwcur. */
|
|
|
|
avail = nm_prev(kring->nr_hwcur, lim) - nm_i;
|
|
if (avail < 0)
|
|
avail += lim + 1;
|
|
avail *= nm_buf_len;
|
|
|
|
/* First pass: While holding the lock on the RX mbuf queue,
|
|
* extract as many mbufs as they fit the available space,
|
|
* and put them in a temporary queue.
|
|
* To avoid performing a per-mbuf division (mlen / nm_buf_len) to
|
|
* to update avail, we do the update in a while loop that we
|
|
* also use to set the RX slots, but without performing the copy. */
|
|
mbq_init(&tmpq);
|
|
mbq_lock(&kring->rx_queue);
|
|
for (n = 0;; n++) {
|
|
m = mbq_peek(&kring->rx_queue);
|
|
if (!m) {
|
|
/* No more packets from the driver. */
|
|
break;
|
|
}
|
|
|
|
mlen = MBUF_LEN(m);
|
|
if (mlen > avail) {
|
|
/* No more space in the ring. */
|
|
break;
|
|
}
|
|
|
|
mbq_dequeue(&kring->rx_queue);
|
|
|
|
while (mlen) {
|
|
copy = nm_buf_len;
|
|
if (mlen < copy) {
|
|
copy = mlen;
|
|
}
|
|
mlen -= copy;
|
|
avail -= nm_buf_len;
|
|
|
|
ring->slot[nm_i].len = copy;
|
|
ring->slot[nm_i].flags = (mlen ? NS_MOREFRAG : 0);
|
|
nm_i = nm_next(nm_i, lim);
|
|
}
|
|
|
|
mbq_enqueue(&tmpq, m);
|
|
}
|
|
mbq_unlock(&kring->rx_queue);
|
|
|
|
/* Second pass: Drain the temporary queue, going over the used RX slots,
|
|
* and perform the copy out of the RX queue lock. */
|
|
nm_i = kring->nr_hwtail;
|
|
|
|
for (;;) {
|
|
void *nmaddr;
|
|
int ofs = 0;
|
|
int morefrag;
|
|
|
|
m = mbq_dequeue(&tmpq);
|
|
if (!m) {
|
|
break;
|
|
}
|
|
|
|
do {
|
|
nmaddr = NMB(na, &ring->slot[nm_i]);
|
|
/* We only check the address here on generic rx rings. */
|
|
if (nmaddr == NETMAP_BUF_BASE(na)) { /* Bad buffer */
|
|
m_freem(m);
|
|
mbq_purge(&tmpq);
|
|
mbq_fini(&tmpq);
|
|
return netmap_ring_reinit(kring);
|
|
}
|
|
|
|
copy = ring->slot[nm_i].len;
|
|
m_copydata(m, ofs, copy, nmaddr);
|
|
ofs += copy;
|
|
morefrag = ring->slot[nm_i].flags & NS_MOREFRAG;
|
|
nm_i = nm_next(nm_i, lim);
|
|
} while (morefrag);
|
|
|
|
m_freem(m);
|
|
}
|
|
|
|
mbq_fini(&tmpq);
|
|
|
|
if (n) {
|
|
kring->nr_hwtail = nm_i;
|
|
IFRATE(rate_ctx.new.rxpkt += n);
|
|
}
|
|
kring->nr_kflags &= ~NKR_PENDINTR;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
generic_netmap_dtor(struct netmap_adapter *na)
|
|
{
|
|
struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na;
|
|
if_t ifp = netmap_generic_getifp(gna);
|
|
struct netmap_adapter *prev_na = gna->prev;
|
|
|
|
if (prev_na != NULL) {
|
|
netmap_adapter_put(prev_na);
|
|
if (nm_iszombie(na)) {
|
|
/*
|
|
* The driver has been removed without releasing
|
|
* the reference so we need to do it here.
|
|
*/
|
|
netmap_adapter_put(prev_na);
|
|
}
|
|
nm_prinf("Native netmap adapter for %s restored", prev_na->name);
|
|
}
|
|
NM_RESTORE_NA(ifp, prev_na);
|
|
na->ifp = NULL;
|
|
nm_prinf("Emulated netmap adapter for %s destroyed", na->name);
|
|
}
|
|
|
|
int
|
|
na_is_generic(struct netmap_adapter *na)
|
|
{
|
|
return na->nm_register == generic_netmap_register;
|
|
}
|
|
|
|
/*
|
|
* generic_netmap_attach() makes it possible to use netmap on
|
|
* a device without native netmap support.
|
|
* This is less performant than native support but potentially
|
|
* faster than raw sockets or similar schemes.
|
|
*
|
|
* In this "emulated" mode, netmap rings do not necessarily
|
|
* have the same size as those in the NIC. We use a default
|
|
* value and possibly override it if the OS has ways to fetch the
|
|
* actual configuration.
|
|
*/
|
|
int
|
|
generic_netmap_attach(if_t ifp)
|
|
{
|
|
struct netmap_adapter *na;
|
|
struct netmap_generic_adapter *gna;
|
|
int retval;
|
|
u_int num_tx_desc, num_rx_desc;
|
|
|
|
#ifdef __FreeBSD__
|
|
if (if_gettype(ifp) == IFT_LOOP) {
|
|
nm_prerr("if_loop is not supported by %s", __func__);
|
|
return EINVAL;
|
|
}
|
|
#endif
|
|
|
|
if (NM_NA_CLASH(ifp)) {
|
|
/* If NA(ifp) is not null but there is no valid netmap
|
|
* adapter it means that someone else is using the same
|
|
* pointer (e.g. ax25_ptr on linux). This happens for
|
|
* instance when also PF_RING is in use. */
|
|
nm_prerr("Error: netmap adapter hook is busy");
|
|
return EBUSY;
|
|
}
|
|
|
|
num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */
|
|
|
|
nm_os_generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */
|
|
if (num_tx_desc == 0 || num_rx_desc == 0) {
|
|
nm_prerr("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc);
|
|
return EINVAL;
|
|
}
|
|
|
|
gna = nm_os_malloc(sizeof(*gna));
|
|
if (gna == NULL) {
|
|
nm_prerr("no memory on attach, give up");
|
|
return ENOMEM;
|
|
}
|
|
na = (struct netmap_adapter *)gna;
|
|
strlcpy(na->name, if_name(ifp), sizeof(na->name));
|
|
na->ifp = ifp;
|
|
na->num_tx_desc = num_tx_desc;
|
|
na->num_rx_desc = num_rx_desc;
|
|
na->rx_buf_maxsize = 32768;
|
|
na->nm_register = &generic_netmap_register;
|
|
na->nm_txsync = &generic_netmap_txsync;
|
|
na->nm_rxsync = &generic_netmap_rxsync;
|
|
na->nm_dtor = &generic_netmap_dtor;
|
|
/* when using generic, NAF_NETMAP_ON is set so we force
|
|
* NAF_SKIP_INTR to use the regular interrupt handler
|
|
*/
|
|
na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS;
|
|
|
|
nm_prdis("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)",
|
|
ifp->num_tx_queues, ifp->real_num_tx_queues,
|
|
ifp->tx_queue_len);
|
|
nm_prdis("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)",
|
|
ifp->num_rx_queues, ifp->real_num_rx_queues);
|
|
|
|
nm_os_generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings);
|
|
|
|
retval = netmap_attach_common(na);
|
|
if (retval) {
|
|
nm_os_free(gna);
|
|
return retval;
|
|
}
|
|
|
|
if (NM_NA_VALID(ifp)) {
|
|
gna->prev = NA(ifp); /* save old na */
|
|
netmap_adapter_get(gna->prev);
|
|
}
|
|
NM_ATTACH_NA(ifp, na);
|
|
|
|
nm_os_generic_set_features(gna);
|
|
|
|
nm_prinf("Emulated adapter for %s created (prev was %s)", na->name,
|
|
gna->prev ? gna->prev->name : "NULL");
|
|
|
|
return retval;
|
|
}
|