d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

netmap: align codebase to the current upstream (sha 8374e1a7e6941)

Browse Source 
Changelist:
    - Move large parts of VALE code to a new file and header netmap_bdg.[ch].
      This is useful to reuse the code within upcoming projects.
    - Improvements and bug fixes to pipes and monitors.
    - Introduce nm_os_onattach(), nm_os_onenter() and nm_os_onexit() to
      handle differences between FreeBSD and Linux.
    - Introduce some new helper functions to handle more host rings and fake
      rings (netmap_all_rings(), netmap_real_rings(), ...)
    - Added new sysctl to enable/disable hw checksum in emulated netmap mode.
    - nm_inject: add support for NS_MOREFRAG

Approved by:	gnn (mentor)
Differential Revision:	https://reviews.freebsd.org/D17364
This commit is contained in:
Vincenzo Maffione 2018-10-23 08:55:16 +00:00
parent 60b905ae2f
commit 2a7db7a63d
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=339639
13 changed files with 2772 additions and 2341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sys/conf/files
View File

@ -2522,6 +2522,7 @@ dev/netmap/netmap_pipe.c optional netmap
dev/netmap/netmap_pt.c optional netmap
dev/netmap/netmap_vale.c optional netmap
dev/netmap/netmap_legacy.c optional netmap
dev/netmap/netmap_bdg.c optional netmap
# compile-with "${NORMAL_C} -Wconversion -Wextra"
dev/nfsmb/nfsmb.c optional nfsmb pci
dev/nge/if_nge.c optional nge

125
sys/dev/netmap/netmap.c
View File

@ -521,6 +521,9 @@ int netmap_generic_txqdisc = 1;
int netmap_generic_ringsize = 1024;
int netmap_generic_rings = 1;
/* Non-zero to enable checksum offloading in NIC drivers */
int netmap_generic_hwcsum = 0;
/* Non-zero if ptnet devices are allowed to use virtio-net headers. */
int ptnet_vnet_hdr = 1;
@ -549,6 +552,9 @@ SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0,
SYSCTL_INT(_dev_netmap, OID_AUTO, admode, CTLFLAG_RW, &netmap_admode, 0,
"Adapter mode. 0 selects the best option available,"
"1 forces native adapter, 2 forces emulated adapter");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_hwcsum, CTLFLAG_RW, &netmap_generic_hwcsum,
0, "Hardware checksums. 0 to disable checksum generation by the NIC (default),"
"1 to enable checksum generation by the NIC");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_mit, CTLFLAG_RW, &netmap_generic_mit,
0, "RX notification interval in nanoseconds");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_ringsize, CTLFLAG_RW,
@ -827,8 +833,8 @@ netmap_krings_create(struct netmap_adapter *na, u_int tailroom)
}
/* account for the (possibly fake) host rings */
n[NR_TX] = na->num_tx_rings + 1;
n[NR_RX] = na->num_rx_rings + 1;
n[NR_TX] = netmap_all_rings(na, NR_TX);
n[NR_RX] = netmap_all_rings(na, NR_RX);
len = (n[NR_TX] + n[NR_RX]) *
(sizeof(struct netmap_kring) + sizeof(struct netmap_kring *))
@ -930,11 +936,14 @@ netmap_krings_delete(struct netmap_adapter *na)
void
netmap_hw_krings_delete(struct netmap_adapter *na)
{
struct mbq *q = &na->rx_rings[na->num_rx_rings]->rx_queue;
u_int lim = netmap_real_rings(na, NR_RX), i;
ND("destroy sw mbq with len %d", mbq_len(q));
mbq_purge(q);
mbq_safe_fini(q);
for (i = nma_get_nrings(na, NR_RX); i < lim; i++) {
struct mbq *q = &NMR(na, NR_RX)[i]->rx_queue;
ND("destroy sw mbq with len %d", mbq_len(q));
mbq_purge(q);
mbq_safe_fini(q);
}
netmap_krings_delete(na);
}
@ -1535,7 +1544,7 @@ netmap_get_na(struct nmreq_header *hdr,
goto out;
/* try to see if this is a bridge port */
error = netmap_get_bdg_na(hdr, na, nmd, create);
error = netmap_get_vale_na(hdr, na, nmd, create);
if (error)
goto out;
@ -1827,7 +1836,7 @@ netmap_interp_ringid(struct netmap_priv_d *priv, uint32_t nr_mode,
}
priv->np_qfirst[t] = (nr_mode == NR_REG_SW ?
nma_get_nrings(na, t) : 0);
priv->np_qlast[t] = nma_get_nrings(na, t) + 1;
priv->np_qlast[t] = netmap_all_rings(na, t);
ND("%s: %s %d %d", nr_mode == NR_REG_SW ? "SW" : "NIC+SW",
nm_txrx2str(t),
priv->np_qfirst[t], priv->np_qlast[t]);
@ -2543,7 +2552,7 @@ netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data,
NMG_LOCK();
hdr->nr_reqtype = NETMAP_REQ_REGISTER;
hdr->nr_body = (uintptr_t)&regreq;
error = netmap_get_bdg_na(hdr, &na, NULL, 0);
error = netmap_get_vale_na(hdr, &na, NULL, 0);
hdr->nr_reqtype = NETMAP_REQ_PORT_HDR_SET;
hdr->nr_body = (uintptr_t)req;
if (na && !error) {
@ -3336,6 +3345,12 @@ netmap_attach_common(struct netmap_adapter *na)
}
na->pdev = na; /* make sure netmap_mem_map() is called */
#endif /* __FreeBSD__ */
if (na->na_flags & NAF_HOST_RINGS) {
if (na->num_host_rx_rings == 0)
na->num_host_rx_rings = 1;
if (na->num_host_tx_rings == 0)
na->num_host_tx_rings = 1;
}
if (na->nm_krings_create == NULL) {
/* we assume that we have been called by a driver,
* since other port types all provide their own
@ -3357,7 +3372,7 @@ netmap_attach_common(struct netmap_adapter *na)
/* no special nm_bdg_attach callback. On VALE
* attach, we need to interpose a bwrap
*/
na->nm_bdg_attach = netmap_bwrap_attach;
na->nm_bdg_attach = netmap_default_bdg_attach;
#endif
return 0;
@ -3399,10 +3414,10 @@ netmap_hw_reg(struct netmap_adapter *na, int onoff)
static void
netmap_hw_dtor(struct netmap_adapter *na)
{
if (nm_iszombie(na) || na->ifp == NULL)
if (na->ifp == NULL)
return;
WNA(na->ifp) = NULL;
NM_DETACH_NA(na->ifp);
}
@ -3426,10 +3441,10 @@ netmap_attach_ext(struct netmap_adapter *arg, size_t size, int override_reg)
}
if (arg == NULL || arg->ifp == NULL)
goto fail;
return EINVAL;
ifp = arg->ifp;
if (NA(ifp) && !NM_NA_VALID(ifp)) {
if (NM_NA_CLASH(ifp)) {
/* If NA(ifp) is not null but there is no valid netmap
* adapter it means that someone else is using the same
* pointer (e.g. ax25_ptr on linux). This happens for
@ -3456,28 +3471,8 @@ netmap_attach_ext(struct netmap_adapter *arg, size_t size, int override_reg)
NM_ATTACH_NA(ifp, &hwna->up);
#ifdef linux
if (ifp->netdev_ops) {
/* prepare a clone of the netdev ops */
#ifndef NETMAP_LINUX_HAVE_NETDEV_OPS
hwna->nm_ndo.ndo_start_xmit = ifp->netdev_ops;
#else
hwna->nm_ndo = *ifp->netdev_ops;
#endif /* NETMAP_LINUX_HAVE_NETDEV_OPS */
}
hwna->nm_ndo.ndo_start_xmit = linux_netmap_start_xmit;
hwna->nm_ndo.ndo_change_mtu = linux_netmap_change_mtu;
if (ifp->ethtool_ops) {
hwna->nm_eto = *ifp->ethtool_ops;
}
hwna->nm_eto.set_ringparam = linux_netmap_set_ringparam;
#ifdef NETMAP_LINUX_HAVE_SET_CHANNELS
hwna->nm_eto.set_channels = linux_netmap_set_channels;
#endif /* NETMAP_LINUX_HAVE_SET_CHANNELS */
if (arg->nm_config == NULL) {
hwna->up.nm_config = netmap_linux_config;
}
#endif /* linux */
nm_os_onattach(ifp);
if (arg->nm_dtor == NULL) {
hwna->up.nm_dtor = netmap_hw_dtor;
}
@ -3545,7 +3540,10 @@ netmap_hw_krings_create(struct netmap_adapter *na)
int ret = netmap_krings_create(na, 0);
if (ret == 0) {
/* initialize the mbq for the sw rx ring */
mbq_safe_init(&na->rx_rings[na->num_rx_rings]->rx_queue);
u_int lim = netmap_real_rings(na, NR_RX), i;
for (i = na->num_rx_rings; i < lim; i++) {
mbq_safe_init(&NMR(na, NR_RX)[i]->rx_queue);
}
ND("initialized sw rx queue %d", na->num_rx_rings);
}
return ret;
@ -3608,8 +3606,14 @@ netmap_transmit(struct ifnet *ifp, struct mbuf *m)
unsigned int txr;
struct mbq *q;
int busy;
u_int i;
i = MBUF_TXQ(m);
if (i >= na->num_host_rx_rings) {
i = i % na->num_host_rx_rings;
}
kring = NMR(na, NR_RX)[nma_get_nrings(na, NR_RX) + i];
kring = na->rx_rings[na->num_rx_rings];
// XXX [Linux] we do not need this lock
// if we follow the down/configure/up protocol -gl
// mtx_lock(&na->core_lock);
@ -3639,8 +3643,15 @@ netmap_transmit(struct ifnet *ifp, struct mbuf *m)
goto done;
}
if (nm_os_mbuf_has_offld(m)) {
RD(1, "%s drop mbuf that needs offloadings", na->name);
if (!netmap_generic_hwcsum) {
if (nm_os_mbuf_has_csum_offld(m)) {
RD(1, "%s drop mbuf that needs checksum offload", na->name);
goto done;
}
}
if (nm_os_mbuf_has_seg_offld(m)) {
RD(1, "%s drop mbuf that needs generic segmentation offload", na->name);
goto done;
}
@ -3845,6 +3856,40 @@ netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done)
return netmap_common_irq(na, q, work_done);
}
/* set/clear native flags and if_transmit/netdev_ops */
void
nm_set_native_flags(struct netmap_adapter *na)
{
struct ifnet *ifp = na->ifp;
/* We do the setup for intercepting packets only if we are the
* first user of this adapapter. */
if (na->active_fds > 0) {
return;
}
na->na_flags |= NAF_NETMAP_ON;
nm_os_onenter(ifp);
nm_update_hostrings_mode(na);
}
void
nm_clear_native_flags(struct netmap_adapter *na)
{
struct ifnet *ifp = na->ifp;
/* We undo the setup for intercepting packets only if we are the
* last user of this adapapter. */
if (na->active_fds > 0) {
return;
}
nm_update_hostrings_mode(na);
nm_os_onexit(ifp);
na->na_flags &= ~NAF_NETMAP_ON;
}
/*
* Module loader and unloader

1827
sys/dev/netmap/netmap_bdg.c Normal file
View File

File diff suppressed because it is too large Load Diff

155
sys/dev/netmap/netmap_bdg.h Normal file
View File

@ -0,0 +1,155 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2013-2018 Universita` di Pisa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _NET_NETMAP_BDG_H_
#define _NET_NETMAP_BDG_H_
#if defined(__FreeBSD__)
#define BDG_RWLOCK_T struct rwlock // struct rwlock
#define BDG_RWINIT(b) \
rw_init_flags(&(b)->bdg_lock, "bdg lock", RW_NOWITNESS)
#define BDG_WLOCK(b) rw_wlock(&(b)->bdg_lock)
#define BDG_WUNLOCK(b) rw_wunlock(&(b)->bdg_lock)
#define BDG_RLOCK(b) rw_rlock(&(b)->bdg_lock)
#define BDG_RTRYLOCK(b) rw_try_rlock(&(b)->bdg_lock)
#define BDG_RUNLOCK(b) rw_runlock(&(b)->bdg_lock)
#define BDG_RWDESTROY(b) rw_destroy(&(b)->bdg_lock)
#endif /* __FreeBSD__ */
/* XXX Should go away after fixing find_bridge() - Michio */
#define NM_BDG_HASH 1024 /* forwarding table entries */
/* XXX revise this */
struct nm_hash_ent {
uint64_t mac; /* the top 2 bytes are the epoch */
uint64_t ports;
};
/* Default size for the Maximum Frame Size. */
#define NM_BDG_MFS_DEFAULT 1514
/*
* nm_bridge is a descriptor for a VALE switch.
* Interfaces for a bridge are all in bdg_ports[].
* The array has fixed size, an empty entry does not terminate
* the search, but lookups only occur on attach/detach so we
* don't mind if they are slow.
*
* The bridge is non blocking on the transmit ports: excess
* packets are dropped if there is no room on the output port.
*
* bdg_lock protects accesses to the bdg_ports array.
* This is a rw lock (or equivalent).
*/
#define NM_BDG_IFNAMSIZ IFNAMSIZ
struct nm_bridge {
/* XXX what is the proper alignment/layout ? */
BDG_RWLOCK_T bdg_lock; /* protects bdg_ports */
int bdg_namelen;
uint32_t bdg_active_ports;
char bdg_basename[NM_BDG_IFNAMSIZ];
/* Indexes of active ports (up to active_ports)
* and all other remaining ports.
*/
uint32_t bdg_port_index[NM_BDG_MAXPORTS];
/* used by netmap_bdg_detach_common() */
uint32_t tmp_bdg_port_index[NM_BDG_MAXPORTS];
struct netmap_vp_adapter *bdg_ports[NM_BDG_MAXPORTS];
/*
* Programmable lookup functions to figure out the destination port.
* It returns either of an index of the destination port,
* NM_BDG_BROADCAST to broadcast this packet, or NM_BDG_NOPORT not to
* forward this packet. ring_nr is the source ring index, and the
* function may overwrite this value to forward this packet to a
* different ring index.
* The function is set by netmap_bdg_regops().
*/
struct netmap_bdg_ops *bdg_ops;
/*
* Contains the data structure used by the bdg_ops.lookup function.
* By default points to *ht which is allocated on attach and used by the default lookup
* otherwise will point to the data structure received by netmap_bdg_regops().
*/
void *private_data;
struct nm_hash_ent *ht;
/* Currently used to specify if the bridge is still in use while empty and
* if it has been put in exclusive mode by an external module, see netmap_bdg_regops()
* and netmap_bdg_create().
*/
#define NM_BDG_ACTIVE 1
#define NM_BDG_EXCLUSIVE 2
uint8_t bdg_flags;
#ifdef CONFIG_NET_NS
struct net *ns;
#endif /* CONFIG_NET_NS */
};
static inline void *
nm_bdg_get_auth_token(struct nm_bridge *b)
{
return b->ht;
}
/* bridge not in exclusive mode ==> always valid
* bridge in exclusive mode (created through netmap_bdg_create()) ==> check authentication token
*/
static inline int
nm_bdg_valid_auth_token(struct nm_bridge *b, void *auth_token)
{
return !(b->bdg_flags & NM_BDG_EXCLUSIVE) || b->ht == auth_token;
}
int netmap_get_bdg_na(struct nmreq_header *hdr, struct netmap_adapter **na,
struct netmap_mem_d *nmd, int create, struct netmap_bdg_ops *ops);
struct nm_bridge *nm_find_bridge(const char *name, int create, struct netmap_bdg_ops *ops);
int netmap_bdg_free(struct nm_bridge *b);
void netmap_bdg_detach_common(struct nm_bridge *b, int hw, int sw);
int netmap_vp_bdg_ctl(struct nmreq_header *hdr, struct netmap_adapter *na);
int netmap_vp_reg(struct netmap_adapter *na, int onoff);
int netmap_bwrap_reg(struct netmap_adapter *, int onoff);
int netmap_vp_reg(struct netmap_adapter *na, int onoff);
int netmap_vp_rxsync(struct netmap_kring *kring, int flags);
int netmap_bwrap_notify(struct netmap_kring *kring, int flags);
int netmap_bwrap_attach_common(struct netmap_adapter *na,
struct netmap_adapter *hwna);
int netmap_bwrap_krings_create_common(struct netmap_adapter *na);
void netmap_bwrap_krings_delete_common(struct netmap_adapter *na);
#define NM_NEED_BWRAP (-2)
#endif /* _NET_NETMAP_BDG_H_ */

38
sys/dev/netmap/netmap_freebsd.c
View File

@ -270,11 +270,17 @@ nm_os_send_up(struct ifnet *ifp, struct mbuf *m, struct mbuf *prev)
}
int
nm_os_mbuf_has_offld(struct mbuf *m)
nm_os_mbuf_has_csum_offld(struct mbuf *m)
{
return m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_SCTP |
CSUM_TCP_IPV6 | CSUM_UDP_IPV6 |
CSUM_SCTP_IPV6 | CSUM_TSO);
CSUM_SCTP_IPV6);
}
int
nm_os_mbuf_has_seg_offld(struct mbuf *m)
{
return m->m_pkthdr.csum_flags & CSUM_TSO;
}
static void
@ -632,7 +638,7 @@ struct nm_os_extmem {
void
nm_os_extmem_delete(struct nm_os_extmem *e)
{
D("freeing %jx bytes", (uintmax_t)e->size);
D("freeing %zx bytes", (size_t)e->size);
vm_map_remove(kernel_map, e->kva, e->kva + e->size);
nm_os_free(e);
}
@ -701,7 +707,7 @@ nm_os_extmem_create(unsigned long p, struct nmreq_pools_info *pi, int *perror)
VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE,
VM_PROT_READ | VM_PROT_WRITE, 0);
if (rv != KERN_SUCCESS) {
D("vm_map_find(%jx) failed", (uintmax_t)e->size);
D("vm_map_find(%zx) failed", (size_t)e->size);
goto out_rel;
}
rv = vm_map_wire(kernel_map, e->kva, e->kva + e->size,
@ -1540,6 +1546,30 @@ freebsd_netmap_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data,
return error;
}
void
nm_os_onattach(struct ifnet *ifp)
{
}
void
nm_os_onenter(struct ifnet *ifp)
{
struct netmap_adapter *na = NA(ifp);
na->if_transmit = ifp->if_transmit;
ifp->if_transmit = netmap_transmit;
ifp->if_capenable |= IFCAP_NETMAP;
}
void
nm_os_onexit(struct ifnet *ifp)
{
struct netmap_adapter *na = NA(ifp);
ifp->if_transmit = na->if_transmit;
ifp->if_capenable &= ~IFCAP_NETMAP;
}
extern struct cdevsw netmap_cdevsw; /* XXX used in netmap.c, should go elsewhere */
struct cdevsw netmap_cdevsw = {
.d_version = D_VERSION,

120
sys/dev/netmap/netmap_generic.c
View File

@ -89,117 +89,6 @@ __FBSDID("$FreeBSD$");
#define MBUF_RXQ(m) ((m)->m_pkthdr.flowid)
#define smp_mb()
/*
* FreeBSD mbuf allocator/deallocator in emulation mode:
*/
#if __FreeBSD_version < 1100000
/*
* For older versions of FreeBSD:
*
* We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE
* so that the destructor, if invoked, will not free the packet.
* In principle we should set the destructor only on demand,
* but since there might be a race we better do it on allocation.
* As a consequence, we also need to set the destructor or we
* would leak buffers.
*/
/* mbuf destructor, also need to change the type to EXT_EXTREF,
* add an M_NOFREE flag, and then clear the flag and
* chain into uma_zfree(zone_pack, mf)
* (or reinstall the buffer ?)
*/
#define SET_MBUF_DESTRUCTOR(m, fn) do { \
(m)->m_ext.ext_free = (void *)fn; \
(m)->m_ext.ext_type = EXT_EXTREF; \
} while (0)
static int
void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2)
{
/* restore original mbuf */
m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1;
m->m_ext.ext_arg1 = NULL;
m->m_ext.ext_type = EXT_PACKET;
m->m_ext.ext_free = NULL;
if (MBUF_REFCNT(m) == 0)
SET_MBUF_REFCNT(m, 1);
uma_zfree(zone_pack, m);
return 0;
}
static inline struct mbuf *
nm_os_get_mbuf(struct ifnet *ifp, int len)
{
struct mbuf *m;
(void)ifp;
m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (m) {
/* m_getcl() (mb_ctor_mbuf) has an assert that checks that
* M_NOFREE flag is not specified as third argument,
* so we have to set M_NOFREE after m_getcl(). */
m->m_flags |= M_NOFREE;
m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save
m->m_ext.ext_free = (void *)void_mbuf_dtor;
m->m_ext.ext_type = EXT_EXTREF;
ND(5, "create m %p refcnt %d", m, MBUF_REFCNT(m));
}
return m;
}
#else /* __FreeBSD_version >= 1100000 */
/*
* Newer versions of FreeBSD, using a straightforward scheme.
*
* We allocate mbufs with m_gethdr(), since the mbuf header is needed
* by the driver. We also attach a customly-provided external storage,
* which in this case is a netmap buffer. When calling m_extadd(), however
* we pass a NULL address, since the real address (and length) will be
* filled in by nm_os_generic_xmit_frame() right before calling
* if_transmit().
*
* The dtor function does nothing, however we need it since mb_free_ext()
* has a KASSERT(), checking that the mbuf dtor function is not NULL.
*/
#if __FreeBSD_version <= 1200050
static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { }
#else /* __FreeBSD_version >= 1200051 */
/* The arg1 and arg2 pointers argument were removed by r324446, which
* in included since version 1200051. */
static void void_mbuf_dtor(struct mbuf *m) { }
#endif /* __FreeBSD_version >= 1200051 */
#define SET_MBUF_DESTRUCTOR(m, fn) do { \
(m)->m_ext.ext_free = (fn != NULL) ? \
(void *)fn : (void *)void_mbuf_dtor; \
} while (0)
static inline struct mbuf *
nm_os_get_mbuf(struct ifnet *ifp, int len)
{
struct mbuf *m;
(void)ifp;
(void)len;
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL) {
return m;
}
m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor,
NULL, NULL, 0, EXT_NET_DRV);
return m;
}
#endif /* __FreeBSD_version >= 1100000 */
#elif defined _WIN32
#include "win_glue.h"
@ -1161,7 +1050,7 @@ generic_netmap_dtor(struct netmap_adapter *na)
}
D("Native netmap adapter %p restored", prev_na);
}
NM_ATTACH_NA(ifp, prev_na);
NM_RESTORE_NA(ifp, prev_na);
/*
* netmap_detach_common(), that it's called after this function,
* overrides WNA(ifp) if na->ifp is not NULL.
@ -1202,7 +1091,7 @@ generic_netmap_attach(struct ifnet *ifp)
}
#endif
if (NA(ifp) && !NM_NA_VALID(ifp)) {
if (NM_NA_CLASH(ifp)) {
/* If NA(ifp) is not null but there is no valid netmap
* adapter it means that someone else is using the same
* pointer (e.g. ax25_ptr on linux). This happens for
@ -1230,6 +1119,7 @@ generic_netmap_attach(struct ifnet *ifp)
na->ifp = ifp;
na->num_tx_desc = num_tx_desc;
na->num_rx_desc = num_rx_desc;
na->rx_buf_maxsize = 32768;
na->nm_register = &generic_netmap_register;
na->nm_txsync = &generic_netmap_txsync;
na->nm_rxsync = &generic_netmap_rxsync;
@ -1253,8 +1143,8 @@ generic_netmap_attach(struct ifnet *ifp)
return retval;
}
gna->prev = NA(ifp); /* save old na */
if (gna->prev != NULL) {
if (NM_NA_VALID(ifp)) {
gna->prev = NA(ifp); /* save old na */
netmap_adapter_get(gna->prev);
}
NM_ATTACH_NA(ifp, na);

327
sys/dev/netmap/netmap_kern.h
View File

@ -275,6 +275,7 @@ struct netmap_adapter;
struct nm_bdg_fwd;
struct nm_bridge;
struct netmap_priv_d;
struct nm_bdg_args;
/* os-specific NM_SELINFO_T initialzation/destruction functions */
void nm_os_selinfo_init(NM_SELINFO_T *);
@ -305,6 +306,12 @@ void *nm_os_realloc(void *, size_t new_size, size_t old_size);
void nm_os_free(void *);
void nm_os_vfree(void *);
/* os specific attach/detach enter/exit-netmap-mode routines */
void nm_os_onattach(struct ifnet *);
void nm_os_ondetach(struct ifnet *);
void nm_os_onenter(struct ifnet *);
void nm_os_onexit(struct ifnet *);
/* passes a packet up to the host stack.
* If the packet is sent (or dropped) immediately it returns NULL,
* otherwise it links the packet to prev and returns m.
@ -313,7 +320,8 @@ void nm_os_vfree(void *);
*/
void *nm_os_send_up(struct ifnet *, struct mbuf *m, struct mbuf *prev);
int nm_os_mbuf_has_offld(struct mbuf *m);
int nm_os_mbuf_has_seg_offld(struct mbuf *m);
int nm_os_mbuf_has_csum_offld(struct mbuf *m);
#include "netmap_mbq.h"
@ -507,11 +515,10 @@ struct netmap_kring {
struct netmap_kring *pipe; /* if this is a pipe ring,
* pointer to the other end
*/
uint32_t pipe_tail; /* hwtail updated by the other end */
#endif /* WITH_PIPES */
#ifdef WITH_VALE
int (*save_notify)(struct netmap_kring *kring, int flags);
#endif
#ifdef WITH_MONITOR
/* array of krings that are monitoring this kring */
@ -634,6 +641,7 @@ struct netmap_lut {
};
struct netmap_vp_adapter; // forward
struct nm_bridge;
/* Struct to be filled by nm_config callbacks. */
struct nm_config_info {
@ -644,6 +652,14 @@ struct nm_config_info {
unsigned rx_buf_maxsize;
};
/*
* default type for the magic field.
* May be overriden in glue code.
*/
#ifndef NM_OS_MAGIC
#define NM_OS_MAGIC uint32_t
#endif /* !NM_OS_MAGIC */
/*
* The "struct netmap_adapter" extends the "struct adapter"
* (or equivalent) device descriptor.
@ -660,7 +676,7 @@ struct netmap_adapter {
* always exists and is at least 32 bits) contains a magic
* value which we can use to detect that the interface is good.
*/
uint32_t magic;
NM_OS_MAGIC magic;
uint32_t na_flags; /* enabled, and other flags */
#define NAF_SKIP_INTR 1 /* use the regular interrupt handler.
* useful during initialization
@ -696,6 +712,8 @@ struct netmap_adapter {
u_int num_rx_rings; /* number of adapter receive rings */
u_int num_tx_rings; /* number of adapter transmit rings */
u_int num_host_rx_rings; /* number of host receive rings */
u_int num_host_tx_rings; /* number of host transmit rings */
u_int num_tx_desc; /* number of descriptor in each queue */
u_int num_rx_desc;
@ -783,7 +801,6 @@ struct netmap_adapter {
int (*nm_config)(struct netmap_adapter *, struct nm_config_info *info);
int (*nm_krings_create)(struct netmap_adapter *);
void (*nm_krings_delete)(struct netmap_adapter *);
#ifdef WITH_VALE
/*
* nm_bdg_attach() initializes the na_vp field to point
* to an adapter that can be attached to a VALE switch. If the
@ -799,7 +816,8 @@ struct netmap_adapter {
* initializations
* Called with NMG_LOCK held.
*/
int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *);
int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *,
struct nm_bridge *);
int (*nm_bdg_ctl)(struct nmreq_header *, struct netmap_adapter *);
/* adapter used to attach this adapter to a VALE switch (if any) */
@ -807,7 +825,6 @@ struct netmap_adapter {
/* adapter used to attach the host rings of this adapter
* to a VALE switch (if any) */
struct netmap_vp_adapter *na_hostvp;
#endif
/* standard refcount to control the lifetime of the adapter
* (it should be equal to the lifetime of the corresponding ifp)
@ -843,6 +860,10 @@ struct netmap_adapter {
unsigned rx_buf_maxsize;
char name[NETMAP_REQ_IFNAMSIZ]; /* used at least by pipes */
#ifdef WITH_MONITOR
unsigned long monitor_id; /* debugging */
#endif
};
static __inline u_int
@ -866,6 +887,12 @@ nma_get_nrings(struct netmap_adapter *na, enum txrx t)
return (t == NR_TX ? na->num_tx_rings : na->num_rx_rings);
}
static __inline u_int
nma_get_host_nrings(struct netmap_adapter *na, enum txrx t)
{
return (t == NR_TX ? na->num_host_tx_rings : na->num_host_rx_rings);
}
static __inline void
nma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v)
{
@ -875,6 +902,15 @@ nma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v)
na->num_rx_rings = v;
}
static __inline void
nma_set_host_nrings(struct netmap_adapter *na, enum txrx t, u_int v)
{
if (t == NR_TX)
na->num_host_tx_rings = v;
else
na->num_host_rx_rings = v;
}
static __inline struct netmap_kring**
NMR(struct netmap_adapter *na, enum txrx t)
{
@ -964,13 +1000,22 @@ struct netmap_generic_adapter { /* emulated device */
};
#endif /* WITH_GENERIC */
static __inline int
static __inline u_int
netmap_real_rings(struct netmap_adapter *na, enum txrx t)
{
return nma_get_nrings(na, t) + !!(na->na_flags & NAF_HOST_RINGS);
return nma_get_nrings(na, t) +
!!(na->na_flags & NAF_HOST_RINGS) * nma_get_host_nrings(na, t);
}
#ifdef WITH_VALE
/* account for fake rings */
static __inline u_int
netmap_all_rings(struct netmap_adapter *na, enum txrx t)
{
return max(nma_get_nrings(na, t) + 1, netmap_real_rings(na, t));
}
int netmap_default_bdg_attach(const char *name, struct netmap_adapter *na,
struct nm_bridge *);
struct nm_bdg_polling_state;
/*
* Bridge wrapper for non VALE ports attached to a VALE switch.
@ -1038,12 +1083,12 @@ struct netmap_bwrap_adapter {
int nm_bdg_ctl_attach(struct nmreq_header *hdr, void *auth_token);
int nm_bdg_ctl_detach(struct nmreq_header *hdr, void *auth_token);
int nm_bdg_polling(struct nmreq_header *hdr);
int netmap_bwrap_attach(const char *name, struct netmap_adapter *);
int netmap_bdg_list(struct nmreq_header *hdr);
#ifdef WITH_VALE
int netmap_vi_create(struct nmreq_header *hdr, int);
int nm_vi_create(struct nmreq_header *);
int nm_vi_destroy(const char *name);
int netmap_bdg_list(struct nmreq_header *hdr);
#else /* !WITH_VALE */
#define netmap_vi_create(hdr, a) (EOPNOTSUPP)
#endif /* WITH_VALE */
@ -1262,7 +1307,6 @@ const char *netmap_bdg_name(struct netmap_vp_adapter *);
#define netmap_ifp_to_vp(_ifp) NULL
#define netmap_ifp_to_host_vp(_ifp) NULL
#define netmap_bdg_idx(_vp) -1
#define netmap_bdg_name(_vp) NULL
#endif /* WITH_VALE */
static inline int
@ -1293,67 +1337,8 @@ nm_update_hostrings_mode(struct netmap_adapter *na)
na->rx_rings[na->num_rx_rings]->nr_pending_mode;
}
/* set/clear native flags and if_transmit/netdev_ops */
static inline void
nm_set_native_flags(struct netmap_adapter *na)
{
struct ifnet *ifp = na->ifp;
/* We do the setup for intercepting packets only if we are the
* first user of this adapapter. */
if (na->active_fds > 0) {
return;
}
na->na_flags |= NAF_NETMAP_ON;
#ifdef IFCAP_NETMAP /* or FreeBSD ? */
ifp->if_capenable |= IFCAP_NETMAP;
#endif
#if defined (__FreeBSD__)
na->if_transmit = ifp->if_transmit;
ifp->if_transmit = netmap_transmit;
#elif defined (_WIN32)
(void)ifp; /* prevent a warning */
#elif defined (linux)
na->if_transmit = (void *)ifp->netdev_ops;
ifp->netdev_ops = &((struct netmap_hw_adapter *)na)->nm_ndo;
((struct netmap_hw_adapter *)na)->save_ethtool = ifp->ethtool_ops;
ifp->ethtool_ops = &((struct netmap_hw_adapter*)na)->nm_eto;
#endif /* linux */
nm_update_hostrings_mode(na);
}
static inline void
nm_clear_native_flags(struct netmap_adapter *na)
{
struct ifnet *ifp = na->ifp;
/* We undo the setup for intercepting packets only if we are the
* last user of this adapapter. */
if (na->active_fds > 0) {
return;
}
nm_update_hostrings_mode(na);
#if defined(__FreeBSD__)
ifp->if_transmit = na->if_transmit;
#elif defined(_WIN32)
(void)ifp; /* prevent a warning */
#else
ifp->netdev_ops = (void *)na->if_transmit;
ifp->ethtool_ops = ((struct netmap_hw_adapter*)na)->save_ethtool;
#endif
na->na_flags &= ~NAF_NETMAP_ON;
#ifdef IFCAP_NETMAP /* or FreeBSD ? */
ifp->if_capenable &= ~IFCAP_NETMAP;
#endif
}
#ifdef linux
int netmap_linux_config(struct netmap_adapter *na,
struct nm_config_info *info);
#endif /* linux */
void nm_set_native_flags(struct netmap_adapter *);
void nm_clear_native_flags(struct netmap_adapter *);
/*
* nm_*sync_prologue() functions are used in ioctl/poll and ptnetmap
@ -1458,7 +1443,6 @@ int netmap_get_hw_na(struct ifnet *ifp,
struct netmap_mem_d *nmd, struct netmap_adapter **na);
#ifdef WITH_VALE
/*
* The following bridge-related functions are used by other
* kernel modules.
@ -1473,39 +1457,49 @@ typedef uint32_t (*bdg_lookup_fn_t)(struct nm_bdg_fwd *ft, uint8_t *ring_nr,
typedef int (*bdg_config_fn_t)(struct nm_ifreq *);
typedef void (*bdg_dtor_fn_t)(const struct netmap_vp_adapter *);
typedef void *(*bdg_update_private_data_fn_t)(void *private_data, void *callback_data, int *error);
typedef int (*bdg_vp_create_fn_t)(struct nmreq_header *hdr,
struct ifnet *ifp, struct netmap_mem_d *nmd,
struct netmap_vp_adapter **ret);
typedef int (*bdg_bwrap_attach_fn_t)(const char *nr_name, struct netmap_adapter *hwna);
struct netmap_bdg_ops {
bdg_lookup_fn_t lookup;
bdg_config_fn_t config;
bdg_dtor_fn_t dtor;
bdg_vp_create_fn_t vp_create;
bdg_bwrap_attach_fn_t bwrap_attach;
char name[IFNAMSIZ];
};
uint32_t netmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring,
struct netmap_vp_adapter *, void *private_data);
int netmap_bwrap_attach(const char *name, struct netmap_adapter *, struct netmap_bdg_ops *);
int netmap_bdg_regops(const char *name, struct netmap_bdg_ops *bdg_ops, void *private_data, void *auth_token);
#define NM_BRIDGES 8 /* number of bridges */
#define NM_BDG_MAXPORTS 254 /* up to 254 */
#define NM_BDG_BROADCAST NM_BDG_MAXPORTS
#define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1)
/* these are redefined in case of no VALE support */
int netmap_get_bdg_na(struct nmreq_header *hdr, struct netmap_adapter **na,
struct netmap_mem_d *nmd, int create);
struct nm_bridge *netmap_init_bridges2(u_int);
void netmap_uninit_bridges2(struct nm_bridge *, u_int);
int netmap_init_bridges(void);
void netmap_uninit_bridges(void);
int netmap_bdg_regops(const char *name, struct netmap_bdg_ops *bdg_ops, void *private_data, void *auth_token);
int nm_bdg_update_private_data(const char *name, bdg_update_private_data_fn_t callback,
void *callback_data, void *auth_token);
int netmap_bdg_config(struct nm_ifreq *nifr);
void *netmap_bdg_create(const char *bdg_name, int *return_status);
int netmap_bdg_destroy(const char *bdg_name, void *auth_token);
#ifdef WITH_VALE
uint32_t netmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring,
struct netmap_vp_adapter *, void *private_data);
/* these are redefined in case of no VALE support */
int netmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na,
struct netmap_mem_d *nmd, int create);
void *netmap_vale_create(const char *bdg_name, int *return_status);
int netmap_vale_destroy(const char *bdg_name, void *auth_token);
#else /* !WITH_VALE */
#define netmap_get_bdg_na(_1, _2, _3, _4) 0
#define netmap_init_bridges(_1) 0
#define netmap_uninit_bridges()
#define netmap_bdg_regops(_1, _2) EINVAL
#define netmap_bdg_learning(_1, _2, _3, _4) 0
#define netmap_get_vale_na(_1, _2, _3, _4) 0
#define netmap_bdg_create(_1, _2) NULL
#define netmap_bdg_destroy(_1, _2) 0
#endif /* !WITH_VALE */
#ifdef WITH_PIPES
@ -1611,6 +1605,7 @@ enum { /* verbose flags */
extern int netmap_txsync_retry;
extern int netmap_flags;
extern int netmap_generic_hwcsum;
extern int netmap_generic_mit;
extern int netmap_generic_ringsize;
extern int netmap_generic_rings;
@ -1620,11 +1615,17 @@ extern int netmap_generic_txqdisc;
extern int ptnetmap_tx_workers;
/*
* NA returns a pointer to the struct netmap adapter from the ifp,
* WNA is used to write it.
* NA returns a pointer to the struct netmap adapter from the ifp.
* WNA is os-specific and must be defined in glue code.
*/
#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp))
/*
* we provide a default implementation of NM_ATTACH_NA/NM_DETACH_NA
* based on the WNA field.
* Glue code may override this by defining its own NM_ATTACH_NA
*/
#ifndef NM_ATTACH_NA
/*
* On old versions of FreeBSD, NA(ifp) is a pspare. On linux we
* overload another pointer in the netdev.
@ -1643,6 +1644,12 @@ extern int ptnetmap_tx_workers;
NA(ifp)->magic = \
((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC; \
} while(0)
#define NM_RESTORE_NA(ifp, na) WNA(ifp) = na;
#define NM_DETACH_NA(ifp) do { WNA(ifp) = NULL; } while (0)
#define NM_NA_CLASH(ifp) (NA(ifp) && !NM_NA_VALID(ifp))
#endif /* !NM_ATTACH_NA */
#define NM_IS_NATIVE(ifp) (NM_NA_VALID(ifp) && NA(ifp)->nm_dtor == netmap_hw_dtor)
@ -1752,17 +1759,24 @@ netmap_unload_map(struct netmap_adapter *na,
}
}
#ifdef NETMAP_LINUX_HAVE_DMASYNC
static inline void
netmap_sync_map(struct netmap_adapter *na,
netmap_sync_map_cpu(struct netmap_adapter *na,
bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t)
{
if (*map) {
if (t == NR_RX)
dma_sync_single_for_cpu(na->pdev, *map, sz,
DMA_FROM_DEVICE);
else
dma_sync_single_for_device(na->pdev, *map, sz,
DMA_TO_DEVICE);
dma_sync_single_for_cpu(na->pdev, *map, sz,
(t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE));
}
}
static inline void
netmap_sync_map_dev(struct netmap_adapter *na,
bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t)
{
if (*map) {
dma_sync_single_for_device(na->pdev, *map, sz,
(t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE));
}
}
@ -1780,6 +1794,10 @@ netmap_reload_map(struct netmap_adapter *na,
*map = dma_map_single(na->pdev, buf, sz,
DMA_BIDIRECTIONAL);
}
#else /* !NETMAP_LINUX_HAVE_DMASYNC */
#define netmap_sync_map_cpu(na, tag, map, sz, t)
#define netmap_sync_map_dev(na, tag, map, sz, t)
#endif /* NETMAP_LINUX_HAVE_DMASYNC */
#endif /* linux */
@ -2221,6 +2239,119 @@ void ptnet_nm_krings_delete(struct netmap_adapter *na);
void ptnet_nm_dtor(struct netmap_adapter *na);
#endif /* WITH_PTNETMAP_GUEST */
#ifdef __FreeBSD__
/*
* FreeBSD mbuf allocator/deallocator in emulation mode:
*/
#if __FreeBSD_version < 1100000
/*
* For older versions of FreeBSD:
*
* We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE
* so that the destructor, if invoked, will not free the packet.
* In principle we should set the destructor only on demand,
* but since there might be a race we better do it on allocation.
* As a consequence, we also need to set the destructor or we
* would leak buffers.
*/
/* mbuf destructor, also need to change the type to EXT_EXTREF,
* add an M_NOFREE flag, and then clear the flag and
* chain into uma_zfree(zone_pack, mf)
* (or reinstall the buffer ?)
*/
#define SET_MBUF_DESTRUCTOR(m, fn) do { \
(m)->m_ext.ext_free = (void *)fn; \
(m)->m_ext.ext_type = EXT_EXTREF; \
} while (0)
static int
void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2)
{
/* restore original mbuf */
m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1;
m->m_ext.ext_arg1 = NULL;
m->m_ext.ext_type = EXT_PACKET;
m->m_ext.ext_free = NULL;
if (MBUF_REFCNT(m) == 0)
SET_MBUF_REFCNT(m, 1);
uma_zfree(zone_pack, m);
return 0;
}
static inline struct mbuf *
nm_os_get_mbuf(struct ifnet *ifp, int len)
{
struct mbuf *m;
(void)ifp;
m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
if (m) {
/* m_getcl() (mb_ctor_mbuf) has an assert that checks that
* M_NOFREE flag is not specified as third argument,
* so we have to set M_NOFREE after m_getcl(). */
m->m_flags |= M_NOFREE;
m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save
m->m_ext.ext_free = (void *)void_mbuf_dtor;
m->m_ext.ext_type = EXT_EXTREF;
ND(5, "create m %p refcnt %d", m, MBUF_REFCNT(m));
}
return m;
}
#else /* __FreeBSD_version >= 1100000 */
/*
* Newer versions of FreeBSD, using a straightforward scheme.
*
* We allocate mbufs with m_gethdr(), since the mbuf header is needed
* by the driver. We also attach a customly-provided external storage,
* which in this case is a netmap buffer. When calling m_extadd(), however
* we pass a NULL address, since the real address (and length) will be
* filled in by nm_os_generic_xmit_frame() right before calling
* if_transmit().
*
* The dtor function does nothing, however we need it since mb_free_ext()
* has a KASSERT(), checking that the mbuf dtor function is not NULL.
*/
#if __FreeBSD_version <= 1200050
static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { }
#else /* __FreeBSD_version >= 1200051 */
/* The arg1 and arg2 pointers argument were removed by r324446, which
* in included since version 1200051. */
static void void_mbuf_dtor(struct mbuf *m) { }
#endif /* __FreeBSD_version >= 1200051 */
#define SET_MBUF_DESTRUCTOR(m, fn) do { \
(m)->m_ext.ext_free = (fn != NULL) ? \
(void *)fn : (void *)void_mbuf_dtor; \
} while (0)
static inline struct mbuf *
nm_os_get_mbuf(struct ifnet *ifp, int len)
{
struct mbuf *m;
(void)ifp;
(void)len;
m = m_gethdr(M_NOWAIT, MT_DATA);
if (m == NULL) {
return m;
}
m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor,
NULL, NULL, 0, EXT_NET_DRV);
return m;
}
#endif /* __FreeBSD_version >= 1100000 */
#endif /* __FreeBSD__ */
struct nmreq_option * nmreq_findoption(struct nmreq_option *, uint16_t);
int nmreq_checkduplicate(struct nmreq_option *);

12
sys/dev/netmap/netmap_mem2.c
View File

@ -1845,7 +1845,7 @@ netmap_free_rings(struct netmap_adapter *na)
for_rx_tx(t) {
u_int i;
for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
for (i = 0; i < netmap_all_rings(na, t); i++) {
struct netmap_kring *kring = NMR(na, t)[i];
struct netmap_ring *ring = kring->ring;
@ -1884,7 +1884,7 @@ netmap_mem2_rings_create(struct netmap_adapter *na)
for_rx_tx(t) {
u_int i;
for (i = 0; i <= nma_get_nrings(na, t); i++) {
for (i = 0; i < netmap_all_rings(na, t); i++) {
struct netmap_kring *kring = NMR(na, t)[i];
struct netmap_ring *ring = kring->ring;
u_int len, ndesc;
@ -1922,7 +1922,7 @@ netmap_mem2_rings_create(struct netmap_adapter *na)
netmap_mem_bufsize(na->nm_mem);
ND("%s h %d c %d t %d", kring->name,
ring->head, ring->cur, ring->tail);
ND("initializing slots for %s_ring", nm_txrx2str(txrx));
ND("initializing slots for %s_ring", nm_txrx2str(t));
if (!(kring->nr_kflags & NKR_FAKERING)) {
/* this is a real ring */
ND("allocating buffers for %s", kring->name);
@ -1980,7 +1980,7 @@ netmap_mem2_if_new(struct netmap_adapter *na, struct netmap_priv_d *priv)
ntot = 0;
for_rx_tx(t) {
/* account for the (eventually fake) host rings */
n[t] = nma_get_nrings(na, t) + 1;
n[t] = netmap_all_rings(na, t);
ntot += n[t];
}
/*
@ -2654,14 +2654,14 @@ netmap_mem_pt_guest_rings_create(struct netmap_adapter *na)
/* point each kring to the corresponding backend ring */
nifp = (struct netmap_if *)((char *)ptnmd->nm_addr + ptif->nifp_offset);
for (i = 0; i <= na->num_tx_rings; i++) {
for (i = 0; i < netmap_all_rings(na, NR_TX); i++) {
struct netmap_kring *kring = na->tx_rings[i];
if (kring->ring)
continue;
kring->ring = (struct netmap_ring *)
((char *)nifp + nifp->ring_ofs[i]);
}
for (i = 0; i <= na->num_rx_rings; i++) {
for (i = 0; i < netmap_all_rings(na, NR_RX); i++) {
struct netmap_kring *kring = na->rx_rings[i];
if (kring->ring)
continue;

231
sys/dev/netmap/netmap_monitor.c
View File

@ -152,6 +152,12 @@ netmap_monitor_txsync(struct netmap_kring *kring, int flags)
static int
netmap_monitor_rxsync(struct netmap_kring *kring, int flags)
{
struct netmap_monitor_adapter *mna =
(struct netmap_monitor_adapter *)kring->na;
if (unlikely(mna->priv.np_na == NULL)) {
/* parent left netmap mode */
return EIO;
}
ND("%s %x", kring->name, flags);
kring->nr_hwcur = kring->rhead;
mb();
@ -164,11 +170,20 @@ static int
netmap_monitor_krings_create(struct netmap_adapter *na)
{
int error = netmap_krings_create(na, 0);
enum txrx t;
if (error)
return error;
/* override the host rings callbacks */
na->tx_rings[na->num_tx_rings]->nm_sync = netmap_monitor_txsync;
na->rx_rings[na->num_rx_rings]->nm_sync = netmap_monitor_rxsync;
for_rx_tx(t) {
int i;
u_int first = nma_get_nrings(na, t);
for (i = 0; i < nma_get_host_nrings(na, t); i++) {
struct netmap_kring *kring = NMR(na, t)[first + i];
kring->nm_sync = t == NR_TX ? netmap_monitor_txsync :
netmap_monitor_rxsync;
}
}
return 0;
}
@ -244,6 +259,48 @@ static int netmap_monitor_parent_txsync(struct netmap_kring *, int);
static int netmap_monitor_parent_rxsync(struct netmap_kring *, int);
static int netmap_monitor_parent_notify(struct netmap_kring *, int);
static void
nm_monitor_intercept_callbacks(struct netmap_kring *kring)
{
ND("intercept callbacks on %s", kring->name);
kring->mon_sync = kring->nm_sync;
kring->mon_notify = kring->nm_notify;
if (kring->tx == NR_TX) {
kring->nm_sync = netmap_monitor_parent_txsync;
} else {
kring->nm_sync = netmap_monitor_parent_rxsync;
kring->nm_notify = netmap_monitor_parent_notify;
kring->mon_tail = kring->nr_hwtail;
}
}
static void
nm_monitor_restore_callbacks(struct netmap_kring *kring)
{
ND("restoring callbacks on %s", kring->name);
kring->nm_sync = kring->mon_sync;
kring->mon_sync = NULL;
if (kring->tx == NR_RX) {
kring->nm_notify = kring->mon_notify;
}
kring->mon_notify = NULL;
}
static struct netmap_kring *
nm_zmon_list_head(struct netmap_kring *mkring, enum txrx t)
{
struct netmap_adapter *na = mkring->na;
struct netmap_kring *kring = mkring;
struct netmap_zmon_list *z = &kring->zmon_list[t];
/* reach the head of the list */
while (nm_is_zmon(na) && z->prev != NULL) {
kring = z->prev;
na = kring->na;
z = &kring->zmon_list[t];
}
return nm_is_zmon(na) ? NULL : kring;
}
/* add the monitor mkring to the list of monitors of kring.
* If this is the first monitor, intercept the callbacks
*/
@ -254,51 +311,34 @@ netmap_monitor_add(struct netmap_kring *mkring, struct netmap_kring *kring, int
enum txrx t = kring->tx;
struct netmap_zmon_list *z = &kring->zmon_list[t];
struct netmap_zmon_list *mz = &mkring->zmon_list[t];
struct netmap_kring *ikring = kring;
/* a zero-copy monitor which is not the first in the list
* must monitor the previous monitor
*/
if (zmon && z->prev != NULL)
kring = z->prev;
ikring = z->prev; /* tail of the list */
/* synchronize with concurrently running nm_sync()s */
nm_kr_stop(kring, NM_KR_LOCKED);
if (nm_monitor_none(kring)) {
/* this is the first monitor, intercept callbacks */
ND("intercept callbacks on %s", kring->name);
kring->mon_sync = kring->nm_sync;
kring->mon_notify = kring->nm_notify;
if (kring->tx == NR_TX) {
kring->nm_sync = netmap_monitor_parent_txsync;
} else {
kring->nm_sync = netmap_monitor_parent_rxsync;
kring->nm_notify = netmap_monitor_parent_notify;
kring->mon_tail = kring->nr_hwtail;
}
if (nm_monitor_none(ikring)) {
/* this is the first monitor, intercept the callbacks */
ND("%s: intercept callbacks on %s", mkring->name, ikring->name);
nm_monitor_intercept_callbacks(ikring);
}
if (zmon) {
/* append the zmon to the list */
struct netmap_monitor_adapter *mna =
(struct netmap_monitor_adapter *)mkring->na;
struct netmap_adapter *pna;
if (z->prev != NULL)
z->prev->zmon_list[t].next = mkring;
mz->prev = z->prev;
z->prev = mkring;
if (z->next == NULL)
z->next = mkring;
/* grap a reference to the previous netmap adapter
ikring->zmon_list[t].next = mkring;
z->prev = mkring; /* new tail */
mz->prev = ikring;
mz->next = NULL;
/* grab a reference to the previous netmap adapter
* in the chain (this may be the monitored port
* or another zero-copy monitor)
*/
pna = kring->na;
netmap_adapter_get(pna);
netmap_adapter_put(mna->priv.np_na);
mna->priv.np_na = pna;
netmap_adapter_get(ikring->na);
} else {
/* make sure the monitor array exists and is big enough */
error = nm_monitor_alloc(kring, kring->n_monitors + 1);
@ -318,29 +358,50 @@ netmap_monitor_add(struct netmap_kring *mkring, struct netmap_kring *kring, int
* If this is the last monitor, restore the original callbacks
*/
static void
netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring)
netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring, enum txrx t)
{
struct netmap_zmon_list *mz = &mkring->zmon_list[kring->tx];
int zmon = nm_is_zmon(mkring->na);
struct netmap_zmon_list *mz = &mkring->zmon_list[t];
struct netmap_kring *ikring = kring;
if (zmon && mz->prev != NULL)
kring = mz->prev;
if (zmon) {
/* get to the head of the list */
kring = nm_zmon_list_head(mkring, t);
ikring = mz->prev;
}
/* synchronize with concurrently running nm_sync()s */
nm_kr_stop(kring, NM_KR_LOCKED);
/* synchronize with concurrently running nm_sync()s
* if kring is NULL (orphaned list) the monitored port
* has exited netmap mode, so there is nothing to stop
*/
if (kring != NULL)
nm_kr_stop(kring, NM_KR_LOCKED);
if (zmon) {
/* remove the monitor from the list */
if (mz->prev != NULL)
mz->prev->zmon_list[kring->tx].next = mz->next;
else
kring->zmon_list[kring->tx].next = mz->next;
if (mz->next != NULL) {
mz->next->zmon_list[kring->tx].prev = mz->prev;
} else {
kring->zmon_list[kring->tx].prev = mz->prev;
mz->next->zmon_list[t].prev = mz->prev;
/* we also need to let the next monitor drop the
* reference to us and grab the reference to the
* previous ring owner, instead
*/
if (mz->prev != NULL)
netmap_adapter_get(mz->prev->na);
netmap_adapter_put(mkring->na);
} else if (kring != NULL) {
/* in the monitored kring, prev is actually the
* pointer to the tail of the list
*/
kring->zmon_list[t].prev =
(mz->prev != kring ? mz->prev : NULL);
}
if (mz->prev != NULL) {
netmap_adapter_put(mz->prev->na);
mz->prev->zmon_list[t].next = mz->next;
}
mz->prev = NULL;
mz->next = NULL;
} else {
/* this is a copy monitor */
uint32_t mon_pos = mkring->mon_pos[kring->tx];
@ -356,21 +417,13 @@ netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring)
}
}
if (nm_monitor_none(kring)) {
if (ikring != NULL && nm_monitor_none(ikring)) {
/* this was the last monitor, restore the callbacks */
ND("%s: restoring sync on %s: %p", mkring->name, kring->name,
kring->mon_sync);
kring->nm_sync = kring->mon_sync;
kring->mon_sync = NULL;
if (kring->tx == NR_RX) {
ND("%s: restoring notify on %s: %p",
mkring->name, kring->name, kring->mon_notify);
kring->nm_notify = kring->mon_notify;
kring->mon_notify = NULL;
}
nm_monitor_restore_callbacks(ikring);
}
nm_kr_start(kring);
if (kring != NULL)
nm_kr_start(kring);
}
@ -389,9 +442,9 @@ netmap_monitor_stop(struct netmap_adapter *na)
for_rx_tx(t) {
u_int i;
for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
for (i = 0; i < netmap_all_rings(na, t); i++) {
struct netmap_kring *kring = NMR(na, t)[i];
struct netmap_kring *zkring;
struct netmap_zmon_list *z = &kring->zmon_list[t];
u_int j;
for (j = 0; j < kring->n_monitors; j++) {
@ -404,29 +457,33 @@ netmap_monitor_stop(struct netmap_adapter *na)
netmap_adapter_put(mna->priv.np_na);
mna->priv.np_na = NULL;
}
kring->monitors[j] = NULL;
}
zkring = kring->zmon_list[kring->tx].next;
if (zkring != NULL) {
struct netmap_monitor_adapter *next =
(struct netmap_monitor_adapter *)zkring->na;
struct netmap_monitor_adapter *this =
(struct netmap_monitor_adapter *)na;
struct netmap_adapter *pna = this->priv.np_na;
/* let the next monitor forget about us */
if (next->priv.np_na != NULL) {
netmap_adapter_put(next->priv.np_na);
if (!nm_is_zmon(na)) {
/* we are the head of at most one list */
struct netmap_kring *zkring;
for (zkring = z->next; zkring != NULL;
zkring = zkring->zmon_list[t].next)
{
struct netmap_monitor_adapter *next =
(struct netmap_monitor_adapter *)zkring->na;
/* let the monitor forget about us */
netmap_adapter_put(next->priv.np_na); /* nop if null */
next->priv.np_na = NULL;
}
if (pna != NULL && nm_is_zmon(na)) {
/* we are a monitor ourselves and we may
* need to pass down the reference to
* the previous adapter in the chain
*/
netmap_adapter_get(pna);
next->priv.np_na = pna;
continue;
}
next->priv.np_na = NULL;
/* orhpan the zmon list */
if (z->next != NULL)
z->next->zmon_list[t].prev = NULL;
z->next = NULL;
z->prev = NULL;
}
if (!nm_monitor_none(kring)) {
kring->n_monitors = 0;
nm_monitor_dealloc(kring);
nm_monitor_restore_callbacks(kring);
}
}
}
@ -455,7 +512,7 @@ netmap_monitor_reg_common(struct netmap_adapter *na, int onoff, int zmon)
return ENXIO;
}
for_rx_tx(t) {
for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
for (i = 0; i < netmap_all_rings(na, t); i++) {
mkring = NMR(na, t)[i];
if (!nm_kring_pending_on(mkring))
continue;
@ -477,7 +534,7 @@ netmap_monitor_reg_common(struct netmap_adapter *na, int onoff, int zmon)
if (na->active_fds == 0)
na->na_flags &= ~NAF_NETMAP_ON;
for_rx_tx(t) {
for (i = 0; i < nma_get_nrings(na, t) + 1; i++) {
for (i = 0; i < netmap_all_rings(na, t); i++) {
mkring = NMR(na, t)[i];
if (!nm_kring_pending_off(mkring))
continue;
@ -495,7 +552,7 @@ netmap_monitor_reg_common(struct netmap_adapter *na, int onoff, int zmon)
continue;
if (mna->flags & nm_txrx2flag(s)) {
kring = NMR(pna, s)[i];
netmap_monitor_del(mkring, kring);
netmap_monitor_del(mkring, kring, s);
}
}
}
@ -593,6 +650,7 @@ netmap_zmon_parent_sync(struct netmap_kring *kring, int flags, enum txrx tx)
ms->len = s->len;
s->len = tmp;
ms->flags = s->flags;
s->flags |= NS_BUF_CHANGED;
beg = nm_next(beg, lim);
@ -710,6 +768,7 @@ netmap_monitor_parent_sync(struct netmap_kring *kring, u_int first_new, int new_
memcpy(dst, src, copy_len);
ms->len = copy_len;
ms->flags = s->flags;
sent++;
beg = nm_next(beg, lim);
@ -836,7 +895,6 @@ netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na,
struct ifnet *ifp = NULL;
int error;
int zcopy = (req->nr_flags & NR_ZCOPY_MON);
char monsuff[10] = "";
if (zcopy) {
req->nr_flags |= (NR_MONITOR_TX | NR_MONITOR_RX);
@ -890,14 +948,11 @@ netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na,
D("ringid error");
goto free_out;
}
if (mna->priv.np_qlast[NR_TX] - mna->priv.np_qfirst[NR_TX] == 1) {
snprintf(monsuff, 10, "-%d", mna->priv.np_qfirst[NR_TX]);
}
snprintf(mna->up.name, sizeof(mna->up.name), "%s%s/%s%s%s", pna->name,
monsuff,
snprintf(mna->up.name, sizeof(mna->up.name), "%s/%s%s%s#%lu", pna->name,
zcopy ? "z" : "",
(req->nr_flags & NR_MONITOR_RX) ? "r" : "",
(req->nr_flags & NR_MONITOR_TX) ? "t" : "");
(req->nr_flags & NR_MONITOR_TX) ? "t" : "",
pna->monitor_id++);
/* the monitor supports the host rings iff the parent does */
mna->up.na_flags |= (pna->na_flags & NAF_HOST_RINGS);

60
sys/dev/netmap/netmap_pipe.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2014-2016 Giuseppe Lettieri
* Copyright (C) 2014-2018 Giuseppe Lettieri
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -185,8 +185,9 @@ int
netmap_pipe_txsync(struct netmap_kring *txkring, int flags)
{
struct netmap_kring *rxkring = txkring->pipe;
u_int k, lim = txkring->nkr_num_slots - 1;
u_int k, lim = txkring->nkr_num_slots - 1, nk;
int m; /* slots to transfer */
int complete; /* did we see a complete packet ? */
struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring;
ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name);
@ -194,6 +195,9 @@ netmap_pipe_txsync(struct netmap_kring *txkring, int flags)
txkring->nr_hwcur, txkring->nr_hwtail,
txkring->rcur, txkring->rhead, txkring->rtail);
/* update the hwtail */
txkring->nr_hwtail = txkring->pipe_tail;
m = txkring->rhead - txkring->nr_hwcur; /* new slots */
if (m < 0)
m += txkring->nkr_num_slots;
@ -203,29 +207,29 @@ netmap_pipe_txsync(struct netmap_kring *txkring, int flags)
return 0;
}
for (k = txkring->nr_hwcur; m; m--, k = nm_next(k, lim)) {
for (k = txkring->nr_hwcur, nk = lim + 1, complete = 0; m;
m--, k = nm_next(k, lim), nk = (complete ? k : nk)) {
struct netmap_slot *rs = &rxring->slot[k];
struct netmap_slot *ts = &txring->slot[k];
rs->len = ts->len;
rs->ptr = ts->ptr;
*rs = *ts;
if (ts->flags & NS_BUF_CHANGED) {
rs->buf_idx = ts->buf_idx;
rs->flags |= NS_BUF_CHANGED;
ts->flags &= ~NS_BUF_CHANGED;
}
complete = !(ts->flags & NS_MOREFRAG);
}
mb(); /* make sure the slots are updated before publishing them */
rxkring->nr_hwtail = k;
txkring->nr_hwcur = k;
ND(20, "TX after : hwcur %d hwtail %d cur %d head %d tail %d k %d",
txkring->nr_hwcur, txkring->nr_hwtail,
txkring->rcur, txkring->rhead, txkring->rtail, k);
rxkring->nm_notify(rxkring, 0);
if (likely(nk <= lim)) {
mb(); /* make sure the slots are updated before publishing them */
rxkring->pipe_tail = nk; /* only publish complete packets */
rxkring->nm_notify(rxkring, 0);
}
return 0;
}
@ -243,6 +247,9 @@ netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags)
rxkring->nr_hwcur, rxkring->nr_hwtail,
rxkring->rcur, rxkring->rhead, rxkring->rtail);
/* update the hwtail */
rxkring->nr_hwtail = rxkring->pipe_tail;
m = rxkring->rhead - rxkring->nr_hwcur; /* released slots */
if (m < 0)
m += rxkring->nkr_num_slots;
@ -264,7 +271,7 @@ netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags)
}
mb(); /* make sure the slots are updated before publishing them */
txkring->nr_hwtail = nm_prev(k, lim);
txkring->pipe_tail = nm_prev(k, lim);
rxkring->nr_hwcur = k;
ND(20, "RX after : hwcur %d hwtail %d cur %d head %d tail %d k %d",
@ -346,14 +353,19 @@ netmap_pipe_krings_create(struct netmap_adapter *na)
if (error)
goto del_krings1;
/* cross link the krings */
/* cross link the krings and initialize the pipe_tails */
for_rx_tx(t) {
enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */
for (i = 0; i < nma_get_nrings(na, t); i++) {
NMR(na, t)[i]->pipe = NMR(ona, r)[i];
NMR(ona, r)[i]->pipe = NMR(na, t)[i];
struct netmap_kring *k1 = NMR(na, t)[i],
*k2 = NMR(ona, r)[i];
k1->pipe = k2;
k2->pipe = k1;
/* mark all peer-adapter rings as fake */
NMR(ona, r)[i]->nr_kflags |= NKR_FAKERING;
k2->nr_kflags |= NKR_FAKERING;
/* init tails */
k1->pipe_tail = k1->nr_hwtail;
k2->pipe_tail = k2->nr_hwtail;
}
}
@ -436,6 +448,16 @@ netmap_pipe_reg(struct netmap_adapter *na, int onoff)
if (nm_kring_pending_on(kring)) {
struct netmap_kring *sring, *dring;
kring->nr_mode = NKR_NETMAP_ON;
if ((kring->nr_kflags & NKR_FAKERING) &&
(kring->pipe->nr_kflags & NKR_FAKERING)) {
/* this is a re-open of a pipe
* end-point kept alive by the other end.
* We need to leave everything as it is
*/
continue;
}
/* copy the buffers from the non-fake ring */
if (kring->nr_kflags & NKR_FAKERING) {
sring = kring->pipe;
@ -556,10 +578,10 @@ netmap_pipe_krings_delete(struct netmap_adapter *na)
if (ring == NULL)
continue;
if (kring->nr_hwtail == kring->nr_hwcur)
ring->slot[kring->nr_hwtail].buf_idx = 0;
if (kring->tx == NR_RX)
ring->slot[kring->pipe_tail].buf_idx = 0;
for (j = nm_next(kring->nr_hwtail, lim);
for (j = nm_next(kring->pipe_tail, lim);
j != kring->nr_hwcur;
j = nm_next(j, lim))
{

2188
sys/dev/netmap/netmap_vale.c
View File

File diff suppressed because it is too large Load Diff

2
sys/net/netmap.h
View File

@ -237,6 +237,8 @@ struct netmap_slot {
* are the number of fragments.
*/
#define NETMAP_MAX_FRAGS 64 /* max number of fragments */
/*
* struct netmap_ring

27
sys/net/netmap_user.h
View File

@ -1029,20 +1029,35 @@ nm_inject(struct nm_desc *d, const void *buf, size_t size)
for (c = 0; c < n ; c++, ri++) {
/* compute current ring to use */
struct netmap_ring *ring;
uint32_t i, idx;
uint32_t i, j, idx;
size_t rem;
if (ri > d->last_tx_ring)
ri = d->first_tx_ring;
ring = NETMAP_TXRING(d->nifp, ri);
if (nm_ring_empty(ring)) {
continue;
rem = size;
j = ring->cur;
while (rem > ring->nr_buf_size && j != ring->tail) {
rem -= ring->nr_buf_size;
j = nm_ring_next(ring, j);
}
if (j == ring->tail && rem > 0)
continue;
i = ring->cur;
while (i != j) {
idx = ring->slot[i].buf_idx;
ring->slot[i].len = ring->nr_buf_size;
ring->slot[i].flags = NS_MOREFRAG;
nm_pkt_copy(buf, NETMAP_BUF(ring, idx), ring->nr_buf_size);
i = nm_ring_next(ring, i);
buf = (char *)buf + ring->nr_buf_size;
}
idx = ring->slot[i].buf_idx;
ring->slot[i].len = size;
nm_pkt_copy(buf, NETMAP_BUF(ring, idx), size);
d->cur_tx_ring = ri;
ring->slot[i].len = rem;
ring->slot[i].flags = 0;
nm_pkt_copy(buf, NETMAP_BUF(ring, idx), rem);
ring->head = ring->cur = nm_ring_next(ring, i);
d->cur_tx_ring = ri;
return size;
}
return 0; /* fail */