1188 lines
33 KiB
C
1188 lines
33 KiB
C
/*
|
|
* Copyright (C) 2016-2018 Vincenzo Maffione
|
|
* Copyright (C) 2015 Stefano Garzarella
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* common headers
|
|
*/
|
|
#if defined(__FreeBSD__)
|
|
#include <sys/cdefs.h>
|
|
#include <sys/param.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/types.h>
|
|
#include <sys/selinfo.h>
|
|
#include <sys/socket.h>
|
|
#include <net/if.h>
|
|
#include <net/if_var.h>
|
|
#include <machine/bus.h>
|
|
|
|
#define usleep_range(_1, _2) \
|
|
pause_sbt("sync-kloop-sleep", SBT_1US * _1, SBT_1US * 1, C_ABSOLUTE)
|
|
|
|
#elif defined(linux)
|
|
#include <bsd_glue.h>
|
|
#include <linux/file.h>
|
|
#include <linux/eventfd.h>
|
|
#endif
|
|
|
|
#include <net/netmap.h>
|
|
#include <dev/netmap/netmap_kern.h>
|
|
#include <net/netmap_virt.h>
|
|
#include <dev/netmap/netmap_mem2.h>
|
|
|
|
/* Support for eventfd-based notifications. */
|
|
#if defined(linux)
|
|
#define SYNC_KLOOP_POLL
|
|
#endif
|
|
|
|
/* Write kring pointers (hwcur, hwtail) to the CSB.
|
|
* This routine is coupled with ptnetmap_guest_read_kring_csb(). */
|
|
static inline void
|
|
sync_kloop_kernel_write(struct nm_csb_ktoa __user *ptr, uint32_t hwcur,
|
|
uint32_t hwtail)
|
|
{
|
|
/* Issue a first store-store barrier to make sure writes to the
|
|
* netmap ring do not overcome updates on ktoa->hwcur and ktoa->hwtail. */
|
|
nm_stst_barrier();
|
|
|
|
/*
|
|
* The same scheme used in nm_sync_kloop_appl_write() applies here.
|
|
* We allow the application to read a value of hwcur more recent than the value
|
|
* of hwtail, since this would anyway result in a consistent view of the
|
|
* ring state (and hwcur can never wraparound hwtail, since hwcur must be
|
|
* behind head).
|
|
*
|
|
* The following memory barrier scheme is used to make this happen:
|
|
*
|
|
* Application Kernel
|
|
*
|
|
* STORE(hwcur) LOAD(hwtail)
|
|
* wmb() <-------------> rmb()
|
|
* STORE(hwtail) LOAD(hwcur)
|
|
*/
|
|
CSB_WRITE(ptr, hwcur, hwcur);
|
|
nm_stst_barrier();
|
|
CSB_WRITE(ptr, hwtail, hwtail);
|
|
}
|
|
|
|
/* Read kring pointers (head, cur, sync_flags) from the CSB.
|
|
* This routine is coupled with ptnetmap_guest_write_kring_csb(). */
|
|
static inline void
|
|
sync_kloop_kernel_read(struct nm_csb_atok __user *ptr,
|
|
struct netmap_ring *shadow_ring,
|
|
uint32_t num_slots)
|
|
{
|
|
/*
|
|
* We place a memory barrier to make sure that the update of head never
|
|
* overtakes the update of cur.
|
|
* (see explanation in sync_kloop_kernel_write).
|
|
*/
|
|
CSB_READ(ptr, head, shadow_ring->head);
|
|
nm_ldld_barrier();
|
|
CSB_READ(ptr, cur, shadow_ring->cur);
|
|
CSB_READ(ptr, sync_flags, shadow_ring->flags);
|
|
|
|
/* Make sure that loads from atok->head and atok->cur are not delayed
|
|
* after the loads from the netmap ring. */
|
|
nm_ldld_barrier();
|
|
}
|
|
|
|
/* Enable or disable application --> kernel kicks. */
|
|
static inline void
|
|
csb_ktoa_kick_enable(struct nm_csb_ktoa __user *csb_ktoa, uint32_t val)
|
|
{
|
|
CSB_WRITE(csb_ktoa, kern_need_kick, val);
|
|
}
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
/* Are application interrupt enabled or disabled? */
|
|
static inline uint32_t
|
|
csb_atok_intr_enabled(struct nm_csb_atok __user *csb_atok)
|
|
{
|
|
uint32_t v;
|
|
|
|
CSB_READ(csb_atok, appl_need_kick, v);
|
|
|
|
return v;
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
static inline void
|
|
sync_kloop_kring_dump(const char *title, const struct netmap_kring *kring)
|
|
{
|
|
nm_prinf("%s, kring %s, hwcur %d, rhead %d, "
|
|
"rcur %d, rtail %d, hwtail %d",
|
|
title, kring->name, kring->nr_hwcur, kring->rhead,
|
|
kring->rcur, kring->rtail, kring->nr_hwtail);
|
|
}
|
|
|
|
/* Arguments for netmap_sync_kloop_tx_ring() and
|
|
* netmap_sync_kloop_rx_ring().
|
|
*/
|
|
struct sync_kloop_ring_args {
|
|
struct netmap_kring *kring;
|
|
struct nm_csb_atok *csb_atok;
|
|
struct nm_csb_ktoa *csb_ktoa;
|
|
#ifdef SYNC_KLOOP_POLL
|
|
struct eventfd_ctx *irq_ctx;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
/* Are we busy waiting rather than using a schedule() loop ? */
|
|
bool busy_wait;
|
|
/* Are we processing in the context of VM exit ? */
|
|
bool direct;
|
|
};
|
|
|
|
static void
|
|
netmap_sync_kloop_tx_ring(const struct sync_kloop_ring_args *a)
|
|
{
|
|
struct netmap_kring *kring = a->kring;
|
|
struct nm_csb_atok *csb_atok = a->csb_atok;
|
|
struct nm_csb_ktoa *csb_ktoa = a->csb_ktoa;
|
|
struct netmap_ring shadow_ring; /* shadow copy of the netmap_ring */
|
|
#ifdef SYNC_KLOOP_POLL
|
|
bool more_txspace = false;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
uint32_t num_slots;
|
|
int batch;
|
|
|
|
if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
|
|
return;
|
|
}
|
|
|
|
num_slots = kring->nkr_num_slots;
|
|
|
|
/* Disable application --> kernel notifications. */
|
|
if (!a->direct) {
|
|
csb_ktoa_kick_enable(csb_ktoa, 0);
|
|
}
|
|
/* Copy the application kring pointers from the CSB */
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
|
|
for (;;) {
|
|
batch = shadow_ring.head - kring->nr_hwcur;
|
|
if (batch < 0)
|
|
batch += num_slots;
|
|
|
|
#ifdef PTN_TX_BATCH_LIM
|
|
if (batch > PTN_TX_BATCH_LIM(num_slots)) {
|
|
/* If application moves ahead too fast, let's cut the move so
|
|
* that we don't exceed our batch limit. */
|
|
uint32_t head_lim = kring->nr_hwcur + PTN_TX_BATCH_LIM(num_slots);
|
|
|
|
if (head_lim >= num_slots)
|
|
head_lim -= num_slots;
|
|
nm_prdis(1, "batch: %d head: %d head_lim: %d", batch, shadow_ring.head,
|
|
head_lim);
|
|
shadow_ring.head = head_lim;
|
|
batch = PTN_TX_BATCH_LIM(num_slots);
|
|
}
|
|
#endif /* PTN_TX_BATCH_LIM */
|
|
|
|
if (nm_kr_txspace(kring) <= (num_slots >> 1)) {
|
|
shadow_ring.flags |= NAF_FORCE_RECLAIM;
|
|
}
|
|
|
|
/* Netmap prologue */
|
|
shadow_ring.tail = kring->rtail;
|
|
if (unlikely(nm_txsync_prologue(kring, &shadow_ring) >= num_slots)) {
|
|
/* Reinit ring and enable notifications. */
|
|
netmap_ring_reinit(kring);
|
|
if (!a->busy_wait) {
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (unlikely(netmap_debug & NM_DEBUG_TXSYNC)) {
|
|
sync_kloop_kring_dump("pre txsync", kring);
|
|
}
|
|
|
|
if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
|
|
if (!a->busy_wait) {
|
|
/* Re-enable notifications. */
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
}
|
|
nm_prerr("txsync() failed");
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Finalize
|
|
* Copy kernel hwcur and hwtail into the CSB for the application sync(), and
|
|
* do the nm_sync_finalize.
|
|
*/
|
|
sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur,
|
|
kring->nr_hwtail);
|
|
if (kring->rtail != kring->nr_hwtail) {
|
|
/* Some more room available in the parent adapter. */
|
|
kring->rtail = kring->nr_hwtail;
|
|
#ifdef SYNC_KLOOP_POLL
|
|
more_txspace = true;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
}
|
|
|
|
if (unlikely(netmap_debug & NM_DEBUG_TXSYNC)) {
|
|
sync_kloop_kring_dump("post txsync", kring);
|
|
}
|
|
|
|
/* Interrupt the application if needed. */
|
|
#ifdef SYNC_KLOOP_POLL
|
|
if (a->irq_ctx && more_txspace && csb_atok_intr_enabled(csb_atok)) {
|
|
/* We could disable kernel --> application kicks here,
|
|
* to avoid spurious interrupts. */
|
|
eventfd_signal(a->irq_ctx, 1);
|
|
more_txspace = false;
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
/* Read CSB to see if there is more work to do. */
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
if (shadow_ring.head == kring->rhead) {
|
|
if (a->busy_wait) {
|
|
break;
|
|
}
|
|
/*
|
|
* No more packets to transmit. We enable notifications and
|
|
* go to sleep, waiting for a kick from the application when new
|
|
* new slots are ready for transmission.
|
|
*/
|
|
/* Re-enable notifications. */
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
/* Double check, with store-load memory barrier. */
|
|
nm_stld_barrier();
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
if (shadow_ring.head != kring->rhead) {
|
|
/* We won the race condition, there are more packets to
|
|
* transmit. Disable notifications and do another cycle */
|
|
csb_ktoa_kick_enable(csb_ktoa, 0);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (nm_kr_txempty(kring)) {
|
|
/* No more available TX slots. We stop waiting for a notification
|
|
* from the backend (netmap_tx_irq). */
|
|
nm_prdis(1, "TX ring");
|
|
break;
|
|
}
|
|
}
|
|
|
|
nm_kr_put(kring);
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
if (a->irq_ctx && more_txspace && csb_atok_intr_enabled(csb_atok)) {
|
|
eventfd_signal(a->irq_ctx, 1);
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
}
|
|
|
|
/* RX cycle without receive any packets */
|
|
#define SYNC_LOOP_RX_DRY_CYCLES_MAX 2
|
|
|
|
static inline int
|
|
sync_kloop_norxslots(struct netmap_kring *kring, uint32_t g_head)
|
|
{
|
|
return (NM_ACCESS_ONCE(kring->nr_hwtail) == nm_prev(g_head,
|
|
kring->nkr_num_slots - 1));
|
|
}
|
|
|
|
static void
|
|
netmap_sync_kloop_rx_ring(const struct sync_kloop_ring_args *a)
|
|
{
|
|
|
|
struct netmap_kring *kring = a->kring;
|
|
struct nm_csb_atok *csb_atok = a->csb_atok;
|
|
struct nm_csb_ktoa *csb_ktoa = a->csb_ktoa;
|
|
struct netmap_ring shadow_ring; /* shadow copy of the netmap_ring */
|
|
int dry_cycles = 0;
|
|
#ifdef SYNC_KLOOP_POLL
|
|
bool some_recvd = false;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
uint32_t num_slots;
|
|
|
|
if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
|
|
return;
|
|
}
|
|
|
|
num_slots = kring->nkr_num_slots;
|
|
|
|
/* Get RX csb_atok and csb_ktoa pointers from the CSB. */
|
|
num_slots = kring->nkr_num_slots;
|
|
|
|
/* Disable notifications. */
|
|
if (!a->direct) {
|
|
csb_ktoa_kick_enable(csb_ktoa, 0);
|
|
}
|
|
/* Copy the application kring pointers from the CSB */
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
|
|
for (;;) {
|
|
uint32_t hwtail;
|
|
|
|
/* Netmap prologue */
|
|
shadow_ring.tail = kring->rtail;
|
|
if (unlikely(nm_rxsync_prologue(kring, &shadow_ring) >= num_slots)) {
|
|
/* Reinit ring and enable notifications. */
|
|
netmap_ring_reinit(kring);
|
|
if (!a->busy_wait) {
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (unlikely(netmap_debug & NM_DEBUG_RXSYNC)) {
|
|
sync_kloop_kring_dump("pre rxsync", kring);
|
|
}
|
|
|
|
if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
|
|
if (!a->busy_wait) {
|
|
/* Re-enable notifications. */
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
}
|
|
nm_prerr("rxsync() failed");
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Finalize
|
|
* Copy kernel hwcur and hwtail into the CSB for the application sync()
|
|
*/
|
|
hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
|
|
sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur, hwtail);
|
|
if (kring->rtail != hwtail) {
|
|
kring->rtail = hwtail;
|
|
#ifdef SYNC_KLOOP_POLL
|
|
some_recvd = true;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
dry_cycles = 0;
|
|
} else {
|
|
dry_cycles++;
|
|
}
|
|
|
|
if (unlikely(netmap_debug & NM_DEBUG_RXSYNC)) {
|
|
sync_kloop_kring_dump("post rxsync", kring);
|
|
}
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
/* Interrupt the application if needed. */
|
|
if (a->irq_ctx && some_recvd && csb_atok_intr_enabled(csb_atok)) {
|
|
/* We could disable kernel --> application kicks here,
|
|
* to avoid spurious interrupts. */
|
|
eventfd_signal(a->irq_ctx, 1);
|
|
some_recvd = false;
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
/* Read CSB to see if there is more work to do. */
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
if (sync_kloop_norxslots(kring, shadow_ring.head)) {
|
|
if (a->busy_wait) {
|
|
break;
|
|
}
|
|
/*
|
|
* No more slots available for reception. We enable notification and
|
|
* go to sleep, waiting for a kick from the application when new receive
|
|
* slots are available.
|
|
*/
|
|
/* Re-enable notifications. */
|
|
csb_ktoa_kick_enable(csb_ktoa, 1);
|
|
/* Double check, with store-load memory barrier. */
|
|
nm_stld_barrier();
|
|
sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
|
|
if (!sync_kloop_norxslots(kring, shadow_ring.head)) {
|
|
/* We won the race condition, more slots are available. Disable
|
|
* notifications and do another cycle. */
|
|
csb_ktoa_kick_enable(csb_ktoa, 0);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
|
|
if (unlikely(hwtail == kring->rhead ||
|
|
dry_cycles >= SYNC_LOOP_RX_DRY_CYCLES_MAX)) {
|
|
/* No more packets to be read from the backend. We stop and
|
|
* wait for a notification from the backend (netmap_rx_irq). */
|
|
nm_prdis(1, "nr_hwtail: %d rhead: %d dry_cycles: %d",
|
|
hwtail, kring->rhead, dry_cycles);
|
|
break;
|
|
}
|
|
}
|
|
|
|
nm_kr_put(kring);
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
/* Interrupt the application if needed. */
|
|
if (a->irq_ctx && some_recvd && csb_atok_intr_enabled(csb_atok)) {
|
|
eventfd_signal(a->irq_ctx, 1);
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
}
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
struct sync_kloop_poll_ctx;
|
|
struct sync_kloop_poll_entry {
|
|
/* Support for receiving notifications from
|
|
* a netmap ring or from the application. */
|
|
struct file *filp;
|
|
wait_queue_t wait;
|
|
wait_queue_head_t *wqh;
|
|
|
|
/* Support for sending notifications to the application. */
|
|
struct eventfd_ctx *irq_ctx;
|
|
struct file *irq_filp;
|
|
|
|
/* Arguments for the ring processing function. Useful
|
|
* in case of custom wake-up function. */
|
|
struct sync_kloop_ring_args *args;
|
|
struct sync_kloop_poll_ctx *parent;
|
|
|
|
};
|
|
|
|
struct sync_kloop_poll_ctx {
|
|
poll_table wait_table;
|
|
unsigned int next_entry;
|
|
int (*next_wake_fun)(wait_queue_t *, unsigned, int, void *);
|
|
unsigned int num_entries;
|
|
unsigned int num_tx_rings;
|
|
unsigned int num_rings;
|
|
/* First num_tx_rings entries are for the TX kicks.
|
|
* Then the RX kicks entries follow. The last two
|
|
* entries are for TX irq, and RX irq. */
|
|
struct sync_kloop_poll_entry entries[0];
|
|
};
|
|
|
|
static void
|
|
sync_kloop_poll_table_queue_proc(struct file *file, wait_queue_head_t *wqh,
|
|
poll_table *pt)
|
|
{
|
|
struct sync_kloop_poll_ctx *poll_ctx =
|
|
container_of(pt, struct sync_kloop_poll_ctx, wait_table);
|
|
struct sync_kloop_poll_entry *entry = poll_ctx->entries +
|
|
poll_ctx->next_entry;
|
|
|
|
BUG_ON(poll_ctx->next_entry >= poll_ctx->num_entries);
|
|
entry->wqh = wqh;
|
|
entry->filp = file;
|
|
/* Use the default wake up function. */
|
|
if (poll_ctx->next_wake_fun == NULL) {
|
|
init_waitqueue_entry(&entry->wait, current);
|
|
} else {
|
|
init_waitqueue_func_entry(&entry->wait,
|
|
poll_ctx->next_wake_fun);
|
|
}
|
|
add_wait_queue(wqh, &entry->wait);
|
|
}
|
|
|
|
static int
|
|
sync_kloop_tx_kick_wake_fun(wait_queue_t *wait, unsigned mode,
|
|
int wake_flags, void *key)
|
|
{
|
|
struct sync_kloop_poll_entry *entry =
|
|
container_of(wait, struct sync_kloop_poll_entry, wait);
|
|
|
|
netmap_sync_kloop_tx_ring(entry->args);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
sync_kloop_tx_irq_wake_fun(wait_queue_t *wait, unsigned mode,
|
|
int wake_flags, void *key)
|
|
{
|
|
struct sync_kloop_poll_entry *entry =
|
|
container_of(wait, struct sync_kloop_poll_entry, wait);
|
|
struct sync_kloop_poll_ctx *poll_ctx = entry->parent;
|
|
int i;
|
|
|
|
for (i = 0; i < poll_ctx->num_tx_rings; i++) {
|
|
struct eventfd_ctx *irq_ctx = poll_ctx->entries[i].irq_ctx;
|
|
|
|
if (irq_ctx) {
|
|
eventfd_signal(irq_ctx, 1);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
sync_kloop_rx_kick_wake_fun(wait_queue_t *wait, unsigned mode,
|
|
int wake_flags, void *key)
|
|
{
|
|
struct sync_kloop_poll_entry *entry =
|
|
container_of(wait, struct sync_kloop_poll_entry, wait);
|
|
|
|
netmap_sync_kloop_rx_ring(entry->args);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
sync_kloop_rx_irq_wake_fun(wait_queue_t *wait, unsigned mode,
|
|
int wake_flags, void *key)
|
|
{
|
|
struct sync_kloop_poll_entry *entry =
|
|
container_of(wait, struct sync_kloop_poll_entry, wait);
|
|
struct sync_kloop_poll_ctx *poll_ctx = entry->parent;
|
|
int i;
|
|
|
|
for (i = poll_ctx->num_tx_rings; i < poll_ctx->num_rings; i++) {
|
|
struct eventfd_ctx *irq_ctx = poll_ctx->entries[i].irq_ctx;
|
|
|
|
if (irq_ctx) {
|
|
eventfd_signal(irq_ctx, 1);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
int
|
|
netmap_sync_kloop(struct netmap_priv_d *priv, struct nmreq_header *hdr)
|
|
{
|
|
struct nmreq_sync_kloop_start *req =
|
|
(struct nmreq_sync_kloop_start *)(uintptr_t)hdr->nr_body;
|
|
struct nmreq_opt_sync_kloop_eventfds *eventfds_opt = NULL;
|
|
#ifdef SYNC_KLOOP_POLL
|
|
struct sync_kloop_poll_ctx *poll_ctx = NULL;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
int num_rx_rings, num_tx_rings, num_rings;
|
|
struct sync_kloop_ring_args *args = NULL;
|
|
uint32_t sleep_us = req->sleep_us;
|
|
struct nm_csb_atok* csb_atok_base;
|
|
struct nm_csb_ktoa* csb_ktoa_base;
|
|
struct netmap_adapter *na;
|
|
struct nmreq_option *opt;
|
|
bool na_could_sleep = false;
|
|
bool busy_wait = true;
|
|
bool direct_tx = false;
|
|
bool direct_rx = false;
|
|
int err = 0;
|
|
int i;
|
|
|
|
if (sleep_us > 1000000) {
|
|
/* We do not accept sleeping for more than a second. */
|
|
return EINVAL;
|
|
}
|
|
|
|
if (priv->np_nifp == NULL) {
|
|
return ENXIO;
|
|
}
|
|
mb(); /* make sure following reads are not from cache */
|
|
|
|
na = priv->np_na;
|
|
if (!nm_netmap_on(na)) {
|
|
return ENXIO;
|
|
}
|
|
|
|
NMG_LOCK();
|
|
/* Make sure the application is working in CSB mode. */
|
|
if (!priv->np_csb_atok_base || !priv->np_csb_ktoa_base) {
|
|
NMG_UNLOCK();
|
|
nm_prerr("sync-kloop on %s requires "
|
|
"NETMAP_REQ_OPT_CSB option", na->name);
|
|
return EINVAL;
|
|
}
|
|
|
|
csb_atok_base = priv->np_csb_atok_base;
|
|
csb_ktoa_base = priv->np_csb_ktoa_base;
|
|
|
|
/* Make sure that no kloop is currently running. */
|
|
if (priv->np_kloop_state & NM_SYNC_KLOOP_RUNNING) {
|
|
err = EBUSY;
|
|
}
|
|
priv->np_kloop_state |= NM_SYNC_KLOOP_RUNNING;
|
|
NMG_UNLOCK();
|
|
if (err) {
|
|
return err;
|
|
}
|
|
|
|
num_rx_rings = priv->np_qlast[NR_RX] - priv->np_qfirst[NR_RX];
|
|
num_tx_rings = priv->np_qlast[NR_TX] - priv->np_qfirst[NR_TX];
|
|
num_rings = num_tx_rings + num_rx_rings;
|
|
|
|
args = nm_os_malloc(num_rings * sizeof(args[0]));
|
|
if (!args) {
|
|
err = ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
/* Prepare the arguments for netmap_sync_kloop_tx_ring()
|
|
* and netmap_sync_kloop_rx_ring(). */
|
|
for (i = 0; i < num_tx_rings; i++) {
|
|
struct sync_kloop_ring_args *a = args + i;
|
|
|
|
a->kring = NMR(na, NR_TX)[i + priv->np_qfirst[NR_TX]];
|
|
a->csb_atok = csb_atok_base + i;
|
|
a->csb_ktoa = csb_ktoa_base + i;
|
|
a->busy_wait = busy_wait;
|
|
a->direct = direct_tx;
|
|
}
|
|
for (i = 0; i < num_rx_rings; i++) {
|
|
struct sync_kloop_ring_args *a = args + num_tx_rings + i;
|
|
|
|
a->kring = NMR(na, NR_RX)[i + priv->np_qfirst[NR_RX]];
|
|
a->csb_atok = csb_atok_base + num_tx_rings + i;
|
|
a->csb_ktoa = csb_ktoa_base + num_tx_rings + i;
|
|
a->busy_wait = busy_wait;
|
|
a->direct = direct_rx;
|
|
}
|
|
|
|
/* Validate notification options. */
|
|
opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_SYNC_KLOOP_MODE);
|
|
if (opt != NULL) {
|
|
struct nmreq_opt_sync_kloop_mode *mode_opt =
|
|
(struct nmreq_opt_sync_kloop_mode *)opt;
|
|
|
|
direct_tx = !!(mode_opt->mode & NM_OPT_SYNC_KLOOP_DIRECT_TX);
|
|
direct_rx = !!(mode_opt->mode & NM_OPT_SYNC_KLOOP_DIRECT_RX);
|
|
if (mode_opt->mode & ~(NM_OPT_SYNC_KLOOP_DIRECT_TX |
|
|
NM_OPT_SYNC_KLOOP_DIRECT_RX)) {
|
|
opt->nro_status = err = EINVAL;
|
|
goto out;
|
|
}
|
|
opt->nro_status = 0;
|
|
}
|
|
opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_SYNC_KLOOP_EVENTFDS);
|
|
if (opt != NULL) {
|
|
if (opt->nro_size != sizeof(*eventfds_opt) +
|
|
sizeof(eventfds_opt->eventfds[0]) * num_rings) {
|
|
/* Option size not consistent with the number of
|
|
* entries. */
|
|
opt->nro_status = err = EINVAL;
|
|
goto out;
|
|
}
|
|
#ifdef SYNC_KLOOP_POLL
|
|
eventfds_opt = (struct nmreq_opt_sync_kloop_eventfds *)opt;
|
|
opt->nro_status = 0;
|
|
|
|
/* Check if some ioeventfd entry is not defined, and force sleep
|
|
* synchronization in that case. */
|
|
busy_wait = false;
|
|
for (i = 0; i < num_rings; i++) {
|
|
if (eventfds_opt->eventfds[i].ioeventfd < 0) {
|
|
busy_wait = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (busy_wait && (direct_tx || direct_rx)) {
|
|
/* For direct processing we need all the
|
|
* ioeventfds to be valid. */
|
|
opt->nro_status = err = EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
/* We need 2 poll entries for TX and RX notifications coming
|
|
* from the netmap adapter, plus one entries per ring for the
|
|
* notifications coming from the application. */
|
|
poll_ctx = nm_os_malloc(sizeof(*poll_ctx) +
|
|
(num_rings + 2) * sizeof(poll_ctx->entries[0]));
|
|
init_poll_funcptr(&poll_ctx->wait_table,
|
|
sync_kloop_poll_table_queue_proc);
|
|
poll_ctx->num_entries = 2 + num_rings;
|
|
poll_ctx->num_tx_rings = num_tx_rings;
|
|
poll_ctx->num_rings = num_rings;
|
|
poll_ctx->next_entry = 0;
|
|
poll_ctx->next_wake_fun = NULL;
|
|
|
|
if (direct_tx && (na->na_flags & NAF_BDG_MAYSLEEP)) {
|
|
/* In direct mode, VALE txsync is called from
|
|
* wake-up context, where it is not possible
|
|
* to sleep.
|
|
*/
|
|
na->na_flags &= ~NAF_BDG_MAYSLEEP;
|
|
na_could_sleep = true;
|
|
}
|
|
|
|
for (i = 0; i < num_rings + 2; i++) {
|
|
poll_ctx->entries[i].args = args + i;
|
|
poll_ctx->entries[i].parent = poll_ctx;
|
|
}
|
|
|
|
/* Poll for notifications coming from the applications through
|
|
* eventfds. */
|
|
for (i = 0; i < num_rings; i++, poll_ctx->next_entry++) {
|
|
struct eventfd_ctx *irq = NULL;
|
|
struct file *filp = NULL;
|
|
unsigned long mask;
|
|
bool tx_ring = (i < num_tx_rings);
|
|
|
|
if (eventfds_opt->eventfds[i].irqfd >= 0) {
|
|
filp = eventfd_fget(
|
|
eventfds_opt->eventfds[i].irqfd);
|
|
if (IS_ERR(filp)) {
|
|
err = PTR_ERR(filp);
|
|
goto out;
|
|
}
|
|
irq = eventfd_ctx_fileget(filp);
|
|
if (IS_ERR(irq)) {
|
|
err = PTR_ERR(irq);
|
|
goto out;
|
|
}
|
|
}
|
|
poll_ctx->entries[i].irq_filp = filp;
|
|
poll_ctx->entries[i].irq_ctx = irq;
|
|
poll_ctx->entries[i].args->busy_wait = busy_wait;
|
|
/* Don't let netmap_sync_kloop_*x_ring() use
|
|
* IRQs in direct mode. */
|
|
poll_ctx->entries[i].args->irq_ctx =
|
|
((tx_ring && direct_tx) ||
|
|
(!tx_ring && direct_rx)) ? NULL :
|
|
poll_ctx->entries[i].irq_ctx;
|
|
poll_ctx->entries[i].args->direct =
|
|
(tx_ring ? direct_tx : direct_rx);
|
|
|
|
if (!busy_wait) {
|
|
filp = eventfd_fget(
|
|
eventfds_opt->eventfds[i].ioeventfd);
|
|
if (IS_ERR(filp)) {
|
|
err = PTR_ERR(filp);
|
|
goto out;
|
|
}
|
|
if (tx_ring && direct_tx) {
|
|
/* Override the wake up function
|
|
* so that it can directly call
|
|
* netmap_sync_kloop_tx_ring().
|
|
*/
|
|
poll_ctx->next_wake_fun =
|
|
sync_kloop_tx_kick_wake_fun;
|
|
} else if (!tx_ring && direct_rx) {
|
|
/* Same for direct RX. */
|
|
poll_ctx->next_wake_fun =
|
|
sync_kloop_rx_kick_wake_fun;
|
|
} else {
|
|
poll_ctx->next_wake_fun = NULL;
|
|
}
|
|
mask = filp->f_op->poll(filp,
|
|
&poll_ctx->wait_table);
|
|
if (mask & POLLERR) {
|
|
err = EINVAL;
|
|
goto out;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Poll for notifications coming from the netmap rings bound to
|
|
* this file descriptor. */
|
|
if (!busy_wait) {
|
|
NMG_LOCK();
|
|
/* In direct mode, override the wake up function so
|
|
* that it can forward the netmap_tx_irq() to the
|
|
* guest. */
|
|
poll_ctx->next_wake_fun = direct_tx ?
|
|
sync_kloop_tx_irq_wake_fun : NULL;
|
|
poll_wait(priv->np_filp, priv->np_si[NR_TX],
|
|
&poll_ctx->wait_table);
|
|
poll_ctx->next_entry++;
|
|
|
|
poll_ctx->next_wake_fun = direct_rx ?
|
|
sync_kloop_rx_irq_wake_fun : NULL;
|
|
poll_wait(priv->np_filp, priv->np_si[NR_RX],
|
|
&poll_ctx->wait_table);
|
|
poll_ctx->next_entry++;
|
|
NMG_UNLOCK();
|
|
}
|
|
#else /* SYNC_KLOOP_POLL */
|
|
opt->nro_status = EOPNOTSUPP;
|
|
goto out;
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
}
|
|
|
|
nm_prinf("kloop busy_wait %u, direct_tx %u, direct_rx %u, "
|
|
"na_could_sleep %u", busy_wait, direct_tx, direct_rx,
|
|
na_could_sleep);
|
|
|
|
/* Main loop. */
|
|
for (;;) {
|
|
if (unlikely(NM_ACCESS_ONCE(priv->np_kloop_state) & NM_SYNC_KLOOP_STOPPING)) {
|
|
break;
|
|
}
|
|
|
|
#ifdef SYNC_KLOOP_POLL
|
|
if (!busy_wait) {
|
|
/* It is important to set the task state as
|
|
* interruptible before processing any TX/RX ring,
|
|
* so that if a notification on ring Y comes after
|
|
* we have processed ring Y, but before we call
|
|
* schedule(), we don't miss it. This is true because
|
|
* the wake up function will change the task state,
|
|
* and therefore the schedule_timeout() call below
|
|
* will observe the change).
|
|
*/
|
|
set_current_state(TASK_INTERRUPTIBLE);
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
/* Process all the TX rings bound to this file descriptor. */
|
|
for (i = 0; !direct_tx && i < num_tx_rings; i++) {
|
|
struct sync_kloop_ring_args *a = args + i;
|
|
netmap_sync_kloop_tx_ring(a);
|
|
}
|
|
|
|
/* Process all the RX rings bound to this file descriptor. */
|
|
for (i = 0; !direct_rx && i < num_rx_rings; i++) {
|
|
struct sync_kloop_ring_args *a = args + num_tx_rings + i;
|
|
netmap_sync_kloop_rx_ring(a);
|
|
}
|
|
|
|
if (busy_wait) {
|
|
/* Default synchronization method: sleep for a while. */
|
|
usleep_range(sleep_us, sleep_us);
|
|
}
|
|
#ifdef SYNC_KLOOP_POLL
|
|
else {
|
|
/* Yield to the scheduler waiting for a notification
|
|
* to come either from netmap or the application. */
|
|
schedule_timeout(msecs_to_jiffies(3000));
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
}
|
|
out:
|
|
#ifdef SYNC_KLOOP_POLL
|
|
if (poll_ctx) {
|
|
/* Stop polling from netmap and the eventfds, and deallocate
|
|
* the poll context. */
|
|
if (!busy_wait) {
|
|
__set_current_state(TASK_RUNNING);
|
|
}
|
|
for (i = 0; i < poll_ctx->next_entry; i++) {
|
|
struct sync_kloop_poll_entry *entry =
|
|
poll_ctx->entries + i;
|
|
|
|
if (entry->wqh)
|
|
remove_wait_queue(entry->wqh, &entry->wait);
|
|
/* We did not get a reference to the eventfds, but
|
|
* don't do that on netmap file descriptors (since
|
|
* a reference was not taken. */
|
|
if (entry->filp && entry->filp != priv->np_filp)
|
|
fput(entry->filp);
|
|
if (entry->irq_ctx)
|
|
eventfd_ctx_put(entry->irq_ctx);
|
|
if (entry->irq_filp)
|
|
fput(entry->irq_filp);
|
|
}
|
|
nm_os_free(poll_ctx);
|
|
poll_ctx = NULL;
|
|
}
|
|
#endif /* SYNC_KLOOP_POLL */
|
|
|
|
if (args) {
|
|
nm_os_free(args);
|
|
args = NULL;
|
|
}
|
|
|
|
/* Reset the kloop state. */
|
|
NMG_LOCK();
|
|
priv->np_kloop_state = 0;
|
|
if (na_could_sleep) {
|
|
na->na_flags |= NAF_BDG_MAYSLEEP;
|
|
}
|
|
NMG_UNLOCK();
|
|
|
|
return err;
|
|
}
|
|
|
|
int
|
|
netmap_sync_kloop_stop(struct netmap_priv_d *priv)
|
|
{
|
|
struct netmap_adapter *na;
|
|
bool running = true;
|
|
int err = 0;
|
|
|
|
if (priv->np_nifp == NULL) {
|
|
return ENXIO;
|
|
}
|
|
mb(); /* make sure following reads are not from cache */
|
|
|
|
na = priv->np_na;
|
|
if (!nm_netmap_on(na)) {
|
|
return ENXIO;
|
|
}
|
|
|
|
/* Set the kloop stopping flag. */
|
|
NMG_LOCK();
|
|
priv->np_kloop_state |= NM_SYNC_KLOOP_STOPPING;
|
|
NMG_UNLOCK();
|
|
|
|
/* Send a notification to the kloop, in case it is blocked in
|
|
* schedule_timeout(). We can use either RX or TX, because the
|
|
* kloop is waiting on both. */
|
|
nm_os_selwakeup(priv->np_si[NR_RX]);
|
|
|
|
/* Wait for the kloop to actually terminate. */
|
|
while (running) {
|
|
usleep_range(1000, 1500);
|
|
NMG_LOCK();
|
|
running = (NM_ACCESS_ONCE(priv->np_kloop_state)
|
|
& NM_SYNC_KLOOP_RUNNING);
|
|
NMG_UNLOCK();
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
#ifdef WITH_PTNETMAP
|
|
/*
|
|
* Guest ptnetmap txsync()/rxsync() routines, used in ptnet device drivers.
|
|
* These routines are reused across the different operating systems supported
|
|
* by netmap.
|
|
*/
|
|
|
|
/*
|
|
* Reconcile host and guest views of the transmit ring.
|
|
*
|
|
* Guest user wants to transmit packets up to the one before ring->head,
|
|
* and guest kernel knows tx_ring->hwcur is the first packet unsent
|
|
* by the host kernel.
|
|
*
|
|
* We push out as many packets as possible, and possibly
|
|
* reclaim buffers from previously completed transmission.
|
|
*
|
|
* Notifications from the host are enabled only if the user guest would
|
|
* block (no space in the ring).
|
|
*/
|
|
bool
|
|
netmap_pt_guest_txsync(struct nm_csb_atok *atok, struct nm_csb_ktoa *ktoa,
|
|
struct netmap_kring *kring, int flags)
|
|
{
|
|
bool notify = false;
|
|
|
|
/* Disable notifications */
|
|
atok->appl_need_kick = 0;
|
|
|
|
/*
|
|
* First part: tell the host to process the new packets,
|
|
* updating the CSB.
|
|
*/
|
|
kring->nr_hwcur = ktoa->hwcur;
|
|
nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
|
|
|
|
/* Ask for a kick from a guest to the host if needed. */
|
|
if (((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
|
|
&& NM_ACCESS_ONCE(ktoa->kern_need_kick)) ||
|
|
(flags & NAF_FORCE_RECLAIM)) {
|
|
atok->sync_flags = flags;
|
|
notify = true;
|
|
}
|
|
|
|
/*
|
|
* Second part: reclaim buffers for completed transmissions.
|
|
*/
|
|
if (nm_kr_wouldblock(kring) || (flags & NAF_FORCE_RECLAIM)) {
|
|
nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
|
|
&kring->nr_hwcur);
|
|
}
|
|
|
|
/*
|
|
* No more room in the ring for new transmissions. The user thread will
|
|
* go to sleep and we need to be notified by the host when more free
|
|
* space is available.
|
|
*/
|
|
if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
|
|
/* Re-enable notifications. */
|
|
atok->appl_need_kick = 1;
|
|
/* Double check, with store-load memory barrier. */
|
|
nm_stld_barrier();
|
|
nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
|
|
&kring->nr_hwcur);
|
|
/* If there is new free space, disable notifications */
|
|
if (unlikely(!nm_kr_wouldblock(kring))) {
|
|
atok->appl_need_kick = 0;
|
|
}
|
|
}
|
|
|
|
nm_prdis(1, "%s CSB(head:%u cur:%u hwtail:%u) KRING(head:%u cur:%u tail:%u)",
|
|
kring->name, atok->head, atok->cur, ktoa->hwtail,
|
|
kring->rhead, kring->rcur, kring->nr_hwtail);
|
|
|
|
return notify;
|
|
}
|
|
|
|
/*
|
|
* Reconcile host and guest view of the receive ring.
|
|
*
|
|
* Update hwcur/hwtail from host (reading from CSB).
|
|
*
|
|
* If guest user has released buffers up to the one before ring->head, we
|
|
* also give them to the host.
|
|
*
|
|
* Notifications from the host are enabled only if the user guest would
|
|
* block (no more completed slots in the ring).
|
|
*/
|
|
bool
|
|
netmap_pt_guest_rxsync(struct nm_csb_atok *atok, struct nm_csb_ktoa *ktoa,
|
|
struct netmap_kring *kring, int flags)
|
|
{
|
|
bool notify = false;
|
|
|
|
/* Disable notifications */
|
|
atok->appl_need_kick = 0;
|
|
|
|
/*
|
|
* First part: import newly received packets, by updating the kring
|
|
* hwtail to the hwtail known from the host (read from the CSB).
|
|
* This also updates the kring hwcur.
|
|
*/
|
|
nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur);
|
|
kring->nr_kflags &= ~NKR_PENDINTR;
|
|
|
|
/*
|
|
* Second part: tell the host about the slots that guest user has
|
|
* released, by updating cur and head in the CSB.
|
|
*/
|
|
if (kring->rhead != kring->nr_hwcur) {
|
|
nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
|
|
}
|
|
|
|
/*
|
|
* No more completed RX slots. The user thread will go to sleep and
|
|
* we need to be notified by the host when more RX slots have been
|
|
* completed.
|
|
*/
|
|
if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
|
|
/* Re-enable notifications. */
|
|
atok->appl_need_kick = 1;
|
|
/* Double check, with store-load memory barrier. */
|
|
nm_stld_barrier();
|
|
nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
|
|
&kring->nr_hwcur);
|
|
/* If there are new slots, disable notifications. */
|
|
if (!nm_kr_wouldblock(kring)) {
|
|
atok->appl_need_kick = 0;
|
|
}
|
|
}
|
|
|
|
/* Ask for a kick from the guest to the host if needed. */
|
|
if ((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
|
|
&& NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
|
|
atok->sync_flags = flags;
|
|
notify = true;
|
|
}
|
|
|
|
nm_prdis(1, "%s CSB(head:%u cur:%u hwtail:%u) KRING(head:%u cur:%u tail:%u)",
|
|
kring->name, atok->head, atok->cur, ktoa->hwtail,
|
|
kring->rhead, kring->rcur, kring->nr_hwtail);
|
|
|
|
return notify;
|
|
}
|
|
|
|
/*
|
|
* Callbacks for ptnet drivers: nm_krings_create, nm_krings_delete, nm_dtor.
|
|
*/
|
|
int
|
|
ptnet_nm_krings_create(struct netmap_adapter *na)
|
|
{
|
|
struct netmap_pt_guest_adapter *ptna =
|
|
(struct netmap_pt_guest_adapter *)na; /* Upcast. */
|
|
struct netmap_adapter *na_nm = &ptna->hwup.up;
|
|
struct netmap_adapter *na_dr = &ptna->dr.up;
|
|
int ret;
|
|
|
|
if (ptna->backend_users) {
|
|
return 0;
|
|
}
|
|
|
|
/* Create krings on the public netmap adapter. */
|
|
ret = netmap_hw_krings_create(na_nm);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
|
|
/* Copy krings into the netmap adapter private to the driver. */
|
|
na_dr->tx_rings = na_nm->tx_rings;
|
|
na_dr->rx_rings = na_nm->rx_rings;
|
|
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
ptnet_nm_krings_delete(struct netmap_adapter *na)
|
|
{
|
|
struct netmap_pt_guest_adapter *ptna =
|
|
(struct netmap_pt_guest_adapter *)na; /* Upcast. */
|
|
struct netmap_adapter *na_nm = &ptna->hwup.up;
|
|
struct netmap_adapter *na_dr = &ptna->dr.up;
|
|
|
|
if (ptna->backend_users) {
|
|
return;
|
|
}
|
|
|
|
na_dr->tx_rings = NULL;
|
|
na_dr->rx_rings = NULL;
|
|
|
|
netmap_hw_krings_delete(na_nm);
|
|
}
|
|
|
|
void
|
|
ptnet_nm_dtor(struct netmap_adapter *na)
|
|
{
|
|
struct netmap_pt_guest_adapter *ptna =
|
|
(struct netmap_pt_guest_adapter *)na;
|
|
|
|
netmap_mem_put(ptna->dr.up.nm_mem);
|
|
memset(&ptna->dr, 0, sizeof(ptna->dr));
|
|
netmap_mem_pt_guest_ifp_del(na->nm_mem, na->ifp);
|
|
}
|
|
|
|
int
|
|
netmap_pt_guest_attach(struct netmap_adapter *arg,
|
|
unsigned int nifp_offset, unsigned int memid)
|
|
{
|
|
struct netmap_pt_guest_adapter *ptna;
|
|
if_t ifp = arg ? arg->ifp : NULL;
|
|
int error;
|
|
|
|
/* get allocator */
|
|
arg->nm_mem = netmap_mem_pt_guest_new(ifp, nifp_offset, memid);
|
|
if (arg->nm_mem == NULL)
|
|
return ENOMEM;
|
|
arg->na_flags |= NAF_MEM_OWNER;
|
|
error = netmap_attach_ext(arg, sizeof(struct netmap_pt_guest_adapter), 1);
|
|
if (error)
|
|
return error;
|
|
|
|
/* get the netmap_pt_guest_adapter */
|
|
ptna = (struct netmap_pt_guest_adapter *) NA(ifp);
|
|
|
|
/* Initialize a separate pass-through netmap adapter that is going to
|
|
* be used by the ptnet driver only, and so never exposed to netmap
|
|
* applications. We only need a subset of the available fields. */
|
|
memset(&ptna->dr, 0, sizeof(ptna->dr));
|
|
ptna->dr.up.ifp = ifp;
|
|
ptna->dr.up.nm_mem = netmap_mem_get(ptna->hwup.up.nm_mem);
|
|
ptna->dr.up.nm_config = ptna->hwup.up.nm_config;
|
|
|
|
ptna->backend_users = 0;
|
|
|
|
return 0;
|
|
}
|
|
|
|
#endif /* WITH_PTNETMAP */
|