freebsd-dev/sys/dev/ena/ena_netmap.c
Marcin Wojtas 358bcc4c6c Add support for ENA NETMAP partial initialization
In NETMAP mode not all queues need to be allocated to NETMAP. Some of
them could be left to the kernel. Configuration is managed by the flags
nr_mode and nr_pending_mode provided per each NETMAP kring.

ENA driver checks those flags and perform proper rings initialization.

Differential Revision: https://reviews.freebsd.org/D21937
Submitted by: Rafal Kozik <rk@semihalf.com>
              Michal Krawczyk <mk@semihalf.com>
Obtained from: Semihalf
Sponsored by:  Amazon, Inc.
2019-10-31 16:02:42 +00:00

1096 lines
29 KiB
C

/*-
* BSD LICENSE
*
* Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#ifdef DEV_NETMAP
#include "ena.h"
#include "ena_netmap.h"
#define ENA_NETMAP_MORE_FRAMES 1
#define ENA_NETMAP_NO_MORE_FRAMES 0
#define ENA_MAX_FRAMES 16384
struct ena_netmap_ctx {
struct netmap_kring *kring;
struct ena_adapter *adapter;
struct netmap_adapter *na;
struct netmap_slot *slots;
struct ena_ring *ring;
struct ena_com_io_cq *io_cq;
struct ena_com_io_sq *io_sq;
u_int nm_i;
uint16_t nt;
uint16_t lim;
};
/* Netmap callbacks */
static int ena_netmap_reg(struct netmap_adapter *, int);
static int ena_netmap_txsync(struct netmap_kring *, int);
static int ena_netmap_rxsync(struct netmap_kring *, int);
/* Helper functions */
static int ena_netmap_tx_frames(struct ena_netmap_ctx *);
static int ena_netmap_tx_frame(struct ena_netmap_ctx *);
static inline uint16_t ena_netmap_count_slots(struct ena_netmap_ctx *);
static inline uint16_t ena_netmap_packet_len(struct netmap_slot *, u_int,
uint16_t);
static int ena_netmap_copy_data(struct netmap_adapter *,
struct netmap_slot *, u_int, uint16_t, uint16_t, void *);
static int ena_netmap_map_single_slot(struct netmap_adapter *,
struct netmap_slot *, bus_dma_tag_t, bus_dmamap_t, void **, uint64_t *);
static int ena_netmap_tx_map_slots(struct ena_netmap_ctx *,
struct ena_tx_buffer *, void **, uint16_t *, uint16_t *);
static void ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *,
struct ena_tx_buffer *);
static void ena_netmap_tx_cleanup(struct ena_netmap_ctx *);
static uint16_t ena_netmap_tx_clean_one(struct ena_netmap_ctx *,
uint16_t);
static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
static int ena_netmap_rx_frames(struct ena_netmap_ctx *);
static int ena_netmap_rx_frame(struct ena_netmap_ctx *);
static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t,
int *);
static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *);
static void ena_netmap_fill_ctx(struct netmap_kring *,
struct ena_netmap_ctx *, uint16_t);
int
ena_netmap_attach(struct ena_adapter *adapter)
{
struct netmap_adapter na;
ena_trace(ENA_NETMAP, "netmap attach\n");
bzero(&na, sizeof(na));
na.na_flags = NAF_MOREFRAG;
na.ifp = adapter->ifp;
na.num_tx_desc = adapter->tx_ring_size;
na.num_rx_desc = adapter->rx_ring_size;
na.num_tx_rings = adapter->num_queues;
na.num_rx_rings = adapter->num_queues;
na.rx_buf_maxsize = adapter->buf_ring_size;
na.nm_txsync = ena_netmap_txsync;
na.nm_rxsync = ena_netmap_rxsync;
na.nm_register = ena_netmap_reg;
return (netmap_attach(&na));
}
int
ena_netmap_alloc_rx_slot(struct ena_adapter *adapter,
struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
{
struct netmap_adapter *na = NA(adapter->ifp);
struct netmap_kring *kring;
struct netmap_ring *ring;
struct netmap_slot *slot;
void *addr;
uint64_t paddr;
int nm_i, qid, head, lim, rc;
/* if previously allocated frag is not used */
if (unlikely(rx_info->netmap_buf_idx != 0))
return (0);
qid = rx_ring->qid;
kring = na->rx_rings[qid];
nm_i = kring->nr_hwcur;
head = kring->rhead;
ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
"rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
if ((nm_i == head) && rx_ring->initialized) {
ena_trace(ENA_NETMAP, "No free slots in netmap ring\n");
return (ENOMEM);
}
ring = kring->ring;
if (ring == NULL) {
device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid);
return (EFAULT);
}
slot = &ring->slot[nm_i];
addr = PNMB(na, slot, &paddr);
if (addr == NETMAP_BUF_BASE(na)) {
device_printf(adapter->pdev, "Bad buff in slot\n");
return (EFAULT);
}
rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
if (rc != 0) {
ena_trace(ENA_WARNING, "DMA mapping error\n");
return (rc);
}
bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
rx_info->ena_buf.paddr = paddr;
rx_info->ena_buf.len = ring->nr_buf_size;
rx_info->mbuf = NULL;
rx_info->netmap_buf_idx = slot->buf_idx;
slot->buf_idx = 0;
lim = kring->nkr_num_slots - 1;
kring->nr_hwcur = nm_next(nm_i, lim);
return (0);
}
void
ena_netmap_free_rx_slot(struct ena_adapter *adapter,
struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
{
struct netmap_adapter *na;
struct netmap_kring *kring;
struct netmap_slot *slot;
int nm_i, qid, lim;
na = NA(adapter->ifp);
if (na == NULL) {
device_printf(adapter->pdev, "netmap adapter is NULL\n");
return;
}
if (na->rx_rings == NULL) {
device_printf(adapter->pdev, "netmap rings are NULL\n");
return;
}
qid = rx_ring->qid;
kring = na->rx_rings[qid];
if (kring == NULL) {
device_printf(adapter->pdev,
"netmap kernel ring %d is NULL\n", qid);
return;
}
lim = kring->nkr_num_slots - 1;
nm_i = nm_prev(kring->nr_hwcur, lim);
if (kring->nr_mode != NKR_NETMAP_ON)
return;
bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
BUS_DMASYNC_POSTREAD);
netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
KASSERT(kring->ring == NULL, ("Netmap Rx ring is NULL\n"));
slot = &kring->ring->slot[nm_i];
ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n");
slot->buf_idx = rx_info->netmap_buf_idx;
slot->flags = NS_BUF_CHANGED;
rx_info->netmap_buf_idx = 0;
kring->nr_hwcur = nm_i;
}
static bool
ena_ring_in_netmap(struct ena_adapter *adapter, int qid, enum txrx x)
{
struct netmap_adapter *na;
struct netmap_kring *kring;
if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
na = NA(adapter->ifp);
kring = (x == NR_RX) ? na->rx_rings[qid] : na->tx_rings[qid];
if (kring->nr_mode == NKR_NETMAP_ON)
return true;
}
return false;
}
bool
ena_tx_ring_in_netmap(struct ena_adapter *adapter, int qid)
{
return ena_ring_in_netmap(adapter, qid, NR_TX);
}
bool
ena_rx_ring_in_netmap(struct ena_adapter *adapter, int qid)
{
return ena_ring_in_netmap(adapter, qid, NR_RX);
}
static void
ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
{
if (!ena_ring_in_netmap(adapter, qid, x))
return;
netmap_reset(NA(adapter->ifp), x, qid, 0);
ena_trace(ENA_NETMAP, "%s ring %d is in netmap mode\n",
(x == NR_TX) ? "Tx" : "Rx", qid);
}
void
ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
{
ena_netmap_reset_ring(adapter, qid, NR_RX);
}
void
ena_netmap_reset_tx_ring(struct ena_adapter *adapter, int qid)
{
ena_netmap_reset_ring(adapter, qid, NR_TX);
}
static int
ena_netmap_reg(struct netmap_adapter *na, int onoff)
{
struct ifnet *ifp = na->ifp;
struct ena_adapter* adapter = ifp->if_softc;
struct netmap_kring *kring;
enum txrx t;
int rc, i;
sx_xlock(&adapter->ioctl_sx);
ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
ena_down(adapter);
if (onoff) {
ena_trace(ENA_NETMAP, "netmap on\n");
for_rx_tx(t) {
for (i = 0; i <= nma_get_nrings(na, t); i++) {
kring = NMR(na, t)[i];
if (nm_kring_pending_on(kring)) {
kring->nr_mode = NKR_NETMAP_ON;
}
}
}
nm_set_native_flags(na);
} else {
ena_trace(ENA_NETMAP, "netmap off\n");
nm_clear_native_flags(na);
for_rx_tx(t) {
for (i = 0; i <= nma_get_nrings(na, t); i++) {
kring = NMR(na, t)[i];
if (nm_kring_pending_off(kring)) {
kring->nr_mode = NKR_NETMAP_OFF;
}
}
}
}
rc = ena_up(adapter);
if (rc != 0) {
ena_trace(ENA_WARNING, "ena_up failed with rc=%d\n", rc);
adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
nm_clear_native_flags(na);
ena_destroy_device(adapter, false);
ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
rc = ena_restore_device(adapter);
}
sx_unlock(&adapter->ioctl_sx);
return (rc);
}
static int
ena_netmap_txsync(struct netmap_kring *kring, int flags)
{
struct ena_netmap_ctx ctx;
int rc = 0;
ena_netmap_fill_ctx(kring, &ctx, ENA_IO_TXQ_IDX(kring->ring_id));
ctx.ring = &ctx.adapter->tx_ring[kring->ring_id];
ENA_RING_MTX_LOCK(ctx.ring);
if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter)))
goto txsync_end;
if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
goto txsync_end;
rc = ena_netmap_tx_frames(&ctx);
ena_netmap_tx_cleanup(&ctx);
txsync_end:
ENA_RING_MTX_UNLOCK(ctx.ring);
return (rc);
}
static int
ena_netmap_tx_frames(struct ena_netmap_ctx *ctx)
{
struct ena_ring *tx_ring = ctx->ring;
int rc = 0;
ctx->nm_i = ctx->kring->nr_hwcur;
ctx->nt = ctx->ring->next_to_use;
__builtin_prefetch(&ctx->slots[ctx->nm_i]);
while (ctx->nm_i != ctx->kring->rhead) {
if ((rc = ena_netmap_tx_frame(ctx)) != 0) {
/*
* When there is no empty space in Tx ring, error is
* still being returned. It should not be passed to the
* netmap, as application knows current ring state from
* netmap ring pointers. Returning error there could
* cause application to exit, but the Tx ring is commonly
* being full.
*/
if (rc == ENA_COM_NO_MEM)
rc = 0;
break;
}
tx_ring->acum_pkts++;
}
/* If any packet was sent... */
if (likely(ctx->nm_i != ctx->kring->nr_hwcur)) {
wmb();
/* ...send the doorbell to the device. */
ena_com_write_sq_doorbell(ctx->io_sq);
counter_u64_add(ctx->ring->tx_stats.doorbells, 1);
tx_ring->acum_pkts = 0;
ctx->ring->next_to_use = ctx->nt;
ctx->kring->nr_hwcur = ctx->nm_i;
}
return (rc);
}
static int
ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
{
struct ena_com_tx_ctx ena_tx_ctx;
struct ena_adapter *adapter;
struct ena_ring *tx_ring;
struct ena_tx_buffer *tx_info;
uint16_t req_id;
uint16_t header_len;
uint16_t packet_len;
int nb_hw_desc;
int rc;
void *push_hdr;
adapter = ctx->adapter;
if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
ena_trace(ENA_WARNING, "Too many slots per packet\n");
return (EINVAL);
}
tx_ring = ctx->ring;
req_id = tx_ring->free_tx_ids[ctx->nt];
tx_info = &tx_ring->tx_buffer_info[req_id];
tx_info->num_of_bufs = 0;
tx_info->nm_info.sockets_used = 0;
rc = ena_netmap_tx_map_slots(ctx, tx_info, &push_hdr, &header_len,
&packet_len);
if (unlikely(rc != 0)) {
device_printf(adapter->pdev, "Failed to map Tx slot\n");
return (rc);
}
bzero(&ena_tx_ctx, sizeof(struct ena_com_tx_ctx));
ena_tx_ctx.ena_bufs = tx_info->bufs;
ena_tx_ctx.push_header = push_hdr;
ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
ena_tx_ctx.req_id = req_id;
ena_tx_ctx.header_len = header_len;
/* There are no any offloads, as the netmap doesn't support them */
if (tx_ring->acum_pkts == DB_THRESHOLD ||
ena_com_is_doorbell_needed(ctx->io_sq, &ena_tx_ctx)) {
wmb();
ena_com_write_sq_doorbell(ctx->io_sq);
counter_u64_add(tx_ring->tx_stats.doorbells, 1);
tx_ring->acum_pkts = 0;
}
rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
if (unlikely(rc != 0)) {
if (likely(rc == ENA_COM_NO_MEM)) {
ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
"Tx ring[%d] is out of space\n", tx_ring->que->id);
} else {
device_printf(adapter->pdev,
"Failed to prepare Tx bufs\n");
}
counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
ena_netmap_unmap_last_socket_chain(ctx, tx_info);
return (rc);
}
counter_enter();
counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
counter_u64_add_protected(tx_ring->tx_stats.bytes, packet_len);
counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
counter_u64_add_protected(adapter->hw_stats.tx_bytes, packet_len);
counter_exit();
tx_info->tx_descs = nb_hw_desc;
ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
for (unsigned int i = 0; i < tx_info->num_of_bufs; i++)
bus_dmamap_sync(adapter->tx_buf_tag,
tx_info->nm_info.map_seg[i], BUS_DMASYNC_PREWRITE);
return (0);
}
static inline uint16_t
ena_netmap_count_slots(struct ena_netmap_ctx *ctx)
{
uint16_t slots = 1;
uint16_t nm = ctx->nm_i;
while ((ctx->slots[nm].flags & NS_MOREFRAG) != 0) {
slots++;
nm = nm_next(nm, ctx->lim);
}
return slots;
}
static inline uint16_t
ena_netmap_packet_len(struct netmap_slot *slots, u_int slot_index,
uint16_t limit)
{
struct netmap_slot *nm_slot;
uint16_t packet_size = 0;
do {
nm_slot = &slots[slot_index];
packet_size += nm_slot->len;
slot_index = nm_next(slot_index, limit);
} while ((nm_slot->flags & NS_MOREFRAG) != 0);
return packet_size;
}
static int
ena_netmap_copy_data(struct netmap_adapter *na, struct netmap_slot *slots,
u_int slot_index, uint16_t limit, uint16_t bytes_to_copy, void *destination)
{
struct netmap_slot *nm_slot;
void *slot_vaddr;
uint16_t packet_size;
uint16_t data_amount;
packet_size = 0;
do {
nm_slot = &slots[slot_index];
slot_vaddr = NMB(na, nm_slot);
if (unlikely(slot_vaddr == NULL))
return (EINVAL);
data_amount = min_t(uint16_t, bytes_to_copy, nm_slot->len);
memcpy(destination, slot_vaddr, data_amount);
bytes_to_copy -= data_amount;
slot_index = nm_next(slot_index, limit);
} while ((nm_slot->flags & NS_MOREFRAG) != 0 && bytes_to_copy > 0);
return (0);
}
static int
ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
{
int rc;
*vaddr = PNMB(na, slot, paddr);
if (unlikely(vaddr == NULL)) {
ena_trace(ENA_ALERT, "Slot address is NULL\n");
return (EINVAL);
}
rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
if (unlikely(rc != 0)) {
ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n",
slot->buf_idx);
return (EINVAL);
}
return (0);
}
static int
ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx,
struct ena_tx_buffer *tx_info, void **push_hdr, uint16_t *header_len,
uint16_t *packet_len)
{
struct netmap_slot *slot;
struct ena_com_buf *ena_buf;
struct ena_adapter *adapter;
struct ena_ring *tx_ring;
struct ena_netmap_tx_info *nm_info;
bus_dmamap_t *nm_maps;
void *vaddr;
uint64_t paddr;
uint32_t *nm_buf_idx;
uint32_t slot_head_len;
uint32_t frag_len;
uint32_t remaining_len;
uint16_t push_len;
uint16_t delta;
int rc;
adapter = ctx->adapter;
tx_ring = ctx->ring;
ena_buf = tx_info->bufs;
nm_info = &tx_info->nm_info;
nm_maps = nm_info->map_seg;
nm_buf_idx = nm_info->socket_buf_idx;
slot = &ctx->slots[ctx->nm_i];
slot_head_len = slot->len;
*packet_len = ena_netmap_packet_len(ctx->slots, ctx->nm_i, ctx->lim);
remaining_len = *packet_len;
delta = 0;
__builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
/*
* When the device is in LLQ mode, the driver will copy
* the header into the device memory space.
* The ena_com layer assumes that the header is in a linear
* memory space.
* This assumption might be wrong since part of the header
* can be in the fragmented buffers.
* First, check if header fits in the first slot. If not, copy
* it to separate buffer that will be holding linearized data.
*/
push_len = min_t(uint32_t, *packet_len,
tx_ring->tx_max_header_size);
*header_len = push_len;
/* If header is in linear space, just point to socket's data. */
if (likely(push_len <= slot_head_len)) {
*push_hdr = NMB(ctx->na, slot);
if (unlikely(push_hdr == NULL)) {
device_printf(adapter->pdev,
"Slot vaddress is NULL\n");
return (EINVAL);
}
/*
* Otherwise, copy whole portion of header from multiple slots
* to intermediate buffer.
*/
} else {
rc = ena_netmap_copy_data(ctx->na,
ctx->slots,
ctx->nm_i,
ctx->lim,
push_len,
tx_ring->push_buf_intermediate_buf);
if (unlikely(rc)) {
device_printf(adapter->pdev,
"Failed to copy data from slots to push_buf\n");
return (EINVAL);
}
*push_hdr = tx_ring->push_buf_intermediate_buf;
counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
delta = push_len - slot_head_len;
}
ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
"slot: %d header_buf->vaddr: %p push_len: %d\n",
slot->buf_idx, *push_hdr, push_len);
/*
* If header was in linear memory space, map for the dma rest of the data
* in the first mbuf of the mbuf chain.
*/
if (slot_head_len > push_len) {
rc = ena_netmap_map_single_slot(ctx->na,
slot,
adapter->tx_buf_tag,
*nm_maps,
&vaddr,
&paddr);
if (unlikely(rc != 0)) {
device_printf(adapter->pdev,
"DMA mapping error\n");
return (rc);
}
nm_maps++;
ena_buf->paddr = paddr + push_len;
ena_buf->len = slot->len - push_len;
ena_buf++;
tx_info->num_of_bufs++;
}
remaining_len -= slot->len;
/* Save buf idx before advancing */
*nm_buf_idx = slot->buf_idx;
nm_buf_idx++;
slot->buf_idx = 0;
/* Advance to the next socket */
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
slot = &ctx->slots[ctx->nm_i];
nm_info->sockets_used++;
/*
* If header is in non linear space (delta > 0), then skip mbufs
* containing header and map the last one containing both header
* and the packet data.
* The first segment is already counted in.
*/
while (delta > 0) {
__builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
frag_len = slot->len;
/*
* If whole segment contains header just move to the
* next one and reduce delta.
*/
if (unlikely(delta >= frag_len)) {
delta -= frag_len;
} else {
/*
* Map the data and then assign it with the
* offsets
*/
rc = ena_netmap_map_single_slot(ctx->na,
slot,
adapter->tx_buf_tag,
*nm_maps,
&vaddr,
&paddr);
if (unlikely(rc != 0)) {
device_printf(adapter->pdev,
"DMA mapping error\n");
goto error_map;
}
nm_maps++;
ena_buf->paddr = paddr + delta;
ena_buf->len = slot->len - delta;
ena_buf++;
tx_info->num_of_bufs++;
delta = 0;
}
remaining_len -= slot->len;
/* Save buf idx before advancing */
*nm_buf_idx = slot->buf_idx;
nm_buf_idx++;
slot->buf_idx = 0;
/* Advance to the next socket */
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
slot = &ctx->slots[ctx->nm_i];
nm_info->sockets_used++;
}
} else {
*push_hdr = NULL;
/*
* header_len is just a hint for the device. Because netmap is
* not giving us any information about packet header length and
* it is not guaranteed that all packet headers will be in the
* 1st slot, setting header_len to 0 is making the device ignore
* this value and resolve header on it's own.
*/
*header_len = 0;
}
/* Map all remaining data (regular routine for non-LLQ mode) */
while (remaining_len > 0) {
__builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
rc = ena_netmap_map_single_slot(ctx->na,
slot,
adapter->tx_buf_tag,
*nm_maps,
&vaddr,
&paddr);
if (unlikely(rc != 0)) {
device_printf(adapter->pdev,
"DMA mapping error\n");
goto error_map;
}
nm_maps++;
ena_buf->paddr = paddr;
ena_buf->len = slot->len;
ena_buf++;
tx_info->num_of_bufs++;
remaining_len -= slot->len;
/* Save buf idx before advancing */
*nm_buf_idx = slot->buf_idx;
nm_buf_idx++;
slot->buf_idx = 0;
/* Advance to the next socket */
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
slot = &ctx->slots[ctx->nm_i];
nm_info->sockets_used++;
}
return (0);
error_map:
ena_netmap_unmap_last_socket_chain(ctx, tx_info);
return (rc);
}
static void
ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *ctx,
struct ena_tx_buffer *tx_info)
{
struct ena_netmap_tx_info *nm_info;
int n;
nm_info = &tx_info->nm_info;
/**
* As the used sockets must not be equal to the buffers used in the LLQ
* mode, they must be treated separately.
* First, unmap the DMA maps.
*/
n = tx_info->num_of_bufs;
while (n--) {
netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
nm_info->map_seg[n]);
}
tx_info->num_of_bufs = 0;
/* Next, retain the sockets back to the userspace */
n = nm_info->sockets_used;
while (n--) {
ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
nm_info->socket_buf_idx[n] = 0;
ctx->nm_i = nm_prev(ctx->nm_i, ctx->lim);
}
nm_info->sockets_used = 0;
}
static void
ena_netmap_tx_cleanup(struct ena_netmap_ctx *ctx)
{
uint16_t req_id;
uint16_t total_tx_descs = 0;
ctx->nm_i = ctx->kring->nr_hwtail;
ctx->nt = ctx->ring->next_to_clean;
/* Reclaim buffers for completed transmissions */
while (ena_com_tx_comp_req_id_get(ctx->io_cq, &req_id) >= 0) {
if (validate_tx_req_id(ctx->ring, req_id) != 0)
break;
total_tx_descs += ena_netmap_tx_clean_one(ctx, req_id);
}
ctx->kring->nr_hwtail = ctx->nm_i;
if (total_tx_descs > 0) {
/* acknowledge completion of sent packets */
ctx->ring->next_to_clean = ctx->nt;
ena_com_comp_ack(ctx->ring->ena_com_io_sq, total_tx_descs);
ena_com_update_dev_comp_head(ctx->ring->ena_com_io_cq);
}
}
static uint16_t
ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id)
{
struct ena_tx_buffer *tx_info;
struct ena_netmap_tx_info *nm_info;
int n;
tx_info = &ctx->ring->tx_buffer_info[req_id];
nm_info = &tx_info->nm_info;
/**
* As the used sockets must not be equal to the buffers used in the LLQ
* mode, they must be treated separately.
* First, unmap the DMA maps.
*/
n = tx_info->num_of_bufs;
for (n = 0; n < tx_info->num_of_bufs; n++) {
netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
nm_info->map_seg[n]);
}
tx_info->num_of_bufs = 0;
/* Next, retain the sockets back to the userspace */
for (n = 0; n < nm_info->sockets_used; n++) {
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0,
"Tx idx is not 0.\n");
ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
nm_info->socket_buf_idx[n] = 0;
}
nm_info->sockets_used = 0;
ctx->ring->free_tx_ids[ctx->nt] = req_id;
ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->lim);
return tx_info->tx_descs;
}
static inline int
validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
{
struct ena_adapter *adapter = tx_ring->adapter;
if (likely(req_id < tx_ring->ring_size))
return (0);
ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id);
counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
return (EFAULT);
}
static int
ena_netmap_rxsync(struct netmap_kring *kring, int flags)
{
struct ena_netmap_ctx ctx;
int rc;
ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id));
ctx.ring = &ctx.adapter->rx_ring[kring->ring_id];
if (ctx.kring->rhead > ctx.lim) {
/* Probably not needed to release slots from RX ring. */
return (netmap_ring_reinit(ctx.kring));
}
if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0))
return (0);
if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
return (0);
if ((rc = ena_netmap_rx_frames(&ctx)) != 0)
return (rc);
ena_netmap_rx_cleanup(&ctx);
return (0);
}
static inline int
ena_netmap_rx_frames(struct ena_netmap_ctx *ctx)
{
int rc = 0;
int frames_counter = 0;
ctx->nt = ctx->ring->next_to_clean;
ctx->nm_i = ctx->kring->nr_hwtail;
while((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) {
frames_counter++;
/* In case of multiple frames, it is not an error. */
rc = 0;
if (frames_counter > ENA_MAX_FRAMES) {
device_printf(ctx->adapter->pdev,
"Driver is stuck in the Rx loop\n");
break;
}
};
ctx->kring->nr_hwtail = ctx->nm_i;
ctx->kring->nr_kflags &= ~NKR_PENDINTR;
ctx->ring->next_to_clean = ctx->nt;
return (rc);
}
static inline int
ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
{
struct ena_com_rx_ctx ena_rx_ctx;
int rc, len = 0;
uint16_t buf, nm;
ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
if (unlikely(rc != 0)) {
ena_trace(ENA_ALERT, "Too many desc from the device.\n");
counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
ctx->adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, ctx->adapter);
return (rc);
}
if (unlikely(ena_rx_ctx.descs == 0))
return (ENA_NETMAP_NO_MORE_FRAMES);
ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
" %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
ena_rx_ctx.hash);
for (buf = 0; buf < ena_rx_ctx.descs; buf++)
if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
break;
/*
* ena_netmap_rx_load_desc doesn't know the number of descriptors.
* It just set flag NS_MOREFRAG to all slots, then here flag of
* last slot is cleared.
*/
ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED;
if (rc != 0) {
goto rx_clear_desc;
}
bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
counter_enter();
counter_u64_add_protected(ctx->ring->rx_stats.bytes, len);
counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len);
counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1);
counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1);
counter_exit();
return (ENA_NETMAP_MORE_FRAMES);
rx_clear_desc:
nm = ctx->nm_i;
/* Remove failed packet from ring */
while(buf--) {
ctx->slots[nm].flags = 0;
ctx->slots[nm].len = 0;
nm = nm_prev(nm, ctx->lim);
}
return (rc);
}
static inline int
ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
{
struct ena_rx_buffer *rx_info;
uint16_t req_id;
int rc;
req_id = ctx->ring->ena_bufs[buf].req_id;
rc = validate_rx_req_id(ctx->ring, req_id);
if (unlikely(rc != 0))
return (rc);
rx_info = &ctx->ring->rx_buffer_info[req_id];
bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
BUS_DMASYNC_POSTREAD);
netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n");
ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
rx_info->netmap_buf_idx = 0;
/*
* Set NS_MOREFRAG to all slots.
* Then ena_netmap_rx_frame clears it from last one.
*/
ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED;
ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
*len += ctx->slots[ctx->nm_i].len;
ctx->ring->free_rx_ids[ctx->nt] = req_id;
ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
rx_info, ctx->slots[ctx->nm_i].buf_idx,
(uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
return (0);
}
static inline void
ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx)
{
int refill_required;
refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur;
if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail)
refill_required -= 1;
if (refill_required == 0)
return;
else if (refill_required < 0)
refill_required += ctx->kring->nkr_num_slots;
ena_refill_rx_bufs(ctx->ring, refill_required);
}
static inline void
ena_netmap_fill_ctx(struct netmap_kring *kring, struct ena_netmap_ctx *ctx,
uint16_t ena_qid)
{
ctx->kring = kring;
ctx->na = kring->na;
ctx->adapter = ctx->na->ifp->if_softc;
ctx->lim = kring->nkr_num_slots - 1;
ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid];
ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid];
ctx->slots = kring->ring->slot;
}
void
ena_netmap_unload(struct ena_adapter *adapter, bus_dmamap_t map)
{
struct netmap_adapter *na = NA(adapter->ifp);
netmap_unload_map(na, adapter->tx_buf_tag, map);
}
#endif /* DEV_NETMAP */