d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/ionic: add optimized non-scattered Rx/Tx

Browse Source 
The code is very similar, but the simple case can skip a few branches
in the hot path. This improves PPS when 10KB mbufs are used.

S/G is enabled on the Rx side by offload DEV_RX_OFFLOAD_SCATTER.
S/G is enabled on the Tx side by offload DEV_TX_OFFLOAD_MULTI_SEGS.

S/G is automatically enabled on the Rx side if the provided mbufs are
too small to hold the maximum possible frame.

To enable S/G in testpmd, add these args:
  --rx-offloads=0x2000 --tx-offloads=0x8000

Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
Signed-off-by: R Mohamed Shah <mohamedshah.r@amd.com>
This commit is contained in:
Andrew Boyer 2022-10-18 12:41:28 -07:00 committed by Ferruh Yigit
parent 07512941de
commit e86a6fcc7c
9 changed files with 1054 additions and 571 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/guides/rel_notes/release_22_11.rst
View File

@ -143,6 +143,7 @@ New Features
* Added support for advertising packet types.
* Added support for descriptor status functions.
* Added Q-in-CMB feature controlled by device option ``ionic_cmb``.
* Added optimized handlers for non-scattered Rx and Tx.
* **Updated Intel iavf driver.**

25
drivers/net/ionic/ionic_ethdev.c
View File

@ -828,8 +828,6 @@ ionic_dev_configure(struct rte_eth_dev *eth_dev)
ionic_lif_configure(lif);
ionic_lif_set_features(lif);
return 0;
}
@ -883,6 +881,13 @@ ionic_dev_start(struct rte_eth_dev *eth_dev)
if (dev_conf->lpbk_mode)
IONIC_PRINT(WARNING, "Loopback mode not supported");
/* Re-set features in case SG flag was added in rx_queue_setup() */
err = ionic_lif_set_features(lif);
if (err) {
IONIC_PRINT(ERR, "Cannot set LIF features: %d", err);
return err;
}
lif->frame_size = eth_dev->data->mtu + IONIC_ETH_OVERHEAD;
err = ionic_lif_change_mtu(lif, eth_dev->data->mtu);
@ -917,6 +922,18 @@ ionic_dev_start(struct rte_eth_dev *eth_dev)
speed);
}
if (lif->hw_features & IONIC_ETH_HW_RX_SG)
eth_dev->rx_pkt_burst = &ionic_recv_pkts_sg;
else
eth_dev->rx_pkt_burst = &ionic_recv_pkts;
if (lif->hw_features & IONIC_ETH_HW_TX_SG)
eth_dev->tx_pkt_burst = &ionic_xmit_pkts_sg;
else
eth_dev->tx_pkt_burst = &ionic_xmit_pkts;
eth_dev->tx_pkt_prepare = &ionic_prep_pkts;
ionic_dev_link_update(eth_dev, 0);
return 0;
@ -980,10 +997,6 @@ eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params)
IONIC_PRINT_CALL();
eth_dev->dev_ops = &ionic_eth_dev_ops;
eth_dev->rx_pkt_burst = &ionic_recv_pkts;
eth_dev->tx_pkt_burst = &ionic_xmit_pkts;
eth_dev->tx_pkt_prepare = &ionic_prep_pkts;
eth_dev->rx_descriptor_status = ionic_dev_rx_descriptor_status;
eth_dev->tx_descriptor_status = ionic_dev_tx_descriptor_status;

61
drivers/net/ionic/ionic_lif.c
View File

@ -755,11 +755,10 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,
struct ionic_rx_qcq **rxq_out)
{
struct ionic_rx_qcq *rxq;
uint16_t flags, seg_size, hdr_seg_size, max_segs, max_segs_fw;
uint16_t flags = 0, seg_size, hdr_seg_size, max_segs, max_segs_fw = 1;
uint32_t max_mtu;
int err;
flags = IONIC_QCQ_F_SG;
if (lif->state & IONIC_LIF_F_Q_IN_CMB)
flags |= IONIC_QCQ_F_CMB;
@ -770,7 +769,18 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,
max_mtu = rte_le_to_cpu_32(lif->adapter->ident.lif.eth.max_mtu);
max_segs_fw = IONIC_RX_MAX_SG_ELEMS + 1;
/* If mbufs are too small to hold received packets, enable SG */
if (max_mtu > hdr_seg_size) {
IONIC_PRINT(NOTICE, "Enabling RX_OFFLOAD_SCATTER");
lif->eth_dev->data->dev_conf.rxmode.offloads |=
RTE_ETH_RX_OFFLOAD_SCATTER;
ionic_lif_configure_rx_sg_offload(lif);
}
if (lif->features & IONIC_ETH_HW_RX_SG) {
flags |= IONIC_QCQ_F_SG;
max_segs_fw = IONIC_RX_MAX_SG_ELEMS + 1;
}
/*
* Calculate how many fragment pointers might be stored in queue.
@ -820,14 +830,17 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,
uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out)
{
struct ionic_tx_qcq *txq;
uint16_t flags, num_segs_fw;
uint16_t flags = 0, num_segs_fw = 1;
int err;
flags = IONIC_QCQ_F_SG;
if (lif->features & IONIC_ETH_HW_TX_SG) {
flags |= IONIC_QCQ_F_SG;
num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1;
}
if (lif->state & IONIC_LIF_F_Q_IN_CMB)
flags |= IONIC_QCQ_F_CMB;
num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1;
IONIC_PRINT(DEBUG, "txq %u num_segs %u", index, num_segs_fw);
err = ionic_qcq_alloc(lif,
IONIC_QTYPE_TXQ,
@ -1561,8 +1574,7 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
.type = q->type,
.ver = lif->qtype_info[q->type].version,
.index = rte_cpu_to_le_32(q->index),
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG |
IONIC_QINIT_F_ENA),
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
.ring_size = rte_log2_u32(q->num_descs),
.ring_base = rte_cpu_to_le_64(q->base_pa),
@ -1572,6 +1584,8 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
};
int err;
if (txq->flags & IONIC_QCQ_F_SG)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
if (txq->flags & IONIC_QCQ_F_CMB)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
@ -1615,8 +1629,7 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
.type = q->type,
.ver = lif->qtype_info[q->type].version,
.index = rte_cpu_to_le_32(q->index),
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG |
IONIC_QINIT_F_ENA),
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
.ring_size = rte_log2_u32(q->num_descs),
.ring_base = rte_cpu_to_le_64(q->base_pa),
@ -1626,6 +1639,8 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
};
int err;
if (rxq->flags & IONIC_QCQ_F_SG)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
if (rxq->flags & IONIC_QCQ_F_CMB)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
@ -1791,6 +1806,20 @@ ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask)
}
}
void
ionic_lif_configure_rx_sg_offload(struct ionic_lif *lif)
{
struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode;
if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) {
lif->features |= IONIC_ETH_HW_RX_SG;
lif->eth_dev->data->scattered_rx = 1;
} else {
lif->features &= ~IONIC_ETH_HW_RX_SG;
lif->eth_dev->data->scattered_rx = 0;
}
}
void
ionic_lif_configure(struct ionic_lif *lif)
{
@ -1836,13 +1865,11 @@ ionic_lif_configure(struct ionic_lif *lif)
else
lif->features &= ~IONIC_ETH_HW_RX_CSUM;
if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) {
lif->features |= IONIC_ETH_HW_RX_SG;
lif->eth_dev->data->scattered_rx = 1;
} else {
lif->features &= ~IONIC_ETH_HW_RX_SG;
lif->eth_dev->data->scattered_rx = 0;
}
/*
* NB: RX_SG may be enabled later during rx_queue_setup() if
* required by the mbuf/mtu configuration
*/
ionic_lif_configure_rx_sg_offload(lif);
/* Covers VLAN_STRIP */
ionic_lif_configure_vlan_offload(lif, RTE_ETH_VLAN_STRIP_MASK);

1
drivers/net/ionic/ionic_lif.h
View File

@ -188,6 +188,7 @@ void ionic_lif_stop(struct ionic_lif *lif);
void ionic_lif_configure(struct ionic_lif *lif);
void ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask);
void ionic_lif_configure_rx_sg_offload(struct ionic_lif *lif);
void ionic_lif_reset(struct ionic_lif *lif);
int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr);

576
drivers/net/ionic/ionic_rxtx.c
View File

@ -2,50 +2,28 @@
* Copyright 2018-2022 Advanced Micro Devices, Inc.
*/
#include <sys/queue.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <stdint.h>
#include <stdarg.h>
#include <unistd.h>
#include <inttypes.h>
#include <rte_byteorder.h>
#include <rte_common.h>
#include <rte_cycles.h>
#include <rte_byteorder.h>
#include <rte_errno.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_interrupts.h>
#include <rte_pci.h>
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_atomic.h>
#include <rte_branch_prediction.h>
#include <rte_mempool.h>
#include <rte_malloc.h>
#include <rte_mbuf.h>
#include <rte_ether.h>
#include <ethdev_driver.h>
#include <rte_prefetch.h>
#include <rte_udp.h>
#include <rte_tcp.h>
#include <rte_sctp.h>
#include <rte_string_fns.h>
#include <rte_errno.h>
#include <rte_ip.h>
#include <rte_net.h>
#include <rte_tcp.h>
#include <rte_ethdev.h>
#include <ethdev_driver.h>
#include "ionic_logs.h"
#include "ionic_mac_api.h"
#include "ionic_ethdev.h"
#include "ionic.h"
#include "ionic_dev.h"
#include "ionic_lif.h"
#include "ionic_ethdev.h"
#include "ionic_rxtx.h"
#include "ionic_logs.h"
static void
ionic_empty_array(void **array, uint32_t cnt, uint16_t idx)
@ -103,60 +81,6 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
}
static __rte_always_inline void
ionic_tx_flush(struct ionic_tx_qcq *txq)
{
struct ionic_cq *cq = &txq->qcq.cq;
struct ionic_queue *q = &txq->qcq.q;
struct rte_mbuf *txm;
struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base;
void **info;
uint32_t i;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
if ((cq->tail_idx & 0x3) == 0)
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
/* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2)));
/* Prefetch next mbuf */
void **next_info =
IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1));
if (next_info[0])
rte_mbuf_prefetch_part2(next_info[0]);
if (next_info[1])
rte_mbuf_prefetch_part2(next_info[1]);
info = IONIC_INFO_PTR(q, q->tail_idx);
for (i = 0; i < q->num_segs; i++) {
txm = info[i];
if (!txm)
break;
if (txq->flags & IONIC_QCQ_F_FAST_FREE)
rte_mempool_put(txm->pool, txm);
else
rte_pktmbuf_free_seg(txm);
info[i] = NULL;
}
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
}
cq_desc = &cq_desc_base[cq->tail_idx];
}
}
void __rte_cold
ionic_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
{
@ -394,7 +318,7 @@ ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
return desc;
}
static int
int
ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
{
struct ionic_queue *q = &txq->qcq.q;
@ -405,7 +329,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
rte_iova_t data_iova;
uint64_t desc_addr = 0, next_addr;
uint16_t desc_len = 0;
uint8_t desc_nsge;
uint8_t desc_nsge = 0;
uint32_t hdrlen;
uint32_t mss = txm->tso_segsz;
uint32_t frag_left = 0;
@ -416,6 +340,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
bool start, done;
bool encap;
bool has_vlan = !!(txm->ol_flags & RTE_MBUF_F_TX_VLAN);
bool use_sgl = !!(txq->flags & IONIC_QCQ_F_SG);
uint16_t vlan_tci = txm->vlan_tci;
uint64_t ol_flags = txm->ol_flags;
@ -438,48 +363,22 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
}
seglen = hdrlen + mss;
left = txm->data_len;
data_iova = rte_mbuf_data_iova(txm);
desc = ionic_tx_tso_next(txq, &elem);
txm_seg = txm;
start = true;
seglen = hdrlen + mss;
/* Chop data up into desc segments */
while (left > 0) {
len = RTE_MIN(seglen, left);
frag_left = seglen - len;
desc_addr = rte_cpu_to_le_64(data_iova + offset);
desc_len = len;
desc_nsge = 0;
left -= len;
offset += len;
if (txm->nb_segs > 1 && frag_left > 0)
continue;
done = (txm->nb_segs == 1 && left == 0);
ionic_tx_tso_post(q, desc, txm,
desc_addr, desc_nsge, desc_len,
hdrlen, mss,
encap,
vlan_tci, has_vlan,
start, done);
desc = ionic_tx_tso_next(txq, &elem);
start = false;
seglen = mss;
}
/* Chop frags into desc segments */
txm_seg = txm->next;
/* Walk the chain of mbufs */
while (txm_seg != NULL) {
offset = 0;
data_iova = rte_mbuf_data_iova(txm_seg);
left = txm_seg->data_len;
/* Split the mbuf data up into multiple descriptors */
while (left > 0) {
next_addr = rte_cpu_to_le_64(data_iova + offset);
if (frag_left > 0) {
if (frag_left > 0 && use_sgl) {
/* Fill previous descriptor's SGE */
len = RTE_MIN(frag_left, left);
frag_left -= len;
elem->addr = next_addr;
@ -487,16 +386,19 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
elem++;
desc_nsge++;
} else {
len = RTE_MIN(mss, left);
frag_left = mss - len;
/* Fill new descriptor's data field */
len = RTE_MIN(seglen, left);
frag_left = seglen - len;
desc_addr = next_addr;
desc_len = len;
desc_nsge = 0;
}
left -= len;
offset += len;
if (txm_seg->next != NULL && frag_left > 0)
continue;
/* Pack the next mbuf's data into the descriptor */
if (txm_seg->next != NULL && frag_left > 0 && use_sgl)
break;
done = (txm_seg->next == NULL && left == 0);
ionic_tx_tso_post(q, desc, txm_seg,
@ -507,6 +409,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
start, done);
desc = ionic_tx_tso_next(txq, &elem);
start = false;
seglen = mss;
}
txm_seg = txm_seg->next;
@ -517,157 +420,6 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
return 0;
}
static __rte_always_inline int
ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
{
struct ionic_queue *q = &txq->qcq.q;
struct ionic_txq_desc *desc, *desc_base = q->base;
struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
struct ionic_txq_sg_elem *elem;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *txm_seg;
void **info;
rte_iova_t data_iova;
uint64_t ol_flags = txm->ol_flags;
uint64_t addr, cmd;
uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
uint8_t flags = 0;
desc = &desc_base[q->head_idx];
info = IONIC_INFO_PTR(q, q->head_idx);
if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_L3)) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
}
if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
}
if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
stats->no_csum++;
if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
}
if (ol_flags & RTE_MBUF_F_TX_VLAN) {
flags |= IONIC_TXQ_DESC_FLAG_VLAN;
desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
}
addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
desc->cmd = rte_cpu_to_le_64(cmd);
desc->len = rte_cpu_to_le_16(txm->data_len);
info[0] = txm;
if (txm->nb_segs > 1) {
txm_seg = txm->next;
elem = sg_desc_base[q->head_idx].elems;
while (txm_seg != NULL) {
/* Stash the mbuf ptr in the array */
info++;
*info = txm_seg;
/* Configure the SGE */
data_iova = rte_mbuf_data_iova(txm_seg);
elem->len = rte_cpu_to_le_16(txm_seg->data_len);
elem->addr = rte_cpu_to_le_64(data_iova);
elem++;
txm_seg = txm_seg->next;
}
}
q->head_idx = Q_NEXT_TO_POST(q, 1);
return 0;
}
uint16_t
ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct ionic_tx_qcq *txq = tx_queue;
struct ionic_queue *q = &txq->qcq.q;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *mbuf;
uint32_t bytes_tx = 0;
uint16_t nb_avail, nb_tx = 0;
int err;
struct ionic_txq_desc *desc_base = q->base;
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[q->head_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
if (tx_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[0]);
rte_mbuf_prefetch_part2(tx_pkts[0]);
}
if (unlikely(ionic_q_space_avail(q) < txq->free_thresh)) {
/* Cleaning old buffers */
ionic_tx_flush(txq);
}
nb_avail = ionic_q_space_avail(q);
if (unlikely(nb_avail < nb_pkts)) {
stats->stop += nb_pkts - nb_avail;
nb_pkts = nb_avail;
}
while (nb_tx < nb_pkts) {
uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[next_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
if (nb_tx + 1 < nb_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
}
mbuf = tx_pkts[nb_tx];
if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
err = ionic_tx_tso(txq, mbuf);
else
err = ionic_tx(txq, mbuf);
if (err) {
stats->drop += nb_pkts - nb_tx;
break;
}
bytes_tx += mbuf->pkt_len;
nb_tx++;
}
if (nb_tx > 0) {
rte_wmb();
ionic_q_flush(q);
stats->packets += nb_tx;
stats->bytes += bytes_tx;
}
return nb_tx;
}
/*********************************************************************
*
* TX prep functions
@ -820,7 +572,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
}
#define IONIC_CSUM_FLAG_MASK (IONIC_RXQ_COMP_CSUM_F_VLAN - 1)
static const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK]
const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK]
__rte_cache_aligned = {
/* IP_BAD set */
[IONIC_RXQ_COMP_CSUM_F_IP_BAD] = RTE_MBUF_F_RX_IP_CKSUM_BAD,
@ -850,7 +602,7 @@ static const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK]
};
/* RTE_PTYPE_UNKNOWN is 0x0 */
static const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK]
const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK]
__rte_cache_aligned = {
[IONIC_PKT_TYPE_NON_IP] = RTE_PTYPE_UNKNOWN,
[IONIC_PKT_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
@ -884,203 +636,6 @@ ionic_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
return ptypes;
}
/*
* Cleans one descriptor. Connects the filled mbufs into a chain.
* Does not advance the tail index.
*/
static __rte_always_inline void
ionic_rx_clean_one(struct ionic_rx_qcq *rxq,
struct ionic_rxq_comp *cq_desc,
struct ionic_rx_service *rx_svc)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm, *rxm_seg, *prev_rxm;
struct ionic_rx_stats *stats = &rxq->stats;
uint64_t pkt_flags = 0;
uint32_t pkt_type;
uint32_t left, i;
uint16_t cq_desc_len;
uint8_t ptype, cflags;
void **info;
cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
info = IONIC_INFO_PTR(q, q->tail_idx);
rxm = info[0];
if (cq_desc->status) {
stats->bad_cq_status++;
return;
}
if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
stats->bad_len++;
return;
}
info[0] = NULL;
/* Set the mbuf metadata based on the cq entry */
rxm->rearm_data[0] = rxq->rearm_data;
rxm->pkt_len = cq_desc_len;
rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len);
left = cq_desc_len - rxm->data_len;
rxm->nb_segs = cq_desc->num_sg_elems + 1;
prev_rxm = rxm;
for (i = 1; i < rxm->nb_segs && left; i++) {
rxm_seg = info[i];
info[i] = NULL;
/* Set the chained mbuf metadata */
rxm_seg->rearm_data[0] = rxq->rearm_seg_data;
rxm_seg->data_len = RTE_MIN(rxq->seg_size, left);
left -= rxm_seg->data_len;
/* Link the mbuf */
prev_rxm->next = rxm_seg;
prev_rxm = rxm_seg;
}
/* Terminate the mbuf chain */
prev_rxm->next = NULL;
/* RSS */
pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
/* Vlan Strip */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
}
/* Checksum */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
pkt_flags |= ionic_csum_flags[cflags];
}
rxm->ol_flags = pkt_flags;
/* Packet Type */
ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
pkt_type = ionic_ptype_table[ptype];
if (pkt_type == RTE_PTYPE_UNKNOWN) {
struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
struct rte_ether_hdr *);
uint16_t ether_type = eth_h->ether_type;
if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
pkt_type = RTE_PTYPE_L2_ETHER_ARP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
stats->mtods++;
}
rxm->packet_type = pkt_type;
rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
rx_svc->nb_rx++;
stats->packets++;
stats->bytes += rxm->pkt_len;
}
/*
* Fills one descriptor with mbufs. Does not advance the head index.
*/
static __rte_always_inline int
ionic_rx_fill_one(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm, *rxm_seg;
struct ionic_rxq_desc *desc, *desc_base = q->base;
struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
rte_iova_t data_iova;
uint32_t i;
void **info;
int ret;
info = IONIC_INFO_PTR(q, q->head_idx);
desc = &desc_base[q->head_idx];
sg_desc = &sg_desc_base[q->head_idx];
/* mbuf is unused => whole chain is unused */
if (unlikely(info[0]))
return 0;
if (rxq->mb_idx == 0) {
ret = rte_mempool_get_bulk(rxq->mb_pool,
(void **)rxq->mbs,
IONIC_MBUF_BULK_ALLOC);
if (ret) {
assert(0);
return -ENOMEM;
}
rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
}
rxm = rxq->mbs[--rxq->mb_idx];
info[0] = rxm;
data_iova = rte_mbuf_data_iova_default(rxm);
desc->addr = rte_cpu_to_le_64(data_iova);
for (i = 1; i < q->num_segs; i++) {
/* mbuf is unused => rest of the chain is unused */
if (info[i])
return 0;
if (rxq->mb_idx == 0) {
ret = rte_mempool_get_bulk(rxq->mb_pool,
(void **)rxq->mbs,
IONIC_MBUF_BULK_ALLOC);
if (ret) {
assert(0);
return -ENOMEM;
}
rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
}
rxm_seg = rxq->mbs[--rxq->mb_idx];
info[i] = rxm_seg;
/* The data_off does not get set to 0 until later */
data_iova = rxm_seg->buf_iova;
sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova);
}
return 0;
}
/*
* Fills all descriptors with mbufs.
*/
static int __rte_cold
ionic_rx_fill(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
uint32_t i;
int err;
for (i = 1; i < q->num_descs; i++) {
err = ionic_rx_fill_one(rxq);
if (err)
return err;
q->head_idx = Q_NEXT_TO_POST(q, 1);
}
ionic_q_flush(q);
return 0;
}
/*
* Perform one-time initialization of descriptor fields
* which will not change for the life of the queue.
@ -1148,10 +703,13 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
if (err)
return err;
/* Allocate buffers for descriptor rings */
if (ionic_rx_fill(rxq) != 0) {
IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
rx_queue_id);
/* Allocate buffers for descriptor ring */
if (rxq->flags & IONIC_QCQ_F_SG)
err = ionic_rx_fill_sg(rxq);
else
err = ionic_rx_fill(rxq);
if (err != 0) {
IONIC_PRINT(ERR, "Could not fill queue %d", rx_queue_id);
return -1;
}
@ -1160,55 +718,6 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
return 0;
}
/*
* Walk the CQ to find completed receive descriptors.
* Any completed descriptor found is refilled.
*/
static __rte_always_inline void
ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
struct ionic_rx_service *rx_svc)
{
struct ionic_cq *cq = &rxq->qcq.cq;
struct ionic_queue *q = &rxq->qcq.q;
struct ionic_rxq_desc *q_desc_base = q->base;
struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
uint32_t work_done = 0;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 8 x 8B bufinfo */
rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8)));
/* Prefetch 4 x 16B comp */
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
/* Prefetch 4 x 16B descriptors */
if (!(rxq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
ionic_rx_clean_one(rxq, cq_desc, rx_svc);
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
(void)ionic_rx_fill_one(rxq);
q->head_idx = Q_NEXT_TO_POST(q, 1);
if (++work_done == work_to_do)
break;
cq_desc = &cq_desc_base[cq->tail_idx];
}
/* Update the queue indices and ring the doorbell */
if (work_done)
ionic_q_flush(q);
}
/*
* Stop Receive Units for specified queue.
*/
@ -1237,21 +746,6 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
return 0;
}
uint16_t
ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct ionic_rx_qcq *rxq = rx_queue;
struct ionic_rx_service rx_svc;
rx_svc.rx_pkts = rx_pkts;
rx_svc.nb_rx = 0;
ionic_rxq_service(rxq, nb_pkts, &rx_svc);
return rx_svc.nb_rx;
}
int
ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
{

46
drivers/net/ionic/ionic_rxtx.h
View File

@ -5,7 +5,19 @@
#ifndef _IONIC_RXTX_H_
#define _IONIC_RXTX_H_
#include <rte_mbuf.h>
#include <stdint.h>
#include "ionic_if.h"
struct ionic_rx_qcq;
struct ionic_tx_qcq;
struct rte_eth_dev;
struct rte_eth_rxconf;
struct rte_eth_rxq_info;
struct rte_eth_txconf;
struct rte_eth_txq_info;
struct rte_mbuf;
struct rte_mempool;
struct ionic_rx_service {
/* cb in */
@ -14,13 +26,12 @@ struct ionic_rx_service {
uint16_t nb_rx;
};
uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
uint16_t ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
#define IONIC_CSUM_FLAG_MASK (IONIC_RXQ_COMP_CSUM_F_VLAN - 1)
extern const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK];
extern const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK];
/* ionic_rxtx.c */
int ionic_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
uint16_t nb_desc, uint32_t socket_id,
const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp);
@ -45,4 +56,25 @@ int ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset);
const uint32_t *ionic_dev_supported_ptypes_get(struct rte_eth_dev *dev);
int ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm);
uint16_t ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
/* ionic_rxtx_simple.c */
uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
int ionic_rx_fill(struct ionic_rx_qcq *rxq);
/* ionic_rxtx_sg.c */
uint16_t ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq);
#endif /* _IONIC_RXTX_H_ */

496
drivers/net/ionic/ionic_rxtx_sg.c Normal file
View File

@ -0,0 +1,496 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2018-2022 Advanced Micro Devices, Inc.
*/
#include <stdio.h>
#include <errno.h>
#include <stdint.h>
#include <assert.h>
#include <rte_common.h>
#include <rte_byteorder.h>
#include <rte_atomic.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
#include <rte_ether.h>
#include <rte_prefetch.h>
#include "ionic.h"
#include "ionic_if.h"
#include "ionic_dev.h"
#include "ionic_lif.h"
#include "ionic_rxtx.h"
static __rte_always_inline void
ionic_tx_flush_sg(struct ionic_tx_qcq *txq)
{
struct ionic_cq *cq = &txq->qcq.cq;
struct ionic_queue *q = &txq->qcq.q;
struct rte_mbuf *txm;
struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base;
void **info;
uint32_t i;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
if ((cq->tail_idx & 0x3) == 0)
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
/* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2)));
/* Prefetch next mbuf */
void **next_info =
IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1));
if (next_info[0])
rte_mbuf_prefetch_part2(next_info[0]);
if (next_info[1])
rte_mbuf_prefetch_part2(next_info[1]);
info = IONIC_INFO_PTR(q, q->tail_idx);
for (i = 0; i < q->num_segs; i++) {
txm = info[i];
if (!txm)
break;
if (txq->flags & IONIC_QCQ_F_FAST_FREE)
rte_mempool_put(txm->pool, txm);
else
rte_pktmbuf_free_seg(txm);
info[i] = NULL;
}
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
}
cq_desc = &cq_desc_base[cq->tail_idx];
}
}
static __rte_always_inline int
ionic_tx_sg(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
{
struct ionic_queue *q = &txq->qcq.q;
struct ionic_txq_desc *desc, *desc_base = q->base;
struct ionic_txq_sg_desc_v1 *sg_desc, *sg_desc_base = q->sg_base;
struct ionic_txq_sg_elem *elem;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *txm_seg;
rte_iova_t data_iova;
void **info;
uint64_t ol_flags = txm->ol_flags;
uint64_t addr, cmd;
uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
uint8_t flags = 0;
desc = &desc_base[q->head_idx];
sg_desc = &sg_desc_base[q->head_idx];
info = IONIC_INFO_PTR(q, q->head_idx);
if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_L3)) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
}
if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
}
if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
stats->no_csum++;
if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
}
if (ol_flags & RTE_MBUF_F_TX_VLAN) {
flags |= IONIC_TXQ_DESC_FLAG_VLAN;
desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
}
addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
desc->cmd = rte_cpu_to_le_64(cmd);
desc->len = rte_cpu_to_le_16(txm->data_len);
info[0] = txm;
if (txm->nb_segs > 1) {
txm_seg = txm->next;
elem = sg_desc->elems;
while (txm_seg != NULL) {
/* Stash the mbuf ptr in the array */
info++;
*info = txm_seg;
/* Configure the SGE */
data_iova = rte_mbuf_data_iova(txm_seg);
elem->len = rte_cpu_to_le_16(txm_seg->data_len);
elem->addr = rte_cpu_to_le_64(data_iova);
elem++;
txm_seg = txm_seg->next;
}
}
q->head_idx = Q_NEXT_TO_POST(q, 1);
return 0;
}
uint16_t
ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct ionic_tx_qcq *txq = tx_queue;
struct ionic_queue *q = &txq->qcq.q;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *mbuf;
uint32_t bytes_tx = 0;
uint16_t nb_avail, nb_tx = 0;
int err;
struct ionic_txq_desc *desc_base = q->base;
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[q->head_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
if (tx_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[0]);
rte_mbuf_prefetch_part2(tx_pkts[0]);
}
if (ionic_q_space_avail(q) < txq->free_thresh) {
/* Cleaning old buffers */
ionic_tx_flush_sg(txq);
}
nb_avail = ionic_q_space_avail(q);
if (nb_avail < nb_pkts) {
stats->stop += nb_pkts - nb_avail;
nb_pkts = nb_avail;
}
while (nb_tx < nb_pkts) {
uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[next_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
if (nb_tx + 1 < nb_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
}
mbuf = tx_pkts[nb_tx];
if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
err = ionic_tx_tso(txq, mbuf);
else
err = ionic_tx_sg(txq, mbuf);
if (err) {
stats->drop += nb_pkts - nb_tx;
break;
}
bytes_tx += mbuf->pkt_len;
nb_tx++;
}
if (nb_tx > 0) {
rte_wmb();
ionic_q_flush(q);
stats->packets += nb_tx;
stats->bytes += bytes_tx;
}
return nb_tx;
}
/*
* Cleans one descriptor. Connects the filled mbufs into a chain.
* Does not advance the tail index.
*/
static __rte_always_inline void
ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq,
struct ionic_rxq_comp *cq_desc,
struct ionic_rx_service *rx_svc)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm;
struct rte_mbuf *rxm_seg, *prev_rxm;
struct ionic_rx_stats *stats = &rxq->stats;
uint64_t pkt_flags = 0;
uint32_t pkt_type;
uint32_t left, i;
uint16_t cq_desc_len;
uint8_t ptype, cflags;
void **info;
cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
info = IONIC_INFO_PTR(q, q->tail_idx);
rxm = info[0];
if (cq_desc->status) {
stats->bad_cq_status++;
return;
}
if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
stats->bad_len++;
return;
}
info[0] = NULL;
/* Set the mbuf metadata based on the cq entry */
rxm->rearm_data[0] = rxq->rearm_data;
rxm->pkt_len = cq_desc_len;
rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len);
left = cq_desc_len - rxm->data_len;
rxm->nb_segs = cq_desc->num_sg_elems + 1;
prev_rxm = rxm;
for (i = 1; i < rxm->nb_segs && left; i++) {
rxm_seg = info[i];
info[i] = NULL;
/* Set the chained mbuf metadata */
rxm_seg->rearm_data[0] = rxq->rearm_seg_data;
rxm_seg->data_len = RTE_MIN(rxq->seg_size, left);
left -= rxm_seg->data_len;
/* Link the mbuf */
prev_rxm->next = rxm_seg;
prev_rxm = rxm_seg;
}
/* Terminate the mbuf chain */
prev_rxm->next = NULL;
/* RSS */
pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
/* Vlan Strip */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
}
/* Checksum */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
pkt_flags |= ionic_csum_flags[cflags];
}
rxm->ol_flags = pkt_flags;
/* Packet Type */
ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
pkt_type = ionic_ptype_table[ptype];
if (pkt_type == RTE_PTYPE_UNKNOWN) {
struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
struct rte_ether_hdr *);
uint16_t ether_type = eth_h->ether_type;
if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
pkt_type = RTE_PTYPE_L2_ETHER_ARP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
stats->mtods++;
} else if (pkt_flags & RTE_MBUF_F_RX_VLAN) {
pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
} else {
pkt_type |= RTE_PTYPE_L2_ETHER;
}
rxm->packet_type = pkt_type;
rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
rx_svc->nb_rx++;
stats->packets++;
stats->bytes += rxm->pkt_len;
}
/*
* Fills one descriptor with mbufs. Does not advance the head index.
*/
static __rte_always_inline int
ionic_rx_fill_one_sg(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm;
struct rte_mbuf *rxm_seg;
struct ionic_rxq_desc *desc, *desc_base = q->base;
struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
rte_iova_t data_iova;
uint32_t i;
void **info;
int ret;
info = IONIC_INFO_PTR(q, q->head_idx);
desc = &desc_base[q->head_idx];
sg_desc = &sg_desc_base[q->head_idx];
/* mbuf is unused => whole chain is unused */
if (info[0])
return 0;
if (rxq->mb_idx == 0) {
ret = rte_mempool_get_bulk(rxq->mb_pool,
(void **)rxq->mbs,
IONIC_MBUF_BULK_ALLOC);
if (ret) {
assert(0);
return -ENOMEM;
}
rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
}
rxm = rxq->mbs[--rxq->mb_idx];
info[0] = rxm;
data_iova = rte_mbuf_data_iova_default(rxm);
desc->addr = rte_cpu_to_le_64(data_iova);
for (i = 1; i < q->num_segs; i++) {
/* mbuf is unused => rest of the chain is unused */
if (info[i])
return 0;
if (rxq->mb_idx == 0) {
ret = rte_mempool_get_bulk(rxq->mb_pool,
(void **)rxq->mbs,
IONIC_MBUF_BULK_ALLOC);
if (ret) {
assert(0);
return -ENOMEM;
}
rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
}
rxm_seg = rxq->mbs[--rxq->mb_idx];
info[i] = rxm_seg;
/* The data_off does not get set to 0 until later */
data_iova = rxm_seg->buf_iova;
sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova);
}
return 0;
}
/*
* Walk the CQ to find completed receive descriptors.
* Any completed descriptor found is refilled.
*/
static __rte_always_inline void
ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
struct ionic_rx_service *rx_svc)
{
struct ionic_cq *cq = &rxq->qcq.cq;
struct ionic_queue *q = &rxq->qcq.q;
struct ionic_rxq_desc *q_desc_base = q->base;
struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
uint32_t work_done = 0;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 8 x 8B bufinfo */
rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8)));
/* Prefetch 4 x 16B comp */
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
/* Prefetch 4 x 16B descriptors */
if (!(rxq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
/* Clean one descriptor */
ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc);
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
/* Fill one descriptor */
(void)ionic_rx_fill_one_sg(rxq);
q->head_idx = Q_NEXT_TO_POST(q, 1);
if (++work_done == work_to_do)
break;
cq_desc = &cq_desc_base[cq->tail_idx];
}
/* Update the queue indices and ring the doorbell */
if (work_done)
ionic_q_flush(q);
}
uint16_t
ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct ionic_rx_qcq *rxq = rx_queue;
struct ionic_rx_service rx_svc;
rx_svc.rx_pkts = rx_pkts;
rx_svc.nb_rx = 0;
ionic_rxq_service_sg(rxq, nb_pkts, &rx_svc);
return rx_svc.nb_rx;
}
/*
* Fills all descriptors with mbufs.
*/
int __rte_cold
ionic_rx_fill_sg(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
uint32_t i;
int err = 0;
for (i = 0; i < q->num_descs - 1u; i++) {
err = ionic_rx_fill_one_sg(rxq);
if (err)
break;
q->head_idx = Q_NEXT_TO_POST(q, 1);
}
ionic_q_flush(q);
return err;
}

417
drivers/net/ionic/ionic_rxtx_simple.c Normal file
View File

@ -0,0 +1,417 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2018-2022 Advanced Micro Devices, Inc.
*/
#include <stdio.h>
#include <errno.h>
#include <stdint.h>
#include <assert.h>
#include <rte_common.h>
#include <rte_byteorder.h>
#include <rte_atomic.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
#include <rte_ether.h>
#include <rte_prefetch.h>
#include "ionic.h"
#include "ionic_if.h"
#include "ionic_dev.h"
#include "ionic_lif.h"
#include "ionic_rxtx.h"
static __rte_always_inline void
ionic_tx_flush(struct ionic_tx_qcq *txq)
{
struct ionic_cq *cq = &txq->qcq.cq;
struct ionic_queue *q = &txq->qcq.q;
struct rte_mbuf *txm;
struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base;
void **info;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 4 x 16B comp at cq->tail_idx + 4 */
if ((cq->tail_idx & 0x3) == 0)
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) {
/* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */
rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]);
/* Prefetch next mbuf */
void **next_info =
&q->info[Q_NEXT_TO_SRVC(q, 1)];
if (next_info[0])
rte_mbuf_prefetch_part2(next_info[0]);
info = &q->info[q->tail_idx];
{
txm = info[0];
if (txq->flags & IONIC_QCQ_F_FAST_FREE)
rte_mempool_put(txm->pool, txm);
else
rte_pktmbuf_free_seg(txm);
info[0] = NULL;
}
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
}
cq_desc = &cq_desc_base[cq->tail_idx];
}
}
static __rte_always_inline int
ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
{
struct ionic_queue *q = &txq->qcq.q;
struct ionic_txq_desc *desc, *desc_base = q->base;
struct ionic_tx_stats *stats = &txq->stats;
void **info;
uint64_t ol_flags = txm->ol_flags;
uint64_t addr, cmd;
uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
uint8_t flags = 0;
if (txm->nb_segs > 1)
return -EINVAL;
desc = &desc_base[q->head_idx];
info = &q->info[q->head_idx];
if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_L3)) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
}
if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) &&
(txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
}
if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
stats->no_csum++;
if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) &&
((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) ||
(ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) {
flags |= IONIC_TXQ_DESC_FLAG_ENCAP;
}
if (ol_flags & RTE_MBUF_F_TX_VLAN) {
flags |= IONIC_TXQ_DESC_FLAG_VLAN;
desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci);
}
addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
cmd = encode_txq_desc_cmd(opcode, flags, 0, addr);
desc->cmd = rte_cpu_to_le_64(cmd);
desc->len = rte_cpu_to_le_16(txm->data_len);
info[0] = txm;
q->head_idx = Q_NEXT_TO_POST(q, 1);
return 0;
}
uint16_t
ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct ionic_tx_qcq *txq = tx_queue;
struct ionic_queue *q = &txq->qcq.q;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *mbuf;
uint32_t bytes_tx = 0;
uint16_t nb_avail, nb_tx = 0;
int err;
struct ionic_txq_desc *desc_base = q->base;
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[q->head_idx]);
rte_prefetch0(&q->info[q->head_idx]);
if (tx_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[0]);
rte_mbuf_prefetch_part2(tx_pkts[0]);
}
if (ionic_q_space_avail(q) < txq->free_thresh) {
/* Cleaning old buffers */
ionic_tx_flush(txq);
}
nb_avail = ionic_q_space_avail(q);
if (nb_avail < nb_pkts) {
stats->stop += nb_pkts - nb_avail;
nb_pkts = nb_avail;
}
while (nb_tx < nb_pkts) {
uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
if (!(txq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&desc_base[next_idx]);
rte_prefetch0(&q->info[next_idx]);
if (nb_tx + 1 < nb_pkts) {
rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]);
rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]);
}
mbuf = tx_pkts[nb_tx];
if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
err = ionic_tx_tso(txq, mbuf);
else
err = ionic_tx(txq, mbuf);
if (err) {
stats->drop += nb_pkts - nb_tx;
break;
}
bytes_tx += mbuf->pkt_len;
nb_tx++;
}
if (nb_tx > 0) {
rte_wmb();
ionic_q_flush(q);
stats->packets += nb_tx;
stats->bytes += bytes_tx;
}
return nb_tx;
}
/*
* Cleans one descriptor. Connects the filled mbufs into a chain.
* Does not advance the tail index.
*/
static __rte_always_inline void
ionic_rx_clean_one(struct ionic_rx_qcq *rxq,
struct ionic_rxq_comp *cq_desc,
struct ionic_rx_service *rx_svc)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm;
struct ionic_rx_stats *stats = &rxq->stats;
uint64_t pkt_flags = 0;
uint32_t pkt_type;
uint16_t cq_desc_len;
uint8_t ptype, cflags;
void **info;
cq_desc_len = rte_le_to_cpu_16(cq_desc->len);
info = &q->info[q->tail_idx];
rxm = info[0];
if (cq_desc->status) {
stats->bad_cq_status++;
return;
}
if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {
stats->bad_len++;
return;
}
info[0] = NULL;
/* Set the mbuf metadata based on the cq entry */
rxm->rearm_data[0] = rxq->rearm_data;
rxm->pkt_len = cq_desc_len;
rxm->data_len = cq_desc_len;
/* RSS */
pkt_flags |= RTE_MBUF_F_RX_RSS_HASH;
rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);
/* Vlan Strip */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci);
}
/* Checksum */
if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK;
pkt_flags |= ionic_csum_flags[cflags];
}
rxm->ol_flags = pkt_flags;
/* Packet Type */
ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK;
pkt_type = ionic_ptype_table[ptype];
if (pkt_type == RTE_PTYPE_UNKNOWN) {
struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
struct rte_ether_hdr *);
uint16_t ether_type = eth_h->ether_type;
if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
pkt_type = RTE_PTYPE_L2_ETHER_ARP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP))
pkt_type = RTE_PTYPE_L2_ETHER_LLDP;
else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588))
pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC;
stats->mtods++;
} else if (pkt_flags & RTE_MBUF_F_RX_VLAN) {
pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
} else {
pkt_type |= RTE_PTYPE_L2_ETHER;
}
rxm->packet_type = pkt_type;
rx_svc->rx_pkts[rx_svc->nb_rx] = rxm;
rx_svc->nb_rx++;
stats->packets++;
stats->bytes += rxm->pkt_len;
}
/*
* Fills one descriptor with mbufs. Does not advance the head index.
*/
static __rte_always_inline int
ionic_rx_fill_one(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
struct rte_mbuf *rxm;
struct ionic_rxq_desc *desc, *desc_base = q->base;
rte_iova_t data_iova;
void **info;
int ret;
info = &q->info[q->head_idx];
desc = &desc_base[q->head_idx];
/* mbuf is unused */
if (info[0])
return 0;
if (rxq->mb_idx == 0) {
ret = rte_mempool_get_bulk(rxq->mb_pool,
(void **)rxq->mbs,
IONIC_MBUF_BULK_ALLOC);
if (ret) {
assert(0);
return -ENOMEM;
}
rxq->mb_idx = IONIC_MBUF_BULK_ALLOC;
}
rxm = rxq->mbs[--rxq->mb_idx];
info[0] = rxm;
data_iova = rte_mbuf_data_iova_default(rxm);
desc->addr = rte_cpu_to_le_64(data_iova);
return 0;
}
/*
* Walk the CQ to find completed receive descriptors.
* Any completed descriptor found is refilled.
*/
static __rte_always_inline void
ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
struct ionic_rx_service *rx_svc)
{
struct ionic_cq *cq = &rxq->qcq.cq;
struct ionic_queue *q = &rxq->qcq.q;
struct ionic_rxq_desc *q_desc_base = q->base;
struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
uint32_t work_done = 0;
cq_desc = &cq_desc_base[cq->tail_idx];
while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
if (cq->tail_idx == 0)
cq->done_color = !cq->done_color;
/* Prefetch 8 x 8B bufinfo */
rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]);
/* Prefetch 4 x 16B comp */
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
/* Prefetch 4 x 16B descriptors */
if (!(rxq->flags & IONIC_QCQ_F_CMB))
rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
/* Clean one descriptor */
ionic_rx_clean_one(rxq, cq_desc, rx_svc);
q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
/* Fill one descriptor */
(void)ionic_rx_fill_one(rxq);
q->head_idx = Q_NEXT_TO_POST(q, 1);
if (++work_done == work_to_do)
break;
cq_desc = &cq_desc_base[cq->tail_idx];
}
/* Update the queue indices and ring the doorbell */
if (work_done)
ionic_q_flush(q);
}
uint16_t
ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct ionic_rx_qcq *rxq = rx_queue;
struct ionic_rx_service rx_svc;
rx_svc.rx_pkts = rx_pkts;
rx_svc.nb_rx = 0;
ionic_rxq_service(rxq, nb_pkts, &rx_svc);
return rx_svc.nb_rx;
}
/*
* Fills all descriptors with mbufs.
*/
int __rte_cold
ionic_rx_fill(struct ionic_rx_qcq *rxq)
{
struct ionic_queue *q = &rxq->qcq.q;
uint32_t i;
int err = 0;
for (i = 0; i < q->num_descs - 1u; i++) {
err = ionic_rx_fill_one(rxq);
if (err)
break;
q->head_idx = Q_NEXT_TO_POST(q, 1);
}
ionic_q_flush(q);
return err;
}

2
drivers/net/ionic/meson.build
View File

@ -16,4 +16,6 @@ sources = files(
'ionic_main.c',
'ionic_rx_filter.c',
'ionic_rxtx.c',
'ionic_rxtx_simple.c',
'ionic_rxtx_sg.c',
)