From e86a6fcc7cf3774bf13b96778afec64df4d7f4ae Mon Sep 17 00:00:00 2001 From: Andrew Boyer Date: Tue, 18 Oct 2022 12:41:28 -0700 Subject: [PATCH] net/ionic: add optimized non-scattered Rx/Tx The code is very similar, but the simple case can skip a few branches in the hot path. This improves PPS when 10KB mbufs are used. S/G is enabled on the Rx side by offload DEV_RX_OFFLOAD_SCATTER. S/G is enabled on the Tx side by offload DEV_TX_OFFLOAD_MULTI_SEGS. S/G is automatically enabled on the Rx side if the provided mbufs are too small to hold the maximum possible frame. To enable S/G in testpmd, add these args: --rx-offloads=0x2000 --tx-offloads=0x8000 Signed-off-by: Andrew Boyer Signed-off-by: R Mohamed Shah --- doc/guides/rel_notes/release_22_11.rst | 1 + drivers/net/ionic/ionic_ethdev.c | 25 +- drivers/net/ionic/ionic_lif.c | 61 ++- drivers/net/ionic/ionic_lif.h | 1 + drivers/net/ionic/ionic_rxtx.c | 576 ++----------------------- drivers/net/ionic/ionic_rxtx.h | 46 +- drivers/net/ionic/ionic_rxtx_sg.c | 496 +++++++++++++++++++++ drivers/net/ionic/ionic_rxtx_simple.c | 417 ++++++++++++++++++ drivers/net/ionic/meson.build | 2 + 9 files changed, 1054 insertions(+), 571 deletions(-) create mode 100644 drivers/net/ionic/ionic_rxtx_sg.c create mode 100644 drivers/net/ionic/ionic_rxtx_simple.c diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst index b7dbfc8f7b..f1640c074b 100644 --- a/doc/guides/rel_notes/release_22_11.rst +++ b/doc/guides/rel_notes/release_22_11.rst @@ -143,6 +143,7 @@ New Features * Added support for advertising packet types. * Added support for descriptor status functions. * Added Q-in-CMB feature controlled by device option ``ionic_cmb``. + * Added optimized handlers for non-scattered Rx and Tx. * **Updated Intel iavf driver.** diff --git a/drivers/net/ionic/ionic_ethdev.c b/drivers/net/ionic/ionic_ethdev.c index 1092c7a921..340fd0cd59 100644 --- a/drivers/net/ionic/ionic_ethdev.c +++ b/drivers/net/ionic/ionic_ethdev.c @@ -828,8 +828,6 @@ ionic_dev_configure(struct rte_eth_dev *eth_dev) ionic_lif_configure(lif); - ionic_lif_set_features(lif); - return 0; } @@ -883,6 +881,13 @@ ionic_dev_start(struct rte_eth_dev *eth_dev) if (dev_conf->lpbk_mode) IONIC_PRINT(WARNING, "Loopback mode not supported"); + /* Re-set features in case SG flag was added in rx_queue_setup() */ + err = ionic_lif_set_features(lif); + if (err) { + IONIC_PRINT(ERR, "Cannot set LIF features: %d", err); + return err; + } + lif->frame_size = eth_dev->data->mtu + IONIC_ETH_OVERHEAD; err = ionic_lif_change_mtu(lif, eth_dev->data->mtu); @@ -917,6 +922,18 @@ ionic_dev_start(struct rte_eth_dev *eth_dev) speed); } + if (lif->hw_features & IONIC_ETH_HW_RX_SG) + eth_dev->rx_pkt_burst = &ionic_recv_pkts_sg; + else + eth_dev->rx_pkt_burst = &ionic_recv_pkts; + + if (lif->hw_features & IONIC_ETH_HW_TX_SG) + eth_dev->tx_pkt_burst = &ionic_xmit_pkts_sg; + else + eth_dev->tx_pkt_burst = &ionic_xmit_pkts; + + eth_dev->tx_pkt_prepare = &ionic_prep_pkts; + ionic_dev_link_update(eth_dev, 0); return 0; @@ -980,10 +997,6 @@ eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params) IONIC_PRINT_CALL(); eth_dev->dev_ops = &ionic_eth_dev_ops; - eth_dev->rx_pkt_burst = &ionic_recv_pkts; - eth_dev->tx_pkt_burst = &ionic_xmit_pkts; - eth_dev->tx_pkt_prepare = &ionic_prep_pkts; - eth_dev->rx_descriptor_status = ionic_dev_rx_descriptor_status; eth_dev->tx_descriptor_status = ionic_dev_tx_descriptor_status; diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c index eeec4860ba..383fff9ffd 100644 --- a/drivers/net/ionic/ionic_lif.c +++ b/drivers/net/ionic/ionic_lif.c @@ -755,11 +755,10 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, struct ionic_rx_qcq **rxq_out) { struct ionic_rx_qcq *rxq; - uint16_t flags, seg_size, hdr_seg_size, max_segs, max_segs_fw; + uint16_t flags = 0, seg_size, hdr_seg_size, max_segs, max_segs_fw = 1; uint32_t max_mtu; int err; - flags = IONIC_QCQ_F_SG; if (lif->state & IONIC_LIF_F_Q_IN_CMB) flags |= IONIC_QCQ_F_CMB; @@ -770,7 +769,18 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, max_mtu = rte_le_to_cpu_32(lif->adapter->ident.lif.eth.max_mtu); - max_segs_fw = IONIC_RX_MAX_SG_ELEMS + 1; + /* If mbufs are too small to hold received packets, enable SG */ + if (max_mtu > hdr_seg_size) { + IONIC_PRINT(NOTICE, "Enabling RX_OFFLOAD_SCATTER"); + lif->eth_dev->data->dev_conf.rxmode.offloads |= + RTE_ETH_RX_OFFLOAD_SCATTER; + ionic_lif_configure_rx_sg_offload(lif); + } + + if (lif->features & IONIC_ETH_HW_RX_SG) { + flags |= IONIC_QCQ_F_SG; + max_segs_fw = IONIC_RX_MAX_SG_ELEMS + 1; + } /* * Calculate how many fragment pointers might be stored in queue. @@ -820,14 +830,17 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index, uint16_t ntxq_descs, struct ionic_tx_qcq **txq_out) { struct ionic_tx_qcq *txq; - uint16_t flags, num_segs_fw; + uint16_t flags = 0, num_segs_fw = 1; int err; - flags = IONIC_QCQ_F_SG; + if (lif->features & IONIC_ETH_HW_TX_SG) { + flags |= IONIC_QCQ_F_SG; + num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1; + } if (lif->state & IONIC_LIF_F_Q_IN_CMB) flags |= IONIC_QCQ_F_CMB; - num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1; + IONIC_PRINT(DEBUG, "txq %u num_segs %u", index, num_segs_fw); err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, @@ -1561,8 +1574,7 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq) .type = q->type, .ver = lif->qtype_info[q->type].version, .index = rte_cpu_to_le_32(q->index), - .flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG | - IONIC_QINIT_F_ENA), + .flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA), .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE), .ring_size = rte_log2_u32(q->num_descs), .ring_base = rte_cpu_to_le_64(q->base_pa), @@ -1572,6 +1584,8 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq) }; int err; + if (txq->flags & IONIC_QCQ_F_SG) + ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG); if (txq->flags & IONIC_QCQ_F_CMB) ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB); @@ -1615,8 +1629,7 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq) .type = q->type, .ver = lif->qtype_info[q->type].version, .index = rte_cpu_to_le_32(q->index), - .flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG | - IONIC_QINIT_F_ENA), + .flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA), .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE), .ring_size = rte_log2_u32(q->num_descs), .ring_base = rte_cpu_to_le_64(q->base_pa), @@ -1626,6 +1639,8 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq) }; int err; + if (rxq->flags & IONIC_QCQ_F_SG) + ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG); if (rxq->flags & IONIC_QCQ_F_CMB) ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB); @@ -1791,6 +1806,20 @@ ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask) } } +void +ionic_lif_configure_rx_sg_offload(struct ionic_lif *lif) +{ + struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode; + + if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { + lif->features |= IONIC_ETH_HW_RX_SG; + lif->eth_dev->data->scattered_rx = 1; + } else { + lif->features &= ~IONIC_ETH_HW_RX_SG; + lif->eth_dev->data->scattered_rx = 0; + } +} + void ionic_lif_configure(struct ionic_lif *lif) { @@ -1836,13 +1865,11 @@ ionic_lif_configure(struct ionic_lif *lif) else lif->features &= ~IONIC_ETH_HW_RX_CSUM; - if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { - lif->features |= IONIC_ETH_HW_RX_SG; - lif->eth_dev->data->scattered_rx = 1; - } else { - lif->features &= ~IONIC_ETH_HW_RX_SG; - lif->eth_dev->data->scattered_rx = 0; - } + /* + * NB: RX_SG may be enabled later during rx_queue_setup() if + * required by the mbuf/mtu configuration + */ + ionic_lif_configure_rx_sg_offload(lif); /* Covers VLAN_STRIP */ ionic_lif_configure_vlan_offload(lif, RTE_ETH_VLAN_STRIP_MASK); diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h index 09ebca7e89..00c8add95c 100644 --- a/drivers/net/ionic/ionic_lif.h +++ b/drivers/net/ionic/ionic_lif.h @@ -188,6 +188,7 @@ void ionic_lif_stop(struct ionic_lif *lif); void ionic_lif_configure(struct ionic_lif *lif); void ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask); +void ionic_lif_configure_rx_sg_offload(struct ionic_lif *lif); void ionic_lif_reset(struct ionic_lif *lif); int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr); diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c index f43f9c1149..d216980450 100644 --- a/drivers/net/ionic/ionic_rxtx.c +++ b/drivers/net/ionic/ionic_rxtx.c @@ -2,50 +2,28 @@ * Copyright 2018-2022 Advanced Micro Devices, Inc. */ -#include #include -#include #include #include #include -#include -#include -#include -#include #include -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include -#include -#include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include -#include "ionic_logs.h" -#include "ionic_mac_api.h" -#include "ionic_ethdev.h" +#include "ionic.h" +#include "ionic_dev.h" #include "ionic_lif.h" +#include "ionic_ethdev.h" #include "ionic_rxtx.h" +#include "ionic_logs.h" static void ionic_empty_array(void **array, uint32_t cnt, uint16_t idx) @@ -103,60 +81,6 @@ ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; } -static __rte_always_inline void -ionic_tx_flush(struct ionic_tx_qcq *txq) -{ - struct ionic_cq *cq = &txq->qcq.cq; - struct ionic_queue *q = &txq->qcq.q; - struct rte_mbuf *txm; - struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; - void **info; - uint32_t i; - - cq_desc = &cq_desc_base[cq->tail_idx]; - - while (color_match(cq_desc->color, cq->done_color)) { - cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); - if (cq->tail_idx == 0) - cq->done_color = !cq->done_color; - - /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ - if ((cq->tail_idx & 0x3) == 0) - rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); - - while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { - /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ - rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2))); - - /* Prefetch next mbuf */ - void **next_info = - IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1)); - if (next_info[0]) - rte_mbuf_prefetch_part2(next_info[0]); - if (next_info[1]) - rte_mbuf_prefetch_part2(next_info[1]); - - info = IONIC_INFO_PTR(q, q->tail_idx); - for (i = 0; i < q->num_segs; i++) { - txm = info[i]; - if (!txm) - break; - - if (txq->flags & IONIC_QCQ_F_FAST_FREE) - rte_mempool_put(txm->pool, txm); - else - rte_pktmbuf_free_seg(txm); - - info[i] = NULL; - } - - q->tail_idx = Q_NEXT_TO_SRVC(q, 1); - } - - cq_desc = &cq_desc_base[cq->tail_idx]; - } -} - void __rte_cold ionic_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) { @@ -394,7 +318,7 @@ ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) return desc; } -static int +int ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) { struct ionic_queue *q = &txq->qcq.q; @@ -405,7 +329,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) rte_iova_t data_iova; uint64_t desc_addr = 0, next_addr; uint16_t desc_len = 0; - uint8_t desc_nsge; + uint8_t desc_nsge = 0; uint32_t hdrlen; uint32_t mss = txm->tso_segsz; uint32_t frag_left = 0; @@ -416,6 +340,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) bool start, done; bool encap; bool has_vlan = !!(txm->ol_flags & RTE_MBUF_F_TX_VLAN); + bool use_sgl = !!(txq->flags & IONIC_QCQ_F_SG); uint16_t vlan_tci = txm->vlan_tci; uint64_t ol_flags = txm->ol_flags; @@ -438,48 +363,22 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; } - seglen = hdrlen + mss; - left = txm->data_len; - data_iova = rte_mbuf_data_iova(txm); - desc = ionic_tx_tso_next(txq, &elem); + txm_seg = txm; start = true; + seglen = hdrlen + mss; - /* Chop data up into desc segments */ - - while (left > 0) { - len = RTE_MIN(seglen, left); - frag_left = seglen - len; - desc_addr = rte_cpu_to_le_64(data_iova + offset); - desc_len = len; - desc_nsge = 0; - left -= len; - offset += len; - if (txm->nb_segs > 1 && frag_left > 0) - continue; - done = (txm->nb_segs == 1 && left == 0); - ionic_tx_tso_post(q, desc, txm, - desc_addr, desc_nsge, desc_len, - hdrlen, mss, - encap, - vlan_tci, has_vlan, - start, done); - desc = ionic_tx_tso_next(txq, &elem); - start = false; - seglen = mss; - } - - /* Chop frags into desc segments */ - - txm_seg = txm->next; + /* Walk the chain of mbufs */ while (txm_seg != NULL) { offset = 0; data_iova = rte_mbuf_data_iova(txm_seg); left = txm_seg->data_len; + /* Split the mbuf data up into multiple descriptors */ while (left > 0) { next_addr = rte_cpu_to_le_64(data_iova + offset); - if (frag_left > 0) { + if (frag_left > 0 && use_sgl) { + /* Fill previous descriptor's SGE */ len = RTE_MIN(frag_left, left); frag_left -= len; elem->addr = next_addr; @@ -487,16 +386,19 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) elem++; desc_nsge++; } else { - len = RTE_MIN(mss, left); - frag_left = mss - len; + /* Fill new descriptor's data field */ + len = RTE_MIN(seglen, left); + frag_left = seglen - len; desc_addr = next_addr; desc_len = len; desc_nsge = 0; } left -= len; offset += len; - if (txm_seg->next != NULL && frag_left > 0) - continue; + + /* Pack the next mbuf's data into the descriptor */ + if (txm_seg->next != NULL && frag_left > 0 && use_sgl) + break; done = (txm_seg->next == NULL && left == 0); ionic_tx_tso_post(q, desc, txm_seg, @@ -507,6 +409,7 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) start, done); desc = ionic_tx_tso_next(txq, &elem); start = false; + seglen = mss; } txm_seg = txm_seg->next; @@ -517,157 +420,6 @@ ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) return 0; } -static __rte_always_inline int -ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) -{ - struct ionic_queue *q = &txq->qcq.q; - struct ionic_txq_desc *desc, *desc_base = q->base; - struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; - struct ionic_txq_sg_elem *elem; - struct ionic_tx_stats *stats = &txq->stats; - struct rte_mbuf *txm_seg; - void **info; - rte_iova_t data_iova; - uint64_t ol_flags = txm->ol_flags; - uint64_t addr, cmd; - uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; - uint8_t flags = 0; - - desc = &desc_base[q->head_idx]; - info = IONIC_INFO_PTR(q, q->head_idx); - - if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && - (txq->flags & IONIC_QCQ_F_CSUM_L3)) { - opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; - flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; - } - - if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && - (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || - ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && - (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { - opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; - flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; - } - - if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) - stats->no_csum++; - - if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || - (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && - ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || - (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { - flags |= IONIC_TXQ_DESC_FLAG_ENCAP; - } - - if (ol_flags & RTE_MBUF_F_TX_VLAN) { - flags |= IONIC_TXQ_DESC_FLAG_VLAN; - desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); - } - - addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); - - cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); - desc->cmd = rte_cpu_to_le_64(cmd); - desc->len = rte_cpu_to_le_16(txm->data_len); - - info[0] = txm; - - if (txm->nb_segs > 1) { - txm_seg = txm->next; - - elem = sg_desc_base[q->head_idx].elems; - - while (txm_seg != NULL) { - /* Stash the mbuf ptr in the array */ - info++; - *info = txm_seg; - - /* Configure the SGE */ - data_iova = rte_mbuf_data_iova(txm_seg); - elem->len = rte_cpu_to_le_16(txm_seg->data_len); - elem->addr = rte_cpu_to_le_64(data_iova); - elem++; - - txm_seg = txm_seg->next; - } - } - - q->head_idx = Q_NEXT_TO_POST(q, 1); - - return 0; -} - -uint16_t -ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) -{ - struct ionic_tx_qcq *txq = tx_queue; - struct ionic_queue *q = &txq->qcq.q; - struct ionic_tx_stats *stats = &txq->stats; - struct rte_mbuf *mbuf; - uint32_t bytes_tx = 0; - uint16_t nb_avail, nb_tx = 0; - int err; - - struct ionic_txq_desc *desc_base = q->base; - if (!(txq->flags & IONIC_QCQ_F_CMB)) - rte_prefetch0(&desc_base[q->head_idx]); - rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx)); - - if (tx_pkts) { - rte_mbuf_prefetch_part1(tx_pkts[0]); - rte_mbuf_prefetch_part2(tx_pkts[0]); - } - - if (unlikely(ionic_q_space_avail(q) < txq->free_thresh)) { - /* Cleaning old buffers */ - ionic_tx_flush(txq); - } - - nb_avail = ionic_q_space_avail(q); - if (unlikely(nb_avail < nb_pkts)) { - stats->stop += nb_pkts - nb_avail; - nb_pkts = nb_avail; - } - - while (nb_tx < nb_pkts) { - uint16_t next_idx = Q_NEXT_TO_POST(q, 1); - if (!(txq->flags & IONIC_QCQ_F_CMB)) - rte_prefetch0(&desc_base[next_idx]); - rte_prefetch0(IONIC_INFO_PTR(q, next_idx)); - - if (nb_tx + 1 < nb_pkts) { - rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); - rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); - } - - mbuf = tx_pkts[nb_tx]; - - if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) - err = ionic_tx_tso(txq, mbuf); - else - err = ionic_tx(txq, mbuf); - if (err) { - stats->drop += nb_pkts - nb_tx; - break; - } - - bytes_tx += mbuf->pkt_len; - nb_tx++; - } - - if (nb_tx > 0) { - rte_wmb(); - ionic_q_flush(q); - - stats->packets += nb_tx; - stats->bytes += bytes_tx; - } - - return nb_tx; -} - /********************************************************************* * * TX prep functions @@ -820,7 +572,7 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, } #define IONIC_CSUM_FLAG_MASK (IONIC_RXQ_COMP_CSUM_F_VLAN - 1) -static const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK] +const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK] __rte_cache_aligned = { /* IP_BAD set */ [IONIC_RXQ_COMP_CSUM_F_IP_BAD] = RTE_MBUF_F_RX_IP_CKSUM_BAD, @@ -850,7 +602,7 @@ static const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK] }; /* RTE_PTYPE_UNKNOWN is 0x0 */ -static const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK] +const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK] __rte_cache_aligned = { [IONIC_PKT_TYPE_NON_IP] = RTE_PTYPE_UNKNOWN, [IONIC_PKT_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, @@ -884,203 +636,6 @@ ionic_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) return ptypes; } -/* - * Cleans one descriptor. Connects the filled mbufs into a chain. - * Does not advance the tail index. - */ -static __rte_always_inline void -ionic_rx_clean_one(struct ionic_rx_qcq *rxq, - struct ionic_rxq_comp *cq_desc, - struct ionic_rx_service *rx_svc) -{ - struct ionic_queue *q = &rxq->qcq.q; - struct rte_mbuf *rxm, *rxm_seg, *prev_rxm; - struct ionic_rx_stats *stats = &rxq->stats; - uint64_t pkt_flags = 0; - uint32_t pkt_type; - uint32_t left, i; - uint16_t cq_desc_len; - uint8_t ptype, cflags; - void **info; - - cq_desc_len = rte_le_to_cpu_16(cq_desc->len); - - info = IONIC_INFO_PTR(q, q->tail_idx); - - rxm = info[0]; - - if (cq_desc->status) { - stats->bad_cq_status++; - return; - } - - if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { - stats->bad_len++; - return; - } - - info[0] = NULL; - - /* Set the mbuf metadata based on the cq entry */ - rxm->rearm_data[0] = rxq->rearm_data; - rxm->pkt_len = cq_desc_len; - rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len); - left = cq_desc_len - rxm->data_len; - rxm->nb_segs = cq_desc->num_sg_elems + 1; - prev_rxm = rxm; - - for (i = 1; i < rxm->nb_segs && left; i++) { - rxm_seg = info[i]; - info[i] = NULL; - - /* Set the chained mbuf metadata */ - rxm_seg->rearm_data[0] = rxq->rearm_seg_data; - rxm_seg->data_len = RTE_MIN(rxq->seg_size, left); - left -= rxm_seg->data_len; - - /* Link the mbuf */ - prev_rxm->next = rxm_seg; - prev_rxm = rxm_seg; - } - - /* Terminate the mbuf chain */ - prev_rxm->next = NULL; - - /* RSS */ - pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; - rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); - - /* Vlan Strip */ - if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { - pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; - rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); - } - - /* Checksum */ - if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { - cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; - pkt_flags |= ionic_csum_flags[cflags]; - } - - rxm->ol_flags = pkt_flags; - - /* Packet Type */ - ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; - pkt_type = ionic_ptype_table[ptype]; - if (pkt_type == RTE_PTYPE_UNKNOWN) { - struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, - struct rte_ether_hdr *); - uint16_t ether_type = eth_h->ether_type; - if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) - pkt_type = RTE_PTYPE_L2_ETHER_ARP; - else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) - pkt_type = RTE_PTYPE_L2_ETHER_LLDP; - else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) - pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; - stats->mtods++; - } - - rxm->packet_type = pkt_type; - - rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; - rx_svc->nb_rx++; - - stats->packets++; - stats->bytes += rxm->pkt_len; -} - -/* - * Fills one descriptor with mbufs. Does not advance the head index. - */ -static __rte_always_inline int -ionic_rx_fill_one(struct ionic_rx_qcq *rxq) -{ - struct ionic_queue *q = &rxq->qcq.q; - struct rte_mbuf *rxm, *rxm_seg; - struct ionic_rxq_desc *desc, *desc_base = q->base; - struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; - rte_iova_t data_iova; - uint32_t i; - void **info; - int ret; - - info = IONIC_INFO_PTR(q, q->head_idx); - desc = &desc_base[q->head_idx]; - sg_desc = &sg_desc_base[q->head_idx]; - - /* mbuf is unused => whole chain is unused */ - if (unlikely(info[0])) - return 0; - - if (rxq->mb_idx == 0) { - ret = rte_mempool_get_bulk(rxq->mb_pool, - (void **)rxq->mbs, - IONIC_MBUF_BULK_ALLOC); - if (ret) { - assert(0); - return -ENOMEM; - } - - rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; - } - - rxm = rxq->mbs[--rxq->mb_idx]; - info[0] = rxm; - - data_iova = rte_mbuf_data_iova_default(rxm); - desc->addr = rte_cpu_to_le_64(data_iova); - - for (i = 1; i < q->num_segs; i++) { - /* mbuf is unused => rest of the chain is unused */ - if (info[i]) - return 0; - - if (rxq->mb_idx == 0) { - ret = rte_mempool_get_bulk(rxq->mb_pool, - (void **)rxq->mbs, - IONIC_MBUF_BULK_ALLOC); - if (ret) { - assert(0); - return -ENOMEM; - } - - rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; - } - - rxm_seg = rxq->mbs[--rxq->mb_idx]; - info[i] = rxm_seg; - - /* The data_off does not get set to 0 until later */ - data_iova = rxm_seg->buf_iova; - sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova); - } - - return 0; -} - -/* - * Fills all descriptors with mbufs. - */ -static int __rte_cold -ionic_rx_fill(struct ionic_rx_qcq *rxq) -{ - struct ionic_queue *q = &rxq->qcq.q; - uint32_t i; - int err; - - for (i = 1; i < q->num_descs; i++) { - err = ionic_rx_fill_one(rxq); - if (err) - return err; - - q->head_idx = Q_NEXT_TO_POST(q, 1); - } - - ionic_q_flush(q); - - return 0; -} - /* * Perform one-time initialization of descriptor fields * which will not change for the life of the queue. @@ -1148,10 +703,13 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) if (err) return err; - /* Allocate buffers for descriptor rings */ - if (ionic_rx_fill(rxq) != 0) { - IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", - rx_queue_id); + /* Allocate buffers for descriptor ring */ + if (rxq->flags & IONIC_QCQ_F_SG) + err = ionic_rx_fill_sg(rxq); + else + err = ionic_rx_fill(rxq); + if (err != 0) { + IONIC_PRINT(ERR, "Could not fill queue %d", rx_queue_id); return -1; } @@ -1160,55 +718,6 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) return 0; } -/* - * Walk the CQ to find completed receive descriptors. - * Any completed descriptor found is refilled. - */ -static __rte_always_inline void -ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, - struct ionic_rx_service *rx_svc) -{ - struct ionic_cq *cq = &rxq->qcq.cq; - struct ionic_queue *q = &rxq->qcq.q; - struct ionic_rxq_desc *q_desc_base = q->base; - struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; - uint32_t work_done = 0; - - cq_desc = &cq_desc_base[cq->tail_idx]; - - while (color_match(cq_desc->pkt_type_color, cq->done_color)) { - cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); - - if (cq->tail_idx == 0) - cq->done_color = !cq->done_color; - - /* Prefetch 8 x 8B bufinfo */ - rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); - /* Prefetch 4 x 16B comp */ - rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); - /* Prefetch 4 x 16B descriptors */ - if (!(rxq->flags & IONIC_QCQ_F_CMB)) - rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); - - ionic_rx_clean_one(rxq, cq_desc, rx_svc); - - q->tail_idx = Q_NEXT_TO_SRVC(q, 1); - - (void)ionic_rx_fill_one(rxq); - - q->head_idx = Q_NEXT_TO_POST(q, 1); - - if (++work_done == work_to_do) - break; - - cq_desc = &cq_desc_base[cq->tail_idx]; - } - - /* Update the queue indices and ring the doorbell */ - if (work_done) - ionic_q_flush(q); -} - /* * Stop Receive Units for specified queue. */ @@ -1237,21 +746,6 @@ ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) return 0; } -uint16_t -ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - struct ionic_rx_qcq *rxq = rx_queue; - struct ionic_rx_service rx_svc; - - rx_svc.rx_pkts = rx_pkts; - rx_svc.nb_rx = 0; - - ionic_rxq_service(rxq, nb_pkts, &rx_svc); - - return rx_svc.nb_rx; -} - int ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) { diff --git a/drivers/net/ionic/ionic_rxtx.h b/drivers/net/ionic/ionic_rxtx.h index 5b9d9eac80..8537141597 100644 --- a/drivers/net/ionic/ionic_rxtx.h +++ b/drivers/net/ionic/ionic_rxtx.h @@ -5,7 +5,19 @@ #ifndef _IONIC_RXTX_H_ #define _IONIC_RXTX_H_ -#include +#include + +#include "ionic_if.h" + +struct ionic_rx_qcq; +struct ionic_tx_qcq; +struct rte_eth_dev; +struct rte_eth_rxconf; +struct rte_eth_rxq_info; +struct rte_eth_txconf; +struct rte_eth_txq_info; +struct rte_mbuf; +struct rte_mempool; struct ionic_rx_service { /* cb in */ @@ -14,13 +26,12 @@ struct ionic_rx_service { uint16_t nb_rx; }; -uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts); -uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); -uint16_t ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); +#define IONIC_CSUM_FLAG_MASK (IONIC_RXQ_COMP_CSUM_F_VLAN - 1) +extern const uint64_t ionic_csum_flags[IONIC_CSUM_FLAG_MASK]; +extern const uint32_t ionic_ptype_table[IONIC_RXQ_COMP_PKT_TYPE_MASK]; + +/* ionic_rxtx.c */ int ionic_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, uint16_t nb_desc, uint32_t socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp); @@ -45,4 +56,25 @@ int ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset); const uint32_t *ionic_dev_supported_ptypes_get(struct rte_eth_dev *dev); +int ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm); + +uint16_t ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + +/* ionic_rxtx_simple.c */ +uint16_t ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + +int ionic_rx_fill(struct ionic_rx_qcq *rxq); + +/* ionic_rxtx_sg.c */ +uint16_t ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + +int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq); + #endif /* _IONIC_RXTX_H_ */ diff --git a/drivers/net/ionic/ionic_rxtx_sg.c b/drivers/net/ionic/ionic_rxtx_sg.c new file mode 100644 index 0000000000..4e4a18e5d7 --- /dev/null +++ b/drivers/net/ionic/ionic_rxtx_sg.c @@ -0,0 +1,496 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018-2022 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ionic.h" +#include "ionic_if.h" +#include "ionic_dev.h" +#include "ionic_lif.h" +#include "ionic_rxtx.h" + +static __rte_always_inline void +ionic_tx_flush_sg(struct ionic_tx_qcq *txq) +{ + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; + struct rte_mbuf *txm; + struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; + void **info; + uint32_t i; + + cq_desc = &cq_desc_base[cq->tail_idx]; + + while (color_match(cq_desc->color, cq->done_color)) { + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); + if (cq->tail_idx == 0) + cq->done_color = !cq->done_color; + + /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ + if ((cq->tail_idx & 0x3) == 0) + rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); + + while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { + /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ + rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2))); + + /* Prefetch next mbuf */ + void **next_info = + IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1)); + if (next_info[0]) + rte_mbuf_prefetch_part2(next_info[0]); + if (next_info[1]) + rte_mbuf_prefetch_part2(next_info[1]); + + info = IONIC_INFO_PTR(q, q->tail_idx); + for (i = 0; i < q->num_segs; i++) { + txm = info[i]; + if (!txm) + break; + + if (txq->flags & IONIC_QCQ_F_FAST_FREE) + rte_mempool_put(txm->pool, txm); + else + rte_pktmbuf_free_seg(txm); + + info[i] = NULL; + } + + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); + } + + cq_desc = &cq_desc_base[cq->tail_idx]; + } +} + +static __rte_always_inline int +ionic_tx_sg(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) +{ + struct ionic_queue *q = &txq->qcq.q; + struct ionic_txq_desc *desc, *desc_base = q->base; + struct ionic_txq_sg_desc_v1 *sg_desc, *sg_desc_base = q->sg_base; + struct ionic_txq_sg_elem *elem; + struct ionic_tx_stats *stats = &txq->stats; + struct rte_mbuf *txm_seg; + rte_iova_t data_iova; + void **info; + uint64_t ol_flags = txm->ol_flags; + uint64_t addr, cmd; + uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; + uint8_t flags = 0; + + desc = &desc_base[q->head_idx]; + sg_desc = &sg_desc_base[q->head_idx]; + info = IONIC_INFO_PTR(q, q->head_idx); + + if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_L3)) { + opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; + flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; + } + + if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || + ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { + opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; + flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; + } + + if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) + stats->no_csum++; + + if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || + (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && + ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || + (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { + flags |= IONIC_TXQ_DESC_FLAG_ENCAP; + } + + if (ol_flags & RTE_MBUF_F_TX_VLAN) { + flags |= IONIC_TXQ_DESC_FLAG_VLAN; + desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); + } + + addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); + + cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); + desc->cmd = rte_cpu_to_le_64(cmd); + desc->len = rte_cpu_to_le_16(txm->data_len); + + info[0] = txm; + + if (txm->nb_segs > 1) { + txm_seg = txm->next; + + elem = sg_desc->elems; + + while (txm_seg != NULL) { + /* Stash the mbuf ptr in the array */ + info++; + *info = txm_seg; + + /* Configure the SGE */ + data_iova = rte_mbuf_data_iova(txm_seg); + elem->len = rte_cpu_to_le_16(txm_seg->data_len); + elem->addr = rte_cpu_to_le_64(data_iova); + elem++; + + txm_seg = txm_seg->next; + } + } + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + return 0; +} + +uint16_t +ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; + struct rte_mbuf *mbuf; + uint32_t bytes_tx = 0; + uint16_t nb_avail, nb_tx = 0; + int err; + + struct ionic_txq_desc *desc_base = q->base; + if (!(txq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&desc_base[q->head_idx]); + rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx)); + + if (tx_pkts) { + rte_mbuf_prefetch_part1(tx_pkts[0]); + rte_mbuf_prefetch_part2(tx_pkts[0]); + } + + if (ionic_q_space_avail(q) < txq->free_thresh) { + /* Cleaning old buffers */ + ionic_tx_flush_sg(txq); + } + + nb_avail = ionic_q_space_avail(q); + if (nb_avail < nb_pkts) { + stats->stop += nb_pkts - nb_avail; + nb_pkts = nb_avail; + } + + while (nb_tx < nb_pkts) { + uint16_t next_idx = Q_NEXT_TO_POST(q, 1); + if (!(txq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&desc_base[next_idx]); + rte_prefetch0(IONIC_INFO_PTR(q, next_idx)); + + if (nb_tx + 1 < nb_pkts) { + rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); + rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); + } + + mbuf = tx_pkts[nb_tx]; + + if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) + err = ionic_tx_tso(txq, mbuf); + else + err = ionic_tx_sg(txq, mbuf); + if (err) { + stats->drop += nb_pkts - nb_tx; + break; + } + + bytes_tx += mbuf->pkt_len; + nb_tx++; + } + + if (nb_tx > 0) { + rte_wmb(); + ionic_q_flush(q); + + stats->packets += nb_tx; + stats->bytes += bytes_tx; + } + + return nb_tx; +} + +/* + * Cleans one descriptor. Connects the filled mbufs into a chain. + * Does not advance the tail index. + */ +static __rte_always_inline void +ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq, + struct ionic_rxq_comp *cq_desc, + struct ionic_rx_service *rx_svc) +{ + struct ionic_queue *q = &rxq->qcq.q; + struct rte_mbuf *rxm; + struct rte_mbuf *rxm_seg, *prev_rxm; + struct ionic_rx_stats *stats = &rxq->stats; + uint64_t pkt_flags = 0; + uint32_t pkt_type; + uint32_t left, i; + uint16_t cq_desc_len; + uint8_t ptype, cflags; + void **info; + + cq_desc_len = rte_le_to_cpu_16(cq_desc->len); + + info = IONIC_INFO_PTR(q, q->tail_idx); + + rxm = info[0]; + + if (cq_desc->status) { + stats->bad_cq_status++; + return; + } + + if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { + stats->bad_len++; + return; + } + + info[0] = NULL; + + /* Set the mbuf metadata based on the cq entry */ + rxm->rearm_data[0] = rxq->rearm_data; + rxm->pkt_len = cq_desc_len; + rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len); + left = cq_desc_len - rxm->data_len; + rxm->nb_segs = cq_desc->num_sg_elems + 1; + + prev_rxm = rxm; + + for (i = 1; i < rxm->nb_segs && left; i++) { + rxm_seg = info[i]; + info[i] = NULL; + + /* Set the chained mbuf metadata */ + rxm_seg->rearm_data[0] = rxq->rearm_seg_data; + rxm_seg->data_len = RTE_MIN(rxq->seg_size, left); + left -= rxm_seg->data_len; + + /* Link the mbuf */ + prev_rxm->next = rxm_seg; + prev_rxm = rxm_seg; + } + + /* Terminate the mbuf chain */ + prev_rxm->next = NULL; + + /* RSS */ + pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; + rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); + + /* Vlan Strip */ + if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { + pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; + rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); + } + + /* Checksum */ + if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { + cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; + pkt_flags |= ionic_csum_flags[cflags]; + } + + rxm->ol_flags = pkt_flags; + + /* Packet Type */ + ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; + pkt_type = ionic_ptype_table[ptype]; + if (pkt_type == RTE_PTYPE_UNKNOWN) { + struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, + struct rte_ether_hdr *); + uint16_t ether_type = eth_h->ether_type; + if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) + pkt_type = RTE_PTYPE_L2_ETHER_ARP; + else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) + pkt_type = RTE_PTYPE_L2_ETHER_LLDP; + else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) + pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; + stats->mtods++; + } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { + pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; + } else { + pkt_type |= RTE_PTYPE_L2_ETHER; + } + + rxm->packet_type = pkt_type; + + rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; + rx_svc->nb_rx++; + + stats->packets++; + stats->bytes += rxm->pkt_len; +} + +/* + * Fills one descriptor with mbufs. Does not advance the head index. + */ +static __rte_always_inline int +ionic_rx_fill_one_sg(struct ionic_rx_qcq *rxq) +{ + struct ionic_queue *q = &rxq->qcq.q; + struct rte_mbuf *rxm; + struct rte_mbuf *rxm_seg; + struct ionic_rxq_desc *desc, *desc_base = q->base; + struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; + rte_iova_t data_iova; + uint32_t i; + void **info; + int ret; + + info = IONIC_INFO_PTR(q, q->head_idx); + desc = &desc_base[q->head_idx]; + sg_desc = &sg_desc_base[q->head_idx]; + + /* mbuf is unused => whole chain is unused */ + if (info[0]) + return 0; + + if (rxq->mb_idx == 0) { + ret = rte_mempool_get_bulk(rxq->mb_pool, + (void **)rxq->mbs, + IONIC_MBUF_BULK_ALLOC); + if (ret) { + assert(0); + return -ENOMEM; + } + + rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; + } + + rxm = rxq->mbs[--rxq->mb_idx]; + info[0] = rxm; + + data_iova = rte_mbuf_data_iova_default(rxm); + desc->addr = rte_cpu_to_le_64(data_iova); + + for (i = 1; i < q->num_segs; i++) { + /* mbuf is unused => rest of the chain is unused */ + if (info[i]) + return 0; + + if (rxq->mb_idx == 0) { + ret = rte_mempool_get_bulk(rxq->mb_pool, + (void **)rxq->mbs, + IONIC_MBUF_BULK_ALLOC); + if (ret) { + assert(0); + return -ENOMEM; + } + + rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; + } + + rxm_seg = rxq->mbs[--rxq->mb_idx]; + info[i] = rxm_seg; + + /* The data_off does not get set to 0 until later */ + data_iova = rxm_seg->buf_iova; + sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova); + } + + return 0; +} + +/* + * Walk the CQ to find completed receive descriptors. + * Any completed descriptor found is refilled. + */ +static __rte_always_inline void +ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do, + struct ionic_rx_service *rx_svc) +{ + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *q_desc_base = q->base; + struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; + uint32_t work_done = 0; + + cq_desc = &cq_desc_base[cq->tail_idx]; + + while (color_match(cq_desc->pkt_type_color, cq->done_color)) { + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); + if (cq->tail_idx == 0) + cq->done_color = !cq->done_color; + + /* Prefetch 8 x 8B bufinfo */ + rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); + /* Prefetch 4 x 16B comp */ + rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); + /* Prefetch 4 x 16B descriptors */ + if (!(rxq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); + + /* Clean one descriptor */ + ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); + + /* Fill one descriptor */ + (void)ionic_rx_fill_one_sg(rxq); + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (++work_done == work_to_do) + break; + + cq_desc = &cq_desc_base[cq->tail_idx]; + } + + /* Update the queue indices and ring the doorbell */ + if (work_done) + ionic_q_flush(q); +} + +uint16_t +ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct ionic_rx_qcq *rxq = rx_queue; + struct ionic_rx_service rx_svc; + + rx_svc.rx_pkts = rx_pkts; + rx_svc.nb_rx = 0; + + ionic_rxq_service_sg(rxq, nb_pkts, &rx_svc); + + return rx_svc.nb_rx; +} + +/* + * Fills all descriptors with mbufs. + */ +int __rte_cold +ionic_rx_fill_sg(struct ionic_rx_qcq *rxq) +{ + struct ionic_queue *q = &rxq->qcq.q; + uint32_t i; + int err = 0; + + for (i = 0; i < q->num_descs - 1u; i++) { + err = ionic_rx_fill_one_sg(rxq); + if (err) + break; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + } + + ionic_q_flush(q); + + return err; +} diff --git a/drivers/net/ionic/ionic_rxtx_simple.c b/drivers/net/ionic/ionic_rxtx_simple.c new file mode 100644 index 0000000000..a191fd3ec9 --- /dev/null +++ b/drivers/net/ionic/ionic_rxtx_simple.c @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2018-2022 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ionic.h" +#include "ionic_if.h" +#include "ionic_dev.h" +#include "ionic_lif.h" +#include "ionic_rxtx.h" + +static __rte_always_inline void +ionic_tx_flush(struct ionic_tx_qcq *txq) +{ + struct ionic_cq *cq = &txq->qcq.cq; + struct ionic_queue *q = &txq->qcq.q; + struct rte_mbuf *txm; + struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; + void **info; + + cq_desc = &cq_desc_base[cq->tail_idx]; + + while (color_match(cq_desc->color, cq->done_color)) { + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); + if (cq->tail_idx == 0) + cq->done_color = !cq->done_color; + + /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ + if ((cq->tail_idx & 0x3) == 0) + rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); + + while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { + /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ + rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]); + + /* Prefetch next mbuf */ + void **next_info = + &q->info[Q_NEXT_TO_SRVC(q, 1)]; + if (next_info[0]) + rte_mbuf_prefetch_part2(next_info[0]); + + info = &q->info[q->tail_idx]; + { + txm = info[0]; + + if (txq->flags & IONIC_QCQ_F_FAST_FREE) + rte_mempool_put(txm->pool, txm); + else + rte_pktmbuf_free_seg(txm); + + info[0] = NULL; + } + + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); + } + + cq_desc = &cq_desc_base[cq->tail_idx]; + } +} + +static __rte_always_inline int +ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) +{ + struct ionic_queue *q = &txq->qcq.q; + struct ionic_txq_desc *desc, *desc_base = q->base; + struct ionic_tx_stats *stats = &txq->stats; + void **info; + uint64_t ol_flags = txm->ol_flags; + uint64_t addr, cmd; + uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; + uint8_t flags = 0; + + if (txm->nb_segs > 1) + return -EINVAL; + + desc = &desc_base[q->head_idx]; + info = &q->info[q->head_idx]; + + if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_L3)) { + opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; + flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; + } + + if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || + ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && + (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { + opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; + flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; + } + + if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) + stats->no_csum++; + + if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || + (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && + ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || + (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { + flags |= IONIC_TXQ_DESC_FLAG_ENCAP; + } + + if (ol_flags & RTE_MBUF_F_TX_VLAN) { + flags |= IONIC_TXQ_DESC_FLAG_VLAN; + desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); + } + + addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); + + cmd = encode_txq_desc_cmd(opcode, flags, 0, addr); + desc->cmd = rte_cpu_to_le_64(cmd); + desc->len = rte_cpu_to_le_16(txm->data_len); + + info[0] = txm; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + return 0; +} + +uint16_t +ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct ionic_tx_qcq *txq = tx_queue; + struct ionic_queue *q = &txq->qcq.q; + struct ionic_tx_stats *stats = &txq->stats; + struct rte_mbuf *mbuf; + uint32_t bytes_tx = 0; + uint16_t nb_avail, nb_tx = 0; + int err; + + struct ionic_txq_desc *desc_base = q->base; + if (!(txq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&desc_base[q->head_idx]); + rte_prefetch0(&q->info[q->head_idx]); + + if (tx_pkts) { + rte_mbuf_prefetch_part1(tx_pkts[0]); + rte_mbuf_prefetch_part2(tx_pkts[0]); + } + + if (ionic_q_space_avail(q) < txq->free_thresh) { + /* Cleaning old buffers */ + ionic_tx_flush(txq); + } + + nb_avail = ionic_q_space_avail(q); + if (nb_avail < nb_pkts) { + stats->stop += nb_pkts - nb_avail; + nb_pkts = nb_avail; + } + + while (nb_tx < nb_pkts) { + uint16_t next_idx = Q_NEXT_TO_POST(q, 1); + if (!(txq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&desc_base[next_idx]); + rte_prefetch0(&q->info[next_idx]); + + if (nb_tx + 1 < nb_pkts) { + rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); + rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); + } + + mbuf = tx_pkts[nb_tx]; + + if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) + err = ionic_tx_tso(txq, mbuf); + else + err = ionic_tx(txq, mbuf); + if (err) { + stats->drop += nb_pkts - nb_tx; + break; + } + + bytes_tx += mbuf->pkt_len; + nb_tx++; + } + + if (nb_tx > 0) { + rte_wmb(); + ionic_q_flush(q); + + stats->packets += nb_tx; + stats->bytes += bytes_tx; + } + + return nb_tx; +} + +/* + * Cleans one descriptor. Connects the filled mbufs into a chain. + * Does not advance the tail index. + */ +static __rte_always_inline void +ionic_rx_clean_one(struct ionic_rx_qcq *rxq, + struct ionic_rxq_comp *cq_desc, + struct ionic_rx_service *rx_svc) +{ + struct ionic_queue *q = &rxq->qcq.q; + struct rte_mbuf *rxm; + struct ionic_rx_stats *stats = &rxq->stats; + uint64_t pkt_flags = 0; + uint32_t pkt_type; + uint16_t cq_desc_len; + uint8_t ptype, cflags; + void **info; + + cq_desc_len = rte_le_to_cpu_16(cq_desc->len); + + info = &q->info[q->tail_idx]; + + rxm = info[0]; + + if (cq_desc->status) { + stats->bad_cq_status++; + return; + } + + if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { + stats->bad_len++; + return; + } + + info[0] = NULL; + + /* Set the mbuf metadata based on the cq entry */ + rxm->rearm_data[0] = rxq->rearm_data; + rxm->pkt_len = cq_desc_len; + rxm->data_len = cq_desc_len; + + /* RSS */ + pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; + rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); + + /* Vlan Strip */ + if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { + pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; + rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); + } + + /* Checksum */ + if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { + cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; + pkt_flags |= ionic_csum_flags[cflags]; + } + + rxm->ol_flags = pkt_flags; + + /* Packet Type */ + ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; + pkt_type = ionic_ptype_table[ptype]; + if (pkt_type == RTE_PTYPE_UNKNOWN) { + struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, + struct rte_ether_hdr *); + uint16_t ether_type = eth_h->ether_type; + if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) + pkt_type = RTE_PTYPE_L2_ETHER_ARP; + else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) + pkt_type = RTE_PTYPE_L2_ETHER_LLDP; + else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) + pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; + stats->mtods++; + } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { + pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; + } else { + pkt_type |= RTE_PTYPE_L2_ETHER; + } + + rxm->packet_type = pkt_type; + + rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; + rx_svc->nb_rx++; + + stats->packets++; + stats->bytes += rxm->pkt_len; +} + +/* + * Fills one descriptor with mbufs. Does not advance the head index. + */ +static __rte_always_inline int +ionic_rx_fill_one(struct ionic_rx_qcq *rxq) +{ + struct ionic_queue *q = &rxq->qcq.q; + struct rte_mbuf *rxm; + struct ionic_rxq_desc *desc, *desc_base = q->base; + rte_iova_t data_iova; + void **info; + int ret; + + info = &q->info[q->head_idx]; + desc = &desc_base[q->head_idx]; + + /* mbuf is unused */ + if (info[0]) + return 0; + + if (rxq->mb_idx == 0) { + ret = rte_mempool_get_bulk(rxq->mb_pool, + (void **)rxq->mbs, + IONIC_MBUF_BULK_ALLOC); + if (ret) { + assert(0); + return -ENOMEM; + } + + rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; + } + + rxm = rxq->mbs[--rxq->mb_idx]; + info[0] = rxm; + + data_iova = rte_mbuf_data_iova_default(rxm); + desc->addr = rte_cpu_to_le_64(data_iova); + + return 0; +} + +/* + * Walk the CQ to find completed receive descriptors. + * Any completed descriptor found is refilled. + */ +static __rte_always_inline void +ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, + struct ionic_rx_service *rx_svc) +{ + struct ionic_cq *cq = &rxq->qcq.cq; + struct ionic_queue *q = &rxq->qcq.q; + struct ionic_rxq_desc *q_desc_base = q->base; + struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; + uint32_t work_done = 0; + + cq_desc = &cq_desc_base[cq->tail_idx]; + + while (color_match(cq_desc->pkt_type_color, cq->done_color)) { + cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); + if (cq->tail_idx == 0) + cq->done_color = !cq->done_color; + + /* Prefetch 8 x 8B bufinfo */ + rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]); + /* Prefetch 4 x 16B comp */ + rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); + /* Prefetch 4 x 16B descriptors */ + if (!(rxq->flags & IONIC_QCQ_F_CMB)) + rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); + + /* Clean one descriptor */ + ionic_rx_clean_one(rxq, cq_desc, rx_svc); + q->tail_idx = Q_NEXT_TO_SRVC(q, 1); + + /* Fill one descriptor */ + (void)ionic_rx_fill_one(rxq); + + q->head_idx = Q_NEXT_TO_POST(q, 1); + + if (++work_done == work_to_do) + break; + + cq_desc = &cq_desc_base[cq->tail_idx]; + } + + /* Update the queue indices and ring the doorbell */ + if (work_done) + ionic_q_flush(q); +} + +uint16_t +ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct ionic_rx_qcq *rxq = rx_queue; + struct ionic_rx_service rx_svc; + + rx_svc.rx_pkts = rx_pkts; + rx_svc.nb_rx = 0; + + ionic_rxq_service(rxq, nb_pkts, &rx_svc); + + return rx_svc.nb_rx; +} + +/* + * Fills all descriptors with mbufs. + */ +int __rte_cold +ionic_rx_fill(struct ionic_rx_qcq *rxq) +{ + struct ionic_queue *q = &rxq->qcq.q; + uint32_t i; + int err = 0; + + for (i = 0; i < q->num_descs - 1u; i++) { + err = ionic_rx_fill_one(rxq); + if (err) + break; + + q->head_idx = Q_NEXT_TO_POST(q, 1); + } + + ionic_q_flush(q); + + return err; +} diff --git a/drivers/net/ionic/meson.build b/drivers/net/ionic/meson.build index 00865c2139..71c7f2311a 100644 --- a/drivers/net/ionic/meson.build +++ b/drivers/net/ionic/meson.build @@ -16,4 +16,6 @@ sources = files( 'ionic_main.c', 'ionic_rx_filter.c', 'ionic_rxtx.c', + 'ionic_rxtx_simple.c', + 'ionic_rxtx_sg.c', )