From b5fcebb24207991028ba1539cd185fa4a9f69d41 Mon Sep 17 00:00:00 2001 From: Gordon Noonan Date: Wed, 12 May 2021 11:28:54 +0100 Subject: [PATCH] net/iavf: use write combining store for tail updates Performance improvement: use a write combining store instead of a regular mmio write to update queue tail registers. Signed-off-by: Gordon Noonan Signed-off-by: Radu Nicolau Acked-by: Qi Zhang --- drivers/common/iavf/iavf_osdep.h | 5 +++++ drivers/net/iavf/iavf_rxtx.c | 6 +++--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 4 ++-- drivers/net/iavf/iavf_rxtx_vec_common.h | 2 +- drivers/net/iavf/iavf_rxtx_vec_sse.c | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/common/iavf/iavf_osdep.h b/drivers/common/iavf/iavf_osdep.h index 76c186a5b8..31d3d809f9 100644 --- a/drivers/common/iavf/iavf_osdep.h +++ b/drivers/common/iavf/iavf_osdep.h @@ -132,6 +132,11 @@ writeq(uint64_t value, volatile void *addr) #define IAVF_PCI_REG_WRITE(reg, value) writel(value, reg) #define IAVF_PCI_REG_WRITE_RELAXED(reg, value) writel_relaxed(value, reg) +#define IAVF_PCI_REG_WC_WRITE(reg, value) \ + rte_write32_wc((rte_cpu_to_le_32(value)), reg) +#define IAVF_PCI_REG_WC_WRITE_RELAXED(reg, value) \ + rte_write32_wc_relaxed((rte_cpu_to_le_32(value)), reg) + #define IAVF_READ_REG(hw, reg) rd32(hw, reg) #define IAVF_WRITE_REG(hw, reg, value) wr32(hw, reg, value) diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c index 6a713df828..fdab80a79d 100644 --- a/drivers/net/iavf/iavf_rxtx.c +++ b/drivers/net/iavf/iavf_rxtx.c @@ -1161,7 +1161,7 @@ iavf_update_rx_tail(struct iavf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id) rxq->port_id, rxq->queue_id, rx_id, nb_hold); rx_id = (uint16_t)((rx_id == 0) ? (rxq->nb_rx_desc - 1) : (rx_id - 1)); - IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); nb_hold = 0; } rxq->nb_rx_hold = nb_hold; @@ -1906,7 +1906,7 @@ iavf_rx_alloc_bufs(struct iavf_rx_queue *rxq) /* Update rx tail register */ rte_wmb(); - IAVF_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger); + IAVF_PCI_REG_WC_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger); rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); @@ -2332,7 +2332,7 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", txq->port_id, txq->queue_id, tx_id, nb_tx); - IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id); + IAVF_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id); txq->tx_tail = tx_id; return nb_tx; diff --git a/drivers/net/iavf/iavf_rxtx_vec_avx2.c b/drivers/net/iavf/iavf_rxtx_vec_avx2.c index f5646d6453..475070e036 100644 --- a/drivers/net/iavf/iavf_rxtx_vec_avx2.c +++ b/drivers/net/iavf/iavf_rxtx_vec_avx2.c @@ -1500,7 +1500,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; - IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); return nb_pkts; } diff --git a/drivers/net/iavf/iavf_rxtx_vec_avx512.c b/drivers/net/iavf/iavf_rxtx_vec_avx512.c index d99de2a8bc..571161c0cd 100644 --- a/drivers/net/iavf/iavf_rxtx_vec_avx512.c +++ b/drivers/net/iavf/iavf_rxtx_vec_avx512.c @@ -156,7 +156,7 @@ iavf_rxq_rearm(struct iavf_rx_queue *rxq) (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ - IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); } #define IAVF_RX_LEN_MASK 0x80808080 @@ -1958,7 +1958,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; - IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); return nb_pkts; } diff --git a/drivers/net/iavf/iavf_rxtx_vec_common.h b/drivers/net/iavf/iavf_rxtx_vec_common.h index 6bb6903a8a..457d6339e1 100644 --- a/drivers/net/iavf/iavf_rxtx_vec_common.h +++ b/drivers/net/iavf/iavf_rxtx_vec_common.h @@ -570,7 +570,7 @@ iavf_rxq_rearm_common(struct iavf_rx_queue *rxq, __rte_unused bool avx512) (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ - IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); } #endif diff --git a/drivers/net/iavf/iavf_rxtx_vec_sse.c b/drivers/net/iavf/iavf_rxtx_vec_sse.c index ef1f17ea21..bf87696fa4 100644 --- a/drivers/net/iavf/iavf_rxtx_vec_sse.c +++ b/drivers/net/iavf/iavf_rxtx_vec_sse.c @@ -88,7 +88,7 @@ iavf_rxq_rearm(struct iavf_rx_queue *rxq) rx_id, rxq->rxrearm_start, rxq->rxrearm_nb); /* Update the tail pointer on the NIC */ - IAVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); } static inline void @@ -1171,7 +1171,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_pkts=%u", txq->port_id, txq->queue_id, tx_id, nb_pkts); - IAVF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + IAVF_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); return nb_pkts; }