d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx4: fix HW memory optimizations careless

Browse Source 
Volatilize all Rx/Tx HW negotiation memories to be sure no compiler
optimization prevents either load or store commands.

Fixes: c3c977bbecbd ("net/mlx4: add Tx bypassing Verbs")
Fixes: 9f57340a8087 ("net/mlx4: restore Rx offloads")
Fixes: 6681b845034c ("net/mlx4: add Rx bypassing Verbs")
Fixes: 62e96ffb93ad ("net/mlx4: fix no Rx interrupts")

Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
This commit is contained in:
Matan Azrad 2017-11-02 16:42:50 +00:00 committed by Ferruh Yigit
parent dae76a678c
commit b68d92b45c
2 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
drivers/net/mlx4/mlx4_prm.h
View File

@ -80,14 +80,14 @@ enum {
/* Send queue information. */ /* Send queue information. */
struct mlx4_sq { struct mlx4_sq {
uint8_t *buf; /**< SQ buffer. */ volatile uint8_t *buf; /**< SQ buffer. */
uint8_t *eob; /**< End of SQ buffer */ volatile uint8_t *eob; /**< End of SQ buffer */
uint32_t head; /**< SQ head counter in units of TXBBS. */ uint32_t head; /**< SQ head counter in units of TXBBS. */
uint32_t tail; /**< SQ tail counter in units of TXBBS. */ uint32_t tail; /**< SQ tail counter in units of TXBBS. */
uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */ uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */
uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */ uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */
uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */ uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */
uint32_t *db; /**< Pointer to the doorbell. */ volatile uint32_t *db; /**< Pointer to the doorbell. */
uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */ uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */
}; };
@ -100,11 +100,11 @@ struct mlx4_sq {
/* Completion queue information. */ /* Completion queue information. */
struct mlx4_cq { struct mlx4_cq {
void *cq_uar; /**< CQ user access region. */ volatile void *cq_uar; /**< CQ user access region. */
void *cq_db_reg; /**< CQ doorbell register. */ volatile void *cq_db_reg; /**< CQ doorbell register. */
uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */ volatile uint32_t *set_ci_db; /**< Pointer to the CQ doorbell. */
uint32_t *arm_db; /**< Pointer to doorbell for arming Rx events. */ volatile uint32_t *arm_db; /**< Arming Rx events doorbell. */
uint8_t *buf; /**< Pointer to the completion queue buffer. */ volatile uint8_t *buf; /**< Pointer to the completion queue buffer. */
uint32_t cqe_cnt; /**< Number of entries in the queue. */ uint32_t cqe_cnt; /**< Number of entries in the queue. */
uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */ uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */
uint32_t cons_index; /**< Last queue entry that was handled. */ uint32_t cons_index; /**< Last queue entry that was handled. */
@ -128,10 +128,10 @@ struct mlx4_cq {
* @return * @return
* Pointer to CQE entry. * Pointer to CQE entry.
*/ */
static inline struct mlx4_cqe * static inline volatile struct mlx4_cqe *
mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index) mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index)
{ {
return (struct mlx4_cqe *)(cq->buf + return (volatile struct mlx4_cqe *)(cq->buf +
((index & (cq->cqe_cnt - 1)) << ((index & (cq->cqe_cnt - 1)) <<
(5 + cq->cqe_64)) + (5 + cq->cqe_64)) +
(cq->cqe_64 << 5)); (cq->cqe_64 << 5));

67
drivers/net/mlx4/mlx4_rxtx.c
View File

@ -69,7 +69,7 @@
* DWORD (32 byte) of a TXBB. * DWORD (32 byte) of a TXBB.
*/ */
struct pv { struct pv {
struct mlx4_wqe_data_seg *dseg; volatile struct mlx4_wqe_data_seg *dseg;
uint32_t val; uint32_t val;
}; };
@ -97,14 +97,15 @@ mlx4_txq_stamp_freed_wqe(struct mlx4_sq *sq, uint16_t index, uint8_t owner)
{ {
uint32_t stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL | uint32_t stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL |
(!!owner << MLX4_SQ_STAMP_SHIFT)); (!!owner << MLX4_SQ_STAMP_SHIFT));
uint8_t *wqe = mlx4_get_send_wqe(sq, (index & sq->txbb_cnt_mask)); volatile uint8_t *wqe = mlx4_get_send_wqe(sq,
uint32_t *ptr = (uint32_t *)wqe; (index & sq->txbb_cnt_mask));
volatile uint32_t *ptr = (volatile uint32_t *)wqe;
int i; int i;
int txbbs_size; int txbbs_size;
int num_txbbs; int num_txbbs;
/* Extract the size from the control segment of the WQE. */ /* Extract the size from the control segment of the WQE. */
num_txbbs = MLX4_SIZE_TO_TXBBS((((struct mlx4_wqe_ctrl_seg *) num_txbbs = MLX4_SIZE_TO_TXBBS((((volatile struct mlx4_wqe_ctrl_seg *)
wqe)->fence_size & 0x3f) << 4); wqe)->fence_size & 0x3f) << 4);
txbbs_size = num_txbbs * MLX4_TXBB_SIZE; txbbs_size = num_txbbs * MLX4_TXBB_SIZE;
/* Optimize the common case when there is no wrap-around. */ /* Optimize the common case when there is no wrap-around. */
@ -119,8 +120,8 @@ mlx4_txq_stamp_freed_wqe(struct mlx4_sq *sq, uint16_t index, uint8_t owner)
for (i = 0; i < txbbs_size; i += MLX4_SQ_STAMP_STRIDE) { for (i = 0; i < txbbs_size; i += MLX4_SQ_STAMP_STRIDE) {
*ptr = stamp; *ptr = stamp;
ptr += MLX4_SQ_STAMP_DWORDS; ptr += MLX4_SQ_STAMP_DWORDS;
if ((uint8_t *)ptr >= sq->eob) { if ((volatile uint8_t *)ptr >= sq->eob) {
ptr = (uint32_t *)sq->buf; ptr = (volatile uint32_t *)sq->buf;
stamp ^= RTE_BE32(0x80000000); stamp ^= RTE_BE32(0x80000000);
} }
} }
@ -149,7 +150,7 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_n,
unsigned int elts_comp = txq->elts_comp; unsigned int elts_comp = txq->elts_comp;
unsigned int elts_tail = txq->elts_tail; unsigned int elts_tail = txq->elts_tail;
struct mlx4_cq *cq = &txq->mcq; struct mlx4_cq *cq = &txq->mcq;
struct mlx4_cqe *cqe; volatile struct mlx4_cqe *cqe;
uint32_t cons_index = cq->cons_index; uint32_t cons_index = cq->cons_index;
uint16_t new_index; uint16_t new_index;
uint16_t nr_txbbs = 0; uint16_t nr_txbbs = 0;
@ -160,7 +161,7 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_n,
* reported by them. * reported by them.
*/ */
do { do {
cqe = (struct mlx4_cqe *)mlx4_get_cqe(cq, cons_index); cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cons_index);
if (unlikely(!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ if (unlikely(!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
!!(cons_index & cq->cqe_cnt))) !!(cons_index & cq->cqe_cnt)))
break; break;
@ -171,8 +172,8 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_n,
#ifndef NDEBUG #ifndef NDEBUG
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) { MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = volatile struct mlx4_err_cqe *cqe_err =
(struct mlx4_err_cqe *)cqe; (volatile struct mlx4_err_cqe *)cqe;
ERROR("%p CQE error - vendor syndrome: 0x%x" ERROR("%p CQE error - vendor syndrome: 0x%x"
" syndrome: 0x%x\n", " syndrome: 0x%x\n",
(void *)txq, cqe_err->vendor_err, (void *)txq, cqe_err->vendor_err,
@ -239,15 +240,15 @@ mlx4_txq_mb2mp(struct rte_mbuf *buf)
static int static int
mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq, mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
struct mlx4_wqe_ctrl_seg **pctrl) volatile struct mlx4_wqe_ctrl_seg **pctrl)
{ {
int wqe_real_size; int wqe_real_size;
int nr_txbbs; int nr_txbbs;
struct pv *pv = (struct pv *)txq->bounce_buf; struct pv *pv = (struct pv *)txq->bounce_buf;
struct mlx4_sq *sq = &txq->msq; struct mlx4_sq *sq = &txq->msq;
uint32_t head_idx = sq->head & sq->txbb_cnt_mask; uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
struct mlx4_wqe_ctrl_seg *ctrl; volatile struct mlx4_wqe_ctrl_seg *ctrl;
struct mlx4_wqe_data_seg *dseg; volatile struct mlx4_wqe_data_seg *dseg;
struct rte_mbuf *sbuf; struct rte_mbuf *sbuf;
uint32_t lkey; uint32_t lkey;
uintptr_t addr; uintptr_t addr;
@ -255,8 +256,8 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
int pv_counter = 0; int pv_counter = 0;
/* Calculate the needed work queue entry size for this packet. */ /* Calculate the needed work queue entry size for this packet. */
wqe_real_size = sizeof(struct mlx4_wqe_ctrl_seg) + wqe_real_size = sizeof(volatile struct mlx4_wqe_ctrl_seg) +
buf->nb_segs * sizeof(struct mlx4_wqe_data_seg); buf->nb_segs * sizeof(volatile struct mlx4_wqe_data_seg);
nr_txbbs = MLX4_SIZE_TO_TXBBS(wqe_real_size); nr_txbbs = MLX4_SIZE_TO_TXBBS(wqe_real_size);
/* /*
* Check that there is room for this WQE in the send queue and that * Check that there is room for this WQE in the send queue and that
@ -268,17 +269,18 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
return -1; return -1;
} }
/* Get the control and data entries of the WQE. */ /* Get the control and data entries of the WQE. */
ctrl = (struct mlx4_wqe_ctrl_seg *)mlx4_get_send_wqe(sq, head_idx); ctrl = (volatile struct mlx4_wqe_ctrl_seg *)
dseg = (struct mlx4_wqe_data_seg *)((uintptr_t)ctrl + mlx4_get_send_wqe(sq, head_idx);
sizeof(struct mlx4_wqe_ctrl_seg)); dseg = (volatile struct mlx4_wqe_data_seg *)
((uintptr_t)ctrl + sizeof(struct mlx4_wqe_ctrl_seg));
*pctrl = ctrl; *pctrl = ctrl;
/* Fill the data segments with buffer information. */ /* Fill the data segments with buffer information. */
for (sbuf = buf; sbuf != NULL; sbuf = sbuf->next, dseg++) { for (sbuf = buf; sbuf != NULL; sbuf = sbuf->next, dseg++) {
addr = rte_pktmbuf_mtod(sbuf, uintptr_t); addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
rte_prefetch0((volatile void *)addr); rte_prefetch0((volatile void *)addr);
/* Handle WQE wraparound. */ /* Handle WQE wraparound. */
if (dseg >= (struct mlx4_wqe_data_seg *)sq->eob) if (dseg >= (volatile struct mlx4_wqe_data_seg *)sq->eob)
dseg = (struct mlx4_wqe_data_seg *)sq->buf; dseg = (volatile struct mlx4_wqe_data_seg *)sq->buf;
dseg->addr = rte_cpu_to_be_64(addr); dseg->addr = rte_cpu_to_be_64(addr);
/* Memory region key (big endian) for this memory pool. */ /* Memory region key (big endian) for this memory pool. */
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf)); lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
@ -395,8 +397,8 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
struct txq_elt *elt_next = &(*txq->elts)[elts_head_next]; struct txq_elt *elt_next = &(*txq->elts)[elts_head_next];
struct txq_elt *elt = &(*txq->elts)[elts_head]; struct txq_elt *elt = &(*txq->elts)[elts_head];
uint32_t owner_opcode = MLX4_OPCODE_SEND; uint32_t owner_opcode = MLX4_OPCODE_SEND;
struct mlx4_wqe_ctrl_seg *ctrl; volatile struct mlx4_wqe_ctrl_seg *ctrl;
struct mlx4_wqe_data_seg *dseg; volatile struct mlx4_wqe_data_seg *dseg;
union { union {
uint32_t flags; uint32_t flags;
uint16_t flags16[2]; uint16_t flags16[2];
@ -433,15 +435,18 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
break; break;
} }
/* Get the control and data entries of the WQE. */ /* Get the control and data entries of the WQE. */
ctrl = (struct mlx4_wqe_ctrl_seg *) ctrl = (volatile struct mlx4_wqe_ctrl_seg *)
mlx4_get_send_wqe(sq, head_idx); mlx4_get_send_wqe(sq, head_idx);
dseg = (struct mlx4_wqe_data_seg *)((uintptr_t)ctrl + dseg = (volatile struct mlx4_wqe_data_seg *)
((uintptr_t)ctrl +
sizeof(struct mlx4_wqe_ctrl_seg)); sizeof(struct mlx4_wqe_ctrl_seg));
addr = rte_pktmbuf_mtod(buf, uintptr_t); addr = rte_pktmbuf_mtod(buf, uintptr_t);
rte_prefetch0((volatile void *)addr); rte_prefetch0((volatile void *)addr);
/* Handle WQE wraparound. */ /* Handle WQE wraparound. */
if (dseg >= (struct mlx4_wqe_data_seg *)sq->eob) if (dseg >=
dseg = (struct mlx4_wqe_data_seg *)sq->buf; (volatile struct mlx4_wqe_data_seg *)sq->eob)
dseg = (volatile struct mlx4_wqe_data_seg *)
sq->buf;
dseg->addr = rte_cpu_to_be_64(addr); dseg->addr = rte_cpu_to_be_64(addr);
/* Memory region key (big endian). */ /* Memory region key (big endian). */
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf)); lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
@ -633,7 +638,7 @@ rxq_cq_to_ol_flags(uint32_t flags, int csum, int csum_l2tun)
* CQE checksum information. * CQE checksum information.
*/ */
static inline uint32_t static inline uint32_t
mlx4_cqe_flags(struct mlx4_cqe *cqe, int csum, int csum_l2tun) mlx4_cqe_flags(volatile struct mlx4_cqe *cqe, int csum, int csum_l2tun)
{ {
uint32_t flags = 0; uint32_t flags = 0;
@ -666,13 +671,13 @@ mlx4_cqe_flags(struct mlx4_cqe *cqe, int csum, int csum_l2tun)
* Number of bytes of the CQE, 0 in case there is no completion. * Number of bytes of the CQE, 0 in case there is no completion.
*/ */
static unsigned int static unsigned int
mlx4_cq_poll_one(struct rxq *rxq, struct mlx4_cqe **out) mlx4_cq_poll_one(struct rxq *rxq, volatile struct mlx4_cqe **out)
{ {
int ret = 0; int ret = 0;
struct mlx4_cqe *cqe = NULL; volatile struct mlx4_cqe *cqe = NULL;
struct mlx4_cq *cq = &rxq->mcq; struct mlx4_cq *cq = &rxq->mcq;
cqe = (struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index); cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index);
if (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ if (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
!!(cq->cons_index & cq->cqe_cnt)) !!(cq->cons_index & cq->cqe_cnt))
goto out; goto out;
@ -717,7 +722,7 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
int len = 0; int len = 0;
while (pkts_n) { while (pkts_n) {
struct mlx4_cqe *cqe; volatile struct mlx4_cqe *cqe;
uint32_t idx = rq_ci & wr_cnt; uint32_t idx = rq_ci & wr_cnt;
struct rte_mbuf *rep = (*rxq->elts)[idx]; struct rte_mbuf *rep = (*rxq->elts)[idx];
volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx]; volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx];