d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx5: add missing sanity checks for Tx completion queue

Browse Source 
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
This commit is contained in:
Yongseok Koh 2017-12-26 19:55:45 -08:00 committed by Ferruh Yigit
parent 36aa55ea57
commit 2eefbec531
5 changed files with 38 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
drivers/net/mlx5/mlx5_rxtx.c
View File

@ -357,6 +357,8 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* Start processing. */ /* Start processing. */
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
max_elts = (elts_n - (elts_head - txq->elts_tail)); max_elts = (elts_n - (elts_head - txq->elts_tail));
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
if (unlikely(!max_wqe)) if (unlikely(!max_wqe))
return 0; return 0;
@ -700,6 +702,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* Save elts_head in unused "immediate" field of WQE. */ /* Save elts_head in unused "immediate" field of WQE. */
last_wqe->ctrl3 = txq->elts_head; last_wqe->ctrl3 = txq->elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi;
#endif
} else { } else {
txq->elts_comp = comp; txq->elts_comp = comp;
} }
@ -818,6 +823,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* Start processing. */ /* Start processing. */
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
max_elts = (elts_n - (elts_head - txq->elts_tail)); max_elts = (elts_n - (elts_head - txq->elts_tail));
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
if (unlikely(!max_wqe)) if (unlikely(!max_wqe))
return 0; return 0;
@ -911,6 +918,9 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
/* Save elts_head in unused "immediate" field of WQE. */ /* Save elts_head in unused "immediate" field of WQE. */
wqe->ctrl[3] = elts_head; wqe->ctrl[3] = elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi;
#endif
} else { } else {
txq->elts_comp = comp; txq->elts_comp = comp;
} }
@ -1042,6 +1052,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
/* Start processing. */ /* Start processing. */
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
max_elts = (elts_n - (elts_head - txq->elts_tail)); max_elts = (elts_n - (elts_head - txq->elts_tail));
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
do { do {
struct rte_mbuf *buf = *(pkts++); struct rte_mbuf *buf = *(pkts++);
uintptr_t addr; uintptr_t addr;
@ -1203,6 +1215,9 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
/* Save elts_head in unused "immediate" field of WQE. */ /* Save elts_head in unused "immediate" field of WQE. */
wqe->ctrl[3] = elts_head; wqe->ctrl[3] = elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi;
#endif
} else { } else {
txq->elts_comp = comp; txq->elts_comp = comp;
} }
@ -1549,7 +1564,9 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
wqe->ctrl[3] = elts_head; wqe->ctrl[3] = elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
txq->mpw_comp = txq->wqe_ci; txq->mpw_comp = txq->wqe_ci;
txq->cq_pi++; #ifndef NDEBUG
++txq->cq_pi;
#endif
} else { } else {
txq->elts_comp += j; txq->elts_comp += j;
} }

2
drivers/net/mlx5/mlx5_rxtx.h
View File

@ -185,7 +185,9 @@ struct mlx5_txq_data {
uint16_t elts_comp; /* Counter since last completion request. */ uint16_t elts_comp; /* Counter since last completion request. */
uint16_t mpw_comp; /* WQ index since last completion request. */ uint16_t mpw_comp; /* WQ index since last completion request. */
uint16_t cq_ci; /* Consumer index for completion queue. */ uint16_t cq_ci; /* Consumer index for completion queue. */
#ifndef NDEBUG
uint16_t cq_pi; /* Producer index for completion queue. */ uint16_t cq_pi; /* Producer index for completion queue. */
#endif
uint16_t wqe_ci; /* Consumer index for work queue. */ uint16_t wqe_ci; /* Consumer index for work queue. */
uint16_t wqe_pi; /* Producer index for work queue. */ uint16_t wqe_pi; /* Producer index for work queue. */
uint16_t elts_n:4; /* (*elts)[] length (in log2). */ uint16_t elts_n:4; /* (*elts)[] length (in log2). */

8
drivers/net/mlx5/mlx5_rxtx_vec_neon.h
View File

@ -135,6 +135,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
assert(elts_n > pkts_n); assert(elts_n > pkts_n);
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
if (unlikely(!pkts_n)) if (unlikely(!pkts_n))
return 0; return 0;
for (n = 0; n < pkts_n; ++n) { for (n = 0; n < pkts_n; ++n) {
@ -205,7 +207,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
wqe->ctrl[2] = rte_cpu_to_be_32(8); wqe->ctrl[2] = rte_cpu_to_be_32(8);
wqe->ctrl[3] = txq->elts_head; wqe->ctrl[3] = txq->elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi; ++txq->cq_pi;
#endif
} }
#ifdef MLX5_PMD_SOFT_COUNTERS #ifdef MLX5_PMD_SOFT_COUNTERS
txq->stats.opackets += n; txq->stats.opackets += n;
@ -269,6 +273,8 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
assert(elts_n > pkts_n); assert(elts_n > pkts_n);
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
max_elts = (elts_n - (elts_head - txq->elts_tail)); max_elts = (elts_n - (elts_head - txq->elts_tail));
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts); pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
if (unlikely(!pkts_n)) if (unlikely(!pkts_n))
@ -306,7 +312,9 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
} else { } else {
/* Request a completion. */ /* Request a completion. */
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi; ++txq->cq_pi;
#endif
comp_req = 8; comp_req = 8;
} }
/* Fill CTRL in the header. */ /* Fill CTRL in the header. */

8
drivers/net/mlx5/mlx5_rxtx_vec_sse.h
View File

@ -135,6 +135,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
assert(elts_n > pkts_n); assert(elts_n > pkts_n);
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
if (unlikely(!pkts_n)) if (unlikely(!pkts_n))
return 0; return 0;
for (n = 0; n < pkts_n; ++n) { for (n = 0; n < pkts_n; ++n) {
@ -206,7 +208,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
wqe->ctrl[2] = rte_cpu_to_be_32(8); wqe->ctrl[2] = rte_cpu_to_be_32(8);
wqe->ctrl[3] = txq->elts_head; wqe->ctrl[3] = txq->elts_head;
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi; ++txq->cq_pi;
#endif
} }
#ifdef MLX5_PMD_SOFT_COUNTERS #ifdef MLX5_PMD_SOFT_COUNTERS
txq->stats.opackets += n; txq->stats.opackets += n;
@ -268,6 +272,8 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
assert(elts_n > pkts_n); assert(elts_n > pkts_n);
mlx5_tx_complete(txq); mlx5_tx_complete(txq);
max_elts = (elts_n - (elts_head - txq->elts_tail)); max_elts = (elts_n - (elts_head - txq->elts_tail));
/* A CQE slot must always be available. */
assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts); pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr); assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr);
@ -307,7 +313,9 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
} else { } else {
/* Request a completion. */ /* Request a completion. */
txq->elts_comp = 0; txq->elts_comp = 0;
#ifndef NDEBUG
++txq->cq_pi; ++txq->cq_pi;
#endif
comp_req = 8; comp_req = 8;
} }
/* Fill CTRL in the header. */ /* Fill CTRL in the header. */

2
drivers/net/mlx5/mlx5_txq.c
View File

@ -418,7 +418,9 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
(volatile struct mlx5_cqe (*)[]) (volatile struct mlx5_cqe (*)[])
(uintptr_t)cq_info.buf; (uintptr_t)cq_info.buf;
txq_data->cq_ci = 0; txq_data->cq_ci = 0;
#ifndef NDEBUG
txq_data->cq_pi = 0; txq_data->cq_pi = 0;
#endif
txq_data->wqe_ci = 0; txq_data->wqe_ci = 0;
txq_data->wqe_pi = 0; txq_data->wqe_pi = 0;
txq_ibv->qp = tmpl.qp; txq_ibv->qp = tmpl.qp;