d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx4: revert resource domain support

Browse Source 
This reverts commit 3e49c148b7.

Resource domains are not part of the standard Verbs interface. The
performance improvement they bring will be restored later through a
different data path implementation.

This commit makes the PMD not rely on the non-standard QP allocation
interface.

Signed-off-by: Moti Haimovsky <motih@mellanox.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
This commit is contained in:
Adrien Mazarguil 2017-09-01 10:06:37 +02:00 committed by Ferruh Yigit
parent fa8551ffa8
commit 72ba7fadbf
3 changed files with 20 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
drivers/net/mlx4/mlx4.c
View File

@ -697,17 +697,6 @@ txq_cleanup(struct txq *txq)
claim_zero(ibv_destroy_qp(txq->qp));
if (txq->cq != NULL)
claim_zero(ibv_destroy_cq(txq->cq));
if (txq->rd != NULL) {
struct ibv_exp_destroy_res_domain_attr attr = {
.comp_mask = 0,
};
assert(txq->priv != NULL);
assert(txq->priv->ctx != NULL);
claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx,
txq->rd,
&attr));
}
for (i = 0; (i != elemof(txq->mp2mr)); ++i) {
if (txq->mp2mr[i].mp == NULL)
break;
@ -1175,9 +1164,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
};
union {
struct ibv_exp_query_intf_params params;
struct ibv_exp_qp_init_attr init;
struct ibv_exp_res_domain_init_attr rd;
struct ibv_exp_cq_init_attr cq;
struct ibv_qp_init_attr init;
struct ibv_qp_attr mod;
} attr;
enum ibv_exp_query_intf_status status;
@ -1191,24 +1178,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
return EINVAL;
}
/* MRs will be registered in mp2mr[] later. */
attr.rd = (struct ibv_exp_res_domain_init_attr){
.comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
IBV_EXP_RES_DOMAIN_MSG_MODEL),
.thread_model = IBV_EXP_THREAD_SINGLE,
.msg_model = IBV_EXP_MSG_HIGH_BW,
};
tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
if (tmpl.rd == NULL) {
ret = ENOMEM;
ERROR("%p: RD creation failure: %s",
(void *)dev, strerror(ret));
goto error;
}
attr.cq = (struct ibv_exp_cq_init_attr){
.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
.res_domain = tmpl.rd,
};
tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0);
if (tmpl.cq == NULL) {
ret = ENOMEM;
ERROR("%p: CQ creation failure: %s",
@ -1219,7 +1189,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
priv->device_attr.max_qp_wr);
DEBUG("priv->device_attr.max_sge is %d",
priv->device_attr.max_sge);
attr.init = (struct ibv_exp_qp_init_attr){
attr.init = (struct ibv_qp_init_attr){
/* CQ to be associated with the send queue. */
.send_cq = tmpl.cq,
/* CQ to be associated with the receive queue. */
@ -1237,12 +1207,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
/* Do *NOT* enable this, completions events are managed per
* TX burst. */
.sq_sig_all = 0,
.pd = priv->pd,
.res_domain = tmpl.rd,
.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
};
tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init);
tmpl.qp = ibv_create_qp(priv->pd, &attr.init);
if (tmpl.qp == NULL) {
ret = (errno ? errno : EINVAL);
ERROR("%p: QP creation failure: %s",
@ -1710,17 +1676,6 @@ rxq_cleanup(struct rxq *rxq)
claim_zero(ibv_destroy_cq(rxq->cq));
if (rxq->channel != NULL)
claim_zero(ibv_destroy_comp_channel(rxq->channel));
if (rxq->rd != NULL) {
struct ibv_exp_destroy_res_domain_attr attr = {
.comp_mask = 0,
};
assert(rxq->priv != NULL);
assert(rxq->priv->ctx != NULL);
claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx,
rxq->rd,
&attr));
}
if (rxq->mr != NULL)
claim_zero(ibv_dereg_mr(rxq->mr));
memset(rxq, 0, sizeof(*rxq));
@ -1890,10 +1845,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* QP pointer or NULL in case of error.
*/
static struct ibv_qp *
rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
struct ibv_exp_res_domain *rd)
rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc)
{
struct ibv_exp_qp_init_attr attr = {
struct ibv_qp_init_attr attr = {
/* CQ to be associated with the send queue. */
.send_cq = cq,
/* CQ to be associated with the receive queue. */
@ -1907,13 +1861,9 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
.max_recv_sge = 1,
},
.qp_type = IBV_QPT_RAW_PACKET,
.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
.pd = priv->pd,
.res_domain = rd,
};
return ibv_exp_create_qp(priv->ctx, &attr);
return ibv_create_qp(priv->pd, &attr);
}
/**
@ -1949,8 +1899,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
struct ibv_qp_attr mod;
union {
struct ibv_exp_query_intf_params params;
struct ibv_exp_cq_init_attr cq;
struct ibv_exp_res_domain_init_attr rd;
} attr;
enum ibv_exp_query_intf_status status;
struct ibv_recv_wr *bad_wr;
@ -1988,19 +1936,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
(void *)dev, strerror(ret));
goto error;
}
attr.rd = (struct ibv_exp_res_domain_init_attr){
.comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
IBV_EXP_RES_DOMAIN_MSG_MODEL),
.thread_model = IBV_EXP_THREAD_SINGLE,
.msg_model = IBV_EXP_MSG_HIGH_BW,
};
tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
if (tmpl.rd == NULL) {
ret = ENOMEM;
ERROR("%p: RD creation failure: %s",
(void *)dev, strerror(ret));
goto error;
}
if (dev->data->dev_conf.intr_conf.rxq) {
tmpl.channel = ibv_create_comp_channel(priv->ctx);
if (tmpl.channel == NULL) {
@ -2011,12 +1946,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
goto error;
}
}
attr.cq = (struct ibv_exp_cq_init_attr){
.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
.res_domain = tmpl.rd,
};
tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0,
&attr.cq);
tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0);
if (tmpl.cq == NULL) {
ret = ENOMEM;
ERROR("%p: CQ creation failure: %s",
@ -2027,7 +1957,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
priv->device_attr.max_qp_wr);
DEBUG("priv->device_attr.max_sge is %d",
priv->device_attr.max_sge);
tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd);
tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc);
if (tmpl.qp == NULL) {
ret = (errno ? errno : EINVAL);
ERROR("%p: QP creation failure: %s",

2
drivers/net/mlx4/mlx4.h
View File

@ -167,7 +167,6 @@ struct rxq {
struct rxq_elt (*elts)[]; /* Rx elements. */
struct mlx4_rxq_stats stats; /* RX queue counters. */
unsigned int socket; /* CPU socket ID for allocations. */
struct ibv_exp_res_domain *rd; /* Resource Domain. */
};
/* TX element. */
@ -204,7 +203,6 @@ struct txq {
unsigned int elts_comp_cd_init; /* Initial value for countdown. */
struct mlx4_txq_stats stats; /* TX queue counters. */
unsigned int socket; /* CPU socket ID for allocations. */
struct ibv_exp_res_domain *rd; /* Resource Domain. */
};
struct rte_flow;

30
drivers/net/mlx4/mlx4_flow.c
View File

@ -752,29 +752,21 @@ mlx4_flow_create_drop_queue(struct priv *priv)
ERROR("Cannot allocate memory for drop struct");
goto err;
}
cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
&(struct ibv_exp_cq_init_attr){
.comp_mask = 0,
});
cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
if (!cq) {
ERROR("Cannot create drop CQ");
goto err_create_cq;
}
qp = ibv_exp_create_qp(priv->ctx,
&(struct ibv_exp_qp_init_attr){
.send_cq = cq,
.recv_cq = cq,
.cap = {
.max_recv_wr = 1,
.max_recv_sge = 1,
},
.qp_type = IBV_QPT_RAW_PACKET,
.comp_mask =
IBV_EXP_QP_INIT_ATTR_PD |
IBV_EXP_QP_INIT_ATTR_PORT,
.pd = priv->pd,
.port_num = priv->port,
});
qp = ibv_create_qp(priv->pd,
&(struct ibv_qp_init_attr){
.send_cq = cq,
.recv_cq = cq,
.cap = {
.max_recv_wr = 1,
.max_recv_sge = 1,
},
.qp_type = IBV_QPT_RAW_PACKET,
});
if (!qp) {
ERROR("Cannot create drop QP");
goto err_create_qp;