net/mana: support Rx interrupts
mana can receive Rx interrupts from kernel through RDMA verbs interface. Implement Rx interrupts in the driver. Signed-off-by: Long Li <longli@microsoft.com>
This commit is contained in:
Long Li
Ferruh Yigit
parent
e350b56889
commit
afd5d17072
@ -7,6 +7,7 @@
|
||||
Speed capabilities = P
|
||||
Link status = P
|
||||
Removal event = Y
|
||||
Rx interrupt = Y
|
||||
Queue start/stop = Y
|
||||
RSS hash = Y
|
||||
L3 checksum offload = Y
|
||||
|
||||
@ -215,7 +215,7 @@ enum {
|
||||
*/
|
||||
int
|
||||
mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
|
||||
uint32_t queue_id, uint32_t tail)
|
||||
uint32_t queue_id, uint32_t tail, uint8_t arm)
|
||||
{
|
||||
uint8_t *addr = db_page;
|
||||
union gdma_doorbell_entry e = {};
|
||||
@ -230,14 +230,14 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
|
||||
case GDMA_QUEUE_RECEIVE:
|
||||
e.rq.id = queue_id;
|
||||
e.rq.tail_ptr = tail;
|
||||
e.rq.wqe_cnt = 1;
|
||||
e.rq.wqe_cnt = arm;
|
||||
addr += DOORBELL_OFFSET_RQ;
|
||||
break;
|
||||
|
||||
case GDMA_QUEUE_COMPLETION:
|
||||
e.cq.id = queue_id;
|
||||
e.cq.tail_ptr = tail;
|
||||
e.cq.arm = 1;
|
||||
e.cq.arm = arm;
|
||||
addr += DOORBELL_OFFSET_CQ;
|
||||
break;
|
||||
|
||||
@ -249,8 +249,8 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
|
||||
/* Ensure all writes are done before ringing doorbell */
|
||||
rte_wmb();
|
||||
|
||||
DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
|
||||
db_page, addr, queue_id, queue_type, tail);
|
||||
DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u",
|
||||
db_page, addr, queue_id, queue_type, tail, arm);
|
||||
|
||||
rte_write64(e.as_uint64, addr);
|
||||
return 0;
|
||||
|
||||
@ -103,7 +103,72 @@ mana_dev_configure(struct rte_eth_dev *dev)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int mana_intr_uninstall(struct mana_priv *priv);
|
||||
static void
|
||||
rx_intr_vec_disable(struct mana_priv *priv)
|
||||
{
|
||||
struct rte_intr_handle *intr_handle = priv->intr_handle;
|
||||
|
||||
rte_intr_free_epoll_fd(intr_handle);
|
||||
rte_intr_vec_list_free(intr_handle);
|
||||
rte_intr_nb_efd_set(intr_handle, 0);
|
||||
}
|
||||
|
||||
static int
|
||||
rx_intr_vec_enable(struct mana_priv *priv)
|
||||
{
|
||||
unsigned int i;
|
||||
unsigned int rxqs_n = priv->dev_data->nb_rx_queues;
|
||||
unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
|
||||
struct rte_intr_handle *intr_handle = priv->intr_handle;
|
||||
int ret;
|
||||
|
||||
rx_intr_vec_disable(priv);
|
||||
|
||||
if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
|
||||
DRV_LOG(ERR, "Failed to allocate memory for interrupt vector");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
struct mana_rxq *rxq = priv->dev_data->rx_queues[i];
|
||||
|
||||
ret = rte_intr_vec_list_index_set(intr_handle, i,
|
||||
RTE_INTR_VEC_RXTX_OFFSET + i);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to set intr vec %u", i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = rte_intr_efds_index_set(intr_handle, i, rxq->channel->fd);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to set FD at intr %u", i);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return rte_intr_nb_efd_set(intr_handle, n);
|
||||
}
|
||||
|
||||
static void
|
||||
rxq_intr_disable(struct mana_priv *priv)
|
||||
{
|
||||
int err = rte_errno;
|
||||
|
||||
rx_intr_vec_disable(priv);
|
||||
rte_errno = err;
|
||||
}
|
||||
|
||||
static int
|
||||
rxq_intr_enable(struct mana_priv *priv)
|
||||
{
|
||||
const struct rte_eth_intr_conf *const intr_conf =
|
||||
&priv->dev_data->dev_conf.intr_conf;
|
||||
|
||||
if (!intr_conf->rxq)
|
||||
return 0;
|
||||
|
||||
return rx_intr_vec_enable(priv);
|
||||
}
|
||||
|
||||
static int
|
||||
mana_dev_start(struct rte_eth_dev *dev)
|
||||
@ -141,8 +206,17 @@ mana_dev_start(struct rte_eth_dev *dev)
|
||||
/* Enable datapath for secondary processes */
|
||||
mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX);
|
||||
|
||||
ret = rxq_intr_enable(priv);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to enable RX interrupts");
|
||||
goto failed_intr;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
failed_intr:
|
||||
mana_stop_rx_queues(dev);
|
||||
|
||||
failed_rx:
|
||||
mana_stop_tx_queues(dev);
|
||||
|
||||
@ -153,9 +227,12 @@ mana_dev_start(struct rte_eth_dev *dev)
|
||||
}
|
||||
|
||||
static int
|
||||
mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
|
||||
mana_dev_stop(struct rte_eth_dev *dev)
|
||||
{
|
||||
int ret;
|
||||
struct mana_priv *priv = dev->data->dev_private;
|
||||
|
||||
rxq_intr_disable(priv);
|
||||
|
||||
dev->tx_pkt_burst = mana_tx_burst_removed;
|
||||
dev->rx_pkt_burst = mana_rx_burst_removed;
|
||||
@ -180,6 +257,8 @@ mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int mana_intr_uninstall(struct mana_priv *priv);
|
||||
|
||||
static int
|
||||
mana_dev_close(struct rte_eth_dev *dev)
|
||||
{
|
||||
@ -614,6 +693,8 @@ static const struct eth_dev_ops mana_dev_ops = {
|
||||
.tx_queue_release = mana_dev_tx_queue_release,
|
||||
.rx_queue_setup = mana_dev_rx_queue_setup,
|
||||
.rx_queue_release = mana_dev_rx_queue_release,
|
||||
.rx_queue_intr_enable = mana_rx_intr_enable,
|
||||
.rx_queue_intr_disable = mana_rx_intr_disable,
|
||||
.link_update = mana_dev_link_update,
|
||||
.stats_get = mana_dev_stats_get,
|
||||
.stats_reset = mana_dev_stats_reset,
|
||||
@ -850,10 +931,22 @@ mana_intr_uninstall(struct mana_priv *priv)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
mana_intr_install(struct mana_priv *priv)
|
||||
int
|
||||
mana_fd_set_non_blocking(int fd)
|
||||
{
|
||||
int ret, flags;
|
||||
int ret = fcntl(fd, F_GETFL);
|
||||
|
||||
if (ret != -1 && !fcntl(fd, F_SETFL, ret | O_NONBLOCK))
|
||||
return 0;
|
||||
|
||||
rte_errno = errno;
|
||||
return -rte_errno;
|
||||
}
|
||||
|
||||
static int
|
||||
mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv)
|
||||
{
|
||||
int ret;
|
||||
struct ibv_context *ctx = priv->ib_ctx;
|
||||
|
||||
priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
|
||||
@ -863,31 +956,35 @@ mana_intr_install(struct mana_priv *priv)
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
rte_intr_fd_set(priv->intr_handle, -1);
|
||||
ret = rte_intr_fd_set(priv->intr_handle, -1);
|
||||
if (ret)
|
||||
goto free_intr;
|
||||
|
||||
flags = fcntl(ctx->async_fd, F_GETFL);
|
||||
ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
|
||||
ret = mana_fd_set_non_blocking(ctx->async_fd);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK");
|
||||
goto free_intr;
|
||||
}
|
||||
|
||||
rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
|
||||
rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
|
||||
ret = rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
|
||||
if (ret)
|
||||
goto free_intr;
|
||||
|
||||
ret = rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
|
||||
if (ret)
|
||||
goto free_intr;
|
||||
|
||||
ret = rte_intr_callback_register(priv->intr_handle,
|
||||
mana_intr_handler, priv);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to register intr callback");
|
||||
rte_intr_fd_set(priv->intr_handle, -1);
|
||||
goto restore_fd;
|
||||
goto free_intr;
|
||||
}
|
||||
|
||||
eth_dev->intr_handle = priv->intr_handle;
|
||||
return 0;
|
||||
|
||||
restore_fd:
|
||||
fcntl(ctx->async_fd, F_SETFL, flags);
|
||||
|
||||
free_intr:
|
||||
rte_intr_instance_free(priv->intr_handle);
|
||||
priv->intr_handle = NULL;
|
||||
@ -1178,7 +1275,7 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
|
||||
rte_eth_copy_pci_info(eth_dev, pci_dev);
|
||||
|
||||
/* Create async interrupt handler */
|
||||
ret = mana_intr_install(priv);
|
||||
ret = mana_intr_install(eth_dev, priv);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to install intr handler");
|
||||
goto failed;
|
||||
|
||||
@ -420,6 +420,7 @@ struct mana_rxq {
|
||||
uint32_t num_desc;
|
||||
struct rte_mempool *mp;
|
||||
struct ibv_cq *cq;
|
||||
struct ibv_comp_channel *channel;
|
||||
struct ibv_wq *wq;
|
||||
|
||||
/* For storing pending requests */
|
||||
@ -453,8 +454,8 @@ extern int mana_logtype_init;
|
||||
#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
|
||||
|
||||
int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
|
||||
uint32_t queue_id, uint32_t tail);
|
||||
int mana_rq_ring_doorbell(struct mana_rxq *rxq);
|
||||
uint32_t queue_id, uint32_t tail, uint8_t arm);
|
||||
int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm);
|
||||
|
||||
int gdma_post_work_request(struct mana_gdma_queue *queue,
|
||||
struct gdma_work_request *work_req,
|
||||
@ -534,4 +535,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
|
||||
void *mana_alloc_verbs_buf(size_t size, void *data);
|
||||
void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
|
||||
|
||||
int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
||||
int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
||||
int mana_fd_set_non_blocking(int fd);
|
||||
|
||||
#endif
|
||||
|
||||
@ -22,7 +22,7 @@ static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = {
|
||||
};
|
||||
|
||||
int
|
||||
mana_rq_ring_doorbell(struct mana_rxq *rxq)
|
||||
mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm)
|
||||
{
|
||||
struct mana_priv *priv = rxq->priv;
|
||||
int ret;
|
||||
@ -37,9 +37,9 @@ mana_rq_ring_doorbell(struct mana_rxq *rxq)
|
||||
}
|
||||
|
||||
ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE,
|
||||
rxq->gdma_rq.id,
|
||||
rxq->gdma_rq.head *
|
||||
GDMA_WQE_ALIGNMENT_UNIT_SIZE);
|
||||
rxq->gdma_rq.id,
|
||||
rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE,
|
||||
arm);
|
||||
|
||||
if (ret)
|
||||
DRV_LOG(ERR, "failed to ring RX doorbell ret %d", ret);
|
||||
@ -121,7 +121,7 @@ mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq)
|
||||
}
|
||||
}
|
||||
|
||||
mana_rq_ring_doorbell(rxq);
|
||||
mana_rq_ring_doorbell(rxq, rxq->num_desc);
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -163,6 +163,14 @@ mana_stop_rx_queues(struct rte_eth_dev *dev)
|
||||
DRV_LOG(ERR,
|
||||
"rx_queue destroy_cq failed %d", ret);
|
||||
rxq->cq = NULL;
|
||||
|
||||
if (rxq->channel) {
|
||||
ret = ibv_destroy_comp_channel(rxq->channel);
|
||||
if (ret)
|
||||
DRV_LOG(ERR, "failed destroy comp %d",
|
||||
ret);
|
||||
rxq->channel = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Drain and free posted WQEs */
|
||||
@ -204,8 +212,24 @@ mana_start_rx_queues(struct rte_eth_dev *dev)
|
||||
.data = (void *)(uintptr_t)rxq->socket,
|
||||
}));
|
||||
|
||||
if (dev->data->dev_conf.intr_conf.rxq) {
|
||||
rxq->channel = ibv_create_comp_channel(priv->ib_ctx);
|
||||
if (!rxq->channel) {
|
||||
ret = -errno;
|
||||
DRV_LOG(ERR, "Queue %d comp channel failed", i);
|
||||
goto fail;
|
||||
}
|
||||
|
||||
ret = mana_fd_set_non_blocking(rxq->channel->fd);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "Failed to set comp non-blocking");
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc,
|
||||
NULL, NULL, 0);
|
||||
NULL, rxq->channel,
|
||||
rxq->channel ? i : 0);
|
||||
if (!rxq->cq) {
|
||||
ret = -errno;
|
||||
DRV_LOG(ERR, "failed to create rx cq queue %d", i);
|
||||
@ -356,7 +380,8 @@ mana_start_rx_queues(struct rte_eth_dev *dev)
|
||||
uint16_t
|
||||
mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
|
||||
{
|
||||
uint16_t pkt_received = 0, cqe_processed = 0;
|
||||
uint16_t pkt_received = 0;
|
||||
uint8_t wqe_posted = 0;
|
||||
struct mana_rxq *rxq = dpdk_rxq;
|
||||
struct mana_priv *priv = rxq->priv;
|
||||
struct gdma_comp comp;
|
||||
@ -442,18 +467,65 @@ mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
|
||||
if (rxq->desc_ring_tail >= rxq->num_desc)
|
||||
rxq->desc_ring_tail = 0;
|
||||
|
||||
cqe_processed++;
|
||||
|
||||
/* Post another request */
|
||||
ret = mana_alloc_and_post_rx_wqe(rxq);
|
||||
if (ret) {
|
||||
DRV_LOG(ERR, "failed to post rx wqe ret=%d", ret);
|
||||
break;
|
||||
}
|
||||
|
||||
wqe_posted++;
|
||||
}
|
||||
|
||||
if (cqe_processed)
|
||||
mana_rq_ring_doorbell(rxq);
|
||||
if (wqe_posted)
|
||||
mana_rq_ring_doorbell(rxq, wqe_posted);
|
||||
|
||||
return pkt_received;
|
||||
}
|
||||
|
||||
static int
|
||||
mana_arm_cq(struct mana_rxq *rxq, uint8_t arm)
|
||||
{
|
||||
struct mana_priv *priv = rxq->priv;
|
||||
uint32_t head = rxq->gdma_cq.head %
|
||||
(rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE);
|
||||
|
||||
DRV_LOG(ERR, "Ringing completion queue ID %u head %u arm %d",
|
||||
rxq->gdma_cq.id, head, arm);
|
||||
|
||||
return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION,
|
||||
rxq->gdma_cq.id, head, arm);
|
||||
}
|
||||
|
||||
int
|
||||
mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
|
||||
{
|
||||
struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
|
||||
|
||||
return mana_arm_cq(rxq, 1);
|
||||
}
|
||||
|
||||
int
|
||||
mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
|
||||
{
|
||||
struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
|
||||
struct ibv_cq *ev_cq;
|
||||
void *ev_ctx;
|
||||
int ret;
|
||||
|
||||
ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx);
|
||||
if (ret)
|
||||
ret = errno;
|
||||
else if (ev_cq != rxq->cq)
|
||||
ret = EINVAL;
|
||||
|
||||
if (ret) {
|
||||
if (ret != EAGAIN)
|
||||
DRV_LOG(ERR, "Can't disable RX intr queue %d",
|
||||
rx_queue_id);
|
||||
} else {
|
||||
ibv_ack_cq_events(rxq->cq, 1);
|
||||
}
|
||||
|
||||
return -ret;
|
||||
}
|
||||
|
||||
@ -406,7 +406,8 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
|
||||
ret = mana_ring_doorbell(db_page, GDMA_QUEUE_SEND,
|
||||
txq->gdma_sq.id,
|
||||
txq->gdma_sq.head *
|
||||
GDMA_WQE_ALIGNMENT_UNIT_SIZE);
|
||||
GDMA_WQE_ALIGNMENT_UNIT_SIZE,
|
||||
0);
|
||||
if (ret)
|
||||
DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user