2018-01-29 13:11:30 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
* Copyright 2015 6WIND S.A.
|
2018-03-20 19:20:35 +00:00
|
|
|
* Copyright 2015 Mellanox Technologies, Ltd
|
2015-10-30 18:52:31 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <errno.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <stdint.h>
|
2017-10-06 15:45:49 +00:00
|
|
|
#include <unistd.h>
|
2019-06-27 15:10:07 +00:00
|
|
|
#include <inttypes.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
|
|
|
|
#include <rte_mbuf.h>
|
|
|
|
#include <rte_malloc.h>
|
2021-01-29 16:48:19 +00:00
|
|
|
#include <ethdev_driver.h>
|
2021-04-12 06:32:24 +00:00
|
|
|
#include <rte_bus_pci.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
#include <rte_common.h>
|
2020-07-19 10:18:10 +00:00
|
|
|
#include <rte_eal_paging.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2020-01-29 12:38:30 +00:00
|
|
|
#include <mlx5_common.h>
|
2020-04-13 21:17:48 +00:00
|
|
|
#include <mlx5_common_mr.h>
|
2020-06-28 09:21:47 +00:00
|
|
|
#include <mlx5_malloc.h>
|
2020-01-29 12:38:27 +00:00
|
|
|
|
2016-06-24 13:17:53 +00:00
|
|
|
#include "mlx5_defs.h"
|
2020-01-29 12:38:27 +00:00
|
|
|
#include "mlx5_utils.h"
|
2015-10-30 18:52:31 +00:00
|
|
|
#include "mlx5.h"
|
2021-04-12 06:32:22 +00:00
|
|
|
#include "mlx5_tx.h"
|
2015-10-30 18:52:31 +00:00
|
|
|
#include "mlx5_rxtx.h"
|
|
|
|
#include "mlx5_autoconf.h"
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Allocate TX queue elements.
|
|
|
|
*
|
2016-06-24 13:17:46 +00:00
|
|
|
* @param txq_ctrl
|
2015-10-30 18:52:31 +00:00
|
|
|
* Pointer to TX queue structure.
|
|
|
|
*/
|
2017-10-09 14:44:48 +00:00
|
|
|
void
|
|
|
|
txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl)
|
2015-10-30 18:52:31 +00:00
|
|
|
{
|
2017-10-09 14:44:48 +00:00
|
|
|
const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
|
2015-10-30 18:52:31 +00:00
|
|
|
unsigned int i;
|
|
|
|
|
2016-06-24 13:17:53 +00:00
|
|
|
for (i = 0; (i != elts_n); ++i)
|
2019-07-21 14:24:57 +00:00
|
|
|
txq_ctrl->txq.elts[i] = NULL;
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs",
|
2019-04-10 18:41:16 +00:00
|
|
|
PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx, elts_n);
|
2016-06-24 13:17:46 +00:00
|
|
|
txq_ctrl->txq.elts_head = 0;
|
|
|
|
txq_ctrl->txq.elts_tail = 0;
|
2016-06-24 13:17:55 +00:00
|
|
|
txq_ctrl->txq.elts_comp = 0;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Free TX queue elements.
|
|
|
|
*
|
2016-06-24 13:17:46 +00:00
|
|
|
* @param txq_ctrl
|
2015-10-30 18:52:31 +00:00
|
|
|
* Pointer to TX queue structure.
|
|
|
|
*/
|
2020-01-09 17:16:05 +00:00
|
|
|
void
|
2017-10-09 14:44:40 +00:00
|
|
|
txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl)
|
2015-10-30 18:52:31 +00:00
|
|
|
{
|
2017-07-06 18:41:06 +00:00
|
|
|
const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
|
|
|
|
const uint16_t elts_m = elts_n - 1;
|
|
|
|
uint16_t elts_head = txq_ctrl->txq.elts_head;
|
|
|
|
uint16_t elts_tail = txq_ctrl->txq.elts_tail;
|
2019-07-21 14:24:57 +00:00
|
|
|
struct rte_mbuf *(*elts)[elts_n] = &txq_ctrl->txq.elts;
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs",
|
2019-04-10 18:41:16 +00:00
|
|
|
PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx);
|
2016-06-24 13:17:46 +00:00
|
|
|
txq_ctrl->txq.elts_head = 0;
|
|
|
|
txq_ctrl->txq.elts_tail = 0;
|
2016-06-24 13:17:55 +00:00
|
|
|
txq_ctrl->txq.elts_comp = 0;
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2016-03-31 09:43:43 +00:00
|
|
|
while (elts_tail != elts_head) {
|
2017-07-06 18:41:06 +00:00
|
|
|
struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(elt != NULL);
|
2017-05-31 17:48:45 +00:00
|
|
|
rte_pktmbuf_free_seg(elt);
|
2020-01-30 16:14:39 +00:00
|
|
|
#ifdef RTE_LIBRTE_MLX5_DEBUG
|
2016-03-31 09:43:43 +00:00
|
|
|
/* Poisoning. */
|
2017-07-06 18:41:06 +00:00
|
|
|
memset(&(*elts)[elts_tail & elts_m],
|
2016-06-24 13:17:53 +00:00
|
|
|
0x77,
|
2017-07-06 18:41:06 +00:00
|
|
|
sizeof((*elts)[elts_tail & elts_m]));
|
2016-03-31 09:43:43 +00:00
|
|
|
#endif
|
2017-07-06 18:41:06 +00:00
|
|
|
++elts_tail;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-10 09:17:00 +00:00
|
|
|
/**
|
|
|
|
* Returns the per-port supported offloads.
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2018-01-10 09:17:00 +00:00
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* Supported Tx offloads.
|
|
|
|
*/
|
|
|
|
uint64_t
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
|
2018-01-10 09:17:00 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2018-01-10 09:17:00 +00:00
|
|
|
uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS |
|
|
|
|
DEV_TX_OFFLOAD_VLAN_INSERT);
|
|
|
|
struct mlx5_dev_config *config = &priv->config;
|
|
|
|
|
|
|
|
if (config->hw_csum)
|
|
|
|
offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
|
|
|
|
DEV_TX_OFFLOAD_UDP_CKSUM |
|
|
|
|
DEV_TX_OFFLOAD_TCP_CKSUM);
|
|
|
|
if (config->tso)
|
|
|
|
offloads |= DEV_TX_OFFLOAD_TCP_TSO;
|
2020-07-16 08:23:15 +00:00
|
|
|
if (config->tx_pp)
|
|
|
|
offloads |= DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP;
|
2018-06-24 06:22:26 +00:00
|
|
|
if (config->swp) {
|
|
|
|
if (config->hw_csum)
|
|
|
|
offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
|
|
|
|
if (config->tso)
|
|
|
|
offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_UDP_TNL_TSO);
|
|
|
|
}
|
2018-01-10 09:17:00 +00:00
|
|
|
if (config->tunnel_en) {
|
|
|
|
if (config->hw_csum)
|
|
|
|
offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
|
|
|
|
if (config->tso)
|
|
|
|
offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
|
2019-11-26 14:06:05 +00:00
|
|
|
DEV_TX_OFFLOAD_GRE_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_GENEVE_TNL_TSO);
|
2018-01-10 09:17:00 +00:00
|
|
|
}
|
2021-01-22 17:12:09 +00:00
|
|
|
if (!config->mprq.enabled)
|
|
|
|
offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
|
2018-01-10 09:17:00 +00:00
|
|
|
return offloads;
|
|
|
|
}
|
|
|
|
|
2020-07-19 15:35:37 +00:00
|
|
|
/* Fetches and drops all SW-owned and error CQEs to synchronize CQ. */
|
|
|
|
static void
|
|
|
|
txq_sync_cq(struct mlx5_txq_data *txq)
|
|
|
|
{
|
|
|
|
volatile struct mlx5_cqe *cqe;
|
|
|
|
int ret, i;
|
|
|
|
|
|
|
|
i = txq->cqe_s;
|
|
|
|
do {
|
|
|
|
cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
|
|
|
|
ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
|
|
|
|
if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
|
|
|
|
if (likely(ret != MLX5_CQE_STATUS_ERR)) {
|
|
|
|
/* No new CQEs in completion queue. */
|
|
|
|
MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
++txq->cq_ci;
|
|
|
|
} while (--i);
|
|
|
|
/* Move all CQEs to HW ownership. */
|
|
|
|
for (i = 0; i < txq->cqe_s; i++) {
|
|
|
|
cqe = &txq->cqes[i];
|
|
|
|
cqe->op_own = MLX5_CQE_INVALIDATE;
|
|
|
|
}
|
|
|
|
/* Resync CQE and WQE (WQ in reset state). */
|
2020-09-23 09:16:37 +00:00
|
|
|
rte_io_wmb();
|
2020-07-19 15:35:37 +00:00
|
|
|
*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
|
2020-11-06 16:58:03 +00:00
|
|
|
txq->cq_pi = txq->cq_ci;
|
2020-09-23 09:16:37 +00:00
|
|
|
rte_io_wmb();
|
2020-07-19 15:35:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Tx queue stop. Device queue goes to the idle state,
|
|
|
|
* all involved mbufs are freed from elts/WQ.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* Tx queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t idx)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_txq_data *txq = (*priv->txqs)[idx];
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl =
|
|
|
|
container_of(txq, struct mlx5_txq_ctrl, txq);
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
|
|
|
|
/* Move QP to RESET state. */
|
2020-10-01 14:09:22 +00:00
|
|
|
ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj, MLX5_TXQ_MOD_RDY2RST,
|
|
|
|
(uint8_t)priv->dev_port);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2020-07-19 15:35:37 +00:00
|
|
|
/* Handle all send completions. */
|
|
|
|
txq_sync_cq(txq);
|
|
|
|
/* Free elts stored in the SQ. */
|
|
|
|
txq_free_elts(txq_ctrl);
|
|
|
|
/* Prevent writing new pkts to SQ by setting no free WQE.*/
|
|
|
|
txq->wqe_ci = txq->wqe_s;
|
|
|
|
txq->wqe_pi = 0;
|
|
|
|
txq->elts_comp = 0;
|
|
|
|
/* Set the actual queue state. */
|
|
|
|
dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Tx queue stop. Device queue goes to the idle state,
|
|
|
|
* all involved mbufs are freed from elts/WQ.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* Tx queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t idx)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2020-09-09 13:01:46 +00:00
|
|
|
if (rte_eth_dev_is_tx_hairpin_queue(dev, idx)) {
|
2020-07-19 15:35:37 +00:00
|
|
|
DRV_LOG(ERR, "Hairpin queue can't be stopped");
|
|
|
|
rte_errno = EINVAL;
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
if (dev->data->tx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STOPPED)
|
|
|
|
return 0;
|
|
|
|
if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
|
|
|
|
ret = mlx5_mp_os_req_queue_control(dev, idx,
|
|
|
|
MLX5_MP_REQ_QUEUE_TX_STOP);
|
|
|
|
} else {
|
|
|
|
ret = mlx5_tx_queue_stop_primary(dev, idx);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Rx queue start. Device queue goes to the ready state,
|
|
|
|
* all required mbufs are allocated and WQ is replenished.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* RX queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t idx)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_txq_data *txq = (*priv->txqs)[idx];
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl =
|
|
|
|
container_of(txq, struct mlx5_txq_ctrl, txq);
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
|
2020-10-01 14:09:22 +00:00
|
|
|
ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
|
2020-10-27 06:43:25 +00:00
|
|
|
MLX5_TXQ_MOD_RST2RDY,
|
2020-10-01 14:09:22 +00:00
|
|
|
(uint8_t)priv->dev_port);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2020-07-19 15:35:37 +00:00
|
|
|
txq_ctrl->txq.wqe_ci = 0;
|
|
|
|
txq_ctrl->txq.wqe_pi = 0;
|
|
|
|
txq_ctrl->txq.elts_comp = 0;
|
|
|
|
/* Set the actual queue state. */
|
|
|
|
dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Rx queue start. Device queue goes to the ready state,
|
|
|
|
* all required mbufs are allocated and WQ is replenished.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* RX queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t idx)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2020-09-09 13:01:46 +00:00
|
|
|
if (rte_eth_dev_is_tx_hairpin_queue(dev, idx)) {
|
2020-07-19 15:35:37 +00:00
|
|
|
DRV_LOG(ERR, "Hairpin queue can't be started");
|
|
|
|
rte_errno = EINVAL;
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
if (dev->data->tx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STARTED)
|
|
|
|
return 0;
|
|
|
|
if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
|
|
|
|
ret = mlx5_mp_os_req_queue_control(dev, idx,
|
|
|
|
MLX5_MP_REQ_QUEUE_TX_START);
|
|
|
|
} else {
|
|
|
|
ret = mlx5_tx_queue_start_primary(dev, idx);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
/**
|
2019-10-30 23:53:15 +00:00
|
|
|
* Tx queue presetup checks.
|
2015-10-30 18:52:31 +00:00
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
2019-10-30 23:53:15 +00:00
|
|
|
* Tx queue index.
|
2015-10-30 18:52:31 +00:00
|
|
|
* @param desc
|
|
|
|
* Number of descriptors to configure in queue.
|
|
|
|
*
|
|
|
|
* @return
|
2018-03-05 12:21:06 +00:00
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
2015-10-30 18:52:31 +00:00
|
|
|
*/
|
2019-10-30 23:53:15 +00:00
|
|
|
static int
|
2020-06-11 17:43:27 +00:00
|
|
|
mlx5_tx_queue_pre_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t *desc)
|
2015-10-30 18:52:31 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2020-06-11 17:43:27 +00:00
|
|
|
if (*desc <= MLX5_TX_COMP_THRESH) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(WARNING,
|
|
|
|
"port %u number of descriptors requested for Tx queue"
|
|
|
|
" %u must be higher than MLX5_TX_COMP_THRESH, using %u"
|
2020-06-11 17:43:27 +00:00
|
|
|
" instead of %u", dev->data->port_id, idx,
|
|
|
|
MLX5_TX_COMP_THRESH + 1, *desc);
|
|
|
|
*desc = MLX5_TX_COMP_THRESH + 1;
|
2016-06-24 13:17:55 +00:00
|
|
|
}
|
2020-06-11 17:43:27 +00:00
|
|
|
if (!rte_is_power_of_2(*desc)) {
|
|
|
|
*desc = 1 << log2above(*desc);
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(WARNING,
|
|
|
|
"port %u increased number of descriptors in Tx queue"
|
|
|
|
" %u to the next power of two (%d)",
|
2020-06-11 17:43:27 +00:00
|
|
|
dev->data->port_id, idx, *desc);
|
2016-06-24 13:17:53 +00:00
|
|
|
}
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u configuring queue %u for %u descriptors",
|
2020-06-11 17:43:27 +00:00
|
|
|
dev->data->port_id, idx, *desc);
|
2015-10-30 18:52:31 +00:00
|
|
|
if (idx >= priv->txqs_n) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(ERR, "port %u Tx queue index out of range (%u >= %u)",
|
|
|
|
dev->data->port_id, idx, priv->txqs_n);
|
2018-03-05 12:21:06 +00:00
|
|
|
rte_errno = EOVERFLOW;
|
|
|
|
return -rte_errno;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
2018-03-05 12:21:04 +00:00
|
|
|
if (!mlx5_txq_releasable(dev, idx)) {
|
2018-03-05 12:21:06 +00:00
|
|
|
rte_errno = EBUSY;
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(ERR, "port %u unable to release queue index %u",
|
|
|
|
dev->data->port_id, idx);
|
2018-03-05 12:21:06 +00:00
|
|
|
return -rte_errno;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_release(dev, idx);
|
2019-10-30 23:53:15 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2020-10-01 14:09:14 +00:00
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
/**
|
|
|
|
* DPDK callback to configure a TX queue.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
* @param desc
|
|
|
|
* Number of descriptors to configure in queue.
|
|
|
|
* @param socket
|
|
|
|
* NUMA socket on which memory must be allocated.
|
|
|
|
* @param[in] conf
|
|
|
|
* Thresholds parameters.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
|
|
|
unsigned int socket, const struct rte_eth_txconf *conf)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_txq_data *txq = (*priv->txqs)[idx];
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl =
|
|
|
|
container_of(txq, struct mlx5_txq_ctrl, txq);
|
|
|
|
int res;
|
|
|
|
|
2020-06-11 17:43:27 +00:00
|
|
|
res = mlx5_tx_queue_pre_setup(dev, idx, &desc);
|
2019-10-30 23:53:15 +00:00
|
|
|
if (res)
|
|
|
|
return res;
|
2018-03-05 12:21:04 +00:00
|
|
|
txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf);
|
2017-10-09 14:44:48 +00:00
|
|
|
if (!txq_ctrl) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(ERR, "port %u unable to allocate queue index %u",
|
|
|
|
dev->data->port_id, idx);
|
2018-03-05 12:21:06 +00:00
|
|
|
return -rte_errno;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u adding Tx queue %u to list",
|
|
|
|
dev->data->port_id, idx);
|
2017-10-09 14:44:47 +00:00
|
|
|
(*priv->txqs)[idx] = &txq_ctrl->txq;
|
2018-03-05 12:21:06 +00:00
|
|
|
return 0;
|
2015-10-30 18:52:31 +00:00
|
|
|
}
|
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
/**
|
|
|
|
* DPDK callback to configure a TX hairpin queue.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device structure.
|
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
* @param desc
|
|
|
|
* Number of descriptors to configure in queue.
|
|
|
|
* @param[in] hairpin_conf
|
|
|
|
* The hairpin binding configuration.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
mlx5_tx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
|
|
|
|
uint16_t desc,
|
|
|
|
const struct rte_eth_hairpin_conf *hairpin_conf)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_txq_data *txq = (*priv->txqs)[idx];
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl =
|
|
|
|
container_of(txq, struct mlx5_txq_ctrl, txq);
|
|
|
|
int res;
|
|
|
|
|
2020-06-11 17:43:27 +00:00
|
|
|
res = mlx5_tx_queue_pre_setup(dev, idx, &desc);
|
2019-10-30 23:53:15 +00:00
|
|
|
if (res)
|
|
|
|
return res;
|
2020-10-26 16:37:41 +00:00
|
|
|
if (hairpin_conf->peer_count != 1) {
|
2019-10-30 23:53:15 +00:00
|
|
|
rte_errno = EINVAL;
|
2020-10-26 16:37:41 +00:00
|
|
|
DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue index %u"
|
|
|
|
" peer count is %u", dev->data->port_id,
|
|
|
|
idx, hairpin_conf->peer_count);
|
2019-10-30 23:53:15 +00:00
|
|
|
return -rte_errno;
|
|
|
|
}
|
2020-10-26 16:37:41 +00:00
|
|
|
if (hairpin_conf->peers[0].port == dev->data->port_id) {
|
|
|
|
if (hairpin_conf->peers[0].queue >= priv->rxqs_n) {
|
|
|
|
rte_errno = EINVAL;
|
|
|
|
DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue"
|
|
|
|
" index %u, Rx %u is larger than %u",
|
|
|
|
dev->data->port_id, idx,
|
|
|
|
hairpin_conf->peers[0].queue, priv->txqs_n);
|
|
|
|
return -rte_errno;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (hairpin_conf->manual_bind == 0 ||
|
|
|
|
hairpin_conf->tx_explicit == 0) {
|
|
|
|
rte_errno = EINVAL;
|
|
|
|
DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue"
|
|
|
|
" index %u peer port %u with attributes %u %u",
|
|
|
|
dev->data->port_id, idx,
|
|
|
|
hairpin_conf->peers[0].port,
|
|
|
|
hairpin_conf->manual_bind,
|
|
|
|
hairpin_conf->tx_explicit);
|
|
|
|
return -rte_errno;
|
|
|
|
}
|
|
|
|
}
|
2019-10-30 23:53:15 +00:00
|
|
|
txq_ctrl = mlx5_txq_hairpin_new(dev, idx, desc, hairpin_conf);
|
|
|
|
if (!txq_ctrl) {
|
|
|
|
DRV_LOG(ERR, "port %u unable to allocate queue index %u",
|
|
|
|
dev->data->port_id, idx);
|
|
|
|
return -rte_errno;
|
|
|
|
}
|
|
|
|
DRV_LOG(DEBUG, "port %u adding Tx queue %u to list",
|
|
|
|
dev->data->port_id, idx);
|
|
|
|
(*priv->txqs)[idx] = &txq_ctrl->txq;
|
2020-07-19 15:35:37 +00:00
|
|
|
dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN;
|
2019-10-30 23:53:15 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
/**
|
|
|
|
* DPDK callback to release a TX queue.
|
|
|
|
*
|
|
|
|
* @param dpdk_txq
|
|
|
|
* Generic TX queue pointer.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
mlx5_tx_queue_release(void *dpdk_txq)
|
|
|
|
{
|
2017-10-09 14:44:40 +00:00
|
|
|
struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq;
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl;
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv;
|
2015-10-30 18:52:31 +00:00
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
if (txq == NULL)
|
|
|
|
return;
|
2017-10-09 14:44:40 +00:00
|
|
|
txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
|
2016-06-24 13:17:53 +00:00
|
|
|
priv = txq_ctrl->priv;
|
2015-10-30 18:52:31 +00:00
|
|
|
for (i = 0; (i != priv->txqs_n); ++i)
|
|
|
|
if ((*priv->txqs)[i] == txq) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u removing Tx queue %u from list",
|
2019-04-10 18:41:16 +00:00
|
|
|
PORT_ID(priv), txq->idx);
|
2020-05-06 18:10:59 +00:00
|
|
|
mlx5_txq_release(ETH_DEV(priv), i);
|
2015-10-30 18:52:31 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2017-10-06 15:45:49 +00:00
|
|
|
|
2019-11-08 15:07:50 +00:00
|
|
|
/**
|
|
|
|
* Configure the doorbell register non-cached attribute.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
* @param page_size
|
|
|
|
* Systme page size
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
txq_uar_ncattr_init(struct mlx5_txq_ctrl *txq_ctrl, size_t page_size)
|
|
|
|
{
|
2019-11-15 11:35:06 +00:00
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
2020-01-13 14:22:37 +00:00
|
|
|
off_t cmd;
|
2019-11-08 15:07:50 +00:00
|
|
|
|
2019-11-15 11:35:06 +00:00
|
|
|
txq_ctrl->txq.db_heu = priv->config.dbnc == MLX5_TXDB_HEURISTIC;
|
2019-11-08 15:07:50 +00:00
|
|
|
txq_ctrl->txq.db_nc = 0;
|
|
|
|
/* Check the doorbell register mapping type. */
|
|
|
|
cmd = txq_ctrl->uar_mmap_offset / page_size;
|
|
|
|
cmd >>= MLX5_UAR_MMAP_CMD_SHIFT;
|
|
|
|
cmd &= MLX5_UAR_MMAP_CMD_MASK;
|
|
|
|
if (cmd == MLX5_MMAP_GET_NC_PAGES_CMD)
|
|
|
|
txq_ctrl->txq.db_nc = 1;
|
|
|
|
}
|
|
|
|
|
2019-04-10 18:41:17 +00:00
|
|
|
/**
|
|
|
|
* Initialize Tx UAR registers for primary process.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
*/
|
2020-10-01 14:09:18 +00:00
|
|
|
void
|
2019-04-10 18:41:17 +00:00
|
|
|
txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
|
|
|
struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv));
|
|
|
|
#ifndef RTE_ARCH_64
|
|
|
|
unsigned int lock_idx;
|
|
|
|
#endif
|
2020-07-19 10:18:10 +00:00
|
|
|
const size_t page_size = rte_mem_page_size();
|
|
|
|
if (page_size == (size_t)-1) {
|
|
|
|
DRV_LOG(ERR, "Failed to get mem page size");
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
}
|
2019-04-10 18:41:17 +00:00
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
if (txq_ctrl->type != MLX5_TXQ_TYPE_STANDARD)
|
|
|
|
return;
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
|
|
|
|
MLX5_ASSERT(ppriv);
|
2019-04-10 18:41:17 +00:00
|
|
|
ppriv->uar_table[txq_ctrl->txq.idx] = txq_ctrl->bf_reg;
|
2019-11-08 15:07:50 +00:00
|
|
|
txq_uar_ncattr_init(txq_ctrl, page_size);
|
2019-04-10 18:41:17 +00:00
|
|
|
#ifndef RTE_ARCH_64
|
|
|
|
/* Assign an UAR lock according to UAR page number */
|
|
|
|
lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
|
|
|
|
MLX5_UAR_PAGE_NUM_MASK;
|
2020-07-16 08:23:06 +00:00
|
|
|
txq_ctrl->txq.uar_lock = &priv->sh->uar_lock[lock_idx];
|
2019-04-10 18:41:17 +00:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Remap UAR register of a Tx queue for secondary process.
|
|
|
|
*
|
|
|
|
* Remapped address is stored at the table in the process private structure of
|
|
|
|
* the device, indexed by queue index.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
* @param fd
|
|
|
|
* Verbs file descriptor to map UAR pages.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
txq_uar_init_secondary(struct mlx5_txq_ctrl *txq_ctrl, int fd)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
|
|
|
struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv));
|
|
|
|
struct mlx5_txq_data *txq = &txq_ctrl->txq;
|
|
|
|
void *addr;
|
|
|
|
uintptr_t uar_va;
|
|
|
|
uintptr_t offset;
|
2020-07-19 10:18:10 +00:00
|
|
|
const size_t page_size = rte_mem_page_size();
|
|
|
|
if (page_size == (size_t)-1) {
|
|
|
|
DRV_LOG(ERR, "Failed to get mem page size");
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
return -rte_errno;
|
|
|
|
}
|
2019-04-10 18:41:17 +00:00
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
if (txq_ctrl->type != MLX5_TXQ_TYPE_STANDARD)
|
|
|
|
return 0;
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(ppriv);
|
2019-04-10 18:41:17 +00:00
|
|
|
/*
|
|
|
|
* As rdma-core, UARs are mapped in size of OS page
|
|
|
|
* size. Ref to libmlx5 function: mlx5_init_context()
|
|
|
|
*/
|
|
|
|
uar_va = (uintptr_t)txq_ctrl->bf_reg;
|
|
|
|
offset = uar_va & (page_size - 1); /* Offset in page. */
|
2020-07-19 10:18:10 +00:00
|
|
|
addr = rte_mem_map(NULL, page_size, RTE_PROT_WRITE, RTE_MAP_SHARED,
|
|
|
|
fd, txq_ctrl->uar_mmap_offset);
|
|
|
|
if (!addr) {
|
2019-04-10 18:41:17 +00:00
|
|
|
DRV_LOG(ERR,
|
|
|
|
"port %u mmap failed for BF reg of txq %u",
|
|
|
|
txq->port_id, txq->idx);
|
|
|
|
rte_errno = ENXIO;
|
|
|
|
return -rte_errno;
|
|
|
|
}
|
|
|
|
addr = RTE_PTR_ADD(addr, offset);
|
|
|
|
ppriv->uar_table[txq->idx] = addr;
|
2019-11-08 15:07:50 +00:00
|
|
|
txq_uar_ncattr_init(txq_ctrl, page_size);
|
2019-04-10 18:41:17 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Unmap UAR register of a Tx queue for secondary process.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
txq_uar_uninit_secondary(struct mlx5_txq_ctrl *txq_ctrl)
|
|
|
|
{
|
|
|
|
struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(txq_ctrl->priv));
|
|
|
|
void *addr;
|
2020-07-19 10:18:10 +00:00
|
|
|
const size_t page_size = rte_mem_page_size();
|
|
|
|
if (page_size == (size_t)-1) {
|
|
|
|
DRV_LOG(ERR, "Failed to get mem page size");
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
}
|
2019-04-10 18:41:17 +00:00
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
if (txq_ctrl->type != MLX5_TXQ_TYPE_STANDARD)
|
|
|
|
return;
|
2019-04-10 18:41:17 +00:00
|
|
|
addr = ppriv->uar_table[txq_ctrl->txq.idx];
|
2020-07-19 10:18:10 +00:00
|
|
|
rte_mem_unmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size);
|
2019-04-10 18:41:17 +00:00
|
|
|
}
|
2017-10-06 15:45:49 +00:00
|
|
|
|
2020-05-28 06:59:49 +00:00
|
|
|
/**
|
|
|
|
* Deinitialize Tx UAR registers for secondary process.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev)
|
|
|
|
{
|
2021-01-24 11:02:04 +00:00
|
|
|
struct mlx5_proc_priv *ppriv = (struct mlx5_proc_priv *)
|
|
|
|
dev->process_private;
|
|
|
|
const size_t page_size = rte_mem_page_size();
|
|
|
|
void *addr;
|
2020-05-28 06:59:49 +00:00
|
|
|
unsigned int i;
|
|
|
|
|
2021-01-24 11:02:04 +00:00
|
|
|
if (page_size == (size_t)-1) {
|
|
|
|
DRV_LOG(ERR, "Failed to get mem page size");
|
|
|
|
return;
|
|
|
|
}
|
2020-05-28 06:59:49 +00:00
|
|
|
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
|
2021-01-24 11:02:04 +00:00
|
|
|
for (i = 0; i != ppriv->uar_table_sz; ++i) {
|
|
|
|
if (!ppriv->uar_table[i])
|
2020-05-28 06:59:49 +00:00
|
|
|
continue;
|
2021-01-24 11:02:04 +00:00
|
|
|
addr = ppriv->uar_table[i];
|
|
|
|
rte_mem_unmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size);
|
|
|
|
|
2020-05-28 06:59:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-06 15:45:49 +00:00
|
|
|
/**
|
2019-04-10 18:41:17 +00:00
|
|
|
* Initialize Tx UAR registers for secondary process.
|
2017-10-06 15:45:49 +00:00
|
|
|
*
|
2019-04-10 18:41:17 +00:00
|
|
|
* @param dev
|
2018-03-05 12:21:04 +00:00
|
|
|
* Pointer to Ethernet device.
|
2017-10-06 15:45:49 +00:00
|
|
|
* @param fd
|
|
|
|
* Verbs file descriptor to map UAR pages.
|
|
|
|
*
|
|
|
|
* @return
|
2018-03-05 12:21:06 +00:00
|
|
|
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
2017-10-06 15:45:49 +00:00
|
|
|
*/
|
|
|
|
int
|
2019-04-10 18:41:17 +00:00
|
|
|
mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd)
|
2017-10-06 15:45:49 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2017-10-09 14:44:40 +00:00
|
|
|
struct mlx5_txq_data *txq;
|
|
|
|
struct mlx5_txq_ctrl *txq_ctrl;
|
2019-04-10 18:41:17 +00:00
|
|
|
unsigned int i;
|
|
|
|
int ret;
|
2017-10-06 15:45:49 +00:00
|
|
|
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
|
2017-10-06 15:45:49 +00:00
|
|
|
for (i = 0; i != priv->txqs_n; ++i) {
|
2018-02-06 09:26:21 +00:00
|
|
|
if (!(*priv->txqs)[i])
|
|
|
|
continue;
|
2017-10-06 15:45:49 +00:00
|
|
|
txq = (*priv->txqs)[i];
|
2017-10-09 14:44:40 +00:00
|
|
|
txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
|
2019-10-30 23:53:15 +00:00
|
|
|
if (txq_ctrl->type != MLX5_TXQ_TYPE_STANDARD)
|
|
|
|
continue;
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(txq->idx == (uint16_t)i);
|
2019-04-10 18:41:17 +00:00
|
|
|
ret = txq_uar_init_secondary(txq_ctrl, fd);
|
|
|
|
if (ret)
|
|
|
|
goto error;
|
2017-10-06 15:45:49 +00:00
|
|
|
}
|
|
|
|
return 0;
|
2019-04-10 18:41:17 +00:00
|
|
|
error:
|
|
|
|
/* Rollback. */
|
|
|
|
do {
|
|
|
|
if (!(*priv->txqs)[i])
|
|
|
|
continue;
|
|
|
|
txq = (*priv->txqs)[i];
|
|
|
|
txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
|
|
|
|
txq_uar_uninit_secondary(txq_ctrl);
|
|
|
|
} while (i--);
|
|
|
|
return -rte_errno;
|
2017-10-06 15:45:49 +00:00
|
|
|
}
|
2017-10-09 14:44:47 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Verify the Verbs Tx queue list is empty
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2017-10-09 14:44:47 +00:00
|
|
|
*
|
2018-03-05 12:21:01 +00:00
|
|
|
* @return
|
|
|
|
* The number of object not released.
|
2017-10-09 14:44:47 +00:00
|
|
|
*/
|
|
|
|
int
|
2019-10-30 23:53:14 +00:00
|
|
|
mlx5_txq_obj_verify(struct rte_eth_dev *dev)
|
2017-10-09 14:44:47 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2017-10-09 14:44:47 +00:00
|
|
|
int ret = 0;
|
2019-10-30 23:53:14 +00:00
|
|
|
struct mlx5_txq_obj *txq_obj;
|
2017-10-09 14:44:47 +00:00
|
|
|
|
2019-10-30 23:53:14 +00:00
|
|
|
LIST_FOREACH(txq_obj, &priv->txqsobj, next) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced",
|
2019-10-30 23:53:14 +00:00
|
|
|
dev->data->port_id, txq_obj->txq_ctrl->txq.idx);
|
2017-10-09 14:44:47 +00:00
|
|
|
++ret;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
2017-10-09 14:44:48 +00:00
|
|
|
|
2019-05-01 01:01:43 +00:00
|
|
|
/**
|
2019-07-18 19:40:52 +00:00
|
|
|
* Calculate the total number of WQEBB for Tx queue.
|
2019-05-01 01:01:43 +00:00
|
|
|
*
|
|
|
|
* Simplified version of calc_sq_size() in rdma-core.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* The number of WQEBB.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl)
|
|
|
|
{
|
|
|
|
unsigned int wqe_size;
|
|
|
|
const unsigned int desc = 1 << txq_ctrl->txq.elts_n;
|
|
|
|
|
2019-07-21 14:24:57 +00:00
|
|
|
wqe_size = MLX5_WQE_CSEG_SIZE +
|
|
|
|
MLX5_WQE_ESEG_SIZE +
|
|
|
|
MLX5_WSEG_SIZE -
|
|
|
|
MLX5_ESEG_MIN_INLINE_SIZE +
|
|
|
|
txq_ctrl->max_inline_data;
|
2019-05-01 01:01:43 +00:00
|
|
|
return rte_align32pow2(wqe_size * desc) / MLX5_WQE_SIZE;
|
|
|
|
}
|
|
|
|
|
2019-10-01 06:53:37 +00:00
|
|
|
/**
|
|
|
|
* Calculate the maximal inline data size for Tx queue.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* The maximal inline data size.
|
|
|
|
*/
|
|
|
|
static unsigned int
|
|
|
|
txq_calc_inline_max(struct mlx5_txq_ctrl *txq_ctrl)
|
|
|
|
{
|
|
|
|
const unsigned int desc = 1 << txq_ctrl->txq.elts_n;
|
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
|
|
|
unsigned int wqe_size;
|
|
|
|
|
2020-06-03 15:05:58 +00:00
|
|
|
wqe_size = priv->sh->device_attr.max_qp_wr / desc;
|
2019-10-01 06:53:37 +00:00
|
|
|
if (!wqe_size)
|
|
|
|
return 0;
|
|
|
|
/*
|
|
|
|
* This calculation is derived from tthe source of
|
|
|
|
* mlx5_calc_send_wqe() in rdma_core library.
|
|
|
|
*/
|
|
|
|
wqe_size = wqe_size * MLX5_WQE_SIZE -
|
|
|
|
MLX5_WQE_CSEG_SIZE -
|
|
|
|
MLX5_WQE_ESEG_SIZE -
|
|
|
|
MLX5_WSEG_SIZE -
|
|
|
|
MLX5_WSEG_SIZE +
|
|
|
|
MLX5_DSEG_MIN_INLINE_SIZE;
|
|
|
|
return wqe_size;
|
|
|
|
}
|
|
|
|
|
2017-10-09 14:44:48 +00:00
|
|
|
/**
|
2018-01-10 09:16:58 +00:00
|
|
|
* Set Tx queue parameters from device configuration.
|
2017-10-09 14:44:48 +00:00
|
|
|
*
|
2018-01-10 09:16:58 +00:00
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
2017-10-09 14:44:48 +00:00
|
|
|
*/
|
2018-01-10 09:16:58 +00:00
|
|
|
static void
|
|
|
|
txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
|
2017-10-09 14:44:48 +00:00
|
|
|
{
|
2019-07-21 14:24:57 +00:00
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
|
|
|
struct mlx5_dev_config *config = &priv->config;
|
|
|
|
unsigned int inlen_send; /* Inline data for ordinary SEND.*/
|
|
|
|
unsigned int inlen_empw; /* Inline data for enhanced MPW. */
|
|
|
|
unsigned int inlen_mode; /* Minimal required Inline data. */
|
|
|
|
unsigned int txqs_inline; /* Min Tx queues to enable inline. */
|
|
|
|
uint64_t dev_txoff = priv->dev_data->dev_conf.txmode.offloads;
|
|
|
|
bool tso = txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO |
|
|
|
|
DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_GRE_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_IP_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_UDP_TNL_TSO);
|
|
|
|
bool vlan_inline;
|
|
|
|
unsigned int temp;
|
|
|
|
|
2021-01-22 17:12:09 +00:00
|
|
|
txq_ctrl->txq.fast_free =
|
|
|
|
!!((txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) &&
|
|
|
|
!(txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_MULTI_SEGS) &&
|
|
|
|
!config->mprq.enabled);
|
2019-07-21 14:24:57 +00:00
|
|
|
if (config->txqs_inline == MLX5_ARG_UNSET)
|
|
|
|
txqs_inline =
|
|
|
|
#if defined(RTE_ARCH_ARM64)
|
2019-09-25 07:53:24 +00:00
|
|
|
(priv->pci_dev->id.device_id ==
|
2019-07-21 14:24:57 +00:00
|
|
|
PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) ?
|
|
|
|
MLX5_INLINE_MAX_TXQS_BLUEFIELD :
|
|
|
|
#endif
|
|
|
|
MLX5_INLINE_MAX_TXQS;
|
|
|
|
else
|
|
|
|
txqs_inline = (unsigned int)config->txqs_inline;
|
|
|
|
inlen_send = (config->txq_inline_max == MLX5_ARG_UNSET) ?
|
|
|
|
MLX5_SEND_DEF_INLINE_LEN :
|
|
|
|
(unsigned int)config->txq_inline_max;
|
|
|
|
inlen_empw = (config->txq_inline_mpw == MLX5_ARG_UNSET) ?
|
|
|
|
MLX5_EMPW_DEF_INLINE_LEN :
|
|
|
|
(unsigned int)config->txq_inline_mpw;
|
|
|
|
inlen_mode = (config->txq_inline_min == MLX5_ARG_UNSET) ?
|
|
|
|
0 : (unsigned int)config->txq_inline_min;
|
2019-11-26 10:45:03 +00:00
|
|
|
if (config->mps != MLX5_MPW_ENHANCED && config->mps != MLX5_MPW)
|
2019-07-21 14:24:57 +00:00
|
|
|
inlen_empw = 0;
|
|
|
|
/*
|
|
|
|
* If there is requested minimal amount of data to inline
|
|
|
|
* we MUST enable inlining. This is a case for ConnectX-4
|
|
|
|
* which usually requires L2 inlined for correct operating
|
2020-02-24 19:52:14 +00:00
|
|
|
* and ConnectX-4 Lx which requires L2-L4 inlined to
|
2019-07-21 14:24:57 +00:00
|
|
|
* support E-Switch Flows.
|
|
|
|
*/
|
|
|
|
if (inlen_mode) {
|
|
|
|
if (inlen_mode <= MLX5_ESEG_MIN_INLINE_SIZE) {
|
|
|
|
/*
|
|
|
|
* Optimize minimal inlining for single
|
|
|
|
* segment packets to fill one WQEBB
|
|
|
|
* without gaps.
|
|
|
|
*/
|
|
|
|
temp = MLX5_ESEG_MIN_INLINE_SIZE;
|
|
|
|
} else {
|
|
|
|
temp = inlen_mode - MLX5_ESEG_MIN_INLINE_SIZE;
|
|
|
|
temp = RTE_ALIGN(temp, MLX5_WSEG_SIZE) +
|
|
|
|
MLX5_ESEG_MIN_INLINE_SIZE;
|
|
|
|
temp = RTE_MIN(temp, MLX5_SEND_MAX_INLINE_LEN);
|
|
|
|
}
|
|
|
|
if (temp != inlen_mode) {
|
|
|
|
DRV_LOG(INFO,
|
|
|
|
"port %u minimal required inline setting"
|
|
|
|
" aligned from %u to %u",
|
|
|
|
PORT_ID(priv), inlen_mode, temp);
|
|
|
|
inlen_mode = temp;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If port is configured to support VLAN insertion and device
|
|
|
|
* does not support this feature by HW (for NICs before ConnectX-5
|
|
|
|
* or in case of wqe_vlan_insert flag is not set) we must enable
|
|
|
|
* data inline on all queues because it is supported by single
|
|
|
|
* tx_burst routine.
|
|
|
|
*/
|
|
|
|
txq_ctrl->txq.vlan_en = config->hw_vlan_insert;
|
|
|
|
vlan_inline = (dev_txoff & DEV_TX_OFFLOAD_VLAN_INSERT) &&
|
|
|
|
!config->hw_vlan_insert;
|
|
|
|
/*
|
|
|
|
* If there are few Tx queues it is prioritized
|
|
|
|
* to save CPU cycles and disable data inlining at all.
|
|
|
|
*/
|
2019-08-05 13:03:52 +00:00
|
|
|
if (inlen_send && priv->txqs_n >= txqs_inline) {
|
2019-07-21 14:24:57 +00:00
|
|
|
/*
|
|
|
|
* The data sent with ordinal MLX5_OPCODE_SEND
|
|
|
|
* may be inlined in Ethernet Segment, align the
|
|
|
|
* length accordingly to fit entire WQEBBs.
|
|
|
|
*/
|
2019-10-01 06:53:37 +00:00
|
|
|
temp = RTE_MAX(inlen_send,
|
|
|
|
MLX5_ESEG_MIN_INLINE_SIZE + MLX5_WQE_DSEG_SIZE);
|
|
|
|
temp -= MLX5_ESEG_MIN_INLINE_SIZE + MLX5_WQE_DSEG_SIZE;
|
|
|
|
temp = RTE_ALIGN(temp, MLX5_WQE_SIZE);
|
|
|
|
temp += MLX5_ESEG_MIN_INLINE_SIZE + MLX5_WQE_DSEG_SIZE;
|
2019-07-21 14:24:57 +00:00
|
|
|
temp = RTE_MIN(temp, MLX5_WQE_SIZE_MAX +
|
|
|
|
MLX5_ESEG_MIN_INLINE_SIZE -
|
|
|
|
MLX5_WQE_CSEG_SIZE -
|
|
|
|
MLX5_WQE_ESEG_SIZE -
|
|
|
|
MLX5_WQE_DSEG_SIZE * 2);
|
|
|
|
temp = RTE_MIN(temp, MLX5_SEND_MAX_INLINE_LEN);
|
|
|
|
temp = RTE_MAX(temp, inlen_mode);
|
|
|
|
if (temp != inlen_send) {
|
|
|
|
DRV_LOG(INFO,
|
|
|
|
"port %u ordinary send inline setting"
|
|
|
|
" aligned from %u to %u",
|
|
|
|
PORT_ID(priv), inlen_send, temp);
|
|
|
|
inlen_send = temp;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Not aligned to cache lines, but to WQEs.
|
|
|
|
* First bytes of data (initial alignment)
|
|
|
|
* is going to be copied explicitly at the
|
|
|
|
* beginning of inlining buffer in Ethernet
|
|
|
|
* Segment.
|
|
|
|
*/
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
|
|
|
|
MLX5_ASSERT(inlen_send <= MLX5_WQE_SIZE_MAX +
|
|
|
|
MLX5_ESEG_MIN_INLINE_SIZE -
|
|
|
|
MLX5_WQE_CSEG_SIZE -
|
|
|
|
MLX5_WQE_ESEG_SIZE -
|
|
|
|
MLX5_WQE_DSEG_SIZE * 2);
|
2019-08-05 13:03:52 +00:00
|
|
|
} else if (inlen_mode) {
|
2019-07-21 14:24:57 +00:00
|
|
|
/*
|
|
|
|
* If minimal inlining is requested we must
|
|
|
|
* enable inlining in general, despite the
|
2019-08-05 13:03:52 +00:00
|
|
|
* number of configured queues. Ignore the
|
|
|
|
* txq_inline_max devarg, this is not
|
|
|
|
* full-featured inline.
|
2019-07-21 14:24:57 +00:00
|
|
|
*/
|
|
|
|
inlen_send = inlen_mode;
|
2019-08-05 13:03:52 +00:00
|
|
|
inlen_empw = 0;
|
|
|
|
} else if (vlan_inline) {
|
|
|
|
/*
|
|
|
|
* Hardware does not report offload for
|
|
|
|
* VLAN insertion, we must enable data inline
|
|
|
|
* to implement feature by software.
|
|
|
|
*/
|
|
|
|
inlen_send = MLX5_ESEG_MIN_INLINE_SIZE;
|
|
|
|
inlen_empw = 0;
|
|
|
|
} else {
|
2019-07-21 14:24:57 +00:00
|
|
|
inlen_send = 0;
|
|
|
|
inlen_empw = 0;
|
|
|
|
}
|
2019-08-05 13:03:52 +00:00
|
|
|
txq_ctrl->txq.inlen_send = inlen_send;
|
|
|
|
txq_ctrl->txq.inlen_mode = inlen_mode;
|
|
|
|
txq_ctrl->txq.inlen_empw = 0;
|
2019-07-21 14:24:57 +00:00
|
|
|
if (inlen_send && inlen_empw && priv->txqs_n >= txqs_inline) {
|
|
|
|
/*
|
|
|
|
* The data sent with MLX5_OPCODE_ENHANCED_MPSW
|
|
|
|
* may be inlined in Data Segment, align the
|
|
|
|
* length accordingly to fit entire WQEBBs.
|
|
|
|
*/
|
2019-10-01 06:53:37 +00:00
|
|
|
temp = RTE_MAX(inlen_empw,
|
|
|
|
MLX5_WQE_SIZE + MLX5_DSEG_MIN_INLINE_SIZE);
|
|
|
|
temp -= MLX5_DSEG_MIN_INLINE_SIZE;
|
|
|
|
temp = RTE_ALIGN(temp, MLX5_WQE_SIZE);
|
|
|
|
temp += MLX5_DSEG_MIN_INLINE_SIZE;
|
2019-07-21 14:24:57 +00:00
|
|
|
temp = RTE_MIN(temp, MLX5_WQE_SIZE_MAX +
|
|
|
|
MLX5_DSEG_MIN_INLINE_SIZE -
|
|
|
|
MLX5_WQE_CSEG_SIZE -
|
|
|
|
MLX5_WQE_ESEG_SIZE -
|
|
|
|
MLX5_WQE_DSEG_SIZE);
|
|
|
|
temp = RTE_MIN(temp, MLX5_EMPW_MAX_INLINE_LEN);
|
|
|
|
if (temp != inlen_empw) {
|
|
|
|
DRV_LOG(INFO,
|
|
|
|
"port %u enhanced empw inline setting"
|
|
|
|
" aligned from %u to %u",
|
|
|
|
PORT_ID(priv), inlen_empw, temp);
|
|
|
|
inlen_empw = temp;
|
|
|
|
}
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(inlen_empw >= MLX5_ESEG_MIN_INLINE_SIZE);
|
|
|
|
MLX5_ASSERT(inlen_empw <= MLX5_WQE_SIZE_MAX +
|
|
|
|
MLX5_DSEG_MIN_INLINE_SIZE -
|
|
|
|
MLX5_WQE_CSEG_SIZE -
|
|
|
|
MLX5_WQE_ESEG_SIZE -
|
|
|
|
MLX5_WQE_DSEG_SIZE);
|
2019-07-21 14:24:57 +00:00
|
|
|
txq_ctrl->txq.inlen_empw = inlen_empw;
|
|
|
|
}
|
|
|
|
txq_ctrl->max_inline_data = RTE_MAX(inlen_send, inlen_empw);
|
|
|
|
if (tso) {
|
|
|
|
txq_ctrl->max_tso_header = MLX5_MAX_TSO_HEADER;
|
|
|
|
txq_ctrl->max_inline_data = RTE_MAX(txq_ctrl->max_inline_data,
|
|
|
|
MLX5_MAX_TSO_HEADER);
|
|
|
|
txq_ctrl->txq.tso_en = 1;
|
|
|
|
}
|
|
|
|
txq_ctrl->txq.tunnel_en = config->tunnel_en | config->swp;
|
|
|
|
txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_UDP_TNL_TSO |
|
|
|
|
DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) &
|
|
|
|
txq_ctrl->txq.offloads) && config->swp;
|
2018-01-10 09:16:58 +00:00
|
|
|
}
|
|
|
|
|
2019-10-01 06:53:37 +00:00
|
|
|
/**
|
|
|
|
* Adjust Tx queue data inline parameters for large queue sizes.
|
|
|
|
* The data inline feature requires multiple WQEs to fit the packets,
|
|
|
|
* and if the large amount of Tx descriptors is requested by application
|
|
|
|
* the total WQE amount may exceed the hardware capabilities. If the
|
|
|
|
* default inline setting are used we can try to adjust these ones and
|
|
|
|
* meet the hardware requirements and not exceed the queue size.
|
|
|
|
*
|
|
|
|
* @param txq_ctrl
|
|
|
|
* Pointer to Tx queue control structure.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* Zero on success, otherwise the parameters can not be adjusted.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
txq_adjust_params(struct mlx5_txq_ctrl *txq_ctrl)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = txq_ctrl->priv;
|
|
|
|
struct mlx5_dev_config *config = &priv->config;
|
|
|
|
unsigned int max_inline;
|
|
|
|
|
|
|
|
max_inline = txq_calc_inline_max(txq_ctrl);
|
|
|
|
if (!txq_ctrl->txq.inlen_send) {
|
|
|
|
/*
|
|
|
|
* Inline data feature is not engaged at all.
|
|
|
|
* There is nothing to adjust.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->max_inline_data <= max_inline) {
|
|
|
|
/*
|
|
|
|
* The requested inline data length does not
|
|
|
|
* exceed queue capabilities.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.inlen_mode > max_inline) {
|
|
|
|
DRV_LOG(ERR,
|
|
|
|
"minimal data inline requirements (%u) are not"
|
|
|
|
" satisfied (%u) on port %u, try the smaller"
|
|
|
|
" Tx queue size (%d)",
|
|
|
|
txq_ctrl->txq.inlen_mode, max_inline,
|
|
|
|
priv->dev_data->port_id,
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr);
|
2019-10-01 06:53:37 +00:00
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.inlen_send > max_inline &&
|
|
|
|
config->txq_inline_max != MLX5_ARG_UNSET &&
|
|
|
|
config->txq_inline_max > (int)max_inline) {
|
|
|
|
DRV_LOG(ERR,
|
|
|
|
"txq_inline_max requirements (%u) are not"
|
|
|
|
" satisfied (%u) on port %u, try the smaller"
|
|
|
|
" Tx queue size (%d)",
|
|
|
|
txq_ctrl->txq.inlen_send, max_inline,
|
|
|
|
priv->dev_data->port_id,
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr);
|
2019-10-01 06:53:37 +00:00
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.inlen_empw > max_inline &&
|
|
|
|
config->txq_inline_mpw != MLX5_ARG_UNSET &&
|
|
|
|
config->txq_inline_mpw > (int)max_inline) {
|
|
|
|
DRV_LOG(ERR,
|
|
|
|
"txq_inline_mpw requirements (%u) are not"
|
|
|
|
" satisfied (%u) on port %u, try the smaller"
|
|
|
|
" Tx queue size (%d)",
|
|
|
|
txq_ctrl->txq.inlen_empw, max_inline,
|
|
|
|
priv->dev_data->port_id,
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr);
|
2019-10-01 06:53:37 +00:00
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.tso_en && max_inline < MLX5_MAX_TSO_HEADER) {
|
|
|
|
DRV_LOG(ERR,
|
|
|
|
"tso header inline requirements (%u) are not"
|
|
|
|
" satisfied (%u) on port %u, try the smaller"
|
|
|
|
" Tx queue size (%d)",
|
|
|
|
MLX5_MAX_TSO_HEADER, max_inline,
|
|
|
|
priv->dev_data->port_id,
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr);
|
2019-10-01 06:53:37 +00:00
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.inlen_send > max_inline) {
|
|
|
|
DRV_LOG(WARNING,
|
|
|
|
"adjust txq_inline_max (%u->%u)"
|
|
|
|
" due to large Tx queue on port %u",
|
|
|
|
txq_ctrl->txq.inlen_send, max_inline,
|
|
|
|
priv->dev_data->port_id);
|
|
|
|
txq_ctrl->txq.inlen_send = max_inline;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->txq.inlen_empw > max_inline) {
|
|
|
|
DRV_LOG(WARNING,
|
|
|
|
"adjust txq_inline_mpw (%u->%u)"
|
|
|
|
"due to large Tx queue on port %u",
|
|
|
|
txq_ctrl->txq.inlen_empw, max_inline,
|
|
|
|
priv->dev_data->port_id);
|
|
|
|
txq_ctrl->txq.inlen_empw = max_inline;
|
|
|
|
}
|
|
|
|
txq_ctrl->max_inline_data = RTE_MAX(txq_ctrl->txq.inlen_send,
|
|
|
|
txq_ctrl->txq.inlen_empw);
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(txq_ctrl->max_inline_data <= max_inline);
|
|
|
|
MLX5_ASSERT(txq_ctrl->txq.inlen_mode <= max_inline);
|
|
|
|
MLX5_ASSERT(txq_ctrl->txq.inlen_mode <= txq_ctrl->txq.inlen_send);
|
|
|
|
MLX5_ASSERT(txq_ctrl->txq.inlen_mode <= txq_ctrl->txq.inlen_empw ||
|
|
|
|
!txq_ctrl->txq.inlen_empw);
|
2019-10-01 06:53:37 +00:00
|
|
|
return 0;
|
|
|
|
error:
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2018-01-10 09:16:58 +00:00
|
|
|
/**
|
|
|
|
* Create a DPDK Tx queue.
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2018-01-10 09:16:58 +00:00
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
* @param desc
|
|
|
|
* Number of descriptors to configure in queue.
|
|
|
|
* @param socket
|
|
|
|
* NUMA socket on which memory must be allocated.
|
|
|
|
* @param[in] conf
|
|
|
|
* Thresholds parameters.
|
|
|
|
*
|
|
|
|
* @return
|
2018-03-05 12:21:06 +00:00
|
|
|
* A DPDK queue object on success, NULL otherwise and rte_errno is set.
|
2018-01-10 09:16:58 +00:00
|
|
|
*/
|
2018-03-05 12:21:00 +00:00
|
|
|
struct mlx5_txq_ctrl *
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
|
|
|
unsigned int socket, const struct rte_eth_txconf *conf)
|
2018-01-10 09:16:58 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2018-01-10 09:16:58 +00:00
|
|
|
struct mlx5_txq_ctrl *tmpl;
|
|
|
|
|
2020-06-28 09:21:47 +00:00
|
|
|
tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl) +
|
|
|
|
desc * sizeof(struct rte_mbuf *), 0, socket);
|
2018-03-05 12:21:06 +00:00
|
|
|
if (!tmpl) {
|
|
|
|
rte_errno = ENOMEM;
|
2018-01-10 09:16:58 +00:00
|
|
|
return NULL;
|
2018-03-05 12:21:06 +00:00
|
|
|
}
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
|
|
|
|
MLX5_MR_BTREE_CACHE_N, socket)) {
|
|
|
|
/* rte_errno is already set. */
|
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
/* Save pointer of global generation number to check memory event. */
|
2020-04-13 21:17:48 +00:00
|
|
|
tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->share_cache.dev_gen;
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(desc > MLX5_TX_COMP_THRESH);
|
2018-05-10 11:56:55 +00:00
|
|
|
tmpl->txq.offloads = conf->offloads |
|
|
|
|
dev->data->dev_conf.txmode.offloads;
|
2018-01-10 09:16:58 +00:00
|
|
|
tmpl->priv = priv;
|
|
|
|
tmpl->socket = socket;
|
|
|
|
tmpl->txq.elts_n = log2above(desc);
|
2019-07-21 14:24:57 +00:00
|
|
|
tmpl->txq.elts_s = desc;
|
|
|
|
tmpl->txq.elts_m = desc - 1;
|
2019-04-10 18:41:17 +00:00
|
|
|
tmpl->txq.port_id = dev->data->port_id;
|
2019-04-10 18:41:16 +00:00
|
|
|
tmpl->txq.idx = idx;
|
2018-01-10 09:16:58 +00:00
|
|
|
txq_set_params(tmpl);
|
2019-10-01 06:53:37 +00:00
|
|
|
if (txq_adjust_params(tmpl))
|
|
|
|
goto error;
|
2019-05-01 01:01:43 +00:00
|
|
|
if (txq_calc_wqebb_cnt(tmpl) >
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr) {
|
2019-05-01 01:01:43 +00:00
|
|
|
DRV_LOG(ERR,
|
|
|
|
"port %u Tx WQEBB count (%d) exceeds the limit (%d),"
|
|
|
|
" try smaller queue size",
|
|
|
|
dev->data->port_id, txq_calc_wqebb_cnt(tmpl),
|
2020-06-03 15:05:58 +00:00
|
|
|
priv->sh->device_attr.max_qp_wr);
|
2019-05-01 01:01:43 +00:00
|
|
|
rte_errno = ENOMEM;
|
|
|
|
goto error;
|
|
|
|
}
|
2020-10-27 15:28:23 +00:00
|
|
|
__atomic_fetch_add(&tmpl->refcnt, 1, __ATOMIC_RELAXED);
|
2019-10-30 23:53:15 +00:00
|
|
|
tmpl->type = MLX5_TXQ_TYPE_STANDARD;
|
2017-10-09 14:44:48 +00:00
|
|
|
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
|
|
|
|
return tmpl;
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
error:
|
2020-12-15 08:48:32 +00:00
|
|
|
mlx5_mr_btree_free(&tmpl->txq.mr_ctrl.cache_bh);
|
2020-06-28 09:21:47 +00:00
|
|
|
mlx5_free(tmpl);
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
return NULL;
|
2017-10-09 14:44:48 +00:00
|
|
|
}
|
|
|
|
|
2019-10-30 23:53:15 +00:00
|
|
|
/**
|
|
|
|
* Create a DPDK Tx hairpin queue.
|
|
|
|
*
|
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
* @param desc
|
|
|
|
* Number of descriptors to configure in queue.
|
|
|
|
* @param hairpin_conf
|
|
|
|
* The hairpin configuration.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* A DPDK queue object on success, NULL otherwise and rte_errno is set.
|
|
|
|
*/
|
|
|
|
struct mlx5_txq_ctrl *
|
|
|
|
mlx5_txq_hairpin_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
|
|
|
const struct rte_eth_hairpin_conf *hairpin_conf)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_txq_ctrl *tmpl;
|
|
|
|
|
2020-06-28 09:21:47 +00:00
|
|
|
tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,
|
|
|
|
SOCKET_ID_ANY);
|
2019-10-30 23:53:15 +00:00
|
|
|
if (!tmpl) {
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
tmpl->priv = priv;
|
|
|
|
tmpl->socket = SOCKET_ID_ANY;
|
|
|
|
tmpl->txq.elts_n = log2above(desc);
|
|
|
|
tmpl->txq.port_id = dev->data->port_id;
|
|
|
|
tmpl->txq.idx = idx;
|
|
|
|
tmpl->hairpin_conf = *hairpin_conf;
|
|
|
|
tmpl->type = MLX5_TXQ_TYPE_HAIRPIN;
|
2020-10-27 15:28:23 +00:00
|
|
|
__atomic_fetch_add(&tmpl->refcnt, 1, __ATOMIC_RELAXED);
|
2019-10-30 23:53:15 +00:00
|
|
|
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
|
|
|
|
return tmpl;
|
|
|
|
}
|
|
|
|
|
2017-10-09 14:44:48 +00:00
|
|
|
/**
|
|
|
|
* Get a Tx queue.
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2017-10-09 14:44:48 +00:00
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* A pointer to the queue if it exists.
|
|
|
|
*/
|
2018-03-05 12:21:00 +00:00
|
|
|
struct mlx5_txq_ctrl *
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
|
2017-10-09 14:44:48 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2020-10-01 14:09:14 +00:00
|
|
|
struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
|
2017-10-09 14:44:48 +00:00
|
|
|
struct mlx5_txq_ctrl *ctrl = NULL;
|
|
|
|
|
2020-10-01 14:09:14 +00:00
|
|
|
if (txq_data) {
|
|
|
|
ctrl = container_of(txq_data, struct mlx5_txq_ctrl, txq);
|
2020-10-27 15:28:23 +00:00
|
|
|
__atomic_fetch_add(&ctrl->refcnt, 1, __ATOMIC_RELAXED);
|
2017-10-09 14:44:48 +00:00
|
|
|
}
|
|
|
|
return ctrl;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Release a Tx queue.
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2017-10-09 14:44:48 +00:00
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
*
|
|
|
|
* @return
|
2018-03-05 12:21:05 +00:00
|
|
|
* 1 while a reference on it exists, 0 when freed.
|
2017-10-09 14:44:48 +00:00
|
|
|
*/
|
|
|
|
int
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
|
2017-10-09 14:44:48 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2020-10-01 14:09:19 +00:00
|
|
|
struct mlx5_txq_ctrl *txq_ctrl;
|
2017-10-09 14:44:48 +00:00
|
|
|
|
|
|
|
if (!(*priv->txqs)[idx])
|
|
|
|
return 0;
|
2020-10-01 14:09:19 +00:00
|
|
|
txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
|
2020-11-03 06:47:44 +00:00
|
|
|
if (__atomic_sub_fetch(&txq_ctrl->refcnt, 1, __ATOMIC_RELAXED) > 1)
|
2020-10-01 14:09:14 +00:00
|
|
|
return 1;
|
2020-10-01 14:09:19 +00:00
|
|
|
if (txq_ctrl->obj) {
|
|
|
|
priv->obj_ops.txq_obj_release(txq_ctrl->obj);
|
|
|
|
LIST_REMOVE(txq_ctrl->obj, next);
|
|
|
|
mlx5_free(txq_ctrl->obj);
|
|
|
|
txq_ctrl->obj = NULL;
|
|
|
|
}
|
|
|
|
if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) {
|
|
|
|
if (txq_ctrl->txq.fcqs) {
|
|
|
|
mlx5_free(txq_ctrl->txq.fcqs);
|
|
|
|
txq_ctrl->txq.fcqs = NULL;
|
|
|
|
}
|
|
|
|
txq_free_elts(txq_ctrl);
|
2020-11-03 06:48:32 +00:00
|
|
|
dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
|
2017-10-09 14:44:48 +00:00
|
|
|
}
|
2020-10-15 06:38:11 +00:00
|
|
|
if (!__atomic_load_n(&txq_ctrl->refcnt, __ATOMIC_RELAXED)) {
|
|
|
|
if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD)
|
|
|
|
mlx5_mr_btree_free(&txq_ctrl->txq.mr_ctrl.cache_bh);
|
|
|
|
LIST_REMOVE(txq_ctrl, next);
|
|
|
|
mlx5_free(txq_ctrl);
|
|
|
|
(*priv->txqs)[idx] = NULL;
|
|
|
|
}
|
2020-10-01 14:09:14 +00:00
|
|
|
return 0;
|
2017-10-09 14:44:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Verify if the queue can be released.
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2017-10-09 14:44:48 +00:00
|
|
|
* @param idx
|
|
|
|
* TX queue index.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* 1 if the queue can be released.
|
|
|
|
*/
|
|
|
|
int
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx)
|
2017-10-09 14:44:48 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2017-10-09 14:44:48 +00:00
|
|
|
struct mlx5_txq_ctrl *txq;
|
|
|
|
|
|
|
|
if (!(*priv->txqs)[idx])
|
|
|
|
return -1;
|
|
|
|
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
|
2020-10-15 06:38:11 +00:00
|
|
|
return (__atomic_load_n(&txq->refcnt, __ATOMIC_RELAXED) == 1);
|
2017-10-09 14:44:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Verify the Tx Queue list is empty
|
|
|
|
*
|
2018-03-05 12:21:04 +00:00
|
|
|
* @param dev
|
|
|
|
* Pointer to Ethernet device.
|
2017-10-09 14:44:48 +00:00
|
|
|
*
|
2018-03-05 12:21:01 +00:00
|
|
|
* @return
|
|
|
|
* The number of object not released.
|
2017-10-09 14:44:48 +00:00
|
|
|
*/
|
|
|
|
int
|
2018-03-05 12:21:04 +00:00
|
|
|
mlx5_txq_verify(struct rte_eth_dev *dev)
|
2017-10-09 14:44:48 +00:00
|
|
|
{
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
2019-04-10 18:41:16 +00:00
|
|
|
struct mlx5_txq_ctrl *txq_ctrl;
|
2017-10-09 14:44:48 +00:00
|
|
|
int ret = 0;
|
|
|
|
|
2019-04-10 18:41:16 +00:00
|
|
|
LIST_FOREACH(txq_ctrl, &priv->txqsctrl, next) {
|
2018-03-13 09:23:56 +00:00
|
|
|
DRV_LOG(DEBUG, "port %u Tx queue %u still referenced",
|
2019-04-10 18:41:16 +00:00
|
|
|
dev->data->port_id, txq_ctrl->txq.idx);
|
2017-10-09 14:44:48 +00:00
|
|
|
++ret;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
2020-07-16 08:23:13 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Set the Tx queue dynamic timestamp (mask and offset)
|
|
|
|
*
|
|
|
|
* @param[in] dev
|
|
|
|
* Pointer to the Ethernet device structure.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
|
|
struct mlx5_dev_ctx_shared *sh = priv->sh;
|
|
|
|
struct mlx5_txq_data *data;
|
|
|
|
int off, nbit;
|
|
|
|
unsigned int i;
|
|
|
|
uint64_t mask = 0;
|
|
|
|
|
|
|
|
nbit = rte_mbuf_dynflag_lookup
|
|
|
|
(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL);
|
|
|
|
off = rte_mbuf_dynfield_lookup
|
|
|
|
(RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL);
|
2020-10-29 03:14:59 +00:00
|
|
|
if (nbit >= 0 && off >= 0 && sh->txpp.refcnt)
|
2020-07-16 08:23:13 +00:00
|
|
|
mask = 1ULL << nbit;
|
|
|
|
for (i = 0; i != priv->txqs_n; ++i) {
|
|
|
|
data = (*priv->txqs)[i];
|
|
|
|
if (!data)
|
|
|
|
continue;
|
|
|
|
data->sh = sh;
|
|
|
|
data->ts_mask = mask;
|
|
|
|
data->ts_offset = off;
|
|
|
|
}
|
|
|
|
}
|