17ed314c6c
Enabling LRO offload per queue makes sense because the user will probably want to allocate different mempool for LRO queues - the LRO mempool mbuf size may be bigger than non LRO mempool. Change the LRO offload to be per queue instead of per port. If one of the queues is with LRO enabled, all the queues will be configured via DevX. If RSS flows direct TCP packets to queues with different LRO enabling, these flows will not be offloaded with LRO. Signed-off-by: Matan Azrad <matan@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
169 lines
3.6 KiB
C
169 lines
3.6 KiB
C
/* SPDX-License-Identifier: BSD-3-Clause
|
|
* Copyright 2017 6WIND S.A.
|
|
* Copyright 2017 Mellanox Technologies, Ltd
|
|
*/
|
|
|
|
#include <assert.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
#include <stdlib.h>
|
|
|
|
/* Verbs header. */
|
|
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
|
|
#ifdef PEDANTIC
|
|
#pragma GCC diagnostic ignored "-Wpedantic"
|
|
#endif
|
|
#include <infiniband/verbs.h>
|
|
#include <infiniband/mlx5dv.h>
|
|
#ifdef PEDANTIC
|
|
#pragma GCC diagnostic error "-Wpedantic"
|
|
#endif
|
|
|
|
#include <rte_mbuf.h>
|
|
#include <rte_mempool.h>
|
|
#include <rte_prefetch.h>
|
|
|
|
#include "mlx5.h"
|
|
#include "mlx5_utils.h"
|
|
#include "mlx5_rxtx.h"
|
|
#include "mlx5_rxtx_vec.h"
|
|
#include "mlx5_autoconf.h"
|
|
#include "mlx5_defs.h"
|
|
#include "mlx5_prm.h"
|
|
|
|
#if defined RTE_ARCH_X86_64
|
|
#include "mlx5_rxtx_vec_sse.h"
|
|
#elif defined RTE_ARCH_ARM64
|
|
#include "mlx5_rxtx_vec_neon.h"
|
|
#else
|
|
#error "This should not be compiled if SIMD instructions are not supported."
|
|
#endif
|
|
|
|
/**
|
|
* Skip error packets.
|
|
*
|
|
* @param rxq
|
|
* Pointer to RX queue structure.
|
|
* @param[out] pkts
|
|
* Array to store received packets.
|
|
* @param pkts_n
|
|
* Maximum number of packets in array.
|
|
*
|
|
* @return
|
|
* Number of packets successfully received (<= pkts_n).
|
|
*/
|
|
static uint16_t
|
|
rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
|
|
uint16_t pkts_n)
|
|
{
|
|
uint16_t n = 0;
|
|
unsigned int i;
|
|
#ifdef MLX5_PMD_SOFT_COUNTERS
|
|
uint32_t err_bytes = 0;
|
|
#endif
|
|
|
|
for (i = 0; i < pkts_n; ++i) {
|
|
struct rte_mbuf *pkt = pkts[i];
|
|
|
|
if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
|
|
#ifdef MLX5_PMD_SOFT_COUNTERS
|
|
err_bytes += PKT_LEN(pkt);
|
|
#endif
|
|
rte_pktmbuf_free_seg(pkt);
|
|
} else {
|
|
pkts[n++] = pkt;
|
|
}
|
|
}
|
|
rxq->stats.idropped += (pkts_n - n);
|
|
#ifdef MLX5_PMD_SOFT_COUNTERS
|
|
/* Correct counters of errored completions. */
|
|
rxq->stats.ipackets -= (pkts_n - n);
|
|
rxq->stats.ibytes -= err_bytes;
|
|
#endif
|
|
mlx5_rx_err_handle(rxq, 1);
|
|
return n;
|
|
}
|
|
|
|
/**
|
|
* DPDK callback for vectorized RX.
|
|
*
|
|
* @param dpdk_rxq
|
|
* Generic pointer to RX queue structure.
|
|
* @param[out] pkts
|
|
* Array to store received packets.
|
|
* @param pkts_n
|
|
* Maximum number of packets in array.
|
|
*
|
|
* @return
|
|
* Number of packets successfully received (<= pkts_n).
|
|
*/
|
|
uint16_t
|
|
mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
|
|
{
|
|
struct mlx5_rxq_data *rxq = dpdk_rxq;
|
|
uint16_t nb_rx;
|
|
uint64_t err = 0;
|
|
|
|
nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
|
|
if (unlikely(err | rxq->err_state))
|
|
nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
|
|
return nb_rx;
|
|
}
|
|
|
|
/**
|
|
* Check a RX queue can support vectorized RX.
|
|
*
|
|
* @param rxq
|
|
* Pointer to RX queue.
|
|
*
|
|
* @return
|
|
* 1 if supported, negative errno value if not.
|
|
*/
|
|
int __attribute__((cold))
|
|
mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
|
|
{
|
|
struct mlx5_rxq_ctrl *ctrl =
|
|
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
|
|
|
|
if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
|
|
return -ENOTSUP;
|
|
if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
|
|
return -ENOTSUP;
|
|
if (rxq->lro)
|
|
return -ENOTSUP;
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* Check a device can support vectorized RX.
|
|
*
|
|
* @param dev
|
|
* Pointer to Ethernet device.
|
|
*
|
|
* @return
|
|
* 1 if supported, negative errno value if not.
|
|
*/
|
|
int __attribute__((cold))
|
|
mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
|
|
{
|
|
struct mlx5_priv *priv = dev->data->dev_private;
|
|
uint16_t i;
|
|
|
|
if (!priv->config.rx_vec_en)
|
|
return -ENOTSUP;
|
|
if (mlx5_mprq_enabled(dev))
|
|
return -ENOTSUP;
|
|
/* All the configured queues should support. */
|
|
for (i = 0; i < priv->rxqs_n; ++i) {
|
|
struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
|
|
|
|
if (!rxq)
|
|
continue;
|
|
if (mlx5_rxq_check_vec_support(rxq) < 0)
|
|
break;
|
|
}
|
|
if (i != priv->rxqs_n)
|
|
return -ENOTSUP;
|
|
return 1;
|
|
}
|