2018-01-29 13:11:30 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
* Copyright 2017 6WIND S.A.
|
2018-03-20 19:20:35 +00:00
|
|
|
* Copyright 2017 Mellanox Technologies, Ltd
|
2017-10-09 18:46:56 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef RTE_PMD_MLX5_RXTX_VEC_H_
|
|
|
|
|
#define RTE_PMD_MLX5_RXTX_VEC_H_
|
|
|
|
|
|
|
|
|
|
#include <rte_common.h>
|
|
|
|
|
#include <rte_mbuf.h>
|
|
|
|
|
|
2020-01-29 12:38:27 +00:00
|
|
|
#include <mlx5_prm.h>
|
|
|
|
|
|
2017-10-09 18:46:56 +00:00
|
|
|
#include "mlx5_autoconf.h"
|
|
|
|
|
|
2018-01-10 09:17:00 +00:00
|
|
|
/* HW checksum offload capabilities of vectorized Tx. */
|
|
|
|
|
#define MLX5_VEC_TX_CKSUM_OFFLOAD_CAP \
|
|
|
|
|
(DEV_TX_OFFLOAD_IPV4_CKSUM | \
|
|
|
|
|
DEV_TX_OFFLOAD_UDP_CKSUM | \
|
|
|
|
|
DEV_TX_OFFLOAD_TCP_CKSUM | \
|
|
|
|
|
DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)
|
|
|
|
|
|
2017-10-09 18:46:56 +00:00
|
|
|
/*
|
|
|
|
|
* Compile time sanity check for vectorized functions.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define S_ASSERT_RTE_MBUF(s) \
|
|
|
|
|
static_assert(s, "A field of struct rte_mbuf is changed")
|
|
|
|
|
#define S_ASSERT_MLX5_CQE(s) \
|
|
|
|
|
static_assert(s, "A field of struct mlx5_cqe is changed")
|
|
|
|
|
|
|
|
|
|
/* rxq_cq_decompress_v() */
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, hash) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
|
|
|
|
|
|
|
|
|
|
/* rxq_cq_to_ptype_oflags_v() */
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, ol_flags) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rearm_data) + 8);
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, rearm_data) ==
|
|
|
|
|
RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
|
|
|
|
|
|
|
|
|
|
/* rxq_burst_v() */
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, pkt_len) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
|
|
|
|
|
S_ASSERT_RTE_MBUF(offsetof(struct rte_mbuf, data_len) ==
|
|
|
|
|
offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
|
2017-10-09 18:47:00 +00:00
|
|
|
#if (RTE_CACHE_LINE_SIZE == 128)
|
|
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 64);
|
|
|
|
|
#else
|
2017-10-09 18:46:56 +00:00
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, pkt_info) == 0);
|
2017-10-09 18:47:00 +00:00
|
|
|
#endif
|
2017-10-09 18:46:56 +00:00
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rx_hash_res) ==
|
|
|
|
|
offsetof(struct mlx5_cqe, pkt_info) + 12);
|
2019-07-22 14:52:21 +00:00
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, rsvd1) + 11 ==
|
2017-10-09 18:46:56 +00:00
|
|
|
offsetof(struct mlx5_cqe, hdr_type_etc));
|
|
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, vlan_info) ==
|
|
|
|
|
offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
|
2019-07-22 14:52:21 +00:00
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, lro_num_seg) + 12 ==
|
2017-10-09 18:46:56 +00:00
|
|
|
offsetof(struct mlx5_cqe, byte_cnt));
|
|
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, sop_drop_qpn) ==
|
|
|
|
|
RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
|
|
|
|
|
S_ASSERT_MLX5_CQE(offsetof(struct mlx5_cqe, op_own) ==
|
|
|
|
|
offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
|
|
|
|
|
|
2017-10-09 18:46:57 +00:00
|
|
|
/**
|
|
|
|
|
* Replenish buffers for RX in bulk.
|
|
|
|
|
*
|
|
|
|
|
* @param rxq
|
|
|
|
|
* Pointer to RX queue structure.
|
|
|
|
|
* @param n
|
|
|
|
|
* Number of buffers to be replenished.
|
|
|
|
|
*/
|
|
|
|
|
static inline void
|
|
|
|
|
mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
|
|
|
|
|
{
|
|
|
|
|
const uint16_t q_n = 1 << rxq->elts_n;
|
|
|
|
|
const uint16_t q_mask = q_n - 1;
|
2017-10-10 14:04:02 +00:00
|
|
|
uint16_t elts_idx = rxq->rq_ci & q_mask;
|
2017-10-09 18:46:57 +00:00
|
|
|
struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
|
2018-05-09 11:13:50 +00:00
|
|
|
volatile struct mlx5_wqe_data_seg *wq =
|
|
|
|
|
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
|
2017-10-09 18:46:57 +00:00
|
|
|
unsigned int i;
|
|
|
|
|
|
2020-01-30 16:14:40 +00:00
|
|
|
MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
|
|
|
|
|
MLX5_ASSERT(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
|
|
|
|
|
MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) >
|
|
|
|
|
MLX5_VPMD_DESCS_PER_LOOP);
|
2017-10-09 18:46:57 +00:00
|
|
|
/* Not to cross queue end. */
|
|
|
|
|
n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
|
|
|
|
|
if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
|
|
|
|
|
rxq->stats.rx_nombuf += n;
|
|
|
|
|
return;
|
|
|
|
|
}
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
for (i = 0; i < n; ++i) {
|
2019-03-25 19:13:10 +00:00
|
|
|
void *buf_addr;
|
2019-01-14 21:16:22 +00:00
|
|
|
|
2019-03-25 19:13:10 +00:00
|
|
|
/*
|
2020-01-20 19:16:26 +00:00
|
|
|
* In order to support the mbufs with external attached
|
|
|
|
|
* data buffer we should use the buf_addr pointer instead of
|
|
|
|
|
* rte_mbuf_buf_addr(). It touches the mbuf itself and may
|
|
|
|
|
* impact the performance.
|
2019-03-25 19:13:10 +00:00
|
|
|
*/
|
|
|
|
|
buf_addr = elts[i]->buf_addr;
|
2019-01-14 21:16:22 +00:00
|
|
|
wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
|
2017-10-09 18:46:57 +00:00
|
|
|
RTE_PKTMBUF_HEADROOM);
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
/* If there's only one MR, no need to replace LKey in WQE. */
|
|
|
|
|
if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
|
|
|
|
|
wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
|
|
|
|
|
}
|
2017-10-09 18:46:57 +00:00
|
|
|
rxq->rq_ci += n;
|
2017-10-10 14:04:02 +00:00
|
|
|
/* Prevent overflowing into consumed mbufs. */
|
|
|
|
|
elts_idx = rxq->rq_ci & q_mask;
|
|
|
|
|
for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
|
|
|
|
|
(*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf;
|
2018-01-25 21:02:49 +00:00
|
|
|
rte_cio_wmb();
|
2017-10-09 18:46:57 +00:00
|
|
|
*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-09 18:46:56 +00:00
|
|
|
#endif /* RTE_PMD_MLX5_RXTX_VEC_H_ */
|
