2018-01-29 13:11:30 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
* Copyright 2015 6WIND S.A.
|
2018-03-20 19:20:35 +00:00
|
|
|
* Copyright 2015 Mellanox Technologies, Ltd
|
2015-10-30 18:52:31 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef RTE_PMD_MLX5_RXTX_H_
|
|
|
|
|
#define RTE_PMD_MLX5_RXTX_H_
|
|
|
|
|
|
2015-10-30 18:55:09 +00:00
|
|
|
#include <stddef.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
#include <stdint.h>
|
2017-10-09 14:44:45 +00:00
|
|
|
#include <sys/queue.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
|
|
|
|
|
/* Verbs header. */
|
|
|
|
|
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
|
|
|
|
|
#ifdef PEDANTIC
|
2016-09-19 14:36:54 +00:00
|
|
|
#pragma GCC diagnostic ignored "-Wpedantic"
|
2015-10-30 18:52:31 +00:00
|
|
|
#endif
|
|
|
|
|
#include <infiniband/verbs.h>
|
2017-09-26 15:38:24 +00:00
|
|
|
#include <infiniband/mlx5dv.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
#ifdef PEDANTIC
|
2016-09-19 14:36:54 +00:00
|
|
|
#pragma GCC diagnostic error "-Wpedantic"
|
2015-10-30 18:52:31 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#include <rte_mbuf.h>
|
|
|
|
|
#include <rte_mempool.h>
|
2016-09-20 08:53:47 +00:00
|
|
|
#include <rte_common.h>
|
2017-08-23 07:10:58 +00:00
|
|
|
#include <rte_hexdump.h>
|
2017-10-09 14:44:45 +00:00
|
|
|
#include <rte_atomic.h>
|
2018-07-12 12:01:31 +00:00
|
|
|
#include <rte_spinlock.h>
|
|
|
|
|
#include <rte_io.h>
|
2019-03-10 08:28:02 +00:00
|
|
|
#include <rte_bus_pci.h>
|
2019-07-22 14:52:10 +00:00
|
|
|
#include <rte_malloc.h>
|
2015-10-30 18:52:31 +00:00
|
|
|
|
|
|
|
|
#include "mlx5_utils.h"
|
|
|
|
|
#include "mlx5.h"
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
#include "mlx5_mr.h"
|
2015-10-30 18:55:16 +00:00
|
|
|
#include "mlx5_autoconf.h"
|
2015-10-30 18:52:31 +00:00
|
|
|
#include "mlx5_defs.h"
|
2016-06-24 13:17:52 +00:00
|
|
|
#include "mlx5_prm.h"
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
#include "mlx5_glue.h"
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2018-07-12 09:31:04 +00:00
|
|
|
/* Support tunnel matching. */
|
2019-08-08 11:38:56 +00:00
|
|
|
#define MLX5_FLOW_TUNNEL 8
|
2018-07-12 09:31:04 +00:00
|
|
|
|
2015-10-30 18:52:36 +00:00
|
|
|
struct mlx5_rxq_stats {
|
|
|
|
|
#ifdef MLX5_PMD_SOFT_COUNTERS
|
|
|
|
|
uint64_t ipackets; /**< Total of successfully received packets. */
|
|
|
|
|
uint64_t ibytes; /**< Total of successfully received bytes. */
|
|
|
|
|
#endif
|
|
|
|
|
uint64_t idropped; /**< Total of packets dropped when RX ring full. */
|
|
|
|
|
uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct mlx5_txq_stats {
|
|
|
|
|
#ifdef MLX5_PMD_SOFT_COUNTERS
|
|
|
|
|
uint64_t opackets; /**< Total of successfully sent packets. */
|
|
|
|
|
uint64_t obytes; /**< Total of successfully sent bytes. */
|
|
|
|
|
#endif
|
2017-09-14 10:50:37 +00:00
|
|
|
uint64_t oerrors; /**< Total number of failed transmitted packets. */
|
2015-10-30 18:52:36 +00:00
|
|
|
};
|
|
|
|
|
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv;
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2016-06-24 13:17:54 +00:00
|
|
|
/* Compressed CQE context. */
|
|
|
|
|
struct rxq_zip {
|
|
|
|
|
uint16_t ai; /* Array index. */
|
|
|
|
|
uint16_t ca; /* Current array index. */
|
|
|
|
|
uint16_t na; /* Next array index. */
|
|
|
|
|
uint16_t cq_ci; /* The next CQE. */
|
|
|
|
|
uint32_t cqe_cnt; /* Number of CQEs. */
|
|
|
|
|
};
|
|
|
|
|
|
2018-05-09 11:13:50 +00:00
|
|
|
/* Multi-Packet RQ buffer header. */
|
|
|
|
|
struct mlx5_mprq_buf {
|
|
|
|
|
struct rte_mempool *mp;
|
|
|
|
|
rte_atomic16_t refcnt; /* Atomically accessed refcnt. */
|
|
|
|
|
uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */
|
2019-07-22 14:52:20 +00:00
|
|
|
struct rte_mbuf_ext_shared_info shinfos[];
|
|
|
|
|
/*
|
|
|
|
|
* Shared information per stride.
|
|
|
|
|
* More memory will be allocated for the first stride head-room and for
|
|
|
|
|
* the strides data.
|
|
|
|
|
*/
|
2018-05-09 11:13:50 +00:00
|
|
|
} __rte_cache_aligned;
|
|
|
|
|
|
|
|
|
|
/* Get pointer to the first stride. */
|
2019-07-22 14:52:20 +00:00
|
|
|
#define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \
|
|
|
|
|
sizeof(struct mlx5_mprq_buf) + \
|
|
|
|
|
(strd_n) * \
|
|
|
|
|
sizeof(struct rte_mbuf_ext_shared_info) + \
|
|
|
|
|
RTE_PKTMBUF_HEADROOM))
|
2018-05-09 11:13:50 +00:00
|
|
|
|
2019-07-22 14:52:18 +00:00
|
|
|
#define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
|
|
|
|
|
#define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
|
|
|
|
|
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
enum mlx5_rxq_err_state {
|
|
|
|
|
MLX5_RXQ_ERR_STATE_NO_ERROR = 0,
|
|
|
|
|
MLX5_RXQ_ERR_STATE_NEED_RESET,
|
|
|
|
|
MLX5_RXQ_ERR_STATE_NEED_READY,
|
|
|
|
|
};
|
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
/* RX queue descriptor. */
|
2017-10-09 14:44:39 +00:00
|
|
|
struct mlx5_rxq_data {
|
2015-10-30 18:52:41 +00:00
|
|
|
unsigned int csum:1; /* Enable checksum offloading. */
|
2017-10-10 14:37:07 +00:00
|
|
|
unsigned int hw_timestamp:1; /* Enable HW timestamp. */
|
2016-03-03 14:26:44 +00:00
|
|
|
unsigned int vlan_strip:1; /* Enable VLAN stripping. */
|
2016-03-17 15:38:56 +00:00
|
|
|
unsigned int crc_present:1; /* CRC must be subtracted. */
|
2019-07-29 11:53:25 +00:00
|
|
|
unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */
|
2016-09-20 08:53:48 +00:00
|
|
|
unsigned int cqe_n:4; /* Log 2 of CQ elements. */
|
2016-09-20 08:53:47 +00:00
|
|
|
unsigned int elts_n:4; /* Log 2 of Mbufs. */
|
2016-09-28 12:11:18 +00:00
|
|
|
unsigned int rss_hash:1; /* RSS hash result is enabled. */
|
2016-12-29 15:15:21 +00:00
|
|
|
unsigned int mark:1; /* Marked flow available on the queue. */
|
2018-05-09 11:13:50 +00:00
|
|
|
unsigned int strd_num_n:5; /* Log 2 of the number of stride. */
|
|
|
|
|
unsigned int strd_sz_n:4; /* Log 2 of stride size. */
|
|
|
|
|
unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
unsigned int err_state:2; /* enum mlx5_rxq_err_state. */
|
2019-07-22 14:52:23 +00:00
|
|
|
unsigned int strd_headroom_en:1; /* Enable mbuf headroom in MPRQ. */
|
2019-07-29 11:53:29 +00:00
|
|
|
unsigned int lro:1; /* Enable LRO. */
|
|
|
|
|
unsigned int :1; /* Remaining bits. */
|
2016-09-20 08:53:47 +00:00
|
|
|
volatile uint32_t *rq_db;
|
|
|
|
|
volatile uint32_t *cq_db;
|
2017-09-29 07:17:24 +00:00
|
|
|
uint16_t port_id;
|
2018-08-23 23:10:33 +00:00
|
|
|
uint32_t rq_ci;
|
2018-06-26 12:39:24 +00:00
|
|
|
uint16_t consumed_strd; /* Number of consumed strides in WQE. */
|
2018-08-23 23:10:33 +00:00
|
|
|
uint32_t rq_pi;
|
|
|
|
|
uint32_t cq_ci;
|
2018-11-15 10:29:20 +00:00
|
|
|
uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */
|
2019-05-30 10:20:31 +00:00
|
|
|
union {
|
|
|
|
|
struct rxq_zip zip; /* Compressed context. */
|
|
|
|
|
uint16_t decompressed;
|
|
|
|
|
/* Number of ready mbufs decompressed from the CQ. */
|
|
|
|
|
};
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
|
2018-05-09 11:13:50 +00:00
|
|
|
uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */
|
|
|
|
|
volatile void *wqes;
|
2016-06-24 13:17:52 +00:00
|
|
|
volatile struct mlx5_cqe(*cqes)[];
|
2018-05-09 11:13:50 +00:00
|
|
|
RTE_STD_C11
|
|
|
|
|
union {
|
|
|
|
|
struct rte_mbuf *(*elts)[];
|
|
|
|
|
struct mlx5_mprq_buf *(*mprq_bufs)[];
|
|
|
|
|
};
|
2016-06-24 13:17:52 +00:00
|
|
|
struct rte_mempool *mp;
|
2018-05-09 11:13:50 +00:00
|
|
|
struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
|
|
|
|
|
struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */
|
2019-04-10 18:41:16 +00:00
|
|
|
uint16_t idx; /* Queue index. */
|
2016-06-24 13:17:52 +00:00
|
|
|
struct mlx5_rxq_stats stats;
|
2017-07-06 18:41:10 +00:00
|
|
|
uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
|
|
|
|
|
struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
|
2017-09-26 15:38:24 +00:00
|
|
|
void *cq_uar; /* CQ user access region. */
|
|
|
|
|
uint32_t cqn; /* CQ number. */
|
|
|
|
|
uint8_t cq_arm_sn; /* CQ arm seq number. */
|
2018-07-12 12:01:31 +00:00
|
|
|
#ifndef RTE_ARCH_64
|
|
|
|
|
rte_spinlock_t *uar_lock_cq;
|
|
|
|
|
/* CQ (UAR) access lock required for 32bit implementations */
|
|
|
|
|
#endif
|
2018-04-23 12:33:03 +00:00
|
|
|
uint32_t tunnel; /* Tunnel information. */
|
2016-06-24 13:17:47 +00:00
|
|
|
} __rte_cache_aligned;
|
|
|
|
|
|
2019-07-22 14:52:11 +00:00
|
|
|
enum mlx5_rxq_obj_type {
|
|
|
|
|
MLX5_RXQ_OBJ_TYPE_IBV, /* mlx5_rxq_obj with ibv_wq. */
|
|
|
|
|
MLX5_RXQ_OBJ_TYPE_DEVX_RQ, /* mlx5_rxq_obj with mlx5_devx_rq. */
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* Verbs/DevX Rx queue elements. */
|
|
|
|
|
struct mlx5_rxq_obj {
|
|
|
|
|
LIST_ENTRY(mlx5_rxq_obj) next; /* Pointer to the next element. */
|
2017-10-09 14:44:46 +00:00
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
|
|
|
|
struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */
|
2016-06-24 13:17:52 +00:00
|
|
|
struct ibv_cq *cq; /* Completion Queue. */
|
2019-07-22 14:52:11 +00:00
|
|
|
enum mlx5_rxq_obj_type type;
|
|
|
|
|
RTE_STD_C11
|
|
|
|
|
union {
|
|
|
|
|
struct ibv_wq *wq; /* Work Queue. */
|
|
|
|
|
struct mlx5_devx_obj *rq; /* DevX object for Rx Queue. */
|
|
|
|
|
};
|
2017-03-14 13:03:09 +00:00
|
|
|
struct ibv_comp_channel *channel;
|
2017-10-09 14:44:46 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* RX queue control descriptor. */
|
|
|
|
|
struct mlx5_rxq_ctrl {
|
2019-04-10 18:41:16 +00:00
|
|
|
struct mlx5_rxq_data rxq; /* Data path structure. */
|
2017-10-09 14:44:49 +00:00
|
|
|
LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
|
|
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
2019-07-22 14:52:11 +00:00
|
|
|
struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv; /* Back pointer to private data. */
|
2017-10-09 14:44:46 +00:00
|
|
|
unsigned int socket; /* CPU socket ID for allocations. */
|
|
|
|
|
unsigned int irq:1; /* Whether IRQ is enabled. */
|
2019-07-24 09:05:10 +00:00
|
|
|
unsigned int dbr_umem_id_valid:1; /* dbr_umem_id holds a valid value. */
|
2018-07-12 09:30:58 +00:00
|
|
|
uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
|
2018-07-12 09:31:04 +00:00
|
|
|
uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
|
2019-05-30 10:20:38 +00:00
|
|
|
uint32_t wqn; /* WQ number. */
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
uint16_t dump_file_n; /* Number of dump files. */
|
2019-07-22 14:52:18 +00:00
|
|
|
uint32_t dbr_umem_id; /* Storing door-bell information, */
|
|
|
|
|
uint64_t dbr_offset; /* needed when freeing door-bell. */
|
|
|
|
|
struct mlx5dv_devx_umem *wq_umem; /* WQ buffer registration info. */
|
2015-10-30 18:52:31 +00:00
|
|
|
};
|
|
|
|
|
|
2019-07-22 14:52:12 +00:00
|
|
|
enum mlx5_ind_tbl_type {
|
|
|
|
|
MLX5_IND_TBL_TYPE_IBV,
|
|
|
|
|
MLX5_IND_TBL_TYPE_DEVX,
|
|
|
|
|
};
|
|
|
|
|
|
2017-10-09 14:44:50 +00:00
|
|
|
/* Indirection table. */
|
2019-07-22 14:52:12 +00:00
|
|
|
struct mlx5_ind_table_obj {
|
|
|
|
|
LIST_ENTRY(mlx5_ind_table_obj) next; /* Pointer to the next element. */
|
2017-10-09 14:44:50 +00:00
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
2019-07-22 14:52:12 +00:00
|
|
|
enum mlx5_ind_tbl_type type;
|
|
|
|
|
RTE_STD_C11
|
|
|
|
|
union {
|
|
|
|
|
struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
|
|
|
|
|
struct mlx5_devx_obj *rqt; /* DevX RQT object. */
|
|
|
|
|
};
|
2018-04-25 15:27:50 +00:00
|
|
|
uint32_t queues_n; /**< Number of queues in the list. */
|
2017-10-09 14:44:50 +00:00
|
|
|
uint16_t queues[]; /**< Queue list. */
|
|
|
|
|
};
|
|
|
|
|
|
2017-10-09 14:44:51 +00:00
|
|
|
/* Hash Rx queue. */
|
|
|
|
|
struct mlx5_hrxq {
|
|
|
|
|
LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */
|
|
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
2019-07-22 14:52:12 +00:00
|
|
|
struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
|
2019-07-22 14:52:13 +00:00
|
|
|
RTE_STD_C11
|
|
|
|
|
union {
|
|
|
|
|
struct ibv_qp *qp; /* Verbs queue pair. */
|
|
|
|
|
struct mlx5_devx_obj *tir; /* DevX TIR object. */
|
|
|
|
|
};
|
2019-04-14 20:17:11 +00:00
|
|
|
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
|
|
|
|
|
void *action; /* DV QP action pointer. */
|
|
|
|
|
#endif
|
2017-10-09 14:44:51 +00:00
|
|
|
uint64_t hash_fields; /* Verbs Hash fields. */
|
2018-04-25 15:27:50 +00:00
|
|
|
uint32_t rss_key_len; /* Hash key length in bytes. */
|
2017-10-09 14:44:51 +00:00
|
|
|
uint8_t rss_key[]; /* Hash key. */
|
|
|
|
|
};
|
|
|
|
|
|
2019-07-21 14:24:57 +00:00
|
|
|
/* TX queue send local data. */
|
|
|
|
|
__extension__
|
|
|
|
|
struct mlx5_txq_local {
|
|
|
|
|
struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
|
|
|
|
|
struct rte_mbuf *mbuf; /* first mbuf to process. */
|
|
|
|
|
uint16_t pkts_copy; /* packets copied to elts. */
|
|
|
|
|
uint16_t pkts_sent; /* packets sent. */
|
2019-08-05 13:03:54 +00:00
|
|
|
uint16_t pkts_loop; /* packets sent on loop entry. */
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t elts_free; /* available elts remain. */
|
|
|
|
|
uint16_t wqe_free; /* available wqe remain. */
|
|
|
|
|
uint16_t mbuf_off; /* data offset in current mbuf. */
|
|
|
|
|
uint16_t mbuf_nseg; /* number of remaining mbuf. */
|
|
|
|
|
};
|
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
/* TX queue descriptor. */
|
2017-06-30 17:19:08 +00:00
|
|
|
__extension__
|
2017-10-09 14:44:40 +00:00
|
|
|
struct mlx5_txq_data {
|
2017-07-06 18:41:06 +00:00
|
|
|
uint16_t elts_head; /* Current counter in (*elts)[]. */
|
|
|
|
|
uint16_t elts_tail; /* Counter of first element awaiting completion. */
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t elts_comp; /* elts index since last completion request. */
|
|
|
|
|
uint16_t elts_s; /* Number of mbuf elements. */
|
|
|
|
|
uint16_t elts_m; /* Mask for mbuf elements indices. */
|
|
|
|
|
/* Fields related to elts mbuf storage. */
|
|
|
|
|
uint16_t wqe_ci; /* Consumer index for work queue. */
|
|
|
|
|
uint16_t wqe_pi; /* Producer index for work queue. */
|
|
|
|
|
uint16_t wqe_s; /* Number of WQ elements. */
|
|
|
|
|
uint16_t wqe_m; /* Mask Number for WQ elements. */
|
|
|
|
|
uint16_t wqe_comp; /* WQE index since last completion request. */
|
|
|
|
|
uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */
|
|
|
|
|
/* WQ related fields. */
|
2016-06-24 13:17:53 +00:00
|
|
|
uint16_t cq_ci; /* Consumer index for completion queue. */
|
2017-12-27 03:55:45 +00:00
|
|
|
#ifndef NDEBUG
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t cq_pi; /* Counter of issued CQE "always" requests. */
|
2017-12-27 03:55:45 +00:00
|
|
|
#endif
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t cqe_s; /* Number of CQ elements. */
|
|
|
|
|
uint16_t cqe_m; /* Mask for CQ indices. */
|
|
|
|
|
/* CQ related fields. */
|
|
|
|
|
uint16_t elts_n:4; /* elts[] length (in log2). */
|
2016-09-20 08:53:48 +00:00
|
|
|
uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
|
2017-03-02 09:01:31 +00:00
|
|
|
uint16_t tso_en:1; /* When set hardware TSO is enabled. */
|
2017-03-02 09:05:44 +00:00
|
|
|
uint16_t tunnel_en:1;
|
|
|
|
|
/* When set TX offload for tunneled packets are supported. */
|
2018-04-08 12:41:20 +00:00
|
|
|
uint16_t swp_en:1; /* Whether SW parser is enabled. */
|
2019-07-21 14:24:57 +00:00
|
|
|
uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
|
|
|
|
|
uint16_t inlen_send; /* Ordinary send data inline size. */
|
|
|
|
|
uint16_t inlen_empw; /* eMPW max packet size to inline. */
|
|
|
|
|
uint16_t inlen_mode; /* Minimal data length to inline. */
|
2016-06-24 13:17:55 +00:00
|
|
|
uint32_t qp_num_8s; /* QP number shifted by 8. */
|
2018-01-10 09:17:00 +00:00
|
|
|
uint64_t offloads; /* Offloads for Tx Queue. */
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
|
2019-07-21 14:24:57 +00:00
|
|
|
struct mlx5_wqe *wqes; /* Work queue. */
|
|
|
|
|
struct mlx5_wqe *wqes_end; /* Work queue array limit. */
|
|
|
|
|
volatile struct mlx5_cqe *cqes; /* Completion queue. */
|
2016-06-24 13:17:53 +00:00
|
|
|
volatile uint32_t *qp_db; /* Work queue doorbell. */
|
|
|
|
|
volatile uint32_t *cq_db; /* Completion queue doorbell. */
|
2019-04-10 18:41:17 +00:00
|
|
|
uint16_t port_id; /* Port ID of device. */
|
2019-04-10 18:41:16 +00:00
|
|
|
uint16_t idx; /* Queue index. */
|
2015-10-30 18:52:36 +00:00
|
|
|
struct mlx5_txq_stats stats; /* TX queue counters. */
|
2018-07-12 12:01:31 +00:00
|
|
|
#ifndef RTE_ARCH_64
|
|
|
|
|
rte_spinlock_t *uar_lock;
|
|
|
|
|
/* UAR access lock required for 32bit implementations */
|
|
|
|
|
#endif
|
2019-07-21 14:24:57 +00:00
|
|
|
struct rte_mbuf *elts[0];
|
|
|
|
|
/* Storage for queued packets, must be the last field. */
|
2016-06-24 13:17:46 +00:00
|
|
|
} __rte_cache_aligned;
|
|
|
|
|
|
2017-10-09 14:44:47 +00:00
|
|
|
/* Verbs Rx queue elements. */
|
|
|
|
|
struct mlx5_txq_ibv {
|
|
|
|
|
LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */
|
|
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
2018-03-13 09:23:55 +00:00
|
|
|
struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */
|
2017-10-09 14:44:47 +00:00
|
|
|
struct ibv_cq *cq; /* Completion Queue. */
|
|
|
|
|
struct ibv_qp *qp; /* Queue Pair. */
|
|
|
|
|
};
|
|
|
|
|
|
2016-06-24 13:17:46 +00:00
|
|
|
/* TX queue control descriptor. */
|
2017-10-09 14:44:40 +00:00
|
|
|
struct mlx5_txq_ctrl {
|
2017-10-09 14:44:48 +00:00
|
|
|
LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
|
|
|
|
|
rte_atomic32_t refcnt; /* Reference counter. */
|
2016-03-03 14:27:12 +00:00
|
|
|
unsigned int socket; /* CPU socket ID for allocations. */
|
2017-10-09 14:44:47 +00:00
|
|
|
unsigned int max_inline_data; /* Max inline data. */
|
|
|
|
|
unsigned int max_tso_header; /* Max TSO header size. */
|
|
|
|
|
struct mlx5_txq_ibv *ibv; /* Verbs queue object. */
|
2019-02-21 09:29:14 +00:00
|
|
|
struct mlx5_priv *priv; /* Back pointer to private data. */
|
2017-10-06 15:45:49 +00:00
|
|
|
off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
|
2019-04-10 18:41:17 +00:00
|
|
|
void *bf_reg; /* BlueFlame register from Verbs. */
|
net/mlx5: handle Tx completion with error
When WQEs are posted to the HW to send packets, the PMD may get a
completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, bad sizes, etc.
that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts and huge complexity.
The error CQEs change the SQ state to error state what causes all the
next posted WQEs to be completed with CQE flush error forever.
Currently, the PMD doesn't handle Tx error CQEs and even may crashed
when one of them appears.
Extend the Tx data-path to detect these error CQEs, to report them by
the statistics error counters, to recover the SQ by moving the state
to ready again and adjusting the management variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily, hence
a dump file with debug information will be created for the first number
of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:37 +00:00
|
|
|
uint16_t dump_file_n; /* Number of dump files. */
|
2019-07-21 14:24:57 +00:00
|
|
|
struct mlx5_txq_data txq; /* Data path structure. */
|
|
|
|
|
/* Must be the last field in the structure, contains elts[]. */
|
2015-10-30 18:52:31 +00:00
|
|
|
};
|
|
|
|
|
|
2019-04-10 18:41:17 +00:00
|
|
|
#define MLX5_TX_BFREG(txq) \
|
|
|
|
|
(MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
|
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
/* mlx5_rxq.c */
|
|
|
|
|
|
2015-10-30 18:55:11 +00:00
|
|
|
extern uint8_t rss_hash_default_key[];
|
|
|
|
|
|
2018-05-09 11:13:50 +00:00
|
|
|
int mlx5_check_mprq_support(struct rte_eth_dev *dev);
|
|
|
|
|
int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq);
|
|
|
|
|
int mlx5_mprq_enabled(struct rte_eth_dev *dev);
|
|
|
|
|
int mlx5_mprq_free_mp(struct rte_eth_dev *dev);
|
|
|
|
|
int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev);
|
2018-03-05 12:20:58 +00:00
|
|
|
int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
|
|
|
|
unsigned int socket, const struct rte_eth_rxconf *conf,
|
|
|
|
|
struct rte_mempool *mp);
|
|
|
|
|
void mlx5_rx_queue_release(void *dpdk_rxq);
|
2018-03-05 12:21:04 +00:00
|
|
|
int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
|
|
|
|
|
void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
|
2017-06-14 11:49:14 +00:00
|
|
|
int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
|
|
|
|
int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
2019-07-22 14:52:18 +00:00
|
|
|
struct mlx5_rxq_obj *mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
|
|
|
|
|
enum mlx5_rxq_obj_type type);
|
2019-07-22 14:52:11 +00:00
|
|
|
int mlx5_rxq_obj_verify(struct rte_eth_dev *dev);
|
2018-03-05 12:21:04 +00:00
|
|
|
struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
|
|
|
|
|
uint16_t desc, unsigned int socket,
|
|
|
|
|
const struct rte_eth_rxconf *conf,
|
|
|
|
|
struct rte_mempool *mp);
|
|
|
|
|
struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_rxq_verify(struct rte_eth_dev *dev);
|
2018-03-05 12:20:58 +00:00
|
|
|
int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
|
2019-07-22 14:52:12 +00:00
|
|
|
int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev);
|
2018-04-25 15:27:50 +00:00
|
|
|
struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
|
|
|
|
|
const uint8_t *rss_key, uint32_t rss_key_len,
|
|
|
|
|
uint64_t hash_fields,
|
2018-08-01 08:37:38 +00:00
|
|
|
const uint16_t *queues, uint32_t queues_n,
|
2019-07-29 11:53:28 +00:00
|
|
|
int tunnel __rte_unused);
|
2018-04-25 15:27:50 +00:00
|
|
|
struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
|
|
|
|
|
const uint8_t *rss_key, uint32_t rss_key_len,
|
|
|
|
|
uint64_t hash_fields,
|
2018-07-12 09:31:01 +00:00
|
|
|
const uint16_t *queues, uint32_t queues_n);
|
2018-03-05 12:21:04 +00:00
|
|
|
int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
|
2019-07-22 14:52:13 +00:00
|
|
|
int mlx5_hrxq_verify(struct rte_eth_dev *dev);
|
2018-07-12 09:30:48 +00:00
|
|
|
struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
|
|
|
|
|
void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
|
2019-07-29 11:53:29 +00:00
|
|
|
uint64_t mlx5_get_rx_port_offloads(void);
|
2018-03-05 12:21:04 +00:00
|
|
|
uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
|
2015-10-30 18:52:31 +00:00
|
|
|
|
|
|
|
|
/* mlx5_txq.c */
|
|
|
|
|
|
2018-03-05 12:20:58 +00:00
|
|
|
int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
|
|
|
|
unsigned int socket, const struct rte_eth_txconf *conf);
|
|
|
|
|
void mlx5_tx_queue_release(void *dpdk_txq);
|
2019-04-10 18:41:17 +00:00
|
|
|
int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
|
2018-03-05 12:21:04 +00:00
|
|
|
struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
|
|
|
|
|
int mlx5_txq_ibv_verify(struct rte_eth_dev *dev);
|
|
|
|
|
struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
|
|
|
|
|
uint16_t desc, unsigned int socket,
|
|
|
|
|
const struct rte_eth_txconf *conf);
|
|
|
|
|
struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
|
|
|
|
|
int mlx5_txq_verify(struct rte_eth_dev *dev);
|
2018-03-05 12:20:58 +00:00
|
|
|
void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
|
2018-03-05 12:21:04 +00:00
|
|
|
uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
|
2015-10-30 18:52:31 +00:00
|
|
|
|
|
|
|
|
/* mlx5_rxtx.c */
|
|
|
|
|
|
2017-07-26 19:29:33 +00:00
|
|
|
extern uint32_t mlx5_ptype_table[];
|
2018-04-08 12:41:20 +00:00
|
|
|
extern uint8_t mlx5_cksum_table[];
|
|
|
|
|
extern uint8_t mlx5_swp_types_table[];
|
2017-07-06 18:41:10 +00:00
|
|
|
|
2017-07-26 19:29:33 +00:00
|
|
|
void mlx5_set_ptype_table(void);
|
2018-04-08 12:41:20 +00:00
|
|
|
void mlx5_set_cksum_table(void);
|
|
|
|
|
void mlx5_set_swp_types_table(void);
|
2019-08-05 13:03:51 +00:00
|
|
|
__rte_noinline int mlx5_tx_error_cqe_handle
|
2019-07-21 14:24:59 +00:00
|
|
|
(struct mlx5_txq_data *restrict txq,
|
|
|
|
|
volatile struct mlx5_err_cqe *err_cqe);
|
2018-03-05 12:20:58 +00:00
|
|
|
uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);
|
|
|
|
|
__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq,
|
|
|
|
|
uint8_t mbuf_prepare);
|
2018-05-09 11:13:50 +00:00
|
|
|
void mlx5_mprq_buf_free_cb(void *addr, void *opaque);
|
|
|
|
|
void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf);
|
|
|
|
|
uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts,
|
|
|
|
|
uint16_t pkts_n);
|
2018-03-05 12:20:58 +00:00
|
|
|
uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
|
|
|
|
|
uint16_t pkts_n);
|
|
|
|
|
uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts,
|
|
|
|
|
uint16_t pkts_n);
|
|
|
|
|
int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
|
|
|
|
|
int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
|
2018-10-27 15:10:55 +00:00
|
|
|
uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
2019-05-30 10:20:32 +00:00
|
|
|
void mlx5_dump_debug_information(const char *path, const char *title,
|
|
|
|
|
const void *buf, unsigned int len);
|
2019-05-30 10:20:38 +00:00
|
|
|
int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
|
|
|
|
|
const struct mlx5_mp_arg_queue_state_modify *sm);
|
2015-10-30 18:52:31 +00:00
|
|
|
|
2017-07-06 18:41:10 +00:00
|
|
|
/* Vectorized version of mlx5_rxtx.c */
|
2018-03-05 12:21:04 +00:00
|
|
|
int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
|
|
|
|
|
int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
|
|
|
|
|
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
|
2018-03-05 12:20:58 +00:00
|
|
|
uint16_t pkts_n);
|
2017-07-06 18:41:10 +00:00
|
|
|
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
/* mlx5_mr.c */
|
|
|
|
|
|
|
|
|
|
void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
|
|
|
|
|
uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
|
2018-11-15 10:29:17 +00:00
|
|
|
uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
|
2018-09-24 18:36:47 +00:00
|
|
|
uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
|
|
|
|
|
struct rte_mempool *mp);
|
2019-03-10 08:28:02 +00:00
|
|
|
int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
|
|
|
|
|
size_t len);
|
|
|
|
|
int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
|
|
|
|
|
size_t len);
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
|
2018-07-12 12:01:31 +00:00
|
|
|
/**
|
|
|
|
|
* Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
|
|
|
|
|
* 64bit architectures.
|
|
|
|
|
*
|
|
|
|
|
* @param val
|
|
|
|
|
* value to write in CPU endian format.
|
|
|
|
|
* @param addr
|
|
|
|
|
* Address to write to.
|
|
|
|
|
* @param lock
|
|
|
|
|
* Address of the lock to use for that UAR access.
|
|
|
|
|
*/
|
|
|
|
|
static __rte_always_inline void
|
2018-11-15 10:29:19 +00:00
|
|
|
__mlx5_uar_write64_relaxed(uint64_t val, void *addr,
|
2018-07-12 12:01:31 +00:00
|
|
|
rte_spinlock_t *lock __rte_unused)
|
|
|
|
|
{
|
|
|
|
|
#ifdef RTE_ARCH_64
|
2018-11-15 10:29:19 +00:00
|
|
|
*(uint64_t *)addr = val;
|
2018-07-12 12:01:31 +00:00
|
|
|
#else /* !RTE_ARCH_64 */
|
|
|
|
|
rte_spinlock_lock(lock);
|
2018-11-15 10:29:19 +00:00
|
|
|
*(uint32_t *)addr = val;
|
2018-07-12 12:01:31 +00:00
|
|
|
rte_io_wmb();
|
2018-11-15 10:29:19 +00:00
|
|
|
*((uint32_t *)addr + 1) = val >> 32;
|
2018-07-12 12:01:31 +00:00
|
|
|
rte_spinlock_unlock(lock);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
|
|
|
|
|
* 64bit architectures while guaranteeing the order of execution with the
|
|
|
|
|
* code being executed.
|
|
|
|
|
*
|
|
|
|
|
* @param val
|
|
|
|
|
* value to write in CPU endian format.
|
|
|
|
|
* @param addr
|
|
|
|
|
* Address to write to.
|
|
|
|
|
* @param lock
|
|
|
|
|
* Address of the lock to use for that UAR access.
|
|
|
|
|
*/
|
|
|
|
|
static __rte_always_inline void
|
2018-11-15 10:29:19 +00:00
|
|
|
__mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
|
2018-07-12 12:01:31 +00:00
|
|
|
{
|
|
|
|
|
rte_io_wmb();
|
|
|
|
|
__mlx5_uar_write64_relaxed(val, addr, lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Assist macros, used instead of directly calling the functions they wrap. */
|
|
|
|
|
#ifdef RTE_ARCH_64
|
|
|
|
|
#define mlx5_uar_write64_relaxed(val, dst, lock) \
|
|
|
|
|
__mlx5_uar_write64_relaxed(val, dst, NULL)
|
|
|
|
|
#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
|
|
|
|
|
#else
|
|
|
|
|
#define mlx5_uar_write64_relaxed(val, dst, lock) \
|
|
|
|
|
__mlx5_uar_write64_relaxed(val, dst, lock)
|
|
|
|
|
#define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
|
|
|
|
|
#endif
|
|
|
|
|
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
/* CQE status. */
|
|
|
|
|
enum mlx5_cqe_status {
|
2019-08-05 13:03:51 +00:00
|
|
|
MLX5_CQE_STATUS_SW_OWN = -1,
|
|
|
|
|
MLX5_CQE_STATUS_HW_OWN = -2,
|
|
|
|
|
MLX5_CQE_STATUS_ERR = -3,
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
};
|
2017-07-06 18:41:10 +00:00
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Check whether CQE is valid.
|
|
|
|
|
*
|
|
|
|
|
* @param cqe
|
|
|
|
|
* Pointer to CQE.
|
|
|
|
|
* @param cqes_n
|
|
|
|
|
* Size of completion queue.
|
|
|
|
|
* @param ci
|
|
|
|
|
* Consumer index.
|
|
|
|
|
*
|
|
|
|
|
* @return
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
* The CQE status.
|
2017-07-06 18:41:10 +00:00
|
|
|
*/
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
static __rte_always_inline enum mlx5_cqe_status
|
|
|
|
|
check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n,
|
|
|
|
|
const uint16_t ci)
|
2017-07-06 18:41:10 +00:00
|
|
|
{
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
const uint16_t idx = ci & cqes_n;
|
|
|
|
|
const uint8_t op_own = cqe->op_own;
|
|
|
|
|
const uint8_t op_owner = MLX5_CQE_OWNER(op_own);
|
|
|
|
|
const uint8_t op_code = MLX5_CQE_OPCODE(op_own);
|
2017-07-06 18:41:10 +00:00
|
|
|
|
|
|
|
|
if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
|
net/mlx5: extend Rx completion with error handling
When WQEs are posted to the HW to receive packets, the PMD may receive
a completion report with error from the HW, aka error CQE which is
associated to a bad WQE.
The error reason may be bad address, wrong lkey, small buffer size,
etc. that can wrongly be configured by the PMD or by the user.
Checking all the optional mistakes to prevent error CQEs doesn't make
sense due to performance impacts, moreover, some error CQEs can be
triggered because of the packets coming from the wire when the DPDK
application has no any control.
Most of the error CQE types change the RQ state to error state what
causes all the next received packets to be dropped by the HW and to be
completed with CQE flush error forever.
The current solution detects these error CQEs and even reports the
errors to the user by the statistics error counters but without
recovery, so if the RQ inserted to the error state it never moves to
ready state again and all the next packets ever will be dropped.
Extend the error CQEs handling for recovery by moving the state to
ready again, and rearranging all the RQ WQEs and the management
variables appropriately.
Sometimes the error CQE root cause is very hard to debug and even may
be related to some corner cases which are not reproducible easily,
hence a dump file with debug information will be created for the first
number of error CQEs, this number can be configured by the PMD probe
parameters.
Cc: stable@dpdk.org
Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
2019-05-30 10:20:36 +00:00
|
|
|
return MLX5_CQE_STATUS_HW_OWN;
|
|
|
|
|
rte_cio_rmb();
|
|
|
|
|
if (unlikely(op_code == MLX5_CQE_RESP_ERR ||
|
|
|
|
|
op_code == MLX5_CQE_REQ_ERR))
|
|
|
|
|
return MLX5_CQE_STATUS_ERR;
|
|
|
|
|
return MLX5_CQE_STATUS_SW_OWN;
|
2017-07-06 18:41:10 +00:00
|
|
|
}
|
|
|
|
|
|
2018-09-24 18:36:47 +00:00
|
|
|
/**
|
|
|
|
|
* Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
|
|
|
|
|
* cloned mbuf is allocated is returned instead.
|
|
|
|
|
*
|
|
|
|
|
* @param buf
|
|
|
|
|
* Pointer to mbuf.
|
|
|
|
|
*
|
|
|
|
|
* @return
|
|
|
|
|
* Memory pool where data is located for given mbuf.
|
|
|
|
|
*/
|
2018-11-15 10:29:17 +00:00
|
|
|
static inline struct rte_mempool *
|
2018-09-24 18:36:47 +00:00
|
|
|
mlx5_mb2mp(struct rte_mbuf *buf)
|
|
|
|
|
{
|
2019-01-14 05:20:54 +00:00
|
|
|
if (unlikely(RTE_MBUF_CLONED(buf)))
|
2018-09-24 18:36:47 +00:00
|
|
|
return rte_mbuf_from_indirect(buf)->pool;
|
|
|
|
|
return buf->pool;
|
|
|
|
|
}
|
|
|
|
|
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
/**
|
|
|
|
|
* Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
|
|
|
|
|
* as mempool is pre-configured and static.
|
|
|
|
|
*
|
|
|
|
|
* @param rxq
|
|
|
|
|
* Pointer to Rx queue structure.
|
|
|
|
|
* @param addr
|
|
|
|
|
* Address to search.
|
|
|
|
|
*
|
|
|
|
|
* @return
|
|
|
|
|
* Searched LKey on success, UINT32_MAX on no match.
|
|
|
|
|
*/
|
2017-07-06 18:41:10 +00:00
|
|
|
static __rte_always_inline uint32_t
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
|
2017-07-06 18:41:10 +00:00
|
|
|
{
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
|
|
|
|
|
uint32_t lkey;
|
|
|
|
|
|
|
|
|
|
/* Linear search on MR cache array. */
|
|
|
|
|
lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
|
|
|
|
|
MLX5_MR_CACHE_N, addr);
|
|
|
|
|
if (likely(lkey != UINT32_MAX))
|
|
|
|
|
return lkey;
|
|
|
|
|
/* Take slower bottom-half (Binary Search) on miss. */
|
|
|
|
|
return mlx5_rx_addr2mr_bh(rxq, addr);
|
2017-07-06 18:41:10 +00:00
|
|
|
}
|
|
|
|
|
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Query LKey from a packet buffer for Tx. If not found, add the mempool.
|
|
|
|
|
*
|
|
|
|
|
* @param txq
|
|
|
|
|
* Pointer to Tx queue structure.
|
|
|
|
|
* @param addr
|
|
|
|
|
* Address to search.
|
|
|
|
|
*
|
|
|
|
|
* @return
|
|
|
|
|
* Searched LKey on success, UINT32_MAX on no match.
|
|
|
|
|
*/
|
|
|
|
|
static __rte_always_inline uint32_t
|
2018-11-15 10:29:17 +00:00
|
|
|
mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
{
|
|
|
|
|
struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
|
2018-11-15 10:29:17 +00:00
|
|
|
uintptr_t addr = (uintptr_t)mb->buf_addr;
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
uint32_t lkey;
|
|
|
|
|
|
|
|
|
|
/* Check generation bit to see if there's any change on existing MRs. */
|
|
|
|
|
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
|
|
|
|
|
mlx5_mr_flush_local_cache(mr_ctrl);
|
|
|
|
|
/* Linear search on MR cache array. */
|
|
|
|
|
lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
|
|
|
|
|
MLX5_MR_CACHE_N, addr);
|
|
|
|
|
if (likely(lkey != UINT32_MAX))
|
|
|
|
|
return lkey;
|
2018-11-15 10:29:17 +00:00
|
|
|
/* Take slower bottom-half on miss. */
|
|
|
|
|
return mlx5_tx_mb2mr_bh(txq, mb);
|
2018-09-24 18:36:47 +00:00
|
|
|
}
|
net/mlx5: add new memory region support
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
2018-05-09 11:09:04 +00:00
|
|
|
|
2017-07-06 18:41:10 +00:00
|
|
|
/**
|
2017-10-25 00:27:25 +00:00
|
|
|
* Ring TX queue doorbell and flush the update if requested.
|
2017-07-06 18:41:10 +00:00
|
|
|
*
|
|
|
|
|
* @param txq
|
|
|
|
|
* Pointer to TX queue structure.
|
|
|
|
|
* @param wqe
|
|
|
|
|
* Pointer to the last WQE posted in the NIC.
|
2017-10-25 00:27:25 +00:00
|
|
|
* @param cond
|
|
|
|
|
* Request for write memory barrier after BlueFlame update.
|
2017-07-06 18:41:10 +00:00
|
|
|
*/
|
|
|
|
|
static __rte_always_inline void
|
2017-10-25 00:27:25 +00:00
|
|
|
mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
|
|
|
|
|
int cond)
|
2017-07-06 18:41:10 +00:00
|
|
|
{
|
2019-04-10 18:41:17 +00:00
|
|
|
uint64_t *dst = MLX5_TX_BFREG(txq);
|
2017-07-06 18:41:10 +00:00
|
|
|
volatile uint64_t *src = ((volatile uint64_t *)wqe);
|
|
|
|
|
|
2018-01-25 21:02:49 +00:00
|
|
|
rte_cio_wmb();
|
2017-09-17 10:42:02 +00:00
|
|
|
*txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
|
2017-07-06 18:41:10 +00:00
|
|
|
/* Ensure ordering between DB record and BF copy. */
|
|
|
|
|
rte_wmb();
|
2018-07-12 12:01:31 +00:00
|
|
|
mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
|
2017-10-25 00:27:25 +00:00
|
|
|
if (cond)
|
|
|
|
|
rte_wmb();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Ring TX queue doorbell and flush the update by write memory barrier.
|
|
|
|
|
*
|
|
|
|
|
* @param txq
|
|
|
|
|
* Pointer to TX queue structure.
|
|
|
|
|
* @param wqe
|
|
|
|
|
* Pointer to the last WQE posted in the NIC.
|
|
|
|
|
*/
|
|
|
|
|
static __rte_always_inline void
|
|
|
|
|
mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
|
|
|
|
|
{
|
|
|
|
|
mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
|
2017-07-06 18:41:10 +00:00
|
|
|
}
|
|
|
|
|
|
2015-10-30 18:52:31 +00:00
|
|
|
#endif /* RTE_PMD_MLX5_RXTX_H_ */
|
