d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx4: add new memory region support

Browse Source 
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.

There are multiple layers for MR search.

L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx4_mr_lookup_cache().

If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx4_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.

If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx4_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.

If L3 misses, a new MR for the address should be created -
mlx4_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.

In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.

To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.

'--legacy-mem' is also supported.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
This commit is contained in:
Yongseok Koh 2018-05-09 04:09:06 -07:00 committed by Ferruh Yigit
parent 2d684b911d
commit 9797bfcce1
9 changed files with 1409 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/guides/nics/mlx4.rst
View File

@ -370,6 +370,12 @@ Performance tuning
The XXX can be different on different systems. Make sure to configure
according to the setpci output.
6. To minimize overhead of searching Memory Regions:
- '--socket-mem' is recommended to pin memory by predictable amount.
- Configure per-lcore cache when creating Mempools for packet buffer.
- Refrain from dynamically allocating/freeing memory in run-time.
Usage example
-------------

26
drivers/net/mlx4/mlx4.c
View File

@ -44,9 +44,15 @@
#include "mlx4.h"
#include "mlx4_glue.h"
#include "mlx4_flow.h"
#include "mlx4_mr.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
struct mlx4_dev_list mlx4_mem_event_cb_list =
LIST_HEAD_INITIALIZER(mlx4_mem_event_cb_list);
rte_rwlock_t mlx4_mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
/** Configuration structure for device arguments. */
struct mlx4_conf {
struct {
@ -92,6 +98,20 @@ mlx4_dev_configure(struct rte_eth_dev *dev)
if (ret)
ERROR("%p: interrupt handler installation failed",
(void *)dev);
/*
* Once the device is added to the list of memory event callback, its
* global MR cache table cannot be expanded on the fly because of
* deadlock. If it overflows, lookup should be done by searching MR list
* linearly, which is slow.
*/
if (mlx4_mr_btree_init(&priv->mr.cache, MLX4_MR_BTREE_CACHE_N * 2,
dev->device->numa_node)) {
/* rte_errno is already set. */
return -rte_errno;
}
rte_rwlock_write_lock(&mlx4_mem_event_rwlock);
LIST_INSERT_HEAD(&mlx4_mem_event_cb_list, priv, mem_event_cb);
rte_rwlock_write_unlock(&mlx4_mem_event_rwlock);
exit:
return ret;
}
@ -125,6 +145,9 @@ mlx4_dev_start(struct rte_eth_dev *dev)
(void *)dev, strerror(-ret));
goto err;
}
#ifndef NDEBUG
mlx4_mr_dump_dev(dev);
#endif
ret = mlx4_rxq_intr_enable(priv);
if (ret) {
ERROR("%p: interrupt handler installation failed",
@ -200,6 +223,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
mlx4_rx_queue_release(dev->data->rx_queues[i]);
for (i = 0; i != dev->data->nb_tx_queues; ++i)
mlx4_tx_queue_release(dev->data->tx_queues[i]);
mlx4_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(mlx4_glue->dealloc_pd(priv->pd));
@ -964,6 +988,8 @@ rte_mlx4_pmd_init(void)
}
mlx4_glue->fork_init();
rte_pci_register(&mlx4_driver);
rte_mem_event_callback_register("MLX4_MEM_EVENT_CB",
mlx4_mr_mem_event_cb, NULL);
}
RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__);

14
drivers/net/mlx4/mlx4.h
View File

@ -23,6 +23,9 @@
#include <rte_ether.h>
#include <rte_interrupts.h>
#include <rte_mempool.h>
#include <rte_rwlock.h>
#include "mlx4_mr.h"
#ifndef IBV_RX_HASH_INNER
/** This is not necessarily defined by supported RDMA core versions. */
@ -66,8 +69,12 @@ struct rxq;
struct txq;
struct rte_flow;
LIST_HEAD(mlx4_dev_list, priv);
LIST_HEAD(mlx4_mr_list, mlx4_mr);
/** Private data structure. */
struct priv {
LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
struct rte_eth_dev *dev; /**< Ethernet device. */
struct ibv_context *ctx; /**< Verbs context. */
struct ibv_device_attr device_attr; /**< Device properties. */
@ -86,6 +93,13 @@ struct priv {
uint64_t hw_rss_sup; /**< Supported RSS hash fields (Verbs format). */
struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
struct {
uint32_t dev_gen; /* Generation number to flush local caches. */
rte_rwlock_t rwlock; /* MR Lock. */
struct mlx4_mr_btree cache; /* Global MR cache table. */
struct mlx4_mr_list mr_list; /* Registered MR list. */
struct mlx4_mr_list mr_free_list; /* Freed MR list. */
} mr;
LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];

1144
drivers/net/mlx4/mlx4_mr.c
View File

File diff suppressed because it is too large Load Diff

122
drivers/net/mlx4/mlx4_mr.h Normal file
View File

@ -0,0 +1,122 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2018 6WIND S.A.
* Copyright 2018 Mellanox Technologies, Ltd
*/
#ifndef RTE_PMD_MLX4_MR_H_
#define RTE_PMD_MLX4_MR_H_
#include <stddef.h>
#include <stdint.h>
#include <sys/queue.h>
/* Verbs headers do not support -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <infiniband/verbs.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
#include <rte_eal_memconfig.h>
#include <rte_ethdev.h>
#include <rte_rwlock.h>
#include <rte_bitmap.h>
/* Size of per-queue MR cache array for linear search. */
#define MLX4_MR_CACHE_N 8
/* Size of MR cache table for binary search. */
#define MLX4_MR_BTREE_CACHE_N 256
/* Memory Region object. */
struct mlx4_mr {
LIST_ENTRY(mlx4_mr) mr; /**< Pointer to the prev/next entry. */
struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
const struct rte_memseg_list *msl;
int ms_base_idx; /* Start index of msl->memseg_arr[]. */
int ms_n; /* Number of memsegs in use. */
uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
};
/* Cache entry for Memory Region. */
struct mlx4_mr_cache {
uintptr_t start; /* Start address of MR. */
uintptr_t end; /* End address of MR. */
uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
} __rte_packed;
/* MR Cache table for Binary search. */
struct mlx4_mr_btree {
uint16_t len; /* Number of entries. */
uint16_t size; /* Total number of entries. */
int overflow; /* Mark failure of table expansion. */
struct mlx4_mr_cache (*table)[];
} __rte_packed;
/* Per-queue MR control descriptor. */
struct mlx4_mr_ctrl {
uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
uint32_t cur_gen; /* Generation number saved to flush caches. */
uint16_t mru; /* Index of last hit entry in top-half cache. */
uint16_t head; /* Index of the oldest entry in top-half cache. */
struct mlx4_mr_cache cache[MLX4_MR_CACHE_N]; /* Cache for top-half. */
struct mlx4_mr_btree cache_bh; /* Cache for bottom-half. */
} __rte_packed;
extern struct mlx4_dev_list mlx4_mem_event_cb_list;
extern rte_rwlock_t mlx4_mem_event_rwlock;
/* First entry must be NULL for comparison. */
#define mlx4_mr_btree_len(bt) ((bt)->len - 1)
int mlx4_mr_btree_init(struct mlx4_mr_btree *bt, int n, int socket);
void mlx4_mr_btree_free(struct mlx4_mr_btree *bt);
void mlx4_mr_btree_dump(struct mlx4_mr_btree *bt);
void mlx4_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
size_t len, void *arg);
int mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
struct rte_mempool *mp);
void mlx4_mr_dump_dev(struct rte_eth_dev *dev);
void mlx4_mr_release(struct rte_eth_dev *dev);
/**
* Look up LKey from given lookup table by linear search. Firstly look up the
* last-hit entry. If miss, the entire array is searched. If found, update the
* last-hit index and return LKey.
*
* @param lkp_tbl
* Pointer to lookup table.
* @param[in,out] cached_idx
* Pointer to last-hit index.
* @param n
* Size of lookup table.
* @param addr
* Search key.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
static __rte_always_inline uint32_t
mlx4_mr_lookup_cache(struct mlx4_mr_cache *lkp_tbl, uint16_t *cached_idx,
uint16_t n, uintptr_t addr)
{
uint16_t idx;
if (likely(addr >= lkp_tbl[*cached_idx].start &&
addr < lkp_tbl[*cached_idx].end))
return lkp_tbl[*cached_idx].lkey;
for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
if (addr >= lkp_tbl[idx].start &&
addr < lkp_tbl[idx].end) {
/* Found. */
*cached_idx = idx;
return lkp_tbl[idx].lkey;
}
}
return UINT32_MAX;
}
#endif /* RTE_PMD_MLX4_MR_H_ */

14
drivers/net/mlx4/mlx4_rxq.c
View File

@ -488,6 +488,7 @@ mlx4_rxq_attach(struct rxq *rxq)
}
struct priv *priv = rxq->priv;
struct rte_eth_dev *dev = priv->dev;
const uint32_t elts_n = 1 << rxq->elts_n;
const uint32_t sges_n = 1 << rxq->sges_n;
struct rte_mbuf *(*elts)[elts_n] = rxq->elts;
@ -552,6 +553,11 @@ mlx4_rxq_attach(struct rxq *rxq)
msg = "failed to obtain device information from WQ/CQ objects";
goto error;
}
/* Pre-register Rx mempool. */
DEBUG("port %u Rx queue %u registering mp %s having %u chunks",
priv->dev->data->port_id, rxq->stats.idx,
rxq->mp->name, rxq->mp->nb_mem_chunks);
mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp);
wqes = (volatile struct mlx4_wqe_data_seg (*)[])
((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset);
for (i = 0; i != RTE_DIM(*elts); ++i) {
@ -583,7 +589,7 @@ mlx4_rxq_attach(struct rxq *rxq)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(buf->data_len),
.lkey = UINT32_MAX,
.lkey = mlx4_rx_mb2mr(rxq, buf),
};
(*elts)[i] = buf;
}
@ -855,6 +861,11 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1 << rxq->sges_n);
goto error;
}
if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh,
MLX4_MR_BTREE_CACHE_N, socket)) {
/* rte_errno is already set. */
goto error;
}
if (dev->data->dev_conf.intr_conf.rxq) {
rxq->channel = mlx4_glue->create_comp_channel(priv->ctx);
if (rxq->channel == NULL) {
@ -912,5 +923,6 @@ mlx4_rx_queue_release(void *dpdk_rxq)
assert(!rxq->rq_db);
if (rxq->channel)
claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel));
mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh);
rte_free(rxq);
}

35
drivers/net/mlx4/mlx4_rxtx.c
View File

@ -343,24 +343,6 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_m,
txq->elts_tail = elts_tail;
}
/**
* Get memory pool (MP) from mbuf. If mbuf is indirect, the pool from which
* the cloned mbuf is allocated is returned instead.
*
* @param buf
* Pointer to mbuf.
*
* @return
* Memory pool where data is located for given mbuf.
*/
static struct rte_mempool *
mlx4_txq_mb2mp(struct rte_mbuf *buf)
{
if (unlikely(RTE_MBUF_INDIRECT(buf)))
return rte_mbuf_from_indirect(buf)->pool;
return buf->pool;
}
/**
* Write Tx data segment to the SQ.
*
@ -378,7 +360,7 @@ mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
uint32_t lkey, uintptr_t addr, rte_be32_t byte_count)
{
dseg->addr = rte_cpu_to_be_64(addr);
dseg->lkey = rte_cpu_to_be_32(lkey);
dseg->lkey = lkey;
#if RTE_CACHE_LINE_SIZE < 64
/*
* Need a barrier here before writing the byte_count
@ -437,7 +419,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
goto txbb_tail_segs;
txbb_head_seg:
/* Memory region key (big endian) for this memory pool. */
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@ -449,7 +431,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
dseg = (volatile struct mlx4_wqe_data_seg *)
sq->buf;
dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
dseg->lkey = rte_cpu_to_be_32(lkey);
dseg->lkey = lkey;
/*
* This data segment starts at the beginning of a new
* TXBB, so we need to postpone its byte_count writing
@ -469,7 +451,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
/* Jump to default if there are more than two segments remaining. */
switch (nb_segs) {
default:
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@ -485,7 +467,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 2:
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@ -501,7 +483,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 1:
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@ -611,7 +593,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
elt->buf = NULL;
break;
}
lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
lkey = mlx4_tx_mb2mr(txq, buf);
if (unlikely(lkey == (uint32_t)-1)) {
/* MR does not exist. */
DEBUG("%p: unable to get MP <-> MR association",
@ -966,6 +948,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* changes.
*/
scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
/* If there's only one MR, no need to replace LKey in WQE. */
if (unlikely(mlx4_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
scat->lkey = mlx4_rx_mb2mr(rxq, rep);
if (len > seg->data_len) {
len -= seg->data_len;
++pkt->nb_segs;

71
drivers/net/mlx4/mlx4_rxtx.h
View File

@ -25,6 +25,7 @@
#include "mlx4.h"
#include "mlx4_prm.h"
#include "mlx4_mr.h"
/** Rx queue counters. */
struct mlx4_rxq_stats {
@ -46,6 +47,7 @@ struct rxq {
uint16_t port_id; /**< Port ID for incoming packets. */
uint16_t sges_n; /**< Number of segments per packet (log2 value). */
uint16_t elts_n; /**< Mbuf queue size (log2 value). */
struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct rte_mbuf *(*elts)[]; /**< Rx elements. */
volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */
volatile uint32_t *rq_db; /**< RQ doorbell record. */
@ -100,6 +102,7 @@ struct txq {
int elts_comp_cd; /**< Countdown for next completion. */
unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
unsigned int elts_n; /**< (*elts)[] length. */
struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct txq_elt (*elts)[]; /**< Tx elements. */
struct mlx4_txq_stats stats; /**< Tx queue counters. */
uint32_t max_inline; /**< Max inline send size. */
@ -155,12 +158,70 @@ int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
const struct rte_eth_txconf *conf);
void mlx4_tx_queue_release(void *dpdk_txq);
static inline uint32_t
mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
/* mlx4_mr.c */
void mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl);
uint32_t mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr);
uint32_t mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr);
/**
* Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
* as mempool is pre-configured and static.
*
* @param rxq
* Pointer to Rx queue structure.
* @param addr
* Address to search.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
static __rte_always_inline uint32_t
mlx4_rx_addr2mr(struct rxq *rxq, uintptr_t addr)
{
(void)txq;
(void)mp;
return UINT32_MAX;
struct mlx4_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
uint32_t lkey;
/* Linear search on MR cache array. */
lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
MLX4_MR_CACHE_N, addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half (Binary Search) on miss. */
return mlx4_rx_addr2mr_bh(rxq, addr);
}
#define mlx4_rx_mb2mr(rxq, mb) mlx4_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
/**
* Query LKey from a packet buffer for Tx. If not found, add the mempool.
*
* @param txq
* Pointer to Tx queue structure.
* @param addr
* Address to search.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
static __rte_always_inline uint32_t
mlx4_tx_addr2mr(struct txq *txq, uintptr_t addr)
{
struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
uint32_t lkey;
/* Check generation bit to see if there's any change on existing MRs. */
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
mlx4_mr_flush_local_cache(mr_ctrl);
/* Linear search on MR cache array. */
lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
MLX4_MR_CACHE_N, addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half (binary search) on miss. */
return mlx4_tx_addr2mr_bh(txq, addr);
}
#define mlx4_tx_mb2mr(rxq, mb) mlx4_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
#endif /* MLX4_RXTX_H_ */

8
drivers/net/mlx4/mlx4_txq.c
View File

@ -316,6 +316,13 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
if (mlx4_mr_btree_init(&txq->mr_ctrl.cache_bh,
MLX4_MR_BTREE_CACHE_N, socket)) {
/* rte_errno is already set. */
goto error;
}
/* Save pointer of global generation number to check memory event. */
txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
dev->data->tx_queues[idx] = txq;
return 0;
@ -356,5 +363,6 @@ mlx4_tx_queue_release(void *dpdk_txq)
claim_zero(mlx4_glue->destroy_qp(txq->qp));
if (txq->cq)
claim_zero(mlx4_glue->destroy_cq(txq->cq));
mlx4_mr_btree_free(&txq->mr_ctrl.cache_bh);
rte_free(txq);
}