d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

common/mlx5: share MR top-half search function

Browse Source 
Add function to search in local liniar cache and use it in the drivers
instead of their functions.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
This commit is contained in:
Michael Baum 2021-10-19 23:55:59 +03:00 committed by Thomas Monjalon
parent 5fbc75ace1
commit fb690f71bd
6 changed files with 77 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/common/mlx5/mlx5_common.h
View File

@ -22,6 +22,7 @@
#include "mlx5_prm.h"
#include "mlx5_devx_cmds.h"
#include "mlx5_common_os.h"
#include "mlx5_common_mr.h"
/* Reported driver name. */
#define MLX5_PCI_DRIVER_NAME "mlx5_pci"
@ -447,6 +448,14 @@ __rte_internal
bool
mlx5_dev_is_pci(const struct rte_device *dev);
/* mlx5_common_mr.c */
__rte_internal
uint32_t
mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
struct mlx5_mr_share_cache *share_cache);
/* mlx5_common_os.c */
int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);

52
drivers/common/mlx5/mlx5_common_mr.c
View File

@ -576,6 +576,8 @@ mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
*
* @param pd
* Pointer to pd of a device (net, regex, vdpa,...).
* @param mp_id
* Multi-process identifier, may be NULL for the primary process.
* @param share_cache
* Pointer to a global shared MR cache.
* @param[out] entry
@ -867,6 +869,8 @@ mlx5_mr_create_primary(void *pd,
*
* @param pd
* Pointer to pd handle of a device (net, regex, vdpa,...).
* @param mp_id
* Multi-process identifier, may be NULL for the primary process.
* @param share_cache
* Pointer to a global shared MR cache.
* @param[out] entry
@ -874,6 +878,8 @@ mlx5_mr_create_primary(void *pd,
* created. If failed to create one, this will not be updated.
* @param addr
* Target virtual address to register.
* @param mr_ext_memseg_en
* Configurable flag about external memory segment enable or not.
*
* @return
* Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
@ -907,6 +913,8 @@ mlx5_mr_create(void *pd, struct mlx5_mp_id *mp_id,
*
* @param pd
* Pointer to pd of a device (net, regex, vdpa,...).
* @param mp_id
* Multi-process identifier, may be NULL for the primary process.
* @param share_cache
* Pointer to a global shared MR cache.
* @param mr_ctrl
@ -916,6 +924,8 @@ mlx5_mr_create(void *pd, struct mlx5_mp_id *mp_id,
* created. If failed to create one, this is not written.
* @param addr
* Search key.
* @param mr_ext_memseg_en
* Configurable flag about external memory segment enable or not.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
@ -971,12 +981,16 @@ mr_lookup_caches(void *pd, struct mlx5_mp_id *mp_id,
*
* @param pd
* Pointer to pd of a device (net, regex, vdpa,...).
* @param mp_id
* Multi-process identifier, may be NULL for the primary process.
* @param share_cache
* Pointer to a global shared MR cache.
* @param mr_ctrl
* Pointer to per-queue MR control structure.
* @param addr
* Search key.
* @param mr_ext_memseg_en
* Configurable flag about external memory segment enable or not.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
@ -1822,3 +1836,41 @@ mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
mr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;
return lkey;
}
/**
* Query LKey from a packet buffer.
*
* @param cdev
* Pointer to the mlx5 device structure.
* @param mp_id
* Multi-process identifier, may be NULL for the primary process.
* @param mr_ctrl
* Pointer to per-queue MR control structure.
* @param mbuf
* Pointer to mbuf.
* @param share_cache
* Pointer to a global shared MR cache.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
uint32_t
mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
struct mlx5_mr_share_cache *share_cache)
{
uint32_t lkey;
uintptr_t addr = (uintptr_t)mbuf->buf_addr;
/* Check generation bit to see if there's any change on existing MRs. */
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
mlx5_mr_flush_local_cache(mr_ctrl);
/* Linear search on MR cache array. */
lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
MLX5_MR_CACHE_N, (uintptr_t)mbuf->buf_addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half on miss. */
return mlx5_mr_addr2mr_bh(cdev->pd, mp_id, share_cache, mr_ctrl,
addr, cdev->config.mr_ext_memseg_en);
}

1
drivers/common/mlx5/version.map
View File

@ -118,6 +118,7 @@ INTERNAL {
mlx5_mr_insert_cache;
mlx5_mr_lookup_cache;
mlx5_mr_lookup_list;
mlx5_mr_mb2mr;
mlx5_free_mr_by_addr;
mlx5_mr_rebuild_cache;
mlx5_mr_release_cache;

38
drivers/compress/mlx5/mlx5_compress.c
View File

@ -435,40 +435,6 @@ static struct rte_compressdev_ops mlx5_compress_ops = {
.stream_free = NULL,
};
/**
* Query LKey from a packet buffer for QP. If not found, add the mempool.
*
* @param priv
* Pointer to the priv object.
* @param addr
* Search key.
* @param mr_ctrl
* Pointer to per-queue MR control structure.
* @param ol_flags
* Mbuf offload features.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
static __rte_always_inline uint32_t
mlx5_compress_addr2mr(struct mlx5_compress_priv *priv, uintptr_t addr,
struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
{
uint32_t lkey;
/* Check generation bit to see if there's any change on existing MRs. */
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
mlx5_mr_flush_local_cache(mr_ctrl);
/* Linear search on MR cache array. */
lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
MLX5_MR_CACHE_N, addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half on miss. */
return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
addr, !!(ol_flags & EXT_ATTACHED_MBUF));
}
static __rte_always_inline uint32_t
mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
volatile struct mlx5_wqe_dseg *restrict dseg,
@ -478,8 +444,8 @@ mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
dseg->bcount = rte_cpu_to_be_32(len);
dseg->lkey = mlx5_compress_addr2mr(qp->priv, addr, &qp->mr_ctrl,
mbuf->ol_flags);
dseg->lkey = mlx5_mr_mb2mr(qp->priv->cdev, 0, &qp->mr_ctrl, mbuf,
&qp->priv->mr_scache);
dseg->pbuf = rte_cpu_to_be_64(addr);
return dseg->lkey;
}

38
drivers/crypto/mlx5/mlx5_crypto.c
View File

@ -303,40 +303,6 @@ mlx5_crypto_get_block_size(struct rte_crypto_op *op)
}
}
/**
* Query LKey from a packet buffer for QP. If not found, add the mempool.
*
* @param priv
* Pointer to the priv object.
* @param addr
* Search key.
* @param mr_ctrl
* Pointer to per-queue MR control structure.
* @param ol_flags
* Mbuf offload features.
*
* @return
* Searched LKey on success, UINT32_MAX on no match.
*/
static __rte_always_inline uint32_t
mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
{
uint32_t lkey;
/* Check generation bit to see if there's any change on existing MRs. */
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
mlx5_mr_flush_local_cache(mr_ctrl);
/* Linear search on MR cache array. */
lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
MLX5_MR_CACHE_N, addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half on miss. */
return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
addr, !!(ol_flags & EXT_ATTACHED_MBUF));
}
static __rte_always_inline uint32_t
mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
@ -350,8 +316,8 @@ mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
*remain -= data_len;
klm->bcount = rte_cpu_to_be_32(data_len);
klm->pbuf = rte_cpu_to_be_64(addr);
klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
mbuf->ol_flags);
klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf,
&priv->mr_scache);
return klm->lkey;
}

34
drivers/regex/mlx5/mlx5_regex_fastpath.c
View File

@ -123,26 +123,12 @@ set_wqe_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode,
* Searched LKey on success, UINT32_MAX on no match.
*/
static inline uint32_t
mlx5_regex_addr2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
struct rte_mbuf *mbuf)
mlx5_regex_mb2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
struct rte_mbuf *mbuf)
{
uintptr_t addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
uint32_t lkey;
/* Check generation bit to see if there's any change on existing MRs. */
if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
mlx5_mr_flush_local_cache(mr_ctrl);
/* Linear search on MR cache array. */
lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
MLX5_MR_CACHE_N, addr);
if (likely(lkey != UINT32_MAX))
return lkey;
/* Take slower bottom-half on miss. */
return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
addr, !!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
return mlx5_mr_mb2mr(priv->cdev, 0, mr_ctrl, mbuf, &priv->mr_scache);
}
static inline void
__prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp_obj,
struct rte_regex_ops *op, struct mlx5_regex_job *job,
@ -194,7 +180,7 @@ prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
struct mlx5_klm klm;
klm.byte_count = rte_pktmbuf_data_len(op->mbuf);
klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, op->mbuf);
klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, op->mbuf);
klm.address = rte_pktmbuf_mtod(op->mbuf, uintptr_t);
__prep_one(priv, qp_obj, op, job, qp_obj->pi, &klm);
qp_obj->db_pi = qp_obj->pi;
@ -317,6 +303,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
uint32_t len = 0;
struct mlx5_klm *mkey_klm = NULL;
struct mlx5_klm klm;
uintptr_t addr;
while (left_ops--)
rte_prefetch0(op[left_ops]);
@ -360,11 +347,12 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
klm.mkey = rte_cpu_to_be_32
(qp->jobs[mkey_job_id].imkey->id);
while (mbuf) {
addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
/* Build indirect mkey seg's KLM. */
mkey_klm->mkey = mlx5_regex_addr2mr
(priv, &qp->mr_ctrl, mbuf);
mkey_klm->address = rte_cpu_to_be_64
(rte_pktmbuf_mtod(mbuf, uintptr_t));
mkey_klm->mkey = mlx5_regex_mb2mr(priv,
&qp->mr_ctrl,
mbuf);
mkey_klm->address = rte_cpu_to_be_64(addr);
mkey_klm->byte_count = rte_cpu_to_be_32
(rte_pktmbuf_data_len(mbuf));
/*
@ -380,7 +368,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
klm.byte_count = scatter_size;
} else {
/* The single mubf case. Build the KLM directly. */
klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, mbuf);
klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, mbuf);
klm.address = rte_pktmbuf_mtod(mbuf, uintptr_t);
klm.byte_count = rte_pktmbuf_data_len(mbuf);
}