d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

common/mlx5: fix doorbell mapping configuration

Browse Source 
UAR mapping type can be affected by the devarg tx_db_nc, which can cause
setting the environment variable MLX5_SHUT_UP_BF.
So, the MLX5_SHUT_UP_BF value and the UAR mapping parameter affect the
UAR cache mode.

Wrongly, the devarg was considered for the MLX5_SHUT_UP_BF but not for
the UAR mapping parameter in all the drivers except the net.

Take the tx_db_nc devarg into account for all the drivers.

Fixes: ca1418ce39 ("common/mlx5: share device context object")
Cc: stable@dpdk.org

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Reviewed-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
This commit is contained in:
Michael Baum 2021-11-03 20:35:11 +02:00 committed by Thomas Monjalon
parent 3f0e54fe00
commit b4371d3d56
6 changed files with 35 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
drivers/common/mlx5/mlx5_common.c
View File

@ -928,30 +928,25 @@ RTE_INIT_PRIO(mlx5_is_haswell_broadwell_cpu, LOG)
/**
* Allocate the User Access Region with DevX on specified device.
* This routine handles the following UAR allocation issues:
*
* @param [in] ctx
* Infiniband device context to perform allocation on.
* @param [in] mapping
* MLX5DV_UAR_ALLOC_TYPE_BF - allocate as cached memory with write-combining
* attributes (if supported by the host), the
* writes to the UAR registers must be followed
* by write memory barrier.
* MLX5DV_UAR_ALLOC_TYPE_NC - allocate as non-cached memory, all writes are
* promoted to the registers immediately, no
* memory barriers needed.
* mapping < 0 - the first attempt is performed with MLX5DV_UAR_ALLOC_TYPE_NC,
* if this fails the next attempt with MLX5DV_UAR_ALLOC_TYPE_BF
* is performed. The drivers specifying negative values should
* always provide the write memory barrier operation after UAR
* register writings.
* If there is no definitions for the MLX5DV_UAR_ALLOC_TYPE_xx (older rdma
* library headers), the caller can specify 0.
* - tries to allocate the UAR with the most appropriate memory mapping
* type from the ones supported by the host.
*
* - tries to allocate the UAR with non-NULL base address OFED 5.0.x and
* Upstream rdma_core before v29 returned the NULL as UAR base address
* if UAR was not the first object in the UAR page.
* It caused the PMD failure and we should try to get another UAR till
* we get the first one with non-NULL base address returned.
*
* @param [in] cdev
* Pointer to mlx5 device structure to perform allocation on its context.
*
* @return
* UAR object pointer on success, NULL otherwise and rte_errno is set.
*/
void *
mlx5_devx_alloc_uar(void *ctx, int mapping)
mlx5_devx_alloc_uar(struct mlx5_common_device *cdev)
{
void *uar;
uint32_t retry, uar_mapping;
@ -960,26 +955,35 @@ mlx5_devx_alloc_uar(void *ctx, int mapping)
for (retry = 0; retry < MLX5_ALLOC_UAR_RETRY; ++retry) {
#ifdef MLX5DV_UAR_ALLOC_TYPE_NC
/* Control the mapping type according to the settings. */
uar_mapping = (mapping < 0) ?
MLX5DV_UAR_ALLOC_TYPE_NC : mapping;
uar_mapping = (cdev->config.dbnc == MLX5_TXDB_NCACHED) ?
MLX5DV_UAR_ALLOC_TYPE_NC : MLX5DV_UAR_ALLOC_TYPE_BF;
#else
/*
* It seems we have no way to control the memory mapping type
* for the UAR, the default "Write-Combining" type is supposed.
*/
uar_mapping = 0;
RTE_SET_USED(mapping);
#endif
uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
#ifdef MLX5DV_UAR_ALLOC_TYPE_NC
if (!uar && mapping < 0) {
if (!uar && uar_mapping == MLX5DV_UAR_ALLOC_TYPE_BF) {
/*
* In some environments like virtual machine the
* Write Combining mapped might be not supported and
* UAR allocation fails. We tried "Non-Cached" mapping
* for the case.
*/
DRV_LOG(DEBUG, "Failed to allocate DevX UAR (BF)");
uar_mapping = MLX5DV_UAR_ALLOC_TYPE_NC;
uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
} else if (!uar && uar_mapping == MLX5DV_UAR_ALLOC_TYPE_NC) {
/*
* If Verbs/kernel does not support "Non-Cached"
* try the "Write-Combining".
*/
DRV_LOG(DEBUG, "Failed to allocate DevX UAR (NC)");
uar_mapping = MLX5DV_UAR_ALLOC_TYPE_BF;
uar = mlx5_glue->devx_alloc_uar(ctx, uar_mapping);
uar = mlx5_glue->devx_alloc_uar(cdev->ctx, uar_mapping);
}
#endif
if (!uar) {

5
drivers/common/mlx5/mlx5_common.h
View File

@ -284,8 +284,6 @@ __rte_internal
void mlx5_translate_port_name(const char *port_name_in,
struct mlx5_switch_info *port_info_out);
void mlx5_glue_constructor(void);
__rte_internal
void *mlx5_devx_alloc_uar(void *ctx, int mapping);
extern uint8_t haswell_broadwell_cpu;
__rte_internal
@ -417,6 +415,9 @@ void
mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
struct rte_mempool *mp);
__rte_internal
void *mlx5_devx_alloc_uar(struct mlx5_common_device *cdev);
/* mlx5_common_os.c */
int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);

2
drivers/compress/mlx5/mlx5_compress.c
View File

@ -683,7 +683,7 @@ mlx5_compress_uar_release(struct mlx5_compress_priv *priv)
static int
mlx5_compress_uar_prepare(struct mlx5_compress_priv *priv)
{
priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
priv->uar = mlx5_devx_alloc_uar(priv->cdev);
if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
NULL) {
rte_errno = errno;

2
drivers/crypto/mlx5/mlx5_crypto.c
View File

@ -736,7 +736,7 @@ mlx5_crypto_uar_release(struct mlx5_crypto_priv *priv)
static int
mlx5_crypto_uar_prepare(struct mlx5_crypto_priv *priv)
{
priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
priv->uar = mlx5_devx_alloc_uar(priv->cdev);
if (priv->uar)
priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
if (priv->uar == NULL || priv->uar_addr == NULL) {

2
drivers/regex/mlx5/mlx5_regex.c
View File

@ -107,7 +107,7 @@ mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
* registers writings, it is safe to allocate UAR with any
* memory mapping type.
*/
priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
priv->uar = mlx5_devx_alloc_uar(priv->cdev);
if (!priv->uar) {
DRV_LOG(ERR, "can't allocate uar.");
rte_errno = ENOMEM;

2
drivers/vdpa/mlx5/mlx5_vdpa_event.c
View File

@ -61,7 +61,7 @@ mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
* registers writings, it is safe to allocate UAR with any
* memory mapping type.
*/
priv->uar = mlx5_devx_alloc_uar(priv->cdev->ctx, -1);
priv->uar = mlx5_devx_alloc_uar(priv->cdev);
if (!priv->uar) {
rte_errno = errno;
DRV_LOG(ERR, "Failed to allocate UAR.");