d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

common/mlx5: support DevX indirect mkey creation

Browse Source 
Add option to create an indirect mkey by the current
mlx5_devx_cmd_mkey_create command.

Indirect mkey points to set of direct mkeys.
By this way, the HW\SW can reference fragmented memory by one object.

Align the net/mlx5 driver usage in the above command.

Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
This commit is contained in:
Matan Azrad 2020-01-29 12:38:34 +00:00 committed by Ferruh Yigit
parent e3b8dabebe
commit 53ec4db032
4 changed files with 69 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
drivers/common/mlx5/mlx5_devx_cmds.c
View File

@ -142,7 +142,11 @@ struct mlx5_devx_obj *
mlx5_devx_cmd_mkey_create(struct ibv_context *ctx,
struct mlx5_devx_mkey_attr *attr)
{
uint32_t in[MLX5_ST_SZ_DW(create_mkey_in)] = {0};
struct mlx5_klm *klm_array = attr->klm_array;
int klm_num = attr->klm_num;
int in_size_dw = MLX5_ST_SZ_DW(create_mkey_in) +
(klm_num ? RTE_ALIGN(klm_num, 4) : 0) * MLX5_ST_SZ_DW(klm);
uint32_t in[in_size_dw];
uint32_t out[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
void *mkc;
struct mlx5_devx_obj *mkey = rte_zmalloc("mkey", sizeof(*mkey), 0);
@ -153,27 +157,52 @@ mlx5_devx_cmd_mkey_create(struct ibv_context *ctx,
rte_errno = ENOMEM;
return NULL;
}
memset(in, 0, in_size_dw * 4);
pgsize = sysconf(_SC_PAGESIZE);
translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16;
MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
if (klm_num > 0) {
int i;
uint8_t *klm = (uint8_t *)MLX5_ADDR_OF(create_mkey_in, in,
klm_pas_mtt);
translation_size = RTE_ALIGN(klm_num, 4);
for (i = 0; i < klm_num; i++) {
MLX5_SET(klm, klm, byte_count, klm_array[i].byte_count);
MLX5_SET(klm, klm, mkey, klm_array[i].mkey);
MLX5_SET64(klm, klm, address, klm_array[i].address);
klm += MLX5_ST_SZ_BYTES(klm);
}
for (; i < (int)translation_size; i++) {
MLX5_SET(klm, klm, mkey, 0x0);
MLX5_SET64(klm, klm, address, 0x0);
klm += MLX5_ST_SZ_BYTES(klm);
}
MLX5_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ?
MLX5_MKC_ACCESS_MODE_KLM_FBS :
MLX5_MKC_ACCESS_MODE_KLM);
MLX5_SET(mkc, mkc, log_page_size, attr->log_entity_size);
} else {
translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16;
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize));
}
MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
translation_size);
MLX5_SET(create_mkey_in, in, mkey_umem_id, attr->umem_id);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access);
MLX5_SET(mkc, mkc, lw, 0x1);
MLX5_SET(mkc, mkc, lr, 0x1);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, pd, attr->pd);
MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
MLX5_SET(mkc, mkc, translations_octword_size, translation_size);
MLX5_SET64(mkc, mkc, start_addr, attr->addr);
MLX5_SET64(mkc, mkc, len, attr->size);
MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize));
mkey->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
sizeof(out));
if (!mkey->obj) {
DRV_LOG(ERR, "Can't create mkey - error %d", errno);
DRV_LOG(ERR, "Can't create %sdirect mkey - error %d\n",
klm_num ? "an in" : "a ", errno);
rte_errno = errno;
rte_free(mkey);
return NULL;

17
drivers/common/mlx5/mlx5_devx_cmds.h
View File

@ -6,6 +6,7 @@
#define RTE_PMD_MLX5_DEVX_CMDS_H_
#include "mlx5_glue.h"
#include "mlx5_prm.h"
/* devX creation object */
@ -14,11 +15,26 @@ struct mlx5_devx_obj {
int id; /* The object ID. */
};
/* UMR memory buffer used to define 1 entry in indirect mkey. */
struct mlx5_klm {
uint32_t byte_count;
uint32_t mkey;
uint64_t address;
};
/* This is limitation of libibverbs: in length variable type is u16. */
#define MLX5_DEVX_MAX_KLM_ENTRIES ((UINT16_MAX - \
MLX5_ST_SZ_DW(create_mkey_in) * 4) / (MLX5_ST_SZ_DW(klm) * 4))
struct mlx5_devx_mkey_attr {
uint64_t addr;
uint64_t size;
uint32_t umem_id;
uint32_t pd;
uint32_t log_entity_size;
uint32_t pg_access:1;
struct mlx5_klm *klm_array;
int klm_num;
};
/* HCA qos attributes. */
@ -216,6 +232,7 @@ struct mlx5_devx_modify_sq_attr {
uint32_t hairpin_peer_vhca:16;
};
/* mlx5_devx_cmds.c */
struct mlx5_devx_obj *mlx5_devx_cmd_flow_counter_alloc(struct ibv_context *ctx,

12
drivers/common/mlx5/mlx5_prm.h
View File

@ -726,6 +726,8 @@ enum {
enum {
MLX5_MKC_ACCESS_MODE_MTT = 0x1,
MLX5_MKC_ACCESS_MODE_KLM = 0x2,
MLX5_MKC_ACCESS_MODE_KLM_FBS = 0x3,
};
/* Flow counters. */
@ -790,6 +792,16 @@ struct mlx5_ifc_query_flow_counter_in_bits {
u8 flow_counter_id[0x20];
};
#define MLX5_MAX_KLM_BYTE_COUNT 0x80000000u
#define MLX5_MIN_KLM_FIXED_BUFFER_SIZE 0x1000u
struct mlx5_ifc_klm_bits {
u8 byte_count[0x20];
u8 mkey[0x20];
u8 address[0x40];
};
struct mlx5_ifc_mkc_bits {
u8 reserved_at_0[0x1];
u8 free[0x1];

4
drivers/net/mlx5/mlx5_flow_dv.c
View File

@ -3885,6 +3885,10 @@ flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
mkey_attr.size = size;
mkey_attr.umem_id = mem_mng->umem->umem_id;
mkey_attr.pd = sh->pdn;
mkey_attr.log_entity_size = 0;
mkey_attr.pg_access = 0;
mkey_attr.klm_array = NULL;
mkey_attr.klm_num = 0;
mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
if (!mem_mng->dm) {
mlx5_glue->devx_umem_dereg(mem_mng->umem);