d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

regex/mlx5: setup fast path

Browse Source 
Allocated and register input/output buffers and metadata.

Signed-off-by: Yuval Avnery <yuvalav@mellanox.com>
Acked-by: Ori Kam <orika@mellanox.com>
This commit is contained in:
Yuval Avnery 2020-07-20 06:26:14 +00:00 committed by Thomas Monjalon
parent 92f2c6a30f
commit 5f41b66d12
6 changed files with 238 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
drivers/common/mlx5/mlx5_prm.h
View File

@ -373,6 +373,42 @@ struct mlx5_cqe {
uint8_t op_own;
};
/* MMO metadata segment */
#define MLX5_OPCODE_MMO 0x2f
#define MLX5_OPC_MOD_MMO_REGEX 0x4
struct mlx5_wqe_metadata_seg {
uint32_t mmo_control_31_0; /* mmo_control_63_32 is in ctrl_seg.imm */
uint32_t lkey;
uint64_t addr;
};
struct mlx5_ifc_regexp_mmo_control_bits {
uint8_t reserved_at_31[0x2];
uint8_t le[0x1];
uint8_t reserved_at_28[0x1];
uint8_t subset_id_0[0xc];
uint8_t reserved_at_16[0x4];
uint8_t subset_id_1[0xc];
uint8_t ctrl[0x4];
uint8_t subset_id_2[0xc];
uint8_t reserved_at_16_1[0x4];
uint8_t subset_id_3[0xc];
};
struct mlx5_ifc_regexp_metadata_bits {
uint8_t rof_version[0x10];
uint8_t latency_count[0x10];
uint8_t instruction_count[0x10];
uint8_t primary_thread_count[0x10];
uint8_t match_count[0x8];
uint8_t detected_match_count[0x8];
uint8_t status[0x10];
uint8_t job_id[0x20];
uint8_t reserved[0x80];
};
/* Adding direct verbs to data-path. */
/* CQ sequence number mask. */

1
drivers/regex/mlx5/Makefile
View File

@ -11,6 +11,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_rxp.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_devx.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_control.c
SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_fastpath.c
# Basic CFLAGS.
CFLAGS += -O3

1
drivers/regex/mlx5/meson.build
View File

@ -14,6 +14,7 @@ sources = files(
'mlx5_rxp.c',
'mlx5_regex_devx.c',
'mlx5_regex_control.c',
'mlx5_regex_fastpath.c',
)
cflags_options = [
'-std=c11',

8
drivers/regex/mlx5/mlx5_regex.h
View File

@ -40,6 +40,11 @@ struct mlx5_regex_qp {
struct mlx5_regex_sq *sqs; /* Pointer to sq array. */
uint16_t nb_obj; /* Number of sq objects. */
struct mlx5_regex_cq cq; /* CQ struct. */
uint32_t free_sqs;
struct mlx5_regex_job *jobs;
struct ibv_mr *metadata;
struct ibv_mr *inputs;
struct ibv_mr *outputs;
};
struct mlx5_regex_db {
@ -92,4 +97,7 @@ int mlx5_devx_regex_database_program(void *ctx, uint8_t engine,
int mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind,
const struct rte_regexdev_qp_conf *cfg);
/* mlx5_regex_fastpath.c */
int mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id);
#endif /* MLX5_REGEX_H */

2
drivers/regex/mlx5/mlx5_regex_control.c
View File

@ -363,6 +363,8 @@ mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind,
goto error;
}
}
mlx5_regexdev_setup_fastpath(priv, qp_ind);
return 0;
error:

190
drivers/regex/mlx5/mlx5_regex_fastpath.c Normal file
View File

@ -0,0 +1,190 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2020 Mellanox Technologies, Ltd
*/
#include <unistd.h>
#include <sys/mman.h>
#include <rte_malloc.h>
#include <rte_log.h>
#include <rte_errno.h>
#include <rte_bus_pci.h>
#include <rte_pci.h>
#include <rte_regexdev_driver.h>
#include <rte_mbuf.h>
#include <infiniband/mlx5dv.h>
#include <mlx5_glue.h>
#include <mlx5_common.h>
#include <mlx5_prm.h>
#include <strings.h>
#include "mlx5_regex_utils.h"
#include "mlx5_rxp.h"
#include "mlx5_regex.h"
#define MLX5_REGEX_METADATA_SIZE 64
#define MLX5_REGEX_MAX_INPUT (1 << 14)
#define MLX5_REGEX_MAX_OUTPUT (1 << 11)
#define MLX5_REGEX_WQE_METADATA_OFFSET 16
#define MLX5_REGEX_WQE_GATHER_OFFSET 32
#define MLX5_REGEX_WQE_SCATTER_OFFSET 48
static inline uint32_t
sq_size_get(struct mlx5_regex_sq *sq)
{
return (1U << sq->log_nb_desc);
}
struct mlx5_regex_job {
uint64_t user_id;
uint8_t *input;
volatile uint8_t *output;
volatile uint8_t *metadata;
} __rte_cached_aligned;
static inline void
set_data_seg(struct mlx5_wqe_data_seg *seg,
uint32_t length, uint32_t lkey,
uintptr_t address)
{
seg->byte_count = rte_cpu_to_be_32(length);
seg->lkey = rte_cpu_to_be_32(lkey);
seg->addr = rte_cpu_to_be_64(address);
}
static inline void
set_metadata_seg(struct mlx5_wqe_metadata_seg *seg,
uint32_t mmo_control_31_0, uint32_t lkey,
uintptr_t address)
{
seg->mmo_control_31_0 = htobe32(mmo_control_31_0);
seg->lkey = rte_cpu_to_be_32(lkey);
seg->addr = rte_cpu_to_be_64(address);
}
static void
setup_sqs(struct mlx5_regex_qp *queue)
{
size_t sqid, entry;
uint32_t job_id;
for (sqid = 0; sqid < queue->nb_obj; sqid++) {
struct mlx5_regex_sq *sq = &queue->sqs[sqid];
uint8_t *wqe = (uint8_t *)sq->wqe;
for (entry = 0 ; entry < sq_size_get(sq); entry++) {
job_id = sqid * sq_size_get(sq) + entry;
struct mlx5_regex_job *job = &queue->jobs[job_id];
set_metadata_seg((struct mlx5_wqe_metadata_seg *)
(wqe + MLX5_REGEX_WQE_METADATA_OFFSET),
0, queue->metadata->lkey,
(uintptr_t)job->metadata);
set_data_seg((struct mlx5_wqe_data_seg *)
(wqe + MLX5_REGEX_WQE_GATHER_OFFSET),
0, queue->inputs->lkey,
(uintptr_t)job->input);
set_data_seg((struct mlx5_wqe_data_seg *)
(wqe + MLX5_REGEX_WQE_SCATTER_OFFSET),
MLX5_REGEX_MAX_OUTPUT,
queue->outputs->lkey,
(uintptr_t)job->output);
wqe += 64;
}
queue->free_sqs |= 1 << sqid;
}
}
static int
setup_buffers(struct mlx5_regex_qp *qp, struct ibv_pd *pd)
{
uint32_t i;
int err;
void *ptr = rte_calloc(__func__, qp->nb_desc,
MLX5_REGEX_METADATA_SIZE,
MLX5_REGEX_METADATA_SIZE);
if (!ptr)
return -ENOMEM;
qp->metadata = mlx5_glue->reg_mr(pd, ptr,
MLX5_REGEX_METADATA_SIZE*qp->nb_desc,
IBV_ACCESS_LOCAL_WRITE);
if (!qp->metadata) {
rte_free(ptr);
return -EINVAL;
}
ptr = rte_calloc(__func__, qp->nb_desc,
MLX5_REGEX_MAX_INPUT,
MLX5_REGEX_MAX_INPUT);
if (!ptr) {
err = -ENOMEM;
goto err_input;
}
qp->inputs = mlx5_glue->reg_mr(pd, ptr,
MLX5_REGEX_MAX_INPUT*qp->nb_desc,
IBV_ACCESS_LOCAL_WRITE);
if (!qp->inputs) {
rte_free(ptr);
err = -EINVAL;
goto err_input;
}
ptr = rte_calloc(__func__, qp->nb_desc,
MLX5_REGEX_MAX_OUTPUT,
MLX5_REGEX_MAX_OUTPUT);
if (!ptr) {
err = -ENOMEM;
goto err_output;
}
qp->outputs = mlx5_glue->reg_mr(pd, ptr,
MLX5_REGEX_MAX_OUTPUT * qp->nb_desc,
IBV_ACCESS_LOCAL_WRITE);
if (!qp->outputs) {
rte_free(ptr);
err = -EINVAL;
goto err_output;
}
/* distribute buffers to jobs */
for (i = 0; i < qp->nb_desc; i++) {
qp->jobs[i].input =
(uint8_t *)qp->inputs->addr +
(i % qp->nb_desc) * MLX5_REGEX_MAX_INPUT;
qp->jobs[i].output =
(uint8_t *)qp->outputs->addr +
(i % qp->nb_desc) * MLX5_REGEX_MAX_OUTPUT;
qp->jobs[i].metadata =
(uint8_t *)qp->metadata->addr +
(i % qp->nb_desc) * MLX5_REGEX_METADATA_SIZE;
}
return 0;
err_output:
ptr = qp->inputs->addr;
rte_free(ptr);
mlx5_glue->dereg_mr(qp->inputs);
err_input:
ptr = qp->metadata->addr;
rte_free(ptr);
mlx5_glue->dereg_mr(qp->metadata);
return err;
}
int
mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id)
{
struct mlx5_regex_qp *qp = &priv->qps[qp_id];
int err;
qp->jobs = rte_calloc(__func__, qp->nb_desc, sizeof(*qp->jobs),
sizeof(*qp->jobs));
if (!qp->jobs)
return -ENOMEM;
err = setup_buffers(qp, priv->pd);
if (err)
return err;
setup_sqs(qp);
return 0;
}