d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

app/regex: support multiple queue pairs

Browse Source 
Up to this commit the regex application used one QP which was assigned a
number of jobs, each with a different segment of a file to parse.  This
commit adds support for multiple QPs assignments. All QPs will be
assigned the same number of jobs, with the same segments of file to
parse. It will enable comparing functionality with different numbers of
QPs. All queues are managed on one core with one thread. This commit
focuses on changing routines API to support multi QPs, mainly, QP scalar
variables are replaced by per-QP struct instance.  The enqueue/dequeue
operations are interleaved as follows:
 enqueue(QP #1)
 enqueue(QP #2)
 ...
 enqueue(QP #n)
 dequeue(QP #1)
 dequeue(QP #2)
 ...
 dequeue(QP #n)

A new parameter 'nb_qps' was added to configure the number of QPs:
 --nb_qps <num of qps>.
If not configured, nb_qps is set to 1 by default.

Signed-off-by: Ophir Munk <ophirmu@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
This commit is contained in:
Ophir Munk 2021-01-10 11:10:19 +00:00 committed by Thomas Monjalon
parent 2d1fb3f2a6
commit 4545bd0088
1 changed files with 203 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
app/test-regex/main.c
View File

@ -33,12 +33,22 @@ enum app_args {
ARG_NUM_OF_JOBS, ARG_NUM_OF_JOBS,
ARG_PERF_MODE, ARG_PERF_MODE,
ARG_NUM_OF_ITERATIONS, ARG_NUM_OF_ITERATIONS,
ARG_NUM_OF_QPS,
}; };
struct job_ctx { struct job_ctx {
struct rte_mbuf *mbuf; struct rte_mbuf *mbuf;
}; };
struct qp_params {
uint32_t total_enqueue;
uint32_t total_dequeue;
uint32_t total_matches;
struct rte_regex_ops **ops;
struct job_ctx *jobs_ctx;
char *buf;
};
static void static void
usage(const char *prog_name) usage(const char *prog_name)
{ {
@ -47,13 +57,15 @@ usage(const char *prog_name)
" --data NAME: data file to use\n" " --data NAME: data file to use\n"
" --nb_jobs: number of jobs to use\n" " --nb_jobs: number of jobs to use\n"
" --perf N: only outputs the performance data\n" " --perf N: only outputs the performance data\n"
" --nb_iter N: number of iteration to run\n", " --nb_iter N: number of iteration to run\n"
" --nb_qps N: number of queues to use\n",
prog_name); prog_name);
} }
static void static void
args_parse(int argc, char **argv, char *rules_file, char *data_file, args_parse(int argc, char **argv, char *rules_file, char *data_file,
uint32_t *nb_jobs, bool *perf_mode, uint32_t *nb_iterations) uint32_t *nb_jobs, bool *perf_mode, uint32_t *nb_iterations,
uint32_t *nb_qps)
{ {
char **argvopt; char **argvopt;
int opt; int opt;
@ -71,6 +83,8 @@ args_parse(int argc, char **argv, char *rules_file, char *data_file,
{ "perf", 0, 0, ARG_PERF_MODE}, { "perf", 0, 0, ARG_PERF_MODE},
/* Number of iterations to run with perf test */ /* Number of iterations to run with perf test */
{ "nb_iter", 1, 0, ARG_NUM_OF_ITERATIONS}, { "nb_iter", 1, 0, ARG_NUM_OF_ITERATIONS},
/* Number of QPs. */
{ "nb_qps", 1, 0, ARG_NUM_OF_QPS},
/* End of options */ /* End of options */
{ 0, 0, 0, 0 } { 0, 0, 0, 0 }
}; };
@ -104,6 +118,9 @@ args_parse(int argc, char **argv, char *rules_file, char *data_file,
case ARG_NUM_OF_ITERATIONS: case ARG_NUM_OF_ITERATIONS:
*nb_iterations = atoi(optarg); *nb_iterations = atoi(optarg);
break; break;
case ARG_NUM_OF_QPS:
*nb_qps = atoi(optarg);
break;
case ARG_HELP: case ARG_HELP:
usage("RegEx test app"); usage("RegEx test app");
break; break;
@ -163,15 +180,17 @@ error:
} }
static int static int
init_port(uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches) init_port(uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches,
uint32_t nb_qps)
{ {
uint16_t id; uint16_t id;
uint16_t qp_id;
uint16_t num_devs; uint16_t num_devs;
char *rules = NULL; char *rules = NULL;
long rules_len; long rules_len;
struct rte_regexdev_info info; struct rte_regexdev_info info;
struct rte_regexdev_config dev_conf = { struct rte_regexdev_config dev_conf = {
.nb_queue_pairs = 1, .nb_queue_pairs = nb_qps,
.nb_groups = 1, .nb_groups = 1,
}; };
struct rte_regexdev_qp_conf qp_conf = { struct rte_regexdev_qp_conf qp_conf = {
@ -203,7 +222,8 @@ init_port(uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches)
*nb_max_matches = info.max_matches; *nb_max_matches = info.max_matches;
*nb_max_payload = info.max_payload_size; *nb_max_payload = info.max_payload_size;
if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F) if (info.regexdev_capa & RTE_REGEXDEV_SUPP_MATCH_AS_END_F)
dev_conf.dev_cfg_flags |= RTE_REGEXDEV_CFG_MATCH_AS_END_F; dev_conf.dev_cfg_flags |=
RTE_REGEXDEV_CFG_MATCH_AS_END_F;
dev_conf.nb_max_matches = info.max_matches; dev_conf.nb_max_matches = info.max_matches;
dev_conf.nb_rules_per_group = info.max_rules_per_group; dev_conf.nb_rules_per_group = info.max_rules_per_group;
dev_conf.rule_db_len = rules_len; dev_conf.rule_db_len = rules_len;
@ -214,12 +234,16 @@ init_port(uint16_t *nb_max_payload, char *rules_file, uint8_t *nb_max_matches)
goto error; goto error;
} }
if (info.regexdev_capa & RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F) if (info.regexdev_capa & RTE_REGEXDEV_CAPA_QUEUE_PAIR_OOS_F)
qp_conf.qp_conf_flags |= RTE_REGEX_QUEUE_PAIR_CFG_OOS_F; qp_conf.qp_conf_flags |=
res = rte_regexdev_queue_pair_setup(id, 0, &qp_conf); RTE_REGEX_QUEUE_PAIR_CFG_OOS_F;
if (res < 0) { for (qp_id = 0; qp_id < nb_qps; qp_id++) {
printf("Error, can't setup queue pair for device %d.\n", res = rte_regexdev_queue_pair_setup(id, qp_id,
id); &qp_conf);
goto error; if (res < 0) {
printf("Error, can't setup queue pair %u for "
"device %d.\n", qp_id, id);
goto error;
}
} }
printf(":: initializing device: %d done\n", id); printf(":: initializing device: %d done\n", id);
} }
@ -239,122 +263,171 @@ extbuf_free_cb(void *addr __rte_unused, void *fcb_opaque __rte_unused)
static int static int
run_regex(uint32_t nb_jobs, run_regex(uint32_t nb_jobs,
uint16_t nb_max_payload, bool perf_mode, uint32_t nb_iterations, uint16_t nb_max_payload, bool perf_mode, uint32_t nb_iterations,
char *data_file, uint8_t nb_max_matches) char *data_file, uint8_t nb_max_matches, uint32_t nb_qps)
{ {
char *buf = NULL; char *buf = NULL;
long buf_len; long buf_len = 0;
long job_len; long job_len = 0;
uint32_t actual_jobs = 0; uint32_t actual_jobs = 0;
uint32_t i; uint32_t i;
struct rte_regex_ops **ops; uint16_t qp_id;
uint16_t dev_id = 0; uint16_t dev_id = 0;
uint16_t qp_id = 0;
uint8_t nb_matches; uint8_t nb_matches;
struct rte_regexdev_match *match; struct rte_regexdev_match *match;
long pos = 0; long pos;
unsigned long d_ind = 0; unsigned long d_ind = 0;
struct rte_mbuf_ext_shared_info shinfo; struct rte_mbuf_ext_shared_info shinfo;
uint32_t total_enqueue = 0;
uint32_t total_dequeue = 0;
uint32_t total_matches = 0;
int res = 0; int res = 0;
time_t start; time_t start;
time_t end; time_t end;
double time; double time;
struct job_ctx *jobs_ctx;
struct rte_mempool *mbuf_mp; struct rte_mempool *mbuf_mp;
struct qp_params *qp;
struct qp_params *qps = NULL;
bool update;
uint16_t qps_used = 0;
shinfo.free_cb = extbuf_free_cb; shinfo.free_cb = extbuf_free_cb;
mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", nb_jobs * nb_qps, 0,
mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool", nb_jobs, 0,
0, MBUF_SIZE, rte_socket_id()); 0, MBUF_SIZE, rte_socket_id());
if (mbuf_mp == NULL) { if (mbuf_mp == NULL) {
printf("Error, can't create memory pool\n"); printf("Error, can't create memory pool\n");
return -ENOMEM; return -ENOMEM;
} }
ops = rte_malloc(NULL, sizeof(*ops) * nb_jobs, 0); qps = rte_malloc(NULL, sizeof(*qps) * nb_qps, 0);
if (!ops) { if (!qps) {
printf("Error, can't allocate memory for ops.\n"); printf("Error, can't allocate memory for QPs\n");
return -ENOMEM; res = -ENOMEM;
goto end;
} }
jobs_ctx = rte_malloc(NULL, sizeof(struct job_ctx)*nb_jobs, 0); for (qp_id = 0; qp_id < nb_qps; qp_id++) {
if (!jobs_ctx) { struct rte_regex_ops **ops;
printf("Error, can't allocate memory for jobs_ctx.\n"); struct job_ctx *jobs_ctx;
return -ENOMEM;
}
/* Allocate the jobs and assign each job with an mbuf. */ qps_used++;
for (i = 0; i < nb_jobs; i++) { qp = &qps[qp_id];
ops[i] = rte_malloc(NULL, sizeof(*ops[0]) + nb_max_matches * qp->jobs_ctx = NULL;
sizeof(struct rte_regexdev_match), 0); qp->buf = NULL;
if (!ops[i]) { qp->ops = ops = rte_malloc(NULL, sizeof(*ops) * nb_jobs, 0);
printf("Error, can't allocate memory for op.\n"); if (!ops) {
printf("Error, can't allocate memory for ops.\n");
res = -ENOMEM; res = -ENOMEM;
goto end; goto end;
} }
ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
if (!ops[i]->mbuf) { qp->jobs_ctx = jobs_ctx =
printf("Error, can't attach mbuf.\n"); rte_malloc(NULL, sizeof(*jobs_ctx) * nb_jobs, 0);
if (!jobs_ctx) {
printf("Error, can't allocate memory for jobs_ctx.\n");
res = -ENOMEM; res = -ENOMEM;
goto end; goto end;
} }
}
buf_len = read_file(data_file, &buf); /* Allocate the jobs and assign each job with an mbuf. */
if (buf_len <= 0) { for (i = 0; i < nb_jobs; i++) {
printf("Error, can't read file, or file is empty.\n"); ops[i] = rte_malloc(NULL, sizeof(*ops[0]) +
res = -EXIT_FAILURE; nb_max_matches *
goto end; sizeof(struct rte_regexdev_match), 0);
} if (!ops[i]) {
printf("Error, can't allocate "
"memory for op.\n");
res = -ENOMEM;
goto end;
}
ops[i]->mbuf = rte_pktmbuf_alloc(mbuf_mp);
if (!ops[i]->mbuf) {
printf("Error, can't attach mbuf.\n");
res = -ENOMEM;
goto end;
}
}
job_len = buf_len / nb_jobs; buf_len = read_file(data_file, &buf);
if (job_len == 0) { if (buf_len <= 0) {
printf("Error, To many jobs, for the given input.\n"); printf("Error, can't read file, or file is empty.\n");
res = -EXIT_FAILURE; res = -EXIT_FAILURE;
goto end; goto end;
} }
if (job_len > nb_max_payload) { job_len = buf_len / nb_jobs;
printf("Error, not enough jobs to cover input.\n"); if (job_len == 0) {
res = -EXIT_FAILURE; printf("Error, To many jobs, for the given input.\n");
goto end; res = -EXIT_FAILURE;
} goto end;
}
/* Assign each mbuf with the data to handle. */ if (job_len > nb_max_payload) {
for (i = 0; (pos < buf_len) && (i < nb_jobs) ; i++) { printf("Error, not enough jobs to cover input.\n");
long act_job_len = RTE_MIN(job_len, buf_len - pos); res = -EXIT_FAILURE;
rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0, goto end;
act_job_len, &shinfo); }
jobs_ctx[i].mbuf = ops[i]->mbuf;
ops[i]->mbuf->data_len = job_len; /* Assign each mbuf with the data to handle. */
ops[i]->mbuf->pkt_len = act_job_len; actual_jobs = 0;
ops[i]->user_id = i; pos = 0;
ops[i]->group_id0 = 1; for (i = 0; (pos < buf_len) && (i < nb_jobs) ; i++) {
pos += act_job_len; long act_job_len = RTE_MIN(job_len, buf_len - pos);
actual_jobs++; rte_pktmbuf_attach_extbuf(ops[i]->mbuf, &buf[pos], 0,
act_job_len, &shinfo);
jobs_ctx[i].mbuf = ops[i]->mbuf;
ops[i]->mbuf->data_len = job_len;
ops[i]->mbuf->pkt_len = act_job_len;
ops[i]->user_id = i;
ops[i]->group_id0 = 1;
pos += act_job_len;
actual_jobs++;
}
qp->buf = buf;
qp->total_matches = 0;
} }
start = clock(); start = clock();
for (i = 0; i < nb_iterations; i++) { for (i = 0; i < nb_iterations; i++) {
total_enqueue = 0; for (qp_id = 0; qp_id < nb_qps; qp_id++) {
total_dequeue = 0; qp = &qps[qp_id];
while (total_dequeue < actual_jobs) { qp->total_enqueue = 0;
struct rte_regex_ops **cur_ops_to_enqueue = ops + qp->total_dequeue = 0;
total_enqueue;
struct rte_regex_ops **cur_ops_to_dequeue = ops +
total_dequeue;
if (actual_jobs - total_enqueue)
total_enqueue += rte_regexdev_enqueue_burst
(dev_id, qp_id, cur_ops_to_enqueue,
actual_jobs - total_enqueue);
total_dequeue += rte_regexdev_dequeue_burst
(dev_id, qp_id, cur_ops_to_dequeue,
total_enqueue - total_dequeue);
} }
do {
update = false;
for (qp_id = 0; qp_id < nb_qps; qp_id++) {
qp = &qps[qp_id];
if (qp->total_dequeue < actual_jobs) {
struct rte_regex_ops **
cur_ops_to_enqueue = qp->ops +
qp->total_enqueue;
if (actual_jobs - qp->total_enqueue)
qp->total_enqueue +=
rte_regexdev_enqueue_burst
(dev_id,
qp_id,
cur_ops_to_enqueue,
actual_jobs -
qp->total_enqueue);
}
}
for (qp_id = 0; qp_id < nb_qps; qp_id++) {
qp = &qps[qp_id];
if (qp->total_dequeue < actual_jobs) {
struct rte_regex_ops **
cur_ops_to_dequeue = qp->ops +
qp->total_dequeue;
qp->total_dequeue +=
rte_regexdev_dequeue_burst
(dev_id,
qp_id,
cur_ops_to_dequeue,
qp->total_enqueue -
qp->total_dequeue);
update = true;
}
}
} while (update);
} }
end = clock(); end = clock();
time = ((double)end - start) / CLOCKS_PER_SEC; time = ((double)end - start) / CLOCKS_PER_SEC;
@ -364,51 +437,59 @@ run_regex(uint32_t nb_jobs,
(((double)actual_jobs * job_len * nb_iterations * 8) / time) / (((double)actual_jobs * job_len * nb_iterations * 8) / time) /
1000000000.0); 1000000000.0);
if (!perf_mode) { if (perf_mode)
goto end;
for (qp_id = 0; qp_id < nb_qps; qp_id++) {
printf("\n############ QP id=%u ############\n", qp_id);
qp = &qps[qp_id];
/* Log results per job. */ /* Log results per job. */
for (d_ind = 0; d_ind < total_dequeue; d_ind++) { for (d_ind = 0; d_ind < qp->total_dequeue; d_ind++) {
nb_matches = ops[d_ind % actual_jobs]->nb_matches; nb_matches = qp->ops[d_ind % actual_jobs]->nb_matches;
printf("Job id %"PRIu64" number of matches = %d\n", printf("Job id %"PRIu64" number of matches = %d\n",
ops[d_ind]->user_id, nb_matches); qp->ops[d_ind]->user_id, nb_matches);
total_matches += nb_matches; qp->total_matches += nb_matches;
match = ops[d_ind % actual_jobs]->matches; match = qp->ops[d_ind % actual_jobs]->matches;
for (i = 0; i < nb_matches; i++) { for (i = 0; i < nb_matches; i++) {
printf("match %d, rule = %d, start = %d,len = %d\n", printf("match %d, rule = %d, "
"start = %d,len = %d\n",
i, match->rule_id, match->start_offset, i, match->rule_id, match->start_offset,
match->len); match->len);
match++; match++;
} }
} }
printf("Total matches = %d\n", total_matches); printf("Total matches = %d\n", qp->total_matches);
printf("All Matches:\n"); printf("All Matches:\n");
/* Log absolute results. */ /* Log absolute results. */
for (d_ind = 0; d_ind < total_dequeue; d_ind++) { for (d_ind = 0; d_ind < qp->total_dequeue; d_ind++) {
nb_matches = ops[d_ind % actual_jobs]->nb_matches; nb_matches = qp->ops[d_ind % actual_jobs]->nb_matches;
total_matches += nb_matches; qp->total_matches += nb_matches;
match = ops[d_ind % actual_jobs]->matches; match = qp->ops[d_ind % actual_jobs]->matches;
for (i = 0; i < nb_matches; i++) { for (i = 0; i < nb_matches; i++) {
printf("start = %ld, len = %d, rule = %d\n", printf("start = %ld, len = %d, rule = %d\n",
match->start_offset + d_ind * job_len, match->start_offset +
match->len, match->rule_id); d_ind * job_len,
match->len, match->rule_id);
match++; match++;
} }
} }
} }
end: end:
for (i = 0; i < actual_jobs; i++) { for (qp_id = 0; qp_id < qps_used; qp_id++) {
if (ops[i]) qp = &qps[qp_id];
rte_free(ops[i]); for (i = 0; i < actual_jobs && qp->ops; i++)
if (jobs_ctx[i].mbuf) rte_free(qp->ops[i]);
rte_pktmbuf_free(jobs_ctx[i].mbuf); rte_free(qp->ops);
qp->ops = NULL;
for (i = 0; i < actual_jobs && qp->jobs_ctx; i++)
rte_pktmbuf_free(qp->jobs_ctx[i].mbuf);
rte_free(qp->jobs_ctx);
qp->jobs_ctx = NULL;
rte_free(qp->buf);
qp->buf = NULL;
} }
rte_free(ops);
rte_free(jobs_ctx);
if (buf)
rte_free(buf);
if (mbuf_mp) if (mbuf_mp)
rte_mempool_free(mbuf_mp); rte_mempool_free(mbuf_mp);
rte_free(qps);
return res; return res;
} }
@ -418,12 +499,14 @@ main(int argc, char **argv)
char rules_file[MAX_FILE_NAME]; char rules_file[MAX_FILE_NAME];
char data_file[MAX_FILE_NAME]; char data_file[MAX_FILE_NAME];
uint32_t nb_jobs = 0; uint32_t nb_jobs = 0;
uint16_t nb_max_payload = 0;
bool perf_mode = 0; bool perf_mode = 0;
uint32_t nb_iterations = 0; uint32_t nb_iterations = 0;
uint8_t nb_max_matches = 0;
int ret; int ret;
uint16_t nb_max_payload = 0;
uint8_t nb_max_matches = 0;
uint32_t nb_qps = 1;
/* Init EAL. */
ret = rte_eal_init(argc, argv); ret = rte_eal_init(argc, argv);
if (ret < 0) if (ret < 0)
rte_exit(EXIT_FAILURE, "EAL init failed\n"); rte_exit(EXIT_FAILURE, "EAL init failed\n");
@ -431,13 +514,16 @@ main(int argc, char **argv)
argv += ret; argv += ret;
if (argc > 1) if (argc > 1)
args_parse(argc, argv, rules_file, data_file, &nb_jobs, args_parse(argc, argv, rules_file, data_file, &nb_jobs,
&perf_mode, &nb_iterations); &perf_mode, &nb_iterations, &nb_qps);
ret = init_port(&nb_max_payload, rules_file, &nb_max_matches); if (nb_qps == 0)
rte_exit(EXIT_FAILURE, "Number of QPs must be greater than 0\n");
ret = init_port(&nb_max_payload, rules_file,
&nb_max_matches, nb_qps);
if (ret < 0) if (ret < 0)
rte_exit(EXIT_FAILURE, "init port failed\n"); rte_exit(EXIT_FAILURE, "init port failed\n");
ret = run_regex(nb_jobs, nb_max_payload, perf_mode, ret = run_regex(nb_jobs, nb_max_payload, perf_mode,
nb_iterations, data_file, nb_max_matches); nb_iterations, data_file, nb_max_matches, nb_qps);
if (ret < 0) { if (ret < 0) {
rte_exit(EXIT_FAILURE, "RegEx function failed\n"); rte_exit(EXIT_FAILURE, "RegEx function failed\n");
} }