d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fio_nvme: defer qpair allocation to file_open callback

Browse Source 
All jobs are created at boot, meaning the setup callback
is invoked for all jobs before any are executed.

But it may be useful to put 'stonewall' parameters in
the job file to execute a bunch of workloads in succession,
starting one workload when the previous one completes.
But since qpairs are created currently during setup, the
total number of workloads that can be expressed is limited
since qpairs for all workloads are allocated up front.

So instead defer allocation of the io qpairs until the
file_open callback.  These don't get called until the
job associated with the 'file' (in this case, the
nvme namespace) is ready to execute.

Note that we cannot free the qpairs in the file_close
callback, since fio may 'close' the file before all
I/O have been completed.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I3c60cf27c3660a3c94042c0de719f5bebdb9b417
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7481
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: <dongx.yi@intel.com>
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Jim Harris 2021-04-19 11:36:29 -07:00
parent 9bacb8629d
commit f69367c788
1 changed files with 25 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
examples/nvme/fio_plugin/fio_plugin.c
View File

@ -319,7 +319,6 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
{
struct thread_data *td = cb_ctx;
struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_nvme_io_qpair_opts qpopts;
struct spdk_fio_ctrlr *fio_ctrlr;
struct spdk_fio_qpair *fio_qpair;
struct spdk_nvme_ns *ns;
@ -398,25 +397,7 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
return;
}
spdk_nvme_ctrlr_get_default_io_qpair_opts(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
qpopts.delay_cmd_submit = true;
if (fio_options->enable_wrr) {
qpopts.qprio = fio_options->wrr_priority;
}
fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
if (!fio_qpair->qpair) {
SPDK_ERRLOG("Cannot allocate nvme io_qpair any more\n");
g_error = true;
free(fio_qpair);
return;
}
if (fio_options->print_qid_mappings == 1) {
log_info("job %s: %s qid %d\n", td->o.name, f->file_name,
spdk_nvme_qpair_get_id(fio_qpair->qpair));
}
f->engine_data = fio_qpair;
fio_qpair->ns = ns;
fio_qpair->f = f;
fio_qpair->fio_ctrlr = fio_ctrlr;
@ -677,6 +658,30 @@ static int spdk_fio_setup(struct thread_data *td)
static int spdk_fio_open(struct thread_data *td, struct fio_file *f)
{
struct spdk_fio_qpair *fio_qpair = f->engine_data;
struct spdk_fio_ctrlr *fio_ctrlr = fio_qpair->fio_ctrlr;
struct spdk_fio_options *fio_options = td->eo;
struct spdk_nvme_io_qpair_opts qpopts;
spdk_nvme_ctrlr_get_default_io_qpair_opts(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
qpopts.delay_cmd_submit = true;
if (fio_options->enable_wrr) {
qpopts.qprio = fio_options->wrr_priority;
}
fio_qpair->qpair = spdk_nvme_ctrlr_alloc_io_qpair(fio_ctrlr->ctrlr, &qpopts, sizeof(qpopts));
if (!fio_qpair->qpair) {
SPDK_ERRLOG("Cannot allocate nvme io_qpair any more\n");
g_error = true;
free(fio_qpair);
return -1;
}
if (fio_options->print_qid_mappings == 1) {
log_info("job %s: %s qid %d\n", td->o.name, f->file_name,
spdk_nvme_qpair_get_id(fio_qpair->qpair));
}
return 0;
}