d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

examples/nvme_fio_plugin: add support for zone append

Browse Source 
Now when we have support for spdk_nvme_zns_zone_append() and
spdk_nvme_zns_zone_appendv(), hook them up in the nvme fio plugin.

Note that fio itself does not have support for zone append,
since unlike SPDK, there is no user facing zone append API in
Linux. Therefore, this new option simply replaces writes with
zone appends in the SPDK fio backend.

This is however still useful for the following reasons:
-Provides a way to test zone append in SPDK.
-By using zone append, we can test with iodepth > 1.

With regular writes, the user can only specify iodepth=1.
This is because for zone namespaces, writes have to target
the write pointer. Having more than one write in flight, per
zone, will lead to I/O errors.

In Linux, it is possible to use fio with iodepth > 1
on zoned namespaces, simply because of the mq-deadline
scheduler, which throttles writes such that there is only
one write in flight, per zone, even if user space has
queued up more.

Since a user might not want to use zone append unconditionally,
even on a namespace that supports it, make this an option
rather than enabling it unconditionally.

Signed-off-by: Niklas Cassel <niklas.cassel@wdc.com>
Change-Id: I028b79f6445bc63b68c97d1370c6f8139779666d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6330
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Niklas Cassel 2021-02-08 13:39:31 +00:00 committed by Tomasz Zawadzki
parent 014baeb8ef
commit 17ef8ec9ac
2 changed files with 65 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
examples/nvme/fio_plugin/README.md
View File

@ -139,6 +139,17 @@ then you can reset all zones before fio start running its jobs by using the engi
--initial_zone_reset=1
## Zone Append
When running FIO against a Zoned Namespace you need to specify --iodepth=1 to avoid
"Zone Invalid Write: The write to a zone was not at the write pointer." I/O errors.
However, if your controller supports Zone Append, you can use the engine option:
--zone_append=1
To send zone append commands instead of write commands to the controller.
When using zone append, you will be able to specify a --iodepth greater than 1.
## Shared Memory Increase
If your device has a lot of zones, fio can give you errors such as:

62
examples/nvme/fio_plugin/fio_plugin.c
View File

@ -93,6 +93,7 @@ struct spdk_fio_options {
char *digest_enable;
int enable_vmd;
int initial_zone_reset;
int zone_append;
};
struct spdk_fio_request {
@ -130,6 +131,7 @@ struct spdk_fio_qpair {
struct spdk_nvme_qpair *qpair;
struct spdk_nvme_ns *ns;
uint32_t io_flags;
bool zone_append_enabled;
bool nvme_pi_enabled;
/* True for DIF and false for DIX, and this is valid only if nvme_pi_enabled is true. */
bool extended_lba;
@ -408,6 +410,18 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
return;
}
if (fio_options->zone_append) {
if (spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&
spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED) {
fprintf(stdout, "Using zone append instead of write\n");
fio_qpair->zone_append_enabled = true;
} else {
SPDK_ERRLOG("zone_append=1 requested, but namespace lacks support\n");
g_error = true;
return;
}
}
f->real_file_size = spdk_nvme_ns_get_size(fio_qpair->ns);
if (f->real_file_size <= 0) {
g_error = true;
@ -688,6 +702,12 @@ static void spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u)
}
}
static inline uint64_t
fio_offset_to_zslba(unsigned long long offset, struct spdk_nvme_ns *ns)
{
return (offset / spdk_nvme_zns_ns_get_zone_size(ns)) * spdk_nvme_zns_ns_get_zone_size_sectors(ns);
}
static int
fio_extended_lba_setup_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)
{
@ -982,15 +1002,31 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
break;
case DDIR_WRITE:
if (!g_spdk_enable_sgl) {
rc = spdk_nvme_ns_cmd_write_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, lba,
lba_count,
spdk_fio_completion_cb, fio_req,
fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
if (!fio_qpair->zone_append_enabled) {
rc = spdk_nvme_ns_cmd_write_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, lba,
lba_count,
spdk_fio_completion_cb, fio_req,
fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
} else {
uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);
rc = spdk_nvme_zns_zone_append_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, zslba,
lba_count,
spdk_fio_completion_cb, fio_req,
fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);
}
} else {
rc = spdk_nvme_ns_cmd_writev_with_md(ns, fio_qpair->qpair, lba,
lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
dif_ctx->apptag_mask, dif_ctx->app_tag);
if (!fio_qpair->zone_append_enabled) {
rc = spdk_nvme_ns_cmd_writev_with_md(ns, fio_qpair->qpair, lba,
lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
dif_ctx->apptag_mask, dif_ctx->app_tag);
} else {
uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);
rc = spdk_nvme_zns_zone_appendv_with_md(ns, fio_qpair->qpair, zslba,
lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,
spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,
dif_ctx->apptag_mask, dif_ctx->app_tag);
}
}
break;
default:
@ -1545,6 +1581,16 @@ static struct fio_option options[] = {
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_INVALID,
},
{
.name = "zone_append",
.lname = "Use zone append instead of write",
.type = FIO_OPT_INT,
.off1 = offsetof(struct spdk_fio_options, zone_append),
.def = "0",
.help = "Use zone append instead of write (zone_append=1 or zone_append=0)",
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_INVALID,
},
{
.name = NULL,
},