diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 559ed4b014..8a59ce26ad 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -6035,6 +6035,7 @@ abort_timeout_sec | Optional | number | Abort execution timeout value no_wr_batching | Optional | boolean | Disable work requests batching (RDMA only) control_msg_num | Optional | number | The number of control messages per poll group (TCP only) disable_mappable_bar0 | Optional | boolean | disable client mmap() of BAR0 (VFIO-USER only) +zcopy | Optional | boolean | Use zero-copy operations if the underlying bdev supports them #### Example diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index 13e9f13081..1d7e3e9888 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -114,6 +114,8 @@ struct spdk_nvmf_transport_opts { */ size_t opts_size; uint32_t acceptor_poll_rate; + /* Use zero-copy operations if the underlying bdev supports them */ + bool zcopy; }; struct spdk_nvmf_listen_opts { diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index c121666351..5e251fbfd2 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -3695,10 +3695,15 @@ nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev bool nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req) { + struct spdk_nvmf_transport *transport = req->qpair->transport; struct spdk_nvmf_ns *ns; req->zcopy_phase = NVMF_ZCOPY_PHASE_NONE; + if (!transport->opts.zcopy) { + return false; + } + if (nvmf_qpair_is_admin_queue(req->qpair)) { /* Admin queue */ return false; diff --git a/lib/nvmf/nvmf_rpc.c b/lib/nvmf/nvmf_rpc.c index a9d6a3b87d..5a6bc80fbd 100644 --- a/lib/nvmf/nvmf_rpc.c +++ b/lib/nvmf/nvmf_rpc.c @@ -1928,6 +1928,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] "abort_timeout_sec", offsetof(struct nvmf_rpc_create_transport_ctx, opts.abort_timeout_sec), spdk_json_decode_uint32, true }, + { + "zcopy", offsetof(struct nvmf_rpc_create_transport_ctx, opts.zcopy), + spdk_json_decode_bool, true + }, { "tgt_name", offsetof(struct nvmf_rpc_create_transport_ctx, tgt_name), spdk_json_decode_string, true diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index ab7e6a7739..f46e2bddfb 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -114,6 +114,7 @@ nvmf_transport_dump_opts(struct spdk_nvmf_transport *transport, struct spdk_json spdk_json_write_named_uint32(w, "num_shared_buffers", opts->num_shared_buffers); spdk_json_write_named_uint32(w, "buf_cache_size", opts->buf_cache_size); spdk_json_write_named_bool(w, "dif_insert_or_strip", opts->dif_insert_or_strip); + spdk_json_write_named_bool(w, "zcopy", opts->zcopy); if (transport->ops->dump_opts) { transport->ops->dump_opts(transport, w); @@ -182,6 +183,7 @@ static void nvmf_transport_opts_copy(struct spdk_nvmf_transport_opts *opts, SET_FIELD(association_timeout); SET_FIELD(transport_specific); SET_FIELD(acceptor_poll_rate); + SET_FIELD(zcopy); /* Do not remove this statement, you should always update this statement when you adding a new field, * and do not forget to add the SET_FIELD statement for your added field. */ diff --git a/scripts/rpc.py b/scripts/rpc.py index cce7a27738..40b2839eb8 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1950,6 +1950,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse p.add_argument('-a', '--max-aq-depth', help='Max number of admin cmds per AQ', type=int) p.add_argument('-n', '--num-shared-buffers', help='The number of pooled data buffers available to the transport', type=int) p.add_argument('-b', '--buf-cache-size', help='The number of shared buffers to reserve for each poll group', type=int) + p.add_argument('-z', '--zcopy', action='store_true', help='''Use zero-copy operations if the + underlying bdev supports them''') p.add_argument('-d', '--num-cqe', help="""The number of CQ entires. Only used when no_srq=true. Relevant only for RDMA transport""", type=int) p.add_argument('-s', '--max-srq-depth', help='Max number of outstanding I/O per SRQ. Relevant only for RDMA transport', type=int) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index 1601fd85f7..ba436ec3eb 100644 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -114,6 +114,7 @@ def nvmf_create_transport(client, **params): max_aq_depth: Max size admin queue per controller (optional) num_shared_buffers: The number of pooled data buffers available to the transport (optional) buf_cache_size: The number of shared buffers to reserve for each poll group (optional) + zcopy: Use zero-copy operations if the underlying bdev supports them (optional) num_cqe: The number of CQ entries to configure CQ size. Only used when no_srq=true - RDMA specific (optional) max_srq_depth: Max number of outstanding I/O per shared receive queue - RDMA specific (optional) no_srq: Boolean flag to disable SRQ even for devices that support it - RDMA specific (optional) diff --git a/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c b/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c index 16a15b14b0..4518522b31 100644 --- a/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c +++ b/test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c @@ -2291,6 +2291,7 @@ static void test_nvmf_ctrlr_use_zcopy(void) { struct spdk_nvmf_subsystem subsystem = {}; + struct spdk_nvmf_transport transport = {}; struct spdk_nvmf_request req = {}; struct spdk_nvmf_qpair qpair = {}; struct spdk_nvmf_ctrlr ctrlr = {}; @@ -2314,10 +2315,13 @@ test_nvmf_ctrlr_use_zcopy(void) ctrlr.subsys = &subsystem; + transport.opts.zcopy = true; + qpair.ctrlr = &ctrlr; qpair.group = &group; qpair.qid = 1; qpair.state = SPDK_NVMF_QPAIR_ACTIVE; + qpair.transport = &transport; group.thread = spdk_get_thread(); group.num_sgroups = 1; @@ -2360,9 +2364,14 @@ test_nvmf_ctrlr_use_zcopy(void) /* ZCOPY Not supported */ CU_ASSERT(nvmf_ctrlr_use_zcopy(&req) == false); + ns.zcopy = true; + + /* ZCOPY disabled on transport level */ + transport.opts.zcopy = false; + CU_ASSERT(nvmf_ctrlr_use_zcopy(&req) == false); + transport.opts.zcopy = true; /* Success */ - ns.zcopy = true; CU_ASSERT(nvmf_ctrlr_use_zcopy(&req)); } @@ -2371,6 +2380,7 @@ test_spdk_nvmf_request_zcopy_start(void) { struct spdk_nvmf_request req = {}; struct spdk_nvmf_qpair qpair = {}; + struct spdk_nvmf_transport transport = {}; struct spdk_nvme_cmd cmd = {}; union nvmf_c2h_msg rsp = {}; struct spdk_nvmf_ctrlr ctrlr = {}; @@ -2402,6 +2412,8 @@ test_spdk_nvmf_request_zcopy_start(void) ctrlr.subsys = (struct spdk_nvmf_subsystem *)&subsystem; ctrlr.listener = &listener; + transport.opts.zcopy = true; + group.thread = spdk_get_thread(); group.num_sgroups = 1; sgroups.state = SPDK_NVMF_SUBSYSTEM_ACTIVE; @@ -2415,6 +2427,7 @@ test_spdk_nvmf_request_zcopy_start(void) qpair.ctrlr = &ctrlr; qpair.group = &group; + qpair.transport = &transport; qpair.qid = 1; qpair.state = SPDK_NVMF_QPAIR_ACTIVE; @@ -2496,6 +2509,7 @@ test_zcopy_read(void) { struct spdk_nvmf_request req = {}; struct spdk_nvmf_qpair qpair = {}; + struct spdk_nvmf_transport transport = {}; struct spdk_nvme_cmd cmd = {}; union nvmf_c2h_msg rsp = {}; struct spdk_nvmf_ctrlr ctrlr = {}; @@ -2527,6 +2541,8 @@ test_zcopy_read(void) ctrlr.subsys = (struct spdk_nvmf_subsystem *)&subsystem; ctrlr.listener = &listener; + transport.opts.zcopy = true; + group.thread = spdk_get_thread(); group.num_sgroups = 1; sgroups.state = SPDK_NVMF_SUBSYSTEM_ACTIVE; @@ -2540,6 +2556,7 @@ test_zcopy_read(void) qpair.ctrlr = &ctrlr; qpair.group = &group; + qpair.transport = &transport; qpair.qid = 1; qpair.state = SPDK_NVMF_QPAIR_ACTIVE; @@ -2583,6 +2600,7 @@ test_zcopy_write(void) { struct spdk_nvmf_request req = {}; struct spdk_nvmf_qpair qpair = {}; + struct spdk_nvmf_transport transport = {}; struct spdk_nvme_cmd cmd = {}; union nvmf_c2h_msg rsp = {}; struct spdk_nvmf_ctrlr ctrlr = {}; @@ -2614,6 +2632,8 @@ test_zcopy_write(void) ctrlr.subsys = (struct spdk_nvmf_subsystem *)&subsystem; ctrlr.listener = &listener; + transport.opts.zcopy = true; + group.thread = spdk_get_thread(); group.num_sgroups = 1; sgroups.state = SPDK_NVMF_SUBSYSTEM_ACTIVE; @@ -2627,6 +2647,7 @@ test_zcopy_write(void) qpair.ctrlr = &ctrlr; qpair.group = &group; + qpair.transport = &transport; qpair.qid = 1; qpair.state = SPDK_NVMF_QPAIR_ACTIVE;