nvmf: Add nvmf_set_crdt RPC

Set the three CRDT values at SPDK_RPC_STARTUP time.

Signed-off-by: Peng Yu <yupeng0921@gmail.com>
Change-Id: I2fb4c4a3e367a4888cfec4658e6bf6899c7ae1f4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8007
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

CHANGELOG.md

@@ -25,6 +25,8 @@ Added `min_cntlid` and `max_cntlid` to `nvmf_create_subsystem` to limit the cont
 
 `spdk_nvmf_request_get_buffers_multi` API is removed.
 
+Added the `nvmf_set_crdt` RPC for setting command retry delay times.
+
 ### nvme
 
 Added a new function `spdk_nvme_ns_cmd_copy` to submit a Simple Copy Command to a Namespace.

doc/jsonrpc.md

@@ -6840,6 +6840,22 @@ Example response:
 }
 ~~~
 
+
+## nvmf_set_crdt {#rpc_nvmf_set_crdt}
+
+Set the 3 CRDT (Command Retry Delay Time) values. For details about
+CRDT, please refer to the NVMe specification. Currently all the
+SPDK nvmf subsystems share the same CRDT values. The default values
+are 0. This rpc can only be invoked in STARTUP stage. All values are
+in units of 100 milliseconds (same as the NVMe specification).
+
+### Parameters
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+crdt1                   | Optional | number      | Command Retry Delay Time 1
+crdt2                   | Optional | number      | Command Retry Delay Time 2
+crdt3                   | Optional | number      | Command Retry Delay Time 3
+
 # Vhost Target {#jsonrpc_components_vhost_tgt}
 
 The following common preconditions need to be met in all target types.

include/spdk/nvmf.h

@@ -70,6 +70,7 @@ struct spdk_nvmf_target_opts {
 	char		name[NVMF_TGT_NAME_MAX_LENGTH];
 	uint32_t	max_subsystems;
 	uint32_t	acceptor_poll_rate;
+	uint16_t	crdt[3];
 };
 
 struct spdk_nvmf_transport_opts {

lib/nvmf/ctrlr.c

@@ -88,6 +88,7 @@ nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp,
 #define SPDK_NVMF_INVALID_CONNECT_DATA(rsp, field)	\
 	nvmf_invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field))
 
+
 static void
 nvmf_ctrlr_stop_keep_alive_timer(struct spdk_nvmf_ctrlr *ctrlr)
 {
@@ -2338,13 +2339,10 @@ spdk_nvmf_ctrlr_identify_ctrlr(struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvme_c
 
 		nvmf_ctrlr_populate_oacs(ctrlr, cdata);
 
-		/*
-		 * FIXME: Set all crdt to 0 currently,
-		 * will provide an API to customize them later.
-		 */
-		cdata->crdt[0] = 0;
-		cdata->crdt[1] = 0;
-		cdata->crdt[2] = 0;
+		assert(subsystem->tgt != NULL);
+		cdata->crdt[0] = subsystem->tgt->crdt[0];
+		cdata->crdt[1] = subsystem->tgt->crdt[1];
+		cdata->crdt[2] = subsystem->tgt->crdt[2];
 
 		SPDK_DEBUGLOG(nvmf, "ext ctrlr data: ioccsz 0x%x\n",
 			      cdata->nvmf_specific.ioccsz);

lib/nvmf/nvmf.c

@@ -283,6 +283,16 @@ spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts)
 		acceptor_poll_rate = opts->acceptor_poll_rate;
 	}
 
+	if (!opts) {
+		tgt->crdt[0] = 0;
+		tgt->crdt[1] = 0;
+		tgt->crdt[2] = 0;
+	} else {
+		tgt->crdt[0] = opts->crdt[0];
+		tgt->crdt[1] = opts->crdt[1];
+		tgt->crdt[2] = opts->crdt[2];
+	}
+
 	tgt->discovery_genctr = 0;
 	TAILQ_INIT(&tgt->transports);
 	TAILQ_INIT(&tgt->poll_groups);
@@ -572,6 +582,15 @@ spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_
 
 	spdk_json_write_object_end(w);
 
+	spdk_json_write_object_begin(w);
+	spdk_json_write_named_string(w, "method", "nvmf_set_crdt");
+	spdk_json_write_named_object_begin(w, "params");
+	spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]);
+	spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]);
+	spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]);
+	spdk_json_write_object_end(w);
+	spdk_json_write_object_end(w);
+
 	/* write transports */
 	TAILQ_FOREACH(transport, &tgt->transports, link) {
 		spdk_json_write_object_begin(w);

lib/nvmf/nvmf_internal.h

@@ -87,6 +87,8 @@ struct spdk_nvmf_tgt {
 	spdk_nvmf_tgt_destroy_done_fn		*destroy_cb_fn;
 	void					*destroy_cb_arg;
 
+	uint16_t				crdt[3];
+
 	TAILQ_ENTRY(spdk_nvmf_tgt)		link;
 };
 

module/event/subsystems/nvmf/event_nvmf.h

@@ -57,6 +57,7 @@ struct spdk_nvmf_tgt_conf {
 extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;
 
 extern uint32_t g_spdk_nvmf_tgt_max_subsystems;
+extern uint16_t g_spdk_nvmf_tgt_crdt[3];
 
 extern struct spdk_nvmf_tgt *g_spdk_nvmf_tgt;
 

module/event/subsystems/nvmf/nvmf_rpc.c

@@ -128,3 +128,43 @@ rpc_nvmf_set_config(struct spdk_jsonrpc_request *request,
 }
 SPDK_RPC_REGISTER("nvmf_set_config", rpc_nvmf_set_config, SPDK_RPC_STARTUP)
 SPDK_RPC_REGISTER_ALIAS_DEPRECATED(nvmf_set_config, set_nvmf_target_config)
+
+struct nvmf_rpc_set_crdt {
+	uint16_t crdt1;
+	uint16_t crdt2;
+	uint16_t crdt3;
+};
+
+static const struct spdk_json_object_decoder rpc_set_crdt_opts_decoders[] = {
+	{"crdt1", offsetof(struct nvmf_rpc_set_crdt, crdt1), spdk_json_decode_uint16, true},
+	{"crdt2", offsetof(struct nvmf_rpc_set_crdt, crdt2), spdk_json_decode_uint16, true},
+	{"crdt3", offsetof(struct nvmf_rpc_set_crdt, crdt3), spdk_json_decode_uint16, true},
+};
+
+static void
+rpc_nvmf_set_crdt(struct spdk_jsonrpc_request *request,
+		  const struct spdk_json_val *params)
+{
+	struct nvmf_rpc_set_crdt rpc_set_crdt;
+
+	rpc_set_crdt.crdt1 = 0;
+	rpc_set_crdt.crdt2 = 0;
+	rpc_set_crdt.crdt3 = 0;
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, rpc_set_crdt_opts_decoders,
+					    SPDK_COUNTOF(rpc_set_crdt_opts_decoders), &rpc_set_crdt)) {
+			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			return;
+		}
+	}
+
+	g_spdk_nvmf_tgt_crdt[0] = rpc_set_crdt.crdt1;
+	g_spdk_nvmf_tgt_crdt[1] = rpc_set_crdt.crdt2;
+	g_spdk_nvmf_tgt_crdt[2] = rpc_set_crdt.crdt3;
+
+	spdk_jsonrpc_send_bool_response(request, true);
+}
+SPDK_RPC_REGISTER("nvmf_set_crdt", rpc_nvmf_set_crdt, SPDK_RPC_STARTUP)

module/event/subsystems/nvmf/nvmf_tgt.c

@@ -65,6 +65,7 @@ struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf = {
 
 struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL;
 uint32_t g_spdk_nvmf_tgt_max_subsystems = 0;
+uint16_t g_spdk_nvmf_tgt_crdt[3] = {0, 0, 0};
 
 static enum nvmf_tgt_state g_tgt_state;
 
@@ -277,6 +278,9 @@ nvmf_tgt_create_target(void)
 
 	opts.max_subsystems = g_spdk_nvmf_tgt_max_subsystems;
 	opts.acceptor_poll_rate = g_spdk_nvmf_tgt_conf.acceptor_poll_rate;
+	opts.crdt[0] = g_spdk_nvmf_tgt_crdt[0];
+	opts.crdt[1] = g_spdk_nvmf_tgt_crdt[1];
+	opts.crdt[2] = g_spdk_nvmf_tgt_crdt[2];
 	g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(&opts);
 	if (!g_spdk_nvmf_tgt) {
 		SPDK_ERRLOG("spdk_nvmf_tgt_create() failed\n");

scripts/rpc.py

@@ -2155,6 +2155,18 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
     p.add_argument('-t', '--tgt_name', help='The name of the parent NVMe-oF target (optional)', type=str)
     p.set_defaults(func=nvmf_get_stats)
 
+    def nvmf_set_crdt(args):
+        print_dict(rpc.nvmf.nvmf_set_crdt(args.client, args.crdt1, args.crdt2, args.crdt3))
+
+    p = subparsers.add_parser(
+        'nvmf_set_crdt',
+        help="""Set the 3 crdt (Command Retry Delay Time) values for NVMf subsystem. All
+        values are in units of 100 milliseconds (same as the NVM Express specification).""")
+    p.add_argument('-t1', '--crdt1', help='Command Retry Delay Time 1, in units of 100 milliseconds', type=int)
+    p.add_argument('-t2', '--crdt2', help='Command Retry Delay Time 2, in units of 100 milliseconds', type=int)
+    p.add_argument('-t3', '--crdt3', help='Command Retry Delay Time 3, in units of 100 milliseconds', type=int)
+    p.set_defaults(func=nvmf_set_crdt)
+
     # pmem
     def bdev_pmem_create_pool(args):
         num_blocks = int((args.total_size * 1024 * 1024) / args.block_size)

scripts/rpc/nvmf.py

@@ -540,3 +540,25 @@ def nvmf_get_stats(client, tgt_name=None):
         }
 
     return client.call('nvmf_get_stats', params)
+
+
+def nvmf_set_crdt(client, crdt1=None, crdt2=None, crdt3=None):
+    """Set the 3 crdt (Command Retry Delay Time) values
+
+    Args:
+        crdt1: Command Retry Delay Time 1
+        crdt2: Command Retry Delay Time 2
+        crdt3: Command Retry Delay Time 3
+
+    Returns:
+        True or False
+    """
+    params = {}
+    if crdt1 is not None:
+        params['crdt1'] = crdt1
+    if crdt2 is not None:
+        params['crdt2'] = crdt2
+    if crdt3 is not None:
+        params['crdt3'] = crdt3
+
+    return client.call('nvmf_set_crdt', params)

test/json_config/config_filter.py

@@ -28,6 +28,7 @@ def filter_methods(do_remove_global_rpcs):
         'nvmf_set_config',
         'nvmf_set_max_subsystems',
         'nvmf_create_transport',
+        'nvmf_set_crdt',
         'bdev_set_options',
         'bdev_wait_for_examine',
         'bdev_nvme_set_options',

test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c

@@ -1499,8 +1499,10 @@ test_get_dif_ctx(void)
 static void
 test_identify_ctrlr(void)
 {
+	struct spdk_nvmf_tgt tgt = {};
 	struct spdk_nvmf_subsystem subsystem = {
-		.subtype = SPDK_NVMF_SUBTYPE_NVME
+		.subtype = SPDK_NVMF_SUBTYPE_NVME,
+		.tgt = &tgt,
 	};
 	struct spdk_nvmf_transport_ops tops = {};
 	struct spdk_nvmf_transport transport = {