From 94345a0a1a00ceaa63aca5d592841146c48dd9ae Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Wed, 19 Feb 2020 19:18:51 +0800 Subject: [PATCH] nvme: Add the priority field in struct spdk_nvme_transport_id Purpose: To set the priority of the NVMe-oF connection especially for TCP connection. For example, the previous example can be: trtype:TCP adrfam:IPv4 traddr:10.67.110.181 trsvcid:4420 With the change, it could be: trtype:TCP adrfam:IPv4 traddr:10.67.110.181 trsvcid:4420 priority:2 The priority is optional. We try to change spdk_nvme_transport_id but not in spdk_nvme_ctrlr_opts since the opts in spdk_nvme_ctrlr_opts will reflect in every nvme ctrlr, this is short of flexibility. Signed-off-by: Ziye Yang Signed-off-by: Sudheer Mogilappagari Change-Id: I1ba364c714a95f2dbeab2b3fcc832b0222b48a15 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1875 Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- CHANGELOG.md | 3 +++ include/spdk/nvme.h | 9 +++++++++ lib/nvme/nvme.c | 10 ++++++++++ lib/nvme/nvme_fabric.c | 8 ++++++-- lib/nvme/nvme_tcp.c | 6 +++++- lib/nvmf/tcp.c | 7 ++++++- module/bdev/nvme/bdev_nvme_rpc.c | 8 ++++++++ scripts/rpc.py | 3 +++ scripts/rpc/bdev.py | 8 ++++++-- test/common/lib/test_sock.c | 5 +++++ test/unit/lib/nvme/nvme.c/nvme_ut.c | 9 +++++++++ test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c | 3 +++ 12 files changed, 73 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8185b72ae..5a24c77759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ New version of OCF comes with API changes and bug fixes Export internal nvme_ctrlr_cmd_security_receive/send() APIs as public APIs with "spdk_" prefix. +Added `priority` field in `spdk_nvme_transport_id`, this field is used to specify the priority +of the NVMe-oF connection, and currently it is used for NVMe-oF tcp connection. + ### copy The copy engine library, modules and public APIs have been renamed. Use of the word `copy` diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index e66a8ddf03..607ecbbc04 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -53,6 +53,8 @@ extern "C" { #define SPDK_NVME_TRANSPORT_NAME_RDMA "RDMA" #define SPDK_NVME_TRANSPORT_NAME_TCP "TCP" +#define SPDK_NVMF_PRIORITY_MAX_LEN 4 + /** * Opaque handle to a controller. Returned by spdk_nvme_probe()'s attach_cb. */ @@ -360,6 +362,13 @@ struct spdk_nvme_transport_id { * Subsystem NQN of the NVMe over Fabrics endpoint. May be a zero length string. */ char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1]; + + /** + * The Transport connection priority of the NVMe-oF endpoint. Currently this is + * only supported by posix based sock implementation on Kernel TCP stack. More + * information of this field can be found from the socket(7) man page. + */ + int priority; }; /** diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index 294f840fdb..e1d9ba1979 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -32,6 +32,7 @@ */ #include "spdk/nvmf_spec.h" +#include "spdk/string.h" #include "nvme_internal.h" #include "nvme_io_msg.h" @@ -975,6 +976,13 @@ spdk_nvme_transport_id_parse(struct spdk_nvme_transport_id *trid, const char *st return -EINVAL; } memcpy(trid->trsvcid, val, val_len + 1); + } else if (strcasecmp(key, "priority") == 0) { + if (val_len > SPDK_NVMF_PRIORITY_MAX_LEN) { + SPDK_ERRLOG("priority length %zu greater than maximum allowed %u\n", + val_len, SPDK_NVMF_PRIORITY_MAX_LEN); + return -EINVAL; + } + trid->priority = spdk_strtol(val, 10); } else if (strcasecmp(key, "subnqn") == 0) { if (val_len > SPDK_NVMF_NQN_MAX_LEN) { SPDK_ERRLOG("subnqn length %zu greater than maximum allowed %u\n", @@ -1046,6 +1054,8 @@ spdk_nvme_host_id_parse(struct spdk_nvme_host_id *hostid, const char *str) continue; } else if (strcasecmp(key, "subnqn") == 0) { continue; + } else if (strcasecmp(key, "priority") == 0) { + continue; } else if (strcasecmp(key, "ns") == 0) { continue; } else if (strcasecmp(key, "hostaddr") == 0) { diff --git a/lib/nvme/nvme_fabric.c b/lib/nvme/nvme_fabric.c index d547dc714b..61b0c9caef 100644 --- a/lib/nvme/nvme_fabric.c +++ b/lib/nvme/nvme_fabric.c @@ -166,7 +166,8 @@ nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint static void nvme_fabric_discover_probe(struct spdk_nvmf_discovery_log_page_entry *entry, - struct spdk_nvme_probe_ctx *probe_ctx) + struct spdk_nvme_probe_ctx *probe_ctx, + int discover_priority) { struct spdk_nvme_transport_id trid; uint8_t *end; @@ -221,6 +222,9 @@ nvme_fabric_discover_probe(struct spdk_nvmf_discovery_log_page_entry *entry, trid.subnqn, trid.trtype, trid.traddr, trid.trsvcid); + /* Copy the priority from the discovery ctrlr */ + trid.priority = discover_priority; + nvme_ctrlr_probe(&trid, probe_ctx, NULL); } @@ -378,7 +382,7 @@ nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr, } for (i = 0; i < numrec; i++) { - nvme_fabric_discover_probe(log_page_entry++, probe_ctx); + nvme_fabric_discover_probe(log_page_entry++, probe_ctx, ctrlr->trid.priority); } remaining_num_rec -= numrec; log_page_offset += numrec * sizeof(struct spdk_nvmf_discovery_log_page_entry); diff --git a/lib/nvme/nvme_tcp.c b/lib/nvme/nvme_tcp.c index 053f2caec4..85871520d2 100644 --- a/lib/nvme/nvme_tcp.c +++ b/lib/nvme/nvme_tcp.c @@ -1487,6 +1487,7 @@ nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpa struct nvme_tcp_qpair *tqpair; int family; long int port; + struct spdk_sock_opts opts; tqpair = nvme_tcp_qpair(qpair); @@ -1528,7 +1529,10 @@ nvme_tcp_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpa return -1; } - tqpair->sock = spdk_sock_connect(ctrlr->trid.traddr, port, NULL); + opts.opts_size = sizeof(opts); + spdk_sock_get_default_opts(&opts); + opts.priority = ctrlr->trid.priority; + tqpair->sock = spdk_sock_connect_ext(ctrlr->trid.traddr, port, NULL, &opts); if (!tqpair->sock) { SPDK_ERRLOG("sock connection error of tqpair=%p with addr=%s, port=%ld\n", tqpair, ctrlr->trid.traddr, port); diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index cac1ac6fc6..e66ad15c66 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -594,6 +594,7 @@ spdk_nvmf_tcp_listen(struct spdk_nvmf_transport *transport, struct spdk_nvmf_tcp_port *port; int trsvcid_int; uint8_t adrfam; + struct spdk_sock_opts opts; ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); @@ -612,7 +613,11 @@ spdk_nvmf_tcp_listen(struct spdk_nvmf_transport *transport, } port->trid = trid; - port->listen_sock = spdk_sock_listen(trid->traddr, trsvcid_int, NULL); + opts.opts_size = sizeof(opts); + spdk_sock_get_default_opts(&opts); + opts.priority = transport->opts.sock_priority; + port->listen_sock = spdk_sock_listen_ext(trid->traddr, trsvcid_int, + NULL, &opts); if (port->listen_sock == NULL) { SPDK_ERRLOG("spdk_sock_listen(%s, %d) failed: %s (%d)\n", trid->traddr, trsvcid_int, diff --git a/module/bdev/nvme/bdev_nvme_rpc.c b/module/bdev/nvme/bdev_nvme_rpc.c index ab82409625..1cda88647a 100644 --- a/module/bdev/nvme/bdev_nvme_rpc.c +++ b/module/bdev/nvme/bdev_nvme_rpc.c @@ -173,6 +173,7 @@ struct rpc_bdev_nvme_attach_controller { char *adrfam; char *traddr; char *trsvcid; + char *priority; char *subnqn; char *hostnqn; char *hostaddr; @@ -189,6 +190,7 @@ free_rpc_bdev_nvme_attach_controller(struct rpc_bdev_nvme_attach_controller *req free(req->adrfam); free(req->traddr); free(req->trsvcid); + free(req->priority); free(req->subnqn); free(req->hostnqn); free(req->hostaddr); @@ -202,6 +204,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_attach_controller_dec {"adrfam", offsetof(struct rpc_bdev_nvme_attach_controller, adrfam), spdk_json_decode_string, true}, {"trsvcid", offsetof(struct rpc_bdev_nvme_attach_controller, trsvcid), spdk_json_decode_string, true}, + {"priority", offsetof(struct rpc_bdev_nvme_attach_controller, priority), spdk_json_decode_string, true}, {"subnqn", offsetof(struct rpc_bdev_nvme_attach_controller, subnqn), spdk_json_decode_string, true}, {"hostnqn", offsetof(struct rpc_bdev_nvme_attach_controller, hostnqn), spdk_json_decode_string, true}, {"hostaddr", offsetof(struct rpc_bdev_nvme_attach_controller, hostaddr), spdk_json_decode_string, true}, @@ -303,6 +306,11 @@ spdk_rpc_bdev_nvme_attach_controller(struct spdk_jsonrpc_request *request, snprintf(trid.trsvcid, sizeof(trid.trsvcid), "%s", ctx->req.trsvcid); } + /* Parse priority for the NVMe-oF transport connection */ + if (ctx->req.priority) { + trid.priority = spdk_strtol(ctx->req.priority, 10); + } + /* Parse subnqn */ if (ctx->req.subnqn) { snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", ctx->req.subnqn); diff --git a/scripts/rpc.py b/scripts/rpc.py index 33f7bc5340..1bbed0ab8d 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -425,6 +425,7 @@ if __name__ == "__main__": traddr=args.traddr, adrfam=args.adrfam, trsvcid=args.trsvcid, + priority=args.priority, subnqn=args.subnqn, hostnqn=args.hostnqn, hostaddr=args.hostaddr, @@ -443,6 +444,8 @@ if __name__ == "__main__": help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host') p.add_argument('-s', '--trsvcid', help='NVMe-oF target trsvcid: e.g., a port number') + p.add_argument('-p', '--priority', + help='NVMe-oF connection priority: e.g., a priority number') p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn') p.add_argument('-q', '--hostnqn', help='NVMe-oF host subnqn') p.add_argument('-i', '--hostaddr', diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index d75ddee645..d193a50cb0 100644 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -432,8 +432,8 @@ def bdev_nvme_set_hotplug(client, enable, period_us=None): @deprecated_alias('construct_nvme_bdev') def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvcid=None, - subnqn=None, hostnqn=None, hostaddr=None, hostsvcid=None, - prchk_reftag=None, prchk_guard=None): + priority=None, subnqn=None, hostnqn=None, hostaddr=None, + hostsvcid=None, prchk_reftag=None, prchk_guard=None): """Construct block device for each NVMe namespace in the attached controller. Args: @@ -442,6 +442,7 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc traddr: transport address (PCI BDF or IP address) adrfam: address family ("IPv4", "IPv6", "IB", or "FC") (optional for PCIe) trsvcid: transport service ID (port number for IP-based addresses; optional for PCIe) + priority: transport connection priority (Sock priority for TCP-based transports; optional) subnqn: subsystem NQN to connect to (optional) hostnqn: NQN to connect from (optional) hostaddr: host transport address (IP address for IP-based transports, NULL for PCIe or FC; optional) @@ -471,6 +472,9 @@ def bdev_nvme_attach_controller(client, name, trtype, traddr, adrfam=None, trsvc if trsvcid: params['trsvcid'] = trsvcid + if priority: + params['priority'] = priority + if subnqn: params['subnqn'] = subnqn diff --git a/test/common/lib/test_sock.c b/test/common/lib/test_sock.c index 8b0b35818b..d2c83b7320 100644 --- a/test/common/lib/test_sock.c +++ b/test/common/lib/test_sock.c @@ -40,8 +40,13 @@ DEFINE_STUB(spdk_sock_getaddr, int, (struct spdk_sock *sock, char *saddr, int sl char *caddr, int clen, uint16_t *cport), 0); DEFINE_STUB(spdk_sock_connect, struct spdk_sock *, (const char *ip, int port, char *impl_name), NULL); +DEFINE_STUB(spdk_sock_connect_ext, struct spdk_sock *, (const char *ip, int port, char *impl_name, + struct spdk_sock_opts *opts), NULL); DEFINE_STUB(spdk_sock_listen, struct spdk_sock *, (const char *ip, int port, char *impl_name), NULL); +DEFINE_STUB(spdk_sock_listen_ext, struct spdk_sock *, (const char *ip, int port, char *impl_name, + struct spdk_sock_opts *opts), NULL); +DEFINE_STUB_V(spdk_sock_get_default_opts, (struct spdk_sock_opts *opts)); DEFINE_STUB(spdk_sock_accept, struct spdk_sock *, (struct spdk_sock *sock), NULL); DEFINE_STUB(spdk_sock_close, int, (struct spdk_sock **sock), 0); DEFINE_STUB(spdk_sock_recv, ssize_t, (struct spdk_sock *sock, void *buf, size_t len), 0); diff --git a/test/unit/lib/nvme/nvme.c/nvme_ut.c b/test/unit/lib/nvme/nvme.c/nvme_ut.c index 2dc301235d..4c7cc34129 100644 --- a/test/unit/lib/nvme/nvme.c/nvme_ut.c +++ b/test/unit/lib/nvme/nvme.c/nvme_ut.c @@ -971,6 +971,15 @@ test_trid_parse_and_compare(void) CU_ASSERT(spdk_nvme_transport_id_parse(&trid1, "trtype=PCIe traddr=0000:04:00.0") == 0); CU_ASSERT(spdk_nvme_transport_id_parse(&trid2, "trtype=PCIe traddr=05:00.0") == 0); CU_ASSERT(spdk_nvme_transport_id_compare(&trid1, &trid2) < 0); + + CU_ASSERT(spdk_nvme_transport_id_parse(&trid1, + "trtype:tcp\n" + "adrfam:ipv4\n" + "traddr:192.168.100.8\n" + "trsvcid:4420\n" + "priority:2\n" + "subnqn:nqn.2014-08.org.nvmexpress.discovery") == 0); + CU_ASSERT(trid1.priority == 2); } static void diff --git a/test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c b/test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c index db8717fc46..c2f23fd407 100644 --- a/test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c +++ b/test/unit/lib/nvme/nvme_tcp.c/nvme_tcp_ut.c @@ -45,6 +45,9 @@ SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME); DEFINE_STUB(nvme_qpair_submit_request, int, (struct spdk_nvme_qpair *qpair, struct nvme_request *req), 0); +DEFINE_STUB(spdk_sock_set_priority, + int, (struct spdk_sock *sock, int priority), 0); + static void test_nvme_tcp_pdu_set_data_buf(void) {