d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: Make ANA state not only per subsystem listener but also per ANA group

Browse Source 
Refine ANA state from per subsystem listener to per subsystem listener
per ANA group.

Add an array of ANA state per ANA group to subsystem listener. The array is
indexed by ANA group ID - 1.

Then in I/O paths, we get ANA state by
    ctrlr->listener->ana_state[ns->anagrpid - 1].

The NVMe specification indicates the existence of NVM subsystem specific
ANA state when FFFFFFFFh is specified as NSID for the Get Features
and the Set Features commands. For these, we return the optimized state.

Update the nvmf_subsystem_get_listeners RPC to return all ANA states
of the underlying ANA groups. The nvmf_subsystem_get_listeners RPC is
not matured and not used in the test code yet. Hence compatibility is
not high priority.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Ia2d4d5361ac01236f595c22765fd35e4c5fdee0e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9064
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Shuhei Matsumoto 2021-08-06 07:20:51 +09:00 committed by Tomasz Zawadzki
parent 07bfc3cbe9
commit 785d10b5c7
5 changed files with 97 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
lib/nvmf/ctrlr.c
View File

@ -1990,7 +1990,7 @@ nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iov
ana_desc.ana_group_id = anagrpid;
ana_desc.num_of_nsid = ctrlr->subsys->ana_group[anagrpid - 1];
ana_desc.ana_state = ctrlr->listener->ana_state;
ana_desc.ana_state = ctrlr->listener->ana_state[anagrpid - 1];
copy_len = spdk_min(sizeof(ana_desc) - offset, length);
copied_len = _copy_buf_to_iovs(&copy_ctx, (const char *)&ana_desc + offset,
@ -2287,6 +2287,7 @@ spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr,
struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys;
struct spdk_nvmf_ns *ns;
uint32_t max_num_blocks;
enum spdk_nvme_ana_state ana_state;
if (cmd->nsid == 0 || cmd->nsid > subsystem->max_nsid) {
SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", cmd->nsid);
@ -2318,10 +2319,12 @@ spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_ctrlr *ctrlr,
}
if (subsystem->flags.ana_reporting) {
assert(ns->anagrpid - 1 < subsystem->max_nsid);
nsdata->anagrpid = ns->anagrpid;
if (ctrlr->listener->ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE ||
ctrlr->listener->ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) {
ana_state = ctrlr->listener->ana_state[ns->anagrpid - 1];
if (ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE ||
ana_state == SPDK_NVME_ANA_PERSISTENT_LOSS_STATE) {
nsdata->nuse = 0;
}
}
@ -2769,6 +2772,28 @@ _nvme_ana_state_to_path_status(enum spdk_nvme_ana_state ana_state)
}
}
/* we have to use the typedef in the function declaration to appease astyle. */
typedef enum spdk_nvme_ana_state spdk_nvme_ana_state_t;
static spdk_nvme_ana_state_t
nvmf_ctrlr_get_ana_state(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid)
{
struct spdk_nvmf_ns *ns;
/* We do not have NVM subsystem specific ANA state. Hence if NSID is either
* SPDK_NVMF_GLOBAL_NS_TAG, invalid, or for inactive namespace, return
* the optimized state.
*/
ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid);
if (ns == NULL) {
return SPDK_NVME_ANA_OPTIMIZED_STATE;
}
assert(ns->anagrpid - 1 < ctrlr->subsys->max_nsid);
return ctrlr->listener->ana_state[ns->anagrpid];
}
static int
nvmf_ctrlr_get_features(struct spdk_nvmf_request *req)
{
@ -2798,7 +2823,7 @@ nvmf_ctrlr_get_features(struct spdk_nvmf_request *req)
/*
* Process Get Features command for non-discovery controller
*/
ana_state = ctrlr->listener->ana_state;
ana_state = nvmf_ctrlr_get_ana_state(ctrlr, cmd->nsid);
switch (ana_state) {
case SPDK_NVME_ANA_INACCESSIBLE_STATE:
case SPDK_NVME_ANA_PERSISTENT_LOSS_STATE:
@ -2894,7 +2919,7 @@ nvmf_ctrlr_set_features(struct spdk_nvmf_request *req)
/*
* Process Set Features command for non-discovery controller
*/
ana_state = ctrlr->listener->ana_state;
ana_state = nvmf_ctrlr_get_ana_state(ctrlr, cmd->nsid);
switch (ana_state) {
case SPDK_NVME_ANA_INACCESSIBLE_STATE:
case SPDK_NVME_ANA_CHANGE_STATE:
@ -3690,10 +3715,17 @@ nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req)
return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
}
/* It will be lower overhead to check if ANA state is optimized or
* non-optimized.
*/
ana_state = ctrlr->listener->ana_state;
ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid);
if (ns == NULL || ns->bdev == NULL) {
SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid);
response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
response->status.dnr = 1;
return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
}
assert(ns->anagrpid - 1 < ctrlr->subsys->max_nsid);
ana_state = ctrlr->listener->ana_state[ns->anagrpid - 1];
if (spdk_unlikely(ana_state != SPDK_NVME_ANA_OPTIMIZED_STATE &&
ana_state != SPDK_NVME_ANA_NON_OPTIMIZED_STATE)) {
SPDK_DEBUGLOG(nvmf, "Fail I/O command due to ANA state %d\n",
@ -3703,14 +3735,6 @@ nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req)
return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
}
ns = _nvmf_subsystem_get_ns(ctrlr->subsys, nsid);
if (ns == NULL || ns->bdev == NULL) {
SPDK_DEBUGLOG(nvmf, "Unsuccessful query for nsid %u\n", cmd->nsid);
response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
response->status.dnr = 1;
return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
}
/* scan-build falsely reporting dereference of null pointer */
assert(group != NULL && group->sgroups != NULL);
ns_info = &group->sgroups[ctrlr->subsys->id].ns_info[nsid - 1];

2
lib/nvmf/nvmf_internal.h
View File

@ -103,7 +103,7 @@ struct spdk_nvmf_subsystem_listener {
void *cb_arg;
struct spdk_nvme_transport_id *trid;
struct spdk_nvmf_transport *transport;
enum spdk_nvme_ana_state ana_state;
enum spdk_nvme_ana_state *ana_state;
uint64_t ana_state_change_count;
TAILQ_ENTRY(spdk_nvmf_subsystem_listener) link;
};

12
lib/nvmf/nvmf_rpc.c
View File

@ -2207,6 +2207,7 @@ dump_nvmf_subsystem_listener(struct spdk_json_write_ctx *w,
{
const struct spdk_nvme_transport_id *trid = listener->trid;
const char *adrfam;
uint32_t i;
spdk_json_write_object_begin(w);
@ -2222,8 +2223,15 @@ dump_nvmf_subsystem_listener(struct spdk_json_write_ctx *w,
spdk_json_write_object_end(w);
if (nvmf_subsystem_get_ana_reporting(listener->subsystem)) {
spdk_json_write_named_string(w, "ana_state",
nvme_ana_state_str(listener->ana_state));
spdk_json_write_named_array_begin(w, "ana_states");
for (i = 0; i < listener->subsystem->max_nsid; i++) {
spdk_json_write_object_begin(w);
spdk_json_write_named_uint32(w, "ana_group", i + 1);
spdk_json_write_named_string(w, "ana_state",
nvme_ana_state_str(listener->ana_state[i]));
spdk_json_write_object_end(w);
}
spdk_json_write_array_end(w);
}
spdk_json_write_object_end(w);

24
lib/nvmf/subsystem.c
View File

@ -348,6 +348,7 @@ _nvmf_subsystem_remove_listener(struct spdk_nvmf_subsystem *subsystem,
}
TAILQ_REMOVE(&subsystem->listeners, listener, link);
free(listener->ana_state);
free(listener);
}
@ -991,6 +992,7 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
struct spdk_nvmf_transport *transport;
struct spdk_nvmf_subsystem_listener *listener;
struct spdk_nvmf_listener *tr_listener;
uint32_t i;
int rc = 0;
assert(cb_fn != NULL);
@ -1033,7 +1035,16 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
listener->cb_fn = cb_fn;
listener->cb_arg = cb_arg;
listener->subsystem = subsystem;
listener->ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener->ana_state = calloc(subsystem->max_nsid, sizeof(enum spdk_nvme_ana_state));
if (!listener->ana_state) {
free(listener);
cb_fn(cb_arg, -ENOMEM);
return;
}
for (i = 0; i < subsystem->max_nsid; i++) {
listener->ana_state[i] = SPDK_NVME_ANA_OPTIMIZED_STATE;
}
if (transport->ops->listen_associate != NULL) {
rc = transport->ops->listen_associate(transport, subsystem, trid);
@ -1570,7 +1581,6 @@ spdk_nvmf_subsystem_add_ns_ext(struct spdk_nvmf_subsystem *subsystem, const char
ns->anagrpid = opts.anagrpid;
subsystem->ana_group[ns->anagrpid - 1]++;
TAILQ_INIT(&ns->registrants);
if (ptpl_file) {
rc = nvmf_ns_load_reservation(ptpl_file, &info);
if (!rc) {
@ -3019,6 +3029,7 @@ nvmf_subsystem_set_ana_state(struct spdk_nvmf_subsystem *subsystem,
{
struct spdk_nvmf_subsystem_listener *listener;
struct subsystem_listener_update_ctx *ctx;
uint32_t i;
assert(cb_fn != NULL);
assert(subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE ||
@ -3048,11 +3059,6 @@ nvmf_subsystem_set_ana_state(struct spdk_nvmf_subsystem *subsystem,
return;
}
if (listener->ana_state == ana_state) {
cb_fn(cb_arg, 0);
return;
}
ctx = calloc(1, sizeof(*ctx));
if (!ctx) {
SPDK_ERRLOG("Unable to allocate context\n");
@ -3060,7 +3066,9 @@ nvmf_subsystem_set_ana_state(struct spdk_nvmf_subsystem *subsystem,
return;
}
listener->ana_state = ana_state;
for (i = 0; i < subsystem->max_nsid; i++) {
listener->ana_state[i] = ana_state;
}
listener->ana_state_change_count++;
ctx->listener = listener;

45
test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c
View File

@ -1052,14 +1052,15 @@ test_set_get_features(void)
{
struct spdk_nvmf_subsystem subsystem = {};
struct spdk_nvmf_qpair admin_qpair = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[3];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_nvmf_ctrlr ctrlr = {
.subsys = &subsystem, .admin_qpair = &admin_qpair, .listener = &listener
};
union nvmf_h2c_msg cmd = {};
union nvmf_c2h_msg rsp = {};
struct spdk_nvmf_ns ns[3];
struct spdk_nvmf_ns *ns_arr[3] = {&ns[0], NULL, &ns[2]};;
struct spdk_nvmf_ns *ns_arr[3] = {&ns[0], NULL, &ns[2]};
struct spdk_nvmf_request req;
int rc;
@ -1067,7 +1068,8 @@ test_set_get_features(void)
ns[2].anagrpid = 3;
subsystem.ns = ns_arr;
subsystem.max_nsid = SPDK_COUNTOF(ns_arr);
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[0] = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[2] = SPDK_NVME_ANA_OPTIMIZED_STATE;
admin_qpair.ctrlr = &ctrlr;
req.qpair = &admin_qpair;
cmd.nvme_cmd.nsid = 1;
@ -1666,7 +1668,8 @@ test_fused_compare_and_write(void)
struct spdk_nvmf_subsystem subsystem = {};
struct spdk_nvmf_ns ns = {};
struct spdk_nvmf_ns *subsys_ns[1] = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[1];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_bdev bdev = {};
struct spdk_nvmf_poll_group group = {};
@ -1682,7 +1685,7 @@ test_fused_compare_and_write(void)
subsys_ns[0] = &ns;
subsystem.ns = (struct spdk_nvmf_ns **)&subsys_ns;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[0] = SPDK_NVME_ANA_OPTIMIZED_STATE;
/* Enable controller */
ctrlr.vcprop.cc.bits.en = 1;
@ -1838,7 +1841,8 @@ test_get_ana_log_page_one_ns_per_anagrp(void)
uint32_t ana_group[3];
struct spdk_nvmf_subsystem subsystem = { .ana_group = ana_group };
struct spdk_nvmf_ctrlr ctrlr = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[3];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_nvmf_ns ns[3];
struct spdk_nvmf_ns *ns_arr[3] = {&ns[0], &ns[1], &ns[2]};
uint64_t offset;
@ -1858,7 +1862,10 @@ test_get_ana_log_page_one_ns_per_anagrp(void)
}
ctrlr.subsys = &subsystem;
ctrlr.listener = &listener;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
for (i = 0; i < 3; i++) {
listener.ana_state[i] = SPDK_NVME_ANA_OPTIMIZED_STATE;
}
for (i = 0; i < 3; i++) {
ns_arr[i]->nsid = i + 1;
@ -1879,7 +1886,7 @@ test_get_ana_log_page_one_ns_per_anagrp(void)
ana_desc->ana_group_id = ns_arr[i]->nsid;
ana_desc->num_of_nsid = 1;
ana_desc->change_count = 0;
ana_desc->ana_state = ctrlr.listener->ana_state;
ana_desc->ana_state = ctrlr.listener->ana_state[i];
ana_desc->nsid[0] = ns_arr[i]->nsid;
memcpy(&expected_page[offset], ana_desc, UT_ANA_DESC_SIZE);
offset += UT_ANA_DESC_SIZE;
@ -1923,7 +1930,8 @@ test_get_ana_log_page_multi_ns_per_anagrp(void)
struct spdk_nvmf_ns *ns_arr[5] = {&ns[0], &ns[1], &ns[2], &ns[3], &ns[4]};
uint32_t ana_group[5] = {0};
struct spdk_nvmf_subsystem subsystem = { .ns = ns_arr, .ana_group = ana_group, };
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[5];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state, };
struct spdk_nvmf_ctrlr ctrlr = { .subsys = &subsystem, .listener = &listener, };
char expected_page[UT_ANA_LOG_PAGE_SIZE] = {0};
char actual_page[UT_ANA_LOG_PAGE_SIZE] = {0};
@ -1938,7 +1946,9 @@ test_get_ana_log_page_multi_ns_per_anagrp(void)
subsystem.max_nsid = 5;
subsystem.ana_group[1] = 3;
subsystem.ana_group[2] = 2;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
for (i = 0; i < 5; i++) {
listener.ana_state[i] = SPDK_NVME_ANA_OPTIMIZED_STATE;
}
for (i = 0; i < 5; i++) {
ns_arr[i]->nsid = i + 1;
@ -2365,7 +2375,8 @@ test_spdk_nvmf_request_zcopy_start(void)
struct spdk_nvmf_subsystem subsystem = {};
struct spdk_nvmf_ns ns = {};
struct spdk_nvmf_ns *subsys_ns[1] = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[1];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_bdev bdev = { .blockcnt = 100, .blocklen = 512};
struct spdk_nvmf_poll_group group = {};
@ -2382,7 +2393,7 @@ test_spdk_nvmf_request_zcopy_start(void)
subsys_ns[0] = &ns;
subsystem.ns = (struct spdk_nvmf_ns **)&subsys_ns;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[0] = SPDK_NVME_ANA_OPTIMIZED_STATE;
/* Enable controller */
ctrlr.vcprop.cc.bits.en = 1;
@ -2489,7 +2500,8 @@ test_zcopy_read(void)
struct spdk_nvmf_subsystem subsystem = {};
struct spdk_nvmf_ns ns = {};
struct spdk_nvmf_ns *subsys_ns[1] = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[1];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_bdev bdev = { .blockcnt = 100, .blocklen = 512};
struct spdk_nvmf_poll_group group = {};
@ -2506,7 +2518,7 @@ test_zcopy_read(void)
subsys_ns[0] = &ns;
subsystem.ns = (struct spdk_nvmf_ns **)&subsys_ns;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[0] = SPDK_NVME_ANA_OPTIMIZED_STATE;
/* Enable controller */
ctrlr.vcprop.cc.bits.en = 1;
@ -2575,7 +2587,8 @@ test_zcopy_write(void)
struct spdk_nvmf_subsystem subsystem = {};
struct spdk_nvmf_ns ns = {};
struct spdk_nvmf_ns *subsys_ns[1] = {};
struct spdk_nvmf_subsystem_listener listener = {};
enum spdk_nvme_ana_state ana_state[1];
struct spdk_nvmf_subsystem_listener listener = { .ana_state = ana_state };
struct spdk_bdev bdev = { .blockcnt = 100, .blocklen = 512};
struct spdk_nvmf_poll_group group = {};
@ -2592,7 +2605,7 @@ test_zcopy_write(void)
subsys_ns[0] = &ns;
subsystem.ns = (struct spdk_nvmf_ns **)&subsys_ns;
listener.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE;
listener.ana_state[0] = SPDK_NVME_ANA_OPTIMIZED_STATE;
/* Enable controller */
ctrlr.vcprop.cc.bits.en = 1;