d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bdev/nvme: Add set_bdev_nvme_options

Browse Source 
Change-Id: I5ab027d5204f9c4991eacaea631367783d1ad115
Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-on: https://review.gerrithub.io/418723
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Seth Howell <seth.howell5141@gmail.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
This commit is contained in:
Pawel Wodkowski 2018-07-09 23:04:33 +02:00 committed by Ben Walker
parent 911c042df8
commit 4bef621e90
11 changed files with 267 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
doc/jsonrpc.md
View File

@ -892,6 +892,48 @@ Example response:
}
~~~
## set_bdev_nvme_options {#rpc_set_bdev_nvme_options}
Set global parameters for all bdev NVMe. This RPC may only be called before SPDK subsystems have been initialized.
### Parameters
Name | Optional | Type | Description
-------------------------- | -------- | ----------- | -----------
action_on_timeout | Optional | string | Action to take on command time out: none, reset or abort
timeout_us | Optional | number | Timeout for each command, in microseconds. If 0, don't track timeouts
retry_count | Optional | number | The number of attempts per I/O before an I/O fails
nvme_adminq_poll_period_us | Optional | number | How often the admin queue is polled for asynchronous events in microsecond
### Example
Example request:
~~~
request:
{
"params": {
"retry_count": 5,
"nvme_adminq_poll_period_us": 2000,
"timeout_us": 10000000,
"action_on_timeout": "reset"
},
"jsonrpc": "2.0",
"method": "set_bdev_nvme_options",
"id": 1
}
~~~
Example response:
~~~
{
"jsonrpc": "2.0",
"id": 1,
"result": true
}
~~~
## construct_nvme_bdev {#rpc_construct_nvme_bdev}
Construct @ref bdev_config_nvme

4
etc/spdk/iscsi.conf.in
View File

@ -108,8 +108,8 @@
# The number of attempts per I/O when an I/O fails. Do not include
# this key to get the default behavior.
RetryCount 4
# Timeout for each command, in seconds. If 0, don't track timeouts.
Timeout 0
# Timeout for each command, in microseconds. If 0, don't track timeouts.
TimeoutUsec 0
# Action to take on command time out. Only valid when Timeout is greater
# than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
# the command, or 'None' to just print a message but do nothing.

4
etc/spdk/nvmf.conf.in
View File

@ -99,8 +99,8 @@
# The number of attempts per I/O when an I/O fails. Do not include
# this key to get the default behavior.
RetryCount 4
# Timeout for each command, in seconds. If 0, don't track timeouts.
Timeout 0
# Timeout for each command, in microseconds. If 0, don't track timeouts.
TimeoutUsec 0
# Action to take on command time out. Only valid when Timeout is greater
# than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
# the command, or 'None' to just print a message but do nothing.

4
etc/spdk/vhost.conf.in
View File

@ -89,8 +89,8 @@
# The number of attempts per I/O when an I/O fails. Do not include
# this key to get the default behavior.
RetryCount 4
# Timeout for each command, in seconds. If 0, don't track timeouts.
Timeout 0
# Timeout for each command, in microseconds. If 0, don't track timeouts.
TimeoutUsec 0
# Action to take on command time out. Only valid when Timeout is greater
# than 0. This may be 'Reset' to reset the controller, 'Abort' to abort
# the command, or 'None' to just print a message but do nothing.

136
lib/bdev/nvme/bdev_nvme.c
View File

@ -94,18 +94,18 @@ struct nvme_probe_ctx {
const char *hostnqn;
};
enum timeout_action {
TIMEOUT_ACTION_NONE = 0,
TIMEOUT_ACTION_RESET,
TIMEOUT_ACTION_ABORT,
static struct spdk_bdev_nvme_opts g_opts = {
.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE,
.timeout_us = 0,
.retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT,
.nvme_adminq_poll_period_us = 1000000ULL,
};
static bool g_bdev_nvme_init_done = false;
static int g_hot_insert_nvme_controller_index = 0;
static enum timeout_action g_action_on_timeout = TIMEOUT_ACTION_NONE;
static uint64_t g_timeout_us = 0;
static int g_nvme_adminq_poll_timeout_us = 0;
static bool g_nvme_hotplug_enabled = false;
static int g_nvme_hotplug_poll_timeout_us = 0;
static bool g_nvme_hotplug_enabled = false;
static struct spdk_poller *g_hotplug_poller;
static char *g_nvme_hostnqn = NULL;
static pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
@ -850,8 +850,8 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
SPDK_WARNLOG("Warning: Detected a timeout. ctrlr=%p qpair=%p cid=%u\n", ctrlr, qpair, cid);
switch (g_action_on_timeout) {
case TIMEOUT_ACTION_ABORT:
switch (g_opts.action_on_timeout) {
case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
if (qpair) {
rc = spdk_nvme_ctrlr_cmd_abort(ctrlr, qpair, cid,
spdk_nvme_abort_cpl, ctrlr);
@ -863,13 +863,13 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
}
/* FALLTHROUGH */
case TIMEOUT_ACTION_RESET:
case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
rc = spdk_nvme_ctrlr_reset(ctrlr);
if (rc) {
SPDK_ERRLOG("Resetting controller failed.\n");
}
break;
case TIMEOUT_ACTION_NONE:
case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
break;
}
}
@ -959,12 +959,12 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr,
}
nvme_ctrlr->adminq_timer_poller = spdk_poller_register(bdev_nvme_poll_adminq, ctrlr,
g_nvme_adminq_poll_timeout_us);
g_opts.nvme_adminq_poll_period_us);
TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, nvme_ctrlr, tailq);
if (g_action_on_timeout != TIMEOUT_ACTION_NONE) {
spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_timeout_us,
if (g_opts.timeout_us > 0 && g_opts.action_on_timeout != SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE) {
spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_opts.timeout_us,
timeout_cb, NULL);
}
@ -1039,6 +1039,24 @@ bdev_nvme_hotplug(void *arg)
return -1;
}
void
spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts)
{
*opts = g_opts;
}
int
spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
{
if (g_bdev_nvme_init_done) {
return -EPERM;
}
g_opts = *opts;
return 0;
}
int
spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
const char *base_name,
@ -1133,12 +1151,14 @@ bdev_nvme_library_init(void)
struct spdk_conf_section *sp;
const char *val;
int rc = 0;
int64_t intval;
size_t i;
struct nvme_probe_ctx *probe_ctx = NULL;
int retry_count;
int timeout;
uint32_t local_nvme_num = 0;
g_bdev_nvme_init_done = true;
sp = spdk_conf_find_section(NULL, "Nvme");
if (sp == NULL) {
goto end;
@ -1160,37 +1180,47 @@ bdev_nvme_library_init(void)
}
}
spdk_nvme_retry_count = retry_count;
g_opts.retry_count = retry_count;
val = spdk_conf_section_get_val(sp, "TimeoutUsec");
if (val != NULL) {
g_timeout_us = strtoll(val, NULL, 10);
intval = strtoll(val, NULL, 10);
if (intval == LLONG_MIN || intval == LLONG_MAX) {
SPDK_ERRLOG("Invalid TimeoutUsec value\n");
rc = -1;
goto end;
} else if (intval < 0) {
intval = 0;
}
} else {
/* Check old name for backward compatibility */
timeout = spdk_conf_section_get_intval(sp, "Timeout");
if (timeout < 0) {
timeout = spdk_conf_section_get_intval(sp, "NvmeTimeoutValue");
if (timeout < 0) {
g_timeout_us = 0;
intval = spdk_conf_section_get_intval(sp, "Timeout");
if (intval < 0) {
intval = spdk_conf_section_get_intval(sp, "NvmeTimeoutValue");
if (intval < 0) {
intval = 0;
} else {
g_timeout_us = timeout * 1000000ULL;
intval *= 1000000ULL;
SPDK_WARNLOG("NvmeTimeoutValue (in seconds) was renamed to TimeoutUsec (in microseconds)\n");
SPDK_WARNLOG("Please update your configuration file\n");
}
} else {
g_timeout_us = timeout * 1000000ULL;
intval *= 1000000ULL;
SPDK_WARNLOG("Timeout (in seconds) was renamed to TimeoutUsec (in microseconds)\n");
SPDK_WARNLOG("Please update your configuration file\n");
}
}
if (g_timeout_us > 0) {
g_opts.timeout_us = intval;
if (g_opts.timeout_us > 0) {
val = spdk_conf_section_get_val(sp, "ActionOnTimeout");
if (val != NULL) {
if (!strcasecmp(val, "Reset")) {
g_action_on_timeout = TIMEOUT_ACTION_RESET;
g_opts.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET;
} else if (!strcasecmp(val, "Abort")) {
g_action_on_timeout = TIMEOUT_ACTION_ABORT;
g_opts.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT;
}
} else {
/* Handle old name for backward compatibility */
@ -1200,15 +1230,15 @@ bdev_nvme_library_init(void)
SPDK_WARNLOG("Please update your configuration file\n");
if (spdk_conf_section_get_boolval(sp, "ResetControllerOnTimeout", false)) {
g_action_on_timeout = TIMEOUT_ACTION_RESET;
g_opts.action_on_timeout = SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET;
}
}
}
}
g_nvme_adminq_poll_timeout_us = spdk_conf_section_get_intval(sp, "AdminPollRate");
if (g_nvme_adminq_poll_timeout_us <= 0) {
g_nvme_adminq_poll_timeout_us = 1000000;
intval = spdk_conf_section_get_intval(sp, "AdminPollRate");
if (intval > 0) {
g_opts.nvme_adminq_poll_period_us = intval;
}
if (spdk_process_is_primary()) {
@ -1221,9 +1251,7 @@ bdev_nvme_library_init(void)
}
g_nvme_hostnqn = spdk_conf_section_get_val(sp, "HostNQN");
if (g_nvme_hostnqn) {
probe_ctx->hostnqn = g_nvme_hostnqn;
}
probe_ctx->hostnqn = g_nvme_hostnqn;
for (i = 0; i < NVME_MAX_CONTROLLERS; i++) {
val = spdk_conf_section_get_nmval(sp, "TransportID", i, 0);
@ -1313,6 +1341,10 @@ bdev_nvme_library_init(void)
}
end:
if (rc == 0) {
spdk_nvme_retry_count = g_opts.retry_count;
}
free(probe_ctx);
return rc;
}
@ -1617,21 +1649,21 @@ bdev_nvme_get_spdk_running_config(FILE *fp)
fprintf(fp, "RetryCount %d\n", spdk_nvme_retry_count);
fprintf(fp, "\n"
"# Timeout for each command, in microseconds. If 0, don't track timeouts.\n");
fprintf(fp, "Timeout %"PRIu64"\n", g_timeout_us);
fprintf(fp, "TimeoutUsec %"PRIu64"\n", g_opts.timeout_us);
fprintf(fp, "\n"
"# Action to take on command time out. Only valid when Timeout is greater\n"
"# than 0. This may be 'Reset' to reset the controller, 'Abort' to abort\n"
"# the command, or 'None' to just print a message but do nothing.\n"
"# Admin command timeouts will always result in a reset.\n");
switch (g_action_on_timeout) {
case TIMEOUT_ACTION_NONE:
switch (g_opts.action_on_timeout) {
case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:
fprintf(fp, "ActionOnTimeout None\n");
break;
case TIMEOUT_ACTION_RESET:
case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:
fprintf(fp, "ActionOnTimeout Reset\n");
break;
case TIMEOUT_ACTION_ABORT:
case SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT:
fprintf(fp, "ActionOnTimeout Abort\n");
break;
}
@ -1639,7 +1671,7 @@ bdev_nvme_get_spdk_running_config(FILE *fp)
fprintf(fp, "\n"
"# Set how often the admin queue is polled for asynchronous events.\n"
"# Units in microseconds.\n");
fprintf(fp, "AdminPollRate %d\n", g_nvme_adminq_poll_timeout_us);
fprintf(fp, "AdminPollRate %"PRIu64"\n", g_opts.nvme_adminq_poll_period_us);
fprintf(fp, "\n"
"# Disable handling of hotplug (runtime insert and remove) events,\n"
"# users can set to Yes if want to enable it.\n"
@ -1662,6 +1694,28 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w)
struct nvme_ctrlr *nvme_ctrlr;
struct spdk_nvme_transport_id *trid;
const char *adrfam;
const char *action;
if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET) {
action = "reset";
} else if (g_opts.action_on_timeout == SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT) {
action = "abort";
} else {
action = "none";
}
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "method", "set_bdev_nvme_options");
spdk_json_write_named_object_begin(w, "params");
spdk_json_write_named_string(w, "action_on_timeout", action);
spdk_json_write_named_uint64(w, "timeout_us", g_opts.timeout_us);
spdk_json_write_named_uint32(w, "retry_count", g_opts.retry_count);
spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
pthread_mutex_lock(&g_bdev_nvme_mutex);
TAILQ_FOREACH(nvme_ctrlr, &g_nvme_ctrlrs, tailq) {

16
lib/bdev/nvme/bdev_nvme.h
View File

@ -42,6 +42,19 @@
#define NVME_MAX_CONTROLLERS 1024
enum spdk_bdev_timeout_action {
SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE = 0,
SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET,
SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT,
};
struct spdk_bdev_nvme_opts {
enum spdk_bdev_timeout_action action_on_timeout;
uint64_t timeout_us;
uint32_t retry_count;
uint64_t nvme_adminq_poll_period_us;
};
struct nvme_ctrlr {
/**
* points to pinned, physically contiguous memory region;
@ -70,6 +83,9 @@ struct nvme_bdev {
struct spdk_nvme_ns *ns;
};
void spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);
int spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);
int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
const char *base_name,
const char **names, size_t *count,

60
lib/bdev/nvme/bdev_nvme_rpc.c
View File

@ -49,6 +49,66 @@ struct open_descriptors {
};
typedef TAILQ_HEAD(, open_descriptors) open_descriptors_t;
static int
rpc_decode_action_on_timeout(const struct spdk_json_val *val, void *out)
{
enum spdk_bdev_timeout_action *action = out;
if (spdk_json_strequal(val, "none") == true) {
*action = SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE;
} else if (spdk_json_strequal(val, "abort") == true) {
*action = SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT;
} else if (spdk_json_strequal(val, "reset") == true) {
*action = SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET;
} else {
SPDK_NOTICELOG("Invalid parameter value: action_on_timeout\n");
return -EINVAL;
}
return 0;
}
static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] = {
{"action_on_timeout", offsetof(struct spdk_bdev_nvme_opts, action_on_timeout), rpc_decode_action_on_timeout, true},
{"timeout_us", offsetof(struct spdk_bdev_nvme_opts, timeout_us), spdk_json_decode_uint64, true},
{"retry_count", offsetof(struct spdk_bdev_nvme_opts, retry_count), spdk_json_decode_uint32, true},
{"nvme_adminq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_adminq_poll_period_us), spdk_json_decode_uint64, true},
};
static void
spdk_rpc_set_bdev_nvme_options(struct spdk_jsonrpc_request *request,
const struct spdk_json_val *params)
{
struct spdk_bdev_nvme_opts opts;
struct spdk_json_write_ctx *w;
int rc;
spdk_bdev_nvme_get_opts(&opts);
if (params && spdk_json_decode_object(params, rpc_bdev_nvme_options_decoders,
SPDK_COUNTOF(rpc_bdev_nvme_options_decoders),
&opts)) {
SPDK_ERRLOG("spdk_json_decode_object failed\n");
rc = -EINVAL;
goto invalid;
}
rc = spdk_bdev_nvme_set_opts(&opts);
if (rc) {
goto invalid;
}
w = spdk_jsonrpc_begin_result(request);
if (w != NULL) {
spdk_json_write_bool(w, true);
spdk_jsonrpc_end_result(request, w);
}
return;
invalid:
spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(-rc));
}
SPDK_RPC_REGISTER("set_bdev_nvme_options", spdk_rpc_set_bdev_nvme_options, SPDK_RPC_STARTUP)
struct rpc_construct_nvme {
char *name;
char *trtype;

20
scripts/rpc.py
View File

@ -218,6 +218,26 @@ if __name__ == "__main__":
p.add_argument('name', help='aio bdev name')
p.set_defaults(func=delete_aio_bdev)
@call_cmd
def set_bdev_nvme_options(args):
rpc.bdev.set_bdev_nvme_options(args.client,
action_on_timeout=args.action_on_timeout,
timeout_s=args.timeout_s,
retry_count=args.retry_count,
nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us)
p = subparsers.add_parser('set_bdev_nvme_options',
help='Set options for the bdev nvme type. This is startup command.')
p.add_argument('-a', '--action-on-timeout',
help="Action to take on command time out. Valid valies are: none, reset, abort")
p.add_argument('-t', '--timeout-us',
help="Timeout for each command, in microseconds. If 0, don't track timeouts.", type=int)
p.add_argument('-n', '--retry-count',
help='the number of attempts per I/O when an I/O fails', type=int)
p.add_argument('-p', '--nvme-adminq-poll-period-us',
help='How often the admin queue is polled for asynchronous events', type=int)
p.set_defaults(func=set_bdev_nvme_options)
@call_cmd
def construct_nvme_bdev(args):
print_array(rpc.bdev.construct_nvme_bdev(args.client,

26
scripts/rpc/bdev.py
View File

@ -147,6 +147,32 @@ def delete_aio_bdev(client, name):
return client.call('delete_aio_bdev', params)
def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry_count=None, nvme_adminq_poll_period_us=None):
"""Set options for the bdev nvme. This is startup command.
Args:
action_on_timeout: action to take on command time out. Valid values are: none, reset, abort (optional)
timeout_us: Timeout for each command, in microseconds. If 0, don't track timeouts (optional)
retry_count: The number of attempts per I/O when an I/O fails (optional)
nvme_adminq_poll_period_us: how often the admin queue is polled for asynchronous events in microsecon (optional)
"""
params = {}
if action_on_timeout:
params['action_on_timeout'] = action_on_timeout
if timeout_us:
params['timeout_us'] = timeout_us
if retry_count:
params['retry_count'] = retry_count
if nvme_adminq_poll_period_us:
params['nvme_adminq_poll_period_us'] = nvme_adminq_poll_period_us
return client.call('set_bdev_nvme_options', params)
def construct_nvme_bdev(client, name, trtype, traddr, adrfam=None, trsvcid=None, subnqn=None):
"""Construct NVMe namespace block device.

1
test/iscsi_tgt/fio/iscsi.conf.in
View File

@ -11,7 +11,6 @@
[Nvme]
RetryCount 4
Timeout 0
ActionOnTimeout None
AdminPollRate 100000
HotplugEnable Yes

3
test/json_config/config_filter.py
View File

@ -8,7 +8,8 @@ def filter_methods(filename, do_remove_startup_rpcs):
'set_iscsi_options',
'set_nvmf_target_config',
'set_nvmf_target_options',
'set_bdev_options'
'set_bdev_options',
'set_bdev_nvme_options'
]
with open(filename) as json_file: