d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bdev/nvme: configure the number of requests allocated for one NVMe I/O queue via RPC

Browse Source 
A single I/O may allocate more than one request, since splitting may be
necessary to conform to the device's maximum transfer size, PRP list
compatibility requirements, or driver-assisted striping.  Very big
I/O request sent from application may get error due to limited resources
in NVMe driver layer, so here we add an optional parameter to make the
parameter can be configured by users.

Fix issue #745.

Change-Id: I7824232c54865b052dcd0ec6e91484c3837fc2c4
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461182
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Changpeng Liu 2019-07-10 01:13:31 -04:00
parent bf7460f548
commit 90520634bd
7 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -133,6 +133,11 @@ a thinly provisioned logical volume. See documentation for complete details.
### nvme
Added an optional parameter `--io-queue-requests` to RPC `set_bdev_nvme_options`, which
can be used to change the number of requests allocated for one NVMe I/O queue. For
very big I/O size, e.g. 128MiB, with this option user will not get an error due to
limited requests in NVMe driver layer.
Added spdk_nvme_ctrlr_get_transport_id() to get the transport ID from a
previously attached controller.

4
doc/jsonrpc.md
View File

@ -1402,6 +1402,7 @@ timeout_us | Optional | number | Timeout for each command,
retry_count | Optional | number | The number of attempts per I/O before an I/O fails
nvme_adminq_poll_period_us | Optional | number | How often the admin queue is polled for asynchronous events in microseconds
nvme_ioq_poll_period_us | Optional | number | How often I/O queues are polled for completions, in microseconds. Default: 0 (as fast as possible).
io_queue_requests | Optional | number | The number of requests allocated for each NVMe I/O queue. Default: 512.
### Example
@ -1414,7 +1415,8 @@ request:
"retry_count": 5,
"nvme_adminq_poll_period_us": 2000,
"timeout_us": 10000000,
"action_on_timeout": "reset"
"action_on_timeout": "reset",
"io_queue_requests" : 2048,
},
"jsonrpc": "2.0",
"method": "set_bdev_nvme_options",

5
lib/bdev/nvme/bdev_nvme.c
View File

@ -107,6 +107,7 @@ static struct spdk_bdev_nvme_opts g_opts = {
.retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT,
.nvme_adminq_poll_period_us = 1000000ULL,
.nvme_ioq_poll_period_us = 0,
.io_queue_requests = 0,
};
#define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL
@ -537,6 +538,8 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf)
spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));
opts.delay_pcie_doorbell = true;
opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);
g_opts.io_queue_requests = opts.io_queue_requests;
ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts));
@ -1098,6 +1101,7 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
return 0;
}
struct set_nvme_hotplug_ctx {
uint64_t period_us;
bool enabled;
@ -2118,6 +2122,7 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w)
spdk_json_write_named_uint32(w, "retry_count", g_opts.retry_count);
spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);
spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);
spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);

1
lib/bdev/nvme/bdev_nvme.h
View File

@ -52,6 +52,7 @@ struct spdk_bdev_nvme_opts {
uint32_t retry_count;
uint64_t nvme_adminq_poll_period_us;
uint64_t nvme_ioq_poll_period_us;
uint32_t io_queue_requests;
};
typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, int rc);

1
lib/bdev/nvme/bdev_nvme_rpc.c
View File

@ -75,6 +75,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] =
{"retry_count", offsetof(struct spdk_bdev_nvme_opts, retry_count), spdk_json_decode_uint32, true},
{"nvme_adminq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_adminq_poll_period_us), spdk_json_decode_uint64, true},
{"nvme_ioq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_ioq_poll_period_us), spdk_json_decode_uint64, true},
{"io_queue_requests", offsetof(struct spdk_bdev_nvme_opts, io_queue_requests), spdk_json_decode_uint32, true},
};
static void

5
scripts/rpc.py
View File

@ -298,7 +298,8 @@ if __name__ == "__main__":
timeout_us=args.timeout_us,
retry_count=args.retry_count,
nvme_adminq_poll_period_us=args.nvme_adminq_poll_period_us,
nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us)
nvme_ioq_poll_period_us=args.nvme_ioq_poll_period_us,
io_queue_requests=args.io_queue_requests)
p = subparsers.add_parser('set_bdev_nvme_options',
help='Set options for the bdev nvme type. This is startup command.')
@ -312,6 +313,8 @@ if __name__ == "__main__":
help='How often the admin queue is polled for asynchronous events', type=int)
p.add_argument('-i', '--nvme-ioq-poll-period-us',
help='How often to poll I/O queues for completions', type=int)
p.add_argument('-s', '--io-queue-requests',
help='The number of requests allocated for each NVMe I/O queue. Default: 512', type=int)
p.set_defaults(func=set_bdev_nvme_options)
def set_bdev_nvme_hotplug(args):

6
scripts/rpc/bdev.py
View File

@ -276,7 +276,7 @@ def delete_aio_bdev(client, name):
def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry_count=None,
nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None):
nvme_adminq_poll_period_us=None, nvme_ioq_poll_period_us=None, io_queue_requests=None):
"""Set options for the bdev nvme. This is startup command.
Args:
@ -285,6 +285,7 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry
retry_count: The number of attempts per I/O when an I/O fails (optional)
nvme_adminq_poll_period_us: How often the admin queue is polled for asynchronous events in microseconds (optional)
nvme_ioq_poll_period_us: How often to poll I/O queues for completions in microseconds (optional)
io_queue_requests: The number of requests allocated for each NVMe I/O queue. Default: 512 (optional)
"""
params = {}
@ -303,6 +304,9 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry
if nvme_ioq_poll_period_us:
params['nvme_ioq_poll_period_us'] = nvme_ioq_poll_period_us
if io_queue_requests:
params['io_queue_requests'] = io_queue_requests
return client.call('set_bdev_nvme_options', params)