d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rdma: add a flag for disabling srq.

Browse Source 
There are cases where srq can be a detriment. Add a flag to allow users
to disable srq even if they have a piece of hardware that supports it.

Change-Id: Ia3be8e8c8e8463964e6ff1c02b07afbf4c3cc8f7
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452271
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Seth Howell 2019-04-26 14:25:20 -07:00 committed by Ben Walker
parent 9d0e11d4a2
commit 350e429a57
7 changed files with 35 additions and 14 deletions
CHANGELOG.md
doc
jsonrpc.md
include/spdk
nvmf.h
lib
event/subsystems/nvmf
nvmf_rpc.c
nvmf
rdma.c
scripts
rpc.py
rpc
nvmf.py

6
CHANGELOG.md

@ -2,6 +2,12 @@
## v19.07: (Upcoming Release)
### NVMe-oF Target
Shared receive queue can now be disabled even for NICs that support it using the
`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared
receive queue is predicated on hardware support when this flag is not used.
## v19.04:
### nvme

1
doc/jsonrpc.md

@ -3486,6 +3486,7 @@ max_aq_depth | Optional | number | Max number of admin cmds per
num_shared_buffers | Optional | number | The number of pooled data buffers available to the transport
buf_cache_size | Optional | number | The number of shared buffers to reserve for each poll group
max_srq_depth | Optional | number | The number of elements in a per-thread shared receive queue (RDMA only)
no_srq | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only)
### Example:

19
include/spdk/nvmf.h

@ -64,15 +64,16 @@ struct spdk_json_write_ctx;
struct spdk_nvmf_transport;
struct spdk_nvmf_transport_opts {
uint16_t max_queue_depth;
uint16_t max_qpairs_per_ctrlr;
uint32_t in_capsule_data_size;
uint32_t max_io_size;
uint32_t io_unit_size;
uint32_t max_aq_depth;
uint32_t num_shared_buffers;
uint32_t buf_cache_size;
uint32_t max_srq_depth;
uint16_t max_queue_depth;
uint16_t max_qpairs_per_ctrlr;
uint32_t in_capsule_data_size;
uint32_t max_io_size;
uint32_t io_unit_size;
uint32_t max_aq_depth;
uint32_t num_shared_buffers;
uint32_t buf_cache_size;
uint32_t max_srq_depth;
bool no_srq;
};
/**

4
lib/event/subsystems/nvmf/nvmf_rpc.c

@ -1452,6 +1452,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[]
"max_srq_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_srq_depth),
spdk_json_decode_uint32, true
},
{
"no_srq", offsetof(struct nvmf_rpc_create_transport_ctx, opts.no_srq),
spdk_json_decode_bool, true
},
};
static void

9
lib/nvmf/rdma.c

@ -2043,6 +2043,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
#define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES)
#define SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS 4095
#define SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE 32
#define SPDK_NVMF_RDMA_DEFAULT_NO_SRQ false;
static void
spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
@ -2056,6 +2057,7 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
opts->num_shared_buffers = SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS;
opts->buf_cache_size = SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE;
opts->max_srq_depth = SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH;
opts->no_srq = SPDK_NVMF_RDMA_DEFAULT_NO_SRQ
}
const struct spdk_mem_map_ops g_nvmf_rdma_map_ops = {
@ -2099,7 +2101,7 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts)
" Transport opts: max_ioq_depth=%d, max_io_size=%d,\n"
" max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n"
" in_capsule_data_size=%d, max_aq_depth=%d,\n"
" num_shared_buffers=%d, max_srq_depth=%d\n",
" num_shared_buffers=%d, max_srq_depth=%d, no_srq=%d\n",
opts->max_queue_depth,
opts->max_io_size,
opts->max_qpairs_per_ctrlr,
@ -2107,7 +2109,8 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts)
opts->in_capsule_data_size,
opts->max_aq_depth,
opts->num_shared_buffers,
opts->max_srq_depth);
opts->max_srq_depth,
opts->no_srq);
/* I/O unit size cannot be larger than max I/O size */
if (opts->io_unit_size > opts->max_io_size) {
@ -2936,7 +2939,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport)
TAILQ_INIT(&poller->qpairs);
TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link);
if (device->attr.max_srq != 0) {
if (transport->opts.no_srq == false && device->attr.max_srq != 0) {
poller->max_srq_depth = transport->opts.max_srq_depth;
memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr));

4
scripts/rpc.py

@ -1401,7 +1401,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
max_aq_depth=args.max_aq_depth,
num_shared_buffers=args.num_shared_buffers,
buf_cache_size=args.buf_cache_size,
max_srq_depth=args.max_srq_depth)
max_srq_depth=args.max_srq_depth,
no_srq=args.no_srq)
p = subparsers.add_parser('nvmf_create_transport', help='Create NVMf transport')
p.add_argument('-t', '--trtype', help='Transport type (ex. RDMA)', type=str, required=True)
@ -1414,6 +1415,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
p.add_argument('-n', '--num-shared-buffers', help='The number of pooled data buffers available to the transport', type=int)
p.add_argument('-b', '--buf-cache-size', help='The number of shared buffers to reserve for each poll group', type=int)
p.add_argument('-s', '--max-srq-depth', help='Max number of outstanding I/O per SRQ. Relevant only for RDMA transport', type=int)
p.add_argument('-r', '--no-srq', action='store_true', help='Disable per-thread shared receive queue. Relevant only for RDMA transport')
p.set_defaults(func=nvmf_create_transport)
def get_nvmf_transports(args):

6
scripts/rpc/nvmf.py

@ -45,7 +45,8 @@ def nvmf_create_transport(client,
max_aq_depth=None,
num_shared_buffers=None,
buf_cache_size=None,
max_srq_depth=None):
max_srq_depth=None,
no_srq=False):
"""NVMf Transport Create options.
Args:
@ -59,6 +60,7 @@ def nvmf_create_transport(client,
num_shared_buffers: The number of pooled data buffers available to the transport (optional)
buf_cache_size: The number of shared buffers to reserve for each poll group (optional)
max_srq_depth: Max number of outstanding I/O per shared receive queue - RDMA specific (optional)
no_srq: Boolean flag to disable SRQ even for devices that support it - RDMA specific (optional)
Returns:
True or False
@ -84,6 +86,8 @@ def nvmf_create_transport(client,
params['buf_cache_size'] = buf_cache_size
if max_srq_depth:
params['max_srq_depth'] = max_srq_depth
if no_srq:
params['no_srq'] = no_srq
return client.call('nvmf_create_transport', params)