d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

iscsi: Set resource pool size to the predetermined values to run with little memory

Browse Source 
Add three parameters, pdu_pool_size, immediate_data_pool_size, and
data_out_pool_size to the RPC iscsi_set_options to run iSCSI target
with little memory.

For some use cases, we want to keep the max number of connections,
but simultaneously we want to reduce the pool size and let I/Os wait
until resource is provided.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I74dc785310b1d985f3e338c1e13fba3a3840d113
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8191
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Shuhei Matsumoto 2021-06-09 12:24:06 +09:00 committed by Tomasz Zawadzki
parent db8b121db0
commit 11d76e8c81
8 changed files with 83 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -29,6 +29,12 @@ Added API `spdk_idxd_submit_copy_crc32c` to perform a CRC32C while copying data.
Added API `spdk_idxd_batch_prep_copy_crc32c` to prepare a batch operation to perform
a CRC32C while copying data.
### iscsi
New parameters, `pdu_pool_size`, `immediate_data_pool_size`, and `data_out_pool_size`,
were added to the RPC `iscsi_set_options` to run iSCSI target with varying amount of
available memory.
### util
`spdk_crc32c_iov_update` function was added to support calculating the crc32c of the

3
doc/jsonrpc.md
View File

@ -4618,6 +4618,9 @@ error_recovery_level | Optional | number | Session specific paramete
allow_duplicated_isid | Optional | boolean | Allow duplicated initiator session ID (default: `false`)
max_large_datain_per_connection | Optional | number | Max number of outstanding split read I/Os per connection (default: 64)
max_r2t_per_connection | Optional | number | Max number of outstanding R2Ts per connection (default: 4)
pdu_pool_size | Optional | number | Number of PDUs in the pool (default: approximately 2 * max_sessions * (max_queue_depth + max_connections_per_session))
immediate_data_pool_size | Optional | number | Number of immediate data buffers in the pool (default: 128 * max_sessions)
data_out_pool_size | Optional | number | Number of data out buffers in the pool (default: 16 * max_sessions)
To load CHAP shared secret file, its path is required to specify explicitly in the parameter `auth_file`.

6
lib/iscsi/iscsi.h
View File

@ -339,6 +339,9 @@ struct spdk_iscsi_opts {
bool AllowDuplicateIsid;
uint32_t MaxLargeDataInPerConnection;
uint32_t MaxR2TPerConnection;
uint32_t pdu_pool_size;
uint32_t immediate_data_pool_size;
uint32_t data_out_pool_size;
};
struct spdk_iscsi_globals {
@ -372,6 +375,9 @@ struct spdk_iscsi_globals {
bool AllowDuplicateIsid;
uint32_t MaxLargeDataInPerConnection;
uint32_t MaxR2TPerConnection;
uint32_t pdu_pool_size;
uint32_t immediate_data_pool_size;
uint32_t data_out_pool_size;
struct spdk_mempool *pdu_pool;
struct spdk_mempool *pdu_immediate_data_pool;

3
lib/iscsi/iscsi_rpc.c
View File

@ -1701,6 +1701,9 @@ static const struct spdk_json_object_decoder rpc_set_iscsi_opts_decoders[] = {
{"allow_duplicated_isid", offsetof(struct spdk_iscsi_opts, AllowDuplicateIsid), spdk_json_decode_bool, true},
{"max_large_datain_per_connection", offsetof(struct spdk_iscsi_opts, MaxLargeDataInPerConnection), spdk_json_decode_uint32, true},
{"max_r2t_per_connection", offsetof(struct spdk_iscsi_opts, MaxR2TPerConnection), spdk_json_decode_uint32, true},
{"pdu_pool_size", offsetof(struct spdk_iscsi_opts, pdu_pool_size), spdk_json_decode_uint32, true},
{"immediate_data_pool_size", offsetof(struct spdk_iscsi_opts, immediate_data_pool_size), spdk_json_decode_uint32, true},
{"data_out_pool_size", offsetof(struct spdk_iscsi_opts, data_out_pool_size), spdk_json_decode_uint32, true},
};
static void

55
lib/iscsi/iscsi_subsystem.c
View File

@ -68,13 +68,6 @@ mobj_ctor(struct spdk_mempool *mp, __attribute__((unused)) void *arg,
~ISCSI_DATA_BUFFER_MASK);
}
#define NUM_PDU_PER_CONNECTION(iscsi) (2 * (iscsi->MaxQueueDepth + \
iscsi->MaxLargeDataInPerConnection + \
2 * iscsi->MaxR2TPerConnection + 8))
#define PDU_POOL_SIZE(iscsi) (iscsi->MaxConnections * NUM_PDU_PER_CONNECTION(iscsi))
#define IMMEDIATE_DATA_POOL_SIZE(iscsi) (iscsi->MaxConnections * 128)
#define DATA_OUT_POOL_SIZE(iscsi) (iscsi->MaxConnections * MAX_DATA_OUT_PER_CONNECTION)
static int
iscsi_initialize_pdu_pool(void)
{
@ -86,7 +79,7 @@ iscsi_initialize_pdu_pool(void)
/* create PDU pool */
iscsi->pdu_pool = spdk_mempool_create("PDU_Pool",
PDU_POOL_SIZE(iscsi),
iscsi->pdu_pool_size,
sizeof(struct spdk_iscsi_pdu),
256, SPDK_ENV_SOCKET_ID_ANY);
if (!iscsi->pdu_pool) {
@ -95,7 +88,7 @@ iscsi_initialize_pdu_pool(void)
}
iscsi->pdu_immediate_data_pool = spdk_mempool_create_ctor("PDU_immediate_data_Pool",
IMMEDIATE_DATA_POOL_SIZE(iscsi),
iscsi->immediate_data_pool_size,
imm_mobj_size, 256,
SPDK_ENV_SOCKET_ID_ANY,
mobj_ctor, NULL);
@ -105,7 +98,7 @@ iscsi_initialize_pdu_pool(void)
}
iscsi->pdu_data_out_pool = spdk_mempool_create_ctor("PDU_data_out_Pool",
DATA_OUT_POOL_SIZE(iscsi),
iscsi->data_out_pool_size,
dout_mobj_size, 256,
SPDK_ENV_SOCKET_ID_ANY,
mobj_ctor, NULL);
@ -201,10 +194,10 @@ iscsi_check_pools(void)
{
struct spdk_iscsi_globals *iscsi = &g_iscsi;
iscsi_check_pool(iscsi->pdu_pool, PDU_POOL_SIZE(iscsi));
iscsi_check_pool(iscsi->pdu_pool, iscsi->pdu_pool_size);
iscsi_check_pool(iscsi->session_pool, SESSION_POOL_SIZE(iscsi));
iscsi_check_pool(iscsi->pdu_immediate_data_pool, IMMEDIATE_DATA_POOL_SIZE(iscsi));
iscsi_check_pool(iscsi->pdu_data_out_pool, DATA_OUT_POOL_SIZE(iscsi));
iscsi_check_pool(iscsi->pdu_immediate_data_pool, iscsi->immediate_data_pool_size);
iscsi_check_pool(iscsi->pdu_data_out_pool, iscsi->data_out_pool_size);
iscsi_check_pool(iscsi->task_pool, DEFAULT_TASK_POOL_SIZE);
}
@ -319,6 +312,13 @@ iscsi_log_globals(void)
g_iscsi.MaxR2TPerConnection);
}
#define NUM_PDU_PER_CONNECTION(opts) (2 * (opts->MaxQueueDepth + \
opts->MaxLargeDataInPerConnection + \
2 * opts->MaxR2TPerConnection + 8))
#define PDU_POOL_SIZE(opts) (opts->MaxSessions * NUM_PDU_PER_CONNECTION(opts))
#define IMMEDIATE_DATA_POOL_SIZE(opts) (opts->MaxSessions * 128)
#define DATA_OUT_POOL_SIZE(opts) (opts->MaxSessions * MAX_DATA_OUT_PER_CONNECTION)
static void
iscsi_opts_init(struct spdk_iscsi_opts *opts)
{
@ -341,6 +341,9 @@ iscsi_opts_init(struct spdk_iscsi_opts *opts)
opts->nodebase = NULL;
opts->MaxLargeDataInPerConnection = DEFAULT_MAX_LARGE_DATAIN_PER_CONNECTION;
opts->MaxR2TPerConnection = DEFAULT_MAXR2T;
opts->pdu_pool_size = PDU_POOL_SIZE(opts);
opts->immediate_data_pool_size = IMMEDIATE_DATA_POOL_SIZE(opts);
opts->data_out_pool_size = DATA_OUT_POOL_SIZE(opts);
}
struct spdk_iscsi_opts *
@ -415,6 +418,9 @@ iscsi_opts_copy(struct spdk_iscsi_opts *src)
dst->chap_group = src->chap_group;
dst->MaxLargeDataInPerConnection = src->MaxLargeDataInPerConnection;
dst->MaxR2TPerConnection = src->MaxR2TPerConnection;
dst->pdu_pool_size = src->pdu_pool_size;
dst->immediate_data_pool_size = src->immediate_data_pool_size;
dst->data_out_pool_size = src->data_out_pool_size;
return dst;
}
@ -504,6 +510,21 @@ iscsi_opts_verify(struct spdk_iscsi_opts *opts)
return -EINVAL;
}
if (opts->pdu_pool_size == 0) {
SPDK_ERRLOG("0 is invalid. pdu_pool_size must be more than 0\n");
return -EINVAL;
}
if (opts->immediate_data_pool_size == 0) {
SPDK_ERRLOG("0 is invalid. immediate_data_pool_size must be more than 0\n");
return -EINVAL;
}
if (opts->data_out_pool_size == 0) {
SPDK_ERRLOG("0 is invalid. data_out_pool_size must be more than 0\n");
return -EINVAL;
}
return 0;
}
@ -549,6 +570,9 @@ iscsi_set_global_params(struct spdk_iscsi_opts *opts)
g_iscsi.chap_group = opts->chap_group;
g_iscsi.MaxLargeDataInPerConnection = opts->MaxLargeDataInPerConnection;
g_iscsi.MaxR2TPerConnection = opts->MaxR2TPerConnection;
g_iscsi.pdu_pool_size = opts->pdu_pool_size;
g_iscsi.immediate_data_pool_size = opts->immediate_data_pool_size;
g_iscsi.data_out_pool_size = opts->data_out_pool_size;
iscsi_log_globals();
@ -1252,6 +1276,11 @@ iscsi_opts_info_json(struct spdk_json_write_ctx *w)
spdk_json_write_named_uint32(w, "max_r2t_per_connection",
g_iscsi.MaxR2TPerConnection);
spdk_json_write_named_uint32(w, "pdu_pool_size", g_iscsi.pdu_pool_size);
spdk_json_write_named_uint32(w, "immediate_data_pool_size",
g_iscsi.immediate_data_pool_size);
spdk_json_write_named_uint32(w, "data_out_pool_size", g_iscsi.data_out_pool_size);
spdk_json_write_object_end(w);
}

8
scripts/rpc.py
View File

@ -975,7 +975,10 @@ if __name__ == "__main__":
error_recovery_level=args.error_recovery_level,
allow_duplicated_isid=args.allow_duplicated_isid,
max_large_datain_per_connection=args.max_large_datain_per_connection,
max_r2t_per_connection=args.max_r2t_per_connection)
max_r2t_per_connection=args.max_r2t_per_connection,
pdu_pool_size=args.pdu_pool_size,
immediate_data_pool_size=args.immediate_data_pool_size,
data_out_pool_size=args.data_out_pool_size)
p = subparsers.add_parser('iscsi_set_options', aliases=['set_iscsi_options'],
help="""Set options of iSCSI subsystem""")
@ -1001,6 +1004,9 @@ if __name__ == "__main__":
p.add_argument('-p', '--allow-duplicated-isid', help='Allow duplicated initiator session ID.', action='store_true')
p.add_argument('-x', '--max-large-datain-per-connection', help='Max number of outstanding split read I/Os per connection', type=int)
p.add_argument('-k', '--max-r2t-per-connection', help='Max number of outstanding R2Ts per connection', type=int)
p.add_argument('-u', '--pdu-pool-size', help='Number of PDUs in the pool', type=int)
p.add_argument('-j', '--immediate_data-pool-size', help='Number of immediate data buffers in the pool', type=int)
p.add_argument('-z', '--data-out-pool-size', help='Number of data out buffers in the pool', type=int)
p.set_defaults(func=iscsi_set_options)
def iscsi_set_discovery_auth(args):

14
scripts/rpc/iscsi.py
View File

@ -22,7 +22,10 @@ def iscsi_set_options(
error_recovery_level=None,
allow_duplicated_isid=None,
max_large_datain_per_connection=None,
max_r2t_per_connection=None):
max_r2t_per_connection=None,
pdu_pool_size=None,
immediate_data_pool_size=None,
data_out_pool_size=None):
"""Set iSCSI target options.
Args:
@ -45,6 +48,9 @@ def iscsi_set_options(
allow_duplicated_isid: Allow duplicated initiator session ID
max_large_datain_per_connection: Max number of outstanding split read I/Os per connection (optional)
max_r2t_per_connection: Max number of outstanding R2Ts per connection (optional)
pdu_pool_size: Number of PDUs in the pool (optional)
immediate_data_pool_size: Number of immediate data buffers in the pool (optional)
data_out_pool_size: Number of data out buffers in the pool (optional)
Returns:
True or False
@ -89,6 +95,12 @@ def iscsi_set_options(
params['max_large_datain_per_connection'] = max_large_datain_per_connection
if max_r2t_per_connection:
params['max_r2t_per_connection'] = max_r2t_per_connection
if pdu_pool_size:
params['pdu_pool_size'] = pdu_pool_size
if immediate_data_pool_size:
params['immediate_data_pool_size'] = immediate_data_pool_size
if data_out_pool_size:
params['data_out_pool_size'] = data_out_pool_size
return client.call('iscsi_set_options', params)

3
test/spdkcli/match_files/spdkcli_iscsi.test.match
View File

@ -9,12 +9,14 @@ o- iscsi .......................................................................
o- global_params ........................................................................................................... [...]
| o- allow_duplicated_isid: False .......................................................................................... [...]
| o- chap_group: 1 ......................................................................................................... [...]
| o- data_out_pool_size: 2048 .............................................................................................. [...]
| o- default_time2retain: 20 ............................................................................................... [...]
| o- default_time2wait: 2 .................................................................................................. [...]
| o- disable_chap: True .................................................................................................... [...]
| o- error_recovery_level: 0 ............................................................................................... [...]
| o- first_burst_length: 8192 .............................................................................................. [...]
| o- immediate_data: True .................................................................................................. [...]
| o- immediate_data_pool_size: 16384 ....................................................................................... [...]
| o- max_connections_per_session: 2 ........................................................................................ [...]
| o- max_large_datain_per_connection: 64 ................................................................................... [...]
| o- max_queue_depth: 64 ................................................................................................... [...]
@ -24,6 +26,7 @@ o- iscsi .......................................................................
| o- node_base: iqn.2016-06.io.spdk ........................................................................................ [...]
| o- nop_in_interval: 30 ................................................................................................... [...]
| o- nop_timeout: 60 ....................................................................................................... [...]
| o- pdu_pool_size: 36864 .................................................................................................. [...]
| o- require_chap: False ................................................................................................... [...]
o- initiator_groups ........................................................................................ [Initiator groups: 2]
| o- initiator_group2 ............................................................................................ [Initiators: 2]