d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

subsystem: make subsystem init in async manner

Browse Source 
The next patch will make bdev modules init
in the async manner.

Change-Id: I4909c80510d786daf54003b99a5925428cf37373
Signed-off-by: Ziye Yang <optimistyzy@gmail.com>
Reviewed-on: https://review.gerrithub.io/362110
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Ziye Yang 2017-05-23 12:31:24 +08:00 committed by Jim Harris
parent 8a44220b1a
commit 6e0d1dcdfa
14 changed files with 155 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/spdk/net.h
View File

@ -45,7 +45,7 @@
#define IDLE_INTERVAL_TIME_IN_US 5000
const char *spdk_net_framework_get_name(void);
int spdk_net_framework_start(void);
void spdk_net_framework_start(void);
void spdk_net_framework_clear_socket_association(int sock);
int spdk_net_framework_fini(void);
int spdk_net_framework_idle_time(void);

6
include/spdk_internal/event.h
View File

@ -55,7 +55,8 @@ uint32_t spdk_event_queue_run_batch(uint32_t lcore);
struct spdk_subsystem {
const char *name;
int (*init)(void);
/* User must call spdk_subsystem_init_next() when they are done with their initialization. */
void (*init)(void);
int (*fini)(void);
void (*config)(FILE *fp);
TAILQ_ENTRY(spdk_subsystem) tailq;
@ -71,8 +72,9 @@ struct spdk_subsystem_depend {
void spdk_add_subsystem(struct spdk_subsystem *subsystem);
void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
int spdk_subsystem_init(void);
void spdk_subsystem_init(void);
int spdk_subsystem_fini(void);
void spdk_subsystem_init_next(int rc);
void spdk_subsystem_config(FILE *fp);
/**

20
lib/bdev/bdev.c
View File

@ -258,7 +258,7 @@ spdk_bdev_mgmt_channel_destroy(void *io_device, void *ctx_buf)
{
}
static int
static void
spdk_bdev_initialize(void)
{
int i, cache_size;
@ -274,7 +274,8 @@ spdk_bdev_initialize(void)
if (g_bdev_mgr.bdev_io_pool == NULL) {
SPDK_ERRLOG("could not allocate spdk_bdev_io pool");
return -1;
rc = -1;
goto end;
}
for (i = 0; i < RTE_MAX_LCORE; i++) {
@ -295,7 +296,8 @@ spdk_bdev_initialize(void)
SPDK_ENV_SOCKET_ID_ANY);
if (!g_bdev_mgr.buf_small_pool) {
SPDK_ERRLOG("create rbuf small pool failed\n");
return -1;
rc = -1;
goto end;
}
cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_env_get_core_count());
@ -306,19 +308,22 @@ spdk_bdev_initialize(void)
SPDK_ENV_SOCKET_ID_ANY);
if (!g_bdev_mgr.buf_large_pool) {
SPDK_ERRLOG("create rbuf large pool failed\n");
return -1;
rc = -1;
goto end;
}
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, tailq) {
rc = bdev_module->module_init();
if (rc) {
return rc;
rc = -1;
goto end;
}
}
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.vbdev_modules, tailq) {
rc = bdev_module->module_init();
if (rc) {
return rc;
rc = -1;
goto end;
}
}
@ -326,7 +331,8 @@ spdk_bdev_initialize(void)
spdk_bdev_mgmt_channel_destroy,
sizeof(struct spdk_bdev_mgmt_channel));
return 0;
end:
spdk_subsystem_init_next(rc);
}
static int

5
lib/copy/copy_engine.c
View File

@ -237,7 +237,7 @@ spdk_copy_engine_module_finish(void)
}
}
static int
static void
spdk_copy_engine_initialize(void)
{
spdk_copy_engine_module_initialize();
@ -247,7 +247,8 @@ spdk_copy_engine_initialize(void)
*/
spdk_io_device_register(&spdk_copy_module_list, copy_create_cb, copy_destroy_cb,
sizeof(struct copy_io_channel));
return 0;
spdk_subsystem_init_next(0);
}
static int

14
lib/event/app.c
View File

@ -410,13 +410,6 @@ spdk_app_init(struct spdk_app_opts *opts)
spdk_trace_set_tpoint_group_mask(tpoint_group_mask);
}
}
rc = spdk_subsystem_init();
if (rc < 0) {
SPDK_ERRLOG("spdk_subsystem_init() failed\n");
spdk_conf_free(g_spdk_app.config);
exit(EXIT_FAILURE);
}
}
int
@ -440,6 +433,13 @@ spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2)
g_spdk_app.rc = 0;
spdk_subsystem_init();
/* Early return if there is error */
if (g_spdk_app.rc) {
return g_spdk_app.rc;
}
event = spdk_event_allocate(rte_get_master_lcore(), start_fn,
arg1, arg2);
/* Queues up the event, but can't run it until the reactors start */

47
lib/event/subsystem.c
View File

@ -41,6 +41,7 @@ static TAILQ_HEAD(spdk_subsystem_list, spdk_subsystem) g_subsystems =
TAILQ_HEAD_INITIALIZER(g_subsystems);
static TAILQ_HEAD(subsystem_depend, spdk_subsystem_depend) g_depends =
TAILQ_HEAD_INITIALIZER(g_depends);
static struct spdk_subsystem *g_next_subsystem;
void
spdk_add_subsystem(struct spdk_subsystem *subsystem)
@ -108,36 +109,56 @@ subsystem_sort(void)
}
}
int
void
spdk_subsystem_init_next(int rc)
{
if (rc) {
spdk_app_stop(rc);
assert(g_next_subsystem != NULL);
SPDK_ERRLOG("Init subsystem %s failed\n", g_next_subsystem->name);
return;
}
if (!g_next_subsystem) {
g_next_subsystem = TAILQ_FIRST(&g_subsystems);
} else {
g_next_subsystem = TAILQ_NEXT(g_next_subsystem, tailq);
}
if (!g_next_subsystem) {
return;
}
if (g_next_subsystem->init) {
g_next_subsystem->init();
} else {
spdk_subsystem_init_next(0);
}
}
void
spdk_subsystem_init(void)
{
int rc = 0;
struct spdk_subsystem *subsystem;
struct spdk_subsystem_depend *dep;
/* Verify that all dependency name and depends_on subsystems are registered */
TAILQ_FOREACH(dep, &g_depends, tailq) {
if (!spdk_subsystem_find(&g_subsystems, dep->name)) {
SPDK_ERRLOG("subsystem %s is missing\n", dep->name);
return -1;
spdk_app_stop(-1);
return;
}
if (!spdk_subsystem_find(&g_subsystems, dep->depends_on)) {
SPDK_ERRLOG("subsystem %s dependency %s is missing\n",
dep->name, dep->depends_on);
return -1;
spdk_app_stop(-1);
return;
}
}
subsystem_sort();
TAILQ_FOREACH(subsystem, &g_subsystems, tailq) {
if (subsystem->init) {
rc = subsystem->init();
if (rc)
return rc;
}
}
return rc;
spdk_subsystem_init_next(0);
}
int

19
lib/iscsi/iscsi_subsystem.c
View File

@ -963,33 +963,37 @@ spdk_iscsi_setup(void *arg1, void *arg2)
spdk_iscsi_acceptor_start();
}
static int
static void
spdk_iscsi_subsystem_init(void)
{
int rc;
int rc = 0;
rc = spdk_iscsi_app_read_parameters();
if (rc < 0) {
SPDK_ERRLOG("spdk_iscsi_app_read_parameters() failed\n");
return -1;
rc = -1;
goto end;
}
rc = spdk_iscsi_initialize_all_pools();
if (rc != 0) {
SPDK_ERRLOG("spdk_initialize_all_pools() failed\n");
return -1;
rc = -1;
goto end;
}
rc = spdk_iscsi_init_tgt_nodes();
if (rc < 0) {
SPDK_ERRLOG("spdk_iscsi_init_tgt_nodes() failed\n");
return -1;
rc = -1;
goto end;
}
rc = spdk_initialize_iscsi_conns();
if (rc < 0) {
SPDK_ERRLOG("spdk_initialize_iscsi_conns() failed\n");
return -1;
rc = -1;
goto end;
}
/*
@ -997,7 +1001,8 @@ spdk_iscsi_subsystem_init(void)
*/
spdk_event_call(spdk_event_allocate(spdk_env_get_current_core(), spdk_iscsi_setup, NULL, NULL));
return 0;
end:
spdk_subsystem_init_next(rc);
}
static int

9
lib/net/interface.c
View File

@ -414,13 +414,14 @@ static void spdk_interface_ip_update(void)
pthread_mutex_unlock(&interface_lock);
}
static int
static void
spdk_interface_init(void)
{
TAILQ_INIT(&g_interface_head);
spdk_prepare_ifc_list();
spdk_get_ifc_ipv4();
return 0;
spdk_subsystem_init_next(0);
}
static int spdk_interface_destroy(void)
@ -461,10 +462,10 @@ void *spdk_interface_get_list(void)
#else /* Not Linux */
static int
static void
spdk_interface_init(void)
{
return 0;
spdk_subsystem_init_next(0);
}
static int

4
lib/net/net_framework_default.c
View File

@ -42,9 +42,9 @@ const char *spdk_net_framework_get_name(void)
}
__attribute__((weak))
int spdk_net_framework_start(void)
void spdk_net_framework_start(void)
{
return 0;
spdk_subsystem_init_next(0);
}
__attribute__((weak))

5
lib/rpc/rpc.c
View File

@ -225,7 +225,7 @@ spdk_rpc_setup(void *arg)
RPC_SELECT_INTERVAL);
}
static int
static void
spdk_rpc_initialize(void)
{
/*
@ -235,7 +235,8 @@ spdk_rpc_initialize(void)
* or RPC commands.
*/
spdk_poller_register(&g_rpc_poller, spdk_rpc_setup, NULL, spdk_env_get_current_core(), 0);
return 0;
spdk_subsystem_init_next(0);
}
static int

11
lib/scsi/scsi.c
View File

@ -97,24 +97,25 @@ spdk_read_config_scsi_parameters(void)
return 0;
}
static int
static void
spdk_scsi_subsystem_init(void)
{
int rc;
int rc = 0;
rc = pthread_mutex_init(&g_spdk_scsi.mutex, NULL);
if (rc != 0) {
SPDK_ERRLOG("mutex_init() failed\n");
return -1;
goto end;
}
rc = spdk_read_config_scsi_parameters();
if (rc < 0) {
SPDK_ERRLOG("spdk_scsi_parameters() failed\n");
return -1;
rc = -1;
}
return rc;
end:
spdk_subsystem_init_next(rc);
}
static int

13
lib/vhost/task.c
View File

@ -129,14 +129,17 @@ spdk_vhost_iovec_free(struct iovec *iov)
rte_mempool_put(g_iov_buffer_pool, iov);
}
static int
static void
spdk_vhost_subsystem_init(void)
{
int rc = 0;
g_task_pool = rte_mempool_create("vhost task pool", 16384, sizeof(struct spdk_vhost_task),
128, 0, NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
if (!g_task_pool) {
SPDK_ERRLOG("create task pool failed\n");
return -1;
rc = -1;
goto end;
}
g_iov_buffer_pool = rte_mempool_create("vhost iov buffer pool", 2048,
@ -144,14 +147,16 @@ spdk_vhost_subsystem_init(void)
128, 0, NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
if (!g_iov_buffer_pool) {
SPDK_ERRLOG("create iov buffer pool failed\n");
return -1;
rc = -1;
goto end;
}
for (int i = 0; i < RTE_MAX_LCORE; i++) {
TAILQ_INIT(&g_need_iovecs[i]);
}
return 0;
end:
spdk_subsystem_init_next(rc);
}
static int

17
test/lib/event/subsystem/subsystem_ut.c
View File

@ -39,6 +39,13 @@
static struct spdk_subsystem g_ut_subsystems[8];
static struct spdk_subsystem_depend g_ut_subsystem_deps[8];
static int global_rc;
void
spdk_app_stop(int rc)
{
global_rc = rc;
}
static void
set_up_subsystem(struct spdk_subsystem *subsystem, const char *name)
@ -79,6 +86,7 @@ subsystem_sort_test_depends_on_single(void)
int i;
char subsystem_name[16];
global_rc = -1;
spdk_subsystem_init();
i = 4;
@ -122,6 +130,7 @@ subsystem_sort_test_depends_on_multiple(void)
spdk_add_subsystem_depend(&g_ut_subsystem_deps[i]);
}
global_rc = -1;
spdk_subsystem_init();
subsystem = TAILQ_FIRST(&g_subsystems);
@ -180,7 +189,9 @@ subsystem_sort_test_missing_dependency(void)
set_up_depends(&g_ut_subsystem_deps[0], "A", "B");
spdk_add_subsystem_depend(&g_ut_subsystem_deps[0]);
CU_ASSERT(spdk_subsystem_init() != 0);
global_rc = -1;
spdk_subsystem_init();
CU_ASSERT(global_rc != 0);
/*
* Dependency from C to A is defined, but C is missing
@ -193,7 +204,9 @@ subsystem_sort_test_missing_dependency(void)
set_up_depends(&g_ut_subsystem_deps[0], "C", "A");
spdk_add_subsystem_depend(&g_ut_subsystem_deps[0]);
CU_ASSERT(spdk_subsystem_init() != 0);
global_rc = -1;
spdk_subsystem_init();
CU_ASSERT(global_rc != 0);
}

74
test/lib/scsi/init/init_ut.c
View File

@ -43,6 +43,8 @@
/* Unit test stubbed bdev subsystem dependency */
SPDK_SUBSYSTEM_REGISTER(bdev, NULL, NULL, NULL)
static int global_rc;
static int
null_init(void)
{
@ -65,6 +67,12 @@ spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend)
{
}
void
spdk_subsystem_init_next(int rc)
{
global_rc = rc;
}
static struct spdk_conf *
spdk_config_init_scsi_params(char *key, char *value)
{
@ -114,17 +122,17 @@ static void
scsi_init_sp_null(void)
{
struct spdk_conf *config;
int rc;
config = spdk_conf_allocate();
SPDK_CU_ASSERT_FATAL(config != NULL);
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* sp = null; set default scsi params */
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_set_as_default(NULL);
@ -136,19 +144,19 @@ scsi_init_set_max_unmap_lba_count_config_param(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.max_unmap_lba_count = 65536 of Scsi section */
config = spdk_config_init_scsi_params("MaxUnmapLbaCount", "65536");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.max_unmap_lba_count == 65536 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.max_unmap_lba_count = 65536;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -158,20 +166,20 @@ scsi_init_set_max_unmap_block_descriptor_count_config_param(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.max_unmap_block_descriptor_count = 1
* of Scsi section */
config = spdk_config_init_scsi_params("MaxUnmapBlockDescriptorCount", "1");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.max_unmap_block_descriptor_count == 1 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.max_unmap_block_descriptor_count = 1;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -181,20 +189,20 @@ scsi_init_set_optimal_unmap_granularity_config_param(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.optimal_unmap_granularity = 0
* of Scsi section */
config = spdk_config_init_scsi_params("OptimalUnmapGranularity", "0");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.optimal_unmap_granularity == 0 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.optimal_unmap_granularity = 0;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -204,20 +212,20 @@ scsi_init_set_unmap_granularity_alignment_config_param(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.unmap_granularity_alignment = 0
* of Scsi section */
config = spdk_config_init_scsi_params("UnmapGranularityAlignment", "0");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.unmap_granularity_alignment == 0 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.unmap_granularity_alignment = 0;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -227,20 +235,20 @@ scsi_init_ugavalid_yes(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.ugavalid = Yes
* of Scsi section */
config = spdk_config_init_scsi_params("Ugavalid", "Yes");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.ugavalid == 1 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.ugavalid = 1;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -250,20 +258,20 @@ scsi_init_ugavalid_no(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.ugavalid = No
* of Scsi section */
config = spdk_config_init_scsi_params("Ugavalid", "No");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.ugavalid == 0 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.ugavalid = 0;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -272,15 +280,15 @@ static void
scsi_init_ugavalid_unknown_value_failure(void)
{
struct spdk_scsi_parameters params;
int rc;
struct spdk_conf *config;
/* set scsi_params.ugavalid = unknown value
* of Scsi section */
config = spdk_config_init_scsi_params("Ugavalid", "unknown value");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
CU_ASSERT_EQUAL(rc, 0);
global_rc = -1;
spdk_scsi_subsystem_init();
CU_ASSERT_EQUAL(global_rc, 0);
/* Assert the scsi_params.ugavalid == DEFAULT_UGAVALID and
* assert the rest of the params are set to their default values */
@ -296,20 +304,20 @@ scsi_init_max_write_same_length(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* set scsi_params.max_write_same_length = 512
* of Scsi section */
config = spdk_config_init_scsi_params("MaxWriteSameLength", "512");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Assert the scsi_params.max_write_same_length == 512 and
* assert the rest of the params are set to their default values */
set_default_scsi_params(&params);
params.max_write_same_length = 512;
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -319,19 +327,19 @@ scsi_init_read_config_scsi_params(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* Set null for item's key and value;
* set default scsi parameters */
config = spdk_config_init_scsi_params("", "");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Sets the default values for all the parameters
* of the Scsi section and returns success */
set_default_scsi_params(&params);
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}
@ -341,20 +349,20 @@ scsi_init_success(void)
{
struct spdk_scsi_parameters params;
struct spdk_conf *config;
int rc;
/* Set null for item's key and value;
* set default scsi parameters */
config = spdk_config_init_scsi_params("", "");
spdk_conf_set_as_default(config);
rc = spdk_scsi_subsystem_init();
global_rc = -1;
spdk_scsi_subsystem_init();
/* Sets the default values for all the parameters
* of the Scsi section, initialize th device
* and returns success */
set_default_scsi_params(&params);
CU_ASSERT(memcmp(&g_spdk_scsi.scsi_params, &params, sizeof(params)) == 0);
CU_ASSERT_EQUAL(rc, 0);
CU_ASSERT_EQUAL(global_rc, 0);
spdk_conf_free(config);
}