nvmf: convert nvmf_tgt into an event subsystem

This is required as part of our effort to build a single target app that
can run e.g. NVMe-oF and iSCSI targets in the same process.

Change-Id: Ic8426df08899070d709e3675a57f47f64c23c5eb
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-on: https://review.gerrithub.io/403218
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

app/nvmf_tgt/Makefile

@@ -38,9 +38,9 @@ include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
 
 APP = nvmf_tgt
 
-C_SRCS := conf.c nvmf_main.c nvmf_tgt.c nvmf_rpc.c
+C_SRCS := nvmf_main.c
 
-SPDK_LIB_LIST = event_bdev event_copy
+SPDK_LIB_LIST = event_bdev event_copy event_nvmf
 SPDK_LIB_LIST += nvmf event log trace conf util bdev copy rpc jsonrpc json
 SPDK_LIB_LIST += app_rpc log_rpc bdev_rpc
 

app/nvmf_tgt/nvmf_main.c

@@ -34,7 +34,6 @@
 #include "spdk/stdinc.h"
 
 #include "spdk/env.h"
-#include "nvmf_tgt.h"
 #include "spdk/event.h"
 #include "spdk/log.h"
 
@@ -51,6 +50,15 @@ nvmf_parse_arg(int ch, char *arg)
 {
 }
 
+static void
+nvmf_tgt_started(void *arg1, void *arg2)
+{
+	if (getenv("MEMZONE_DUMP") != NULL) {
+		spdk_memzone_dump(stdout);
+		fflush(stdout);
+	}
+}
+
 int
 main(int argc, char **argv)
 {
@@ -68,7 +76,8 @@ main(int argc, char **argv)
 		exit(rc);
 	}
 
-	rc = spdk_nvmf_tgt_start(&opts);
+	/* Blocks until the application is exiting */
+	rc = spdk_app_start(&opts, nvmf_tgt_started, NULL, NULL);
 
 	return rc;
 }

lib/event/subsystems/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += bdev copy iscsi nbd net scsi vhost
+DIRS-y += bdev copy iscsi nbd net nvmf scsi vhost
 
 .PHONY: all clean $(DIRS-y)
 

lib/event/subsystems/nvmf/Makefile

@@ -0,0 +1,40 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+C_SRCS = conf.c nvmf_rpc.c nvmf_tgt.c
+LIBNAME = event_nvmf
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/event/subsystems/nvmf/conf.c

@@ -31,9 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "spdk/stdinc.h"
-
-#include "nvmf_tgt.h"
+#include "event_nvmf.h"
 
 #include "spdk/conf.h"
 #include "spdk/log.h"

lib/event/subsystems/nvmf/event_nvmf.h

@@ -38,7 +38,8 @@
 
 #include "spdk/nvmf.h"
 #include "spdk/queue.h"
-#include "spdk/event.h"
+
+#include "spdk_internal/event.h"
 
 struct rpc_listen_address {
 	char *transport;

lib/event/subsystems/nvmf/nvmf_rpc.c

@@ -31,7 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "spdk/stdinc.h"
+#include "event_nvmf.h"
 
 #include "spdk/bdev.h"
 #include "spdk/log.h"
@@ -42,7 +42,6 @@
 #include "spdk/string.h"
 #include "spdk/util.h"
 
-#include "nvmf_tgt.h"
 static int
 json_write_hex_str(struct spdk_json_write_ctx *w, const void *data, size_t size)
 {

lib/event/subsystems/nvmf/nvmf_tgt.c

@@ -31,9 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "spdk/stdinc.h"
-
-#include "nvmf_tgt.h"
+#include "event_nvmf.h"
 
 #include "spdk/bdev.h"
 #include "spdk/event.h"
@@ -54,7 +52,7 @@ static size_t g_active_poll_groups = 0;
 
 static struct spdk_poller *g_acceptor_poller = NULL;
 
-static void nvmf_tgt_advance_state(void *arg1, void *arg2);
+static void nvmf_tgt_advance_state(void);
 
 static void
 _spdk_nvmf_shutdown_cb(void *arg1, void *arg2)
@@ -70,16 +68,12 @@ _spdk_nvmf_shutdown_cb(void *arg1, void *arg2)
 	}
 
 	g_tgt.state = NVMF_TGT_FINI_STOP_ACCEPTOR;
-	nvmf_tgt_advance_state(NULL, NULL);
+	nvmf_tgt_advance_state();
 }
 
 static void
-spdk_nvmf_shutdown_cb(void)
+spdk_nvmf_subsystem_fini(void)
 {
-	printf("\n=========================\n");
-	printf("   NVMF shutdown signal\n");
-	printf("=========================\n");
-
 	/* Always let the first core to handle the case */
 	if (spdk_env_get_current_core() != spdk_env_get_first_core()) {
 		spdk_event_call(spdk_event_allocate(spdk_env_get_first_core(),
@@ -151,7 +145,7 @@ static void
 nvmf_tgt_destroy_poll_group_done(void *ctx)
 {
 	g_tgt.state = NVMF_TGT_FINI_FREE_RESOURCES;
-	nvmf_tgt_advance_state(NULL, NULL);
+	nvmf_tgt_advance_state();
 }
 
 static void
@@ -173,7 +167,7 @@ static void
 nvmf_tgt_create_poll_group_done(void *ctx)
 {
 	g_tgt.state = NVMF_TGT_INIT_START_SUBSYSTEMS;
-	nvmf_tgt_advance_state(NULL, NULL);
+	nvmf_tgt_advance_state();
 }
 
 static void
@@ -204,7 +198,7 @@ nvmf_tgt_subsystem_started(struct spdk_nvmf_subsystem *subsystem,
 	}
 
 	g_tgt.state = NVMF_TGT_INIT_START_ACCEPTOR;
-	nvmf_tgt_advance_state(NULL, NULL);
+	nvmf_tgt_advance_state();
 }
 
 static void
@@ -219,11 +213,11 @@ nvmf_tgt_subsystem_stopped(struct spdk_nvmf_subsystem *subsystem,
 	}
 
 	g_tgt.state = NVMF_TGT_FINI_DESTROY_POLL_GROUPS;
-	nvmf_tgt_advance_state(NULL, NULL);
+	nvmf_tgt_advance_state();
 }
 
 static void
-nvmf_tgt_advance_state(void *arg1, void *arg2)
+nvmf_tgt_advance_state(void)
 {
 	enum nvmf_tgt_state prev_state;
 	int rc = -1;
@@ -284,10 +278,7 @@ nvmf_tgt_advance_state(void *arg1, void *arg2)
 			g_tgt.state = NVMF_TGT_RUNNING;
 			break;
 		case NVMF_TGT_RUNNING:
-			if (getenv("MEMZONE_DUMP") != NULL) {
-				spdk_memzone_dump(stdout);
-				fflush(stdout);
-			}
+			spdk_subsystem_init_next(0);
 			break;
 		case NVMF_TGT_FINI_STOP_ACCEPTOR:
 			spdk_poller_unregister(&g_acceptor_poller);
@@ -316,30 +307,23 @@ nvmf_tgt_advance_state(void *arg1, void *arg2)
 			g_tgt.state = NVMF_TGT_STOPPED;
 			break;
 		case NVMF_TGT_STOPPED:
-			spdk_app_stop(0);
+			spdk_subsystem_fini_next();
 			return;
 		case NVMF_TGT_ERROR:
-			spdk_app_stop(rc);
+			spdk_subsystem_init_next(rc);
 			return;
 		}
 
 	} while (g_tgt.state != prev_state);
 }
 
-int
-spdk_nvmf_tgt_start(struct spdk_app_opts *opts)
+static void
+spdk_nvmf_subsystem_init(void)
 {
-	int rc;
-
-	opts->shutdown_cb = spdk_nvmf_shutdown_cb;
-
-	/* Blocks until the application is exiting */
-	rc = spdk_app_start(opts, nvmf_tgt_advance_state, NULL, NULL);
-	if (rc) {
-		SPDK_ERRLOG("spdk_app_start() retn non-zero\n");
-	}
-
-	spdk_app_fini();
-
-	return rc;
+	g_tgt.state = NVMF_TGT_INIT_NONE;
+	nvmf_tgt_advance_state();
 }
+
+SPDK_SUBSYSTEM_REGISTER(nvmf, spdk_nvmf_subsystem_init, spdk_nvmf_subsystem_fini,
+			NULL)
+SPDK_SUBSYSTEM_DEPEND(nvmf, bdev)