module/raid: raid level specific start/stop callbacks

Add a function pointer to raid_bdev_module to be called before the bdev
is registered and a corresponding one to be called before the bdev is
unregistered. This allows setting up the bdev parameters and any custom
initialization/cleanup required for the raid module.

Change-Id: Ib9fe8f0365ca47f499a50630f582399e7bb9fd0f
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/472714
Community-CI: Broadcom SPDK FC-NVMe CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

module/bdev/raid/bdev_raid.c

@@ -272,6 +272,9 @@ raid_bdev_destruct(void *ctxt)
 
 	if (g_shutdown_started) {
 		TAILQ_REMOVE(&g_raid_bdev_configured_list, raid_bdev, state_link);
+		if (raid_bdev->module->stop != NULL) {
+			raid_bdev->module->stop(raid_bdev);
+		}
 		raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
 		TAILQ_INSERT_TAIL(&g_raid_bdev_offline_list, raid_bdev, state_link);
 	}
@@ -1301,7 +1304,6 @@ static int
 raid_bdev_configure(struct raid_bdev *raid_bdev)
 {
 	uint32_t		blocklen;
-	uint64_t		min_blockcnt;
 	struct spdk_bdev	*raid_bdev_gen;
 	int rc = 0;
 	uint8_t i;
@@ -1310,13 +1312,7 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
 	assert(raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs);
 
 	blocklen = raid_bdev->base_bdev_info[0].bdev->blocklen;
-	min_blockcnt = raid_bdev->base_bdev_info[0].bdev->blockcnt;
 	for (i = 1; i < raid_bdev->num_base_bdevs; i++) {
-		/* Calculate minimum block count from all base bdevs */
-		if (raid_bdev->base_bdev_info[i].bdev->blockcnt < min_blockcnt) {
-			min_blockcnt = raid_bdev->base_bdev_info[i].bdev->blockcnt;
-		}
-
 		/* Check blocklen for all base bdevs that it should be same */
 		if (blocklen != raid_bdev->base_bdev_info[i].bdev->blocklen) {
 			/*
@@ -1337,36 +1333,25 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
 
 	raid_bdev_gen = &raid_bdev->bdev;
 	raid_bdev_gen->blocklen = blocklen;
-	if (raid_bdev->num_base_bdevs > 1) {
+
-		raid_bdev_gen->optimal_io_boundary = raid_bdev->strip_size;
+	rc = raid_bdev->module->start(raid_bdev);
-		raid_bdev_gen->split_on_optimal_io_boundary = true;
+	if (rc != 0) {
-	} else {
+		SPDK_ERRLOG("raid module startup callback failed\n");
-		/* Do not need to split reads/writes on single bdev RAID modules. */
+		return rc;
-		raid_bdev_gen->optimal_io_boundary = 0;
-		raid_bdev_gen->split_on_optimal_io_boundary = false;
 	}
-
-	/*
-	 * RAID bdev logic is for striping so take the minimum block count based
-	 * approach where total block count of raid bdev is the number of base
-	 * bdev times the minimum block count of any base bdev
-	 */
-	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID, "min blockcount %lu,  numbasedev %u, strip size shift %u\n",
-		      min_blockcnt,
-		      raid_bdev->num_base_bdevs, raid_bdev->strip_size_shift);
-	raid_bdev_gen->blockcnt = ((min_blockcnt >> raid_bdev->strip_size_shift) <<
-				   raid_bdev->strip_size_shift)  * raid_bdev->num_base_bdevs;
-	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID, "io device register %p\n", raid_bdev);
-	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID, "blockcnt %lu, blocklen %u\n", raid_bdev_gen->blockcnt,
-		      raid_bdev_gen->blocklen);
-
 	raid_bdev->state = RAID_BDEV_STATE_ONLINE;
+	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID, "io device register %p\n", raid_bdev);
+	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID, "blockcnt %lu, blocklen %u\n",
+		      raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
 	spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
 				sizeof(struct raid_bdev_io_channel),
 				raid_bdev->bdev.name);
 	rc = spdk_bdev_register(raid_bdev_gen);
 	if (rc != 0) {
 		SPDK_ERRLOG("Unable to register raid bdev and stay at configuring state\n");
+		if (raid_bdev->module->stop != NULL) {
+			raid_bdev->module->stop(raid_bdev);
+		}
 		spdk_io_device_unregister(raid_bdev, NULL);
 		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
 		return rc;
@@ -1405,6 +1390,9 @@ raid_bdev_deconfigure(struct raid_bdev *raid_bdev, raid_bdev_destruct_cb cb_fn,
 
 	assert(raid_bdev->num_base_bdevs == raid_bdev->num_base_bdevs_discovered);
 	TAILQ_REMOVE(&g_raid_bdev_configured_list, raid_bdev, state_link);
+	if (raid_bdev->module->stop != NULL) {
+		raid_bdev->module->stop(raid_bdev);
+	}
 	raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
 	assert(raid_bdev->num_base_bdevs_discovered);
 	TAILQ_INSERT_TAIL(&g_raid_bdev_offline_list, raid_bdev, state_link);

module/bdev/raid/bdev_raid.h

@@ -253,6 +253,21 @@ struct raid_bdev_module {
 	/* RAID level implemented by this module */
 	enum raid_level level;
 
+	/*
+	 * Called when the raid is starting, right before changing the state to
+	 * online and registering the bdev. Parameters of the bdev like blockcnt
+	 * should be set here.
+	 *
+	 * Non-zero return value will abort the startup process.
+	 */
+	int (*start)(struct raid_bdev *raid_bdev);
+
+	/*
+	 * Called when the raid is stopping, right before changing the state to
+	 * offline and unregistering the bdev. Optional.
+	 */
+	void (*stop)(struct raid_bdev *raid_bdev);
+
 	/* Handler for R/W requests */
 	void (*submit_rw_request)(struct raid_bdev_io *raid_io);
 

module/bdev/raid/raid0.c

@@ -340,8 +340,44 @@ raid0_submit_null_payload_request(struct raid_bdev_io *raid_io)
 	}
 }
 
+static int raid0_start(struct raid_bdev *raid_bdev)
+{
+	uint64_t min_blockcnt;
+	uint8_t i;
+
+	min_blockcnt = raid_bdev->base_bdev_info[0].bdev->blockcnt;
+	for (i = 1; i < raid_bdev->num_base_bdevs; i++) {
+		/* Calculate minimum block count from all base bdevs */
+		if (raid_bdev->base_bdev_info[i].bdev->blockcnt < min_blockcnt) {
+			min_blockcnt = raid_bdev->base_bdev_info[i].bdev->blockcnt;
+		}
+	}
+
+	/*
+	 * Take the minimum block count based approach where total block count
+	 * of raid bdev is the number of base bdev times the minimum block count
+	 * of any base bdev.
+	 */
+	SPDK_DEBUGLOG(SPDK_LOG_BDEV_RAID0, "min blockcount %lu,  numbasedev %u, strip size shift %u\n",
+		      min_blockcnt, raid_bdev->num_base_bdevs, raid_bdev->strip_size_shift);
+	raid_bdev->bdev.blockcnt = ((min_blockcnt >> raid_bdev->strip_size_shift) <<
+				    raid_bdev->strip_size_shift)  * raid_bdev->num_base_bdevs;
+
+	if (raid_bdev->num_base_bdevs > 1) {
+		raid_bdev->bdev.optimal_io_boundary = raid_bdev->strip_size;
+		raid_bdev->bdev.split_on_optimal_io_boundary = true;
+	} else {
+		/* Do not need to split reads/writes on single bdev RAID modules. */
+		raid_bdev->bdev.optimal_io_boundary = 0;
+		raid_bdev->bdev.split_on_optimal_io_boundary = false;
+	}
+
+	return 0;
+}
+
 static struct raid_bdev_module g_raid0_module = {
 	.level = RAID0,
+	.start = raid0_start,
 	.submit_rw_request = raid0_submit_rw_request,
 	.submit_null_payload_request = raid0_submit_null_payload_request,
 };