bdev: add API to claim block devices

Claim the block devices used by iSCSI LUNs and NVMe-oF subsystems so
they can't accidentally be reused.

This will also be used by virtual block devices to allow layering of
bdevs.

Change-Id: I5384923fbf24f13f4ce720a797c5a628053d49f4
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

include/spdk/bdev.h

@@ -43,6 +43,7 @@
 #include <stddef.h>  /* for offsetof */
 #include <sys/uio.h> /* for struct iovec */
 #include <stdbool.h>
+#include <pthread.h>
 
 #include "spdk/event.h"
 #include "spdk/queue.h"
@@ -100,6 +101,9 @@ struct spdk_bdev {
 	/** generation value used by block device reset */
 	uint32_t gencnt;
 
+	/** Mutex protecting claimed */
+	pthread_mutex_t mutex;
+
 	/** True if another blockdev or a LUN is using this device */
 	bool claimed;
 
@@ -285,6 +289,28 @@ void spdk_bdev_unregister(struct spdk_bdev *bdev);
 struct spdk_bdev *spdk_bdev_first(void);
 struct spdk_bdev *spdk_bdev_next(struct spdk_bdev *prev);
 
+/**
+ * Claim ownership of a block device.
+ *
+ * User applications and virtual blockdevs may use this to mediate access to bdevs.
+ *
+ * When the ownership of the bdev is no longer needed, the user should call spdk_bdev_unclaim().
+ *
+ * \param bdev Block device to claim.
+ * \return true if the caller claimed the bdev, or false if it was already claimed by another user.
+ */
+bool spdk_bdev_claim(struct spdk_bdev *bdev);
+
+/**
+ * Release claim of ownership of a block device.
+ *
+ * When a bdev reference acquired with spdk_bdev_claim() is no longer needed, the user should
+ * release the claim using spdk_bdev_unclaim().
+ *
+ * \param bdev Block device to release.
+ */
+void spdk_bdev_unclaim(struct spdk_bdev *bdev);
+
 bool spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type);
 
 int spdk_bdev_dump_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w);

lib/bdev/bdev.c

@@ -854,6 +854,9 @@ spdk_bdev_register(struct spdk_bdev *bdev)
 	/* initialize the reset generation value to zero */
 	bdev->gencnt = 0;
 
+	pthread_mutex_init(&bdev->mutex, NULL);
+	bdev->claimed = false;
+
 	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name);
 	TAILQ_INSERT_TAIL(&spdk_bdev_list, bdev, link);
 }
@@ -866,12 +869,46 @@ spdk_bdev_unregister(struct spdk_bdev *bdev)
 	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name);
 	TAILQ_REMOVE(&spdk_bdev_list, bdev, link);
 
+	pthread_mutex_destroy(&bdev->mutex);
+
 	rc = bdev->fn_table->destruct(bdev->ctxt);
 	if (rc < 0) {
 		SPDK_ERRLOG("destruct failed\n");
 	}
 }
 
+bool
+spdk_bdev_claim(struct spdk_bdev *bdev)
+{
+	bool success;
+
+	pthread_mutex_lock(&bdev->mutex);
+
+	if (!bdev->claimed) {
+		/* Take ownership of bdev. */
+		bdev->claimed = true;
+		success = true;
+	} else {
+		/* bdev is already claimed. */
+		success = false;
+	}
+
+	pthread_mutex_unlock(&bdev->mutex);
+
+	return success;
+}
+
+void
+spdk_bdev_unclaim(struct spdk_bdev *bdev)
+{
+	pthread_mutex_lock(&bdev->mutex);
+
+	assert(bdev->claimed);
+	bdev->claimed = false;
+
+	pthread_mutex_unlock(&bdev->mutex);
+}
+
 void
 spdk_bdev_io_get_rbuf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_rbuf_cb cb)
 {

lib/nvmf/subsystem.c

@@ -43,6 +43,7 @@
 #include "spdk/trace.h"
 #include "spdk/nvmf_spec.h"
 
+#include "spdk_internal/bdev.h"
 #include "spdk_internal/log.h"
 
 static TAILQ_HEAD(, spdk_nvmf_subsystem) g_subsystems = TAILQ_HEAD_INITIALIZER(g_subsystems);
@@ -420,6 +421,12 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 {
 	int i = 0;
 
+	if (!spdk_bdev_claim(bdev)) {
+		SPDK_ERRLOG("Subsystem %s: bdev %s is already claimed\n",
+			    subsystem->subnqn, bdev->name);
+		return -1;
+	}
+
 	assert(subsystem->mode == NVMF_SUBSYSTEM_MODE_VIRTUAL);
 	while (i < MAX_VIRTUAL_NAMESPACE && subsystem->dev.virt.ns_list[i]) {
 		i++;

lib/nvmf/virtual.c

@@ -534,7 +534,11 @@ nvmf_virtual_ctrlr_process_io_cmd(struct spdk_nvmf_request *req)
 static void
 nvmf_virtual_ctrlr_detach(struct spdk_nvmf_subsystem *subsystem)
 {
-	return;
+	uint32_t i;
+
+	for (i = 0; i < subsystem->dev.virt.ns_count; i++) {
+		spdk_bdev_unclaim(subsystem->dev.virt.ns_list[i]);
+	}
 }
 
 const struct spdk_nvmf_ctrlr_ops spdk_nvmf_virtual_ctrlr_ops = {

lib/scsi/lun.c

@@ -289,6 +289,12 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev)
 		return NULL;
 	}
 
+	if (!spdk_bdev_claim(bdev)) {
+		SPDK_ERRLOG("LUN %s: bdev %s is already claimed\n", name, bdev->name);
+		free(lun);
+		return NULL;
+	}
+
 	TAILQ_INIT(&lun->tasks);
 	TAILQ_INIT(&lun->pending_tasks);
 
@@ -308,6 +314,7 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev)
 static int
 spdk_scsi_lun_destruct(struct spdk_scsi_lun *lun)
 {
+	spdk_bdev_unclaim(lun->bdev);
 	spdk_scsi_lun_db_delete(lun);
 
 	free(lun);

test/lib/bdev/bdevio/bdevio.c

@@ -110,7 +110,7 @@ bdevio_construct_targets(void)
 	bdev = spdk_bdev_first();
 	while (bdev != NULL) {
 
-		if (bdev->claimed) {
+		if (!spdk_bdev_claim(bdev)) {
 			bdev = spdk_bdev_next(bdev);
 			continue;
 		}
@@ -147,6 +147,7 @@ bdevio_cleanup_targets(void)
 	target = g_io_targets;
 	while (target != NULL) {
 		execute_spdk_function(__put_io_channel, target, NULL);
+		spdk_bdev_unclaim(target->bdev);
 		g_io_targets = target->next;
 		free(target);
 		target = g_io_targets;

test/lib/bdev/bdevperf/bdevperf.c

@@ -120,7 +120,7 @@ bdevperf_construct_targets(void)
 	bdev = spdk_bdev_first();
 	while (bdev != NULL) {
 
-		if (bdev->claimed) {
+		if (!spdk_bdev_claim(bdev)) {
 			bdev = spdk_bdev_next(bdev);
 			continue;
 		}
@@ -168,6 +168,7 @@ end_run(void *arg1, void *arg2)
 	struct io_target *target = arg1;
 
 	spdk_put_io_channel(target->ch);
+	spdk_bdev_unclaim(target->bdev);
 	if (--g_target_count == 0) {
 		if (g_show_performance_real_time) {
 			spdk_poller_unregister(&g_perf_timer, NULL);

test/lib/nvmf/subsystem/subsystem_ut.c

@@ -111,6 +111,12 @@ spdk_nvmf_session_poll(struct spdk_nvmf_session *session)
 	return -1;
 }
 
+bool
+spdk_bdev_claim(struct spdk_bdev *bdev)
+{
+	return true;
+}
+
 static void
 nvmf_test_create_subsystem(void)
 {

test/lib/scsi/lun/lun_ut.c

@@ -108,6 +108,17 @@ spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)
 	return -1;
 }
 
+bool
+spdk_bdev_claim(struct spdk_bdev *bdev)
+{
+	return true;
+}
+
+void
+spdk_bdev_unclaim(struct spdk_bdev *bdev)
+{
+}
+
 void spdk_scsi_dev_queue_mgmt_task(struct spdk_scsi_dev *dev,
 				   struct spdk_scsi_task *task)
 {