diff --git a/include/spdk/bdev.h b/include/spdk/bdev.h index e1da49cea8..82c3e11a1f 100644 --- a/include/spdk/bdev.h +++ b/include/spdk/bdev.h @@ -55,10 +55,20 @@ #define SPDK_BDEV_MAX_NAME_LENGTH 16 #define SPDK_BDEV_MAX_PRODUCT_NAME_LENGTH 50 +typedef void (*spdk_bdev_remove_cb_t)(void *remove_ctx); + struct spdk_bdev_io; struct spdk_bdev_fn_table; struct spdk_json_write_ctx; +/** Blockdev status */ +enum spdk_bdev_status { + SPDK_BDEV_STATUS_INVALID, + SPDK_BDEV_STATUS_UNCLAIMED, + SPDK_BDEV_STATUS_CLAIMED, + SPDK_BDEV_STATUS_REMOVING, +}; + /** * \brief SPDK block device. * @@ -104,8 +114,14 @@ struct spdk_bdev { /** Mutex protecting claimed */ pthread_mutex_t mutex; - /** True if another blockdev or a LUN is using this device */ - bool claimed; + /** The bdev status */ + enum spdk_bdev_status status; + + /** Remove callback function pointer to upper level stack */ + spdk_bdev_remove_cb_t remove_cb; + + /** Callback context for hot remove the device */ + void *remove_ctx; TAILQ_ENTRY(spdk_bdev) link; }; @@ -305,9 +321,11 @@ struct spdk_bdev *spdk_bdev_next(struct spdk_bdev *prev); * When the ownership of the bdev is no longer needed, the user should call spdk_bdev_unclaim(). * * \param bdev Block device to claim. + * \param remove_cb callback function for hot remove the device. + * \param remove_ctx param for hot removal callback function. * \return true if the caller claimed the bdev, or false if it was already claimed by another user. */ -bool spdk_bdev_claim(struct spdk_bdev *bdev); +bool spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, void *remove_ctx); /** * Release claim of ownership of a block device. diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 0fd59aa0e7..ea234651a7 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -546,6 +546,7 @@ spdk_bdev_read(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { return NULL; } @@ -585,6 +586,7 @@ spdk_bdev_readv(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { return NULL; } @@ -621,6 +623,7 @@ spdk_bdev_write(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) { return NULL; } @@ -659,6 +662,7 @@ spdk_bdev_writev(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); if (spdk_bdev_io_valid(bdev, offset, len) != 0) { return NULL; } @@ -695,6 +699,7 @@ spdk_bdev_unmap(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); if (bdesc_count == 0) { SPDK_ERRLOG("Invalid bdesc_count 0\n"); return NULL; @@ -735,6 +740,7 @@ spdk_bdev_flush(struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); bdev_io = spdk_bdev_get_io(); if (!bdev_io) { SPDK_ERRLOG("bdev_io memory allocation failed duing flush\n"); @@ -763,6 +769,7 @@ spdk_bdev_reset(struct spdk_bdev *bdev, enum spdk_bdev_reset_type reset_type, struct spdk_bdev_io *bdev_io; int rc; + assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED); bdev_io = spdk_bdev_get_io(); if (!bdev_io) { SPDK_ERRLOG("bdev_io memory allocation failed duing reset\n"); @@ -921,8 +928,7 @@ spdk_bdev_register(struct spdk_bdev *bdev) bdev->gencnt = 0; pthread_mutex_init(&bdev->mutex, NULL); - bdev->claimed = false; - + bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name); TAILQ_INSERT_TAIL(&spdk_bdev_list, bdev, link); } @@ -933,7 +939,22 @@ spdk_bdev_unregister(struct spdk_bdev *bdev) int rc; SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name); + + pthread_mutex_lock(&bdev->mutex); + assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_UNCLAIMED); + if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) { + if (bdev->remove_cb) { + bdev->status = SPDK_BDEV_STATUS_REMOVING; + pthread_mutex_unlock(&bdev->mutex); + bdev->remove_cb(bdev->remove_ctx); + return; + } else { + bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; + } + } + TAILQ_REMOVE(&spdk_bdev_list, bdev, link); + pthread_mutex_unlock(&bdev->mutex); pthread_mutex_destroy(&bdev->mutex); @@ -944,15 +965,18 @@ spdk_bdev_unregister(struct spdk_bdev *bdev) } bool -spdk_bdev_claim(struct spdk_bdev *bdev) +spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, + void *remove_ctx) { bool success; pthread_mutex_lock(&bdev->mutex); - if (!bdev->claimed) { + if (bdev->status != SPDK_BDEV_STATUS_CLAIMED) { /* Take ownership of bdev. */ - bdev->claimed = true; + bdev->remove_cb = remove_cb; + bdev->remove_ctx = remove_ctx; + bdev->status = SPDK_BDEV_STATUS_CLAIMED; success = true; } else { /* bdev is already claimed. */ @@ -967,12 +991,21 @@ spdk_bdev_claim(struct spdk_bdev *bdev) void spdk_bdev_unclaim(struct spdk_bdev *bdev) { + bool do_unregister = false; + pthread_mutex_lock(&bdev->mutex); - - assert(bdev->claimed); - bdev->claimed = false; - + assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_REMOVING); + if (bdev->status == SPDK_BDEV_STATUS_REMOVING) { + do_unregister = true; + } + bdev->remove_cb = NULL; + bdev->remove_ctx = NULL; + bdev->status = SPDK_BDEV_STATUS_UNCLAIMED; pthread_mutex_unlock(&bdev->mutex); + + if (do_unregister == true) { + spdk_bdev_unregister(bdev); + } } void diff --git a/lib/bdev/rpc/bdev_rpc.c b/lib/bdev/rpc/bdev_rpc.c index c183a0a0d6..bdc9b42fc7 100644 --- a/lib/bdev/rpc/bdev_rpc.c +++ b/lib/bdev/rpc/bdev_rpc.c @@ -72,7 +72,11 @@ spdk_rpc_get_bdevs(struct spdk_jsonrpc_server_conn *conn, spdk_json_write_uint64(w, bdev->blockcnt); spdk_json_write_name(w, "claimed"); - spdk_json_write_bool(w, bdev->claimed); + if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) { + spdk_json_write_bool(w, true); + } else { + spdk_json_write_bool(w, false); + } spdk_json_write_name(w, "driver_specific"); spdk_json_write_object_begin(w); diff --git a/lib/bdev/split/vbdev_split.c b/lib/bdev/split/vbdev_split.c index 5d861a43fd..597ed424ea 100644 --- a/lib/bdev/split/vbdev_split.c +++ b/lib/bdev/split/vbdev_split.c @@ -241,7 +241,7 @@ vbdev_split_create(struct spdk_bdev *base_bdev, uint64_t split_count, uint64_t s int rc; struct split_base *split_base; - if (!spdk_bdev_claim(base_bdev)) { + if (!spdk_bdev_claim(base_bdev, NULL, NULL)) { SPDK_ERRLOG("Split bdev %s is already claimed\n", base_bdev->name); return -1; } diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index fae797da11..6477e5aef5 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -412,7 +412,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd { int i = 0; - if (!spdk_bdev_claim(bdev)) { + if (!spdk_bdev_claim(bdev, NULL, NULL)) { SPDK_ERRLOG("Subsystem %s: bdev %s is already claimed\n", subsystem->subnqn, bdev->name); return -1; diff --git a/lib/scsi/lun.c b/lib/scsi/lun.c index 48afcab8fd..4c4c0f7030 100644 --- a/lib/scsi/lun.c +++ b/lib/scsi/lun.c @@ -290,7 +290,7 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev) return NULL; } - if (!spdk_bdev_claim(bdev)) { + if (!spdk_bdev_claim(bdev, NULL, NULL)) { SPDK_ERRLOG("LUN %s: bdev %s is already claimed\n", name, bdev->name); free(lun); return NULL; diff --git a/test/lib/bdev/bdevio/bdevio.c b/test/lib/bdev/bdevio/bdevio.c index 74da38e05c..c28b2d6ab2 100644 --- a/test/lib/bdev/bdevio/bdevio.c +++ b/test/lib/bdev/bdevio/bdevio.c @@ -112,7 +112,7 @@ bdevio_construct_targets(void) bdev = spdk_bdev_first(); while (bdev != NULL) { - if (!spdk_bdev_claim(bdev)) { + if (!spdk_bdev_claim(bdev, NULL, NULL)) { bdev = spdk_bdev_next(bdev); continue; } diff --git a/test/lib/bdev/bdevperf/bdevperf.c b/test/lib/bdev/bdevperf/bdevperf.c index 28d59fbbfe..97836f580d 100644 --- a/test/lib/bdev/bdevperf/bdevperf.c +++ b/test/lib/bdev/bdevperf/bdevperf.c @@ -122,7 +122,7 @@ bdevperf_construct_targets(void) bdev = spdk_bdev_first(); while (bdev != NULL) { - if (!spdk_bdev_claim(bdev)) { + if (!spdk_bdev_claim(bdev, NULL, NULL)) { bdev = spdk_bdev_next(bdev); continue; } diff --git a/test/lib/nvmf/subsystem/subsystem_ut.c b/test/lib/nvmf/subsystem/subsystem_ut.c index 883755923d..51bc6acc2c 100644 --- a/test/lib/nvmf/subsystem/subsystem_ut.c +++ b/test/lib/nvmf/subsystem/subsystem_ut.c @@ -161,7 +161,8 @@ spdk_nvmf_session_poll(struct spdk_nvmf_session *session) } bool -spdk_bdev_claim(struct spdk_bdev *bdev) +spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, + void *remove_ctx) { return true; } diff --git a/test/lib/scsi/lun/lun_ut.c b/test/lib/scsi/lun/lun_ut.c index 388fa234d7..7ce214a4ac 100644 --- a/test/lib/scsi/lun/lun_ut.c +++ b/test/lib/scsi/lun/lun_ut.c @@ -109,7 +109,8 @@ spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) } bool -spdk_bdev_claim(struct spdk_bdev *bdev) +spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, + void *remove_ctx) { return true; }