d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bdev: add spdk_bdev_open/close

Browse Source 
Retire the old claim/unclaim semantics in favor of
open/close.  Clients must now open a bdev to get
an spdk_bdev_desc, then pass this desc to get an
I/O channel.

This allows multiple clients to open a bdev,
although only one may open a bdev with write
access.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I4d319f1278170124169a8a75fd791e926b3f7171

Reviewed-on: https://review.gerrithub.io/367611
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Jim Harris 2017-06-29 11:23:50 -07:00
parent 7347136f8b
commit 57d174ff67
24 changed files with 423 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
include/spdk/bdev.h
View File

@ -63,8 +63,7 @@ struct spdk_json_write_ctx;
/** Blockdev status */
enum spdk_bdev_status {
SPDK_BDEV_STATUS_INVALID,
SPDK_BDEV_STATUS_UNCLAIMED,
SPDK_BDEV_STATUS_CLAIMED,
SPDK_BDEV_STATUS_READY,
SPDK_BDEV_STATUS_REMOVING,
};
@ -75,6 +74,11 @@ enum spdk_bdev_status {
*/
struct spdk_bdev;
/**
* \brief Handle to an opened SPDK block device.
*/
struct spdk_bdev_desc;
/** Blockdev I/O type */
enum spdk_bdev_io_type {
SPDK_BDEV_IO_TYPE_READ = 1,
@ -139,29 +143,26 @@ struct spdk_bdev *spdk_bdev_next(struct spdk_bdev *prev);
*/
struct spdk_bdev *spdk_bdev_first_leaf(void);
struct spdk_bdev *spdk_bdev_next_leaf(struct spdk_bdev *prev);
/**
* Claim ownership of a block device.
*
* User applications and virtual blockdevs may use this to mediate access to bdevs.
*
* When the ownership of the bdev is no longer needed, the user should call spdk_bdev_unclaim().
*
* \param bdev Block device to claim.
* \param remove_cb callback function for hot remove the device.
* \param remove_ctx param for hot removal callback function.
* \return true if the caller claimed the bdev, or false if it was already claimed by another user.
*/
bool spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, void *remove_ctx);
/**
* Release claim of ownership of a block device.
* Open a block device for I/O operations.
*
* When a bdev reference acquired with spdk_bdev_claim() is no longer needed, the user should
* release the claim using spdk_bdev_unclaim().
*
* \param bdev Block device to release.
* \param bdev Block device to open.
* \param write true is read/write access requested, false if read-only
* \param remove_cb callback function for hot remove the device.
* \param remove_ctx param for hot removal callback function.
* \param desc output parameter for the descriptor when operation is successful
* \return 0 if operation is successful, suitable errno value otherwise
*/
void spdk_bdev_unclaim(struct spdk_bdev *bdev);
int spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx, struct spdk_bdev_desc **desc);
/**
* Close a previously opened block device.
*
* \param desc Block device descriptor to close.
*/
void spdk_bdev_close(struct spdk_bdev_desc *desc);
bool spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type);
@ -229,15 +230,16 @@ size_t spdk_bdev_get_buf_align(const struct spdk_bdev *bdev);
bool spdk_bdev_has_write_cache(const struct spdk_bdev *bdev);
/**
* Obtain an I/O channel for this block device. I/O channels are bound to threads,
* so the resulting I/O channel may only be used from the thread it was originally
* obtained from.
* Obtain an I/O channel for the block device opened by the specified
* descriptor. I/O channels are bound to threads, so the resulting I/O
* channel may only be used from the thread it was originally obtained
* from.
*
* \param bdev Block device
* \param desc Block device descriptor
*
* \return A handle to the I/O channel or NULL on failure.
*/
struct spdk_io_channel *spdk_bdev_get_io_channel(struct spdk_bdev *bdev);
struct spdk_io_channel *spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc);
/**
* Submit a read request to the bdev on the given channel.

1
include/spdk/nvmf.h
View File

@ -113,6 +113,7 @@ struct spdk_nvmf_subsystem {
struct {
char sn[MAX_SN_LEN + 1];
struct spdk_bdev *ns_list[MAX_VIRTUAL_NAMESPACE];
struct spdk_bdev_desc *desc[MAX_VIRTUAL_NAMESPACE];
struct spdk_io_channel *ch[MAX_VIRTUAL_NAMESPACE];
uint32_t max_nsid;
} virt;

9
include/spdk_internal/bdev.h
View File

@ -211,11 +211,12 @@ struct spdk_bdev {
TAILQ_ENTRY(spdk_bdev) vbdev_link;
/** Remove callback function pointer to upper level stack */
spdk_bdev_remove_cb_t remove_cb;
bool bdev_opened_for_write;
/** Callback context for hot remove the device */
void *remove_ctx;
uint32_t vbdevs_opened_for_write;
/** List of open descriptors for this block device. */
TAILQ_HEAD(, spdk_bdev_desc) open_descs;
TAILQ_ENTRY(spdk_bdev) link;

135
lib/bdev/bdev.c
View File

@ -96,6 +96,14 @@ struct spdk_bdev_mgmt_channel {
need_buf_tailq_t need_buf_large;
};
struct spdk_bdev_desc {
struct spdk_bdev *bdev;
spdk_bdev_remove_cb_t remove_cb;
void *remove_ctx;
bool write;
TAILQ_ENTRY(spdk_bdev_desc) link;
};
struct spdk_bdev_channel {
struct spdk_bdev *bdev;
@ -702,9 +710,9 @@ spdk_bdev_channel_destroy(void *io_device, void *ctx_buf)
}
struct spdk_io_channel *
spdk_bdev_get_io_channel(struct spdk_bdev *bdev)
spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
{
return spdk_get_io_channel(bdev);
return spdk_get_io_channel(desc->bdev);
}
const char *
@ -785,7 +793,6 @@ spdk_bdev_read(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {
return -EINVAL;
}
@ -825,7 +832,6 @@ spdk_bdev_readv(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {
return -EINVAL;
}
@ -862,7 +868,6 @@ spdk_bdev_write(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {
return -EINVAL;
}
@ -902,7 +907,6 @@ spdk_bdev_writev(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
if (spdk_bdev_io_valid(bdev, offset, len) != 0) {
return -EINVAL;
}
@ -940,7 +944,6 @@ spdk_bdev_unmap(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
if (bdesc_count == 0) {
SPDK_ERRLOG("Invalid bdesc_count 0\n");
return -EINVAL;
@ -982,7 +985,6 @@ spdk_bdev_flush(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
int rc;
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
bdev_io = spdk_bdev_get_io();
if (!bdev_io) {
SPDK_ERRLOG("bdev_io memory allocation failed duing flush\n");
@ -1070,8 +1072,6 @@ spdk_bdev_reset(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);
bdev_io = spdk_bdev_get_io();
if (!bdev_io) {
SPDK_ERRLOG("bdev_io memory allocation failed duing reset\n");
@ -1360,8 +1360,13 @@ _spdk_bdev_register(struct spdk_bdev *bdev)
{
struct spdk_bdev_module_if *vbdev_module;
bdev->status = SPDK_BDEV_STATUS_READY;
/* initialize the reset generation value to zero */
bdev->gencnt = 0;
TAILQ_INIT(&bdev->open_descs);
bdev->bdev_opened_for_write = false;
bdev->vbdevs_opened_for_write = 0;
TAILQ_INIT(&bdev->vbdevs);
TAILQ_INIT(&bdev->base_bdevs);
@ -1373,7 +1378,6 @@ _spdk_bdev_register(struct spdk_bdev *bdev)
sizeof(struct spdk_bdev_channel));
pthread_mutex_init(&bdev->mutex, NULL);
bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name);
TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, link);
@ -1406,23 +1410,28 @@ spdk_vbdev_register(struct spdk_bdev *vbdev, struct spdk_bdev **base_bdevs, int
void
spdk_bdev_unregister(struct spdk_bdev *bdev)
{
struct spdk_bdev_desc *desc, *tmp;
int rc;
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name);
pthread_mutex_lock(&bdev->mutex);
assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_UNCLAIMED);
if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) {
if (bdev->remove_cb) {
bdev->status = SPDK_BDEV_STATUS_REMOVING;
bdev->status = SPDK_BDEV_STATUS_REMOVING;
TAILQ_FOREACH_SAFE(desc, &bdev->open_descs, link, tmp) {
if (desc->remove_cb) {
pthread_mutex_unlock(&bdev->mutex);
bdev->remove_cb(bdev->remove_ctx);
return;
} else {
bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;
desc->remove_cb(desc->remove_ctx);
pthread_mutex_lock(&bdev->mutex);
}
}
if (!TAILQ_EMPTY(&bdev->open_descs)) {
pthread_mutex_unlock(&bdev->mutex);
return;
}
TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, link);
pthread_mutex_unlock(&bdev->mutex);
@ -1448,43 +1457,93 @@ spdk_vbdev_unregister(struct spdk_bdev *vbdev)
spdk_bdev_unregister(vbdev);
}
bool
spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx)
static bool
__is_bdev_opened_for_write(struct spdk_bdev *bdev)
{
bool success;
struct spdk_bdev *base;
if (bdev->bdev_opened_for_write) {
return true;
}
TAILQ_FOREACH(base, &bdev->base_bdevs, base_bdev_link) {
if (__is_bdev_opened_for_write(base)) {
return true;
}
}
return false;
}
static void
__modify_write_counts(struct spdk_bdev *bdev, int mod)
{
struct spdk_bdev *base;
TAILQ_FOREACH(base, &bdev->base_bdevs, base_bdev_link) {
base->vbdevs_opened_for_write += mod;
__modify_write_counts(base, mod);
}
}
int
spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx, struct spdk_bdev_desc **_desc)
{
struct spdk_bdev_desc *desc;
desc = calloc(1, sizeof(*desc));
if (desc == NULL) {
return -ENOMEM;
}
pthread_mutex_lock(&bdev->mutex);
if (bdev->status != SPDK_BDEV_STATUS_CLAIMED) {
/* Take ownership of bdev. */
bdev->remove_cb = remove_cb;
bdev->remove_ctx = remove_ctx;
bdev->status = SPDK_BDEV_STATUS_CLAIMED;
success = true;
} else {
/* bdev is already claimed. */
success = false;
if (write && (__is_bdev_opened_for_write(bdev) || bdev->vbdevs_opened_for_write > 0)) {
SPDK_ERRLOG("failed, %s (or one of its virtual bdevs) already opened for write\n", bdev->name);
free(desc);
pthread_mutex_unlock(&bdev->mutex);
return -EPERM;
}
TAILQ_INSERT_TAIL(&bdev->open_descs, desc, link);
if (write) {
bdev->bdev_opened_for_write = true;
__modify_write_counts(bdev, 1);
}
desc->bdev = bdev;
desc->remove_cb = remove_cb;
desc->remove_ctx = remove_ctx;
desc->write = write;
*_desc = desc;
pthread_mutex_unlock(&bdev->mutex);
return success;
return 0;
}
void
spdk_bdev_unclaim(struct spdk_bdev *bdev)
spdk_bdev_close(struct spdk_bdev_desc *desc)
{
struct spdk_bdev *bdev = desc->bdev;
bool do_unregister = false;
pthread_mutex_lock(&bdev->mutex);
assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_REMOVING);
if (desc->write) {
assert(bdev->bdev_opened_for_write);
bdev->bdev_opened_for_write = false;
__modify_write_counts(bdev, -1);
}
TAILQ_REMOVE(&bdev->open_descs, desc, link);
free(desc);
if (bdev->status == SPDK_BDEV_STATUS_REMOVING) {
do_unregister = true;
}
bdev->remove_cb = NULL;
bdev->remove_ctx = NULL;
bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;
pthread_mutex_unlock(&bdev->mutex);
if (do_unregister == true) {

6
lib/bdev/error/vbdev_error.c
View File

@ -157,7 +157,6 @@ vbdev_error_free(struct vbdev_error_disk *error_disk)
TAILQ_REMOVE(&g_vbdev_error_disks, error_disk, tailq);
spdk_bdev_unclaim(error_disk->base_bdev);
vbdev_error_disk_free(error_disk);
}
@ -217,11 +216,6 @@ spdk_vbdev_error_create(struct spdk_bdev *base_bdev)
struct vbdev_error_disk *disk;
int rc;
if (!spdk_bdev_claim(base_bdev, NULL, NULL)) {
SPDK_ERRLOG("Error bdev %s is already claimed\n", base_bdev->name);
return -1;
}
disk = calloc(1, sizeof(*disk));
if (!disk) {
SPDK_ERRLOG("Memory allocation failure\n");

11
lib/bdev/rpc/bdev_rpc.c
View File

@ -72,12 +72,11 @@ spdk_rpc_get_bdevs(struct spdk_jsonrpc_server_conn *conn,
spdk_json_write_name(w, "num_blocks");
spdk_json_write_uint64(w, spdk_bdev_get_num_blocks(bdev));
spdk_json_write_name(w, "claimed");
if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) {
spdk_json_write_bool(w, true);
} else {
spdk_json_write_bool(w, false);
}
spdk_json_write_name(w, "bdev_opened_for_write");
spdk_json_write_bool(w, bdev->bdev_opened_for_write);
spdk_json_write_name(w, "vbdevs_opened_for_write");
spdk_json_write_uint32(w, bdev->vbdevs_opened_for_write);
spdk_json_write_name(w, "driver_specific");
spdk_json_write_object_begin(w);

6
lib/bdev/split/vbdev_split.c
View File

@ -153,7 +153,6 @@ static void
vbdev_split_base_put_ref(struct split_base *split_base)
{
if (__sync_sub_and_fetch(&split_base->ref, 1) == 0) {
spdk_bdev_unclaim(split_base->base_bdev);
free(split_base);
}
}
@ -239,11 +238,6 @@ vbdev_split_create(struct spdk_bdev *base_bdev, uint64_t split_count, uint64_t s
int rc;
struct split_base *split_base;
if (!spdk_bdev_claim(base_bdev, NULL, NULL)) {
SPDK_ERRLOG("Split bdev %s is already claimed\n", spdk_bdev_get_name(base_bdev));
return -1;
}
if (split_size_mb) {
if (((split_size_mb * mb) % base_bdev->blocklen) != 0) {
SPDK_ERRLOG("Split size %" PRIu64 " MB is not possible with block size "

15
lib/blob/bdev/blob_bdev.c
View File

@ -43,6 +43,7 @@
struct blob_bdev {
struct spdk_bs_dev bs_dev;
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc;
};
static inline struct spdk_bdev *
@ -120,9 +121,9 @@ bdev_blob_unmap(struct spdk_bs_dev *dev, struct spdk_io_channel *channel, uint64
static struct spdk_io_channel *
bdev_blob_create_channel(struct spdk_bs_dev *dev)
{
struct spdk_bdev *bdev = __get_bdev(dev);
struct blob_bdev *blob_bdev = (struct blob_bdev *)dev;
return spdk_bdev_get_io_channel(bdev);
return spdk_bdev_get_io_channel(blob_bdev->desc);
}
static void
@ -141,6 +142,8 @@ struct spdk_bs_dev *
spdk_bdev_create_bs_dev(struct spdk_bdev *bdev)
{
struct blob_bdev *b;
struct spdk_bdev_desc *desc;
int rc;
b = calloc(1, sizeof(*b));
@ -149,7 +152,15 @@ spdk_bdev_create_bs_dev(struct spdk_bdev *bdev)
return NULL;
}
rc = spdk_bdev_open(bdev, true, NULL, NULL, &desc);
if (rc != 0) {
SPDK_ERRLOG("could not open bdev, error=%d\n", rc);
free(b);
return NULL;
}
b->bdev = bdev;
b->desc = desc;
b->bs_dev.blockcnt = spdk_bdev_get_num_blocks(bdev);
b->bs_dev.blocklen = spdk_bdev_get_block_size(bdev);
b->bs_dev.create_channel = bdev_blob_create_channel;

9
lib/nvmf/subsystem.c
View File

@ -391,6 +391,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
uint32_t nsid)
{
uint32_t i;
int rc;
assert(subsystem->mode == NVMF_SUBSYSTEM_MODE_VIRTUAL);
@ -420,9 +421,11 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
}
}
if (!spdk_bdev_claim(bdev, spdk_nvmf_ctrlr_hot_remove, subsystem)) {
SPDK_ERRLOG("Subsystem %s: bdev %s is already claimed\n",
subsystem->subnqn, spdk_bdev_get_name(bdev));
rc = spdk_bdev_open(bdev, true, spdk_nvmf_ctrlr_hot_remove, subsystem,
&subsystem->dev.virt.desc[i]);
if (rc != 0) {
SPDK_ERRLOG("Subsystem %s: bdev %s cannot be opened, error=%d\n",
subsystem->subnqn, spdk_bdev_get_name(bdev), rc);
return 0;
}

4
lib/nvmf/virtual.c
View File

@ -616,7 +616,7 @@ nvmf_virtual_ctrlr_attach(struct spdk_nvmf_subsystem *subsystem)
continue;
}
ch = spdk_bdev_get_io_channel(bdev);
ch = spdk_bdev_get_io_channel(subsystem->dev.virt.desc[i]);
if (ch == NULL) {
SPDK_ERRLOG("io_channel allocation failed\n");
return -1;
@ -635,7 +635,7 @@ nvmf_virtual_ctrlr_detach(struct spdk_nvmf_subsystem *subsystem)
for (i = 0; i < subsystem->dev.virt.max_nsid; i++) {
if (subsystem->dev.virt.ns_list[i]) {
spdk_put_io_channel(subsystem->dev.virt.ch[i]);
spdk_bdev_unclaim(subsystem->dev.virt.ns_list[i]);
spdk_bdev_close(subsystem->dev.virt.desc[i]);
subsystem->dev.virt.ch[i] = NULL;
subsystem->dev.virt.ns_list[i] = NULL;
}

5
lib/rocksdb/env_spdk.cc
View File

@ -477,11 +477,6 @@ spdk_rocksdb_run(void *arg1, void *arg2)
exit(1);
}
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
SPDK_ERRLOG("could not claim bdev %s\n", g_bdev_name.c_str());
exit(1);
}
g_bs_dev = spdk_bdev_create_bs_dev(bdev);
printf("using bdev %s\n", g_bdev_name.c_str());
spdk_fs_load(g_bs_dev, __send_request, fs_load_cb, NULL);

12
lib/scsi/lun.c
View File

@ -278,8 +278,10 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev,
return NULL;
}
if (!spdk_bdev_claim(bdev, spdk_scsi_lun_hot_remove, lun)) {
SPDK_ERRLOG("LUN %s: bdev %s is already claimed\n", name, spdk_bdev_get_name(bdev));
rc = spdk_bdev_open(bdev, true, spdk_scsi_lun_hot_remove, lun, &lun->bdev_desc);
if (rc != 0) {
SPDK_ERRLOG("LUN %s: bdev %s cannot be opened, error=%d\n", name, spdk_bdev_get_name(bdev), rc);
free(lun);
return NULL;
}
@ -295,7 +297,7 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev,
rc = spdk_scsi_lun_db_add(lun);
if (rc < 0) {
SPDK_ERRLOG("Unable to add LUN %s to DB\n", lun->name);
spdk_bdev_unclaim(bdev);
spdk_bdev_close(lun->bdev_desc);
free(lun);
return NULL;
}
@ -306,7 +308,7 @@ spdk_scsi_lun_construct(const char *name, struct spdk_bdev *bdev,
int
spdk_scsi_lun_destruct(struct spdk_scsi_lun *lun)
{
spdk_bdev_unclaim(lun->bdev);
spdk_bdev_close(lun->bdev_desc);
spdk_poller_unregister(&lun->hotplug_poller, NULL);
spdk_scsi_lun_db_delete(lun);
@ -379,7 +381,7 @@ int spdk_scsi_lun_allocate_io_channel(struct spdk_scsi_lun *lun)
lun->lcore = spdk_env_get_current_core();
lun->io_channel = spdk_bdev_get_io_channel(lun->bdev);
lun->io_channel = spdk_bdev_get_io_channel(lun->bdev_desc);
if (lun->io_channel == NULL) {
return -1;
}

3
lib/scsi/scsi_internal.h
View File

@ -82,6 +82,9 @@ struct spdk_scsi_lun {
/** The blockdev associated with this LUN. */
struct spdk_bdev *bdev;
/** Descriptor for opened block device. */
struct spdk_bdev_desc *bdev_desc;
/** I/O channel for the blockdev associated with this LUN. */
struct spdk_io_channel *io_channel;

14
lib/vhost/vhost_blk.c
View File

@ -62,6 +62,7 @@ struct spdk_vhost_blk_task {
struct spdk_vhost_blk_dev {
struct spdk_vhost_dev vdev;
struct spdk_bdev *bdev;
struct spdk_bdev_desc *bdev_desc;
struct spdk_io_channel *bdev_io_channel;
struct spdk_poller *requestq_poller;
struct spdk_ring *tasks_pool;
@ -341,7 +342,7 @@ add_vdev_cb(void *arg)
spdk_vhost_dev_mem_register(&bvdev->vdev);
if (bvdev->bdev) {
bvdev->bdev_io_channel = spdk_bdev_get_io_channel(bvdev->bdev);
bvdev->bdev_io_channel = spdk_bdev_get_io_channel(bvdev->bdev_desc);
if (!bvdev->bdev_io_channel) {
SPDK_ERRLOG("Controller %s: IO channel allocation failed\n", vdev->name);
abort();
@ -571,9 +572,10 @@ spdk_vhost_blk_construct(const char *name, uint64_t cpumask, const char *dev_nam
return -1;
}
if (spdk_bdev_claim(bdev, bdev_remove_cb, bvdev) == false) {
SPDK_ERRLOG("Controller %s: failed to claim bdev '%s'\n",
name, dev_name);
ret = spdk_bdev_open(bdev, true, bdev_remove_cb, bvdev, &bvdev->bdev_desc);
if (ret != 0) {
SPDK_ERRLOG("Controller %s: could not open bdev '%s', error=%d\n",
name, dev_name, ret);
goto err;
}
@ -582,7 +584,7 @@ spdk_vhost_blk_construct(const char *name, uint64_t cpumask, const char *dev_nam
ret = spdk_vhost_dev_construct(&bvdev->vdev, name, cpumask, SPDK_VHOST_DEV_T_BLK,
&vhost_blk_device_backend);
if (ret != 0) {
spdk_bdev_unclaim(bdev);
spdk_bdev_close(bvdev->bdev_desc);
goto err;
}
@ -605,7 +607,7 @@ spdk_vhost_blk_destroy(struct spdk_vhost_dev *vdev)
return -EINVAL;
}
spdk_bdev_unclaim(bvdev->bdev);
spdk_bdev_close(bvdev->bdev_desc);
bvdev->bdev = NULL;
SPDK_NOTICELOG("Controller %s: removed device\n", vdev->name);

20
test/lib/bdev/bdevio/bdevio.c
View File

@ -54,6 +54,7 @@ pthread_cond_t g_test_cond;
struct io_target {
struct spdk_bdev *bdev;
struct spdk_bdev_desc *bdev_desc;
struct spdk_io_channel *ch;
struct io_target *next;
};
@ -94,7 +95,7 @@ __get_io_channel(void *arg1, void *arg2)
{
struct io_target *target = arg1;
target->ch = spdk_bdev_get_io_channel(target->bdev);
target->ch = spdk_bdev_get_io_channel(target->bdev_desc);
wake_ut_thread();
}
@ -103,6 +104,7 @@ bdevio_construct_targets(void)
{
struct spdk_bdev *bdev;
struct io_target *target;
int rc;
printf("I/O targets:\n");
@ -111,7 +113,15 @@ bdevio_construct_targets(void)
uint64_t num_blocks = spdk_bdev_get_num_blocks(bdev);
uint32_t block_size = spdk_bdev_get_block_size(bdev);
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
target = malloc(sizeof(struct io_target));
if (target == NULL) {
return -ENOMEM;
}
rc = spdk_bdev_open(bdev, true, NULL, NULL, &target->bdev_desc);
if (rc != 0) {
free(target);
SPDK_ERRLOG("Could not open leaf bdev %s, error=%d\n", spdk_bdev_get_name(bdev), rc);
bdev = spdk_bdev_next_leaf(bdev);
continue;
}
@ -121,10 +131,6 @@ bdevio_construct_targets(void)
num_blocks, block_size,
(num_blocks * block_size + 1024 * 1024 - 1) / (1024 * 1024));
target = malloc(sizeof(struct io_target));
if (target == NULL) {
return -ENOMEM;
}
target->bdev = bdev;
target->next = g_io_targets;
execute_spdk_function(__get_io_channel, target, NULL);
@ -153,7 +159,7 @@ bdevio_cleanup_targets(void)
target = g_io_targets;
while (target != NULL) {
execute_spdk_function(__put_io_channel, target, NULL);
spdk_bdev_unclaim(target->bdev);
spdk_bdev_close(target->bdev_desc);
g_io_targets = target->next;
free(target);
target = g_io_targets;

19
test/lib/bdev/bdevperf/bdevperf.c
View File

@ -76,6 +76,7 @@ static void bdevperf_submit_single(struct io_target *target);
struct io_target {
struct spdk_bdev *bdev;
struct spdk_bdev_desc *bdev_desc;
struct spdk_io_channel *ch;
struct io_target *next;
unsigned lcore;
@ -116,15 +117,11 @@ bdevperf_construct_targets(void)
struct spdk_bdev *bdev;
struct io_target *target;
size_t align;
int rc;
bdev = spdk_bdev_first_leaf();
while (bdev != NULL) {
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
bdev = spdk_bdev_next_leaf(bdev);
continue;
}
if (g_unmap && !spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_UNMAP)) {
printf("Skipping %s because it does not support unmap\n", spdk_bdev_get_name(bdev));
bdev = spdk_bdev_next_leaf(bdev);
@ -137,6 +134,14 @@ bdevperf_construct_targets(void)
/* Return immediately because all mallocs will presumably fail after this */
return;
}
rc = spdk_bdev_open(bdev, true, NULL, NULL, &target->bdev_desc);
if (rc != 0) {
SPDK_ERRLOG("Could not open leaf bdev %s, error=%d\n", spdk_bdev_get_name(bdev), rc);
bdev = spdk_bdev_next_leaf(bdev);
continue;
}
target->bdev = bdev;
/* Mapping each target to lcore */
index = g_target_count % spdk_env_get_core_count();
@ -171,7 +176,7 @@ end_run(void *arg1, void *arg2)
struct io_target *target = arg1;
spdk_put_io_channel(target->ch);
spdk_bdev_unclaim(target->bdev);
spdk_bdev_close(target->bdev_desc);
if (--g_target_count == 0) {
if (g_show_performance_real_time) {
spdk_poller_unregister(&g_perf_timer, NULL);
@ -452,7 +457,7 @@ bdevperf_submit_on_core(void *arg1, void *arg2)
/* Submit initial I/O for each block device. Each time one
* completes, another will be submitted. */
while (target != NULL) {
target->ch = spdk_bdev_get_io_channel(target->bdev);
target->ch = spdk_bdev_get_io_channel(target->bdev_desc);
/* Start a timer to stop this I/O chain when the run is over */
spdk_poller_register(&target->run_timer, end_target, target, target->lcore,

10
test/lib/bdev/nbd/nbd.c
View File

@ -74,6 +74,7 @@ struct nbd_io {
struct nbd_disk {
struct spdk_bdev *bdev;
struct spdk_bdev_desc *bdev_desc;
struct spdk_io_channel *ch;
int fd;
struct spdk_poller *poller;
@ -108,7 +109,7 @@ static void
nbd_shutdown(void)
{
spdk_put_io_channel(g_nbd_disk.ch);
spdk_bdev_unclaim(g_nbd_disk.bdev);
spdk_bdev_close(g_nbd_disk.bdev_desc);
close(g_nbd_disk.fd);
spdk_app_stop(0);
}
@ -355,14 +356,15 @@ nbd_start(void *arg1, void *arg2)
return;
}
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
SPDK_ERRLOG("could not claim bdev %s\n", g_bdev_name);
rc = spdk_bdev_open(bdev, true, NULL, NULL, &g_nbd_disk.bdev_desc);
if (rc != 0) {
SPDK_ERRLOG("could not open bdev %s, error=%d\n", g_bdev_name, rc);
spdk_app_stop(-1);
return;
}
g_nbd_disk.bdev = bdev;
g_nbd_disk.ch = spdk_bdev_get_io_channel(bdev);
g_nbd_disk.ch = spdk_bdev_get_io_channel(g_nbd_disk.bdev_desc);
rc = socketpair(AF_UNIX, SOCK_STREAM, 0, sp);
if (rc != 0) {

5
test/lib/blobfs/fuse/fuse.c
View File

@ -246,11 +246,6 @@ construct_targets(void)
exit(1);
}
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev));
exit(1);
}
g_bs_dev = spdk_bdev_create_bs_dev(bdev);
printf("Mounting BlobFS on bdev %s\n", spdk_bdev_get_name(bdev));

6
test/lib/blobfs/mkfs/mkfs.c
View File

@ -79,12 +79,6 @@ spdk_mkfs_run(void *arg1, void *arg2)
return;
}
if (!spdk_bdev_claim(bdev, NULL, NULL)) {
SPDK_ERRLOG("could not claim bdev %s\n", g_bdev_name);
spdk_app_stop(-1);
return;
}
printf("Initializing filesystem on bdev %s...", g_bdev_name);
fflush(stdout);
g_bs_dev = spdk_bdev_create_bs_dev(bdev);

204
test/unit/lib/bdev/bdev.c/bdev_ut.c
View File

@ -99,9 +99,207 @@ null_clean(void)
return 0;
}
static void
bdev_test(void)
static int
stub_destruct(void *ctx)
{
return 0;
}
static struct spdk_bdev_fn_table fn_table = {
.destruct = stub_destruct,
};
static struct spdk_bdev *
allocate_bdev(char *name)
{
struct spdk_bdev *bdev;
bdev = calloc(1, sizeof(*bdev));
SPDK_CU_ASSERT_FATAL(bdev != NULL);
bdev->name = name;
bdev->fn_table = &fn_table;
spdk_bdev_register(bdev);
CU_ASSERT(TAILQ_EMPTY(&bdev->base_bdevs));
CU_ASSERT(TAILQ_EMPTY(&bdev->vbdevs));
return bdev;
}
static struct spdk_bdev *
allocate_vbdev(char *name, struct spdk_bdev *base1, struct spdk_bdev *base2)
{
struct spdk_bdev *bdev;
struct spdk_bdev *array[2];
bdev = calloc(1, sizeof(*bdev));
SPDK_CU_ASSERT_FATAL(bdev != NULL);
bdev->name = name;
bdev->fn_table = &fn_table;
/* vbdev must have at least one base bdev */
CU_ASSERT(base1 != NULL);
array[0] = base1;
array[1] = base2;
spdk_vbdev_register(bdev, array, base2 == NULL ? 1 : 2);
CU_ASSERT(!TAILQ_EMPTY(&bdev->base_bdevs));
CU_ASSERT(TAILQ_EMPTY(&bdev->vbdevs));
return bdev;
}
static void
free_bdev(struct spdk_bdev *bdev)
{
spdk_bdev_unregister(bdev);
free(bdev);
}
static void
free_vbdev(struct spdk_bdev *bdev)
{
spdk_vbdev_unregister(bdev);
free(bdev);
}
static void
open_write_test(void)
{
struct spdk_bdev *bdev[8];
struct spdk_bdev_desc *desc[8];
int rc;
/*
* Create a tree of bdevs to test various open w/ write cases.
*
* bdev0 through bdev2 are physical block devices, such as NVMe
* namespaces or Ceph block devices.
*
* bdev3 is a virtual bdev with multiple base bdevs. This models
* caching or RAID use cases.
*
* bdev4 through bdev6 are all virtual bdevs with the same base
* bdev. This models partitioning or logical volume use cases.
*
* bdev7 is a virtual bdev with multiple base bdevs, but these
* base bdevs are themselves virtual bdevs.
*
* bdev7
* |
* +----------+
* | |
* bdev3 bdev4 bdev5 bdev6
* | | | |
* +---+---+ +-------+-------+
* | | |
* bdev0 bdev1 bdev2
*/
bdev[0] = allocate_bdev("bdev0");
bdev[1] = allocate_bdev("bdev1");
bdev[2] = allocate_bdev("bdev2");
bdev[3] = allocate_vbdev("bdev3", bdev[0], bdev[1]);
bdev[4] = allocate_vbdev("bdev4", bdev[2], NULL);
bdev[5] = allocate_vbdev("bdev5", bdev[2], NULL);
bdev[6] = allocate_vbdev("bdev6", bdev[2], NULL);
bdev[7] = allocate_vbdev("bdev7", bdev[3], bdev[4]);
/* Open bdev0 read-only. This should succeed. */
rc = spdk_bdev_open(bdev[0], false, NULL, NULL, &desc[0]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[0] != NULL);
spdk_bdev_close(desc[0]);
/* Open bdev1 read/write. This should succeed. */
rc = spdk_bdev_open(bdev[1], true, NULL, NULL, &desc[1]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[1] != NULL);
/*
* Open bdev3 read/write. This should fail, since one of its
* base bdevs have been explicitly opened read/write.
*/
rc = spdk_bdev_open(bdev[3], true, NULL, NULL, &desc[3]);
CU_ASSERT(rc == -EPERM);
/* Open bdev3 read-only. This should succeed. */
rc = spdk_bdev_open(bdev[3], false, NULL, NULL, &desc[3]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[3] != NULL);
spdk_bdev_close(desc[3]);
/*
* Open bdev7 read/write. This should fail, since one of its
* base bdevs have been explicitly opened read/write. This
* test ensures the bdev code traverses through multiple levels
* of base bdevs.
*/
rc = spdk_bdev_open(bdev[7], true, NULL, NULL, &desc[7]);
CU_ASSERT(rc == -EPERM);
/* Open bdev7 read-only. This should succeed. */
rc = spdk_bdev_open(bdev[7], false, NULL, NULL, &desc[7]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[7] != NULL);
spdk_bdev_close(desc[7]);
/* Reset tree by closing remaining descriptors. */
spdk_bdev_close(desc[1]);
/* Open bdev7 read/write. This should succeed. */
rc = spdk_bdev_open(bdev[7], true, NULL, NULL, &desc[7]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[7] != NULL);
/*
* Open bdev4 read/write. This should fail, since one of its
* virtual bdevs has been explicitly opened read/write.
*/
rc = spdk_bdev_open(bdev[4], true, NULL, NULL, &desc[4]);
CU_ASSERT(rc == -EPERM);
/* Open bdev4 read-only. This should succeed. */
rc = spdk_bdev_open(bdev[4], false, NULL, NULL, &desc[4]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[4] != NULL);
spdk_bdev_close(desc[4]);
/*
* Open bdev2 read/write. This should fail, since one of its
* virtual bdevs has been explicitly opened read/write. This
* test ensures the bdev code traverses through multiple levels
* of virtual bdevs.
*/
rc = spdk_bdev_open(bdev[2], true, NULL, NULL, &desc[2]);
CU_ASSERT(rc == -EPERM);
/* Open bdev2 read-only. This should succeed. */
rc = spdk_bdev_open(bdev[2], false, NULL, NULL, &desc[2]);
CU_ASSERT(rc == 0);
CU_ASSERT(desc[2] != NULL);
spdk_bdev_close(desc[2]);
/* Reset tree by closing remaining descriptors. */
spdk_bdev_close(desc[7]);
free_vbdev(bdev[7]);
free_vbdev(bdev[3]);
free_vbdev(bdev[4]);
free_vbdev(bdev[5]);
free_vbdev(bdev[6]);
free_bdev(bdev[0]);
free_bdev(bdev[1]);
free_bdev(bdev[2]);
}
int
@ -121,7 +319,7 @@ main(int argc, char **argv)
}
if (
CU_add_test(suite, "bdev", bdev_test) == NULL
CU_add_test(suite, "open_write", open_write_test) == NULL
) {
CU_cleanup_registry();
return CU_get_error();

8
test/unit/lib/nvmf/discovery.c/discovery_ut.c
View File

@ -89,11 +89,11 @@ spdk_nvmf_listen_addr_cleanup(struct spdk_nvmf_listen_addr *addr)
return;
}
bool
spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx)
int
spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx, struct spdk_bdev_desc **desc)
{
return true;
return 0;
}
const char *

8
test/unit/lib/nvmf/subsystem.c/subsystem_ut.c
View File

@ -155,11 +155,11 @@ spdk_nvmf_session_poll(struct spdk_nvmf_session *session)
return -1;
}
bool
spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx)
int
spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx, struct spdk_bdev_desc **desc)
{
return true;
return 0;
}
const char *

4
test/unit/lib/nvmf/virtual.c/virtual_ut.c
View File

@ -137,7 +137,7 @@ spdk_bdev_get_num_blocks(const struct spdk_bdev *bdev)
}
struct spdk_io_channel *
spdk_bdev_get_io_channel(struct spdk_bdev *bdev)
spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
{
return NULL;
}
@ -198,7 +198,7 @@ int spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)
return -1;
}
void spdk_bdev_unclaim(struct spdk_bdev *bdev)
void spdk_bdev_close(struct spdk_bdev_desc *desc)
{
}

12
test/unit/lib/scsi/lun.c/lun_ut.c
View File

@ -137,15 +137,15 @@ spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)
return -1;
}
bool
spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx)
int
spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
void *remove_ctx, struct spdk_bdev_desc **desc)
{
return true;
return 0;
}
void
spdk_bdev_unclaim(struct spdk_bdev *bdev)
spdk_bdev_close(struct spdk_bdev_desc *desc)
{
}
@ -191,7 +191,7 @@ spdk_bdev_scsi_execute(struct spdk_bdev *bdev, struct spdk_scsi_task *task)
}
struct spdk_io_channel *
spdk_bdev_get_io_channel(struct spdk_bdev *bdev)
spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
{
return NULL;
}