d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

vbdev: close the base bdev on its opened thread

Browse Source 
With the JSON configure, the base device will be opened on the
same thread (RPC thread) to handle the JSON operation. Later the
virtual device upon the base device can be opened on another
thread. At the time of virtual device destruction, the base
device is also closed at the thread where opening the virtual
device, it is actually different than the original thread where
this base device is opened through the JSON configure.

Add a thread here to record the exact thread where the base
device is opened and then later route it back to handle the base
device close operation.

Change-Id: Id1299bb0d86db57bce48b9787d7f248d29a6f345
Signed-off-by: GangCao <gang.cao@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/974
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
GangCao 2020-02-21 09:58:08 -05:00 committed by Tomasz Zawadzki
parent ae86baeba2
commit b3be320de4
9 changed files with 191 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/bdev/part.c
View File

@ -39,6 +39,7 @@
#include "spdk/likely.h"
#include "spdk/log.h"
#include "spdk/string.h"
#include "spdk/thread.h"
#include "spdk/bdev_module.h"
@ -55,6 +56,7 @@ struct spdk_bdev_part_base {
struct bdev_part_tailq *tailq;
spdk_io_channel_create_cb ch_create_cb;
spdk_io_channel_destroy_cb ch_destroy_cb;
struct spdk_thread *thread;
};
struct spdk_bdev *
@ -87,12 +89,24 @@ spdk_bdev_part_base_get_bdev_name(struct spdk_bdev_part_base *part_base)
return part_base->bdev->name;
}
static void
_spdk_bdev_part_base_free(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
void
spdk_bdev_part_base_free(struct spdk_bdev_part_base *base)
{
if (base->desc) {
spdk_bdev_close(base->desc);
base->desc = NULL;
/* Close the underlying bdev on its same opened thread. */
if (base->thread && base->thread != spdk_get_thread()) {
spdk_thread_send_msg(base->thread, _spdk_bdev_part_base_free, base->desc);
} else {
spdk_bdev_close(base->desc);
}
}
if (base->base_free_fn != NULL) {
@ -442,6 +456,9 @@ struct spdk_bdev_part_base *
return NULL;
}
/* Save the thread where the base device is opened */
base->thread = spdk_get_thread();
return base;
}

59
module/bdev/compress/vbdev_compress.c
View File

@ -128,6 +128,7 @@ struct vbdev_compress {
bool orphaned; /* base bdev claimed but comp_bdev not registered */
TAILQ_HEAD(, vbdev_comp_op) queued_comp_ops;
TAILQ_ENTRY(vbdev_compress) link;
struct spdk_thread *thread; /* thread where base device is opened */
};
static TAILQ_HEAD(, vbdev_compress) g_vbdev_comp = TAILQ_HEAD_INITIALIZER(g_vbdev_comp);
@ -909,6 +910,14 @@ _device_unregister_cb(void *io_device)
free(comp_bdev);
}
static void
_vbdev_compress_destruct_cb(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
static void
vbdev_compress_destruct_cb(void *cb_arg, int reduce_errno)
{
@ -919,7 +928,12 @@ vbdev_compress_destruct_cb(void *cb_arg, int reduce_errno)
} else {
TAILQ_REMOVE(&g_vbdev_comp, comp_bdev, link);
spdk_bdev_module_release_bdev(comp_bdev->base_bdev);
spdk_bdev_close(comp_bdev->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (comp_bdev->thread && comp_bdev->thread != spdk_get_thread()) {
spdk_thread_send_msg(comp_bdev->thread, _vbdev_compress_destruct_cb, comp_bdev->base_desc);
} else {
spdk_bdev_close(comp_bdev->base_desc);
}
comp_bdev->vol = NULL;
if (comp_bdev->orphaned == false) {
spdk_io_device_unregister(comp_bdev, _device_unregister_cb);
@ -1076,6 +1090,14 @@ vbdev_compress_config_json(struct spdk_json_write_ctx *w)
return 0;
}
static void
_vbdev_reduce_init_cb(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
/* Callback from reduce for when init is complete. We'll pass the vbdev_comp struct
* used for initial metadata operations to claim where it will be further filled out
* and added to the global list.
@ -1087,7 +1109,12 @@ vbdev_reduce_init_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno
/* We're done with metadata operations */
spdk_put_io_channel(meta_ctx->base_ch);
spdk_bdev_close(meta_ctx->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (meta_ctx->thread && meta_ctx->thread != spdk_get_thread()) {
spdk_thread_send_msg(meta_ctx->thread, _vbdev_reduce_init_cb, meta_ctx->base_desc);
} else {
spdk_bdev_close(meta_ctx->base_desc);
}
meta_ctx->base_desc = NULL;
if (reduce_errno == 0) {
@ -1311,6 +1338,10 @@ vbdev_init_reduce(struct spdk_bdev *bdev, const char *pm_path)
free(meta_ctx);
return -EINVAL;
}
/* Save the thread where the base device is opened */
meta_ctx->thread = spdk_get_thread();
meta_ctx->base_ch = spdk_bdev_get_io_channel(meta_ctx->base_desc);
spdk_reduce_vol_init(&meta_ctx->params, &meta_ctx->backing_dev,
@ -1569,6 +1600,9 @@ vbdev_compress_claim(struct vbdev_compress *comp_bdev)
goto error_open;
}
/* Save the thread where the base device is opened */
comp_bdev->thread = spdk_get_thread();
spdk_io_device_register(comp_bdev, comp_bdev_ch_create_cb, comp_bdev_ch_destroy_cb,
sizeof(struct comp_io_channel),
comp_bdev->comp_bdev.name);
@ -1632,6 +1666,14 @@ bdev_compress_delete(const char *name, spdk_delete_compress_complete cb_fn, void
}
}
static void
_vbdev_reduce_load_cb(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
/* Callback from reduce for then load is complete. We'll pass the vbdev_comp struct
* used for initial metadata operations to claim where it will be further filled out
* and added to the global list.
@ -1644,7 +1686,12 @@ vbdev_reduce_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno
/* Done with metadata operations */
spdk_put_io_channel(meta_ctx->base_ch);
spdk_bdev_close(meta_ctx->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (meta_ctx->thread && meta_ctx->thread != spdk_get_thread()) {
spdk_thread_send_msg(meta_ctx->thread, _vbdev_reduce_load_cb, meta_ctx->base_desc);
} else {
spdk_bdev_close(meta_ctx->base_desc);
}
meta_ctx->base_desc = NULL;
if (reduce_errno != 0 && reduce_errno != -ENOENT) {
@ -1676,6 +1723,9 @@ vbdev_reduce_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno
goto err;
}
/* Save the thread where the base device is opened */
meta_ctx->thread = spdk_get_thread();
meta_ctx->comp_bdev.module = &compress_if;
pthread_mutex_init(&meta_ctx->reduce_lock, NULL);
rc = spdk_bdev_module_claim_bdev(meta_ctx->base_bdev, meta_ctx->base_desc,
@ -1736,6 +1786,9 @@ vbdev_compress_examine(struct spdk_bdev *bdev)
return;
}
/* Save the thread where the base device is opened */
meta_ctx->thread = spdk_get_thread();
meta_ctx->base_ch = spdk_bdev_get_io_channel(meta_ctx->base_desc);
spdk_reduce_vol_load(&meta_ctx->backing_dev, vbdev_reduce_load_cb, meta_ctx);
}

21
module/bdev/crypto/vbdev_crypto.c
View File

@ -173,6 +173,7 @@ struct vbdev_crypto {
struct rte_cryptodev_sym_session *session_decrypt; /* decryption session for this bdev */
struct rte_crypto_sym_xform cipher_xform; /* crypto control struct for this bdev */
TAILQ_ENTRY(vbdev_crypto) link;
struct spdk_thread *thread; /* thread where base device is opened */
};
static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto);
@ -1202,6 +1203,15 @@ _device_unregister_cb(void *io_device)
free(crypto_bdev);
}
/* Wrapper for the bdev close operation. */
static void
_vbdev_crypto_destruct(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
/* Called after we've unregistered following a hot remove callback.
* Our finish entry point will be called next.
*/
@ -1216,8 +1226,12 @@ vbdev_crypto_destruct(void *ctx)
/* Unclaim the underlying bdev. */
spdk_bdev_module_release_bdev(crypto_bdev->base_bdev);
/* Close the underlying bdev. */
spdk_bdev_close(crypto_bdev->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (crypto_bdev->thread && crypto_bdev->thread != spdk_get_thread()) {
spdk_thread_send_msg(crypto_bdev->thread, _vbdev_crypto_destruct, crypto_bdev->base_desc);
} else {
spdk_bdev_close(crypto_bdev->base_desc);
}
/* Unregister the io_device. */
spdk_io_device_unregister(crypto_bdev, _device_unregister_cb);
@ -1826,6 +1840,9 @@ vbdev_crypto_claim(struct spdk_bdev *bdev)
goto error_open;
}
/* Save the thread where the base device is opened */
vbdev->thread = spdk_get_thread();
rc = spdk_bdev_module_claim_bdev(bdev, vbdev->base_desc, vbdev->crypto_bdev.module);
if (rc) {
SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev));

20
module/bdev/delay/vbdev_delay.c
View File

@ -87,6 +87,7 @@ struct vbdev_delay {
uint64_t average_write_latency_ticks; /* the average write delay */
uint64_t p99_write_latency_ticks; /* the p99 write delay */
TAILQ_ENTRY(vbdev_delay) link;
struct spdk_thread *thread; /* thread where base device is opened */
};
static TAILQ_HEAD(, vbdev_delay) g_delay_nodes = TAILQ_HEAD_INITIALIZER(g_delay_nodes);
@ -129,6 +130,14 @@ _device_unregister_cb(void *io_device)
free(delay_node);
}
static void
_vbdev_delay_destruct(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
static int
vbdev_delay_destruct(void *ctx)
{
@ -143,8 +152,12 @@ vbdev_delay_destruct(void *ctx)
/* Unclaim the underlying bdev. */
spdk_bdev_module_release_bdev(delay_node->base_bdev);
/* Close the underlying bdev. */
spdk_bdev_close(delay_node->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (delay_node->thread && delay_node->thread != spdk_get_thread()) {
spdk_thread_send_msg(delay_node->thread, _vbdev_delay_destruct, delay_node->base_desc);
} else {
spdk_bdev_close(delay_node->base_desc);
}
/* Unregister the io_device. */
spdk_io_device_unregister(delay_node, _device_unregister_cb);
@ -653,6 +666,9 @@ vbdev_delay_register(struct spdk_bdev *bdev)
goto error_unregister;
}
/* Save the thread where the base device is opened */
delay_node->thread = spdk_get_thread();
rc = spdk_bdev_module_claim_bdev(bdev, delay_node->base_desc, delay_node->delay_bdev.module);
if (rc) {
SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev));

19
module/bdev/nvme/bdev_nvme_rpc.c
View File

@ -49,6 +49,7 @@ struct open_descriptors {
void *desc;
struct spdk_bdev *bdev;
TAILQ_ENTRY(open_descriptors) tqlst;
struct spdk_thread *thread;
};
typedef TAILQ_HEAD(, open_descriptors) open_descriptors_t;
@ -515,6 +516,14 @@ struct firmware_update_info {
struct rpc_apply_firmware *req;
};
static void
_apply_firmware_cleanup(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
static void
apply_firmware_cleanup(void *cb_arg)
{
@ -540,7 +549,12 @@ apply_firmware_cleanup(void *cb_arg)
TAILQ_FOREACH_SAFE(opt, &firm_ctx->desc_head, tqlst, tmp) {
TAILQ_REMOVE(&firm_ctx->desc_head, opt, tqlst);
spdk_bdev_close(opt->desc);
/* Close the underlying bdev on its same opened thread. */
if (opt->thread && opt->thread != spdk_get_thread()) {
spdk_thread_send_msg(opt->thread, _apply_firmware_cleanup, opt->desc);
} else {
spdk_bdev_close(opt->desc);
}
free(opt);
}
free(firm_ctx);
@ -715,6 +729,9 @@ spdk_rpc_bdev_nvme_apply_firmware(struct spdk_jsonrpc_request *request,
return;
}
/* Save the thread where the base device is opened */
opt->thread = spdk_get_thread();
opt->desc = desc;
opt->bdev = bdev;
TAILQ_INSERT_TAIL(&firm_ctx->desc_head, opt, tqlst);

36
module/bdev/ocf/vbdev_ocf.c
View File

@ -157,6 +157,14 @@ get_other_cache_instance(struct vbdev_ocf *vbdev)
return NULL;
}
static void
_remove_base_bdev(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
/* Close and unclaim base bdev */
static void
remove_base_bdev(struct vbdev_ocf_base *base)
@ -167,7 +175,12 @@ remove_base_bdev(struct vbdev_ocf_base *base)
}
spdk_bdev_module_release_bdev(base->bdev);
spdk_bdev_close(base->desc);
/* Close the underlying bdev on its same opened thread. */
if (base->thread && base->thread != spdk_get_thread()) {
spdk_thread_send_msg(base->thread, _remove_base_bdev, base->desc);
} else {
spdk_bdev_close(base->desc);
}
base->attached = false;
}
}
@ -1378,6 +1391,9 @@ attach_base(struct vbdev_ocf_base *base)
return -ENOMEM;
}
/* Save the thread where the base device is opened */
base->thread = spdk_get_thread();
base->attached = true;
return status;
}
@ -1489,6 +1505,14 @@ struct metadata_probe_ctx {
int refcnt;
};
static void
_examine_ctx_put(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
static void
examine_ctx_put(struct metadata_probe_ctx *ctx)
{
@ -1505,7 +1529,12 @@ examine_ctx_put(struct metadata_probe_ctx *ctx)
}
if (ctx->base.desc) {
spdk_bdev_close(ctx->base.desc);
/* Close the underlying bdev on its same opened thread. */
if (ctx->base.thread && ctx->base.thread != spdk_get_thread()) {
spdk_thread_send_msg(ctx->base.thread, _examine_ctx_put, ctx->base.desc);
} else {
spdk_bdev_close(ctx->base.desc);
}
}
if (ctx->volume) {
@ -1689,6 +1718,9 @@ vbdev_ocf_examine_disk(struct spdk_bdev *bdev)
return;
}
/* Save the thread where the base device is opened */
ctx->base.thread = spdk_get_thread();
ocf_metadata_probe(vbdev_ocf_ctx, ctx->volume, metadata_probe_cb, ctx);
}

3
module/bdev/ocf/vbdev_ocf.h
View File

@ -145,6 +145,9 @@ struct vbdev_ocf_base {
/* Reference to main vbdev */
struct vbdev_ocf *parent;
/* thread where base device is opened */
struct spdk_thread *thread;
};
/*

7
module/bdev/passthru/vbdev_passthru.c
View File

@ -133,6 +133,7 @@ static void
_vbdev_passthru_destruct(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
@ -154,7 +155,11 @@ vbdev_passthru_destruct(void *ctx)
spdk_bdev_module_release_bdev(pt_node->base_bdev);
/* Close the underlying bdev on its same opened thread. */
spdk_thread_send_msg(pt_node->thread, _vbdev_passthru_destruct, pt_node->base_desc);
if (pt_node->thread && pt_node->thread != spdk_get_thread()) {
spdk_thread_send_msg(pt_node->thread, _vbdev_passthru_destruct, pt_node->base_desc);
} else {
spdk_bdev_close(pt_node->base_desc);
}
/* Unregister the io_device. */
spdk_io_device_unregister(pt_node, _device_unregister_cb);

20
module/bdev/zone_block/vbdev_zone_block.c
View File

@ -85,6 +85,7 @@ struct bdev_zone_block {
uint64_t zone_capacity; /* zone capacity */
uint64_t zone_shift; /* log2 of zone_size */
TAILQ_ENTRY(bdev_zone_block) link;
struct spdk_thread *thread; /* thread where base device is opened */
};
static TAILQ_HEAD(, bdev_zone_block) g_bdev_nodes = TAILQ_HEAD_INITIALIZER(g_bdev_nodes);
@ -165,6 +166,14 @@ _device_unregister_cb(void *io_device)
free(bdev_node);
}
static void
_zone_block_destruct(void *ctx)
{
struct spdk_bdev_desc *desc = ctx;
spdk_bdev_close(desc);
}
static int
zone_block_destruct(void *ctx)
{
@ -175,8 +184,12 @@ zone_block_destruct(void *ctx)
/* Unclaim the underlying bdev. */
spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(bdev_node->base_desc));
/* Close the underlying bdev. */
spdk_bdev_close(bdev_node->base_desc);
/* Close the underlying bdev on its same opened thread. */
if (bdev_node->thread && bdev_node->thread != spdk_get_thread()) {
spdk_thread_send_msg(bdev_node->thread, _zone_block_destruct, bdev_node->base_desc);
} else {
spdk_bdev_close(bdev_node->base_desc);
}
/* Unregister the io_device. */
spdk_io_device_unregister(bdev_node, _device_unregister_cb);
@ -797,6 +810,9 @@ zone_block_register(struct spdk_bdev *base_bdev)
goto open_failed;
}
/* Save the thread where the base device is opened */
bdev_node->thread = spdk_get_thread();
rc = spdk_bdev_module_claim_bdev(base_bdev, bdev_node->base_desc, bdev_node->bdev.module);
if (rc) {
SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(base_bdev));