d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bdev_nvme: add synchronization to reset.

Browse Source 
The generic bdev layer currently has a lot of snychronization built into
spdk_bdev_reset. However, in a couple patches I am going to introduce a
few instances where I call bdev_nvme_reset directly from this module.
The reason I call bdev_nvme_reset directly from this module is so that I
don't have to open a descriptor to the bdev in the module itself.

In order to be able to call bdev_nvme_reset from both this module and in
response to a bdev_io, we need to synchronize and queue reset requests.

Change-Id: I7ece41119cba705a7481d365d20a1eb746a80f64
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/473754
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell 2019-11-08 14:26:09 -07:00 committed by Tomasz Zawadzki
parent a4d9919b67
commit f57b098e7c
2 changed files with 59 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
module/bdev/nvme/bdev_nvme.c
View File

@ -54,13 +54,14 @@ static void bdev_nvme_get_spdk_running_config(FILE *fp);
static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
struct nvme_io_channel {
struct spdk_nvme_qpair *qpair;
struct spdk_poller *poller;
struct spdk_nvme_qpair *qpair;
struct spdk_poller *poller;
TAILQ_HEAD(, spdk_bdev_io) pending_resets;
bool collect_spin_stat;
uint64_t spin_ticks;
uint64_t start_ticks;
uint64_t end_ticks;
bool collect_spin_stat;
uint64_t spin_ticks;
uint64_t start_ticks;
uint64_t end_ticks;
};
struct nvme_bdev_io {
@ -147,6 +148,7 @@ static int bdev_nvme_io_passthru_md(struct nvme_bdev *nbdev, struct spdk_io_chan
struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, void *md_buf, size_t md_len);
static int nvme_ctrlr_create_bdev(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr,
struct nvme_bdev_ns *nvme_ns);
static int bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio);
struct spdk_nvme_qpair *
spdk_bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch)
@ -280,14 +282,47 @@ bdev_nvme_flush(struct nvme_bdev *nbdev, struct nvme_bdev_io *bio,
return 0;
}
static void
_bdev_nvme_complete_pending_resets(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(_ch);
struct spdk_bdev_io *bdev_io;
enum spdk_bdev_io_status status = SPDK_BDEV_IO_STATUS_SUCCESS;
/* A NULL ctx means success. */
if (spdk_io_channel_iter_get_ctx(i) != NULL) {
status = SPDK_BDEV_IO_STATUS_FAILED;
}
while (!TAILQ_EMPTY(&nvme_ch->pending_resets)) {
bdev_io = TAILQ_FIRST(&nvme_ch->pending_resets);
TAILQ_REMOVE(&nvme_ch->pending_resets, bdev_io, module_link);
spdk_bdev_io_complete(bdev_io, status);
}
spdk_for_each_channel_continue(i, 0);
}
static void
_bdev_nvme_reset_complete(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, int rc)
{
/* we are using the for_each_channel cb_arg like a return code here. */
/* If it's zero, we succeeded, otherwise, the reset failed. */
void *cb_arg = NULL;
if (rc) {
cb_arg = (void *)0x1;
SPDK_ERRLOG("Resetting controller failed.\n");
} else {
SPDK_NOTICELOG("Resetting controller successful.\n");
}
__atomic_clear(&nvme_bdev_ctrlr->resetting, __ATOMIC_RELAXED);
/* Make sure we clear any pending resets before returning. */
spdk_for_each_channel(nvme_bdev_ctrlr,
_bdev_nvme_complete_pending_resets,
cb_arg, NULL);
}
static void
@ -377,6 +412,21 @@ _bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
static int
bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio)
{
struct spdk_io_channel *ch;
struct nvme_io_channel *nvme_ch;
if (__atomic_test_and_set(&nvme_bdev_ctrlr->resetting, __ATOMIC_RELAXED)) {
SPDK_NOTICELOG("Unable to perform reset, already in progress.\n");
if (bio) {
ch = spdk_get_io_channel(nvme_bdev_ctrlr);
assert(ch != NULL);
nvme_ch = spdk_io_channel_get_ctx(ch);
TAILQ_INSERT_TAIL(&nvme_ch->pending_resets, spdk_bdev_io_from_ctx(bio), module_link);
spdk_put_io_channel(ch);
}
return 0;
}
/* First, delete all NVMe I/O queue pairs. */
spdk_for_each_channel(nvme_bdev_ctrlr,
_bdev_nvme_reset_destroy_qpair,
@ -588,6 +638,8 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf)
}
ch->poller = spdk_poller_register(bdev_nvme_poll, ch, g_opts.nvme_ioq_poll_period_us);
TAILQ_INIT(&ch->pending_resets);
return 0;
}

1
module/bdev/nvme/common.h
View File

@ -62,6 +62,7 @@ struct nvme_bdev_ctrlr {
struct spdk_nvme_transport_id trid;
char *name;
int ref;
bool resetting;
bool destruct;
/**
* PI check flags. This flags is set to NVMe controllers created only