d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: give qpair_disconnect an asynchronous api.

Browse Source 
qpair_disconnect has previously presented an entirely synchronous API.
However, it relies on other asynchronous operations to complete its
task. By giving it an asynchronous API, we can avoid possible race
conditions. Patch 1 of several.

Change-Id: If9e26ee70ae5d6c0273750226b4408a8e4587e19
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/417345
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Seth Howell 2018-06-29 12:09:47 -07:00 committed by Jim Harris
parent 82c64e2a41
commit 09e3f4e3db
4 changed files with 72 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/spdk/nvmf.h
View File

@ -191,12 +191,20 @@ int spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group,
int spdk_nvmf_poll_group_remove(struct spdk_nvmf_poll_group *group,
struct spdk_nvmf_qpair *qpair);
typedef void (*nvmf_qpair_disconnect_cb)(void *ctx);
/**
* Disconnect an NVMe-oF qpair
*
* \param qpair The NVMe-oF qpair to disconnect.
* \param cb_fn The function to call upon completion of the disconnect.
* \param ctx The context to pass to the callback function.
*
* \return 0 upon success.
* \return -ENOMEM if the function specific context could not be allocated.
*/
void spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair);
int spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn,
void *ctx);
/**
* Create an NVMe-oF subsystem.

72
lib/nvmf/nvmf.c
View File

@ -56,6 +56,13 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF)
#define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024
#define SPDK_NVMF_DEFAULT_IO_UNIT_SIZE 131072
struct nvmf_qpair_disconnect_ctx {
struct spdk_nvmf_qpair *qpair;
struct spdk_nvmf_ctrlr *ctrlr;
nvmf_qpair_disconnect_cb cb_fn;
void *ctx;
};
void
spdk_nvmf_tgt_opts_init(struct spdk_nvmf_tgt_opts *opts)
{
@ -136,7 +143,7 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
spdk_poller_unregister(&group->poller);
TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qptmp) {
spdk_nvmf_qpair_disconnect(qpair);
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
}
TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) {
@ -601,17 +608,24 @@ spdk_nvmf_poll_group_remove(struct spdk_nvmf_poll_group *group,
}
static void
_spdk_nvmf_ctrlr_free(void *ctx)
_spdk_nvmf_ctrlr_free_from_qpair(void *ctx)
{
struct spdk_nvmf_ctrlr *ctrlr = ctx;
struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr;
spdk_nvmf_ctrlr_destruct(ctrlr);
if (qpair_ctx->cb_fn) {
qpair_ctx->cb_fn(qpair_ctx->ctx);
}
free(qpair_ctx);
}
static void
_spdk_nvmf_qpair_destroy(void *ctx, int status)
{
struct spdk_nvmf_qpair *qpair = ctx;
struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair;
struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr;
uint16_t qid = qpair->qid;
uint32_t count;
@ -624,6 +638,10 @@ _spdk_nvmf_qpair_destroy(void *ctx, int status)
spdk_nvmf_transport_qpair_fini(qpair);
if (!ctrlr) {
if (qpair_ctx->cb_fn) {
qpair_ctx->cb_fn(qpair_ctx->ctx);
}
free(qpair_ctx);
return;
}
@ -635,20 +653,31 @@ _spdk_nvmf_qpair_destroy(void *ctx, int status)
if (count == 0) {
/* If this was the last queue pair on the controller, also send a message
* to the subsystem to remove the controller. */
spdk_thread_send_msg(ctrlr->subsys->thread, _spdk_nvmf_ctrlr_free, ctrlr);
qpair_ctx->ctrlr = ctrlr;
spdk_thread_send_msg(ctrlr->subsys->thread, _spdk_nvmf_ctrlr_free_from_qpair, qpair_ctx);
} else {
if (qpair_ctx->cb_fn) {
qpair_ctx->cb_fn(qpair_ctx->ctx);
}
free(qpair_ctx);
}
}
static void
_spdk_nvmf_qpair_deactivate(void *ctx)
{
struct spdk_nvmf_qpair *qpair = ctx;
struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair;
if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING ||
qpair->state == SPDK_NVMF_QPAIR_INACTIVE) {
/* This can occur if the connection is killed by the target,
* which results in a notification that the connection
* died. */
if (qpair_ctx->cb_fn) {
qpair_ctx->cb_fn(qpair_ctx->ctx);
}
free(qpair_ctx);
return;
}
@ -658,22 +687,34 @@ _spdk_nvmf_qpair_deactivate(void *ctx)
/* Check for outstanding I/O */
if (!TAILQ_EMPTY(&qpair->outstanding)) {
qpair->state_cb = _spdk_nvmf_qpair_destroy;
qpair->state_cb_arg = qpair;
qpair->state_cb_arg = qpair_ctx;
return;
}
_spdk_nvmf_qpair_destroy(qpair, 0);
_spdk_nvmf_qpair_destroy(qpair_ctx, 0);
}
void
spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair)
int
spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx)
{
struct nvmf_qpair_disconnect_ctx *qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));
if (!qpair_ctx) {
SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");
return -ENOMEM;
}
qpair_ctx->qpair = qpair;
qpair_ctx->cb_fn = cb_fn;
qpair_ctx->ctx = ctx;
if (qpair->group->thread == spdk_get_thread()) {
_spdk_nvmf_qpair_deactivate(qpair);
_spdk_nvmf_qpair_deactivate(qpair_ctx);
} else {
/* Send a message to the thread that owns this qpair */
spdk_thread_send_msg(qpair->group->thread, _spdk_nvmf_qpair_deactivate, qpair);
spdk_thread_send_msg(qpair->group->thread, _spdk_nvmf_qpair_deactivate, qpair_ctx);
}
return 0;
}
int
@ -821,14 +862,19 @@ spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
{
struct spdk_nvmf_qpair *qpair, *tmp;
struct spdk_nvmf_subsystem_poll_group *sgroup;
int rc = 0;
uint32_t nsid;
TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, tmp) {
if (qpair->ctrlr->subsys == subsystem) {
spdk_nvmf_qpair_disconnect(qpair);
rc += spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
}
}
if (rc != 0) {
return -1;
}
sgroup = &group->sgroups[subsystem->id];
sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;

2
lib/nvmf/rdma.c
View File

@ -780,7 +780,7 @@ nvmf_rdma_disconnect(struct rdma_cm_event *evt)
/* ack the disconnect event before rdma_destroy_id */
rdma_ack_cm_event(evt);
spdk_nvmf_qpair_disconnect(qpair);
spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);
return 0;
}

5
test/unit/lib/nvmf/request.c/request_ut.c
View File

@ -112,9 +112,10 @@ struct spdk_nvme_ns *spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint3
return NULL;
}
void
spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair)
int
spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx)
{
return 0;
}
static void