nvme: Add spdk_nvme_ctrlr_reset_async

Implement an async variant of spdk_nvme_ctrlr_reset(). This initial
implementation only allocates a context and returns it to the caller,
relying on the caller to poll the context to execute the existing
spdk_nvme_ctrlr_reset() implementation.

Wire up spdk_nvme_ctrlr_reset() to use this async variant to verify
that NVMe controller reset still works.

Signed-off-by: Jonathan Teh <jonathan.teh@mayadata.io>
Change-Id: I75d4b75dbf5897db452ee65286aef5a4eb839fca
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8330
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>

CHANGELOG.md

@@ -89,6 +89,9 @@ A new option `disable_read_ana_log_page` was added to struct `spdk_nvme_ctrlr_op
 reading ANA log page. The upper layer is expected to read ANA log page instead if `true`.
 The default value is `false`.
 
+New APIs, `spdk_nvme_ctrlr_reset_async` and `spdk_nvme_ctrlr_reset_poll_async`, have been added to
+reset a controller asynchronously.
+
 ### rpc
 
 New RPC `bdev_rbd_register_cluster` and `bdev_rbd_unregister_cluster` was added, it allows to create

include/spdk/nvme.h

@@ -1017,6 +1017,41 @@ void spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr,
  */
 int spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr);
 
+struct spdk_nvme_ctrlr_reset_ctx;
+
+/**
+ * Create a context object that can be polled to perform a full hardware reset of the NVMe controller.
+ *
+ * The function will set the controller reset context on success, user must call
+ * spdk_nvme_ctrlr_reset_poll_async() until it returns a value other than -EAGAIN.
+ *
+ * \param ctrlr Opaque handle to NVMe controller.
+ * \param reset_ctx Double pointer to reset context.
+ *
+ * \return 0 on success.
+ * \return -ENOMEM if context could not be allocated.
+ * \return -ENXIO if controller has been removed.
+ *
+ */
+int spdk_nvme_ctrlr_reset_async(struct spdk_nvme_ctrlr *ctrlr,
+				struct spdk_nvme_ctrlr_reset_ctx **reset_ctx);
+
+/**
+ * Proceed with resetting contoller associated with the controller reset context.
+ *
+ * The controller reset context is one returned from a previous call to
+ * spdk_nvme_ctrlr_reset_async().  Users must call this function on the
+ * controller reset context until it returns a value other than -EAGAIN.
+ *
+ * \param ctrlr_reset_ctx Context used to track controller reset actions.
+ *
+ * \return 0 if all controller reset operations are complete; the ctrlr_reset_ctx
+ * is also freed and no longer valid.
+ * \return -EAGAIN if there are still pending controller reset operations; user must call
+ * spdk_nvme_ctrlr_reset_poll_async again to continue progress.
+ */
+int spdk_nvme_ctrlr_reset_poll_async(struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx);
+
 /**
  * Perform a NVMe subsystem reset.
  *

lib/nvme/nvme_ctrlr.c

@@ -1390,8 +1390,8 @@ nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr)
 	}
 }
 
-int
-spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
+static int
+nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
 {
 	int rc = 0, rc_tmp = 0;
 	struct spdk_nvme_qpair	*qpair;
@@ -1485,6 +1485,70 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
 	return rc;
 }
 
+static void
+nvme_ctrlr_reset_ctx_init(struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx,
+			  struct spdk_nvme_ctrlr *ctrlr)
+{
+	ctrlr_reset_ctx->ctrlr = ctrlr;
+}
+
+static int
+nvme_ctrlr_reset_poll_async(struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx)
+{
+	return nvme_ctrlr_reset(ctrlr_reset_ctx->ctrlr);
+}
+
+int
+spdk_nvme_ctrlr_reset_poll_async(struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx)
+{
+	int rc;
+	if (!ctrlr_reset_ctx) {
+		return -EINVAL;
+	}
+	rc = nvme_ctrlr_reset_poll_async(ctrlr_reset_ctx);
+	if (rc == -EAGAIN) {
+		return rc;
+	}
+
+	free(ctrlr_reset_ctx);
+	return rc;
+}
+
+int
+spdk_nvme_ctrlr_reset_async(struct spdk_nvme_ctrlr *ctrlr,
+			    struct spdk_nvme_ctrlr_reset_ctx **reset_ctx)
+{
+	struct spdk_nvme_ctrlr_reset_ctx *ctrlr_reset_ctx;
+
+	ctrlr_reset_ctx = calloc(1, sizeof(*ctrlr_reset_ctx));
+	if (!ctrlr_reset_ctx) {
+		return -ENOMEM;
+	}
+
+	nvme_ctrlr_reset_ctx_init(ctrlr_reset_ctx, ctrlr);
+	*reset_ctx = ctrlr_reset_ctx;
+
+	return 0;
+}
+
+int
+spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct spdk_nvme_ctrlr_reset_ctx reset_ctx = {};
+	int rc = 0;
+
+	nvme_ctrlr_reset_ctx_init(&reset_ctx, ctrlr);
+
+	while (true) {
+		rc = nvme_ctrlr_reset_poll_async(&reset_ctx);
+		if (rc != -EAGAIN) {
+			break;
+		}
+	}
+
+	return rc;
+}
+
 int
 spdk_nvme_ctrlr_reset_subsystem(struct spdk_nvme_ctrlr *ctrlr)
 {

lib/nvme/nvme_internal.h

@@ -919,6 +919,10 @@ struct spdk_nvme_detach_ctx {
 	TAILQ_HEAD(, nvme_ctrlr_detach_ctx)	head;
 };
 
+struct spdk_nvme_ctrlr_reset_ctx {
+	struct spdk_nvme_ctrlr			*ctrlr;
+};
+
 struct nvme_driver {
 	pthread_mutex_t			lock;
 
@@ -1043,7 +1047,6 @@ void	nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
 int	nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
 				       struct nvme_ctrlr_detach_ctx *ctx);
 void	nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
-int	nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr);
 int	nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
 void	nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx,
 			     struct spdk_nvme_ctrlr *ctrlr);

lib/nvme/spdk_nvme.map

@@ -36,6 +36,8 @@
 	spdk_nvme_ctrlr_set_trid;
 	spdk_nvme_ctrlr_reset_subsystem;
 	spdk_nvme_ctrlr_reset;
+	spdk_nvme_ctrlr_reset_async;
+	spdk_nvme_ctrlr_reset_poll_async;
 	spdk_nvme_ctrlr_fail;
 	spdk_nvme_ctrlr_is_failed;
 	spdk_nvme_ctrlr_get_data;