vhost: added API to call external spdk_events on vdev reactor

The next step towards fixing synchronization issues between RPC and
vhost controller reactor.

The general idea is to hide spdk_vhost_dev_find declaration, so that
vdev-related functions can't be (incorrectly) called from outside of
the vhost controller reactor. The vdev pointer should only be acquired
via spdk_vhost_call_external_event.

Change-Id: I689226c2271f81624b2651bae3b960b7efb1c01b
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/371098
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

include/spdk/vhost.h

@@ -80,4 +80,21 @@ int spdk_vhost_blk_destroy(struct spdk_vhost_dev *dev);
 struct spdk_bdev *spdk_vhost_blk_get_dev(struct spdk_vhost_dev *ctrlr);
 bool spdk_vhost_blk_get_readonly(struct spdk_vhost_dev *vdev);
 
+/**
+ * Call function on reactor of given vhost controller.
+ * If controller is not in use, the event will be called
+ * right away on the caller's thread.
+ *
+ * This function is thread safe.
+ *
+ * \param ctrlr_name name of the vhost controller to run
+ * this event on
+ * \param fn function to be called. The first parameter
+ * of callback function is either actual spdk_vhost_dev
+ * pointer or NULL in case vdev with given name doesn't
+ * exist. The second param is user provided argument *arg*.
+ * \param arg parameter to be passed to *fn*.
+ */
+void spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, void *arg);
+
 #endif /* SPDK_VHOST_H */

lib/vhost/vhost.c

@@ -548,6 +548,26 @@ spdk_vhost_event_cb(void *arg1, void *arg2)
 	sem_post(&ctx->sem);
 }
 
+static void
+spdk_vhost_event_async_fn(void *arg1, void *arg2)
+{
+	struct spdk_vhost_dev_event_ctx *ctx = arg1;
+	struct spdk_vhost_dev *vdev;
+
+	pthread_mutex_lock(&g_spdk_vhost_mutex);
+	vdev = spdk_vhost_dev_find(ctx->ctrlr_name);
+	if (vdev != ctx->vdev) {
+		/* vdev has been changed after enqueuing this event */
+		vdev = NULL;
+	}
+
+	ctx->cb_fn(vdev, arg2);
+	pthread_mutex_unlock(&g_spdk_vhost_mutex);
+
+	free(ctx->ctrlr_name);
+	free(ctx);
+}
+
 int
 spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, void *arg,
 		      unsigned timeout_sec, const char *errmsg)
@@ -585,6 +605,29 @@ spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, vo
 	return ev_ctx.response;
 }
 
+static int
+spdk_vhost_event_async_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, void *arg)
+{
+	struct spdk_vhost_dev_event_ctx *ev_ctx;
+	struct spdk_event *ev;
+
+	ev_ctx = calloc(1, sizeof(*ev_ctx));
+	if (ev_ctx == NULL) {
+		SPDK_ERRLOG("Failed to alloc vhost event.\n");
+		return -ENOMEM;
+	}
+
+	ev_ctx->vdev = vdev;
+	ev_ctx->ctrlr_name = strdup(vdev->name);
+	ev_ctx->cb_fn = cb_fn;
+
+	ev = spdk_event_allocate(vdev->lcore, spdk_vhost_event_async_fn, ev_ctx, arg);
+	assert(ev);
+	spdk_event_call(ev);
+
+	return 0;
+}
+
 static void
 destroy_device(int vid)
 {
@@ -822,4 +865,27 @@ destroy_connection(int vid)
 	pthread_mutex_unlock(&g_spdk_vhost_mutex);
 }
 
+void
+spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, void *arg)
+{
+	struct spdk_vhost_dev *vdev;
+
+	pthread_mutex_lock(&g_spdk_vhost_mutex);
+	vdev = spdk_vhost_dev_find(ctrlr_name);
+
+	if (vdev == NULL) {
+		pthread_mutex_unlock(&g_spdk_vhost_mutex);
+		fn(NULL, arg);
+		return;
+	}
+
+	if (vdev->lcore == -1) {
+		fn(vdev, arg);
+	} else {
+		spdk_vhost_event_async_send(vdev, fn, arg);
+	}
+
+	pthread_mutex_unlock(&g_spdk_vhost_mutex);
+}
+
 SPDK_LOG_REGISTER_TRACE_FLAG("vhost_ring", SPDK_TRACE_VHOST_RING)

lib/vhost/vhost_blk.c

@@ -451,13 +451,7 @@ bdev_remove_cb(void *remove_ctx)
 {
 	struct spdk_vhost_blk_dev *bvdev = remove_ctx;
 
-	if (bvdev->vdev.lcore != -1 && (uint32_t)bvdev->vdev.lcore != spdk_env_get_current_core()) {
-		/* Call on proper core. */
-		spdk_vhost_event_send(&bvdev->vdev, _bdev_remove_cb, bvdev, 1, "vhost blk hot remove");
-		return;
-	}
-
-	_bdev_remove_cb(&bvdev->vdev, bvdev);
+	spdk_vhost_call_external_event(bvdev->vdev.name, _bdev_remove_cb, bvdev);
 }