vhost-blk: apply interrupt

If interrupt mode is set, related poller functions will be
registered to interrupt handler instead of poller.
interrupt_tgt can work as vhost-blk target to support VM.

Change-Id: I3a15f9a63532f44fe0d2f0cb69b0efdd72431d10
Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/4277
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

examples/interrupt_tgt/Makefile

@@ -55,6 +55,9 @@ SPDK_LIB_LIST += bdev_malloc bdev_passthru bdev_error bdev_gpt bdev_split bdev_r
 SPDK_LIB_LIST += bdev_lvol lvol blob_bdev blob
 # blobfs libraries
 SPDK_LIB_LIST += blobfs blobfs_bdev
+# vhost blk related libraries.
+SPDK_LIB_LIST += vhost event_vhost
+SPDK_LIB_LIST += scsi event_scsi # vhost-scsi is not supported, just because vhost lib requires scsi related libs
 
 ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV))
 SPDK_LIB_LIST += env_dpdk_rpc

examples/interrupt_tgt/interrupt_tgt.c

@@ -34,17 +34,22 @@
 #include "spdk/stdinc.h"
 #include "spdk/conf.h"
 #include "spdk/event.h"
+#include "spdk/vhost.h"
 
 static void
 interrupt_tgt_usage(void)
 {
 	printf(" -E                        Set interrupt mode\n");
+	printf(" -S <path>                 directory where to create vhost sockets (default: pwd)\n");
 }
 
 static int
 interrupt_tgt_parse_arg(int ch, char *arg)
 {
 	switch (ch) {
+	case 'S':
+		spdk_vhost_set_socket_path(arg);
+		break;
 	case 'E':
 		spdk_interrupt_mode_enable();
 		break;
@@ -68,7 +73,7 @@ main(int argc, char *argv[])
 	spdk_app_opts_init(&opts);
 	opts.name = "interrupt_tgt";
 
-	if ((rc = spdk_app_parse_args(argc, argv, &opts, "E", NULL,
+	if ((rc = spdk_app_parse_args(argc, argv, &opts, "S:E", NULL,
 				      interrupt_tgt_parse_arg, interrupt_tgt_usage)) !=
 	    SPDK_APP_PARSE_ARGS_SUCCESS) {
 		exit(rc);

lib/vhost/vhost.c

@@ -199,6 +199,38 @@ vhost_vq_avail_ring_get(struct spdk_vhost_virtqueue *virtqueue, uint16_t *reqs,
 	}
 
 	count = spdk_min(count, reqs_len);
+	if (virtqueue->vsession && virtqueue->vsession->interrupt_mode) {
+		/* if completed IO number is larger than SPDK_AIO_QUEUE_DEPTH,
+		 * io_getevent should be called again to ensure all completed IO are processed.
+		 */
+		int rc;
+		uint64_t num_events;
+
+		rc = read(vring->kickfd, &num_events, sizeof(num_events));
+		if (rc < 0) {
+			SPDK_ERRLOG("failed to acknowledge kickfd: %s.\n", spdk_strerror(errno));
+			return -errno;
+		}
+
+		if ((uint16_t)(avail_idx - last_idx) != num_events) {
+			SPDK_DEBUGLOG(vhost_ring,
+				      "virtqueue gets %d reqs, but kickfd shows %lu reqs\n",
+				      avail_idx - last_idx, num_events);
+		}
+
+		if (num_events > count) {
+			SPDK_DEBUGLOG(vhost_ring,
+				      "virtqueue kickfd shows %lu reqs, take %d, send notice for other reqs\n",
+				      num_events, reqs_len);
+			num_events -= count;
+			rc = write(vring->kickfd, &num_events, sizeof(num_events));
+			if (rc < 0) {
+				SPDK_ERRLOG("failed to kick vring: %s.\n", spdk_strerror(errno));
+				return -errno;
+			}
+		}
+	}
+
 	virtqueue->last_avail_idx += count;
 	for (i = 0; i < count; i++) {
 		reqs[i] = vring->avail->ring[(last_idx + i) & size_mask];
@@ -500,6 +532,14 @@ vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession,
 	rte_vhost_clr_inflight_desc_split(vsession->vid, vq_idx, virtqueue->last_used_idx, id);
 
 	virtqueue->used_req_cnt++;
+
+	if (vsession->interrupt_mode) {
+		if (virtqueue->vring.desc == NULL || vhost_vq_event_is_suppressed(virtqueue)) {
+			return;
+		}
+
+		vhost_vq_used_signal(vsession, virtqueue);
+	}
 }
 
 void
@@ -1194,6 +1234,10 @@ vhost_start_device_cb(int vid)
 		goto out;
 	}
 
+	if (spdk_interrupt_mode_is_enabled()) {
+		vsession->interrupt_mode = true;
+	}
+
 	vdev = vsession->vdev;
 	if (vsession->started) {
 		/* already started, nothing to do */
@@ -1242,11 +1286,15 @@ vhost_start_device_cb(int vid)
 			q->packed.used_phase = q->last_used_idx >> 15;
 			q->last_used_idx = q->last_used_idx & 0x7FFF;
 
-			/* Disable I/O submission notifications, we'll be polling. */
-			q->vring.device_event->flags = VRING_PACKED_EVENT_FLAG_DISABLE;
+			if (!vsession->interrupt_mode) {
+				/* Disable I/O submission notifications, we'll be polling. */
+				q->vring.device_event->flags = VRING_PACKED_EVENT_FLAG_DISABLE;
+			}
 		} else {
-			/* Disable I/O submission notifications, we'll be polling. */
-			q->vring.used->flags = VRING_USED_F_NO_NOTIFY;
+			if (!vsession->interrupt_mode) {
+				/* Disable I/O submission notifications, we'll be polling. */
+				q->vring.used->flags = VRING_USED_F_NO_NOTIFY;
+			}
 		}
 
 		q->packed.packed_ring = packed_ring;

lib/vhost/vhost_blk.c

@@ -690,6 +690,14 @@ _vdev_vq_worker(struct spdk_vhost_virtqueue *vq)
 
 }
 
+static int
+vdev_vq_worker(void *arg)
+{
+	struct spdk_vhost_virtqueue *vq = arg;
+
+	return _vdev_vq_worker(vq);
+}
+
 static int
 vdev_worker(void *arg)
 {
@@ -788,6 +796,14 @@ _no_bdev_vdev_vq_worker(struct spdk_vhost_virtqueue *vq)
 	return SPDK_POLLER_BUSY;
 }
 
+static int
+no_bdev_vdev_vq_worker(void *arg)
+{
+	struct spdk_vhost_virtqueue *vq = arg;
+
+	return _no_bdev_vdev_vq_worker(vq);
+}
+
 static int
 no_bdev_vdev_worker(void *arg)
 {
@@ -802,6 +818,55 @@ no_bdev_vdev_worker(void *arg)
 	return SPDK_POLLER_BUSY;
 }
 
+static void
+vhost_blk_session_unregister_interrupts(struct spdk_vhost_blk_session *bvsession)
+{
+	struct spdk_vhost_session *vsession = &bvsession->vsession;
+	struct spdk_vhost_virtqueue *vq;
+	int i;
+
+	SPDK_DEBUGLOG(vhost_blk, "unregister virtqueues interrupt\n");
+	for (i = 0; i < vsession->max_queues; i++) {
+		vq = &vsession->virtqueue[i];
+		if (vq->intr == NULL) {
+			break;
+		}
+
+		SPDK_DEBUGLOG(vhost_blk, "unregister vq[%d]'s kickfd is %d\n",
+			      i, vq->vring.kickfd);
+		spdk_interrupt_unregister(&vq->intr);
+	}
+}
+
+static int
+vhost_blk_session_register_interrupts(struct spdk_vhost_blk_session *bvsession,
+				      spdk_interrupt_fn fn)
+{
+	struct spdk_vhost_session *vsession = &bvsession->vsession;
+	struct spdk_vhost_virtqueue *vq = NULL;
+	int i;
+
+	SPDK_DEBUGLOG(vhost_blk, "Register virtqueues interrupt\n");
+	for (i = 0; i < vsession->max_queues; i++) {
+		vq = &vsession->virtqueue[i];
+		SPDK_DEBUGLOG(vhost_blk, "Register vq[%d]'s kickfd is %d\n",
+			      i, vq->vring.kickfd);
+
+		vq->intr = SPDK_INTERRUPT_REGISTER(vq->vring.kickfd, fn, vq);
+		if (vq->intr == NULL) {
+			SPDK_ERRLOG("Fail to register req notifier handler.\n");
+			goto err;
+		}
+	}
+
+	return 0;
+
+err:
+	vhost_blk_session_unregister_interrupts(bvsession);
+
+	return -1;
+}
+
 static struct spdk_vhost_blk_dev *
 to_blk_dev(struct spdk_vhost_dev *vdev)
 {
@@ -863,6 +928,7 @@ vhost_session_bdev_remove_cb(struct spdk_vhost_dev *vdev,
 			     void *ctx)
 {
 	struct spdk_vhost_blk_session *bvsession;
+	int rc;
 
 	bvsession = (struct spdk_vhost_blk_session *)vsession;
 	if (bvsession->requestq_poller) {
@@ -870,6 +936,16 @@ vhost_session_bdev_remove_cb(struct spdk_vhost_dev *vdev,
 		bvsession->requestq_poller = SPDK_POLLER_REGISTER(no_bdev_vdev_worker, bvsession, 0);
 	}
 
+	if (vsession->virtqueue[0].intr) {
+		vhost_blk_session_unregister_interrupts(bvsession);
+		rc = vhost_blk_session_register_interrupts(bvsession, no_bdev_vdev_vq_worker);
+		if (rc) {
+			SPDK_ERRLOG("%s: Interrupt register failed\n", vsession->name);
+			return -1;
+		}
+
+	}
+
 	return 0;
 }
 
@@ -1013,10 +1089,22 @@ vhost_blk_start_cb(struct spdk_vhost_dev *vdev,
 		}
 	}
 
-	bvsession->requestq_poller = SPDK_POLLER_REGISTER(bvdev->bdev ? vdev_worker : no_bdev_vdev_worker,
-				     bvsession, 0);
-	SPDK_INFOLOG(vhost, "%s: started poller on lcore %d\n",
-		     vsession->name, spdk_env_get_current_core());
+	if (spdk_interrupt_mode_is_enabled()) {
+		rc = vhost_blk_session_register_interrupts(bvsession,
+				bvdev->bdev ? vdev_vq_worker : no_bdev_vdev_vq_worker);
+		if (rc) {
+			SPDK_ERRLOG("%s: Interrupt register failed\n", vsession->name);
+			goto out;
+		}
+		SPDK_INFOLOG(vhost, "%s: started interrupt source on lcore %d\n",
+			     vsession->name, spdk_env_get_current_core());
+	} else {
+		bvsession->requestq_poller = SPDK_POLLER_REGISTER(bvdev->bdev ? vdev_worker : no_bdev_vdev_worker,
+					     bvsession, 0);
+		SPDK_INFOLOG(vhost, "%s: started poller on lcore %d\n",
+			     vsession->name, spdk_env_get_current_core());
+	}
+
 out:
 	vhost_session_start_done(vsession, rc);
 	return rc;
@@ -1072,6 +1160,11 @@ vhost_blk_stop_cb(struct spdk_vhost_dev *vdev,
 	struct spdk_vhost_blk_session *bvsession = to_blk_session(vsession);
 
 	spdk_poller_unregister(&bvsession->requestq_poller);
+
+	if (vsession->virtqueue[0].intr) {
+		vhost_blk_session_unregister_interrupts(bvsession);
+	}
+
 	bvsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb,
 				 bvsession, 1000);
 	return 0;

lib/vhost/vhost_internal.h

@@ -124,6 +124,8 @@ struct spdk_vhost_virtqueue {
 	uint32_t vring_idx;
 
 	struct spdk_vhost_session *vsession;
+
+	struct spdk_interrupt *intr;
 } __attribute((aligned(SPDK_CACHE_LINE_SIZE)));
 
 struct spdk_vhost_session {
@@ -141,6 +143,7 @@ struct spdk_vhost_session {
 	bool started;
 	bool needs_restart;
 	bool forced_polling;
+	bool interrupt_mode;
 
 	struct rte_vhost_memory *mem;
 

test/unit/lib/vhost/vhost.c/vhost_ut.c

@@ -319,7 +319,7 @@ remove_controller_test(void)
 static void
 vq_avail_ring_get_test(void)
 {
-	struct spdk_vhost_virtqueue vq;
+	struct spdk_vhost_virtqueue vq = {};
 	uint16_t avail_mem[34];
 	uint16_t reqs[32];
 	uint16_t reqs_len, ret, i;