diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 227d5489b8..d5f97cccbc 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -546,7 +546,7 @@ void vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, uint16_t num_descs, uint16_t buffer_id, - uint32_t length) + uint32_t length, uint16_t inflight_head) { struct vring_packed_desc *desc = &virtqueue->vring.desc_packed[virtqueue->last_used_idx]; bool used, avail; @@ -587,6 +587,8 @@ vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, * written to the descriptor. */ spdk_smp_wmb(); + + rte_vhost_set_last_inflight_io_packed(vsession->vid, virtqueue->vring_idx, inflight_head); /* To mark a desc as used, the device sets the F_USED bit in flags to match * the internal Device ring wrap counter. It also sets the F_AVAIL bit to * match the same value. @@ -596,6 +598,7 @@ vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, } else { desc->flags &= ~VRING_DESC_F_AVAIL_USED; } + rte_vhost_clr_inflight_desc_packed(vsession->vid, virtqueue->vring_idx, inflight_head); vhost_log_used_vring_elem(vsession, virtqueue, virtqueue->last_used_idx); virtqueue->last_used_idx += num_descs; diff --git a/lib/vhost/vhost_blk.c b/lib/vhost/vhost_blk.c index 6867d579aa..4fa11d2732 100644 --- a/lib/vhost/vhost_blk.c +++ b/lib/vhost/vhost_blk.c @@ -72,6 +72,7 @@ struct spdk_vhost_blk_task { uint16_t req_idx; uint16_t num_descs; uint16_t buffer_id; + uint16_t inflight_head; /* for io wait */ struct spdk_bdev_io_wait_entry bdev_io_wait; @@ -141,7 +142,8 @@ blk_task_enqueue(struct spdk_vhost_blk_task *task) if (task->vq->packed.packed_ring) { vhost_vq_packed_ring_enqueue(&task->bvsession->vsession, task->vq, task->num_descs, - task->buffer_id, task->used_len); + task->buffer_id, task->used_len, + task->inflight_head); } else { vhost_vq_used_ring_enqueue(&task->bvsession->vsession, task->vq, task->req_idx, task->used_len); @@ -578,6 +580,10 @@ process_blk_task(struct spdk_vhost_virtqueue *vq, uint16_t req_idx) task->req_idx = req_idx; task->num_descs = num_descs; task->buffer_id = task_idx; + + rte_vhost_set_inflight_desc_packed(task->bvsession->vsession.vid, vq->vring_idx, + req_idx, (req_idx + num_descs - 1) % vq->vring.size, + &task->inflight_head); } task->bvsession->vsession.task_cnt++; @@ -752,7 +758,8 @@ no_bdev_process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_ SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n", vsession->name, req_idx); vhost_vq_packed_ring_enqueue(vsession, vq, num_descs, - task->buffer_id, task->used_len); + task->buffer_id, task->used_len, + task->inflight_head); return; } @@ -769,7 +776,8 @@ no_bdev_process_packed_vq(struct spdk_vhost_blk_session *bvsession, struct spdk_ task->used = false; vhost_vq_packed_ring_enqueue(vsession, vq, num_descs, - task->buffer_id, task->used_len); + task->buffer_id, task->used_len, + task->inflight_head); } static int diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h index 319735d5c5..b2a282402c 100644 --- a/lib/vhost/vhost_internal.h +++ b/lib/vhost/vhost_internal.h @@ -324,11 +324,12 @@ void vhost_vq_used_ring_enqueue(struct spdk_vhost_session *vsession, * \buffer_id descriptor buffer ID. * \length device write length. Specify the length of the buffer that has been initialized * (written to) by the device + * \inflight_head the head idx of this IO inflight desc chain. */ void vhost_vq_packed_ring_enqueue(struct spdk_vhost_session *vsession, struct spdk_vhost_virtqueue *virtqueue, uint16_t num_descs, uint16_t buffer_id, - uint32_t length); + uint32_t length, uint16_t inflight_head); /** * Get subsequent descriptor from given table. diff --git a/test/unit/lib/vhost/vhost.c/vhost_ut.c b/test/unit/lib/vhost/vhost.c/vhost_ut.c index e845bb1ff8..9a1c30a9ce 100644 --- a/test/unit/lib/vhost/vhost.c/vhost_ut.c +++ b/test/unit/lib/vhost/vhost.c/vhost_ut.c @@ -75,6 +75,10 @@ DEFINE_STUB(rte_vhost_set_last_inflight_io_split, int, (int vid, uint16_t vring_idx, uint16_t idx), 0); DEFINE_STUB(rte_vhost_clr_inflight_desc_split, int, (int vid, uint16_t vring_idx, uint16_t last_used_idx, uint16_t idx), 0); +DEFINE_STUB(rte_vhost_set_last_inflight_io_packed, int, + (int vid, uint16_t vring_idx, uint16_t head), 0); +DEFINE_STUB(rte_vhost_clr_inflight_desc_packed, int, + (int vid, uint16_t vring_idx, uint16_t head), 0); DEFINE_STUB_V(rte_vhost_log_write, (int vid, uint64_t addr, uint64_t len)); DEFINE_STUB_V(vhost_session_mem_register, (struct rte_vhost_memory *mem)); DEFINE_STUB_V(vhost_session_mem_unregister, (struct rte_vhost_memory *mem)); @@ -473,9 +477,9 @@ vq_packed_ring_test(void) } /* Host complete them out of order: 1, 0, 2. */ - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1); - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1); - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1, 0); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1, 0); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1, 0); /* Host has got all the available request but only complete three requests */ CU_ASSERT(vq.last_avail_idx == 0); @@ -511,10 +515,10 @@ vq_packed_ring_test(void) CU_ASSERT(vq.packed.avail_phase == 0); /* Host complete all the requests */ - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1); - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1); - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 3, 1); - vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 1, 1, 0); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 0, 1, 0); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 3, 1, 0); + vhost_vq_packed_ring_enqueue(&vs, &vq, 1, 2, 1, 0); CU_ASSERT(vq.last_used_idx == vq.last_avail_idx); CU_ASSERT(vq.packed.used_phase == vq.packed.avail_phase);