diff --git a/lib/librte_vhost/vdpa.c b/lib/librte_vhost/vdpa.c index e915488432..24a6698e91 100644 --- a/lib/librte_vhost/vdpa.c +++ b/lib/librte_vhost/vdpa.c @@ -181,7 +181,7 @@ rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) return -1; if (unlikely(dlen < vq->desc[desc_id].len)) { - idesc = alloc_copy_ind_table(dev, vq, + idesc = vhost_alloc_copy_ind_table(dev, vq, vq->desc[desc_id].addr, vq->desc[desc_id].len); if (unlikely(!idesc)) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 7d427b60a5..981837b5dd 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -200,6 +200,39 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, } } +void * +vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return NULL; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return NULL; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + void cleanup_vq(struct vhost_virtqueue *vq, int destroy) { diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index dd27c24c17..884befa858 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -488,6 +488,9 @@ void vhost_backend_cleanup(struct virtio_net *dev); uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t iova, uint64_t *len, uint8_t perm); +void *vhost_alloc_copy_ind_table(struct virtio_net *dev, + struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len); int vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq); void vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq); @@ -601,39 +604,6 @@ vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) eventfd_write(vq->callfd, (eventfd_t)1); } -static __rte_always_inline void * -alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t desc_addr, uint64_t desc_len) -{ - void *idesc; - uint64_t src, dst; - uint64_t len, remain = desc_len; - - idesc = rte_malloc(__func__, desc_len, 0); - if (unlikely(!idesc)) - return 0; - - dst = (uint64_t)(uintptr_t)idesc; - - while (remain) { - len = remain; - src = vhost_iova_to_vva(dev, vq, desc_addr, &len, - VHOST_ACCESS_RO); - if (unlikely(!src || !len)) { - rte_free(idesc); - return 0; - } - - rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); - - remain -= len; - dst += len; - desc_addr += len; - } - - return idesc; -} - static __rte_always_inline void free_ind_table(void *idesc) { diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 9ddaa10212..50aeaa0301 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -337,7 +337,7 @@ fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, * The indirect desc table is not contiguous * in process VA space, we have to copy it. */ - idesc = alloc_copy_ind_table(dev, vq, + idesc = vhost_alloc_copy_ind_table(dev, vq, vq->desc[idx].addr, vq->desc[idx].len); if (unlikely(!idesc)) return -1; @@ -454,7 +454,8 @@ fill_vec_buf_packed_indirect(struct virtio_net *dev, * The indirect desc table is not contiguous * in process VA space, we have to copy it. */ - idescs = alloc_copy_ind_table(dev, vq, desc->addr, desc->len); + idescs = vhost_alloc_copy_ind_table(dev, + vq, desc->addr, desc->len); if (unlikely(!idescs)) return -1; @@ -610,6 +611,36 @@ reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } +static __rte_noinline void +copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct buf_vector *buf_vec, + struct virtio_net_hdr_mrg_rxbuf *hdr) +{ + uint64_t len; + uint64_t remain = dev->vhost_hlen; + uint64_t src = (uint64_t)(uintptr_t)hdr, dst; + uint64_t iova = buf_vec->buf_iova; + + while (remain) { + len = RTE_MIN(remain, + buf_vec->buf_len); + dst = buf_vec->buf_addr; + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, + len); + + PRINT_PACKET(dev, (uintptr_t)dst, + (uint32_t)len, 0); + vhost_log_cache_write(dev, vq, + iova, len); + + remain -= len; + iova += len; + src += len; + buf_vec++; + } +} + static __rte_always_inline int copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct buf_vector *buf_vec, @@ -703,30 +734,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, num_buffers); if (unlikely(hdr == &tmp_hdr)) { - uint64_t len; - uint64_t remain = dev->vhost_hlen; - uint64_t src = (uint64_t)(uintptr_t)hdr, dst; - uint64_t iova = buf_vec[0].buf_iova; - uint16_t hdr_vec_idx = 0; - - while (remain) { - len = RTE_MIN(remain, - buf_vec[hdr_vec_idx].buf_len); - dst = buf_vec[hdr_vec_idx].buf_addr; - rte_memcpy((void *)(uintptr_t)dst, - (void *)(uintptr_t)src, - len); - - PRINT_PACKET(dev, (uintptr_t)dst, - (uint32_t)len, 0); - vhost_log_cache_write(dev, vq, - iova, len); - - remain -= len; - iova += len; - src += len; - hdr_vec_idx++; - } + copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr); } else { PRINT_PACKET(dev, (uintptr_t)hdr_addr, dev->vhost_hlen, 0); @@ -1064,6 +1072,27 @@ vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) } } +static __rte_noinline void +copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr, + struct buf_vector *buf_vec) +{ + uint64_t len; + uint64_t remain = sizeof(struct virtio_net_hdr); + uint64_t src; + uint64_t dst = (uint64_t)(uintptr_t)hdr; + + while (remain) { + len = RTE_MIN(remain, buf_vec->buf_len); + src = buf_vec->buf_addr; + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + buf_vec++; + } +} + static __rte_always_inline int copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct buf_vector *buf_vec, uint16_t nr_vec, @@ -1095,28 +1124,11 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, if (virtio_net_with_host_offload(dev)) { if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) { - uint64_t len; - uint64_t remain = sizeof(struct virtio_net_hdr); - uint64_t src; - uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr; - uint16_t hdr_vec_idx = 0; - /* * No luck, the virtio-net header doesn't fit * in a contiguous virtual area. */ - while (remain) { - len = RTE_MIN(remain, - buf_vec[hdr_vec_idx].buf_len); - src = buf_vec[hdr_vec_idx].buf_addr; - rte_memcpy((void *)(uintptr_t)dst, - (void *)(uintptr_t)src, len); - - remain -= len; - dst += len; - hdr_vec_idx++; - } - + copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec); hdr = &tmp_hdr; } else { hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);