d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
releases
numam-dpdk/drivers/net/virtio/virtio_rxtx_packed_neon.h
Maxime Coquelin ba55c94a7e net/virtio: revert forcing IOVA as VA mode for virtio-user 
This patch removes the simplification in Virtio descriptors
handling, where their buffer addresses are IOVAs for Virtio
PCI devices, and VA-only for Virtio-user devices, which
added a requirement on Virtio-user that it only supported
IOVA as VA.

This change introduced a regression for applications using
Virtio-user and other physical PMDs that require IOVA as PA
because they don't use an IOMMU.

This patch reverts to the old behaviour, but needed to be
reworked because of the refactoring that happened in v21.02.

Fixes: 17043a2909 ("net/virtio: force IOVA as VA mode for virtio-user")
Cc: stable@dpdk.org

Reported-by: Olivier Matz <olivier.matz@6wind.com>
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Tested-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: David Marchand <david.marchand@redhat.com>
2021-09-30 12:58:09 +02:00

294 lines
9.0 KiB
C
Raw Permalink Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Arm Corporation
*/
#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <rte_net.h>
#include <rte_vect.h>
#include "virtio_ethdev.h"
#include "virtio.h"
#include "virtio_rxtx_packed.h"
#include "virtqueue.h"
static inline int
virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
struct rte_mbuf **tx_pkts)
{
struct virtqueue *vq = virtnet_txq_to_vq(txvq);
uint16_t head_size = vq->hw->vtnet_hdr_size;
uint16_t idx = vq->vq_avail_idx;
struct virtio_net_hdr *hdr;
struct vq_desc_extra *dxp;
struct vring_packed_desc *p_desc;
uint16_t i;
if (idx & PACKED_BATCH_MASK)
return -1;
if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
return -1;
/* Map four refcnt and nb_segs from mbufs to one NEON register. */
uint8x16_t ref_seg_msk = {
2, 3, 4, 5,
10, 11, 12, 13,
18, 19, 20, 21,
26, 27, 28, 29
};
/* Map four data_off from mbufs to one NEON register. */
uint8x8_t data_msk = {
0, 1,
8, 9,
16, 17,
24, 25
};
uint16x8_t net_hdr_msk = {
0xFFFF, 0xFFFF,
0, 0, 0, 0
};
uint16x4_t pkts[PACKED_BATCH_SIZE];
uint8x16x2_t mbuf;
/* Load four mbufs rearm data. */
RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
pkts[0] = vld1_u16((uint16_t *)&tx_pkts[0]->rearm_data);
pkts[1] = vld1_u16((uint16_t *)&tx_pkts[1]->rearm_data);
pkts[2] = vld1_u16((uint16_t *)&tx_pkts[2]->rearm_data);
pkts[3] = vld1_u16((uint16_t *)&tx_pkts[3]->rearm_data);
mbuf.val[0] = vreinterpretq_u8_u16(vcombine_u16(pkts[0], pkts[1]));
mbuf.val[1] = vreinterpretq_u8_u16(vcombine_u16(pkts[2], pkts[3]));
/* refcnt = 1 and nb_segs = 1 */
uint32x4_t def_ref_seg = vdupq_n_u32(0x10001);
/* Check refcnt and nb_segs. */
uint32x4_t ref_seg = vreinterpretq_u32_u8(vqtbl2q_u8(mbuf, ref_seg_msk));
uint64x2_t cmp1 = vreinterpretq_u64_u32(~vceqq_u32(ref_seg, def_ref_seg));
if (unlikely(vgetq_lane_u64(cmp1, 0) || vgetq_lane_u64(cmp1, 1)))
return -1;
/* Check headroom is enough. */
uint16x4_t head_rooms = vdup_n_u16(head_size);
RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
offsetof(struct rte_mbuf, rearm_data));
uint16x4_t data_offset = vreinterpret_u16_u8(vqtbl2_u8(mbuf, data_msk));
uint64x1_t cmp2 = vreinterpret_u64_u16(vclt_u16(data_offset, head_rooms));
if (unlikely(vget_lane_u64(cmp2, 0)))
return -1;
virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
dxp = &vq->vq_descx[idx + i];
dxp->ndescs = 1;
dxp->cookie = tx_pkts[i];
}
virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
tx_pkts[i]->data_off -= head_size;
tx_pkts[i]->data_len += head_size;
}
uint64x2x2_t desc[PACKED_BATCH_SIZE / 2];
uint64x2_t base_addr0 = {
VIRTIO_MBUF_ADDR(tx_pkts[0], vq) + tx_pkts[0]->data_off,
VIRTIO_MBUF_ADDR(tx_pkts[1], vq) + tx_pkts[1]->data_off
};
uint64x2_t base_addr1 = {
VIRTIO_MBUF_ADDR(tx_pkts[2], vq) + tx_pkts[2]->data_off,
VIRTIO_MBUF_ADDR(tx_pkts[3], vq) + tx_pkts[3]->data_off
};
desc[0].val[0] = base_addr0;
desc[1].val[0] = base_addr1;
uint64_t flags = (uint64_t)vq->vq_packed.cached_flags << FLAGS_LEN_BITS_OFFSET;
uint64x2_t tx_desc0 = {
flags | (uint64_t)idx << ID_BITS_OFFSET | tx_pkts[0]->data_len,
flags | (uint64_t)(idx + 1) << ID_BITS_OFFSET | tx_pkts[1]->data_len
};
uint64x2_t tx_desc1 = {
flags | (uint64_t)(idx + 2) << ID_BITS_OFFSET | tx_pkts[2]->data_len,
flags | (uint64_t)(idx + 3) << ID_BITS_OFFSET | tx_pkts[3]->data_len
};
desc[0].val[1] = tx_desc0;
desc[1].val[1] = tx_desc1;
if (!vq->hw->has_tx_offload) {
virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
struct virtio_net_hdr *, -head_size);
/* Clear net hdr. */
uint16x8_t v_hdr = vld1q_u16((void *)hdr);
vst1q_u16((void *)hdr, vandq_u16(v_hdr, net_hdr_msk));
}
} else {
virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
struct virtio_net_hdr *, -head_size);
virtqueue_xmit_offload(hdr, tx_pkts[i]);
}
}
/* Enqueue packet buffers. */
p_desc = &vq->vq_packed.ring.desc[idx];
vst2q_u64((uint64_t *)p_desc, desc[0]);
vst2q_u64((uint64_t *)(p_desc + 2), desc[1]);
virtio_update_batch_stats(&txvq->stats, tx_pkts[0]->pkt_len,
tx_pkts[1]->pkt_len, tx_pkts[2]->pkt_len,
tx_pkts[3]->pkt_len);
vq->vq_avail_idx += PACKED_BATCH_SIZE;
vq->vq_free_cnt -= PACKED_BATCH_SIZE;
if (vq->vq_avail_idx >= vq->vq_nentries) {
vq->vq_avail_idx -= vq->vq_nentries;
vq->vq_packed.cached_flags ^=
VRING_PACKED_DESC_F_AVAIL_USED;
}
return 0;
}
static inline int
virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
struct rte_mbuf **rx_pkts)
{
struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
struct virtio_hw *hw = vq->hw;
uint16_t head_size = hw->vtnet_hdr_size;
uint16_t id = vq->vq_used_cons_idx;
struct vring_packed_desc *p_desc;
uint16_t i;
if (id & PACKED_BATCH_MASK)
return -1;
if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
return -1;
/* Map packed descriptor to mbuf fields. */
uint8x16_t shuf_msk1 = {
0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
0, 1, /* octet 1~0, low 16 bits pkt_len */
0xFF, 0xFF, /* skip high 16 bits of pkt_len, zero out */
0, 1, /* octet 1~0, 16 bits data_len */
0xFF, 0xFF, /* vlan tci set as unknown */
0xFF, 0xFF, 0xFF, 0xFF
};
uint8x16_t shuf_msk2 = {
0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
8, 9, /* octet 9~8, low 16 bits pkt_len */
0xFF, 0xFF, /* skip high 16 bits of pkt_len, zero out */
8, 9, /* octet 9~8, 16 bits data_len */
0xFF, 0xFF, /* vlan tci set as unknown */
0xFF, 0xFF, 0xFF, 0xFF
};
/* Subtract the header length. */
uint16x8_t len_adjust = {
0, 0, /* ignore pkt_type field */
head_size, /* sub head_size on pkt_len */
0, /* ignore high 16 bits of pkt_len */
head_size, /* sub head_size on data_len */
0, 0, 0 /* ignore non-length fields */
};
uint64x2_t desc[PACKED_BATCH_SIZE / 2];
uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2];
uint64x2_t pkt_mb[PACKED_BATCH_SIZE];
p_desc = &vq->vq_packed.ring.desc[id];
/* Load high 64 bits of packed descriptor 0,1. */
desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1];
/* Load high 64 bits of packed descriptor 2,3. */
desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1];
/* Only care avail/used bits. */
uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK);
/* Extract high 32 bits of packed descriptor (id, flags). */
uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]),
vreinterpretq_u32_u64(desc[1]));
uint32x4_t v_flag = vandq_u32(v_desc, v_mask);
uint32x4_t v_used_flag = vdupq_n_u32(0);
if (vq->vq_packed.used_wrap_counter)
v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK);
uint64x2_t desc_stats = vreinterpretq_u64_u32(~vceqq_u32(v_flag, v_used_flag));
/* Check all descs are used. */
if (unlikely(vgetq_lane_u64(desc_stats, 0) || vgetq_lane_u64(desc_stats, 1)))
return -1;
/* Load 2 mbuf pointers per time. */
mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]);
vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]);
mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]);
vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]);
/**
* Update data length and packet length for descriptor.
* structure of pkt_mb:
* --------------------------------------------------------------------
* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
* --------------------------------------------------------------------
*/
pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[0]), shuf_msk1));
pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[0]), shuf_msk2));
pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[1]), shuf_msk1));
pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[1]), shuf_msk2));
pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]);
vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]);
vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]);
vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]);
if (hw->has_rx_offload) {
virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
char *addr = (char *)rx_pkts[i]->buf_addr +
RTE_PKTMBUF_HEADROOM - head_size;
virtio_vec_rx_offload(rx_pkts[i],
(struct virtio_net_hdr *)addr);
}
}
virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
rx_pkts[3]->pkt_len);
vq->vq_free_cnt += PACKED_BATCH_SIZE;
vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
if (vq->vq_used_cons_idx >= vq->vq_nentries) {
vq->vq_used_cons_idx -= vq->vq_nentries;
vq->vq_packed.used_wrap_counter ^= 1;
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink