d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/ixgbe: eliminate mbuf write on rearm

Browse Source 
With the mbuf rework, we now have 8 contiguous bytes to be rearmed in the
mbuf just before the 8-bytes of olflags. If we don't do the rearm write
inside the descriptor ring replenishment function, and delay it to
receiving the packet, we can do a single 16B write inside the RX function
to set both the rearm data, and the flags together.

Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
This commit is contained in:
Konstantin Ananyev 2017-04-10 16:59:41 +01:00 committed by Ferruh Yigit
parent 309f95f2b6
commit f160666a10
1 changed files with 52 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
View File

@ -82,19 +82,23 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
__m128i vaddr0, vaddr1;
uintptr_t p0, p1;
mb0 = rxep[0].mbuf;
mb1 = rxep[1].mbuf;
/*
* Flush mbuf with pkt template.
* Data to be rearmed is 6 bytes long.
*/
p0 = (uintptr_t)&mb0->rearm_data;
*(uint64_t *)p0 = rxq->mbuf_initializer;
p1 = (uintptr_t)&mb1->rearm_data;
*(uint64_t *)p1 = rxq->mbuf_initializer;
#ifndef RTE_IXGBE_RX_OLFLAGS_ENABLE
{
uintptr_t p0, p1;
/*
* Flush mbuf with pkt template.
* Data to be rearmed is 6 bytes long.
*/
p0 = (uintptr_t)&mb0->rearm_data;
*(uint64_t *)p0 = rxq->mbuf_initializer;
p1 = (uintptr_t)&mb1->rearm_data;
*(uint64_t *)p1 = rxq->mbuf_initializer;
}
#endif
/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr));
@ -139,14 +143,11 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
#ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE
static inline void
desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags,
struct rte_mbuf **rx_pkts)
{
__m128i ptype0, ptype1, vtag0, vtag1, csum;
union {
uint16_t e[4];
uint64_t dword;
} vol;
__m128i rearm0, rearm1, rearm2, rearm3;
/* mask everything except rss type */
const __m128i rsstype_msk = _mm_set_epi16(
@ -225,12 +226,40 @@ desc_to_olflags_v(__m128i descs[4], uint8_t vlan_flags,
vtag1 = _mm_or_si128(vtag0, vtag1);
vtag1 = _mm_or_si128(ptype0, vtag1);
vol.dword = _mm_cvtsi128_si64(vtag1);
rx_pkts[0]->ol_flags = vol.e[0];
rx_pkts[1]->ol_flags = vol.e[1];
rx_pkts[2]->ol_flags = vol.e[2];
rx_pkts[3]->ol_flags = vol.e[3];
/*
* At this point, we have the 4 sets of flags in the low 64-bits
* of vtag1 (4x16).
* We want to extract these, and merge them with the mbuf init data
* so we can do a single 16-byte write to the mbuf to set the flags
* and all the other initialization fields. Extracting the
* appropriate flags means that we have to do a shift and blend for
* each mbuf before we do the write.
*/
#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10);
rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10);
rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10);
rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10);
#else
rearm0 = _mm_slli_si128(vtag1, 14);
rearm1 = _mm_slli_si128(vtag1, 12);
rearm2 = _mm_slli_si128(vtag1, 10);
rearm3 = _mm_slli_si128(vtag1, 8);
rearm0 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm0, 48));
rearm1 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm1, 48));
rearm2 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm2, 48));
rearm3 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm3, 48));
#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
_mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0);
_mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1);
_mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2);
_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
}
#else
#define desc_to_olflags_v(desc, vlan_flags, rx_pkts) do { \
@ -265,6 +294,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
0, 0 /* ignore pkt_type field */
);
__m128i dd_check, eop_check;
__m128i mbuf_init;
uint8_t vlan_flags;
/* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */
@ -310,6 +340,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
0xFF, 0xFF
);
mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);
/* Cache is empty -> need to scan the buffer rings, but first move
* the next 'n' mbufs into the cache
*/
@ -382,7 +414,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]);
/* set ol_flags with vlan packet type */
desc_to_olflags_v(descs, vlan_flags, &rx_pkts[pos]);
desc_to_olflags_v(descs, mbuf_init, vlan_flags, &rx_pkts[pos]);
/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);