diff --git a/examples/l3fwd/l3fwd_em.c b/examples/l3fwd/l3fwd_em.c index a203dc9e46..35de31157e 100644 --- a/examples/l3fwd/l3fwd_em.c +++ b/examples/l3fwd/l3fwd_em.c @@ -860,10 +860,15 @@ em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, int i, nb_enq = 0, nb_deq = 0; struct lcore_conf *lconf; unsigned int lcore_id; + uint16_t *dst_ports; if (event_p_id < 0) return; + dst_ports = rte_zmalloc("", sizeof(uint16_t) * evt_rsrc->vector_size, + RTE_CACHE_LINE_SIZE); + if (dst_ports == NULL) + return; lcore_id = rte_lcore_id(); lconf = &lcore_conf[lcore_id]; @@ -885,13 +890,12 @@ em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, } #if defined RTE_ARCH_X86 || defined __ARM_NEON - l3fwd_em_process_event_vector(events[i].vec, lconf); + l3fwd_em_process_event_vector(events[i].vec, lconf, + dst_ports); #else l3fwd_em_no_opt_process_event_vector(events[i].vec, - lconf); + lconf, dst_ports); #endif - if (flags & L3FWD_EVENT_TX_DIRECT) - event_vector_txq_set(events[i].vec, 0); } if (flags & L3FWD_EVENT_TX_ENQ) { @@ -915,6 +919,7 @@ em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc, l3fwd_event_worker_cleanup(event_d_id, event_p_id, events, nb_enq, nb_deq, 1); + rte_free(dst_ports); } int __rte_noinline diff --git a/examples/l3fwd/l3fwd_em.h b/examples/l3fwd/l3fwd_em.h index fe2ee59f6a..7d051fc076 100644 --- a/examples/l3fwd/l3fwd_em.h +++ b/examples/l3fwd/l3fwd_em.h @@ -100,7 +100,7 @@ l3fwd_em_simple_forward(struct rte_mbuf *m, uint16_t portid, } } -static __rte_always_inline void +static __rte_always_inline uint16_t l3fwd_em_simple_process(struct rte_mbuf *m, struct lcore_conf *qconf) { struct rte_ether_hdr *eth_hdr; @@ -117,6 +117,8 @@ l3fwd_em_simple_process(struct rte_mbuf *m, struct lcore_conf *qconf) m->port = l3fwd_em_handle_ipv6(m, m->port, eth_hdr, qconf); else m->port = BAD_PORT; + + return m->port; } /* @@ -179,7 +181,8 @@ l3fwd_em_no_opt_process_events(int nb_rx, struct rte_event **events, static inline void l3fwd_em_no_opt_process_event_vector(struct rte_event_vector *vec, - struct lcore_conf *qconf) + struct lcore_conf *qconf, + uint16_t *dst_ports) { struct rte_mbuf **mbufs = vec->mbufs; int32_t i; @@ -188,30 +191,20 @@ l3fwd_em_no_opt_process_event_vector(struct rte_event_vector *vec, for (i = 0; i < PREFETCH_OFFSET && i < vec->nb_elem; i++) rte_prefetch0(rte_pktmbuf_mtod(mbufs[i], void *)); - /* Process first packet to init vector attributes */ - l3fwd_em_simple_process(mbufs[0], qconf); - if (vec->attr_valid) { - if (mbufs[0]->port != BAD_PORT) - vec->port = mbufs[0]->port; - else - vec->attr_valid = 0; - } - /* * Prefetch and forward already prefetched packets. */ - for (i = 1; i < (vec->nb_elem - PREFETCH_OFFSET); i++) { + for (i = 0; i < (vec->nb_elem - PREFETCH_OFFSET); i++) { rte_prefetch0( rte_pktmbuf_mtod(mbufs[i + PREFETCH_OFFSET], void *)); - l3fwd_em_simple_process(mbufs[i], qconf); - event_vector_attr_validate(vec, mbufs[i]); + dst_ports[i] = l3fwd_em_simple_process(mbufs[i], qconf); } /* Forward remaining prefetched packets */ - for (; i < vec->nb_elem; i++) { - l3fwd_em_simple_process(mbufs[i], qconf); - event_vector_attr_validate(vec, mbufs[i]); - } + for (; i < vec->nb_elem; i++) + dst_ports[i] = l3fwd_em_simple_process(mbufs[i], qconf); + + process_event_vector(vec, dst_ports); } #endif /* __L3FWD_EM_H__ */ diff --git a/examples/l3fwd/l3fwd_em_hlm.h b/examples/l3fwd/l3fwd_em_hlm.h index 12b997e477..2e11eefad7 100644 --- a/examples/l3fwd/l3fwd_em_hlm.h +++ b/examples/l3fwd/l3fwd_em_hlm.h @@ -332,70 +332,20 @@ l3fwd_em_process_events(int nb_rx, struct rte_event **ev, static inline void l3fwd_em_process_event_vector(struct rte_event_vector *vec, - struct lcore_conf *qconf) + struct lcore_conf *qconf, uint16_t *dst_port) { - struct rte_mbuf **mbufs = vec->mbufs; - uint16_t dst_port[MAX_PKT_BURST]; - int32_t i, j, n, pos; - - for (j = 0; j < EM_HASH_LOOKUP_COUNT && j < vec->nb_elem; j++) - rte_prefetch0( - rte_pktmbuf_mtod(mbufs[j], struct rte_ether_hdr *) + 1); + uint16_t i; if (vec->attr_valid) - vec->port = em_get_dst_port(qconf, mbufs[0], mbufs[0]->port); + l3fwd_em_process_packets(vec->nb_elem, vec->mbufs, dst_port, + vec->port, qconf, 1); + else + for (i = 0; i < vec->nb_elem; i++) + l3fwd_em_process_packets(1, &vec->mbufs[i], + &dst_port[i], + vec->mbufs[i]->port, qconf, 1); - n = RTE_ALIGN_FLOOR(vec->nb_elem, EM_HASH_LOOKUP_COUNT); - for (j = 0; j < n; j += EM_HASH_LOOKUP_COUNT) { - uint32_t pkt_type = - RTE_PTYPE_L3_MASK | RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP; - uint32_t l3_type, tcp_or_udp; - - for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) - pkt_type &= mbufs[j + i]->packet_type; - - l3_type = pkt_type & RTE_PTYPE_L3_MASK; - tcp_or_udp = pkt_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP); - - for (i = 0, pos = j + EM_HASH_LOOKUP_COUNT; - i < EM_HASH_LOOKUP_COUNT && pos < vec->nb_elem; - i++, pos++) { - rte_prefetch0(rte_pktmbuf_mtod(mbufs[pos], - struct rte_ether_hdr *) + - 1); - } - - if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) { - em_get_dst_port_ipv4xN_events(qconf, &mbufs[j], - &dst_port[j]); - } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) { - em_get_dst_port_ipv6xN_events(qconf, &mbufs[j], - &dst_port[j]); - } else { - for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { - mbufs[j + i]->port = - em_get_dst_port(qconf, mbufs[j + i], - mbufs[j + i]->port); - process_packet(mbufs[j + i], - &mbufs[j + i]->port); - event_vector_attr_validate(vec, mbufs[j + i]); - } - continue; - } - processx4_step3(&mbufs[j], &dst_port[j]); - - for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) { - mbufs[j + i]->port = dst_port[j + i]; - event_vector_attr_validate(vec, mbufs[j + i]); - } - } - - for (; j < vec->nb_elem; j++) { - mbufs[j]->port = - em_get_dst_port(qconf, mbufs[j], mbufs[j]->port); - process_packet(mbufs[j], &mbufs[j]->port); - event_vector_attr_validate(vec, mbufs[j]); - } + process_event_vector(vec, dst_port); } #endif /* __L3FWD_EM_HLM_H__ */ diff --git a/examples/l3fwd/l3fwd_em_sequential.h b/examples/l3fwd/l3fwd_em_sequential.h index d2f75edb8a..067f23889a 100644 --- a/examples/l3fwd/l3fwd_em_sequential.h +++ b/examples/l3fwd/l3fwd_em_sequential.h @@ -113,39 +113,48 @@ l3fwd_em_process_events(int nb_rx, struct rte_event **events, for (i = 1, j = 0; j < nb_rx; i++, j++) { struct rte_mbuf *mbuf = events[j]->mbuf; + uint16_t port; if (i < nb_rx) { rte_prefetch0(rte_pktmbuf_mtod( events[i]->mbuf, struct rte_ether_hdr *) + 1); } + port = mbuf->port; mbuf->port = em_get_dst_port(qconf, mbuf, mbuf->port); process_packet(mbuf, &mbuf->port); + if (mbuf->port == BAD_PORT) + mbuf->port = port; } } static inline void l3fwd_em_process_event_vector(struct rte_event_vector *vec, - struct lcore_conf *qconf) + struct lcore_conf *qconf, uint16_t *dst_ports) { + const uint8_t attr_valid = vec->attr_valid; struct rte_mbuf **mbufs = vec->mbufs; int32_t i, j; rte_prefetch0(rte_pktmbuf_mtod(mbufs[0], struct rte_ether_hdr *) + 1); - if (vec->attr_valid) - vec->port = em_get_dst_port(qconf, mbufs[0], mbufs[0]->port); - for (i = 0, j = 1; i < vec->nb_elem; i++, j++) { if (j < vec->nb_elem) rte_prefetch0(rte_pktmbuf_mtod(mbufs[j], struct rte_ether_hdr *) + 1); - mbufs[i]->port = - em_get_dst_port(qconf, mbufs[i], mbufs[i]->port); - process_packet(mbufs[i], &mbufs[i]->port); - event_vector_attr_validate(vec, mbufs[i]); + dst_ports[i] = em_get_dst_port(qconf, mbufs[i], + attr_valid ? vec->port : + mbufs[i]->port); } + j = RTE_ALIGN_FLOOR(vec->nb_elem, FWDSTEP); + + for (i = 0; i != j; i += FWDSTEP) + processx4_step3(&vec->mbufs[i], &dst_ports[i]); + for (; i < vec->nb_elem; i++) + process_packet(vec->mbufs[i], &dst_ports[i]); + + process_event_vector(vec, dst_ports); } #endif /* __L3FWD_EM_SEQUENTIAL_H__ */ diff --git a/examples/l3fwd/l3fwd_event.h b/examples/l3fwd/l3fwd_event.h index 3fe38aada0..e21817c36b 100644 --- a/examples/l3fwd/l3fwd_event.h +++ b/examples/l3fwd/l3fwd_event.h @@ -103,27 +103,6 @@ process_dst_port(uint16_t *dst_ports, uint16_t nb_elem) } #endif -static inline void -event_vector_attr_validate(struct rte_event_vector *vec, struct rte_mbuf *mbuf) -{ - /* l3fwd application only changes mbuf port while processing */ - if (vec->attr_valid && (vec->port != mbuf->port)) - vec->attr_valid = 0; -} - -static inline void -event_vector_txq_set(struct rte_event_vector *vec, uint16_t txq) -{ - if (vec->attr_valid) { - vec->queue = txq; - } else { - int i; - - for (i = 0; i < vec->nb_elem; i++) - rte_event_eth_tx_adapter_txq_set(vec->mbufs[i], txq); - } -} - static inline uint16_t filter_bad_packets(struct rte_mbuf **mbufs, uint16_t *dst_port, uint16_t nb_pkts)