diff --git a/iokernel/dpdk.c b/iokernel/dpdk.c index 4bbf792..2d3e915 100644 --- a/iokernel/dpdk.c +++ b/iokernel/dpdk.c @@ -106,10 +106,8 @@ static inline int dpdk_port_init(uint8_t port, struct rte_mempool *mbuf_pool) txconf = &dev_info.default_txconf; txconf->tx_rs_thresh = 64; txconf->tx_free_thresh = 64; -#if 0 txconf->txq_flags &= ~(ETH_TXQ_FLAGS_NOXSUMUDP | ETH_TXQ_FLAGS_NOXSUMTCP); -#endif /* Allocate and set up 1 TX queue per Ethernet port. */ for (q = 0; q < tx_rings; q++) { diff --git a/iokernel/tx.c b/iokernel/tx.c index a65997e..75e1186 100644 --- a/iokernel/tx.c +++ b/iokernel/tx.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -60,7 +61,6 @@ static void tx_prepare_tx_mbuf(struct rte_mbuf *buf, buf->data_len = net_hdr->len; buf->ol_flags = 0; -#if 0 if (net_hdr->olflags != 0) { if (net_hdr->olflags & OLFLAG_IP_CHKSUM) buf->ol_flags |= PKT_TX_IP_CKSUM; @@ -71,10 +71,10 @@ static void tx_prepare_tx_mbuf(struct rte_mbuf *buf, if (net_hdr->olflags & OLFLAG_IPV6) buf->ol_flags |= PKT_TX_IPV6; + buf->l4_len = sizeof(struct tcp_hdr); buf->l3_len = sizeof(struct ipv4_hdr); buf->l2_len = ETHER_HDR_LEN; } -#endif /* initialize the private data, used to send completion events */ priv_data = tx_pktmbuf_get_priv(buf); diff --git a/ixgbe.patch b/ixgbe.patch index dca88ab..ed86272 100644 --- a/ixgbe.patch +++ b/ixgbe.patch @@ -1,5 +1,5 @@ diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c -index 6c582b4be..2579e344f 100644 +index 6c582b4be..049b738a2 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.c +++ b/drivers/net/ixgbe/ixgbe_rxtx.c @@ -554,10 +554,10 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) @@ -15,6 +15,77 @@ index 6c582b4be..2579e344f 100644 #endif /* Reset transmit descriptors after they have been used */ +@@ -2371,6 +2371,45 @@ static const struct ixgbe_txq_ops def_txq_ops = { + .reset = ixgbe_reset_tx_queue, + }; + ++static uint16_t ++install_txq_ctx(struct ixgbe_tx_queue *txq, uint16_t slot, ++ const uint64_t flags, const union ixgbe_tx_offload tx_offload) ++{ ++ ++ /* This must be called before any packets are placed in the ring */ ++ ASSERT(txq->tx_tail == 0); ++ ASSERT(((struct ixgbe_tx_entry_v *)txq->sw_ring_v)->mbuf == NULL); ++ ASSERT(txq->nb_tx_free >= txq->tx_free_thresh); ++ ASSERT(txq->nb_tx_desc > 0); ++ ASSERT(txq->nb_tx_free >= 1); ++ ASSERT(slot < IXGBE_CTX_NUM); ++ ++ ++ uint64_t tx_ol_req = flags & IXGBE_TX_OFFLOAD_MASK; ++ ++ volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring; ++ volatile struct ixgbe_adv_tx_context_desc *ctx_txd; ++ ctx_txd = (volatile struct ixgbe_adv_tx_context_desc *)txr; ++ ++ txq->ctx_curr = slot; ++ ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, ++ tx_offload, NULL); ++ ++ txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + 1); ++ txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - 1); ++ ++ PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", ++ (unsigned) txq->port_id, (unsigned) txq->queue_id, ++ (unsigned) 1, (unsigned) 1); ++ IXGBE_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, 1); ++ txq->tx_tail = 1; ++ return 1; ++} ++ ++ ++#define IXGBE_SIMPLE_FLAGS_OFFLOAD \ ++ (IXGBE_SIMPLE_FLAGS & ~(ETH_TXQ_FLAGS_NOXSUMUDP | ETH_TXQ_FLAGS_NOXSUMTCP)) ++ + /* Takes an ethdev and a queue and sets up the tx function to be used based on + * the queue parameters. Used in tx_queue_setup by primary process and then + * in dev_init by secondary process when attaching to an existing ethdev. +@@ -2379,7 +2418,7 @@ void __attribute__((cold)) + ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) + { + /* Use a simple Tx queue (no offloads, no multi segs) if possible */ +- if (((txq->txq_flags & IXGBE_SIMPLE_FLAGS) == IXGBE_SIMPLE_FLAGS) && ++ if (((txq->txq_flags & IXGBE_SIMPLE_FLAGS_OFFLOAD) == IXGBE_SIMPLE_FLAGS_OFFLOAD) && + #ifdef RTE_LIBRTE_SECURITY + !(txq->using_ipsec) && + #endif +@@ -2591,6 +2630,15 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, + + dev->data->tx_queues[queue_idx] = txq; + ++ uint64_t olflags = PKT_TX_IP_CKSUM | PKT_TX_IPV4 | PKT_TX_TCP_CKSUM; ++ union ixgbe_tx_offload offloads; ++ memset(&offloads, 0, sizeof(offloads)); ++ offloads.l2_len = ETHER_HDR_LEN; ++ offloads.l3_len = sizeof(struct ipv4_hdr); ++ offloads.l4_len = sizeof(struct tcp_hdr); ++ ++ if (install_txq_ctx(txq, 0, olflags, offloads) != 1) ++ return -EINVAL; + + return 0; + } diff --git a/drivers/net/ixgbe/ixgbe_rxtx.h b/drivers/net/ixgbe/ixgbe_rxtx.h index 69c718bca..9d3fdd0dd 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.h @@ -37,3 +108,45 @@ index 69c718bca..9d3fdd0dd 100644 #define RTE_IXGBE_MAX_RX_BURST RTE_IXGBE_RXQ_REARM_THRESH #endif +diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +index 414840a2b..950beebda 100644 +--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h ++++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +@@ -94,8 +94,8 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = &txq->sw_ring_v[txq->tx_next_dd - (n - 1)]; +- m = rte_pktmbuf_prefree_seg(txep[0].mbuf); +- if (likely(m != NULL)) { ++ if (likely(txep[0].mbuf && ++ (m = rte_pktmbuf_prefree_seg(txep[0].mbuf)) != NULL)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < n; i++) { +diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +index c9ba48246..0a6bc6b06 100644 +--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c ++++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +@@ -618,7 +618,21 @@ static inline void + vtx1(volatile union ixgbe_adv_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) + { +- __m128i descriptor = _mm_set_epi64x((uint64_t)pkt->pkt_len << 46 | ++ ++ /* Set Packet Length */ ++ uint64_t top_flags = (uint64_t)pkt->pkt_len << 46; ++ ++ /* Set IXGBE_TXD_POPTS_IXSM */ ++ top_flags |= (pkt->ol_flags & PKT_TX_IP_CKSUM) >> 14; ++ RTE_BUILD_BUG_ON( ++ PKT_TX_IP_CKSUM >> 14 != (uint64_t)IXGBE_TXD_POPTS_IXSM << 40); ++ ++ /* Set IXGBE_TXD_POPTS_TXSM */ ++ top_flags |= (pkt->ol_flags & PKT_TX_TCP_CKSUM) >> 11; ++ RTE_BUILD_BUG_ON( ++ PKT_TX_TCP_CKSUM >> 11 != (uint64_t)IXGBE_TXD_POPTS_TXSM << 40); ++ ++ __m128i descriptor = _mm_set_epi64x(top_flags | + flags | pkt->data_len, + pkt->buf_iova + pkt->data_off); + _mm_store_si128((__m128i *)&txdp->read, descriptor); diff --git a/runtime/net/core.c b/runtime/net/core.c index 92ba860..5be3714 100644 --- a/runtime/net/core.c +++ b/runtime/net/core.c @@ -380,9 +380,6 @@ static void net_push_iphdr(struct mbuf *m, uint8_t proto, uint32_t daddr) iphdr->chksum = 0; iphdr->saddr = hton32(netcfg.addr); iphdr->daddr = hton32(daddr); - - /* calculate IP checksum */ - iphdr->chksum = chksum_internet((void *)iphdr, sizeof(*iphdr)); } static uint32_t net_get_ip_route(uint32_t daddr) diff --git a/runtime/net/tcp_out.c b/runtime/net/tcp_out.c index 47e3033..d4f9cd4 100644 --- a/runtime/net/tcp_out.c +++ b/runtime/net/tcp_out.c @@ -53,6 +53,8 @@ int tcp_tx_raw_rst(struct netaddr laddr, struct netaddr raddr, tcp_seq seq) if (unlikely((!m))) return -ENOMEM; + m->txflags = OLFLAG_TCP_CHKSUM; + /* write the tcp header */ tcphdr = mbuf_push_hdr(m, *tcphdr); tcphdr->sport = hton16(laddr.port); @@ -90,6 +92,8 @@ int tcp_tx_raw_rst_ack(struct netaddr laddr, struct netaddr raddr, if (unlikely((!m))) return -ENOMEM; + m->txflags = OLFLAG_TCP_CHKSUM; + /* write the tcp header */ tcphdr = mbuf_push_hdr(m, *tcphdr); tcphdr->sport = hton16(laddr.port); @@ -123,6 +127,8 @@ int tcp_tx_ack(tcpconn_t *c) if (unlikely(!m)) return -ENOMEM; + m->txflags = OLFLAG_TCP_CHKSUM; + tcphdr = tcp_push_tcphdr(m, c, TCP_ACK); tcphdr->seq = hton32(load_acquire(&c->pcb.snd_nxt)); @@ -156,6 +162,8 @@ int tcp_tx_ctl(tcpconn_t *c, uint8_t flags) if (unlikely(!m)) return -ENOMEM; + m->txflags = OLFLAG_TCP_CHKSUM; + m->seg_seq = c->pcb.snd_nxt; m->seg_end = c->pcb.snd_nxt + 1; tcphdr = tcp_push_tcphdr(m, c, flags); @@ -223,6 +231,7 @@ ssize_t tcp_tx_buf(tcpconn_t *c, const void *buf, size_t len, bool push) ret = -ENOBUFS; break; } + m->txflags = OLFLAG_TCP_CHKSUM; seglen = min(end - pos, TCP_MSS); m->seg_seq = c->pcb.snd_nxt; m->seg_end = c->pcb.snd_nxt + seglen;