d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx5: support generic tunnel offloading

Browse Source 
This commit adds support for generic tunnel TSO and checksum offload.
PMD will compute the inner/outer headers offset according to the
mbuf fields. Hardware will do calculation based on offsets and types.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
Acked-by: Yongseok Koh <yskoh@mellanox.com>
This commit is contained in:
Xueming Li 2018-04-08 20:41:20 +08:00 committed by Ferruh Yigit
parent 593f472c40
commit 5f8ba81c42
11 changed files with 233 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/net/mlx5/Makefile
View File

@ -111,6 +111,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
infiniband/verbs.h \ infiniband/verbs.h \
enum IBV_WQ_FLAG_RX_END_PADDING \ enum IBV_WQ_FLAG_RX_END_PADDING \
$(AUTOCONF_OUTPUT) $(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
HAVE_IBV_MLX5_MOD_SWP \
infiniband/mlx5dv.h \
enum MLX5DV_CONTEXT_MASK_SWP \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \ $Q sh -- '$<' '$@' \
HAVE_IBV_MLX5_MOD_MPW \ HAVE_IBV_MLX5_MOD_MPW \
infiniband/mlx5dv.h \ infiniband/mlx5dv.h \

14
drivers/net/mlx5/mlx5.c
View File

@ -621,6 +621,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
unsigned int mps; unsigned int mps;
unsigned int cqe_comp; unsigned int cqe_comp;
unsigned int tunnel_en = 0; unsigned int tunnel_en = 0;
unsigned int swp = 0;
int idx; int idx;
int i; int i;
struct mlx5dv_context attrs_out = {0}; struct mlx5dv_context attrs_out = {0};
@ -696,6 +697,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
} }
ibv_dev = list[i]; ibv_dev = list[i];
DRV_LOG(DEBUG, "device opened"); DRV_LOG(DEBUG, "device opened");
#ifdef HAVE_IBV_MLX5_MOD_SWP
attrs_out.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
#endif
/* /*
* Multi-packet send is supported by ConnectX-4 Lx PF as well * Multi-packet send is supported by ConnectX-4 Lx PF as well
* as all ConnectX-5 devices. * as all ConnectX-5 devices.
@ -716,6 +720,11 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
DRV_LOG(DEBUG, "MPW isn't supported"); DRV_LOG(DEBUG, "MPW isn't supported");
mps = MLX5_MPW_DISABLED; mps = MLX5_MPW_DISABLED;
} }
#ifdef HAVE_IBV_MLX5_MOD_SWP
if (attrs_out.comp_mask | MLX5DV_CONTEXT_MASK_SWP)
swp = attrs_out.sw_parsing_caps.sw_parsing_offloads;
DRV_LOG(DEBUG, "SWP support: %u", swp);
#endif
if (RTE_CACHE_LINE_SIZE == 128 && if (RTE_CACHE_LINE_SIZE == 128 &&
!(attrs_out.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP)) !(attrs_out.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
cqe_comp = 0; cqe_comp = 0;
@ -763,6 +772,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
.txqs_inline = MLX5_ARG_UNSET, .txqs_inline = MLX5_ARG_UNSET,
.inline_max_packet_sz = MLX5_ARG_UNSET, .inline_max_packet_sz = MLX5_ARG_UNSET,
.vf_nl_en = 1, .vf_nl_en = 1,
.swp = !!swp,
}; };
len = snprintf(name, sizeof(name), PCI_PRI_FMT, len = snprintf(name, sizeof(name), PCI_PRI_FMT,
@ -1227,8 +1237,10 @@ RTE_INIT(rte_mlx5_pmd_init);
static void static void
rte_mlx5_pmd_init(void) rte_mlx5_pmd_init(void)
{ {
/* Build the static table for ptype conversion. */ /* Build the static tables for Verbs conversion. */
mlx5_set_ptype_table(); mlx5_set_ptype_table();
mlx5_set_cksum_table();
mlx5_set_swp_types_table();
/* /*
* RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
* huge pages. Calling ibv_fork_init() during init allows * huge pages. Calling ibv_fork_init() during init allows

1
drivers/net/mlx5/mlx5.h
View File

@ -89,6 +89,7 @@ struct mlx5_dev_config {
unsigned int rx_vec_en:1; /* Rx vector is enabled. */ unsigned int rx_vec_en:1; /* Rx vector is enabled. */
unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */ unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */
unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */ unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
unsigned int ind_table_max_size; /* Maximum indirection table size. */ unsigned int ind_table_max_size; /* Maximum indirection table size. */
int txq_inline; /* Maximum packet size for inlining. */ int txq_inline; /* Maximum packet size for inlining. */

5
drivers/net/mlx5/mlx5_ethdev.c
View File

@ -1090,11 +1090,14 @@ mlx5_select_tx_function(struct rte_eth_dev *dev)
int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO | int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
DEV_TX_OFFLOAD_VXLAN_TNL_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
DEV_TX_OFFLOAD_GRE_TNL_TSO)); DEV_TX_OFFLOAD_GRE_TNL_TSO));
int swp = !!(tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
DEV_TX_OFFLOAD_UDP_TNL_TSO |
DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM));
int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT); int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT);
assert(priv != NULL); assert(priv != NULL);
/* Select appropriate TX function. */ /* Select appropriate TX function. */
if (vlan_insert || tso) if (vlan_insert || tso || swp)
return tx_pkt_burst; return tx_pkt_burst;
if (config->mps == MLX5_MPW_ENHANCED) { if (config->mps == MLX5_MPW_ENHANCED) {
if (mlx5_check_vec_tx_support(dev) > 0) { if (mlx5_check_vec_tx_support(dev) > 0) {

24
drivers/net/mlx5/mlx5_prm.h
View File

@ -107,6 +107,30 @@
/* Inner L4 checksum offload (Tunneled packets only). */ /* Inner L4 checksum offload (Tunneled packets only). */
#define MLX5_ETH_WQE_L4_INNER_CSUM (1u << 5) #define MLX5_ETH_WQE_L4_INNER_CSUM (1u << 5)
/* Outer L4 type is TCP. */
#define MLX5_ETH_WQE_L4_OUTER_TCP (0u << 5)
/* Outer L4 type is UDP. */
#define MLX5_ETH_WQE_L4_OUTER_UDP (1u << 5)
/* Outer L3 type is IPV4. */
#define MLX5_ETH_WQE_L3_OUTER_IPV4 (0u << 4)
/* Outer L3 type is IPV6. */
#define MLX5_ETH_WQE_L3_OUTER_IPV6 (1u << 4)
/* Inner L4 type is TCP. */
#define MLX5_ETH_WQE_L4_INNER_TCP (0u << 1)
/* Inner L4 type is UDP. */
#define MLX5_ETH_WQE_L4_INNER_UDP (1u << 1)
/* Inner L3 type is IPV4. */
#define MLX5_ETH_WQE_L3_INNER_IPV4 (0u << 0)
/* Inner L3 type is IPV6. */
#define MLX5_ETH_WQE_L3_INNER_IPV6 (1u << 0)
/* Is flow mark valid. */ /* Is flow mark valid. */
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
#define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff00) #define MLX5_FLOW_MARK_IS_VALID(val) ((val) & 0xffffff00)

122
drivers/net/mlx5/mlx5_rxtx.c
View File

@ -47,6 +47,9 @@ uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
[0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */ [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */
}; };
uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned;
uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
/** /**
* Build a table to translate Rx completion flags to packet type. * Build a table to translate Rx completion flags to packet type.
* *
@ -202,6 +205,74 @@ mlx5_set_ptype_table(void)
RTE_PTYPE_INNER_L4_UDP; RTE_PTYPE_INNER_L4_UDP;
} }
/**
* Build a table to translate packet to checksum type of Verbs.
*/
void
mlx5_set_cksum_table(void)
{
unsigned int i;
uint8_t v;
/*
* The index should have:
* bit[0] = PKT_TX_TCP_SEG
* bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
* bit[4] = PKT_TX_IP_CKSUM
* bit[8] = PKT_TX_OUTER_IP_CKSUM
* bit[9] = tunnel
*/
for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) {
v = 0;
if (i & (1 << 9)) {
/* Tunneled packet. */
if (i & (1 << 8)) /* Outer IP. */
v |= MLX5_ETH_WQE_L3_CSUM;
if (i & (1 << 4)) /* Inner IP. */
v |= MLX5_ETH_WQE_L3_INNER_CSUM;
if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
v |= MLX5_ETH_WQE_L4_INNER_CSUM;
} else {
/* No tunnel. */
if (i & (1 << 4)) /* IP. */
v |= MLX5_ETH_WQE_L3_CSUM;
if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */
v |= MLX5_ETH_WQE_L4_CSUM;
}
mlx5_cksum_table[i] = v;
}
}
/**
* Build a table to translate packet type of mbuf to SWP type of Verbs.
*/
void
mlx5_set_swp_types_table(void)
{
unsigned int i;
uint8_t v;
/*
* The index should have:
* bit[0:1] = PKT_TX_L4_MASK
* bit[4] = PKT_TX_IPV6
* bit[8] = PKT_TX_OUTER_IPV6
* bit[9] = PKT_TX_OUTER_UDP
*/
for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) {
v = 0;
if (i & (1 << 8))
v |= MLX5_ETH_WQE_L3_OUTER_IPV6;
if (i & (1 << 9))
v |= MLX5_ETH_WQE_L4_OUTER_UDP;
if (i & (1 << 4))
v |= MLX5_ETH_WQE_L3_INNER_IPV6;
if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52))
v |= MLX5_ETH_WQE_L4_INNER_UDP;
mlx5_swp_types_table[i] = v;
}
}
/** /**
* Return the size of tailroom of WQ. * Return the size of tailroom of WQ.
* *
@ -267,7 +338,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
static int static int
inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf, inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
uint32_t *length, uint32_t *length,
uint8_t *cs_flags,
uintptr_t *addr, uintptr_t *addr,
uint16_t *pkt_inline_sz, uint16_t *pkt_inline_sz,
uint8_t **raw, uint8_t **raw,
@ -279,9 +349,8 @@ inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
(1 << txq->wqe_n) * MLX5_WQE_SIZE); (1 << txq->wqe_n) * MLX5_WQE_SIZE);
unsigned int copy_b; unsigned int copy_b;
uint8_t vlan_sz = (buf->ol_flags & PKT_TX_VLAN_PKT) ? 4 : 0; uint8_t vlan_sz = (buf->ol_flags & PKT_TX_VLAN_PKT) ? 4 : 0;
const uint8_t tunneled = txq->tunnel_en && const uint8_t tunneled = txq->tunnel_en && (buf->ol_flags &
(buf->ol_flags & (PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_MASK);
PKT_TX_TUNNEL_VXLAN));
uint16_t n_wqe; uint16_t n_wqe;
*tso_segsz = buf->tso_segsz; *tso_segsz = buf->tso_segsz;
@ -290,19 +359,15 @@ inline_tso(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
txq->stats.oerrors++; txq->stats.oerrors++;
return -EINVAL; return -EINVAL;
} }
if (tunneled) { if (tunneled)
*tso_header_sz += buf->outer_l2_len + buf->outer_l3_len; *tso_header_sz += buf->outer_l2_len + buf->outer_l3_len;
*cs_flags |= MLX5_ETH_WQE_L4_INNER_CSUM; /* First seg must contain all TSO headers. */
} else { if (unlikely(*tso_header_sz > MLX5_MAX_TSO_HEADER) ||
*cs_flags |= MLX5_ETH_WQE_L4_CSUM; *tso_header_sz > DATA_LEN(buf)) {
}
if (unlikely(*tso_header_sz > MLX5_MAX_TSO_HEADER)) {
txq->stats.oerrors++; txq->stats.oerrors++;
return -EINVAL; return -EINVAL;
} }
copy_b = *tso_header_sz - *pkt_inline_sz; copy_b = *tso_header_sz - *pkt_inline_sz;
/* First seg must contain all TSO headers. */
assert(copy_b <= *length);
if (!copy_b || ((end - (uintptr_t)*raw) < copy_b)) if (!copy_b || ((end - (uintptr_t)*raw) < copy_b))
return -EAGAIN; return -EAGAIN;
n_wqe = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4; n_wqe = (MLX5_WQE_DS(copy_b) - 1 + 3) / 4;
@ -435,7 +500,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
if (unlikely(!max_wqe)) if (unlikely(!max_wqe))
return 0; return 0;
do { do {
struct rte_mbuf *buf = NULL; struct rte_mbuf *buf = *pkts; /* First_seg. */
uint8_t *raw; uint8_t *raw;
volatile struct mlx5_wqe_v *wqe = NULL; volatile struct mlx5_wqe_v *wqe = NULL;
volatile rte_v128u32_t *dseg = NULL; volatile rte_v128u32_t *dseg = NULL;
@ -447,15 +512,16 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
uint16_t tso_header_sz = 0; uint16_t tso_header_sz = 0;
uint16_t ehdr; uint16_t ehdr;
uint8_t cs_flags; uint8_t cs_flags;
uint64_t tso = 0; uint8_t tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
uint8_t is_vlan = !!(buf->ol_flags & PKT_TX_VLAN_PKT);
uint32_t swp_offsets = 0;
uint8_t swp_types = 0;
uint16_t tso_segsz = 0; uint16_t tso_segsz = 0;
#ifdef MLX5_PMD_SOFT_COUNTERS #ifdef MLX5_PMD_SOFT_COUNTERS
uint32_t total_length = 0; uint32_t total_length = 0;
#endif #endif
int ret; int ret;
/* first_seg */
buf = *pkts;
segs_n = buf->nb_segs; segs_n = buf->nb_segs;
/* /*
* Make sure there is enough room to store this packet and * Make sure there is enough room to store this packet and
@ -490,10 +556,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
if (pkts_n - i > 1) if (pkts_n - i > 1)
rte_prefetch0( rte_prefetch0(
rte_pktmbuf_mtod(*(pkts + 1), volatile void *)); rte_pktmbuf_mtod(*(pkts + 1), volatile void *));
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
txq_mbuf_to_swp(txq, buf, tso, is_vlan,
(uint8_t *)&swp_offsets, &swp_types);
raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE; raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE;
/* Replace the Ethernet type by the VLAN if necessary. */ /* Replace the Ethernet type by the VLAN if necessary. */
if (buf->ol_flags & PKT_TX_VLAN_PKT) { if (is_vlan) {
uint32_t vlan = rte_cpu_to_be_32(0x81000000 | uint32_t vlan = rte_cpu_to_be_32(0x81000000 |
buf->vlan_tci); buf->vlan_tci);
unsigned int len = 2 * ETHER_ADDR_LEN - 2; unsigned int len = 2 * ETHER_ADDR_LEN - 2;
@ -516,9 +584,8 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
addr += pkt_inline_sz; addr += pkt_inline_sz;
} }
raw += MLX5_WQE_DWORD_SIZE; raw += MLX5_WQE_DWORD_SIZE;
tso = txq->tso_en && (buf->ol_flags & PKT_TX_TCP_SEG);
if (tso) { if (tso) {
ret = inline_tso(txq, buf, &length, &cs_flags, ret = inline_tso(txq, buf, &length,
&addr, &pkt_inline_sz, &addr, &pkt_inline_sz,
&raw, &max_wqe, &raw, &max_wqe,
&tso_segsz, &tso_header_sz); &tso_segsz, &tso_header_sz);
@ -695,8 +762,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
0, 0,
}; };
wqe->eseg = (rte_v128u32_t){ wqe->eseg = (rte_v128u32_t){
0, swp_offsets,
cs_flags | (rte_cpu_to_be_16(tso_segsz) << 16), cs_flags | (swp_types << 8) |
(rte_cpu_to_be_16(tso_segsz) << 16),
0, 0,
(ehdr << 16) | rte_cpu_to_be_16(tso_header_sz), (ehdr << 16) | rte_cpu_to_be_16(tso_header_sz),
}; };
@ -709,8 +777,8 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
0, 0,
}; };
wqe->eseg = (rte_v128u32_t){ wqe->eseg = (rte_v128u32_t){
0, swp_offsets,
cs_flags, cs_flags | (swp_types << 8),
0, 0,
(ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz), (ehdr << 16) | rte_cpu_to_be_16(pkt_inline_sz),
}; };
@ -882,7 +950,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
} }
max_elts -= segs_n; max_elts -= segs_n;
--pkts_n; --pkts_n;
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
/* Retrieve packet information. */ /* Retrieve packet information. */
length = PKT_LEN(buf); length = PKT_LEN(buf);
assert(length); assert(length);
@ -1114,7 +1182,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
* iteration. * iteration.
*/ */
max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
/* Retrieve packet information. */ /* Retrieve packet information. */
length = PKT_LEN(buf); length = PKT_LEN(buf);
/* Start new session if packet differs. */ /* Start new session if packet differs. */
@ -1391,7 +1459,7 @@ txq_burst_empw(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
/* Make sure there is enough room to store this packet. */ /* Make sure there is enough room to store this packet. */
if (max_elts - j == 0) if (max_elts - j == 0)
break; break;
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
/* Retrieve packet information. */ /* Retrieve packet information. */
length = PKT_LEN(buf); length = PKT_LEN(buf);
/* Start new session if: /* Start new session if:

98
drivers/net/mlx5/mlx5_rxtx.h
View File

@ -168,6 +168,7 @@ struct mlx5_txq_data {
uint16_t tso_en:1; /* When set hardware TSO is enabled. */ uint16_t tso_en:1; /* When set hardware TSO is enabled. */
uint16_t tunnel_en:1; uint16_t tunnel_en:1;
/* When set TX offload for tunneled packets are supported. */ /* When set TX offload for tunneled packets are supported. */
uint16_t swp_en:1; /* Whether SW parser is enabled. */
uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
@ -280,8 +281,12 @@ uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
/* mlx5_rxtx.c */ /* mlx5_rxtx.c */
extern uint32_t mlx5_ptype_table[]; extern uint32_t mlx5_ptype_table[];
extern uint8_t mlx5_cksum_table[];
extern uint8_t mlx5_swp_types_table[];
void mlx5_set_ptype_table(void); void mlx5_set_ptype_table(void);
void mlx5_set_cksum_table(void);
void mlx5_set_swp_types_table(void);
uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n); uint16_t pkts_n);
uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts,
@ -614,38 +619,89 @@ mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
} }
/** /**
* Convert the Checksum offloads to Verbs. * Convert mbuf to Verb SWP.
* *
* @param txq_data * @param txq_data
* Pointer to the Tx queue. * Pointer to the Tx queue.
* @param buf * @param buf
* Pointer to the mbuf. * Pointer to the mbuf.
* @param tso
* TSO offloads enabled.
* @param vlan
* VLAN offloads enabled
* @param offsets
* Pointer to the SWP header offsets.
* @param swp_types
* Pointer to the SWP header types.
*/
static __rte_always_inline void
txq_mbuf_to_swp(struct mlx5_txq_data *txq, struct rte_mbuf *buf,
uint8_t tso, uint64_t vlan,
uint8_t *offsets, uint8_t *swp_types)
{
uint64_t tunnel = buf->ol_flags & PKT_TX_TUNNEL_MASK;
uint16_t idx;
uint16_t off;
const uint64_t ol_flags_mask = PKT_TX_L4_MASK | PKT_TX_IPV6 |
PKT_TX_OUTER_IPV6;
if (likely(!tunnel || !txq->swp_en ||
(tunnel != PKT_TX_TUNNEL_UDP && tunnel != PKT_TX_TUNNEL_IP)))
return;
/*
* The index should have:
* bit[0:1] = PKT_TX_L4_MASK
* bit[4] = PKT_TX_IPV6
* bit[8] = PKT_TX_OUTER_IPV6
* bit[9] = PKT_TX_OUTER_UDP
*/
idx = (buf->ol_flags & ol_flags_mask) >> 52;
if (tunnel == PKT_TX_TUNNEL_UDP)
idx |= 1 << 9;
*swp_types = mlx5_swp_types_table[idx];
/* swp offsets. */
off = buf->outer_l2_len + (vlan ? 4 : 0); /* Outer L3 offset. */
if (tso || (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM))
offsets[1] = off >> 1;
off += buf->outer_l3_len; /* Outer L4 offset. */
if (tunnel == PKT_TX_TUNNEL_UDP)
offsets[0] = off >> 1;
off += buf->l2_len; /* Inner L3 offset. */
if (tso || (buf->ol_flags & PKT_TX_IP_CKSUM))
offsets[3] = off >> 1;
off += buf->l3_len; /* Inner L4 offset. */
if (tso || ((buf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) ||
((buf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM))
offsets[2] = off >> 1;
}
/**
* Convert the Checksum offloads to Verbs.
*
* @param buf
* Pointer to the mbuf.
* *
* @return * @return
* the converted cs_flags. * Converted checksum flags.
*/ */
static __rte_always_inline uint8_t static __rte_always_inline uint8_t
txq_ol_cksum_to_cs(struct mlx5_txq_data *txq_data, struct rte_mbuf *buf) txq_ol_cksum_to_cs(struct rte_mbuf *buf)
{ {
uint8_t cs_flags = 0; uint32_t idx;
uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK);
const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK |
PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM;
/* Should we enable HW CKSUM offload */ /*
if (buf->ol_flags & * The index should have:
(PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | * bit[0] = PKT_TX_TCP_SEG
PKT_TX_OUTER_IP_CKSUM)) { * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
if (txq_data->tunnel_en && * bit[4] = PKT_TX_IP_CKSUM
(buf->ol_flags & * bit[8] = PKT_TX_OUTER_IP_CKSUM
(PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN))) { * bit[9] = tunnel
cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | */
MLX5_ETH_WQE_L4_INNER_CSUM; idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9);
if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM) return mlx5_cksum_table[idx];
cs_flags |= MLX5_ETH_WQE_L3_CSUM;
} else {
cs_flags = MLX5_ETH_WQE_L3_CSUM |
MLX5_ETH_WQE_L4_CSUM;
}
}
return cs_flags;
} }
/** /**

9
drivers/net/mlx5/mlx5_rxtx_vec.c
View File

@ -42,8 +42,6 @@
/** /**
* Count the number of packets having same ol_flags and calculate cs_flags. * Count the number of packets having same ol_flags and calculate cs_flags.
* *
* @param txq
* Pointer to TX queue structure.
* @param pkts * @param pkts
* Pointer to array of packets. * Pointer to array of packets.
* @param pkts_n * @param pkts_n
@ -55,8 +53,7 @@
* Number of packets having same ol_flags. * Number of packets having same ol_flags.
*/ */
static inline unsigned int static inline unsigned int
txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags)
uint16_t pkts_n, uint8_t *cs_flags)
{ {
unsigned int pos; unsigned int pos;
const uint64_t ol_mask = const uint64_t ol_mask =
@ -70,7 +67,7 @@ txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
for (pos = 1; pos < pkts_n; ++pos) for (pos = 1; pos < pkts_n; ++pos)
if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
break; break;
*cs_flags = txq_ol_cksum_to_cs(txq, pkts[0]); *cs_flags = txq_ol_cksum_to_cs(pkts[0]);
return pos; return pos;
} }
@ -141,7 +138,7 @@ mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
n = txq_count_contig_single_seg(&pkts[nb_tx], n); n = txq_count_contig_single_seg(&pkts[nb_tx], n);
if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP)
n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags);
ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
nb_tx += ret; nb_tx += ret;
if (!ret) if (!ret)

2
drivers/net/mlx5/mlx5_rxtx_vec_neon.h
View File

@ -142,7 +142,7 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
break; break;
wqe = &((volatile struct mlx5_wqe64 *) wqe = &((volatile struct mlx5_wqe64 *)
txq->wqes)[wqe_ci & wq_mask].hdr; txq->wqes)[wqe_ci & wq_mask].hdr;
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
/* Title WQEBB pointer. */ /* Title WQEBB pointer. */
t_wqe = (uint8x16_t *)wqe; t_wqe = (uint8x16_t *)wqe;
dseg = (uint8_t *)(wqe + 1); dseg = (uint8_t *)(wqe + 1);

2
drivers/net/mlx5/mlx5_rxtx_vec_sse.h
View File

@ -144,7 +144,7 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
} }
wqe = &((volatile struct mlx5_wqe64 *) wqe = &((volatile struct mlx5_wqe64 *)
txq->wqes)[wqe_ci & wq_mask].hdr; txq->wqes)[wqe_ci & wq_mask].hdr;
cs_flags = txq_ol_cksum_to_cs(txq, buf); cs_flags = txq_ol_cksum_to_cs(buf);
/* Title WQEBB pointer. */ /* Title WQEBB pointer. */
t_wqe = (__m128i *)wqe; t_wqe = (__m128i *)wqe;
dseg = (__m128i *)(wqe + 1); dseg = (__m128i *)(wqe + 1);

10
drivers/net/mlx5/mlx5_txq.c
View File

@ -119,6 +119,9 @@ mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
if (config->tso) if (config->tso)
offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
DEV_TX_OFFLOAD_GRE_TNL_TSO); DEV_TX_OFFLOAD_GRE_TNL_TSO);
if (config->swp)
offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO |
DEV_TX_OFFLOAD_UDP_TNL_TSO);
} }
return offloads; return offloads;
} }
@ -686,7 +689,9 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
int is_empw_func = is_empw_burst_func(tx_pkt_burst); int is_empw_func = is_empw_burst_func(tx_pkt_burst);
int tso = !!(txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO | int tso = !!(txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO |
DEV_TX_OFFLOAD_VXLAN_TNL_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
DEV_TX_OFFLOAD_GRE_TNL_TSO)); DEV_TX_OFFLOAD_GRE_TNL_TSO |
DEV_TX_OFFLOAD_IP_TNL_TSO |
DEV_TX_OFFLOAD_UDP_TNL_TSO));
txq_inline = (config->txq_inline == MLX5_ARG_UNSET) ? txq_inline = (config->txq_inline == MLX5_ARG_UNSET) ?
0 : config->txq_inline; 0 : config->txq_inline;
@ -767,6 +772,9 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
txq_ctrl->txq.tso_en = 1; txq_ctrl->txq.tso_en = 1;
} }
txq_ctrl->txq.tunnel_en = config->tunnel_en; txq_ctrl->txq.tunnel_en = config->tunnel_en;
txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO |
DEV_TX_OFFLOAD_UDP_TNL_TSO) &
txq_ctrl->txq.offloads) && config->swp;
} }
/** /**