From dd8c20eee47263a0b8bed08b5c51e1cc78b48aee Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Sat, 22 Jan 2022 21:18:15 +0530 Subject: [PATCH] net/cnxk: add cn9k template Tx functions to build Add cn9k segregated Tx and event Tx functions to build, add macros to make future modifications simpler. Signed-off-by: Pavan Nikhilesh Acked-by: Jerin Jacob --- drivers/event/cnxk/cn9k_eventdev.c | 28 +- drivers/event/cnxk/cn9k_worker.h | 58 +- drivers/event/cnxk/cn9k_worker_dual_tx_enq.c | 23 - .../event/cnxk/cn9k_worker_dual_tx_enq_seg.c | 23 - drivers/event/cnxk/cn9k_worker_tx_enq.c | 23 - drivers/event/cnxk/cn9k_worker_tx_enq_seg.c | 23 - drivers/event/cnxk/meson.build | 42 +- drivers/net/cnxk/cn9k_tx.h | 510 +++++++++--------- drivers/net/cnxk/cn9k_tx_mseg.c | 25 - drivers/net/cnxk/cn9k_tx_select.c | 59 ++ drivers/net/cnxk/cn9k_tx_vec.c | 25 - drivers/net/cnxk/cn9k_tx_vec_mseg.c | 24 - drivers/net/cnxk/meson.build | 40 +- 13 files changed, 447 insertions(+), 456 deletions(-) delete mode 100644 drivers/event/cnxk/cn9k_worker_dual_tx_enq.c delete mode 100644 drivers/event/cnxk/cn9k_worker_dual_tx_enq_seg.c delete mode 100644 drivers/event/cnxk/cn9k_worker_tx_enq.c delete mode 100644 drivers/event/cnxk/cn9k_worker_tx_enq_seg.c delete mode 100644 drivers/net/cnxk/cn9k_tx_mseg.c create mode 100644 drivers/net/cnxk/cn9k_tx_select.c delete mode 100644 drivers/net/cnxk/cn9k_tx_vec.c delete mode 100644 drivers/net/cnxk/cn9k_tx_vec_mseg.c diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c index 237170d6c1..4611936b7f 100644 --- a/drivers/event/cnxk/cn9k_eventdev.c +++ b/drivers/event/cnxk/cn9k_eventdev.c @@ -13,13 +13,7 @@ deq_op = deq_ops[dev->rx_offloads & (NIX_RX_OFFLOAD_MAX - 1)] #define CN9K_SET_EVDEV_ENQ_OP(dev, enq_op, enq_ops) \ - (enq_op = enq_ops[!!(dev->tx_offloads & NIX_TX_OFFLOAD_SECURITY_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSO_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] \ - [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)]) + enq_op = enq_ops[dev->tx_offloads & (NIX_TX_OFFLOAD_MAX - 1)] static int cn9k_sso_hws_link(void *arg, void *port, uint16_t *map, uint16_t nb_link) @@ -504,30 +498,26 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev) }; /* Tx modes */ - const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq[2][2][2][2][2][2][2] = { -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_tx_adptr_enq_##name, + const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_sso_hws_tx_adptr_enq_##name, NIX_TX_FASTPATH_MODES #undef T }; - const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq_seg[2][2][2][2][2][2][2] = { -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_tx_adptr_enq_seg_##name, + const event_tx_adapter_enqueue_t sso_hws_tx_adptr_enq_seg[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_sso_hws_tx_adptr_enq_seg_##name, NIX_TX_FASTPATH_MODES #undef T }; - const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq[2][2][2][2][2][2][2] = { -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_tx_adptr_enq_##name, + const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_sso_hws_dual_tx_adptr_enq_##name, NIX_TX_FASTPATH_MODES #undef T }; - const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq_seg[2][2][2][2][2][2][2] = { -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_tx_adptr_enq_seg_##name, + const event_tx_adapter_enqueue_t sso_hws_dual_tx_adptr_enq_seg[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_sso_hws_dual_tx_adptr_enq_seg_##name, NIX_TX_FASTPATH_MODES #undef T }; diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h index 8f7c09f596..c99e459c1b 100644 --- a/drivers/event/cnxk/cn9k_worker.h +++ b/drivers/event/cnxk/cn9k_worker.h @@ -833,7 +833,7 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, return 1; } -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ +#define T(name, sz, flags) \ uint16_t __rte_hot cn9k_sso_hws_tx_adptr_enq_##name( \ void *port, struct rte_event ev[], uint16_t nb_events); \ uint16_t __rte_hot cn9k_sso_hws_tx_adptr_enq_seg_##name( \ @@ -846,4 +846,60 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, NIX_TX_FASTPATH_MODES #undef T +#define SSO_TX(fn, sz, flags) \ + uint16_t __rte_hot fn(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ + { \ + struct cn9k_sso_hws *ws = port; \ + uint64_t cmd[sz]; \ + RTE_SET_USED(nb_events); \ + return cn9k_sso_hws_event_tx( \ + ws->base, &ev[0], cmd, \ + (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ + ws->tx_adptr_data, \ + flags); \ + } + +#define SSO_TX_SEG(fn, sz, flags) \ + uint16_t __rte_hot fn(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ + { \ + uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ + struct cn9k_sso_hws *ws = port; \ + RTE_SET_USED(nb_events); \ + return cn9k_sso_hws_event_tx( \ + ws->base, &ev[0], cmd, \ + (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ + ws->tx_adptr_data, \ + (flags) | NIX_TX_MULTI_SEG_F); \ + } + +#define SSO_DUAL_TX(fn, sz, flags) \ + uint16_t __rte_hot fn(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ + { \ + struct cn9k_sso_hws_dual *ws = port; \ + uint64_t cmd[sz]; \ + RTE_SET_USED(nb_events); \ + return cn9k_sso_hws_event_tx( \ + ws->base[!ws->vws], &ev[0], cmd, \ + (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ + ws->tx_adptr_data, \ + flags); \ + } + +#define SSO_DUAL_TX_SEG(fn, sz, flags) \ + uint16_t __rte_hot fn(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ + { \ + uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ + struct cn9k_sso_hws_dual *ws = port; \ + RTE_SET_USED(nb_events); \ + return cn9k_sso_hws_event_tx( \ + ws->base[!ws->vws], &ev[0], cmd, \ + (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ + ws->tx_adptr_data, \ + (flags) | NIX_TX_MULTI_SEG_F); \ + } + #endif diff --git a/drivers/event/cnxk/cn9k_worker_dual_tx_enq.c b/drivers/event/cnxk/cn9k_worker_dual_tx_enq.c deleted file mode 100644 index db045d0e01..0000000000 --- a/drivers/event/cnxk/cn9k_worker_dual_tx_enq.c +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_worker.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_hot cn9k_sso_hws_dual_tx_adptr_enq_##name( \ - void *port, struct rte_event ev[], uint16_t nb_events) \ - { \ - struct cn9k_sso_hws_dual *ws = port; \ - uint64_t cmd[sz]; \ - \ - RTE_SET_USED(nb_events); \ - return cn9k_sso_hws_event_tx( \ - ws->base[!ws->vws], &ev[0], cmd, \ - (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ - ws->tx_adptr_data, \ - flags); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/event/cnxk/cn9k_worker_dual_tx_enq_seg.c b/drivers/event/cnxk/cn9k_worker_dual_tx_enq_seg.c deleted file mode 100644 index 95d711f9b3..0000000000 --- a/drivers/event/cnxk/cn9k_worker_dual_tx_enq_seg.c +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_worker.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_hot cn9k_sso_hws_dual_tx_adptr_enq_seg_##name( \ - void *port, struct rte_event ev[], uint16_t nb_events) \ - { \ - uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ - struct cn9k_sso_hws_dual *ws = port; \ - \ - RTE_SET_USED(nb_events); \ - return cn9k_sso_hws_event_tx( \ - ws->base[!ws->vws], &ev[0], cmd, \ - (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ - ws->tx_adptr_data, \ - (flags) | NIX_TX_MULTI_SEG_F); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/event/cnxk/cn9k_worker_tx_enq.c b/drivers/event/cnxk/cn9k_worker_tx_enq.c deleted file mode 100644 index 026cef8e2b..0000000000 --- a/drivers/event/cnxk/cn9k_worker_tx_enq.c +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_worker.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_hot cn9k_sso_hws_tx_adptr_enq_##name( \ - void *port, struct rte_event ev[], uint16_t nb_events) \ - { \ - struct cn9k_sso_hws *ws = port; \ - uint64_t cmd[sz]; \ - \ - RTE_SET_USED(nb_events); \ - return cn9k_sso_hws_event_tx( \ - ws->base, &ev[0], cmd, \ - (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ - ws->tx_adptr_data, \ - flags); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/event/cnxk/cn9k_worker_tx_enq_seg.c b/drivers/event/cnxk/cn9k_worker_tx_enq_seg.c deleted file mode 100644 index 97cd7c7d8e..0000000000 --- a/drivers/event/cnxk/cn9k_worker_tx_enq_seg.c +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_worker.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_hot cn9k_sso_hws_tx_adptr_enq_seg_##name( \ - void *port, struct rte_event ev[], uint16_t nb_events) \ - { \ - uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ - struct cn9k_sso_hws *ws = port; \ - \ - RTE_SET_USED(nb_events); \ - return cn9k_sso_hws_event_tx( \ - ws->base, &ev[0], cmd, \ - (const uint64_t(*)[RTE_MAX_QUEUES_PER_PORT]) & \ - ws->tx_adptr_data, \ - (flags) | NIX_TX_MULTI_SEG_F); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/event/cnxk/meson.build b/drivers/event/cnxk/meson.build index 27697d2ece..c39515dadc 100644 --- a/drivers/event/cnxk/meson.build +++ b/drivers/event/cnxk/meson.build @@ -11,10 +11,6 @@ endif sources = files( 'cn9k_eventdev.c', 'cn9k_worker.c', - 'cn9k_worker_tx_enq.c', - 'cn9k_worker_tx_enq_seg.c', - 'cn9k_worker_dual_tx_enq.c', - 'cn9k_worker_dual_tx_enq_seg.c', 'cn10k_eventdev.c', 'cn10k_worker.c', 'cn10k_worker_deq.c', @@ -293,6 +289,44 @@ sources += files( 'deq/cn9k/deq_112_127_dual_ca_tmo_seg_burst.c', ) +sources += files( + 'tx/cn9k/tx_0_15.c', + 'tx/cn9k/tx_16_31.c', + 'tx/cn9k/tx_32_47.c', + 'tx/cn9k/tx_48_63.c', + 'tx/cn9k/tx_64_79.c', + 'tx/cn9k/tx_80_95.c', + 'tx/cn9k/tx_96_111.c', + 'tx/cn9k/tx_112_127.c', + 'tx/cn9k/tx_0_15_seg.c', + 'tx/cn9k/tx_16_31_seg.c', + 'tx/cn9k/tx_32_47_seg.c', + 'tx/cn9k/tx_48_63_seg.c', + 'tx/cn9k/tx_64_79_seg.c', + 'tx/cn9k/tx_80_95_seg.c', + 'tx/cn9k/tx_96_111_seg.c', + 'tx/cn9k/tx_112_127_seg.c', +) + +sources += files( + 'tx/cn9k/tx_0_15_dual.c', + 'tx/cn9k/tx_16_31_dual.c', + 'tx/cn9k/tx_32_47_dual.c', + 'tx/cn9k/tx_48_63_dual.c', + 'tx/cn9k/tx_64_79_dual.c', + 'tx/cn9k/tx_80_95_dual.c', + 'tx/cn9k/tx_96_111_dual.c', + 'tx/cn9k/tx_112_127_dual.c', + 'tx/cn9k/tx_0_15_dual_seg.c', + 'tx/cn9k/tx_16_31_dual_seg.c', + 'tx/cn9k/tx_32_47_dual_seg.c', + 'tx/cn9k/tx_48_63_dual_seg.c', + 'tx/cn9k/tx_64_79_dual_seg.c', + 'tx/cn9k/tx_80_95_dual_seg.c', + 'tx/cn9k/tx_96_111_dual_seg.c', + 'tx/cn9k/tx_112_127_dual_seg.c', +) + extra_flags = ['-flax-vector-conversions', '-Wno-strict-aliasing'] foreach flag: extra_flags if cc.has_argument(flag) diff --git a/drivers/net/cnxk/cn9k_tx.h b/drivers/net/cnxk/cn9k_tx.h index 070a7d9439..8564dd85ee 100644 --- a/drivers/net/cnxk/cn9k_tx.h +++ b/drivers/net/cnxk/cn9k_tx.h @@ -14,6 +14,7 @@ #define NIX_TX_OFFLOAD_TSO_F BIT(4) #define NIX_TX_OFFLOAD_TSTAMP_F BIT(5) #define NIX_TX_OFFLOAD_SECURITY_F BIT(6) +#define NIX_TX_OFFLOAD_MAX (NIX_TX_OFFLOAD_SECURITY_F << 1) /* Flags to control xmit_prepare function. * Defining it from backwards to denote its been @@ -1822,279 +1823,264 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, #define T_SEC_F NIX_TX_OFFLOAD_SECURITY_F /* [T_SEC_F] [TSP] [TSO] [NOFF] [VLAN] [OL3OL4CSUM] [L3L4CSUM] */ -#define NIX_TX_FASTPATH_MODES \ -T(no_offload, 0, 0, 0, 0, 0, 0, 0, 4, \ - NIX_TX_OFFLOAD_NONE) \ -T(l3l4csum, 0, 0, 0, 0, 0, 0, 1, 4, \ - L3L4CSUM_F) \ -T(ol3ol4csum, 0, 0, 0, 0, 0, 1, 0, 4, \ - OL3OL4CSUM_F) \ -T(ol3ol4csum_l3l4csum, 0, 0, 0, 0, 0, 1, 1, 4, \ - OL3OL4CSUM_F | L3L4CSUM_F) \ -T(vlan, 0, 0, 0, 0, 1, 0, 0, 6, \ - VLAN_F) \ -T(vlan_l3l4csum, 0, 0, 0, 0, 1, 0, 1, 6, \ - VLAN_F | L3L4CSUM_F) \ -T(vlan_ol3ol4csum, 0, 0, 0, 0, 1, 1, 0, 6, \ - VLAN_F | OL3OL4CSUM_F) \ -T(vlan_ol3ol4csum_l3l4csum, 0, 0, 0, 0, 1, 1, 1, 6, \ - VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(noff, 0, 0, 0, 1, 0, 0, 0, 4, \ - NOFF_F) \ -T(noff_l3l4csum, 0, 0, 0, 1, 0, 0, 1, 4, \ - NOFF_F | L3L4CSUM_F) \ -T(noff_ol3ol4csum, 0, 0, 0, 1, 0, 1, 0, 4, \ - NOFF_F | OL3OL4CSUM_F) \ -T(noff_ol3ol4csum_l3l4csum, 0, 0, 0, 1, 0, 1, 1, 4, \ - NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(noff_vlan, 0, 0, 0, 1, 1, 0, 0, 6, \ - NOFF_F | VLAN_F) \ -T(noff_vlan_l3l4csum, 0, 0, 0, 1, 1, 0, 1, 6, \ - NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(noff_vlan_ol3ol4csum, 0, 0, 0, 1, 1, 1, 0, 6, \ - NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(noff_vlan_ol3ol4csum_l3l4csum, 0, 0, 0, 1, 1, 1, 1, 6, \ - NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(tso, 0, 0, 1, 0, 0, 0, 0, 6, \ - TSO_F) \ -T(tso_l3l4csum, 0, 0, 1, 0, 0, 0, 1, 6, \ - TSO_F | L3L4CSUM_F) \ -T(tso_ol3ol4csum, 0, 0, 1, 0, 0, 1, 0, 6, \ - TSO_F | OL3OL4CSUM_F) \ -T(tso_ol3ol4csum_l3l4csum, 0, 0, 1, 0, 0, 1, 1, 6, \ - TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(tso_vlan, 0, 0, 1, 0, 1, 0, 0, 6, \ - TSO_F | VLAN_F) \ -T(tso_vlan_l3l4csum, 0, 0, 1, 0, 1, 0, 1, 6, \ - TSO_F | VLAN_F | L3L4CSUM_F) \ -T(tso_vlan_ol3ol4csum, 0, 0, 1, 0, 1, 1, 0, 6, \ - TSO_F | VLAN_F | OL3OL4CSUM_F) \ -T(tso_vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 0, 1, 1, 1, 6, \ - TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(tso_noff, 0, 0, 1, 1, 0, 0, 0, 6, \ - TSO_F | NOFF_F) \ -T(tso_noff_l3l4csum, 0, 0, 1, 1, 0, 0, 1, 6, \ - TSO_F | NOFF_F | L3L4CSUM_F) \ -T(tso_noff_ol3ol4csum, 0, 0, 1, 1, 0, 1, 0, 6, \ - TSO_F | NOFF_F | OL3OL4CSUM_F) \ -T(tso_noff_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 0, 1, 1, 6, \ - TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(tso_noff_vlan, 0, 0, 1, 1, 1, 0, 0, 6, \ - TSO_F | NOFF_F | VLAN_F) \ -T(tso_noff_vlan_l3l4csum, 0, 0, 1, 1, 1, 0, 1, 6, \ - TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(tso_noff_vlan_ol3ol4csum, 0, 0, 1, 1, 1, 1, 0, 6, \ - TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(tso_noff_vlan_ol3ol4csum_l3l4csum, 0, 0, 1, 1, 1, 1, 1, 6, \ - TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts, 0, 1, 0, 0, 0, 0, 0, 8, \ - TSP_F) \ -T(ts_l3l4csum, 0, 1, 0, 0, 0, 0, 1, 8, \ - TSP_F | L3L4CSUM_F) \ -T(ts_ol3ol4csum, 0, 1, 0, 0, 0, 1, 0, 8, \ - TSP_F | OL3OL4CSUM_F) \ -T(ts_ol3ol4csum_l3l4csum, 0, 1, 0, 0, 0, 1, 1, 8, \ - TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_vlan, 0, 1, 0, 0, 1, 0, 0, 8, \ - TSP_F | VLAN_F) \ -T(ts_vlan_l3l4csum, 0, 1, 0, 0, 1, 0, 1, 8, \ - TSP_F | VLAN_F | L3L4CSUM_F) \ -T(ts_vlan_ol3ol4csum, 0, 1, 0, 0, 1, 1, 0, 8, \ - TSP_F | VLAN_F | OL3OL4CSUM_F) \ -T(ts_vlan_ol3ol4csum_l3l4csum, 0, 1, 0, 0, 1, 1, 1, 8, \ - TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_noff, 0, 1, 0, 1, 0, 0, 0, 8, \ - TSP_F | NOFF_F) \ -T(ts_noff_l3l4csum, 0, 1, 0, 1, 0, 0, 1, 8, \ - TSP_F | NOFF_F | L3L4CSUM_F) \ -T(ts_noff_ol3ol4csum, 0, 1, 0, 1, 0, 1, 0, 8, \ - TSP_F | NOFF_F | OL3OL4CSUM_F) \ -T(ts_noff_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 0, 1, 1, 8, \ - TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_noff_vlan, 0, 1, 0, 1, 1, 0, 0, 8, \ - TSP_F | NOFF_F | VLAN_F) \ -T(ts_noff_vlan_l3l4csum, 0, 1, 0, 1, 1, 0, 1, 8, \ - TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(ts_noff_vlan_ol3ol4csum, 0, 1, 0, 1, 1, 1, 0, 8, \ - TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(ts_noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 0, 1, 1, 1, 1, 8, \ - TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_tso, 0, 1, 1, 0, 0, 0, 0, 8, \ - TSP_F | TSO_F) \ -T(ts_tso_l3l4csum, 0, 1, 1, 0, 0, 0, 1, 8, \ - TSP_F | TSO_F | L3L4CSUM_F) \ -T(ts_tso_ol3ol4csum, 0, 1, 1, 0, 0, 1, 0, 8, \ - TSP_F | TSO_F | OL3OL4CSUM_F) \ -T(ts_tso_ol3ol4csum_l3l4csum, 0, 1, 1, 0, 0, 1, 1, 8, \ - TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_tso_vlan, 0, 1, 1, 0, 1, 0, 0, 8, \ - TSP_F | TSO_F | VLAN_F) \ -T(ts_tso_vlan_l3l4csum, 0, 1, 1, 0, 1, 0, 1, 8, \ - TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \ -T(ts_tso_vlan_ol3ol4csum, 0, 1, 1, 0, 1, 1, 0, 8, \ - TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ -T(ts_tso_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 0, 1, 1, 1, 8, \ - TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_tso_noff, 0, 1, 1, 1, 0, 0, 0, 8, \ - TSP_F | TSO_F | NOFF_F) \ -T(ts_tso_noff_l3l4csum, 0, 1, 1, 1, 0, 0, 1, 8, \ - TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \ -T(ts_tso_noff_ol3ol4csum, 0, 1, 1, 1, 0, 1, 0, 8, \ - TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ -T(ts_tso_noff_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 0, 1, 1, 8, \ - TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(ts_tso_noff_vlan, 0, 1, 1, 1, 1, 0, 0, 8, \ - TSP_F | TSO_F | NOFF_F | VLAN_F) \ -T(ts_tso_noff_vlan_l3l4csum, 0, 1, 1, 1, 1, 0, 1, 8, \ - TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(ts_tso_noff_vlan_ol3ol4csum, 0, 1, 1, 1, 1, 1, 0, 8, \ - TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 0, 1, 1, 1, 1, 1, 1, 8, \ - TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\ -T(sec, 1, 0, 0, 0, 0, 0, 0, 4, \ - T_SEC_F) \ -T(sec_l3l4csum, 1, 0, 0, 0, 0, 0, 1, 4, \ - T_SEC_F | L3L4CSUM_F) \ -T(sec_ol3ol4csum, 1, 0, 0, 0, 0, 1, 0, 4, \ - T_SEC_F | OL3OL4CSUM_F) \ -T(sec_ol3ol4csum_l3l4csum, 1, 0, 0, 0, 0, 1, 1, 4, \ - T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_vlan, 1, 0, 0, 0, 1, 0, 0, 6, \ - T_SEC_F | VLAN_F) \ -T(sec_vlan_l3l4csum, 1, 0, 0, 0, 1, 0, 1, 6, \ - T_SEC_F | VLAN_F | L3L4CSUM_F) \ -T(sec_vlan_ol3ol4csum, 1, 0, 0, 0, 1, 1, 0, 6, \ - T_SEC_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_vlan_ol3ol4csum_l3l4csum, 1, 0, 0, 0, 1, 1, 1, 6, \ - T_SEC_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_noff, 1, 0, 0, 1, 0, 0, 0, 4, \ - T_SEC_F | NOFF_F) \ -T(sec_noff_l3l4csum, 1, 0, 0, 1, 0, 0, 1, 4, \ - T_SEC_F | NOFF_F | L3L4CSUM_F) \ -T(sec_noff_ol3ol4csum, 1, 0, 0, 1, 0, 1, 0, 4, \ - T_SEC_F | NOFF_F | OL3OL4CSUM_F) \ -T(sec_noff_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 0, 1, 1, 4, \ - T_SEC_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_noff_vlan, 1, 0, 0, 1, 1, 0, 0, 6, \ - T_SEC_F | NOFF_F | VLAN_F) \ -T(sec_noff_vlan_l3l4csum, 1, 0, 0, 1, 1, 0, 1, 6, \ - T_SEC_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(sec_noff_vlan_ol3ol4csum, 1, 0, 0, 1, 1, 1, 0, 6, \ - T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_noff_vlan_ol3ol4csum_l3l4csum, 1, 0, 0, 1, 1, 1, 1, 6, \ - T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_tso, 1, 0, 1, 0, 0, 0, 0, 6, \ - T_SEC_F | TSO_F) \ -T(sec_tso_l3l4csum, 1, 0, 1, 0, 0, 0, 1, 6, \ - T_SEC_F | TSO_F | L3L4CSUM_F) \ -T(sec_tso_ol3ol4csum, 1, 0, 1, 0, 0, 1, 0, 6, \ - T_SEC_F | TSO_F | OL3OL4CSUM_F) \ -T(sec_tso_ol3ol4csum_l3l4csum, 1, 0, 1, 0, 0, 1, 1, 6, \ - T_SEC_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_tso_vlan, 1, 0, 1, 0, 1, 0, 0, 6, \ - T_SEC_F | TSO_F | VLAN_F) \ -T(sec_tso_vlan_l3l4csum, 1, 0, 1, 0, 1, 0, 1, 6, \ - T_SEC_F | TSO_F | VLAN_F | L3L4CSUM_F) \ -T(sec_tso_vlan_ol3ol4csum, 1, 0, 1, 0, 1, 1, 0, 6, \ - T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_tso_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 0, 1, 1, 1, 6, \ - T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_tso_noff, 1, 0, 1, 1, 0, 0, 0, 6, \ - T_SEC_F | TSO_F | NOFF_F) \ -T(sec_tso_noff_l3l4csum, 1, 0, 1, 1, 0, 0, 1, 6, \ - T_SEC_F | TSO_F | NOFF_F | L3L4CSUM_F) \ -T(sec_tso_noff_ol3ol4csum, 1, 0, 1, 1, 0, 1, 0, 6, \ - T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ -T(sec_tso_noff_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 0, 1, 1, 6, \ - T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_tso_noff_vlan, 1, 0, 1, 1, 1, 0, 0, 6, \ - T_SEC_F | TSO_F | NOFF_F | VLAN_F) \ -T(sec_tso_noff_vlan_l3l4csum, 1, 0, 1, 1, 1, 0, 1, 6, \ - T_SEC_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(sec_tso_noff_vlan_ol3ol4csum, 1, 0, 1, 1, 1, 1, 0, 6, \ - T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 0, 1, 1, 1, 1, 1, 6, \ - T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\ -T(sec_ts, 1, 1, 0, 0, 0, 0, 0, 8, \ - T_SEC_F | TSP_F) \ -T(sec_ts_l3l4csum, 1, 1, 0, 0, 0, 0, 1, 8, \ - T_SEC_F | TSP_F | L3L4CSUM_F) \ -T(sec_ts_ol3ol4csum, 1, 1, 0, 0, 0, 1, 0, 8, \ - T_SEC_F | TSP_F | OL3OL4CSUM_F) \ -T(sec_ts_ol3ol4csum_l3l4csum, 1, 1, 0, 0, 0, 1, 1, 8, \ - T_SEC_F | TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_vlan, 1, 1, 0, 0, 1, 0, 0, 8, \ - T_SEC_F | TSP_F | VLAN_F) \ -T(sec_ts_vlan_l3l4csum, 1, 1, 0, 0, 1, 0, 1, 8, \ - T_SEC_F | TSP_F | VLAN_F | L3L4CSUM_F) \ -T(sec_ts_vlan_ol3ol4csum, 1, 1, 0, 0, 1, 1, 0, 8, \ - T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_ts_vlan_ol3ol4csum_l3l4csum, 1, 1, 0, 0, 1, 1, 1, 8, \ - T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_noff, 1, 1, 0, 1, 0, 0, 0, 8, \ - T_SEC_F | TSP_F | NOFF_F) \ -T(sec_ts_noff_l3l4csum, 1, 1, 0, 1, 0, 0, 1, 8, \ - T_SEC_F | TSP_F | NOFF_F | L3L4CSUM_F) \ -T(sec_ts_noff_ol3ol4csum, 1, 1, 0, 1, 0, 1, 0, 8, \ - T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F) \ -T(sec_ts_noff_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 0, 1, 1, 8, \ - T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_noff_vlan, 1, 1, 0, 1, 1, 0, 0, 8, \ - T_SEC_F | TSP_F | NOFF_F | VLAN_F) \ -T(sec_ts_noff_vlan_l3l4csum, 1, 1, 0, 1, 1, 0, 1, 8, \ - T_SEC_F | TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(sec_ts_noff_vlan_ol3ol4csum, 1, 1, 0, 1, 1, 1, 0, 8, \ - T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_ts_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 0, 1, 1, 1, 1, 8, \ - T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F)\ -T(sec_ts_tso, 1, 1, 1, 0, 0, 0, 0, 8, \ - T_SEC_F | TSP_F | TSO_F) \ -T(sec_ts_tso_l3l4csum, 1, 1, 1, 0, 0, 0, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | L3L4CSUM_F) \ -T(sec_ts_tso_ol3ol4csum, 1, 1, 1, 0, 0, 1, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F) \ -T(sec_ts_tso_ol3ol4csum_l3l4csum, 1, 1, 1, 0, 0, 1, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_tso_vlan, 1, 1, 1, 0, 1, 0, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | VLAN_F) \ -T(sec_ts_tso_vlan_l3l4csum, 1, 1, 1, 0, 1, 0, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \ -T(sec_ts_tso_vlan_ol3ol4csum, 1, 1, 1, 0, 1, 1, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ -T(sec_ts_tso_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 0, 1, 1, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_tso_noff, 1, 1, 1, 1, 0, 0, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F) \ -T(sec_ts_tso_noff_l3l4csum, 1, 1, 1, 1, 0, 0, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \ -T(sec_ts_tso_noff_ol3ol4csum, 1, 1, 1, 1, 0, 1, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ -T(sec_ts_tso_noff_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 0, 1, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ -T(sec_ts_tso_noff_vlan, 1, 1, 1, 1, 1, 0, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F) \ -T(sec_ts_tso_noff_vlan_l3l4csum, 1, 1, 1, 1, 1, 0, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ -T(sec_ts_tso_noff_vlan_ol3ol4csum, 1, 1, 1, 1, 1, 1, 0, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F)\ -T(sec_ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 1, 1, 1, 1, 1, 1, 1, 8, \ - T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | \ - L3L4CSUM_F) +#define NIX_TX_FASTPATH_MODES_0_15 \ + T(no_offload, 4, NIX_TX_OFFLOAD_NONE) \ + T(l3l4csum, 4, L3L4CSUM_F) \ + T(ol3ol4csum, 4, OL3OL4CSUM_F) \ + T(ol3ol4csum_l3l4csum, 4, OL3OL4CSUM_F | L3L4CSUM_F) \ + T(vlan, 6, VLAN_F) \ + T(vlan_l3l4csum, 6, VLAN_F | L3L4CSUM_F) \ + T(vlan_ol3ol4csum, 6, VLAN_F | OL3OL4CSUM_F) \ + T(vlan_ol3ol4csum_l3l4csum, 6, VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(noff, 4, NOFF_F) \ + T(noff_l3l4csum, 4, NOFF_F | L3L4CSUM_F) \ + T(noff_ol3ol4csum, 4, NOFF_F | OL3OL4CSUM_F) \ + T(noff_ol3ol4csum_l3l4csum, 4, NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(noff_vlan, 6, NOFF_F | VLAN_F) \ + T(noff_vlan_l3l4csum, 6, NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(noff_vlan_ol3ol4csum, 6, NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(noff_vlan_ol3ol4csum_l3l4csum, 6, \ + NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ +#define NIX_TX_FASTPATH_MODES_16_31 \ + T(tso, 6, TSO_F) \ + T(tso_l3l4csum, 6, TSO_F | L3L4CSUM_F) \ + T(tso_ol3ol4csum, 6, TSO_F | OL3OL4CSUM_F) \ + T(tso_ol3ol4csum_l3l4csum, 6, TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(tso_vlan, 6, TSO_F | VLAN_F) \ + T(tso_vlan_l3l4csum, 6, TSO_F | VLAN_F | L3L4CSUM_F) \ + T(tso_vlan_ol3ol4csum, 6, TSO_F | VLAN_F | OL3OL4CSUM_F) \ + T(tso_vlan_ol3ol4csum_l3l4csum, 6, \ + TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(tso_noff, 6, TSO_F | NOFF_F) \ + T(tso_noff_l3l4csum, 6, TSO_F | NOFF_F | L3L4CSUM_F) \ + T(tso_noff_ol3ol4csum, 6, TSO_F | NOFF_F | OL3OL4CSUM_F) \ + T(tso_noff_ol3ol4csum_l3l4csum, 6, \ + TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(tso_noff_vlan, 6, TSO_F | NOFF_F | VLAN_F) \ + T(tso_noff_vlan_l3l4csum, 6, TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(tso_noff_vlan_ol3ol4csum, 6, TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(tso_noff_vlan_ol3ol4csum_l3l4csum, 6, \ + TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_32_47 \ + T(ts, 8, TSP_F) \ + T(ts_l3l4csum, 8, TSP_F | L3L4CSUM_F) \ + T(ts_ol3ol4csum, 8, TSP_F | OL3OL4CSUM_F) \ + T(ts_ol3ol4csum_l3l4csum, 8, TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_vlan, 8, TSP_F | VLAN_F) \ + T(ts_vlan_l3l4csum, 8, TSP_F | VLAN_F | L3L4CSUM_F) \ + T(ts_vlan_ol3ol4csum, 8, TSP_F | VLAN_F | OL3OL4CSUM_F) \ + T(ts_vlan_ol3ol4csum_l3l4csum, 8, \ + TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_noff, 8, TSP_F | NOFF_F) \ + T(ts_noff_l3l4csum, 8, TSP_F | NOFF_F | L3L4CSUM_F) \ + T(ts_noff_ol3ol4csum, 8, TSP_F | NOFF_F | OL3OL4CSUM_F) \ + T(ts_noff_ol3ol4csum_l3l4csum, 8, \ + TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_noff_vlan, 8, TSP_F | NOFF_F | VLAN_F) \ + T(ts_noff_vlan_l3l4csum, 8, TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(ts_noff_vlan_ol3ol4csum, 8, TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(ts_noff_vlan_ol3ol4csum_l3l4csum, 8, \ + TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_48_63 \ + T(ts_tso, 8, TSP_F | TSO_F) \ + T(ts_tso_l3l4csum, 8, TSP_F | TSO_F | L3L4CSUM_F) \ + T(ts_tso_ol3ol4csum, 8, TSP_F | TSO_F | OL3OL4CSUM_F) \ + T(ts_tso_ol3ol4csum_l3l4csum, 8, \ + TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_tso_vlan, 8, TSP_F | TSO_F | VLAN_F) \ + T(ts_tso_vlan_l3l4csum, 8, TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \ + T(ts_tso_vlan_ol3ol4csum, 8, TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ + T(ts_tso_vlan_ol3ol4csum_l3l4csum, 8, \ + TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_tso_noff, 8, TSP_F | TSO_F | NOFF_F) \ + T(ts_tso_noff_l3l4csum, 8, TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \ + T(ts_tso_noff_ol3ol4csum, 8, TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ + T(ts_tso_noff_ol3ol4csum_l3l4csum, 8, \ + TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(ts_tso_noff_vlan, 8, TSP_F | TSO_F | NOFF_F | VLAN_F) \ + T(ts_tso_noff_vlan_l3l4csum, 8, \ + TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(ts_tso_noff_vlan_ol3ol4csum, 8, \ + TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 8, \ + TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_64_79 \ + T(sec, 4, T_SEC_F) \ + T(sec_l3l4csum, 4, T_SEC_F | L3L4CSUM_F) \ + T(sec_ol3ol4csum, 4, T_SEC_F | OL3OL4CSUM_F) \ + T(sec_ol3ol4csum_l3l4csum, 4, T_SEC_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_vlan, 6, T_SEC_F | VLAN_F) \ + T(sec_vlan_l3l4csum, 6, T_SEC_F | VLAN_F | L3L4CSUM_F) \ + T(sec_vlan_ol3ol4csum, 6, T_SEC_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_vlan_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_noff, 4, T_SEC_F | NOFF_F) \ + T(sec_noff_l3l4csum, 4, T_SEC_F | NOFF_F | L3L4CSUM_F) \ + T(sec_noff_ol3ol4csum, 4, T_SEC_F | NOFF_F | OL3OL4CSUM_F) \ + T(sec_noff_ol3ol4csum_l3l4csum, 4, \ + T_SEC_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_noff_vlan, 6, T_SEC_F | NOFF_F | VLAN_F) \ + T(sec_noff_vlan_l3l4csum, 6, T_SEC_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(sec_noff_vlan_ol3ol4csum, 6, \ + T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_noff_vlan_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_80_95 \ + T(sec_tso, 6, T_SEC_F | TSO_F) \ + T(sec_tso_l3l4csum, 6, T_SEC_F | TSO_F | L3L4CSUM_F) \ + T(sec_tso_ol3ol4csum, 6, T_SEC_F | TSO_F | OL3OL4CSUM_F) \ + T(sec_tso_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_tso_vlan, 6, T_SEC_F | TSO_F | VLAN_F) \ + T(sec_tso_vlan_l3l4csum, 6, T_SEC_F | TSO_F | VLAN_F | L3L4CSUM_F) \ + T(sec_tso_vlan_ol3ol4csum, 6, T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_tso_vlan_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_tso_noff, 6, T_SEC_F | TSO_F | NOFF_F) \ + T(sec_tso_noff_l3l4csum, 6, T_SEC_F | TSO_F | NOFF_F | L3L4CSUM_F) \ + T(sec_tso_noff_ol3ol4csum, 6, T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ + T(sec_tso_noff_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_tso_noff_vlan, 6, T_SEC_F | TSO_F | NOFF_F | VLAN_F) \ + T(sec_tso_noff_vlan_l3l4csum, 6, \ + T_SEC_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(sec_tso_noff_vlan_ol3ol4csum, 6, \ + T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_tso_noff_vlan_ol3ol4csum_l3l4csum, 6, \ + T_SEC_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_96_111 \ + T(sec_ts, 8, T_SEC_F | TSP_F) \ + T(sec_ts_l3l4csum, 8, T_SEC_F | TSP_F | L3L4CSUM_F) \ + T(sec_ts_ol3ol4csum, 8, T_SEC_F | TSP_F | OL3OL4CSUM_F) \ + T(sec_ts_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_vlan, 8, T_SEC_F | TSP_F | VLAN_F) \ + T(sec_ts_vlan_l3l4csum, 8, T_SEC_F | TSP_F | VLAN_F | L3L4CSUM_F) \ + T(sec_ts_vlan_ol3ol4csum, 8, T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_ts_vlan_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_noff, 8, T_SEC_F | TSP_F | NOFF_F) \ + T(sec_ts_noff_l3l4csum, 8, T_SEC_F | TSP_F | NOFF_F | L3L4CSUM_F) \ + T(sec_ts_noff_ol3ol4csum, 8, T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F) \ + T(sec_ts_noff_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_noff_vlan, 8, T_SEC_F | TSP_F | NOFF_F | VLAN_F) \ + T(sec_ts_noff_vlan_l3l4csum, 8, \ + T_SEC_F | TSP_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(sec_ts_noff_vlan_ol3ol4csum, 8, \ + T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_ts_noff_vlan_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES_112_127 \ + T(sec_ts_tso, 8, T_SEC_F | TSP_F | TSO_F) \ + T(sec_ts_tso_l3l4csum, 8, T_SEC_F | TSP_F | TSO_F | L3L4CSUM_F) \ + T(sec_ts_tso_ol3ol4csum, 8, T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F) \ + T(sec_ts_tso_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_tso_vlan, 8, T_SEC_F | TSP_F | TSO_F | VLAN_F) \ + T(sec_ts_tso_vlan_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | VLAN_F | L3L4CSUM_F) \ + T(sec_ts_tso_vlan_ol3ol4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_ts_tso_vlan_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | VLAN_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_tso_noff, 8, T_SEC_F | TSP_F | TSO_F | NOFF_F) \ + T(sec_ts_tso_noff_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | L3L4CSUM_F) \ + T(sec_ts_tso_noff_ol3ol4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F) \ + T(sec_ts_tso_noff_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F) \ + T(sec_ts_tso_noff_vlan, 8, T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F) \ + T(sec_ts_tso_noff_vlan_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | L3L4CSUM_F) \ + T(sec_ts_tso_noff_vlan_ol3ol4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F) \ + T(sec_ts_tso_noff_vlan_ol3ol4csum_l3l4csum, 8, \ + T_SEC_F | TSP_F | TSO_F | NOFF_F | VLAN_F | OL3OL4CSUM_F | \ + L3L4CSUM_F) + +#define NIX_TX_FASTPATH_MODES \ + NIX_TX_FASTPATH_MODES_0_15 \ + NIX_TX_FASTPATH_MODES_16_31 \ + NIX_TX_FASTPATH_MODES_32_47 \ + NIX_TX_FASTPATH_MODES_48_63 \ + NIX_TX_FASTPATH_MODES_64_79 \ + NIX_TX_FASTPATH_MODES_80_95 \ + NIX_TX_FASTPATH_MODES_96_111 \ + NIX_TX_FASTPATH_MODES_112_127 + +#define T(name, sz, flags) \ uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_mseg_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_vec_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); \ - \ uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_vec_mseg_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts); NIX_TX_FASTPATH_MODES #undef T +#define NIX_TX_XMIT(fn, sz, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ + { \ + uint64_t cmd[sz]; \ + /* For TSO inner checksum is a must */ \ + if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ + !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ + return 0; \ + return cn9k_nix_xmit_pkts(tx_queue, tx_pkts, pkts, cmd, \ + flags); \ + } + +#define NIX_TX_XMIT_MSEG(fn, sz, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ + { \ + uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ + /* For TSO inner checksum is a must */ \ + if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ + !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ + return 0; \ + return cn9k_nix_xmit_pkts_mseg(tx_queue, tx_pkts, pkts, cmd, \ + (flags) | NIX_TX_MULTI_SEG_F); \ + } + +#define NIX_TX_XMIT_VEC(fn, sz, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ + { \ + uint64_t cmd[sz]; \ + /* For TSO inner checksum is a must */ \ + if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ + !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ + return 0; \ + return cn9k_nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, \ + (flags)); \ + } + +#define NIX_TX_XMIT_VEC_MSEG(fn, sz, flags) \ + uint16_t __rte_noinline __rte_hot fn( \ + void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ + { \ + uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ + /* For TSO inner checksum is a must */ \ + if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ + !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ + return 0; \ + return cn9k_nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, \ + (flags) | \ + NIX_TX_MULTI_SEG_F); \ + } + #endif /* __CN9K_TX_H__ */ diff --git a/drivers/net/cnxk/cn9k_tx_mseg.c b/drivers/net/cnxk/cn9k_tx_mseg.c deleted file mode 100644 index 37cba78e71..0000000000 --- a/drivers/net/cnxk/cn9k_tx_mseg.c +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_ethdev.h" -#include "cn9k_tx.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_noinline __rte_hot \ - cn9k_nix_xmit_pkts_mseg_##name(void *tx_queue, \ - struct rte_mbuf **tx_pkts, \ - uint16_t pkts) \ - { \ - uint64_t cmd[(sz) + CNXK_NIX_TX_MSEG_SG_DWORDS - 2]; \ - \ - /* For TSO inner checksum is a must */ \ - if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ - !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ - return 0; \ - return cn9k_nix_xmit_pkts_mseg(tx_queue, tx_pkts, pkts, cmd, \ - (flags) | NIX_TX_MULTI_SEG_F); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/net/cnxk/cn9k_tx_select.c b/drivers/net/cnxk/cn9k_tx_select.c new file mode 100644 index 0000000000..6d40422f68 --- /dev/null +++ b/drivers/net/cnxk/cn9k_tx_select.c @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2021 Marvell. + */ + +#include "cn9k_ethdev.h" +#include "cn9k_tx.h" + +static inline void +pick_tx_func(struct rte_eth_dev *eth_dev, + const eth_tx_burst_t tx_burst[NIX_TX_OFFLOAD_MAX]) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + + /* [TS] [TSO] [NOFF] [VLAN] [OL3_OL4_CSUM] [IL3_IL4_CSUM] */ + eth_dev->tx_pkt_burst = + tx_burst[dev->tx_offload_flags & (NIX_TX_OFFLOAD_MAX - 1)]; +} + +void +cn9k_eth_set_tx_function(struct rte_eth_dev *eth_dev) +{ + struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); + + const eth_tx_burst_t nix_eth_tx_burst[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_nix_xmit_pkts_##name, + NIX_TX_FASTPATH_MODES +#undef T + }; + + const eth_tx_burst_t nix_eth_tx_burst_mseg[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_nix_xmit_pkts_mseg_##name, + NIX_TX_FASTPATH_MODES +#undef T + }; + + const eth_tx_burst_t nix_eth_tx_vec_burst[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_nix_xmit_pkts_vec_##name, + NIX_TX_FASTPATH_MODES +#undef T + }; + + const eth_tx_burst_t nix_eth_tx_vec_burst_mseg[NIX_TX_OFFLOAD_MAX] = { +#define T(name, sz, flags)[flags] = cn9k_nix_xmit_pkts_vec_mseg_##name, + NIX_TX_FASTPATH_MODES +#undef T + }; + + if (dev->scalar_ena) { + pick_tx_func(eth_dev, nix_eth_tx_burst); + if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) + pick_tx_func(eth_dev, nix_eth_tx_burst_mseg); + } else { + pick_tx_func(eth_dev, nix_eth_tx_vec_burst); + if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) + pick_tx_func(eth_dev, nix_eth_tx_vec_burst_mseg); + } + + rte_mb(); +} diff --git a/drivers/net/cnxk/cn9k_tx_vec.c b/drivers/net/cnxk/cn9k_tx_vec.c deleted file mode 100644 index b424f95895..0000000000 --- a/drivers/net/cnxk/cn9k_tx_vec.c +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_ethdev.h" -#include "cn9k_tx.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_noinline __rte_hot \ - cn9k_nix_xmit_pkts_vec_##name(void *tx_queue, \ - struct rte_mbuf **tx_pkts, \ - uint16_t pkts) \ - { \ - uint64_t cmd[sz]; \ - \ - /* For TSO inner checksum is a must */ \ - if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ - !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ - return 0; \ - return cn9k_nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, \ - (flags)); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/net/cnxk/cn9k_tx_vec_mseg.c b/drivers/net/cnxk/cn9k_tx_vec_mseg.c deleted file mode 100644 index 5fdf0a9062..0000000000 --- a/drivers/net/cnxk/cn9k_tx_vec_mseg.c +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(C) 2021 Marvell. - */ - -#include "cn9k_ethdev.h" -#include "cn9k_tx.h" - -#define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ - uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_vec_mseg_##name( \ - void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ - { \ - uint64_t cmd[sz]; \ - \ - /* For TSO inner checksum is a must */ \ - if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \ - !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \ - return 0; \ - return cn9k_nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, \ - (flags) | \ - NIX_TX_MULTI_SEG_F); \ - } - -NIX_TX_FASTPATH_MODES -#undef T diff --git a/drivers/net/cnxk/meson.build b/drivers/net/cnxk/meson.build index fcf8140686..01bd5442b5 100644 --- a/drivers/net/cnxk/meson.build +++ b/drivers/net/cnxk/meson.build @@ -29,10 +29,7 @@ sources += files( 'cn9k_ethdev_sec.c', 'cn9k_rte_flow.c', 'cn9k_rx_select.c', - 'cn9k_tx.c', - 'cn9k_tx_mseg.c', - 'cn9k_tx_vec.c', - 'cn9k_tx_vec_mseg.c', + 'cn9k_tx_select.c', ) sources += files( @@ -70,6 +67,41 @@ sources += files( 'rx/cn9k/rx_112_127_vec_mseg.c', ) +sources += files( + 'tx/cn9k/tx_0_15.c', + 'tx/cn9k/tx_16_31.c', + 'tx/cn9k/tx_32_47.c', + 'tx/cn9k/tx_48_63.c', + 'tx/cn9k/tx_64_79.c', + 'tx/cn9k/tx_80_95.c', + 'tx/cn9k/tx_96_111.c', + 'tx/cn9k/tx_112_127.c', + 'tx/cn9k/tx_0_15_mseg.c', + 'tx/cn9k/tx_16_31_mseg.c', + 'tx/cn9k/tx_32_47_mseg.c', + 'tx/cn9k/tx_48_63_mseg.c', + 'tx/cn9k/tx_64_79_mseg.c', + 'tx/cn9k/tx_80_95_mseg.c', + 'tx/cn9k/tx_96_111_mseg.c', + 'tx/cn9k/tx_112_127_mseg.c', + 'tx/cn9k/tx_0_15_vec.c', + 'tx/cn9k/tx_16_31_vec.c', + 'tx/cn9k/tx_32_47_vec.c', + 'tx/cn9k/tx_48_63_vec.c', + 'tx/cn9k/tx_64_79_vec.c', + 'tx/cn9k/tx_80_95_vec.c', + 'tx/cn9k/tx_96_111_vec.c', + 'tx/cn9k/tx_112_127_vec.c', + 'tx/cn9k/tx_0_15_vec_mseg.c', + 'tx/cn9k/tx_16_31_vec_mseg.c', + 'tx/cn9k/tx_32_47_vec_mseg.c', + 'tx/cn9k/tx_48_63_vec_mseg.c', + 'tx/cn9k/tx_64_79_vec_mseg.c', + 'tx/cn9k/tx_80_95_vec_mseg.c', + 'tx/cn9k/tx_96_111_vec_mseg.c', + 'tx/cn9k/tx_112_127_vec_mseg.c', +) + # CN10K sources += files( 'cn10k_ethdev.c',