d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/iavf: rework Tx path

Browse Source 
Rework the Tx path and Tx descriptor usage in order to
allow for better use of offload flags and to facilitate enabling of
inline crypto offload feature.

Signed-off-by: Declan Doherty <declan.doherty@intel.com>
Signed-off-by: Abhijit Sinha <abhijit.sinha@intel.com>
Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
Acked-by: Jingjing Wu <jingjing.wu@intel.com>
This commit is contained in:
Radu Nicolau 2021-10-28 17:04:55 +01:00 committed by Qi Zhang
parent 993f0d4d62
commit 1e728b0112
3 changed files with 426 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

529
drivers/net/iavf/iavf_rxtx.c
View File

@ -1054,27 +1054,34 @@ iavf_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union iavf_rx_desc *rxdp)
static inline void
iavf_flex_rxd_to_vlan_tci(struct rte_mbuf *mb,
volatile union iavf_rx_flex_desc *rxdp,
uint8_t rx_flags)
volatile union iavf_rx_flex_desc *rxdp)
{
uint16_t vlan_tci = 0;
if (rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1 &&
rte_le_to_cpu_64(rxdp->wb.status_error0) &
(1 << IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S))
vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag1);
if (rte_le_to_cpu_64(rxdp->wb.status_error0) &
(1 << IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
mb->ol_flags |= RTE_MBUF_F_RX_VLAN |
RTE_MBUF_F_RX_VLAN_STRIPPED;
mb->vlan_tci =
rte_le_to_cpu_16(rxdp->wb.l2tag1);
} else {
mb->vlan_tci = 0;
}
#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
if (rx_flags & IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2 &&
rte_le_to_cpu_16(rxdp->wb.status_error1) &
(1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S))
vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd);
#endif
if (vlan_tci) {
mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
mb->vlan_tci = vlan_tci;
if (rte_le_to_cpu_16(rxdp->wb.status_error1) &
(1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) {
mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED |
RTE_MBUF_F_RX_QINQ |
RTE_MBUF_F_RX_VLAN_STRIPPED |
RTE_MBUF_F_RX_VLAN;
mb->vlan_tci_outer = mb->vlan_tci;
mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd);
PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
rte_le_to_cpu_16(rxdp->wb.l2tag2_1st),
rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd));
} else {
mb->vlan_tci_outer = 0;
}
#endif
}
/* Translate the rx descriptor status and error fields to pkt flags */
@ -1394,7 +1401,7 @@ iavf_recv_pkts_flex_rxd(void *rx_queue,
rxm->ol_flags = 0;
rxm->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
iavf_flex_rxd_to_vlan_tci(rxm, &rxd, rxq->rx_flags);
iavf_flex_rxd_to_vlan_tci(rxm, &rxd);
rxq->rxd_to_pkt_fields(rxq, rxm, &rxd);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
rxm->ol_flags |= pkt_flags;
@ -1536,7 +1543,7 @@ iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
first_seg->ol_flags = 0;
first_seg->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
iavf_flex_rxd_to_vlan_tci(first_seg, &rxd, rxq->rx_flags);
iavf_flex_rxd_to_vlan_tci(first_seg, &rxd);
rxq->rxd_to_pkt_fields(rxq, first_seg, &rxd);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
@ -1774,7 +1781,7 @@ iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq)
mb->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)];
iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j], rxq->rx_flags);
iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]);
rxq->rxd_to_pkt_fields(rxq, mb, &rxdp[j]);
stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0);
@ -2068,190 +2075,298 @@ iavf_xmit_cleanup(struct iavf_tx_queue *txq)
return 0;
}
/* Check if the context descriptor is needed for TX offloading */
static inline uint16_t
iavf_calc_context_desc(uint64_t flags, uint8_t vlan_flag)
static inline void
iavf_fill_ctx_desc_cmd_field(volatile uint64_t *field, struct rte_mbuf *m)
{
if (flags & RTE_MBUF_F_TX_TCP_SEG)
return 1;
if (flags & RTE_MBUF_F_TX_VLAN &&
vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2)
return 1;
return 0;
uint64_t cmd = 0;
/* TSO enabled */
if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
cmd = IAVF_TX_CTX_DESC_TSO << IAVF_TXD_DATA_QW1_CMD_SHIFT;
/* Time Sync - Currently not supported */
/* Outer L2 TAG 2 Insertion - Currently not supported */
/* Inner L2 TAG 2 Insertion - Currently not supported */
*field |= cmd;
}
static inline void
iavf_txd_enable_checksum(uint64_t ol_flags,
uint32_t *td_cmd,
uint32_t *td_offset,
union iavf_tx_offload tx_offload)
iavf_fill_ctx_desc_tunnelling_field(volatile uint64_t *qw0,
const struct rte_mbuf *m)
{
/* Set MACLEN */
*td_offset |= (tx_offload.l2_len >> 1) <<
IAVF_TX_DESC_LENGTH_MACLEN_SHIFT;
uint64_t eip_typ = IAVF_TX_CTX_DESC_EIPT_NONE;
uint64_t eip_len = 0;
uint64_t eip_noinc = 0;
/* Default - IP_ID is increment in each segment of LSO */
/* Enable L3 checksum offloads */
if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
*td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM;
*td_offset |= (tx_offload.l3_len >> 2) <<
IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
} else if (ol_flags & RTE_MBUF_F_TX_IPV4) {
*td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4;
*td_offset |= (tx_offload.l3_len >> 2) <<
IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
} else if (ol_flags & RTE_MBUF_F_TX_IPV6) {
*td_cmd |= IAVF_TX_DESC_CMD_IIPT_IPV6;
*td_offset |= (tx_offload.l3_len >> 2) <<
IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
switch (m->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
RTE_MBUF_F_TX_OUTER_IPV6 |
RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
case RTE_MBUF_F_TX_OUTER_IPV4:
eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_NO_CHECKSUM_OFFLOAD;
eip_len = m->outer_l3_len >> 2;
break;
case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_CHECKSUM_OFFLOAD;
eip_len = m->outer_l3_len >> 2;
break;
case RTE_MBUF_F_TX_OUTER_IPV6:
eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV6;
eip_len = m->outer_l3_len >> 2;
break;
}
if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
*td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
*td_offset |= (tx_offload.l4_len >> 2) <<
*qw0 = eip_typ << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_SHIFT |
eip_len << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_SHIFT |
eip_noinc << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_SHIFT;
}
static inline uint16_t
iavf_fill_ctx_desc_segmentation_field(volatile uint64_t *field,
struct rte_mbuf *m)
{
uint64_t segmentation_field = 0;
uint64_t total_length = 0;
total_length = m->pkt_len - (m->l2_len + m->l3_len + m->l4_len);
if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
total_length -= m->outer_l3_len;
#ifdef RTE_LIBRTE_IAVF_DEBUG_TX
if (!m->l4_len || !m->tso_segsz)
PMD_TX_LOG(DEBUG, "L4 length %d, LSO Segment size %d",
m->l4_len, m->tso_segsz);
if (m->tso_segsz < 88)
PMD_TX_LOG(DEBUG, "LSO Segment size %d is less than minimum %d",
m->tso_segsz, 88);
#endif
segmentation_field =
(((uint64_t)total_length << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) &
IAVF_TXD_CTX_QW1_TSO_LEN_MASK) |
(((uint64_t)m->tso_segsz << IAVF_TXD_CTX_QW1_MSS_SHIFT) &
IAVF_TXD_CTX_QW1_MSS_MASK);
*field |= segmentation_field;
return total_length;
}
struct iavf_tx_context_desc_qws {
__le64 qw0;
__le64 qw1;
};
static inline void
iavf_fill_context_desc(volatile struct iavf_tx_context_desc *desc,
struct rte_mbuf *m, uint16_t *tlen)
{
volatile struct iavf_tx_context_desc_qws *desc_qws =
(volatile struct iavf_tx_context_desc_qws *)desc;
/* fill descriptor type field */
desc_qws->qw1 = IAVF_TX_DESC_DTYPE_CONTEXT;
/* fill command field */
iavf_fill_ctx_desc_cmd_field(&desc_qws->qw1, m);
/* fill segmentation field */
if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) {
*tlen = iavf_fill_ctx_desc_segmentation_field(&desc_qws->qw1,
m);
}
/* fill tunnelling field */
if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
iavf_fill_ctx_desc_tunnelling_field(&desc_qws->qw0, m);
else
desc_qws->qw0 = 0;
desc_qws->qw0 = rte_cpu_to_le_64(desc_qws->qw0);
desc_qws->qw1 = rte_cpu_to_le_64(desc_qws->qw1);
}
static inline void
iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1,
struct rte_mbuf *m)
{
uint64_t command = 0;
uint64_t offset = 0;
uint64_t l2tag1 = 0;
*qw1 = IAVF_TX_DESC_DTYPE_DATA;
command = (uint64_t)IAVF_TX_DESC_CMD_ICRC;
/* Descriptor based VLAN insertion */
if (m->ol_flags & RTE_MBUF_F_TX_VLAN) {
command |= (uint64_t)IAVF_TX_DESC_CMD_IL2TAG1;
l2tag1 |= m->vlan_tci;
}
/* Set MACLEN */
offset |= (m->l2_len >> 1) << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT;
/* Enable L3 checksum offloading inner */
if (m->ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_IPV4)) {
command |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM;
offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
} else if (m->ol_flags & RTE_MBUF_F_TX_IPV4) {
command |= IAVF_TX_DESC_CMD_IIPT_IPV4;
offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
} else if (m->ol_flags & RTE_MBUF_F_TX_IPV6) {
command |= IAVF_TX_DESC_CMD_IIPT_IPV6;
offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
}
if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
offset |= (m->l4_len >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
return;
}
/* Enable L4 checksum offloads */
switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
switch (m->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
case RTE_MBUF_F_TX_TCP_CKSUM:
*td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
*td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
break;
case RTE_MBUF_F_TX_SCTP_CKSUM:
*td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP;
*td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
command |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP;
offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
break;
case RTE_MBUF_F_TX_UDP_CKSUM:
*td_cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP;
*td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
break;
default:
command |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP;
offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
break;
}
*qw1 = rte_cpu_to_le_64((((uint64_t)command <<
IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) |
(((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) &
IAVF_TXD_DATA_QW1_OFFSET_MASK) |
((uint64_t)l2tag1 << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT));
}
/* set TSO context descriptor
* support IP -> L4 and IP -> IP -> L4
*/
static inline uint64_t
iavf_set_tso_ctx(struct rte_mbuf *mbuf, union iavf_tx_offload tx_offload)
static inline void
iavf_fill_data_desc(volatile struct iavf_tx_desc *desc,
struct rte_mbuf *m, uint64_t desc_template,
uint16_t tlen, uint16_t ipseclen)
{
uint64_t ctx_desc = 0;
uint32_t cd_cmd, hdr_len, cd_tso_len;
uint32_t hdrlen = m->l2_len;
uint32_t bufsz = 0;
if (!tx_offload.l4_len) {
PMD_TX_LOG(DEBUG, "L4 length set to 0");
return ctx_desc;
/* fill data descriptor qw1 from template */
desc->cmd_type_offset_bsz = desc_template;
/* set data buffer address */
desc->buffer_addr = rte_mbuf_data_iova(m);
/* calculate data buffer size less set header lengths */
if ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) &&
(m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG |
RTE_MBUF_F_TX_UDP_SEG))) {
hdrlen += m->outer_l3_len;
if (m->ol_flags & RTE_MBUF_F_TX_L4_MASK)
hdrlen += m->l3_len + m->l4_len;
else
hdrlen += m->l3_len;
if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)
hdrlen += ipseclen;
bufsz = hdrlen + tlen;
} else {
bufsz = m->data_len;
}
hdr_len = tx_offload.l2_len +
tx_offload.l3_len +
tx_offload.l4_len;
/* set data buffer size */
desc->cmd_type_offset_bsz |=
(((uint64_t)bufsz << IAVF_TXD_DATA_QW1_TX_BUF_SZ_SHIFT) &
IAVF_TXD_DATA_QW1_TX_BUF_SZ_MASK);
cd_cmd = IAVF_TX_CTX_DESC_TSO;
cd_tso_len = mbuf->pkt_len - hdr_len;
ctx_desc |= ((uint64_t)cd_cmd << IAVF_TXD_CTX_QW1_CMD_SHIFT) |
((uint64_t)cd_tso_len << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) |
((uint64_t)mbuf->tso_segsz << IAVF_TXD_CTX_QW1_MSS_SHIFT);
return ctx_desc;
desc->buffer_addr = rte_cpu_to_le_64(desc->buffer_addr);
desc->cmd_type_offset_bsz = rte_cpu_to_le_64(desc->cmd_type_offset_bsz);
}
/* Construct the tx flags */
static inline uint64_t
iavf_build_ctob(uint32_t td_cmd, uint32_t td_offset, unsigned int size,
uint32_t td_tag)
{
return rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DATA |
((uint64_t)td_cmd << IAVF_TXD_QW1_CMD_SHIFT) |
((uint64_t)td_offset <<
IAVF_TXD_QW1_OFFSET_SHIFT) |
((uint64_t)size <<
IAVF_TXD_QW1_TX_BUF_SZ_SHIFT) |
((uint64_t)td_tag <<
IAVF_TXD_QW1_L2TAG1_SHIFT));
}
/* TX function */
uint16_t
iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
volatile struct iavf_tx_desc *txd;
volatile struct iavf_tx_desc *txr;
struct iavf_tx_queue *txq;
struct iavf_tx_entry *sw_ring;
struct iavf_tx_queue *txq = tx_queue;
volatile struct iavf_tx_desc *txr = txq->tx_ring;
struct iavf_tx_entry *txe_ring = txq->sw_ring;
struct iavf_tx_entry *txe, *txn;
struct rte_mbuf *tx_pkt;
struct rte_mbuf *m_seg;
uint16_t tx_id;
uint16_t nb_tx;
uint32_t td_cmd;
uint32_t td_offset;
uint32_t td_tag;
uint64_t ol_flags;
uint16_t nb_used;
uint16_t nb_ctx;
uint16_t tx_last;
uint16_t slen;
uint64_t buf_dma_addr;
uint16_t cd_l2tag2 = 0;
union iavf_tx_offload tx_offload = {0};
struct rte_mbuf *mb, *mb_seg;
uint16_t desc_idx, desc_idx_last;
uint16_t idx;
txq = tx_queue;
sw_ring = txq->sw_ring;
txr = txq->tx_ring;
tx_id = txq->tx_tail;
txe = &sw_ring[tx_id];
/* Check if the descriptor ring needs to be cleaned. */
if (txq->nb_free < txq->free_thresh)
(void)iavf_xmit_cleanup(txq);
iavf_xmit_cleanup(txq);
for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
td_cmd = 0;
td_tag = 0;
td_offset = 0;
desc_idx = txq->tx_tail;
txe = &txe_ring[desc_idx];
#ifdef RTE_LIBRTE_IAVF_DEBUG_TX_DESC_RING
iavf_dump_tx_entry_ring(txq);
iavf_dump_tx_desc_ring(txq);
#endif
for (idx = 0; idx < nb_pkts; idx++) {
volatile struct iavf_tx_desc *ddesc;
uint16_t nb_desc_ctx;
uint16_t nb_desc_data, nb_desc_required;
uint16_t tlen = 0, ipseclen = 0;
uint64_t ddesc_template = 0;
uint64_t ddesc_cmd = 0;
mb = tx_pkts[idx];
tx_pkt = *tx_pkts++;
RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
ol_flags = tx_pkt->ol_flags;
tx_offload.l2_len = tx_pkt->l2_len;
tx_offload.l3_len = tx_pkt->l3_len;
tx_offload.l4_len = tx_pkt->l4_len;
tx_offload.tso_segsz = tx_pkt->tso_segsz;
/* Calculate the number of context descriptors needed. */
nb_ctx = iavf_calc_context_desc(ol_flags, txq->vlan_flag);
nb_desc_data = mb->nb_segs;
nb_desc_ctx = !!(mb->ol_flags &
(RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG |
RTE_MBUF_F_TX_TUNNEL_MASK));
/* The number of descriptors that must be allocated for
/**
* The number of descriptors that must be allocated for
* a packet equals to the number of the segments of that
* packet plus 1 context descriptor if needed.
* packet plus the context and ipsec descriptors if needed.
*/
nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
tx_last = (uint16_t)(tx_id + nb_used - 1);
nb_desc_required = nb_desc_data + nb_desc_ctx;
/* Circular ring */
if (tx_last >= txq->nb_tx_desc)
tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
desc_idx_last = (uint16_t)(desc_idx + nb_desc_required - 1);
PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u"
" tx_first=%u tx_last=%u",
txq->port_id, txq->queue_id, tx_id, tx_last);
/* wrap descriptor ring */
if (desc_idx_last >= txq->nb_tx_desc)
desc_idx_last =
(uint16_t)(desc_idx_last - txq->nb_tx_desc);
if (nb_used > txq->nb_free) {
PMD_TX_LOG(DEBUG,
"port_id=%u queue_id=%u tx_first=%u tx_last=%u",
txq->port_id, txq->queue_id, desc_idx, desc_idx_last);
if (nb_desc_required > txq->nb_free) {
if (iavf_xmit_cleanup(txq)) {
if (nb_tx == 0)
if (idx == 0)
return 0;
goto end_of_tx;
}
if (unlikely(nb_used > txq->rs_thresh)) {
while (nb_used > txq->nb_free) {
if (unlikely(nb_desc_required > txq->rs_thresh)) {
while (nb_desc_required > txq->nb_free) {
if (iavf_xmit_cleanup(txq)) {
if (nb_tx == 0)
if (idx == 0)
return 0;
goto end_of_tx;
}
@ -2259,122 +2374,94 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
}
}
/* Descriptor based VLAN insertion */
if (ol_flags & RTE_MBUF_F_TX_VLAN &&
txq->vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1) {
td_cmd |= IAVF_TX_DESC_CMD_IL2TAG1;
td_tag = tx_pkt->vlan_tci;
}
iavf_build_data_desc_cmd_offset_fields(&ddesc_template, mb);
/* According to datasheet, the bit2 is reserved and must be
* set to 1.
*/
td_cmd |= 0x04;
/* Enable checksum offloading */
if (ol_flags & IAVF_TX_CKSUM_OFFLOAD_MASK)
iavf_txd_enable_checksum(ol_flags, &td_cmd,
&td_offset, tx_offload);
if (nb_ctx) {
/* Setup TX context descriptor if required */
uint64_t cd_type_cmd_tso_mss =
IAVF_TX_DESC_DTYPE_CONTEXT;
volatile struct iavf_tx_context_desc *ctx_txd =
if (nb_desc_ctx) {
volatile struct iavf_tx_context_desc *ctx_desc =
(volatile struct iavf_tx_context_desc *)
&txr[tx_id];
&txr[desc_idx];
/* clear QW0 or the previous writeback value
* may impact next write
*/
*(volatile uint64_t *)ctx_txd = 0;
*(volatile uint64_t *)ctx_desc = 0;
txn = &sw_ring[txe->next_id];
txn = &txe_ring[txe->next_id];
RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
if (txe->mbuf) {
rte_pktmbuf_free_seg(txe->mbuf);
txe->mbuf = NULL;
}
/* TSO enabled */
if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
cd_type_cmd_tso_mss |=
iavf_set_tso_ctx(tx_pkt, tx_offload);
iavf_fill_context_desc(ctx_desc, mb, &tlen);
IAVF_DUMP_TX_DESC(txq, ctx_desc, desc_idx);
if (ol_flags & RTE_MBUF_F_TX_VLAN &&
txq->vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) {
cd_type_cmd_tso_mss |= IAVF_TX_CTX_DESC_IL2TAG2
<< IAVF_TXD_CTX_QW1_CMD_SHIFT;
cd_l2tag2 = tx_pkt->vlan_tci;
txe->last_id = desc_idx_last;
desc_idx = txe->next_id;
txe = txn;
}
ctx_txd->type_cmd_tso_mss =
rte_cpu_to_le_64(cd_type_cmd_tso_mss);
ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
IAVF_DUMP_TX_DESC(txq, &txr[tx_id], tx_id);
txe->last_id = tx_last;
tx_id = txe->next_id;
txe = txn;
}
m_seg = tx_pkt;
mb_seg = mb;
do {
txd = &txr[tx_id];
txn = &sw_ring[txe->next_id];
ddesc = (volatile struct iavf_tx_desc *)
&txr[desc_idx];
txn = &txe_ring[txe->next_id];
RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
if (txe->mbuf)
rte_pktmbuf_free_seg(txe->mbuf);
txe->mbuf = m_seg;
/* Setup TX Descriptor */
slen = m_seg->data_len;
buf_dma_addr = rte_mbuf_data_iova(m_seg);
txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
txd->cmd_type_offset_bsz = iavf_build_ctob(td_cmd,
td_offset,
slen,
td_tag);
txe->mbuf = mb_seg;
iavf_fill_data_desc(ddesc, mb_seg,
ddesc_template, tlen, ipseclen);
IAVF_DUMP_TX_DESC(txq, txd, tx_id);
txe->last_id = tx_last;
tx_id = txe->next_id;
IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx);
txe->last_id = desc_idx_last;
desc_idx = txe->next_id;
txe = txn;
m_seg = m_seg->next;
} while (m_seg);
mb_seg = mb_seg->next;
} while (mb_seg);
/* The last packet data descriptor needs End Of Packet (EOP) */
td_cmd |= IAVF_TX_DESC_CMD_EOP;
txq->nb_used = (uint16_t)(txq->nb_used + nb_used);
txq->nb_free = (uint16_t)(txq->nb_free - nb_used);
ddesc_cmd = IAVF_TX_DESC_CMD_EOP;
txq->nb_used = (uint16_t)(txq->nb_used + nb_desc_required);
txq->nb_free = (uint16_t)(txq->nb_free - nb_desc_required);
if (txq->nb_used >= txq->rs_thresh) {
PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id="
"%4u (port=%d queue=%d)",
tx_last, txq->port_id, txq->queue_id);
desc_idx_last, txq->port_id, txq->queue_id);
td_cmd |= IAVF_TX_DESC_CMD_RS;
ddesc_cmd |= IAVF_TX_DESC_CMD_RS;
/* Update txq RS bit counters */
txq->nb_used = 0;
}
txd->cmd_type_offset_bsz |=
rte_cpu_to_le_64(((uint64_t)td_cmd) <<
IAVF_TXD_QW1_CMD_SHIFT);
IAVF_DUMP_TX_DESC(txq, txd, tx_id);
ddesc->cmd_type_offset_bsz |= rte_cpu_to_le_64(ddesc_cmd <<
IAVF_TXD_DATA_QW1_CMD_SHIFT);
IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx - 1);
}
end_of_tx:
rte_wmb();
PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
txq->port_id, txq->queue_id, tx_id, nb_tx);
txq->port_id, txq->queue_id, desc_idx, idx);
IAVF_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id);
txq->tx_tail = tx_id;
IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, desc_idx);
txq->tx_tail = desc_idx;
return nb_tx;
return idx;
}
/* Check if the packet with vlan user priority is transmitted in the

117
drivers/net/iavf/iavf_rxtx.h
View File

@ -403,6 +403,112 @@ enum iavf_rx_flex_desc_status_error_1_bits {
IAVF_RX_FLEX_DESC_STATUS1_LAST /* this entry must be last!!! */
};
#define IAVF_TXD_DATA_QW1_DTYPE_SHIFT (0)
#define IAVF_TXD_DATA_QW1_DTYPE_MASK (0xFUL << IAVF_TXD_QW1_DTYPE_SHIFT)
#define IAVF_TXD_DATA_QW1_CMD_SHIFT (4)
#define IAVF_TXD_DATA_QW1_CMD_MASK (0x3FFUL << IAVF_TXD_DATA_QW1_CMD_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_SHIFT (16)
#define IAVF_TXD_DATA_QW1_OFFSET_MASK (0x3FFFFULL << \
IAVF_TXD_DATA_QW1_OFFSET_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_MACLEN_SHIFT (IAVF_TXD_DATA_QW1_OFFSET_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_MACLEN_MASK \
(0x7FUL << IAVF_TXD_DATA_QW1_OFFSET_MACLEN_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_IPLEN_SHIFT \
(IAVF_TXD_DATA_QW1_OFFSET_SHIFT + IAVF_TX_DESC_LENGTH_IPLEN_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_IPLEN_MASK \
(0x7FUL << IAVF_TXD_DATA_QW1_OFFSET_IPLEN_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_L4LEN_SHIFT \
(IAVF_TXD_DATA_QW1_OFFSET_SHIFT + IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT)
#define IAVF_TXD_DATA_QW1_OFFSET_L4LEN_MASK \
(0xFUL << IAVF_TXD_DATA_QW1_OFFSET_L4LEN_SHIFT)
#define IAVF_TXD_DATA_QW1_MACLEN_MASK \
(0x7FUL << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT)
#define IAVF_TXD_DATA_QW1_IPLEN_MASK \
(0x7FUL << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT)
#define IAVF_TXD_DATA_QW1_L4LEN_MASK \
(0xFUL << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT)
#define IAVF_TXD_DATA_QW1_FCLEN_MASK \
(0xFUL << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT)
#define IAVF_TXD_DATA_QW1_TX_BUF_SZ_SHIFT (34)
#define IAVF_TXD_DATA_QW1_TX_BUF_SZ_MASK \
(0x3FFFULL << IAVF_TXD_DATA_QW1_TX_BUF_SZ_SHIFT)
#define IAVF_TXD_DATA_QW1_L2TAG1_SHIFT (48)
#define IAVF_TXD_DATA_QW1_L2TAG1_MASK \
(0xFFFFULL << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT)
#define IAVF_TXD_CTX_QW1_IPSEC_PARAMS_CIPHERBLK_SHIFT (11)
#define IAVF_TXD_CTX_QW1_IPSEC_PARAMS_CIPHERBLK_MASK \
(0x7UL << IAVF_TXD_CTX_QW1_IPSEC_PARAMS_CIPHERBLK_SHIFT)
#define IAVF_TXD_CTX_QW1_IPSEC_PARAMS_ICVLEN_SHIFT (14)
#define IAVF_TXD_CTX_QW1_IPSEC_PARAMS_ICVLEN_MASK \
(0xFUL << IAVF_TXD_CTX_QW1_IPSEC_PARAMS_ICVLEN_SHIFT)
#define IAVF_TXD_CTX_QW1_SEG_PARAMS_TLEN_SHIFT (30)
#define IAVF_TXD_CTX_QW1_SEG_PARAMS_TLEN_MASK \
(0x3FFFFUL << IAVF_TXD_CTX_QW1_SEG_PARAMS_TLEN_SHIFT)
#define IAVF_TXD_CTX_QW1_TSYNC_PARAMS_TLEN_SHIFT (30)
#define IAVF_TXD_CTX_QW1_TSYNC_PARAMS_TLEN_MASK \
(0x3FUL << IAVF_TXD_CTX_QW1_SEG_PARAMS_TLEN_SHIFT)
#define IAVF_TXD_CTX_QW1_SEG_PARAMS_MSS_SHIFT (50)
#define IAVF_TXD_CTX_QW1_SEG_PARAMS_MSS_MASK \
(0x3FFFUL << IAVF_TXD_CTX_QW1_SEG_PARAMS_MSS_SHIFT)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_SHIFT (0)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_MASK (0x3UL)
enum iavf_tx_ctx_desc_tunnel_external_ip_type {
IAVF_TX_CTX_DESC_EIPT_NONE,
IAVF_TX_CTX_DESC_EIPT_IPV6,
IAVF_TX_CTX_DESC_EIPT_IPV4_NO_CHECKSUM_OFFLOAD,
IAVF_TX_CTX_DESC_EIPT_IPV4_CHECKSUM_OFFLOAD
};
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_SHIFT (2)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_MASK (0x7FUL)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4TUNT_SHIFT (9)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4TUNT_MASK (0x3UL)
enum iavf_tx_ctx_desc_tunnel_l4_tunnel_type {
IAVF_TX_CTX_DESC_L4_TUN_TYP_NO_UDP_GRE,
IAVF_TX_CTX_DESC_L4_TUN_TYP_UDP,
IAVF_TX_CTX_DESC_L4_TUN_TYP_GRE
};
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_SHIFT (11)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_MASK (0x1UL)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4TUNLEN_SHIFT (12)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4TUNLEN_MASK (0x7FUL)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_DECTTL_SHIFT (19)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_DECTTL_MASK (0xFUL)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4T_CS_SHIFT (23)
#define IAVF_TXD_CTX_QW0_TUN_PARAMS_L4T_CS_MASK (0x1UL)
#define IAVF_TXD_CTX_QW0_L2TAG2_PARAM (32)
#define IAVF_TXD_CTX_QW0_L2TAG2_MASK (0xFFFFUL)
#define IAVF_RX_FLEX_DESC_IPSEC_CRYPTO_SAID_MASK (0xFFFFF)
/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF) /* 10-bits */
/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF) /* 10-bits */
@ -553,9 +659,10 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
const volatile struct iavf_tx_desc *tx_desc = desc;
enum iavf_tx_desc_dtype_value type;
type = (enum iavf_tx_desc_dtype_value)rte_le_to_cpu_64(
tx_desc->cmd_type_offset_bsz &
rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK));
type = (enum iavf_tx_desc_dtype_value)
rte_le_to_cpu_64(tx_desc->cmd_type_offset_bsz &
rte_cpu_to_le_64(IAVF_TXD_DATA_QW1_DTYPE_MASK));
switch (type) {
case IAVF_TX_DESC_DTYPE_DATA:
name = "Tx_data_desc";
@ -569,8 +676,8 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
}
printf("Queue %d %s %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n",
txq->queue_id, name, tx_id, tx_desc->buffer_addr,
tx_desc->cmd_type_offset_bsz);
txq->queue_id, name, tx_id, tx_desc->buffer_addr,
tx_desc->cmd_type_offset_bsz);
}
#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \

10
drivers/net/iavf/iavf_rxtx_vec_sse.c
View File

@ -363,10 +363,12 @@ static inline void
flex_desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts,
const uint32_t *type_table)
{
const __m128i ptype_mask = _mm_set_epi16(0, IAVF_RX_FLEX_DESC_PTYPE_M,
0, IAVF_RX_FLEX_DESC_PTYPE_M,
0, IAVF_RX_FLEX_DESC_PTYPE_M,
0, IAVF_RX_FLEX_DESC_PTYPE_M);
const __m128i ptype_mask =
_mm_set_epi16(IAVF_RX_FLEX_DESC_PTYPE_M, 0x0,
IAVF_RX_FLEX_DESC_PTYPE_M, 0x0,
IAVF_RX_FLEX_DESC_PTYPE_M, 0x0,
IAVF_RX_FLEX_DESC_PTYPE_M, 0x0);
__m128i ptype_01 = _mm_unpacklo_epi32(descs[0], descs[1]);
__m128i ptype_23 = _mm_unpacklo_epi32(descs[2], descs[3]);
__m128i ptype_all = _mm_unpacklo_epi64(ptype_01, ptype_23);