The logic that sets iri_vtag and M_VLANTAG does not handle the case where the 802.11q VLAN tag is 0. Fix this issue across the iflib drivers. While there, also improve and align the VLAN tag check extraction, by moving it outside the RX descriptor loop, eliminating a local variable and additional checks. PR: 260068 Reviewed by: kbowling, gallatin Reported by: erj MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D33156
965 lines
26 KiB
C
965 lines
26 KiB
C
/******************************************************************************
|
|
|
|
Copyright (c) 2013-2018, Intel Corporation
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions are met:
|
|
|
|
1. Redistributions of source code must retain the above copyright notice,
|
|
this list of conditions and the following disclaimer.
|
|
|
|
2. Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in the
|
|
documentation and/or other materials provided with the distribution.
|
|
|
|
3. Neither the name of the Intel Corporation nor the names of its
|
|
contributors may be used to endorse or promote products derived from
|
|
this software without specific prior written permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
******************************************************************************/
|
|
/*$FreeBSD$*/
|
|
|
|
/*
|
|
** IXL driver TX/RX Routines:
|
|
** This was seperated to allow usage by
|
|
** both the PF and VF drivers.
|
|
*/
|
|
|
|
#ifndef IXL_STANDALONE_BUILD
|
|
#include "opt_inet.h"
|
|
#include "opt_inet6.h"
|
|
#include "opt_rss.h"
|
|
#endif
|
|
|
|
#include "ixl.h"
|
|
|
|
#ifdef RSS
|
|
#include <net/rss_config.h>
|
|
#endif
|
|
|
|
/* Local Prototypes */
|
|
static u8 ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype);
|
|
|
|
static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi);
|
|
static void ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
|
|
static int ixl_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear);
|
|
static int ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear);
|
|
|
|
static void ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru);
|
|
static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
|
|
qidx_t pidx);
|
|
static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
|
|
qidx_t budget);
|
|
static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
|
|
|
|
struct if_txrx ixl_txrx_hwb = {
|
|
ixl_isc_txd_encap,
|
|
ixl_isc_txd_flush,
|
|
ixl_isc_txd_credits_update_hwb,
|
|
ixl_isc_rxd_available,
|
|
ixl_isc_rxd_pkt_get,
|
|
ixl_isc_rxd_refill,
|
|
ixl_isc_rxd_flush,
|
|
NULL
|
|
};
|
|
|
|
struct if_txrx ixl_txrx_dwb = {
|
|
ixl_isc_txd_encap,
|
|
ixl_isc_txd_flush,
|
|
ixl_isc_txd_credits_update_dwb,
|
|
ixl_isc_rxd_available,
|
|
ixl_isc_rxd_pkt_get,
|
|
ixl_isc_rxd_refill,
|
|
ixl_isc_rxd_flush,
|
|
NULL
|
|
};
|
|
|
|
/*
|
|
* @key key is saved into this parameter
|
|
*/
|
|
void
|
|
ixl_get_default_rss_key(u32 *key)
|
|
{
|
|
MPASS(key != NULL);
|
|
|
|
u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687,
|
|
0x183cfd8c, 0xce880440, 0x580cbc3c,
|
|
0x35897377, 0x328b25e1, 0x4fa98922,
|
|
0xb7d90c14, 0xd5bad70d, 0xcd15a2c1,
|
|
0x0, 0x0, 0x0};
|
|
|
|
bcopy(rss_seed, key, IXL_RSS_KEY_SIZE);
|
|
}
|
|
|
|
/**
|
|
* i40e_vc_stat_str - convert virtchnl status err code to a string
|
|
* @hw: pointer to the HW structure
|
|
* @stat_err: the status error code to convert
|
|
**/
|
|
const char *
|
|
i40e_vc_stat_str(struct i40e_hw *hw, enum virtchnl_status_code stat_err)
|
|
{
|
|
switch (stat_err) {
|
|
case VIRTCHNL_STATUS_SUCCESS:
|
|
return "OK";
|
|
case VIRTCHNL_ERR_PARAM:
|
|
return "VIRTCHNL_ERR_PARAM";
|
|
case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
|
|
return "VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH";
|
|
case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
|
|
return "VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR";
|
|
case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
|
|
return "VIRTCHNL_STATUS_ERR_INVALID_VF_ID";
|
|
case VIRTCHNL_STATUS_NOT_SUPPORTED:
|
|
return "VIRTCHNL_STATUS_NOT_SUPPORTED";
|
|
}
|
|
|
|
snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err);
|
|
return hw->err_str;
|
|
}
|
|
|
|
void
|
|
ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...)
|
|
{
|
|
va_list args;
|
|
|
|
if (!(mask & enabled_mask))
|
|
return;
|
|
|
|
/* Re-implement device_printf() */
|
|
device_print_prettyname(dev);
|
|
va_start(args, fmt);
|
|
vprintf(fmt, args);
|
|
va_end(args);
|
|
}
|
|
|
|
static bool
|
|
ixl_is_tx_desc_done(struct tx_ring *txr, int idx)
|
|
{
|
|
return (((txr->tx_base[idx].cmd_type_offset_bsz >> I40E_TXD_QW1_DTYPE_SHIFT)
|
|
& I40E_TXD_QW1_DTYPE_MASK) == I40E_TX_DESC_DTYPE_DESC_DONE);
|
|
}
|
|
|
|
static int
|
|
ixl_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, if_pkt_info_t pi)
|
|
{
|
|
int count, curseg, i, hlen, segsz, seglen, tsolen;
|
|
|
|
if (nsegs <= IXL_MAX_TX_SEGS-2)
|
|
return (0);
|
|
segsz = pi->ipi_tso_segsz;
|
|
curseg = count = 0;
|
|
|
|
hlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
|
|
tsolen = pi->ipi_len - hlen;
|
|
|
|
i = 0;
|
|
curseg = segs[0].ds_len;
|
|
while (hlen > 0) {
|
|
count++;
|
|
if (count > IXL_MAX_TX_SEGS - 2)
|
|
return (1);
|
|
if (curseg == 0) {
|
|
i++;
|
|
if (__predict_false(i == nsegs))
|
|
return (1);
|
|
|
|
curseg = segs[i].ds_len;
|
|
}
|
|
seglen = min(curseg, hlen);
|
|
curseg -= seglen;
|
|
hlen -= seglen;
|
|
// printf("H:seglen = %d, count=%d\n", seglen, count);
|
|
}
|
|
while (tsolen > 0) {
|
|
segsz = pi->ipi_tso_segsz;
|
|
while (segsz > 0 && tsolen != 0) {
|
|
count++;
|
|
if (count > IXL_MAX_TX_SEGS - 2) {
|
|
// printf("bad: count = %d\n", count);
|
|
return (1);
|
|
}
|
|
if (curseg == 0) {
|
|
i++;
|
|
if (__predict_false(i == nsegs)) {
|
|
// printf("bad: tsolen = %d", tsolen);
|
|
return (1);
|
|
}
|
|
curseg = segs[i].ds_len;
|
|
}
|
|
seglen = min(curseg, segsz);
|
|
segsz -= seglen;
|
|
curseg -= seglen;
|
|
tsolen -= seglen;
|
|
// printf("D:seglen = %d, count=%d\n", seglen, count);
|
|
}
|
|
count = 0;
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* Setup descriptor for hw offloads
|
|
*
|
|
**********************************************************************/
|
|
|
|
static void
|
|
ixl_tx_setup_offload(struct ixl_tx_queue *que,
|
|
if_pkt_info_t pi, u32 *cmd, u32 *off)
|
|
{
|
|
switch (pi->ipi_etype) {
|
|
#ifdef INET
|
|
case ETHERTYPE_IP:
|
|
if (pi->ipi_csum_flags & IXL_CSUM_IPV4)
|
|
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
|
|
else
|
|
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
|
|
break;
|
|
#endif
|
|
#ifdef INET6
|
|
case ETHERTYPE_IPV6:
|
|
*cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
|
|
break;
|
|
#endif
|
|
default:
|
|
break;
|
|
}
|
|
|
|
*off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
|
|
*off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
|
|
|
|
switch (pi->ipi_ipproto) {
|
|
case IPPROTO_TCP:
|
|
if (pi->ipi_csum_flags & IXL_CSUM_TCP) {
|
|
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
|
|
*off |= (pi->ipi_tcp_hlen >> 2) <<
|
|
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
|
|
/* Check for NO_HEAD MDD event */
|
|
MPASS(pi->ipi_tcp_hlen != 0);
|
|
}
|
|
break;
|
|
case IPPROTO_UDP:
|
|
if (pi->ipi_csum_flags & IXL_CSUM_UDP) {
|
|
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
|
|
*off |= (sizeof(struct udphdr) >> 2) <<
|
|
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
|
|
}
|
|
break;
|
|
case IPPROTO_SCTP:
|
|
if (pi->ipi_csum_flags & IXL_CSUM_SCTP) {
|
|
*cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
|
|
*off |= (sizeof(struct sctphdr) >> 2) <<
|
|
I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
|
|
}
|
|
/* Fall Thru */
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**********************************************************************
|
|
*
|
|
* Setup context for hardware segmentation offload (TSO)
|
|
*
|
|
**********************************************************************/
|
|
static int
|
|
ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi)
|
|
{
|
|
if_softc_ctx_t scctx;
|
|
struct i40e_tx_context_desc *TXD;
|
|
u32 cmd, mss, type, tsolen;
|
|
int idx, total_hdr_len;
|
|
u64 type_cmd_tso_mss;
|
|
|
|
idx = pi->ipi_pidx;
|
|
TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx];
|
|
total_hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
|
|
tsolen = pi->ipi_len - total_hdr_len;
|
|
scctx = txr->que->vsi->shared;
|
|
|
|
type = I40E_TX_DESC_DTYPE_CONTEXT;
|
|
cmd = I40E_TX_CTX_DESC_TSO;
|
|
/*
|
|
* TSO MSS must not be less than 64; this prevents a
|
|
* BAD_LSO_MSS MDD event when the MSS is too small.
|
|
*/
|
|
if (pi->ipi_tso_segsz < IXL_MIN_TSO_MSS) {
|
|
txr->mss_too_small++;
|
|
pi->ipi_tso_segsz = IXL_MIN_TSO_MSS;
|
|
}
|
|
mss = pi->ipi_tso_segsz;
|
|
|
|
/* Check for BAD_LS0_MSS MDD event (mss too large) */
|
|
MPASS(mss <= IXL_MAX_TSO_MSS);
|
|
/* Check for NO_HEAD MDD event (header lengths are 0) */
|
|
MPASS(pi->ipi_ehdrlen != 0);
|
|
MPASS(pi->ipi_ip_hlen != 0);
|
|
/* Partial check for BAD_LSO_LEN MDD event */
|
|
MPASS(tsolen != 0);
|
|
/* Partial check for WRONG_SIZE MDD event (during TSO) */
|
|
MPASS(total_hdr_len + mss <= IXL_MAX_FRAME);
|
|
|
|
type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) |
|
|
((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
|
|
((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
|
|
((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
|
|
TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss);
|
|
|
|
TXD->tunneling_params = htole32(0);
|
|
txr->que->tso++;
|
|
|
|
return ((idx + 1) & (scctx->isc_ntxd[0]-1));
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* This routine maps the mbufs to tx descriptors, allowing the
|
|
* TX engine to transmit the packets.
|
|
* - return 0 on success, positive on failure
|
|
*
|
|
**********************************************************************/
|
|
#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
|
|
|
|
static int
|
|
ixl_isc_txd_encap(void *arg, if_pkt_info_t pi)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct ixl_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx];
|
|
struct tx_ring *txr = &que->txr;
|
|
int nsegs = pi->ipi_nsegs;
|
|
bus_dma_segment_t *segs = pi->ipi_segs;
|
|
struct i40e_tx_desc *txd = NULL;
|
|
int i, j, mask, pidx_last;
|
|
u32 cmd, off, tx_intr;
|
|
|
|
cmd = off = 0;
|
|
i = pi->ipi_pidx;
|
|
|
|
tx_intr = (pi->ipi_flags & IPI_TX_INTR);
|
|
|
|
/* Set up the TSO/CSUM offload */
|
|
if (pi->ipi_csum_flags & CSUM_OFFLOAD) {
|
|
/* Set up the TSO context descriptor if required */
|
|
if (pi->ipi_csum_flags & CSUM_TSO) {
|
|
/* Prevent MAX_BUFF MDD event (for TSO) */
|
|
if (ixl_tso_detect_sparse(segs, nsegs, pi))
|
|
return (EFBIG);
|
|
i = ixl_tso_setup(txr, pi);
|
|
}
|
|
ixl_tx_setup_offload(que, pi, &cmd, &off);
|
|
}
|
|
if (pi->ipi_mflags & M_VLANTAG)
|
|
cmd |= I40E_TX_DESC_CMD_IL2TAG1;
|
|
|
|
cmd |= I40E_TX_DESC_CMD_ICRC;
|
|
mask = scctx->isc_ntxd[0] - 1;
|
|
/* Check for WRONG_SIZE MDD event */
|
|
MPASS(pi->ipi_len >= IXL_MIN_FRAME);
|
|
#ifdef INVARIANTS
|
|
if (!(pi->ipi_csum_flags & CSUM_TSO))
|
|
MPASS(pi->ipi_len <= IXL_MAX_FRAME);
|
|
#endif
|
|
for (j = 0; j < nsegs; j++) {
|
|
bus_size_t seglen;
|
|
|
|
txd = &txr->tx_base[i];
|
|
seglen = segs[j].ds_len;
|
|
|
|
/* Check for ZERO_BSIZE MDD event */
|
|
MPASS(seglen != 0);
|
|
|
|
txd->buffer_addr = htole64(segs[j].ds_addr);
|
|
txd->cmd_type_offset_bsz =
|
|
htole64(I40E_TX_DESC_DTYPE_DATA
|
|
| ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT)
|
|
| ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT)
|
|
| ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)
|
|
| ((u64)htole16(pi->ipi_vtag) << I40E_TXD_QW1_L2TAG1_SHIFT));
|
|
|
|
txr->tx_bytes += seglen;
|
|
pidx_last = i;
|
|
i = (i+1) & mask;
|
|
}
|
|
/* Set the last descriptor for report */
|
|
txd->cmd_type_offset_bsz |=
|
|
htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT));
|
|
/* Add to report status array (if using TX interrupts) */
|
|
if (!vsi->enable_head_writeback && tx_intr) {
|
|
txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
|
|
txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & mask;
|
|
MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
|
|
}
|
|
pi->ipi_new_pidx = i;
|
|
|
|
++txr->tx_packets;
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
struct tx_ring *txr = &vsi->tx_queues[txqid].txr;
|
|
|
|
/*
|
|
* Advance the Transmit Descriptor Tail (Tdt), this tells the
|
|
* hardware that this frame is available to transmit.
|
|
*/
|
|
/* Check for ENDLESS_TX MDD event */
|
|
MPASS(pidx < vsi->shared->isc_ntxd[0]);
|
|
wr32(vsi->hw, txr->tail, pidx);
|
|
}
|
|
|
|
|
|
/*********************************************************************
|
|
*
|
|
* (Re)Initialize a queue transmit ring by clearing its memory.
|
|
*
|
|
**********************************************************************/
|
|
void
|
|
ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que)
|
|
{
|
|
struct tx_ring *txr = &que->txr;
|
|
|
|
/* Clear the old ring contents */
|
|
bzero((void *)txr->tx_base,
|
|
(sizeof(struct i40e_tx_desc)) *
|
|
(vsi->shared->isc_ntxd[0] + (vsi->enable_head_writeback ? 1 : 0)));
|
|
|
|
wr32(vsi->hw, txr->tail, 0);
|
|
}
|
|
|
|
/*
|
|
* ixl_get_tx_head - Retrieve the value from the
|
|
* location the HW records its HEAD index
|
|
*/
|
|
static inline u32
|
|
ixl_get_tx_head(struct ixl_tx_queue *que)
|
|
{
|
|
if_softc_ctx_t scctx = que->vsi->shared;
|
|
struct tx_ring *txr = &que->txr;
|
|
void *head = &txr->tx_base[scctx->isc_ntxd[0]];
|
|
|
|
return LE32_TO_CPU(*(volatile __le32 *)head);
|
|
}
|
|
|
|
static int
|
|
ixl_isc_txd_credits_update_hwb(void *arg, uint16_t qid, bool clear)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct ixl_tx_queue *que = &vsi->tx_queues[qid];
|
|
struct tx_ring *txr = &que->txr;
|
|
int head, credits;
|
|
|
|
/* Get the Head WB value */
|
|
head = ixl_get_tx_head(que);
|
|
|
|
credits = head - txr->tx_cidx_processed;
|
|
if (credits < 0)
|
|
credits += scctx->isc_ntxd[0];
|
|
if (clear)
|
|
txr->tx_cidx_processed = head;
|
|
|
|
return (credits);
|
|
}
|
|
|
|
static int
|
|
ixl_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid];
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct tx_ring *txr = &tx_que->txr;
|
|
|
|
qidx_t processed = 0;
|
|
qidx_t cur, prev, ntxd, rs_cidx;
|
|
int32_t delta;
|
|
bool is_done;
|
|
|
|
rs_cidx = txr->tx_rs_cidx;
|
|
#if 0
|
|
device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) rs_cidx %d, txr->tx_rs_pidx %d\n", __func__,
|
|
txr->me, rs_cidx, txr->tx_rs_pidx);
|
|
#endif
|
|
if (rs_cidx == txr->tx_rs_pidx)
|
|
return (0);
|
|
cur = txr->tx_rsq[rs_cidx];
|
|
MPASS(cur != QIDX_INVALID);
|
|
is_done = ixl_is_tx_desc_done(txr, cur);
|
|
|
|
if (!is_done)
|
|
return (0);
|
|
|
|
/* If clear is false just let caller know that there
|
|
* are descriptors to reclaim */
|
|
if (!clear)
|
|
return (1);
|
|
|
|
prev = txr->tx_cidx_processed;
|
|
ntxd = scctx->isc_ntxd[0];
|
|
do {
|
|
MPASS(prev != cur);
|
|
delta = (int32_t)cur - (int32_t)prev;
|
|
if (delta < 0)
|
|
delta += ntxd;
|
|
MPASS(delta > 0);
|
|
#if 0
|
|
device_printf(iflib_get_dev(vsi->ctx),
|
|
"%s: (q%d) cidx_processed=%u cur=%u clear=%d delta=%d\n",
|
|
__func__, txr->me, prev, cur, clear, delta);
|
|
#endif
|
|
processed += delta;
|
|
prev = cur;
|
|
rs_cidx = (rs_cidx + 1) & (ntxd-1);
|
|
if (rs_cidx == txr->tx_rs_pidx)
|
|
break;
|
|
cur = txr->tx_rsq[rs_cidx];
|
|
MPASS(cur != QIDX_INVALID);
|
|
is_done = ixl_is_tx_desc_done(txr, cur);
|
|
} while (is_done);
|
|
|
|
txr->tx_rs_cidx = rs_cidx;
|
|
txr->tx_cidx_processed = prev;
|
|
|
|
#if 0
|
|
device_printf(iflib_get_dev(vsi->ctx), "%s: (q%d) processed %d\n", __func__, txr->me, processed);
|
|
#endif
|
|
return (processed);
|
|
}
|
|
|
|
static void
|
|
ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct rx_ring *rxr = &((vsi->rx_queues[iru->iru_qsidx]).rxr);
|
|
uint64_t *paddrs;
|
|
uint32_t next_pidx, pidx;
|
|
uint16_t count;
|
|
int i;
|
|
|
|
paddrs = iru->iru_paddrs;
|
|
pidx = iru->iru_pidx;
|
|
count = iru->iru_count;
|
|
|
|
for (i = 0, next_pidx = pidx; i < count; i++) {
|
|
rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
|
|
if (++next_pidx == scctx->isc_nrxd[0])
|
|
next_pidx = 0;
|
|
}
|
|
}
|
|
|
|
static void
|
|
ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr;
|
|
|
|
wr32(vsi->hw, rxr->tail, pidx);
|
|
}
|
|
|
|
static int
|
|
ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr;
|
|
union i40e_rx_desc *rxd;
|
|
u64 qword;
|
|
uint32_t status;
|
|
int cnt, i, nrxd;
|
|
|
|
nrxd = vsi->shared->isc_nrxd[0];
|
|
|
|
for (cnt = 0, i = idx; cnt < nrxd - 1 && cnt <= budget;) {
|
|
rxd = &rxr->rx_base[i];
|
|
qword = le64toh(rxd->wb.qword1.status_error_len);
|
|
status = (qword & I40E_RXD_QW1_STATUS_MASK)
|
|
>> I40E_RXD_QW1_STATUS_SHIFT;
|
|
|
|
if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0)
|
|
break;
|
|
if (++i == nrxd)
|
|
i = 0;
|
|
if (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))
|
|
cnt++;
|
|
}
|
|
|
|
return (cnt);
|
|
}
|
|
|
|
/*
|
|
** i40e_ptype_to_hash: parse the packet type
|
|
** to determine the appropriate hash.
|
|
*/
|
|
static inline int
|
|
ixl_ptype_to_hash(u8 ptype)
|
|
{
|
|
struct i40e_rx_ptype_decoded decoded;
|
|
|
|
decoded = decode_rx_desc_ptype(ptype);
|
|
|
|
if (!decoded.known)
|
|
return M_HASHTYPE_OPAQUE;
|
|
|
|
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2)
|
|
return M_HASHTYPE_OPAQUE;
|
|
|
|
/* Note: anything that gets to this point is IP */
|
|
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) {
|
|
switch (decoded.inner_prot) {
|
|
case I40E_RX_PTYPE_INNER_PROT_TCP:
|
|
return M_HASHTYPE_RSS_TCP_IPV6;
|
|
case I40E_RX_PTYPE_INNER_PROT_UDP:
|
|
return M_HASHTYPE_RSS_UDP_IPV6;
|
|
default:
|
|
return M_HASHTYPE_RSS_IPV6;
|
|
}
|
|
}
|
|
if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) {
|
|
switch (decoded.inner_prot) {
|
|
case I40E_RX_PTYPE_INNER_PROT_TCP:
|
|
return M_HASHTYPE_RSS_TCP_IPV4;
|
|
case I40E_RX_PTYPE_INNER_PROT_UDP:
|
|
return M_HASHTYPE_RSS_UDP_IPV4;
|
|
default:
|
|
return M_HASHTYPE_RSS_IPV4;
|
|
}
|
|
}
|
|
/* We should never get here!! */
|
|
return M_HASHTYPE_OPAQUE;
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* This routine executes in ithread context. It sends data which has been
|
|
* dma'ed into host memory to upper layer.
|
|
*
|
|
* Returns 0 upon success, errno on failure
|
|
*
|
|
*********************************************************************/
|
|
static int
|
|
ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
|
|
{
|
|
struct ixl_vsi *vsi = arg;
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct ixl_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx];
|
|
struct rx_ring *rxr = &que->rxr;
|
|
union i40e_rx_desc *cur;
|
|
u32 status, error;
|
|
u16 plen;
|
|
u64 qword;
|
|
u8 ptype;
|
|
bool eop;
|
|
int i, cidx;
|
|
|
|
cidx = ri->iri_cidx;
|
|
i = 0;
|
|
do {
|
|
/* 5 descriptor receive limit */
|
|
MPASS(i < IXL_MAX_RX_SEGS);
|
|
|
|
cur = &rxr->rx_base[cidx];
|
|
qword = le64toh(cur->wb.qword1.status_error_len);
|
|
status = (qword & I40E_RXD_QW1_STATUS_MASK)
|
|
>> I40E_RXD_QW1_STATUS_SHIFT;
|
|
error = (qword & I40E_RXD_QW1_ERROR_MASK)
|
|
>> I40E_RXD_QW1_ERROR_SHIFT;
|
|
plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
|
|
>> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
|
|
ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
|
|
>> I40E_RXD_QW1_PTYPE_SHIFT;
|
|
|
|
/* we should never be called without a valid descriptor */
|
|
MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0);
|
|
|
|
ri->iri_len += plen;
|
|
rxr->rx_bytes += plen;
|
|
|
|
cur->wb.qword1.status_error_len = 0;
|
|
eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
|
|
|
|
/*
|
|
** Make sure bad packets are discarded,
|
|
** note that only EOP descriptor has valid
|
|
** error results.
|
|
*/
|
|
if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
|
|
rxr->desc_errs++;
|
|
return (EBADMSG);
|
|
}
|
|
ri->iri_frags[i].irf_flid = 0;
|
|
ri->iri_frags[i].irf_idx = cidx;
|
|
ri->iri_frags[i].irf_len = plen;
|
|
if (++cidx == vsi->shared->isc_nrxd[0])
|
|
cidx = 0;
|
|
i++;
|
|
} while (!eop);
|
|
|
|
/* capture data for dynamic ITR adjustment */
|
|
rxr->packets++;
|
|
rxr->rx_packets++;
|
|
|
|
if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
|
|
rxr->csum_errs += ixl_rx_checksum(ri, status, error, ptype);
|
|
ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss);
|
|
ri->iri_rsstype = ixl_ptype_to_hash(ptype);
|
|
if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
|
|
ri->iri_vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
|
|
ri->iri_flags |= M_VLANTAG;
|
|
}
|
|
ri->iri_nfrags = i;
|
|
return (0);
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* Verify that the hardware indicated that the checksum is valid.
|
|
* Inform the stack about the status of checksum so that stack
|
|
* doesn't spend time verifying the checksum.
|
|
*
|
|
*********************************************************************/
|
|
static u8
|
|
ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype)
|
|
{
|
|
struct i40e_rx_ptype_decoded decoded;
|
|
|
|
ri->iri_csum_flags = 0;
|
|
|
|
/* No L3 or L4 checksum was calculated */
|
|
if (!(status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
|
|
return (0);
|
|
|
|
decoded = decode_rx_desc_ptype(ptype);
|
|
|
|
/* IPv6 with extension headers likely have bad csum */
|
|
if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
|
|
decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) {
|
|
if (status &
|
|
(1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) {
|
|
ri->iri_csum_flags = 0;
|
|
return (1);
|
|
}
|
|
}
|
|
|
|
ri->iri_csum_flags |= CSUM_L3_CALC;
|
|
|
|
/* IPv4 checksum error */
|
|
if (error & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT))
|
|
return (1);
|
|
|
|
ri->iri_csum_flags |= CSUM_L3_VALID;
|
|
ri->iri_csum_flags |= CSUM_L4_CALC;
|
|
|
|
/* L4 checksum error */
|
|
if (error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))
|
|
return (1);
|
|
|
|
ri->iri_csum_flags |= CSUM_L4_VALID;
|
|
ri->iri_csum_data |= htons(0xffff);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/* Set Report Status queue fields to 0 */
|
|
void
|
|
ixl_init_tx_rsqs(struct ixl_vsi *vsi)
|
|
{
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct ixl_tx_queue *tx_que;
|
|
int i, j;
|
|
|
|
for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) {
|
|
struct tx_ring *txr = &tx_que->txr;
|
|
|
|
txr->tx_rs_cidx = txr->tx_rs_pidx;
|
|
|
|
/* Initialize the last processed descriptor to be the end of
|
|
* the ring, rather than the start, so that we avoid an
|
|
* off-by-one error when calculating how many descriptors are
|
|
* done in the credits_update function.
|
|
*/
|
|
txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
|
|
|
|
for (j = 0; j < scctx->isc_ntxd[0]; j++)
|
|
txr->tx_rsq[j] = QIDX_INVALID;
|
|
}
|
|
}
|
|
|
|
void
|
|
ixl_init_tx_cidx(struct ixl_vsi *vsi)
|
|
{
|
|
if_softc_ctx_t scctx = vsi->shared;
|
|
struct ixl_tx_queue *tx_que;
|
|
int i;
|
|
|
|
for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) {
|
|
struct tx_ring *txr = &tx_que->txr;
|
|
|
|
txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Input: bitmap of enum virtchnl_link_speed
|
|
*/
|
|
u64
|
|
ixl_max_vc_speed_to_value(u8 link_speeds)
|
|
{
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_40GB)
|
|
return IF_Gbps(40);
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_25GB)
|
|
return IF_Gbps(25);
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_20GB)
|
|
return IF_Gbps(20);
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_10GB)
|
|
return IF_Gbps(10);
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_1GB)
|
|
return IF_Gbps(1);
|
|
if (link_speeds & VIRTCHNL_LINK_SPEED_100MB)
|
|
return IF_Mbps(100);
|
|
else
|
|
/* Minimum supported link speed */
|
|
return IF_Mbps(100);
|
|
}
|
|
|
|
void
|
|
ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi,
|
|
struct sysctl_ctx_list *ctx, const char *sysctl_name)
|
|
{
|
|
struct sysctl_oid *tree;
|
|
struct sysctl_oid_list *child;
|
|
struct sysctl_oid_list *vsi_list;
|
|
|
|
tree = device_get_sysctl_tree(dev);
|
|
child = SYSCTL_CHILDREN(tree);
|
|
vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
|
|
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "VSI Number");
|
|
vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
|
|
|
|
ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
|
|
}
|
|
|
|
void
|
|
ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
|
|
struct sysctl_oid_list *child,
|
|
struct i40e_eth_stats *eth_stats)
|
|
{
|
|
struct ixl_sysctl_info ctls[] =
|
|
{
|
|
{ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
|
|
{ð_stats->rx_unicast, "ucast_pkts_rcvd",
|
|
"Unicast Packets Received"},
|
|
{ð_stats->rx_multicast, "mcast_pkts_rcvd",
|
|
"Multicast Packets Received"},
|
|
{ð_stats->rx_broadcast, "bcast_pkts_rcvd",
|
|
"Broadcast Packets Received"},
|
|
{ð_stats->rx_discards, "rx_discards", "Discarded RX packets"},
|
|
{ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
|
|
{ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
|
|
{ð_stats->tx_multicast, "mcast_pkts_txd",
|
|
"Multicast Packets Transmitted"},
|
|
{ð_stats->tx_broadcast, "bcast_pkts_txd",
|
|
"Broadcast Packets Transmitted"},
|
|
// end
|
|
{0,0,0}
|
|
};
|
|
|
|
struct ixl_sysctl_info *entry = ctls;
|
|
while (entry->stat != 0)
|
|
{
|
|
SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
|
|
CTLFLAG_RD, entry->stat,
|
|
entry->description);
|
|
entry++;
|
|
}
|
|
}
|
|
|
|
void
|
|
ixl_vsi_add_queues_stats(struct ixl_vsi *vsi, struct sysctl_ctx_list *ctx)
|
|
{
|
|
struct sysctl_oid_list *vsi_list, *queue_list;
|
|
struct sysctl_oid *queue_node;
|
|
char queue_namebuf[IXL_QUEUE_NAME_LEN];
|
|
|
|
struct ixl_rx_queue *rx_que;
|
|
struct ixl_tx_queue *tx_que;
|
|
struct tx_ring *txr;
|
|
struct rx_ring *rxr;
|
|
|
|
vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
|
|
|
|
/* Queue statistics */
|
|
for (int q = 0; q < vsi->num_rx_queues; q++) {
|
|
bzero(queue_namebuf, sizeof(queue_namebuf));
|
|
snprintf(queue_namebuf, sizeof(queue_namebuf), "rxq%02d", q);
|
|
queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
|
|
OID_AUTO, queue_namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE,
|
|
NULL, "RX Queue #");
|
|
queue_list = SYSCTL_CHILDREN(queue_node);
|
|
|
|
rx_que = &(vsi->rx_queues[q]);
|
|
rxr = &(rx_que->rxr);
|
|
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
|
|
CTLFLAG_RD, &(rx_que->irqs),
|
|
"irqs on this queue (both Tx and Rx)");
|
|
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets",
|
|
CTLFLAG_RD, &(rxr->rx_packets),
|
|
"Queue Packets Received");
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes",
|
|
CTLFLAG_RD, &(rxr->rx_bytes),
|
|
"Queue Bytes Received");
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "desc_err",
|
|
CTLFLAG_RD, &(rxr->desc_errs),
|
|
"Queue Rx Descriptor Errors");
|
|
SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr",
|
|
CTLFLAG_RD, &(rxr->itr), 0,
|
|
"Queue Rx ITR Interval");
|
|
}
|
|
for (int q = 0; q < vsi->num_tx_queues; q++) {
|
|
bzero(queue_namebuf, sizeof(queue_namebuf));
|
|
snprintf(queue_namebuf, sizeof(queue_namebuf), "txq%02d", q);
|
|
queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
|
|
OID_AUTO, queue_namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE,
|
|
NULL, "TX Queue #");
|
|
queue_list = SYSCTL_CHILDREN(queue_node);
|
|
|
|
tx_que = &(vsi->tx_queues[q]);
|
|
txr = &(tx_que->txr);
|
|
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso",
|
|
CTLFLAG_RD, &(tx_que->tso),
|
|
"TSO");
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small",
|
|
CTLFLAG_RD, &(txr->mss_too_small),
|
|
"TSO sends with an MSS less than 64");
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets",
|
|
CTLFLAG_RD, &(txr->tx_packets),
|
|
"Queue Packets Transmitted");
|
|
SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes",
|
|
CTLFLAG_RD, &(txr->tx_bytes),
|
|
"Queue Bytes Transmitted");
|
|
SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr",
|
|
CTLFLAG_RD, &(txr->itr), 0,
|
|
"Queue Tx ITR Interval");
|
|
}
|
|
}
|