571 lines
16 KiB
C
571 lines
16 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*
|
|
* Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "if_em.h"
|
|
|
|
#ifdef RSS
|
|
#include <net/rss_config.h>
|
|
#include <netinet/in_rss.h>
|
|
#endif
|
|
|
|
#ifdef VERBOSE_DEBUG
|
|
#define DPRINTF device_printf
|
|
#else
|
|
#define DPRINTF(...)
|
|
#endif
|
|
|
|
/*********************************************************************
|
|
* Local Function prototypes
|
|
*********************************************************************/
|
|
static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi);
|
|
static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
|
|
static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
|
|
|
|
static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru);
|
|
|
|
static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
|
|
qidx_t pidx);
|
|
static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
|
|
qidx_t budget);
|
|
|
|
static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
|
|
|
|
static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi,
|
|
uint32_t *cmd_type_len, uint32_t *olinfo_status);
|
|
static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi,
|
|
uint32_t *cmd_type_len, uint32_t *olinfo_status);
|
|
|
|
static void igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype);
|
|
static int igb_determine_rsstype(uint16_t pkt_info);
|
|
|
|
extern void igb_if_enable_intr(if_ctx_t ctx);
|
|
extern int em_intr(void *arg);
|
|
|
|
struct if_txrx igb_txrx = {
|
|
.ift_txd_encap = igb_isc_txd_encap,
|
|
.ift_txd_flush = igb_isc_txd_flush,
|
|
.ift_txd_credits_update = igb_isc_txd_credits_update,
|
|
.ift_rxd_available = igb_isc_rxd_available,
|
|
.ift_rxd_pkt_get = igb_isc_rxd_pkt_get,
|
|
.ift_rxd_refill = igb_isc_rxd_refill,
|
|
.ift_rxd_flush = igb_isc_rxd_flush,
|
|
.ift_legacy_intr = em_intr
|
|
};
|
|
|
|
/**********************************************************************
|
|
*
|
|
* Setup work for hardware segmentation offload (TSO) on
|
|
* adapters using advanced tx descriptors
|
|
*
|
|
**********************************************************************/
|
|
static int
|
|
igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
|
|
uint32_t *olinfo_status)
|
|
{
|
|
struct e1000_adv_tx_context_desc *TXD;
|
|
struct e1000_softc *sc = txr->sc;
|
|
uint32_t type_tucmd_mlhl = 0, vlan_macip_lens = 0;
|
|
uint32_t mss_l4len_idx = 0;
|
|
uint32_t paylen;
|
|
|
|
switch(pi->ipi_etype) {
|
|
case ETHERTYPE_IPV6:
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
|
|
break;
|
|
case ETHERTYPE_IP:
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
|
|
/* Tell transmit desc to also do IPv4 checksum. */
|
|
*olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
|
|
break;
|
|
default:
|
|
panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
|
|
__func__, ntohs(pi->ipi_etype));
|
|
break;
|
|
}
|
|
|
|
TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
|
|
|
|
/* This is used in the transmit desc in encap */
|
|
paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen;
|
|
|
|
/* VLAN MACLEN IPLEN */
|
|
if (pi->ipi_mflags & M_VLANTAG) {
|
|
vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT);
|
|
}
|
|
|
|
vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
|
|
vlan_macip_lens |= pi->ipi_ip_hlen;
|
|
TXD->vlan_macip_lens = htole32(vlan_macip_lens);
|
|
|
|
/* ADV DTYPE TUCMD */
|
|
type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
|
|
TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
|
|
|
|
/* MSS L4LEN IDX */
|
|
mss_l4len_idx |= (pi->ipi_tso_segsz << E1000_ADVTXD_MSS_SHIFT);
|
|
mss_l4len_idx |= (pi->ipi_tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
|
|
/* 82575 needs the queue index added */
|
|
if (sc->hw.mac.type == e1000_82575)
|
|
mss_l4len_idx |= txr->me << 4;
|
|
TXD->mss_l4len_idx = htole32(mss_l4len_idx);
|
|
|
|
TXD->u.seqnum_seed = htole32(0);
|
|
*cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
|
|
*olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
|
|
*olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
|
|
|
|
return (1);
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* Advanced Context Descriptor setup for VLAN, CSUM or TSO
|
|
*
|
|
**********************************************************************/
|
|
static int
|
|
igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
|
|
uint32_t *olinfo_status)
|
|
{
|
|
struct e1000_adv_tx_context_desc *TXD;
|
|
struct e1000_softc *sc = txr->sc;
|
|
uint32_t vlan_macip_lens, type_tucmd_mlhl;
|
|
uint32_t mss_l4len_idx;
|
|
mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0;
|
|
|
|
/* First check if TSO is to be used */
|
|
if (pi->ipi_csum_flags & CSUM_TSO)
|
|
return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status));
|
|
|
|
/* Indicate the whole packet as payload when not doing TSO */
|
|
*olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT;
|
|
|
|
/* Now ready a context descriptor */
|
|
TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
|
|
|
|
/*
|
|
** In advanced descriptors the vlan tag must
|
|
** be placed into the context descriptor. Hence
|
|
** we need to make one even if not doing offloads.
|
|
*/
|
|
if (pi->ipi_mflags & M_VLANTAG) {
|
|
vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT);
|
|
} else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) {
|
|
return (0);
|
|
}
|
|
|
|
/* Set the ether header length */
|
|
vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
|
|
|
|
switch(pi->ipi_etype) {
|
|
case ETHERTYPE_IP:
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
|
|
break;
|
|
case ETHERTYPE_IPV6:
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
vlan_macip_lens |= pi->ipi_ip_hlen;
|
|
type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
|
|
|
|
switch (pi->ipi_ipproto) {
|
|
case IPPROTO_TCP:
|
|
if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) {
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
|
|
*olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
|
|
}
|
|
break;
|
|
case IPPROTO_UDP:
|
|
if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) {
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
|
|
*olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
|
|
}
|
|
break;
|
|
case IPPROTO_SCTP:
|
|
if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) {
|
|
type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
|
|
*olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
/* 82575 needs the queue index added */
|
|
if (sc->hw.mac.type == e1000_82575)
|
|
mss_l4len_idx = txr->me << 4;
|
|
|
|
/* Now copy bits into descriptor */
|
|
TXD->vlan_macip_lens = htole32(vlan_macip_lens);
|
|
TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
|
|
TXD->u.seqnum_seed = htole32(0);
|
|
TXD->mss_l4len_idx = htole32(mss_l4len_idx);
|
|
|
|
return (1);
|
|
}
|
|
|
|
static int
|
|
igb_isc_txd_encap(void *arg, if_pkt_info_t pi)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
if_softc_ctx_t scctx = sc->shared;
|
|
struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
|
|
struct tx_ring *txr = &que->txr;
|
|
int nsegs = pi->ipi_nsegs;
|
|
bus_dma_segment_t *segs = pi->ipi_segs;
|
|
union e1000_adv_tx_desc *txd = NULL;
|
|
int i, j, pidx_last;
|
|
uint32_t olinfo_status, cmd_type_len, txd_flags;
|
|
qidx_t ntxd;
|
|
|
|
pidx_last = olinfo_status = 0;
|
|
/* Basic descriptor defines */
|
|
cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
|
|
E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
|
|
|
|
if (pi->ipi_mflags & M_VLANTAG)
|
|
cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
|
|
|
|
i = pi->ipi_pidx;
|
|
ntxd = scctx->isc_ntxd[0];
|
|
txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_ADVTXD_DCMD_RS : 0;
|
|
/* Consume the first descriptor */
|
|
i += igb_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status);
|
|
if (i == scctx->isc_ntxd[0])
|
|
i = 0;
|
|
|
|
/* 82575 needs the queue index added */
|
|
if (sc->hw.mac.type == e1000_82575)
|
|
olinfo_status |= txr->me << 4;
|
|
|
|
for (j = 0; j < nsegs; j++) {
|
|
bus_size_t seglen;
|
|
bus_addr_t segaddr;
|
|
|
|
txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
|
|
seglen = segs[j].ds_len;
|
|
segaddr = htole64(segs[j].ds_addr);
|
|
|
|
txd->read.buffer_addr = segaddr;
|
|
txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
|
|
cmd_type_len | seglen);
|
|
txd->read.olinfo_status = htole32(olinfo_status);
|
|
pidx_last = i;
|
|
if (++i == scctx->isc_ntxd[0]) {
|
|
i = 0;
|
|
}
|
|
}
|
|
if (txd_flags) {
|
|
txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
|
|
txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
|
|
MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
|
|
}
|
|
|
|
txd->read.cmd_type_len |= htole32(E1000_TXD_CMD_EOP | txd_flags);
|
|
pi->ipi_new_pidx = i;
|
|
|
|
return (0);
|
|
}
|
|
|
|
static void
|
|
igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
struct em_tx_queue *que = &sc->tx_queues[txqid];
|
|
struct tx_ring *txr = &que->txr;
|
|
|
|
E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
|
|
}
|
|
|
|
static int
|
|
igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
if_softc_ctx_t scctx = sc->shared;
|
|
struct em_tx_queue *que = &sc->tx_queues[txqid];
|
|
struct tx_ring *txr = &que->txr;
|
|
|
|
qidx_t processed = 0;
|
|
int updated;
|
|
qidx_t cur, prev, ntxd, rs_cidx;
|
|
int32_t delta;
|
|
uint8_t status;
|
|
|
|
rs_cidx = txr->tx_rs_cidx;
|
|
if (rs_cidx == txr->tx_rs_pidx)
|
|
return (0);
|
|
cur = txr->tx_rsq[rs_cidx];
|
|
status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status;
|
|
updated = !!(status & E1000_TXD_STAT_DD);
|
|
|
|
if (!updated)
|
|
return (0);
|
|
|
|
/* If clear is false just let caller know that there
|
|
* are descriptors to reclaim */
|
|
if (!clear)
|
|
return (1);
|
|
|
|
prev = txr->tx_cidx_processed;
|
|
ntxd = scctx->isc_ntxd[0];
|
|
do {
|
|
MPASS(prev != cur);
|
|
delta = (int32_t)cur - (int32_t)prev;
|
|
if (delta < 0)
|
|
delta += ntxd;
|
|
MPASS(delta > 0);
|
|
|
|
processed += delta;
|
|
prev = cur;
|
|
rs_cidx = (rs_cidx + 1) & (ntxd-1);
|
|
if (rs_cidx == txr->tx_rs_pidx)
|
|
break;
|
|
cur = txr->tx_rsq[rs_cidx];
|
|
status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status;
|
|
} while ((status & E1000_TXD_STAT_DD));
|
|
|
|
txr->tx_rs_cidx = rs_cidx;
|
|
txr->tx_cidx_processed = prev;
|
|
return (processed);
|
|
}
|
|
|
|
static void
|
|
igb_isc_rxd_refill(void *arg, if_rxd_update_t iru)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
if_softc_ctx_t scctx = sc->shared;
|
|
uint16_t rxqid = iru->iru_qsidx;
|
|
struct em_rx_queue *que = &sc->rx_queues[rxqid];
|
|
union e1000_adv_rx_desc *rxd;
|
|
struct rx_ring *rxr = &que->rxr;
|
|
uint64_t *paddrs;
|
|
uint32_t next_pidx, pidx;
|
|
uint16_t count;
|
|
int i;
|
|
|
|
paddrs = iru->iru_paddrs;
|
|
pidx = iru->iru_pidx;
|
|
count = iru->iru_count;
|
|
|
|
for (i = 0, next_pidx = pidx; i < count; i++) {
|
|
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[next_pidx];
|
|
|
|
rxd->read.pkt_addr = htole64(paddrs[i]);
|
|
if (++next_pidx == scctx->isc_nrxd[0])
|
|
next_pidx = 0;
|
|
}
|
|
}
|
|
|
|
static void
|
|
igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
struct em_rx_queue *que = &sc->rx_queues[rxqid];
|
|
struct rx_ring *rxr = &que->rxr;
|
|
|
|
E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
|
|
}
|
|
|
|
static int
|
|
igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
if_softc_ctx_t scctx = sc->shared;
|
|
struct em_rx_queue *que = &sc->rx_queues[rxqid];
|
|
struct rx_ring *rxr = &que->rxr;
|
|
union e1000_adv_rx_desc *rxd;
|
|
uint32_t staterr = 0;
|
|
int cnt, i;
|
|
|
|
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
|
|
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i];
|
|
staterr = le32toh(rxd->wb.upper.status_error);
|
|
|
|
if ((staterr & E1000_RXD_STAT_DD) == 0)
|
|
break;
|
|
if (++i == scctx->isc_nrxd[0])
|
|
i = 0;
|
|
if (staterr & E1000_RXD_STAT_EOP)
|
|
cnt++;
|
|
}
|
|
return (cnt);
|
|
}
|
|
|
|
/****************************************************************
|
|
* Routine sends data which has been dma'ed into host memory
|
|
* to upper layer. Initialize ri structure.
|
|
*
|
|
* Returns 0 upon success, errno on failure
|
|
***************************************************************/
|
|
|
|
static int
|
|
igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
|
|
{
|
|
struct e1000_softc *sc = arg;
|
|
if_softc_ctx_t scctx = sc->shared;
|
|
struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
|
|
struct rx_ring *rxr = &que->rxr;
|
|
union e1000_adv_rx_desc *rxd;
|
|
|
|
uint16_t pkt_info, len;
|
|
uint32_t ptype, staterr;
|
|
int i, cidx;
|
|
bool eop;
|
|
|
|
staterr = i = 0;
|
|
cidx = ri->iri_cidx;
|
|
|
|
do {
|
|
rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx];
|
|
staterr = le32toh(rxd->wb.upper.status_error);
|
|
pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info);
|
|
|
|
MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
|
|
|
|
len = le16toh(rxd->wb.upper.length);
|
|
ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
|
|
|
|
ri->iri_len += len;
|
|
rxr->rx_bytes += ri->iri_len;
|
|
|
|
rxd->wb.upper.status_error = 0;
|
|
eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
|
|
|
|
/* Make sure bad packets are discarded */
|
|
if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
|
|
sc->dropped_pkts++;
|
|
++rxr->rx_discarded;
|
|
return (EBADMSG);
|
|
}
|
|
ri->iri_frags[i].irf_flid = 0;
|
|
ri->iri_frags[i].irf_idx = cidx;
|
|
ri->iri_frags[i].irf_len = len;
|
|
|
|
if (++cidx == scctx->isc_nrxd[0])
|
|
cidx = 0;
|
|
#ifdef notyet
|
|
if (rxr->hdr_split == true) {
|
|
ri->iri_frags[i].irf_flid = 1;
|
|
ri->iri_frags[i].irf_idx = cidx;
|
|
if (++cidx == scctx->isc_nrxd[0])
|
|
cidx = 0;
|
|
}
|
|
#endif
|
|
i++;
|
|
} while (!eop);
|
|
|
|
rxr->rx_packets++;
|
|
|
|
if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
|
|
igb_rx_checksum(staterr, ri, ptype);
|
|
|
|
if (staterr & E1000_RXD_STAT_VP) {
|
|
if (((sc->hw.mac.type == e1000_i350) ||
|
|
(sc->hw.mac.type == e1000_i354)) &&
|
|
(staterr & E1000_RXDEXT_STATERR_LB))
|
|
ri->iri_vtag = be16toh(rxd->wb.upper.vlan);
|
|
else
|
|
ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
|
|
ri->iri_flags |= M_VLANTAG;
|
|
}
|
|
|
|
ri->iri_flowid =
|
|
le32toh(rxd->wb.lower.hi_dword.rss);
|
|
ri->iri_rsstype = igb_determine_rsstype(pkt_info);
|
|
ri->iri_nfrags = i;
|
|
|
|
return (0);
|
|
}
|
|
|
|
/*********************************************************************
|
|
*
|
|
* Verify that the hardware indicated that the checksum is valid.
|
|
* Inform the stack about the status of checksum so that stack
|
|
* doesn't spend time verifying the checksum.
|
|
*
|
|
*********************************************************************/
|
|
static void
|
|
igb_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
|
|
{
|
|
uint16_t status = (uint16_t)staterr;
|
|
uint8_t errors = (uint8_t)(staterr >> 24);
|
|
|
|
if (__predict_false(status & E1000_RXD_STAT_IXSM))
|
|
return;
|
|
|
|
/* If there is a layer 3 or 4 error we are done */
|
|
if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE)))
|
|
return;
|
|
|
|
/* IP Checksum Good */
|
|
if (status & E1000_RXD_STAT_IPCS)
|
|
ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
|
|
|
|
/* Valid L4E checksum */
|
|
if (__predict_true(status &
|
|
(E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
|
|
/* SCTP header present */
|
|
if (__predict_false((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
|
|
(ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)) {
|
|
ri->iri_csum_flags |= CSUM_SCTP_VALID;
|
|
} else {
|
|
ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
|
|
ri->iri_csum_data = htons(0xffff);
|
|
}
|
|
}
|
|
}
|
|
|
|
/********************************************************************
|
|
*
|
|
* Parse the packet type to determine the appropriate hash
|
|
*
|
|
******************************************************************/
|
|
static int
|
|
igb_determine_rsstype(uint16_t pkt_info)
|
|
{
|
|
switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
|
|
case E1000_RXDADV_RSSTYPE_IPV4_TCP:
|
|
return M_HASHTYPE_RSS_TCP_IPV4;
|
|
case E1000_RXDADV_RSSTYPE_IPV4:
|
|
return M_HASHTYPE_RSS_IPV4;
|
|
case E1000_RXDADV_RSSTYPE_IPV6_TCP:
|
|
return M_HASHTYPE_RSS_TCP_IPV6;
|
|
case E1000_RXDADV_RSSTYPE_IPV6_EX:
|
|
return M_HASHTYPE_RSS_IPV6_EX;
|
|
case E1000_RXDADV_RSSTYPE_IPV6:
|
|
return M_HASHTYPE_RSS_IPV6;
|
|
case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
|
|
return M_HASHTYPE_RSS_TCP_IPV6_EX;
|
|
default:
|
|
return M_HASHTYPE_OPAQUE;
|
|
}
|
|
}
|