Introduce HW TSO support for VNIC
This feature was added in Pass2.0. Significantly improves VNIC's TCP performance on Tx. Reviewed by: wma Obtained from: Semihalf Sponsored by: Cavium Differential Revision: https://reviews.freebsd.org/D5424
This commit is contained in:
parent
3bdcfead19
commit
af8fe8f169
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=296039
@ -292,6 +292,7 @@ struct nicvf {
|
||||
uint8_t max_queues;
|
||||
struct resource *reg_base;
|
||||
boolean_t link_up;
|
||||
boolean_t hw_tso;
|
||||
uint8_t duplex;
|
||||
uint32_t speed;
|
||||
uint8_t cpi_alg;
|
||||
|
@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/if_vlan_var.h>
|
||||
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/ip.h>
|
||||
#include <netinet/if_ether.h>
|
||||
#include <netinet/tcp_lro.h>
|
||||
|
||||
@ -194,6 +195,9 @@ nicvf_attach(device_t dev)
|
||||
nic->pnicvf = nic;
|
||||
|
||||
NICVF_CORE_LOCK_INIT(nic);
|
||||
/* Enable HW TSO on Pass2 */
|
||||
if (!pass1_silicon(dev))
|
||||
nic->hw_tso = TRUE;
|
||||
|
||||
rid = VNIC_VF_REG_RID;
|
||||
nic->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
|
||||
@ -356,6 +360,14 @@ nicvf_setup_ifnet(struct nicvf *nic)
|
||||
/* Set the default values */
|
||||
if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0);
|
||||
if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
|
||||
if (nic->hw_tso) {
|
||||
/* TSO */
|
||||
if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0);
|
||||
/* TSO parameters */
|
||||
ifp->if_hw_tsomax = NICVF_TSO_MAXSIZE;
|
||||
ifp->if_hw_tsomaxsegcount = NICVF_TSO_NSEGS;
|
||||
ifp->if_hw_tsomaxsegsize = MCLBYTES;
|
||||
}
|
||||
/* IP/TCP/UDP HW checksums */
|
||||
if_setcapabilitiesbit(ifp, IFCAP_HWCSUM, 0);
|
||||
if_setcapabilitiesbit(ifp, IFCAP_HWSTATS, 0);
|
||||
@ -364,7 +376,8 @@ nicvf_setup_ifnet(struct nicvf *nic)
|
||||
*/
|
||||
if_clearhwassist(ifp);
|
||||
if_sethwassistbits(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP), 0);
|
||||
|
||||
if (nic->hw_tso)
|
||||
if_sethwassistbits(ifp, (CSUM_TSO), 0);
|
||||
if_setcapenable(ifp, if_getcapabilities(ifp));
|
||||
|
||||
return (0);
|
||||
@ -513,6 +526,8 @@ nicvf_if_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
|
||||
ifp->if_capenable ^= IFCAP_TXCSUM;
|
||||
if (mask & IFCAP_RXCSUM)
|
||||
ifp->if_capenable ^= IFCAP_RXCSUM;
|
||||
if ((mask & IFCAP_TSO4) && nic->hw_tso)
|
||||
ifp->if_capenable ^= IFCAP_TSO4;
|
||||
if (mask & IFCAP_LRO) {
|
||||
/*
|
||||
* Lock the driver for a moment to avoid
|
||||
|
@ -1070,8 +1070,8 @@ nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len,
|
||||
BUS_SPACE_MAXADDR, /* lowaddr */
|
||||
BUS_SPACE_MAXADDR, /* highaddr */
|
||||
NULL, NULL, /* filtfunc, filtfuncarg */
|
||||
NICVF_TXBUF_MAXSIZE, /* maxsize */
|
||||
NICVF_TXBUF_NSEGS, /* nsegments */
|
||||
NICVF_TSO_MAXSIZE, /* maxsize */
|
||||
NICVF_TSO_NSEGS, /* nsegments */
|
||||
MCLBYTES, /* maxsegsize */
|
||||
0, /* flags */
|
||||
NULL, NULL, /* lockfunc, lockfuncarg */
|
||||
@ -1727,14 +1727,18 @@ static __inline int
|
||||
nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
|
||||
int subdesc_cnt, struct mbuf *mbuf, int len)
|
||||
{
|
||||
struct nicvf *nic;
|
||||
struct sq_hdr_subdesc *hdr;
|
||||
struct ether_vlan_header *eh;
|
||||
#ifdef INET
|
||||
struct ip *ip;
|
||||
struct tcphdr *th;
|
||||
#endif
|
||||
uint16_t etype;
|
||||
int ehdrlen, iphlen, poff;
|
||||
|
||||
nic = sq->nic;
|
||||
|
||||
hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
|
||||
sq->snd_buff[qentry].mbuf = mbuf;
|
||||
|
||||
@ -1746,18 +1750,25 @@ nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
|
||||
hdr->subdesc_cnt = subdesc_cnt;
|
||||
hdr->tot_len = len;
|
||||
|
||||
if (mbuf->m_pkthdr.csum_flags != 0) {
|
||||
hdr->csum_l3 = 1; /* Enable IP csum calculation */
|
||||
|
||||
eh = mtod(mbuf, struct ether_vlan_header *);
|
||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
|
||||
ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
|
||||
etype = ntohs(eh->evl_proto);
|
||||
} else {
|
||||
ehdrlen = ETHER_HDR_LEN;
|
||||
etype = ntohs(eh->evl_encap_proto);
|
||||
}
|
||||
eh = mtod(mbuf, struct ether_vlan_header *);
|
||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
|
||||
ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
|
||||
etype = ntohs(eh->evl_proto);
|
||||
} else {
|
||||
ehdrlen = ETHER_HDR_LEN;
|
||||
etype = ntohs(eh->evl_encap_proto);
|
||||
}
|
||||
|
||||
switch (etype) {
|
||||
#ifdef INET6
|
||||
case ETHERTYPE_IPV6:
|
||||
/* ARM64TODO: Add support for IPv6 */
|
||||
hdr->csum_l3 = 0;
|
||||
sq->snd_buff[qentry].mbuf = NULL;
|
||||
return (ENXIO);
|
||||
#endif
|
||||
#ifdef INET
|
||||
case ETHERTYPE_IP:
|
||||
if (mbuf->m_len < ehdrlen + sizeof(struct ip)) {
|
||||
mbuf = m_pullup(mbuf, ehdrlen + sizeof(struct ip));
|
||||
sq->snd_buff[qentry].mbuf = mbuf;
|
||||
@ -1765,21 +1776,13 @@ nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
|
||||
return (ENOBUFS);
|
||||
}
|
||||
|
||||
switch (etype) {
|
||||
#ifdef INET6
|
||||
case ETHERTYPE_IPV6:
|
||||
/* ARM64TODO: Add support for IPv6 */
|
||||
hdr->csum_l3 = 0;
|
||||
sq->snd_buff[qentry].mbuf = NULL;
|
||||
return (ENXIO);
|
||||
#endif
|
||||
#ifdef INET
|
||||
case ETHERTYPE_IP:
|
||||
ip = (struct ip *)(mbuf->m_data + ehdrlen);
|
||||
ip->ip_sum = 0;
|
||||
iphlen = ip->ip_hl << 2;
|
||||
poff = ehdrlen + iphlen;
|
||||
ip = (struct ip *)(mbuf->m_data + ehdrlen);
|
||||
ip->ip_sum = 0;
|
||||
iphlen = ip->ip_hl << 2;
|
||||
poff = ehdrlen + iphlen;
|
||||
|
||||
if (mbuf->m_pkthdr.csum_flags != 0) {
|
||||
hdr->csum_l3 = 1; /* Enable IP csum calculation */
|
||||
switch (ip->ip_p) {
|
||||
case IPPROTO_TCP:
|
||||
if ((mbuf->m_pkthdr.csum_flags & CSUM_TCP) == 0)
|
||||
@ -1820,17 +1823,28 @@ nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
|
||||
default:
|
||||
break;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
hdr->csum_l3 = 0;
|
||||
return (0);
|
||||
hdr->l3_offset = ehdrlen;
|
||||
hdr->l4_offset = ehdrlen + iphlen;
|
||||
}
|
||||
|
||||
hdr->l3_offset = ehdrlen;
|
||||
hdr->l4_offset = ehdrlen + iphlen;
|
||||
} else
|
||||
if ((mbuf->m_pkthdr.tso_segsz != 0) && nic->hw_tso) {
|
||||
/*
|
||||
* Extract ip again as m_data could have been modified.
|
||||
*/
|
||||
ip = (struct ip *)(mbuf->m_data + ehdrlen);
|
||||
th = (struct tcphdr *)((caddr_t)ip + iphlen);
|
||||
|
||||
hdr->tso = 1;
|
||||
hdr->tso_start = ehdrlen + iphlen + (th->th_off * 4);
|
||||
hdr->tso_max_paysize = mbuf->m_pkthdr.tso_segsz;
|
||||
hdr->inner_l3_offset = ehdrlen - 2;
|
||||
nic->drv_stats.tx_tso++;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
hdr->csum_l3 = 0;
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -1859,10 +1873,11 @@ int
|
||||
nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf *mbuf)
|
||||
{
|
||||
bus_dma_segment_t segs[256];
|
||||
struct nicvf *nic;
|
||||
struct snd_buff *snd_buff;
|
||||
size_t seg;
|
||||
int nsegs, qentry;
|
||||
int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT - 1;
|
||||
int subdesc_cnt;
|
||||
int err;
|
||||
|
||||
NICVF_TX_LOCK_ASSERT(sq);
|
||||
@ -1880,7 +1895,11 @@ nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf *mbuf)
|
||||
}
|
||||
|
||||
/* Set how many subdescriptors is required */
|
||||
subdesc_cnt += nsegs;
|
||||
nic = sq->nic;
|
||||
if (mbuf->m_pkthdr.tso_segsz != 0 && nic->hw_tso)
|
||||
subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;
|
||||
else
|
||||
subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT + nsegs - 1;
|
||||
|
||||
if (subdesc_cnt > sq->free_cnt) {
|
||||
/* ARM64TODO: Add mbuf defragmentation if we lack descriptors */
|
||||
|
@ -131,10 +131,13 @@
|
||||
#define NICVF_RCV_BUF_ALIGN_LEN(addr) \
|
||||
(NICVF_ALIGNED_ADDR((addr), NICVF_RCV_BUF_ALIGN_BYTES) - (addr))
|
||||
|
||||
#define NICVF_TXBUF_MAXSIZE 9212 /* Total max payload without TSO */
|
||||
#define NICVF_TXBUF_MAXSIZE NIC_HW_MAX_FRS /* Total max payload without TSO */
|
||||
#define NICVF_TXBUF_NSEGS 256 /* Single command is at most 256 buffers
|
||||
(hdr + 255 subcmds) */
|
||||
|
||||
/* TSO-related definitions */
|
||||
#define NICVF_TSO_MAXSIZE IP_MAXPACKET
|
||||
#define NICVF_TSO_NSEGS NICVF_TXBUF_NSEGS
|
||||
#define NICVF_TSO_HEADER_SIZE 128
|
||||
|
||||
/* Queue enable/disable */
|
||||
#define NICVF_SQ_EN (1UL << 19)
|
||||
|
@ -565,25 +565,28 @@ struct sq_hdr_subdesc {
|
||||
uint64_t subdesc_cnt:8;
|
||||
uint64_t csum_l4:2;
|
||||
uint64_t csum_l3:1;
|
||||
uint64_t rsvd0:5;
|
||||
uint64_t csum_inner_l4:2;
|
||||
uint64_t csum_inner_l3:1;
|
||||
uint64_t rsvd0:2;
|
||||
uint64_t l4_offset:8;
|
||||
uint64_t l3_offset:8;
|
||||
uint64_t rsvd1:4;
|
||||
uint64_t tot_len:20; /* W0 */
|
||||
|
||||
uint64_t tso_sdc_cont:8;
|
||||
uint64_t tso_sdc_first:8;
|
||||
uint64_t tso_l4_offset:8;
|
||||
uint64_t tso_flags_last:12;
|
||||
uint64_t tso_flags_first:12;
|
||||
uint64_t rsvd2:2;
|
||||
uint64_t rsvd2:24;
|
||||
uint64_t inner_l4_offset:8;
|
||||
uint64_t inner_l3_offset:8;
|
||||
uint64_t tso_start:8;
|
||||
uint64_t rsvd3:2;
|
||||
uint64_t tso_max_paysize:14; /* W1 */
|
||||
#elif defined(__LITTLE_ENDIAN_BITFIELD)
|
||||
uint64_t tot_len:20;
|
||||
uint64_t rsvd1:4;
|
||||
uint64_t l3_offset:8;
|
||||
uint64_t l4_offset:8;
|
||||
uint64_t rsvd0:5;
|
||||
uint64_t rsvd0:2;
|
||||
uint64_t csum_inner_l3:1;
|
||||
uint64_t csum_inner_l4:2;
|
||||
uint64_t csum_l3:1;
|
||||
uint64_t csum_l4:2;
|
||||
uint64_t subdesc_cnt:8;
|
||||
@ -594,12 +597,11 @@ struct sq_hdr_subdesc {
|
||||
uint64_t subdesc_type:4; /* W0 */
|
||||
|
||||
uint64_t tso_max_paysize:14;
|
||||
uint64_t rsvd2:2;
|
||||
uint64_t tso_flags_first:12;
|
||||
uint64_t tso_flags_last:12;
|
||||
uint64_t tso_l4_offset:8;
|
||||
uint64_t tso_sdc_first:8;
|
||||
uint64_t tso_sdc_cont:8; /* W1 */
|
||||
uint64_t rsvd3:2;
|
||||
uint64_t tso_start:8;
|
||||
uint64_t inner_l3_offset:8;
|
||||
uint64_t inner_l4_offset:8;
|
||||
uint64_t rsvd2:24;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user