d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hyperv/hn: Set the TCP ACK/data segment aggregation limit

Browse Source 
Set TCP ACK append limit to 1, i.e. aggregate 2 ACKs at most.  Aggregating
anything more than 2 hurts TCP sending performance in hyperv.  This
significantly improves the TCP sending performance when the number of
concurrent connetion is low (2~8).  And it greatly stabilizes the TCP
sending performance in other cases.

Set TCP data segments aggregation length limit to 37500.  Without this
limitation, hn(4) could aggregate ~45 TCP data segments for each
connection (even at 64 or more connections) before dispatching them to
socket code; large aggregation slows down ACK sending and eventually
hurts/destabilizes TCP reception performance.  This setting stabilizes
and improves TCP reception performance for >4 concurrent connections
significantly.

Make them sysctls so they could be adjusted.

Reviewed by:	adrian, gallatin (previous version), hselasky (previous version)
Approved by:	adrian (mentor)
MFC after:	1 week
Sponsored by:	Microsoft OSTC
Differential Revision:	https://reviews.freebsd.org/D5185
This commit is contained in:
Sepherosa Ziehau 2016-02-18 04:59:37 +00:00
parent 7ae3d4bf54
commit aa7f74113e
2 changed files with 53 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sys/dev/hyperv/netvsc/hv_net_vsc.h
View File

@ -1030,7 +1030,6 @@ typedef struct hn_softc {
struct task hn_txeof_task;
struct lro_ctrl hn_lro;
int hn_lro_hiwat;
/* Trust csum verification on host side */
int hn_trust_hcsum; /* HN_TRUST_HCSUM_ */

94
sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
View File

@ -176,14 +176,11 @@ struct hn_txdesc {
#define HN_CSUM_ASSIST_WIN8 (CSUM_TCP)
#define HN_CSUM_ASSIST (CSUM_IP | CSUM_UDP | CSUM_TCP)
/* XXX move to netinet/tcp_lro.h */
#define HN_LRO_HIWAT_MAX 65535
#define HN_LRO_HIWAT_DEF HN_LRO_HIWAT_MAX
#define HN_LRO_LENLIM_DEF (25 * ETHERMTU)
/* YYY 2*MTU is a bit rough, but should be good enough. */
#define HN_LRO_HIWAT_MTULIM(ifp) (2 * (ifp)->if_mtu)
#define HN_LRO_HIWAT_ISVALID(sc, hiwat) \
((hiwat) >= HN_LRO_HIWAT_MTULIM((sc)->hn_ifp) || \
(hiwat) <= HN_LRO_HIWAT_MAX)
#define HN_LRO_LENLIM_MIN(ifp) (2 * (ifp)->if_mtu)
#define HN_LRO_ACKCNT_DEF 1
/*
* Be aware that this sleepable mutex will exhibit WITNESS errors when
@ -253,9 +250,8 @@ static void hn_start(struct ifnet *ifp);
static void hn_start_txeof(struct ifnet *ifp);
static int hn_ifmedia_upd(struct ifnet *ifp);
static void hn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr);
#ifdef HN_LRO_HIWAT
static int hn_lro_hiwat_sysctl(SYSCTL_HANDLER_ARGS);
#endif
static int hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS);
static int hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS);
static int hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS);
static int hn_tx_chimney_size_sysctl(SYSCTL_HANDLER_ARGS);
static int hn_check_iplen(const struct mbuf *, int);
@ -265,15 +261,6 @@ static void hn_start_taskfunc(void *xsc, int pending);
static void hn_txeof_taskfunc(void *xsc, int pending);
static int hn_encap(struct hn_softc *, struct hn_txdesc *, struct mbuf **);
static __inline void
hn_set_lro_hiwat(struct hn_softc *sc, int hiwat)
{
sc->hn_lro_hiwat = hiwat;
#ifdef HN_LRO_HIWAT
sc->hn_lro.lro_hiwat = sc->hn_lro_hiwat;
#endif
}
static int
hn_ifmedia_upd(struct ifnet *ifp __unused)
{
@ -358,7 +345,6 @@ netvsc_attach(device_t dev)
bzero(sc, sizeof(hn_softc_t));
sc->hn_unit = unit;
sc->hn_dev = dev;
sc->hn_lro_hiwat = HN_LRO_HIWAT_DEF;
sc->hn_direct_tx_size = hn_direct_tx_size;
if (hn_trust_hosttcp)
sc->hn_trust_hcsum |= HN_TRUST_HCSUM_TCP;
@ -442,9 +428,8 @@ netvsc_attach(device_t dev)
/* Driver private LRO settings */
sc->hn_lro.ifp = ifp;
#endif
#ifdef HN_LRO_HIWAT
sc->hn_lro.lro_hiwat = sc->hn_lro_hiwat;
#endif
sc->hn_lro.lro_length_lim = HN_LRO_LENLIM_DEF;
sc->hn_lro.lro_ackcnt_lim = HN_LRO_ACKCNT_DEF;
#endif /* INET || INET6 */
#if __FreeBSD_version >= 1100045
@ -480,11 +465,12 @@ netvsc_attach(device_t dev)
CTLFLAG_RW, &sc->hn_lro.lro_flushed, 0, "LRO flushed");
SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "lro_tried",
CTLFLAG_RW, &sc->hn_lro_tried, "# of LRO tries");
#ifdef HN_LRO_HIWAT
SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_hiwat",
CTLTYPE_INT | CTLFLAG_RW, sc, 0, hn_lro_hiwat_sysctl,
"I", "LRO high watermark");
#endif
SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_length_lim",
CTLTYPE_UINT | CTLFLAG_RW, sc, 0, hn_lro_lenlim_sysctl, "IU",
"Max # of data bytes to be aggregated by LRO");
SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "lro_ackcnt_lim",
CTLTYPE_INT | CTLFLAG_RW, sc, 0, hn_lro_ackcnt_sysctl, "I",
"Max # of ACKs to be aggregated by LRO");
SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "trust_hosttcp",
CTLTYPE_INT | CTLFLAG_RW, sc, HN_TRUST_HCSUM_TCP,
hn_trust_hcsum_sysctl, "I",
@ -1410,12 +1396,13 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
/* Obtain and record requested MTU */
ifp->if_mtu = ifr->ifr_mtu;
/*
* Make sure that LRO high watermark is still valid,
* after MTU change (the 2*MTU limit).
* Make sure that LRO aggregation length limit is still
* valid, after the MTU change.
*/
if (!HN_LRO_HIWAT_ISVALID(sc, sc->hn_lro_hiwat))
hn_set_lro_hiwat(sc, HN_LRO_HIWAT_MTULIM(ifp));
if (sc->hn_lro.lro_length_lim < HN_LRO_LENLIM_MIN(ifp))
sc->hn_lro.lro_length_lim = HN_LRO_LENLIM_MIN(ifp);
do {
NV_LOCK(sc);
@ -1722,26 +1709,51 @@ hn_watchdog(struct ifnet *ifp)
}
#endif
#ifdef HN_LRO_HIWAT
static int
hn_lro_hiwat_sysctl(SYSCTL_HANDLER_ARGS)
hn_lro_lenlim_sysctl(SYSCTL_HANDLER_ARGS)
{
struct hn_softc *sc = arg1;
int hiwat, error;
unsigned int lenlim;
int error;
hiwat = sc->hn_lro_hiwat;
error = sysctl_handle_int(oidp, &hiwat, 0, req);
lenlim = sc->hn_lro.lro_length_lim;
error = sysctl_handle_int(oidp, &lenlim, 0, req);
if (error || req->newptr == NULL)
return error;
if (!HN_LRO_HIWAT_ISVALID(sc, hiwat))
if (lenlim < HN_LRO_LENLIM_MIN(sc->hn_ifp) ||
lenlim > TCP_LRO_LENGTH_MAX)
return EINVAL;
if (sc->hn_lro_hiwat != hiwat)
hn_set_lro_hiwat(sc, hiwat);
sc->hn_lro.lro_length_lim = lenlim;
return 0;
}
static int
hn_lro_ackcnt_sysctl(SYSCTL_HANDLER_ARGS)
{
struct hn_softc *sc = arg1;
int ackcnt, error;
/*
* lro_ackcnt_lim is append count limit,
* +1 to turn it into aggregation limit.
*/
ackcnt = sc->hn_lro.lro_ackcnt_lim + 1;
error = sysctl_handle_int(oidp, &ackcnt, 0, req);
if (error || req->newptr == NULL)
return error;
if (ackcnt < 2 || ackcnt > (TCP_LRO_ACKCNT_MAX + 1))
return EINVAL;
/*
* Convert aggregation limit back to append
* count limit.
*/
sc->hn_lro.lro_ackcnt_lim = ackcnt - 1;
return 0;
}
#endif /* HN_LRO_HIWAT */
static int
hn_trust_hcsum_sysctl(SYSCTL_HANDLER_ARGS)