From c48d20d7c75383ca2a4a100998b919f88b3e0675 Mon Sep 17 00:00:00 2001
From: Sepherosa Ziehau <sephe@FreeBSD.org>
Date: Tue, 12 Jan 2016 01:50:56 +0000
Subject: [PATCH] hyperv/hn: Avoid mbuf cluster allocation, if the packet is
 small.

This one mainly avoids mbuf cluster allocation for TCP ACKs during
TCP sending tests.  And it gives me ~200Mbps improvement (4.7Gbps
-> 4.9Gbps), when running iperf3 TCP sending test w/ 16 connections.

While I'm here, nuke the unnecessary zeroing out pkthdr.csum_flags.

Reviewed by:		adrain
Approved by:		adrian (mentor)
Sponsored by:		Microsoft OSTC
Differential Revision:	https://reviews.freebsd.org/D4853
---
 sys/dev/hyperv/netvsc/hv_net_vsc.h            |  1 +
 sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c | 55 ++++++++++---------
 2 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/sys/dev/hyperv/netvsc/hv_net_vsc.h b/sys/dev/hyperv/netvsc/hv_net_vsc.h
index bc34898be08c..b1d1e37cedca 100644
--- a/sys/dev/hyperv/netvsc/hv_net_vsc.h
+++ b/sys/dev/hyperv/netvsc/hv_net_vsc.h
@@ -1011,6 +1011,7 @@ typedef struct hn_softc {
 	u_long		hn_csum_tcp;
 	u_long		hn_csum_trusted;
 	u_long		hn_lro_tried;
+	u_long		hn_small_pkts;
 } hn_softc_t;
 
 
diff --git a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
index aae5881e9287..4225a431eb93 100644
--- a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
+++ b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
@@ -469,6 +469,8 @@ netvsc_attach(device_t dev)
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "csum_trusted",
 	    CTLFLAG_RW, &sc->hn_csum_trusted,
 	    "# of TCP segements that we trust host's csum verification");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "small_pkts",
+	    CTLFLAG_RW, &sc->hn_small_pkts, "# of small packets received");
 
 	if (unit == 0) {
 		struct sysctl_ctx_list *dc_ctx;
@@ -1022,35 +1024,38 @@ netvsc_recv(struct hv_device *device_ctx, netvsc_packet *packet,
 	 */
 	if (packet->tot_data_buf_len > (ifp->if_mtu + ETHER_HDR_LEN)) {
 		return (0);
+	} else if (packet->tot_data_buf_len <= MHLEN) {
+		m_new = m_gethdr(M_NOWAIT, MT_DATA);
+		if (m_new == NULL)
+			return (0);
+		memcpy(mtod(m_new, void *), packet->data,
+		    packet->tot_data_buf_len);
+		m_new->m_pkthdr.len = m_new->m_len = packet->tot_data_buf_len;
+		sc->hn_small_pkts++;
+	} else {
+		/*
+		 * Get an mbuf with a cluster.  For packets 2K or less,
+		 * get a standard 2K cluster.  For anything larger, get a
+		 * 4K cluster.  Any buffers larger than 4K can cause problems
+		 * if looped around to the Hyper-V TX channel, so avoid them.
+		 */
+		size = MCLBYTES;
+		if (packet->tot_data_buf_len > MCLBYTES) {
+			/* 4096 */
+			size = MJUMPAGESIZE;
+		}
+
+		m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
+		if (m_new == NULL) {
+			device_printf(dev, "alloc mbuf failed.\n");
+			return (0);
+		}
+
+		hv_m_append(m_new, packet->tot_data_buf_len, packet->data);
 	}
-
-	/*
-	 * Get an mbuf with a cluster.  For packets 2K or less,
-	 * get a standard 2K cluster.  For anything larger, get a
-	 * 4K cluster.  Any buffers larger than 4K can cause problems
-	 * if looped around to the Hyper-V TX channel, so avoid them.
-	 */
-	size = MCLBYTES;
-
-	if (packet->tot_data_buf_len > MCLBYTES) {
-		/* 4096 */
-		size = MJUMPAGESIZE;
-	}
-
-	m_new = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
-
-	if (m_new == NULL) {
-		device_printf(dev, "alloc mbuf failed.\n");
-		return (0);
-	}
-
-	hv_m_append(m_new, packet->tot_data_buf_len,
-			packet->data);
-
 	m_new->m_pkthdr.rcvif = ifp;
 
 	/* receive side checksum offload */
-	m_new->m_pkthdr.csum_flags = 0;
 	if (NULL != csum_info) {
 		/* IP csum offload */
 		if (csum_info->receive.ip_csum_succeeded) {