hyperv/hn: Apply VF's RSS setting

Since in Azure SYN and SYN|ACK go through the synthetic parts while the
rest of the same TCP flow goes through the VF, apply VF's RSS settings
to synthetic parts to have a consistent hash value/type for the same TCP
flow.

MFC after:	1 week
Sponsored by:	Microsoft
Differential Revision:	https://reviews.freebsd.org/D12333

sys/dev/hyperv/netvsc/hn_rndis.c

@@ -502,7 +502,7 @@ hn_rndis_query_rsscaps(struct hn_softc *sc, int *rxr_cnt0)
 
 	/* Commit! */
 	sc->hn_rss_ind_size = indsz;
-	sc->hn_rss_hash = hash_func | hash_types;
+	sc->hn_rss_hcap = hash_func | hash_types;
 	*rxr_cnt0 = rxr_cnt;
 	return (0);
 }

sys/dev/hyperv/netvsc/if_hn.c

@@ -284,6 +284,8 @@ static void			hn_xpnt_vf_init_taskfunc(void *, int);
 static void			hn_xpnt_vf_init(struct hn_softc *);
 static void			hn_xpnt_vf_setenable(struct hn_softc *);
 static void			hn_xpnt_vf_setdisable(struct hn_softc *, bool);
+static void			hn_vf_rss_fixup(struct hn_softc *, bool);
+static void			hn_vf_rss_restore(struct hn_softc *);
 
 static int			hn_rndis_rxinfo(const void *, int,
 				    struct hn_rxinfo *);
@@ -327,6 +329,8 @@ static int			hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS);
 static int			hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS);
 #endif
 static int			hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS);
+static int			hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS);
+static int			hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS);
 static int			hn_txagg_size_sysctl(SYSCTL_HANDLER_ARGS);
 static int			hn_txagg_pkts_sysctl(SYSCTL_HANDLER_ARGS);
 static int			hn_txagg_pktmax_sysctl(SYSCTL_HANDLER_ARGS);
@@ -386,8 +390,11 @@ static int			hn_rxfilter_config(struct hn_softc *);
 static int			hn_rss_reconfig(struct hn_softc *);
 #endif
 static void			hn_rss_ind_fixup(struct hn_softc *);
+static void			hn_rss_mbuf_hash(struct hn_softc *, uint32_t);
 static int			hn_rxpkt(struct hn_rx_ring *, const void *,
 				    int, const struct hn_rxinfo *);
+static uint32_t			hn_rss_type_fromndis(uint32_t);
+static uint32_t			hn_rss_type_tondis(uint32_t);
 
 static int			hn_tx_ring_create(struct hn_softc *, int);
 static void			hn_tx_ring_destroy(struct hn_tx_ring *);
@@ -1134,11 +1141,13 @@ hn_rxvf_change(struct hn_softc *sc, struct ifnet *ifp, bool rxvf)
 	hn_rxvf_set(sc, rxvf ? ifp : NULL);
 
 	if (rxvf) {
+		hn_vf_rss_fixup(sc, true);
 		hn_suspend_mgmt(sc);
 		sc->hn_link_flags &=
 		    ~(HN_LINK_FLAG_LINKUP | HN_LINK_FLAG_NETCHG);
 		if_link_state_change(hn_ifp, LINK_STATE_DOWN);
 	} else {
+		hn_vf_rss_restore(sc);
 		hn_resume_mgmt(sc);
 	}
 
@@ -1335,6 +1344,248 @@ hn_mtu_change_fixup(struct hn_softc *sc)
 #endif
 }
 
+static uint32_t
+hn_rss_type_fromndis(uint32_t rss_hash)
+{
+	uint32_t types = 0;
+
+	if (rss_hash & NDIS_HASH_IPV4)
+		types |= RSS_TYPE_IPV4;
+	if (rss_hash & NDIS_HASH_TCP_IPV4)
+		types |= RSS_TYPE_TCP_IPV4;
+	if (rss_hash & NDIS_HASH_IPV6)
+		types |= RSS_TYPE_IPV6;
+	if (rss_hash & NDIS_HASH_IPV6_EX)
+		types |= RSS_TYPE_IPV6_EX;
+	if (rss_hash & NDIS_HASH_TCP_IPV6)
+		types |= RSS_TYPE_TCP_IPV6;
+	if (rss_hash & NDIS_HASH_TCP_IPV6_EX)
+		types |= RSS_TYPE_TCP_IPV6_EX;
+	return (types);
+}
+
+static uint32_t
+hn_rss_type_tondis(uint32_t types)
+{
+	uint32_t rss_hash = 0;
+
+	KASSERT((types &
+	(RSS_TYPE_UDP_IPV4 | RSS_TYPE_UDP_IPV6 | RSS_TYPE_UDP_IPV6_EX)) == 0,
+	("UDP4, UDP6 and UDP6EX are not supported"));
+
+	if (types & RSS_TYPE_IPV4)
+		rss_hash |= NDIS_HASH_IPV4;
+	if (types & RSS_TYPE_TCP_IPV4)
+		rss_hash |= NDIS_HASH_TCP_IPV4;
+	if (types & RSS_TYPE_IPV6)
+		rss_hash |= NDIS_HASH_IPV6;
+	if (types & RSS_TYPE_IPV6_EX)
+		rss_hash |= NDIS_HASH_IPV6_EX;
+	if (types & RSS_TYPE_TCP_IPV6)
+		rss_hash |= NDIS_HASH_TCP_IPV6;
+	if (types & RSS_TYPE_TCP_IPV6_EX)
+		rss_hash |= NDIS_HASH_TCP_IPV6_EX;
+	return (rss_hash);
+}
+
+static void
+hn_rss_mbuf_hash(struct hn_softc *sc, uint32_t mbuf_hash)
+{
+	int i;
+
+	HN_LOCK_ASSERT(sc);
+
+	for (i = 0; i < sc->hn_rx_ring_cnt; ++i)
+		sc->hn_rx_ring[i].hn_mbuf_hash = mbuf_hash;
+}
+
+static void
+hn_vf_rss_fixup(struct hn_softc *sc, bool reconf)
+{
+	struct ifnet *ifp, *vf_ifp;
+	struct ifrsshash ifrh;
+	struct ifrsskey ifrk;
+	int error;
+	uint32_t my_types, diff_types, mbuf_types = 0;
+
+	HN_LOCK_ASSERT(sc);
+	KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED,
+	    ("%s: synthetic parts are not attached", sc->hn_ifp->if_xname));
+
+	if (sc->hn_rx_ring_inuse == 1) {
+		/* No RSS on synthetic parts; done. */
+		return;
+	}
+	if ((sc->hn_rss_hcap & NDIS_HASH_FUNCTION_TOEPLITZ) == 0) {
+		/* Synthetic parts do not support Toeplitz; done. */
+		return;
+	}
+
+	ifp = sc->hn_ifp;
+	vf_ifp = sc->hn_vf_ifp;
+
+	/*
+	 * Extract VF's RSS key.  Only 40 bytes key for Toeplitz is
+	 * supported.
+	 */
+	memset(&ifrk, 0, sizeof(ifrk));
+	strlcpy(ifrk.ifrk_name, vf_ifp->if_xname, sizeof(ifrk.ifrk_name));
+	error = vf_ifp->if_ioctl(vf_ifp, SIOCGIFRSSKEY, (caddr_t)&ifrk);
+	if (error) {
+		if_printf(ifp, "%s SIOCGRSSKEY failed: %d\n",
+		    vf_ifp->if_xname, error);
+		goto done;
+	}
+	if (ifrk.ifrk_func != RSS_FUNC_TOEPLITZ) {
+		if_printf(ifp, "%s RSS function %u is not Toeplitz\n",
+		    vf_ifp->if_xname, ifrk.ifrk_func);
+		goto done;
+	}
+	if (ifrk.ifrk_keylen != NDIS_HASH_KEYSIZE_TOEPLITZ) {
+		if_printf(ifp, "%s invalid RSS Toeplitz key length %d\n",
+		    vf_ifp->if_xname, ifrk.ifrk_keylen);
+		goto done;
+	}
+
+	/*
+	 * Extract VF's RSS hash.  Only Toeplitz is supported.
+	 */
+	memset(&ifrh, 0, sizeof(ifrh));
+	strlcpy(ifrh.ifrh_name, vf_ifp->if_xname, sizeof(ifrh.ifrh_name));
+	error = vf_ifp->if_ioctl(vf_ifp, SIOCGIFRSSHASH, (caddr_t)&ifrh);
+	if (error) {
+		if_printf(ifp, "%s SIOCGRSSHASH failed: %d\n",
+		    vf_ifp->if_xname, error);
+		goto done;
+	}
+	if (ifrh.ifrh_func != RSS_FUNC_TOEPLITZ) {
+		if_printf(ifp, "%s RSS function %u is not Toeplitz\n",
+		    vf_ifp->if_xname, ifrh.ifrh_func);
+		goto done;
+	}
+
+	my_types = hn_rss_type_fromndis(sc->hn_rss_hcap);
+	if ((ifrh.ifrh_types & my_types) == 0) {
+		/* This disables RSS; ignore it then */
+		if_printf(ifp, "%s intersection of RSS types failed.  "
+		    "VF %#x, mine %#x\n", vf_ifp->if_xname,
+		    ifrh.ifrh_types, my_types);
+		goto done;
+	}
+
+	diff_types = my_types ^ ifrh.ifrh_types;
+	my_types &= ifrh.ifrh_types;
+	mbuf_types = my_types;
+
+	/*
+	 * Detect RSS hash value/type confliction.
+	 *
+	 * NOTE:
+	 * We don't disable the hash type, but stop delivery the hash
+	 * value/type through mbufs on RX path.
+	 */
+	if ((my_types & RSS_TYPE_IPV4) &&
+	    (diff_types & ifrh.ifrh_types &
+	     (RSS_TYPE_TCP_IPV4 | RSS_TYPE_UDP_IPV4))) {
+		/* Conflict; disable IPV4 hash type/value delivery. */
+		if_printf(ifp, "disable IPV4 mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_IPV4;
+	}
+	if ((my_types & RSS_TYPE_IPV6) &&
+	    (diff_types & ifrh.ifrh_types &
+	     (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 |
+	      RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX |
+	      RSS_TYPE_IPV6_EX))) {
+		/* Conflict; disable IPV6 hash type/value delivery. */
+		if_printf(ifp, "disable IPV6 mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_IPV6;
+	}
+	if ((my_types & RSS_TYPE_IPV6_EX) &&
+	    (diff_types & ifrh.ifrh_types &
+	     (RSS_TYPE_TCP_IPV6 | RSS_TYPE_UDP_IPV6 |
+	      RSS_TYPE_TCP_IPV6_EX | RSS_TYPE_UDP_IPV6_EX |
+	      RSS_TYPE_IPV6))) {
+		/* Conflict; disable IPV6_EX hash type/value delivery. */
+		if_printf(ifp, "disable IPV6_EX mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_IPV6_EX;
+	}
+	if ((my_types & RSS_TYPE_TCP_IPV6) &&
+	    (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6_EX)) {
+		/* Conflict; disable TCP_IPV6 hash type/value delivery. */
+		if_printf(ifp, "disable TCP_IPV6 mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_TCP_IPV6;
+	}
+	if ((my_types & RSS_TYPE_TCP_IPV6_EX) &&
+	    (diff_types & ifrh.ifrh_types & RSS_TYPE_TCP_IPV6)) {
+		/* Conflict; disable TCP_IPV6_EX hash type/value delivery. */
+		if_printf(ifp, "disable TCP_IPV6_EX mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_TCP_IPV6_EX;
+	}
+	if ((my_types & RSS_TYPE_UDP_IPV6) &&
+	    (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6_EX)) {
+		/* Conflict; disable UDP_IPV6 hash type/value delivery. */
+		if_printf(ifp, "disable UDP_IPV6 mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_UDP_IPV6;
+	}
+	if ((my_types & RSS_TYPE_UDP_IPV6_EX) &&
+	    (diff_types & ifrh.ifrh_types & RSS_TYPE_UDP_IPV6)) {
+		/* Conflict; disable UDP_IPV6_EX hash type/value delivery. */
+		if_printf(ifp, "disable UDP_IPV6_EX mbuf hash delivery\n");
+		mbuf_types &= ~RSS_TYPE_UDP_IPV6_EX;
+	}
+
+	/*
+	 * Indirect table does not matter.
+	 */
+
+	sc->hn_rss_hash = (sc->hn_rss_hcap & NDIS_HASH_FUNCTION_MASK) |
+	    hn_rss_type_tondis(my_types);
+	memcpy(sc->hn_rss.rss_key, ifrk.ifrk_key, sizeof(sc->hn_rss.rss_key));
+	sc->hn_flags |= HN_FLAG_HAS_RSSKEY;
+
+	if (reconf) {
+		error = hn_rss_reconfig(sc);
+		if (error) {
+			/* XXX roll-back? */
+			if_printf(ifp, "hn_rss_reconfig failed: %d\n", error);
+			/* XXX keep going. */
+		}
+	}
+done:
+	/* Hash deliverability for mbufs. */
+	hn_rss_mbuf_hash(sc, hn_rss_type_tondis(mbuf_types));
+}
+
+static void
+hn_vf_rss_restore(struct hn_softc *sc)
+{
+
+	HN_LOCK_ASSERT(sc);
+	KASSERT(sc->hn_flags & HN_FLAG_SYNTH_ATTACHED,
+	    ("%s: synthetic parts are not attached", sc->hn_ifp->if_xname));
+
+	if (sc->hn_rx_ring_inuse == 1)
+		goto done;
+
+	/*
+	 * Restore hash types.  Key does _not_ matter.
+	 */
+	if (sc->hn_rss_hash != sc->hn_rss_hcap) {
+		int error;
+
+		sc->hn_rss_hash = sc->hn_rss_hcap;
+		error = hn_rss_reconfig(sc);
+		if (error) {
+			if_printf(sc->hn_ifp, "hn_rss_reconfig failed: %d\n",
+			    error);
+			/* XXX keep going. */
+		}
+	}
+done:
+	/* Hash deliverability for mbufs. */
+	hn_rss_mbuf_hash(sc, NDIS_HASH_ALL);
+}
+
 static void
 hn_xpnt_vf_setready(struct hn_softc *sc)
 {
@@ -1501,6 +1752,13 @@ hn_xpnt_vf_init(struct hn_softc *sc)
 	 */
 	hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_VF);
 
+	/*
+	 * NOTE:
+	 * Fixup RSS related bits _after_ the VF is brought up, since
+	 * many VFs generate RSS key during it's initialization.
+	 */
+	hn_vf_rss_fixup(sc, true);
+
 	/* Mark transparent mode VF as enabled. */
 	hn_xpnt_vf_setenable(sc);
 }
@@ -1659,7 +1917,8 @@ hn_ifnet_detevent(void *xsc, struct ifnet *ifp)
 		ifp->if_input = sc->hn_vf_input;
 		sc->hn_vf_input = NULL;
 
-		if (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)
+		if ((sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) &&
+		    (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED))
 			hn_nvs_set_datapath(sc, HN_NVS_DATAPATH_SYNTH);
 
 		if (sc->hn_vf_rdytick == 0) {
@@ -1681,11 +1940,18 @@ hn_ifnet_detevent(void *xsc, struct ifnet *ifp)
 			sc->hn_ifp->if_hw_tsomaxsegsize = sc->hn_saved_tsosegsz;
 		}
 
-		/*
+		if (sc->hn_flags & HN_FLAG_SYNTH_ATTACHED) {
-		 * Resume link status management, which was suspended
+			/*
-		 * by hn_ifnet_attevent().
+			 * Restore RSS settings.
-		 */
+			 */
-		hn_resume_mgmt(sc);
+			hn_vf_rss_restore(sc);
+
+			/*
+			 * Resume link status management, which was suspended
+			 * by hn_ifnet_attevent().
+			 */
+			hn_resume_mgmt(sc);
+		}
 	}
 
 	/* Mark transparent mode VF as disabled. */
@@ -1935,6 +2201,12 @@ hn_attach(device_t dev)
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hash",
 	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
 	    hn_rss_hash_sysctl, "A", "RSS hash");
+	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_hashcap",
+	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+	    hn_rss_hcap_sysctl, "A", "RSS hash capabilities");
+	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "mbuf_hash",
+	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
+	    hn_rss_mbuf_sysctl, "A", "RSS hash for mbufs");
 	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rss_ind_size",
 	    CTLFLAG_RD, &sc->hn_rss_ind_size, 0, "RSS indirect entry count");
 #ifndef RSS
@@ -3026,14 +3298,21 @@ hn_rxpkt(struct hn_rx_ring *rxr, const void *data, int dlen,
 {
 	struct ifnet *ifp, *hn_ifp = rxr->hn_ifp;
 	struct mbuf *m_new;
-	int size, do_lro = 0, do_csum = 1;
+	int size, do_lro = 0, do_csum = 1, is_vf = 0;
-	int hash_type;
+	int hash_type = M_HASHTYPE_NONE;
 
-	/*
+	ifp = hn_ifp;
-	 * If the non-transparent mode VF is active, inject this packet
+	if (rxr->hn_rxvf_ifp != NULL) {
-	 * into the VF.
+		/*
-	 */
+		 * Non-transparent mode VF; pretend this packet is from
-	ifp = rxr->hn_rxvf_ifp ? rxr->hn_rxvf_ifp : hn_ifp;
+		 * the VF.
+		 */
+		ifp = rxr->hn_rxvf_ifp;
+		is_vf = 1;
+	} else if (rxr->hn_rx_flags & HN_RX_FLAG_XPNT_VF) {
+		/* Transparent mode VF. */
+		is_vf = 1;
+	}
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 		/*
@@ -3186,16 +3465,6 @@ skip:
 	 * If VF is activated (tranparent/non-transparent mode does not
 	 * matter here).
 	 *
-	 * - Don't setup mbuf hash, if 'options RSS' is set.
-	 *
-	 *   In Azure, when VF is activated, TCP SYN and SYN|ACK go
-	 *   through hn(4) while the rest of segments and ACKs belonging
-	 *   to the same TCP 4-tuple go through the VF.  So don't setup
-	 *   mbuf hash, if a VF is activated and 'options RSS' is not
-	 *   enabled.  hn(4) and the VF may use neither the same RSS
-	 *   hash key nor the same RSS hash function, so the hash value
-	 *   for packets belonging to the same flow could be different!
-	 *
 	 * - Disable LRO
 	 *
 	 *   hn(4) will only receive broadcast packets, multicast packets,
@@ -3206,20 +3475,23 @@ skip:
 	 *   all, since the LRO flush will use hn(4) as the receiving
 	 *   interface; i.e. hn_ifp->if_input(hn_ifp, m).
 	 */
-	if (hn_ifp != ifp || (rxr->hn_rx_flags & HN_RX_FLAG_XPNT_VF)) {
+	if (is_vf)
-		do_lro = 0;	/* disable LRO. */
+		do_lro = 0;
-#ifndef RSS
-		goto skip_hash;	/* skip mbuf hash setup */
-#endif
-	}
 
+	/*
+	 * If VF is activated (tranparent/non-transparent mode does not
+	 * matter here), do _not_ mess with unsupported hash types or
+	 * functions.
+	 */
 	if (info->hash_info != HN_NDIS_HASH_INFO_INVALID) {
 		rxr->hn_rss_pkts++;
 		m_new->m_pkthdr.flowid = info->hash_value;
-		hash_type = M_HASHTYPE_OPAQUE_HASH;
+		if (!is_vf)
+			hash_type = M_HASHTYPE_OPAQUE_HASH;
 		if ((info->hash_info & NDIS_HASH_FUNCTION_MASK) ==
 		    NDIS_HASH_FUNCTION_TOEPLITZ) {
-			uint32_t type = (info->hash_info & NDIS_HASH_TYPE_MASK);
+			uint32_t type = (info->hash_info & NDIS_HASH_TYPE_MASK &
+			    rxr->hn_mbuf_hash);
 
 			/*
 			 * NOTE:
@@ -3256,15 +3528,12 @@ skip:
 				break;
 			}
 		}
-	} else {
+	} else if (!is_vf) {
 		m_new->m_pkthdr.flowid = rxr->hn_rx_idx;
 		hash_type = M_HASHTYPE_OPAQUE;
 	}
 	M_HASHTYPE_SET(m_new, hash_type);
 
-#ifndef RSS
-skip_hash:
-#endif
 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 	if (hn_ifp != ifp) {
 		const struct ether_header *eh;
@@ -3589,20 +3858,7 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
 		else
 			ifrh->ifrh_func = RSS_FUNC_PRIVATE;
-
+		ifrh->ifrh_types = hn_rss_type_fromndis(sc->hn_rss_hash);
-		ifrh->ifrh_types = 0;
-		if (sc->hn_rss_hash & NDIS_HASH_IPV4)
-			ifrh->ifrh_types |= RSS_TYPE_IPV4;
-		if (sc->hn_rss_hash & NDIS_HASH_TCP_IPV4)
-			ifrh->ifrh_types |= RSS_TYPE_TCP_IPV4;
-		if (sc->hn_rss_hash & NDIS_HASH_IPV6)
-			ifrh->ifrh_types |= RSS_TYPE_IPV6;
-		if (sc->hn_rss_hash & NDIS_HASH_IPV6_EX)
-			ifrh->ifrh_types |= RSS_TYPE_IPV6_EX;
-		if (sc->hn_rss_hash & NDIS_HASH_TCP_IPV6)
-			ifrh->ifrh_types |= RSS_TYPE_TCP_IPV6;
-		if (sc->hn_rss_hash & NDIS_HASH_TCP_IPV6_EX)
-			ifrh->ifrh_types |= RSS_TYPE_TCP_IPV6_EX;
 		HN_UNLOCK(sc);
 		break;
 
@@ -4154,6 +4410,16 @@ hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS)
 	if (error || req->newptr == NULL)
 		goto back;
 
+	if ((sc->hn_flags & HN_FLAG_RXVF) ||
+	    (hn_xpnt_vf && sc->hn_vf_ifp != NULL)) {
+		/*
+		 * RSS key is synchronized w/ VF's, don't allow users
+		 * to change it.
+		 */
+		error = EBUSY;
+		goto back;
+	}
+
 	error = SYSCTL_IN(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key));
 	if (error)
 		goto back;
@@ -4219,6 +4485,34 @@ hn_rss_hash_sysctl(SYSCTL_HANDLER_ARGS)
 	return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req);
 }
 
+static int
+hn_rss_hcap_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct hn_softc *sc = arg1;
+	char hash_str[128];
+	uint32_t hash;
+
+	HN_LOCK(sc);
+	hash = sc->hn_rss_hcap;
+	HN_UNLOCK(sc);
+	snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS);
+	return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req);
+}
+
+static int
+hn_rss_mbuf_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct hn_softc *sc = arg1;
+	char hash_str[128];
+	uint32_t hash;
+
+	HN_LOCK(sc);
+	hash = sc->hn_rx_ring[0].hn_mbuf_hash;
+	HN_UNLOCK(sc);
+	snprintf(hash_str, sizeof(hash_str), "%b", hash, NDIS_HASH_BITS);
+	return sysctl_handle_string(oidp, hash_str, sizeof(hash_str), req);
+}
+
 static int
 hn_vf_sysctl(SYSCTL_HANDLER_ARGS)
 {
@@ -4519,6 +4813,7 @@ hn_create_rx_data(struct hn_softc *sc, int ring_cnt)
 			rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_UDP;
 		if (hn_trust_hostip)
 			rxr->hn_trust_hcsum |= HN_TRUST_HCSUM_IP;
+		rxr->hn_mbuf_hash = NDIS_HASH_ALL;
 		rxr->hn_ifp = sc->hn_ifp;
 		if (i < sc->hn_tx_ring_cnt)
 			rxr->hn_txr = &sc->hn_tx_ring[i];
@@ -5980,6 +6275,7 @@ hn_synth_attach(struct hn_softc *sc, int mtu)
 	/* Clear RSS stuffs. */
 	sc->hn_rss_ind_size = 0;
 	sc->hn_rss_hash = 0;
+	sc->hn_rss_hcap = 0;
 
 	/*
 	 * Attach the primary channel _before_ attaching NVS and RNDIS.
@@ -6098,6 +6394,12 @@ hn_synth_attach(struct hn_softc *sc, int mtu)
 		hn_rss_ind_fixup(sc);
 	}
 
+	sc->hn_rss_hash = sc->hn_rss_hcap;
+	if ((sc->hn_flags & HN_FLAG_RXVF) ||
+	    (sc->hn_xvf_flags & HN_XVFFLAG_ENABLED)) {
+		/* NOTE: Don't reconfigure RSS; will do immediately. */
+		hn_vf_rss_fixup(sc, false);
+	}
 	error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE);
 	if (error)
 		goto failed;

sys/dev/hyperv/netvsc/if_hnvar.h

@@ -64,6 +64,7 @@ struct hn_rx_ring {
 	void		*hn_pktbuf;
 	int		hn_pktbuf_len;
 	int		hn_rx_flags;	/* HN_RX_FLAG_ */
+	uint32_t	hn_mbuf_hash;	/* NDIS_HASH_ */
 	uint8_t		*hn_rxbuf;	/* shadow sc->hn_rxbuf */
 	int		hn_rx_idx;
 
@@ -238,7 +239,8 @@ struct hn_softc {
 	uint32_t		hn_rndis_agg_align;
 
 	int			hn_rss_ind_size;
-	uint32_t		hn_rss_hash;	/* NDIS_HASH_ */
+	uint32_t		hn_rss_hash;	/* setting, NDIS_HASH_ */
+	uint32_t		hn_rss_hcap;	/* caps, NDIS_HASH_ */
 	struct ndis_rssprm_toeplitz hn_rss;
 
 	eventhandler_tag	hn_ifaddr_evthand;

sys/dev/hyperv/netvsc/ndis.h

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 Microsoft Corp.
+ * Copyright (c) 2016-2017 Microsoft Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,6 +57,13 @@
 #define	NDIS_HASH_TCP_IPV6		0x00001000
 #define	NDIS_HASH_TCP_IPV6_EX		0x00002000
 
+#define	NDIS_HASH_ALL			(NDIS_HASH_IPV4 |	\
+					 NDIS_HASH_TCP_IPV4 |	\
+					 NDIS_HASH_IPV6 |	\
+					 NDIS_HASH_IPV6_EX |	\
+					 NDIS_HASH_TCP_IPV6 |	\
+					 NDIS_HASH_TCP_IPV6_EX)
+
 /* Hash description for use with printf(9) %b identifier. */
 #define	NDIS_HASH_BITS			\
 	"\20\1TOEPLITZ\11IP4\12TCP4\13IP6\14IP6EX\15TCP6\16TCP6EX"