From 7d93aa7db1e12ce3fc0d5aca0245219d7a8a8d9d Mon Sep 17 00:00:00 2001
From: scottl <scottl@FreeBSD.org>
Date: Tue, 15 Apr 2014 08:08:44 +0000
Subject: [PATCH] MFC r261558

Add a new sysctl, dev.cxgbe.N.rsrv_noflow, and a companion tunable,
hw.cxgbe.rsrv_noflow.  When set, queue 0 of the port is reserved for
TX packets without a flowid.  The hash value of packets with a flowid
is bumped up by 1.  The intent is to provide a private queue for
link-level packets like LACP that is unlikely to overflow or suffer
deep queue latency.
---
 sys/dev/cxgbe/adapter.h |  1 +
 sys/dev/cxgbe/t4_main.c | 36 +++++++++++++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 9bf047fb4cc3..d13bbd15f8dd 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -208,6 +208,7 @@ struct port_info {
 	/* These need to be int as they are used in sysctl */
 	int ntxq;	/* # of tx queues */
 	int first_txq;	/* index of first tx queue */
+	int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */
 	int nrxq;	/* # of rx queues */
 	int first_rxq;	/* index of first rx queue */
 #ifdef TCP_OFFLOAD
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 3fb4320e188b..cf47df3271aa 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -197,6 +197,9 @@ TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 static int t4_nrxq1g = -1;
 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 
+static int t4_rsrv_noflowq = 0;
+TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
+
 #ifdef TCP_OFFLOAD
 #define NOFLDTXQ_10G 8
 static int t4_nofldtxq10g = -1;
@@ -299,6 +302,7 @@ struct intrs_and_queues {
 	int nrxq10g;		/* # of NIC rxq's for each 10G port */
 	int ntxq1g;		/* # of NIC txq's for each 1G port */
 	int nrxq1g;		/* # of NIC rxq's for each 1G port */
+	int rsrv_noflowq;	/* Flag whether to reserve queue 0 */
 #ifdef TCP_OFFLOAD
 	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
 	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
@@ -375,6 +379,7 @@ static int cxgbe_sysctls(struct port_info *);
 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
+static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
@@ -782,6 +787,11 @@ t4_attach(device_t dev)
 			pi->ntxq = iaq.ntxq1g;
 		}
 
+		if (pi->ntxq > 1)
+			pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
+		else
+			pi->rsrv_noflowq = 0;
+
 		rqidx += pi->nrxq;
 		tqidx += pi->ntxq;
 
@@ -1267,7 +1277,8 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
 	}
 
 	if (m->m_flags & M_FLOWID)
-		txq += (m->m_pkthdr.flowid % pi->ntxq);
+		txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
+		    + pi->rsrv_noflowq);
 	br = txq->br;
 
 	if (TXQ_TRYLOCK(txq) == 0) {
@@ -1719,6 +1730,7 @@ cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
 	iaq->ntxq1g = t4_ntxq1g;
 	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
 	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
+	iaq->rsrv_noflowq = t4_rsrv_noflowq;
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
 		iaq->nofldtxq10g = t4_nofldtxq10g;
@@ -4498,6 +4510,9 @@ cxgbe_sysctls(struct port_info *pi)
 	    &pi->first_rxq, 0, "index of first rx queue");
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
 	    &pi->first_txq, 0, "index of first tx queue");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
+	    CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
+	    "Reserve queue 0 for non-flowid packets");
 
 #ifdef TCP_OFFLOAD
 	if (is_offload(pi->adapter)) {
@@ -4751,6 +4766,25 @@ sysctl_btphy(SYSCTL_HANDLER_ARGS)
 	return (rc);
 }
 
+static int
+sysctl_noflowq(SYSCTL_HANDLER_ARGS)
+{
+	struct port_info *pi = arg1;
+	int rc, val;
+
+	val = pi->rsrv_noflowq;
+	rc = sysctl_handle_int(oidp, &val, 0, req);
+	if (rc != 0 || req->newptr == NULL)
+		return (rc);
+
+	if ((val >= 1) && (pi->ntxq > 1))
+		pi->rsrv_noflowq = 1;
+	else
+		pi->rsrv_noflowq = 0;
+
+	return (rc);
+}
+
 static int
 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
 {