cxgbe(4): Allow the driver to specify a burst size when configuring a

traffic class for rate limiting.

Add experimental knobs that allow the user to specify a default pktsize
and burstsize for traffic classes associated with a port:

dev.<ifname>.<instance>.tc.pktsize
dev.<ifname>.<instance>.tc.burstsize

Sponsored by:	Chelsio Communications

sys/dev/cxgbe/adapter.h

@@ -249,6 +249,7 @@ struct tx_cl_rl_params {
 	enum fw_sched_params_mode mode;		/* aggr or per-flow */
 	uint32_t maxrate;
 	uint16_t pktsize;
+	uint16_t burstsize;
 };
 
 /* Tx scheduler parameters for a channel/port */
@@ -259,7 +260,9 @@ struct tx_sched_params {
 	/* Class WRR */
 	/* XXX */
 
-	/* Class Rate Limiter */
+	/* Class Rate Limiter (including the default pktsize and burstsize). */
+	int pktsize;
+	int burstsize;
 	struct tx_cl_rl_params cl_rl[];
 };
 

sys/dev/cxgbe/common/common.h

@@ -814,7 +814,7 @@ int t4_sched_config(struct adapter *adapter, int type, int minmaxen,
 int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 		    int rateunit, int ratemode, int channel, int cl,
 		    int minrate, int maxrate, int weight, int pktsize,
-		    int sleep_ok);
+		    int burstsize, int sleep_ok);
 int t4_sched_params_ch_rl(struct adapter *adapter, int channel, int ratemode,
 			  unsigned int maxrate, int sleep_ok);
 int t4_sched_params_cl_wrr(struct adapter *adapter, int channel, int cl,

sys/dev/cxgbe/common/t4_hw.c

@@ -9803,7 +9803,7 @@ int t4_sched_config(struct adapter *adapter, int type, int minmaxen,
 int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 		    int rateunit, int ratemode, int channel, int cl,
 		    int minrate, int maxrate, int weight, int pktsize,
-		    int sleep_ok)
+		    int burstsize, int sleep_ok)
 {
 	struct fw_sched_cmd cmd;
 
@@ -9825,6 +9825,7 @@ int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 	cmd.u.params.max = cpu_to_be32(maxrate);
 	cmd.u.params.weight = cpu_to_be16(weight);
 	cmd.u.params.pktsize = cpu_to_be16(pktsize);
+	cmd.u.params.burstsize = cpu_to_be16(burstsize);
 
 	return t4_wr_mbox_meat(adapter,adapter->mbox, &cmd, sizeof(cmd),
 			       NULL, sleep_ok);

sys/dev/cxgbe/t4_main.c

@@ -6008,6 +6008,13 @@ cxgbe_sysctls(struct port_info *pi)
 	 */
 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL,
 	    "Tx scheduler traffic classes (cl_rl)");
+	children2 = SYSCTL_CHILDREN(oid);
+	SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize",
+	    CTLFLAG_RW, &pi->sched_params->pktsize, 0,
+	    "pktsize for per-flow cl-rl (0 means up to the driver )");
+	SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize",
+	    CTLFLAG_RW, &pi->sched_params->burstsize, 0,
+	    "burstsize for per-flow cl-rl (0 means up to the driver)");
 	for (i = 0; i < sc->chip_params->nsched_cls; i++) {
 		struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
 

sys/dev/cxgbe/t4_sched.c

@@ -211,7 +211,7 @@ set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p,
 	}
 	rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode,
 	    fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate,
-	    p->weight, p->pktsize, sleep_ok);
+	    p->weight, p->pktsize, 0, sleep_ok);
 	end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD);
 
 	if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
@@ -258,7 +258,7 @@ update_tx_sched(void *context, int pending)
 			rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED,
 			    FW_SCHED_PARAMS_LEVEL_CL_RL, tc->mode, tc->rateunit,
 			    tc->ratemode, pi->tx_chan, j, 0, tc->maxrate, 0,
-			    tc->pktsize, 1);
+			    tc->pktsize, tc->burstsize, 1);
 			end_synchronized_op(sc, 0);
 
 			mtx_lock(&sc->tc_lock);
@@ -481,7 +481,7 @@ int
 t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
     int *tc_idx)
 {
-	int rc = 0, fa = -1, i;
+	int rc = 0, fa = -1, i, pktsize, burstsize;
 	bool update;
 	struct tx_cl_rl_params *tc;
 	struct port_info *pi;
@@ -489,7 +489,16 @@ t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
 	MPASS(port_id >= 0 && port_id < sc->params.nports);
 
 	pi = sc->port[port_id];
+	if (pi->sched_params->pktsize > 0)
+		pktsize = pi->sched_params->pktsize;
+	else
+		pktsize = pi->vi[0].ifp->if_mtu;
+	if (pi->sched_params->burstsize > 0)
+		burstsize = pi->sched_params->burstsize;
+	else
+		burstsize = pktsize * 4;
 	tc = &pi->sched_params->cl_rl[0];
+
 	update = false;
 	mtx_lock(&sc->tc_lock);
 	for (i = 0; i < sc->chip_params->nsched_cls; i++, tc++) {
@@ -499,8 +508,8 @@ t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
 		if (tc->ratemode == FW_SCHED_PARAMS_RATE_ABS &&
 		    tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE &&
 		    tc->mode == FW_SCHED_PARAMS_MODE_FLOW &&
-		    tc->maxrate == maxrate &&
+		    tc->maxrate == maxrate && tc->pktsize == pktsize &&
-		    tc->pktsize == pi->vi[0].ifp->if_mtu) {
+		    tc->burstsize == burstsize) {
 			tc->refcount++;
 			*tc_idx = i;
 			if ((tc->flags & (CLRL_ERR | CLRL_ASYNC | CLRL_SYNC)) ==
@@ -519,7 +528,8 @@ t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
 		tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
 		tc->mode = FW_SCHED_PARAMS_MODE_FLOW;
 		tc->maxrate = maxrate;
-		tc->pktsize = pi->vi[0].ifp->if_mtu;
+		tc->pktsize = pktsize;
+		tc->burstsize = burstsize;
 		*tc_idx = fa;
 		update = true;
 	} else {