From 6eb3180fb29ecd49c35559ab0af16211c2026e92 Mon Sep 17 00:00:00 2001 From: Navdeep Parhar Date: Thu, 4 Jul 2013 21:19:01 +0000 Subject: [PATCH] - Make note of interface MTU change if the rx queues exist, and not just when the interface is up. - Add a tunable to control the TOE's rx coalesce feature (enabled by default as it always has been). Consider the interface MTU or the coalesce size when deciding which cluster zone to use to fill the offload rx queue's free list. The tunable is: dev.{t4nex,t5nex}..toe.rx_coalesce MFC after: 1 day --- sys/dev/cxgbe/adapter.h | 1 - sys/dev/cxgbe/offload.h | 1 + sys/dev/cxgbe/t4_main.c | 9 +++++++-- sys/dev/cxgbe/t4_sge.c | 35 +++++++++++++++++++++++++++++++--- sys/dev/cxgbe/tom/t4_connect.c | 3 ++- sys/dev/cxgbe/tom/t4_listen.c | 3 ++- 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index df0976a766be..38af49f2612a 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -132,7 +132,6 @@ enum { #else FL_BUF_SIZES = 3, /* cluster, jumbo9k, jumbo16k */ #endif - OFLD_BUF_SIZE = MJUM16BYTES, /* size of fl buffer for TOE rxq */ CTRL_EQ_QSIZE = 128, diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h index 60907756f3a8..7f4b38e72e2b 100644 --- a/sys/dev/cxgbe/offload.h +++ b/sys/dev/cxgbe/offload.h @@ -140,6 +140,7 @@ struct tom_tunables { int ddp; int indsz; int ddp_thres; + int rx_coalesce; }; int t4_register_uld(struct uld_info *); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 6f96390edb26..fde79c41a128 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -1072,9 +1072,10 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) if (rc) return (rc); ifp->if_mtu = mtu; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (pi->flags & PORT_INIT_DONE) { t4_update_fl_bufsize(ifp); - rc = update_mac_settings(pi, XGMAC_MTU); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + rc = update_mac_settings(pi, XGMAC_MTU); } end_synchronized_op(sc, 0); break; @@ -4381,6 +4382,10 @@ t4_sysctls(struct adapter *sc) G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, &sc->tt.ddp_thres, 0, "DDP threshold"); + + sc->tt.rx_coalesce = 1; + SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", + CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); } #endif diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 89eee2d073c8..586137a83afd 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -659,6 +659,18 @@ mtu_to_bufsize(int mtu) return (bufsize); } +#ifdef TCP_OFFLOAD +static inline int +mtu_to_bufsize_toe(struct adapter *sc, int mtu) +{ + + if (sc->tt.rx_coalesce) + return (G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2))); + + return (mtu); +} +#endif + int t4_setup_port_queues(struct port_info *pi) { @@ -673,9 +685,10 @@ t4_setup_port_queues(struct port_info *pi) #endif char name[16]; struct adapter *sc = pi->adapter; + struct ifnet *ifp = pi->ifp; struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev); struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); - int bufsize = mtu_to_bufsize(pi->ifp->if_mtu); + int bufsize; oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD, NULL, "rx queues"); @@ -696,6 +709,7 @@ t4_setup_port_queues(struct port_info *pi) * a) initialize iq and fl * b) allocate queue iff it will take direct interrupts. */ + bufsize = mtu_to_bufsize(ifp->if_mtu); for_each_rxq(pi, i, rxq) { init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq, @@ -719,6 +733,7 @@ t4_setup_port_queues(struct port_info *pi) } #ifdef TCP_OFFLOAD + bufsize = mtu_to_bufsize_toe(sc, ifp->if_mtu); for_each_ofld_rxq(pi, i, ofld_rxq) { init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, @@ -726,7 +741,7 @@ t4_setup_port_queues(struct port_info *pi) snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", device_get_nameunit(pi->dev), i); - init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, OFLD_BUF_SIZE, name); + init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, name); if (sc->flags & INTR_DIRECT || (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) { @@ -1557,9 +1572,13 @@ t4_update_fl_bufsize(struct ifnet *ifp) { struct port_info *pi = ifp->if_softc; struct sge_rxq *rxq; +#ifdef TCP_OFFLOAD + struct sge_ofld_rxq *ofld_rxq; +#endif struct sge_fl *fl; - int i, bufsize = mtu_to_bufsize(ifp->if_mtu); + int i, bufsize; + bufsize = mtu_to_bufsize(ifp->if_mtu); for_each_rxq(pi, i, rxq) { fl = &rxq->fl; @@ -1567,6 +1586,16 @@ t4_update_fl_bufsize(struct ifnet *ifp) set_fl_tag_idx(fl, bufsize); FL_UNLOCK(fl); } +#ifdef TCP_OFFLOAD + bufsize = mtu_to_bufsize_toe(pi->adapter, ifp->if_mtu); + for_each_ofld_rxq(pi, i, ofld_rxq) { + fl = &ofld_rxq->fl; + + FL_LOCK(fl); + set_fl_tag_idx(fl, bufsize); + FL_UNLOCK(fl); + } +#endif } int diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c index 11cb4fc28948..08267cdf9f52 100644 --- a/sys/dev/cxgbe/tom/t4_connect.c +++ b/sys/dev/cxgbe/tom/t4_connect.c @@ -251,7 +251,8 @@ calc_opt2a(struct socket *so, struct toepcb *toep) opt2 |= F_T5_OPT_2_VALID; opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */ } - opt2 |= V_RX_COALESCE(M_RX_COALESCE); + if (sc->tt.rx_coalesce) + opt2 |= V_RX_COALESCE(M_RX_COALESCE); #ifdef USE_DDP_RX_FLOW_CONTROL if (toep->ulp_mode == ULP_MODE_TCPDDP) diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c index 7259a299fe8e..9e1dc8041d54 100644 --- a/sys/dev/cxgbe/tom/t4_listen.c +++ b/sys/dev/cxgbe/tom/t4_listen.c @@ -1021,7 +1021,8 @@ calc_opt2p(struct adapter *sc, struct port_info *pi, int rxqid, opt2 |= F_T5_OPT_2_VALID; opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */ } - opt2 |= V_RX_COALESCE(M_RX_COALESCE); + if (sc->tt.rx_coalesce) + opt2 |= V_RX_COALESCE(M_RX_COALESCE); #ifdef USE_DDP_RX_FLOW_CONTROL if (ulp_mode == ULP_MODE_TCPDDP)