diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 9742da01124c..2cc7aa69a044 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -29,6 +29,8 @@ __FBSDID("$FreeBSD$"); #include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_ratelimit.h" #ifdef TCP_OFFLOAD #include <sys/param.h> @@ -186,6 +188,76 @@ send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) t4_wrq_tx(sc, wr); } +#ifdef RATELIMIT +/* + * Input is Bytes/second (so_max_pacing-rate), chip counts in Kilobits/second. + */ +static int +update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) +{ + int tc_idx, rc; + const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000; + const int port_id = toep->vi->pi->port_id; + + CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps); + + if (kbps == 0) { + /* unbind */ + tc_idx = -1; + } else { + rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx); + if (rc != 0) + return (rc); + MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls); + } + + if (toep->tc_idx != tc_idx) { + struct wrqe *wr; + struct fw_flowc_wr *flowc; + int nparams = 1, flowclen, flowclen16; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + + flowclen = sizeof(*flowc) + nparams * sizeof(struct + fw_flowc_mnemval); + flowclen16 = howmany(flowclen, 16); + if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 || + (wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq)) == NULL) { + if (tc_idx >= 0) + t4_release_cl_rl_kbps(sc, port_id, tc_idx); + return (ENOMEM); + } + + flowc = wrtod(wr); + memset(flowc, 0, wr->wr_len); + + flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | + V_FW_FLOWC_WR_NPARAMS(nparams)); + flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) | + V_FW_WR_FLOWID(toep->tid)); + + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; + if (tc_idx == -1) + flowc->mnemval[0].val = htobe32(0xff); + else + flowc->mnemval[0].val = htobe32(tc_idx); + + txsd->tx_credits = flowclen16; + txsd->plen = 0; + toep->tx_credits -= txsd->tx_credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + t4_wrq_tx(sc, wr); + } + + if (toep->tc_idx >= 0) + t4_release_cl_rl_kbps(sc, port_id, toep->tc_idx); + toep->tc_idx = tc_idx; + + return (0); +} +#endif + void send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) { @@ -619,7 +691,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int tx_credits, shove, compl, sowwakeup; - struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + struct ofld_tx_sdesc *txsd; bool aiotx_mbuf_seen; INP_WLOCK_ASSERT(inp); @@ -638,6 +710,13 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; +#ifdef RATELIMIT + if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && + (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { + inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; + } +#endif + /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. @@ -648,6 +727,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) return; } + txsd = &toep->txsd[toep->txsd_pidx]; do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c index e66210ec1195..64d8fd092a6e 100644 --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" +#include "opt_ratelimit.h" #include <sys/param.h> #include <sys/types.h> @@ -156,6 +157,7 @@ alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags) refcount_init(&toep->refcount, 1); toep->td = sc->tom_softc; toep->vi = vi; + toep->tc_idx = -1; toep->tx_total = tx_credits; toep->tx_credits = tx_credits; toep->ofld_txq = &sc->sge.ofld_txq[txqid]; @@ -312,6 +314,10 @@ release_offload_resources(struct toepcb *toep) if (toep->ce) release_lip(td, toep->ce); +#ifdef RATELIMIT + if (toep->tc_idx != -1) + t4_release_cl_rl_kbps(sc, toep->vi->pi->port_id, toep->tc_idx); +#endif mtx_lock(&td->toep_list_lock); TAILQ_REMOVE(&td->toep_list, toep, link); mtx_unlock(&td->toep_list_lock); diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index 5a774fd824ae..452d99ba46eb 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -149,6 +149,7 @@ struct toepcb { struct l2t_entry *l2te; /* L2 table entry used by this connection */ struct clip_entry *ce; /* CLIP table entry used by this tid */ int tid; /* Connection identifier */ + int tc_idx; /* traffic class that this tid is bound to */ /* tx credit handling */ u_int tx_total; /* total tx WR credits (in 16B units) */ diff --git a/sys/modules/cxgbe/tom/Makefile b/sys/modules/cxgbe/tom/Makefile index 96daf2c441a8..f7abba94b96e 100644 --- a/sys/modules/cxgbe/tom/Makefile +++ b/sys/modules/cxgbe/tom/Makefile @@ -10,6 +10,7 @@ SRCS= bus_if.h SRCS+= device_if.h SRCS+= opt_inet.h SRCS+= opt_inet6.h +SRCS+= opt_ratelimit.h SRCS+= pci_if.h SRCS+= t4_connect.c SRCS+= t4_cpl_io.c