d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cxgbe(4): Create two variants of service_iq, one for queues with

Browse Source 
freelists and one for those without.

MFH:		3 weeks
Sponsored by:	Chelsio Communications
This commit is contained in:
Navdeep Parhar 2018-08-11 04:55:47 +00:00
parent e5ae53901a
commit 3098bcfc05
4 changed files with 253 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sys/dev/cxgbe/adapter.h
View File

@ -670,6 +670,7 @@ struct sge_wrq {
#define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
struct sge_nm_rxq {
volatile int nm_state; /* NM_OFF, NM_ON, or NM_BUSY */
struct vi_info *vi;
struct iq_desc *iq_desc;
@ -797,7 +798,6 @@ struct adapter {
struct irq {
struct resource *res;
int rid;
volatile int nm_state; /* NM_OFF, NM_ON, or NM_BUSY */
void *tag;
struct sge_rxq *rxq;
struct sge_nm_rxq *nm_rxq;
@ -1186,9 +1186,10 @@ void release_tid(struct adapter *, int, struct sge_wrq *);
#ifdef DEV_NETMAP
/* t4_netmap.c */
struct sge_nm_rxq;
void cxgbe_nm_attach(struct vi_info *);
void cxgbe_nm_detach(struct vi_info *);
void t4_nm_intr(void *);
void service_nm_rxq(struct sge_nm_rxq *);
#endif
/* t4_sge.c */
@ -1207,7 +1208,10 @@ int t4_setup_vi_queues(struct vi_info *);
int t4_teardown_vi_queues(struct vi_info *);
void t4_intr_all(void *);
void t4_intr(void *);
#ifdef DEV_NETMAP
void t4_nm_intr(void *);
void t4_vi_intr(void *);
#endif
void t4_intr_err(void *);
void t4_intr_evt(void *);
void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);

25
sys/dev/cxgbe/t4_main.c
View File

@ -684,8 +684,8 @@ struct {
#ifdef TCP_OFFLOAD
/*
* service_iq() has an iq and needs the fl. Offset of fl from the iq should be
* exactly the same for both rxq and ofld_rxq.
* service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should
* be exactly the same for both rxq and ofld_rxq.
*/
CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
@ -4819,9 +4819,26 @@ t4_setup_intr_handlers(struct adapter *sc)
#ifdef DEV_NETMAP
if (q < vi->nnmrxq)
irq->nm_rxq = nm_rxq++;
if (irq->nm_rxq != NULL &&
irq->rxq == NULL) {
/* Netmap rx only */
rc = t4_alloc_irq(sc, irq, rid,
t4_nm_intr, irq->nm_rxq, s);
}
if (irq->nm_rxq != NULL &&
irq->rxq != NULL) {
/* NIC and Netmap rx */
rc = t4_alloc_irq(sc, irq, rid,
t4_vi_intr, irq, s);
}
#endif
rc = t4_alloc_irq(sc, irq, rid,
t4_vi_intr, irq, s);
if (irq->rxq != NULL &&
irq->nm_rxq == NULL) {
/* NIC rx only */
rc = t4_alloc_irq(sc, irq, rid,
t4_intr, irq->rxq, s);
}
if (rc != 0)
return (rc);
#ifdef RSS

11
sys/dev/cxgbe/t4_netmap.c
View File

@ -358,8 +358,6 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp,
nm_set_native_flags(na);
for_each_nm_rxq(vi, i, nm_rxq) {
struct irq *irq = &sc->irq[vi->first_intr + i];
kring = na->rx_rings[nm_rxq->nid];
if (!nm_kring_pending_on(kring) ||
nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID)
@ -387,7 +385,7 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp,
t4_write_reg(sc, sc->sge_kdoorbell_reg,
nm_rxq->fl_db_val | V_PIDX(j));
atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON);
atomic_cmpset_int(&nm_rxq->nm_state, NM_OFF, NM_ON);
}
for_each_nm_txq(vi, i, nm_txq) {
@ -459,14 +457,12 @@ cxgbe_netmap_off(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp,
free_nm_txq_hwq(vi, nm_txq);
}
for_each_nm_rxq(vi, i, nm_rxq) {
struct irq *irq = &sc->irq[vi->first_intr + i];
kring = na->rx_rings[nm_rxq->nid];
if (!nm_kring_pending_off(kring) ||
nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID)
continue;
while (!atomic_cmpset_int(&irq->nm_state, NM_ON, NM_OFF))
while (!atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_OFF))
pause("nmst", 1);
free_nm_rxq_hwq(vi, nm_rxq);
@ -955,9 +951,8 @@ handle_nm_sge_egr_update(struct adapter *sc, struct ifnet *ifp,
}
void
t4_nm_intr(void *arg)
service_nm_rxq(struct sge_nm_rxq *nm_rxq)
{
struct sge_nm_rxq *nm_rxq = arg;
struct vi_info *vi = nm_rxq->vi;
struct adapter *sc = vi->pi->adapter;
struct ifnet *ifp = vi->ifp;

355
sys/dev/cxgbe/t4_sge.c
View File

@ -204,6 +204,7 @@ struct sgl {
};
static int service_iq(struct sge_iq *, int);
static int service_iq_fl(struct sge_iq *, int);
static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
@ -1333,8 +1334,12 @@ t4_teardown_vi_queues(struct vi_info *vi)
}
/*
* Deals with errors and the firmware event queue. All data rx queues forward
* their interrupt to the firmware event queue.
* Interrupt handler when the driver is using only 1 interrupt. This is a very
* unusual scenario.
*
* a) Deals with errors, if any.
* b) Services firmware event queue, which is taking interrupts for all other
* queues.
*/
void
t4_intr_all(void *arg)
@ -1342,14 +1347,16 @@ t4_intr_all(void *arg)
struct adapter *sc = arg;
struct sge_iq *fwq = &sc->sge.fwq;
MPASS(sc->intr_count == 1);
t4_intr_err(arg);
if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
service_iq(fwq, 0);
atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
}
t4_intr_evt(fwq);
}
/* Deals with error interrupts */
/*
* Interrupt handler for errors (installed directly when multiple interrupts are
* being used, or called by t4_intr_all).
*/
void
t4_intr_err(void *arg)
{
@ -1359,6 +1366,10 @@ t4_intr_err(void *arg)
t4_slow_intr_handler(sc);
}
/*
* Interrupt handler for iq-only queues. The firmware event queue is the only
* such queue right now.
*/
void
t4_intr_evt(void *arg)
{
@ -1370,90 +1381,74 @@ t4_intr_evt(void *arg)
}
}
/*
* Interrupt handler for iq+fl queues.
*/
void
t4_intr(void *arg)
{
struct sge_iq *iq = arg;
if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
service_iq(iq, 0);
service_iq_fl(iq, 0);
atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
}
}
#ifdef DEV_NETMAP
/*
* Interrupt handler for netmap rx queues.
*/
void
t4_nm_intr(void *arg)
{
struct sge_nm_rxq *nm_rxq = arg;
if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
service_nm_rxq(nm_rxq);
atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
}
}
/*
* Interrupt handler for vectors shared between NIC and netmap rx queues.
*/
void
t4_vi_intr(void *arg)
{
struct irq *irq = arg;
#ifdef DEV_NETMAP
if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) {
t4_nm_intr(irq->nm_rxq);
atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON);
}
MPASS(irq->nm_rxq != NULL);
t4_nm_intr(irq->nm_rxq);
MPASS(irq->rxq != NULL);
t4_intr(irq->rxq);
}
#endif
if (irq->rxq != NULL)
t4_intr(irq->rxq);
}
static inline int
sort_before_lro(struct lro_ctrl *lro)
{
return (lro->lro_mbuf_max != 0);
}
/*
* Deals with anything and everything on the given ingress queue.
* Deals with interrupts on an iq-only (no freelist) queue.
*/
static int
service_iq(struct sge_iq *iq, int budget)
{
struct sge_iq *q;
struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
struct sge_fl *fl; /* Use iff IQ_HAS_FL */
struct adapter *sc = iq->adapter;
struct iq_desc *d = &iq->desc[iq->cidx];
int ndescs = 0, limit;
int rsp_type, refill;
int rsp_type;
uint32_t lq;
uint16_t fl_hw_cidx;
struct mbuf *m0;
STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
#if defined(INET) || defined(INET6)
const struct timeval lro_timeout = {0, sc->lro_timeout};
struct lro_ctrl *lro = &rxq->lro;
#endif
KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
KASSERT((iq->flags & IQ_HAS_FL) == 0,
("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
iq->flags));
MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
limit = budget ? budget : iq->qsize / 16;
if (iq->flags & IQ_HAS_FL) {
fl = &rxq->fl;
fl_hw_cidx = fl->hw_cidx; /* stable snapshot */
} else {
fl = NULL;
fl_hw_cidx = 0; /* to silence gcc warning */
}
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_ADJ_CREDIT) {
MPASS(sort_before_lro(lro));
iq->flags &= ~IQ_ADJ_CREDIT;
if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
tcp_lro_flush_all(lro);
t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
V_INGRESSQID((u32)iq->cntxt_id) |
V_SEINTARM(iq->intr_params));
return (0);
}
ndescs = 1;
}
#else
MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
#endif
/*
* We always come back and check the descriptor ring for new indirect
* interrupts and other responses after running a single handler.
@ -1463,74 +1458,40 @@ service_iq(struct sge_iq *iq, int budget)
rmb();
refill = 0;
m0 = NULL;
rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
lq = be32toh(d->rsp.pldbuflen_qid);
switch (rsp_type) {
case X_RSPD_TYPE_FLBUF:
panic("%s: data for an iq (%p) with no freelist",
__func__, iq);
KASSERT(iq->flags & IQ_HAS_FL,
("%s: data for an iq (%p) with no freelist",
__func__, iq));
m0 = get_fl_payload(sc, fl, lq);
if (__predict_false(m0 == NULL))
goto process_iql;
refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
#ifdef T4_PKT_TIMESTAMP
/*
* 60 bit timestamp for the payload is
* *(uint64_t *)m0->m_pktdat. Note that it is
* in the leading free-space in the mbuf. The
* kernel can clobber it during a pullup,
* m_copymdata, etc. You need to make sure that
* the mbuf reaches you unmolested if you care
* about the timestamp.
*/
*(uint64_t *)m0->m_pktdat =
be64toh(ctrl->u.last_flit) &
0xfffffffffffffff;
#endif
/* fall through */
/* NOTREACHED */
case X_RSPD_TYPE_CPL:
KASSERT(d->rss.opcode < NUM_CPL_CMDS,
("%s: bad opcode %02x.", __func__,
d->rss.opcode));
t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL);
break;
case X_RSPD_TYPE_INTR:
/*
* Interrupts should be forwarded only to queues
* that are not forwarding their interrupts.
* This means service_iq can recurse but only 1
* level deep.
*/
KASSERT(budget == 0,
("%s: budget %u, rsp_type %u", __func__,
budget, rsp_type));
/*
* There are 1K interrupt-capable queues (qids 0
* through 1023). A response type indicating a
* forwarded interrupt with a qid >= 1K is an
* iWARP async notification.
*/
if (lq >= 1024) {
t4_an_handler(iq, &d->rsp);
break;
}
if (__predict_true(lq >= 1024)) {
t4_an_handler(iq, &d->rsp);
break;
}
q = sc->sge.iqmap[lq - sc->sge.iq_start -
sc->sge.iq_base];
if (atomic_cmpset_int(&q->state, IQS_IDLE,
IQS_BUSY)) {
if (service_iq(q, q->qsize / 16) == 0) {
if (service_iq_fl(q, q->qsize / 16) == 0) {
atomic_cmpset_int(&q->state,
IQS_BUSY, IQS_IDLE);
} else {
@ -1563,33 +1524,12 @@ service_iq(struct sge_iq *iq, int budget)
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
ndescs = 0;
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_LRO_ENABLED &&
!sort_before_lro(lro) &&
sc->lro_timeout != 0) {
tcp_lro_flush_inactive(lro,
&lro_timeout);
}
#endif
if (budget) {
if (iq->flags & IQ_HAS_FL) {
FL_LOCK(fl);
refill_fl(sc, fl, 32);
FL_UNLOCK(fl);
}
return (EINPROGRESS);
}
}
if (refill) {
FL_LOCK(fl);
refill_fl(sc, fl, 32);
FL_UNLOCK(fl);
fl_hw_cidx = fl->hw_cidx;
}
}
process_iql:
if (STAILQ_EMPTY(&iql))
break;
@ -1599,12 +1539,167 @@ service_iq(struct sge_iq *iq, int budget)
*/
q = STAILQ_FIRST(&iql);
STAILQ_REMOVE_HEAD(&iql, link);
if (service_iq(q, q->qsize / 8) == 0)
if (service_iq_fl(q, q->qsize / 8) == 0)
atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
else
STAILQ_INSERT_TAIL(&iql, q, link);
}
t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
return (0);
}
static inline int
sort_before_lro(struct lro_ctrl *lro)
{
return (lro->lro_mbuf_max != 0);
}
/*
* Deals with interrupts on an iq+fl queue.
*/
static int
service_iq_fl(struct sge_iq *iq, int budget)
{
struct sge_rxq *rxq = iq_to_rxq(iq);
struct sge_fl *fl;
struct adapter *sc = iq->adapter;
struct iq_desc *d = &iq->desc[iq->cidx];
int ndescs = 0, limit;
int rsp_type, refill, starved;
uint32_t lq;
uint16_t fl_hw_cidx;
struct mbuf *m0;
#if defined(INET) || defined(INET6)
const struct timeval lro_timeout = {0, sc->lro_timeout};
struct lro_ctrl *lro = &rxq->lro;
#endif
KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
MPASS(iq->flags & IQ_HAS_FL);
limit = budget ? budget : iq->qsize / 16;
fl = &rxq->fl;
fl_hw_cidx = fl->hw_cidx; /* stable snapshot */
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_ADJ_CREDIT) {
MPASS(sort_before_lro(lro));
iq->flags &= ~IQ_ADJ_CREDIT;
if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
tcp_lro_flush_all(lro);
t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
V_INGRESSQID((u32)iq->cntxt_id) |
V_SEINTARM(iq->intr_params));
return (0);
}
ndescs = 1;
}
#else
MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
#endif
while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
rmb();
refill = 0;
m0 = NULL;
rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
lq = be32toh(d->rsp.pldbuflen_qid);
switch (rsp_type) {
case X_RSPD_TYPE_FLBUF:
m0 = get_fl_payload(sc, fl, lq);
if (__predict_false(m0 == NULL))
goto out;
refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
#ifdef T4_PKT_TIMESTAMP
/*
* 60 bit timestamp for the payload is
* *(uint64_t *)m0->m_pktdat. Note that it is
* in the leading free-space in the mbuf. The
* kernel can clobber it during a pullup,
* m_copymdata, etc. You need to make sure that
* the mbuf reaches you unmolested if you care
* about the timestamp.
*/
*(uint64_t *)m0->m_pktdat =
be64toh(ctrl->u.last_flit) & 0xfffffffffffffff;
#endif
/* fall through */
case X_RSPD_TYPE_CPL:
KASSERT(d->rss.opcode < NUM_CPL_CMDS,
("%s: bad opcode %02x.", __func__, d->rss.opcode));
t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
break;
case X_RSPD_TYPE_INTR:
/*
* There are 1K interrupt-capable queues (qids 0
* through 1023). A response type indicating a
* forwarded interrupt with a qid >= 1K is an
* iWARP async notification. That is the only
* acceptable indirect interrupt on this queue.
*/
if (__predict_false(lq < 1024)) {
panic("%s: indirect interrupt on iq_fl %p "
"with qid %u", __func__, iq, lq);
}
t4_an_handler(iq, &d->rsp);
break;
default:
KASSERT(0, ("%s: illegal response type %d on iq %p",
__func__, rsp_type, iq));
log(LOG_ERR, "%s: illegal response type %d on iq %p",
device_get_nameunit(sc->dev), rsp_type, iq);
break;
}
d++;
if (__predict_false(++iq->cidx == iq->sidx)) {
iq->cidx = 0;
iq->gen ^= F_RSPD_GEN;
d = &iq->desc[0];
}
if (__predict_false(++ndescs == limit)) {
t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
V_INGRESSQID(iq->cntxt_id) |
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
ndescs = 0;
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_LRO_ENABLED &&
!sort_before_lro(lro) &&
sc->lro_timeout != 0) {
tcp_lro_flush_inactive(lro, &lro_timeout);
}
#endif
if (budget) {
FL_LOCK(fl);
refill_fl(sc, fl, 32);
FL_UNLOCK(fl);
return (EINPROGRESS);
}
}
if (refill) {
FL_LOCK(fl);
refill_fl(sc, fl, 32);
FL_UNLOCK(fl);
fl_hw_cidx = fl->hw_cidx;
}
}
out:
#if defined(INET) || defined(INET6)
if (iq->flags & IQ_LRO_ENABLED) {
if (ndescs > 0 && lro->lro_mbuf_count > 8) {
@ -1621,15 +1716,11 @@ service_iq(struct sge_iq *iq, int budget)
t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
if (iq->flags & IQ_HAS_FL) {
int starved;
FL_LOCK(fl);
starved = refill_fl(sc, fl, 64);
FL_UNLOCK(fl);
if (__predict_false(starved != 0))
add_fl_to_sfl(sc, fl);
}
FL_LOCK(fl);
starved = refill_fl(sc, fl, 64);
FL_UNLOCK(fl);
if (__predict_false(starved != 0))
add_fl_to_sfl(sc, fl);
return (0);
}