d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Have ifmp_ring_enqueue() abdicate instead of switch to a consumer

Browse Source 
Move TX out of the enqueue() path. As a result, we need
to have ifmp_ring_check_drainage() pick up from the abdicate state.

We also need to either enqueue the TX task, or check drainage
after calling ifmp_ring_enqueue() to ensure it's sent.

This change results in a 30% small packet forwarding improvement.

Reviewed by:	olivier, sbruno
Approved by:	sbruno (mentor)
Sponsored by:	Limelight Networks
Differential Revision:	https://reviews.freebsd.org/D12439
This commit is contained in:
Stephen Hurd 2017-09-23 16:46:30 +00:00
parent 290e7bac6e
commit 1225d9da9f
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=323954
2 changed files with 7 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
sys/net/iflib.c
View File

@ -3515,8 +3515,7 @@ _task_fn_tx(void *context)
}
if (txq->ift_db_pending)
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE);
else
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
if (ctx->ifc_flags & IFC_LEGACY)
IFDI_INTR_ENABLE(ctx);
else {
@ -3718,16 +3717,14 @@ iflib_if_transmit(if_t ifp, struct mbuf *m)
DBG_COUNTER_INC(tx_seen);
err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE);
GROUPTASK_ENQUEUE(&txq->ift_task);
if (err) {
GROUPTASK_ENQUEUE(&txq->ift_task);
/* support forthcoming later */
#ifdef DRIVER_BACKPRESSURE
txq->ift_closed = TRUE;
#endif
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
m_freem(m);
} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) {
GROUPTASK_ENQUEUE(&txq->ift_task);
}
return (err);

14
sys/net/mp_ring.c
View File

@ -454,18 +454,12 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget)
do {
os.state = ns.state = r->state;
ns.pidx_tail = pidx_stop;
ns.flags = BUSY;
if (os.flags == IDLE)
ns.flags = ABDICATED;
} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
critical_exit();
counter_u64_add(r->enqueues, n);
/*
* Turn into a consumer if some other thread isn't active as a consumer
* already.
*/
if (os.flags != BUSY)
drain_ring_lockless(r, ns, os.flags, budget);
return (0);
}
#endif
@ -476,7 +470,9 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budget)
union ring_state os, ns;
os.state = r->state;
if (os.flags != STALLED || os.pidx_head != os.pidx_tail || r->can_drain(r) == 0)
if ((os.flags != STALLED && os.flags != ABDICATED) || // Only continue in STALLED and ABDICATED
os.pidx_head != os.pidx_tail || // Require work to be available
(os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left
return;
MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */