Have ifmp_ring_enqueue() abdicate instead of switch to a consumer
Move TX out of the enqueue() path. As a result, we need to have ifmp_ring_check_drainage() pick up from the abdicate state. We also need to either enqueue the TX task, or check drainage after calling ifmp_ring_enqueue() to ensure it's sent. This change results in a 30% small packet forwarding improvement. Reviewed by: olivier, sbruno Approved by: sbruno (mentor) Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D12439
This commit is contained in:
parent
290e7bac6e
commit
1225d9da9f
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=323954
@ -3515,8 +3515,7 @@ _task_fn_tx(void *context)
|
||||
}
|
||||
if (txq->ift_db_pending)
|
||||
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE);
|
||||
else
|
||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
|
||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
|
||||
if (ctx->ifc_flags & IFC_LEGACY)
|
||||
IFDI_INTR_ENABLE(ctx);
|
||||
else {
|
||||
@ -3718,16 +3717,14 @@ iflib_if_transmit(if_t ifp, struct mbuf *m)
|
||||
DBG_COUNTER_INC(tx_seen);
|
||||
err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE);
|
||||
|
||||
GROUPTASK_ENQUEUE(&txq->ift_task);
|
||||
if (err) {
|
||||
GROUPTASK_ENQUEUE(&txq->ift_task);
|
||||
/* support forthcoming later */
|
||||
#ifdef DRIVER_BACKPRESSURE
|
||||
txq->ift_closed = TRUE;
|
||||
#endif
|
||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
|
||||
m_freem(m);
|
||||
} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) {
|
||||
GROUPTASK_ENQUEUE(&txq->ift_task);
|
||||
}
|
||||
|
||||
return (err);
|
||||
|
@ -454,18 +454,12 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget)
|
||||
do {
|
||||
os.state = ns.state = r->state;
|
||||
ns.pidx_tail = pidx_stop;
|
||||
ns.flags = BUSY;
|
||||
if (os.flags == IDLE)
|
||||
ns.flags = ABDICATED;
|
||||
} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
|
||||
critical_exit();
|
||||
counter_u64_add(r->enqueues, n);
|
||||
|
||||
/*
|
||||
* Turn into a consumer if some other thread isn't active as a consumer
|
||||
* already.
|
||||
*/
|
||||
if (os.flags != BUSY)
|
||||
drain_ring_lockless(r, ns, os.flags, budget);
|
||||
|
||||
return (0);
|
||||
}
|
||||
#endif
|
||||
@ -476,7 +470,9 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budget)
|
||||
union ring_state os, ns;
|
||||
|
||||
os.state = r->state;
|
||||
if (os.flags != STALLED || os.pidx_head != os.pidx_tail || r->can_drain(r) == 0)
|
||||
if ((os.flags != STALLED && os.flags != ABDICATED) || // Only continue in STALLED and ABDICATED
|
||||
os.pidx_head != os.pidx_tail || // Require work to be available
|
||||
(os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left
|
||||
return;
|
||||
|
||||
MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */
|
||||
|
Loading…
Reference in New Issue
Block a user