iflib: Fix some nits in the rx refill code.
- Get rid of the ifl_vm_addrs array. It is not used by any existing consumer, so we are just dirtying a couple of cache lines for no reason. - Use uma_zalloc(fl->ifl_zone) instead of m_cljget(). Otherwise m_cljget() is doing unnecessary work to look up the correct zone, when iflib already knows what that zone is. - ifl_gen is only used when INVARIANTS is on, so make that more clear. - Fix some style nits and inconsistencies. Reviewed by: gallatin Tested by: pho MFC after: 2 weeks Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D25490
This commit is contained in:
parent
a363e1d4d0
commit
b256d25c50
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=362963
@ -206,8 +206,6 @@ struct iflib_ctx {
|
|||||||
#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
|
#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
|
||||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill
|
#define isc_rxd_refill ifc_txrx.ift_rxd_refill
|
||||||
#define isc_rxd_flush ifc_txrx.ift_rxd_flush
|
#define isc_rxd_flush ifc_txrx.ift_rxd_flush
|
||||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill
|
|
||||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill
|
|
||||||
#define isc_legacy_intr ifc_txrx.ift_legacy_intr
|
#define isc_legacy_intr ifc_txrx.ift_legacy_intr
|
||||||
eventhandler_tag ifc_vlan_attach_event;
|
eventhandler_tag ifc_vlan_attach_event;
|
||||||
eventhandler_tag ifc_vlan_detach_event;
|
eventhandler_tag ifc_vlan_detach_event;
|
||||||
@ -392,8 +390,7 @@ struct iflib_fl {
|
|||||||
bus_dma_tag_t ifl_buf_tag;
|
bus_dma_tag_t ifl_buf_tag;
|
||||||
iflib_dma_info_t ifl_ifdi;
|
iflib_dma_info_t ifl_ifdi;
|
||||||
uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
|
uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
|
||||||
caddr_t ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
|
qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
|
||||||
qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
|
|
||||||
} __aligned(CACHE_LINE_SIZE);
|
} __aligned(CACHE_LINE_SIZE);
|
||||||
|
|
||||||
static inline qidx_t
|
static inline qidx_t
|
||||||
@ -854,7 +851,6 @@ netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, boo
|
|||||||
if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
|
if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
|
||||||
return netmap_ring_reinit(kring);
|
return netmap_ring_reinit(kring);
|
||||||
|
|
||||||
fl->ifl_vm_addrs[tmp_pidx] = addr;
|
|
||||||
if (__predict_false(init)) {
|
if (__predict_false(init)) {
|
||||||
netmap_load_map(na, fl->ifl_buf_tag,
|
netmap_load_map(na, fl->ifl_buf_tag,
|
||||||
map[nic_i], addr);
|
map[nic_i], addr);
|
||||||
@ -1295,7 +1291,6 @@ iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
|
|||||||
|
|
||||||
fl = &rxq->ifr_fl[flid];
|
fl = &rxq->ifr_fl[flid];
|
||||||
iru->iru_paddrs = fl->ifl_bus_addrs;
|
iru->iru_paddrs = fl->ifl_bus_addrs;
|
||||||
iru->iru_vaddrs = &fl->ifl_vm_addrs[0];
|
|
||||||
iru->iru_idxs = fl->ifl_rxd_idxs;
|
iru->iru_idxs = fl->ifl_rxd_idxs;
|
||||||
iru->iru_qsidx = rxq->ifr_id;
|
iru->iru_qsidx = rxq->ifr_id;
|
||||||
iru->iru_buf_size = fl->ifl_buf_size;
|
iru->iru_buf_size = fl->ifl_buf_size;
|
||||||
@ -1916,7 +1911,7 @@ _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* _iflib_fl_refill - refill an rxq free-buffer list
|
* iflib_fl_refill - refill an rxq free-buffer list
|
||||||
* @ctx: the iflib context
|
* @ctx: the iflib context
|
||||||
* @fl: the free list to refill
|
* @fl: the free list to refill
|
||||||
* @count: the number of new buffers to allocate
|
* @count: the number of new buffers to allocate
|
||||||
@ -1925,7 +1920,7 @@ _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
|
|||||||
* The caller must assure that @count does not exceed the queue's capacity.
|
* The caller must assure that @count does not exceed the queue's capacity.
|
||||||
*/
|
*/
|
||||||
static uint8_t
|
static uint8_t
|
||||||
_iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
||||||
{
|
{
|
||||||
struct if_rxd_update iru;
|
struct if_rxd_update iru;
|
||||||
struct rxq_refill_cb_arg cb_arg;
|
struct rxq_refill_cb_arg cb_arg;
|
||||||
@ -1962,12 +1957,13 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
|||||||
if (n > 8)
|
if (n > 8)
|
||||||
DBG_COUNTER_INC(fl_refills_large);
|
DBG_COUNTER_INC(fl_refills_large);
|
||||||
iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
|
iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
|
||||||
while (n--) {
|
while (n-- > 0) {
|
||||||
/*
|
/*
|
||||||
* We allocate an uninitialized mbuf + cluster, mbuf is
|
* We allocate an uninitialized mbuf + cluster, mbuf is
|
||||||
* initialized after rx.
|
* initialized after rx.
|
||||||
*
|
*
|
||||||
* If the cluster is still set then we know a minimum sized packet was received
|
* If the cluster is still set then we know a minimum sized
|
||||||
|
* packet was received
|
||||||
*/
|
*/
|
||||||
bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,
|
bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,
|
||||||
&frag_idx);
|
&frag_idx);
|
||||||
@ -1975,7 +1971,7 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
|||||||
bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
|
bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
|
||||||
MPASS(frag_idx >= 0);
|
MPASS(frag_idx >= 0);
|
||||||
if ((cl = sd_cl[frag_idx]) == NULL) {
|
if ((cl = sd_cl[frag_idx]) == NULL) {
|
||||||
cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size);
|
cl = uma_zalloc(fl->ifl_zone, M_NOWAIT);
|
||||||
if (__predict_false(cl == NULL))
|
if (__predict_false(cl == NULL))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
@ -1989,7 +1985,7 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr;
|
sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr;
|
||||||
sd_cl[frag_idx] = cl;
|
sd_cl[frag_idx] = cl;
|
||||||
#if MEMORY_LOGGING
|
#if MEMORY_LOGGING
|
||||||
fl->ifl_cl_enqueued++;
|
fl->ifl_cl_enqueued++;
|
||||||
@ -2014,22 +2010,23 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
|||||||
DBG_COUNTER_INC(rx_allocs);
|
DBG_COUNTER_INC(rx_allocs);
|
||||||
fl->ifl_rxd_idxs[i] = frag_idx;
|
fl->ifl_rxd_idxs[i] = frag_idx;
|
||||||
fl->ifl_bus_addrs[i] = bus_addr;
|
fl->ifl_bus_addrs[i] = bus_addr;
|
||||||
fl->ifl_vm_addrs[i] = cl;
|
|
||||||
credits++;
|
credits++;
|
||||||
i++;
|
i++;
|
||||||
MPASS(credits <= fl->ifl_size);
|
MPASS(credits <= fl->ifl_size);
|
||||||
if (++idx == fl->ifl_size) {
|
if (++idx == fl->ifl_size) {
|
||||||
|
#ifdef INVARIANTS
|
||||||
fl->ifl_gen = 1;
|
fl->ifl_gen = 1;
|
||||||
|
#endif
|
||||||
idx = 0;
|
idx = 0;
|
||||||
}
|
}
|
||||||
if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
|
if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
|
||||||
iru.iru_pidx = pidx;
|
iru.iru_pidx = pidx;
|
||||||
iru.iru_count = i;
|
iru.iru_count = i;
|
||||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
|
ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
|
||||||
i = 0;
|
|
||||||
pidx = idx;
|
|
||||||
fl->ifl_pidx = idx;
|
fl->ifl_pidx = idx;
|
||||||
fl->ifl_credits = credits;
|
fl->ifl_credits = credits;
|
||||||
|
pidx = idx;
|
||||||
|
i = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2063,8 +2060,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
|
|||||||
return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY);
|
return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY);
|
||||||
}
|
}
|
||||||
|
|
||||||
static __inline uint8_t
|
static inline uint8_t
|
||||||
__iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl)
|
iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl)
|
||||||
{
|
{
|
||||||
/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
|
/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
|
||||||
int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
|
int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
|
||||||
@ -2076,7 +2073,7 @@ __iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl)
|
|||||||
MPASS(reclaimable == delta);
|
MPASS(reclaimable == delta);
|
||||||
|
|
||||||
if (reclaimable > 0)
|
if (reclaimable > 0)
|
||||||
return (_iflib_fl_refill(ctx, fl, reclaimable));
|
return (iflib_fl_refill(ctx, fl, reclaimable));
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2107,22 +2104,20 @@ iflib_fl_bufs_free(iflib_fl_t fl)
|
|||||||
bus_dmamap_sync(fl->ifl_buf_tag, sd_map,
|
bus_dmamap_sync(fl->ifl_buf_tag, sd_map,
|
||||||
BUS_DMASYNC_POSTREAD);
|
BUS_DMASYNC_POSTREAD);
|
||||||
bus_dmamap_unload(fl->ifl_buf_tag, sd_map);
|
bus_dmamap_unload(fl->ifl_buf_tag, sd_map);
|
||||||
if (*sd_cl != NULL)
|
uma_zfree(fl->ifl_zone, *sd_cl);
|
||||||
uma_zfree(fl->ifl_zone, *sd_cl);
|
*sd_cl = NULL;
|
||||||
if (*sd_m != NULL) {
|
if (*sd_m != NULL) {
|
||||||
m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
|
m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
|
||||||
uma_zfree(zone_mbuf, *sd_m);
|
uma_zfree(zone_mbuf, *sd_m);
|
||||||
|
*sd_m = NULL;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
MPASS(*sd_cl == NULL);
|
|
||||||
MPASS(*sd_m == NULL);
|
MPASS(*sd_m == NULL);
|
||||||
}
|
}
|
||||||
#if MEMORY_LOGGING
|
#if MEMORY_LOGGING
|
||||||
fl->ifl_m_dequeued++;
|
fl->ifl_m_dequeued++;
|
||||||
fl->ifl_cl_dequeued++;
|
fl->ifl_cl_dequeued++;
|
||||||
#endif
|
#endif
|
||||||
*sd_cl = NULL;
|
|
||||||
*sd_m = NULL;
|
|
||||||
}
|
}
|
||||||
#ifdef INVARIANTS
|
#ifdef INVARIANTS
|
||||||
for (i = 0; i < fl->ifl_size; i++) {
|
for (i = 0; i < fl->ifl_size; i++) {
|
||||||
@ -2176,7 +2171,7 @@ iflib_fl_setup(iflib_fl_t fl)
|
|||||||
/* avoid pre-allocating zillions of clusters to an idle card
|
/* avoid pre-allocating zillions of clusters to an idle card
|
||||||
* potentially speeding up attach
|
* potentially speeding up attach
|
||||||
*/
|
*/
|
||||||
(void) _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
|
(void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
|
||||||
MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
|
MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
|
||||||
if (min(128, fl->ifl_size) != fl->ifl_credits)
|
if (min(128, fl->ifl_size) != fl->ifl_credits)
|
||||||
return (ENOBUFS);
|
return (ENOBUFS);
|
||||||
@ -2791,7 +2786,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
|
|||||||
cidxp = &rxq->ifr_fl[0].ifl_cidx;
|
cidxp = &rxq->ifr_fl[0].ifl_cidx;
|
||||||
if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
|
if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
|
||||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
|
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
|
||||||
retval |= __iflib_fl_refill_all(ctx, fl);
|
retval |= iflib_fl_refill_all(ctx, fl);
|
||||||
DBG_COUNTER_INC(rx_unavail);
|
DBG_COUNTER_INC(rx_unavail);
|
||||||
return (retval);
|
return (retval);
|
||||||
}
|
}
|
||||||
@ -2851,7 +2846,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
|
|||||||
CURVNET_RESTORE();
|
CURVNET_RESTORE();
|
||||||
/* make sure that we can refill faster than drain */
|
/* make sure that we can refill faster than drain */
|
||||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
|
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
|
||||||
retval |= __iflib_fl_refill_all(ctx, fl);
|
retval |= iflib_fl_refill_all(ctx, fl);
|
||||||
|
|
||||||
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
|
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
|
||||||
if (lro_enabled)
|
if (lro_enabled)
|
||||||
|
@ -95,7 +95,6 @@ typedef struct if_rxd_info {
|
|||||||
|
|
||||||
typedef struct if_rxd_update {
|
typedef struct if_rxd_update {
|
||||||
uint64_t *iru_paddrs;
|
uint64_t *iru_paddrs;
|
||||||
caddr_t *iru_vaddrs;
|
|
||||||
qidx_t *iru_idxs;
|
qidx_t *iru_idxs;
|
||||||
qidx_t iru_pidx;
|
qidx_t iru_pidx;
|
||||||
uint16_t iru_qsidx;
|
uint16_t iru_qsidx;
|
||||||
|
Loading…
Reference in New Issue
Block a user