From d0428dd51cb76234d1d95cd80c1d247adfa0c2bb Mon Sep 17 00:00:00 2001 From: sephe Date: Fri, 1 Apr 2016 06:28:33 +0000 Subject: [PATCH] tcp/lro: Use tcp_lro_flush_all in device drivers to avoid code duplication And factor out tcp_lro_rx_done, which deduplicates the same logic with netinet/tcp_lro.c Reviewed by: gallatin (1st version), hps, zbb, np, Dexuan Cui Sponsored by: Microsoft OSTC Differential Revision: https://reviews.freebsd.org/D5725 --- sys/dev/cxgb/cxgb_sge.c | 6 +---- sys/dev/cxgbe/t4_sge.c | 7 +---- sys/dev/e1000/if_igb.c | 6 +---- sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c | 7 +---- sys/dev/ixgbe/ix_txrx.c | 6 +---- sys/dev/ixl/ixl_txrx.c | 6 +---- sys/dev/mlx5/mlx5_en/mlx5_en_rx.c | 8 +----- sys/dev/mxge/if_mxge.c | 6 +---- sys/dev/oce/oce_if.c | 6 +---- sys/dev/qlxgb/qla_isr.c | 7 +---- sys/dev/qlxge/qls_isr.c | 7 +---- sys/dev/vnic/nicvf_queues.c | 6 +---- sys/dev/vxge/vxge.c | 9 ++----- sys/dev/xen/netfront/netfront.c | 7 +---- sys/netinet/tcp_lro.c | 27 ++++++++++++------- sys/ofed/drivers/net/mlx4/en_rx.c | 8 +----- 16 files changed, 33 insertions(+), 96 deletions(-) diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index a622fcfef566..c83bd66379e7 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -2976,11 +2976,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) #if defined(INET6) || defined(INET) /* Flush LRO */ - while (!SLIST_EMPTY(&lro_ctrl->lro_active)) { - struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active); - SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next); - tcp_lro_flush(lro_ctrl, queued); - } + tcp_lro_flush_all(lro_ctrl); #endif if (sleeping) diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 7f3d390d08c5..daf3b8a23644 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -1397,13 +1397,8 @@ process_iql: #if defined(INET) || defined(INET6) if (iq->flags & IQ_LRO_ENABLED) { struct lro_ctrl *lro = &rxq->lro; - struct lro_entry *l; - while (!SLIST_EMPTY(&lro->lro_active)) { - l = SLIST_FIRST(&lro->lro_active); - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, l); - } + tcp_lro_flush_all(lro); } #endif diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c index 805f66bef8f0..93b33adc1d09 100644 --- a/sys/dev/e1000/if_igb.c +++ b/sys/dev/e1000/if_igb.c @@ -4974,7 +4974,6 @@ igb_rxeof(struct igb_queue *que, int count, int *done) struct rx_ring *rxr = que->rxr; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; - struct lro_entry *queued; int i, processed = 0, rxdone = 0; u32 ptype, staterr = 0; union e1000_adv_rx_desc *cur; @@ -5202,10 +5201,7 @@ next_desc: /* * Flush any outstanding LRO work */ - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); if (done != NULL) *done += rxdone; diff --git a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c index 69c4eb528a8f..acc49b4540cb 100644 --- a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c +++ b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c @@ -778,13 +778,8 @@ netvsc_channel_rollup(struct hv_vmbus_channel *chan) struct hn_tx_ring *txr = chan->hv_chan_txr; #if defined(INET) || defined(INET6) struct hn_rx_ring *rxr = chan->hv_chan_rxr; - struct lro_ctrl *lro = &rxr->hn_lro; - struct lro_entry *queued; - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(&rxr->hn_lro); #endif /* diff --git a/sys/dev/ixgbe/ix_txrx.c b/sys/dev/ixgbe/ix_txrx.c index 00e5dc8cf0d4..66bcb255332b 100644 --- a/sys/dev/ixgbe/ix_txrx.c +++ b/sys/dev/ixgbe/ix_txrx.c @@ -1753,7 +1753,6 @@ ixgbe_rxeof(struct ix_queue *que) struct rx_ring *rxr = que->rxr; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; - struct lro_entry *queued; int i, nextp, processed = 0; u32 staterr = 0; u32 count = adapter->rx_process_limit; @@ -2003,10 +2002,7 @@ next_desc: /* * Flush any outstanding LRO work */ - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); IXGBE_RX_UNLOCK(rxr); diff --git a/sys/dev/ixl/ixl_txrx.c b/sys/dev/ixl/ixl_txrx.c index 0ae7433e8e8d..ca6e252b3fea 100644 --- a/sys/dev/ixl/ixl_txrx.c +++ b/sys/dev/ixl/ixl_txrx.c @@ -1511,7 +1511,6 @@ ixl_rxeof(struct ixl_queue *que, int count) struct ifnet *ifp = vsi->ifp; #if defined(INET6) || defined(INET) struct lro_ctrl *lro = &rxr->lro; - struct lro_entry *queued; #endif int i, nextp, processed = 0; union i40e_rx_desc *cur; @@ -1735,10 +1734,7 @@ next_desc: /* * Flush any outstanding LRO work */ - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); #endif IXL_RX_UNLOCK(rxr); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c index 48339be526bd..230dfcc333a1 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -322,9 +322,6 @@ mlx5e_decompress_cqes(struct mlx5e_cq *cq) static int mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) { -#ifndef HAVE_TURBO_LRO - struct lro_entry *queued; -#endif int i; for (i = 0; i < budget; i++) { @@ -399,10 +396,7 @@ wq_ll_pop: /* ensure cq space is freed before enabling more cqes */ wmb(); #ifndef HAVE_TURBO_LRO - while ((queued = SLIST_FIRST(&rq->lro.lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&rq->lro.lro_active, next); - tcp_lro_flush(&rq->lro, queued); - } + tcp_lro_flush_all(&rq->lro); #endif return (i); } diff --git a/sys/dev/mxge/if_mxge.c b/sys/dev/mxge/if_mxge.c index ddbaf89c8230..f78b34b2a233 100644 --- a/sys/dev/mxge/if_mxge.c +++ b/sys/dev/mxge/if_mxge.c @@ -2819,11 +2819,7 @@ mxge_clean_rx_done(struct mxge_slice_state *ss) break; } #if defined(INET) || defined (INET6) - while (!SLIST_EMPTY(&ss->lc.lro_active)) { - struct lro_entry *lro = SLIST_FIRST(&ss->lc.lro_active); - SLIST_REMOVE_HEAD(&ss->lc.lro_active, next); - tcp_lro_flush(&ss->lc, lro); - } + tcp_lro_flush_all(&ss->lc); #endif } diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c index f0cce5f6e0d1..370461291234 100644 --- a/sys/dev/oce/oce_if.c +++ b/sys/dev/oce/oce_if.c @@ -1497,16 +1497,12 @@ static void oce_rx_flush_lro(struct oce_rq *rq) { struct lro_ctrl *lro = &rq->lro; - struct lro_entry *queued; POCE_SOFTC sc = (POCE_SOFTC) rq->parent; if (!IF_LRO_ENABLED(sc)) return; - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); rq->lro_pkts_queued = 0; return; diff --git a/sys/dev/qlxgb/qla_isr.c b/sys/dev/qlxgb/qla_isr.c index 3169fd16a152..983fc12fb92a 100644 --- a/sys/dev/qlxgb/qla_isr.c +++ b/sys/dev/qlxgb/qla_isr.c @@ -267,7 +267,6 @@ qla_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count) uint32_t comp_idx, desc_count; q80_stat_desc_t *sdesc; struct lro_ctrl *lro; - struct lro_entry *queued; uint32_t ret = 0; dev = ha->pci_dev; @@ -324,11 +323,7 @@ qla_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count) } } - while((!SLIST_EMPTY(&lro->lro_active))) { - queued = SLIST_FIRST(&lro->lro_active); - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); if (hw->sds[sds_idx].sdsr_next != comp_idx) { QL_UPDATE_SDS_CONSUMER_INDEX(ha, sds_idx, comp_idx); diff --git a/sys/dev/qlxge/qls_isr.c b/sys/dev/qlxge/qls_isr.c index 03b3a3c52152..4cca895f389f 100644 --- a/sys/dev/qlxge/qls_isr.c +++ b/sys/dev/qlxge/qls_isr.c @@ -232,7 +232,6 @@ qls_cq_isr(qla_host_t *ha, uint32_t cq_idx) uint32_t i, cq_comp_idx; int ret = 0, tx_comp_done = 0; struct lro_ctrl *lro; - struct lro_entry *queued; cq_b = ha->rx_ring[cq_idx].cq_base_vaddr; lro = &ha->rx_ring[cq_idx].lro; @@ -287,11 +286,7 @@ qls_cq_isr(qla_host_t *ha, uint32_t cq_idx) } } - while((!SLIST_EMPTY(&lro->lro_active))) { - queued = SLIST_FIRST(&lro->lro_active); - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); ha->rx_ring[cq_idx].cq_next = cq_comp_idx; diff --git a/sys/dev/vnic/nicvf_queues.c b/sys/dev/vnic/nicvf_queues.c index 2aeed4de2d77..9b46d7cb999d 100644 --- a/sys/dev/vnic/nicvf_queues.c +++ b/sys/dev/vnic/nicvf_queues.c @@ -746,7 +746,6 @@ nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) struct rcv_queue *rq; struct cqe_rx_t *cq_desc; struct lro_ctrl *lro; - struct lro_entry *queued; int rq_idx; int cmp_err; @@ -831,10 +830,7 @@ out: rq_idx = cq_idx; rq = &nic->qs->rq[rq_idx]; lro = &rq->lro; - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); NICVF_CMP_UNLOCK(cq); diff --git a/sys/dev/vxge/vxge.c b/sys/dev/vxge/vxge.c index 9a3cab654f95..84320f6d614e 100644 --- a/sys/dev/vxge/vxge.c +++ b/sys/dev/vxge/vxge.c @@ -996,7 +996,6 @@ vxge_rx_compl(vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, vxge_vpath_t *vpath = (vxge_vpath_t *) userdata; vxge_dev_t *vdev = vpath->vdev; - struct lro_entry *queued = NULL; struct lro_ctrl *lro = &vpath->lro; /* get the interface pointer */ @@ -1083,12 +1082,8 @@ vxge_rx_compl(vxge_hal_vpath_h vpath_handle, vxge_hal_rxd_h rxdh, &dtr_priv, &t_code) == VXGE_HAL_OK); /* Flush any outstanding LRO work */ - if (vpath->lro_enable && vpath->lro.lro_cnt) { - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } - } + if (vpath->lro_enable && vpath->lro.lro_cnt) + tcp_lro_flush_all(lro); return (status); } diff --git a/sys/dev/xen/netfront/netfront.c b/sys/dev/xen/netfront/netfront.c index 22ca81c45ccf..7add70ac5c1d 100644 --- a/sys/dev/xen/netfront/netfront.c +++ b/sys/dev/xen/netfront/netfront.c @@ -1202,7 +1202,6 @@ xn_rxeof(struct netfront_rxq *rxq) struct netfront_info *np = rxq->info; #if (defined(INET) || defined(INET6)) struct lro_ctrl *lro = &rxq->lro; - struct lro_entry *queued; #endif struct netfront_rx_info rinfo; struct netif_rx_response *rx = &rinfo.rx; @@ -1296,11 +1295,7 @@ xn_rxeof(struct netfront_rxq *rxq) /* * Flush any outstanding LRO work */ - while (!SLIST_EMPTY(&lro->lro_active)) { - queued = SLIST_FIRST(&lro->lro_active); - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } + tcp_lro_flush_all(lro); #endif xn_alloc_rx_buffers(rxq); diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c index f9d7f891780b..c817747957e3 100644 --- a/sys/netinet/tcp_lro.c +++ b/sys/netinet/tcp_lro.c @@ -67,6 +67,8 @@ static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures"); #define TCP_LRO_INVALID_CSUM 0x0000 #endif +static void tcp_lro_rx_done(struct lro_ctrl *lc); + int tcp_lro_init(struct lro_ctrl *lc) { @@ -226,6 +228,17 @@ tcp_lro_rx_csum_fixup(struct lro_entry *le, void *l3hdr, struct tcphdr *th, } #endif +static void +tcp_lro_rx_done(struct lro_ctrl *lc) +{ + struct lro_entry *le; + + while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) { + SLIST_REMOVE_HEAD(&lc->lro_active, next); + tcp_lro_flush(lc, le); + } +} + void tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout) { @@ -362,13 +375,12 @@ done: void tcp_lro_flush_all(struct lro_ctrl *lc) { - struct lro_entry *le; uint32_t hashtype; uint32_t flowid; unsigned x; /* check if no mbufs to flush */ - if (__predict_false(lc->lro_mbuf_count == 0)) + if (lc->lro_mbuf_count == 0) goto done; /* sort all mbufs according to stream */ @@ -390,10 +402,7 @@ tcp_lro_flush_all(struct lro_ctrl *lc) hashtype = M_HASHTYPE_GET(mb); /* flush active streams */ - while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lc->lro_active, next); - tcp_lro_flush(lc, le); - } + tcp_lro_rx_done(lc); } #ifdef TCP_LRO_RESET_SEQUENCE /* reset sequence number */ @@ -409,10 +418,8 @@ tcp_lro_flush_all(struct lro_ctrl *lc) } done: /* flush active streams */ - while ((le = SLIST_FIRST(&lc->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lc->lro_active, next); - tcp_lro_flush(lc, le); - } + tcp_lro_rx_done(lc); + lc->lro_mbuf_count = 0; } diff --git a/sys/ofed/drivers/net/mlx4/en_rx.c b/sys/ofed/drivers/net/mlx4/en_rx.c index 16bf597b01a2..b29096d56deb 100644 --- a/sys/ofed/drivers/net/mlx4/en_rx.c +++ b/sys/ofed/drivers/net/mlx4/en_rx.c @@ -561,9 +561,6 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud struct mbuf *mb; struct mlx4_cq *mcq = &cq->mcq; struct mlx4_cqe *buf = cq->buf; -#ifdef INET - struct lro_entry *queued; -#endif int index; unsigned int length; int polled = 0; @@ -669,10 +666,7 @@ next: /* Flush all pending IP reassembly sessions */ out: #ifdef INET - while ((queued = SLIST_FIRST(&ring->lro.lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&ring->lro.lro_active, next); - tcp_lro_flush(&ring->lro, queued); - } + tcp_lro_flush_all(&ring->lro); #endif AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled); mcq->cons_index = cons_index;