d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- move WR_LEN in to cxgb_adapter.h add PIO_LEN to make intent clearer

Browse Source 
- move cxgb_tx_common in to cxgb_multiq.c and rename to cxgb_tx
- move cxgb_tx_common dependencies
- further simplify cxgb_dequeue_packet for the non-multiqueue case
- only launch one service thread per port in the non-multiq case
- remove dead cleaning code from cxgb_sge.c
- simplify PIO case substantially in by returning directly from mbuf collapse
  and just using m_copydata
- remove gratuitous m_gethdr in the rx path
- clarify freeing of mbufs in collapse
This commit is contained in:
Kip Macy 2008-01-15 03:27:42 +00:00
parent 74aba11713
commit 60f1e27625
6 changed files with 192 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
sys/dev/cxgb/cxgb_adapter.h
View File

@ -149,13 +149,24 @@ enum { /* adapter flags */
#define FL_Q_SIZE 4096
#define JUMBO_Q_SIZE 1024
#define RSPQ_Q_SIZE 1024
#if 0
#define TX_ETH_Q_SIZE 1024
#else
#define TX_ETH_Q_SIZE 64
#endif
enum { TXQ_ETH = 0,
TXQ_OFLD = 1,
TXQ_CTRL = 2, };
/*
* work request size in bytes
*/
#define WR_LEN (WR_FLITS * 8)
#define PIO_LEN (WR_LEN - sizeof(struct cpl_tx_pkt))
/* careful, the following are set on priv_flags and must not collide with
* IFF_ flags!
*/
@ -288,7 +299,6 @@ struct sge_txq {
struct mtx lock;
struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1];
bus_dma_segment_t txq_segs[TX_MAX_SEGS];
struct mbuf *txq_m_vec[TX_WR_COUNT_MAX];
#define TXQ_NAME_LEN 32
char lockbuf[TXQ_NAME_LEN];
};
@ -610,10 +620,7 @@ void cxgb_pcpu_shutdown_threads(struct adapter *sc);
void cxgb_pcpu_startup_threads(struct adapter *sc);
int process_responses(adapter_t *adap, struct sge_qset *qs, int budget);
int cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax);
void t3_free_qset(adapter_t *sc, struct sge_qset *q);
int cxgb_dequeue_packet(struct ifnet *, struct sge_txq *, struct mbuf **);
void cxgb_start(struct ifnet *ifp);
void refill_fl_service(adapter_t *adap, struct sge_fl *fl);
int reclaim_completed_tx(struct sge_txq *q, int reclaim_min);
#endif

73
sys/dev/cxgb/cxgb_main.c
View File

@ -281,21 +281,6 @@ struct cxgb_ident {
static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
static __inline void
check_pkt_coalesce(struct sge_qset *qs)
{
struct adapter *sc;
struct sge_txq *txq;
txq = &qs->txq[TXQ_ETH];
sc = qs->port->adapter;
if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2))))
sc->tunq_fill[qs->idx] = 0;
else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2))))
sc->tunq_fill[qs->idx] = 1;
}
static __inline char
t3rev2char(struct adapter *adapter)
{
@ -1865,64 +1850,6 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
return (error);
}
int
cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax)
{
struct sge_txq *txq;
int err, in_use_init, count;
struct mbuf **m_vec;
txq = &qs->txq[TXQ_ETH];
m_vec = txq->txq_m_vec;
in_use_init = txq->in_use;
err = 0;
while ((txq->in_use - in_use_init < txmax) &&
(txq->size > txq->in_use + TX_MAX_DESC)) {
check_pkt_coalesce(qs);
count = cxgb_dequeue_packet(ifp, txq, m_vec);
if (count == 0)
break;
ETHER_BPF_MTAP(ifp, m_vec[0]);
if ((err = t3_encap(qs, m_vec, count)) != 0)
break;
txq->txq_enqueued += count;
}
#if 0 /* !MULTIQ */
if (__predict_false(err)) {
if (err == ENOMEM) {
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IFQ_LOCK(&ifp->if_snd);
IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]);
IFQ_UNLOCK(&ifp->if_snd);
}
}
if (err == 0 && m_vec[0] == NULL) {
err = ENOBUFS;
}
else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
(ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
setbit(&qs->txq_stopped, TXQ_ETH);
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
err = ENOSPC;
}
#else
if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) {
err = ENOSPC;
setbit(&qs->txq_stopped, TXQ_ETH);
}
if (err == ENOMEM) {
int i;
/*
* Sub-optimal :-/
*/
for (i = 0; i < count; i++)
m_freem(m_vec[i]);
}
#endif
return (err);
}
static int
cxgb_media_change(struct ifnet *ifp)
{

166
sys/dev/cxgb/cxgb_multiq.c
View File

@ -115,6 +115,9 @@ static void cxgb_pcpu_start_proc(void *arg);
#ifdef IFNET_MULTIQUEUE
static int cxgb_pcpu_cookie_to_qidx(struct port_info *, uint32_t cookie);
#endif
static int cxgb_tx(struct sge_qset *qs, uint32_t txmax);
static inline int
cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
{
@ -124,7 +127,6 @@ cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
#ifndef IFNET_MULTIQUEUE
panic("not expecting enqueue without multiqueue");
#endif
KASSERT(m != NULL, ("null mbuf"));
KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
if (qs->qs_flags & QS_EXITING) {
@ -164,8 +166,8 @@ cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m)
return (err);
}
int
cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_vec)
static int
cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
{
struct mbuf *m;
struct sge_qset *qs;
@ -176,10 +178,16 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v
if (txq->immpkt != NULL)
panic("immediate packet set");
#endif
mtx_assert(&txq->lock, MA_OWNED);
IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m);
if (m == NULL)
return (0);
m_vec[0] = m;
return (1);
#endif
coalesced = count = size = 0;
qs = txq_to_qset(txq, TXQ_ETH);
if (qs->qs_flags & QS_EXITING)
@ -193,39 +201,20 @@ cxgb_dequeue_packet(struct ifnet *unused, struct sge_txq *txq, struct mbuf **m_v
}
sc = qs->port->adapter;
#ifndef IFNET_MULTIQUEUE
/*
* This is terrible from a cache and locking efficiency standpoint
* but then again ... so is ifnet.
*/
while (((qs->qs_flags & QS_EXITING) == 0) && !IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd) && !buf_ring_full(&txq->txq_mr)) {
struct mbuf *m = NULL;
IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m);
if (m) {
KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
if (buf_ring_enqueue(&txq->txq_mr, m))
panic("ring full");
} else
break;
}
#endif
m = buf_ring_dequeue(&txq->txq_mr);
if (m == NULL)
if (m == NULL)
return (0);
buf_ring_scan(&txq->txq_mr, m, __FILE__, __LINE__);
KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
m_vec[0] = m;
if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX || m->m_next != NULL ||
(cxgb_pcpu_tx_coalesce == 0)) {
return (1);
}
#ifndef IFNET_MULTIQUEUE
panic("coalesce not supported yet");
#endif
count = 1;
KASSERT(m->m_type == MT_DATA,
("m=%p is bad mbuf type %d from ring cons=%d prod=%d", m,
m->m_type, txq->txq_mr.br_cons, txq->txq_mr.br_prod));
m_vec[0] = m;
if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX ||
m->m_next != NULL || (cxgb_pcpu_tx_coalesce == 0)) {
return (count);
}
size = m->m_pkthdr.len;
for (m = buf_ring_peek(&txq->txq_mr); m != NULL;
m = buf_ring_peek(&txq->txq_mr)) {
@ -381,13 +370,15 @@ cxgb_pcpu_free(struct sge_qset *qs)
{
struct mbuf *m;
struct sge_txq *txq = &qs->txq[TXQ_ETH];
mtx_lock(&txq->lock);
while ((m = mbufq_dequeue(&txq->sendq)) != NULL)
m_freem(m);
while ((m = buf_ring_dequeue(&txq->txq_mr)) != NULL)
m_freem(m);
t3_free_tx_desc_all(txq);
mtx_unlock(&txq->lock);
}
static int
@ -400,6 +391,7 @@ cxgb_pcpu_reclaim_tx(struct sge_txq *txq)
KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d",
qs->qs_cpuid, curcpu));
#endif
mtx_assert(&txq->lock, MA_OWNED);
reclaimable = desc_reclaimable(txq);
if (reclaimable == 0)
@ -429,6 +421,8 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
sc = pi->adapter;
txq = &qs->txq[TXQ_ETH];
mtx_assert(&txq->lock, MA_OWNED);
retry:
if (!pi->link_config.link_ok)
initerr = ENXIO;
@ -474,10 +468,12 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
DPRINTF("stopped=%d flush=%d max_desc=%d\n",
stopped, flush, max_desc);
err = flush ? cxgb_tx_common(qs->port->ifp, qs, max_desc) : ENOSPC;
err = flush ? cxgb_tx(qs, max_desc) : ENOSPC;
if ((tx_flush && flush && err == 0) && !buf_ring_empty(&txq->txq_mr)) {
if ((tx_flush && flush && err == 0) &&
(!buf_ring_empty(&txq->txq_mr) ||
!IFQ_DRV_IS_EMPTY(&pi->ifp->if_snd))) {
struct thread *td = curthread;
if (++i > 1) {
@ -525,7 +521,8 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
txq = &qs->txq[TXQ_ETH];
if (((sc->tunq_coalesce == 0) || (buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) ||
if (((sc->tunq_coalesce == 0) ||
(buf_ring_count(&txq->txq_mr) >= TX_WR_COUNT_MAX) ||
(cxgb_pcpu_tx_coalesce == 0)) && mtx_trylock(&txq->lock)) {
if (cxgb_debug)
printf("doing immediate transmit\n");
@ -658,12 +655,18 @@ cxgb_pcpu_cookie_to_qidx(struct port_info *pi, uint32_t cookie)
void
cxgb_pcpu_startup_threads(struct adapter *sc)
{
int i, j;
int i, j, nqsets;
struct proc *p;
for (i = 0; i < (sc)->params.nports; ++i) {
struct port_info *pi = adap2pinfo(sc, i);
#ifdef IFNET_MULTIQUEUE
nqsets = pi->nqsets;
#else
nqsets = 1;
#endif
for (j = 0; j < pi->nqsets; ++j) {
struct sge_qset *qs;
@ -701,3 +704,84 @@ cxgb_pcpu_shutdown_threads(struct adapter *sc)
}
}
}
static __inline void
check_pkt_coalesce(struct sge_qset *qs)
{
struct adapter *sc;
struct sge_txq *txq;
txq = &qs->txq[TXQ_ETH];
sc = qs->port->adapter;
if (sc->tunq_fill[qs->idx] && (txq->in_use < (txq->size - (txq->size>>2))))
sc->tunq_fill[qs->idx] = 0;
else if (!sc->tunq_fill[qs->idx] && (txq->in_use > (txq->size - (txq->size>>2))))
sc->tunq_fill[qs->idx] = 1;
}
static int
cxgb_tx(struct sge_qset *qs, uint32_t txmax)
{
struct sge_txq *txq;
struct ifnet *ifp = qs->port->ifp;
int i, err, in_use_init, count;
struct mbuf *m_vec[TX_WR_COUNT_MAX];
txq = &qs->txq[TXQ_ETH];
ifp = qs->port->ifp;
in_use_init = txq->in_use;
err = 0;
for (i = 0; i < TX_WR_COUNT_MAX; i++)
m_vec[i] = NULL;
mtx_assert(&txq->lock, MA_OWNED);
while ((txq->in_use - in_use_init < txmax) &&
(txq->size > txq->in_use + TX_MAX_DESC)) {
check_pkt_coalesce(qs);
count = cxgb_dequeue_packet(txq, m_vec);
if (count == 0) {
err = ENOBUFS;
break;
}
ETHER_BPF_MTAP(ifp, m_vec[0]);
if ((err = t3_encap(qs, m_vec, count)) != 0)
break;
txq->txq_enqueued += count;
m_vec[0] = NULL;
}
#if 0 /* !MULTIQ */
if (__predict_false(err)) {
if (err == ENOMEM) {
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
IFQ_LOCK(&ifp->if_snd);
IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]);
IFQ_UNLOCK(&ifp->if_snd);
}
}
else if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC) &&
(ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
setbit(&qs->txq_stopped, TXQ_ETH);
ifp->if_drv_flags |= IFF_DRV_OACTIVE;
err = ENOSPC;
}
#else
if ((err == 0) && (txq->size <= txq->in_use + TX_MAX_DESC)) {
err = ENOSPC;
setbit(&qs->txq_stopped, TXQ_ETH);
}
if (err == ENOMEM) {
int i;
/*
* Sub-optimal :-/
*/
printf("ENOMEM!!!");
for (i = 0; i < count; i++)
m_freem(m_vec[i]);
}
#endif
return (err);
}

125
sys/dev/cxgb/cxgb_sge.c
View File

@ -26,6 +26,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***************************************************************************/
#define DEBUG_BUFRING
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
@ -72,7 +74,7 @@ __FBSDID("$FreeBSD$");
int txq_fills = 0;
static int bogus_imm = 0;
static int recycle_enable = 1;
static int recycle_enable = 0;
extern int cxgb_txq_buf_ring_size;
int cxgb_cached_allocations;
int cxgb_cached;
@ -92,10 +94,6 @@ extern int cxgb_use_16k_clusters;
*/
#define TX_RECLAIM_PERIOD (hz >> 1)
/*
* work request size in bytes
*/
#define WR_LEN (WR_FLITS * 8)
/*
* Values for sge_txq.flags
@ -218,13 +216,6 @@ reclaim_completed_tx_(struct sge_txq *q, int reclaim_min)
return (reclaim);
}
int
reclaim_completed_tx(struct sge_txq *q, int reclaim_min)
{
return reclaim_completed_tx_(q, reclaim_min);
}
/**
* should_restart_tx - are there enough resources to restart a Tx queue?
* @q: the Tx queue
@ -672,8 +663,7 @@ alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size,
if (sw_size) {
len = nelem * sw_size;
s = malloc(len, M_DEVBUF, M_WAITOK);
bzero(s, len);
s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO);
*(void **)sdesc = s;
}
if (parent_entry_tag == NULL)
@ -736,17 +726,16 @@ sge_timer_cb(void *arg)
struct sge_qset *qs;
struct sge_txq *txq;
int i, j;
int reclaim_eth, reclaim_ofl, refill_rx;
int reclaim_ofl, refill_rx;
for (i = 0; i < sc->params.nports; i++)
for (j = 0; j < sc->port[i].nqsets; j++) {
qs = &sc->sge.qs[i + j];
txq = &qs->txq[0];
reclaim_eth = txq[TXQ_ETH].processed - txq[TXQ_ETH].cleaned;
reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned;
refill_rx = ((qs->fl[0].credits < qs->fl[0].size) ||
(qs->fl[1].credits < qs->fl[1].size));
if (reclaim_eth || reclaim_ofl || refill_rx) {
if (reclaim_ofl || refill_rx) {
pi = &sc->port[i];
taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task);
break;
@ -834,31 +823,14 @@ refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits)
static __inline void
sge_txq_reclaim_(struct sge_txq *txq, int force)
{
int reclaimable, n;
struct port_info *pi;
pi = txq->port;
reclaim_more:
n = 0;
if ((reclaimable = desc_reclaimable(txq)) < 16)
if (desc_reclaimable(txq) < 16)
return;
if (mtx_trylock(&txq->lock) == 0)
return;
n = reclaim_completed_tx_(txq, 16);
reclaim_completed_tx_(txq, 16);
mtx_unlock(&txq->lock);
if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
txq->size - txq->in_use >= TX_START_MAX_DESC) {
struct sge_qset *qs = txq_to_qset(txq, TXQ_ETH);
txq_fills++;
pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
clrbit(&qs->txq_stopped, TXQ_ETH);
taskqueue_enqueue(pi->tq, &pi->start_task);
}
if (n)
goto reclaim_more;
}
static void
@ -886,8 +858,6 @@ sge_timer_reclaim(void *arg, int ncount)
#endif
for (i = 0; i < nqsets; i++) {
qs = &sc->sge.qs[i];
txq = &qs->txq[TXQ_ETH];
sge_txq_reclaim_(txq, FALSE);
txq = &qs->txq[TXQ_OFLD];
sge_txq_reclaim_(txq, FALSE);
@ -1184,7 +1154,7 @@ write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs
* is freed all clusters will be freed
* with it
*/
txsd->mi.mi_base = NULL;
KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p", txsd->mi.mi_base));
wrp = (struct work_request_hdr *)txd;
wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
V_WR_SGLSFLT(1)) | wr_hi;
@ -1244,6 +1214,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
DPRINTF("t3_encap cpu=%d ", curcpu);
mi = NULL;
pi = qs->port;
sc = pi->adapter;
txq = &qs->txq[TXQ_ETH];
@ -1252,9 +1223,13 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
sgl = txq->txq_sgl;
segs = txq->txq_segs;
m0 = *m;
DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset);
DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan);
if (cxgb_debug)
printf("mi_base=%p cidx=%d pidx=%d\n\n", txsd->mi.mi_base, txq->cidx, txq->pidx);
mtx_assert(&txq->lock, MA_OWNED);
cntrl = V_TXPKT_INTF(pi->txpkt_intf);
/*
* XXX need to add VLAN support for 6.x
@ -1263,7 +1238,9 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
if (m0->m_pkthdr.csum_flags & (CSUM_TSO))
tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz);
#endif
KASSERT(txsd->mi.mi_base == NULL, ("overwrting valid entry mi_base==%p",
txsd->mi.mi_base));
if (count > 1) {
panic("count > 1 not support in CVS\n");
if ((err = busdma_map_sg_vec(m, &m0, segs, count)))
@ -1276,13 +1253,10 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
}
KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d count=%d", nsegs, count));
if (m0->m_type == MT_DATA)
DPRINTF("mbuf type=%d tags:%d head=%p", m0->m_type, !SLIST_EMPTY(&m0->m_pkthdr.tags),
SLIST_FIRST(&m0->m_pkthdr.tags));
mi_collapse_mbuf(&txsd->mi, m0);
mi = &txsd->mi;
if (m0->m_pkthdr.len > PIO_LEN) {
mi_collapse_mbuf(&txsd->mi, m0);
mi = &txsd->mi;
}
if (count > 1) {
struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd;
int i, fidx;
@ -1373,25 +1347,11 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
mlen = m0->m_pkthdr.len;
cpl->len = htonl(mlen | 0x80000000);
if (mlen <= WR_LEN - sizeof(*cpl)) {
if (mlen <= PIO_LEN) {
txq_prod(txq, 1, &txqs);
DPRINTF("mlen==%d max=%ld\n", mlen, (WR_LEN - sizeof(*cpl)));
if (mi->mi_type != MT_IOVEC &&
mi->mi_type != MT_CLIOVEC)
memcpy(&txd->flit[2], mi->mi_data, mlen);
else {
/*
* XXX mbuf_iovec
*/
#if 0
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
#endif
printf("bailing on m_copydata\n");
}
m_freem_iovec(&txsd->mi);
txsd->mi.mi_base = NULL;
m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]);
m_freem(m0);
m0 = NULL;
flits = (mlen + 7) / 8 + 2;
cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) |
V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) |
@ -1797,13 +1757,18 @@ t3_sge_stop(adapter_t *sc)
for (nqsets = i = 0; i < (sc)->params.nports; i++)
nqsets += sc->port[i].nqsets;
#ifdef notyet
/*
*
* XXX
*/
for (i = 0; i < nqsets; ++i) {
struct sge_qset *qs = &sc->sge.qs[i];
taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
}
#endif
}
/**
@ -1839,7 +1804,8 @@ t3_free_tx_desc(struct sge_txq *q, int reclaimable)
bus_dmamap_unload(q->entry_tag, txsd->map);
txsd->flags &= ~TX_SW_DESC_MAPPED;
}
m_freem_iovec(&txsd->mi);
m_freem_iovec(&txsd->mi);
buf_ring_scan(&q->txq_mr, txsd->mi.mi_base, __FILE__, __LINE__);
txsd->mi.mi_base = NULL;
#if defined(DIAGNOSTIC) && 0
@ -2513,7 +2479,7 @@ init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
static int
get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m)
struct t3_mbuf_hdr *mh, struct rsp_desc *r)
{
unsigned int len_cq = ntohl(r->len_cq);
@ -2522,6 +2488,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
uint32_t len = G_RSPD_LEN(len_cq);
uint32_t flags = ntohl(r->flags);
uint8_t sopeop = G_RSPD_SOP_EOP(flags);
struct mbuf *m;
uint32_t *ref;
int ret = 0;
@ -2536,13 +2503,13 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
cl = mtod(m0, void *);
memcpy(cl, sd->data, len);
recycle_rx_buf(adap, fl, fl->cidx);
*m = m0;
m = m0;
} else {
skip_recycle:
int flags = 0;
bus_dmamap_unload(fl->entry_tag, sd->map);
cl = sd->rxsd_cl;
*m = m0 = (struct mbuf *)cl;
m = m0 = (struct mbuf *)cl;
m0->m_len = len;
if ((sopeop == RSPQ_SOP_EOP) ||
@ -2561,8 +2528,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
case RSPQ_NSOP_NEOP:
DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m));
if (mh->mh_tail == NULL) {
if (cxgb_debug)
printf("discarding intermediate descriptor entry\n");
printf("discarding intermediate descriptor entry\n");
m_freem(m);
break;
}
@ -2798,18 +2764,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
#ifdef DISABLE_MBUF_IOVEC
struct mbuf *m;
m = m_gethdr(M_DONTWAIT, MT_DATA);
if (m == NULL) {
log(LOG_WARNING, "failed to get mbuf for packet\n");
budget_left--;
break;
} else {
m->m_next = m->m_nextpkt = NULL;
}
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m);
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r);
#else
eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r);
#ifdef IFNET_MULTIQUEUE

4
sys/dev/cxgb/sys/mvec.h
View File

@ -232,13 +232,15 @@ m_freem_iovec(struct mbuf_iovec *mi)
struct mbuf *m;
switch (mi->mi_type) {
case EXT_MBUF:
m_free_fast((struct mbuf *)mi->mi_base);
break;
case EXT_IOVEC:
case EXT_CLIOVEC:
case EXT_JMPIOVEC:
m = (struct mbuf *)mi->mi_base;
m_free_iovec(m, mi->mi_type);
break;
case EXT_MBUF:
case EXT_CLUSTER:
case EXT_JUMBOP:
case EXT_JUMBO9:

59
sys/dev/cxgb/sys/uipc_mvec.c
View File

@ -162,7 +162,7 @@ _mcl_collapse_mbuf(struct mbuf_iovec *mi, struct mbuf *m)
}
KASSERT(mi->mi_len != 0, ("miov has len 0"));
KASSERT(mi->mi_type > 0, ("mi_type is invalid"));
KASSERT(mi->mi_base, ("mi_base is invalid"));
return (n);
}
@ -204,40 +204,15 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs)
if (n->m_flags & M_PKTHDR && !SLIST_EMPTY(&n->m_pkthdr.tags))
m_tag_delete_chain(n, NULL);
if (n->m_pkthdr.len <= PIO_LEN)
return (0);
retry:
seg_count = 0;
if (n->m_next == NULL) {
busdma_map_mbuf_fast(n, segs);
*nsegs = 1;
return (0);
}
if (n->m_pkthdr.len <= 104) {
caddr_t data;
if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL)
return (ENOMEM);
data = m0->m_data;
memcpy(m0, n, sizeof(struct m_hdr) + sizeof(struct pkthdr));
m0->m_data = data;
m0->m_len = n->m_pkthdr.len;
m0->m_flags &= ~M_EXT;
m0->m_next = NULL;
m0->m_type = n->m_type;
n->m_flags &= ~M_PKTHDR;
while (n) {
memcpy(data, n->m_data, n->m_len);
data += n->m_len;
n = n->m_next;
}
m_freem(*m);
n = m0;
*m = n;
goto retry;
}
while (n && seg_count < TX_MAX_SEGS) {
marray[seg_count] = n;
@ -249,18 +224,6 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs)
n = n->m_next;
}
#if 0
/*
* XXX needs more careful consideration
*/
if (__predict_false(seg_count == 1)) {
n = marray[0];
if (n != *m)
/* XXX */
goto retry;
}
#endif
if (seg_count == 0) {
if (cxgb_debug)
printf("empty segment chain\n");
@ -302,16 +265,20 @@ busdma_map_sg_collapse(struct mbuf **m, bus_dma_segment_t *segs, int *nsegs)
}
n = *m;
while (n) {
if (((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT) &&
(n->m_len > 0) && (n->m_ext.ext_type != EXT_PACKET) )
if (n->m_ext.ext_type == EXT_PACKET)
goto skip;
else if (n->m_len == 0)
/* do nothing */;
else if ((n->m_flags & (M_EXT|M_NOFREE)) == M_EXT)
n->m_flags &= ~M_EXT;
else if ((n->m_len > 0) || (n->m_ext.ext_type == EXT_PACKET)) {
n = n->m_next;
continue;
}
else
goto skip;
mhead = n->m_next;
m_free(n);
n = mhead;
continue;
skip:
n = n->m_next;
}
*nsegs = seg_count;
*m = m0;