d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cxgbe(4): Updates to the base NIC driver and t4_tom to support the iSCSI

Browse Source 
offload driver.  These changes come from projects/cxl_iscsi.
This commit is contained in:
np 2015-12-26 00:26:02 +00:00
parent 6f44a590da
commit 77193eb303
7 changed files with 177 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sys/dev/cxgbe/adapter.h
View File

@ -766,7 +766,7 @@ struct adapter {
void *tom_softc; /* (struct tom_data *) */ void *tom_softc; /* (struct tom_data *) */
struct tom_tunables tt; struct tom_tunables tt;
void *iwarp_softc; /* (struct c4iw_dev *) */ void *iwarp_softc; /* (struct c4iw_dev *) */
void *iscsi_softc; void *iscsi_ulp_softc; /* (struct cxgbei_data *) */
#endif #endif
struct l2t_data *l2t; /* L2 table */ struct l2t_data *l2t; /* L2 table */
struct tid_info tids; struct tid_info tids;

2
sys/dev/cxgbe/offload.h
View File

@ -156,7 +156,7 @@ int t4_register_uld(struct uld_info *);
int t4_unregister_uld(struct uld_info *); int t4_unregister_uld(struct uld_info *);
int t4_activate_uld(struct adapter *, int); int t4_activate_uld(struct adapter *, int);
int t4_deactivate_uld(struct adapter *, int); int t4_deactivate_uld(struct adapter *, int);
void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *); void t4_iscsi_init(struct adapter *, u_int, const u_int *);
int uld_active(struct adapter *, int); int uld_active(struct adapter *, int);
#endif #endif
#endif #endif

5
sys/dev/cxgbe/t4_main.c
View File

@ -8801,11 +8801,8 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
#ifdef TCP_OFFLOAD #ifdef TCP_OFFLOAD
void void
t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
const unsigned int *pgsz_order)
{ {
struct vi_info *vi = ifp->if_softc;
struct adapter *sc = vi->pi->adapter;
t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |

349
sys/dev/cxgbe/tom/t4_cpl_io.c
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2012 Chelsio Communications, Inc. * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
* All rights reserved. * All rights reserved.
* Written by: Navdeep Parhar <np@FreeBSD.org> * Written by: Navdeep Parhar <np@FreeBSD.org>
* *
@ -71,33 +71,6 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc);
VNET_DECLARE(int, tcp_autorcvbuf_max); VNET_DECLARE(int, tcp_autorcvbuf_max);
#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
/*
* For ULP connections HW may add headers, e.g., for digests, that aren't part
* of the messages sent by the host but that are part of the TCP payload and
* therefore consume TCP sequence space. Tx connection parameters that
* operate in TCP sequence space are affected by the HW additions and need to
* compensate for them to accurately track TCP sequence numbers. This array
* contains the compensating extra lengths for ULP packets. It is indexed by
* a packet's ULP submode.
*/
const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8};
/*
* Return the length of any HW additions that will be made to a Tx packet.
* Such additions can happen for some types of ULP packets.
*/
static inline unsigned int
ulp_extra_len(struct mbuf *m, int *ulp_mode)
{
struct m_tag *mtag;
if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL)
return (0);
*ulp_mode = *((int *)(mtag + 1));
return (t4_ulp_extra_len[*ulp_mode & 3]);
}
void void
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
{ {
@ -384,13 +357,10 @@ t4_rcvd(struct toedev *tod, struct tcpcb *tp)
KASSERT(toep->sb_cc >= sbused(sb), KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).", ("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc)); __func__, sb, sbused(sb), toep->sb_cc));
if (toep->ulp_mode == ULP_MODE_ISCSI) {
toep->rx_credits += toep->sb_cc; toep->rx_credits += toep->sb_cc - sbused(sb);
toep->sb_cc = 0; toep->sb_cc = sbused(sb);
} else {
toep->rx_credits += toep->sb_cc - sbused(sb);
toep->sb_cc = sbused(sb);
}
if (toep->rx_credits > 0 && if (toep->rx_credits > 0 &&
(tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
(toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
@ -490,25 +460,16 @@ max_dsgl_nsegs(int tx_credits)
static inline void static inline void
write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign) unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign)
{ {
struct fw_ofld_tx_data_wr *txwr = dst; struct fw_ofld_tx_data_wr *txwr = dst;
unsigned int wr_ulp_mode;
txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
V_FW_WR_IMMDLEN(immdlen)); V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
V_FW_WR_LEN16(credits)); V_FW_WR_LEN16(credits));
txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) |
/* for iscsi, the mode & submode setting is per-packet */ V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove));
if (toep->ulp_mode == ULP_MODE_ISCSI)
wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) |
V_TX_ULP_SUBMODE(ulp_mode & 3);
else
wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode);
txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/
V_TX_SHOVE(shove));
txwr->plen = htobe32(plen); txwr->plen = htobe32(plen);
if (txalign > 0) { if (txalign > 0) {
@ -616,6 +577,9 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
toep->ulp_mode == ULP_MODE_RDMA, toep->ulp_mode == ULP_MODE_RDMA,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
return;
/* /*
* This function doesn't resume by itself. Someone else must clear the * This function doesn't resume by itself. Someone else must clear the
* flag and call this function. * flag and call this function.
@ -802,56 +766,70 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
close_conn(sc, toep); close_conn(sc, toep);
} }
/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ static inline void
void rqdrop_locked(struct mbufq *q, int plen)
t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
{ {
struct mbuf *sndptr, *m = NULL; struct mbuf *m;
while (plen > 0) {
m = mbufq_dequeue(q);
/* Too many credits. */
MPASS(m != NULL);
M_ASSERTPKTHDR(m);
/* Partial credits. */
MPASS(plen >= m->m_pkthdr.len);
plen -= m->m_pkthdr.len;
m_freem(m);
}
}
void
t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop)
{
struct mbuf *sndptr, *m;
struct fw_ofld_tx_data_wr *txwr; struct fw_ofld_tx_data_wr *txwr;
struct wrqe *wr; struct wrqe *wr;
unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
u_int adjusted_plen, ulp_submode;
struct inpcb *inp = toep->inp; struct inpcb *inp = toep->inp;
struct tcpcb *tp; struct tcpcb *tp = intotcpcb(inp);
struct socket *so; int tx_credits, shove;
struct sockbuf *sb; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0; struct mbufq *pduq = &toep->ulp_pduq;
int shove, compl; static const u_int ulp_extra_len[] = {0, 4, 4, 8};
struct ofld_tx_sdesc *txsd;
INP_WLOCK_ASSERT(inp); INP_WLOCK_ASSERT(inp);
if (toep->flags & TPF_ABORT_SHUTDOWN)
return;
tp = intotcpcb(inp);
so = inp->inp_socket;
sb = &so->so_snd;
txsd = &toep->txsd[toep->txsd_pidx];
KASSERT(toep->flags & TPF_FLOWC_WR_SENT, KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(toep->ulp_mode == ULP_MODE_ISCSI,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
return;
/* /*
* This function doesn't resume by itself. Someone else must clear the * This function doesn't resume by itself. Someone else must clear the
* flag and call this function. * flag and call this function.
*/ */
if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
KASSERT(drop == 0,
("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
return; return;
}
sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); if (drop)
if (!qlen) rqdrop_locked(&toep->ulp_pdu_reclaimq, drop);
return;
while ((sndptr = mbufq_first(pduq)) != NULL) {
M_ASSERTPKTHDR(sndptr);
do {
tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
max_imm = max_imm_payload(tx_credits); max_imm = max_imm_payload(tx_credits);
max_nsegs = max_dsgl_nsegs(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits);
if (drop) {
t4_cpl_iscsi_callback(toep->td, toep, &drop,
CPL_FW4_ACK);
drop = 0;
}
plen = 0; plen = 0;
nsegs = 0; nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
@ -861,7 +839,10 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
nsegs += n; nsegs += n;
plen += m->m_len; plen += m->m_len;
/* This mbuf sent us _over_ the nsegs limit, return */ /*
* This mbuf would send us _over_ the nsegs limit.
* Suspend tx because the PDU can't be sent out.
*/
if (plen > max_imm && nsegs > max_nsegs) { if (plen > max_imm && nsegs > max_nsegs) {
toep->flags |= TPF_TX_SUSPENDED; toep->flags |= TPF_TX_SUSPENDED;
return; return;
@ -869,30 +850,35 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
if (max_nsegs_1mbuf < n) if (max_nsegs_1mbuf < n)
max_nsegs_1mbuf = n; max_nsegs_1mbuf = n;
/* This mbuf put us right at the max_nsegs limit */
if (plen > max_imm && nsegs == max_nsegs) {
toep->flags |= TPF_TX_SUSPENDED;
return;
}
}
shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
/* nothing to send */
if (plen == 0) {
KASSERT(m == NULL,
("%s: nothing to send, but m != NULL", __func__));
break;
} }
if (__predict_false(toep->flags & TPF_FIN_SENT)) if (__predict_false(toep->flags & TPF_FIN_SENT))
panic("%s: excess tx.", __func__); panic("%s: excess tx.", __func__);
ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode); /*
* We have a PDU to send. All of it goes out in one WR so 'm'
* is NULL. A PDU's length is always a multiple of 4.
*/
MPASS(m == NULL);
MPASS((plen & 3) == 0);
MPASS(sndptr->m_pkthdr.len == plen);
shove = !(tp->t_flags & TF_MORETOCOME);
ulp_submode = mbuf_ulp_submode(sndptr);
MPASS(ulp_submode < nitems(ulp_extra_len));
/*
* plen doesn't include header and data digests, which are
* generated and inserted in the right places by the TOE, but
* they do occupy TCP sequence space and need to be accounted
* for.
*/
adjusted_plen = plen + ulp_extra_len[ulp_submode];
if (plen <= max_imm) { if (plen <= max_imm) {
/* Immediate data tx */ /* Immediate data tx */
wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
toep->ofld_txq); toep->ofld_txq);
if (wr == NULL) { if (wr == NULL) {
/* XXX: how will we recover from this? */ /* XXX: how will we recover from this? */
@ -901,16 +887,17 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
} }
txwr = wrtod(wr); txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16); credits = howmany(wr->wr_len, 16);
write_tx_wr(txwr, toep, plen, ulp_len, credits, shove, write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
ulp_mode, 0); shove, ulp_submode, sc->tt.tx_align);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
nsegs = 0;
} else { } else {
int wr_len; int wr_len;
/* DSGL tx */ /* DSGL tx */
wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq); wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
if (wr == NULL) { if (wr == NULL) {
/* XXX: how will we recover from this? */ /* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED; toep->flags |= TPF_TX_SUSPENDED;
@ -918,8 +905,8 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
} }
txwr = wrtod(wr); txwr = wrtod(wr);
credits = howmany(wr_len, 16); credits = howmany(wr_len, 16);
write_tx_wr(txwr, toep, 0, ulp_len, credits, shove, write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
ulp_mode, 0); shove, ulp_submode, sc->tt.tx_align);
write_tx_sgl(txwr + 1, sndptr, m, nsegs, write_tx_sgl(txwr + 1, sndptr, m, nsegs,
max_nsegs_1mbuf); max_nsegs_1mbuf);
if (wr_len & 0xf) { if (wr_len & 0xf) {
@ -932,28 +919,26 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
KASSERT(toep->tx_credits >= credits, KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__)); ("%s: not enough credits", __func__));
m = mbufq_dequeue(pduq);
MPASS(m == sndptr);
mbufq_enqueue(&toep->ulp_pdu_reclaimq, m);
toep->tx_credits -= credits; toep->tx_credits -= credits;
toep->tx_nocompl += credits; toep->tx_nocompl += credits;
toep->plen_nocompl += plen; toep->plen_nocompl += plen;
if (toep->tx_credits <= toep->tx_total * 3 / 8 && if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
toep->tx_nocompl >= toep->tx_total / 4) toep->tx_nocompl >= toep->tx_total / 4) {
compl = 1;
if (compl) {
txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
toep->tx_nocompl = 0; toep->tx_nocompl = 0;
toep->plen_nocompl = 0; toep->plen_nocompl = 0;
} }
tp->snd_nxt += ulp_len;
tp->snd_max += ulp_len;
/* goto next mbuf */ tp->snd_nxt += adjusted_plen;
sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); tp->snd_max += adjusted_plen;
toep->flags |= TPF_TX_DATA_SENT; toep->flags |= TPF_TX_DATA_SENT;
if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
toep->flags |= TPF_TX_SUSPENDED; toep->flags |= TPF_TX_SUSPENDED;
}
KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
txsd->plen = plen; txsd->plen = plen;
@ -966,10 +951,10 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
toep->txsd_avail--; toep->txsd_avail--;
t4_l2t_send(sc, wr, toep->l2te); t4_l2t_send(sc, wr, toep->l2te);
} while (m != NULL); }
/* Send a FIN if requested, but only if there's no more data to send */ /* Send a FIN if requested, but only if there are no more PDUs to send */
if (m == NULL && toep->flags & TPF_SEND_FIN) if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
close_conn(sc, toep); close_conn(sc, toep);
} }
@ -987,7 +972,10 @@ t4_tod_output(struct toedev *tod, struct tcpcb *tp)
("%s: inp %p dropped.", __func__, inp)); ("%s: inp %p dropped.", __func__, inp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
t4_push_frames(sc, toep, 0); if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, 0);
else
t4_push_frames(sc, toep, 0);
return (0); return (0);
} }
@ -1009,7 +997,7 @@ t4_send_fin(struct toedev *tod, struct tcpcb *tp)
toep->flags |= TPF_SEND_FIN; toep->flags |= TPF_SEND_FIN;
if (tp->t_state >= TCPS_ESTABLISHED) { if (tp->t_state >= TCPS_ESTABLISHED) {
if (toep->ulp_mode == ULP_MODE_ISCSI) if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_ulp_push_frames(sc, toep, 0); t4_push_pdus(sc, toep, 0);
else else
t4_push_frames(sc, toep, 0); t4_push_frames(sc, toep, 0);
} }
@ -1250,91 +1238,6 @@ abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
} }
} }
int
cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
/*
* tom_cpl_iscsi_callback -
* iscsi and tom would share the following cpl messages, so when any of these
* message is received, after tom is done with processing it, the messages
* needs to be forwarded to iscsi for further processing:
* - CPL_SET_TCB_RPL
* - CPL_RX_DATA_DDP
*/
void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
unsigned int);
struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
/*
* Check if the handler function is set for a given CPL
* return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
*/
int
t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
{
MPASS(opcode < nitems(sc->cpl_handler));
return (sc->cpl_handler[opcode] &&
sc->cpl_handler[opcode] != cpl_not_handled);
}
/*
* set the tom_cpl_iscsi_callback function, this function should be used
* whenever both toe and iscsi need to process the same cpl msg.
*/
void
t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *,
void *, unsigned int))
{
tom_cpl_iscsi_callback = fp;
}
void
t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
unsigned int, int *qlen))
{
tom_queue_iscsi_callback = fp;
}
int
t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
unsigned int opcode)
{
struct socket *so;
if (opcode == CPL_FW4_ACK)
so = toep->inp->inp_socket;
else {
INP_WLOCK(toep->inp);
so = toep->inp->inp_socket;
INP_WUNLOCK(toep->inp);
}
if (tom_cpl_iscsi_callback && so) {
if (toep->ulp_mode == ULP_MODE_ISCSI) {
tom_cpl_iscsi_callback(td, so, m, opcode);
return (0);
}
}
return (1);
}
struct mbuf *
t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
unsigned int cmd, int *qlen)
{
if (tom_queue_iscsi_callback && so) {
if (toep->ulp_mode == ULP_MODE_ISCSI)
return (tom_queue_iscsi_callback(so, cmd, qlen));
}
return (NULL);
}
/* /*
* TCP RST from the peer, timeout, or some other such critical error. * TCP RST from the peer, timeout, or some other such critical error.
*/ */
@ -1733,21 +1636,34 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
toep->tx_credits >= toep->tx_total / 4) { toep->tx_credits >= toep->tx_total / 4) {
toep->flags &= ~TPF_TX_SUSPENDED; toep->flags &= ~TPF_TX_SUSPENDED;
if (toep->ulp_mode == ULP_MODE_ISCSI) if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_ulp_push_frames(sc, toep, plen); t4_push_pdus(sc, toep, plen);
else else
t4_push_frames(sc, toep, plen); t4_push_frames(sc, toep, plen);
} else if (plen > 0) { } else if (plen > 0) {
struct sockbuf *sb = &so->so_snd; struct sockbuf *sb = &so->so_snd;
int sbu;
if (toep->ulp_mode == ULP_MODE_ISCSI) SOCKBUF_LOCK(sb);
t4_cpl_iscsi_callback(toep->td, toep, &plen, sbu = sbused(sb);
CPL_FW4_ACK); if (toep->ulp_mode == ULP_MODE_ISCSI) {
else {
SOCKBUF_LOCK(sb); if (__predict_false(sbu > 0)) {
/*
* The data trasmitted before the tid's ULP mode
* changed to ISCSI is still in so_snd.
* Incoming credits should account for so_snd
* first.
*/
sbdrop_locked(sb, min(sbu, plen));
plen -= min(sbu, plen);
}
sowwakeup_locked(so); /* unlocks so_snd */
rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
} else {
sbdrop_locked(sb, plen); sbdrop_locked(sb, plen);
sowwakeup_locked(so); sowwakeup_locked(so); /* unlocks so_snd */
SOCKBUF_UNLOCK_ASSERT(sb);
} }
SOCKBUF_UNLOCK_ASSERT(sb);
} }
INP_WUNLOCK(inp); INP_WUNLOCK(inp);
@ -1771,14 +1687,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
if (is_ftid(sc, tid)) if (is_ftid(sc, tid))
return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
else {
struct toepcb *toep = lookup_tid(sc, tid);
t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); /*
return (0); * TOM and/or other ULPs don't request replies for CPL_SET_TCB or
} * CPL_SET_TCB_FIELD requests. This can easily change and when it does
* the dispatch code will go here.
*/
#ifdef INVARIANTS
panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
tid, iq);
#else
log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
__func__, tid, iq);
#endif
CXGBE_UNIMPLEMENTED(__func__); return (0);
} }
void void

14
sys/dev/cxgbe/tom/t4_ddp.c
View File

@ -513,7 +513,6 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl); unsigned int tid = GET_TID(cpl);
uint32_t vld; uint32_t vld;
struct toepcb *toep = lookup_tid(sc, tid); struct toepcb *toep = lookup_tid(sc, tid);
struct tom_data *td = toep->td;
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
@ -525,16 +524,11 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
panic("%s: DDP error 0x%x (tid %d, toep %p)", panic("%s: DDP error 0x%x (tid %d, toep %p)",
__func__, vld, tid, toep); __func__, vld, tid, toep);
} }
if (toep->ulp_mode == ULP_MODE_ISCSI) { if (toep->ulp_mode == ULP_MODE_ISCSI) {
m = m_get(M_NOWAIT, MT_DATA); sc->cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m);
if (m == NULL) return (0);
CXGBE_UNIMPLEMENTED("mbuf alloc failure"); }
memcpy(mtod(m, unsigned char *), cpl,
sizeof(struct cpl_rx_data_ddp));
if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP))
return (0);
m_freem(m);
}
handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));

12
sys/dev/cxgbe/tom/t4_tom.c
View File

@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/ktr.h> #include <sys/ktr.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/limits.h>
#include <sys/module.h> #include <sys/module.h>
#include <sys/protosw.h> #include <sys/protosw.h>
#include <sys/domain.h> #include <sys/domain.h>
@ -158,6 +159,8 @@ alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags)
toep->ofld_txq = &sc->sge.ofld_txq[txqid]; toep->ofld_txq = &sc->sge.ofld_txq[txqid];
toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid];
toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; toep->ctrlq = &sc->sge.ctrlq[pi->port_id];
mbufq_init(&toep->ulp_pduq, INT_MAX);
mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX);
toep->txsd_total = txsd_total; toep->txsd_total = txsd_total;
toep->txsd_avail = txsd_total; toep->txsd_avail = txsd_total;
toep->txsd_pidx = 0; toep->txsd_pidx = 0;
@ -273,6 +276,14 @@ release_offload_resources(struct toepcb *toep)
CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)", CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)",
__func__, toep, tid, toep->l2te, toep->ce); __func__, toep, tid, toep->l2te, toep->ce);
/*
* These queues should have been emptied at approximately the same time
* that a normal connection's socket's so_snd would have been purged or
* drained. Do _not_ clean up here.
*/
MPASS(mbufq_len(&toep->ulp_pduq) == 0);
MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0);
if (toep->ulp_mode == ULP_MODE_TCPDDP) if (toep->ulp_mode == ULP_MODE_TCPDDP)
release_ddp_resources(toep); release_ddp_resources(toep);
@ -380,6 +391,7 @@ final_cpl_received(struct toepcb *toep)
toep->inp = NULL; toep->inp = NULL;
toep->flags &= ~TPF_CPL_PENDING; toep->flags &= ~TPF_CPL_PENDING;
mbufq_drain(&toep->ulp_pdu_reclaimq);
if (!(toep->flags & TPF_ATTACHED)) if (!(toep->flags & TPF_ATTACHED))
release_offload_resources(toep); release_offload_resources(toep);

38
sys/dev/cxgbe/tom/t4_tom.h
View File

@ -1,5 +1,5 @@
/*- /*-
* Copyright (c) 2012 Chelsio Communications, Inc. * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
* All rights reserved. * All rights reserved.
* Written by: Navdeep Parhar <np@FreeBSD.org> * Written by: Navdeep Parhar <np@FreeBSD.org>
* *
@ -115,6 +115,10 @@ struct toepcb {
int rx_credits; /* rx credits (in bytes) to be returned to hw */ int rx_credits; /* rx credits (in bytes) to be returned to hw */
u_int ulp_mode; /* ULP mode */ u_int ulp_mode; /* ULP mode */
void *ulpcb;
void *ulpcb2;
struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */
struct mbufq ulp_pdu_reclaimq;
u_int ddp_flags; u_int ddp_flags;
struct ddp_buffer *db[2]; struct ddp_buffer *db[2];
@ -220,6 +224,22 @@ td_adapter(struct tom_data *td)
return (td->tod.tod_softc); return (td->tod.tod_softc);
} }
static inline void
set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode)
{
M_ASSERTPKTHDR(m);
m->m_pkthdr.PH_per.eight[0] = ulp_submode;
}
static inline uint8_t
mbuf_ulp_submode(struct mbuf *m)
{
M_ASSERTPKTHDR(m);
return (m->m_pkthdr.PH_per.eight[0]);
}
/* t4_tom.c */ /* t4_tom.c */
struct toepcb *alloc_toepcb(struct vi_info *, int, int, int); struct toepcb *alloc_toepcb(struct vi_info *, int, int, int);
void free_toepcb(struct toepcb *); void free_toepcb(struct toepcb *);
@ -275,6 +295,7 @@ int t4_send_rst(struct toedev *, struct tcpcb *);
void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t, void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t,
uint64_t, uint64_t); uint64_t, uint64_t);
void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop); void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop);
/* t4_ddp.c */ /* t4_ddp.c */
void t4_init_ddp(struct adapter *, struct tom_data *); void t4_init_ddp(struct adapter *, struct tom_data *);
@ -287,19 +308,4 @@ void handle_ddp_close(struct toepcb *, struct tcpcb *, struct sockbuf *,
uint32_t); uint32_t);
void insert_ddp_data(struct toepcb *, uint32_t); void insert_ddp_data(struct toepcb *, uint32_t);
/* ULP related */
#define CXGBE_ISCSI_MBUF_TAG 50
int t4tom_cpl_handler_registered(struct adapter *, unsigned int);
void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *,
struct socket *, void *, unsigned int));
void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
unsigned int, int *));
void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int);
int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t);
struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t,
int *);
extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *,
void *, unsigned int);
extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int,
int *);
#endif #endif