cxgbe(4): Use opaque cookies or tid range-checks to determine the

intended recipient of a CPL when it can't be determined solely from the
opcode.  Retire the per-queue handlers for such CPLs in favor of the new
scheme.

Sponsored by:	Chelsio Communications
This commit is contained in:
Navdeep Parhar 2018-04-30 15:18:38 +00:00
parent 44af5666d9
commit 4535e8046f
7 changed files with 136 additions and 149 deletions

View File

@ -365,6 +365,19 @@ enum {
NM_BUSY = 2,
};
enum {
CPL_COOKIE_RESERVED = 0,
CPL_COOKIE_FILTER,
CPL_COOKIE_DDP0,
CPL_COOKIE_DDP1,
CPL_COOKIE_TOM,
CPL_COOKIE_AVAILABLE1,
CPL_COOKIE_AVAILABLE2,
CPL_COOKIE_AVAILABLE3,
NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */
};
struct sge_iq;
struct rss_header;
typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
@ -379,8 +392,6 @@ struct sge_iq {
uint32_t flags;
volatile int state;
struct adapter *adapter;
cpl_handler_t set_tcb_rpl;
cpl_handler_t l2t_write_rpl;
struct iq_desc *desc; /* KVA of descriptor ring */
int8_t intr_pktc_idx; /* packet count threshold index */
uint8_t gen; /* generation bit */
@ -1203,9 +1214,10 @@ int parse_pkt(struct adapter *, struct mbuf **);
void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *);
void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *);
int tnl_cong(struct port_info *, int);
int t4_register_an_handler(an_handler_t);
int t4_register_fw_msg_handler(int, fw_msg_handler_t);
int t4_register_cpl_handler(int, cpl_handler_t);
void t4_register_an_handler(an_handler_t);
void t4_register_fw_msg_handler(int, fw_msg_handler_t);
void t4_register_cpl_handler(int, cpl_handler_t);
void t4_register_shared_cpl_handler(int, cpl_handler_t, int);
/* t4_tracer.c */
struct t4_tracer;

View File

@ -617,8 +617,6 @@ static int del_filter(struct adapter *, struct t4_filter *);
static void clear_filter(struct filter_entry *);
static int set_filter_wr(struct adapter *, int);
static int del_filter_wr(struct adapter *, int);
static int set_tcb_rpl(struct sge_iq *, const struct rss_header *,
struct mbuf *);
static int get_sge_context(struct adapter *, struct t4_sge_context *);
static int load_fw(struct adapter *, struct t4_data *);
static int load_cfg(struct adapter *, struct t4_data *);
@ -9269,22 +9267,6 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
static int
set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
MPASS(iq->set_tcb_rpl != NULL);
return (iq->set_tcb_rpl(iq, rss, m));
}
static int
l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
MPASS(iq->l2t_write_rpl != NULL);
return (iq->l2t_write_rpl(iq, rss, m));
}
static int
get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
{
@ -10556,8 +10538,10 @@ mod_event(module_t mod, int cmd, void *arg)
sx_xlock(&mlu);
if (loaded++ == 0) {
t4_sge_modload();
t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl);
t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl);
t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
t4_filter_rpl, CPL_COOKIE_FILTER);
t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL,
do_l2t_write_rpl, CPL_COOKIE_FILTER);
t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
sx_init(&t4_list_lock, "T4/T5 adapters");

View File

@ -285,98 +285,122 @@ static counter_u64_t extfree_rels;
an_handler_t t4_an_handler;
fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES];
cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES];
static int
an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
{
#ifdef INVARIANTS
panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
#else
log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
__func__, iq, ctrl);
#endif
return (EDOOFUS);
}
int
void
t4_register_an_handler(an_handler_t h)
{
uintptr_t *loc, new;
uintptr_t *loc;
new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
loc = (uintptr_t *) &t4_an_handler;
atomic_store_rel_ptr(loc, new);
MPASS(h == NULL || t4_an_handler == NULL);
return (0);
loc = (uintptr_t *)&t4_an_handler;
atomic_store_rel_ptr(loc, (uintptr_t)h);
}
static int
fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
{
const struct cpl_fw6_msg *cpl =
__containerof(rpl, struct cpl_fw6_msg, data[0]);
#ifdef INVARIANTS
panic("%s: fw_msg type %d", __func__, cpl->type);
#else
log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
#endif
return (EDOOFUS);
}
int
void
t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
{
uintptr_t *loc, new;
if (type >= nitems(t4_fw_msg_handler))
return (EINVAL);
uintptr_t *loc;
MPASS(type < nitems(t4_fw_msg_handler));
MPASS(h == NULL || t4_fw_msg_handler[type] == NULL);
/*
* These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
* handler dispatch table. Reject any attempt to install a handler for
* this subtype.
*/
if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
return (EINVAL);
MPASS(type != FW_TYPE_RSSCPL);
MPASS(type != FW6_TYPE_RSSCPL);
new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
loc = (uintptr_t *) &t4_fw_msg_handler[type];
atomic_store_rel_ptr(loc, new);
loc = (uintptr_t *)&t4_fw_msg_handler[type];
atomic_store_rel_ptr(loc, (uintptr_t)h);
}
return (0);
void
t4_register_cpl_handler(int opcode, cpl_handler_t h)
{
uintptr_t *loc;
MPASS(opcode < nitems(t4_cpl_handler));
MPASS(h == NULL || t4_cpl_handler[opcode] == NULL);
loc = (uintptr_t *)&t4_cpl_handler[opcode];
atomic_store_rel_ptr(loc, (uintptr_t)h);
}
static int
cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
struct mbuf *m)
{
const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
u_int tid;
int cookie;
#ifdef INVARIANTS
panic("%s: opcode 0x%02x on iq %p with payload %p",
__func__, rss->opcode, iq, m);
#else
log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
__func__, rss->opcode, iq, m);
m_freem(m);
#endif
return (EDOOFUS);
MPASS(m == NULL);
tid = GET_TID(cpl);
if (is_ftid(iq->adapter, tid)) {
/*
* The return code for filter-write is put in the CPL cookie so
* we have to rely on the hardware tid (is_ftid) to determine
* that this is a response to a filter.
*/
cookie = CPL_COOKIE_FILTER;
} else {
cookie = G_COOKIE(cpl->cookie);
}
MPASS(cookie > CPL_COOKIE_RESERVED);
MPASS(cookie < nitems(set_tcb_rpl_handlers));
return (set_tcb_rpl_handlers[cookie](iq, rss, m));
}
int
t4_register_cpl_handler(int opcode, cpl_handler_t h)
static int
l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
struct mbuf *m)
{
uintptr_t *loc, new;
const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
unsigned int cookie;
if (opcode >= nitems(t4_cpl_handler))
return (EINVAL);
MPASS(m == NULL);
new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
loc = (uintptr_t *) &t4_cpl_handler[opcode];
atomic_store_rel_ptr(loc, new);
cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER;
return (l2t_write_rpl_handlers[cookie](iq, rss, m));
}
return (0);
static void
t4_init_shared_cpl_handlers(void)
{
t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler);
t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler);
}
void
t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
{
uintptr_t *loc;
MPASS(opcode < nitems(t4_cpl_handler));
MPASS(cookie > CPL_COOKIE_RESERVED);
MPASS(cookie < NUM_CPL_COOKIES);
MPASS(t4_cpl_handler[opcode] != NULL);
switch (opcode) {
case CPL_SET_TCB_RPL:
loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie];
break;
case CPL_L2T_WRITE_RPL:
loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie];
break;
default:
MPASS(0);
return;
}
MPASS(h == NULL || *loc == (uintptr_t)NULL);
atomic_store_rel_ptr(loc, (uintptr_t)h);
}
/*
@ -385,7 +409,6 @@ t4_register_cpl_handler(int opcode, cpl_handler_t h)
void
t4_sge_modload(void)
{
int i;
if (fl_pktshift < 0 || fl_pktshift > 7) {
printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
@ -425,12 +448,7 @@ t4_sge_modload(void)
counter_u64_zero(extfree_refs);
counter_u64_zero(extfree_rels);
t4_an_handler = an_not_handled;
for (i = 0; i < nitems(t4_fw_msg_handler); i++)
t4_fw_msg_handler[i] = fw_msg_not_handled;
for (i = 0; i < nitems(t4_cpl_handler); i++)
t4_cpl_handler[i] = cpl_not_handled;
t4_init_shared_cpl_handlers();
t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
@ -2901,11 +2919,8 @@ alloc_fwq(struct adapter *sc)
init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE);
if (sc->flags & IS_VF)
intr_idx = 0;
else {
else
intr_idx = sc->intr_count > 1 ? 1 : 0;
fwq->set_tcb_rpl = t4_filter_rpl;
fwq->l2t_write_rpl = do_l2t_write_rpl;
}
rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1);
if (rc != 0) {
device_printf(sc->dev,

View File

@ -1902,44 +1902,6 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
return (0);
}
int
do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_SET_TCB_RPL,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
MPASS(iq != &sc->sge.fwq);
toep = lookup_tid(sc, tid);
if (toep->ulp_mode == ULP_MODE_TCPDDP) {
handle_ddp_tcb_rpl(toep, cpl);
return (0);
}
/*
* TOM and/or other ULPs don't request replies for CPL_SET_TCB or
* CPL_SET_TCB_FIELD requests. This can easily change and when it does
* the dispatch code will go here.
*/
#ifdef INVARIANTS
panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
tid, iq);
#else
log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
__func__, tid, iq);
#endif
return (0);
}
void
t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
@ -1949,6 +1911,9 @@ t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
struct ofld_tx_sdesc *txsd;
MPASS((cookie & ~M_COOKIE) == 0);
if (reply) {
MPASS(cookie != CPL_COOKIE_RESERVED);
}
wr = alloc_wrqe(sizeof(*req), wrq);
if (wr == NULL) {

View File

@ -626,11 +626,17 @@ enum {
DDP_BUF1_INVALIDATED
};
void
handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl)
CTASSERT(DDP_BUF0_INVALIDATED == CPL_COOKIE_DDP0);
static int
do_ddp_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
unsigned int db_idx;
struct inpcb *inp = toep->inp;
struct toepcb *toep;
struct inpcb *inp;
struct ddp_buffer *db;
struct kaiocb *job;
long copied;
@ -638,6 +644,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl)
if (cpl->status != CPL_ERR_NONE)
panic("XXX: tcp_rpl failed: %d", cpl->status);
toep = lookup_tid(sc, tid);
inp = toep->inp;
switch (cpl->cookie) {
case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF0_INVALIDATED):
case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF1_INVALIDATED):
@ -645,6 +653,7 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl)
* XXX: This duplicates a lot of code with handle_ddp_data().
*/
db_idx = G_COOKIE(cpl->cookie) - DDP_BUF0_INVALIDATED;
MPASS(db_idx < nitems(toep->ddp.db));
INP_WLOCK(inp);
DDP_LOCK(toep);
db = &toep->ddp.db[db_idx];
@ -689,6 +698,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl)
panic("XXX: unknown tcb_rpl offset %#x, cookie %#x",
G_WORD(cpl->cookie), G_COOKIE(cpl->cookie));
}
return (0);
}
void
@ -1941,6 +1952,10 @@ void
t4_ddp_mod_load(void)
{
t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl,
CPL_COOKIE_DDP0);
t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl,
CPL_COOKIE_DDP1);
t4_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp);
t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
TAILQ_INIT(&ddp_orphan_pagesets);

View File

@ -1330,8 +1330,7 @@ t4_tom_activate(struct adapter *sc)
struct tom_data *td;
struct toedev *tod;
struct vi_info *vi;
struct sge_ofld_rxq *ofld_rxq;
int i, j, rc, v;
int i, rc, v;
ASSERT_SYNCHRONIZED_OP(sc);
@ -1398,10 +1397,6 @@ t4_tom_activate(struct adapter *sc)
for_each_port(sc, i) {
for_each_vi(sc->port[i], v, vi) {
TOEDEV(vi->ifp) = &td->tod;
for_each_ofld_rxq(vi, j, ofld_rxq) {
ofld_rxq->iq.set_tcb_rpl = do_set_tcb_rpl;
ofld_rxq->iq.l2t_write_rpl = do_l2t_write_rpl2;
}
}
}
@ -1504,6 +1499,8 @@ t4_tom_mod_load(void)
struct protosw *tcp_protosw, *tcp6_protosw;
/* CPL handlers */
t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl2,
CPL_COOKIE_TOM);
t4_init_connect_cpl_handlers();
t4_init_listen_cpl_handlers();
t4_init_cpl_io_handlers();
@ -1568,6 +1565,7 @@ t4_tom_mod_unload(void)
t4_uninit_connect_cpl_handlers();
t4_uninit_listen_cpl_handlers();
t4_uninit_cpl_io_handlers();
t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, NULL, CPL_COOKIE_TOM);
return (0);
}

View File

@ -390,7 +390,6 @@ void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *,
uint16_t, uint64_t, uint64_t, int, int);
void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop);
void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop);
int do_set_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
/* t4_ddp.c */
int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int,
@ -416,7 +415,6 @@ void ddp_queue_toep(struct toepcb *);
void release_ddp_resources(struct toepcb *toep);
void handle_ddp_close(struct toepcb *, struct tcpcb *, uint32_t);
void handle_ddp_indicate(struct toepcb *);
void handle_ddp_tcb_rpl(struct toepcb *, const struct cpl_set_tcb_rpl *);
void insert_ddp_data(struct toepcb *, uint32_t);
const struct offload_settings *lookup_offload_policy(struct adapter *, int,
struct mbuf *, uint16_t, struct inpcb *);