cxgbe(4): Add support for high priority filters on T6+. They have their

own region in the TCAM starting with T6, unlike previous chips where
they were in the same region as normal filters.

These filters "hit" before anything else in the LE's lookup.  The exact
order is:
a) High priority filters
b) TOE's active region (TCAM and/or hash)
c) Servers (TOE hw listeners)
d) Normal filters

MFC after:	1 week
Sponsored by:	Chelsio Communications
This commit is contained in:
np 2018-08-09 14:19:47 +00:00
parent 2f30606f2f
commit 963bc8b754
6 changed files with 243 additions and 68 deletions

View File

@ -441,6 +441,13 @@ static inline int is_ftid(const struct adapter *sc, u_int tid)
tid <= sc->tids.ftid_end);
}
static inline int is_hpftid(const struct adapter *sc, u_int tid)
{
return (sc->tids.nhpftids > 0 && tid >= sc->tids.hpftid_base &&
tid <= sc->tids.hpftid_end);
}
static inline int is_etid(const struct adapter *sc, u_int tid)
{

View File

@ -133,7 +133,12 @@ struct tid_info {
u_int ftid_base;
u_int ftid_end;
u_int nhpftids;
u_int hpftid_base;
u_int hpftid_end;
u_int ntids;
u_int tid_base;
u_int netids;
u_int etid_base;
@ -150,10 +155,13 @@ struct tid_info {
union aopen_entry *afree;
u_int atids_in_use;
/* High priority filters and normal filters share the lock and cv. */
struct mtx ftid_lock __aligned(CACHE_LINE_SIZE);
struct cv ftid_cv;
struct filter_entry *ftid_tab;
struct filter_entry *hpftid_tab;
u_int ftids_in_use;
u_int hpftids_in_use;
/*
* hashfilter and TOE are mutually exclusive and both use ntids and

View File

@ -64,12 +64,20 @@ struct filter_entry {
};
static void free_filter_resources(struct filter_entry *);
static int get_tcamfilter(struct adapter *, struct t4_filter *);
static int get_hashfilter(struct adapter *, struct t4_filter *);
static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
struct l2t_entry *, struct smt_entry *);
static int del_hashfilter(struct adapter *, struct t4_filter *);
static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
static inline bool
separate_hpfilter_region(struct adapter *sc)
{
return (chip_id(sc) >= CHELSIO_T6);
}
static int
alloc_hftid_tab(struct tid_info *t, int flags)
{
@ -302,7 +310,7 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
if (rc)
return (rc);
if (sc->tids.ftids_in_use > 0) {
if (sc->tids.ftids_in_use > 0 || sc->tids.hpftids_in_use > 0) {
rc = EBUSY;
goto done;
}
@ -343,39 +351,10 @@ get_filter_hits(struct adapter *sc, uint32_t tid)
int
get_filter(struct adapter *sc, struct t4_filter *t)
{
int i, nfilters = sc->tids.nftids;
struct filter_entry *f;
if (t->fs.hash)
return (get_hashfilter(sc, t));
if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
t->idx >= nfilters) {
t->idx = 0xffffffff;
return (0);
}
mtx_lock(&sc->tids.ftid_lock);
f = &sc->tids.ftid_tab[t->idx];
for (i = t->idx; i < nfilters; i++, f++) {
if (f->valid) {
MPASS(f->tid == sc->tids.ftid_base + i);
t->idx = i;
t->l2tidx = f->l2te ? f->l2te->idx : 0;
t->smtidx = f->smt ? f->smt->idx : 0;
if (f->fs.hitcnts)
t->hits = get_filter_hits(sc, f->tid);
else
t->hits = UINT64_MAX;
t->fs = f->fs;
goto done;
}
}
t->idx = 0xffffffff;
done:
mtx_unlock(&sc->tids.ftid_lock);
return (0);
else
return (get_tcamfilter(sc, t));
}
static int
@ -387,15 +366,23 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
u_int vnic_vld, vnic_vld_mask;
struct wrq_cookie cookie;
int i, rc, busy, locked;
u_int tid;
const int ntids = t->fs.type ? 4 : 1;
MPASS(!t->fs.hash);
MPASS(t->idx < sc->tids.nftids);
/* Already validated against fconf, iconf */
MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
f = &sc->tids.ftid_tab[t->idx];
if (separate_hpfilter_region(sc) && t->fs.prio) {
MPASS(t->idx < sc->tids.nhpftids);
f = &sc->tids.hpftid_tab[t->idx];
tid = sc->tids.hpftid_base + t->idx;
} else {
MPASS(t->idx < sc->tids.nftids);
f = &sc->tids.ftid_tab[t->idx];
tid = sc->tids.ftid_base + t->idx;
}
rc = busy = locked = 0;
mtx_lock(&sc->tids.ftid_lock);
for (i = 0; i < ntids; i++) {
@ -418,7 +405,10 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
rc = ENOMEM;
else {
f->pending = 1;
sc->tids.ftids_in_use++;
if (separate_hpfilter_region(sc) && t->fs.prio)
sc->tids.hpftids_in_use++;
else
sc->tids.ftids_in_use++;
}
}
mtx_unlock(&sc->tids.ftid_lock);
@ -434,7 +424,7 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
* Can't fail now. A set-filter WR will definitely be sent.
*/
f->tid = sc->tids.ftid_base + t->idx;
f->tid = tid;
f->fs = t->fs;
f->l2te = l2te;
f->smt = smt;
@ -649,10 +639,17 @@ set_filter(struct adapter *sc, struct t4_filter *t)
if (rc != 0)
return (rc);
} else {
if (ti->nftids == 0)
return (ENOTSUP);
if (t->idx >= ti->nftids)
return (EINVAL);
if (separate_hpfilter_region(sc) && t->fs.prio) {
if (ti->nhpftids == 0)
return (ENOTSUP);
if (t->idx >= ti->nhpftids)
return (EINVAL);
} else {
if (ti->nftids == 0)
return (ENOTSUP);
if (t->idx >= ti->nftids)
return (EINVAL);
}
/* IPv6 filter idx must be 4 aligned */
if (t->fs.type == 1 &&
((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
@ -702,17 +699,37 @@ set_filter(struct adapter *sc, struct t4_filter *t)
if (rc != 0)
goto done;
}
} else if (separate_hpfilter_region(sc) && t->fs.prio &&
__predict_false(ti->hpftid_tab == NULL)) {
MPASS(ti->nhpftids != 0);
KASSERT(ti->hpftids_in_use == 0,
("%s: no memory allocated but hpftids_in_use is %u",
__func__, ti->hpftids_in_use));
ti->hpftid_tab = malloc(sizeof(struct filter_entry) *
ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO);
if (ti->hpftid_tab == NULL) {
rc = ENOMEM;
goto done;
}
if (!mtx_initialized(&sc->tids.ftid_lock)) {
mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
cv_init(&ti->ftid_cv, "t4fcv");
}
} else if (__predict_false(ti->ftid_tab == NULL)) {
MPASS(ti->nftids != 0);
KASSERT(ti->ftids_in_use == 0,
("%s: no memory allocated but ftids_in_use > 0", __func__));
("%s: no memory allocated but ftids_in_use is %u",
__func__, ti->ftids_in_use));
ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
M_CXGBE, M_NOWAIT | M_ZERO);
if (ti->ftid_tab == NULL) {
rc = ENOMEM;
goto done;
}
mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
cv_init(&ti->ftid_cv, "t4fcv");
if (!mtx_initialized(&sc->tids.ftid_lock)) {
mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
cv_init(&ti->ftid_cv, "t4fcv");
}
}
done:
end_synchronized_op(sc, 0);
@ -768,16 +785,32 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t)
struct filter_entry *f;
struct fw_filter_wr *fwr;
struct wrq_cookie cookie;
int rc;
MPASS(sc->tids.ftid_tab != NULL);
MPASS(sc->tids.nftids > 0);
if (t->idx >= sc->tids.nftids)
return (EINVAL);
int rc, nfilters;
#ifdef INVARIANTS
u_int tid_base;
#endif
mtx_lock(&sc->tids.ftid_lock);
f = &sc->tids.ftid_tab[t->idx];
if (separate_hpfilter_region(sc) && t->fs.prio) {
nfilters = sc->tids.nhpftids;
f = sc->tids.hpftid_tab;
#ifdef INVARIANTS
tid_base = sc->tids.hpftid_base;
#endif
} else {
nfilters = sc->tids.nftids;
f = sc->tids.ftid_tab;
#ifdef INVARIANTS
tid_base = sc->tids.ftid_base;
#endif
}
MPASS(f != NULL); /* Caller checked this. */
if (t->idx >= nfilters) {
rc = EINVAL;
goto done;
}
f += t->idx;
if (f->locked) {
rc = EPERM;
goto done;
@ -790,7 +823,7 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t)
rc = EINVAL;
goto done;
}
MPASS(f->tid == sc->tids.ftid_base + t->idx);
MPASS(f->tid == tid_base + t->idx);
fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
if (fwr == NULL) {
rc = ENOMEM;
@ -833,6 +866,9 @@ del_filter(struct adapter *sc, struct t4_filter *t)
if (t->fs.hash) {
if (sc->tids.hftid_tab != NULL)
return (del_hashfilter(sc, t));
} else if (separate_hpfilter_region(sc) && t->fs.prio) {
if (sc->tids.hpftid_tab != NULL)
return (del_tcamfilter(sc, t));
} else {
if (sc->tids.ftid_tab != NULL)
return (del_tcamfilter(sc, t));
@ -899,21 +935,28 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
u_int tid = GET_TID(rpl);
u_int rc, cleanup, idx;
u_int rc, idx;
struct filter_entry *f;
KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
rss->opcode));
MPASS(is_ftid(sc, tid));
cleanup = 0;
idx = tid - sc->tids.ftid_base;
f = &sc->tids.ftid_tab[idx];
if (is_hpftid(sc, tid)) {
idx = tid - sc->tids.hpftid_base;
f = &sc->tids.hpftid_tab[idx];
} else if (is_ftid(sc, tid)) {
idx = tid - sc->tids.ftid_base;
f = &sc->tids.ftid_tab[idx];
} else
panic("%s: FW reply for invalid TID %d.", __func__, tid);
MPASS(f->tid == tid);
rc = G_COOKIE(rpl->cookie);
mtx_lock(&sc->tids.ftid_lock);
KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
__func__, rc, idx));
__func__, rc, tid));
switch(rc) {
case FW_FILTER_WR_FLT_ADDED:
/* set-filter succeeded */
@ -936,7 +979,10 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
/* set-filter failed due to lack of SMT space. */
MPASS(f->valid == 0);
free_filter_resources(f);
sc->tids.ftids_in_use--;
if (separate_hpfilter_region(sc) && f->fs.prio)
sc->tids.hpftids_in_use--;
else
sc->tids.ftids_in_use--;
break;
case FW_FILTER_WR_SUCCESS:
case FW_FILTER_WR_EINVAL:
@ -1076,12 +1122,70 @@ t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
return (0);
}
static int
get_tcamfilter(struct adapter *sc, struct t4_filter *t)
{
int i, nfilters;
struct filter_entry *f;
u_int in_use;
#ifdef INVARIANTS
u_int tid_base;
#endif
MPASS(!t->fs.hash);
if (separate_hpfilter_region(sc) && t->fs.prio) {
nfilters = sc->tids.nhpftids;
f = sc->tids.hpftid_tab;
in_use = sc->tids.hpftids_in_use;
#ifdef INVARIANTS
tid_base = sc->tids.hpftid_base;
#endif
} else {
nfilters = sc->tids.nftids;
f = sc->tids.ftid_tab;
in_use = sc->tids.ftids_in_use;
#ifdef INVARIANTS
tid_base = sc->tids.ftid_base;
#endif
}
if (in_use == 0 || f == NULL || t->idx >= nfilters) {
t->idx = 0xffffffff;
return (0);
}
f += t->idx;
mtx_lock(&sc->tids.ftid_lock);
for (i = t->idx; i < nfilters; i++, f++) {
if (f->valid) {
MPASS(f->tid == tid_base + i);
t->idx = i;
t->l2tidx = f->l2te ? f->l2te->idx : 0;
t->smtidx = f->smt ? f->smt->idx : 0;
if (f->fs.hitcnts)
t->hits = get_filter_hits(sc, f->tid);
else
t->hits = UINT64_MAX;
t->fs = f->fs;
goto done;
}
}
t->idx = 0xffffffff;
done:
mtx_unlock(&sc->tids.ftid_lock);
return (0);
}
static int
get_hashfilter(struct adapter *sc, struct t4_filter *t)
{
int i, nfilters = sc->tids.ntids;
struct filter_entry *f;
MPASS(t->fs.hash);
if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL ||
t->idx >= nfilters) {
t->idx = 0xffffffff;

View File

@ -1404,6 +1404,7 @@ t4_detach_common(device_t dev)
free(sc->sge.iqmap, M_CXGBE);
free(sc->sge.eqmap, M_CXGBE);
free(sc->tids.ftid_tab, M_CXGBE);
free(sc->tids.hpftid_tab, M_CXGBE);
if (sc->tids.hftid_tab)
free_hftid_tab(&sc->tids);
free(sc->tids.atid_tab, M_CXGBE);
@ -3853,6 +3854,47 @@ get_params__post_init(struct adapter *sc)
__func__, sc->vres.l2t.size, L2T_SIZE));
sc->params.core_vdd = val[6];
if (chip_id(sc) >= CHELSIO_T6) {
#ifdef INVARIANTS
if (sc->params.fw_vers >=
(V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
V_FW_HDR_FW_VER_MICRO(1) | V_FW_HDR_FW_VER_BUILD(0))) {
/*
* Note that the code to enable the region should run
* before t4_fw_initialize and not here. This is just a
* reminder to add said code.
*/
device_printf(sc->dev,
"hpfilter region not enabled.\n");
}
#endif
sc->tids.tid_base = t4_read_reg(sc,
A_LE_DB_ACTIVE_TABLE_START_INDEX);
param[0] = FW_PARAM_PFVF(HPFILTER_START);
param[1] = FW_PARAM_PFVF(HPFILTER_END);
rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query hpfilter parameters: %d.\n", rc);
return (rc);
}
if ((int)val[1] > (int)val[0]) {
sc->tids.hpftid_base = val[0];
sc->tids.hpftid_end = val[1];
sc->tids.nhpftids = val[1] - val[0] + 1;
/*
* These should go off if the layout changes and the
* driver needs to catch up.
*/
MPASS(sc->tids.hpftid_base == 0);
MPASS(sc->tids.tid_base == sc->tids.nhpftids);
}
}
/*
* MPSBGMAP is queried separately because only recent firmwares support
* it as a parameter and we don't want the compound query above to fail
@ -3916,6 +3958,12 @@ get_params__post_init(struct adapter *sc)
return (rc);
}
sc->tids.ntids = val[0];
if (sc->params.fw_vers <
(V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) {
MPASS(sc->tids.ntids >= sc->tids.nhpftids);
sc->tids.ntids -= sc->tids.nhpftids;
}
sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
sc->params.hash_filter = 1;
}
@ -8127,6 +8175,11 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
t->atids_in_use);
}
if (t->nhpftids) {
sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
}
if (t->ntids) {
sbuf_printf(sb, "TID range: ");
if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
@ -8141,10 +8194,10 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
}
if (b)
sbuf_printf(sb, "0-%u, ", b - 1);
sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1);
sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
} else
sbuf_printf(sb, "0-%u", t->ntids - 1);
sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1);
sbuf_printf(sb, ", in use: %u\n",
atomic_load_acq_int(&t->tids_in_use));
}
@ -8155,8 +8208,8 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
}
if (t->nftids) {
sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
t->ftid_base + t->nftids - 1);
sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
t->ftid_end, t->ftids_in_use);
}
if (t->netids) {

View File

@ -367,7 +367,7 @@ set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
MPASS(m == NULL);
tid = GET_TID(cpl);
if (is_ftid(iq->adapter, tid)) {
if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) {
/*
* The return code for filter-write is put in the CPL cookie so
* we have to rely on the hardware tid (is_ftid) to determine

View File

@ -510,7 +510,10 @@ insert_tid(struct adapter *sc, int tid, void *ctx, int ntids)
{
struct tid_info *t = &sc->tids;
t->tid_tab[tid] = ctx;
MPASS(tid >= t->tid_base);
MPASS(tid - t->tid_base < t->ntids);
t->tid_tab[tid - t->tid_base] = ctx;
atomic_add_int(&t->tids_in_use, ntids);
}
@ -519,7 +522,7 @@ lookup_tid(struct adapter *sc, int tid)
{
struct tid_info *t = &sc->tids;
return (t->tid_tab[tid]);
return (t->tid_tab[tid - t->tid_base]);
}
void
@ -527,7 +530,7 @@ update_tid(struct adapter *sc, int tid, void *ctx)
{
struct tid_info *t = &sc->tids;
t->tid_tab[tid] = ctx;
t->tid_tab[tid - t->tid_base] = ctx;
}
void
@ -535,7 +538,7 @@ remove_tid(struct adapter *sc, int tid, int ntids)
{
struct tid_info *t = &sc->tids;
t->tid_tab[tid] = NULL;
t->tid_tab[tid - t->tid_base] = NULL;
atomic_subtract_int(&t->tids_in_use, ntids);
}