d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a switch structure for send tags.

Browse Source 
Move the type and function pointers for operations on existing send
tags (modify, query, next, free) out of 'struct ifnet' and into a new
'struct if_snd_tag_sw'.  A pointer to this structure is added to the
generic part of send tags and is initialized by m_snd_tag_init()
(which now accepts a switch structure as a new argument in place of
the type).

Previously, device driver ifnet methods switched on the type to call
type-specific functions.  Now, those type-specific functions are saved
in the switch structure and invoked directly.  In addition, this more
gracefully permits multiple implementations of the same tag within a
driver.  In particular, NIC TLS for future Chelsio adapters will use a
different implementation than the existing NIC TLS support for T6
adapters.

Reviewed by:	gallatin, hselasky, kib (older version)
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D31572
This commit is contained in:
John Baldwin 2021-09-14 11:43:41 -07:00
parent cf4670fe0b
commit c782ea8bb5
21 changed files with 263 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sys/dev/cxgbe/adapter.h
View File

@ -1292,7 +1292,6 @@ void t4_os_dump_devlog(struct adapter *);
/* t4_kern_tls.c */
int cxgbe_tls_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
void cxgbe_tls_tag_free(struct m_snd_tag *);
void t6_ktls_modload(void);
void t6_ktls_modunload(void);
int t6_ktls_try(struct ifnet *, struct socket *, struct ktls_session *);
@ -1409,9 +1408,6 @@ void t4_free_etid_table(struct adapter *);
struct cxgbe_rate_tag *lookup_etid(struct adapter *, int);
int cxgbe_rate_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
int cxgbe_rate_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *);
int cxgbe_rate_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *);
void cxgbe_rate_tag_free(struct m_snd_tag *);
void cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *);
void cxgbe_ratelimit_query(struct ifnet *, struct if_ratelimit_query_results *);
#endif

10
sys/dev/cxgbe/crypto/t4_kern_tls.c
View File

@ -102,9 +102,15 @@ struct tlspcb {
bool open_pending;
};
static void cxgbe_tls_tag_free(struct m_snd_tag *mst);
static int ktls_setup_keys(struct tlspcb *tlsp,
const struct ktls_session *tls, struct sge_txq *txq);
static const struct if_snd_tag_sw cxgbe_tls_tag_sw = {
.snd_tag_free = cxgbe_tls_tag_free,
.type = IF_SND_TAG_TYPE_TLS
};
static inline struct tlspcb *
mst_to_tls(struct m_snd_tag *t)
{
@ -122,7 +128,7 @@ alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
if (tlsp == NULL)
return (NULL);
m_snd_tag_init(&tlsp->com, ifp, IF_SND_TAG_TYPE_TLS);
m_snd_tag_init(&tlsp->com, ifp, &cxgbe_tls_tag_sw);
tlsp->vi = vi;
tlsp->sc = sc;
tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id];
@ -2071,7 +2077,7 @@ t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs,
return (totdesc);
}
void
static void
cxgbe_tls_tag_free(struct m_snd_tag *mst)
{
struct adapter *sc;

60
sys/dev/cxgbe/t4_main.c
View File

@ -253,11 +253,6 @@ static void cxgbe_qflush(struct ifnet *);
#if defined(KERN_TLS) || defined(RATELIMIT)
static int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
static int cxgbe_snd_tag_modify(struct m_snd_tag *,
union if_snd_tag_modify_params *);
static int cxgbe_snd_tag_query(struct m_snd_tag *,
union if_snd_tag_query_params *);
static void cxgbe_snd_tag_free(struct m_snd_tag *);
#endif
MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
@ -2453,9 +2448,6 @@ cxgbe_vi_attach(device_t dev, struct vi_info *vi)
ifp->if_get_counter = cxgbe_get_counter;
#if defined(KERN_TLS) || defined(RATELIMIT)
ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc;
ifp->if_snd_tag_modify = cxgbe_snd_tag_modify;
ifp->if_snd_tag_query = cxgbe_snd_tag_query;
ifp->if_snd_tag_free = cxgbe_snd_tag_free;
#endif
#ifdef RATELIMIT
ifp->if_ratelimit_query = cxgbe_ratelimit_query;
@ -2926,7 +2918,7 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
}
#ifdef RATELIMIT
if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) {
if (m->m_pkthdr.snd_tag->type == IF_SND_TAG_TYPE_RATE_LIMIT)
if (m->m_pkthdr.snd_tag->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT)
return (ethofld_transmit(ifp, m));
}
#endif
@ -3109,56 +3101,6 @@ cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
}
return (error);
}
static int
cxgbe_snd_tag_modify(struct m_snd_tag *mst,
union if_snd_tag_modify_params *params)
{
switch (mst->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
return (cxgbe_rate_tag_modify(mst, params));
#endif
default:
return (EOPNOTSUPP);
}
}
static int
cxgbe_snd_tag_query(struct m_snd_tag *mst,
union if_snd_tag_query_params *params)
{
switch (mst->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
return (cxgbe_rate_tag_query(mst, params));
#endif
default:
return (EOPNOTSUPP);
}
}
static void
cxgbe_snd_tag_free(struct m_snd_tag *mst)
{
switch (mst->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
cxgbe_rate_tag_free(mst);
return;
#endif
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
cxgbe_tls_tag_free(mst);
return;
#endif
default:
panic("shouldn't get here");
}
}
#endif
/*

22
sys/dev/cxgbe/t4_sched.c
View File

@ -44,7 +44,6 @@ __FBSDID("$FreeBSD$");
#include "common/t4_regs_values.h"
#include "common/t4_msg.h"
static int
in_range(int val, int lo, int hi)
{
@ -785,6 +784,19 @@ free_etid(struct adapter *sc, int etid)
mtx_unlock(&t->etid_lock);
}
static int cxgbe_rate_tag_modify(struct m_snd_tag *,
union if_snd_tag_modify_params *);
static int cxgbe_rate_tag_query(struct m_snd_tag *,
union if_snd_tag_query_params *);
static void cxgbe_rate_tag_free(struct m_snd_tag *);
static const struct if_snd_tag_sw cxgbe_rate_tag_sw = {
.snd_tag_modify = cxgbe_rate_tag_modify,
.snd_tag_query = cxgbe_rate_tag_query,
.snd_tag_free = cxgbe_rate_tag_free,
.type = IF_SND_TAG_TYPE_RATE_LIMIT
};
int
cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
struct m_snd_tag **pt)
@ -819,7 +831,7 @@ cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF);
mbufq_init(&cst->pending_tx, INT_MAX);
mbufq_init(&cst->pending_fwack, INT_MAX);
m_snd_tag_init(&cst->com, ifp, IF_SND_TAG_TYPE_RATE_LIMIT);
m_snd_tag_init(&cst->com, ifp, &cxgbe_rate_tag_sw);
cst->flags |= EO_FLOWC_PENDING | EO_SND_TAG_REF;
cst->adapter = sc;
cst->port_id = pi->port_id;
@ -843,7 +855,7 @@ cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
/*
* Change in parameters, no change in ifp.
*/
int
static int
cxgbe_rate_tag_modify(struct m_snd_tag *mst,
union if_snd_tag_modify_params *params)
{
@ -869,7 +881,7 @@ cxgbe_rate_tag_modify(struct m_snd_tag *mst,
return (0);
}
int
static int
cxgbe_rate_tag_query(struct m_snd_tag *mst,
union if_snd_tag_query_params *params)
{
@ -908,7 +920,7 @@ cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *cst)
free(cst, M_CXGBE);
}
void
static void
cxgbe_rate_tag_free(struct m_snd_tag *mst)
{
struct cxgbe_rate_tag *cst = mst_to_crt(mst);

4
sys/dev/cxgbe/t4_sge.c
View File

@ -2369,7 +2369,7 @@ static inline int
needs_eo(struct m_snd_tag *mst)
{
return (mst != NULL && mst->type == IF_SND_TAG_TYPE_RATE_LIMIT);
return (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT);
}
#endif
@ -2712,7 +2712,7 @@ parse_pkt(struct mbuf **mp, bool vm_wr)
mst = NULL;
#endif
#ifdef KERN_TLS
if (mst != NULL && mst->type == IF_SND_TAG_TYPE_TLS) {
if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) {
int len16;
cflags |= MC_TLS;

5
sys/dev/mlx5/mlx5_en/en.h
View File

@ -1216,9 +1216,4 @@ int mlx5e_update_buf_lossy(struct mlx5e_priv *priv);
int mlx5e_fec_update(struct mlx5e_priv *priv);
int mlx5e_hw_temperature_update(struct mlx5e_priv *priv);
if_snd_tag_alloc_t mlx5e_ul_snd_tag_alloc;
if_snd_tag_modify_t mlx5e_ul_snd_tag_modify;
if_snd_tag_query_t mlx5e_ul_snd_tag_query;
if_snd_tag_free_t mlx5e_ul_snd_tag_free;
#endif /* _MLX5_EN_H_ */

3
sys/dev/mlx5/mlx5_en/en_hw_tls.h
View File

@ -97,8 +97,5 @@ void mlx5e_tls_cleanup(struct mlx5e_priv *);
int mlx5e_sq_tls_xmit(struct mlx5e_sq *, struct mlx5e_xmit_args *, struct mbuf **);
if_snd_tag_alloc_t mlx5e_tls_snd_tag_alloc;
if_snd_tag_modify_t mlx5e_tls_snd_tag_modify;
if_snd_tag_query_t mlx5e_tls_snd_tag_query;
if_snd_tag_free_t mlx5e_tls_snd_tag_free;
#endif /* _MLX5_TLS_H_ */

3
sys/dev/mlx5/mlx5_en/en_rl.h
View File

@ -168,8 +168,5 @@ void mlx5e_rl_cleanup(struct mlx5e_priv *priv);
void mlx5e_rl_refresh_sq_inline(struct mlx5e_rl_priv_data *rl);
if_snd_tag_alloc_t mlx5e_rl_snd_tag_alloc;
if_snd_tag_modify_t mlx5e_rl_snd_tag_modify;
if_snd_tag_query_t mlx5e_rl_snd_tag_query;
if_snd_tag_free_t mlx5e_rl_snd_tag_free;
#endif /* __MLX5_EN_RL_H__ */

73
sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
View File

@ -37,6 +37,27 @@
#ifdef KERN_TLS
#ifdef RATELIMIT
static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify;
#endif
static if_snd_tag_query_t mlx5e_tls_snd_tag_query;
static if_snd_tag_free_t mlx5e_tls_snd_tag_free;
static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = {
.snd_tag_query = mlx5e_tls_snd_tag_query,
.snd_tag_free = mlx5e_tls_snd_tag_free,
.type = IF_SND_TAG_TYPE_TLS
};
#ifdef RATELIMIT
static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = {
.snd_tag_modify = mlx5e_tls_rl_snd_tag_modify,
.snd_tag_query = mlx5e_tls_snd_tag_query,
.snd_tag_free = mlx5e_tls_snd_tag_free,
.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT
};
#endif
MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
/* software TLS context */
@ -281,6 +302,7 @@ mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
struct m_snd_tag **ppmt)
{
union if_snd_tag_alloc_params rl_params;
const struct if_snd_tag_sw *snd_tag_sw;
struct mlx5e_priv *priv;
struct mlx5e_tls_tag *ptag;
const struct tls_session_params *en;
@ -384,10 +406,12 @@ mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw;
break;
#endif
case IF_SND_TAG_TYPE_TLS:
rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
snd_tag_sw = &mlx5e_tls_snd_tag_sw;
break;
default:
error = EOPNOTSUPP;
@ -400,7 +424,7 @@ mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
/* store pointer to mbuf tag */
MPASS(ptag->tag.refcount == 0);
m_snd_tag_init(&ptag->tag, ifp, params->hdr.type);
m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw);
*ppmt = &ptag->tag;
queue_work(priv->tls.wq, &ptag->work);
@ -413,53 +437,32 @@ mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
return (error);
}
int
mlx5e_tls_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
{
#ifdef RATELIMIT
static int
mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
{
union if_snd_tag_modify_params rl_params;
struct mlx5e_tls_tag *ptag =
container_of(pmt, struct mlx5e_tls_tag, tag);
int error;
#endif
switch (pmt->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
memset(&rl_params, 0, sizeof(rl_params));
rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
error = ptag->rl_tag->ifp->if_snd_tag_modify(ptag->rl_tag,
&rl_params);
return (error);
#endif
default:
return (EOPNOTSUPP);
}
memset(&rl_params, 0, sizeof(rl_params));
rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params);
return (error);
}
#endif
int
static int
mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
{
struct mlx5e_tls_tag *ptag =
container_of(pmt, struct mlx5e_tls_tag, tag);
int error;
switch (pmt->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
#endif
case IF_SND_TAG_TYPE_TLS:
error = ptag->rl_tag->ifp->if_snd_tag_query(ptag->rl_tag,
params);
break;
default:
error = EOPNOTSUPP;
break;
}
return (error);
return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params));
}
void
static void
mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
{
struct mlx5e_tls_tag *ptag =
@ -690,9 +693,9 @@ mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf
if (
#ifdef RATELIMIT
ptag->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
#endif
ptag->type != IF_SND_TAG_TYPE_TLS)
ptag->sw->type != IF_SND_TAG_TYPE_TLS)
return (MLX5E_TLS_CONTINUE);
ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);

93
sys/dev/mlx5/mlx5_en/mlx5_en_main.c
View File

@ -36,6 +36,8 @@
#include <net/debugnet.h>
static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
static if_snd_tag_free_t mlx5e_ul_snd_tag_free;
struct mlx5e_channel_param {
struct mlx5e_rq_param rq;
@ -346,6 +348,12 @@ static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX
},
};
static const struct if_snd_tag_sw mlx5e_ul_snd_tag_sw = {
.snd_tag_query = mlx5e_ul_snd_tag_query,
.snd_tag_free = mlx5e_ul_snd_tag_free,
.type = IF_SND_TAG_TYPE_UNLIMITED
};
DEBUGNET_DEFINE(mlx5_en);
MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
@ -2113,7 +2121,7 @@ mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
c->ix = ix;
/* setup send tag */
m_snd_tag_init(&c->tag, c->priv->ifp, IF_SND_TAG_TYPE_UNLIMITED);
m_snd_tag_init(&c->tag, c->priv->ifp, &mlx5e_ul_snd_tag_sw);
init_completion(&c->completion);
@ -4174,7 +4182,7 @@ mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
PRIV_UNLOCK(priv);
}
int
static int
mlx5e_ul_snd_tag_alloc(struct ifnet *ifp,
union if_snd_tag_alloc_params *params,
struct m_snd_tag **ppmt)
@ -4216,7 +4224,7 @@ mlx5e_ul_snd_tag_alloc(struct ifnet *ifp,
}
}
int
static int
mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
{
struct mlx5e_channel *pch =
@ -4227,7 +4235,7 @@ mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *par
return (0);
}
void
static void
mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
{
struct mlx5e_channel *pch =
@ -4262,52 +4270,6 @@ mlx5e_snd_tag_alloc(struct ifnet *ifp,
}
}
static int
mlx5e_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
{
switch (pmt->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
return (mlx5e_rl_snd_tag_modify(pmt, params));
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
return (mlx5e_tls_snd_tag_modify(pmt, params));
#endif
#endif
case IF_SND_TAG_TYPE_UNLIMITED:
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
#endif
default:
return (EOPNOTSUPP);
}
}
static int
mlx5e_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
{
switch (pmt->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
return (mlx5e_rl_snd_tag_query(pmt, params));
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
return (mlx5e_tls_snd_tag_query(pmt, params));
#endif
#endif
case IF_SND_TAG_TYPE_UNLIMITED:
return (mlx5e_ul_snd_tag_query(pmt, params));
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
return (mlx5e_tls_snd_tag_query(pmt, params));
#endif
default:
return (EOPNOTSUPP);
}
}
#ifdef RATELIMIT
#define NUM_HDWR_RATES_MLX 13
static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
@ -4352,34 +4314,6 @@ mlx5e_ratelimit_query(struct ifnet *ifp __unused, struct if_ratelimit_query_resu
}
#endif
static void
mlx5e_snd_tag_free(struct m_snd_tag *pmt)
{
switch (pmt->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
mlx5e_rl_snd_tag_free(pmt);
break;
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
mlx5e_tls_snd_tag_free(pmt);
break;
#endif
#endif
case IF_SND_TAG_TYPE_UNLIMITED:
mlx5e_ul_snd_tag_free(pmt);
break;
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
mlx5e_tls_snd_tag_free(pmt);
break;
#endif
default:
break;
}
}
static void
mlx5e_ifm_add(struct mlx5e_priv *priv, int type)
{
@ -4467,9 +4401,6 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
#endif
ifp->if_capabilities |= IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO;
ifp->if_snd_tag_alloc = mlx5e_snd_tag_alloc;
ifp->if_snd_tag_free = mlx5e_snd_tag_free;
ifp->if_snd_tag_modify = mlx5e_snd_tag_modify;
ifp->if_snd_tag_query = mlx5e_snd_tag_query;
#ifdef RATELIMIT
ifp->if_ratelimit_query = mlx5e_ratelimit_query;
#endif

19
sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
View File

@ -38,6 +38,16 @@ static void mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data *rl, uns
struct sysctl_oid *node, const char *name, const char *desc);
static int mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *, uint64_t value);
static int mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data *, uint64_t value);
static if_snd_tag_modify_t mlx5e_rl_snd_tag_modify;
static if_snd_tag_query_t mlx5e_rl_snd_tag_query;
static if_snd_tag_free_t mlx5e_rl_snd_tag_free;
static const struct if_snd_tag_sw mlx5e_rl_snd_tag_sw = {
.snd_tag_modify = mlx5e_rl_snd_tag_modify,
.snd_tag_query = mlx5e_rl_snd_tag_query,
.snd_tag_free = mlx5e_rl_snd_tag_free,
.type = IF_SND_TAG_TYPE_RATE_LIMIT
};
static void
mlx5e_rl_build_sq_param(struct mlx5e_rl_priv_data *rl,
@ -830,7 +840,6 @@ mlx5e_rl_init(struct mlx5e_priv *priv)
for (i = 0; i < rl->param.tx_channels_per_worker_def; i++) {
struct mlx5e_rl_channel *channel = rlw->channels + i;
channel->worker = rlw;
channel->tag.type = IF_SND_TAG_TYPE_RATE_LIMIT;
STAILQ_INSERT_TAIL(&rlw->index_list_head, channel, entry);
}
MLX5E_RL_WORKER_UNLOCK(rlw);
@ -1110,14 +1119,14 @@ mlx5e_rl_snd_tag_alloc(struct ifnet *ifp,
/* store pointer to mbuf tag */
MPASS(channel->tag.refcount == 0);
m_snd_tag_init(&channel->tag, ifp, IF_SND_TAG_TYPE_RATE_LIMIT);
m_snd_tag_init(&channel->tag, ifp, &mlx5e_rl_snd_tag_sw);
*ppmt = &channel->tag;
done:
return (error);
}
int
static int
mlx5e_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
{
struct mlx5e_rl_channel *channel =
@ -1126,7 +1135,7 @@ mlx5e_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *p
return (mlx5e_rl_modify(channel->worker, channel, params->rate_limit.max_rate));
}
int
static int
mlx5e_rl_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
{
struct mlx5e_rl_channel *channel =
@ -1135,7 +1144,7 @@ mlx5e_rl_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *par
return (mlx5e_rl_query(channel->worker, channel, params));
}
void
static void
mlx5e_rl_snd_tag_free(struct m_snd_tag *pmt)
{
struct mlx5e_rl_channel *channel =

2
sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
View File

@ -98,7 +98,7 @@ mlx5e_select_queue_by_send_tag(struct ifnet *ifp, struct mbuf *mb)
top:
#endif
/* get pointer to sendqueue */
switch (mb_tag->type) {
switch (mb_tag->sw->type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_RATE_LIMIT:
sq = container_of(mb_tag,

7
sys/kern/kern_mbuf.c
View File

@ -1585,13 +1585,14 @@ m_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
}
void
m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp, u_int type)
m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp,
const struct if_snd_tag_sw *sw)
{
if_ref(ifp);
mst->ifp = ifp;
refcount_init(&mst->refcount, 1);
mst->type = type;
mst->sw = sw;
counter_u64_add(snd_tag_count, 1);
}
@ -1601,7 +1602,7 @@ m_snd_tag_destroy(struct m_snd_tag *mst)
struct ifnet *ifp;
ifp = mst->ifp;
ifp->if_snd_tag_free(mst);
mst->sw->snd_tag_free(mst);
if_rele(ifp);
counter_u64_add(snd_tag_count, -1);
}

6
sys/kern/uipc_ktls.c
View File

@ -1459,7 +1459,6 @@ ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate)
.rate_limit.flags = M_NOWAIT,
};
struct m_snd_tag *mst;
struct ifnet *ifp;
/* Can't get to the inp, but it should be locked. */
/* INP_LOCK_ASSERT(inp); */
@ -1477,11 +1476,10 @@ ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate)
}
MPASS(tls->snd_tag != NULL);
MPASS(tls->snd_tag->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT);
MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT);
mst = tls->snd_tag;
ifp = mst->ifp;
return (ifp->if_snd_tag_modify(mst, &params));
return (mst->sw->snd_tag_modify(mst, &params));
}
#endif
#endif

20
sys/net/if_dead.c
View File

@ -109,23 +109,6 @@ ifdead_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
return (EOPNOTSUPP);
}
static int
ifdead_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
{
return (EOPNOTSUPP);
}
static int
ifdead_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
{
return (EOPNOTSUPP);
}
static void
ifdead_snd_tag_free(struct m_snd_tag *pmt)
{
}
static void
ifdead_ratelimit_query(struct ifnet *ifp __unused,
struct if_ratelimit_query_results *q)
@ -156,8 +139,5 @@ if_dead(struct ifnet *ifp)
ifp->if_transmit = ifdead_transmit;
ifp->if_get_counter = ifdead_get_counter;
ifp->if_snd_tag_alloc = ifdead_snd_tag_alloc;
ifp->if_snd_tag_modify = ifdead_snd_tag_modify;
ifp->if_snd_tag_query = ifdead_snd_tag_query;
ifp->if_snd_tag_free = ifdead_snd_tag_free;
ifp->if_ratelimit_query = ifdead_ratelimit_query;
}

72
sys/net/if_lagg.c
View File

@ -583,10 +583,6 @@ lagg_clone_create(struct if_clone *ifc, int unit, caddr_t params)
ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
#if defined(KERN_TLS) || defined(RATELIMIT)
ifp->if_snd_tag_alloc = lagg_snd_tag_alloc;
ifp->if_snd_tag_modify = lagg_snd_tag_modify;
ifp->if_snd_tag_query = lagg_snd_tag_query;
ifp->if_snd_tag_free = lagg_snd_tag_free;
ifp->if_next_snd_tag = lagg_next_snd_tag;
ifp->if_ratelimit_query = lagg_ratelimit_query;
#endif
ifp->if_capenable = ifp->if_capabilities = IFCAP_HWSTATS;
@ -1741,6 +1737,44 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
}
#if defined(KERN_TLS) || defined(RATELIMIT)
#ifdef RATELIMIT
static const struct if_snd_tag_sw lagg_snd_tag_ul_sw = {
.snd_tag_modify = lagg_snd_tag_modify,
.snd_tag_query = lagg_snd_tag_query,
.snd_tag_free = lagg_snd_tag_free,
.next_snd_tag = lagg_next_snd_tag,
.type = IF_SND_TAG_TYPE_UNLIMITED
};
static const struct if_snd_tag_sw lagg_snd_tag_rl_sw = {
.snd_tag_modify = lagg_snd_tag_modify,
.snd_tag_query = lagg_snd_tag_query,
.snd_tag_free = lagg_snd_tag_free,
.next_snd_tag = lagg_next_snd_tag,
.type = IF_SND_TAG_TYPE_RATE_LIMIT
};
#endif
#ifdef KERN_TLS
static const struct if_snd_tag_sw lagg_snd_tag_tls_sw = {
.snd_tag_modify = lagg_snd_tag_modify,
.snd_tag_query = lagg_snd_tag_query,
.snd_tag_free = lagg_snd_tag_free,
.next_snd_tag = lagg_next_snd_tag,
.type = IF_SND_TAG_TYPE_TLS
};
#ifdef RATELIMIT
static const struct if_snd_tag_sw lagg_snd_tag_tls_rl_sw = {
.snd_tag_modify = lagg_snd_tag_modify,
.snd_tag_query = lagg_snd_tag_query,
.snd_tag_free = lagg_snd_tag_free,
.next_snd_tag = lagg_next_snd_tag,
.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT
};
#endif
#endif
static inline struct lagg_snd_tag *
mst_to_lst(struct m_snd_tag *mst)
{
@ -1796,6 +1830,7 @@ lagg_snd_tag_alloc(struct ifnet *ifp,
struct m_snd_tag **ppmt)
{
struct epoch_tracker et;
const struct if_snd_tag_sw *sw;
struct lagg_snd_tag *lst;
struct lagg_softc *sc;
struct lagg_port *lp;
@ -1804,6 +1839,29 @@ lagg_snd_tag_alloc(struct ifnet *ifp,
sc = ifp->if_softc;
switch (params->hdr.type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_UNLIMITED:
sw = &lagg_snd_tag_ul_sw;
break;
case IF_SND_TAG_TYPE_RATE_LIMIT:
sw = &lagg_snd_tag_rl_sw;
break;
#endif
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
sw = &lagg_snd_tag_tls_sw;
break;
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
sw = &lagg_snd_tag_tls_rl_sw;
break;
#endif
#endif
default:
return (EOPNOTSUPP);
}
NET_EPOCH_ENTER(et);
lp = lookup_snd_tag_port(ifp, params->hdr.flowid,
params->hdr.flowtype, params->hdr.numa_domain);
@ -1832,7 +1890,7 @@ lagg_snd_tag_alloc(struct ifnet *ifp,
return (error);
}
m_snd_tag_init(&lst->com, ifp, lst->tag->type);
m_snd_tag_init(&lst->com, ifp, sw);
*ppmt = &lst->com;
return (0);
@ -1854,7 +1912,7 @@ lagg_snd_tag_modify(struct m_snd_tag *mst,
struct lagg_snd_tag *lst;
lst = mst_to_lst(mst);
return (lst->tag->ifp->if_snd_tag_modify(lst->tag, params));
return (lst->tag->sw->snd_tag_modify(lst->tag, params));
}
static int
@ -1864,7 +1922,7 @@ lagg_snd_tag_query(struct m_snd_tag *mst,
struct lagg_snd_tag *lst;
lst = mst_to_lst(mst);
return (lst->tag->ifp->if_snd_tag_query(lst->tag, params));
return (lst->tag->sw->snd_tag_query(lst->tag, params));
}
static void

27
sys/net/if_var.h
View File

@ -249,6 +249,21 @@ union if_snd_tag_query_params {
struct if_snd_tag_rate_limit_params tls_rate_limit;
};
typedef int (if_snd_tag_alloc_t)(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
typedef int (if_snd_tag_modify_t)(struct m_snd_tag *, union if_snd_tag_modify_params *);
typedef int (if_snd_tag_query_t)(struct m_snd_tag *, union if_snd_tag_query_params *);
typedef void (if_snd_tag_free_t)(struct m_snd_tag *);
typedef struct m_snd_tag *(if_next_send_tag_t)(struct m_snd_tag *);
struct if_snd_tag_sw {
if_snd_tag_modify_t *snd_tag_modify;
if_snd_tag_query_t *snd_tag_query;
if_snd_tag_free_t *snd_tag_free;
if_next_send_tag_t *next_snd_tag;
u_int type; /* One of IF_SND_TAG_TYPE_*. */
};
/* Query return flags */
#define RT_NOSUPPORT 0x00000000 /* Not supported */
#define RT_IS_INDIRECT 0x00000001 /*
@ -273,12 +288,6 @@ struct if_ratelimit_query_results {
uint32_t min_segment_burst; /* The amount the adapter bursts at each send */
};
typedef int (if_snd_tag_alloc_t)(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
typedef int (if_snd_tag_modify_t)(struct m_snd_tag *, union if_snd_tag_modify_params *);
typedef int (if_snd_tag_query_t)(struct m_snd_tag *, union if_snd_tag_query_params *);
typedef void (if_snd_tag_free_t)(struct m_snd_tag *);
typedef struct m_snd_tag *(if_next_send_tag_t)(struct m_snd_tag *);
typedef void (if_ratelimit_query_t)(struct ifnet *,
struct if_ratelimit_query_results *);
typedef int (if_ratelimit_setup_t)(struct ifnet *, uint64_t, uint32_t);
@ -420,10 +429,8 @@ struct ifnet {
* Network adapter send tag support:
*/
if_snd_tag_alloc_t *if_snd_tag_alloc;
if_snd_tag_modify_t *if_snd_tag_modify;
if_snd_tag_query_t *if_snd_tag_query;
if_snd_tag_free_t *if_snd_tag_free;
if_next_send_tag_t *if_next_snd_tag;
/* Ratelimit (packet pacing) */
if_ratelimit_query_t *if_ratelimit_query;
if_ratelimit_setup_t *if_ratelimit_setup;

72
sys/net/if_vlan.c
View File

@ -336,6 +336,44 @@ VNET_DEFINE_STATIC(struct if_clone *, vlan_cloner);
#define V_vlan_cloner VNET(vlan_cloner)
#endif
#ifdef RATELIMIT
static const struct if_snd_tag_sw vlan_snd_tag_ul_sw = {
.snd_tag_modify = vlan_snd_tag_modify,
.snd_tag_query = vlan_snd_tag_query,
.snd_tag_free = vlan_snd_tag_free,
.next_snd_tag = vlan_next_snd_tag,
.type = IF_SND_TAG_TYPE_UNLIMITED
};
static const struct if_snd_tag_sw vlan_snd_tag_rl_sw = {
.snd_tag_modify = vlan_snd_tag_modify,
.snd_tag_query = vlan_snd_tag_query,
.snd_tag_free = vlan_snd_tag_free,
.next_snd_tag = vlan_next_snd_tag,
.type = IF_SND_TAG_TYPE_RATE_LIMIT
};
#endif
#ifdef KERN_TLS
static const struct if_snd_tag_sw vlan_snd_tag_tls_sw = {
.snd_tag_modify = vlan_snd_tag_modify,
.snd_tag_query = vlan_snd_tag_query,
.snd_tag_free = vlan_snd_tag_free,
.next_snd_tag = vlan_next_snd_tag,
.type = IF_SND_TAG_TYPE_TLS
};
#ifdef RATELIMIT
static const struct if_snd_tag_sw vlan_snd_tag_tls_rl_sw = {
.snd_tag_modify = vlan_snd_tag_modify,
.snd_tag_query = vlan_snd_tag_query,
.snd_tag_free = vlan_snd_tag_free,
.next_snd_tag = vlan_next_snd_tag,
.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT
};
#endif
#endif
static void
vlan_mc_free(struct epoch_context *ctx)
{
@ -1114,10 +1152,6 @@ vlan_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params)
ifp->if_ioctl = vlan_ioctl;
#if defined(KERN_TLS) || defined(RATELIMIT)
ifp->if_snd_tag_alloc = vlan_snd_tag_alloc;
ifp->if_snd_tag_modify = vlan_snd_tag_modify;
ifp->if_snd_tag_query = vlan_snd_tag_query;
ifp->if_snd_tag_free = vlan_snd_tag_free;
ifp->if_next_snd_tag = vlan_next_snd_tag;
ifp->if_ratelimit_query = vlan_ratelimit_query;
#endif
ifp->if_flags = VLAN_IFFLAGS;
@ -2126,11 +2160,35 @@ vlan_snd_tag_alloc(struct ifnet *ifp,
struct m_snd_tag **ppmt)
{
struct epoch_tracker et;
const struct if_snd_tag_sw *sw;
struct vlan_snd_tag *vst;
struct ifvlan *ifv;
struct ifnet *parent;
int error;
switch (params->hdr.type) {
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_UNLIMITED:
sw = &vlan_snd_tag_ul_sw;
break;
case IF_SND_TAG_TYPE_RATE_LIMIT:
sw = &vlan_snd_tag_rl_sw;
break;
#endif
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
sw = &vlan_snd_tag_tls_sw;
break;
#ifdef RATELIMIT
case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
sw = &vlan_snd_tag_tls_rl_sw;
break;
#endif
#endif
default:
return (EOPNOTSUPP);
}
NET_EPOCH_ENTER(et);
ifv = ifp->if_softc;
if (ifv->ifv_trunk != NULL)
@ -2157,7 +2215,7 @@ vlan_snd_tag_alloc(struct ifnet *ifp,
return (error);
}
m_snd_tag_init(&vst->com, ifp, vst->tag->type);
m_snd_tag_init(&vst->com, ifp, sw);
*ppmt = &vst->com;
return (0);
@ -2179,7 +2237,7 @@ vlan_snd_tag_modify(struct m_snd_tag *mst,
struct vlan_snd_tag *vst;
vst = mst_to_vst(mst);
return (vst->tag->ifp->if_snd_tag_modify(vst->tag, params));
return (vst->tag->sw->snd_tag_modify(vst->tag, params));
}
static int
@ -2189,7 +2247,7 @@ vlan_snd_tag_query(struct m_snd_tag *mst,
struct vlan_snd_tag *vst;
vst = mst_to_vst(mst);
return (vst->tag->ifp->if_snd_tag_query(vst->tag, params));
return (vst->tag->sw->snd_tag_query(vst->tag, params));
}
static void

31
sys/netinet/in_pcb.c
View File

@ -3293,21 +3293,16 @@ in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
.rate_limit.flags = M_NOWAIT,
};
struct m_snd_tag *mst;
struct ifnet *ifp;
int error;
mst = inp->inp_snd_tag;
if (mst == NULL)
return (EINVAL);
ifp = mst->ifp;
if (ifp == NULL)
return (EINVAL);
if (ifp->if_snd_tag_modify == NULL) {
if (mst->sw->snd_tag_modify == NULL) {
error = EOPNOTSUPP;
} else {
error = ifp->if_snd_tag_modify(mst, &params);
error = mst->sw->snd_tag_modify(mst, &params);
}
return (error);
}
@ -3321,22 +3316,17 @@ in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
{
union if_snd_tag_query_params params = { };
struct m_snd_tag *mst;
struct ifnet *ifp;
int error;
mst = inp->inp_snd_tag;
if (mst == NULL)
return (EINVAL);
ifp = mst->ifp;
if (ifp == NULL)
return (EINVAL);
if (ifp->if_snd_tag_query == NULL) {
if (mst->sw->snd_tag_query == NULL) {
error = EOPNOTSUPP;
} else {
error = ifp->if_snd_tag_query(mst, &params);
if (error == 0 && p_max_pacing_rate != NULL)
error = mst->sw->snd_tag_query(mst, &params);
if (error == 0 && p_max_pacing_rate != NULL)
*p_max_pacing_rate = params.rate_limit.max_rate;
}
return (error);
@ -3351,22 +3341,17 @@ in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
{
union if_snd_tag_query_params params = { };
struct m_snd_tag *mst;
struct ifnet *ifp;
int error;
mst = inp->inp_snd_tag;
if (mst == NULL)
return (EINVAL);
ifp = mst->ifp;
if (ifp == NULL)
return (EINVAL);
if (ifp->if_snd_tag_query == NULL)
if (mst->sw->snd_tag_query == NULL)
return (EOPNOTSUPP);
error = ifp->if_snd_tag_query(mst, &params);
if (error == 0 && p_txqueue_level != NULL)
error = mst->sw->snd_tag_query(mst, &params);
if (error == 0 && p_txqueue_level != NULL)
*p_txqueue_level = params.rate_limit.queue_level;
return (error);
}

8
sys/netinet/tcp_ratelimit.c
View File

@ -1065,8 +1065,8 @@ rt_find_real_interface(struct ifnet *ifp, struct inpcb *inp, int *error)
return (NULL);
}
ntag = tag;
while(ntag->ifp->if_next_snd_tag != NULL) {
ntag = ntag->ifp->if_next_snd_tag(ntag);
while (ntag->sw->next_snd_tag != NULL) {
ntag = ntag->sw->next_snd_tag(ntag);
}
tifp = ntag->ifp;
m_snd_tag_rele(tag);
@ -1360,7 +1360,7 @@ tcp_set_pacing_rate(struct tcpcb *tp, struct ifnet *ifp,
* send tag. This will convert the existing
* tag to a TLS ratelimit tag.
*/
MPASS(tls->snd_tag->type == IF_SND_TAG_TYPE_TLS);
MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS);
ktls_output_eagain(tp->t_inpcb, tls);
}
#endif
@ -1405,7 +1405,7 @@ tcp_chg_pacing_rate(const struct tcp_hwrate_limit_table *crte,
tls = tp->t_inpcb->inp_socket->so_snd.sb_tls_info;
MPASS(tls->mode == TCP_TLS_MODE_IFNET);
if (tls->snd_tag != NULL &&
tls->snd_tag->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT) {
tls->snd_tag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT) {
/*
* NIC probably doesn't support ratelimit TLS
* tags if it didn't allocate one when an

7
sys/sys/mbuf.h
View File

@ -140,10 +140,12 @@ struct m_tag {
* Static network interface owned tag.
* Allocated through ifp->if_snd_tag_alloc().
*/
struct if_snd_tag_sw;
struct m_snd_tag {
struct ifnet *ifp; /* network interface tag belongs to */
const struct if_snd_tag_sw *sw;
volatile u_int refcount;
u_int type; /* One of IF_SND_TAG_TYPE_*. */
};
/*
@ -849,7 +851,8 @@ int m_unmapped_uiomove(const struct mbuf *, int, struct uio *,
struct mbuf *m_unshare(struct mbuf *, int);
int m_snd_tag_alloc(struct ifnet *,
union if_snd_tag_alloc_params *, struct m_snd_tag **);
void m_snd_tag_init(struct m_snd_tag *, struct ifnet *, u_int);
void m_snd_tag_init(struct m_snd_tag *, struct ifnet *,
const struct if_snd_tag_sw *);
void m_snd_tag_destroy(struct m_snd_tag *);
static __inline int