Add a TOE KTLS mode and a TOE hook for allocating TLS sessions.
This adds the glue to allocate TLS sessions and invokes it from the TLS enable socket option handler. This also adds some counters for active TOE sessions. The TOE KTLS mode is returned by getsockopt(TLSTX_TLS_MODE) when TOE KTLS is in use on a socket, but cannot be set via setsockopt(). To simplify various checks, a TLS session now includes an explicit 'mode' member set to the value returned by TLSTX_TLS_MODE. Various places that used to check 'sw_encrypt' against NULL to determine software vs ifnet (NIC) TLS now check 'mode' instead. Reviewed by: np, gallatin Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D21891
This commit is contained in:
parent
fa43c5d49e
commit
9e14430d46
@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/vnode.h>
|
||||
|
||||
#include <net/vnet.h>
|
||||
#include <netinet/tcp.h>
|
||||
|
||||
#include <security/audit/audit.h>
|
||||
#include <security/mac/mac_framework.h>
|
||||
@ -295,7 +296,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, int error)
|
||||
|
||||
mb_free_notready(sfio->m, sfio->npages);
|
||||
#ifdef KERN_TLS
|
||||
} else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) {
|
||||
} else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
|
||||
/*
|
||||
* I/O operation is complete, but we still need to
|
||||
* encrypt. We cannot do this in the interrupt thread
|
||||
@ -1028,7 +1029,7 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
|
||||
*/
|
||||
free(sfio, M_TEMP);
|
||||
#ifdef KERN_TLS
|
||||
if (tls != NULL && tls->sw_encrypt != NULL) {
|
||||
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
|
||||
error = (*so->so_proto->pr_usrreqs->pru_send)
|
||||
(so, PRUS_NOTREADY, m, NULL, NULL, td);
|
||||
soref(so);
|
||||
|
@ -63,6 +63,9 @@ __FBSDID("$FreeBSD$");
|
||||
#include <netinet/in_pcb.h>
|
||||
#endif
|
||||
#include <netinet/tcp_var.h>
|
||||
#ifdef TCP_OFFLOAD
|
||||
#include <netinet/tcp_offload.h>
|
||||
#endif
|
||||
#include <opencrypto/xform.h>
|
||||
#include <vm/uma_dbg.h>
|
||||
#include <vm/vm.h>
|
||||
@ -161,6 +164,10 @@ SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD, 0,
|
||||
"Software TLS session stats");
|
||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0,
|
||||
"Hardware (ifnet) TLS session stats");
|
||||
#ifdef TCP_OFFLOAD
|
||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0,
|
||||
"TOE TLS session stats");
|
||||
#endif
|
||||
|
||||
static counter_u64_t ktls_sw_cbc;
|
||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc,
|
||||
@ -199,6 +206,18 @@ SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN,
|
||||
&ktls_ifnet_permitted, 1,
|
||||
"Whether to permit hardware (ifnet) TLS sessions");
|
||||
|
||||
#ifdef TCP_OFFLOAD
|
||||
static counter_u64_t ktls_toe_cbc;
|
||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD,
|
||||
&ktls_toe_cbc,
|
||||
"Active number of TOE TLS sessions using AES-CBC");
|
||||
|
||||
static counter_u64_t ktls_toe_gcm;
|
||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD,
|
||||
&ktls_toe_gcm,
|
||||
"Active number of TOE TLS sessions using AES-GCM");
|
||||
#endif
|
||||
|
||||
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS");
|
||||
|
||||
static void ktls_cleanup(struct ktls_session *tls);
|
||||
@ -325,6 +344,10 @@ ktls_init(void *dummy __unused)
|
||||
ktls_ifnet_reset = counter_u64_alloc(M_WAITOK);
|
||||
ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK);
|
||||
ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK);
|
||||
#ifdef TCP_OFFLOAD
|
||||
ktls_toe_cbc = counter_u64_alloc(M_WAITOK);
|
||||
ktls_toe_gcm = counter_u64_alloc(M_WAITOK);
|
||||
#endif
|
||||
|
||||
rm_init(&ktls_backends_lock, "ktls backends");
|
||||
LIST_INIT(&ktls_backends);
|
||||
@ -607,7 +630,8 @@ ktls_cleanup(struct ktls_session *tls)
|
||||
{
|
||||
|
||||
counter_u64_add(ktls_offload_active, -1);
|
||||
if (tls->free != NULL) {
|
||||
switch (tls->mode) {
|
||||
case TCP_TLS_MODE_SW:
|
||||
MPASS(tls->be != NULL);
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
@ -618,7 +642,8 @@ ktls_cleanup(struct ktls_session *tls)
|
||||
break;
|
||||
}
|
||||
tls->free(tls);
|
||||
} else if (tls->snd_tag != NULL) {
|
||||
break;
|
||||
case TCP_TLS_MODE_IFNET:
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
counter_u64_add(ktls_ifnet_cbc, -1);
|
||||
@ -628,6 +653,19 @@ ktls_cleanup(struct ktls_session *tls)
|
||||
break;
|
||||
}
|
||||
m_snd_tag_rele(tls->snd_tag);
|
||||
break;
|
||||
#ifdef TCP_OFFLOAD
|
||||
case TCP_TLS_MODE_TOE:
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
counter_u64_add(ktls_toe_cbc, -1);
|
||||
break;
|
||||
case CRYPTO_AES_NIST_GCM_16:
|
||||
counter_u64_add(ktls_toe_gcm, -1);
|
||||
break;
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
}
|
||||
if (tls->params.auth_key != NULL) {
|
||||
explicit_bzero(tls->params.auth_key, tls->params.auth_key_len);
|
||||
@ -646,6 +684,52 @@ ktls_cleanup(struct ktls_session *tls)
|
||||
}
|
||||
|
||||
#if defined(INET) || defined(INET6)
|
||||
|
||||
#ifdef TCP_OFFLOAD
|
||||
static int
|
||||
ktls_try_toe(struct socket *so, struct ktls_session *tls)
|
||||
{
|
||||
struct inpcb *inp;
|
||||
struct tcpcb *tp;
|
||||
int error;
|
||||
|
||||
inp = so->so_pcb;
|
||||
INP_WLOCK(inp);
|
||||
if (inp->inp_flags2 & INP_FREED) {
|
||||
INP_WUNLOCK(inp);
|
||||
return (ECONNRESET);
|
||||
}
|
||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
|
||||
INP_WUNLOCK(inp);
|
||||
return (ECONNRESET);
|
||||
}
|
||||
if (inp->inp_socket == NULL) {
|
||||
INP_WUNLOCK(inp);
|
||||
return (ECONNRESET);
|
||||
}
|
||||
tp = intotcpcb(inp);
|
||||
if (tp->tod == NULL) {
|
||||
INP_WUNLOCK(inp);
|
||||
return (EOPNOTSUPP);
|
||||
}
|
||||
|
||||
error = tcp_offload_alloc_tls_session(tp, tls);
|
||||
INP_WUNLOCK(inp);
|
||||
if (error == 0) {
|
||||
tls->mode = TCP_TLS_MODE_TOE;
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
counter_u64_add(ktls_toe_cbc, 1);
|
||||
break;
|
||||
case CRYPTO_AES_NIST_GCM_16:
|
||||
counter_u64_add(ktls_toe_gcm, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Common code used when first enabling ifnet TLS on a connection or
|
||||
* when allocating a new ifnet TLS session due to a routing change.
|
||||
@ -744,6 +828,7 @@ ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force)
|
||||
|
||||
error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
|
||||
if (error == 0) {
|
||||
tls->mode = TCP_TLS_MODE_IFNET;
|
||||
tls->snd_tag = mst;
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
@ -787,6 +872,7 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls)
|
||||
rm_runlock(&ktls_backends_lock, &prio);
|
||||
if (be == NULL)
|
||||
return (EOPNOTSUPP);
|
||||
tls->mode = TCP_TLS_MODE_SW;
|
||||
switch (tls->params.cipher_algorithm) {
|
||||
case CRYPTO_AES_CBC:
|
||||
counter_u64_add(ktls_sw_cbc, 1);
|
||||
@ -834,8 +920,12 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
|
||||
if (error)
|
||||
return (error);
|
||||
|
||||
/* Prefer ifnet TLS over software TLS. */
|
||||
error = ktls_try_ifnet(so, tls, false);
|
||||
/* Prefer TOE -> ifnet TLS -> software TLS. */
|
||||
#ifdef TCP_OFFLOAD
|
||||
error = ktls_try_toe(so, tls);
|
||||
if (error)
|
||||
#endif
|
||||
error = ktls_try_ifnet(so, tls, false);
|
||||
if (error)
|
||||
error = ktls_try_sw(so, tls);
|
||||
|
||||
@ -852,7 +942,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
|
||||
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_tls_info = tls;
|
||||
if (tls->sw_encrypt == NULL)
|
||||
if (tls->mode != TCP_TLS_MODE_SW)
|
||||
so->so_snd.sb_flags |= SB_TLS_IFNET;
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sbunlock(&so->so_snd);
|
||||
@ -875,10 +965,8 @@ ktls_get_tx_mode(struct socket *so)
|
||||
tls = so->so_snd.sb_tls_info;
|
||||
if (tls == NULL)
|
||||
mode = TCP_TLS_MODE_NONE;
|
||||
else if (tls->sw_encrypt != NULL)
|
||||
mode = TCP_TLS_MODE_SW;
|
||||
else
|
||||
mode = TCP_TLS_MODE_IFNET;
|
||||
mode = tls->mode;
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
return (mode);
|
||||
}
|
||||
@ -893,7 +981,13 @@ ktls_set_tx_mode(struct socket *so, int mode)
|
||||
struct inpcb *inp;
|
||||
int error;
|
||||
|
||||
MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET);
|
||||
switch (mode) {
|
||||
case TCP_TLS_MODE_SW:
|
||||
case TCP_TLS_MODE_IFNET:
|
||||
break;
|
||||
default:
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
inp = so->so_pcb;
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
@ -904,8 +998,7 @@ ktls_set_tx_mode(struct socket *so, int mode)
|
||||
return (0);
|
||||
}
|
||||
|
||||
if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) ||
|
||||
(tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) {
|
||||
if (tls->mode == mode) {
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
return (0);
|
||||
}
|
||||
@ -952,7 +1045,7 @@ ktls_set_tx_mode(struct socket *so, int mode)
|
||||
|
||||
SOCKBUF_LOCK(&so->so_snd);
|
||||
so->so_snd.sb_tls_info = tls_new;
|
||||
if (tls_new->sw_encrypt == NULL)
|
||||
if (tls_new->mode != TCP_TLS_MODE_SW)
|
||||
so->so_snd.sb_flags |= SB_TLS_IFNET;
|
||||
SOCKBUF_UNLOCK(&so->so_snd);
|
||||
sbunlock(&so->so_snd);
|
||||
@ -1238,7 +1331,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
|
||||
* When using ifnet TLS, unencrypted TLS records are
|
||||
* sent down the stack to the NIC.
|
||||
*/
|
||||
if (tls->sw_encrypt != NULL) {
|
||||
if (tls->mode == TCP_TLS_MODE_SW) {
|
||||
m->m_flags |= M_NOTREADY;
|
||||
pgs->nrdy = pgs->npgs;
|
||||
*enq_cnt += pgs->npgs;
|
||||
@ -1278,7 +1371,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
|
||||
|
||||
pgs = m->m_ext.ext_pgs;
|
||||
|
||||
KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf"));
|
||||
KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
|
||||
|
||||
pgs->enc_cnt = page_count;
|
||||
pgs->mbuf = m;
|
||||
|
@ -1491,7 +1491,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
|
||||
tls_pruflag = 0;
|
||||
tls = ktls_hold(so->so_snd.sb_tls_info);
|
||||
if (tls != NULL) {
|
||||
if (tls->sw_encrypt != NULL)
|
||||
if (tls->mode == TCP_TLS_MODE_SW)
|
||||
tls_pruflag = PRUS_NOTREADY;
|
||||
|
||||
if (control != NULL) {
|
||||
@ -1659,7 +1659,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
|
||||
}
|
||||
|
||||
#ifdef KERN_TLS
|
||||
if (tls != NULL && tls->sw_encrypt != NULL) {
|
||||
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
|
||||
/*
|
||||
* Note that error is intentionally
|
||||
* ignored.
|
||||
|
@ -357,6 +357,7 @@ struct tcp_function_set {
|
||||
#define TCP_TLS_MODE_NONE 0
|
||||
#define TCP_TLS_MODE_SW 1
|
||||
#define TCP_TLS_MODE_IFNET 2
|
||||
#define TCP_TLS_MODE_TOE 3
|
||||
|
||||
/*
|
||||
* TCP Control message types
|
||||
|
@ -178,6 +178,17 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti)
|
||||
tod->tod_tcp_info(tod, tp, ti);
|
||||
}
|
||||
|
||||
int
|
||||
tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
|
||||
{
|
||||
struct toedev *tod = tp->tod;
|
||||
|
||||
KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
|
||||
INP_WLOCK_ASSERT(tp->t_inpcb);
|
||||
|
||||
return (tod->tod_alloc_tls_session(tod, tp, tls));
|
||||
}
|
||||
|
||||
void
|
||||
tcp_offload_detach(struct tcpcb *tp)
|
||||
{
|
||||
|
@ -46,6 +46,7 @@ int tcp_offload_output(struct tcpcb *);
|
||||
void tcp_offload_rcvd(struct tcpcb *);
|
||||
void tcp_offload_ctloutput(struct tcpcb *, int, int);
|
||||
void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
|
||||
int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
|
||||
void tcp_offload_detach(struct tcpcb *);
|
||||
|
||||
#endif
|
||||
|
@ -1936,8 +1936,6 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
|
||||
error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
|
||||
if (error)
|
||||
return (error);
|
||||
if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET)
|
||||
return (EINVAL);
|
||||
|
||||
INP_WLOCK_RECHECK(inp);
|
||||
error = ktls_set_tx_mode(so, ui);
|
||||
|
@ -191,6 +191,14 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused,
|
||||
return;
|
||||
}
|
||||
|
||||
static int
|
||||
toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
|
||||
struct ktls_session *tls __unused)
|
||||
{
|
||||
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
/*
|
||||
* Inform one or more TOE devices about a listening socket.
|
||||
*/
|
||||
@ -281,6 +289,7 @@ init_toedev(struct toedev *tod)
|
||||
tod->tod_offload_socket = toedev_offload_socket;
|
||||
tod->tod_ctloutput = toedev_ctloutput;
|
||||
tod->tod_tcp_info = toedev_tcp_info;
|
||||
tod->tod_alloc_tls_session = toedev_alloc_tls_session;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -41,6 +41,7 @@ struct tcpopt;
|
||||
struct tcphdr;
|
||||
struct in_conninfo;
|
||||
struct tcp_info;
|
||||
struct ktls_session;
|
||||
|
||||
struct toedev {
|
||||
TAILQ_ENTRY(toedev) link; /* glue for toedev_list */
|
||||
@ -108,6 +109,10 @@ struct toedev {
|
||||
/* Update software state */
|
||||
void (*tod_tcp_info)(struct toedev *, struct tcpcb *,
|
||||
struct tcp_info *);
|
||||
|
||||
/* Create a TLS session */
|
||||
int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
|
||||
struct ktls_session *);
|
||||
};
|
||||
|
||||
typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);
|
||||
|
@ -156,6 +156,7 @@ struct ktls_session {
|
||||
struct tls_session_params params;
|
||||
u_int wq_index;
|
||||
volatile u_int refcount;
|
||||
int mode;
|
||||
|
||||
struct task reset_tag_task;
|
||||
struct inpcb *inp;
|
||||
|
Loading…
Reference in New Issue
Block a user