Initial support for kernel offload of TLS receive.
- Add a new TCP_RXTLS_ENABLE socket option to set the encryption and authentication algorithms and keys as well as the initial sequence number. - When reading from a socket using KTLS receive, applications must use recvmsg(). Each successful call to recvmsg() will return a single TLS record. A new TCP control message, TLS_GET_RECORD, will contain the TLS record header of the decrypted record. The regular message buffer passed to recvmsg() will receive the decrypted payload. This is similar to the interface used by Linux's KTLS RX except that Linux does not return the full TLS header in the control message. - Add plumbing to the TOE KTLS interface to request either transmit or receive KTLS sessions. - When a socket is using receive KTLS, redirect reads from soreceive_stream() into soreceive_generic(). - Note that this interface is currently only defined for TLS 1.1 and 1.2, though I believe we will be able to reuse the same interface and structures for 1.3.
This commit is contained in:
parent
ec1db6e13d
commit
f1f9347546
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=360408
@ -34,7 +34,7 @@
|
||||
.\" From: @(#)tcp.4 8.1 (Berkeley) 6/5/93
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd April 16, 2020
|
||||
.Dd April 27, 2020
|
||||
.Dt TCP 4
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -319,14 +319,11 @@ control message.
|
||||
The payload of this control message is a single byte holding the desired
|
||||
TLS record type.
|
||||
.Pp
|
||||
Data read from this socket will still be encrypted and must be parsed by
|
||||
a TLS-aware consumer.
|
||||
.Pp
|
||||
At present, only a single key may be set on a socket.
|
||||
At present, only a single transmit key may be set on a socket.
|
||||
As such, users of this option must disable rekeying.
|
||||
.It Dv TCP_TXTLS_MODE
|
||||
The integer argument can be used to get or set the current TLS mode of a
|
||||
socket.
|
||||
The integer argument can be used to get or set the current TLS transmit mode
|
||||
of a socket.
|
||||
Setting the mode can only used to toggle between software and NIC TLS after
|
||||
TLS has been initially enabled via the
|
||||
.Dv TCP_TXTLS_ENABLE
|
||||
@ -344,6 +341,33 @@ TLS records are encrypted by the network interface card (NIC).
|
||||
.It Dv TCP_TLS_MODE_TOE
|
||||
TLS records are encrypted by the NIC using a TCP offload engine (TOE).
|
||||
.El
|
||||
.It Dv TCP_RXTLS_ENABLE
|
||||
Enable in-kernel TLS for data read from this socket.
|
||||
The
|
||||
.Vt struct tls_so_enable
|
||||
argument defines the encryption and authentication algorithms and keys
|
||||
used to decrypt the socket data.
|
||||
.Pp
|
||||
Each received TLS record must be read from the socket using
|
||||
.Xr recvmsg 2 .
|
||||
Each received TLS record will contain a
|
||||
.Dv TLS_GET_RECORD
|
||||
control message along with the decrypted payload.
|
||||
The control message contains a
|
||||
.Vt struct tls_get_record
|
||||
which includes fields from the TLS record header.
|
||||
If a corrupted TLS record is received,
|
||||
recvmsg 2
|
||||
will fail with
|
||||
.Dv EBADMSG .
|
||||
.Pp
|
||||
At present, only a single receive key may be set on a socket.
|
||||
As such, users of this option must disable rekeying.
|
||||
.It Dv TCP_RXTLS_MODE
|
||||
The integer argument can be used to get the current TLS receive mode
|
||||
of a socket.
|
||||
The available modes are the same as for
|
||||
.Dv TCP_TXTLS_MODE .
|
||||
.El
|
||||
.Pp
|
||||
The option level for the
|
||||
|
@ -40,6 +40,9 @@ __FBSDID("$FreeBSD$");
|
||||
#include <sys/systm.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/ktr.h>
|
||||
#ifdef KERN_TLS
|
||||
#include <sys/ktls.h>
|
||||
#endif
|
||||
#include <sys/lock.h>
|
||||
#include <sys/limits.h>
|
||||
#include <sys/module.h>
|
||||
@ -814,13 +817,16 @@ t4_tcp_info(struct toedev *tod, struct tcpcb *tp, struct tcp_info *ti)
|
||||
#ifdef KERN_TLS
|
||||
static int
|
||||
t4_alloc_tls_session(struct toedev *tod, struct tcpcb *tp,
|
||||
struct ktls_session *tls)
|
||||
struct ktls_session *tls, int direction)
|
||||
{
|
||||
struct toepcb *toep = tp->t_toe;
|
||||
|
||||
INP_WLOCK_ASSERT(tp->t_inpcb);
|
||||
MPASS(tls != NULL);
|
||||
|
||||
if (direction != KTLS_TX)
|
||||
return (EOPNOTSUPP);
|
||||
|
||||
return (tls_alloc_ktls(toep, tls));
|
||||
}
|
||||
#endif
|
||||
|
@ -702,7 +702,7 @@ ktls_cleanup(struct ktls_session *tls)
|
||||
|
||||
#ifdef TCP_OFFLOAD
|
||||
static int
|
||||
ktls_try_toe(struct socket *so, struct ktls_session *tls)
|
||||
ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
|
||||
{
|
||||
struct inpcb *inp;
|
||||
struct tcpcb *tp;
|
||||
@ -728,7 +728,7 @@ ktls_try_toe(struct socket *so, struct ktls_session *tls)
|
||||
return (EOPNOTSUPP);
|
||||
}
|
||||
|
||||
error = tcp_offload_alloc_tls_session(tp, tls);
|
||||
error = tcp_offload_alloc_tls_session(tp, tls, direction);
|
||||
INP_WUNLOCK(inp);
|
||||
if (error == 0) {
|
||||
tls->mode = TCP_TLS_MODE_TOE;
|
||||
@ -900,6 +900,60 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls)
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
ktls_enable_rx(struct socket *so, struct tls_enable *en)
|
||||
{
|
||||
struct ktls_session *tls;
|
||||
int error;
|
||||
|
||||
if (!ktls_offload_enable)
|
||||
return (ENOTSUP);
|
||||
|
||||
counter_u64_add(ktls_offload_enable_calls, 1);
|
||||
|
||||
/*
|
||||
* This should always be true since only the TCP socket option
|
||||
* invokes this function.
|
||||
*/
|
||||
if (so->so_proto->pr_protocol != IPPROTO_TCP)
|
||||
return (EINVAL);
|
||||
|
||||
/*
|
||||
* XXX: Don't overwrite existing sessions. We should permit
|
||||
* this to support rekeying in the future.
|
||||
*/
|
||||
if (so->so_rcv.sb_tls_info != NULL)
|
||||
return (EALREADY);
|
||||
|
||||
if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
|
||||
return (ENOTSUP);
|
||||
|
||||
error = ktls_create_session(so, en, &tls);
|
||||
if (error)
|
||||
return (error);
|
||||
|
||||
/* TLS RX offload is only supported on TOE currently. */
|
||||
#ifdef TCP_OFFLOAD
|
||||
error = ktls_try_toe(so, tls, KTLS_RX);
|
||||
#else
|
||||
error = EOPNOTSUPP;
|
||||
#endif
|
||||
|
||||
if (error) {
|
||||
ktls_cleanup(tls);
|
||||
return (error);
|
||||
}
|
||||
|
||||
/* Mark the socket as using TLS offload. */
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
so->so_rcv.sb_tls_info = tls;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
|
||||
counter_u64_add(ktls_offload_total, 1);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
ktls_enable_tx(struct socket *so, struct tls_enable *en)
|
||||
{
|
||||
@ -938,7 +992,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
|
||||
|
||||
/* Prefer TOE -> ifnet TLS -> software TLS. */
|
||||
#ifdef TCP_OFFLOAD
|
||||
error = ktls_try_toe(so, tls);
|
||||
error = ktls_try_toe(so, tls, KTLS_TX);
|
||||
if (error)
|
||||
#endif
|
||||
error = ktls_try_ifnet(so, tls, false);
|
||||
@ -969,6 +1023,25 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
ktls_get_rx_mode(struct socket *so)
|
||||
{
|
||||
struct ktls_session *tls;
|
||||
struct inpcb *inp;
|
||||
int mode;
|
||||
|
||||
inp = so->so_pcb;
|
||||
INP_WLOCK_ASSERT(inp);
|
||||
SOCKBUF_LOCK(&so->so_rcv);
|
||||
tls = so->so_rcv.sb_tls_info;
|
||||
if (tls == NULL)
|
||||
mode = TCP_TLS_MODE_NONE;
|
||||
else
|
||||
mode = tls->mode;
|
||||
SOCKBUF_UNLOCK(&so->so_rcv);
|
||||
return (mode);
|
||||
}
|
||||
|
||||
int
|
||||
ktls_get_tx_mode(struct socket *so)
|
||||
{
|
||||
|
@ -2372,12 +2372,34 @@ soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio,
|
||||
|
||||
sb = &so->so_rcv;
|
||||
|
||||
#ifdef KERN_TLS
|
||||
/*
|
||||
* KTLS store TLS records as records with a control message to
|
||||
* describe the framing.
|
||||
*
|
||||
* We check once here before acquiring locks to optimize the
|
||||
* common case.
|
||||
*/
|
||||
if (sb->sb_tls_info != NULL)
|
||||
return (soreceive_generic(so, psa, uio, mp0, controlp,
|
||||
flagsp));
|
||||
#endif
|
||||
|
||||
/* Prevent other readers from entering the socket. */
|
||||
error = sblock(sb, SBLOCKWAIT(flags));
|
||||
if (error)
|
||||
return (error);
|
||||
SOCKBUF_LOCK(sb);
|
||||
|
||||
#ifdef KERN_TLS
|
||||
if (sb->sb_tls_info != NULL) {
|
||||
SOCKBUF_UNLOCK(sb);
|
||||
sbunlock(sb);
|
||||
return (soreceive_generic(so, psa, uio, mp0, controlp,
|
||||
flagsp));
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Easy one, no space to copyout anything. */
|
||||
if (uio->uio_resid == 0) {
|
||||
error = EINVAL;
|
||||
|
@ -178,6 +178,8 @@ struct tcphdr {
|
||||
device */
|
||||
#define TCP_TXTLS_ENABLE 39 /* TLS framing and encryption for transmit */
|
||||
#define TCP_TXTLS_MODE 40 /* Transmit TLS mode */
|
||||
#define TCP_RXTLS_ENABLE 41 /* TLS framing and encryption for receive */
|
||||
#define TCP_RXTLS_MODE 42 /* Receive TLS mode */
|
||||
#define TCP_CONGESTION 64 /* get/set congestion control algorithm */
|
||||
#define TCP_CCALGOOPT 65 /* get/set cc algorithm specific options */
|
||||
#define TCP_DELACK 72 /* socket option for delayed ack */
|
||||
@ -388,6 +390,7 @@ struct tcp_function_set {
|
||||
* TCP Control message types
|
||||
*/
|
||||
#define TLS_SET_RECORD_TYPE 1
|
||||
#define TLS_GET_RECORD 2
|
||||
|
||||
/*
|
||||
* TCP specific variables of interest for tp->t_stats stats(9) accounting.
|
||||
|
@ -198,14 +198,15 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti)
|
||||
}
|
||||
|
||||
int
|
||||
tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
|
||||
tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls,
|
||||
int direction)
|
||||
{
|
||||
struct toedev *tod = tp->tod;
|
||||
|
||||
KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
|
||||
INP_WLOCK_ASSERT(tp->t_inpcb);
|
||||
|
||||
return (tod->tod_alloc_tls_session(tod, tp, tls));
|
||||
return (tod->tod_alloc_tls_session(tod, tp, tls, direction));
|
||||
}
|
||||
|
||||
void
|
||||
|
@ -46,7 +46,7 @@ int tcp_offload_output(struct tcpcb *);
|
||||
void tcp_offload_rcvd(struct tcpcb *);
|
||||
void tcp_offload_ctloutput(struct tcpcb *, int, int);
|
||||
void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
|
||||
int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
|
||||
int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *, int);
|
||||
void tcp_offload_detach(struct tcpcb *);
|
||||
|
||||
#endif
|
||||
|
@ -2080,6 +2080,14 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
|
||||
error = ktls_set_tx_mode(so, ui);
|
||||
INP_WUNLOCK(inp);
|
||||
break;
|
||||
case TCP_RXTLS_ENABLE:
|
||||
INP_WUNLOCK(inp);
|
||||
error = sooptcopyin(sopt, &tls, sizeof(tls),
|
||||
sizeof(tls));
|
||||
if (error)
|
||||
break;
|
||||
error = ktls_enable_rx(so, &tls);
|
||||
break;
|
||||
#endif
|
||||
|
||||
case TCP_KEEPIDLE:
|
||||
@ -2418,6 +2426,11 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
|
||||
INP_WUNLOCK(inp);
|
||||
error = sooptcopyout(sopt, &optval, sizeof(optval));
|
||||
break;
|
||||
case TCP_RXTLS_MODE:
|
||||
optval = ktls_get_rx_mode(so);
|
||||
INP_WUNLOCK(inp);
|
||||
error = sooptcopyout(sopt, &optval, sizeof(optval));
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
INP_WUNLOCK(inp);
|
||||
|
@ -193,7 +193,7 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused,
|
||||
|
||||
static int
|
||||
toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
|
||||
struct ktls_session *tls __unused)
|
||||
struct ktls_session *tls __unused, int direction __unused)
|
||||
{
|
||||
|
||||
return (EINVAL);
|
||||
|
@ -113,7 +113,7 @@ struct toedev {
|
||||
|
||||
/* Create a TLS session */
|
||||
int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
|
||||
struct ktls_session *);
|
||||
struct ktls_session *, int);
|
||||
};
|
||||
|
||||
typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);
|
||||
|
@ -98,7 +98,7 @@ struct tls_mac_data {
|
||||
#define TLS_MINOR_VER_TWO 3 /* 3, 3 */
|
||||
#define TLS_MINOR_VER_THREE 4 /* 3, 4 */
|
||||
|
||||
/* For TCP_TXTLS_ENABLE */
|
||||
/* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */
|
||||
#ifdef _KERNEL
|
||||
struct tls_enable_v0 {
|
||||
const uint8_t *cipher_key;
|
||||
@ -130,6 +130,17 @@ struct tls_enable {
|
||||
uint8_t rec_seq[8];
|
||||
};
|
||||
|
||||
/* Structure for TLS_GET_RECORD. */
|
||||
struct tls_get_record {
|
||||
/* TLS record header. */
|
||||
uint8_t tls_type;
|
||||
uint8_t tls_vmajor;
|
||||
uint8_t tls_vminor;
|
||||
uint16_t tls_length;
|
||||
};
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
struct tls_session_params {
|
||||
uint8_t *cipher_key;
|
||||
uint8_t *auth_key;
|
||||
@ -148,7 +159,9 @@ struct tls_session_params {
|
||||
uint8_t flags;
|
||||
};
|
||||
|
||||
#ifdef _KERNEL
|
||||
/* Used in APIs to request RX vs TX sessions. */
|
||||
#define KTLS_TX 1
|
||||
#define KTLS_RX 2
|
||||
|
||||
#define KTLS_API_VERSION 6
|
||||
|
||||
@ -192,6 +205,7 @@ struct ktls_session {
|
||||
|
||||
int ktls_crypto_backend_register(struct ktls_crypto_backend *be);
|
||||
int ktls_crypto_backend_deregister(struct ktls_crypto_backend *be);
|
||||
int ktls_enable_rx(struct socket *so, struct tls_enable *en);
|
||||
int ktls_enable_tx(struct socket *so, struct tls_enable *en);
|
||||
void ktls_destroy(struct ktls_session *tls);
|
||||
void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
|
||||
@ -199,6 +213,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
|
||||
void ktls_seq(struct sockbuf *sb, struct mbuf *m);
|
||||
void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
|
||||
void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
|
||||
int ktls_get_rx_mode(struct socket *so);
|
||||
int ktls_set_tx_mode(struct socket *so, int mode);
|
||||
int ktls_get_tx_mode(struct socket *so);
|
||||
int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls);
|
||||
|
Loading…
Reference in New Issue
Block a user