Initial support for kernel offload of TLS receive.

- Add a new TCP_RXTLS_ENABLE socket option to set the encryption and
  authentication algorithms and keys as well as the initial sequence
  number.

- When reading from a socket using KTLS receive, applications must use
  recvmsg().  Each successful call to recvmsg() will return a single
  TLS record.  A new TCP control message, TLS_GET_RECORD, will contain
  the TLS record header of the decrypted record.  The regular message
  buffer passed to recvmsg() will receive the decrypted payload.  This
  is similar to the interface used by Linux's KTLS RX except that
  Linux does not return the full TLS header in the control message.

- Add plumbing to the TOE KTLS interface to request either transmit
  or receive KTLS sessions.

- When a socket is using receive KTLS, redirect reads from
  soreceive_stream() into soreceive_generic().

- Note that this interface is currently only defined for TLS 1.1 and
  1.2, though I believe we will be able to reuse the same interface
  and structures for 1.3.

share/man/man4/tcp.4

@@ -34,7 +34,7 @@
 .\"     From: @(#)tcp.4	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd April 16, 2020
+.Dd April 27, 2020
 .Dt TCP 4
 .Os
 .Sh NAME
@@ -319,14 +319,11 @@ control message.
 The payload of this control message is a single byte holding the desired
 TLS record type.
 .Pp
-Data read from this socket will still be encrypted and must be parsed by
-a TLS-aware consumer.
-.Pp
-At present, only a single key may be set on a socket.
+At present, only a single transmit key may be set on a socket.
 As such, users of this option must disable rekeying.
 .It Dv TCP_TXTLS_MODE
-The integer argument can be used to get or set the current TLS mode of a
-socket.
+The integer argument can be used to get or set the current TLS transmit mode
+of a socket.
 Setting the mode can only used to toggle between software and NIC TLS after
 TLS has been initially enabled via the
 .Dv TCP_TXTLS_ENABLE
@@ -344,6 +341,33 @@ TLS records are encrypted by the network interface card (NIC).
 .It Dv TCP_TLS_MODE_TOE
 TLS records are encrypted by the NIC using a TCP offload engine (TOE).
 .El
+.It Dv TCP_RXTLS_ENABLE
+Enable in-kernel TLS for data read from this socket.
+The
+.Vt struct tls_so_enable
+argument defines the encryption and authentication algorithms and keys
+used to decrypt the socket data.
+.Pp
+Each received TLS record must be read from the socket using
+.Xr recvmsg 2 .
+Each received TLS record will contain a
+.Dv TLS_GET_RECORD
+control message along with the decrypted payload.
+The control message contains a
+.Vt struct tls_get_record
+which includes fields from the TLS record header.
+If a corrupted TLS record is received,
+recvmsg 2
+will fail with
+.Dv EBADMSG .
+.Pp
+At present, only a single receive key may be set on a socket.
+As such, users of this option must disable rekeying.
+.It Dv TCP_RXTLS_MODE
+The integer argument can be used to get the current TLS receive mode
+of a socket.
+The available modes are the same as for
+.Dv TCP_TXTLS_MODE .
 .El
 .Pp
 The option level for the

sys/dev/cxgbe/tom/t4_tom.c

@@ -40,6 +40,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
+#ifdef KERN_TLS
+#include <sys/ktls.h>
+#endif
 #include <sys/lock.h>
 #include <sys/limits.h>
 #include <sys/module.h>
@@ -814,13 +817,16 @@ t4_tcp_info(struct toedev *tod, struct tcpcb *tp, struct tcp_info *ti)
 #ifdef KERN_TLS
 static int
 t4_alloc_tls_session(struct toedev *tod, struct tcpcb *tp,
-    struct ktls_session *tls)
+    struct ktls_session *tls, int direction)
 {
 	struct toepcb *toep = tp->t_toe;
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 	MPASS(tls != NULL);
 
+	if (direction != KTLS_TX)
+		return (EOPNOTSUPP);
+
 	return (tls_alloc_ktls(toep, tls));
 }
 #endif

sys/kern/uipc_ktls.c

@@ -702,7 +702,7 @@ ktls_cleanup(struct ktls_session *tls)
 
 #ifdef TCP_OFFLOAD
 static int
-ktls_try_toe(struct socket *so, struct ktls_session *tls)
+ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
 {
 	struct inpcb *inp;
 	struct tcpcb *tp;
@@ -728,7 +728,7 @@ ktls_try_toe(struct socket *so, struct ktls_session *tls)
 		return (EOPNOTSUPP);
 	}
 
-	error = tcp_offload_alloc_tls_session(tp, tls);
+	error = tcp_offload_alloc_tls_session(tp, tls, direction);
 	INP_WUNLOCK(inp);
 	if (error == 0) {
 		tls->mode = TCP_TLS_MODE_TOE;
@@ -900,6 +900,60 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls)
 	return (0);
 }
 
+int
+ktls_enable_rx(struct socket *so, struct tls_enable *en)
+{
+	struct ktls_session *tls;
+	int error;
+
+	if (!ktls_offload_enable)
+		return (ENOTSUP);
+
+	counter_u64_add(ktls_offload_enable_calls, 1);
+
+	/*
+	 * This should always be true since only the TCP socket option
+	 * invokes this function.
+	 */
+	if (so->so_proto->pr_protocol != IPPROTO_TCP)
+		return (EINVAL);
+
+	/*
+	 * XXX: Don't overwrite existing sessions.  We should permit
+	 * this to support rekeying in the future.
+	 */
+	if (so->so_rcv.sb_tls_info != NULL)
+		return (EALREADY);
+
+	if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
+		return (ENOTSUP);
+
+	error = ktls_create_session(so, en, &tls);
+	if (error)
+		return (error);
+
+	/* TLS RX offload is only supported on TOE currently. */
+#ifdef TCP_OFFLOAD
+	error = ktls_try_toe(so, tls, KTLS_RX);
+#else
+	error = EOPNOTSUPP;
+#endif
+
+	if (error) {
+		ktls_cleanup(tls);
+		return (error);
+	}
+
+	/* Mark the socket as using TLS offload. */
+	SOCKBUF_LOCK(&so->so_rcv);
+	so->so_rcv.sb_tls_info = tls;
+	SOCKBUF_UNLOCK(&so->so_rcv);
+
+	counter_u64_add(ktls_offload_total, 1);
+
+	return (0);
+}
+
 int
 ktls_enable_tx(struct socket *so, struct tls_enable *en)
 {
@@ -938,7 +992,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 
 	/* Prefer TOE -> ifnet TLS -> software TLS. */
 #ifdef TCP_OFFLOAD
-	error = ktls_try_toe(so, tls);
+	error = ktls_try_toe(so, tls, KTLS_TX);
 	if (error)
 #endif
 		error = ktls_try_ifnet(so, tls, false);
@@ -969,6 +1023,25 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en)
 	return (0);
 }
 
+int
+ktls_get_rx_mode(struct socket *so)
+{
+	struct ktls_session *tls;
+	struct inpcb *inp;
+	int mode;
+
+	inp = so->so_pcb;
+	INP_WLOCK_ASSERT(inp);
+	SOCKBUF_LOCK(&so->so_rcv);
+	tls = so->so_rcv.sb_tls_info;
+	if (tls == NULL)
+		mode = TCP_TLS_MODE_NONE;
+	else
+		mode = tls->mode;
+	SOCKBUF_UNLOCK(&so->so_rcv);
+	return (mode);
+}
+
 int
 ktls_get_tx_mode(struct socket *so)
 {

sys/kern/uipc_socket.c

@@ -2372,12 +2372,34 @@ soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio,
 
 	sb = &so->so_rcv;
 
+#ifdef KERN_TLS
+	/*
+	 * KTLS store TLS records as records with a control message to
+	 * describe the framing.
+	 *
+	 * We check once here before acquiring locks to optimize the
+	 * common case.
+	 */
+	if (sb->sb_tls_info != NULL)
+		return (soreceive_generic(so, psa, uio, mp0, controlp,
+		    flagsp));
+#endif
+
 	/* Prevent other readers from entering the socket. */
 	error = sblock(sb, SBLOCKWAIT(flags));
 	if (error)
 		return (error);
 	SOCKBUF_LOCK(sb);
 
+#ifdef KERN_TLS
+	if (sb->sb_tls_info != NULL) {
+		SOCKBUF_UNLOCK(sb);
+		sbunlock(sb);
+		return (soreceive_generic(so, psa, uio, mp0, controlp,
+		    flagsp));
+	}
+#endif
+
 	/* Easy one, no space to copyout anything. */
 	if (uio->uio_resid == 0) {
 		error = EINVAL;

sys/netinet/tcp.h

@@ -178,6 +178,8 @@ struct tcphdr {
 				   device */
 #define	TCP_TXTLS_ENABLE 39	/* TLS framing and encryption for transmit */
 #define	TCP_TXTLS_MODE	40	/* Transmit TLS mode */
+#define	TCP_RXTLS_ENABLE 41	/* TLS framing and encryption for receive */
+#define	TCP_RXTLS_MODE	42	/* Receive TLS mode */
 #define	TCP_CONGESTION	64	/* get/set congestion control algorithm */
 #define	TCP_CCALGOOPT	65	/* get/set cc algorithm specific options */
 #define TCP_DELACK  	72	/* socket option for delayed ack */
@@ -388,6 +390,7 @@ struct tcp_function_set {
  * TCP Control message types
  */
 #define	TLS_SET_RECORD_TYPE	1
+#define	TLS_GET_RECORD		2
 
 /*
  * TCP specific variables of interest for tp->t_stats stats(9) accounting.

sys/netinet/tcp_offload.c

@@ -198,14 +198,15 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti)
 }
 
 int
-tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls)
+tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls,
+    int direction)
 {
 	struct toedev *tod = tp->tod;
 
 	KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp));
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
-	return (tod->tod_alloc_tls_session(tod, tp, tls));
+	return (tod->tod_alloc_tls_session(tod, tp, tls, direction));
 }
 
 void

sys/netinet/tcp_offload.h

@@ -46,7 +46,7 @@ int  tcp_offload_output(struct tcpcb *);
 void tcp_offload_rcvd(struct tcpcb *);
 void tcp_offload_ctloutput(struct tcpcb *, int, int);
 void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *);
-int  tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *);
+int  tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *, int);
 void tcp_offload_detach(struct tcpcb *);
 
 #endif

sys/netinet/tcp_usrreq.c

@@ -2080,6 +2080,14 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
 			error = ktls_set_tx_mode(so, ui);
 			INP_WUNLOCK(inp);
 			break;
+		case TCP_RXTLS_ENABLE:
+			INP_WUNLOCK(inp);
+			error = sooptcopyin(sopt, &tls, sizeof(tls),
+			    sizeof(tls));
+			if (error)
+				break;
+			error = ktls_enable_rx(so, &tls);
+			break;
 #endif
 
 		case TCP_KEEPIDLE:
@@ -2418,6 +2426,11 @@ tcp_default_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp
 			INP_WUNLOCK(inp);
 			error = sooptcopyout(sopt, &optval, sizeof(optval));
 			break;
+		case TCP_RXTLS_MODE:
+			optval = ktls_get_rx_mode(so);
+			INP_WUNLOCK(inp);
+			error = sooptcopyout(sopt, &optval, sizeof(optval));
+			break;
 #endif
 		default:
 			INP_WUNLOCK(inp);

sys/netinet/toecore.c

@@ -193,7 +193,7 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused,
 
 static int
 toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused,
-    struct ktls_session *tls __unused)
+    struct ktls_session *tls __unused, int direction __unused)
 {
 
 	return (EINVAL);

sys/netinet/toecore.h

@@ -113,7 +113,7 @@ struct toedev {
 
 	/* Create a TLS session */
 	int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *,
-	    struct ktls_session *);
+	    struct ktls_session *, int);
 };
 
 typedef	void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *);

sys/sys/ktls.h

@@ -98,7 +98,7 @@ struct tls_mac_data {
 #define	TLS_MINOR_VER_TWO	3	/* 3, 3 */
 #define	TLS_MINOR_VER_THREE	4	/* 3, 4 */
 
-/* For TCP_TXTLS_ENABLE */
+/* For TCP_TXTLS_ENABLE and TCP_RXTLS_ENABLE. */
 #ifdef _KERNEL
 struct tls_enable_v0 {
 	const uint8_t *cipher_key;
@@ -130,6 +130,17 @@ struct tls_enable {
 	uint8_t rec_seq[8];
 };
 
+/* Structure for TLS_GET_RECORD. */
+struct tls_get_record {
+	/* TLS record header. */
+	uint8_t  tls_type;
+	uint8_t  tls_vmajor;
+	uint8_t  tls_vminor;
+	uint16_t tls_length;
+};
+
+#ifdef _KERNEL
+
 struct tls_session_params {
 	uint8_t *cipher_key;
 	uint8_t *auth_key;
@@ -148,7 +159,9 @@ struct tls_session_params {
 	uint8_t flags;
 };
 
-#ifdef _KERNEL
+/* Used in APIs to request RX vs TX sessions. */
+#define	KTLS_TX		1
+#define	KTLS_RX		2
 
 #define	KTLS_API_VERSION 6
 
@@ -192,6 +205,7 @@ struct ktls_session {
 
 int ktls_crypto_backend_register(struct ktls_crypto_backend *be);
 int ktls_crypto_backend_deregister(struct ktls_crypto_backend *be);
+int ktls_enable_rx(struct socket *so, struct tls_enable *en);
 int ktls_enable_tx(struct socket *so, struct tls_enable *en);
 void ktls_destroy(struct ktls_session *tls);
 void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
@@ -199,6 +213,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
 void ktls_seq(struct sockbuf *sb, struct mbuf *m);
 void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
 void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
+int ktls_get_rx_mode(struct socket *so);
 int ktls_set_tx_mode(struct socket *so, int mode);
 int ktls_get_tx_mode(struct socket *so);
 int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls);