From 74a68313b503940158a2e8e8f02626d7cdbdaff9 Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Fri, 10 Sep 2021 17:20:39 -0400
Subject: [PATCH] socket: Add macros to lock socket buffers using socket
 references

Since commit c67f3b8b78e50c6df7c057d6cf108e4d6b4312d0 the sockbuf
mutexes belong to the containing socket.  Sockbufs contain a pointer to
a mutex, which by default is initialized to the corresponding mutexes in
the socket.  The SOCKBUF_LOCK() etc. macros operate on this pointer.
However, the pointer is clobbered by listen(2) so it's not safe to use
them unless one is sure that the socket is not a listening socket.

This change introduces a new set of macros which lock socket buffers
through the socket.  This is a bit cheaper since it removes the pointer
indirection, and allows one to safely lock socket buffers and then check
for a listening socket.

For MFC, these macros should be reimplemented in terms of the existing
socket buffer layout.

Reviewed by:	tuexen, gallatin, jhb
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D31900
---
 sys/sys/sockbuf.h   |  7 +++++--
 sys/sys/socketvar.h | 30 ++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h
index 3b345870bd5f..3fc9b10cd240 100644
--- a/sys/sys/sockbuf.h
+++ b/sys/sys/sockbuf.h
@@ -118,8 +118,11 @@ struct sockbuf {
 #ifdef _KERNEL
 
 /*
- * Per-socket buffer mutex used to protect most fields in the socket
- * buffer.
+ * Per-socket buffer mutex used to protect most fields in the socket buffer.
+ * These make use of the mutex pointer embedded in struct sockbuf, which
+ * currently just references mutexes in the containing socket.  The
+ * SOCK_SENDBUF_LOCK() etc. macros can be used instead of or in combination with
+ * these locking macros.
  */
 #define	SOCKBUF_MTX(_sb)		((_sb)->sb_mtx)
 #define	SOCKBUF_LOCK_INIT(_sb, _name) \
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index 69dd1706e366..57913f7bbd65 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -77,8 +77,8 @@ enum socket_qstate {
  * Locking key to struct socket:
  * (a) constant after allocation, no locking required.
  * (b) locked by SOCK_LOCK(so).
- * (cr) locked by SOCKBUF_LOCK(&so->so_rcv).
- * (cs) locked by SOCKBUF_LOCK(&so->so_snd).
+ * (cr) locked by SOCK_RECVBUF_LOCK(so)/SOCKBUF_LOCK(&so->so_rcv).
+ * (cs) locked by SOCK_SENDBUF_LOCK(so)/SOCKBUF_LOCK(&so->so_snd).
  * (e) locked by SOLISTEN_LOCK() of corresponding listening socket.
  * (f) not locked since integer reads/writes are atomic.
  * (g) used only as a sleep/wakeup address, no value.
@@ -255,6 +255,32 @@ struct socket {
 	    ("%s: %p not listening", __func__, (sol)));			\
 } while (0)
 
+/*
+ * Socket buffer locks.  These manipulate the same mutexes as SOCKBUF_LOCK()
+ * and related macros.
+ */
+#define	SOCK_RECVBUF_MTX(so)						\
+	(&(so)->so_rcv_mtx)
+#define	SOCK_RECVBUF_LOCK(so)						\
+	mtx_lock(SOCK_RECVBUF_MTX(so))
+#define	SOCK_RECVBUF_UNLOCK(so)						\
+	mtx_unlock(SOCK_RECVBUF_MTX(so))
+#define	SOCK_RECVBUF_LOCK_ASSERT(so)					\
+	mtx_assert(SOCK_RECVBUF_MTX(so), MA_OWNED)
+#define	SOCK_RECVBUF_UNLOCK_ASSERT(so)					\
+	mtx_assert(SOCK_RECVBUF_MTX(so), MA_NOTOWNED)
+
+#define	SOCK_SENDBUF_MTX(so)						\
+	(&(so)->so_snd_mtx)
+#define	SOCK_SENDBUF_LOCK(so)						\
+	mtx_lock(SOCK_SENDBUF_MTX(so))
+#define	SOCK_SENDBUF_UNLOCK(so)						\
+	mtx_unlock(SOCK_SENDBUF_MTX(so))
+#define	SOCK_SENDBUF_LOCK_ASSERT(so)					\
+	mtx_assert(SOCK_SENDBUF_MTX(so), MA_OWNED)
+#define	SOCK_SENDBUF_UNLOCK_ASSERT(so)					\
+	mtx_assert(SOCK_SENDBUF_MTX(so), MA_NOTOWNED)
+
 /*
  * Macros for sockets and socket buffering.
  */