Introduce a netisr to deliver kernel-generated routing, avoiding

recursive entering of the socket code from the routing code:

- Modify rt_dispatch() to bundle up the sockaddr family, if any,
  associated with a pending mbuf to dispatch to routing sockets, in
  an m_tag on the mbuf.

- Allocate NETISR_ROUTE for use by routing sockets.

- Introduce rtsintrq, an ifqueue to be used by the netisr, and
  introduce rts_input(), a function to unbundle the tagged sockaddr
  and inject the mbuf and address into raw_input(), which previously
  occurred in rt_dispatch().

- Introduce rts_init() to initialize rtsintrq, its mutex, and
  register the netisr.  Perform this at the same point in system
  initialization as setup of the domains.

This change introduces asynchrony between the generation of a
pending routing socket message and delivery to sockets for use
by userspace.  It avoids socket->routing->rtsock->socket use and
helps to avoid lock order reversals between the routing code and
socket code (in particular, raw socket control blocks), as route
locks are held over calls to rt_dispatch().

Reviewed by:		"George V.Neville-Neil" <gnn@neville-neil.com>
Conceptual head nod by:	sam

sys/net/netisr.h

@@ -50,6 +50,7 @@
  */
 #define	NETISR_POLL	0		/* polling callback, must be first */
 #define	NETISR_IP	2		/* same as AF_INET */
+#define	NETISR_ROUTE	14		/* routing socket */
 #define	NETISR_AARP	15		/* Appletalk ARP */
 #define	NETISR_ATALK2	16		/* Appletalk phase 2 */
 #define	NETISR_ATALK1	17		/* Appletalk phase 1 */

sys/net/rtsock.c

@@ -45,6 +45,7 @@
 #include <sys/systm.h>
 
 #include <net/if.h>
+#include <net/netisr.h>
 #include <net/raw_cb.h>
 #include <net/route.h>
 
@@ -71,6 +72,8 @@ MTX_SYSINIT(rtsock, &rtsock_mtx, "rtsock route_cb lock", MTX_DEF);
 #define	RTSOCK_UNLOCK()	mtx_unlock(&rtsock_mtx)
 #define	RTSOCK_LOCK_ASSERT()	mtx_assert(&rtsock_mtx, MA_OWNED)
 
+static struct	ifqueue rtsintrq;
+
 struct walkarg {
 	int	w_tmemsize;
 	int	w_op, w_arg;
@@ -78,6 +81,7 @@ struct walkarg {
 	struct sysctl_req *w_req;
 };
 
+static void	rts_input(struct mbuf *m);
 static struct mbuf *rt_msg1(int type, struct rt_addrinfo *rtinfo);
 static int	rt_msg2(int type, struct rt_addrinfo *rtinfo,
 			caddr_t cp, struct walkarg *w);
@@ -93,6 +97,35 @@ static void	rt_getmetrics(const struct rt_metrics_lite *in,
 			struct rt_metrics *out);
 static void	rt_dispatch(struct mbuf *, const struct sockaddr *);
 
+static void
+rts_init(void)
+{
+
+	rtsintrq.ifq_maxlen = IFQ_MAXLEN;
+	mtx_init(&rtsintrq.ifq_mtx, "rts_inq", NULL, MTX_DEF);
+	netisr_register(NETISR_ROUTE, rts_input, &rtsintrq, NETISR_MPSAFE);
+}
+SYSINIT(rtsock, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, rts_init, 0)
+
+static void
+rts_input(struct mbuf *m)
+{
+	struct sockproto route_proto;
+	unsigned short *family;
+	struct m_tag *tag;
+
+	route_proto.sp_family = PF_ROUTE;
+	tag = m_tag_find(m, PACKET_TAG_RTSOCKFAM, NULL);
+	if (tag != NULL) {
+		family = (unsigned short *)(tag + 1);
+		route_proto.sp_protocol = *family;
+		m_tag_delete(m, tag);
+	} else
+		route_proto.sp_protocol = 0;
+
+	raw_input(m, &route_proto, &route_src, &route_dst);
+}
+
 /*
  * It really doesn't make any sense at all for this code to share much
  * with raw_usrreq.c, since its functionality is so restricted.  XXX
@@ -919,11 +952,26 @@ rt_ifannouncemsg(struct ifnet *ifp, int what)
 static void
 rt_dispatch(struct mbuf *m, const struct sockaddr *sa)
 {
-	struct sockproto route_proto;
+	unsigned short *family;
+	struct m_tag *tag;
 
-	route_proto.sp_family = PF_ROUTE;
-	route_proto.sp_protocol = sa ?  sa->sa_family : 0;
-	raw_input(m, &route_proto, &route_src, &route_dst);
+	/*
+	 * Preserve the family from the sockaddr, if any, in an m_tag for
+	 * use when injecting the mbuf into the routing socket buffer from
+	 * the netisr.
+	 */
+	if (sa != NULL) {
+		tag = m_tag_get(PACKET_TAG_RTSOCKFAM, sizeof(unsigned short),
+		    M_NOWAIT);
+		if (tag == NULL) {
+			m_freem(m);
+			return;
+		}
+		family = (unsigned short *)(tag + 1);
+		*family = sa ? sa->sa_family : 0;
+		m_tag_prepend(m, tag);
+	}
+	netisr_queue(NETISR_ROUTE, m);
 }
 
 /*

sys/sys/mbuf.h

@@ -631,6 +631,7 @@ struct	mbuf	*m_uiotombuf(struct uio *, int, int);
 #define	PACKET_TAG_PF_FRAGCACHE			22 /* PF fragment cached */
 #define	PACKET_TAG_PF_QID			23 /* PF ALTQ queue id */
 #define	PACKET_TAG_PF_TAG			24 /* PF tagged */
+#define	PACKET_TAG_RTSOCKFAM			25 /* rtsock sa family */
 
 /* Packet tag routines. */
 struct	m_tag	*m_tag_alloc(u_int32_t, int, int, int);