IPv4: experimental changes to allow net 0/8, 240/4, part of 127/8

Combined changes to allow experimentation with net 0/8 (network 0),
240/4 (Experimental/"Class E"), and part of the loopback net 127/8
(all but 127.0/16).  All changes are disabled by default, and can be
enabled by the following sysctls:

    net.inet.ip.allow_net0=1
    net.inet.ip.allow_net240=1
    net.inet.ip.loopback_prefixlen=16

When enabled, the corresponding addresses can be used as normal
unicast IP addresses, both as endpoints and when forwarding.

Add descriptions of the new sysctls to inet.4.

Add <machine/param.h> to vnet.h, as CACHE_LINE_SIZE is undefined in
various C files when in.h includes vnet.h.

The proposals motivating this experimentation can be found in

    https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-0
    https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-240
    https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-127

Reviewed by:	rgrimes, pauamma_gundo.com; previous versions melifaro, glebius
Differential Revision: https://reviews.freebsd.org/D35741

share/man/man4/inet.4

@@ -284,6 +284,17 @@ Integer: maximum number of fragments the host will accept and hold
 in the reassembly queue for a packet.
 0 means that the host will not accept any fragmented packets for the VNET.
 This is a per-VNET limit.
+.It Va ip.allow_net0
+Boolean: allow experimental use of addresses in 0.0.0.0/8 as endpoints,
+and allow forwarding of packets with these addresses.
+.It Va ip.allow_net240
+Boolean: allow experimental use of addresses in 240.0.0.0/4 as endpoints,
+and allow forwarding of packets with these addresses.
+.It Va ip.loopback_prefixlen
+Integer: prefix length of the address space reserved for loopback purposes.
+The default is 8, meaning that 127.0.0.0/8 is reserved for loopback,
+and cannot be sent, received, or forwarded on a non-loopback interface.
+Use of other values is experimental.
 .El
 .Sh SEE ALSO
 .Xr ioctl 2 ,

sys/net/vnet.h

@@ -65,6 +65,7 @@
  * as required for libkvm.
  */
 #if defined(_KERNEL) || defined(_WANT_VNET)
+#include <machine/param.h>	/* for CACHE_LINE_SIZE */
 #include <sys/queue.h>
 
 struct vnet {

sys/netinet/in.c

@@ -97,6 +97,28 @@ SYSCTL_BOOL(_net_inet_ip, OID_AUTO, broadcast_lowest, CTLFLAG_VNET | CTLFLAG_RW,
 	&VNET_NAME(broadcast_lowest), 0,
 	"Treat lowest address on a subnet (host 0) as broadcast");
 
+VNET_DEFINE(bool, ip_allow_net240) = false;
+#define	V_ip_allow_net240		VNET(ip_allow_net240)
+SYSCTL_BOOL(_net_inet_ip, OID_AUTO, allow_net240,
+	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_allow_net240), 0,
+	"Allow use of Experimental addresses, aka Class E (240/4)");
+/* see https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-240 */
+
+VNET_DEFINE(bool, ip_allow_net0) = false;
+SYSCTL_BOOL(_net_inet_ip, OID_AUTO, allow_net0,
+	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip_allow_net0), 0,
+	"Allow use of addresses in network 0/8");
+/* see https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-0 */
+
+VNET_DEFINE(uint32_t, in_loopback_mask) = IN_LOOPBACK_MASK_DFLT;
+#define	V_in_loopback_mask	VNET(in_loopback_mask)
+static int sysctl_loopback_prefixlen(SYSCTL_HANDLER_ARGS);
+SYSCTL_PROC(_net_inet_ip, OID_AUTO, loopback_prefixlen,
+	CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
+	NULL, 0, sysctl_loopback_prefixlen, "I",
+	"Prefix length of address space reserved for loopback");
+/* see https://datatracker.ietf.org/doc/draft-schoen-intarea-unicast-127 */
+
 VNET_DECLARE(struct inpcbinfo, ripcbinfo);
 #define	V_ripcbinfo			VNET(ripcbinfo)
 
@@ -251,12 +273,36 @@ in_canforward(struct in_addr in)
 {
 	u_long i = ntohl(in.s_addr);
 
-	if (IN_EXPERIMENTAL(i) || IN_MULTICAST(i) || IN_LINKLOCAL(i) ||
+	if (IN_MULTICAST(i) || IN_LINKLOCAL(i) || IN_LOOPBACK(i))
-	    IN_ZERONET(i) || IN_LOOPBACK(i))
+		return (0);
+	if (IN_EXPERIMENTAL(i) && !V_ip_allow_net240)
+		return (0);
+	if (IN_ZERONET(i) && !V_ip_allow_net0)
 		return (0);
 	return (1);
 }
 
+/*
+ * Sysctl to manage prefix of reserved loopback network; translate
+ * to/from mask.  The mask is always contiguous high-order 1 bits
+ * followed by all 0 bits.
+ */
+static int
+sysctl_loopback_prefixlen(SYSCTL_HANDLER_ARGS)
+{
+	int error, preflen;
+
+	/* ffs is 1-based; compensate. */
+	preflen = 33 - ffs(V_in_loopback_mask);
+	error = sysctl_handle_int(oidp, &preflen, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	if (preflen < 8 || preflen > 32)
+		return (EINVAL);
+	V_in_loopback_mask = 0xffffffff << (32 - preflen);
+	return (0);
+}
+
 /*
  * Trim a mask in a sockaddr
  */

sys/netinet/in.h

@@ -383,7 +383,13 @@ __END_DECLS
 #define	IN_BADCLASS(i)		(((in_addr_t)(i) & 0xf0000000) == 0xf0000000)
 
 #define IN_LINKLOCAL(i)		(((in_addr_t)(i) & 0xffff0000) == 0xa9fe0000)
+#ifdef _KERNEL
+#define IN_LOOPBACK(i) \
+    (((in_addr_t)(i) & V_in_loopback_mask) == 0x7f000000)
+#define IN_LOOPBACK_MASK_DFLT	0xff000000
+#else
 #define IN_LOOPBACK(i)		(((in_addr_t)(i) & 0xff000000) == 0x7f000000)
+#endif
 #define IN_ZERONET(i)		(((in_addr_t)(i) & 0xff000000) == 0)
 
 #define	IN_PRIVATE(i)	((((in_addr_t)(i) & 0xff000000) == 0x0a000000) || \
@@ -414,6 +420,18 @@ __END_DECLS
 
 #define	IN_RFC3021_MASK		((in_addr_t)0xfffffffe)
 
+#ifdef _KERNEL
+#include <net/vnet.h>
+
+VNET_DECLARE(bool, ip_allow_net0);
+VNET_DECLARE(bool, ip_allow_net240);
+/* Address space reserved for loopback */
+VNET_DECLARE(uint32_t, in_loopback_mask);
+#define	V_ip_allow_net0		VNET(ip_allow_net0)
+#define	V_ip_allow_net240	VNET(ip_allow_net240)
+#define	V_in_loopback_mask	VNET(in_loopback_mask)
+#endif
+
 /*
  * Options for use with [gs]etsockopt at the IP level.
  * First word of comment is data type; bool is stored in int.

sys/netinet/ip_icmp.c

@@ -775,8 +775,8 @@ icmp_reflect(struct mbuf *m)
 	NET_EPOCH_ASSERT();
 
 	if (IN_MULTICAST(ntohl(ip->ip_src.s_addr)) ||
-	    IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) ||
+	    (IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) && !V_ip_allow_net240) ||
-	    IN_ZERONET(ntohl(ip->ip_src.s_addr)) ) {
+	    (IN_ZERONET(ntohl(ip->ip_src.s_addr)) && !V_ip_allow_net0) ) {
 		m_freem(m);	/* Bad return address */
 		ICMPSTAT_INC(icps_badaddr);
 		goto done;	/* Ip_output() will check for broadcast */