MFP r274295:

* Move interface route cleanup to route.c:rt_flushifroutes()
* Convert most of "for (fibnum = 0; fibnum < rt_numfibs; fibnum++)" users
  to use new rt_foreach_fib() instead of hand-rolling cycles.

sys/net/if.c

@@ -166,7 +166,6 @@ static int	if_setflag(struct ifnet *, int, int, int *, int);
 static int	if_transmit(struct ifnet *ifp, struct mbuf *m);
 static void	if_unroute(struct ifnet *, int flag, int fam);
 static void	link_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
-static int	if_rtdel(struct radix_node *, void *);
 static int	ifhwioctl(u_long, struct ifnet *, caddr_t, struct thread *);
 static int	if_delmulti_locked(struct ifnet *, struct ifmultiaddr *, int);
 static void	do_link_state_change(void *, int);
@@ -885,8 +884,7 @@ static void
 if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
 {
 	struct ifaddr *ifa;
-	struct radix_node_head	*rnh;
-	int i, j;
+	int i;
 	struct domain *dp;
  	struct ifnet *iter;
  	int found = 0;
@@ -974,23 +972,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp)
 		}
 	}
 
-	/*
-	 * Delete all remaining routes using this interface
-	 * Unfortuneatly the only way to do this is to slog through
-	 * the entire routing table looking for routes which point
-	 * to this interface...oh well...
-	 */
-	for (i = 1; i <= AF_MAX; i++) {
-		for (j = 0; j < rt_numfibs; j++) {
-			rnh = rt_tables_get_rnh(j, i);
-			if (rnh == NULL)
-				continue;
-			RADIX_NODE_HEAD_LOCK(rnh);
-			(void) rnh->rnh_walktree(rnh, if_rtdel, ifp);
-			RADIX_NODE_HEAD_UNLOCK(rnh);
-		}
-	}
-
+	rt_flushifroutes(ifp);
 	if_delgroups(ifp);
 
 	/*
@@ -1410,49 +1392,6 @@ if_getgroupmembers(struct ifgroupreq *data)
 	return (0);
 }
 
-/*
- * Delete Routes for a Network Interface
- *
- * Called for each routing entry via the rnh->rnh_walktree() call above
- * to delete all route entries referencing a detaching network interface.
- *
- * Arguments:
- *	rn	pointer to node in the routing table
- *	arg	argument passed to rnh->rnh_walktree() - detaching interface
- *
- * Returns:
- *	0	successful
- *	errno	failed - reason indicated
- *
- */
-static int
-if_rtdel(struct radix_node *rn, void *arg)
-{
-	struct rtentry	*rt = (struct rtentry *)rn;
-	struct ifnet	*ifp = arg;
-	int		err;
-
-	if (rt->rt_ifp == ifp) {
-
-		/*
-		 * Protect (sorta) against walktree recursion problems
-		 * with cloned routes
-		 */
-		if ((rt->rt_flags & RTF_UP) == 0)
-			return (0);
-
-		err = rtrequest_fib(RTM_DELETE, rt_key(rt), rt->rt_gateway,
-				rt_mask(rt),
-				rt->rt_flags|RTF_RNH_LOCKED|RTF_PINNED,
-				(struct rtentry **) NULL, rt->rt_fibnum);
-		if (err) {
-			log(LOG_WARNING, "if_rtdel: error %d\n", err);
-		}
-	}
-
-	return (0);
-}
-
 /*
  * Return counter values from counter(9)s stored in ifnet.
  */

sys/net/route.c

@@ -139,6 +139,7 @@ static VNET_DEFINE(uma_zone_t, rtzone);		/* Routing table UMA zone. */
 static int rtrequest1_fib_change(struct radix_node_head *, struct rt_addrinfo *,
     struct rtentry **, u_int);
 static void rt_setmetrics(const struct rt_addrinfo *, struct rtentry *);
+static int rt_ifdelroute(struct rtentry *rt, void *arg);
 
 struct if_mtuinfo
 {
@@ -811,6 +812,96 @@ rtrequest_fib(int req,
 	return rtrequest1_fib(req, &info, ret_nrt, fibnum);
 }
 
+
+void
+rt_foreach_fib(int af, rt_setwarg_t *setwa_f, rt_walktree_f_t *wa_f, void *arg)
+{
+	struct radix_node_head *rnh;
+	uint32_t fibnum;
+	int i;
+
+	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+		/* Do we want some specific family? */
+		if (af != AF_UNSPEC) {
+			rnh = rt_tables_get_rnh(fibnum, af);
+			if (rnh == NULL)
+				continue;
+			if (setwa_f != NULL)
+				setwa_f(rnh, fibnum, i, arg);
+
+			RADIX_NODE_HEAD_LOCK(rnh);
+			rnh->rnh_walktree(rnh, (walktree_f_t *)wa_f, arg);
+			RADIX_NODE_HEAD_UNLOCK(rnh);
+			continue;
+		}
+
+		for (i = 1; i <= AF_MAX; i++) {
+			rnh = rt_tables_get_rnh(fibnum, i);
+			if (rnh == NULL)
+				continue;
+			if (setwa_f != NULL)
+				setwa_f(rnh, fibnum, i, arg);
+
+			RADIX_NODE_HEAD_LOCK(rnh);
+			rnh->rnh_walktree(rnh, (walktree_f_t *)wa_f, arg);
+			RADIX_NODE_HEAD_UNLOCK(rnh);
+		}
+	}
+}
+
+/*
+ * Delete Routes for a Network Interface
+ *
+ * Called for each routing entry via the rnh->rnh_walktree() call above
+ * to delete all route entries referencing a detaching network interface.
+ *
+ * Arguments:
+ *	rt	pointer to rtentry
+ *	arg	argument passed to rnh->rnh_walktree() - detaching interface
+ *
+ * Returns:
+ *	0	successful
+ *	errno	failed - reason indicated
+ */
+static int
+rt_ifdelroute(struct rtentry *rt, void *arg)
+{
+	struct ifnet	*ifp = arg;
+	int		err;
+
+	if (rt->rt_ifp != ifp)
+		return (0);
+
+	/*
+	 * Protect (sorta) against walktree recursion problems
+	 * with cloned routes
+	 */
+	if ((rt->rt_flags & RTF_UP) == 0)
+		return (0);
+
+	err = rtrequest_fib(RTM_DELETE, rt_key(rt), rt->rt_gateway,
+			rt_mask(rt),
+			rt->rt_flags | RTF_RNH_LOCKED | RTF_PINNED,
+			(struct rtentry **) NULL, rt->rt_fibnum);
+	if (err != 0)
+		log(LOG_WARNING, "rt_ifdelroute: error %d\n", err);
+
+	return (0);
+}
+
+/*
+ * Delete all remaining routes using this interface
+ * Unfortuneatly the only way to do this is to slog through
+ * the entire routing table looking for routes which point
+ * to this interface...oh well...
+ */
+void
+rt_flushifroutes(struct ifnet *ifp)
+{
+
+	rt_foreach_fib(AF_UNSPEC, NULL, rt_ifdelroute, ifp);
+}
+
 /*
  * These (questionable) definitions of apparent local variables apply
  * to the next two functions.  XXXXXX!!!

sys/net/route.h

@@ -380,6 +380,11 @@ void	 rtfree(struct rtentry *);
 int	 rt_check(struct rtentry **, struct rtentry **, struct sockaddr *);
 void	rt_updatemtu(struct ifnet *);
 
+typedef int rt_walktree_f_t(struct rtentry *, void *);
+typedef void rt_setwarg_t(struct radix_node_head *, uint32_t, int, void *);
+void	rt_foreach_fib(int af, rt_setwarg_t *, rt_walktree_f_t *, void *);
+void	rt_flushifroutes(struct ifnet *ifp);
+
 /* XXX MRT COMPAT VERSIONS THAT SET UNIVERSE to 0 */
 /* Thes are used by old code not yet converted to use multiple FIBS */
 int	 rt_getifa(struct rt_addrinfo *);

sys/netinet/in_rmx.c

@@ -53,6 +53,11 @@ extern int	in_inithead(void **head, int off);
 extern int	in_detachhead(void **head, int off);
 #endif
 
+static void in_setifarnh(struct radix_node_head *rnh, uint32_t fibnum,
+    int af, void *_arg);
+static void in_rtqtimo_setrnh(struct radix_node_head *rnh, uint32_t fibnum,
+    int af, void *_arg);
+
 /*
  * Do what we need to do when inserting a route.
  */
@@ -153,10 +158,9 @@ struct in_ifadown_arg {
 };
 
 static int
-in_ifadownkill(struct radix_node *rn, void *xap)
+in_ifadownkill(struct rtentry *rt, void *xap)
 {
 	struct in_ifadown_arg *ap = xap;
-	struct rtentry *rt = (struct rtentry *)rn;
 
 	RT_LOCK(rt);
 	if (rt->rt_ifa == ap->ifa &&
@@ -189,26 +193,30 @@ in_ifadownkill(struct radix_node *rn, void *xap)
 	return 0;
 }
 
+static void
+in_setifarnh(struct radix_node_head *rnh, uint32_t fibnum, int af,
+    void *_arg)
+{
+	struct in_ifadown_arg *arg;
+
+	arg = (struct in_ifadown_arg *)_arg;
+
+	arg->rnh = rnh;
+}
+
 void
 in_ifadown(struct ifaddr *ifa, int delete)
 {
 	struct in_ifadown_arg arg;
-	struct radix_node_head *rnh;
-	int	fibnum;
 
 	KASSERT(ifa->ifa_addr->sa_family == AF_INET,
 	    ("%s: wrong family", __func__));
 
-	for ( fibnum = 0; fibnum < rt_numfibs; fibnum++) {
-		rnh = rt_tables_get_rnh(fibnum, AF_INET);
-		arg.rnh = rnh;
-		arg.ifa = ifa;
-		arg.del = delete;
-		RADIX_NODE_HEAD_LOCK(rnh);
-		rnh->rnh_walktree(rnh, in_ifadownkill, &arg);
-		RADIX_NODE_HEAD_UNLOCK(rnh);
-		ifa->ifa_flags &= ~IFA_ROUTE;		/* XXXlocking? */
-	}
+	arg.ifa = ifa;
+	arg.del = delete;
+
+	rt_foreach_fib(AF_INET, in_setifarnh, in_ifadownkill, &arg);
+	ifa->ifa_flags &= ~IFA_ROUTE;		/* XXXlocking? */
 }
 
 /*

sys/netinet6/in6_rmx.c

@@ -189,15 +189,10 @@ static VNET_DEFINE(struct callout, rtq_mtutimer);
 #define	V_rtq_mtutimer			VNET(rtq_mtutimer)
 
 static int
-in6_mtuexpire(struct radix_node *rn, void *rock)
+in6_mtuexpire(struct rtentry *rt, void *rock)
 {
-	struct rtentry *rt = (struct rtentry *)rn;
 	struct mtuex_arg *ap = rock;
 
-	/* sanity */
-	if (!rt)
-		panic("rt == NULL in in6_mtuexpire");
-
 	if (rt->rt_expire && !(rt->rt_flags & RTF_PROBEMTU)) {
 		if (rt->rt_expire <= time_uptime) {
 			rt->rt_flags |= RTF_PROBEMTU;
@@ -206,36 +201,29 @@ in6_mtuexpire(struct radix_node *rn, void *rock)
 		}
 	}
 
-	return 0;
+	return (0);
 }
 
 #define	MTUTIMO_DEFAULT	(60*1)
 
 static void
-in6_mtutimo_one(struct radix_node_head *rnh)
+in6_mtutimo_setwa(struct radix_node_head *rnh, uint32_t fibum, int af, void *_arg)
 {
-	struct mtuex_arg arg;
+	struct mtuex_arg *arg;
 
-	arg.rnh = rnh;
-	arg.nextstop = time_uptime + MTUTIMO_DEFAULT;
-	RADIX_NODE_HEAD_LOCK(rnh);
-	rnh->rnh_walktree(rnh, in6_mtuexpire, &arg);
-	RADIX_NODE_HEAD_UNLOCK(rnh);
+	arg = (struct mtuex_arg *)_arg;
+
+	arg->rnh = rnh;
 }
 
 static void
 in6_mtutimo(void *rock)
 {
 	CURVNET_SET_QUIET((struct vnet *) rock);
-	struct radix_node_head *rnh;
 	struct timeval atv;
-	u_int fibnum;
+	struct mtuex_arg arg;
 
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
-		rnh = rt_tables_get_rnh(fibnum, AF_INET6);
-		if (rnh != NULL)
-			in6_mtutimo_one(rnh);
-	}
+	rt_foreach_fib(AF_INET6, in6_mtutimo_setwa, in6_mtuexpire, &arg);
 
 	atv.tv_sec = MTUTIMO_DEFAULT;
 	atv.tv_usec = 0;

sys/netinet6/nd6_rtr.c

@@ -88,7 +88,7 @@ static void in6_init_address_ltimes(struct nd_prefix *,
 static int nd6_prefix_onlink(struct nd_prefix *);
 static int nd6_prefix_offlink(struct nd_prefix *);
 
-static int rt6_deleteroute(struct radix_node *, void *);
+static int rt6_deleteroute(struct rtentry *, void *);
 
 VNET_DECLARE(int, nd6_recalc_reachtm_interval);
 #define	V_nd6_recalc_reachtm_interval	VNET(nd6_recalc_reachtm_interval)
@@ -2067,30 +2067,19 @@ in6_init_address_ltimes(struct nd_prefix *new, struct in6_addrlifetime *lt6)
 void
 rt6_flush(struct in6_addr *gateway, struct ifnet *ifp)
 {
-	struct radix_node_head *rnh;
-	u_int fibnum;
 
 	/* We'll care only link-local addresses */
 	if (!IN6_IS_ADDR_LINKLOCAL(gateway))
 		return;
 
 	/* XXX Do we really need to walk any but the default FIB? */
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
-		rnh = rt_tables_get_rnh(fibnum, AF_INET6);
-		if (rnh == NULL)
-			continue;
-
-		RADIX_NODE_HEAD_LOCK(rnh);
-		rnh->rnh_walktree(rnh, rt6_deleteroute, (void *)gateway);
-		RADIX_NODE_HEAD_UNLOCK(rnh);
-	}
+	rt_foreach_fib(AF_INET6, NULL, rt6_deleteroute, (void *)gateway);
 }
 
 static int
-rt6_deleteroute(struct radix_node *rn, void *arg)
+rt6_deleteroute(struct rtentry *rt, void *arg)
 {
 #define SIN6(s)	((struct sockaddr_in6 *)s)
-	struct rtentry *rt = (struct rtentry *)rn;
 	struct in6_addr *gate = (struct in6_addr *)arg;
 
 	if (rt->rt_gateway == NULL || rt->rt_gateway->sa_family != AF_INET6)