Refactor rt_addrmsg() and rt_routemsg().

Summary:
* Refactor rt_addrmsg(): make V_rt_add_addr_allfibs decision locally.
* Fix rt_routemsg() and multipath by accepting nexthop instead of interface pointer.
* Refactor rtsock_routemsg(): avoid accessing rtentry fields directly.
* Simplify in_addprefix() by moving prefix search to a separate  function.

Reviewers: #network

Subscribers: imp, ae, bz

Differential Revision: https://reviews.freebsd.org/D28011

sys/net/route.c

@@ -710,10 +710,13 @@ rt_addrmsg(int cmd, struct ifaddr *ifa, int fibnum)
 
 	KASSERT(cmd == RTM_ADD || cmd == RTM_DELETE,
 	    ("unexpected cmd %d", cmd));
-	KASSERT(fibnum == RT_ALL_FIBS || (fibnum >= 0 && fibnum < rt_numfibs),
+	KASSERT((fibnum >= 0 && fibnum < rt_numfibs),
 	    ("%s: fib out of range 0 <=%d<%d", __func__, fibnum, rt_numfibs));
 
 	EVENTHANDLER_DIRECT_INVOKE(rt_addrmsg, ifa, cmd);
+
+	if (V_rt_add_addr_allfibs)
+		fibnum = RT_ALL_FIBS;
 	return (rtsock_addrmsg(cmd, ifa, fibnum));
 }
 
@@ -721,13 +724,13 @@ rt_addrmsg(int cmd, struct ifaddr *ifa, int fibnum)
  * Announce kernel-originated route addition/removal to rtsock based on @rt data.
  * cmd: RTM_ cmd
  * @rt: valid rtentry
- * @ifp: target route interface
+ * @nh: nhop object to announce
  * @fibnum: fib id or RT_ALL_FIBS
  *
  * Returns 0 on success.
  */
 int
-rt_routemsg(int cmd, struct rtentry *rt, struct ifnet *ifp, int rti_addrs,
+rt_routemsg(int cmd, struct rtentry *rt, struct nhop_object *nh,
     int fibnum)
 {
 
@@ -739,7 +742,7 @@ rt_routemsg(int cmd, struct rtentry *rt, struct ifnet *ifp, int rti_addrs,
 
 	KASSERT(rt_key(rt) != NULL, (":%s: rt_key must be supplied", __func__));
 
-	return (rtsock_routemsg(cmd, rt, ifp, 0, fibnum));
+	return (rtsock_routemsg(cmd, rt, nh, fibnum));
 }
 
 /*
@@ -775,16 +778,16 @@ rt_newaddrmsg_fib(int cmd, struct ifaddr *ifa, struct rtentry *rt, int fibnum)
 
 	KASSERT(cmd == RTM_ADD || cmd == RTM_DELETE,
 		("unexpected cmd %u", cmd));
-	KASSERT(fibnum == RT_ALL_FIBS || (fibnum >= 0 && fibnum < rt_numfibs),
+	KASSERT((fibnum >= 0 && fibnum < rt_numfibs),
 	    ("%s: fib out of range 0 <=%d<%d", __func__, fibnum, rt_numfibs));
 
 	if (cmd == RTM_ADD) {
 		rt_addrmsg(cmd, ifa, fibnum);
 		if (rt != NULL)
-			rt_routemsg(cmd, rt, ifa->ifa_ifp, 0, fibnum);
+			rt_routemsg(cmd, rt, nhop_select(rt->rt_nhop, 0), fibnum);
 	} else {
 		if (rt != NULL)
-			rt_routemsg(cmd, rt, ifa->ifa_ifp, 0, fibnum);
+			rt_routemsg(cmd, rt, nhop_select(rt->rt_nhop, 0), fibnum);
 		rt_addrmsg(cmd, ifa, fibnum);
 	}
 }

sys/net/route.h

@@ -417,7 +417,7 @@ void	 rt_missmsg(int, struct rt_addrinfo *, int, int);
 void	 rt_missmsg_fib(int, struct rt_addrinfo *, int, int, int);
 void	 rt_newaddrmsg_fib(int, struct ifaddr *, struct rtentry *, int);
 int	 rt_addrmsg(int, struct ifaddr *, int);
-int	 rt_routemsg(int, struct rtentry *, struct ifnet *ifp, int, int);
+int	 rt_routemsg(int, struct rtentry *, struct nhop_object *, int);
 int	 rt_routemsg_info(int, struct rt_addrinfo *, int);
 void	 rt_newmaddrmsg(int, struct ifmultiaddr *);
 void 	 rt_maskedcopy(struct sockaddr *, struct sockaddr *, struct sockaddr *);
@@ -425,10 +425,6 @@ struct rib_head *rt_table_init(int, int, u_int);
 void	rt_table_destroy(struct rib_head *);
 u_int	rt_tables_get_gen(uint32_t table, sa_family_t family);
 
-int	rtsock_addrmsg(int, struct ifaddr *, int);
-int	rtsock_routemsg(int, struct rtentry *, struct ifnet *ifp, int, int);
-int	rtsock_routemsg_info(int, struct rt_addrinfo *, int);
-
 struct sockaddr *rtsock_fix_netmask(const struct sockaddr *dst,
 	    const struct sockaddr *smask, struct sockaddr_storage *dmask);
 

sys/net/route/route_ctl.c

@@ -244,6 +244,16 @@ rt_is_host(const struct rtentry *rt)
 	return (rt->rte_flags & RTF_HOST);
 }
 
+sa_family_t
+rt_get_family(const struct rtentry *rt)
+{
+	const struct sockaddr *dst;
+
+	dst = (const struct sockaddr *)rt_key_const(rt);
+
+	return (dst->sa_family);
+}
+
 /*
  * Returns pointer to nexthop or nexthop group
  * associated with @rt
@@ -1322,11 +1332,10 @@ rib_walk_del(u_int fibnum, int family, rib_filter_f_t *filter_f, void *arg, bool
 				nhg = (struct nhgrp_object *)nh;
 				wn = nhgrp_get_nhops(nhg, &num_nhops);
 				for (int i = 0; i < num_nhops; i++)
-					rt_routemsg(RTM_DELETE, rt,
+					rt_routemsg(RTM_DELETE, rt, wn[i].nh, fibnum);
-					    wn[i].nh->nh_ifp, 0, fibnum);
 			} else
 #endif
-			rt_routemsg(RTM_DELETE, rt, nh->nh_ifp, 0, fibnum);
+			rt_routemsg(RTM_DELETE, rt, nh, fibnum);
 		}
 		rtfree(rt);
 	}

sys/net/route/route_ctl.h

@@ -100,6 +100,7 @@ const struct rtentry *rib_lookup_lpm(uint32_t fibnum, int family,
 
 /* rtentry accessors */
 bool rt_is_host(const struct rtentry *rt);
+sa_family_t rt_get_family(const struct rtentry *);
 struct nhop_object *rt_get_raw_nhop(const struct rtentry *rt);
 #ifdef INET
 struct in_addr;

sys/net/route/route_var.h

@@ -306,6 +306,13 @@ void nhgrp_ref_object(struct nhgrp_object *nhg);
 uint32_t nhgrp_get_idx(const struct nhgrp_object *nhg);
 void nhgrp_free(struct nhgrp_object *nhg);
 
+/* rtsock */
+int rtsock_routemsg(int cmd, struct rtentry *rt, struct nhop_object *nh,
+    int fibnum);
+int rtsock_routemsg_info(int cmd, struct rt_addrinfo *info, int fibnum);
+int rtsock_addrmsg(int cmd, struct ifaddr *ifa, int fibnum);
+
+
 /* lookup_framework.c */
 void fib_grow_rtables(uint32_t new_num_tables);
 int fib_select_algo_initial(struct rib_head *rh);

sys/net/rtsock.c

@@ -1591,23 +1591,25 @@ rtsock_addrmsg(int cmd, struct ifaddr *ifa, int fibnum)
  * Returns 0 on success.
  */
 int
-rtsock_routemsg(int cmd, struct rtentry *rt, struct ifnet *ifp, int rti_addrs,
+rtsock_routemsg(int cmd, struct rtentry *rt, struct nhop_object *nh,
     int fibnum)
 {
-	struct sockaddr_storage ss;
+	union sockaddr_union dst, mask;
 	struct rt_addrinfo info;
-	struct nhop_object *nh;
 
 	if (V_route_cb.any_count == 0)
 		return (0);
 
-	nh = rt->rt_nhop;
+	int family = rt_get_family(rt);
+	init_sockaddrs_family(family, &dst.sa, &mask.sa);
+	export_rtaddrs(rt, &dst.sa, &mask.sa);
+
 	bzero((caddr_t)&info, sizeof(info));
-	info.rti_info[RTAX_DST] = rt_key(rt);
+	info.rti_info[RTAX_DST] = &dst.sa;
-	info.rti_info[RTAX_NETMASK] = rtsock_fix_netmask(rt_key(rt), rt_mask(rt), &ss);
+	info.rti_info[RTAX_NETMASK] = &mask.sa;
 	info.rti_info[RTAX_GATEWAY] = &nh->gw_sa;
 	info.rti_flags = rt->rte_flags | nhop_get_rtflags(nh);
-	info.rti_ifp = ifp;
+	info.rti_ifp = nh->nh_ifp;
 
 	return (rtsock_routemsg_info(cmd, &info, fibnum));
 }

sys/netinet/in.c

@@ -714,15 +714,15 @@ in_gifaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct thread *td)
 	    ? RTF_HOST : 0)
 
 /*
- * Check if we have a route for the given prefix already or add one accordingly.
+ * Check if we have a route for the given prefix already.
  */
-int
+static bool
-in_addprefix(struct in_ifaddr *target, int flags)
+in_hasrtprefix(struct in_ifaddr *target, int flags)
 {
 	struct rm_priotracker in_ifa_tracker;
 	struct in_ifaddr *ia;
 	struct in_addr prefix, mask, p, m;
-	int error;
+	bool result = false;
 
 	if ((flags & RTF_HOST) != 0) {
 		prefix = target->ia_dstaddr.sin_addr;
@@ -758,21 +758,29 @@ in_addprefix(struct in_ifaddr *target, int flags)
 		 * interface address, we are done here.
 		 */
 		if (ia->ia_flags & IFA_ROUTE) {
-			if (V_nosameprefix) {
+			result = true;
+			break;
+		}
+	}
 	IN_IFADDR_RUNLOCK(&in_ifa_tracker);
-				return (EEXIST);
-			} else {
-				int fibnum;
 
-				fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS :
+	return (result);
-					target->ia_ifp->if_fib;
+}
-				rt_addrmsg(RTM_ADD, &target->ia_ifa, fibnum);
+
-				IN_IFADDR_RUNLOCK(&in_ifa_tracker);
+int
+in_addprefix(struct in_ifaddr *target, int flags)
+{
+	int error;
+
+	if (in_hasrtprefix(target, flags)) {
+		if (V_nosameprefix)
+			return (EEXIST);
+		else {
+			rt_addrmsg(RTM_ADD, &target->ia_ifa,
+			    target->ia_ifp->if_fib);
 			return (0);
 		}
 	}
-	}
-	IN_IFADDR_RUNLOCK(&in_ifa_tracker);
 
 	/*
 	 * No-one seem to have this prefix route, so we try to insert it.
@@ -869,11 +877,7 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 	}
 
 	if ((target->ia_flags & IFA_ROUTE) == 0) {
-		int fibnum;
+		rt_addrmsg(RTM_DELETE, &target->ia_ifa, target->ia_ifp->if_fib);
-
-		fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS :
-			target->ia_ifp->if_fib;
-		rt_addrmsg(RTM_DELETE, &target->ia_ifa, fibnum);
 
 		/*
 		 * Removing address from !IFF_UP interface or

sys/netinet6/in6.c

@@ -186,7 +186,7 @@ in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
 	if (cmd != RTM_DELETE)
 		info.rti_ifp = V_loif;
 
-	fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS : ia62ifa(ia)->ifa_ifp->if_fib;
+	fibnum = ia62ifa(ia)->ifa_ifp->if_fib;
 
 	if (cmd == RTM_ADD) {
 		rt_addrmsg(cmd, &ia->ia_ifa, fibnum);

sys/netinet6/nd6_rtr.c

@@ -698,8 +698,10 @@ defrouter_addreq(struct nd_defrouter *new)
 
 	NET_EPOCH_ASSERT();
 	error = rib_action(fibnum, RTM_ADD, &info, &rc);
-	if (rc.rc_rt != NULL)
+	if (rc.rc_rt != NULL) {
-		rt_routemsg(RTM_ADD, rc.rc_rt, new->ifp, 0, fibnum);
+		struct nhop_object *nh = nhop_select(rc.rc_nh_new, 0);
+		rt_routemsg(RTM_ADD, rc.rc_rt, nh, fibnum);
+	}
 	if (error == 0)
 		new->installed = 1;
 }
@@ -736,8 +738,10 @@ defrouter_delreq(struct nd_defrouter *dr)
 
 	NET_EPOCH_ENTER(et);
 	rib_action(fibnum, RTM_DELETE, &info, &rc);
-	if (rc.rc_rt != NULL)
+	if (rc.rc_rt != NULL) {
-		rt_routemsg(RTM_DELETE, rc.rc_rt, dr->ifp, 0, fibnum);
+		struct nhop_object *nh = nhop_select(rc.rc_nh_old, 0);
+		rt_routemsg(RTM_DELETE, rc.rc_rt, nh, fibnum);
+	}
 	NET_EPOCH_EXIT(et);
 
 	dr->installed = 0;
@@ -2077,7 +2081,8 @@ nd6_prefix_onlink_rtrequest(struct nd_prefix *pr, struct ifaddr *ifa)
 		}
 
 		pr->ndpr_stateflags |= NDPRF_ONLINK;
-		rt_routemsg(RTM_ADD, rc.rc_rt, pr->ndpr_ifp, 0, fibnum);
+		struct nhop_object *nh = nhop_select(rc.rc_nh_new, 0);
+		rt_routemsg(RTM_ADD, rc.rc_rt, nh, fibnum);
 	}
 
 	/* Return the last error we got. */
@@ -2224,7 +2229,8 @@ nd6_prefix_offlink(struct nd_prefix *pr)
 		}
 
 		/* report route deletion to the routing socket. */
-		rt_routemsg(RTM_DELETE, rc.rc_rt, ifp, 0, fibnum);
+		struct nhop_object *nh = nhop_select(rc.rc_nh_old, 0);
+		rt_routemsg(RTM_DELETE, rc.rc_rt, nh, fibnum);
 	}
 	NET_EPOCH_EXIT(et);
 	error = a_failure;