* Retire abstract llentry_free() in favor of lltable_drop_entry_queue()

and explicit calls to RTENTRY_FREE_LOCKED()
* Use lltable_prefix_free() in arp_ifscrub to be consistent with nd6.
* Rename <lltable_|llt>_delete function to _delete_addr() to note that
   this function is used to external callers. Make this function maintain
   its own locking.
* Use lookup/unlink/clear call chain from internal callers instead of
    delete_addr.
* Fix LLE_DELETED flag handling

sys/net/if_llatbl.c

@@ -189,21 +189,17 @@ llentries_unlink(struct lltable *llt, struct llentries *head)
 }
 
 /*
- * Deletes an address from the address table.
- * This function is called by the timer functions
- * such as arptimer() and nd6_llinfo_timer(), and
- * the caller does the locking.
+ * Helper function user to drop all mbufs in hold queue.
  *
  * Returns the number of held packets, if any, that were dropped.
  */
 size_t
-llentry_free(struct llentry *lle)
+lltable_drop_entry_queue(struct llentry *lle)
 {
 	size_t pkts_dropped;
 	struct mbuf *next;
 
 	LLE_WLOCK_ASSERT(lle);
-	KASSERT((lle->la_flags & LLE_LINKED) == 0, ("Freeing linked lle"));
 
 	pkts_dropped = 0;
 	while ((lle->la_numheld > 0) && (lle->la_hold != NULL)) {
@@ -218,8 +214,6 @@ llentry_free(struct llentry *lle)
 		("%s: la_numheld %d > 0, pkts_droped %zd", __func__,
 		 lle->la_numheld, pkts_dropped));
 
-	LLE_FREE_LOCKED(lle);
-
 	return (pkts_dropped);
 }
 
@@ -522,9 +516,7 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 		break;
 
 	case RTM_DELETE:
-		IF_AFDATA_CFG_WLOCK(ifp);
-		error = (llt->llt_delete(llt, 0, dst));
-		IF_AFDATA_CFG_WUNLOCK(ifp);
+		error = lltable_delete_addr(llt, 0, dst);
 		return (error == 0 ? 0 : ENOENT);
 
 	default:

sys/net/if_llatbl.h

@@ -151,7 +151,7 @@ typedef	struct llentry *(llt_lookup_t)(struct lltable *, u_int flags,
     const struct sockaddr *l3addr);
 typedef	struct llentry *(llt_create_t)(struct lltable *, u_int flags,
     const struct sockaddr *l3addr);
-typedef	int (llt_delete_t)(struct lltable *, u_int flags,
+typedef	int (llt_delete_addr_t)(struct lltable *, u_int flags,
     const struct sockaddr *l3addr);
 typedef int (llt_dump_entry_t)(struct lltable *, struct llentry *,
     struct sysctl_req *);
@@ -175,7 +175,7 @@ struct lltable {
 
 	llt_lookup_t		*llt_lookup;
 	llt_create_t		*llt_create;
-	llt_delete_t		*llt_delete;
+	llt_delete_addr_t	*llt_delete_addr;
 	llt_dump_entry_t	*llt_dump_entry;
 	llt_hash_t		*llt_hash;
 	llt_match_prefix_t	*llt_match_prefix;
@@ -219,10 +219,12 @@ void		lltable_drain(int);
 #endif
 int		lltable_sysctl_dumparp(int, struct sysctl_req *);
 
-size_t		llentry_free(struct llentry *);
 struct llentry  *llentry_alloc(struct ifnet *, struct lltable *,
 		    struct sockaddr_storage *);
 
+/* helper functions */
+size_t		lltable_drop_entry_queue(struct llentry *);
+
 /*
  * Generic link layer address lookup function.
  */
@@ -243,11 +245,11 @@ lltable_create_lle(struct lltable *llt, u_int flags,
 }
 
 static __inline int
-lltable_delete_lle(struct lltable *llt, u_int flags,
+lltable_delete_addr(struct lltable *llt, u_int flags,
     const struct sockaddr *l3addr)
 {
 
-	return llt->llt_delete(llt, flags, l3addr);
+	return llt->llt_delete_addr(llt, flags, l3addr);
 }
 
 static __inline void

sys/netinet/if_ether.c

@@ -162,16 +162,19 @@ static const struct netisr_handler arp_nh = {
 void
 arp_ifscrub(struct ifnet *ifp, uint32_t addr)
 {
-	struct sockaddr_in addr4;
+	struct sockaddr_in addr4, mask4;
 
 	bzero((void *)&addr4, sizeof(addr4));
 	addr4.sin_len    = sizeof(addr4);
 	addr4.sin_family = AF_INET;
 	addr4.sin_addr.s_addr = addr;
-	IF_AFDATA_CFG_WLOCK(ifp);
-	lltable_delete_lle(LLTABLE(ifp), LLE_IFADDR,
-	    (struct sockaddr *)&addr4);
-	IF_AFDATA_CFG_WUNLOCK(ifp);
+	bzero(&mask4, sizeof(mask4));
+	mask4.sin_len    = sizeof(mask4);
+	mask4.sin_family = AF_INET;
+	mask4.sin_addr.s_addr = INADDR_ANY;
+
+	lltable_prefix_free(AF_INET, (struct sockaddr *)&addr4,
+	    (struct sockaddr *)&mask4, LLE_STATIC);
 }
 #endif
 
@@ -305,9 +308,14 @@ arp_lltable_clear_entry(struct lltable *llt, struct llentry *lle)
 		}
 	}
 
-	/* Finally, free entry */
-	pkts_dropped = llentry_free(lle);
+	lle->la_flags |= LLE_DELETED;
+
+	/* Drop hold queue */
+	pkts_dropped = lltable_drop_entry_queue(lle);
 	ARPSTAT_ADD(dropped, pkts_dropped);
+
+	/* Finally, free entry */
+	LLE_FREE_LOCKED(lle);
 }
 
 /*
@@ -1208,8 +1216,9 @@ arp_update_lle(struct arphdr *ah, struct in_addr isaddr, struct ifnet *ifp,
 void
 arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa)
 {
-	struct llentry *lle;
+	struct llentry *lle, *lle_tmp;
 	struct in_addr addr;
+	struct lltable *llt;
 
 	if (ifa->ifa_carp != NULL)
 		return;
@@ -1238,6 +1247,7 @@ arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa)
 	}
 
 	IF_AFDATA_CFG_WLOCK(ifp);
+	llt = LLTABLE(ifp);
 
 	/* Lock or new shiny lle */
 	LLE_WLOCK(lle);
@@ -1247,18 +1257,26 @@ arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa)
 	 * Instead of dealing with callouts/flags/etc we simply
 	 * delete it and add new one.
 	 */
-	lltable_delete_lle(LLTABLE(ifp), LLE_IFADDR,
+	lle_tmp = lltable_lookup_lle(llt, LLE_EXCLUSIVE,
 	    (struct sockaddr *)IA_SIN(ifa));
 
 	IF_AFDATA_RUN_WLOCK(ifp);
+	if (lle_tmp != NULL)
+		lltable_unlink_entry(llt, lle_tmp);
 	bcopy(IF_LLADDR(ifp), &lle->ll_addr, ifp->if_addrlen);
 	lle->la_flags |= (LLE_VALID | LLE_STATIC);
 	lle->r_flags |= RLLE_VALID;
-	lltable_link_entry(LLTABLE(ifp), lle);
+	lltable_link_entry(llt, lle);
 	IF_AFDATA_RUN_WUNLOCK(ifp);
 
 	IF_AFDATA_CFG_WUNLOCK(ifp);
+	/* XXX: eventhandler */
 	LLE_WUNLOCK(lle);
+
+	if (lle_tmp != NULL) {
+		/* XXX: eventhandler */
+		llt->llt_clear_entry(llt, lle_tmp);
+	}
 }
 
 void

sys/netinet/in.c

@@ -1137,33 +1137,38 @@ in_lltable_delete(struct lltable *llt, u_int flags,
 	struct ifnet *ifp = llt->llt_ifp;
 	struct llentry *lle;
 
-	IF_AFDATA_CFG_WLOCK_ASSERT(ifp);
+	IF_AFDATA_CFG_UNLOCK_ASSERT(ifp);
 	KASSERT(l3addr->sa_family == AF_INET,
 	    ("sin_family %d", l3addr->sa_family));
 
+	IF_AFDATA_CFG_WLOCK(ifp);
 	lle = in_lltable_find_dst(llt, sin->sin_addr);
 	if (lle == NULL) {
+		IF_AFDATA_CFG_WUNLOCK(ifp);
 #ifdef DIAGNOSTIC
-		log(LOG_INFO, "interface address is missing from cache = %p  in delete\n", lle);
+		log(LOG_INFO, "interface address is missing from cache = %p\n",
+		    lle);
 #endif
 		return (ENOENT);
 	}
 
-	if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) {
-		LLE_WLOCK(lle);
-		lle->la_flags |= LLE_DELETED;
-		EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
-		IF_AFDATA_RUN_WLOCK(ifp);
-		lltable_unlink_entry(llt, lle);
-		IF_AFDATA_RUN_WUNLOCK(ifp);
+	/* Skipping LLE_IFADDR record */
+	if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) {
+		IF_AFDATA_CFG_WUNLOCK(ifp);
+		return (0);
+	}
+
+	LLE_WLOCK(lle);
+	IF_AFDATA_RUN_WLOCK(ifp);
+	lltable_unlink_entry(llt, lle);
+	IF_AFDATA_RUN_WUNLOCK(ifp);
+	IF_AFDATA_CFG_WUNLOCK(ifp);
+
+	EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
 #ifdef DIAGNOSTIC
 		log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
 #endif
-		if ((lle->la_flags & (LLE_STATIC | LLE_IFADDR)) == LLE_STATIC)
-			llentry_free(lle);
-		else
-			LLE_WUNLOCK(lle);
-	}
+	llt->llt_clear_entry(llt, lle);
 
 	return (0);
 }
@@ -1304,7 +1309,7 @@ in_domifattach(struct ifnet *ifp)
 
 	llt->llt_lookup = in_lltable_lookup;
 	llt->llt_create = in_lltable_create;
-	llt->llt_delete = in_lltable_delete;
+	llt->llt_delete_addr = in_lltable_delete;
 	llt->llt_dump_entry = in_lltable_dump_entry;
 	llt->llt_hash = in_lltable_hash;
 	llt->llt_clear_entry = arp_lltable_clear_entry;

sys/netinet6/in6.c

@@ -2173,31 +2173,40 @@ in6_lltable_delete(struct lltable *llt, u_int flags,
 	const struct sockaddr *l3addr)
 {
 	const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
+	struct ifnet *ifp;
 	struct llentry *lle;
 
-	IF_AFDATA_CFG_WLOCK_ASSERT(llt->llt_ifp);
+	ifp = llt->llt_ifp;
+
+	IF_AFDATA_CFG_UNLOCK_ASSERT(ifp);
 	KASSERT(l3addr->sa_family == AF_INET6,
 	    ("sin_family %d", l3addr->sa_family));
 
+	IF_AFDATA_CFG_WLOCK(ifp);
 	lle = in6_lltable_find_dst(llt, &sin6->sin6_addr);
 
-	if (lle == NULL)
+	if (lle == NULL) {
+		IF_AFDATA_CFG_WUNLOCK(ifp);
 		return (ENOENT);
-
-	if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) {
-		LLE_WLOCK(lle);
-		lle->la_flags |= LLE_DELETED;
-		IF_AFDATA_RUN_WLOCK(llt->llt_ifp);
-		lltable_unlink_entry(llt, lle);
-		IF_AFDATA_RUN_WUNLOCK(llt->llt_ifp);
-#ifdef DIAGNOSTIC
-		log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
-#endif
-		if ((lle->la_flags & (LLE_STATIC | LLE_IFADDR)) == LLE_STATIC)
-			llentry_free(lle);
-		else
-			LLE_WUNLOCK(lle);
 	}
+
+	/* Skipping LLE_IFADDR record */
+	if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) {
+		IF_AFDATA_CFG_WUNLOCK(ifp);
+		return (0);
+	}
+
+	LLE_WLOCK(lle);
+	IF_AFDATA_RUN_WLOCK(ifp);
+	lltable_unlink_entry(llt, lle);
+	IF_AFDATA_RUN_WUNLOCK(ifp);
+	IF_AFDATA_CFG_WUNLOCK(ifp);
+
+#ifdef DIAGNOSTIC
+	log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
+#endif
+	EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
+	llt->llt_clear_entry(llt, lle);
 	
 	return (0);
 }
@@ -2365,7 +2374,7 @@ in6_domifattach(struct ifnet *ifp)
 
 	llt->llt_lookup = in6_lltable_lookup;
 	llt->llt_create = in6_lltable_create;
-	llt->llt_delete = in6_lltable_delete;
+	llt->llt_delete_addr = in6_lltable_delete;
 	llt->llt_dump_entry = in6_lltable_dump_entry;
 	llt->llt_hash = in6_lltable_hash;
 	llt->llt_clear_entry = nd6_lltable_clear_entry;

sys/netinet6/nd6.c

@@ -1081,7 +1081,7 @@ nd6_free(struct llentry *ln, int gc)
 	if ((ln->la_flags & LLE_DELETED) != 0) {
 		/* Unlinked entry. Stop timer/callout. */
 		nd6_llinfo_settimer_locked(ln, -1);
-		llentry_free(ln);
+		LLE_FREE_LOCKED(ln);
 		return;
 	}
 
@@ -1132,8 +1132,13 @@ nd6_lltable_clear_entry(struct lltable *llt, struct llentry *ln)
 	/* Check if default router needs to be recalculated */
 	nd6_check_recalc_defrtr(llt, ln);
 
+	/* Drop hold queue */
+	lltable_drop_entry_queue(ln);
+
+	ln->la_flags |= LLE_DELETED;
+
 	/* Finally, free entry */
-	llentry_free(ln);
+	LLE_FREE_LOCKED(ln);
 }
 
 /*
@@ -2460,7 +2465,8 @@ int
 nd6_add_ifa_lle(struct in6_ifaddr *ia)
 {
 	struct ifnet *ifp;
-	struct llentry *ln;
+	struct llentry *ln, *ln_tmp;
+	struct lltable *llt;
 
 	ifp = ia->ia_ifa.ifa_ifp;
 	ia->ia_ifa.ifa_rtrequest = nd6_rtrequest;
@@ -2476,20 +2482,36 @@ nd6_add_ifa_lle(struct in6_ifaddr *ia)
 	ln->ln_state = ND6_LLINFO_REACHABLE;
 
 	IF_AFDATA_CFG_WLOCK(ifp);
+	llt = LLTABLE6(ifp);
 	/* Lock or new shiny lle */
 	LLE_WLOCK(ln);
 
-	lltable_delete_lle(LLTABLE6(ifp), LLE_IFADDR,
+	/*
+	 * Check if we already have some corresponding entry.
+	 * Instead of dealing with callouts/flags/etc we simply
+	 * delete it and add new one.
+	 */
+	ln_tmp = lltable_lookup_lle(llt, LLE_EXCLUSIVE,
 	    (struct sockaddr *)&ia->ia_addr);
 
 	bcopy(IF_LLADDR(ifp), &ln->ll_addr, ifp->if_addrlen);
 	/* Finally, link our lle to the list */
 	IF_AFDATA_RUN_WLOCK(ifp);
-	lltable_link_entry(LLTABLE6(ifp), ln);
+	if (ln_tmp != NULL)
+		lltable_unlink_entry(llt, ln_tmp);
+	lltable_link_entry(llt, ln);
 	IF_AFDATA_RUN_WUNLOCK(ifp);
 	IF_AFDATA_CFG_WUNLOCK(ifp);
 
+	/* XXX: event handler? */
 	LLE_WUNLOCK(ln);
+
+	if (ln_tmp != NULL) {
+		/* XXX: event handler ? */
+		llt->llt_clear_entry(llt, ln_tmp);
+	}
+
+
 	in6_newaddrmsg(ia, RTM_ADD);
 	return (0);
 }