Add a new network interface flag, IFF_NEEDSGIANT, which will allow

device drivers to declare that the ifp->if_start() method implemented
by the driver requires Giant in order to operate correctly.

Add a 'struct task' to 'struct ifnet' that can be used to execute a
deferred ifp->if_start() in the event that if_start needs to be called
in a Giant-free environment.  To do this, introduce if_start(), a
wrapper function for ifp->if_start().  If the interface can run MPSAFE,
it directly dispatches into the interface start routine.  If it can't
run MPSAFE, we're running with debug.mpsafenet != 0, and Giant isn't
currently held, the task is queued to execute in a swi holding Giant
via if_start_deferred().

Modify if_handoff() to use if_start() instead of direct dispatch.
Modify 802.11 to use if_start() instead of direct dispatch.

This is intended to provide increased compatibility for non-MPSAFE
network device drivers in the presence of Giant-free operation via
asynchronous dispatch.  However, this commit does not mark any network
interfaces as IFF_NEEDSGIANT.

sys/net/if.c

@@ -50,6 +50,7 @@
 #include <sys/sockio.h>
 #include <sys/syslog.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 #include <sys/domain.h>
 #include <sys/jail.h>
 #include <machine/stdarg.h>
@@ -92,6 +93,7 @@ static void	if_unroute(struct ifnet *, int flag, int fam);
 static void	link_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
 static int	if_rtdel(struct radix_node *, void *);
 static int	ifhwioctl(u_long, struct ifnet *, caddr_t, struct thread *);
+static void	if_start_deferred(void *context, int pending);
 #ifdef INET6
 /*
  * XXX: declare here to avoid to include many inet6 related files..
@@ -365,6 +367,7 @@ if_attach(struct ifnet *ifp)
 	struct sockaddr_dl *sdl;
 	struct ifaddr *ifa;
 
+	TASK_INIT(&ifp->if_starttask, 0, if_start_deferred, ifp);
 	IF_AFDATA_LOCK_INIT(ifp);
 	ifp->if_afdata_initialized = 0;
 	IFNET_WLOCK();
@@ -983,6 +986,10 @@ if_qflush(struct ifaltq *ifq)
  * Handle interface watchdog timer routines.  Called
  * from softclock, we decrement timers (if set) and
  * call the appropriate interface routine on expiration.
+ *
+ * XXXRW: Note that because timeouts run with Giant, if_watchdog() is called
+ * holding Giant.  If we switch to an MPSAFE callout, we likely need to grab
+ * Giant before entering if_watchdog() on an IFF_NEEDSGIANT interface.
  */
 static void
 if_slowtimo(void *arg)
@@ -1839,5 +1846,47 @@ if_printf(struct ifnet *ifp, const char * fmt, ...)
 	return (retval);
 }
 
+/*
+ * When an interface is marked IFF_NEEDSGIANT, its if_start() routine cannot
+ * be called without Giant.  However, we often can't acquire the Giant lock
+ * at those points; instead, we run it via a task queue that holds Giant via
+ * if_start_deferred.
+ *
+ * XXXRW: We need to make sure that the ifnet isn't fully detached until any
+ * outstanding if_start_deferred() tasks that will run after the free.  This
+ * probably means waiting in if_detach().
+ */
+void
+if_start(struct ifnet *ifp)
+{
+
+	NET_ASSERT_GIANT();
+                
+        if ((ifp->if_flags & IFF_NEEDSGIANT) != 0 && debug_mpsafenet != 0) {
+                if (mtx_owned(&Giant))
+                        (*(ifp)->if_start)(ifp);
+                else
+			taskqueue_enqueue(taskqueue_swi_giant,
+			    &ifp->if_starttask);
+        } else
+                (*(ifp)->if_start)(ifp);
+}
+
+static void
+if_start_deferred(void *context, int pending)
+{
+	struct ifnet *ifp;
+
+	/*
+	 * This code must be entered with Giant, and should never run if
+	 * we're not running with debug.mpsafenet.
+	 */
+	KASSERT(debug_mpsafenet != 0, ("if_start_deferred: debug.mpsafenet"));
+	GIANT_REQUIRED;
+
+	ifp = (struct ifnet *)context;
+	(ifp->if_start)(ifp);
+}
+
 SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers");
 SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management");

sys/net/if.h

@@ -126,6 +126,7 @@ struct if_data {
 #define	IFF_PPROMISC	0x20000		/* user-requested promisc mode */
 #define	IFF_MONITOR	0x40000		/* user-requested monitor mode */
 #define	IFF_STATICARP	0x80000		/* static ARP */
+#define	IFF_NEEDSGIANT	0x100000	/* hold Giant over if_start calls */
 
 /* flags set internally only: */
 #define	IFF_CANTCHANGE \

sys/net/if_ethersubr.c

@@ -888,6 +888,8 @@ ether_ifattach(struct ifnet *ifp, const u_int8_t *llc)
 			break; 
 	if (i != ifp->if_addrlen)
 		if_printf(ifp, "Ethernet address: %6D\n", llc, ":");
+	if (debug_mpsafenet && (ifp->if_flags & IFF_NEEDSGIANT) != 0)
+		if_printf(ifp, "if_start running deferred for Giant\n");
 }
 
 /*

sys/net/if_var.h

@@ -79,6 +79,7 @@ struct	ether_header;
 #include <sys/lock.h>		/* XXX */
 #include <sys/mutex.h>		/* XXX */
 #include <sys/event.h>		/* XXX */
+#include <sys/_task.h>
 
 #define	IF_DUNIT_NONE	-1
 
@@ -191,6 +192,7 @@ struct ifnet {
 	void	*if_afdata[AF_MAX];
 	int	if_afdata_initialized;
 	struct	mtx if_afdata_mtx;
+	struct	task if_starttask;	/* task for IFF_NEEDSGIANT */
 };
 
 typedef void if_init_f_t(void *);
@@ -329,6 +331,8 @@ EVENTHANDLER_DECLARE(ifnet_departure_event, ifnet_departure_event_handler_t);
 #define	IF_HANDOFF_ADJ(ifq, m, ifp, adj)	\
 	if_handoff((struct ifqueue *)ifq, m, ifp, adj)
 
+void	if_start(struct ifnet *);
+
 static __inline int
 if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust)
 {
@@ -350,7 +354,7 @@ if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust)
 	_IF_ENQUEUE(ifq, m);
 	IF_UNLOCK(ifq);
 	if (ifp != NULL && !active)
-		(*ifp->if_start)(ifp);
+		if_start(ifp);
 	return (1);
 }
 #if 1 /* ALTQ */
@@ -474,7 +478,7 @@ do {									\
 		if (mflags & M_MCAST)					\
 			(ifp)->if_omcasts++;				\
 		if (((ifp)->if_flags & IFF_OACTIVE) == 0)		\
-			(*(ifp)->if_start)(ifp);			\
+			if_start(ifp);					\
 	}								\
 } while (0)
 

sys/net80211/ieee80211_output.c

@@ -130,7 +130,7 @@ ieee80211_mgmt_output(struct ifnet *ifp, struct ieee80211_node *ni,
 
 	IF_ENQUEUE(&ic->ic_mgtq, m);
 	ifp->if_timer = 1;
-	(*ifp->if_start)(ifp);
+	if_start(ifp);
 	return 0;
 }
 

sys/net80211/ieee80211_proto.c

@@ -513,7 +513,7 @@ ieee80211_newstate(struct ieee80211com *ic, enum ieee80211_state nstate, int mgt
 					IEEE80211_RATE2MBS(ni->ni_rates.rs_rates[ni->ni_txrate]));
 			}
 			ic->ic_mgt_timer = 0;
-			(*ifp->if_start)(ifp);
+			if_start(ifp);
 			break;
 		}
 		break;