netmap: restore hwofs and support it in iflib

Restore the hwofs functionality temporarily disabled by
7ba6ecf216 to prevent issues with iflib.
This patch brings the necessary changes to iflib to
enable howfs to allow interface restarts without
disrupting netmap applications actively using its
rings.
After this change, it becomes possible for multiple
non-cooperating netmap applications to use non-overlapping
subsets of the available netmap rings without clashing
with each other.

PR:		252453
MFC after:	1 week

sys/dev/netmap/netmap.c

@@ -640,7 +640,7 @@ void
 netmap_disable_all_rings(struct ifnet *ifp)
 {
 	if (NM_NA_VALID(ifp)) {
-		netmap_set_all_rings(NA(ifp), NM_KR_STOPPED);
+		netmap_set_all_rings(NA(ifp), NM_KR_LOCKED);
 	}
 }
 
@@ -4072,12 +4072,13 @@ netmap_transmit(struct ifnet *ifp, struct mbuf *m)
 /*
  * Reset function to be called by the driver routines when reinitializing
  * a hardware ring. The driver is in charge of locking to protect the kring
- * while this operation is being performed.
+ * while this operation is being performed. This is normally achieved by
- * This is normally done by calling netmap_disable_all_rings() before
+ * calling netmap_disable_all_rings() before triggering a reset.
- * triggering a reset.
  * If the kring is not in netmap mode, return NULL to inform the caller
  * that this is the case.
- * If the kring is in netmap mode, reset the kring indices to 0.
+ * If the kring is in netmap mode, set hwofs so that the netmap indices
+ * seen by userspace (head/cut/tail) do not change, although the internal
+ * NIC indices have been reset to 0.
  * In any case, adjust kring->nr_mode.
  */
 struct netmap_slot *
@@ -4085,6 +4086,7 @@ netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
 	u_int new_cur)
 {
 	struct netmap_kring *kring;
+	u_int new_hwtail, new_hwofs;
 
 	if (!nm_native_on(na)) {
 		nm_prdis("interface not in native netmap mode");
@@ -4094,26 +4096,45 @@ netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
 	if (tx == NR_TX) {
 		if (n >= na->num_tx_rings)
 			return NULL;
-
 		kring = na->tx_rings[n];
-
+		/*
+		 * Set hwofs to rhead, so that slots[rhead] is mapped to
+		 * the NIC internal slot 0, and thus the netmap buffer
+		 * at rhead is the next to be transmitted. Transmissions
+		 * that were pending before the reset are considered as
+		 * sent, so that we can have hwcur = rhead. All the slots
+		 * are now owned by the user, so we can also reinit hwtail.
+		 */
+		new_hwofs = kring->rhead;
+		new_hwtail = nm_prev(kring->rhead, kring->nkr_num_slots - 1);
 	} else {
 		if (n >= na->num_rx_rings)
 			return NULL;
 		kring = na->rx_rings[n];
+		/*
+		 * Set hwofs to hwtail, so that slots[hwtail] is mapped to
+		 * the NIC internal slot 0, and thus the netmap buffer
+		 * at hwtail is the next to be given to the NIC.
+		 * Unread slots (the ones in [rhead,hwtail[) are owned by
+		 * the user, and thus the caller cannot give them
+		 * to the NIC right now.
+		 */
+		new_hwofs = kring->nr_hwtail;
+		new_hwtail = kring->nr_hwtail;
 	}
 	if (kring->nr_pending_mode == NKR_NETMAP_OFF) {
 		kring->nr_mode = NKR_NETMAP_OFF;
 		return NULL;
 	}
 	if (netmap_verbose) {
-	    nm_prinf("%s, was: hc %u h %u c %u ht %u", kring->name,
+	    nm_prinf("%s, hc %u->%u, ht %u->%u, ho %u->%u", kring->name,
-	        kring->nr_hwcur, kring->ring->head,
+	        kring->nr_hwcur, kring->rhead,
-	        kring->ring->cur, kring->nr_hwtail);
+	        kring->nr_hwtail, new_hwtail,
+		kring->nkr_hwofs, new_hwofs);
 	}
-	/* For the moment being, nkr_hwofs is not used. */
+	kring->nr_hwcur = kring->rhead;
-	kring->rhead = kring->rcur = kring->nr_hwcur = kring->nkr_hwofs = 0;
+	kring->nr_hwtail = new_hwtail;
-	kring->nr_hwtail = (tx == NR_TX) ? (kring->nkr_num_slots - 1) : 0;
+	kring->nkr_hwofs = new_hwofs;
 
 	/*
 	 * Wakeup on the individual and global selwait

sys/net/iflib.c

@@ -834,13 +834,13 @@ netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init)
 {
 	struct netmap_adapter *na = kring->na;
 	u_int const lim = kring->nkr_num_slots - 1;
-	u_int nm_i = kring->nr_hwcur;
 	struct netmap_ring *ring = kring->ring;
 	bus_dmamap_t *map;
 	struct if_rxd_update iru;
 	if_ctx_t ctx = rxq->ifr_ctx;
 	iflib_fl_t fl = &rxq->ifr_fl[0];
 	u_int nic_i_first, nic_i;
+	u_int nm_i;
 	int i, n;
 #if IFLIB_DEBUG_COUNTERS
 	int rf_count = 0;
@@ -849,31 +849,42 @@ netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init)
 	/*
 	 * This function is used both at initialization and in rxsync.
 	 * At initialization we need to prepare (with isc_rxd_refill())
-	 * all the (N) netmap buffers in the ring, in such a way to keep
+	 * all the netmap buffers currently owned by the kernel, in
-	 * fl->ifl_pidx and kring->nr_hwcur in sync (except for
+	 * such a way to keep fl->ifl_pidx and kring->nr_hwcur in sync
-	 * kring->nkr_hwofs); at rxsync time, both indexes point to the
+	 * (except for kring->nkr_hwofs). These may be less than
-	 * next buffer to be refilled.
+	 * kring->nkr_num_slots if netmap_reset() was called while
+	 * an application using the kring that still owned some
+	 * buffers.
+	 * At rxsync time, both indexes point to the next buffer to be
+	 * refilled.
 	 * In any case we publish (with isc_rxd_flush()) up to
 	 * (fl->ifl_pidx - 1) % N (included), to avoid the NIC tail/prod
 	 * pointer to overrun the head/cons pointer, although this is
 	 * not necessary for some NICs (e.g. vmx).
 	 */
-	if (__predict_false(init))
+	if (__predict_false(init)) {
-		n = kring->nkr_num_slots;
+		n = kring->nkr_num_slots - nm_kr_rxspace(kring);
-	else {
+	} else {
-		n = kring->rhead - nm_i;
+		n = kring->rhead - kring->nr_hwcur;
 		if (n == 0)
 			return (0); /* Nothing to do. */
 		if (n < 0)
 			n += kring->nkr_num_slots;
 	}
 
-	/* Start to refill from nr_hwcur, publishing n buffers. */
 	iru_init(&iru, rxq, 0 /* flid */);
 	map = fl->ifl_sds.ifsd_map;
 	nic_i = fl->ifl_pidx;
-	MPASS(!init || nic_i == 0); /* on init/reset, nic_i must be 0 */
+	nm_i = netmap_idx_n2k(kring, nic_i);
-	MPASS(nic_i == netmap_idx_k2n(kring, nm_i));
+	if (__predict_false(init)) {
+		/*
+		 * On init/reset, nic_i must be 0, and we must
+		 * start to refill from hwtail (see netmap_reset()).
+		 */
+		MPASS(nic_i == 0);
+		MPASS(nm_i == kring->nr_hwtail);
+	} else
+		MPASS(nm_i == kring->nr_hwcur);
 	DBG_COUNTER_INC(fl_refills);
 	while (n > 0) {
 #if IFLIB_DEBUG_COUNTERS
@@ -913,7 +924,10 @@ netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init)
 		ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 	}
 	fl->ifl_pidx = nic_i;
-	MPASS(!init || nic_i == 0); /* on init/reset nic_i wraps around to 0 */
+	/*
+	 * At the end of the loop we must have refilled everything
+	 * we could possibly refill.
+	 */
 	MPASS(nm_i == kring->rhead);
 	kring->nr_hwcur = nm_i;