iflib: Free resources in a consistent order during detach

Memory and PCI resources are freed with no particular order.  This could
cause use-after-frees when detaching following a failed attach.  For
instance, iflib_tx_structures_free() frees ctx->ifc_txqs[] but
iflib_tqg_detach() attempts to access this array. Similarly, adapter
queues gets freed by IFDI_QUEUES_FREE() but IFDI_DETACH() attempts to
access adapter queues to free PCI resources.

MFC after:	2 weeks
Sponsored by:	NetApp, Inc.
Differential Revision:	https://reviews.freebsd.org/D27634

sys/dev/e1000/if_em.c

@@ -1106,6 +1106,7 @@ em_if_attach_post(if_ctx_t ctx)
 	/* Setup OS specific network interface */
 	error = em_setup_interface(ctx);
 	if (error != 0) {
+		device_printf(adapter->dev, "Interface setup failed: %d\n", error);
 		goto err_late;
 	}
 
@@ -1123,14 +1124,10 @@ em_if_attach_post(if_ctx_t ctx)
 
 	INIT_DEBUGOUT("em_if_attach_post: end");
 
-	return (error);
+	return (0);
 
 err_late:
-	em_release_hw_control(adapter);
+	/* upon attach_post() error, iflib calls _if_detach() to free resources. */
-	em_free_pci_resources(ctx);
-	em_if_queues_free(ctx);
-	free(adapter->mta, M_DEVBUF);
-
 	return (error);
 }
 
@@ -1155,6 +1152,8 @@ em_if_detach(if_ctx_t ctx)
 	em_release_manageability(adapter);
 	em_release_hw_control(adapter);
 	em_free_pci_resources(ctx);
+	free(adapter->mta, M_DEVBUF);
+	adapter->mta = NULL;
 
 	return (0);
 }
@@ -2981,12 +2980,6 @@ em_if_queues_free(if_ctx_t ctx)
 		free(adapter->rx_queues, M_DEVBUF);
 		adapter->rx_queues = NULL;
 	}
-
-	em_release_hw_control(adapter);
-
-	if (adapter->mta != NULL) {
-		free(adapter->mta, M_DEVBUF);
-	}
 }
 
 /*********************************************************************

sys/dev/ixl/if_ixl.c

@@ -1253,7 +1253,7 @@ ixl_if_queues_free(if_ctx_t ctx)
 	struct ixl_pf *pf = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &pf->vsi;
 
-	if (!vsi->enable_head_writeback) {
+	if (vsi->tx_queues != NULL && !vsi->enable_head_writeback) {
 		struct ixl_tx_queue *que;
 		int i = 0;
 

sys/net/iflib.c

@@ -4900,7 +4900,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct
 		device_printf(dev,
 		    "Cannot use iflib with only 1 MSI-X interrupt!\n");
 		err = ENODEV;
-		goto fail_intr_free;
+		goto fail_queues;
 	}
 
 	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet);
@@ -4936,13 +4936,14 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct
 
 fail_detach:
 	ether_ifdetach(ctx->ifc_ifp);
-fail_intr_free:
-	iflib_free_intr_mem(ctx);
 fail_queues:
+	iflib_tqg_detach(ctx);
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
-	iflib_tqg_detach(ctx);
 	IFDI_DETACH(ctx);
+	IFDI_QUEUES_FREE(ctx);
+fail_intr_free:
+	iflib_free_intr_mem(ctx);
 fail_unlock:
 	CTX_UNLOCK(ctx);
 	iflib_deregister(ctx);
@@ -5139,11 +5140,12 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp,
 fail_detach:
 	ether_ifdetach(ctx->ifc_ifp);
 fail_queues:
+	iflib_tqg_detach(ctx);
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
-	iflib_tqg_detach(ctx);
 fail_iflib_detach:
 	IFDI_DETACH(ctx);
+	IFDI_QUEUES_FREE(ctx);
 fail_unlock:
 	CTX_UNLOCK(ctx);
 	iflib_deregister(ctx);
@@ -5173,6 +5175,8 @@ iflib_pseudo_deregister(if_ctx_t ctx)
 	iflib_tqg_detach(ctx);
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
+	IFDI_DETACH(ctx);
+	IFDI_QUEUES_FREE(ctx);
 
 	iflib_deregister(ctx);
 
@@ -5233,8 +5237,12 @@ iflib_device_deregister(if_ctx_t ctx)
 		led_destroy(ctx->ifc_led_dev);
 
 	iflib_tqg_detach(ctx);
+	iflib_tx_structures_free(ctx);
+	iflib_rx_structures_free(ctx);
+
 	CTX_LOCK(ctx);
 	IFDI_DETACH(ctx);
+	IFDI_QUEUES_FREE(ctx);
 	CTX_UNLOCK(ctx);
 
 	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
@@ -5242,9 +5250,6 @@ iflib_device_deregister(if_ctx_t ctx)
 
 	bus_generic_detach(dev);
 
-	iflib_tx_structures_free(ctx);
-	iflib_rx_structures_free(ctx);
-
 	iflib_deregister(ctx);
 
 	device_set_softc(ctx->ifc_dev, NULL);
@@ -5828,7 +5833,6 @@ iflib_tx_structures_free(if_ctx_t ctx)
 	}
 	free(ctx->ifc_txqs, M_IFLIB);
 	ctx->ifc_txqs = NULL;
-	IFDI_QUEUES_FREE(ctx);
 }
 
 /*********************************************************************