lib/vmd: detach devices during shutdown

Added spdk_vmd_fini(), which detaches all PCI devices acquired by the
VMD subsystem.

Fixes #1148

Change-Id: I43218ef5f9a764546b655c28688897fb91b779cb
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/482852
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## v20.04: (Upcoming Release)
 
+### vmd
+A new function, `spdk_vmd_fini`, has been added. It releases all resources acquired by the VMD
+library through the `spdk_vmd_init` call.
+
 ## v20.01
 
 ### bdev

app/spdk_lspci/spdk_lspci.c

@@ -117,5 +117,7 @@ main(int argc, char **argv)
 		dev = spdk_pci_get_next_device(dev);
 	}
 
+	spdk_vmd_fini();
+
 	return 0;
 }

examples/nvme/fio_plugin/fio_plugin.c

@@ -934,6 +934,7 @@ static void spdk_fio_cleanup(struct thread_data *td)
 {
 	struct spdk_fio_thread	*fio_thread = td->io_ops_data;
 	struct spdk_fio_qpair	*fio_qpair, *fio_qpair_tmp;
+	struct spdk_fio_options *fio_options = td->eo;
 
 	fio_qpair = fio_thread->fio_qpair;
 	while (fio_qpair != NULL) {
@@ -959,6 +960,10 @@ static void spdk_fio_cleanup(struct thread_data *td)
 			fio_ctrlr = fio_ctrlr_tmp;
 		}
 		g_ctrlr = NULL;
+
+		if (fio_options->enable_vmd) {
+			spdk_vmd_fini();
+		}
 	}
 	pthread_mutex_unlock(&g_mutex);
 	if (!g_ctrlr) {

examples/nvme/hello_world/hello_world.c

@@ -426,5 +426,9 @@ int main(int argc, char **argv)
 	printf("Initialization complete.\n");
 	hello_world();
 	cleanup();
+	if (g_vmd) {
+		spdk_vmd_fini();
+	}
+
 	return 0;
 }

examples/nvme/identify/identify.c

@@ -1817,5 +1817,9 @@ int main(int argc, char **argv)
 		fprintf(stderr, "No NVMe controllers found.\n");
 	}
 
+	if (g_vmd) {
+		spdk_vmd_fini();
+	}
+
 	return 0;
 }

examples/nvme/perf/perf.c

@@ -1967,6 +1967,10 @@ unregister_controllers(void)
 		free(entry);
 		entry = next;
 	}
+
+	if (g_vmd) {
+		spdk_vmd_fini();
+	}
 }
 
 static int

examples/vmd/led/led.c

@@ -208,5 +208,7 @@ main(int argc, char **argv)
 		}
 	}
 
+	spdk_vmd_fini();
+
 	return status;
 }

examples/vmd/lsvmd/lsvmd.c

@@ -104,5 +104,7 @@ int main(int argc, char **argv)
 		}
 	}
 
+	spdk_vmd_fini();
+
 	return rc;
 }

include/spdk/vmd.h

@@ -57,6 +57,11 @@ extern "C" {
  */
 int spdk_vmd_init(void);
 
+/**
+ * Release any resources allocated by the VMD library via spdk_vmd_init().
+ */
+void spdk_vmd_fini(void);
+
 /**
  * Returns a list of nvme devices found on the given vmd pci BDF.
  *

lib/vmd/vmd.c

@@ -681,7 +681,7 @@ vmd_create_new_bus(struct vmd_pci_bus *parent, struct vmd_pci_device *bridge, ui
 	bridge->pci.addr.bus = new_bus->bus_number;
 	bridge->pci.addr.dev = bridge->devfn;
 	bridge->pci.addr.func = 0;
-	bridge->pci.addr.domain = parent->vmd->pci.addr.domain;
+	bridge->pci.addr.domain = parent->vmd->pci->addr.domain;
 
 	return new_bus;
 }
@@ -911,7 +911,7 @@ vmd_dev_init(struct vmd_pci_device *dev)
 	if (vmd_is_supported_device(dev)) {
 		spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->pci.addr);
 		SPDK_DEBUGLOG(SPDK_LOG_VMD, "Initalizing NVMe device at %s\n", bdf);
-		dev->pci.parent = &dev->bus->vmd->pci;
+		dev->pci.parent = dev->bus->vmd->pci;
 		spdk_pci_hook_device(spdk_pci_nvme_get_driver(), &dev->pci);
 	}
 }
@@ -1118,8 +1118,8 @@ vmd_scan_pcibus(struct vmd_pci_bus *bus)
 	SPDK_DEBUGLOG(SPDK_LOG_VMD, "VMD scan found %u END DEVICES\n", g_end_device_count);
 
 	SPDK_INFOLOG(SPDK_LOG_VMD, "PCIe devices attached to VMD %04x:%02x:%02x:%x...\n",
-		     bus->vmd->pci.addr.domain, bus->vmd->pci.addr.bus,
-		     bus->vmd->pci.addr.dev, bus->vmd->pci.addr.func);
+		     bus->vmd->pci->addr.domain, bus->vmd->pci->addr.bus,
+		     bus->vmd->pci->addr.dev, bus->vmd->pci->addr.func);
 
 	TAILQ_FOREACH(bus_entry, &bus->vmd->bus_list, tailq) {
 		if (bus_entry->self != NULL) {
@@ -1165,7 +1165,7 @@ vmd_enumerate_devices(struct vmd_adapter *vmd)
 	vmd->vmd_bus.vmd = vmd;
 	vmd->vmd_bus.secondary_bus = vmd->vmd_bus.subordinate_bus = 0;
 	vmd->vmd_bus.primary_bus = vmd->vmd_bus.bus_number = 0;
-	vmd->vmd_bus.domain = vmd->pci.addr.domain;
+	vmd->vmd_bus.domain = vmd->pci->addr.domain;
 
 	return vmd_scan_pcibus(&vmd->vmd_bus);
 }
@@ -1213,7 +1213,7 @@ vmd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
 
 	/* map vmd bars */
 	i = vmd_c->count;
-	vmd_c->vmd[i].pci = *pci_dev;
+	vmd_c->vmd[i].pci = pci_dev;
 	vmd_c->vmd[i].vmd_index = i;
 	vmd_c->vmd[i].domain =
 		(pci_dev->addr.bus << 16) | (pci_dev->addr.dev << 8) | pci_dev->addr.func;
@@ -1253,7 +1253,7 @@ spdk_vmd_pci_device_list(struct spdk_pci_addr vmd_addr, struct spdk_pci_device *
 	}
 
 	for (int i = 0; i < MAX_VMD_TARGET; ++i) {
-		if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci.addr) == 0) {
+		if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci->addr) == 0) {
 			TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) {
 				TAILQ_FOREACH(dev, &bus->dev_list, tailq) {
 					nvme_list[cnt++] = dev->pci;
@@ -1364,4 +1364,14 @@ spdk_vmd_init(void)
 	return spdk_pci_enumerate(spdk_pci_vmd_get_driver(), vmd_enum_cb, &g_vmd_container);
 }
 
+void
+spdk_vmd_fini(void)
+{
+	uint32_t i;
+
+	for (i = 0; i < g_vmd_container.count; ++i) {
+		spdk_pci_device_detach(g_vmd_container.vmd[i].pci);
+	}
+}
+
 SPDK_LOG_REGISTER_COMPONENT("vmd", SPDK_LOG_VMD)

lib/vmd/vmd.h

@@ -138,7 +138,7 @@ struct vmd_pci_device {
  * The VMD adapter
  */
 struct vmd_adapter {
-	struct spdk_pci_device pci;
+	struct spdk_pci_device *pci;
 	uint32_t domain;
 	/* physical and virtual VMD bars */
 	uint64_t cfgbar, cfgbar_size;

module/event/subsystems/vmd/vmd.c

@@ -100,6 +100,8 @@ spdk_vmd_subsystem_fini(void)
 {
 	spdk_poller_unregister(&g_hotplug_poller);
 
+	spdk_vmd_fini();
+
 	spdk_subsystem_fini_next();
 }