From ce6171d465e5795fc2087a1ff666fd2a80707115 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Mon, 27 Jan 2020 12:09:36 +0100 Subject: [PATCH] lib/vmd: detach devices during shutdown Added spdk_vmd_fini(), which detaches all PCI devices acquired by the VMD subsystem. Fixes #1148 Change-Id: I43218ef5f9a764546b655c28688897fb91b779cb Signed-off-by: Konrad Sztyber Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/482852 Community-CI: SPDK CI Jenkins Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris --- CHANGELOG.md | 4 ++++ app/spdk_lspci/spdk_lspci.c | 2 ++ examples/nvme/fio_plugin/fio_plugin.c | 5 +++++ examples/nvme/hello_world/hello_world.c | 4 ++++ examples/nvme/identify/identify.c | 4 ++++ examples/nvme/perf/perf.c | 4 ++++ examples/vmd/led/led.c | 2 ++ examples/vmd/lsvmd/lsvmd.c | 2 ++ include/spdk/vmd.h | 5 +++++ lib/vmd/vmd.c | 24 +++++++++++++++++------- lib/vmd/vmd.h | 2 +- module/event/subsystems/vmd/vmd.c | 2 ++ 12 files changed, 52 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d63b6bc41..1067217970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## v20.04: (Upcoming Release) +### vmd +A new function, `spdk_vmd_fini`, has been added. It releases all resources acquired by the VMD +library through the `spdk_vmd_init` call. + ## v20.01 ### bdev diff --git a/app/spdk_lspci/spdk_lspci.c b/app/spdk_lspci/spdk_lspci.c index c005857e29..b976facb5d 100644 --- a/app/spdk_lspci/spdk_lspci.c +++ b/app/spdk_lspci/spdk_lspci.c @@ -117,5 +117,7 @@ main(int argc, char **argv) dev = spdk_pci_get_next_device(dev); } + spdk_vmd_fini(); + return 0; } diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c index d42f63528b..543bb1b594 100644 --- a/examples/nvme/fio_plugin/fio_plugin.c +++ b/examples/nvme/fio_plugin/fio_plugin.c @@ -934,6 +934,7 @@ static void spdk_fio_cleanup(struct thread_data *td) { struct spdk_fio_thread *fio_thread = td->io_ops_data; struct spdk_fio_qpair *fio_qpair, *fio_qpair_tmp; + struct spdk_fio_options *fio_options = td->eo; fio_qpair = fio_thread->fio_qpair; while (fio_qpair != NULL) { @@ -959,6 +960,10 @@ static void spdk_fio_cleanup(struct thread_data *td) fio_ctrlr = fio_ctrlr_tmp; } g_ctrlr = NULL; + + if (fio_options->enable_vmd) { + spdk_vmd_fini(); + } } pthread_mutex_unlock(&g_mutex); if (!g_ctrlr) { diff --git a/examples/nvme/hello_world/hello_world.c b/examples/nvme/hello_world/hello_world.c index bf0318fd11..c6d2a892e4 100644 --- a/examples/nvme/hello_world/hello_world.c +++ b/examples/nvme/hello_world/hello_world.c @@ -426,5 +426,9 @@ int main(int argc, char **argv) printf("Initialization complete.\n"); hello_world(); cleanup(); + if (g_vmd) { + spdk_vmd_fini(); + } + return 0; } diff --git a/examples/nvme/identify/identify.c b/examples/nvme/identify/identify.c index 481fbe9ffe..c81bbac3dd 100644 --- a/examples/nvme/identify/identify.c +++ b/examples/nvme/identify/identify.c @@ -1817,5 +1817,9 @@ int main(int argc, char **argv) fprintf(stderr, "No NVMe controllers found.\n"); } + if (g_vmd) { + spdk_vmd_fini(); + } + return 0; } diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c index df6ee7e5aa..1268dc1ca4 100644 --- a/examples/nvme/perf/perf.c +++ b/examples/nvme/perf/perf.c @@ -1967,6 +1967,10 @@ unregister_controllers(void) free(entry); entry = next; } + + if (g_vmd) { + spdk_vmd_fini(); + } } static int diff --git a/examples/vmd/led/led.c b/examples/vmd/led/led.c index fb31d8093c..e014de7044 100644 --- a/examples/vmd/led/led.c +++ b/examples/vmd/led/led.c @@ -208,5 +208,7 @@ main(int argc, char **argv) } } + spdk_vmd_fini(); + return status; } diff --git a/examples/vmd/lsvmd/lsvmd.c b/examples/vmd/lsvmd/lsvmd.c index d8eef62556..b083bc049f 100644 --- a/examples/vmd/lsvmd/lsvmd.c +++ b/examples/vmd/lsvmd/lsvmd.c @@ -104,5 +104,7 @@ int main(int argc, char **argv) } } + spdk_vmd_fini(); + return rc; } diff --git a/include/spdk/vmd.h b/include/spdk/vmd.h index 45c7bce0cf..100aa893ce 100644 --- a/include/spdk/vmd.h +++ b/include/spdk/vmd.h @@ -57,6 +57,11 @@ extern "C" { */ int spdk_vmd_init(void); +/** + * Release any resources allocated by the VMD library via spdk_vmd_init(). + */ +void spdk_vmd_fini(void); + /** * Returns a list of nvme devices found on the given vmd pci BDF. * diff --git a/lib/vmd/vmd.c b/lib/vmd/vmd.c index e6200da246..7fae7d4214 100644 --- a/lib/vmd/vmd.c +++ b/lib/vmd/vmd.c @@ -681,7 +681,7 @@ vmd_create_new_bus(struct vmd_pci_bus *parent, struct vmd_pci_device *bridge, ui bridge->pci.addr.bus = new_bus->bus_number; bridge->pci.addr.dev = bridge->devfn; bridge->pci.addr.func = 0; - bridge->pci.addr.domain = parent->vmd->pci.addr.domain; + bridge->pci.addr.domain = parent->vmd->pci->addr.domain; return new_bus; } @@ -911,7 +911,7 @@ vmd_dev_init(struct vmd_pci_device *dev) if (vmd_is_supported_device(dev)) { spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->pci.addr); SPDK_DEBUGLOG(SPDK_LOG_VMD, "Initalizing NVMe device at %s\n", bdf); - dev->pci.parent = &dev->bus->vmd->pci; + dev->pci.parent = dev->bus->vmd->pci; spdk_pci_hook_device(spdk_pci_nvme_get_driver(), &dev->pci); } } @@ -1118,8 +1118,8 @@ vmd_scan_pcibus(struct vmd_pci_bus *bus) SPDK_DEBUGLOG(SPDK_LOG_VMD, "VMD scan found %u END DEVICES\n", g_end_device_count); SPDK_INFOLOG(SPDK_LOG_VMD, "PCIe devices attached to VMD %04x:%02x:%02x:%x...\n", - bus->vmd->pci.addr.domain, bus->vmd->pci.addr.bus, - bus->vmd->pci.addr.dev, bus->vmd->pci.addr.func); + bus->vmd->pci->addr.domain, bus->vmd->pci->addr.bus, + bus->vmd->pci->addr.dev, bus->vmd->pci->addr.func); TAILQ_FOREACH(bus_entry, &bus->vmd->bus_list, tailq) { if (bus_entry->self != NULL) { @@ -1165,7 +1165,7 @@ vmd_enumerate_devices(struct vmd_adapter *vmd) vmd->vmd_bus.vmd = vmd; vmd->vmd_bus.secondary_bus = vmd->vmd_bus.subordinate_bus = 0; vmd->vmd_bus.primary_bus = vmd->vmd_bus.bus_number = 0; - vmd->vmd_bus.domain = vmd->pci.addr.domain; + vmd->vmd_bus.domain = vmd->pci->addr.domain; return vmd_scan_pcibus(&vmd->vmd_bus); } @@ -1213,7 +1213,7 @@ vmd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) /* map vmd bars */ i = vmd_c->count; - vmd_c->vmd[i].pci = *pci_dev; + vmd_c->vmd[i].pci = pci_dev; vmd_c->vmd[i].vmd_index = i; vmd_c->vmd[i].domain = (pci_dev->addr.bus << 16) | (pci_dev->addr.dev << 8) | pci_dev->addr.func; @@ -1253,7 +1253,7 @@ spdk_vmd_pci_device_list(struct spdk_pci_addr vmd_addr, struct spdk_pci_device * } for (int i = 0; i < MAX_VMD_TARGET; ++i) { - if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci.addr) == 0) { + if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci->addr) == 0) { TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) { TAILQ_FOREACH(dev, &bus->dev_list, tailq) { nvme_list[cnt++] = dev->pci; @@ -1364,4 +1364,14 @@ spdk_vmd_init(void) return spdk_pci_enumerate(spdk_pci_vmd_get_driver(), vmd_enum_cb, &g_vmd_container); } +void +spdk_vmd_fini(void) +{ + uint32_t i; + + for (i = 0; i < g_vmd_container.count; ++i) { + spdk_pci_device_detach(g_vmd_container.vmd[i].pci); + } +} + SPDK_LOG_REGISTER_COMPONENT("vmd", SPDK_LOG_VMD) diff --git a/lib/vmd/vmd.h b/lib/vmd/vmd.h index ea5d41ad5e..46490a6f7b 100644 --- a/lib/vmd/vmd.h +++ b/lib/vmd/vmd.h @@ -138,7 +138,7 @@ struct vmd_pci_device { * The VMD adapter */ struct vmd_adapter { - struct spdk_pci_device pci; + struct spdk_pci_device *pci; uint32_t domain; /* physical and virtual VMD bars */ uint64_t cfgbar, cfgbar_size; diff --git a/module/event/subsystems/vmd/vmd.c b/module/event/subsystems/vmd/vmd.c index 49e3f4a7a7..6180f7f39b 100644 --- a/module/event/subsystems/vmd/vmd.c +++ b/module/event/subsystems/vmd/vmd.c @@ -100,6 +100,8 @@ spdk_vmd_subsystem_fini(void) { spdk_poller_unregister(&g_hotplug_poller); + spdk_vmd_fini(); + spdk_subsystem_fini_next(); }