From b941b2983ad03dfae40951ce46ec04cc67729e23 Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Fri, 21 Jun 2019 07:59:11 +0200 Subject: [PATCH] env_dpdk/pci: don't hotplug devices directly on the dpdk intr thread To safely access the global pci device list on an spdk thread, we'll need not to modify this list on any other thread. When device gets hotplugged on a dpdk thread, it will be now inserted into a new global tailq that can be accessed only under g_pci_mutex. Then any subsequently called public pci function will add it to the regular device tailq. Change-Id: I9cb9d6b24fd731641fd764d0da71bedab38824c9 Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/458932 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/pci.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index ace21dff4e..0cb5fd8ab4 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -48,6 +48,9 @@ static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER; static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); +/* devices hotplugged on a dpdk thread */ +static TAILQ_HEAD(, spdk_pci_device) g_pci_hotplugged_devices = + TAILQ_HEAD_INITIALIZER(g_pci_hotplugged_devices); static TAILQ_HEAD(, spdk_pci_driver) g_pci_drivers = TAILQ_HEAD_INITIALIZER(g_pci_drivers); static int @@ -174,6 +177,7 @@ cleanup_pci_devices(void) { struct spdk_pci_device *dev, *tmp; + /* cleanup removed devices */ TAILQ_FOREACH_SAFE(dev, &g_pci_devices, internal.tailq, tmp) { if (!dev->internal.removed) { continue; @@ -183,6 +187,13 @@ cleanup_pci_devices(void) TAILQ_REMOVE(&g_pci_devices, dev, internal.tailq); free(dev); } + + /* add newly-attached devices */ + TAILQ_FOREACH_SAFE(dev, &g_pci_hotplugged_devices, internal.tailq, tmp) { + TAILQ_REMOVE(&g_pci_hotplugged_devices, dev, internal.tailq); + TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq); + spdk_vtophys_pci_device_added(dev->dev_handle); + } } void @@ -292,8 +303,7 @@ spdk_pci_device_init(struct rte_pci_driver *_drv, } pthread_mutex_lock(&g_pci_mutex); - TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq); - spdk_vtophys_pci_device_added(dev->dev_handle); + TAILQ_INSERT_TAIL(&g_pci_hotplugged_devices, dev, internal.tailq); pthread_mutex_unlock(&g_pci_mutex); return 0; } @@ -396,6 +406,10 @@ spdk_pci_device_attach(struct spdk_pci_driver *driver, driver->cb_arg = NULL; driver->cb_fn = NULL; + + pthread_mutex_lock(&g_pci_mutex); + cleanup_pci_devices(); + pthread_mutex_unlock(&g_pci_mutex); return rc == 0 ? 0 : -1; } @@ -446,6 +460,10 @@ spdk_pci_enumerate(struct spdk_pci_driver *driver, driver->cb_arg = NULL; driver->cb_fn = NULL; + + pthread_mutex_lock(&g_pci_mutex); + cleanup_pci_devices(); + pthread_mutex_unlock(&g_pci_mutex); return 0; }