pci: register dpdk pci drivers right after init

DPDK 18.11+ does its best to ensure all devices are
equally attached or detached in all processes within
a shared memory group. For SPDK it means that if
a device is hotplugged in the primary, then DPDK will
automatically send an IPC hotplug request to all other
processes. Those other processes may not have the same
SPDK PCI driver registered and may fail to attach the
device. DPDK will send back the failure status and the
primary process will also fail to hotplug its device.
To prevent that, we need to pre-register the pci
drivers on env init.

We register the drivers just after the EAL init
because we don't want the matching devices to be picked
up by the initial bus probe in DPDK. That's for 2 reasons:

 1) we don't want to attach *all* available devices
 2) devices attached from non-SPDK context (that is,
    outside of the spdk attach or enumerate functions)
    will still fail to attach - the entire attaching
    process will only take significant amount of time
    and will bloat the log with useless status messages

Change-Id: I7b4c3a2e355f98ea755649f789137f5a727bc935
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/434415
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

lib/env_dpdk/env_internal.h

@@ -72,6 +72,12 @@ extern struct rte_pci_bus rte_pci_bus;
 #define MASK_4KB	((1ULL << SHIFT_4KB) - 1)
 #define VALUE_4KB	(1 << SHIFT_4KB)
 
+#define SPDK_PMD_REGISTER_PCI(pci_drv)									\
+__attribute__((constructor)) static void pci_drv ## _register(void)					\
+{													\
+	spdk_pci_driver_register(&pci_drv);								\
+}
+
 struct spdk_pci_device {
 	struct rte_pci_device		*dev_handle;
 	struct spdk_pci_driver		*driver;
@@ -87,8 +93,10 @@ struct spdk_pci_driver {
 	spdk_pci_enum_cb		cb_fn;
 	void				*cb_arg;
 	bool				is_registered;
+	TAILQ_ENTRY(spdk_pci_driver)	tailq;
 };
 
+void spdk_pci_driver_register(struct spdk_pci_driver *driver);
 int spdk_pci_device_init(struct rte_pci_driver *driver, struct rte_pci_device *device);
 int spdk_pci_device_fini(struct rte_pci_device *device);
 
@@ -96,6 +104,7 @@ int spdk_pci_enumerate(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb,
 int spdk_pci_device_attach(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx,
 			   struct spdk_pci_addr *pci_address);
 
+void spdk_pci_init(void);
 int spdk_mem_map_init(void);
 int spdk_vtophys_init(void);
 

lib/env_dpdk/init.c

@@ -396,6 +396,8 @@ int spdk_env_init(const struct spdk_env_opts *opts)
 		spdk_env_unlink_shared_files();
 	}
 
+	spdk_pci_init();
+
 	if (spdk_mem_map_init() < 0) {
 		fprintf(stderr, "Failed to allocate mem_map\n");
 		return -1;

lib/env_dpdk/pci.c

@@ -42,6 +42,40 @@
 
 static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER;
 static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices);
+static TAILQ_HEAD(, spdk_pci_driver) g_pci_drivers = TAILQ_HEAD_INITIALIZER(g_pci_drivers);
+
+void
+spdk_pci_driver_register(struct spdk_pci_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&g_pci_drivers, driver, tailq);
+}
+
+void
+spdk_pci_init(void)
+{
+#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)
+	struct spdk_pci_driver *driver;
+
+	/* We need to pre-register pci drivers for the pci devices to be
+	 * attachable in multi-process with DPDK 18.11+.
+	 *
+	 * DPDK 18.11+ does its best to ensure all devices are equally
+	 * attached or detached in all processes within a shared memory group.
+	 * For SPDK it means that if a device is hotplugged in the primary,
+	 * then DPDK will automatically send an IPC hotplug request to all other
+	 * processes. Those other processes may not have the same SPDK PCI
+	 * driver registered and may fail to attach the device. DPDK will send
+	 * back the failure status, and the the primary process will also fail
+	 * to hotplug the device. To prevent that, we need to pre-register the
+	 * pci drivers here.
+	 */
+	TAILQ_FOREACH(driver, &g_pci_drivers, tailq) {
+		assert(!driver->is_registered);
+		driver->is_registered = true;
+		rte_pci_register(&driver->driver);
+	}
+#endif
+}
 
 int
 spdk_pci_device_init(struct rte_pci_driver *_drv,

lib/env_dpdk/pci_ioat.c

@@ -114,3 +114,5 @@ spdk_pci_ioat_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_ioat_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_ioat_pci_drv);

lib/env_dpdk/pci_nvme.c

@@ -76,3 +76,5 @@ spdk_pci_nvme_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_nvme_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_nvme_pci_drv);

lib/env_dpdk/pci_virtio.c

@@ -71,3 +71,5 @@ spdk_pci_virtio_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx)
 {
 	return spdk_pci_enumerate(&g_virtio_pci_drv, enum_cb, enum_ctx);
 }
+
+SPDK_PMD_REGISTER_PCI(g_virtio_pci_drv);