nvme: add PCI BDF in spdk_nvme_ctrlr to check whether same ctrlr

Change-Id: Ic8eb395bbfcc688e9c999a6d0026b70c24d386e3
Signed-off-by: GangCao <gang.cao@intel.com>

include/spdk/env.h

@@ -152,6 +152,13 @@ enum spdk_pci_device_type {
 	SPDK_PCI_DEVICE_IOAT,
 };
 
+struct spdk_pci_addr {
+	uint16_t			domain;
+	uint8_t				bus;
+	uint8_t				dev;
+	uint8_t				func;
+};
+
 typedef int (*spdk_pci_enum_cb)(void *enum_ctx, struct spdk_pci_device *pci_dev);
 
 int spdk_pci_enumerate(enum spdk_pci_device_type type,
@@ -182,6 +189,8 @@ int spdk_pci_device_cfg_write16(struct spdk_pci_device *dev, uint16_t value, uin
 int spdk_pci_device_cfg_read32(struct spdk_pci_device *dev, uint32_t *value, uint32_t offset);
 int spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, uint32_t offset);
 
+bool spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr);
+
 #ifdef __cplusplus
 }
 #endif

lib/env/pci.c

@@ -385,6 +385,15 @@ spdk_pci_device_get_serial_number(struct spdk_pci_device *dev, char *sn, size_t
 	return -1;
 }
 
+bool
+spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr)
+{
+	return ((spdk_pci_device_get_domain(dev) == addr->domain) &&
+		(spdk_pci_device_get_bus(dev) == addr->bus) &&
+		(spdk_pci_device_get_dev(dev) == addr->dev) &&
+		(spdk_pci_device_get_func(dev) == addr->func));
+}
+
 #ifdef __linux__
 int
 spdk_pci_device_claim(struct spdk_pci_device *dev)

lib/nvme/nvme.c

@@ -243,10 +243,11 @@ nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
 
 	/* Verify that this controller is not already attached */
 	TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) {
-		/* NOTE: This assumes that the PCI abstraction layer will use the same device handle
-		 *  across enumerations; we could compare by BDF instead if this is not true.
+		/* NOTE: In the case like multi-process environment where the device handle is
+		 * different per each process, we compare by BDF to determine whether it is the
+		 * same controller.
 		 */
-		if (pci_dev == ctrlr->devhandle) {
+		if (spdk_pci_device_compare_addr(pci_dev, &ctrlr->pci_addr)) {
 			return 0;
 		}
 	}

lib/nvme/nvme_ctrlr.c

@@ -1140,6 +1140,12 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr, void *devhandle)
 
 	pthread_mutex_init_recursive(&ctrlr->ctrlr_lock);
 
+	/* Save the PCI address */
+	ctrlr->pci_addr.domain = spdk_pci_device_get_domain(devhandle);
+	ctrlr->pci_addr.bus = spdk_pci_device_get_bus(devhandle);
+	ctrlr->pci_addr.dev = spdk_pci_device_get_dev(devhandle);
+	ctrlr->pci_addr.func = spdk_pci_device_get_func(devhandle);
+
 	return 0;
 }
 

lib/nvme/nvme_internal.h

@@ -459,6 +459,9 @@ struct spdk_nvme_ctrlr {
 	uint64_t			cmb_size;
 	/** Current offset of controller memory buffer */
 	uint64_t			cmb_current_offset;
+
+	/** PCI address including domain, bus, device and function */
+	struct spdk_pci_addr		pci_addr;
 };
 
 struct nvme_driver {

test/lib/nvme/unit/nvme_c/nvme_ut.c

@@ -76,6 +76,12 @@ spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
 	memset(opts, 0, sizeof(*opts));
 }
 
+bool
+spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr)
+{
+	return true;
+}
+
 static void
 test_opc_data_transfer(void)
 {

test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c

@@ -51,6 +51,10 @@ static uint16_t g_pci_vendor_id;
 static uint16_t g_pci_device_id;
 static uint16_t g_pci_subvendor_id;
 static uint16_t g_pci_subdevice_id;
+static uint16_t g_pci_domain;
+static uint8_t g_pci_bus;
+static uint8_t g_pci_dev;
+static uint8_t g_pci_func;
 
 uint64_t g_ut_tsc = 0;
 struct spdk_nvme_registers g_ut_nvme_regs = {};
@@ -112,6 +116,36 @@ spdk_pci_device_get_subdevice_id(struct spdk_pci_device *dev)
 	return g_pci_subdevice_id;
 }
 
+uint16_t
+spdk_pci_device_get_domain(struct spdk_pci_device *dev)
+{
+	return g_pci_domain;
+}
+
+uint8_t
+spdk_pci_device_get_bus(struct spdk_pci_device *dev)
+{
+	return g_pci_bus;
+}
+
+uint8_t
+spdk_pci_device_get_dev(struct spdk_pci_device *dev)
+{
+	return g_pci_dev;
+}
+
+uint8_t
+spdk_pci_device_get_func(struct spdk_pci_device *dev)
+{
+	return g_pci_func;
+}
+
+bool
+spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr)
+{
+	return true;
+}
+
 int nvme_qpair_construct(struct spdk_nvme_qpair *qpair, uint16_t id,
 			 uint16_t num_entries, uint16_t num_trackers,
 			 struct spdk_nvme_ctrlr *ctrlr)

test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c

@@ -80,6 +80,12 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
 	return 0;
 }
 
+bool
+spdk_pci_device_compare_addr(struct spdk_pci_device *dev, struct spdk_pci_addr *addr)
+{
+	return true;
+}
+
 void
 spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
 {