diff --git a/examples/ioat/perf/perf.c b/examples/ioat/perf/perf.c index 125de094a2..7f85a25878 100644 --- a/examples/ioat/perf/perf.c +++ b/examples/ioat/perf/perf.c @@ -142,70 +142,52 @@ ioat_done(void *cb_arg) } } +static bool +probe_cb(void *cb_ctx, void *pdev) +{ + struct pci_device *pci_dev = pdev; + + printf(" Found matching device at %d:%d:%d " + "vendor:0x%04x device:0x%04x\n name:%s\n", + pci_dev->bus, pci_dev->dev, pci_dev->func, + pci_dev->vendor_id, pci_dev->device_id, + pci_device_get_device_name(pci_dev)); + + if (pci_device_has_non_uio_driver(pci_dev)) { + printf("Device has non-uio kernel driver, skipping...\n"); + return false; + } + + return true; +} + +static void +attach_cb(void *cb_ctx, void *pdev, struct ioat_channel *ioat) +{ + struct ioat_device *dev; + + dev = rte_malloc(NULL, sizeof(*dev), 0); + if (dev == NULL) { + printf("Failed to allocate device struct\n"); + return; + } + + dev->ioat = ioat; + TAILQ_INSERT_TAIL(&g_devices, dev, tailq); +} + static int ioat_init(void) { - struct pci_device_iterator *iter; - struct pci_device *pci_dev; - int err = 0; - struct pci_id_match match; - struct ioat_device *dev; - pci_system_init(); TAILQ_INIT(&g_devices); - match.vendor_id = PCI_MATCH_ANY; - match.subvendor_id = PCI_MATCH_ANY; - match.subdevice_id = PCI_MATCH_ANY; - match.device_id = PCI_MATCH_ANY; - match.device_class = 0x088000; - match.device_class_mask = 0xFFFFFF; - - iter = pci_id_match_iterator_create(&match); - - while ((pci_dev = pci_device_next(iter)) != NULL) { - /* Check if the PCI devices is a supported IOAT channel. */ - if (!(ioat_pci_device_match_id(pci_dev->vendor_id, - pci_dev->device_id))) { - continue; - } - - printf(" Found matching device at %d:%d:%d " - "vendor:0x%04x device:0x%04x\n name:%s\n", - pci_dev->bus, pci_dev->dev, pci_dev->func, - pci_dev->vendor_id, pci_dev->device_id, - pci_device_get_device_name(pci_dev)); - - if (pci_device_has_non_uio_driver(pci_dev)) { - printf("Device has non-uio kernel driver, skipping...\n"); - continue; - } - - pci_device_probe(pci_dev); - - dev = rte_malloc(NULL, sizeof(*dev), 0); - if (dev == NULL) { - printf("Failed to allocate device struct\n"); - err = -1; - goto cleanup; - } - - dev->ioat = ioat_attach(pci_dev); - if (dev->ioat == NULL) { - rte_free(dev); - /* Likely no device found. */ - err = -1; - goto cleanup; - } - TAILQ_INSERT_TAIL(&g_devices, dev, tailq); + if (ioat_probe(NULL, probe_cb, attach_cb) != 0) { + fprintf(stderr, "ioat_probe() failed\n"); + return 1; } -cleanup: - pci_iterator_destroy(iter); - if (err != 0) { - ioat_exit(); - } - return err; + return 0; } static void diff --git a/examples/ioat/verify/verify.c b/examples/ioat/verify/verify.c index 75fc5ccc56..b439079144 100644 --- a/examples/ioat/verify/verify.c +++ b/examples/ioat/verify/verify.c @@ -198,71 +198,53 @@ ioat_done(void *cb_arg) } } +static bool +probe_cb(void *cb_ctx, void *pdev) +{ + struct pci_device *pci_dev = pdev; + + printf(" Found matching device at %d:%d:%d " + "vendor:0x%04x device:0x%04x\n name:%s\n", + pci_dev->bus, pci_dev->dev, pci_dev->func, + pci_dev->vendor_id, pci_dev->device_id, + pci_device_get_device_name(pci_dev)); + + if (pci_device_has_non_uio_driver(pci_dev)) { + printf("Device has non-uio kernel driver, skipping...\n"); + return false; + } + + return true; +} + +static void +attach_cb(void *cb_ctx, void *pdev, struct ioat_channel *ioat) +{ + struct ioat_device *dev; + + dev = malloc(sizeof(*dev)); + if (dev == NULL) { + printf("Failed to allocate device struct\n"); + return; + } + memset(dev, 0, sizeof(*dev)); + + dev->ioat = ioat; + TAILQ_INSERT_TAIL(&g_devices, dev, tailq); +} + static int ioat_init(void) { - struct pci_device_iterator *iter; - struct pci_device *pci_dev; - int err = 0; - struct pci_id_match match; - struct ioat_device *dev; - pci_system_init(); TAILQ_INIT(&g_devices); - match.vendor_id = PCI_MATCH_ANY; - match.subvendor_id = PCI_MATCH_ANY; - match.subdevice_id = PCI_MATCH_ANY; - match.device_id = PCI_MATCH_ANY; - match.device_class = 0x088000; - match.device_class_mask = 0xFFFFFF; - - iter = pci_id_match_iterator_create(&match); - - while ((pci_dev = pci_device_next(iter)) != NULL) { - /* Check if the PCI devices is a supported IOAT channel. */ - if (!(ioat_pci_device_match_id(pci_dev->vendor_id, - pci_dev->device_id))) { - continue; - } - - printf(" Found matching device at %d:%d:%d " - "vendor:0x%04x device:0x%04x\n name:%s\n", - pci_dev->bus, pci_dev->dev, pci_dev->func, - pci_dev->vendor_id, pci_dev->device_id, - pci_device_get_device_name(pci_dev)); - - if (pci_device_has_non_uio_driver(pci_dev)) { - printf("Device has non-uio kernel driver, skipping...\n"); - continue; - } - - pci_device_probe(pci_dev); - - dev = malloc(sizeof(*dev)); - if (dev == NULL) { - printf("Failed to allocate device struct\n"); - err = -1; - goto cleanup; - } - memset(dev, 0, sizeof(*dev)); - - dev->ioat = ioat_attach(pci_dev); - if (dev->ioat == NULL) { - free(dev); - /* Likely no device found. */ - err = -1; - goto cleanup; - } - TAILQ_INSERT_TAIL(&g_devices, dev, tailq); + if (ioat_probe(NULL, probe_cb, attach_cb) != 0) { + fprintf(stderr, "ioat_probe() failed\n"); + return 1; } -cleanup: - pci_iterator_destroy(iter); - if (err != 0) { - ioat_exit(); - } - return err; + return 0; } static void diff --git a/include/spdk/ioat.h b/include/spdk/ioat.h index be3ee464d1..37e3277a4c 100644 --- a/include/spdk/ioat.h +++ b/include/spdk/ioat.h @@ -42,26 +42,46 @@ #include #include "spdk/pci.h" +/** + * Opaque handle for a single I/OAT channel returned by \ref ioat_probe(). + */ +struct ioat_channel; + /** * Signature for callback function invoked when a request is completed. */ typedef void (*ioat_callback_t)(void *arg); /** - * Returns true if vendor_id and device_id match a known IOAT PCI device ID. + * Callback for ioat_probe() enumeration. + * + * \return true to attach to this device. */ -bool ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id); +typedef bool (*ioat_probe_cb)(void *cb_ctx, void *pci_dev); /** - * Attach an I/OAT PCI device to the I/OAT userspace driver. + * Callback for ioat_probe() to report a device that has been attached to the userspace I/OAT driver. + */ +typedef void (*ioat_attach_cb)(void *cb_ctx, void *pci_dev, struct ioat_channel *ioat); + +/** + * \brief Enumerate the I/OAT devices attached to the system and attach the userspace I/OAT driver + * to them if desired. * - * To stop using the the device and release its associated resources, + * \param probe_cb will be called once per I/OAT device found in the system. + * \param attach_cb will be called for devices for which probe_cb returned true once the I/OAT + * controller has been attached to the userspace driver. + * + * If called more than once, only devices that are not already attached to the SPDK I/OAT driver + * will be reported. + * + * To stop using the the controller and release its associated resources, * call \ref ioat_detach with the ioat_channel instance returned by this function. */ -struct ioat_channel *ioat_attach(void *device); +int ioat_probe(void *cb_ctx, ioat_probe_cb probe_cb, ioat_attach_cb attach_cb); /** - * Detaches specified device returned by \ref ioat_attach() from the I/OAT driver. + * Detaches specified device returned by \ref ioat_probe() from the I/OAT driver. */ int ioat_detach(struct ioat_channel *ioat); diff --git a/lib/ioat/ioat.c b/lib/ioat/ioat.c index b20b625752..b1e0c7d042 100644 --- a/lib/ioat/ioat.c +++ b/lib/ioat/ioat.c @@ -45,77 +45,14 @@ static __thread struct ioat_channel *ioat_thread_channel; struct ioat_driver { ioat_mutex_t lock; + TAILQ_HEAD(, ioat_channel) attached_chans; }; static struct ioat_driver g_ioat_driver = { .lock = IOAT_MUTEX_INITIALIZER, + .attached_chans = TAILQ_HEAD_INITIALIZER(g_ioat_driver.attached_chans), }; -struct pci_device_id { - uint16_t vendor; - uint16_t device; -}; - -static const struct pci_device_id ioat_pci_table[] = { - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2}, - {PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3}, -}; - -bool -ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id) -{ - size_t i; - const struct pci_device_id *ids; - - for (i = 0; i < sizeof(ioat_pci_table) / sizeof(struct pci_device_id); i++) { - ids = &ioat_pci_table[i]; - if (ids->device == device_id && ids->vendor == vendor_id) { - return true; - } - } - return false; -} - static uint64_t ioat_get_chansts(struct ioat_channel *ioat) { @@ -526,10 +463,10 @@ ioat_channel_start(struct ioat_channel *ioat) return 0; } -struct ioat_channel * +/* Caller must hold g_ioat_driver.lock */ +static struct ioat_channel * ioat_attach(void *device) { - struct ioat_driver *driver = &g_ioat_driver; struct ioat_channel *ioat; uint32_t cmd_reg; @@ -551,13 +488,74 @@ ioat_attach(void *device) return NULL; } - ioat_mutex_lock(&driver->lock); SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next); - ioat_mutex_unlock(&driver->lock); return ioat; } +struct ioat_enum_ctx { + ioat_probe_cb probe_cb; + ioat_attach_cb attach_cb; + void *cb_ctx; +}; + +/* This function must only be called while holding g_ioat_driver.lock */ +static int +ioat_enum_cb(void *ctx, void *pci_dev) +{ + struct ioat_enum_ctx *enum_ctx = ctx; + struct ioat_channel *ioat; + + /* Verify that this device is not already attached */ + TAILQ_FOREACH(ioat, &g_ioat_driver.attached_chans, tailq) { + /* + * NOTE: This assumes that the PCI abstraction layer will use the same device handle + * across enumerations; we could compare by BDF instead if this is not true. + */ + if (pci_dev == ioat->device) { + return 0; + } + } + + if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) { + /* + * Since I/OAT init is relatively quick, just perform the full init during probing. + * If this turns out to be a bottleneck later, this can be changed to work like + * NVMe with a list of devices to initialize in parallel. + */ + ioat = ioat_attach(pci_dev); + if (ioat == NULL) { + ioat_printf(NULL, "ioat_attach() failed\n"); + return -1; + } + + TAILQ_INSERT_TAIL(&g_ioat_driver.attached_chans, ioat, tailq); + + enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, ioat); + } + + return 0; +} + +int +ioat_probe(void *cb_ctx, ioat_probe_cb probe_cb, ioat_attach_cb attach_cb) +{ + int rc; + struct ioat_enum_ctx enum_ctx; + + ioat_mutex_lock(&g_ioat_driver.lock); + + enum_ctx.probe_cb = probe_cb; + enum_ctx.attach_cb = attach_cb; + enum_ctx.cb_ctx = cb_ctx; + + rc = ioat_pci_enumerate(ioat_enum_cb, &enum_ctx); + + ioat_mutex_unlock(&g_ioat_driver.lock); + + return rc; +} + int ioat_detach(struct ioat_channel *ioat) { @@ -568,6 +566,7 @@ ioat_detach(struct ioat_channel *ioat) */ ioat_mutex_lock(&driver->lock); SLIST_REMOVE(&ioat_free_channels, ioat, ioat_channel, next); + TAILQ_REMOVE(&driver->attached_chans, ioat, tailq); ioat_mutex_unlock(&driver->lock); ioat_channel_destruct(ioat); diff --git a/lib/ioat/ioat_impl.h b/lib/ioat/ioat_impl.h index b08889772b..69ae39b7b5 100644 --- a/lib/ioat/ioat_impl.h +++ b/lib/ioat/ioat_impl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include "ioat_pci.h" +#include "ioat_pci.h" + /** * \file * @@ -64,6 +67,99 @@ ioat_zmalloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr) #define ioat_printf(chan, fmt, args...) printf(fmt, ##args) #ifdef USE_PCIACCESS + +static inline bool +ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id) +{ + if (vendor_id != PCI_VENDOR_ID_INTEL) { + return false; + } + + switch (device_id) { + case PCI_DEVICE_ID_INTEL_IOAT_SNB0: + case PCI_DEVICE_ID_INTEL_IOAT_SNB1: + case PCI_DEVICE_ID_INTEL_IOAT_SNB2: + case PCI_DEVICE_ID_INTEL_IOAT_SNB3: + case PCI_DEVICE_ID_INTEL_IOAT_SNB4: + case PCI_DEVICE_ID_INTEL_IOAT_SNB5: + case PCI_DEVICE_ID_INTEL_IOAT_SNB6: + case PCI_DEVICE_ID_INTEL_IOAT_SNB7: + case PCI_DEVICE_ID_INTEL_IOAT_IVB0: + case PCI_DEVICE_ID_INTEL_IOAT_IVB1: + case PCI_DEVICE_ID_INTEL_IOAT_IVB2: + case PCI_DEVICE_ID_INTEL_IOAT_IVB3: + case PCI_DEVICE_ID_INTEL_IOAT_IVB4: + case PCI_DEVICE_ID_INTEL_IOAT_IVB5: + case PCI_DEVICE_ID_INTEL_IOAT_IVB6: + case PCI_DEVICE_ID_INTEL_IOAT_IVB7: + case PCI_DEVICE_ID_INTEL_IOAT_HSW0: + case PCI_DEVICE_ID_INTEL_IOAT_HSW1: + case PCI_DEVICE_ID_INTEL_IOAT_HSW2: + case PCI_DEVICE_ID_INTEL_IOAT_HSW3: + case PCI_DEVICE_ID_INTEL_IOAT_HSW4: + case PCI_DEVICE_ID_INTEL_IOAT_HSW5: + case PCI_DEVICE_ID_INTEL_IOAT_HSW6: + case PCI_DEVICE_ID_INTEL_IOAT_HSW7: + case PCI_DEVICE_ID_INTEL_IOAT_BDX0: + case PCI_DEVICE_ID_INTEL_IOAT_BDX1: + case PCI_DEVICE_ID_INTEL_IOAT_BDX2: + case PCI_DEVICE_ID_INTEL_IOAT_BDX3: + case PCI_DEVICE_ID_INTEL_IOAT_BDX4: + case PCI_DEVICE_ID_INTEL_IOAT_BDX5: + case PCI_DEVICE_ID_INTEL_IOAT_BDX6: + case PCI_DEVICE_ID_INTEL_IOAT_BDX7: + case PCI_DEVICE_ID_INTEL_IOAT_BDX8: + case PCI_DEVICE_ID_INTEL_IOAT_BDX9: + case PCI_DEVICE_ID_INTEL_IOAT_BWD0: + case PCI_DEVICE_ID_INTEL_IOAT_BWD1: + case PCI_DEVICE_ID_INTEL_IOAT_BWD2: + case PCI_DEVICE_ID_INTEL_IOAT_BWD3: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: + return true; + } + + return false; +} + +static inline int +ioat_pci_enumerate(int (*enum_cb)(void *enum_ctx, void *pci_dev), void *enum_ctx) +{ + struct pci_device_iterator *pci_dev_iter; + struct pci_device *pci_dev; + struct pci_id_match match; + int rc; + + match.vendor_id = PCI_VENDOR_ID_INTEL; + match.subvendor_id = PCI_MATCH_ANY; + match.subdevice_id = PCI_MATCH_ANY; + match.device_id = PCI_MATCH_ANY; + match.device_class = 0x088000; + match.device_class_mask = 0xFFFFFF; + + pci_dev_iter = pci_id_match_iterator_create(&match); + + rc = 0; + while ((pci_dev = pci_device_next(pci_dev_iter))) { + if (!(ioat_pci_device_match_id(pci_dev->vendor_id, + pci_dev->device_id))) { + continue; + } + + pci_device_probe(pci_dev); + + if (enum_cb(enum_ctx, pci_dev)) { + rc = -1; + } + } + + pci_iterator_destroy(pci_dev_iter); + + return rc; +} + /** * */ diff --git a/lib/ioat/ioat_internal.h b/lib/ioat/ioat_internal.h index a2e3f2ccf9..1d39a5158e 100644 --- a/lib/ioat/ioat_internal.h +++ b/lib/ioat/ioat_internal.h @@ -75,6 +75,9 @@ struct ioat_channel { union ioat_hw_descriptor *hw_ring; uint64_t hw_ring_phys_addr; uint32_t dma_capabilities; + + /* tailq entry for attached_chans */ + TAILQ_ENTRY(ioat_channel) tailq; }; static inline uint32_t diff --git a/test/lib/ioat/unit/ioat_impl.h b/test/lib/ioat/unit/ioat_impl.h index 7172924f09..a4da5cbbe1 100644 --- a/test/lib/ioat/unit/ioat_impl.h +++ b/test/lib/ioat/unit/ioat_impl.h @@ -24,6 +24,13 @@ ioat_zmalloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr) #define ioat_delay_us(us) ioat_noop() #define ioat_assert(check) assert(check) #define ioat_printf(chan, fmt, args...) printf(fmt, ##args) + +static inline int +ioat_pci_enumerate(int (*enum_cb)(void *enum_ctx, void *pci_dev), void *enum_ctx) +{ + return -1; +} + #define ioat_pcicfg_read32(handle, var, offset) do { *(var) = 0xFFFFFFFFu; } while (0) #define ioat_pcicfg_write32(handle, var, offset) do { (void)(var); } while (0)