From 82e5f6b658f92a6d613398875367badc948f8b99 Mon Sep 17 00:00:00 2001 From: Thomas Monjalon Date: Mon, 8 Nov 2021 18:57:59 +0000 Subject: [PATCH] gpudev: add child device representing a device context The computing device may operate in some isolated contexts. Memory and processing are isolated in a silo represented by a child device. The context is provided as an opaque by the caller of rte_gpu_add_child(). Signed-off-by: Thomas Monjalon --- app/test-gpudev/main.c | 3 +- doc/guides/prog_guide/gpudev.rst | 12 ++++++ lib/gpudev/gpudev.c | 45 +++++++++++++++++++- lib/gpudev/gpudev_driver.h | 2 +- lib/gpudev/rte_gpudev.h | 71 +++++++++++++++++++++++++++++--- lib/gpudev/version.map | 1 + 6 files changed, 125 insertions(+), 9 deletions(-) diff --git a/app/test-gpudev/main.c b/app/test-gpudev/main.c index 15b6e4e2a2..438cfdac54 100644 --- a/app/test-gpudev/main.c +++ b/app/test-gpudev/main.c @@ -88,8 +88,9 @@ main(int argc, char **argv) if (rte_gpu_info_get(gpu_id, &ginfo)) rte_exit(EXIT_FAILURE, "rte_gpu_info_get error - bye\n"); - printf("\tGPU ID %d\n\t\tGPU Bus ID %s NUMA node %d Tot memory %.02f MB, Tot processors %d\n", + printf("\tGPU ID %d\n\t\tparent ID %d GPU Bus ID %s NUMA node %d Tot memory %.02f MB, Tot processors %d\n", ginfo.dev_id, + ginfo.parent, ginfo.name, ginfo.numa_node, (((float)ginfo.total_memory)/(float)1024)/(float)1024, diff --git a/doc/guides/prog_guide/gpudev.rst b/doc/guides/prog_guide/gpudev.rst index 6ea7239159..7694639489 100644 --- a/doc/guides/prog_guide/gpudev.rst +++ b/doc/guides/prog_guide/gpudev.rst @@ -34,3 +34,15 @@ This library provides a number of features: API Overview ------------ + +Child Device +~~~~~~~~~~~~ + +By default, DPDK PCIe module detects and registers physical GPU devices +in the system. +With the gpudev library is also possible to add additional non-physical devices +through an ``uint64_t`` generic handler (e.g. CUDA Driver context) +that will be registered internally by the driver as an additional device (child) +connected to a physical device (parent). +Each device (parent or child) is represented through a ID +required to indicate which device a given operation should be executed on. diff --git a/lib/gpudev/gpudev.c b/lib/gpudev/gpudev.c index c25f9edb8b..4ed8426ef5 100644 --- a/lib/gpudev/gpudev.c +++ b/lib/gpudev/gpudev.c @@ -81,13 +81,22 @@ rte_gpu_is_valid(int16_t dev_id) return false; } +static bool +gpu_match_parent(int16_t dev_id, int16_t parent) +{ + if (parent == RTE_GPU_ID_ANY) + return true; + return gpus[dev_id].info.parent == parent; +} + int16_t -rte_gpu_find_next(int16_t dev_id) +rte_gpu_find_next(int16_t dev_id, int16_t parent) { if (dev_id < 0) dev_id = 0; while (dev_id < gpu_max && - gpus[dev_id].state == RTE_GPU_STATE_UNUSED) + (gpus[dev_id].state == RTE_GPU_STATE_UNUSED || + !gpu_match_parent(dev_id, parent))) dev_id++; if (dev_id >= gpu_max) @@ -178,6 +187,7 @@ rte_gpu_allocate(const char *name) dev->info.name = dev->name; dev->info.dev_id = dev_id; dev->info.numa_node = -1; + dev->info.parent = RTE_GPU_ID_NONE; TAILQ_INIT(&dev->callbacks); gpu_count++; @@ -186,6 +196,28 @@ rte_gpu_allocate(const char *name) return dev; } +int16_t +rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context) +{ + struct rte_gpu *dev; + + if (!rte_gpu_is_valid(parent)) { + GPU_LOG(ERR, "add child to invalid parent ID %d", parent); + rte_errno = ENODEV; + return -rte_errno; + } + + dev = rte_gpu_allocate(name); + if (dev == NULL) + return -rte_errno; + + dev->info.parent = parent; + dev->info.context = child_context; + + rte_gpu_complete_new(dev); + return dev->info.dev_id; +} + void rte_gpu_complete_new(struct rte_gpu *dev) { @@ -200,10 +232,19 @@ rte_gpu_complete_new(struct rte_gpu *dev) int rte_gpu_release(struct rte_gpu *dev) { + int16_t dev_id, child; + if (dev == NULL) { rte_errno = ENODEV; return -rte_errno; } + dev_id = dev->info.dev_id; + RTE_GPU_FOREACH_CHILD(child, dev_id) { + GPU_LOG(ERR, "cannot release device %d with child %d", + dev_id, child); + rte_errno = EBUSY; + return -rte_errno; + } GPU_LOG(DEBUG, "free device %s (id %d)", dev->info.name, dev->info.dev_id); diff --git a/lib/gpudev/gpudev_driver.h b/lib/gpudev/gpudev_driver.h index 2a7089aa52..4d0077161c 100644 --- a/lib/gpudev/gpudev_driver.h +++ b/lib/gpudev/gpudev_driver.h @@ -31,7 +31,7 @@ typedef int (rte_gpu_info_get_t)(struct rte_gpu *dev, struct rte_gpu_info *info) struct rte_gpu_ops { /* Get device info. If NULL, info is just copied. */ rte_gpu_info_get_t *dev_info_get; - /* Close device. */ + /* Close device or child context. */ rte_gpu_close_t *dev_close; }; diff --git a/lib/gpudev/rte_gpudev.h b/lib/gpudev/rte_gpudev.h index e1702fbfe4..df75dbdbab 100644 --- a/lib/gpudev/rte_gpudev.h +++ b/lib/gpudev/rte_gpudev.h @@ -41,8 +41,12 @@ extern "C" { struct rte_gpu_info { /** Unique identifier name. */ const char *name; + /** Opaque handler of the device context. */ + uint64_t context; /** Device ID. */ int16_t dev_id; + /** ID of the parent device, RTE_GPU_ID_NONE if no parent */ + int16_t parent; /** Total processors available on device. */ uint32_t processor_count; /** Total memory available on device. */ @@ -110,6 +114,33 @@ uint16_t rte_gpu_count_avail(void); __rte_experimental bool rte_gpu_is_valid(int16_t dev_id); +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Create a virtual device representing a context in the parent device. + * + * @param name + * Unique string to identify the device. + * @param parent + * Device ID of the parent. + * @param child_context + * Opaque context handler. + * + * @return + * Device ID of the new created child, -rte_errno otherwise: + * - EINVAL if empty name + * - ENAMETOOLONG if long name + * - EEXIST if existing device name + * - ENODEV if invalid parent + * - EPERM if secondary process + * - ENOENT if too many devices + * - ENOMEM if out of space + */ +__rte_experimental +int16_t rte_gpu_add_child(const char *name, + int16_t parent, uint64_t child_context); + /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. @@ -118,13 +149,17 @@ bool rte_gpu_is_valid(int16_t dev_id); * * @param dev_id * The initial device ID to start the research. + * @param parent + * The device ID of the parent. + * RTE_GPU_ID_NONE means no parent. + * RTE_GPU_ID_ANY means no or any parent. * * @return * Next device ID corresponding to a valid and initialized computing device, * RTE_GPU_ID_NONE if there is none. */ __rte_experimental -int16_t rte_gpu_find_next(int16_t dev_id); +int16_t rte_gpu_find_next(int16_t dev_id, int16_t parent); /** * @warning @@ -136,15 +171,41 @@ int16_t rte_gpu_find_next(int16_t dev_id); * The ID of the next possible valid device, usually 0 to iterate all. */ #define RTE_GPU_FOREACH(dev_id) \ - for (dev_id = rte_gpu_find_next(0); \ - dev_id > 0; \ - dev_id = rte_gpu_find_next(dev_id + 1)) + RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_ANY) /** * @warning * @b EXPERIMENTAL: this API may change without prior notice. * - * Close device. + * Macro to iterate over all valid computing devices having no parent. + * + * @param dev_id + * The ID of the next possible valid device, usually 0 to iterate all. + */ +#define RTE_GPU_FOREACH_PARENT(dev_id) \ + RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_NONE) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Macro to iterate over all valid children of a computing device parent. + * + * @param dev_id + * The ID of the next possible valid device, usually 0 to iterate all. + * @param parent + * The device ID of the parent. + */ +#define RTE_GPU_FOREACH_CHILD(dev_id, parent) \ + for (dev_id = rte_gpu_find_next(0, parent); \ + dev_id >= 0; \ + dev_id = rte_gpu_find_next(dev_id + 1, parent)) + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice. + * + * Close device or child context. * All resources are released. * * @param dev_id diff --git a/lib/gpudev/version.map b/lib/gpudev/version.map index b3b6b76c1c..4a934ed933 100644 --- a/lib/gpudev/version.map +++ b/lib/gpudev/version.map @@ -2,6 +2,7 @@ EXPERIMENTAL { global: # added in 21.11 + rte_gpu_add_child; rte_gpu_callback_register; rte_gpu_callback_unregister; rte_gpu_close;