drivers/bus: set device NUMA node to unknown by default

The dev->device.numa_node field is set by each bus driver for
every device it manages to indicate on which NUMA node this device lies.

When this information is unknown, the assigned value is not consistent
across the bus drivers.

Set the default value to SOCKET_ID_ANY (-1) by all bus drivers
when the NUMA information is unavailable. This change impacts
rte_eth_dev_socket_id() in the same manner.

Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
This commit is contained in:
Olivier Matz 2022-10-04 16:58:50 +02:00 committed by David Marchand
parent 4cba3e114b
commit 7dcd73e379
18 changed files with 48 additions and 55 deletions

View File

@ -33,13 +33,6 @@ Deprecation Notices
``__atomic_thread_fence`` must be used for patches that need to be merged in
20.08 onwards. This change will not introduce any performance degradation.
* bus: The ``dev->device.numa_node`` field is set by each bus driver for
every device it manages to indicate on which NUMA node this device lies.
When this information is unknown, the assigned value is not consistent
across the bus drivers.
In DPDK 22.11, the default value will be set to -1 by all bus drivers
when the NUMA information is unavailable.
* kni: The KNI kernel module and library are not recommended for use by new
applications - other technologies such as virtio-user are recommended instead.
Following the DPDK technical board

View File

@ -220,6 +220,12 @@ API Changes
Application can use ``devtools/cocci/prefix_mbuf_offload_flags.cocci``
to replace all occurrences of old mbuf flags in C code.
* bus: Changed the device numa node to -1 when NUMA information is unavailable.
The ``dev->device.numa_node`` field is set by each bus driver for
every device it manages to indicate on which NUMA node this device lies.
When this information is unknown, the assigned value was not consistent
across the bus drivers. This similarly impacts ``rte_eth_dev_socket_id()``.
* bus: Registering a bus has been marked as an internal API.
External users may still register their bus using the ``bus_driver.h``
driver header (see ``enable_driver_sdk`` meson option).

View File

@ -105,12 +105,8 @@ rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
return -1;
}
if (dev->device.numa_node < 0) {
if (rte_socket_count() > 1)
AUXILIARY_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
dev->name);
dev->device.numa_node = 0;
}
if (dev->device.numa_node < 0 && rte_socket_count() > 1)
RTE_LOG(INFO, EAL, "Device %s is not NUMA-aware\n", dev->name);
iova_mode = rte_eal_iova_mode();
if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&

View File

@ -40,14 +40,11 @@ auxiliary_scan_one(const char *dirname, const char *name)
/* Get NUMA node, default to 0 if not present */
snprintf(filename, sizeof(filename), "%s/%s/numa_node",
dirname, name);
if (access(filename, F_OK) != -1) {
if (eal_parse_sysfs_value(filename, &tmp) == 0)
dev->device.numa_node = tmp;
else
dev->device.numa_node = -1;
} else {
dev->device.numa_node = 0;
}
if (access(filename, F_OK) == 0 &&
eal_parse_sysfs_value(filename, &tmp) == 0)
dev->device.numa_node = tmp;
else
dev->device.numa_node = SOCKET_ID_ANY;
auxiliary_on_scan(dev);

View File

@ -179,6 +179,7 @@ dpaa_create_device_list(void)
}
dev->device.bus = &rte_dpaa_bus.bus;
dev->device.numa_node = SOCKET_ID_ANY;
/* Allocate interrupt handle instance */
dev->intr_handle =

View File

@ -156,6 +156,7 @@ scan_one_fslmc_device(char *dev_name)
}
dev->device.bus = &rte_fslmc_bus.bus;
dev->device.numa_node = SOCKET_ID_ANY;
/* Allocate interrupt instance */
dev->intr_handle =

View File

@ -246,7 +246,7 @@ pci_scan_one(int dev_pci_fd, struct pci_conf *conf)
dev->max_vfs = 0;
/* FreeBSD has no NUMA support (yet) */
dev->device.numa_node = 0;
dev->device.numa_node = SOCKET_ID_ANY;
pci_common_set(dev);

View File

@ -283,17 +283,13 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
}
/* get numa node, default to 0 if not present */
snprintf(filename, sizeof(filename), "%s/numa_node",
dirname);
snprintf(filename, sizeof(filename), "%s/numa_node", dirname);
if (access(filename, F_OK) != -1) {
if (eal_parse_sysfs_value(filename, &tmp) == 0)
dev->device.numa_node = tmp;
else
dev->device.numa_node = -1;
} else {
dev->device.numa_node = 0;
}
if (access(filename, F_OK) == 0 &&
eal_parse_sysfs_value(filename, &tmp) == 0)
dev->device.numa_node = tmp;
else
dev->device.numa_node = SOCKET_ID_ANY;
pci_common_set(dev);

View File

@ -236,12 +236,8 @@ rte_pci_probe_one_driver(struct rte_pci_driver *dr,
return 1;
}
if (dev->device.numa_node < 0) {
if (rte_socket_count() > 1)
RTE_LOG(INFO, EAL, "Device %s is not NUMA-aware, defaulting socket to 0\n",
dev->name);
dev->device.numa_node = 0;
}
if (dev->device.numa_node < 0 && rte_socket_count() > 1)
RTE_LOG(INFO, EAL, "Device %s is not NUMA-aware\n", dev->name);
already_probed = rte_dev_is_probed(&dev->device);
if (already_probed && !(dr->drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {

View File

@ -249,7 +249,6 @@ get_device_resource_info(HDEVINFO dev_info,
DWORD error = GetLastError();
if (error == ERROR_NOT_FOUND) {
/* On older CPUs, NUMA is not bound to PCIe locality. */
dev->device.numa_node = 0;
return ERROR_SUCCESS;
}
RTE_LOG_WIN32_ERR("SetupDiGetDevicePropertyW"

View File

@ -293,7 +293,6 @@ vmbus_scan_one(const char *name)
goto error;
dev->device.numa_node = tmp;
} else {
/* if no NUMA support, set default to 0 */
dev->device.numa_node = SOCKET_ID_ANY;
}

View File

@ -111,12 +111,8 @@ vmbus_probe_one_driver(struct rte_vmbus_driver *dr,
/* reference driver structure */
dev->driver = dr;
if (dev->device.numa_node < 0) {
if (rte_socket_count() > 1)
VMBUS_LOG(INFO, "Device %s is not NUMA-aware, defaulting socket to 0",
guid);
dev->device.numa_node = 0;
}
if (dev->device.numa_node < 0 && rte_socket_count() > 1)
VMBUS_LOG(INFO, "Device %s is not NUMA-aware", guid);
/* call the driver probe() function */
VMBUS_LOG(INFO, " probe driver: %s", dr->driver.name);

View File

@ -12,6 +12,7 @@
#include <dev_driver.h>
#include <rte_devargs.h>
#include <rte_eal.h>
#include <rte_memory.h>
#include <rte_log.h>
#include <rte_dmadev_pmd.h>
#include <rte_string_fns.h>
@ -322,7 +323,7 @@ dsa_scan(void)
while ((wq = readdir(dev_dir)) != NULL) {
struct rte_dsa_device *dev;
int numa_node = -1;
int numa_node = SOCKET_ID_ANY;
if (strncmp(wq->d_name, "wq", 2) != 0)
continue;

View File

@ -231,7 +231,7 @@ lcore_rx(struct lcore_params *p)
if ((enabled_port_mask & (1 << port)) == 0)
continue;
if (rte_eth_dev_socket_id(port) > 0 &&
if (rte_eth_dev_socket_id(port) >= 0 &&
rte_eth_dev_socket_id(port) != socket_id)
printf("WARNING, port %u is on remote NUMA node to "
"RX thread.\n\tPerformance will not "
@ -406,7 +406,7 @@ lcore_tx(struct rte_ring *in_r)
if ((enabled_port_mask & (1 << port)) == 0)
continue;
if (rte_eth_dev_socket_id(port) > 0 &&
if (rte_eth_dev_socket_id(port) >= 0 &&
rte_eth_dev_socket_id(port) != socket_id)
printf("WARNING, port %u is on remote NUMA node to "
"TX thread.\n\tPerformance will not "

View File

@ -818,6 +818,8 @@ main(int argc, char *argv[])
printf("\nWARNING: Too many lcores enabled. Only 1 used.\n");
socket_id = rte_eth_dev_socket_id(0);
if (socket_id == SOCKET_ID_ANY)
socket_id = rte_lcore_to_socket_id(rte_get_next_lcore(-1, 0, 0));
/* Memory allocation. 8< */
size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct flow_classifier_acl));

View File

@ -249,7 +249,7 @@ lcore_main(void)
uint16_t port;
RTE_ETH_FOREACH_DEV(port)
if (rte_eth_dev_socket_id(port) > 0 &&
if (rte_eth_dev_socket_id(port) >= 0 &&
rte_eth_dev_socket_id(port) !=
(int)rte_socket_id())
printf("WARNING, port %u is on remote NUMA node to "

View File

@ -561,8 +561,16 @@ rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
int
rte_eth_dev_socket_id(uint16_t port_id)
{
RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
return rte_eth_devices[port_id].data->numa_node;
int socket_id = SOCKET_ID_ANY;
if (!rte_eth_dev_is_valid_port(port_id)) {
rte_errno = EINVAL;
} else {
socket_id = rte_eth_devices[port_id].data->numa_node;
if (socket_id == SOCKET_ID_ANY)
rte_errno = 0;
}
return socket_id;
}
void *

View File

@ -2442,9 +2442,11 @@ int rte_eth_hairpin_unbind(uint16_t tx_port, uint16_t rx_port);
* @param port_id
* The port identifier of the Ethernet device
* @return
* The NUMA socket ID to which the Ethernet device is connected or
* a default of zero if the socket could not be determined.
* -1 is returned is the port_id value is out of range.
* - The NUMA socket ID which the Ethernet device is connected to.
* - -1 (which translates to SOCKET_ID_ANY) if the socket could not be
* determined. rte_errno is then set to:
* - EINVAL is the port_id is invalid,
* - 0 is the socket could not be determined,
*/
int rte_eth_dev_socket_id(uint16_t port_id);