d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

eal: restrict control threads to startup CPU affinity

Browse Source 
Spawning the ctrl threads on anything that is not part of the eal
coremask is not that polite to the rest of the system, especially
when you took good care to pin your processes on cpu resources with
tools like taskset (linux) / cpuset (freebsd).

Rather than introduce yet another eal options to control on which cpu
those ctrl threads are created, let's take the startup cpu affinity
as a reference and remove the eal coremask from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity
is lost.

Introduced a RTE_CPU_AND macro to abstract the differences between linux
and freebsd respective macros.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048

$ procstat -S 1057
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1057 100131 testpmd             -                     2    1 2
 1057 100140 testpmd             eal-intr-thread       1    1 0-1
 1057 100141 testpmd             rte_mp_handle         1    1 0-1
 1057 100142 testpmd             lcore-slave-3         3    1 3

$ cpuset -l 1,2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048

$ procstat -S 1061
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1061 100131 testpmd             -                     2    2 2
 1061 100144 testpmd             eal-intr-thread       1    2 1
 1061 100145 testpmd             rte_mp_handle         1    2 1
 1061 100147 testpmd             lcore-slave-3         3    2 3

$ cpuset -l 2,3 ./build/app/testpmd -l 2,3 --no-huge --no-pci -m 512 \
 -- -i --total-num-mbufs=2048

$ procstat -S 1065
  PID    TID COMM                TDNAME              CPU CSID CPU MASK
 1065 100131 testpmd             -                     2    2 2
 1065 100148 testpmd             eal-intr-thread       2    2 2
 1065 100149 testpmd             rte_mp_handle         2    2 2
 1065 100150 testpmd             lcore-slave-3         3    2 3

Fixes: d651ee4919 ("eal: set affinity for control threads")
Cc: stable@dpdk.org

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
This commit is contained in:
David Marchand 2019-02-19 21:41:11 +01:00 committed by Thomas Monjalon
parent 759b9be661
commit c3568ea376
5 changed files with 70 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
doc/guides/prog_guide/env_abstraction_layer.rst
View File

@ -498,6 +498,28 @@ Those TLS include *_cpuset* and *_socket_id*:
* *_socket_id* stores the NUMA node of the CPU set. If the CPUs in CPU set belong to different NUMA node, the *_socket_id* will be set to SOCKET_ID_ANY.
Control Thread API
~~~~~~~~~~~~~~~~~~
It is possible to create Control Threads using the public API
``rte_ctrl_thread_create()``.
Those threads can be used for management/infrastructure tasks and are used
internally by DPDK for multi process support and interrupt handling.
Those threads will be scheduled on CPUs part of the original process CPU
affinity from which the dataplane and service lcores are excluded.
For example, on a 8 CPUs system, starting a dpdk application with -l 2,3
(dataplane cores), then depending on the affinity configuration which can be
controlled with tools like taskset (Linux) or cpuset (FreeBSD),
- with no affinity configuration, the Control Threads will end up on
0-1,4-7 CPUs.
- with affinity restricted to 2-4, the Control Threads will end up on
CPU 4.
- with affinity restricted to 2-3, the Control Threads will end up on
CPU 2 (master lcore, which is the default when no CPU is available).
.. _known_issue_label:
Known Issues

28
lib/librte_eal/common/eal_common_options.c
View File

@ -217,6 +217,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
internal_cfg->create_uio_dev = 0;
internal_cfg->iova_mode = RTE_IOVA_DC;
internal_cfg->user_mbuf_pool_ops_name = NULL;
CPU_ZERO(&internal_cfg->ctrl_cpuset);
internal_cfg->init_complete = 0;
}
@ -1359,6 +1360,31 @@ eal_auto_detect_cores(struct rte_config *cfg)
cfg->lcore_count -= removed;
}
static void
compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
{
rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
rte_cpuset_t default_set;
unsigned int lcore_id;
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (eal_cpu_detected(lcore_id) &&
rte_lcore_has_role(lcore_id, ROLE_OFF)) {
CPU_SET(lcore_id, cpuset);
}
}
if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
&default_set))
CPU_ZERO(&default_set);
RTE_CPU_AND(cpuset, cpuset, &default_set);
/* if no detected CPU is off, use master core */
if (!CPU_COUNT(cpuset))
CPU_SET(rte_get_master_lcore(), cpuset);
}
int
eal_cleanup_config(struct internal_config *internal_cfg)
{
@ -1392,6 +1418,8 @@ eal_adjust_config(struct internal_config *internal_cfg)
lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
}
compute_ctrl_threads_cpuset(internal_cfg);
/* if no memory amounts were requested, this will result in 0 and
* will be overridden later, right after eal_hugepage_info_init() */
for (i = 0; i < RTE_MAX_NUMA_NODES; i++)

21
lib/librte_eal/common/eal_common_thread.c
View File

@ -16,6 +16,7 @@
#include <rte_memory.h>
#include <rte_log.h>
#include "eal_internal_cfg.h"
#include "eal_private.h"
#include "eal_thread.h"
@ -168,10 +169,9 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name,
const pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg)
{
rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
struct rte_thread_ctrl_params *params;
unsigned int lcore_id;
rte_cpuset_t cpuset;
int cpu_found, ret;
int ret;
params = malloc(sizeof(*params));
if (!params)
@ -195,20 +195,7 @@ rte_ctrl_thread_create(pthread_t *thread, const char *name,
"Cannot set name for ctrl thread\n");
}
cpu_found = 0;
CPU_ZERO(&cpuset);
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (eal_cpu_detected(lcore_id) &&
rte_lcore_has_role(lcore_id, ROLE_OFF)) {
CPU_SET(lcore_id, &cpuset);
cpu_found = 1;
}
}
/* if no detected cpu is off, use master core */
if (!cpu_found)
CPU_SET(rte_get_master_lcore(), &cpuset);
ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
if (ret)
goto fail;

3
lib/librte_eal/common/eal_internal_cfg.h
View File

@ -13,6 +13,8 @@
#include <rte_eal.h>
#include <rte_pci_dev_feature_defs.h>
#include "eal_thread.h"
#define MAX_HUGEPAGE_SIZES 3 /**< support up to 3 page sizes */
/*
@ -73,6 +75,7 @@ struct internal_config {
unsigned num_hugepage_sizes; /**< how many sizes on this system */
struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
enum rte_iova_mode iova_mode ; /**< Set IOVA mode on this system */
rte_cpuset_t ctrl_cpuset; /**< cpuset for ctrl threads */
volatile unsigned int init_complete;
/**< indicates whether EAL has completed initialization */
};

17
lib/librte_eal/common/include/rte_lcore.h
View File

@ -23,10 +23,18 @@ extern "C" {
#define LCORE_ID_ANY UINT32_MAX /**< Any lcore. */
#if defined(__linux__)
typedef cpu_set_t rte_cpuset_t;
typedef cpu_set_t rte_cpuset_t;
#define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
#elif defined(__FreeBSD__)
#include <pthread_np.h>
typedef cpuset_t rte_cpuset_t;
typedef cpuset_t rte_cpuset_t;
#define RTE_CPU_AND(dst, src1, src2) do \
{ \
cpuset_t tmp; \
CPU_COPY(src1, &tmp); \
CPU_AND(&tmp, src2); \
CPU_COPY(&tmp, dst); \
} while (0)
#endif
/**
@ -280,8 +288,9 @@ int rte_thread_setname(pthread_t id, const char *name);
* Create a control thread.
*
* Wrapper to pthread_create(), pthread_setname_np() and
* pthread_setaffinity_np(). The dataplane and service lcores are
* excluded from the affinity of the new thread.
* pthread_setaffinity_np(). The affinity of the new thread is based
* on the CPU affinity retrieved at the time rte_eal_init() was called,
* the dataplane and service lcores are then excluded.
*
* @param thread
* Filled with the thread id of the new created thread.