d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

eal: fix control thread affinity with --lcores

Browse Source 
The ctrl thread cpu affinity setting has been broken when using --lcores.

Using -l/-c options makes each lcore associated to a physical cpu in a 1:1
fashion.
On the contrary, when using --lcores, each lcore cpu affinity can be set
to a list of any online cpu on the system.

To handle both cases, each lcore cpu affinity is considered and removed
from the process startup cpu affinity.

Introduced macros to manipulate dpdk cpu sets in both Linux and FreeBSD.

Examples on a 8 cores Linux system:

$ cd /sys/fs/cgroup/cpuset/
$ mkdir dpdk
$ cd dpdk
$ echo 4-7 > cpuset.cpus
$ echo 0 > cpuset.mems
$ echo $$ > tasks

Before the fix:
$ ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

8427	cpu_list=4-5,7	testpmd
8428	cpu_list=4-6	eal-intr-thread
8429	cpu_list=4-6	rte_mp_handle
8430	cpu_list=4-5,7	lcore-slave-7

$ taskset -c 7 \
 ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

EAL: Detected 8 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Failed to create thread for interrupt handling
EAL: FATAL: Cannot init interrupt-handling thread
EAL: Cannot init interrupt-handling thread
PANIC in main():
Cannot init EAL

After the fix:
$ ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

15214	cpu_list=4-5,7	testpmd
15215	cpu_list=6	eal-intr-thread
15216	cpu_list=6	rte_mp_handle
15217	cpu_list=4-5,7	lcore-slave-7

$ taskset -c 7 \
 ./master/app/testpmd --master-lcore 0 --lcores '(0,7)@(7,4,5)' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

15297	cpu_list=4-5,7	testpmd
15298	cpu_list=4-5,7	eal-intr-thread
15299	cpu_list=4-5,7	rte_mp_handle
15300	cpu_list=4-5,7	lcore-slave-7

Bugzilla ID: 322
Fixes: c3568ea376 ("eal: restrict control threads to startup CPU affinity")
Cc: stable@dpdk.org

Reported-by: Johan Källström <johan.kallstrom@ericsson.com>
Signed-off-by: David Marchand <david.marchand@redhat.com>
This commit is contained in:
David Marchand 2019-07-30 17:05:22 +02:00 committed by Thomas Monjalon
parent d7f936190e
commit b6be16acfe
2 changed files with 38 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/librte_eal/common/eal_common_options.c
View File

@ -1452,11 +1452,11 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
unsigned int lcore_id;
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (eal_cpu_detected(lcore_id) &&
rte_lcore_has_role(lcore_id, ROLE_OFF)) {
CPU_SET(lcore_id, cpuset);
}
if (rte_lcore_has_role(lcore_id, ROLE_OFF))
continue;
RTE_CPU_OR(cpuset, cpuset, &lcore_config[lcore_id].cpuset);
}
RTE_CPU_NOT(cpuset, cpuset);
if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
&default_set))
@ -1464,9 +1464,11 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
RTE_CPU_AND(cpuset, cpuset, &default_set);
/* if no detected CPU is off, use master core */
if (!CPU_COUNT(cpuset))
CPU_SET(rte_get_master_lcore(), cpuset);
/* if no remaining cpu, use master lcore cpu affinity */
if (!CPU_COUNT(cpuset)) {
memcpy(cpuset, &lcore_config[rte_get_master_lcore()].cpuset,
sizeof(*cpuset));
}
}
int

29
lib/librte_eal/common/include/rte_lcore.h
View File

@ -25,6 +25,20 @@ extern "C" {
#if defined(__linux__)
typedef cpu_set_t rte_cpuset_t;
#define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
#define RTE_CPU_OR(dst, src1, src2) CPU_OR(dst, src1, src2)
#define RTE_CPU_FILL(set) do \
{ \
unsigned int i; \
CPU_ZERO(set); \
for (i = 0; i < CPU_SETSIZE; i++) \
CPU_SET(i, set); \
} while (0)
#define RTE_CPU_NOT(dst, src) do \
{ \
cpu_set_t tmp; \
RTE_CPU_FILL(&tmp); \
CPU_XOR(dst, &tmp, src); \
} while (0)
#elif defined(__FreeBSD__)
#include <pthread_np.h>
typedef cpuset_t rte_cpuset_t;
@ -35,6 +49,21 @@ typedef cpuset_t rte_cpuset_t;
CPU_AND(&tmp, src2); \
CPU_COPY(&tmp, dst); \
} while (0)
#define RTE_CPU_OR(dst, src1, src2) do \
{ \
cpuset_t tmp; \
CPU_COPY(src1, &tmp); \
CPU_OR(&tmp, src2); \
CPU_COPY(&tmp, dst); \
} while (0)
#define RTE_CPU_FILL(set) CPU_FILL(set)
#define RTE_CPU_NOT(dst, src) do \
{ \
cpuset_t tmp; \
CPU_FILL(&tmp); \
CPU_NAND(&tmp, src); \
CPU_COPY(&tmp, dst); \
} while (0)
#endif
/**