d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optionally bind ktls threads to NUMA domains

Browse Source 
When ktls_bind_thread is 2, we pick a ktls worker thread that is
bound to the same domain as the TCP connection associated with
the socket. We use roughly the same code as netinet/tcp_hpts.c to
do this. This allows crypto to run on the same domain as the TCP
connection is associated with. Assuming TCP_REUSPORT_LB_NUMA
(D21636) is in place & in use, this ensures that the crypto source
and destination buffers are local to the same NUMA domain as we're
running crypto on.

This change (when TCP_REUSPORT_LB_NUMA, D21636, is used) reduces
cross-domain traffic from over 37% down to about 13% as measured
by pcm.x on a dual-socket Xeon using nginx and a Netflix workload.

Reviewed by:	jhb
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D21648
This commit is contained in:
Andrew Gallatin 2020-12-19 21:46:09 +00:00
parent f6d234d870
commit 02bc3865aa
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=368818
1 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
sys/kern/uipc_ktls.c
View File

@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/domainset.h>
#include <sys/ktls.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
@ -83,6 +84,12 @@ struct ktls_wq {
bool running;
} __aligned(CACHE_LINE_SIZE);
struct ktls_domain_info {
int count;
int cpu[MAXCPU];
};
struct ktls_domain_info ktls_domains[MAXMEMDOM];
static struct ktls_wq *ktls_wq;
static struct proc *ktls_proc;
LIST_HEAD(, ktls_crypto_backend) ktls_backends;
@ -316,6 +323,9 @@ static u_int
ktls_get_cpu(struct socket *so)
{
struct inpcb *inp;
#ifdef NUMA
struct ktls_domain_info *di;
#endif
u_int cpuid;
inp = sotoinpcb(so);
@ -330,7 +340,13 @@ ktls_get_cpu(struct socket *so)
* serialization provided by having the same connection use
* the same queue.
*/
cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads];
#ifdef NUMA
if (ktls_bind_threads > 1 && inp->inp_numa_domain != M_NODOM) {
di = &ktls_domains[inp->inp_numa_domain];
cpuid = di->cpu[inp->inp_flowid % di->count];
} else
#endif
cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads];
return (cpuid);
}
#endif
@ -341,7 +357,7 @@ ktls_init(void *dummy __unused)
struct thread *td;
struct pcpu *pc;
cpuset_t mask;
int error, i;
int count, domain, error, i;
ktls_tasks_active = counter_u64_alloc(M_WAITOK);
ktls_cnt_tx_queued = counter_u64_alloc(M_WAITOK);
@ -397,7 +413,11 @@ ktls_init(void *dummy __unused)
if (ktls_bind_threads) {
if (ktls_bind_threads > 1) {
pc = pcpu_find(i);
CPU_COPY(&cpuset_domain[pc->pc_domain], &mask);
domain = pc->pc_domain;
CPU_COPY(&cpuset_domain[domain], &mask);
count = ktls_domains[domain].count;
ktls_domains[domain].cpu[count] = i;
ktls_domains[domain].count++;
} else {
CPU_SETOF(i, &mask);
}
@ -410,6 +430,18 @@ ktls_init(void *dummy __unused)
ktls_cpuid_lookup[ktls_number_threads] = i;
ktls_number_threads++;
}
/*
* If we somehow have an empty domain, fall back to choosing
* among all KTLS threads.
*/
for (i = 0; i < vm_ndomains; i++) {
if (ktls_domains[i].count == 0) {
ktls_bind_threads = 0;
break;
}
}
printf("KTLS: Initialized %d threads\n", ktls_number_threads);
}
SYSINIT(ktls, SI_SUB_SMP + 1, SI_ORDER_ANY, ktls_init, NULL);
@ -2093,6 +2125,10 @@ ktls_work_thread(void *ctx)
STAILQ_HEAD(, mbuf) local_m_head;
STAILQ_HEAD(, socket) local_so_head;
if (ktls_bind_threads > 1) {
curthread->td_domain.dr_policy =
DOMAINSET_PREF(PCPU_GET(domain));
}
#if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
fpu_kern_thread(0);
#endif