d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Apply MADV_FREE to exec_map entries only after a lowmem event.

Browse Source 
This effectively provides the same benefit as applying MADV_FREE inline
upon every execve, since the page daemon invokes lowmem handlers prior to
scanning the inactive queue. It also has less overhead; the cost of
applying MADV_FREE is very noticeable on many-CPU systems since it includes
that of a TLB shootdown of global PTEs. For instance, this change nearly
halves the system CPU usage during a buildkernel on a 128-vCPU EC2
instance (with some other patches applied).

Benchmarked by:	cperciva (earlier version)
Reviewed by:	kib
MFC after:	2 weeks
Differential Revision:	https://reviews.freebsd.org/D9586
This commit is contained in:
Mark Johnston 2017-02-15 01:50:58 +00:00
parent 2fce30fa8f
commit c6a4ba5a38
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=313756
1 changed files with 49 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
sys/kern/kern_exec.c
View File

@ -1320,6 +1320,7 @@ exec_copyin_data_fds(struct thread *td, struct image_args *args,
struct exec_args_kva {
vm_offset_t addr;
u_int gen;
SLIST_ENTRY(exec_args_kva) next;
};
@ -1327,6 +1328,7 @@ static DPCPU_DEFINE(struct exec_args_kva *, exec_args_kva);
static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist;
static struct mtx exec_args_kva_mtx;
static u_int exec_args_gen;
static void
exec_prealloc_args_kva(void *arg __unused)
@ -1339,6 +1341,7 @@ exec_prealloc_args_kva(void *arg __unused)
for (i = 0; i < exec_map_entries; i++) {
argkva = malloc(sizeof(*argkva), M_PARGS, M_WAITOK);
argkva->addr = kmap_alloc_wait(exec_map, exec_map_entry_size);
argkva->gen = exec_args_gen;
SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next);
}
}
@ -1364,15 +1367,16 @@ exec_alloc_args_kva(void **cookie)
}
static void
exec_free_args_kva(void *cookie)
exec_release_args_kva(struct exec_args_kva *argkva, u_int gen)
{
struct exec_args_kva *argkva;
vm_offset_t base;
argkva = cookie;
base = argkva->addr;
vm_map_madvise(exec_map, base, base + exec_map_entry_size, MADV_FREE);
if (argkva->gen != gen) {
vm_map_madvise(exec_map, base, base + exec_map_entry_size,
MADV_FREE);
argkva->gen = gen;
}
if (!atomic_cmpset_ptr((uintptr_t *)DPCPU_PTR(exec_args_kva),
(uintptr_t)NULL, (uintptr_t)argkva)) {
mtx_lock(&exec_args_kva_mtx);
@ -1382,6 +1386,46 @@ exec_free_args_kva(void *cookie)
}
}
static void
exec_free_args_kva(void *cookie)
{
exec_release_args_kva(cookie, exec_args_gen);
}
static void
exec_args_kva_lowmem(void *arg __unused)
{
SLIST_HEAD(, exec_args_kva) head;
struct exec_args_kva *argkva;
u_int gen;
int i;
gen = atomic_fetchadd_int(&exec_args_gen, 1) + 1;
/*
* Force an madvise of each KVA range. Any currently allocated ranges
* will have MADV_FREE applied once they are freed.
*/
SLIST_INIT(&head);
mtx_lock(&exec_args_kva_mtx);
SLIST_SWAP(&head, &exec_args_kva_freelist, exec_args_kva);
mtx_unlock(&exec_args_kva_mtx);
while ((argkva = SLIST_FIRST(&head)) != NULL) {
SLIST_REMOVE_HEAD(&head, next);
exec_release_args_kva(argkva, gen);
}
CPU_FOREACH(i) {
argkva = (void *)atomic_readandclear_ptr(
(uintptr_t *)DPCPU_ID_PTR(i, exec_args_kva));
if (argkva != NULL)
exec_release_args_kva(argkva, gen);
}
}
EVENTHANDLER_DEFINE(vm_lowmem, exec_args_kva_lowmem, NULL,
EVENTHANDLER_PRI_ANY);
/*
* Allocate temporary demand-paged, zero-filled memory for the file name,
* argument, and environment strings.