Apply MADV_FREE to exec_map entries only after a lowmem event.
This effectively provides the same benefit as applying MADV_FREE inline upon every execve, since the page daemon invokes lowmem handlers prior to scanning the inactive queue. It also has less overhead; the cost of applying MADV_FREE is very noticeable on many-CPU systems since it includes that of a TLB shootdown of global PTEs. For instance, this change nearly halves the system CPU usage during a buildkernel on a 128-vCPU EC2 instance (with some other patches applied). Benchmarked by: cperciva (earlier version) Reviewed by: kib MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D9586
This commit is contained in:
parent
2fce30fa8f
commit
c6a4ba5a38
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=313756
@ -1320,6 +1320,7 @@ exec_copyin_data_fds(struct thread *td, struct image_args *args,
|
||||
|
||||
struct exec_args_kva {
|
||||
vm_offset_t addr;
|
||||
u_int gen;
|
||||
SLIST_ENTRY(exec_args_kva) next;
|
||||
};
|
||||
|
||||
@ -1327,6 +1328,7 @@ static DPCPU_DEFINE(struct exec_args_kva *, exec_args_kva);
|
||||
|
||||
static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist;
|
||||
static struct mtx exec_args_kva_mtx;
|
||||
static u_int exec_args_gen;
|
||||
|
||||
static void
|
||||
exec_prealloc_args_kva(void *arg __unused)
|
||||
@ -1339,6 +1341,7 @@ exec_prealloc_args_kva(void *arg __unused)
|
||||
for (i = 0; i < exec_map_entries; i++) {
|
||||
argkva = malloc(sizeof(*argkva), M_PARGS, M_WAITOK);
|
||||
argkva->addr = kmap_alloc_wait(exec_map, exec_map_entry_size);
|
||||
argkva->gen = exec_args_gen;
|
||||
SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next);
|
||||
}
|
||||
}
|
||||
@ -1364,15 +1367,16 @@ exec_alloc_args_kva(void **cookie)
|
||||
}
|
||||
|
||||
static void
|
||||
exec_free_args_kva(void *cookie)
|
||||
exec_release_args_kva(struct exec_args_kva *argkva, u_int gen)
|
||||
{
|
||||
struct exec_args_kva *argkva;
|
||||
vm_offset_t base;
|
||||
|
||||
argkva = cookie;
|
||||
base = argkva->addr;
|
||||
|
||||
vm_map_madvise(exec_map, base, base + exec_map_entry_size, MADV_FREE);
|
||||
if (argkva->gen != gen) {
|
||||
vm_map_madvise(exec_map, base, base + exec_map_entry_size,
|
||||
MADV_FREE);
|
||||
argkva->gen = gen;
|
||||
}
|
||||
if (!atomic_cmpset_ptr((uintptr_t *)DPCPU_PTR(exec_args_kva),
|
||||
(uintptr_t)NULL, (uintptr_t)argkva)) {
|
||||
mtx_lock(&exec_args_kva_mtx);
|
||||
@ -1382,6 +1386,46 @@ exec_free_args_kva(void *cookie)
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
exec_free_args_kva(void *cookie)
|
||||
{
|
||||
|
||||
exec_release_args_kva(cookie, exec_args_gen);
|
||||
}
|
||||
|
||||
static void
|
||||
exec_args_kva_lowmem(void *arg __unused)
|
||||
{
|
||||
SLIST_HEAD(, exec_args_kva) head;
|
||||
struct exec_args_kva *argkva;
|
||||
u_int gen;
|
||||
int i;
|
||||
|
||||
gen = atomic_fetchadd_int(&exec_args_gen, 1) + 1;
|
||||
|
||||
/*
|
||||
* Force an madvise of each KVA range. Any currently allocated ranges
|
||||
* will have MADV_FREE applied once they are freed.
|
||||
*/
|
||||
SLIST_INIT(&head);
|
||||
mtx_lock(&exec_args_kva_mtx);
|
||||
SLIST_SWAP(&head, &exec_args_kva_freelist, exec_args_kva);
|
||||
mtx_unlock(&exec_args_kva_mtx);
|
||||
while ((argkva = SLIST_FIRST(&head)) != NULL) {
|
||||
SLIST_REMOVE_HEAD(&head, next);
|
||||
exec_release_args_kva(argkva, gen);
|
||||
}
|
||||
|
||||
CPU_FOREACH(i) {
|
||||
argkva = (void *)atomic_readandclear_ptr(
|
||||
(uintptr_t *)DPCPU_ID_PTR(i, exec_args_kva));
|
||||
if (argkva != NULL)
|
||||
exec_release_args_kva(argkva, gen);
|
||||
}
|
||||
}
|
||||
EVENTHANDLER_DEFINE(vm_lowmem, exec_args_kva_lowmem, NULL,
|
||||
EVENTHANDLER_PRI_ANY);
|
||||
|
||||
/*
|
||||
* Allocate temporary demand-paged, zero-filled memory for the file name,
|
||||
* argument, and environment strings.
|
||||
|
Loading…
Reference in New Issue
Block a user