Fix two problems with the page daemon control loop.
Both issues caused the page daemon to erroneously go to sleep when applications are consuming free pages at a high rate, leaving the application threads blocked in VM_WAIT. 1) After completing an inactive queue scan, concurrent allocations may have prevented the page daemon from meeting the v_free_min threshold. In this case, the page daemon was going to sleep even when the inactive queue contained plenty of clean pages. 2) pagedaemon_wakeup() may be called without the free queues lock held. This can lead to a lost wakeup if a call occurs after the page daemon clears vm_pageout_wanted but before going to sleep. Fix 1) by ensuring that we start a new inactive queue scan immediately if v_free_count < v_free_min after a prior scan. Fix 2) by adding a new subroutine, pagedaemon_wait(), called from vm_wait() and vm_waitpfault(). It wakes up the page daemon if either vm_pages_needed or vm_pageout_wanted is false, and atomically sleeps on v_free_count. Reported by: jeff Reviewed by: alc MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D13424
This commit is contained in:
parent
e42df78a1a
commit
280d15cd0a
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=327168
@ -2661,15 +2661,9 @@ _vm_wait(void)
|
||||
msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
|
||||
PDROP | PSWP, "VMWait", 0);
|
||||
} else {
|
||||
if (__predict_false(pageproc == NULL))
|
||||
if (pageproc == NULL)
|
||||
panic("vm_wait in early boot");
|
||||
if (!vm_pageout_wanted) {
|
||||
vm_pageout_wanted = true;
|
||||
wakeup(&vm_pageout_wanted);
|
||||
}
|
||||
vm_pages_needed = true;
|
||||
msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
|
||||
"vmwait", 0);
|
||||
pagedaemon_wait(PVM, "vmwait");
|
||||
}
|
||||
}
|
||||
|
||||
@ -2699,7 +2693,6 @@ vm_page_alloc_fail(vm_object_t object, int req)
|
||||
|
||||
atomic_add_int(&vm_pageout_deficit,
|
||||
max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
|
||||
pagedaemon_wakeup();
|
||||
if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) {
|
||||
if (object != NULL)
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
@ -2708,8 +2701,10 @@ vm_page_alloc_fail(vm_object_t object, int req)
|
||||
VM_OBJECT_WLOCK(object);
|
||||
if (req & VM_ALLOC_WAITOK)
|
||||
return (EAGAIN);
|
||||
} else
|
||||
} else {
|
||||
mtx_unlock(&vm_page_queue_free_mtx);
|
||||
pagedaemon_wakeup();
|
||||
}
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -2728,13 +2723,7 @@ vm_waitpfault(void)
|
||||
{
|
||||
|
||||
mtx_lock(&vm_page_queue_free_mtx);
|
||||
if (!vm_pageout_wanted) {
|
||||
vm_pageout_wanted = true;
|
||||
wakeup(&vm_pageout_wanted);
|
||||
}
|
||||
vm_pages_needed = true;
|
||||
msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
|
||||
"pfault", 0);
|
||||
pagedaemon_wait(PUSER, "pfault");
|
||||
}
|
||||
|
||||
struct vm_pagequeue *
|
||||
|
@ -1829,10 +1829,14 @@ vm_pageout_worker(void *arg)
|
||||
pass++;
|
||||
} else {
|
||||
/*
|
||||
* Yes. Sleep until pages need to be reclaimed or
|
||||
* Yes. If threads are still sleeping in VM_WAIT
|
||||
* then we immediately start a new scan. Otherwise,
|
||||
* sleep until the next wakeup or until pages need to
|
||||
* have their reference stats updated.
|
||||
*/
|
||||
if (mtx_sleep(&vm_pageout_wanted,
|
||||
if (vm_pages_needed) {
|
||||
mtx_unlock(&vm_page_queue_free_mtx);
|
||||
} else if (mtx_sleep(&vm_pageout_wanted,
|
||||
&vm_page_queue_free_mtx, PDROP | PVM, "psleep",
|
||||
hz) == 0) {
|
||||
VM_CNT_INC(v_pdwakeups);
|
||||
@ -1940,17 +1944,42 @@ vm_pageout(void)
|
||||
}
|
||||
|
||||
/*
|
||||
* Unless the free page queue lock is held by the caller, this function
|
||||
* should be regarded as advisory. Specifically, the caller should
|
||||
* not msleep() on &vm_cnt.v_free_count following this function unless
|
||||
* the free page queue lock is held until the msleep() is performed.
|
||||
* Perform an advisory wakeup of the page daemon.
|
||||
*/
|
||||
void
|
||||
pagedaemon_wakeup(void)
|
||||
{
|
||||
|
||||
mtx_assert(&vm_page_queue_free_mtx, MA_NOTOWNED);
|
||||
|
||||
if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
|
||||
vm_pageout_wanted = true;
|
||||
wakeup(&vm_pageout_wanted);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Wake up the page daemon and wait for it to reclaim free pages.
|
||||
*
|
||||
* This function returns with the free queues mutex unlocked.
|
||||
*/
|
||||
void
|
||||
pagedaemon_wait(int pri, const char *wmesg)
|
||||
{
|
||||
|
||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
|
||||
|
||||
/*
|
||||
* vm_pageout_wanted may have been set by an advisory wakeup, but if the
|
||||
* page daemon is running on a CPU, the wakeup will have been lost.
|
||||
* Thus, deliver a potentially spurious wakeup to ensure that the page
|
||||
* daemon has been notified of the shortage.
|
||||
*/
|
||||
if (!vm_pageout_wanted || !vm_pages_needed) {
|
||||
vm_pageout_wanted = true;
|
||||
wakeup(&vm_pageout_wanted);
|
||||
}
|
||||
vm_pages_needed = true;
|
||||
msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | pri,
|
||||
wmesg, 0);
|
||||
}
|
||||
|
@ -96,11 +96,12 @@ extern bool vm_pages_needed;
|
||||
* Signal pageout-daemon and wait for it.
|
||||
*/
|
||||
|
||||
extern void pagedaemon_wakeup(void);
|
||||
void pagedaemon_wait(int pri, const char *wmesg);
|
||||
void pagedaemon_wakeup(void);
|
||||
#define VM_WAIT vm_wait()
|
||||
#define VM_WAITPFAULT vm_waitpfault()
|
||||
extern void vm_wait(void);
|
||||
extern void vm_waitpfault(void);
|
||||
void vm_wait(void);
|
||||
void vm_waitpfault(void);
|
||||
|
||||
#ifdef _KERNEL
|
||||
int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *);
|
||||
|
Loading…
Reference in New Issue
Block a user