d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix a horribly suboptimal algorithm in the vm_daemon.

Browse Source 
In order to determine what to page out, the vm_daemon checks
reference bits on all pages belonging to all processes.  Unfortunately,
the algorithm used reacted badly with shared pages; each shared page
would be checked once per process sharing it; this caused an O(N^2)
growth of tlb invalidations.  The algorithm has been changed so that
each page will be checked only 16 times.

Prior to this change, a fork/sleepbomb of 1300 processes could cause
the vm_daemon to take over 60 seconds to complete, effectively
freezing the system for that time period.  With this change
in place, the vm_daemon completes in less than a second.  Any system
with hundreds of processes sharing pages should benefit from this change.

Note that the vm_daemon is only run when the system is under extreme
memory pressure.  It is likely that many people with loaded systems saw
no symptoms of this problem until they reached the point where swapping
began.

Special thanks go to dillon, peter, and Chuck Cranor, who helped me
get up to speed with vm internals.

PR:		33542, 20393
Reviewed by:	dillon
MFC after:	1 week
This commit is contained in:
Mike Silbersack 2002-02-27 18:03:02 +00:00
parent 01dc2ef5b5
commit 7f3a40933b
11 changed files with 169 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
sys/alpha/alpha/pmap.c
View File

@ -2745,15 +2745,19 @@ pmap_pageable(pmap, sva, eva, pageable)
}
/*
* this routine returns true if a physical page resides
* in the given pmap.
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap, m)
pmap_page_exists_quick(pmap, m)
pmap_t pmap;
vm_page_t m;
{
register pv_entry_t pv;
pv_entry_t pv;
int loops = 0;
int s;
if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
@ -2769,6 +2773,9 @@ pmap_page_exists(pmap, m)
splx(s);
return TRUE;
}
loops++;
if (loops >= 16)
break;
}
splx(s);
return (FALSE);
@ -2932,7 +2939,14 @@ pmap_phys_address(ppn)
/*
* pmap_ts_referenced:
*
* Return the count of reference bits for a page, clearing all of them.
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)

27
sys/amd64/amd64/pmap.c
View File

@ -2933,15 +2933,19 @@ pmap_pageable(pmap, sva, eva, pageable)
}
/*
* this routine returns true if a physical page resides
* in the given pmap.
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap, m)
pmap_page_exists_quick(pmap, m)
pmap_t pmap;
vm_page_t m;
{
register pv_entry_t pv;
pv_entry_t pv;
int loops = 0;
int s;
if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
@ -2949,14 +2953,14 @@ pmap_page_exists(pmap, m)
s = splvm();
/*
* Not found, check current mappings returning immediately if found.
*/
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
splx(s);
return TRUE;
}
loops++;
if (loops >= 16)
break;
}
splx(s);
return (FALSE);
@ -3186,7 +3190,14 @@ pmap_phys_address(ppn)
/*
* pmap_ts_referenced:
*
* Return the count of reference bits for a page, clearing all of them.
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)

27
sys/i386/i386/pmap.c
View File

@ -2933,15 +2933,19 @@ pmap_pageable(pmap, sva, eva, pageable)
}
/*
* this routine returns true if a physical page resides
* in the given pmap.
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap, m)
pmap_page_exists_quick(pmap, m)
pmap_t pmap;
vm_page_t m;
{
register pv_entry_t pv;
pv_entry_t pv;
int loops = 0;
int s;
if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
@ -2949,14 +2953,14 @@ pmap_page_exists(pmap, m)
s = splvm();
/*
* Not found, check current mappings returning immediately if found.
*/
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pmap) {
splx(s);
return TRUE;
}
loops++;
if (loops >= 16)
break;
}
splx(s);
return (FALSE);
@ -3186,7 +3190,14 @@ pmap_phys_address(ppn)
/*
* pmap_ts_referenced:
*
* Return the count of reference bits for a page, clearing all of them.
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)

25
sys/ia64/ia64/pmap.c
View File

@ -2152,13 +2152,17 @@ pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
}
/*
* this routine returns true if a physical page resides
* in the given pmap.
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap_t pmap, vm_page_t m)
pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
{
register pv_entry_t pv;
pv_entry_t pv;
int loops = 0;
int s;
if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
@ -2174,6 +2178,9 @@ pmap_page_exists(pmap_t pmap, vm_page_t m)
splx(s);
return TRUE;
}
loops++;
if (loops >= 16)
break;
}
splx(s);
return (FALSE);
@ -2269,8 +2276,14 @@ pmap_phys_address(int ppn)
/*
* pmap_ts_referenced:
*
* Return the count of reference bits for a page, clearing all of them.
*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)

22
sys/powerpc/aim/mmu_oea.c
View File

@ -936,6 +936,19 @@ pmap_clear_reference(vm_page_t m)
TODO;
}
/*
* pmap_ts_referenced:
*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)
{
@ -1160,8 +1173,15 @@ pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
{
}
/*
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap_t pmap, vm_page_t m)
pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
{
TODO;
return (0);

22
sys/powerpc/powerpc/mmu_oea.c
View File

@ -936,6 +936,19 @@ pmap_clear_reference(vm_page_t m)
TODO;
}
/*
* pmap_ts_referenced:
*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)
{
@ -1160,8 +1173,15 @@ pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
{
}
/*
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap_t pmap, vm_page_t m)
pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
{
TODO;
return (0);

22
sys/powerpc/powerpc/pmap.c
View File

@ -936,6 +936,19 @@ pmap_clear_reference(vm_page_t m)
TODO;
}
/*
* pmap_ts_referenced:
*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)
{
@ -1160,8 +1173,15 @@ pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
{
}
/*
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap_t pmap, vm_page_t m)
pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
{
TODO;
return (0);

21
sys/sparc64/sparc64/pmap.c
View File

@ -1602,10 +1602,14 @@ pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
}
/*
* Return true of a physical page resided in the given pmap.
* Returns true if the pmap's pv is one of the first
* 16 pvs linked to from this page. This count may
* be changed upwards or downwards in the future; it
* is only necessary that true be returned for a small
* subset of pmaps for proper page aging.
*/
boolean_t
pmap_page_exists(pmap_t pm, vm_page_t m)
pmap_page_exists_quick(pmap_t pm, vm_page_t m)
{
if (m->flags & PG_FICTITIOUS)
@ -1682,6 +1686,19 @@ pmap_phys_address(int ppn)
return (sparc64_ptob(ppn));
}
/*
* pmap_ts_referenced:
*
* Return a count of reference bits for a page, clearing those bits.
* It is not necessary for every reference bit to be cleared, but it
* is necessary that 0 only be returned when there are truly no
* reference bits set.
*
* XXX: The exact number of bits to check and clear is a matter that
* should be tested and standardized at some point in the future for
* optimal aging of shared pages.
*/
int
pmap_ts_referenced(vm_page_t m)
{

9
sys/sparc64/sparc64/pv.c
View File

@ -215,15 +215,24 @@ pv_bit_test(vm_page_t m, u_long bits)
return (FALSE);
}
/*
* See pmap_page_exists_quick for operational explanation of
* pv_page_exists.
*/
int
pv_page_exists(pmap_t pm, vm_page_t m)
{
pv_entry_t pv;
int loops = 0;
TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
if (pv->pv_pmap == pm) {
return (TRUE);
}
loops++;
if (loops >= 16)
break;
}
return (FALSE);
}

2
sys/vm/pmap.h
View File

@ -115,7 +115,7 @@ vm_offset_t pmap_map __P((vm_offset_t *, vm_offset_t, vm_offset_t, int));
void pmap_object_init_pt __P((pmap_t pmap, vm_offset_t addr,
vm_object_t object, vm_pindex_t pindex, vm_offset_t size,
int pagelimit));
boolean_t pmap_page_exists __P((pmap_t pmap, vm_page_t m));
boolean_t pmap_page_exists_quick __P((pmap_t pmap, vm_page_t m));
void pmap_page_protect __P((vm_page_t m, vm_prot_t prot));
void pmap_pageable __P((pmap_t, vm_offset_t, vm_offset_t,
boolean_t));

2
sys/vm/vm_pageout.c
View File

@ -494,7 +494,7 @@ vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only)
p->hold_count != 0 ||
p->busy != 0 ||
(p->flags & (PG_BUSY|PG_UNMANAGED)) ||
!pmap_page_exists(vm_map_pmap(map), p)) {
!pmap_page_exists_quick(vm_map_pmap(map), p)) {
p = next;
continue;
}