From ffc82b0a70cb3a4952d29251c5685bacefbf1424 Mon Sep 17 00:00:00 2001 From: John Dyson Date: Sun, 1 Mar 1998 04:18:54 +0000 Subject: [PATCH] 1) Use a more consistent page wait methodology. 2) Do not unnecessarily force page blocking when paging pages out. 3) Further improve swap pager performance and correctness, including fixing the paging in progress deadlock (except in severe I/O error conditions.) 4) Enable vfs_ioopt=1 as a default. 5) Fix and enable the page prezeroing in SMP mode. All in all, SMP systems especially should show a significant improvement in "snappyness." --- sys/amd64/amd64/mp_machdep.c | 4 +-- sys/amd64/amd64/mptable.c | 4 +-- sys/amd64/amd64/pmap.c | 44 ++++++----------------- sys/amd64/include/mptable.h | 4 +-- sys/i386/i386/mp_machdep.c | 4 +-- sys/i386/i386/mptable.c | 4 +-- sys/i386/i386/pmap.c | 44 ++++++----------------- sys/i386/include/mptable.h | 4 +-- sys/kern/subr_smp.c | 4 +-- sys/kern/vfs_bio.c | 31 ++++------------ sys/kern/vfs_export.c | 4 +-- sys/kern/vfs_subr.c | 4 +-- sys/vm/swap_pager.c | 63 ++++++++++++++++----------------- sys/vm/vm_fault.c | 13 +++---- sys/vm/vm_kern.c | 4 +-- sys/vm/vm_object.c | 68 ++++++++++++------------------------ sys/vm/vm_page.c | 34 ++++++++++++------ sys/vm/vm_page.h | 21 +++++------ sys/vm/vm_pageout.c | 9 ++--- sys/vm/vnode_pager.c | 13 ++----- 20 files changed, 152 insertions(+), 228 deletions(-) diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 7efe94da8d13..7ad037f15ad8 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.185 1998/02/10 17:30:26 eivind Exp $ + * $Id: pmap.c,v 1.186 1998/02/12 22:00:01 bde Exp $ */ /* @@ -824,14 +824,8 @@ pmap_page_lookup(object, pindex) vm_page_t m; retry: m = vm_page_lookup(object, pindex); - if (m) { - if (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "pplookp", 0); - goto retry; - } - } - + if (m && vm_page_sleep(m, "pplookp", NULL)) + goto retry; return m; } @@ -1012,14 +1006,7 @@ static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { int s; - if (m->flags & PG_BUSY) { - s = splvm(); - while (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "pmuwpt", 0); - } - splx(s); - } + while (vm_page_sleep(m, "pmuwpt", NULL)); if (m->hold_count == 0) { vm_offset_t pteva; @@ -1155,7 +1142,7 @@ pmap_pinit(pmap) ptdpg->wire_count = 1; ++cnt.v_wire_count; - ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ + ptdpg->flags &= ~(PG_MAPPED | PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg)); @@ -1189,16 +1176,10 @@ pmap_release_free_page(pmap, p) * page-table pages. Those pages are zero now, and * might as well be placed directly into the zero queue. */ - s = splvm(); - if (p->flags & PG_BUSY) { - p->flags |= PG_WANTED; - tsleep(p, PVM, "pmaprl", 0); - splx(s); + if (vm_page_sleep(p, "pmaprl", NULL)) return 0; - } p->flags |= PG_BUSY; - splx(s); /* * Remove the page table page from the processes address space. @@ -1296,7 +1277,7 @@ _pmap_allocpte(pmap, ptepindex) } m->valid = VM_PAGE_BITS_ALL; - m->flags &= ~(PG_ZERO|PG_BUSY); + m->flags &= ~(PG_ZERO | PG_BUSY); m->flags |= PG_MAPPED; return m; @@ -1564,7 +1545,8 @@ pmap_collect() { m = ppv->pv_vm_page; if ((pa = VM_PAGE_TO_PHYS(m)) == 0) continue; - if (m->wire_count || m->hold_count || m->busy || (m->flags & PG_BUSY)) + if (m->wire_count || m->hold_count || m->busy || + (m->flags & PG_BUSY)) continue; pmap_remove_all(pa); } @@ -2329,15 +2311,11 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)]) return; - s = splhigh(); retry: p = vm_page_lookup(object, pindex); - if (p && (p->flags & PG_BUSY)) { - tsleep(p, PVM, "init4p", 0); + if (p && vm_page_sleep(p, "init4p", NULL)) goto retry; - } - splx(s); - + if (p == NULL) { p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); if (p == NULL) diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 7efe94da8d13..7ad037f15ad8 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.185 1998/02/10 17:30:26 eivind Exp $ + * $Id: pmap.c,v 1.186 1998/02/12 22:00:01 bde Exp $ */ /* @@ -824,14 +824,8 @@ pmap_page_lookup(object, pindex) vm_page_t m; retry: m = vm_page_lookup(object, pindex); - if (m) { - if (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "pplookp", 0); - goto retry; - } - } - + if (m && vm_page_sleep(m, "pplookp", NULL)) + goto retry; return m; } @@ -1012,14 +1006,7 @@ static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { int s; - if (m->flags & PG_BUSY) { - s = splvm(); - while (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "pmuwpt", 0); - } - splx(s); - } + while (vm_page_sleep(m, "pmuwpt", NULL)); if (m->hold_count == 0) { vm_offset_t pteva; @@ -1155,7 +1142,7 @@ pmap_pinit(pmap) ptdpg->wire_count = 1; ++cnt.v_wire_count; - ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ + ptdpg->flags &= ~(PG_MAPPED | PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg)); @@ -1189,16 +1176,10 @@ pmap_release_free_page(pmap, p) * page-table pages. Those pages are zero now, and * might as well be placed directly into the zero queue. */ - s = splvm(); - if (p->flags & PG_BUSY) { - p->flags |= PG_WANTED; - tsleep(p, PVM, "pmaprl", 0); - splx(s); + if (vm_page_sleep(p, "pmaprl", NULL)) return 0; - } p->flags |= PG_BUSY; - splx(s); /* * Remove the page table page from the processes address space. @@ -1296,7 +1277,7 @@ _pmap_allocpte(pmap, ptepindex) } m->valid = VM_PAGE_BITS_ALL; - m->flags &= ~(PG_ZERO|PG_BUSY); + m->flags &= ~(PG_ZERO | PG_BUSY); m->flags |= PG_MAPPED; return m; @@ -1564,7 +1545,8 @@ pmap_collect() { m = ppv->pv_vm_page; if ((pa = VM_PAGE_TO_PHYS(m)) == 0) continue; - if (m->wire_count || m->hold_count || m->busy || (m->flags & PG_BUSY)) + if (m->wire_count || m->hold_count || m->busy || + (m->flags & PG_BUSY)) continue; pmap_remove_all(pa); } @@ -2329,15 +2311,11 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)]) return; - s = splhigh(); retry: p = vm_page_lookup(object, pindex); - if (p && (p->flags & PG_BUSY)) { - tsleep(p, PVM, "init4p", 0); + if (p && vm_page_sleep(p, "init4p", NULL)) goto retry; - } - splx(s); - + if (p == NULL) { p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); if (p == NULL) diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 0b5a16262d8b..d532becb7b1a 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.64 1997/12/15 02:18:20 tegge Exp $ + * $Id: mp_machdep.c,v 1.65 1998/02/09 06:08:13 eivind Exp $ */ #include "opt_smp.h" @@ -1991,7 +1991,7 @@ int invltlb_ok = 0; /* throttle smp_invltlb() till safe */ SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); /* Warning: Do not staticize. Used from swtch.s */ -int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ +int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, &do_page_zero_idle, 0, ""); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 1c3997872a8d..f92023eeb287 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -11,7 +11,7 @@ * 2. Absolutely no warranty of function or purpose is made by the author * John S. Dyson. * - * $Id: vfs_bio.c,v 1.150 1998/02/09 06:09:30 eivind Exp $ + * $Id: vfs_bio.c,v 1.151 1998/02/11 20:06:48 dg Exp $ */ /* @@ -1643,12 +1643,7 @@ allocbuf(struct buf * bp, int size) if (m == bogus_page) panic("allocbuf: bogus page found"); #endif - s = splvm(); - while ((m->flags & PG_BUSY) || (m->busy != 0)) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "biodep", 0); - } - splx(s); + vm_page_sleep(m, "biodep", &m->busy); bp->b_pages[i] = NULL; vm_page_unwire(m); @@ -1940,11 +1935,9 @@ biodone(register struct buf * bp) #endif panic("biodone: page busy < 0\n"); } + m->flags |= PG_BUSY; --m->busy; - if ((m->busy == 0) && (m->flags & PG_WANTED)) { - m->flags &= ~PG_WANTED; - wakeup(m); - } + PAGE_WAKEUP(m); --obj->paging_in_progress; foff += resid; iosize -= resid; @@ -2045,11 +2038,9 @@ vfs_unbusy_pages(struct buf * bp) pmap_qenter(trunc_page(bp->b_data), bp->b_pages, bp->b_npages); } --obj->paging_in_progress; + m->flags |= PG_BUSY; --m->busy; - if ((m->busy == 0) && (m->flags & PG_WANTED)) { - m->flags &= ~PG_WANTED; - wakeup(m); - } + PAGE_WAKEUP(m); } if (obj->paging_in_progress == 0 && (obj->flags & OBJ_PIPWNT)) { @@ -2162,16 +2153,8 @@ vfs_busy_pages(struct buf * bp, int clear_modify) retry: for (i = 0; i < bp->b_npages; i++) { vm_page_t m = bp->b_pages[i]; - - if (m && (m->flags & PG_BUSY)) { - s = splvm(); - while (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "vbpage", 0); - } - splx(s); + if (vm_page_sleep(m, "vbpage", NULL)) goto retry; - } } for (i = 0; i < bp->b_npages; i++, foff += PAGE_SIZE) { diff --git a/sys/kern/vfs_export.c b/sys/kern/vfs_export.c index 07ede04b9169..09adde1658db 100644 --- a/sys/kern/vfs_export.c +++ b/sys/kern/vfs_export.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95 - * $Id: vfs_subr.c,v 1.133 1998/02/10 02:54:24 kato Exp $ + * $Id: vfs_subr.c,v 1.134 1998/02/23 06:59:52 dyson Exp $ */ /* @@ -111,7 +111,7 @@ SYSCTL_INT(_debug, OID_AUTO, wantfreevnodes, CTLFLAG_RW, &wantfreevnodes, 0, "") static u_long freevnodes = 0; SYSCTL_INT(_debug, OID_AUTO, freevnodes, CTLFLAG_RD, &freevnodes, 0, ""); -int vfs_ioopt = 0; +int vfs_ioopt = 1; SYSCTL_INT(_vfs, OID_AUTO, ioopt, CTLFLAG_RW, &vfs_ioopt, 0, ""); struct mntlist mountlist; /* mounted filesystem list */ diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index 07ede04b9169..09adde1658db 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95 - * $Id: vfs_subr.c,v 1.133 1998/02/10 02:54:24 kato Exp $ + * $Id: vfs_subr.c,v 1.134 1998/02/23 06:59:52 dyson Exp $ */ /* @@ -111,7 +111,7 @@ SYSCTL_INT(_debug, OID_AUTO, wantfreevnodes, CTLFLAG_RW, &wantfreevnodes, 0, "") static u_long freevnodes = 0; SYSCTL_INT(_debug, OID_AUTO, freevnodes, CTLFLAG_RD, &freevnodes, 0, ""); -int vfs_ioopt = 0; +int vfs_ioopt = 1; SYSCTL_INT(_vfs, OID_AUTO, ioopt, CTLFLAG_RW, &vfs_ioopt, 0, ""); struct mntlist mountlist; /* mounted filesystem list */ diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 05d8a32776fd..828eab9e5d1e 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -39,7 +39,7 @@ * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ * * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 - * $Id: swap_pager.c,v 1.89 1998/02/23 08:22:24 dyson Exp $ + * $Id: swap_pager.c,v 1.90 1998/02/25 03:55:47 dyson Exp $ */ /* @@ -60,7 +60,7 @@ #include #ifndef MAX_PAGEOUT_CLUSTER -#define MAX_PAGEOUT_CLUSTER 8 +#define MAX_PAGEOUT_CLUSTER 16 #endif #ifndef NPENDINGIO @@ -165,7 +165,6 @@ static int swap_pager_block_offset __P((vm_pindex_t pindex)); static daddr_t *swap_pager_diskaddr __P((vm_object_t object, vm_pindex_t pindex, int *valid)); static void swap_pager_finish __P((swp_clean_t spc)); -static void swap_pager_freepage __P((vm_page_t m)); static void swap_pager_free_swap __P((vm_object_t object)); static void swap_pager_freeswapspace __P((vm_object_t object, unsigned int from, @@ -859,17 +858,6 @@ swap_pager_haspage(object, pindex, before, after) return (FALSE); } -/* - * swap_pager_freepage is a convienience routine that clears the busy - * bit and deallocates a page. - */ -static void -swap_pager_freepage(m) - vm_page_t m; -{ - vm_page_free(m); -} - /* * Wakeup based upon spc state */ @@ -914,9 +902,11 @@ swap_pager_ridpages(m, count, reqpage) { int i; - for (i = 0; i < count; i++) - if (i != reqpage) - swap_pager_freepage(m[i]); + for (i = 0; i < count; i++) { + if (i != reqpage) { + vm_page_free(m[i]); + } + } } /* @@ -967,7 +957,7 @@ swap_pager_getpages(object, m, count, reqpage) return (VM_PAGER_FAIL); } for (j = i; j < count; j++) { - swap_pager_freepage(m[j]); + vm_page_free(m[j]); } count = i; break; @@ -997,7 +987,7 @@ swap_pager_getpages(object, m, count, reqpage) (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || ((reqaddr[i] / dmmax) != reqdskregion)) { failed = 1; - swap_pager_freepage(m[i]); + vm_page_free(m[i]); if (first == 0) first = i + 1; } @@ -1013,7 +1003,7 @@ swap_pager_getpages(object, m, count, reqpage) (reqaddr[i] != (reqaddr[reqpage] + (i - reqpage) * btodb(PAGE_SIZE))) || ((reqaddr[i] / dmmax) != reqdskregion)) { failed = 1; - swap_pager_freepage(m[i]); + vm_page_free(m[i]); if (last == count) last = i; } @@ -1427,7 +1417,6 @@ swap_pager_putpages(object, m, count, sync, rtvals) bp->b_bcount = PAGE_SIZE * ix; bp->b_bufsize = PAGE_SIZE * ix; - s = splvm(); swapdev_vp->v_numoutput++; @@ -1560,7 +1549,7 @@ swap_pager_sync() return; } -static void +void swap_pager_finish(spc) register swp_clean_t spc; { @@ -1569,7 +1558,7 @@ swap_pager_finish(spc) vm_page_t *ma; ma = spc->spc_m; - object = ma[spc->spc_first]->object; + object = spc->spc_object; lastidx = spc->spc_first + spc->spc_count; s = splvm(); @@ -1589,6 +1578,8 @@ swap_pager_finish(spc) printf("swap_pager_finish: I/O error, clean of page %lx failed\n", (u_long) VM_PAGE_TO_PHYS(ma[i])); ma[i]->dirty = VM_PAGE_BITS_ALL; + ma[i]->flags |= PG_BUSY; + ma[i]->busy--; PAGE_WAKEUP(ma[i]); } @@ -1603,8 +1594,9 @@ swap_pager_finish(spc) for (i = spc->spc_first; i < lastidx; i++) { if ((ma[i]->queue != PQ_ACTIVE) && ((ma[i]->flags & PG_WANTED) || - pmap_ts_referenced(VM_PAGE_TO_PHYS(ma[i])))) + pmap_ts_referenced(VM_PAGE_TO_PHYS(ma[i])))) { vm_page_activate(ma[i]); + } } } @@ -1622,16 +1614,18 @@ static void swap_pager_iodone(bp) register struct buf *bp; { - int i, s; + int i, s, lastidx; register swp_clean_t spc; vm_object_t object; + vm_page_t *ma; + s = splvm(); spc = (swp_clean_t) bp->b_spc; TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); TAILQ_INSERT_TAIL(&swap_pager_done, spc, spc_list); - object = bp->b_pages[0]->object; + object = spc->spc_object; #if defined(DIAGNOSTIC) if (object->paging_in_progress < spc->spc_count) @@ -1645,19 +1639,22 @@ swap_pager_iodone(bp) (bp->b_flags & B_READ) ? "pagein" : "pageout", (u_long) bp->b_blkno, bp->b_bcount, bp->b_error); } else { - for (i = 0; i < bp->b_npages; i++) { - /* - * we wakeup any processes that are waiting on these pages. - */ - PAGE_WAKEUP(bp->b_pages[i]); - } - object->paging_in_progress -= spc->spc_count; if ((object->paging_in_progress == 0) && (object->flags & OBJ_PIPWNT)) { object->flags &= ~OBJ_PIPWNT; wakeup(object); } + ma = spc->spc_m; + lastidx = spc->spc_first + spc->spc_count; + for (i = spc->spc_first; i < lastidx; i++) { + /* + * we wakeup any processes that are waiting on these pages. + */ + ma[i]->flags |= PG_BUSY; + ma[i]->busy--; + PAGE_WAKEUP(ma[i]); + } } if (bp->b_vp) diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index bb352bc502e9..a986aebe30cd 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -66,7 +66,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_fault.c,v 1.79 1998/02/06 12:14:22 eivind Exp $ + * $Id: vm_fault.c,v 1.80 1998/02/09 06:11:23 eivind Exp $ */ /* @@ -139,6 +139,7 @@ vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags) vm_page_t marray[VM_FAULT_READ]; int hardfault = 0; int faultcount; + int pagewaitbits; struct vnode *vp = NULL; struct proc *p = curproc; /* XXX */ @@ -181,6 +182,7 @@ vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags) } + pagewaitbits = PG_BUSY; RetryFault:; faultcount = 0; @@ -286,7 +288,6 @@ RetryFault:; /* * See whether this page is resident */ - while (TRUE) { if (object->flags & OBJ_DEAD) { @@ -301,12 +302,12 @@ RetryFault:; * If the page is being brought in, wait for it and * then retry. */ - if ((m->flags & PG_BUSY) || m->busy) { + if ((m->flags & PG_BUSY) || (m->busy && (m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL)) { int s; UNLOCK_THINGS; s = splvm(); - if (((m->flags & PG_BUSY) || m->busy)) { + if ((m->flags & PG_BUSY) || (m->busy && (m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL)) { m->flags |= PG_WANTED | PG_REFERENCED; cnt.v_intrans++; tsleep(m, PSWP, "vmpfw", 0); @@ -331,11 +332,11 @@ RetryFault:; } m->flags |= PG_BUSY; - if (((m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) && m->object != kernel_object && m->object != kmem_object) { goto readrest; } + break; } if (((object->type != OBJT_DEFAULT) && @@ -398,7 +399,7 @@ RetryFault:; if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL)) break; if (mt->busy || - (mt->flags & (PG_BUSY|PG_FICTITIOUS)) || + (mt->flags & (PG_BUSY | PG_FICTITIOUS)) || mt->hold_count || mt->wire_count) continue; diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index 2f4485af7590..179a9ccb2a6c 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_kern.c,v 1.44 1998/02/06 12:14:23 eivind Exp $ + * $Id: vm_kern.c,v 1.45 1998/02/23 08:22:29 dyson Exp $ */ /* @@ -182,7 +182,7 @@ kmem_alloc(map, size) VM_ALLOC_ZERO | VM_ALLOC_RETRY); if ((mem->flags & PG_ZERO) == 0) vm_page_zero_fill(mem); - mem->flags &= ~(PG_BUSY|PG_ZERO); + mem->flags &= ~(PG_BUSY | PG_ZERO); mem->valid = VM_PAGE_BITS_ALL; } diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index ad339d467d1c..fd63b8d644bd 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_object.c,v 1.113 1998/02/09 06:11:30 eivind Exp $ + * $Id: vm_object.c,v 1.114 1998/02/25 03:55:50 dyson Exp $ */ /* @@ -334,20 +334,19 @@ vm_object_deallocate(object) robject->ref_count++; retry: - if (robject->paging_in_progress || object->paging_in_progress) { + if (robject->paging_in_progress || + object->paging_in_progress) { vm_object_pip_sleep(robject, "objde1"); - if (robject->paging_in_progress) { - if (robject->type == OBJT_SWAP) { - swap_pager_sync(); - goto retry; - } + if (robject->paging_in_progress && + robject->type == OBJT_SWAP) { + swap_pager_sync(); + goto retry; } vm_object_pip_sleep(object, "objde2"); - if (object->paging_in_progress) { - if (object->type == OBJT_SWAP) { - swap_pager_sync(); - } + if (object->paging_in_progress && + object->type == OBJT_SWAP) { + swap_pager_sync(); } goto retry; } @@ -569,7 +568,7 @@ vm_object_page_clean(object, start, end, syncio) s = splvm(); while ((p->flags & PG_BUSY) || p->busy) { - p->flags |= PG_WANTED|PG_REFERENCED; + p->flags |= PG_WANTED | PG_REFERENCED; tsleep(p, PVM, "vpcwai", 0); if (object->generation != curgeneration) { splx(s); @@ -581,7 +580,8 @@ vm_object_page_clean(object, start, end, syncio) for(i=1;iflags & PG_BUSY) || - (tp->flags & PG_CLEANCHK) == 0) + (tp->flags & PG_CLEANCHK) == 0 || + (tp->busy != 0)) break; if((tp->queue - tp->pc) == PQ_CACHE) { tp->flags &= ~PG_CLEANCHK; @@ -605,7 +605,8 @@ vm_object_page_clean(object, start, end, syncio) for(i = 1; i < chkb;i++) { if (tp = vm_page_lookup(object, pi - i)) { if ((tp->flags & PG_BUSY) || - (tp->flags & PG_CLEANCHK) == 0) + (tp->flags & PG_CLEANCHK) == 0 || + (tp->busy != 0)) break; if((tp->queue - tp->pc) == PQ_CACHE) { tp->flags &= ~PG_CLEANCHK; @@ -810,15 +811,8 @@ vm_object_madvise(object, pindex, count, advise) continue; } - if (m->busy || (m->flags & PG_BUSY)) { - s = splvm(); - if (m->busy || (m->flags & PG_BUSY)) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "madvpw", 0); - } - splx(s); - goto relookup; - } + if (vm_page_sleep(m, "madvpo", &m->busy)) + goto relookup; if (advise == MADV_WILLNEED) { vm_page_activate(m); @@ -1200,9 +1194,8 @@ vm_object_collapse(object) for (p = TAILQ_FIRST(&backing_object->memq); p; p = TAILQ_NEXT(p, listq)) { - p->flags |= PG_BUSY; - new_pindex = p->pindex - backing_offset_index; + p->flags |= PG_BUSY; /* * If the parent has a page here, or if this @@ -1317,16 +1310,9 @@ vm_object_page_remove(object, start, end, clean_only) * The busy flags are only cleared at * interrupt -- minimize the spl transitions */ - if ((p->flags & PG_BUSY) || p->busy) { - s = splvm(); - if ((p->flags & PG_BUSY) || p->busy) { - p->flags |= PG_WANTED; - tsleep(p, PVM, "vmopar", 0); - splx(s); - goto again; - } - splx(s); - } + + if (vm_page_sleep(p, "vmopar", &p->busy)) + goto again; if (clean_only) { vm_page_test_dirty(p); @@ -1355,16 +1341,8 @@ vm_object_page_remove(object, start, end, clean_only) * The busy flags are only cleared at * interrupt -- minimize the spl transitions */ - if ((p->flags & PG_BUSY) || p->busy) { - s = splvm(); - if ((p->flags & PG_BUSY) || p->busy) { - p->flags |= PG_WANTED; - tsleep(p, PVM, "vmopar", 0); - splx(s); - goto again; - } - splx(s); - } + if (vm_page_sleep(p, "vmopar", &p->busy)) + goto again; if (clean_only) { vm_page_test_dirty(p); diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index c9edfd80659f..439fc3d7c57f 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 - * $Id: vm_page.c,v 1.92 1998/02/06 12:14:27 eivind Exp $ + * $Id: vm_page.c,v 1.93 1998/02/09 06:11:32 eivind Exp $ */ /* @@ -917,7 +917,7 @@ vm_page_alloc(object, pindex, page_req) (*pq->lcnt)--; oldobject = NULL; if (qtype == PQ_ZERO) { - m->flags = PG_ZERO|PG_BUSY; + m->flags = PG_ZERO | PG_BUSY; } else if (qtype == PQ_CACHE) { oldobject = m->object; m->flags |= PG_BUSY; @@ -983,6 +983,22 @@ vm_wait() splx(s); } +int +vm_page_sleep(vm_page_t m, char *msg, char *busy) { + vm_object_t object = m->object; + int generation = object->generation; + if ((busy && *busy) || (m->flags & PG_BUSY)) { + int s; + s = splvm(); + if ((busy && *busy) || (m->flags & PG_BUSY)) { + m->flags |= PG_WANTED; + tsleep(m, PVM, msg, 800); + } + splx(s); + } + return ((generation != object->generation) || (busy && *busy) || + (m->flags & PG_BUSY)); +} /* * vm_page_activate: @@ -1058,6 +1074,8 @@ vm_page_freechk_and_unqueue(m) return 0; } + m->valid = 0; + if (m->wire_count != 0) { #if !defined(MAX_PERF) if (m->wire_count > 1) { @@ -1516,11 +1534,8 @@ contigmalloc1(size, type, flags, low, high, alignment, boundary, map) } next = TAILQ_NEXT(m, pageq); - if (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "vpctw0", 0); + if (vm_page_sleep(m, "vpctw0", &m->busy)) goto again1; - } vm_page_test_dirty(m); if (m->dirty) { if (m->object->type == OBJT_VNODE) { @@ -1530,6 +1545,7 @@ contigmalloc1(size, type, flags, low, high, alignment, boundary, map) goto again1; } else if (m->object->type == OBJT_SWAP || m->object->type == OBJT_DEFAULT) { + m->flags |= PG_BUSY; vm_page_protect(m, VM_PROT_NONE); vm_pageout_flush(&m, 1, 0); goto again1; @@ -1548,11 +1564,8 @@ contigmalloc1(size, type, flags, low, high, alignment, boundary, map) } next = TAILQ_NEXT(m, pageq); - if (m->flags & PG_BUSY) { - m->flags |= PG_WANTED; - tsleep(m, PVM, "vpctw1", 0); + if (vm_page_sleep(m, "vpctw1", &m->busy)) goto again1; - } vm_page_test_dirty(m); if (m->dirty) { if (m->object->type == OBJT_VNODE) { @@ -1562,6 +1575,7 @@ contigmalloc1(size, type, flags, low, high, alignment, boundary, map) goto again1; } else if (m->object->type == OBJT_SWAP || m->object->type == OBJT_DEFAULT) { + m->flags |= PG_BUSY; vm_page_protect(m, VM_PROT_NONE); vm_pageout_flush(&m, 1, 0); goto again1; diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 8a819bb015e4..b9241dc8a307 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_page.h,v 1.35 1997/02/22 09:48:32 peter Exp $ + * $Id: vm_page.h,v 1.36 1998/02/05 03:32:47 dyson Exp $ */ /* @@ -264,17 +264,17 @@ extern vm_offset_t last_phys_addr; /* physical address for last_page */ */ #define PAGE_ASSERT_WAIT(m, interruptible) { \ - (m)->flags |= PG_WANTED; \ - assert_wait((int) (m), (interruptible)); \ - } + (m)->flags |= PG_WANTED; \ + assert_wait((int) (m), (interruptible)); \ +} #define PAGE_WAKEUP(m) { \ - (m)->flags &= ~PG_BUSY; \ - if ((m)->flags & PG_WANTED) { \ - (m)->flags &= ~PG_WANTED; \ - wakeup((caddr_t) (m)); \ - } \ - } + (m)->flags &= ~PG_BUSY; \ + if (((m)->flags & PG_WANTED) && ((m)->busy == 0)) { \ + (m)->flags &= ~PG_WANTED; \ + wakeup((m)); \ + } \ +} #if PAGE_SIZE == 4096 #define VM_PAGE_BITS_ALL 0xff @@ -316,6 +316,7 @@ int vm_page_bits __P((int, int)); vm_page_t vm_page_list_find __P((int, int)); int vm_page_queue_index __P((vm_offset_t, int)); vm_page_t vm_page_select __P((vm_object_t, vm_pindex_t, int)); +int vm_page_sleep(vm_page_t m, char *msg, char *busy); /* * Keep page from being freed by the page daemon diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 6eb7927bba2d..b614c3d55d0d 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -65,7 +65,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_pageout.c,v 1.115 1998/02/23 08:22:37 dyson Exp $ + * $Id: vm_pageout.c,v 1.116 1998/02/24 10:16:23 dyson Exp $ */ /* @@ -190,7 +190,7 @@ SYSCTL_INT(_vm, OID_AUTO, max_page_launder, CTLFLAG_RW, &max_page_launder, 0, ""); -#define VM_PAGEOUT_PAGE_COUNT 8 +#define VM_PAGEOUT_PAGE_COUNT 16 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT; int vm_page_max_wired; /* XXX max # of wired pages system-wide */ @@ -352,7 +352,7 @@ vm_pageout_clean(m, sync) * we allow reads during pageouts... */ for (i = page_base; i < (page_base + pageout_count); i++) { - mc[i]->flags |= PG_BUSY; + mc[i]->busy++; vm_page_protect(mc[i], VM_PROT_READ); } @@ -409,7 +409,6 @@ vm_pageout_flush(mc, count, sync) break; } - /* * If the operation is still going, leave the page busy to * block all other accesses. Also, leave the paging in @@ -418,6 +417,8 @@ vm_pageout_flush(mc, count, sync) */ if (pageout_status[i] != VM_PAGER_PEND) { vm_object_pip_wakeup(object); + mt->flags |= PG_BUSY; + mt->busy--; PAGE_WAKEUP(mt); } } diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index ae934c1c35cd..33c69a3c7135 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -38,7 +38,7 @@ * SUCH DAMAGE. * * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 - * $Id: vnode_pager.c,v 1.86 1998/02/25 03:55:53 dyson Exp $ + * $Id: vnode_pager.c,v 1.87 1998/02/26 06:39:58 msmith Exp $ */ /* @@ -893,15 +893,8 @@ vnode_pager_generic_putpages(vp, m, bytecount, sync, rtvals) printf("vnode_pager_putpages: residual I/O %d at %ld\n", auio.uio_resid, m[0]->pindex); } - for (i = 0; i < count; i++) { - m[i]->busy--; - if (i < ncount) { - rtvals[i] = VM_PAGER_OK; - } - if ((m[i]->busy == 0) && (m[i]->flags & PG_WANTED)) { - vm_page_activate(m[i]); - wakeup(m[i]); - } + for (i = 0; i < ncount; i++) { + rtvals[i] = VM_PAGER_OK; } return rtvals[0]; }