diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 11284ab57fdc..7f29adb78476 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -101,13 +101,6 @@ __FBSDID("$FreeBSD$"); #define MSYNC_FLUSH_HARDSEQ 0x01 #define MSYNC_FLUSH_SOFTSEQ 0x02 -/* - * msync / VM object flushing optimizations - */ -static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ; -SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, CTLFLAG_RW, &msync_flush_flags, 0, - "Enable sequential iteration optimization"); - static int old_msync; SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0, "Use old (insecure) msync behavior"); @@ -762,276 +755,167 @@ vm_object_terminate(vm_object_t object) * The object must be locked. */ void -vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags) +vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, + int flags) { - vm_page_t p, np; - vm_pindex_t tstart, tend; - vm_pindex_t pi; - int clearobjflags; - int pagerflags; - int curgeneration; + vm_page_t np, p; + vm_pindex_t pi, tend; + int clearobjflags, curgeneration, n, pagerflags; mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED); VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); - if ((object->flags & OBJ_MIGHTBEDIRTY) == 0) + if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || + object->resident_page_count == 0) return; KASSERT(object->type == OBJT_VNODE, ("Not a vnode object")); - pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; - pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0; + pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ? + VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK; + pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0; + + tend = (end == 0) ? object->size : end; vm_object_set_flag(object, OBJ_CLEANING); - tstart = start; - if (end == 0) { - tend = object->size; - } else { - tend = end; - } - /* - * If the caller is smart and only msync()s a range he knows is - * dirty, we may be able to avoid an object scan. This results in - * a phenominal improvement in performance. We cannot do this - * as a matter of course because the object may be huge - e.g. - * the size might be in the gigabytes or terrabytes. - */ - if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) { - vm_pindex_t tscan; - int scanlimit; - int scanreset; - - scanreset = object->resident_page_count / EASY_SCAN_FACTOR; - if (scanreset < 16) - scanreset = 16; - pagerflags |= VM_PAGER_IGNORE_CLEANCHK; - - scanlimit = scanreset; - tscan = tstart; - while (tscan < tend) { - curgeneration = object->generation; - p = vm_page_lookup(object, tscan); - if (p == NULL || p->valid == 0) { - if (--scanlimit == 0) - break; - ++tscan; - continue; - } - vm_page_test_dirty(p); - if (p->dirty == 0) { - if (--scanlimit == 0) - break; - ++tscan; - continue; - } - /* - * If we have been asked to skip nosync pages and - * this is a nosync page, we can't continue. - */ - if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) { - if (--scanlimit == 0) - break; - ++tscan; - continue; - } - scanlimit = scanreset; - - /* - * This returns 0 if it was unable to busy the first - * page (i.e. had to sleep). - */ - tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags); - - } - - /* - * If everything was dirty and we flushed it successfully, - * and the requested range is not the entire object, we - * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can - * return immediately. - */ - if (tscan >= tend && (tstart || tend < object->size)) { - vm_object_clear_flag(object, OBJ_CLEANING); - return; - } - pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK; - } - - /* - * Generally set CLEANCHK interlock and make the page read-only so - * we can then clear the object flags. + * Make the page read-only so we can then clear the object flags. * * However, if this is a nosync mmap then the object is likely to * stay dirty so do not mess with the page and do not clear the * object flags. */ clearobjflags = 1; - TAILQ_FOREACH(p, &object->memq, listq) { - p->oflags |= VPO_CLEANCHK; - if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) + for (p = vm_page_find_least(object, start); + p != NULL && p->pindex < tend; p = TAILQ_NEXT(p, listq)) { + if ((flags & OBJPC_NOSYNC) != 0 && + (p->oflags & VPO_NOSYNC) != 0) clearobjflags = 0; else pmap_remove_write(p); } - if (clearobjflags && (tstart == 0) && (tend == object->size)) + if (clearobjflags && (start == 0) && (tend == object->size)) vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); rescan: + p = vm_page_find_least(object, start); curgeneration = object->generation; - for (p = TAILQ_FIRST(&object->memq); p; p = np) { - int n; - - np = TAILQ_NEXT(p, listq); - -again: + for (; p != NULL; p = np) { pi = p->pindex; - if ((p->oflags & VPO_CLEANCHK) == 0 || - (pi < tstart) || (pi >= tend) || - p->valid == 0) { - p->oflags &= ~VPO_CLEANCHK; + if (pi >= tend) + break; + np = TAILQ_NEXT(p, listq); + if (p->valid == 0) continue; + while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { + if (object->generation != curgeneration) + goto rescan; } - vm_page_test_dirty(p); - if (p->dirty == 0) { - p->oflags &= ~VPO_CLEANCHK; + if (p->dirty == 0) continue; - } + /* * If we have been asked to skip nosync pages and this is a * nosync page, skip it. Note that the object flags were * not cleared in this case so we do not have to set them. */ - if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) { - p->oflags &= ~VPO_CLEANCHK; + if ((flags & OBJPC_NOSYNC) != 0 && + (p->oflags & VPO_NOSYNC) != 0) continue; - } n = vm_object_page_collect_flush(object, p, - curgeneration, pagerflags); - if (n == 0) - goto rescan; - + curgeneration, pagerflags); + KASSERT(n > 0, ("vm_object_page_collect_flush failed")); if (object->generation != curgeneration) goto rescan; - - /* - * Try to optimize the next page. If we can't we pick up - * our (random) scan where we left off. - */ - if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) - if ((p = vm_page_lookup(object, pi + n)) != NULL) - goto again; + np = vm_page_find_least(object, pi + n); } #if 0 - VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc); + VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); #endif vm_object_clear_flag(object, OBJ_CLEANING); - return; } static int -vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags) +vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, + int pagerflags) { int runlen; int maxf; int chkb; int maxb; - int i; + int i, index; vm_pindex_t pi; vm_page_t maf[vm_pageout_page_count]; vm_page_t mab[vm_pageout_page_count]; vm_page_t ma[vm_pageout_page_count]; + vm_page_t tp, p1; mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED); vm_page_lock_assert(p, MA_NOTOWNED); VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); pi = p->pindex; - while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { - if (object->generation != curgeneration) { - return(0); - } - } maxf = 0; - for(i = 1; i < vm_pageout_page_count; i++) { - vm_page_t tp; - - if ((tp = vm_page_lookup(object, pi + i)) != NULL) { - if ((tp->oflags & VPO_BUSY) || - ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && - (tp->oflags & VPO_CLEANCHK) == 0) || - (tp->busy != 0)) - break; - vm_page_test_dirty(tp); - if (tp->dirty == 0) { - tp->oflags &= ~VPO_CLEANCHK; - break; - } - maf[ i - 1 ] = tp; - maxf++; - continue; - } - break; + for (i = 1, p1 = p; i < vm_pageout_page_count; i++) { + tp = vm_page_next(p1); + if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0) + break; + vm_page_test_dirty(tp); + if (tp->dirty == 0) + break; + maf[i - 1] = p1 = tp; + maxf++; } maxb = 0; chkb = vm_pageout_page_count - maxf; - if (chkb) { - for(i = 1; i < chkb;i++) { - vm_page_t tp; - - if ((tp = vm_page_lookup(object, pi - i)) != NULL) { - if ((tp->oflags & VPO_BUSY) || - ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && - (tp->oflags & VPO_CLEANCHK) == 0) || - (tp->busy != 0)) - break; - vm_page_test_dirty(tp); - if (tp->dirty == 0) { - tp->oflags &= ~VPO_CLEANCHK; - break; - } - mab[ i - 1 ] = tp; - maxb++; - continue; - } + for (i = 1, p1 = p; i < chkb; i++) { + tp = vm_page_prev(p1); + if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0) break; - } + vm_page_test_dirty(tp); + if (tp->dirty == 0) + break; + mab[i - 1] = p1 = tp; + maxb++; } - for(i = 0; i < maxb; i++) { - int index = (maxb - i) - 1; + for (i = 0; i < maxb; i++) { + index = (maxb - i) - 1; ma[index] = mab[i]; - ma[index]->oflags &= ~VPO_CLEANCHK; } - p->oflags &= ~VPO_CLEANCHK; ma[maxb] = p; - for(i = 0; i < maxf; i++) { - int index = (maxb + i) + 1; + for (i = 0; i < maxf; i++) { + index = (maxb + i) + 1; ma[index] = maf[i]; - ma[index]->oflags &= ~VPO_CLEANCHK; } runlen = maxb + maxf + 1; vm_pageout_flush(ma, runlen, pagerflags); for (i = 0; i < runlen; i++) { - if (ma[i]->dirty) { - pmap_remove_write(ma[i]); - ma[i]->oflags |= VPO_CLEANCHK; - + if (ma[i]->dirty != 0) { + KASSERT((ma[i]->flags & PG_WRITEABLE) == 0, + ("vm_object_page_collect_flush: page %p is not write protected", + ma[i])); + } + } + for (i = 0; i < maxf; i++) { + if (ma[i + maxb + 1]->dirty != 0) { /* * maxf will end up being the actual number of pages * we wrote out contiguously, non-inclusive of the * first page. We do not count look-behind pages. */ - if (i >= maxb + 1 && (maxf > i - maxb - 1)) - maxf = i - maxb - 1; + if (maxf > i) { + maxf = i; + break; + } } } - return(maxf + 1); + return (maxf + 1); } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index c769a7b24a38..1b2f39a7de18 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -143,7 +143,6 @@ struct vm_page { */ #define VPO_BUSY 0x0001 /* page is in transit */ #define VPO_WANTED 0x0002 /* someone is waiting for page */ -#define VPO_CLEANCHK 0x0100 /* page will be checked for cleaning */ #define VPO_SWAPINPROG 0x0200 /* swap I/O in progress on page */ #define VPO_NOSYNC 0x0400 /* do not collect for syncer */ diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h index 75d8f0a873ac..519af4277ff8 100644 --- a/sys/vm/vm_pager.h +++ b/sys/vm/vm_pager.h @@ -90,7 +90,6 @@ extern struct pagerops sgpagerops; #define VM_PAGER_PUT_SYNC 0x0001 #define VM_PAGER_PUT_INVAL 0x0002 -#define VM_PAGER_IGNORE_CLEANCHK 0x0004 #define VM_PAGER_CLUSTER_OK 0x0008 #ifdef _KERNEL