Reimplement vm_object_page_clean(), using the fact that vm object memq
is ordered by page index. This greatly simplifies the implementation, since we no longer need to mark the pages with VPO_CLEANCHK to denote the progress. It is enough to remember the current position by index before dropping the object lock. Remove VPO_CLEANCHK and VM_PAGER_IGNORE_CLEANCHK as unused. Garbage-collect vm.msync_flush_flags sysctl. Suggested and reviewed by: alc Tested by: pho
This commit is contained in:
Konstantin Belousov
parent
b382c10a57
commit
e239bb9730
@ -101,13 +101,6 @@ __FBSDID("$FreeBSD$");
|
||||
#define MSYNC_FLUSH_HARDSEQ 0x01
|
||||
#define MSYNC_FLUSH_SOFTSEQ 0x02
|
||||
|
||||
/*
|
||||
* msync / VM object flushing optimizations
|
||||
*/
|
||||
static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ;
|
||||
SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, CTLFLAG_RW, &msync_flush_flags, 0,
|
||||
"Enable sequential iteration optimization");
|
||||
|
||||
static int old_msync;
|
||||
SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
|
||||
"Use old (insecure) msync behavior");
|
||||
@ -762,276 +755,167 @@ vm_object_terminate(vm_object_t object)
|
||||
* The object must be locked.
|
||||
*/
|
||||
void
|
||||
vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags)
|
||||
vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
|
||||
int flags)
|
||||
{
|
||||
vm_page_t p, np;
|
||||
vm_pindex_t tstart, tend;
|
||||
vm_pindex_t pi;
|
||||
int clearobjflags;
|
||||
int pagerflags;
|
||||
int curgeneration;
|
||||
vm_page_t np, p;
|
||||
vm_pindex_t pi, tend;
|
||||
int clearobjflags, curgeneration, n, pagerflags;
|
||||
|
||||
mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
|
||||
VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
|
||||
if ((object->flags & OBJ_MIGHTBEDIRTY) == 0)
|
||||
if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
|
||||
object->resident_page_count == 0)
|
||||
return;
|
||||
KASSERT(object->type == OBJT_VNODE, ("Not a vnode object"));
|
||||
|
||||
pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
|
||||
pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
|
||||
pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
|
||||
VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
|
||||
pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
|
||||
|
||||
tend = (end == 0) ? object->size : end;
|
||||
|
||||
vm_object_set_flag(object, OBJ_CLEANING);
|
||||
|
||||
tstart = start;
|
||||
if (end == 0) {
|
||||
tend = object->size;
|
||||
} else {
|
||||
tend = end;
|
||||
}
|
||||
|
||||
/*
|
||||
* If the caller is smart and only msync()s a range he knows is
|
||||
* dirty, we may be able to avoid an object scan. This results in
|
||||
* a phenominal improvement in performance. We cannot do this
|
||||
* as a matter of course because the object may be huge - e.g.
|
||||
* the size might be in the gigabytes or terrabytes.
|
||||
*/
|
||||
if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) {
|
||||
vm_pindex_t tscan;
|
||||
int scanlimit;
|
||||
int scanreset;
|
||||
|
||||
scanreset = object->resident_page_count / EASY_SCAN_FACTOR;
|
||||
if (scanreset < 16)
|
||||
scanreset = 16;
|
||||
pagerflags |= VM_PAGER_IGNORE_CLEANCHK;
|
||||
|
||||
scanlimit = scanreset;
|
||||
tscan = tstart;
|
||||
while (tscan < tend) {
|
||||
curgeneration = object->generation;
|
||||
p = vm_page_lookup(object, tscan);
|
||||
if (p == NULL || p->valid == 0) {
|
||||
if (--scanlimit == 0)
|
||||
break;
|
||||
++tscan;
|
||||
continue;
|
||||
}
|
||||
vm_page_test_dirty(p);
|
||||
if (p->dirty == 0) {
|
||||
if (--scanlimit == 0)
|
||||
break;
|
||||
++tscan;
|
||||
continue;
|
||||
}
|
||||
/*
|
||||
* If we have been asked to skip nosync pages and
|
||||
* this is a nosync page, we can't continue.
|
||||
*/
|
||||
if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
|
||||
if (--scanlimit == 0)
|
||||
break;
|
||||
++tscan;
|
||||
continue;
|
||||
}
|
||||
scanlimit = scanreset;
|
||||
|
||||
/*
|
||||
* This returns 0 if it was unable to busy the first
|
||||
* page (i.e. had to sleep).
|
||||
*/
|
||||
tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags);
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* If everything was dirty and we flushed it successfully,
|
||||
* and the requested range is not the entire object, we
|
||||
* don't have to mess with CLEANCHK or MIGHTBEDIRTY and can
|
||||
* return immediately.
|
||||
*/
|
||||
if (tscan >= tend && (tstart || tend < object->size)) {
|
||||
vm_object_clear_flag(object, OBJ_CLEANING);
|
||||
return;
|
||||
}
|
||||
pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK;
|
||||
}
|
||||
|
||||
/*
|
||||
* Generally set CLEANCHK interlock and make the page read-only so
|
||||
* we can then clear the object flags.
|
||||
* Make the page read-only so we can then clear the object flags.
|
||||
*
|
||||
* However, if this is a nosync mmap then the object is likely to
|
||||
* stay dirty so do not mess with the page and do not clear the
|
||||
* object flags.
|
||||
*/
|
||||
clearobjflags = 1;
|
||||
TAILQ_FOREACH(p, &object->memq, listq) {
|
||||
p->oflags |= VPO_CLEANCHK;
|
||||
if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC))
|
||||
for (p = vm_page_find_least(object, start);
|
||||
p != NULL && p->pindex < tend; p = TAILQ_NEXT(p, listq)) {
|
||||
if ((flags & OBJPC_NOSYNC) != 0 &&
|
||||
(p->oflags & VPO_NOSYNC) != 0)
|
||||
clearobjflags = 0;
|
||||
else
|
||||
pmap_remove_write(p);
|
||||
}
|
||||
|
||||
if (clearobjflags && (tstart == 0) && (tend == object->size))
|
||||
if (clearobjflags && (start == 0) && (tend == object->size))
|
||||
vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
|
||||
|
||||
rescan:
|
||||
p = vm_page_find_least(object, start);
|
||||
curgeneration = object->generation;
|
||||
|
||||
for (p = TAILQ_FIRST(&object->memq); p; p = np) {
|
||||
int n;
|
||||
|
||||
np = TAILQ_NEXT(p, listq);
|
||||
|
||||
again:
|
||||
for (; p != NULL; p = np) {
|
||||
pi = p->pindex;
|
||||
if ((p->oflags & VPO_CLEANCHK) == 0 ||
|
||||
(pi < tstart) || (pi >= tend) ||
|
||||
p->valid == 0) {
|
||||
p->oflags &= ~VPO_CLEANCHK;
|
||||
if (pi >= tend)
|
||||
break;
|
||||
np = TAILQ_NEXT(p, listq);
|
||||
if (p->valid == 0)
|
||||
continue;
|
||||
while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
|
||||
if (object->generation != curgeneration)
|
||||
goto rescan;
|
||||
}
|
||||
|
||||
vm_page_test_dirty(p);
|
||||
if (p->dirty == 0) {
|
||||
p->oflags &= ~VPO_CLEANCHK;
|
||||
if (p->dirty == 0)
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* If we have been asked to skip nosync pages and this is a
|
||||
* nosync page, skip it. Note that the object flags were
|
||||
* not cleared in this case so we do not have to set them.
|
||||
*/
|
||||
if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
|
||||
p->oflags &= ~VPO_CLEANCHK;
|
||||
if ((flags & OBJPC_NOSYNC) != 0 &&
|
||||
(p->oflags & VPO_NOSYNC) != 0)
|
||||
continue;
|
||||
}
|
||||
|
||||
n = vm_object_page_collect_flush(object, p,
|
||||
curgeneration, pagerflags);
|
||||
if (n == 0)
|
||||
goto rescan;
|
||||
|
||||
curgeneration, pagerflags);
|
||||
KASSERT(n > 0, ("vm_object_page_collect_flush failed"));
|
||||
if (object->generation != curgeneration)
|
||||
goto rescan;
|
||||
|
||||
/*
|
||||
* Try to optimize the next page. If we can't we pick up
|
||||
* our (random) scan where we left off.
|
||||
*/
|
||||
if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ)
|
||||
if ((p = vm_page_lookup(object, pi + n)) != NULL)
|
||||
goto again;
|
||||
np = vm_page_find_least(object, pi + n);
|
||||
}
|
||||
#if 0
|
||||
VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
|
||||
VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
|
||||
#endif
|
||||
|
||||
vm_object_clear_flag(object, OBJ_CLEANING);
|
||||
return;
|
||||
}
|
||||
|
||||
static int
|
||||
vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags)
|
||||
vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration,
|
||||
int pagerflags)
|
||||
{
|
||||
int runlen;
|
||||
int maxf;
|
||||
int chkb;
|
||||
int maxb;
|
||||
int i;
|
||||
int i, index;
|
||||
vm_pindex_t pi;
|
||||
vm_page_t maf[vm_pageout_page_count];
|
||||
vm_page_t mab[vm_pageout_page_count];
|
||||
vm_page_t ma[vm_pageout_page_count];
|
||||
vm_page_t tp, p1;
|
||||
|
||||
mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
|
||||
vm_page_lock_assert(p, MA_NOTOWNED);
|
||||
VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
|
||||
pi = p->pindex;
|
||||
while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
|
||||
if (object->generation != curgeneration) {
|
||||
return(0);
|
||||
}
|
||||
}
|
||||
maxf = 0;
|
||||
for(i = 1; i < vm_pageout_page_count; i++) {
|
||||
vm_page_t tp;
|
||||
|
||||
if ((tp = vm_page_lookup(object, pi + i)) != NULL) {
|
||||
if ((tp->oflags & VPO_BUSY) ||
|
||||
((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
|
||||
(tp->oflags & VPO_CLEANCHK) == 0) ||
|
||||
(tp->busy != 0))
|
||||
break;
|
||||
vm_page_test_dirty(tp);
|
||||
if (tp->dirty == 0) {
|
||||
tp->oflags &= ~VPO_CLEANCHK;
|
||||
break;
|
||||
}
|
||||
maf[ i - 1 ] = tp;
|
||||
maxf++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
for (i = 1, p1 = p; i < vm_pageout_page_count; i++) {
|
||||
tp = vm_page_next(p1);
|
||||
if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0)
|
||||
break;
|
||||
vm_page_test_dirty(tp);
|
||||
if (tp->dirty == 0)
|
||||
break;
|
||||
maf[i - 1] = p1 = tp;
|
||||
maxf++;
|
||||
}
|
||||
|
||||
maxb = 0;
|
||||
chkb = vm_pageout_page_count - maxf;
|
||||
if (chkb) {
|
||||
for(i = 1; i < chkb;i++) {
|
||||
vm_page_t tp;
|
||||
|
||||
if ((tp = vm_page_lookup(object, pi - i)) != NULL) {
|
||||
if ((tp->oflags & VPO_BUSY) ||
|
||||
((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
|
||||
(tp->oflags & VPO_CLEANCHK) == 0) ||
|
||||
(tp->busy != 0))
|
||||
break;
|
||||
vm_page_test_dirty(tp);
|
||||
if (tp->dirty == 0) {
|
||||
tp->oflags &= ~VPO_CLEANCHK;
|
||||
break;
|
||||
}
|
||||
mab[ i - 1 ] = tp;
|
||||
maxb++;
|
||||
continue;
|
||||
}
|
||||
for (i = 1, p1 = p; i < chkb; i++) {
|
||||
tp = vm_page_prev(p1);
|
||||
if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0)
|
||||
break;
|
||||
}
|
||||
vm_page_test_dirty(tp);
|
||||
if (tp->dirty == 0)
|
||||
break;
|
||||
mab[i - 1] = p1 = tp;
|
||||
maxb++;
|
||||
}
|
||||
|
||||
for(i = 0; i < maxb; i++) {
|
||||
int index = (maxb - i) - 1;
|
||||
for (i = 0; i < maxb; i++) {
|
||||
index = (maxb - i) - 1;
|
||||
ma[index] = mab[i];
|
||||
ma[index]->oflags &= ~VPO_CLEANCHK;
|
||||
}
|
||||
p->oflags &= ~VPO_CLEANCHK;
|
||||
ma[maxb] = p;
|
||||
for(i = 0; i < maxf; i++) {
|
||||
int index = (maxb + i) + 1;
|
||||
for (i = 0; i < maxf; i++) {
|
||||
index = (maxb + i) + 1;
|
||||
ma[index] = maf[i];
|
||||
ma[index]->oflags &= ~VPO_CLEANCHK;
|
||||
}
|
||||
runlen = maxb + maxf + 1;
|
||||
|
||||
vm_pageout_flush(ma, runlen, pagerflags);
|
||||
for (i = 0; i < runlen; i++) {
|
||||
if (ma[i]->dirty) {
|
||||
pmap_remove_write(ma[i]);
|
||||
ma[i]->oflags |= VPO_CLEANCHK;
|
||||
|
||||
if (ma[i]->dirty != 0) {
|
||||
KASSERT((ma[i]->flags & PG_WRITEABLE) == 0,
|
||||
("vm_object_page_collect_flush: page %p is not write protected",
|
||||
ma[i]));
|
||||
}
|
||||
}
|
||||
for (i = 0; i < maxf; i++) {
|
||||
if (ma[i + maxb + 1]->dirty != 0) {
|
||||
/*
|
||||
* maxf will end up being the actual number of pages
|
||||
* we wrote out contiguously, non-inclusive of the
|
||||
* first page. We do not count look-behind pages.
|
||||
*/
|
||||
if (i >= maxb + 1 && (maxf > i - maxb - 1))
|
||||
maxf = i - maxb - 1;
|
||||
if (maxf > i) {
|
||||
maxf = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return(maxf + 1);
|
||||
return (maxf + 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -143,7 +143,6 @@ struct vm_page {
|
||||
*/
|
||||
#define VPO_BUSY 0x0001 /* page is in transit */
|
||||
#define VPO_WANTED 0x0002 /* someone is waiting for page */
|
||||
#define VPO_CLEANCHK 0x0100 /* page will be checked for cleaning */
|
||||
#define VPO_SWAPINPROG 0x0200 /* swap I/O in progress on page */
|
||||
#define VPO_NOSYNC 0x0400 /* do not collect for syncer */
|
||||
|
||||
|
||||
@ -90,7 +90,6 @@ extern struct pagerops sgpagerops;
|
||||
|
||||
#define VM_PAGER_PUT_SYNC 0x0001
|
||||
#define VM_PAGER_PUT_INVAL 0x0002
|
||||
#define VM_PAGER_IGNORE_CLEANCHK 0x0004
|
||||
#define VM_PAGER_CLUSTER_OK 0x0008
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
Loading…
Reference in New Issue
Block a user