Defer and aggregate swap_pager_meta_build frees.
Before swp_pager_meta_build replaces an old swapblk with an new one, it frees the old one. To allow such freeing of blocks to be aggregated, have swp_pager_meta_build return the old swap block, and make the caller responsible for freeing it. Define a pair of short static functions, swp_pager_init_freerange and swp_pager_update_freerange, to do the initialization and updating of blk addresses and counters used in aggregating blocks to be freed. Submitted by: Doug Moore <dougm@rice.edu> Reviewed by: kib, markj (an earlier version) Tested by: pho MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D13707
This commit is contained in:
Alan Cox
parent
2dc4dbb967
commit
78f1deeffe
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=337443
@ -403,11 +403,32 @@ static daddr_t swp_pager_getswapspace(int npages);
|
||||
/*
|
||||
* Metadata functions
|
||||
*/
|
||||
static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
|
||||
static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
|
||||
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
|
||||
static void swp_pager_meta_free_all(vm_object_t);
|
||||
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
|
||||
|
||||
static void
|
||||
swp_pager_init_freerange(daddr_t *start, daddr_t *num)
|
||||
{
|
||||
|
||||
*start = SWAPBLK_NONE;
|
||||
*num = 0;
|
||||
}
|
||||
|
||||
static void
|
||||
swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr)
|
||||
{
|
||||
|
||||
if (*start + *num == addr) {
|
||||
(*num)++;
|
||||
} else {
|
||||
swp_pager_freeswapspace(*start, *num);
|
||||
*start = addr;
|
||||
*num = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void *
|
||||
swblk_trie_alloc(struct pctrie *ptree)
|
||||
{
|
||||
@ -861,7 +882,9 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
|
||||
int n = 0;
|
||||
daddr_t blk = SWAPBLK_NONE;
|
||||
vm_pindex_t beg = start; /* save start index */
|
||||
daddr_t addr, n_free, s_free;
|
||||
|
||||
swp_pager_init_freerange(&s_free, &n_free);
|
||||
VM_OBJECT_WLOCK(object);
|
||||
while (size) {
|
||||
if (n == 0) {
|
||||
@ -875,12 +898,15 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
|
||||
}
|
||||
}
|
||||
}
|
||||
swp_pager_meta_build(object, start, blk);
|
||||
addr = swp_pager_meta_build(object, start, blk);
|
||||
if (addr != SWAPBLK_NONE)
|
||||
swp_pager_update_freerange(&s_free, &n_free, addr);
|
||||
--size;
|
||||
++start;
|
||||
++blk;
|
||||
--n;
|
||||
}
|
||||
swp_pager_freeswapspace(s_free, n_free);
|
||||
swp_pager_meta_free(object, start, n);
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
return (0);
|
||||
@ -910,7 +936,7 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
|
||||
vm_pindex_t offset, int destroysource)
|
||||
{
|
||||
vm_pindex_t i;
|
||||
daddr_t dstaddr, first_free, num_free, srcaddr;
|
||||
daddr_t dstaddr, n_free, s_free, srcaddr;
|
||||
|
||||
VM_OBJECT_ASSERT_WLOCKED(srcobject);
|
||||
VM_OBJECT_ASSERT_WLOCKED(dstobject);
|
||||
@ -937,42 +963,38 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
|
||||
/*
|
||||
* Transfer source to destination.
|
||||
*/
|
||||
first_free = SWAPBLK_NONE;
|
||||
num_free = 0;
|
||||
swp_pager_init_freerange(&s_free, &n_free);
|
||||
for (i = 0; i < dstobject->size; ++i) {
|
||||
srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);
|
||||
if (srcaddr == SWAPBLK_NONE)
|
||||
continue;
|
||||
dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
|
||||
if (dstaddr == SWAPBLK_NONE) {
|
||||
/*
|
||||
* Destination has no swapblk and is not resident,
|
||||
* copy source.
|
||||
*
|
||||
* swp_pager_meta_build() can sleep.
|
||||
*/
|
||||
vm_object_pip_add(srcobject, 1);
|
||||
VM_OBJECT_WUNLOCK(srcobject);
|
||||
vm_object_pip_add(dstobject, 1);
|
||||
swp_pager_meta_build(dstobject, i, srcaddr);
|
||||
vm_object_pip_wakeup(dstobject);
|
||||
VM_OBJECT_WLOCK(srcobject);
|
||||
vm_object_pip_wakeup(srcobject);
|
||||
} else {
|
||||
if (dstaddr != SWAPBLK_NONE) {
|
||||
/*
|
||||
* Destination has valid swapblk or it is represented
|
||||
* by a resident page. We destroy the sourceblock.
|
||||
* by a resident page. We destroy the source block.
|
||||
*/
|
||||
if (first_free + num_free == srcaddr)
|
||||
num_free++;
|
||||
else {
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
first_free = srcaddr;
|
||||
num_free = 1;
|
||||
}
|
||||
swp_pager_update_freerange(&s_free, &n_free, srcaddr);
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* Destination has no swapblk and is not resident,
|
||||
* copy source.
|
||||
*
|
||||
* swp_pager_meta_build() can sleep.
|
||||
*/
|
||||
vm_object_pip_add(srcobject, 1);
|
||||
VM_OBJECT_WUNLOCK(srcobject);
|
||||
vm_object_pip_add(dstobject, 1);
|
||||
dstaddr = swp_pager_meta_build(dstobject, i, srcaddr);
|
||||
KASSERT(dstaddr == SWAPBLK_NONE,
|
||||
("Unexpected destination swapblk"));
|
||||
vm_object_pip_wakeup(dstobject);
|
||||
VM_OBJECT_WLOCK(srcobject);
|
||||
vm_object_pip_wakeup(srcobject);
|
||||
}
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
swp_pager_freeswapspace(s_free, n_free);
|
||||
|
||||
/*
|
||||
* Free left over swap blocks in source.
|
||||
@ -1307,7 +1329,9 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
|
||||
{
|
||||
int i, n;
|
||||
boolean_t sync;
|
||||
daddr_t addr, n_free, s_free;
|
||||
|
||||
swp_pager_init_freerange(&s_free, &n_free);
|
||||
if (count && ma[0]->object != object) {
|
||||
panic("swap_pager_putpages: object mismatch %p/%p",
|
||||
object,
|
||||
@ -1322,8 +1346,11 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
|
||||
* check for bogus sysops
|
||||
* force sync if not pageout process
|
||||
*/
|
||||
if (object->type != OBJT_SWAP)
|
||||
swp_pager_meta_build(object, 0, SWAPBLK_NONE);
|
||||
if (object->type != OBJT_SWAP) {
|
||||
addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE);
|
||||
KASSERT(addr == SWAPBLK_NONE,
|
||||
("unexpected object swap block"));
|
||||
}
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
|
||||
n = 0;
|
||||
@ -1391,11 +1418,11 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
|
||||
for (j = 0; j < n; ++j) {
|
||||
vm_page_t mreq = ma[i+j];
|
||||
|
||||
swp_pager_meta_build(
|
||||
mreq->object,
|
||||
mreq->pindex,
|
||||
blk + j
|
||||
);
|
||||
addr = swp_pager_meta_build(mreq->object, mreq->pindex,
|
||||
blk + j);
|
||||
if (addr != SWAPBLK_NONE)
|
||||
swp_pager_update_freerange(&s_free, &n_free,
|
||||
addr);
|
||||
MPASS(mreq->dirty == VM_PAGE_BITS_ALL);
|
||||
mreq->oflags |= VPO_SWAPINPROG;
|
||||
bp->b_pages[j] = mreq;
|
||||
@ -1453,6 +1480,7 @@ swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
|
||||
swp_pager_async_iodone(bp);
|
||||
}
|
||||
VM_OBJECT_WLOCK(object);
|
||||
swp_pager_freeswapspace(s_free, n_free);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1783,14 +1811,15 @@ swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
|
||||
*
|
||||
* The specified swapblk is added to the object's swap metadata. If
|
||||
* the swapblk is not valid, it is freed instead. Any previously
|
||||
* assigned swapblk is freed.
|
||||
* assigned swapblk is returned.
|
||||
*/
|
||||
static void
|
||||
static daddr_t
|
||||
swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
|
||||
{
|
||||
static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted;
|
||||
struct swblk *sb, *sb1;
|
||||
vm_pindex_t modpi, rdpi;
|
||||
daddr_t prev_swapblk;
|
||||
int error, i;
|
||||
|
||||
VM_OBJECT_ASSERT_WLOCKED(object);
|
||||
@ -1815,7 +1844,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
|
||||
sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);
|
||||
if (sb == NULL) {
|
||||
if (swapblk == SWAPBLK_NONE)
|
||||
return;
|
||||
return (SWAPBLK_NONE);
|
||||
for (;;) {
|
||||
sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc ==
|
||||
pageproc ? M_USE_RESERVE : 0));
|
||||
@ -1882,9 +1911,8 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
|
||||
MPASS(sb->p == rdpi);
|
||||
|
||||
modpi = pindex % SWAP_META_PAGES;
|
||||
/* Delete prior contents of metadata. */
|
||||
if (sb->d[modpi] != SWAPBLK_NONE)
|
||||
swp_pager_freeswapspace(sb->d[modpi], 1);
|
||||
/* Return prior contents of metadata. */
|
||||
prev_swapblk = sb->d[modpi];
|
||||
/* Enter block into metadata. */
|
||||
sb->d[modpi] = swapblk;
|
||||
|
||||
@ -1896,6 +1924,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
return (prev_swapblk);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1912,7 +1941,7 @@ static void
|
||||
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
|
||||
{
|
||||
struct swblk *sb;
|
||||
daddr_t first_free, num_free;
|
||||
daddr_t n_free, s_free;
|
||||
vm_pindex_t last;
|
||||
int i, limit, start;
|
||||
|
||||
@ -1920,8 +1949,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
|
||||
if (object->type != OBJT_SWAP || count == 0)
|
||||
return;
|
||||
|
||||
first_free = SWAPBLK_NONE;
|
||||
num_free = 0;
|
||||
swp_pager_init_freerange(&s_free, &n_free);
|
||||
last = pindex + count;
|
||||
for (;;) {
|
||||
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
|
||||
@ -1934,13 +1962,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
|
||||
for (i = start; i < limit; i++) {
|
||||
if (sb->d[i] == SWAPBLK_NONE)
|
||||
continue;
|
||||
if (first_free + num_free == sb->d[i])
|
||||
num_free++;
|
||||
else {
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
first_free = sb->d[i];
|
||||
num_free = 1;
|
||||
}
|
||||
swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
|
||||
sb->d[i] = SWAPBLK_NONE;
|
||||
}
|
||||
if (swp_pager_swblk_empty(sb, 0, start) &&
|
||||
@ -1951,7 +1973,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
|
||||
}
|
||||
pindex = sb->p + SWAP_META_PAGES;
|
||||
}
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
swp_pager_freeswapspace(s_free, n_free);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1964,7 +1986,7 @@ static void
|
||||
swp_pager_meta_free_all(vm_object_t object)
|
||||
{
|
||||
struct swblk *sb;
|
||||
daddr_t first_free, num_free;
|
||||
daddr_t n_free, s_free;
|
||||
vm_pindex_t pindex;
|
||||
int i;
|
||||
|
||||
@ -1972,26 +1994,19 @@ swp_pager_meta_free_all(vm_object_t object)
|
||||
if (object->type != OBJT_SWAP)
|
||||
return;
|
||||
|
||||
first_free = SWAPBLK_NONE;
|
||||
num_free = 0;
|
||||
swp_pager_init_freerange(&s_free, &n_free);
|
||||
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
|
||||
&object->un_pager.swp.swp_blks, pindex)) != NULL;) {
|
||||
pindex = sb->p + SWAP_META_PAGES;
|
||||
for (i = 0; i < SWAP_META_PAGES; i++) {
|
||||
if (sb->d[i] == SWAPBLK_NONE)
|
||||
continue;
|
||||
if (first_free + num_free == sb->d[i])
|
||||
num_free++;
|
||||
else {
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
first_free = sb->d[i];
|
||||
num_free = 1;
|
||||
}
|
||||
swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
|
||||
}
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
swp_pager_freeswapspace(s_free, n_free);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user