When the swap pager allocates space on disk, it requests contiguous
blocks in a single call to blist_alloc(). However, when it frees that space, it previously called blist_free() on each block, one at a time. With this change, the swap pager identifies ranges of contiguous blocks to be freed, and calls blist_free() once per range. In one extreme case, that is described in the review, the time to perform an munmap(2) was reduced by 55%. Submitted by: Doug Moore <dougm@rice.edu> Reviewed by: kib MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D12397
This commit is contained in:
parent
b4cc39e785
commit
230869e051
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=326329
@ -390,6 +390,7 @@ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0,
|
||||
|
||||
static void swp_sizecheck(void);
|
||||
static void swp_pager_async_iodone(struct buf *bp);
|
||||
static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit);
|
||||
static int swapongeom(struct vnode *);
|
||||
static int swaponvp(struct thread *, struct vnode *, u_long);
|
||||
static int swapoff_one(struct swdevt *sp, struct ucred *cred);
|
||||
@ -397,7 +398,7 @@ static int swapoff_one(struct swdevt *sp, struct ucred *cred);
|
||||
/*
|
||||
* Swap bitmap functions
|
||||
*/
|
||||
static void swp_pager_freeswapspace(daddr_t blk, int npages);
|
||||
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
|
||||
static daddr_t swp_pager_getswapspace(int npages);
|
||||
|
||||
/*
|
||||
@ -768,10 +769,12 @@ swp_pager_strategy(struct buf *bp)
|
||||
* This routine may not sleep.
|
||||
*/
|
||||
static void
|
||||
swp_pager_freeswapspace(daddr_t blk, int npages)
|
||||
swp_pager_freeswapspace(daddr_t blk, daddr_t npages)
|
||||
{
|
||||
struct swdevt *sp;
|
||||
|
||||
if (npages == 0)
|
||||
return;
|
||||
mtx_lock(&sw_dev_mtx);
|
||||
TAILQ_FOREACH(sp, &swtailq, sw_list) {
|
||||
if (blk >= sp->sw_first && blk < sp->sw_end) {
|
||||
@ -1760,6 +1763,22 @@ swap_pager_swapoff(struct swdevt *sp)
|
||||
* appropriate tracking counters.
|
||||
*/
|
||||
|
||||
/*
|
||||
* SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free?
|
||||
*/
|
||||
static bool
|
||||
swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
|
||||
{
|
||||
int i;
|
||||
|
||||
MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES);
|
||||
for (i = start; i < limit; i++) {
|
||||
if (sb->d[i] != SWAPBLK_NONE)
|
||||
return (false);
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
/*
|
||||
* SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
|
||||
*
|
||||
@ -1876,16 +1895,10 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
|
||||
/*
|
||||
* Free the swblk if we end up with the empty page run.
|
||||
*/
|
||||
if (swapblk == SWAPBLK_NONE) {
|
||||
for (i = 0; i < SWAP_META_PAGES; i++) {
|
||||
if (sb->d[i] != SWAPBLK_NONE)
|
||||
break;
|
||||
}
|
||||
if (i == SWAP_META_PAGES) {
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
|
||||
rdpi);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
if (swapblk == SWAPBLK_NONE &&
|
||||
swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1903,37 +1916,46 @@ static void
|
||||
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
|
||||
{
|
||||
struct swblk *sb;
|
||||
daddr_t first_free, num_free;
|
||||
vm_pindex_t last;
|
||||
int i;
|
||||
bool empty;
|
||||
int i, limit, start;
|
||||
|
||||
VM_OBJECT_ASSERT_WLOCKED(object);
|
||||
if (object->type != OBJT_SWAP || count == 0)
|
||||
return;
|
||||
|
||||
last = pindex + count - 1;
|
||||
first_free = SWAPBLK_NONE;
|
||||
num_free = 0;
|
||||
last = pindex + count;
|
||||
for (;;) {
|
||||
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
|
||||
rounddown(pindex, SWAP_META_PAGES));
|
||||
if (sb == NULL || sb->p > last)
|
||||
if (sb == NULL || sb->p >= last)
|
||||
break;
|
||||
empty = true;
|
||||
for (i = 0; i < SWAP_META_PAGES; i++) {
|
||||
start = pindex > sb->p ? pindex - sb->p : 0;
|
||||
limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
|
||||
SWAP_META_PAGES;
|
||||
for (i = start; i < limit; i++) {
|
||||
if (sb->d[i] == SWAPBLK_NONE)
|
||||
continue;
|
||||
if (pindex <= sb->p + i && sb->p + i <= last) {
|
||||
swp_pager_freeswapspace(sb->d[i], 1);
|
||||
sb->d[i] = SWAPBLK_NONE;
|
||||
} else
|
||||
empty = false;
|
||||
if (first_free + num_free == sb->d[i])
|
||||
num_free++;
|
||||
else {
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
first_free = sb->d[i];
|
||||
num_free = 1;
|
||||
}
|
||||
sb->d[i] = SWAPBLK_NONE;
|
||||
}
|
||||
pindex = sb->p + SWAP_META_PAGES;
|
||||
if (empty) {
|
||||
if (swp_pager_swblk_empty(sb, 0, start) &&
|
||||
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
|
||||
sb->p);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
pindex = sb->p + SWAP_META_PAGES;
|
||||
}
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1946,6 +1968,7 @@ static void
|
||||
swp_pager_meta_free_all(vm_object_t object)
|
||||
{
|
||||
struct swblk *sb;
|
||||
daddr_t first_free, num_free;
|
||||
vm_pindex_t pindex;
|
||||
int i;
|
||||
|
||||
@ -1953,16 +1976,26 @@ swp_pager_meta_free_all(vm_object_t object)
|
||||
if (object->type != OBJT_SWAP)
|
||||
return;
|
||||
|
||||
first_free = SWAPBLK_NONE;
|
||||
num_free = 0;
|
||||
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
|
||||
&object->un_pager.swp.swp_blks, pindex)) != NULL;) {
|
||||
pindex = sb->p + SWAP_META_PAGES;
|
||||
for (i = 0; i < SWAP_META_PAGES; i++) {
|
||||
if (sb->d[i] != SWAPBLK_NONE)
|
||||
swp_pager_freeswapspace(sb->d[i], 1);
|
||||
if (sb->d[i] == SWAPBLK_NONE)
|
||||
continue;
|
||||
if (first_free + num_free == sb->d[i])
|
||||
num_free++;
|
||||
else {
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
first_free = sb->d[i];
|
||||
num_free = 1;
|
||||
}
|
||||
}
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
|
||||
uma_zfree(swblk_zone, sb);
|
||||
}
|
||||
swp_pager_freeswapspace(first_free, num_free);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1987,7 +2020,6 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
|
||||
{
|
||||
struct swblk *sb;
|
||||
daddr_t r1;
|
||||
int i;
|
||||
|
||||
if ((flags & (SWM_FREE | SWM_POP)) != 0)
|
||||
VM_OBJECT_ASSERT_WLOCKED(object);
|
||||
@ -2010,11 +2042,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
|
||||
return (SWAPBLK_NONE);
|
||||
if ((flags & (SWM_FREE | SWM_POP)) != 0) {
|
||||
sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
|
||||
for (i = 0; i < SWAP_META_PAGES; i++) {
|
||||
if (sb->d[i] != SWAPBLK_NONE)
|
||||
break;
|
||||
}
|
||||
if (i == SWAP_META_PAGES) {
|
||||
if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
|
||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
|
||||
rounddown(pindex, SWAP_META_PAGES));
|
||||
uma_zfree(swblk_zone, sb);
|
||||
|
Loading…
Reference in New Issue
Block a user