d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

When the swap pager allocates space on disk, it requests contiguous

Browse Source 
blocks in a single call to blist_alloc().  However, when it frees
that space, it previously called blist_free() on each block, one at a
time.  With this change, the swap pager identifies ranges of
contiguous blocks to be freed, and calls blist_free() once per
range.  In one extreme case, that is described in the review, the time
to perform an munmap(2) was reduced by 55%.

Submitted by:	Doug Moore <dougm@rice.edu>
Reviewed by:	kib
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D12397
This commit is contained in:
Alan Cox 2017-11-28 17:46:03 +00:00
parent b4cc39e785
commit 230869e051
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=326329
1 changed files with 61 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
sys/vm/swap_pager.c
View File

@ -390,6 +390,7 @@ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0,
static void swp_sizecheck(void);
static void swp_pager_async_iodone(struct buf *bp);
static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit);
static int swapongeom(struct vnode *);
static int swaponvp(struct thread *, struct vnode *, u_long);
static int swapoff_one(struct swdevt *sp, struct ucred *cred);
@ -397,7 +398,7 @@ static int swapoff_one(struct swdevt *sp, struct ucred *cred);
/*
* Swap bitmap functions
*/
static void swp_pager_freeswapspace(daddr_t blk, int npages);
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
static daddr_t swp_pager_getswapspace(int npages);
/*
@ -768,10 +769,12 @@ swp_pager_strategy(struct buf *bp)
* This routine may not sleep.
*/
static void
swp_pager_freeswapspace(daddr_t blk, int npages)
swp_pager_freeswapspace(daddr_t blk, daddr_t npages)
{
struct swdevt *sp;
if (npages == 0)
return;
mtx_lock(&sw_dev_mtx);
TAILQ_FOREACH(sp, &swtailq, sw_list) {
if (blk >= sp->sw_first && blk < sp->sw_end) {
@ -1760,6 +1763,22 @@ swap_pager_swapoff(struct swdevt *sp)
* appropriate tracking counters.
*/
/*
* SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free?
*/
static bool
swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
{
int i;
MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES);
for (i = start; i < limit; i++) {
if (sb->d[i] != SWAPBLK_NONE)
return (false);
}
return (true);
}
/*
* SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
*
@ -1876,16 +1895,10 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
/*
* Free the swblk if we end up with the empty page run.
*/
if (swapblk == SWAPBLK_NONE) {
for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->d[i] != SWAPBLK_NONE)
break;
}
if (i == SWAP_META_PAGES) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
rdpi);
uma_zfree(swblk_zone, sb);
}
if (swapblk == SWAPBLK_NONE &&
swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
uma_zfree(swblk_zone, sb);
}
}
@ -1903,37 +1916,46 @@ static void
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
{
struct swblk *sb;
daddr_t first_free, num_free;
vm_pindex_t last;
int i;
bool empty;
int i, limit, start;
VM_OBJECT_ASSERT_WLOCKED(object);
if (object->type != OBJT_SWAP || count == 0)
return;
last = pindex + count - 1;
first_free = SWAPBLK_NONE;
num_free = 0;
last = pindex + count;
for (;;) {
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));
if (sb == NULL || sb->p > last)
if (sb == NULL || sb->p >= last)
break;
empty = true;
for (i = 0; i < SWAP_META_PAGES; i++) {
start = pindex > sb->p ? pindex - sb->p : 0;
limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
SWAP_META_PAGES;
for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)
continue;
if (pindex <= sb->p + i && sb->p + i <= last) {
swp_pager_freeswapspace(sb->d[i], 1);
sb->d[i] = SWAPBLK_NONE;
} else
empty = false;
if (first_free + num_free == sb->d[i])
num_free++;
else {
swp_pager_freeswapspace(first_free, num_free);
first_free = sb->d[i];
num_free = 1;
}
sb->d[i] = SWAPBLK_NONE;
}
pindex = sb->p + SWAP_META_PAGES;
if (empty) {
if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
sb->p);
uma_zfree(swblk_zone, sb);
}
pindex = sb->p + SWAP_META_PAGES;
}
swp_pager_freeswapspace(first_free, num_free);
}
/*
@ -1946,6 +1968,7 @@ static void
swp_pager_meta_free_all(vm_object_t object)
{
struct swblk *sb;
daddr_t first_free, num_free;
vm_pindex_t pindex;
int i;
@ -1953,16 +1976,26 @@ swp_pager_meta_free_all(vm_object_t object)
if (object->type != OBJT_SWAP)
return;
first_free = SWAPBLK_NONE;
num_free = 0;
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pindex)) != NULL;) {
pindex = sb->p + SWAP_META_PAGES;
for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->d[i] != SWAPBLK_NONE)
swp_pager_freeswapspace(sb->d[i], 1);
if (sb->d[i] == SWAPBLK_NONE)
continue;
if (first_free + num_free == sb->d[i])
num_free++;
else {
swp_pager_freeswapspace(first_free, num_free);
first_free = sb->d[i];
num_free = 1;
}
}
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
uma_zfree(swblk_zone, sb);
}
swp_pager_freeswapspace(first_free, num_free);
}
/*
@ -1987,7 +2020,6 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
{
struct swblk *sb;
daddr_t r1;
int i;
if ((flags & (SWM_FREE | SWM_POP)) != 0)
VM_OBJECT_ASSERT_WLOCKED(object);
@ -2010,11 +2042,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
return (SWAPBLK_NONE);
if ((flags & (SWM_FREE | SWM_POP)) != 0) {
sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->d[i] != SWAPBLK_NONE)
break;
}
if (i == SWAP_META_PAGES) {
if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));
uma_zfree(swblk_zone, sb);