d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a blocking counter KPI.

Browse Source 
refcount(9) was recently extended to support waiting on a refcount to
drop to zero, as this was needed for a lockless VM object
paging-in-progress counter.  However, this adds overhead to all uses of
refcount(9) and doesn't really match traditional refcounting semantics:
once a counter has dropped to zero, the protected object may be freed at
any point and it is not safe to dereference the counter.

This change removes that extension and instead adds a new set of KPIs,
blockcount_*, for use by VM object PIP and busy.

Reviewed by:	jeff, kib, mjg
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D23723
This commit is contained in:
Mark Johnston 2020-02-28 16:05:18 +00:00
parent f37cc7137d
commit c99d0c5801
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=358432
10 changed files with 303 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
sys/kern/kern_synch.c
View File

@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/blockcount.h>
#include <sys/condvar.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
@ -52,7 +53,6 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/refcount.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
@ -336,81 +336,6 @@ pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
(flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags));
}
/*
* Potentially release the last reference for refcount. Check for
* unlikely conditions and signal the caller as to whether it was
* the final ref.
*/
bool
refcount_release_last(volatile u_int *count, u_int n, u_int old)
{
u_int waiter;
waiter = old & REFCOUNT_WAITER;
old = REFCOUNT_COUNT(old);
if (__predict_false(n > old || REFCOUNT_SATURATED(old))) {
/*
* Avoid multiple destructor invocations if underflow occurred.
* This is not perfect since the memory backing the containing
* object may already have been reallocated.
*/
_refcount_update_saturated(count);
return (false);
}
/*
* Attempt to atomically clear the waiter bit. Wakeup waiters
* if we are successful.
*/
if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0))
wakeup(__DEVOLATILE(u_int *, count));
/*
* Last reference. Signal the user to call the destructor.
*
* Ensure that the destructor sees all updates. This synchronizes
* with release fences from all routines which drop the count.
*/
atomic_thread_fence_acq();
return (true);
}
/*
* Wait for a refcount wakeup. This does not guarantee that the ref is still
* zero on return and may be subject to transient wakeups. Callers wanting
* a precise answer should use refcount_wait().
*/
void
_refcount_sleep(volatile u_int *count, struct lock_object *lock,
const char *wmesg, int pri)
{
void *wchan;
u_int old;
if (REFCOUNT_COUNT(*count) == 0) {
if (lock != NULL)
LOCK_CLASS(lock)->lc_unlock(lock);
return;
}
wchan = __DEVOLATILE(void *, count);
sleepq_lock(wchan);
if (lock != NULL)
LOCK_CLASS(lock)->lc_unlock(lock);
old = *count;
for (;;) {
if (REFCOUNT_COUNT(old) == 0) {
sleepq_release(wchan);
return;
}
if (old & REFCOUNT_WAITER)
break;
if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER))
break;
}
sleepq_add(wchan, NULL, wmesg, 0, 0);
sleepq_wait(wchan, pri);
}
/*
* Make all threads sleeping on the specified identifier runnable.
*/
@ -459,6 +384,82 @@ wakeup_any(const void *ident)
kick_proc0();
}
/*
* Signal sleeping waiters after the counter has reached zero.
*/
void
_blockcount_wakeup(blockcount_t *bc, u_int old)
{
KASSERT(_BLOCKCOUNT_WAITERS(old),
("%s: no waiters on %p", __func__, bc));
if (atomic_cmpset_int(&bc->__count, _BLOCKCOUNT_WAITERS_FLAG, 0))
wakeup(bc);
}
/*
* Wait for a wakeup. This does not guarantee that the count is still zero on
* return and may be subject to transient wakeups. Callers wanting a precise
* answer should use blockcount_wait() with an interlock.
*
* Return 0 if there is no work to wait for, and 1 if we slept waiting for work
* to complete. In the latter case the counter value must be re-read.
*/
int
_blockcount_sleep(blockcount_t *bc, struct lock_object *lock, const char *wmesg,
int prio)
{
void *wchan;
uintptr_t lock_state;
u_int old;
int ret;
KASSERT(lock != &Giant.lock_object,
("%s: cannot use Giant as the interlock", __func__));
/*
* Synchronize with the fence in blockcount_release(). If we end up
* waiting, the sleepqueue lock acquisition will provide the required
* side effects.
*
* If there is no work to wait for, but waiters are present, try to put
* ourselves to sleep to avoid jumping ahead.
*/
if (atomic_load_acq_int(&bc->__count) == 0) {
if (lock != NULL && (prio & PDROP) != 0)
LOCK_CLASS(lock)->lc_unlock(lock);
return (0);
}
lock_state = 0;
wchan = bc;
sleepq_lock(wchan);
DROP_GIANT();
if (lock != NULL)
lock_state = LOCK_CLASS(lock)->lc_unlock(lock);
old = blockcount_read(bc);
do {
if (_BLOCKCOUNT_COUNT(old) == 0) {
sleepq_release(wchan);
ret = 0;
goto out;
}
if (_BLOCKCOUNT_WAITERS(old))
break;
} while (!atomic_fcmpset_int(&bc->__count, &old,
old | _BLOCKCOUNT_WAITERS_FLAG));
sleepq_add(wchan, NULL, wmesg, 0, 0);
sleepq_wait(wchan, prio);
ret = 1;
out:
PICKUP_GIANT();
if (lock != NULL && (prio & PDROP) == 0)
LOCK_CLASS(lock)->lc_lock(lock, lock_state);
return (ret);
}
static void
kdb_switch(void)
{

4
sys/kern/vfs_bio.c
View File

@ -2854,9 +2854,9 @@ vfs_vmio_iodone(struct buf *bp)
bool bogus;
obj = bp->b_bufobj->bo_object;
KASSERT(REFCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages,
KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages,
("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages));
blockcount_read(&obj->paging_in_progress), bp->b_npages));
vp = bp->b_vp;
VNPASS(vp->v_holdcnt > 0, vp);

52
sys/sys/_blockcount.h Normal file
View File

@ -0,0 +1,52 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2020 The FreeBSD Foundation
*
* This software was developed by Mark Johnston under sponsorship from
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef __SYS__BLOCKCOUNT_H__
#define __SYS__BLOCKCOUNT_H__
#include <machine/atomic.h>
typedef struct _blockcount {
unsigned int __count;
} blockcount_t;
#define _BLOCKCOUNT_WAITERS_FLAG (1U << 31)
#define _BLOCKCOUNT_COUNT(c) ((c) & ~_BLOCKCOUNT_WAITERS_FLAG)
#define _BLOCKCOUNT_WAITERS(c) (((c) & _BLOCKCOUNT_WAITERS_FLAG) != 0)
static inline unsigned int
blockcount_read(blockcount_t *count)
{
return (_BLOCKCOUNT_COUNT(atomic_load_int(&count->__count)));
}
#endif /* !__SYS__BLOCKCOUNT_H__ */

95
sys/sys/blockcount.h Normal file
View File

@ -0,0 +1,95 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2005 John Baldwin <jhb@FreeBSD.org>
* Copyright (c) 2020 The FreeBSD Foundation
*
* Portions of this software were developed by Mark Johnston under
* sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef __SYS_BLOCKCOUNT_H__
#define __SYS_BLOCKCOUNT_H__
#ifdef _KERNEL
#include <sys/systm.h>
#include <sys/_blockcount.h>
struct lock_object;
int _blockcount_sleep(blockcount_t *bc, struct lock_object *, const char *wmesg,
int prio);
void _blockcount_wakeup(blockcount_t *bc, u_int old);
static __inline void
blockcount_init(blockcount_t *bc)
{
atomic_store_int(&bc->__count, 0);
}
static __inline void
blockcount_acquire(blockcount_t *bc, u_int n)
{
#ifdef INVARIANTS
u_int old;
old = atomic_fetchadd_int(&bc->__count, n);
KASSERT(old + n > old, ("%s: counter overflow %p", __func__, bc));
#else
atomic_add_int(&bc->__count, n);
#endif
}
static __inline void
blockcount_release(blockcount_t *bc, u_int n)
{
u_int old;
atomic_thread_fence_rel();
old = atomic_fetchadd_int(&bc->__count, -n);
KASSERT(old >= n, ("%s: counter underflow %p", __func__, bc));
if (_BLOCKCOUNT_COUNT(old) == n && _BLOCKCOUNT_WAITERS(old))
_blockcount_wakeup(bc, old);
}
static __inline void
_blockcount_wait(blockcount_t *bc, struct lock_object *lo, const char *wmesg,
int prio)
{
KASSERT((prio & PDROP) == 0, ("%s: invalid prio %x", __func__, prio));
while (_blockcount_sleep(bc, lo, wmesg, prio) != 0)
;
}
#define blockcount_sleep(bc, lo, wmesg, prio) \
_blockcount_sleep((bc), (struct lock_object *)(lo), (wmesg), (prio))
#define blockcount_wait(bc, lo, wmesg, prio) \
_blockcount_wait((bc), (struct lock_object *)(lo), (wmesg), (prio))
#endif /* _KERNEL */
#endif /* !__SYS_BLOCKCOUNT_H__ */

107
sys/sys/refcount.h
View File

@ -34,18 +34,14 @@
#ifdef _KERNEL
#include <sys/systm.h>
#include <sys/_blockcount.h>
#else
#include <stdbool.h>
#define KASSERT(exp, msg) /* */
#endif
#define REFCOUNT_WAITER (1U << 31) /* Refcount has waiter. */
#define REFCOUNT_SATURATION_VALUE (3U << 29)
#define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0)
#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER)
bool refcount_release_last(volatile u_int *count, u_int n, u_int old);
#define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0)
#define REFCOUNT_SATURATION_VALUE (3U << 30)
/*
* Attempt to handle reference count overflow and underflow. Force the counter
@ -111,56 +107,6 @@ refcount_acquire_checked(volatile u_int *count)
}
}
static __inline bool
refcount_releasen(volatile u_int *count, u_int n)
{
u_int old;
KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
("refcount_releasen: n=%u too large", n));
/*
* Paired with acquire fence in refcount_release_last.
*/
atomic_thread_fence_rel();
old = atomic_fetchadd_int(count, -n);
if (__predict_false(n >= REFCOUNT_COUNT(old) ||
REFCOUNT_SATURATED(old)))
return (refcount_release_last(count, n, old));
return (false);
}
static __inline bool
refcount_release(volatile u_int *count)
{
return (refcount_releasen(count, 1));
}
#ifdef _KERNEL
struct lock_object;
void _refcount_sleep(volatile u_int *count, struct lock_object *,
const char *wmesg, int prio);
static __inline void
refcount_sleep(volatile u_int *count, const char *wmesg, int prio)
{
_refcount_sleep(count, NULL, wmesg, prio);
}
#define refcount_sleep_interlock(count, lock, wmesg, prio) \
_refcount_sleep((count), (struct lock_object *)(lock), (wmesg), (prio))
static __inline void
refcount_wait(volatile u_int *count, const char *wmesg, int prio)
{
while (*count != 0)
refcount_sleep(count, wmesg, prio);
}
#endif
/*
* This functions returns non-zero if the refcount was
* incremented. Else zero is returned.
@ -172,7 +118,7 @@ refcount_acquire_if_gt(volatile u_int *count, u_int n)
old = *count;
for (;;) {
if (REFCOUNT_COUNT(old) <= n)
if (old <= n)
return (false);
if (__predict_false(REFCOUNT_SATURATED(old)))
return (true);
@ -185,7 +131,41 @@ static __inline __result_use_check bool
refcount_acquire_if_not_zero(volatile u_int *count)
{
return refcount_acquire_if_gt(count, 0);
return (refcount_acquire_if_gt(count, 0));
}
static __inline bool
refcount_releasen(volatile u_int *count, u_int n)
{
u_int old;
KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
("refcount_releasen: n=%u too large", n));
atomic_thread_fence_rel();
old = atomic_fetchadd_int(count, -n);
if (__predict_false(old < n || REFCOUNT_SATURATED(old))) {
_refcount_update_saturated(count);
return (false);
}
if (old > n)
return (false);
/*
* Last reference. Signal the user to call the destructor.
*
* Ensure that the destructor sees all updates. This synchronizes with
* release fences from all routines which drop the count.
*/
atomic_thread_fence_acq();
return (true);
}
static __inline bool
refcount_release(volatile u_int *count)
{
return (refcount_releasen(count, 1));
}
static __inline __result_use_check bool
@ -197,12 +177,12 @@ refcount_release_if_gt(volatile u_int *count, u_int n)
("refcount_release_if_gt: Use refcount_release for final ref"));
old = *count;
for (;;) {
if (REFCOUNT_COUNT(old) <= n)
if (old <= n)
return (false);
if (__predict_false(REFCOUNT_SATURATED(old)))
return (true);
/*
* Paired with acquire fence in refcount_release_last.
* Paired with acquire fence in refcount_releasen().
*/
if (atomic_fcmpset_rel_int(count, &old, old - 1))
return (true);
@ -213,6 +193,7 @@ static __inline __result_use_check bool
refcount_release_if_not_last(volatile u_int *count)
{
return refcount_release_if_gt(count, 1);
return (refcount_release_if_gt(count, 1));
}
#endif /* ! __SYS_REFCOUNT_H__ */
#endif /* !__SYS_REFCOUNT_H__ */

4
sys/vm/vm_fault.c
View File

@ -377,7 +377,7 @@ vm_fault_restore_map_lock(struct faultstate *fs)
{
VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0);
if (!vm_map_trylock_read(fs->map)) {
VM_OBJECT_WUNLOCK(fs->first_object);
@ -428,7 +428,7 @@ vm_fault_populate(struct faultstate *fs)
MPASS(fs->object == fs->first_object);
VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0);
MPASS(fs->first_object->backing_object == NULL);
MPASS(fs->lookup_still_valid);

51
sys/vm/vm_object.c
View File

@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/blockcount.h>
#include <sys/cpuset.h>
#include <sys/lock.h>
#include <sys/mman.h>
@ -201,12 +202,11 @@ vm_object_zdtor(void *mem, int size, void *arg)
("object %p has reservations",
object));
#endif
KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0,
KASSERT(blockcount_read(&object->paging_in_progress) == 0,
("object %p paging_in_progress = %d",
object, REFCOUNT_COUNT(object->paging_in_progress)));
KASSERT(object->busy == 0,
("object %p busy = %d",
object, object->busy));
object, blockcount_read(&object->paging_in_progress)));
KASSERT(!vm_object_busied(object),
("object %p busy = %d", object, blockcount_read(&object->busy)));
KASSERT(object->resident_page_count == 0,
("object %p resident_page_count = %d",
object, object->resident_page_count));
@ -231,8 +231,8 @@ vm_object_zinit(void *mem, int size, int flags)
object->type = OBJT_DEAD;
vm_radix_init(&object->rtree);
refcount_init(&object->ref_count, 0);
refcount_init(&object->paging_in_progress, 0);
refcount_init(&object->busy, 0);
blockcount_init(&object->paging_in_progress);
blockcount_init(&object->busy);
object->resident_page_count = 0;
object->shadow_count = 0;
object->flags = OBJ_DEAD;
@ -363,34 +363,36 @@ void
vm_object_pip_add(vm_object_t object, short i)
{
refcount_acquiren(&object->paging_in_progress, i);
if (i > 0)
blockcount_acquire(&object->paging_in_progress, i);
}
void
vm_object_pip_wakeup(vm_object_t object)
{
refcount_release(&object->paging_in_progress);
vm_object_pip_wakeupn(object, 1);
}
void
vm_object_pip_wakeupn(vm_object_t object, short i)
{
refcount_releasen(&object->paging_in_progress, i);
if (i > 0)
blockcount_release(&object->paging_in_progress, i);
}
/*
* Atomically drop the interlock and wait for pip to drain. This protects
* from sleep/wakeup races due to identity changes. The lock is not
* re-acquired on return.
* Atomically drop the object lock and wait for pip to drain. This protects
* from sleep/wakeup races due to identity changes. The lock is not re-acquired
* on return.
*/
static void
vm_object_pip_sleep(vm_object_t object, const char *waitid)
{
refcount_sleep_interlock(&object->paging_in_progress,
&object->lock, waitid, PVM);
(void)blockcount_sleep(&object->paging_in_progress, &object->lock,
waitid, PVM | PDROP);
}
void
@ -399,10 +401,8 @@ vm_object_pip_wait(vm_object_t object, const char *waitid)
VM_OBJECT_ASSERT_WLOCKED(object);
while (REFCOUNT_COUNT(object->paging_in_progress) > 0) {
vm_object_pip_sleep(object, waitid);
VM_OBJECT_WLOCK(object);
}
blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
PVM);
}
void
@ -411,8 +411,7 @@ vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
VM_OBJECT_ASSERT_UNLOCKED(object);
while (REFCOUNT_COUNT(object->paging_in_progress) > 0)
refcount_wait(&object->paging_in_progress, waitid, PVM);
blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
}
/*
@ -955,7 +954,7 @@ vm_object_terminate(vm_object_t object)
*/
vm_object_pip_wait(object, "objtrm");
KASSERT(!REFCOUNT_COUNT(object->paging_in_progress),
KASSERT(!blockcount_read(&object->paging_in_progress),
("vm_object_terminate: pageout in progress"));
KASSERT(object->ref_count == 0,
@ -2458,7 +2457,7 @@ vm_object_busy(vm_object_t obj)
VM_OBJECT_ASSERT_LOCKED(obj);
refcount_acquire(&obj->busy);
blockcount_acquire(&obj->busy, 1);
/* The fence is required to order loads of page busy. */
atomic_thread_fence_acq_rel();
}
@ -2467,8 +2466,7 @@ void
vm_object_unbusy(vm_object_t obj)
{
refcount_release(&obj->busy);
blockcount_release(&obj->busy, 1);
}
void
@ -2477,8 +2475,7 @@ vm_object_busy_wait(vm_object_t obj, const char *wmesg)
VM_OBJECT_ASSERT_UNLOCKED(obj);
if (obj->busy)
refcount_sleep(&obj->busy, wmesg, PVM);
(void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
}
/*

9
sys/vm/vm_object.h
View File

@ -70,6 +70,7 @@
#define _VM_OBJECT_
#include <sys/queue.h>
#include <sys/_blockcount.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pctrie.h>
@ -113,8 +114,8 @@ struct vm_object {
objtype_t type; /* type of pager */
u_short flags; /* see below */
u_short pg_color; /* (c) color of first page in obj */
volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */
volatile u_int busy; /* (a) object is busy, disallow page busy. */
blockcount_t paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */
blockcount_t busy; /* (a) object is busy, disallow page busy. */
int resident_page_count; /* number of resident pages */
struct vm_object *backing_object; /* object that I'm a shadow of */
vm_ooffset_t backing_object_offset;/* Offset in backing object */
@ -265,7 +266,7 @@ extern struct vm_object kernel_object_store;
lock_class_rw.lc_lock(&(object)->lock.lock_object, (state))
#define VM_OBJECT_ASSERT_PAGING(object) \
KASSERT((object)->paging_in_progress != 0, \
KASSERT(blockcount_read(&(object)->paging_in_progress) != 0, \
("vm_object %p is not paging", object))
#define VM_OBJECT_ASSERT_REFERENCE(object) \
KASSERT((object)->reference_count != 0, \
@ -348,7 +349,7 @@ static inline bool
vm_object_busied(vm_object_t object)
{
return (object->busy != 0);
return (blockcount_read(&object->busy) != 0);
}
#define VM_OBJECT_ASSERT_BUSY(object) MPASS(vm_object_busied((object)))

2
sys/vm/vm_pager.h
View File

@ -168,7 +168,7 @@ vm_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
MPASS((object->flags & OBJ_POPULATE) != 0);
MPASS(pidx < object->size);
MPASS(object->paging_in_progress > 0);
MPASS(blockcount_read(&object->paging_in_progress) > 0);
return ((*pagertab[object->type]->pgo_populate)(object, pidx,
fault_type, max_prot, first, last));
}

2
sys/vm/vm_swapout.c
View File

@ -218,7 +218,7 @@ vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,
goto unlock_return;
VM_OBJECT_ASSERT_LOCKED(object);
if ((object->flags & OBJ_UNMANAGED) != 0 ||
REFCOUNT_COUNT(object->paging_in_progress) > 0)
blockcount_read(&object->paging_in_progress) > 0)
goto unlock_return;
unmap = true;