From c99d0c5801ce22a9976e491d04ce7e1b8996dfdd Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 28 Feb 2020 16:05:18 +0000 Subject: [PATCH] Add a blocking counter KPI. refcount(9) was recently extended to support waiting on a refcount to drop to zero, as this was needed for a lockless VM object paging-in-progress counter. However, this adds overhead to all uses of refcount(9) and doesn't really match traditional refcounting semantics: once a counter has dropped to zero, the protected object may be freed at any point and it is not safe to dereference the counter. This change removes that extension and instead adds a new set of KPIs, blockcount_*, for use by VM object PIP and busy. Reviewed by: jeff, kib, mjg Tested by: pho Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D23723 --- sys/kern/kern_synch.c | 153 +++++++++++++++++++++--------------------- sys/kern/vfs_bio.c | 4 +- sys/sys/_blockcount.h | 52 ++++++++++++++ sys/sys/blockcount.h | 95 ++++++++++++++++++++++++++ sys/sys/refcount.h | 107 ++++++++++++----------------- sys/vm/vm_fault.c | 4 +- sys/vm/vm_object.c | 51 +++++++------- sys/vm/vm_object.h | 9 +-- sys/vm/vm_pager.h | 2 +- sys/vm/vm_swapout.c | 2 +- 10 files changed, 303 insertions(+), 176 deletions(-) create mode 100644 sys/sys/_blockcount.h create mode 100644 sys/sys/blockcount.h diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 435f1a63a523..9c210b39d2d9 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -52,7 +53,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include @@ -336,81 +336,6 @@ pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) (flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags)); } -/* - * Potentially release the last reference for refcount. Check for - * unlikely conditions and signal the caller as to whether it was - * the final ref. - */ -bool -refcount_release_last(volatile u_int *count, u_int n, u_int old) -{ - u_int waiter; - - waiter = old & REFCOUNT_WAITER; - old = REFCOUNT_COUNT(old); - if (__predict_false(n > old || REFCOUNT_SATURATED(old))) { - /* - * Avoid multiple destructor invocations if underflow occurred. - * This is not perfect since the memory backing the containing - * object may already have been reallocated. - */ - _refcount_update_saturated(count); - return (false); - } - - /* - * Attempt to atomically clear the waiter bit. Wakeup waiters - * if we are successful. - */ - if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0)) - wakeup(__DEVOLATILE(u_int *, count)); - - /* - * Last reference. Signal the user to call the destructor. - * - * Ensure that the destructor sees all updates. This synchronizes - * with release fences from all routines which drop the count. - */ - atomic_thread_fence_acq(); - return (true); -} - -/* - * Wait for a refcount wakeup. This does not guarantee that the ref is still - * zero on return and may be subject to transient wakeups. Callers wanting - * a precise answer should use refcount_wait(). - */ -void -_refcount_sleep(volatile u_int *count, struct lock_object *lock, - const char *wmesg, int pri) -{ - void *wchan; - u_int old; - - if (REFCOUNT_COUNT(*count) == 0) { - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - return; - } - wchan = __DEVOLATILE(void *, count); - sleepq_lock(wchan); - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - old = *count; - for (;;) { - if (REFCOUNT_COUNT(old) == 0) { - sleepq_release(wchan); - return; - } - if (old & REFCOUNT_WAITER) - break; - if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER)) - break; - } - sleepq_add(wchan, NULL, wmesg, 0, 0); - sleepq_wait(wchan, pri); -} - /* * Make all threads sleeping on the specified identifier runnable. */ @@ -459,6 +384,82 @@ wakeup_any(const void *ident) kick_proc0(); } +/* + * Signal sleeping waiters after the counter has reached zero. + */ +void +_blockcount_wakeup(blockcount_t *bc, u_int old) +{ + + KASSERT(_BLOCKCOUNT_WAITERS(old), + ("%s: no waiters on %p", __func__, bc)); + + if (atomic_cmpset_int(&bc->__count, _BLOCKCOUNT_WAITERS_FLAG, 0)) + wakeup(bc); +} + +/* + * Wait for a wakeup. This does not guarantee that the count is still zero on + * return and may be subject to transient wakeups. Callers wanting a precise + * answer should use blockcount_wait() with an interlock. + * + * Return 0 if there is no work to wait for, and 1 if we slept waiting for work + * to complete. In the latter case the counter value must be re-read. + */ +int +_blockcount_sleep(blockcount_t *bc, struct lock_object *lock, const char *wmesg, + int prio) +{ + void *wchan; + uintptr_t lock_state; + u_int old; + int ret; + + KASSERT(lock != &Giant.lock_object, + ("%s: cannot use Giant as the interlock", __func__)); + + /* + * Synchronize with the fence in blockcount_release(). If we end up + * waiting, the sleepqueue lock acquisition will provide the required + * side effects. + * + * If there is no work to wait for, but waiters are present, try to put + * ourselves to sleep to avoid jumping ahead. + */ + if (atomic_load_acq_int(&bc->__count) == 0) { + if (lock != NULL && (prio & PDROP) != 0) + LOCK_CLASS(lock)->lc_unlock(lock); + return (0); + } + lock_state = 0; + wchan = bc; + sleepq_lock(wchan); + DROP_GIANT(); + if (lock != NULL) + lock_state = LOCK_CLASS(lock)->lc_unlock(lock); + old = blockcount_read(bc); + do { + if (_BLOCKCOUNT_COUNT(old) == 0) { + sleepq_release(wchan); + ret = 0; + goto out; + } + if (_BLOCKCOUNT_WAITERS(old)) + break; + } while (!atomic_fcmpset_int(&bc->__count, &old, + old | _BLOCKCOUNT_WAITERS_FLAG)); + sleepq_add(wchan, NULL, wmesg, 0, 0); + sleepq_wait(wchan, prio); + ret = 1; + +out: + PICKUP_GIANT(); + if (lock != NULL && (prio & PDROP) == 0) + LOCK_CLASS(lock)->lc_lock(lock, lock_state); + + return (ret); +} + static void kdb_switch(void) { diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 0d95776676c5..a4da72d208fe 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -2854,9 +2854,9 @@ vfs_vmio_iodone(struct buf *bp) bool bogus; obj = bp->b_bufobj->bo_object; - KASSERT(REFCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages, + KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages, ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)", - REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages)); + blockcount_read(&obj->paging_in_progress), bp->b_npages)); vp = bp->b_vp; VNPASS(vp->v_holdcnt > 0, vp); diff --git a/sys/sys/_blockcount.h b/sys/sys/_blockcount.h new file mode 100644 index 000000000000..e4cb7921f8b5 --- /dev/null +++ b/sys/sys/_blockcount.h @@ -0,0 +1,52 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS__BLOCKCOUNT_H__ +#define __SYS__BLOCKCOUNT_H__ + +#include + +typedef struct _blockcount { + unsigned int __count; +} blockcount_t; + +#define _BLOCKCOUNT_WAITERS_FLAG (1U << 31) +#define _BLOCKCOUNT_COUNT(c) ((c) & ~_BLOCKCOUNT_WAITERS_FLAG) +#define _BLOCKCOUNT_WAITERS(c) (((c) & _BLOCKCOUNT_WAITERS_FLAG) != 0) + +static inline unsigned int +blockcount_read(blockcount_t *count) +{ + return (_BLOCKCOUNT_COUNT(atomic_load_int(&count->__count))); +} + +#endif /* !__SYS__BLOCKCOUNT_H__ */ diff --git a/sys/sys/blockcount.h b/sys/sys/blockcount.h new file mode 100644 index 000000000000..7e7ebb7818ac --- /dev/null +++ b/sys/sys/blockcount.h @@ -0,0 +1,95 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2005 John Baldwin + * Copyright (c) 2020 The FreeBSD Foundation + * + * Portions of this software were developed by Mark Johnston under + * sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS_BLOCKCOUNT_H__ +#define __SYS_BLOCKCOUNT_H__ + +#ifdef _KERNEL + +#include +#include + +struct lock_object; + +int _blockcount_sleep(blockcount_t *bc, struct lock_object *, const char *wmesg, + int prio); +void _blockcount_wakeup(blockcount_t *bc, u_int old); + +static __inline void +blockcount_init(blockcount_t *bc) +{ + atomic_store_int(&bc->__count, 0); +} + +static __inline void +blockcount_acquire(blockcount_t *bc, u_int n) +{ +#ifdef INVARIANTS + u_int old; + + old = atomic_fetchadd_int(&bc->__count, n); + KASSERT(old + n > old, ("%s: counter overflow %p", __func__, bc)); +#else + atomic_add_int(&bc->__count, n); +#endif +} + +static __inline void +blockcount_release(blockcount_t *bc, u_int n) +{ + u_int old; + + atomic_thread_fence_rel(); + old = atomic_fetchadd_int(&bc->__count, -n); + KASSERT(old >= n, ("%s: counter underflow %p", __func__, bc)); + if (_BLOCKCOUNT_COUNT(old) == n && _BLOCKCOUNT_WAITERS(old)) + _blockcount_wakeup(bc, old); +} + +static __inline void +_blockcount_wait(blockcount_t *bc, struct lock_object *lo, const char *wmesg, + int prio) +{ + KASSERT((prio & PDROP) == 0, ("%s: invalid prio %x", __func__, prio)); + + while (_blockcount_sleep(bc, lo, wmesg, prio) != 0) + ; +} + +#define blockcount_sleep(bc, lo, wmesg, prio) \ + _blockcount_sleep((bc), (struct lock_object *)(lo), (wmesg), (prio)) +#define blockcount_wait(bc, lo, wmesg, prio) \ + _blockcount_wait((bc), (struct lock_object *)(lo), (wmesg), (prio)) + +#endif /* _KERNEL */ +#endif /* !__SYS_BLOCKCOUNT_H__ */ diff --git a/sys/sys/refcount.h b/sys/sys/refcount.h index a3becb5558ab..28f0b20ee648 100644 --- a/sys/sys/refcount.h +++ b/sys/sys/refcount.h @@ -34,18 +34,14 @@ #ifdef _KERNEL #include +#include #else #include #define KASSERT(exp, msg) /* */ #endif -#define REFCOUNT_WAITER (1U << 31) /* Refcount has waiter. */ -#define REFCOUNT_SATURATION_VALUE (3U << 29) - -#define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0) -#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER) - -bool refcount_release_last(volatile u_int *count, u_int n, u_int old); +#define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0) +#define REFCOUNT_SATURATION_VALUE (3U << 30) /* * Attempt to handle reference count overflow and underflow. Force the counter @@ -111,56 +107,6 @@ refcount_acquire_checked(volatile u_int *count) } } -static __inline bool -refcount_releasen(volatile u_int *count, u_int n) -{ - u_int old; - - KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, - ("refcount_releasen: n=%u too large", n)); - - /* - * Paired with acquire fence in refcount_release_last. - */ - atomic_thread_fence_rel(); - old = atomic_fetchadd_int(count, -n); - if (__predict_false(n >= REFCOUNT_COUNT(old) || - REFCOUNT_SATURATED(old))) - return (refcount_release_last(count, n, old)); - return (false); -} - -static __inline bool -refcount_release(volatile u_int *count) -{ - - return (refcount_releasen(count, 1)); -} - -#ifdef _KERNEL -struct lock_object; -void _refcount_sleep(volatile u_int *count, struct lock_object *, - const char *wmesg, int prio); - -static __inline void -refcount_sleep(volatile u_int *count, const char *wmesg, int prio) -{ - - _refcount_sleep(count, NULL, wmesg, prio); -} - -#define refcount_sleep_interlock(count, lock, wmesg, prio) \ - _refcount_sleep((count), (struct lock_object *)(lock), (wmesg), (prio)) - -static __inline void -refcount_wait(volatile u_int *count, const char *wmesg, int prio) -{ - - while (*count != 0) - refcount_sleep(count, wmesg, prio); -} -#endif - /* * This functions returns non-zero if the refcount was * incremented. Else zero is returned. @@ -172,7 +118,7 @@ refcount_acquire_if_gt(volatile u_int *count, u_int n) old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); @@ -185,7 +131,41 @@ static __inline __result_use_check bool refcount_acquire_if_not_zero(volatile u_int *count) { - return refcount_acquire_if_gt(count, 0); + return (refcount_acquire_if_gt(count, 0)); +} + +static __inline bool +refcount_releasen(volatile u_int *count, u_int n) +{ + u_int old; + + KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, + ("refcount_releasen: n=%u too large", n)); + + atomic_thread_fence_rel(); + old = atomic_fetchadd_int(count, -n); + if (__predict_false(old < n || REFCOUNT_SATURATED(old))) { + _refcount_update_saturated(count); + return (false); + } + if (old > n) + return (false); + + /* + * Last reference. Signal the user to call the destructor. + * + * Ensure that the destructor sees all updates. This synchronizes with + * release fences from all routines which drop the count. + */ + atomic_thread_fence_acq(); + return (true); +} + +static __inline bool +refcount_release(volatile u_int *count) +{ + + return (refcount_releasen(count, 1)); } static __inline __result_use_check bool @@ -197,12 +177,12 @@ refcount_release_if_gt(volatile u_int *count, u_int n) ("refcount_release_if_gt: Use refcount_release for final ref")); old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); /* - * Paired with acquire fence in refcount_release_last. + * Paired with acquire fence in refcount_releasen(). */ if (atomic_fcmpset_rel_int(count, &old, old - 1)) return (true); @@ -213,6 +193,7 @@ static __inline __result_use_check bool refcount_release_if_not_last(volatile u_int *count) { - return refcount_release_if_gt(count, 1); + return (refcount_release_if_gt(count, 1)); } -#endif /* ! __SYS_REFCOUNT_H__ */ + +#endif /* !__SYS_REFCOUNT_H__ */ diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index b99b430af4fe..22c9b54e4b8b 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -377,7 +377,7 @@ vm_fault_restore_map_lock(struct faultstate *fs) { VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); if (!vm_map_trylock_read(fs->map)) { VM_OBJECT_WUNLOCK(fs->first_object); @@ -428,7 +428,7 @@ vm_fault_populate(struct faultstate *fs) MPASS(fs->object == fs->first_object); VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); MPASS(fs->first_object->backing_object == NULL); MPASS(fs->lookup_still_valid); diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 0169d06b730c..85a59287fb01 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -201,12 +202,11 @@ vm_object_zdtor(void *mem, int size, void *arg) ("object %p has reservations", object)); #endif - KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0, + KASSERT(blockcount_read(&object->paging_in_progress) == 0, ("object %p paging_in_progress = %d", - object, REFCOUNT_COUNT(object->paging_in_progress))); - KASSERT(object->busy == 0, - ("object %p busy = %d", - object, object->busy)); + object, blockcount_read(&object->paging_in_progress))); + KASSERT(!vm_object_busied(object), + ("object %p busy = %d", object, blockcount_read(&object->busy))); KASSERT(object->resident_page_count == 0, ("object %p resident_page_count = %d", object, object->resident_page_count)); @@ -231,8 +231,8 @@ vm_object_zinit(void *mem, int size, int flags) object->type = OBJT_DEAD; vm_radix_init(&object->rtree); refcount_init(&object->ref_count, 0); - refcount_init(&object->paging_in_progress, 0); - refcount_init(&object->busy, 0); + blockcount_init(&object->paging_in_progress); + blockcount_init(&object->busy); object->resident_page_count = 0; object->shadow_count = 0; object->flags = OBJ_DEAD; @@ -363,34 +363,36 @@ void vm_object_pip_add(vm_object_t object, short i) { - refcount_acquiren(&object->paging_in_progress, i); + if (i > 0) + blockcount_acquire(&object->paging_in_progress, i); } void vm_object_pip_wakeup(vm_object_t object) { - refcount_release(&object->paging_in_progress); + vm_object_pip_wakeupn(object, 1); } void vm_object_pip_wakeupn(vm_object_t object, short i) { - refcount_releasen(&object->paging_in_progress, i); + if (i > 0) + blockcount_release(&object->paging_in_progress, i); } /* - * Atomically drop the interlock and wait for pip to drain. This protects - * from sleep/wakeup races due to identity changes. The lock is not - * re-acquired on return. + * Atomically drop the object lock and wait for pip to drain. This protects + * from sleep/wakeup races due to identity changes. The lock is not re-acquired + * on return. */ static void vm_object_pip_sleep(vm_object_t object, const char *waitid) { - refcount_sleep_interlock(&object->paging_in_progress, - &object->lock, waitid, PVM); + (void)blockcount_sleep(&object->paging_in_progress, &object->lock, + waitid, PVM | PDROP); } void @@ -399,10 +401,8 @@ vm_object_pip_wait(vm_object_t object, const char *waitid) VM_OBJECT_ASSERT_WLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) { - vm_object_pip_sleep(object, waitid); - VM_OBJECT_WLOCK(object); - } + blockcount_wait(&object->paging_in_progress, &object->lock, waitid, + PVM); } void @@ -411,8 +411,7 @@ vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid) VM_OBJECT_ASSERT_UNLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) - refcount_wait(&object->paging_in_progress, waitid, PVM); + blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM); } /* @@ -955,7 +954,7 @@ vm_object_terminate(vm_object_t object) */ vm_object_pip_wait(object, "objtrm"); - KASSERT(!REFCOUNT_COUNT(object->paging_in_progress), + KASSERT(!blockcount_read(&object->paging_in_progress), ("vm_object_terminate: pageout in progress")); KASSERT(object->ref_count == 0, @@ -2458,7 +2457,7 @@ vm_object_busy(vm_object_t obj) VM_OBJECT_ASSERT_LOCKED(obj); - refcount_acquire(&obj->busy); + blockcount_acquire(&obj->busy, 1); /* The fence is required to order loads of page busy. */ atomic_thread_fence_acq_rel(); } @@ -2467,8 +2466,7 @@ void vm_object_unbusy(vm_object_t obj) { - - refcount_release(&obj->busy); + blockcount_release(&obj->busy, 1); } void @@ -2477,8 +2475,7 @@ vm_object_busy_wait(vm_object_t obj, const char *wmesg) VM_OBJECT_ASSERT_UNLOCKED(obj); - if (obj->busy) - refcount_sleep(&obj->busy, wmesg, PVM); + (void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM); } /* diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h index 71ddbe85bf3d..007e945daa77 100644 --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -70,6 +70,7 @@ #define _VM_OBJECT_ #include +#include #include #include #include @@ -113,8 +114,8 @@ struct vm_object { objtype_t type; /* type of pager */ u_short flags; /* see below */ u_short pg_color; /* (c) color of first page in obj */ - volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */ - volatile u_int busy; /* (a) object is busy, disallow page busy. */ + blockcount_t paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */ + blockcount_t busy; /* (a) object is busy, disallow page busy. */ int resident_page_count; /* number of resident pages */ struct vm_object *backing_object; /* object that I'm a shadow of */ vm_ooffset_t backing_object_offset;/* Offset in backing object */ @@ -265,7 +266,7 @@ extern struct vm_object kernel_object_store; lock_class_rw.lc_lock(&(object)->lock.lock_object, (state)) #define VM_OBJECT_ASSERT_PAGING(object) \ - KASSERT((object)->paging_in_progress != 0, \ + KASSERT(blockcount_read(&(object)->paging_in_progress) != 0, \ ("vm_object %p is not paging", object)) #define VM_OBJECT_ASSERT_REFERENCE(object) \ KASSERT((object)->reference_count != 0, \ @@ -348,7 +349,7 @@ static inline bool vm_object_busied(vm_object_t object) { - return (object->busy != 0); + return (blockcount_read(&object->busy) != 0); } #define VM_OBJECT_ASSERT_BUSY(object) MPASS(vm_object_busied((object))) diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h index 5ceaaf5bc8a4..a1177510a256 100644 --- a/sys/vm/vm_pager.h +++ b/sys/vm/vm_pager.h @@ -168,7 +168,7 @@ vm_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type, MPASS((object->flags & OBJ_POPULATE) != 0); MPASS(pidx < object->size); - MPASS(object->paging_in_progress > 0); + MPASS(blockcount_read(&object->paging_in_progress) > 0); return ((*pagertab[object->type]->pgo_populate)(object, pidx, fault_type, max_prot, first, last)); } diff --git a/sys/vm/vm_swapout.c b/sys/vm/vm_swapout.c index 6dd8f75d3865..bf4a248d7188 100644 --- a/sys/vm/vm_swapout.c +++ b/sys/vm/vm_swapout.c @@ -218,7 +218,7 @@ vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object, goto unlock_return; VM_OBJECT_ASSERT_LOCKED(object); if ((object->flags & OBJ_UNMANAGED) != 0 || - REFCOUNT_COUNT(object->paging_in_progress) > 0) + blockcount_read(&object->paging_in_progress) > 0) goto unlock_return; unmap = true;