vm_page: Consolidate page busy sleep mechanisms
- Modify vm_page_busy_sleep() and vm_page_busy_sleep_unlocked() to take a VM_ALLOC_* flag indicating whether to sleep on shared-busy, and fix up callers. - Modify vm_page_busy_sleep() to return a status indicating whether the object lock was dropped, and fix up callers. - Convert callers of vm_page_sleep_if_busy() to use vm_page_busy_sleep() instead. - Remove vm_page_sleep_if_(x)busy(). No functional change intended. Obtained from: jeff (object_concurrency patches) Reviewed by: kib MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D32947
This commit is contained in:
parent
b0acc3f11b
commit
87b646630c
@ -40,6 +40,9 @@
|
||||
# xargs -n1 | sort | uniq -d;
|
||||
# done
|
||||
|
||||
# 20211115: vm_page_sleep_if_busy removed
|
||||
OLD_FILES+=share/man/man9/vm_page_sleep_if_busy.9.gz
|
||||
|
||||
# 20211113: new clang import which bumps version from 12.0.1 to 13.0.0.
|
||||
OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/algorithm
|
||||
OLD_FILES+=usr/lib/clang/12.0.1/include/cuda_wrappers/complex
|
||||
|
@ -2354,7 +2354,6 @@ MLINKS+=vm_page_busy.9 vm_page_busied.9 \
|
||||
vm_page_busy.9 vm_page_busy_sleep.9 \
|
||||
vm_page_busy.9 vm_page_sbusied.9 \
|
||||
vm_page_busy.9 vm_page_sbusy.9 \
|
||||
vm_page_busy.9 vm_page_sleep_if_busy.9 \
|
||||
vm_page_busy.9 vm_page_sunbusy.9 \
|
||||
vm_page_busy.9 vm_page_trysbusy.9 \
|
||||
vm_page_busy.9 vm_page_tryxbusy.9 \
|
||||
|
@ -24,7 +24,7 @@
|
||||
.\" DAMAGE.
|
||||
.\"
|
||||
.\" $FreeBSD$
|
||||
.Dd August 07, 2013
|
||||
.Dd November 11, 2021
|
||||
.Dt VM_PAGE_BUSY 9
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -33,7 +33,6 @@
|
||||
.Nm vm_page_busy_sleep ,
|
||||
.Nm vm_page_sbusied ,
|
||||
.Nm vm_page_sbusy ,
|
||||
.Nm vm_page_sleep_if_busy ,
|
||||
.Nm vm_page_sunbusy ,
|
||||
.Nm vm_page_trysbusy ,
|
||||
.Nm vm_page_tryxbusy ,
|
||||
@ -52,14 +51,12 @@
|
||||
.Fn vm_page_busied "vm_page_t m"
|
||||
.Ft void
|
||||
.Fn vm_page_busy_downgrade "vm_page_t m"
|
||||
.Ft void
|
||||
.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg"
|
||||
.Ft bool
|
||||
.Fn vm_page_busy_sleep "vm_page_t m" "const char *msg" "int allocflags"
|
||||
.Ft int
|
||||
.Fn vm_page_sbusied "vm_page_t m"
|
||||
.Ft void
|
||||
.Fn vm_page_sbusy "vm_page_t m"
|
||||
.Ft int
|
||||
.Fn vm_page_sleep_if_busy "vm_page_t m" "const char *msg"
|
||||
.Ft void
|
||||
.Fn vm_page_sunbusy "vm_page_t m"
|
||||
.Ft int
|
||||
@ -114,8 +111,23 @@ from an exclusive busy state to a shared busy state.
|
||||
.Pp
|
||||
The
|
||||
.Fn vm_page_busy_sleep
|
||||
function puts the invoking thread to sleep using the appropriate
|
||||
waitchannels for the busy mechanism.
|
||||
checks the busy state of the page
|
||||
.Fa m
|
||||
and puts the invoking thread to sleep if the page is busy.
|
||||
The VM object for the page must be locked.
|
||||
The
|
||||
.Fa allocflags
|
||||
parameter must be either
|
||||
.Dv 0 ,
|
||||
in which case the function will sleep if the page is busied,
|
||||
or
|
||||
.Dv VM_ALLOC_IGN_SBUSY ,
|
||||
in which case the function will sleep only if the page is exclusively
|
||||
busied.
|
||||
A return value of true indicates that the invoking thread was put to
|
||||
sleep and that the object was unlocked.
|
||||
A return value of false indicates that the invoking thread did not sleep
|
||||
and the object remains locked.
|
||||
The parameter
|
||||
.Fa msg
|
||||
is a string describing the sleep condition for userland tools.
|
||||
@ -133,18 +145,6 @@ function shared busies
|
||||
.Fa m .
|
||||
.Pp
|
||||
The
|
||||
.Fn vm_page_sleep_if_busy
|
||||
function puts the invoking thread to sleep, using the appropriate
|
||||
waitchannels for the busy mechanism, if
|
||||
.Fa m .
|
||||
is busied in either exclusive or shared mode.
|
||||
If the invoking thread slept a non-zero value is returned, otherwise
|
||||
0 is returned.
|
||||
The parameter
|
||||
.Fa msg
|
||||
is a string describing the sleep condition for userland tools.
|
||||
.Pp
|
||||
The
|
||||
.Fn vm_page_sunbusy
|
||||
function shared unbusies
|
||||
.Fa m .
|
||||
|
@ -1294,9 +1294,8 @@ vm_fault_busy_sleep(struct faultstate *fs)
|
||||
}
|
||||
vm_object_pip_wakeup(fs->object);
|
||||
unlock_map(fs);
|
||||
if (fs->m == vm_page_lookup(fs->object, fs->pindex))
|
||||
vm_page_busy_sleep(fs->m, "vmpfw", false);
|
||||
else
|
||||
if (fs->m != vm_page_lookup(fs->object, fs->pindex) ||
|
||||
!vm_page_busy_sleep(fs->m, "vmpfw", 0))
|
||||
VM_OBJECT_WUNLOCK(fs->object);
|
||||
VM_CNT_INC(v_intrans);
|
||||
vm_object_deallocate(fs->first_object);
|
||||
|
@ -1397,7 +1397,8 @@ vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
|
||||
*/
|
||||
vm_page_aflag_set(tm, PGA_REFERENCED);
|
||||
}
|
||||
vm_page_busy_sleep(tm, "madvpo", false);
|
||||
if (!vm_page_busy_sleep(tm, "madvpo", 0))
|
||||
VM_OBJECT_WUNLOCK(tobject);
|
||||
goto relookup;
|
||||
}
|
||||
vm_page_advise(tm, advice);
|
||||
@ -1581,7 +1582,8 @@ vm_object_split(vm_map_entry_t entry)
|
||||
*/
|
||||
if (vm_page_tryxbusy(m) == 0) {
|
||||
VM_OBJECT_WUNLOCK(new_object);
|
||||
vm_page_sleep_if_busy(m, "spltwt");
|
||||
if (vm_page_busy_sleep(m, "spltwt", 0))
|
||||
VM_OBJECT_WLOCK(orig_object);
|
||||
VM_OBJECT_WLOCK(new_object);
|
||||
goto retry;
|
||||
}
|
||||
@ -1667,14 +1669,17 @@ vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
VM_OBJECT_WUNLOCK(backing_object);
|
||||
vm_radix_wait();
|
||||
VM_OBJECT_WLOCK(object);
|
||||
} else if (p->object == object) {
|
||||
VM_OBJECT_WUNLOCK(backing_object);
|
||||
if (vm_page_busy_sleep(p, "vmocol", 0))
|
||||
VM_OBJECT_WLOCK(object);
|
||||
} else {
|
||||
if (p->object == object)
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
if (!vm_page_busy_sleep(p, "vmocol", 0))
|
||||
VM_OBJECT_WUNLOCK(backing_object);
|
||||
else
|
||||
VM_OBJECT_WUNLOCK(object);
|
||||
vm_page_busy_sleep(p, "vmocol", false);
|
||||
VM_OBJECT_WLOCK(object);
|
||||
}
|
||||
VM_OBJECT_WLOCK(object);
|
||||
VM_OBJECT_WLOCK(backing_object);
|
||||
return (TAILQ_FIRST(&backing_object->memq));
|
||||
}
|
||||
@ -2103,7 +2108,8 @@ vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
|
||||
* not specified.
|
||||
*/
|
||||
if (vm_page_tryxbusy(p) == 0) {
|
||||
vm_page_sleep_if_busy(p, "vmopar");
|
||||
if (vm_page_busy_sleep(p, "vmopar", 0))
|
||||
VM_OBJECT_WLOCK(object);
|
||||
goto again;
|
||||
}
|
||||
if (vm_page_wired(p)) {
|
||||
@ -2407,7 +2413,10 @@ vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
|
||||
VM_OBJECT_RUNLOCK(tobject);
|
||||
tobject = t1object;
|
||||
}
|
||||
vm_page_busy_sleep(tm, "unwbo", true);
|
||||
tobject = tm->object;
|
||||
if (!vm_page_busy_sleep(tm, "unwbo",
|
||||
VM_ALLOC_IGN_SBUSY))
|
||||
VM_OBJECT_RUNLOCK(tobject);
|
||||
goto again;
|
||||
}
|
||||
vm_page_unwire(tm, queue);
|
||||
|
@ -1000,24 +1000,26 @@ vm_page_sunbusy(vm_page_t m)
|
||||
* vm_page_busy_sleep:
|
||||
*
|
||||
* Sleep if the page is busy, using the page pointer as wchan.
|
||||
* This is used to implement the hard-path of busying mechanism.
|
||||
* This is used to implement the hard-path of the busying mechanism.
|
||||
*
|
||||
* If nonshared is true, sleep only if the page is xbusy.
|
||||
* If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
|
||||
* will not sleep if the page is shared-busy.
|
||||
*
|
||||
* The object lock must be held on entry and will be released on exit.
|
||||
* The object lock must be held on entry.
|
||||
*
|
||||
* Returns true if it slept and dropped the object lock, or false
|
||||
* if there was no sleep and the lock is still held.
|
||||
*/
|
||||
void
|
||||
vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
|
||||
bool
|
||||
vm_page_busy_sleep(vm_page_t m, const char *wmesg, int allocflags)
|
||||
{
|
||||
vm_object_t obj;
|
||||
|
||||
obj = m->object;
|
||||
VM_OBJECT_ASSERT_LOCKED(obj);
|
||||
vm_page_lock_assert(m, MA_NOTOWNED);
|
||||
|
||||
if (!_vm_page_busy_sleep(obj, m, m->pindex, wmesg,
|
||||
nonshared ? VM_ALLOC_SBUSY : 0 , true))
|
||||
VM_OBJECT_DROP(obj);
|
||||
return (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, allocflags,
|
||||
true));
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1026,21 +1028,19 @@ vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
|
||||
* Sleep if the page is busy, using the page pointer as wchan.
|
||||
* This is used to implement the hard-path of busying mechanism.
|
||||
*
|
||||
* If nonshared is true, sleep only if the page is xbusy.
|
||||
* If VM_ALLOC_IGN_SBUSY is specified in allocflags, the function
|
||||
* will not sleep if the page is shared-busy.
|
||||
*
|
||||
* The object lock must not be held on entry. The operation will
|
||||
* return if the page changes identity.
|
||||
*/
|
||||
void
|
||||
vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
|
||||
const char *wmesg, bool nonshared)
|
||||
const char *wmesg, int allocflags)
|
||||
{
|
||||
|
||||
VM_OBJECT_ASSERT_UNLOCKED(obj);
|
||||
vm_page_lock_assert(m, MA_NOTOWNED);
|
||||
|
||||
_vm_page_busy_sleep(obj, m, pindex, wmesg,
|
||||
nonshared ? VM_ALLOC_SBUSY : 0, false);
|
||||
(void)_vm_page_busy_sleep(obj, m, pindex, wmesg, allocflags, false);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -1050,6 +1050,8 @@ vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
|
||||
* lockstate against parameters. Returns true if it sleeps and
|
||||
* false otherwise.
|
||||
*
|
||||
* allocflags uses VM_ALLOC_* flags to specify the lock required.
|
||||
*
|
||||
* If locked is true the lock will be dropped for any true returns
|
||||
* and held for any false returns.
|
||||
*/
|
||||
@ -1395,71 +1397,6 @@ vm_page_free_invalid(vm_page_t m)
|
||||
vm_page_free(m);
|
||||
}
|
||||
|
||||
/*
|
||||
* vm_page_sleep_if_busy:
|
||||
*
|
||||
* Sleep and release the object lock if the page is busied.
|
||||
* Returns TRUE if the thread slept.
|
||||
*
|
||||
* The given page must be unlocked and object containing it must
|
||||
* be locked.
|
||||
*/
|
||||
int
|
||||
vm_page_sleep_if_busy(vm_page_t m, const char *wmesg)
|
||||
{
|
||||
vm_object_t obj;
|
||||
|
||||
vm_page_lock_assert(m, MA_NOTOWNED);
|
||||
VM_OBJECT_ASSERT_WLOCKED(m->object);
|
||||
|
||||
/*
|
||||
* The page-specific object must be cached because page
|
||||
* identity can change during the sleep, causing the
|
||||
* re-lock of a different object.
|
||||
* It is assumed that a reference to the object is already
|
||||
* held by the callers.
|
||||
*/
|
||||
obj = m->object;
|
||||
if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, 0, true)) {
|
||||
VM_OBJECT_WLOCK(obj);
|
||||
return (TRUE);
|
||||
}
|
||||
return (FALSE);
|
||||
}
|
||||
|
||||
/*
|
||||
* vm_page_sleep_if_xbusy:
|
||||
*
|
||||
* Sleep and release the object lock if the page is xbusied.
|
||||
* Returns TRUE if the thread slept.
|
||||
*
|
||||
* The given page must be unlocked and object containing it must
|
||||
* be locked.
|
||||
*/
|
||||
int
|
||||
vm_page_sleep_if_xbusy(vm_page_t m, const char *wmesg)
|
||||
{
|
||||
vm_object_t obj;
|
||||
|
||||
vm_page_lock_assert(m, MA_NOTOWNED);
|
||||
VM_OBJECT_ASSERT_WLOCKED(m->object);
|
||||
|
||||
/*
|
||||
* The page-specific object must be cached because page
|
||||
* identity can change during the sleep, causing the
|
||||
* re-lock of a different object.
|
||||
* It is assumed that a reference to the object is already
|
||||
* held by the callers.
|
||||
*/
|
||||
obj = m->object;
|
||||
if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, VM_ALLOC_SBUSY,
|
||||
true)) {
|
||||
VM_OBJECT_WLOCK(obj);
|
||||
return (TRUE);
|
||||
}
|
||||
return (FALSE);
|
||||
}
|
||||
|
||||
/*
|
||||
* vm_page_dirty_KBI: [ internal use only ]
|
||||
*
|
||||
|
@ -599,9 +599,9 @@ malloc2vm_flags(int malloc_flags)
|
||||
bool vm_page_busy_acquire(vm_page_t m, int allocflags);
|
||||
void vm_page_busy_downgrade(vm_page_t m);
|
||||
int vm_page_busy_tryupgrade(vm_page_t m);
|
||||
void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
|
||||
bool vm_page_busy_sleep(vm_page_t m, const char *msg, int allocflags);
|
||||
void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
|
||||
vm_pindex_t pindex, const char *wmesg, bool nonshared);
|
||||
vm_pindex_t pindex, const char *wmesg, int allocflags);
|
||||
void vm_page_free(vm_page_t m);
|
||||
void vm_page_free_zero(vm_page_t m);
|
||||
|
||||
@ -685,8 +685,6 @@ vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
|
||||
vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
|
||||
vm_page_bits_t vm_page_set_dirty(vm_page_t m);
|
||||
void vm_page_set_valid_range(vm_page_t m, int base, int size);
|
||||
int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
|
||||
int vm_page_sleep_if_xbusy(vm_page_t m, const char *msg);
|
||||
vm_offset_t vm_page_startup(vm_offset_t vaddr);
|
||||
void vm_page_sunbusy(vm_page_t m);
|
||||
bool vm_page_try_remove_all(vm_page_t m);
|
||||
|
Loading…
Reference in New Issue
Block a user