Use the trick of performing the atomic operation on the contained aligned
word to handle the dirty mask updates in vm_page_clear_dirty_mask(). Remove the vm page queue lock around vm_page_dirty() call in vm_fault_hold() the sole purpose of which was to protect dirty on architectures which does not provide short or byte-wide atomics. Reviewed by: alc, attilio Tested by: flo (sparc64) MFC after: 2 weeks
This commit is contained in:
parent
7778ab7e0c
commit
abb9b935ca
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=225840
@ -1090,18 +1090,10 @@ vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
|
||||
* performed through an unmanaged mapping or by a DMA
|
||||
* operation.
|
||||
*
|
||||
* The object lock is not held here. Therefore, like
|
||||
* a pmap operation, the page queues lock may be
|
||||
* required in order to call vm_page_dirty(). See
|
||||
* vm_page_clear_dirty_mask().
|
||||
* The object lock is not held here.
|
||||
* See vm_page_clear_dirty_mask().
|
||||
*/
|
||||
#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
|
||||
vm_page_dirty(*mp);
|
||||
#else
|
||||
vm_page_lock_queues();
|
||||
vm_page_dirty(*mp);
|
||||
vm_page_unlock_queues();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
if (pmap_failed) {
|
||||
|
@ -745,9 +745,9 @@ vm_page_sleep(vm_page_t m, const char *msg)
|
||||
*
|
||||
* Set all bits in the page's dirty field.
|
||||
*
|
||||
* The object containing the specified page must be locked if the call is
|
||||
* made from the machine-independent layer. If, however, the call is
|
||||
* made from the pmap layer, then the page queues lock may be required.
|
||||
* The object containing the specified page must be locked if the
|
||||
* call is made from the machine-independent layer.
|
||||
*
|
||||
* See vm_page_clear_dirty_mask().
|
||||
*/
|
||||
void
|
||||
@ -2339,44 +2339,53 @@ vm_page_set_valid(vm_page_t m, int base, int size)
|
||||
static __inline void
|
||||
vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
|
||||
{
|
||||
uintptr_t addr;
|
||||
#if PAGE_SIZE < 16384
|
||||
int shift;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* If the object is locked and the page is neither VPO_BUSY nor
|
||||
* PGA_WRITEABLE, then the page's dirty field cannot possibly be
|
||||
* set by a concurrent pmap operation.
|
||||
* set by a concurrent pmap operation.
|
||||
*
|
||||
*/
|
||||
VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
|
||||
if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
|
||||
m->dirty &= ~pagebits;
|
||||
else {
|
||||
#if defined(__amd64__) || defined(__i386__) || defined(__ia64__)
|
||||
/*
|
||||
* On the aforementioned architectures, the page queues lock
|
||||
* is not required by the following read-modify-write
|
||||
* operation. The combination of the object's lock and an
|
||||
* atomic operation suffice. Moreover, the pmap layer on
|
||||
* these architectures can call vm_page_dirty() without
|
||||
* holding the page queues lock.
|
||||
* The pmap layer can call vm_page_dirty() without
|
||||
* holding a distinguished lock. The combination of
|
||||
* the object's lock and an atomic operation suffice
|
||||
* to guarantee consistency of the page dirty field.
|
||||
*
|
||||
* For PAGE_SIZE == 32768 case, compiler already
|
||||
* properly aligns the dirty field, so no forcible
|
||||
* alignment is needed. Only require existence of
|
||||
* atomic_clear_64 when page size if 32768.
|
||||
*/
|
||||
#if PAGE_SIZE == 4096
|
||||
atomic_clear_char(&m->dirty, pagebits);
|
||||
#elif PAGE_SIZE == 8192
|
||||
atomic_clear_short(&m->dirty, pagebits);
|
||||
addr = (uintptr_t)&m->dirty;
|
||||
#if PAGE_SIZE == 32768
|
||||
#error pagebits too short
|
||||
atomic_clear_64((uint64_t *)addr, pagebits);
|
||||
#elif PAGE_SIZE == 16384
|
||||
atomic_clear_int(&m->dirty, pagebits);
|
||||
#else
|
||||
#error "PAGE_SIZE is not supported."
|
||||
#endif
|
||||
#else
|
||||
atomic_clear_32((uint32_t *)addr, pagebits);
|
||||
#else /* PAGE_SIZE <= 8192 */
|
||||
/*
|
||||
* Otherwise, the page queues lock is required to ensure that
|
||||
* a concurrent pmap operation does not set the page's dirty
|
||||
* field during the following read-modify-write operation.
|
||||
* Use a trick to perform an 32bit atomic on the
|
||||
* contained aligned word, to not depend on existence
|
||||
* of the atomic_clear_{8, 16}.
|
||||
*/
|
||||
vm_page_lock_queues();
|
||||
m->dirty &= ~pagebits;
|
||||
vm_page_unlock_queues();
|
||||
shift = addr & (sizeof(uint32_t) - 1);
|
||||
#if BYTE_ORDER == BIG_ENDIAN
|
||||
shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
|
||||
#else
|
||||
shift *= NBBY;
|
||||
#endif
|
||||
addr &= ~(sizeof(uint32_t) - 1);
|
||||
atomic_clear_32((uint32_t *)addr, pagebits << shift);
|
||||
#endif /* PAGE_SIZE */
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -94,21 +94,21 @@
|
||||
* object that the page belongs to (O), the pool lock for the page (P),
|
||||
* or the lock for either the free or paging queues (Q). If a field is
|
||||
* annotated below with two of these locks, then holding either lock is
|
||||
* sufficient for read access, but both locks are required for write
|
||||
* sufficient for read access, but both locks are required for write
|
||||
* access.
|
||||
*
|
||||
* In contrast, the synchronization of accesses to the page's dirty field
|
||||
* is machine dependent (M). In the machine-independent layer, the lock
|
||||
* on the object that the page belongs to must be held in order to
|
||||
* operate on the field. However, the pmap layer is permitted to set
|
||||
* all bits within the field without holding that lock. Therefore, if
|
||||
* the underlying architecture does not support atomic read-modify-write
|
||||
* operations on the field's type, then the machine-independent layer
|
||||
* must also hold the page queues lock when performing read-modify-write
|
||||
* operations and the pmap layer must hold the page queues lock when
|
||||
* setting the field. In the machine-independent layer, the
|
||||
* implementation of read-modify-write operations on the field is
|
||||
* encapsulated in vm_page_clear_dirty_mask().
|
||||
* In contrast, the synchronization of accesses to the page's
|
||||
* dirty field is machine dependent (M). In the
|
||||
* machine-independent layer, the lock on the object that the
|
||||
* page belongs to must be held in order to operate on the field.
|
||||
* However, the pmap layer is permitted to set all bits within
|
||||
* the field without holding that lock. If the underlying
|
||||
* architecture does not support atomic read-modify-write
|
||||
* operations on the field's type, then the machine-independent
|
||||
* layer uses 32bit atomic on the aligned 32bit word that
|
||||
* contains the dirty field. In the machine-independent layer,
|
||||
* the implementation of read-modify-write operations on the
|
||||
* field is encapsulated in vm_page_clear_dirty_mask().
|
||||
*/
|
||||
|
||||
TAILQ_HEAD(pglist, vm_page);
|
||||
|
Loading…
Reference in New Issue
Block a user