From 9689d5e5ee12fcabc1aeb63224dbe1137986c236 Mon Sep 17 00:00:00 2001 From: Brian Feldman Date: Mon, 9 Aug 2004 19:52:29 +0000 Subject: [PATCH] Revamp VM map wiring. * Allow no-fault wiring/unwiring to succeed for consistency; however, the wired count remains at zero, so it's a special case. * Fix issues inside vm_map_wire() and vm_map_unwire() where the exact state of user wiring (one or zero) and system wiring (zero or more) could be confused; for example, system unwiring could succeed in removing a user wire, instead of being an error. * Require all mappings to be unwired before they are deleted. When VM space is still wired upon deletion, it will be waited upon for the following unwire. This makes vslock(9) work rather than allowing kernel-locked memory to be deleted out from underneath of its consumer as it would before. --- sys/vm/vm_kern.c | 4 ++++ sys/vm/vm_map.c | 46 ++++++++++++++++++++++++++++++++-------------- sys/vm/vm_map.h | 14 ++++++++++++++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index 5231d2ca1089..27ab5b6b8095 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -206,7 +206,11 @@ kmem_free(map, addr, size) vm_offset_t addr; vm_size_t size; { + int error; + error = vm_map_unwire(map, trunc_page(addr), round_page(addr + size), + VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_HOLESOK); + KASSERT(error == 0, ("kmem_free could not vm_map_unwire")); (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); } diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index c3546552de02..e4dfccc93bb8 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1606,6 +1606,8 @@ vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, * vm_map_unwire: * * Implements both kernel and user unwiring. + * + * Ignores MAP_NOFAULT (wired_count == 0 is okay) for kernel unwiring. */ int vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, @@ -1698,8 +1700,9 @@ vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, /* * If system unwiring, require that the entry is system wired. */ - if (!user_unwire && entry->wired_count < ((entry->eflags & - MAP_ENTRY_USER_WIRED) ? 2 : 1)) { + if (user_unwire ? vm_map_entry_user_wired_count(entry) == 0 : + (vm_map_entry_system_wired_count(entry) == 0 && + (entry->eflags & MAP_ENTRY_NOFAULT) == 0)) { end = entry->end; rv = KERN_INVALID_ARGUMENT; goto done; @@ -1722,8 +1725,11 @@ vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, (entry->eflags & MAP_ENTRY_USER_WIRED))) { if (user_unwire) entry->eflags &= ~MAP_ENTRY_USER_WIRED; - entry->wired_count--; - if (entry->wired_count == 0) { + if (user_unwire || + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) + entry->wired_count--; + if (entry->wired_count == 0 && + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) { /* * Retain the map lock. */ @@ -1831,10 +1837,14 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, */ entry->eflags |= MAP_ENTRY_IN_TRANSITION; /* - * + * The wired_count keeps track of zero or one user + * wirings. It also keeps track of system wirings if + * MAP_ENTRY_NOFAULT is not set. */ - if (entry->wired_count == 0) { - entry->wired_count++; + if ((user_wire ? vm_map_entry_user_wired_count(entry) == 0 : + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) && + entry->wired_count++ == 0 && + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) { saved_start = entry->start; saved_end = entry->end; fictitious = entry->object.vm_object != NULL && @@ -1883,9 +1893,6 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, end = entry->end; goto done; } - } else if (!user_wire || - (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { - entry->wired_count++; } /* * Check the map for holes in the specified region. @@ -1913,6 +1920,11 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, entry = first_entry; while (entry != &map->header && entry->start < end) { if (rv == KERN_SUCCESS) { + /* + * The MAP_ENTRY_USER_WIRED may already have been + * set. If so, this statement has no effect + * (and wired_count will not have changed). + */ if (user_wire) entry->eflags |= MAP_ENTRY_USER_WIRED; } else if (entry->wired_count == -1) { @@ -1922,10 +1934,11 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, */ entry->wired_count = 0; } else { - if (!user_wire || - (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) + if (user_wire || + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) entry->wired_count--; - if (entry->wired_count == 0) { + if (entry->wired_count == 0 && + (entry->eflags & MAP_ENTRY_NOFAULT) == 0) { /* * Retain the map lock. */ @@ -2053,6 +2066,9 @@ vm_map_sync( static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) { + KASSERT(vm_map_entry_user_wired_count(entry) == 1 && + vm_map_entry_system_wired_count(entry) == 0, + ("system/user wiring mistake")); vm_fault_unwire(map, entry->start, entry->end, entry->object.vm_object != NULL && entry->object.vm_object->type == OBJT_DEVICE); @@ -2137,8 +2153,10 @@ vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) /* * Wait for wiring or unwiring of an entry to complete. + * Also wait for any system wirings to disappear. */ - if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0) { + if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 || + vm_map_entry_system_wired_count(entry) != 0) { unsigned int last_timestamp; vm_offset_t saved_start; vm_map_entry_t tmp_entry; diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h index 8616c60c237f..6531ff0b6485 100644 --- a/sys/vm/vm_map.h +++ b/sys/vm/vm_map.h @@ -142,6 +142,20 @@ vm_map_entry_behavior(vm_map_entry_t entry) { return (entry->eflags & MAP_ENTRY_BEHAV_MASK); } + +static __inline int +vm_map_entry_user_wired_count(vm_map_entry_t entry) +{ + if (entry->eflags & MAP_ENTRY_USER_WIRED) + return (1); + return (0); +} + +static __inline int +vm_map_entry_system_wired_count(vm_map_entry_t entry) +{ + return (entry->wired_count - vm_map_entry_user_wired_count(entry)); +} #endif /* _KERNEL */ /*