diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index f5807c166244..1263f0c677a7 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -87,7 +87,7 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include +#include #ifdef HWPMC_HOOKS #include @@ -2502,7 +2502,7 @@ vn_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, * writecount, then undo that now. */ if (writecounted) - vnode_pager_release_writecount(object, 0, size); + vm_pager_release_writecount(object, 0, size); vm_object_deallocate(object); } #ifdef HWPMC_HOOKS diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index fa05627b55cc..4ea49c7aa4a2 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -376,6 +376,10 @@ static boolean_t static void swap_pager_init(void); static void swap_pager_unswapped(vm_page_t); static void swap_pager_swapoff(struct swdevt *sp); +static void swap_pager_update_writecount(vm_object_t object, + vm_offset_t start, vm_offset_t end); +static void swap_pager_release_writecount(vm_object_t object, + vm_offset_t start, vm_offset_t end); struct pagerops swappagerops = { .pgo_init = swap_pager_init, /* early system initialization of pager */ @@ -386,6 +390,8 @@ struct pagerops swappagerops = { .pgo_putpages = swap_pager_putpages, /* pageout */ .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ + .pgo_update_writecount = swap_pager_update_writecount, + .pgo_release_writecount = swap_pager_release_writecount, }; /* @@ -611,6 +617,7 @@ swap_pager_alloc_init(void *handle, struct ucred *cred, vm_ooffset_t size, object = vm_object_allocate(OBJT_SWAP, OFF_TO_IDX(offset + PAGE_MASK + size)); + object->un_pager.swp.writemappings = 0; object->handle = handle; if (cred != NULL) { object->cred = cred; @@ -1903,6 +1910,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) atomic_thread_fence_rel(); object->type = OBJT_SWAP; + object->un_pager.swp.writemappings = 0; KASSERT(object->handle == NULL, ("default pager with handle")); } @@ -2991,3 +2999,27 @@ sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) return (0); } + +static void +swap_pager_update_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + VM_OBJECT_WLOCK(object); + KASSERT((object->flags & OBJ_NOSPLIT) != 0, + ("Splittable object with writecount")); + object->un_pager.swp.writemappings += (vm_ooffset_t)end - start; + VM_OBJECT_WUNLOCK(object); +} + +static void +swap_pager_release_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + VM_OBJECT_WLOCK(object); + KASSERT((object->flags & OBJ_NOSPLIT) != 0, + ("Splittable object with writecount")); + object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; + VM_OBJECT_WUNLOCK(object); +} diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index e33fbc248d16..50289e5a34cd 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -575,10 +575,10 @@ vm_map_process_deferred(void) td->td_map_def_user = NULL; while (entry != NULL) { next = entry->next; - MPASS((entry->eflags & (MAP_ENTRY_VN_WRITECNT | - MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_VN_WRITECNT | + MPASS((entry->eflags & (MAP_ENTRY_WRITECNT | + MAP_ENTRY_VN_EXEC)) != (MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC)); - if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) { + if ((entry->eflags & MAP_ENTRY_WRITECNT) != 0) { /* * Decrement the object's writemappings and * possibly the vnode's v_writecount. @@ -587,7 +587,7 @@ vm_map_process_deferred(void) ("Submap with writecount")); object = entry->object.vm_object; KASSERT(object != NULL, ("No object for writecount")); - vnode_pager_release_writecount(object, entry->start, + vm_pager_release_writecount(object, entry->start, entry->end); } vm_map_entry_set_vnode_text(entry, false); @@ -1481,8 +1481,8 @@ vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, protoeflags |= MAP_ENTRY_GROWS_DOWN; if (cow & MAP_STACK_GROWS_UP) protoeflags |= MAP_ENTRY_GROWS_UP; - if (cow & MAP_VN_WRITECOUNT) - protoeflags |= MAP_ENTRY_VN_WRITECNT; + if (cow & MAP_WRITECOUNT) + protoeflags |= MAP_ENTRY_WRITECNT; if (cow & MAP_VN_EXEC) protoeflags |= MAP_ENTRY_VN_EXEC; if ((cow & MAP_CREATE_GUARD) != 0) @@ -2211,7 +2211,7 @@ _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) vm_map_entry_set_vnode_text(new_entry, true); /* * The object->un_pager.vnp.writemappings for the - * object of MAP_ENTRY_VN_WRITECNT type entry shall be + * object of MAP_ENTRY_WRITECNT type entry shall be * kept as is here. The virtual pages are * re-distributed among the clipped entries, so the sum is * left the same. @@ -3783,20 +3783,20 @@ vm_map_copy_entry( dst_entry->eflags |= MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY; dst_entry->offset = src_entry->offset; - if (src_entry->eflags & MAP_ENTRY_VN_WRITECNT) { + if (src_entry->eflags & MAP_ENTRY_WRITECNT) { /* - * MAP_ENTRY_VN_WRITECNT cannot + * MAP_ENTRY_WRITECNT cannot * indicate write reference from * src_entry, since the entry is * marked as needs copy. Allocate a * fake entry that is used to - * decrement object->un_pager.vnp.writecount + * decrement object->un_pager writecount * at the appropriate time. Attach * fake_entry to the deferred list. */ fake_entry = vm_map_entry_create(dst_map); - fake_entry->eflags = MAP_ENTRY_VN_WRITECNT; - src_entry->eflags &= ~MAP_ENTRY_VN_WRITECNT; + fake_entry->eflags = MAP_ENTRY_WRITECNT; + src_entry->eflags &= ~MAP_ENTRY_WRITECNT; vm_object_reference(src_object); fake_entry->object.vm_object = src_object; fake_entry->start = src_entry->start; @@ -3971,7 +3971,7 @@ vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) * not relock it later for the assertion * correctness. */ - if (old_entry->eflags & MAP_ENTRY_VN_WRITECNT && + if (old_entry->eflags & MAP_ENTRY_WRITECNT && object->type == OBJT_VNODE) { KASSERT(((struct vnode *)object->handle)-> v_writecount > 0, @@ -3991,8 +3991,8 @@ vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) MAP_ENTRY_IN_TRANSITION); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; - if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) { - vnode_pager_update_writecount(object, + if (new_entry->eflags & MAP_ENTRY_WRITECNT) { + vm_pager_update_writecount(object, new_entry->start, new_entry->end); } vm_map_entry_set_vnode_text(new_entry, true); @@ -4023,7 +4023,7 @@ vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) * Copied entry is COW over the old object. */ new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED | - MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_VN_WRITECNT); + MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_WRITECNT); new_entry->wiring_thread = NULL; new_entry->wired_count = 0; new_entry->object.vm_object = NULL; @@ -4047,7 +4047,7 @@ vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) new_entry->end = old_entry->end; new_entry->eflags = old_entry->eflags & ~(MAP_ENTRY_USER_WIRED | MAP_ENTRY_IN_TRANSITION | - MAP_ENTRY_VN_WRITECNT | MAP_ENTRY_VN_EXEC); + MAP_ENTRY_WRITECNT | MAP_ENTRY_VN_EXEC); new_entry->protection = old_entry->protection; new_entry->max_protection = old_entry->max_protection; new_entry->inheritance = VM_INHERIT_ZERO; diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h index 453011a0daca..964d1da1d9a9 100644 --- a/sys/vm/vm_map.h +++ b/sys/vm/vm_map.h @@ -144,7 +144,7 @@ struct vm_map_entry { #define MAP_ENTRY_GROWS_UP 0x00002000 /* bottom-up stacks */ #define MAP_ENTRY_WIRE_SKIPPED 0x00004000 -#define MAP_ENTRY_VN_WRITECNT 0x00008000 /* writeable vnode +#define MAP_ENTRY_WRITECNT 0x00008000 /* tracked writeable mapping */ #define MAP_ENTRY_GUARD 0x00010000 #define MAP_ENTRY_STACK_GAP_DN 0x00020000 @@ -348,7 +348,7 @@ long vmspace_resident_count(struct vmspace *vmspace); #define MAP_CREATE_GUARD 0x00000080 #define MAP_DISABLE_COREDUMP 0x00000100 #define MAP_PREFAULT_MADVISE 0x00000200 /* from (user) madvise request */ -#define MAP_VN_WRITECOUNT 0x00000400 +#define MAP_WRITECOUNT 0x00000400 #define MAP_REMAP 0x00000800 #define MAP_STACK_GROWS_DOWN 0x00001000 #define MAP_STACK_GROWS_UP 0x00002000 diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index 31941b09e86c..23bdde163cec 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -1289,7 +1289,7 @@ vm_mmap_vnode(struct thread *td, vm_size_t objsize, } if (writex) { *writecounted = TRUE; - vnode_pager_update_writecount(obj, 0, objsize); + vm_pager_update_writecount(obj, 0, objsize); } } else { error = EINVAL; @@ -1345,7 +1345,7 @@ vm_mmap_vnode(struct thread *td, vm_size_t objsize, done: if (error != 0 && *writecounted) { *writecounted = FALSE; - vnode_pager_update_writecount(obj, objsize, 0); + vm_pager_update_writecount(obj, objsize, 0); } vput(vp); return (error); @@ -1480,7 +1480,7 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, * writecount, then undo that now. */ if (writecounted) - vnode_pager_release_writecount(object, 0, size); + vm_pager_release_writecount(object, 0, size); vm_object_deallocate(object); } return (error); @@ -1569,7 +1569,7 @@ vm_mmap_object(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, if (flags & MAP_SHARED) docow |= MAP_INHERIT_SHARE; if (writecounted) - docow |= MAP_VN_WRITECOUNT; + docow |= MAP_WRITECOUNT; if (flags & MAP_STACK) { if (object != NULL) return (EINVAL); diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h index 3910e3d19528..49cd99f28d4e 100644 --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -167,6 +167,7 @@ struct vm_object { struct { void *swp_tmpfs; struct pctrie swp_blks; + vm_ooffset_t writemappings; } swp; } un_pager; struct ucred *cred; diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h index 4e21de830692..6aa737890026 100644 --- a/sys/vm/vm_pager.h +++ b/sys/vm/vm_pager.h @@ -61,6 +61,7 @@ typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *); typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t, vm_pindex_t *, vm_pindex_t *); typedef void pgo_pageunswapped_t(vm_page_t); +typedef void pgo_writecount_t(vm_object_t, vm_offset_t, vm_offset_t); struct pagerops { pgo_init_t *pgo_init; /* Initialize pager. */ @@ -72,6 +73,9 @@ struct pagerops { pgo_haspage_t *pgo_haspage; /* Query page. */ pgo_populate_t *pgo_populate; /* Bulk spec pagein. */ pgo_pageunswapped_t *pgo_pageunswapped; + /* Operations for specialized writecount handling */ + pgo_writecount_t *pgo_update_writecount; + pgo_writecount_t *pgo_release_writecount; }; extern struct pagerops defaultpagerops; @@ -192,6 +196,26 @@ vm_pager_page_unswapped(vm_page_t m) (*pagertab[m->object->type]->pgo_pageunswapped)(m); } +static __inline void +vm_pager_update_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + if (pagertab[object->type]->pgo_update_writecount) + pagertab[object->type]->pgo_update_writecount(object, start, + end); +} + +static __inline void +vm_pager_release_writecount(vm_object_t object, vm_offset_t start, + vm_offset_t end) +{ + + if (pagertab[object->type]->pgo_release_writecount) + pagertab[object->type]->pgo_release_writecount(object, start, + end); +} + struct cdev_pager_ops { int (*cdev_pg_fault)(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres); diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index a12a4b80defd..cd12522fd617 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -100,6 +100,10 @@ static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *cred); static int vnode_pager_generic_getpages_done(struct buf *); static void vnode_pager_generic_getpages_done_async(struct buf *); +static void vnode_pager_update_writecount(vm_object_t, vm_offset_t, + vm_offset_t); +static void vnode_pager_release_writecount(vm_object_t, vm_offset_t, + vm_offset_t); struct pagerops vnodepagerops = { .pgo_alloc = vnode_pager_alloc, @@ -108,6 +112,8 @@ struct pagerops vnodepagerops = { .pgo_getpages_async = vnode_pager_getpages_async, .pgo_putpages = vnode_pager_putpages, .pgo_haspage = vnode_pager_haspage, + .pgo_update_writecount = vnode_pager_update_writecount, + .pgo_release_writecount = vnode_pager_release_writecount, }; static struct domainset *vnode_domainset = NULL; @@ -1500,7 +1506,7 @@ vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, off_t eof, VM_OBJECT_WUNLOCK(obj); } -void +static void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { @@ -1529,7 +1535,7 @@ vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, VM_OBJECT_WUNLOCK(object); } -void +static void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, vm_offset_t end) { diff --git a/sys/vm/vnode_pager.h b/sys/vm/vnode_pager.h index 1d6452fb6c30..821da2c98906 100644 --- a/sys/vm/vnode_pager.h +++ b/sys/vm/vnode_pager.h @@ -50,12 +50,8 @@ int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int vnode_pager_local_getpages(struct vop_getpages_args *ap); int vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap); int vnode_pager_putpages_ioflags(int pager_flags); -void vnode_pager_release_writecount(vm_object_t object, vm_offset_t start, - vm_offset_t end); void vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written, off_t eof, int lpos); -void vnode_pager_update_writecount(vm_object_t object, vm_offset_t start, - vm_offset_t end); #endif /* _KERNEL */ #endif /* _VNODE_PAGER_ */