diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index 14960c885734..1401b6254202 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -52,8 +52,10 @@ __FBSDID("$FreeBSD$"); #include #endif #include +#include #include #include +#include #include #include @@ -75,13 +77,25 @@ int vm_ndomains = 1; struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX]; int vm_phys_nsegs; -#define VM_PHYS_FICTITIOUS_NSEGS 8 -static struct vm_phys_fictitious_seg { +struct vm_phys_fictitious_seg; +static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, + struct vm_phys_fictitious_seg *); + +RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree = + RB_INITIALIZER(_vm_phys_fictitious_tree); + +struct vm_phys_fictitious_seg { + RB_ENTRY(vm_phys_fictitious_seg) node; + /* Memory region data */ vm_paddr_t start; vm_paddr_t end; vm_page_t first_page; -} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS]; -static struct mtx vm_phys_fictitious_reg_mtx; +}; + +RB_GENERATE_STATIC(fict_tree, vm_phys_fictitious_seg, node, + vm_phys_fictitious_cmp); + +static struct rwlock vm_phys_fictitious_reg_lock; MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); static struct vm_freelist @@ -113,6 +127,47 @@ static int vm_phys_paddr_to_segind(vm_paddr_t pa); static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order); +/* + * Red-black tree helpers for vm fictitious range management. + */ +static inline int +vm_phys_fictitious_in_range(struct vm_phys_fictitious_seg *p, + struct vm_phys_fictitious_seg *range) +{ + + KASSERT(range->start != 0 && range->end != 0, + ("Invalid range passed on search for vm_fictitious page")); + if (p->start >= range->end) + return (1); + if (p->start < range->start) + return (-1); + + return (0); +} + +static int +vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *p1, + struct vm_phys_fictitious_seg *p2) +{ + + /* Check if this is a search for a page */ + if (p1->end == 0) + return (vm_phys_fictitious_in_range(p1, p2)); + + KASSERT(p2->end != 0, + ("Invalid range passed as second parameter to vm fictitious comparison")); + + /* Searching to add a new range */ + if (p1->end <= p2->start) + return (-1); + if (p1->start >= p2->end) + return (1); + + panic("Trying to add overlapping vm fictitious ranges:\n" + "[%#jx:%#jx] and [%#jx:%#jx]", (uintmax_t)p1->start, + (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end); +} + static __inline int vm_rr_selectdomain(void) { @@ -353,7 +408,7 @@ vm_phys_init(void) } } } - mtx_init(&vm_phys_fictitious_reg_mtx, "vmfctr", NULL, MTX_DEF); + rw_init(&vm_phys_fictitious_reg_lock, "vmfctr"); } /* @@ -517,20 +572,22 @@ vm_phys_paddr_to_vm_page(vm_paddr_t pa) vm_page_t vm_phys_fictitious_to_vm_page(vm_paddr_t pa) { - struct vm_phys_fictitious_seg *seg; + struct vm_phys_fictitious_seg tmp, *seg; vm_page_t m; - int segind; m = NULL; - for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { - seg = &vm_phys_fictitious_segs[segind]; - if (pa >= seg->start && pa < seg->end) { - m = &seg->first_page[atop(pa - seg->start)]; - KASSERT((m->flags & PG_FICTITIOUS) != 0, - ("%p not fictitious", m)); - break; - } - } + tmp.start = pa; + tmp.end = 0; + + rw_rlock(&vm_phys_fictitious_reg_lock); + seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); + rw_runlock(&vm_phys_fictitious_reg_lock); + if (seg == NULL) + return (NULL); + + m = &seg->first_page[atop(pa - seg->start)]; + KASSERT((m->flags & PG_FICTITIOUS) != 0, ("%p not fictitious", m)); + return (m); } @@ -541,10 +598,8 @@ vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, struct vm_phys_fictitious_seg *seg; vm_page_t fp; long i, page_count; - int segind; #ifdef VM_PHYSSEG_DENSE long pi; - boolean_t malloced; #endif page_count = (end - start) / PAGE_SIZE; @@ -555,46 +610,34 @@ vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, if (atop(end) >= vm_page_array_size + first_page) return (EINVAL); fp = &vm_page_array[pi - first_page]; - malloced = FALSE; } else #endif { fp = malloc(page_count * sizeof(struct vm_page), M_FICT_PAGES, M_WAITOK | M_ZERO); -#ifdef VM_PHYSSEG_DENSE - malloced = TRUE; -#endif } for (i = 0; i < page_count; i++) { vm_page_initfake(&fp[i], start + PAGE_SIZE * i, memattr); fp[i].oflags &= ~VPO_UNMANAGED; fp[i].busy_lock = VPB_UNBUSIED; } - mtx_lock(&vm_phys_fictitious_reg_mtx); - for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { - seg = &vm_phys_fictitious_segs[segind]; - if (seg->start == 0 && seg->end == 0) { - seg->start = start; - seg->end = end; - seg->first_page = fp; - mtx_unlock(&vm_phys_fictitious_reg_mtx); - return (0); - } - } - mtx_unlock(&vm_phys_fictitious_reg_mtx); -#ifdef VM_PHYSSEG_DENSE - if (malloced) -#endif - free(fp, M_FICT_PAGES); - return (EBUSY); + + seg = malloc(sizeof(*seg), M_FICT_PAGES, M_WAITOK | M_ZERO); + seg->start = start; + seg->end = end; + seg->first_page = fp; + + rw_wlock(&vm_phys_fictitious_reg_lock); + RB_INSERT(fict_tree, &vm_phys_fictitious_tree, seg); + rw_wunlock(&vm_phys_fictitious_reg_lock); + + return (0); } void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) { - struct vm_phys_fictitious_seg *seg; - vm_page_t fp; - int segind; + struct vm_phys_fictitious_seg *seg, tmp; #ifdef VM_PHYSSEG_DENSE long pi; #endif @@ -602,24 +645,24 @@ vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) #ifdef VM_PHYSSEG_DENSE pi = atop(start); #endif + tmp.start = start; + tmp.end = 0; - mtx_lock(&vm_phys_fictitious_reg_mtx); - for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { - seg = &vm_phys_fictitious_segs[segind]; - if (seg->start == start && seg->end == end) { - seg->start = seg->end = 0; - fp = seg->first_page; - seg->first_page = NULL; - mtx_unlock(&vm_phys_fictitious_reg_mtx); -#ifdef VM_PHYSSEG_DENSE - if (pi < first_page || atop(end) >= vm_page_array_size) -#endif - free(fp, M_FICT_PAGES); - return; - } + rw_wlock(&vm_phys_fictitious_reg_lock); + seg = RB_FIND(fict_tree, &vm_phys_fictitious_tree, &tmp); + if (seg->start != start || seg->end != end) { + rw_wunlock(&vm_phys_fictitious_reg_lock); + panic( + "Unregistering not registered fictitious range [%#jx:%#jx]", + (uintmax_t)start, (uintmax_t)end); } - mtx_unlock(&vm_phys_fictitious_reg_mtx); - KASSERT(0, ("Unregistering not registered fictitious range")); + RB_REMOVE(fict_tree, &vm_phys_fictitious_tree, seg); + rw_wunlock(&vm_phys_fictitious_reg_lock); +#ifdef VM_PHYSSEG_DENSE + if (pi < first_page || atop(end) >= vm_page_array_size) +#endif + free(seg->first_page, M_FICT_PAGES); + free(seg, M_FICT_PAGES); } /*