From 9a8196ce190e6a13ae50849b3384496e7aa57b0c Mon Sep 17 00:00:00 2001 From: Nathan Whitehorn Date: Fri, 19 Jan 2018 17:46:31 +0000 Subject: [PATCH] Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be evaluated at runtime to determine if the architecture has a direct map; if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or 1, the compiler can remove the conditional logic. As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had similar things but spelled differently. 32-bit MIPS has a partial direct-map that maps poorly to this concept and is unchanged. Reviewed by: kib Suggestions from: marius, alc, kib Runtime tested on: amd64, powerpc64, powerpc, mips64 --- sys/amd64/include/vmparam.h | 1 + sys/arm/include/vmparam.h | 2 + sys/arm64/include/vmparam.h | 1 + sys/compat/linuxkpi/common/src/linux_page.c | 46 ++++++++++----------- sys/dev/efidev/efirt.c | 5 +++ sys/i386/include/vmparam.h | 2 + sys/kern/subr_sfbuf.c | 16 ++----- sys/mips/include/vmparam.h | 6 +++ sys/powerpc/include/vmparam.h | 32 +++++++++----- sys/riscv/include/vmparam.h | 1 + sys/sparc64/include/vmparam.h | 8 ++-- sys/sys/sf_buf.h | 16 +++---- sys/vm/vm_page.c | 2 +- 13 files changed, 78 insertions(+), 60 deletions(-) diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h index 6d63201c6581..bb8c3221eb6f 100644 --- a/sys/amd64/include/vmparam.h +++ b/sys/amd64/include/vmparam.h @@ -190,6 +190,7 @@ * because the result is not actually accessed until later, but the early * vt fb startup needs to be reworked. */ +#define PMAP_HAS_DMAP 1 #define PHYS_TO_DMAP(x) ({ \ KASSERT(dmaplimit == 0 || (x) < dmaplimit, \ ("physical address %#jx not covered by the DMAP", \ diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h index ba43e9e91e6e..c57168fb748d 100644 --- a/sys/arm/include/vmparam.h +++ b/sys/arm/include/vmparam.h @@ -187,6 +187,8 @@ extern vm_offset_t vm_max_kernel_address; #define SFBUF #define SFBUF_MAP +#define PMAP_HAS_DMAP 0 + #define DEVMAP_MAX_VADDR ARM_VECTORS_HIGH #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h index 0df07bd314b5..7f0e531e25a6 100644 --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -176,6 +176,7 @@ #define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \ (va) < (dmap_max_addr)) +#define PMAP_HAS_DMAP 1 #define PHYS_TO_DMAP(pa) \ ({ \ KASSERT(PHYS_IN_DMAP(pa), \ diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c index b7d79da46ce6..798e496ce47b 100644 --- a/sys/compat/linuxkpi/common/src/linux_page.c +++ b/sys/compat/linuxkpi/common/src/linux_page.c @@ -63,19 +63,14 @@ __FBSDID("$FreeBSD$"); #include #include -#if defined(__amd64__) || defined(__aarch64__) || defined(__riscv) -#define LINUXKPI_HAVE_DMAP -#else -#undef LINUXKPI_HAVE_DMAP -#endif - void * linux_page_address(struct page *page) { if (page->object != kmem_object && page->object != kernel_object) { -#ifdef LINUXKPI_HAVE_DMAP - return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))); +#ifdef PHYS_TO_DMAP + return (PMAP_HAS_DMAP ? + ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))) : NULL); #else return (NULL); #endif @@ -87,7 +82,8 @@ linux_page_address(struct page *page) vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order) { -#ifdef LINUXKPI_HAVE_DMAP +#ifdef PHYS_TO_DMAP + KASSERT(PMAP_HAS_DMAP, ("Direct map unavailable")); unsigned long npages = 1UL << order; int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL); @@ -145,23 +141,27 @@ linux_alloc_pages(gfp_t flags, unsigned int order) void linux_free_pages(vm_page_t page, unsigned int order) { -#ifdef LINUXKPI_HAVE_DMAP - unsigned long npages = 1UL << order; - unsigned long x; +#ifdef PHYS_TO_DMAP + if (PMAP_HAS_DMAP) { + unsigned long npages = 1UL << order; + unsigned long x; - for (x = 0; x != npages; x++) { - vm_page_t pgo = page + x; + for (x = 0; x != npages; x++) { + vm_page_t pgo = page + x; - vm_page_lock(pgo); - vm_page_free(pgo); - vm_page_unlock(pgo); + vm_page_lock(pgo); + vm_page_free(pgo); + vm_page_unlock(pgo); + } + } else { +#endif + vm_offset_t vaddr; + + vaddr = (vm_offset_t)page_address(page); + + linux_free_kmem(vaddr, order); +#ifdef PHYS_TO_DMAP } -#else - vm_offset_t vaddr; - - vaddr = (vm_offset_t)page_address(page); - - linux_free_kmem(vaddr, order); #endif } diff --git a/sys/dev/efidev/efirt.c b/sys/dev/efidev/efirt.c index 9f54494d68fa..381da270dc4f 100644 --- a/sys/dev/efidev/efirt.c +++ b/sys/dev/efidev/efirt.c @@ -114,6 +114,11 @@ efi_init(void) printf("EFI systbl not available\n"); return (0); } + if (!PMAP_HAS_DMAP) { + if (bootverbose) + printf("EFI systbl requires direct map\n"); + return (0); + } efi_systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys); if (efi_systbl->st_hdr.th_sig != EFI_SYSTBL_SIG) { efi_systbl = NULL; diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h index d3ec935efb3a..57cfd2edfd7d 100644 --- a/sys/i386/include/vmparam.h +++ b/sys/i386/include/vmparam.h @@ -202,4 +202,6 @@ #define SFBUF_CPUSET #define SFBUF_PROCESS_PAGE +#define PMAP_HAS_DMAP 0 + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/kern/subr_sfbuf.c b/sys/kern/subr_sfbuf.c index fa1aa4224c1e..2553de2e0681 100644 --- a/sys/kern/subr_sfbuf.c +++ b/sys/kern/subr_sfbuf.c @@ -88,10 +88,8 @@ sf_buf_init(void *arg) vm_offset_t sf_base; int i; -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) + if (PMAP_HAS_DMAP) return; -#endif nsfbufs = NSFBUFS; TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs); @@ -119,10 +117,8 @@ sf_buf_alloc(struct vm_page *m, int flags) struct sf_buf *sf; int error; -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) + if (PMAP_HAS_DMAP) return ((struct sf_buf *)m); -#endif KASSERT(curthread->td_pinned > 0 || (flags & SFB_CPUPRIVATE) == 0, ("sf_buf_alloc(SFB_CPUPRIVATE): curthread not pinned")); @@ -181,10 +177,8 @@ void sf_buf_free(struct sf_buf *sf) { -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) + if (PMAP_HAS_DMAP) return; -#endif mtx_lock(&sf_buf_lock); sf->ref_count--; @@ -205,10 +199,8 @@ void sf_buf_ref(struct sf_buf *sf) { -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) + if (PMAP_HAS_DMAP) return; -#endif mtx_lock(&sf_buf_lock); KASSERT(sf->ref_count > 0, ("%s: sf %p not allocated", __func__, sf)); diff --git a/sys/mips/include/vmparam.h b/sys/mips/include/vmparam.h index 7fe6f4fa5d7e..305224dc7b87 100644 --- a/sys/mips/include/vmparam.h +++ b/sys/mips/include/vmparam.h @@ -189,6 +189,12 @@ #ifndef __mips_n64 #define SFBUF #define SFBUF_MAP +#define PMAP_HAS_DMAP 0 +#else +#define PMAP_HAS_DMAP 1 #endif +#define PHYS_TO_DMAP(x) MIPS_PHYS_TO_DIRECT(x) +#define DMAP_TO_PHYS(x) MIPS_DIRECT_TO_PHYS(x) + #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/powerpc/include/vmparam.h b/sys/powerpc/include/vmparam.h index 08415c9960df..e079024e0196 100644 --- a/sys/powerpc/include/vmparam.h +++ b/sys/powerpc/include/vmparam.h @@ -37,6 +37,10 @@ #ifndef _MACHINE_VMPARAM_H_ #define _MACHINE_VMPARAM_H_ +#ifndef LOCORE +#include +#endif + #define USRSTACK SHAREDPAGE #ifndef MAXTSIZ @@ -236,17 +240,25 @@ struct pmap_physseg { */ #define SFBUF #define SFBUF_NOMD -#define SFBUF_OPTIONAL_DIRECT_MAP hw_direct_map -#define SFBUF_PHYS_DMAP(x) (x) /* - * We (usually) have a direct map of all physical memory. All - * uses of this macro must be gated by a check on hw_direct_map! - * The location of the direct map may not be 1:1 in future, so use - * of the macro is recommended; it may also grow an assert that hw_direct_map - * is set. + * We (usually) have a direct map of all physical memory, so provide + * a macro to use to get the kernel VA address for a given PA. Returns + * 0 if the direct map is unavailable. The location of the direct map + * may not be 1:1 in future, so use of the macro is recommended. */ -#define PHYS_TO_DMAP(x) x -#define DMAP_TO_PHYS(x) x - +#ifdef __powerpc64__ +#define DMAP_BASE_ADDRESS 0x0000000000000000UL +#else +#define DMAP_BASE_ADDRESS 0x00000000UL +#endif + +#define PMAP_HAS_DMAP (hw_direct_map) +#define PHYS_TO_DMAP(x) ({ \ + KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \ + (x) | DMAP_BASE_ADDRESS; }) +#define DMAP_TO_PHYS(x) ({ \ + KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \ + (x) &~ DMAP_BASE_ADDRESS; }) + #endif /* _MACHINE_VMPARAM_H_ */ diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h index faa3c1efa519..bf0da4daa411 100644 --- a/sys/riscv/include/vmparam.h +++ b/sys/riscv/include/vmparam.h @@ -170,6 +170,7 @@ #define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \ (va) < (dmap_max_addr)) +#define PMAP_HAS_DMAP 1 #define PHYS_TO_DMAP(pa) \ ({ \ KASSERT(PHYS_IN_DMAP(pa), \ diff --git a/sys/sparc64/include/vmparam.h b/sys/sparc64/include/vmparam.h index 68dd0861dd56..ef04b702f882 100644 --- a/sys/sparc64/include/vmparam.h +++ b/sys/sparc64/include/vmparam.h @@ -240,10 +240,12 @@ extern vm_offset_t vm_max_kernel_address; */ #define ZERO_REGION_SIZE PAGE_SIZE +#include + #define SFBUF #define SFBUF_MAP -#define SFBUF_OPTIONAL_DIRECT_MAP dcache_color_ignore -#include -#define SFBUF_PHYS_DMAP(x) TLB_PHYS_TO_DIRECT(x) + +#define PMAP_HAS_DMAP dcache_color_ignore +#define PHYS_TO_DMAP(x) (TLB_PHYS_TO_DIRECT(x)) #endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/sys/sf_buf.h b/sys/sys/sf_buf.h index 08f1d9d7af0b..2a2505c78bf0 100644 --- a/sys/sys/sf_buf.h +++ b/sys/sys/sf_buf.h @@ -77,9 +77,6 @@ struct sfstat { /* sendfile statistics */ * that do no invalidate cache on the rest of CPUs. * SFBUF_NOMD This machine doesn't have machine/sf_buf.h * - * SFBUF_OPTIONAL_DIRECT_MAP Value of this define is used as boolean - * variable that tells whether machine is - * capable of direct map or not at runtime. * SFBUF_MAP This machine provides its own sf_buf_map() and * sf_buf_unmap(). * SFBUF_PROCESS_PAGE This machine provides sf_buf_process_page() @@ -109,9 +106,6 @@ struct sf_buf; #ifndef SFBUF_NOMD #include #endif -#ifdef SFBUF_OPTIONAL_DIRECT_MAP -#include -#endif #ifdef SFBUF struct sf_buf *sf_buf_alloc(struct vm_page *, int); @@ -121,9 +115,9 @@ void sf_buf_ref(struct sf_buf *); static inline vm_offset_t sf_buf_kva(struct sf_buf *sf) { -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) - return (SFBUF_PHYS_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf))); +#ifdef PMAP_HAS_DMAP + if (PMAP_HAS_DMAP) + return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf))); #endif return (sf->kva); @@ -132,8 +126,8 @@ sf_buf_kva(struct sf_buf *sf) static inline vm_page_t sf_buf_page(struct sf_buf *sf) { -#ifdef SFBUF_OPTIONAL_DIRECT_MAP - if (SFBUF_OPTIONAL_DIRECT_MAP) +#ifdef PMAP_HAS_DMAP + if (PMAP_HAS_DMAP) return ((vm_page_t)sf); #endif diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 6635d17a1ef9..37007af6a684 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -2937,7 +2937,7 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked) { #if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP) - if ((m->flags & PG_ZERO) != 0) { + if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) { uint64_t *p; int i; p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));