Remove SFBUF_OPTIONAL_DIRECT_MAP and such hacks, replacing them across the

kernel by PHYS_TO_DMAP() as previously present on amd64, arm64, riscv, and
powerpc64. This introduces a new MI macro (PMAP_HAS_DMAP) that can be
evaluated at runtime to determine if the architecture has a direct map;
if it does not (or does) unconditionally and PMAP_HAS_DMAP is either 0 or
1, the compiler can remove the conditional logic.

As part of this, implement PHYS_TO_DMAP() on sparc64 and mips64, which had
similar things but spelled differently. 32-bit MIPS has a partial direct-map
that maps poorly to this concept and is unchanged.

Reviewed by:		kib
Suggestions from:	marius, alc, kib
Runtime tested on:	amd64, powerpc64, powerpc, mips64

sys/amd64/include/vmparam.h

@@ -190,6 +190,7 @@
  * because the result is not actually accessed until later, but the early
  * vt fb startup needs to be reworked.
  */
+#define	PMAP_HAS_DMAP	1
 #define	PHYS_TO_DMAP(x)	({						\
 	KASSERT(dmaplimit == 0 || (x) < dmaplimit,			\
 	    ("physical address %#jx not covered by the DMAP",		\

sys/arm/include/vmparam.h

@@ -187,6 +187,8 @@ extern vm_offset_t vm_max_kernel_address;
 #define	SFBUF
 #define	SFBUF_MAP
 
+#define	PMAP_HAS_DMAP	0
+
 #define	DEVMAP_MAX_VADDR	ARM_VECTORS_HIGH
 
 #endif	/* _MACHINE_VMPARAM_H_ */

sys/arm64/include/vmparam.h

@@ -176,6 +176,7 @@
 #define	VIRT_IN_DMAP(va)	((va) >= DMAP_MIN_ADDRESS && \
     (va) < (dmap_max_addr))
 
+#define	PMAP_HAS_DMAP	1
 #define	PHYS_TO_DMAP(pa)						\
 ({									\
 	KASSERT(PHYS_IN_DMAP(pa),					\

sys/compat/linuxkpi/common/src/linux_page.c

@@ -63,19 +63,14 @@ __FBSDID("$FreeBSD$");
 #include <linux/preempt.h>
 #include <linux/fs.h>
 
-#if defined(__amd64__) || defined(__aarch64__) || defined(__riscv)
-#define	LINUXKPI_HAVE_DMAP
-#else
-#undef	LINUXKPI_HAVE_DMAP
-#endif
-
 void *
 linux_page_address(struct page *page)
 {
 
 	if (page->object != kmem_object && page->object != kernel_object) {
-#ifdef LINUXKPI_HAVE_DMAP
+#ifdef PHYS_TO_DMAP
-		return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page)));
+		return (PMAP_HAS_DMAP ?
+		    ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))) : NULL);
 #else
 		return (NULL);
 #endif
@@ -87,7 +82,8 @@ linux_page_address(struct page *page)
 vm_page_t
 linux_alloc_pages(gfp_t flags, unsigned int order)
 {
-#ifdef LINUXKPI_HAVE_DMAP
+#ifdef PHYS_TO_DMAP
+	KASSERT(PMAP_HAS_DMAP, ("Direct map unavailable"));
 	unsigned long npages = 1UL << order;
 	int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ |
 	    VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL);
@@ -145,23 +141,27 @@ linux_alloc_pages(gfp_t flags, unsigned int order)
 void
 linux_free_pages(vm_page_t page, unsigned int order)
 {
-#ifdef LINUXKPI_HAVE_DMAP
+#ifdef PHYS_TO_DMAP
-	unsigned long npages = 1UL << order;
+	if (PMAP_HAS_DMAP) {
-	unsigned long x;
+		unsigned long npages = 1UL << order;
+		unsigned long x;
 
-	for (x = 0; x != npages; x++) {
+		for (x = 0; x != npages; x++) {
-		vm_page_t pgo = page + x;
+			vm_page_t pgo = page + x;
 
-		vm_page_lock(pgo);
+			vm_page_lock(pgo);
-		vm_page_free(pgo);
+			vm_page_free(pgo);
-		vm_page_unlock(pgo);
+			vm_page_unlock(pgo);
+		}
+	} else {
+#endif
+		vm_offset_t vaddr;
+
+		vaddr = (vm_offset_t)page_address(page);
+
+		linux_free_kmem(vaddr, order);
+#ifdef PHYS_TO_DMAP
 	}
-#else
-	vm_offset_t vaddr;
-
-	vaddr = (vm_offset_t)page_address(page);
-
-	linux_free_kmem(vaddr, order);
 #endif
 }
 

sys/dev/efidev/efirt.c

@@ -114,6 +114,11 @@ efi_init(void)
 			printf("EFI systbl not available\n");
 		return (0);
 	}
+	if (!PMAP_HAS_DMAP) {
+		if (bootverbose)
+			printf("EFI systbl requires direct map\n");
+		return (0);
+	}
 	efi_systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys);
 	if (efi_systbl->st_hdr.th_sig != EFI_SYSTBL_SIG) {
 		efi_systbl = NULL;

sys/i386/include/vmparam.h

@@ -202,4 +202,6 @@
 #define	SFBUF_CPUSET
 #define	SFBUF_PROCESS_PAGE
 
+#define	PMAP_HAS_DMAP	0
+
 #endif /* _MACHINE_VMPARAM_H_ */

sys/kern/subr_sfbuf.c

@@ -88,10 +88,8 @@ sf_buf_init(void *arg)
 	vm_offset_t sf_base;
 	int i;
 
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+	if (PMAP_HAS_DMAP)
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
 		return;
-#endif
 
 	nsfbufs = NSFBUFS;
 	TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
@@ -119,10 +117,8 @@ sf_buf_alloc(struct vm_page *m, int flags)
 	struct sf_buf *sf;
 	int error;
 
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+	if (PMAP_HAS_DMAP)
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
 		return ((struct sf_buf *)m);
-#endif
 
 	KASSERT(curthread->td_pinned > 0 || (flags & SFB_CPUPRIVATE) == 0,
 	    ("sf_buf_alloc(SFB_CPUPRIVATE): curthread not pinned"));
@@ -181,10 +177,8 @@ void
 sf_buf_free(struct sf_buf *sf)
 {
 
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+	if (PMAP_HAS_DMAP)
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
 		return;
-#endif
 
 	mtx_lock(&sf_buf_lock);
 	sf->ref_count--;
@@ -205,10 +199,8 @@ void
 sf_buf_ref(struct sf_buf *sf)
 {
 
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+	if (PMAP_HAS_DMAP)
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
 		return;
-#endif
 
 	mtx_lock(&sf_buf_lock);
 	KASSERT(sf->ref_count > 0, ("%s: sf %p not allocated", __func__, sf));

sys/mips/include/vmparam.h

@@ -189,6 +189,12 @@
 #ifndef __mips_n64
 #define	SFBUF
 #define	SFBUF_MAP
+#define	PMAP_HAS_DMAP	0
+#else
+#define	PMAP_HAS_DMAP	1
 #endif
 
+#define	PHYS_TO_DMAP(x)	MIPS_PHYS_TO_DIRECT(x)
+#define	DMAP_TO_PHYS(x)	MIPS_DIRECT_TO_PHYS(x)
+
 #endif /* !_MACHINE_VMPARAM_H_ */

sys/powerpc/include/vmparam.h

@@ -37,6 +37,10 @@
 #ifndef _MACHINE_VMPARAM_H_
 #define	_MACHINE_VMPARAM_H_
 
+#ifndef LOCORE
+#include <machine/md_var.h>
+#endif
+
 #define	USRSTACK	SHAREDPAGE
 
 #ifndef	MAXTSIZ
@@ -236,17 +240,25 @@ struct pmap_physseg {
  */
 #define	SFBUF
 #define	SFBUF_NOMD
-#define	SFBUF_OPTIONAL_DIRECT_MAP	hw_direct_map
-#define	SFBUF_PHYS_DMAP(x)		(x)
 
 /*
- * We (usually) have a direct map of all physical memory. All
+ * We (usually) have a direct map of all physical memory, so provide
- * uses of this macro must be gated by a check on hw_direct_map!
+ * a macro to use to get the kernel VA address for a given PA. Returns
- * The location of the direct map may not be 1:1 in future, so use
+ * 0 if the direct map is unavailable. The location of the direct map
- * of the macro is recommended; it may also grow an assert that hw_direct_map
+ * may not be 1:1 in future, so use of the macro is recommended.
- * is set.
  */
-#define PHYS_TO_DMAP(x) x
+#ifdef __powerpc64__
-#define DMAP_TO_PHYS(x) x
+#define	DMAP_BASE_ADDRESS	0x0000000000000000UL
- 
+#else
+#define	DMAP_BASE_ADDRESS	0x00000000UL
+#endif
+
+#define	PMAP_HAS_DMAP	(hw_direct_map)
+#define PHYS_TO_DMAP(x) ({						\
+	KASSERT(hw_direct_map, ("Direct map not provided by PMAP"));	\
+	(x) | DMAP_BASE_ADDRESS; })
+#define DMAP_TO_PHYS(x) ({						\
+	KASSERT(hw_direct_map, ("Direct map not provided by PMAP"));	\
+	(x) &~ DMAP_BASE_ADDRESS; })
+
 #endif /* _MACHINE_VMPARAM_H_ */

sys/riscv/include/vmparam.h

@@ -170,6 +170,7 @@
 #define	VIRT_IN_DMAP(va)	((va) >= DMAP_MIN_ADDRESS && \
     (va) < (dmap_max_addr))
 
+#define	PMAP_HAS_DMAP	1
 #define	PHYS_TO_DMAP(pa)						\
 ({									\
 	KASSERT(PHYS_IN_DMAP(pa),					\

sys/sparc64/include/vmparam.h

@@ -240,10 +240,12 @@ extern vm_offset_t vm_max_kernel_address;
  */
 #define	ZERO_REGION_SIZE	PAGE_SIZE
 
+#include <machine/tlb.h>
+
 #define	SFBUF
 #define	SFBUF_MAP
-#define	SFBUF_OPTIONAL_DIRECT_MAP	dcache_color_ignore
+
-#include <machine/tlb.h>
+#define	PMAP_HAS_DMAP	dcache_color_ignore
-#define	SFBUF_PHYS_DMAP(x)		TLB_PHYS_TO_DIRECT(x)
+#define	PHYS_TO_DMAP(x)	(TLB_PHYS_TO_DIRECT(x))
 
 #endif /* !_MACHINE_VMPARAM_H_ */

sys/sys/sf_buf.h

@@ -77,9 +77,6 @@ struct sfstat {				/* sendfile statistics */
  *			that do no invalidate cache on the rest of CPUs.
  * SFBUF_NOMD		This machine doesn't have machine/sf_buf.h
  *
- * SFBUF_OPTIONAL_DIRECT_MAP	Value of this define is used as boolean
- *				variable that tells whether machine is
- *				capable of direct map or not at runtime.
  * SFBUF_MAP		This machine provides its own sf_buf_map() and
  *			sf_buf_unmap().
  * SFBUF_PROCESS_PAGE	This machine provides sf_buf_process_page()
@@ -109,9 +106,6 @@ struct sf_buf;
 #ifndef SFBUF_NOMD
 #include <machine/sf_buf.h>
 #endif
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
-#include <machine/md_var.h>
-#endif
 
 #ifdef SFBUF
 struct sf_buf *sf_buf_alloc(struct vm_page *, int);
@@ -121,9 +115,9 @@ void sf_buf_ref(struct sf_buf *);
 static inline vm_offset_t
 sf_buf_kva(struct sf_buf *sf)
 {
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+#ifdef PMAP_HAS_DMAP
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
+	if (PMAP_HAS_DMAP)
-		return (SFBUF_PHYS_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf)));
+		return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf)));
 #endif
 
         return (sf->kva);
@@ -132,8 +126,8 @@ sf_buf_kva(struct sf_buf *sf)
 static inline vm_page_t
 sf_buf_page(struct sf_buf *sf)
 {
-#ifdef SFBUF_OPTIONAL_DIRECT_MAP
+#ifdef PMAP_HAS_DMAP
-	if (SFBUF_OPTIONAL_DIRECT_MAP)
+	if (PMAP_HAS_DMAP)
 		return ((vm_page_t)sf);
 #endif
 

sys/vm/vm_page.c

@@ -2937,7 +2937,7 @@ vm_page_free_prep(vm_page_t m, bool pagequeue_locked)
 {
 
 #if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
-	if ((m->flags & PG_ZERO) != 0) {
+	if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
 		uint64_t *p;
 		int i;
 		p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));