diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 004b4a7656af..dcd6e5dc8021 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -215,6 +215,8 @@ struct msgbuf *msgbufp = NULL;
 
 static struct rwlock_padalign pvh_global_lock;
 
+vm_paddr_t dmap_phys_base;	/* The start of the dmap region */
+
 /*
  * Data for the pv entry allocation mechanism
  */
@@ -446,18 +448,19 @@ pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va)
 }
 
 static void
-pmap_bootstrap_dmap(vm_offset_t l1pt)
+pmap_bootstrap_dmap(vm_offset_t l1pt, vm_paddr_t kernstart)
 {
 	vm_offset_t va;
 	vm_paddr_t pa;
 	pd_entry_t *l1;
 	u_int l1_slot;
 
+	pa = dmap_phys_base = kernstart & ~L1_OFFSET;
 	va = DMAP_MIN_ADDRESS;
 	l1 = (pd_entry_t *)l1pt;
 	l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS);
 
-	for (pa = 0; va < DMAP_MAX_ADDRESS;
+	for (; va < DMAP_MAX_ADDRESS;
 	    pa += L1_SIZE, va += L1_SIZE, l1_slot++) {
 		KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index"));
 
@@ -548,7 +551,8 @@ pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen)
 	pt_entry_t *l2;
 	vm_offset_t va, freemempos;
 	vm_offset_t dpcpu, msgbufpv;
-	vm_paddr_t pa;
+	vm_paddr_t pa, min_pa;
+	int i;
 
 	kern_delta = KERNBASE - kernstart;
 	physmem = 0;
@@ -566,8 +570,23 @@ pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen)
 	 */
 	rw_init(&pvh_global_lock, "pmap pv global");
 
+	/* Assume the address we were loaded to is a valid physical address */
+	min_pa = KERNBASE - kern_delta;
+
+	/*
+	 * Find the minimum physical address. physmap is sorted,
+	 * but may contain empty ranges.
+	 */
+	for (i = 0; i < (physmap_idx * 2); i += 2) {
+		if (physmap[i] == physmap[i + 1])
+			continue;
+		if (physmap[i] <= min_pa)
+			min_pa = physmap[i];
+		break;
+	}
+
 	/* Create a direct map region early so we can use it for pa -> va */
-	pmap_bootstrap_dmap(l1pt);
+	pmap_bootstrap_dmap(l1pt, min_pa);
 
 	va = KERNBASE;
 	pa = KERNBASE - kern_delta;
diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h
index 3c14af8d101a..e3c1d74bb438 100644
--- a/sys/arm64/include/vmparam.h
+++ b/sys/arm64/include/vmparam.h
@@ -160,11 +160,13 @@
 #define	DMAP_MIN_ADDRESS	(0xffffffc000000000UL)
 #define	DMAP_MAX_ADDRESS	(0xffffffdfffffffffUL)
 
-#define	DMAP_MIN_PHYSADDR	(0x0000000000000000UL)
-#define	DMAP_MAX_PHYSADDR	(DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS)
+extern vm_paddr_t dmap_phys_base;
+#define	DMAP_MIN_PHYSADDR	(dmap_phys_base)
+#define	DMAP_MAX_PHYSADDR	(dmap_phys_base + (DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS))
 
 /* True if pa is in the dmap range */
-#define	PHYS_IN_DMAP(pa)	((pa) <= DMAP_MAX_PHYSADDR)
+#define	PHYS_IN_DMAP(pa)	((pa) >= DMAP_MIN_PHYSADDR && \
+    (pa) <= DMAP_MAX_PHYSADDR)
 /* True if va is in the dmap range */
 #define	VIRT_IN_DMAP(va)	((va) >= DMAP_MIN_ADDRESS && \
     (va) <= DMAP_MAX_ADDRESS)
@@ -174,7 +176,7 @@
 	KASSERT(PHYS_IN_DMAP(pa),					\
 	    ("%s: PA out of range, PA: 0x%lx", __func__,		\
 	    (vm_paddr_t)(pa)));						\
-	(pa) | DMAP_MIN_ADDRESS;					\
+	((pa) - dmap_phys_base) | DMAP_MIN_ADDRESS;			\
 })
 
 #define	DMAP_TO_PHYS(va)						\
@@ -182,7 +184,7 @@
 	KASSERT(VIRT_IN_DMAP(va),					\
 	    ("%s: VA out of range, VA: 0x%lx", __func__,		\
 	    (vm_offset_t)(va)));					\
-	(va) & ~DMAP_MIN_ADDRESS;					\
+	((va) & ~DMAP_MIN_ADDRESS) + dmap_phys_base;			\
 })
 
 #define	VM_MIN_USER_ADDRESS	(0x0000000000000000UL)