diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c
index 1b4b583b3c09..5c2cb2f25d2e 100644
--- a/sys/dev/md/md.c
+++ b/sys/dev/md/md.c
@@ -1146,8 +1146,6 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
 			VM_OBJECT_WUNLOCK(sc->object);
 		}
 		if (m != NULL) {
-			vm_page_xunbusy(m);
-
 			/*
 			 * The page may be deactivated prior to setting
 			 * PGA_REFERENCED, but in this case it will be
@@ -1157,6 +1155,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
 				vm_page_reference(m);
 			else
 				vm_page_activate(m);
+			vm_page_xunbusy(m);
 		}
 
 		/* Actions on further pages start at offset 0 */