diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 9c4890f2c303..db4db2ea85fd 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -3096,7 +3096,8 @@ vm_page_set_invalid(vm_page_t m, int base, int size)
 		bits = VM_PAGE_BITS_ALL;
 	else
 		bits = vm_page_bits(base, size);
-	if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
+	if (object->ref_count != 0 && m->valid == VM_PAGE_BITS_ALL &&
+	    bits != 0)
 		pmap_remove_all(m);
 	KASSERT((bits == 0 && m->valid == VM_PAGE_BITS_ALL) ||
 	    !pmap_page_is_mapped(m),