diff --git a/sys/dev/usb/controller/ehci_imx.c b/sys/dev/usb/controller/ehci_imx.c
index c116cb9b31ee..20d040845243 100644
--- a/sys/dev/usb/controller/ehci_imx.c
+++ b/sys/dev/usb/controller/ehci_imx.c
@@ -268,6 +268,7 @@ struct imx_ehci_softc {
 	device_t	dev;
 	struct resource	*ehci_mem_res;	/* EHCI core regs. */
 	struct resource	*ehci_irq_res;	/* EHCI core IRQ. */ 
+	bool		usb_mem_allocated;
 };
 
 static struct ofw_compat_data compat_data[] = {
@@ -331,7 +332,8 @@ imx_ehci_detach(device_t dev)
 		bus_release_resource(dev, SYS_RES_MEMORY, 0,
 		    sc->ehci_mem_res);
 
-	usb_bus_mem_free_all(&esc->sc_bus, &ehci_iterate_hw_softc);
+	if (sc->usb_mem_allocated)
+		usb_bus_mem_free_all(&esc->sc_bus, &ehci_iterate_hw_softc);
 
 	/* During module unload there are lots of children leftover */
 	device_delete_children(dev);
@@ -413,6 +415,7 @@ imx_ehci_attach(device_t dev)
 		err = ENOMEM;
 		goto out;
 	}
+	sc->usb_mem_allocated = true;
 
 	/*
 	 * Set handle to USB related registers subregion used by