diff --git a/lib/Makefile b/lib/Makefile
index 81ace36ba990..3f30917173af 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -215,7 +215,7 @@ SUBDIR.${MK_BHYVE}+=	libvmmapi
 .if ${MACHINE_ARCH} != "powerpc"
 SUBDIR.${MK_OPENMP}+=	libomp
 .endif
-.if !defined(COMPAT_32BIT)
+.if !defined(COMPAT_32BIT) && ${MK_USB} != "no"
 SUBDIR.${MK_OPENSSH}+=	libcbor libfido2
 .endif
 SUBDIR.${MK_OPENSSL}+=	libmp