From 8a5d94f94dc26ec417c6aa149c34cc5794ab7362 Mon Sep 17 00:00:00 2001
From: Warner Losh <imp@FreeBSD.org>
Date: Fri, 4 Aug 2017 03:40:01 +0000
Subject: [PATCH] Make nvd vs nda choice boot-time rather than build-time

Introduce hw.nvme.use_nvd tunable. This tunable allows both nvd and
nda to be installed in the kernel, while allowing only one of them to
create devices. This is an all-or-nothing setting, and you can't
change it after boot-time. However, it will allow easier A/B testing.

Differential Revision: https://reviews.freebsd.org/D11825
---
 sys/conf/files             | 6 +++---
 sys/conf/files.amd64       | 2 +-
 sys/dev/nvd/nvd.c          | 5 +++++
 sys/dev/nvme/nvme.h        | 2 ++
 sys/dev/nvme/nvme_sim.c    | 4 ++++
 sys/dev/nvme/nvme_sysctl.c | 6 ++++++
 6 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/sys/conf/files b/sys/conf/files
index b0565ee2d642..a07a59e7468c 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -86,9 +86,9 @@ cam/cam_xpt.c			optional scbus
 cam/ata/ata_all.c		optional scbus
 cam/ata/ata_xpt.c		optional scbus
 cam/ata/ata_pmp.c		optional scbus
-cam/nvme/nvme_all.c		optional scbus nvme !nvd
-cam/nvme/nvme_da.c		optional scbus nvme da !nvd
-cam/nvme/nvme_xpt.c		optional scbus nvme !nvd
+cam/nvme/nvme_all.c		optional scbus nvme
+cam/nvme/nvme_da.c		optional scbus nvme da
+cam/nvme/nvme_xpt.c		optional scbus nvme
 cam/scsi/scsi_xpt.c		optional scbus
 cam/scsi/scsi_all.c		optional scbus
 cam/scsi/scsi_cd.c		optional cd
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 72afbfa889c0..a9b926f83c97 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -341,7 +341,7 @@ dev/nvme/nvme_ctrlr_cmd.c	optional	nvme
 dev/nvme/nvme_ns.c		optional	nvme
 dev/nvme/nvme_ns_cmd.c		optional	nvme
 dev/nvme/nvme_qpair.c		optional	nvme
-dev/nvme/nvme_sim.c		optional	nvme scbus !nvd
+dev/nvme/nvme_sim.c		optional	nvme scbus
 dev/nvme/nvme_sysctl.c		optional	nvme
 dev/nvme/nvme_test.c		optional	nvme
 dev/nvme/nvme_util.c		optional	nvme
diff --git a/sys/dev/nvd/nvd.c b/sys/dev/nvd/nvd.c
index bfb3d264d1e1..cebd36d15af5 100644
--- a/sys/dev/nvd/nvd.c
+++ b/sys/dev/nvd/nvd.c
@@ -134,6 +134,8 @@ MODULE_DEPEND(nvd, nvme, 1, 1, 1);
 static int
 nvd_load()
 {
+	if (!nvme_use_nvd)
+		return 0;
 
 	TAILQ_INIT(&ctrlr_head);
 	TAILQ_INIT(&disk_head);
@@ -150,6 +152,9 @@ nvd_unload()
 	struct nvd_controller	*ctrlr;
 	struct nvd_disk		*disk;
 
+	if (!nvme_use_nvd)
+		return;
+
 	while (!TAILQ_EMPTY(&ctrlr_head)) {
 		ctrlr = TAILQ_FIRST(&ctrlr_head);
 		TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
diff --git a/sys/dev/nvme/nvme.h b/sys/dev/nvme/nvme.h
index aa640b37d4c4..1939afd74333 100644
--- a/sys/dev/nvme/nvme.h
+++ b/sys/dev/nvme/nvme.h
@@ -1003,6 +1003,8 @@ void	nvme_ns_trim_cmd(struct nvme_command *cmd, uint16_t nsid,
 	cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE;
 }
 
+extern int nvme_use_nvd;
+
 #endif /* _KERNEL */
 
 #endif /* __NVME_H__ */
diff --git a/sys/dev/nvme/nvme_sim.c b/sys/dev/nvme/nvme_sim.c
index 08ced0a24732..145b0d423f70 100644
--- a/sys/dev/nvme/nvme_sim.c
+++ b/sys/dev/nvme/nvme_sim.c
@@ -371,6 +371,8 @@ struct nvme_consumer *consumer_cookie;
 static void
 nvme_sim_init(void)
 {
+	if (nvme_use_nvd)
+		return;
 
 	consumer_cookie = nvme_register_consumer(nvme_sim_new_ns,
 	    nvme_sim_new_controller, NULL, nvme_sim_controller_fail);
@@ -382,6 +384,8 @@ SYSINIT(nvme_sim_register, SI_SUB_DRIVERS, SI_ORDER_ANY,
 static void
 nvme_sim_uninit(void)
 {
+	if (nvme_use_nvd)
+		return;
 	/* XXX Cleanup */
 
 	nvme_unregister_consumer(consumer_cookie);
diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c
index 44b0ab7dd76d..b76df116cc29 100644
--- a/sys/dev/nvme/nvme_sysctl.c
+++ b/sys/dev/nvme/nvme_sysctl.c
@@ -33,6 +33,12 @@ __FBSDID("$FreeBSD$");
 
 #include "nvme_private.h"
 
+int nvme_use_nvd = 1;
+
+SYSCTL_NODE(_hw, OID_AUTO, nvme, CTLFLAG_RD, 0, "NVMe sysctl tunables");
+SYSCTL_INT(_hw_nvme, OID_AUTO, use_nvd, CTLFLAG_RDTUN,
+    &nvme_use_nvd, 1, "1 = Create NVD devices, 0 = Create NDA devices");
+
 /*
  * CTLTYPE_S64 and sysctl_handle_64 were added in r217616.  Define these
  *  explicitly here for older kernels that don't include the r217616