Dynamically load .so modules to expand functionality

o Dynamically load all the .so files found in /libexec/nvmecontrol and
  /usr/local/libexec/nvmecontrol.
o Link nvmecontrol -rdynamic so that its symbols are visible to the
  libraries we load.
o Create concatinated linker sets that we dynamically expand.
o Add the linked-in top and logpage linker sets to the mirrors for them
  and add those sets to the mirrors when we load a new .so.
o Add some macros to help hide the names of the linker sets.
o Update the man page.

Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D18455

fold

etc/mtree/BSD.root.dist

@@ -88,6 +88,8 @@
         ..
         geom
         ..
+        nvmecontrol
+        ..
     ..
     libexec
         resolvconf

sbin/nvmecontrol/Makefile

@@ -6,6 +6,7 @@ SRCS=	nvmecontrol.c devlist.c firmware.c format.c identify.c identify_ext.c logp
 	perftest.c reset.c ns.c nvme_util.c power.c nc_util.c
 SRCS+=	wdc.c intel.c
 MAN=	nvmecontrol.8
+LDFLAGS+= -rdynamic
 
 .PATH:	${SRCTOP}/sys/dev/nvme
 

sbin/nvmecontrol/logpage.c

@@ -48,13 +48,13 @@ __FBSDID("$FreeBSD$");
 
 #include "nvmecontrol.h"
 
-SET_DECLARE(logpage, struct logpage_function);
-
 #define LOGPAGE_USAGE							       \
 	"logpage <-p page_id> [-b] [-v vendor] [-x] <controller id|namespace id>\n"  \
 
 #define MAX_FW_SLOTS	(7)
 
+SET_CONCAT_DEF(logpage, struct logpage_function);
+
 const char *
 kv_lookup(const struct kv_name *kv, size_t kv_count, uint32_t key)
 {
@@ -332,7 +332,7 @@ logpage_help(void)
 	fprintf(stderr, "\n");
 	fprintf(stderr, "%-8s %-10s %s\n", "Page", "Vendor","Page Name");
 	fprintf(stderr, "-------- ---------- ----------\n");
-	for (f = SET_BEGIN(logpage); f < SET_LIMIT(logpage); f++) {
+	for (f = logpage_begin(); f < logpage_limit(); f++) {
 		v = (*f)->vendor == NULL ? "-" : (*f)->vendor;
 		fprintf(stderr, "0x%02x     %-10s %s\n", (*f)->log_page, v, (*f)->name);
 	}
@@ -438,7 +438,7 @@ logpage(struct nvme_function *nf, int argc, char *argv[])
 		 * the page is vendor specific, don't match the print function
 		 * unless the vendors match.
 		 */
-		for (f = SET_BEGIN(logpage); f < SET_LIMIT(logpage); f++) {
+		for (f = logpage_begin(); f < logpage_limit(); f++) {
 			if ((*f)->vendor != NULL && vendor != NULL &&
 			    strcmp((*f)->vendor, vendor) != 0)
 				continue;

sbin/nvmecontrol/ns.c

@@ -41,7 +41,7 @@ __FBSDID("$FreeBSD$");
 
 #include "nvmecontrol.h"
 
-SET_DECLARE(ns, struct nvme_function);
+NVME_CMD_DECLARE(ns, struct nvme_function);
 
 #define NS_USAGE							\
 	"ns (create|delete|attach|detach)\n"

sbin/nvmecontrol/nvmecontrol.8

@@ -34,7 +34,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 12, 2018
+.Dd December 7, 2018
 .Dt NVMECONTROL 8
 .Os
 .Sh NAME
@@ -230,6 +230,19 @@ Set the current power mode.
 .Dl nvmecontrol power nvme0
 .Pp
 Get the current power mode.
+.Sh DYNAMIC LOADING
+The directories
+.Pa /libexec/nvmecontrol
+and
+.Pa /usr/local/libexec/nvmecontrol
+are scanned for any .so files.
+These files are loaded.
+The members of the
+.Va top
+linker set are added to the top-level commands.
+The members of the
+.Va logpage
+linker set are added to the logpage parsers.
 .Sh HISTORY
 The
 .Nm

sbin/nvmecontrol/nvmecontrol.c

@@ -34,6 +34,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 
 #include <ctype.h>
+#include <dlfcn.h>
+#include <dirent.h>
 #include <err.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -47,7 +49,7 @@ __FBSDID("$FreeBSD$");
 
 #include "nvmecontrol.h"
 
-SET_DECLARE(top, struct nvme_function);
+SET_CONCAT_DEF(top, struct nvme_function);
 
 static void
 print_usage(const struct nvme_function *f)
@@ -116,6 +118,32 @@ dispatch_set(int argc, char *argv[], struct nvme_function **tbl,
 	gen_usage_set(tbl, tbl_limit);
 }
 
+void
+set_concat_add(struct set_concat *m, void *b, void *e)
+{
+	void **bp, **ep;
+	int add_n, cur_n;
+
+	if (b == NULL)
+		return;
+	/*
+	 * Args are really pointers to arrays of pointers, but C's
+	 * casting rules kinda suck since you can't directly cast
+	 * struct foo ** to a void **.
+	 */
+	bp = (void **)b;
+	ep = (void **)e;
+	add_n = ep - bp;
+	cur_n = 0;
+	if (m->begin != NULL)
+		cur_n = m->limit - m->begin;
+	m->begin = reallocarray(m->begin, cur_n + add_n, sizeof(void *));
+	if (m->begin == NULL)
+		err(1, "expanding concat set");
+	memcpy(m->begin + cur_n, bp, add_n * sizeof(void *));
+	m->limit = m->begin + cur_n + add_n;
+}
+
 static void
 print_bytes(void *data, uint32_t length)
 {
@@ -260,14 +288,64 @@ parse_ns_str(const char *ns_str, char *ctrlr_str, uint32_t *nsid)
 	snprintf(ctrlr_str, nsloc - ns_str + 1, "%s", ns_str);
 }
 
+/*
+ * Loads all the .so's from the specified directory.
+ */
+static void
+load_dir(const char *dir)
+{
+	DIR *d;
+	struct dirent *dent;
+	char *path = NULL;
+	void *h;
+
+	d = opendir(dir);
+	if (d == NULL)
+		return;
+	for (dent = readdir(d); dent != NULL; dent = readdir(d)) {
+		if (strcmp(".so", dent->d_name + dent->d_namlen - 3) != 0)
+			continue;
+		asprintf(&path, "%s/%s", dir, dent->d_name);
+		if (path == NULL)
+			err(1, "Can't malloc for path, giving up.");
+		if ((h = dlopen(path, RTLD_NOW | RTLD_GLOBAL)) == NULL)
+			warnx("Can't load %s: %s", path, dlerror());
+		else {
+			/*
+			 * Add in the top (for cli commands) and logpage (for
+			 * logpage parsing) linker sets. We have to do this by
+			 * hand because linker sets aren't automatically merged.
+			 */
+			void *begin, *limit;
+			begin = dlsym(h, "__start_set_top");
+			limit = dlsym(h, "__stop_set_top");
+			if (begin)
+				add_to_top(begin, limit);
+			begin = dlsym(h, "__start_set_logpage");
+			limit = dlsym(h, "__stop_set_logpage");
+			if (begin)
+				add_to_logpage(begin, limit);
+		}
+		free(path);
+		path = NULL;
+	}
+	closedir(d);
+}
+
 int
 main(int argc, char *argv[])
 {
 
-	if (argc < 2)
-		gen_usage_set(SET_BEGIN(top), SET_LIMIT(top));
+	add_to_top(NVME_CMD_BEGIN(top), NVME_CMD_LIMIT(top));
+	add_to_logpage(NVME_LOGPAGE_BEGIN, NVME_LOGPAGE_LIMIT);
 
-	DISPATCH(argc, argv, top);
+	load_dir("/lib/nvmecontrol");
+	load_dir("/usr/local/lib/nvmecontrol");
+
+	if (argc < 2)
+		gen_usage_set(top_begin(), top_limit());
+
+	dispatch_set(argc, argv, top_begin(), top_limit());
 
 	return (0);
 }

sbin/nvmecontrol/nvmecontrol.h

@@ -43,11 +43,15 @@ struct nvme_function {
 	const char	*usage;
 };
 
-#define NVME_CMDSET(set, sym)	DATA_SET(set, sym)
+#define NVME_SETNAME(set)	set
+#define	NVME_CMDSET(set, sym)	DATA_SET(NVME_SETNAME(set), sym)
 #define NVME_COMMAND(set, nam, function, usage_str)			\
 	static struct nvme_function function ## _nvme_cmd =		\
 	{ .name = #nam, .fn = function, .usage = usage_str };		\
 	NVME_CMDSET(set, function ## _nvme_cmd)
+#define NVME_CMD_BEGIN(set)	SET_BEGIN(NVME_SETNAME(set))
+#define NVME_CMD_LIMIT(set)	SET_LIMIT(NVME_SETNAME(set))
+#define NVME_CMD_DECLARE(set, t) SET_DECLARE(NVME_SETNAME(set), t)
 
 typedef void (*print_fn_t)(const struct nvme_controller_data *cdata, void *buf, uint32_t size);
 
@@ -59,7 +63,8 @@ struct logpage_function {
 	size_t		size;
 };
 
-#define NVME_LOGPAGESET(sym)		DATA_SET(logpage, sym)
+
+#define NVME_LOGPAGESET(sym)		DATA_SET(NVME_SETNAME(logpage), sym)
 #define NVME_LOGPAGE(unique, lp, vend, nam, fn, sz)			\
 	static struct logpage_function unique ## _lpf = {		\
 		.log_page = lp,						\
@@ -69,6 +74,9 @@ struct logpage_function {
 		.size = sz,						\
 	} ;								\
 	NVME_LOGPAGESET(unique ## _lpf)
+#define NVME_LOGPAGE_BEGIN	SET_BEGIN(NVME_SETNAME(logpage))
+#define NVME_LOGPAGE_LIMIT	SET_LIMIT(NVME_SETNAME(logpage))
+#define NVME_LOGPAGE_DECLARE(t)		SET_DECLARE(NVME_SETNAME(logpage), t)
 
 #define DEFAULT_SIZE	(4096)
 struct kv_name {
@@ -78,6 +86,27 @@ struct kv_name {
 
 const char *kv_lookup(const struct kv_name *kv, size_t kv_count, uint32_t key);
 
+NVME_CMD_DECLARE(top, struct nvme_function);
+NVME_LOGPAGE_DECLARE(struct logpage_function);
+
+struct set_concat {
+	void **begin;
+	void **limit;
+};
+void set_concat_add(struct set_concat *m, void *begin, void *end);
+#define SET_CONCAT_DEF(set, t) 							\
+static struct set_concat set ## _concat;					\
+static inline t **set ## _begin() { return ((t **)set ## _concat.begin); }	\
+static inline t **set ## _limit() { return ((t **)set ## _concat.limit); }	\
+void add_to_ ## set(t **b, t **e)						\
+{										\
+	set_concat_add(&set ## _concat, b, e);					\
+}
+#define SET_CONCAT_DECL(set, t)							\
+	void add_to_ ## set(t **b, t **e)
+SET_CONCAT_DECL(top, struct nvme_function);
+SET_CONCAT_DECL(logpage, struct logpage_function);
+
 #define NVME_CTRLR_PREFIX	"nvme"
 #define NVME_NS_PREFIX		"ns"
 
@@ -95,7 +124,7 @@ void dispatch_set(int argc, char *argv[], struct nvme_function **tbl,
     struct nvme_function **tbl_limit);
 
 #define DISPATCH(argc, argv, set)	\
-	dispatch_set(argc, argv, SET_BEGIN(set), SET_LIMIT(set))
+	dispatch_set(argc, argv, NVME_CMD_BEGIN(set), NVME_CMD_LIMIT(set))
 
 /* Utility Routines */
 /*

sbin/nvmecontrol/wdc.c

@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
 #define WDC_USAGE							       \
 	"wdc (cap-diag)\n"
 
-SET_DECLARE(wdc, struct nvme_function);
+NVME_CMD_DECLARE(wdc, struct nvme_function);
 
 #define WDC_NVME_TOC_SIZE	8
 

share/man/man7/hier.7

@@ -152,6 +152,10 @@ Capsicum support libraries
 class-specific libraries for the
 .Xr geom 8
 utility
+.It Pa nvmecontrol/
+vendor-specific libraries to extend the
+.Xr nvmecontrol 8
+utility
 .El
 .It Pa /libexec/
 critical system utilities needed for binaries in