malloc: export kernel zones instead of relying on them being power-of-2

Reviewed by:	markj (previous version)
Differential Revision:	https://reviews.freebsd.org/D27026

lib/libmemstat/memstat.h

@@ -117,6 +117,13 @@ int	memstat_kvm_all(struct memory_type_list *list, void *kvm_handle);
 int	memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle);
 int	memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle);
 
+/*
+ * General malloc routines.
+ */
+size_t	memstat_malloc_zone_get_count(void);
+size_t	memstat_malloc_zone_get_size(size_t n);
+int	memstat_malloc_zone_used(const struct memory_type *mtp, size_t n);
+
 /*
  * Accessor methods for struct memory_type.
  */

lib/libmemstat/memstat_malloc.c

@@ -44,10 +44,22 @@
 #include "memstat.h"
 #include "memstat_internal.h"
 
+static int memstat_malloc_zone_count;
+static int memstat_malloc_zone_sizes[32];
+
+static int	memstat_malloc_zone_init(void);
+static int	memstat_malloc_zone_init_kvm(kvm_t *kvm);
+
 static struct nlist namelist[] = {
 #define	X_KMEMSTATISTICS	0
 	{ .n_name = "_kmemstatistics" },
-#define	X_MP_MAXCPUS		1
+#define	X_KMEMZONES		1
+	{ .n_name = "_kmemzones" },
+#define	X_NUMZONES		2
+	{ .n_name = "_numzones" },
+#define	X_VM_MALLOC_ZONE_COUNT	3
+	{ .n_name = "_vm_malloc_zone_count" },
+#define	X_MP_MAXCPUS		4
 	{ .n_name = "_mp_maxcpus" },
 	{ .n_name = "" },
 };
@@ -111,6 +123,11 @@ memstat_sysctl_malloc(struct memory_type_list *list, int flags)
 		return (-1);
 	}
 
+	if (memstat_malloc_zone_init() == -1) {
+		list->mtl_error = MEMSTAT_ERROR_VERSION;
+		return (-1);
+	}
+
 	size = sizeof(*mthp) + count * (sizeof(*mthp) + sizeof(*mtsp) *
 	    maxcpus);
 
@@ -333,6 +350,12 @@ memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
 		return (-1);
 	}
 
+	ret = memstat_malloc_zone_init_kvm(kvm);
+	if (ret != 0) {
+		list->mtl_error = ret;
+		return (-1);
+	}
+
 	mp_ncpus = kvm_getncpus(kvm);
 
 	for (typep = kmemstatistics; typep != NULL; typep = type.ks_next) {
@@ -416,3 +439,109 @@ memstat_kvm_malloc(struct memory_type_list *list, void *kvm_handle)
 
 	return (0);
 }
+
+static int
+memstat_malloc_zone_init(void)
+{
+	size_t size;
+
+	size = sizeof(memstat_malloc_zone_count);
+	if (sysctlbyname("vm.malloc.zone_count", &memstat_malloc_zone_count,
+	    &size, NULL, 0) < 0) {
+		return (-1);
+	}
+
+	if (memstat_malloc_zone_count > (int)nitems(memstat_malloc_zone_sizes)) {
+		return (-1);
+	}
+
+	size = sizeof(memstat_malloc_zone_sizes);
+	if (sysctlbyname("vm.malloc.zone_sizes", &memstat_malloc_zone_sizes,
+	    &size, NULL, 0) < 0) {
+		return (-1);
+	}
+
+	return (0);
+}
+
+/*
+ * Copied from kern_malloc.c
+ *
+ * kz_zone is an array sized at compilation time, the size is exported in
+ * "numzones". Below we need to iterate kz_size.
+ */
+struct memstat_kmemzone {
+	int kz_size;
+	const char *kz_name;
+	void *kz_zone[1];
+};
+
+static int
+memstat_malloc_zone_init_kvm(kvm_t *kvm)
+{
+	struct memstat_kmemzone *kmemzones, *kz;
+	int numzones, objsize, allocsize, ret;
+	int i;
+
+	ret = kread_symbol(kvm, X_VM_MALLOC_ZONE_COUNT,
+	    &memstat_malloc_zone_count, sizeof(memstat_malloc_zone_count), 0);
+	if (ret != 0) {
+		return (ret);
+	}
+
+	ret = kread_symbol(kvm, X_NUMZONES, &numzones, sizeof(numzones), 0);
+	if (ret != 0) {
+		return (ret);
+	}
+
+	objsize = __offsetof(struct memstat_kmemzone, kz_zone) +
+	    sizeof(void *) * numzones;
+
+	allocsize = objsize * memstat_malloc_zone_count;
+	kmemzones = malloc(allocsize);
+	if (kmemzones == NULL) {
+		return (MEMSTAT_ERROR_NOMEMORY);
+	}
+	ret = kread_symbol(kvm, X_KMEMZONES, kmemzones, allocsize, 0);
+	if (ret != 0) {
+		free(kmemzones);
+		return (ret);
+	}
+
+	kz = kmemzones;
+	for (i = 0; i < (int)nitems(memstat_malloc_zone_sizes); i++) {
+		memstat_malloc_zone_sizes[i] = kz->kz_size;
+		kz = (struct memstat_kmemzone *)((char *)kz + objsize);
+	}
+
+	free(kmemzones);
+	return (0);
+}
+
+size_t
+memstat_malloc_zone_get_count(void)
+{
+
+	return (memstat_malloc_zone_count);
+}
+
+size_t
+memstat_malloc_zone_get_size(size_t n)
+{
+
+	if (n >= nitems(memstat_malloc_zone_sizes)) {
+		return (-1);
+	}
+
+	return (memstat_malloc_zone_sizes[n]);
+}
+
+int
+memstat_malloc_zone_used(const struct memory_type *mtp, size_t n)
+{
+
+	if (memstat_get_sizemask(mtp) & (1 << n))
+		return (1);
+
+	return (0);
+}

share/man/man9/malloc.9

@@ -29,7 +29,7 @@
 .\" $NetBSD: malloc.9,v 1.3 1996/11/11 00:05:11 lukem Exp $
 .\" $FreeBSD$
 .\"
-.Dd August 28, 2020
+.Dd October 30, 2020
 .Dt MALLOC 9
 .Os
 .Sh NAME
@@ -57,6 +57,8 @@
 .Fn reallocf "void *addr" "size_t size" "struct malloc_type *type" "int flags"
 .Ft size_t
 .Fn malloc_usable_size "const void *addr"
+.Ft void *
+.Fn malloc_exec "size_t size" "struct malloc_type *type" "int flags"
 .Fn MALLOC_DECLARE type
 .In sys/param.h
 .In sys/malloc.h
@@ -66,6 +68,8 @@
 .In sys/domainset.h
 .Ft void *
 .Fn malloc_domainset "size_t size" "struct malloc_type *type" "struct domainset *ds" "int flags"
+.Ft void *
+.Fn malloc_domainset_exec "size_t size" "struct malloc_type *type" "struct domainset *ds" "int flags"
 .Sh DESCRIPTION
 The
 .Fn malloc
@@ -82,6 +86,13 @@ See
 .Xr domainset 9
 for some example policies.
 .Pp
+Both
+.Fn malloc_exec
+and
+.Fn malloc_domainset_exec
+can be used to return executable memory.
+Not all platforms enforce a distinction between executable and non-executable memory.
+.Pp
 The
 .Fn mallocarray
 function allocates uninitialized memory in kernel address space for an
@@ -214,11 +225,6 @@ This option should only be used in combination with
 .Dv M_NOWAIT
 when an allocation failure cannot be tolerated by the caller without
 catastrophic effects on the system.
-.It Dv M_EXEC
-Indicates that the system should allocate executable memory.
-If this flag is not set, the system will not allocate executable memory.
-Not all platforms enforce a distinction between executable and
-non-executable memory.
 .El
 .Pp
 Exactly one of either

sys/kern/kern_malloc.c

@@ -147,6 +147,8 @@ static int numzones = MALLOC_DEBUG_MAXZONES;
  * Small malloc(9) memory allocations are allocated from a set of UMA buckets
  * of various sizes.
  *
+ * Warning: the layout of the struct is duplicated in libmemstat for KVM support.
+ *
  * XXX: The comment here used to read "These won't be powers of two for
  * long."  It's possible that a significant amount of wasted memory could be
  * recovered by tuning the sizes of these buckets.
@@ -213,6 +215,19 @@ SYSCTL_PROC(_vm, OID_AUTO, kmem_map_free,
     CTLFLAG_RD | CTLTYPE_ULONG | CTLFLAG_MPSAFE, NULL, 0,
     sysctl_kmem_map_free, "LU", "Free space in kmem");
 
+static SYSCTL_NODE(_vm, OID_AUTO, malloc, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+    "Malloc information");
+
+static u_int vm_malloc_zone_count = nitems(kmemzones);
+SYSCTL_UINT(_vm_malloc, OID_AUTO, zone_count,
+    CTLFLAG_RD, &vm_malloc_zone_count, 0,
+    "Number of malloc zones");
+
+static int sysctl_vm_malloc_zone_sizes(SYSCTL_HANDLER_ARGS);
+SYSCTL_PROC(_vm_malloc, OID_AUTO, zone_sizes,
+    CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, NULL, 0,
+    sysctl_vm_malloc_zone_sizes, "S", "Zone sizes used by malloc");
+
 /*
  * The malloc_mtx protects the kmemstatistics linked list.
  */
@@ -274,6 +289,19 @@ sysctl_kmem_map_free(SYSCTL_HANDLER_ARGS)
 	return (sysctl_handle_long(oidp, &size, 0, req));
 }
 
+static int
+sysctl_vm_malloc_zone_sizes(SYSCTL_HANDLER_ARGS)
+{
+	int sizes[nitems(kmemzones)];
+	int i;
+
+	for (i = 0; i < nitems(kmemzones); i++) {
+		sizes[i] = kmemzones[i].kz_size;
+	}
+
+	return (SYSCTL_OUT(req, &sizes, sizeof(sizes)));
+}
+
 /*
  * malloc(9) uma zone separation -- sub-page buffer overruns in one
  * malloc type will affect only a subset of other malloc types.

usr.bin/vmstat/vmstat.c

@@ -1407,7 +1407,8 @@ domemstat_malloc(void)
 {
 	struct memory_type_list *mtlp;
 	struct memory_type *mtp;
-	int error, first, i;
+	size_t i, zones;
+	int error, first;
 
 	mtlp = memstat_mtl_alloc();
 	if (mtlp == NULL) {
@@ -1435,6 +1436,7 @@ domemstat_malloc(void)
 	xo_emit("{T:/%13s} {T:/%5s} {T:/%6s} {T:/%7s} {T:/%8s}  {T:Size(s)}\n",
 	    "Type", "InUse", "MemUse", "HighUse", "Requests");
 	xo_open_list("memory");
+	zones = memstat_malloc_zone_get_count();
 	for (mtp = memstat_mtl_first(mtlp); mtp != NULL;
 	    mtp = memstat_mtl_next(mtp)) {
 		if (memstat_get_numallocs(mtp) == 0 &&
@@ -1449,11 +1451,11 @@ domemstat_malloc(void)
 		    (uintmax_t)memstat_get_numallocs(mtp));
 		first = 1;
 		xo_open_list("size");
-		for (i = 0; i < 32; i++) {
+		for (i = 0; i < zones; i++) {
-			if (memstat_get_sizemask(mtp) & (1 << i)) {
+			if (memstat_malloc_zone_used(mtp, i)) {
 				if (!first)
 					xo_emit(",");
-				xo_emit("{l:size/%d}", 1 << (i + 4));
+				xo_emit("{l:size/%d}", memstat_malloc_zone_get_size(i));
 				first = 0;
 			}
 		}