Introduce a new sysctl, kern.malloc_stats, which exports kernel malloc

statistics via a binary structure stream:

- Add structure 'malloc_type_stream_header', which defines a stream
  version, definition of MAXCPUS used in the stream, and a number of
  malloc_type records in the stream.

- Add structure 'malloc_type_header', which defines the name of the
  malloc type being reported on.

- When the sysctl is queried, return a stream header, followed by a
  series of type descriptions, each consisting of a type header
  followed by a series of MAXCPUS malloc_type_stats structures holding
  per-CPU allocation information.  Typical values of MAXCPUS will be 1
  (UP compiled kernel) and 16 (SMP compiled kernel).

This query mechanism allows user space monitoring tools to extract
memory allocation statistics in a machine-readable form, and to do so
at a per-CPU granularity, allowing monitoring of allocation patterns
across CPUs in order to better understand the distribution of work and
memory flow over multiple CPUs.

While here:

- Bump statistics width to uint64_t, and hard code using fixed-width
  type in order to be more sure about structure layout in the stream.
  We allocate and free a lot of memory.

- Add kmemcount, a counter of the number of registered malloc types,
  in order to avoid excessive manual counting of types.  Export via a
  new sysctl to allow user-space code to better size buffers.

- De-XXX comment on no longer maintaining the high watermark in old
  sysctl monitoring code.

A follow-up commit of libmemstat(3), a library to monitor kernel memory
allocation, will occur in the next few days.  Likewise, similar changes
to UMA.

sys/kern/kern_malloc.c

@@ -94,6 +94,7 @@ static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
 static struct malloc_type *kmemstatistics;
 static char *kmembase;
 static char *kmemlimit;
+static int kmemcount;
 
 #define KMEM_ZSHIFT	4
 #define KMEM_ZBASE	16
@@ -169,6 +170,7 @@ static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
 #endif
 
 static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
+static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
 
 /* time_uptime of last malloc(9) failure */
 static time_t t_malloc_fail;
@@ -593,6 +595,7 @@ malloc_init(void *data)
 	mtx_lock(&malloc_mtx);
 	mtp->ks_next = kmemstatistics;
 	kmemstatistics = mtp;
+	kmemcount++;
 	mtx_unlock(&malloc_mtx);
 }
 
@@ -615,6 +618,7 @@ malloc_uninit(void *data)
 		}
 	} else
 		kmemstatistics = mtp->ks_next;
+	kmemcount--;
 	mtx_unlock(&malloc_mtx);
 	uma_zfree(mt_zone, mtip);
 }
@@ -639,8 +643,7 @@ sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
 
 	/* Guess at how much room is needed. */
 	mtx_lock(&malloc_mtx);
-	for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next)
-		cnt++;
+	cnt = kmemcount;
 	mtx_unlock(&malloc_mtx);
 
 	bufsize = linesize * (cnt + 1);
@@ -686,10 +689,10 @@ sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
 			temp_bytes = 0;
 
 		/*
-		 * XXXRW: High-waterwark is no longer easily available, so
-		 * we just print '-' for that column.
+		 * High-waterwark is no longer easily available, so we just
+		 * print '-' for that column.
 		 */
-		sbuf_printf(&sbuf, "%13s%6lu%6luK       -%9lu",
+		sbuf_printf(&sbuf, "%13s%6lu%6luK       -%9llu",
 		    mtp->ks_shortdesc,
 		    temp_allocs,
 		    (temp_bytes + 1023) / 1024,
@@ -723,6 +726,91 @@ sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
 SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD,
     NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats");
 
+static int
+sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS)
+{
+	struct malloc_type_stream_header mtsh;
+	struct malloc_type_internal *mtip;
+	struct malloc_type_header mth;
+	struct malloc_type *mtp;
+	int buflen, count, error, i;
+	struct sbuf sbuf;
+	char *buffer;
+
+	mtx_lock(&malloc_mtx);
+restart:
+	mtx_assert(&malloc_mtx, MA_OWNED);
+	count = kmemcount;
+	mtx_unlock(&malloc_mtx);
+	buflen = sizeof(mtsh) + count * (sizeof(mth) +
+	    sizeof(struct malloc_type_stats) * MAXCPU) + 1;
+	buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO);
+	mtx_lock(&malloc_mtx);
+	if (count < kmemcount) {
+		free(buffer, M_TEMP);
+		goto restart;
+	}
+
+	sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN);
+
+	/*
+	 * Insert stream header.
+	 */
+	bzero(&mtsh, sizeof(mtsh));
+	mtsh.mtsh_version = MALLOC_TYPE_STREAM_VERSION;
+	mtsh.mtsh_maxcpus = MAXCPU;
+	mtsh.mtsh_count = kmemcount;
+	if (sbuf_bcat(&sbuf, &mtsh, sizeof(mtsh)) < 0) {
+		mtx_unlock(&malloc_mtx);
+		error = ENOMEM;
+		goto out;
+	}
+
+	/*
+	 * Insert alternating sequence of type headers and type statistics.
+	 */
+	for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
+		mtip = (struct malloc_type_internal *)mtp->ks_handle;
+
+		/*
+		 * Insert type header.
+		 */
+		bzero(&mth, sizeof(mth));
+		strlcpy(mth.mth_name, mtp->ks_shortdesc, MALLOC_MAX_NAME);
+		if (sbuf_bcat(&sbuf, &mth, sizeof(mth)) < 0) {
+			mtx_unlock(&malloc_mtx);
+			error = ENOMEM;
+			goto out;
+		}
+
+		/*
+		 * Insert type statistics for each CPU.
+		 */
+		for (i = 0; i < MAXCPU; i++) {
+			if (sbuf_bcat(&sbuf, &mtip->mti_stats[i],
+			    sizeof(mtip->mti_stats[i])) < 0) {
+				mtx_unlock(&malloc_mtx);
+				error = ENOMEM;
+				goto out;
+			}
+		}
+	}
+	mtx_unlock(&malloc_mtx);
+	sbuf_finish(&sbuf);
+	error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf));
+out:
+	sbuf_delete(&sbuf);
+	free(buffer, M_TEMP);
+	return (error);
+}
+
+SYSCTL_PROC(_kern, OID_AUTO, malloc_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
+    0, 0, sysctl_kern_malloc_stats, "s,malloc_type_ustats",
+    "Return malloc types");
+
+SYSCTL_INT(_kern, OID_AUTO, malloc_count, CTLFLAG_RD, &kmemcount, 0,
+    "Count of kernel malloc types");
+
 #ifdef MALLOC_PROFILE
 
 static int

sys/sys/malloc.h

@@ -70,14 +70,14 @@
  * monitoring app should take into account.
  */
 struct malloc_type_stats {
-	u_long	mts_memalloced;		/* Bytes allocated on CPU. */
-	u_long	mts_memfreed;		/* Bytes freed on CPU. */
-	u_long	mts_numallocs;		/* Number of allocates on CPU. */
-	u_long	mts_numfrees;		/* number of frees on CPU. */
-	u_long	mts_size;		/* Bitmask of sizes allocated on CPU. */
-	u_long	_mts_reserved1;		/* Reserved field. */
-	u_long	_mts_reserved2;		/* Reserved field. */
-	u_long	_mts_reserved3;		/* Reserved field. */
+	uint64_t	mts_memalloced;	/* Bytes allocated on CPU. */
+	uint64_t	mts_memfreed;	/* Bytes freed on CPU. */
+	uint64_t	mts_numallocs;	/* Number of allocates on CPU. */
+	uint64_t	mts_numfrees;	/* number of frees on CPU. */
+	uint64_t	mts_size;	/* Bitmask of sizes allocated on CPU. */
+	uint64_t	_mts_reserved1;	/* Reserved field. */
+	uint64_t	_mts_reserved2;	/* Reserved field. */
+	uint64_t	_mts_reserved3;	/* Reserved field. */
 };
 
 struct malloc_type_internal {
@@ -116,6 +116,26 @@ struct malloc_type {
 	u_int		 _mtx_recurse;
 };
 
+/*
+ * Statistics structure headers for user space.  The kern.malloc sysctl
+ * exposes a structure stream consisting of a stream header, then a series of
+ * malloc type headers and statistics structures (quantity maxcpus).  For
+ * convenience, the kernel will provide the current value of maxcpus at the
+ * head of the stream.
+ */
+#define	MALLOC_TYPE_STREAM_VERSION	0x00000001
+struct malloc_type_stream_header {
+	uint32_t	mtsh_version;	/* Stream format version. */
+	uint32_t	mtsh_maxcpus;	/* Value of MAXCPU for stream. */
+	uint32_t	mtsh_count;	/* Number of records. */
+	uint32_t	_mtsh_pad;	/* Pad/reserved field. */
+};
+
+#define	MALLOC_MAX_NAME	32
+struct malloc_type_header {
+	char				mth_name[MALLOC_MAX_NAME];
+};
+
 #ifdef _KERNEL
 #define	MALLOC_DEFINE(type, shortdesc, longdesc)			\
 	struct malloc_type type[1] = {					\