mallocarray(9): panic if the requested allocation would overflow

Additionally, move the overflow check logic out to WOULD_OVERFLOW() for
consumers to have a common means of testing for overflowing allocations.
WOULD_OVERFLOW() should be a secondary check -- on 64-bit platforms, just
because an allocation won't overflow size_t does not mean it is a sane size
to request.  Callers should be imposing reasonable allocation limits far,
far, below overflow.

Discussed with:	emaste, jhb, kp
Sponsored by:	Dell EMC Isilon

share/man/man9/malloc.9

@@ -29,7 +29,7 @@
 .\" $NetBSD: malloc.9,v 1.3 1996/11/11 00:05:11 lukem Exp $
 .\" $FreeBSD$
 .\"
-.Dd November 19, 2015
+.Dd January 10, 2018
 .Dt MALLOC 9
 .Os
 .Sh NAME
@@ -154,6 +154,7 @@ If the request cannot be immediately fulfilled, the current process is put
 to sleep to wait for resources to be released by other processes.
 The
 .Fn malloc ,
+.Fn mallocarray ,
 .Fn realloc ,
 and
 .Fn reallocf
@@ -162,15 +163,13 @@ functions cannot return
 if
 .Dv M_WAITOK
 is specified.
-The
-.Fn mallocarray
-function can return
-.Dv NULL
 if the multiplication of
 .Fa nmemb
 and
 .Fa size
-would cause an integer overflow.
+would cause an integer overflow, the
+.Fn mallocarray
+function induces a panic.
 .It Dv M_USE_RESERVE
 Indicates that the system can use its reserve of memory to satisfy the
 request.

sys/kern/kern_malloc.c

@@ -535,18 +535,12 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
 	return ((void *) va);
 }
 
-/*
- * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
- * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
- */
-#define MUL_NO_OVERFLOW		(1UL << (sizeof(size_t) * 8 / 2))
 void *
 mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags)
 {
 
-	if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
-	    nmemb > 0 && SIZE_MAX / nmemb < size)
-		return (NULL);
+	if (WOULD_OVERFLOW(nmemb, size))
+		panic("mallocarray: %zu * %zu overflowed", nmemb, size);
 
 	return (malloc(size * nmemb, type, flags));
 }

sys/sys/malloc.h

@@ -41,6 +41,7 @@
 #include <sys/queue.h>
 #include <sys/_lock.h>
 #include <sys/_mutex.h>
+#include <machine/_limits.h>
 
 #define	MINALLOCSIZE	UMA_SMALLEST_UNIT
 
@@ -192,6 +193,20 @@ void	*reallocf(void *addr, unsigned long size, struct malloc_type *type,
 	    int flags) __result_use_check __alloc_size(2);
 
 struct malloc_type *malloc_desc2type(const char *desc);
+
+/*
+ * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX
+ * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW
+ */
+#define MUL_NO_OVERFLOW		(1UL << (sizeof(size_t) * 8 / 2))
+static inline bool
+WOULD_OVERFLOW(size_t nmemb, size_t size)
+{
+
+	return ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) &&
+	    nmemb > 0 && __SIZE_T_MAX / nmemb < size);
+}
+#undef MUL_NO_OVERFLOW
 #endif /* _KERNEL */
 
 #endif /* !_SYS_MALLOC_H_ */