If the size passed to {,v}s{w,n}printf is larger than INT_MAX+1

(i.e., the return value would overflow), set errno to EOVERFLOW
and return an error.  This improves the chances that buggy
applications -- for instance, ones that pass in a negative integer
as the size due to a bogus calculation -- will fail in safe ways.
Returning an error in these situations is specified by POSIX, but
POSIX appears to have an off-by-one error that isn't duplicated in
this change.

Previously, some of these functions would silently cap the size at
INT_MAX+1, and others would exit with an error after writing more
than INT_MAX characters.

PR:		39256
MFC after:	2 weeks

lib/libc/stdio/snprintf.c

@@ -41,6 +41,7 @@ static char sccsid[] = "@(#)snprintf.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdarg.h>
@@ -59,8 +60,11 @@ snprintf(char * __restrict str, size_t n, char const * __restrict fmt, ...)
 	on = n;
 	if (n != 0)
 		n--;
-	if (n > INT_MAX)
-		n = INT_MAX;
+	if (n > INT_MAX) {
+		errno = EOVERFLOW;
+		*str = '\0';
+		return (EOF);
+	}
 	va_start(ap, fmt);
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;
@@ -84,8 +88,11 @@ snprintf_l(char * __restrict str, size_t n, locale_t locale,
 	on = n;
 	if (n != 0)
 		n--;
-	if (n > INT_MAX)
-		n = INT_MAX;
+	if (n > INT_MAX) {
+		errno = EOVERFLOW;
+		*str = '\0';
+		return (EOF);
+	}
 	va_start(ap, fmt);
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;

lib/libc/stdio/vfprintf.c

@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 
 #include <ctype.h>
+#include <errno.h>
 #include <limits.h>
 #include <locale.h>
 #include <stddef.h>
@@ -480,6 +481,7 @@ __vfprintf(FILE *fp, locale_t locale, const char *fmt0, va_list ap)
 		if ((n = fmt - cp) != 0) {
 			if ((unsigned)ret + n > INT_MAX) {
 				ret = EOF;
+				errno = EOVERFLOW;
 				goto error;
 			}
 			PRINT(cp, n);
@@ -935,6 +937,7 @@ number:			if ((dprec = prec) >= 0)
 		prsize = width > realsz ? width : realsz;
 		if ((unsigned)ret + prsize > INT_MAX) {
 			ret = EOF;
+			errno = EOVERFLOW;
 			goto error;
 		}
 

lib/libc/stdio/vfwprintf.c

@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 
 #include <ctype.h>
+#include <errno.h>
 #include <limits.h>
 #include <locale.h>
 #include <stdarg.h>
@@ -553,6 +554,7 @@ __vfwprintf(FILE *fp, locale_t locale, const wchar_t *fmt0, va_list ap)
 		if ((n = fmt - cp) != 0) {
 			if ((unsigned)ret + n > INT_MAX) {
 				ret = EOF;
+				errno = EOVERFLOW;
 				goto error;
 			}
 			PRINT(cp, n);
@@ -1003,6 +1005,7 @@ number:			if ((dprec = prec) >= 0)
 		prsize = width > realsz ? width : realsz;
 		if ((unsigned)ret + prsize > INT_MAX) {
 			ret = EOF;
+			errno = EOVERFLOW;
 			goto error;
 		}
 

lib/libc/stdio/vsnprintf.c

@@ -41,6 +41,7 @@ static char sccsid[] = "@(#)vsnprintf.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
 #include <limits.h>
 #include <stdio.h>
 #include "local.h"
@@ -59,8 +60,11 @@ vsnprintf_l(char * __restrict str, size_t n, locale_t locale,
 	on = n;
 	if (n != 0)
 		n--;
-	if (n > INT_MAX)
-		n = INT_MAX;
+	if (n > INT_MAX) {
+		errno = EOVERFLOW;
+		*str = '\0';
+		return (EOF);
+	}
 	/* Stdio internals do not deal correctly with zero length buffer */
 	if (n == 0) {
 		if (on > 0)

lib/libc/stdio/vswprintf.c

@@ -39,6 +39,7 @@ __FBSDID("FreeBSD: src/lib/libc/stdio/vasprintf.c,v 1.16 2002/08/21 16:19:57 mik
 __FBSDID("$FreeBSD$");
 
 #include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <wchar.h>
@@ -61,6 +62,11 @@ vswprintf_l(wchar_t * __restrict s, size_t n, locale_t locale,
 		errno = EINVAL;
 		return (-1);
 	}
+	if (n - 1 > INT_MAX) {
+		errno = EOVERFLOW;
+		*s = L'\0';
+		return (-1);
+	}
 
 	f._flags = __SWR | __SSTR | __SALC;
 	f._bf._base = f._p = (unsigned char *)malloc(128);