File descriptors are an int, but our stdio FILE object uses a short to hold

them.  Thus, any fd whose value is greater than SHRT_MAX is handled
incorrectly (the short value is sign-extended when converted to an int).
An unpleasant side effect is that if fopen() opens a file and gets a
backing fd that is greater than SHRT_MAX, fclose() will fail and the file
descriptor will be leaked.  Better handle this by fixing fopen(), fdopen(),
and freopen() to fail attempts to use a fd greater than SHRT_MAX with
EMFILE.

At some point in the future we should look at expanding the file descriptor
in FILE to an int, but that is a bit complicated due to ABI issues.

MFC after:	1 week
Discussed on:	arch
Reviewed by:	wollman

lib/libc/stdio/fdopen.c

@@ -57,6 +57,18 @@ fdopen(fd, mode)
 	if (nofile == 0)
 		nofile = getdtablesize();
 
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (fd > SHRT_MAX) {
+		errno = EMFILE;
+		return (NULL);
+	}
+
 	if ((flags = __sflags(mode, &oflags)) == 0)
 		return (NULL);
 

lib/libc/stdio/fopen.c

@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <errno.h>
 #include "un-namespace.h"
@@ -63,6 +64,18 @@ fopen(file, mode)
 		fp->_flags = 0;			/* release */
 		return (NULL);
 	}
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (f > SHRT_MAX) {
+		_close(f);
+		errno = EMFILE;
+		return (NULL);
+	}
 	fp->_file = f;
 	fp->_flags = flags;
 	fp->_cookie = fp;

lib/libc/stdio/freopen.c

@@ -203,6 +203,20 @@ finish:
 		}
 	}
 
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (f > SHRT_MAX) {
+		fp->_flags = 0;		/* set it free */
+		FUNLOCKFILE(fp);
+		errno = EMFILE;
+		return (NULL);
+	}
+
 	fp->_flags = flags;
 	fp->_file = f;
 	fp->_cookie = fp;