libc: Fix size range check in setvbuf

From enh at google.com via openbsd-tech mailing list via pfg@:

The existing test is wrong for LP64, where size_t has twice as many
relevant bits as int, not just one. (Found by inspection by
rprichard.)
This commit is contained in:
Ed Maste 2022-10-03 14:24:42 -04:00
parent 89e5ef8917
commit 9515313b26

View File

@ -39,6 +39,7 @@ static char sccsid[] = "@(#)setvbuf.c 8.2 (Berkeley) 11/16/93";
__FBSDID("$FreeBSD$");
#include "namespace.h"
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include "un-namespace.h"
@ -62,7 +63,7 @@ setvbuf(FILE * __restrict fp, char * __restrict buf, int mode, size_t size)
* when setting _IONBF.
*/
if (mode != _IONBF)
if ((mode != _IOFBF && mode != _IOLBF) || (int)size < 0)
if ((mode != _IOFBF && mode != _IOLBF) || size > INT_MAX)
return (EOF);
FLOCKFILE_CANCELSAFE(fp);