- Return NULL and set errno to EINVAL if size is 0 (as required by POSIX).

Update the manpage to reflect this change.
- Always set the current position to the first null-byte when opening in append
mode. This makes the implementation compatible with glibc's. Update the test
suite.

Reported by:	pho
Approved by:	cognet

lib/libc/stdio/fmemopen.c

@@ -56,6 +56,14 @@ fmemopen(void * __restrict buf, size_t size, const char * __restrict mode)
 	FILE *f;
 	int flags, rc;
 
+	/*
+	 * POSIX says we shall return EINVAL if size is 0.
+	 */
+	if (size == 0) {
+		errno = EINVAL;
+		return (NULL);
+	}
+
 	/*
 	 * Retrieve the flags as used by open(2) from the mode argument, and
 	 * validate them.
@@ -119,14 +127,7 @@ fmemopen(void * __restrict buf, size_t size, const char * __restrict mode)
 	 */
 	switch (mode[0]) {
 	case 'a':
-		if (ck->bin) {
-			/*
-			 * This isn't useful, since the buffer isn't allowed
-			 * to grow.
-			 */
-			ck->off = ck->len = size;
-		} else
-			ck->off = ck->len = strnlen(ck->buf, ck->size);
+		ck->off = ck->len = strnlen(ck->buf, ck->size);
 		break;
 	case 'r':
 		ck->len = size;

lib/libc/stdio/fopen.3

@@ -302,6 +302,15 @@ for any of the errors specified for the routines
 .Xr fclose 3
 and
 .Xr fflush 3 .
+.Pp
+The
+.Fn fmemopen
+function
+may also fail and set
+.Va errno
+if the
+.Fa size
+argument is 0.
 .Sh SEE ALSO
 .Xr open 2 ,
 .Xr fclose 3 ,

tools/regression/lib/libc/stdio/test-fmemopen.c

@@ -138,6 +138,13 @@ test_autoalloc()
 	/* Close the FILE *. */
 	rc = fclose(fp);
 	assert(rc == 0);
+
+	/* Open a FILE * using a wrong mode */
+	fp = fmemopen(NULL, 512, "r");
+	assert(fp == NULL);
+
+	fp = fmemopen(NULL, 512, "w");
+	assert(fp == NULL);
 }
 
 void
@@ -241,6 +248,44 @@ test_binary()
 	assert(rc == 0);
 }
 
+void
+test_append_binary_pos()
+{
+	/*
+	 * For compatibility with other implementations (glibc), we set the
+	 * position to 0 when opening an automatically allocated binary stream
+	 * for appending.
+	 */
+
+	FILE *fp;
+
+	fp = fmemopen(NULL, 16, "ab+");
+	assert(ftell(fp) == 0L);
+	fclose(fp);
+
+	/*
+	 * Make sure that a pre-allocated buffer behaves correctly.
+	 */
+	char buf[] = "Hello";
+	fp = fmemopen(buf, sizeof(buf), "ab+");
+	assert(ftell(fp) == 5);
+	fclose(fp);
+}
+
+void
+test_size_0()
+{
+	/*
+	 * POSIX mandates that we return EINVAL if size is 0
+	 */
+
+	FILE *fp;
+
+	fp = fmemopen(NULL, 0, "r+");
+	assert(fp == NULL);
+	assert(errno == EINVAL);
+}
+
 int
 main(void)
 {
@@ -248,5 +293,7 @@ main(void)
 	test_preexisting();
 	test_data_length();
 	test_binary();
+	test_append_binary_pos();
+	test_size_0();
 	return (0);
 }