It was reported that when using nss_ldap, getgrent(3) would behave

incorrectly when encountering `large' groups (many members and/or many
long member names).  The reporter tracked this down to the glibc NSS
module compatibility code (nss_compat.c): it would prematurely record
that a NSS module was finished iterating through its database in some
cases.

Two aspects are corrected:

1. nss_compat.c recorded that a NSS module was finished iterating
   whenever the module reported something other than SUCCESS.  The
   correct logic is to continue iteration when the module reports
   either SUCCESS or RETURN.  The __nss_compat_getgrent_r and
   __nss_compat_getpwent_r routines are updated to reflect this.

2. An internal helper macro __nss_compat_result is used to map glibc
   NSS status codes to BSD NSS status codes (e.g. NSS_STATUS_SUCCESS ->
   NS_SUCCESS).  It provided the obvious mapping.

   When a NSS routine is called with a too-small buffer, the
   convention in the BSD NSS code is to report RETURN.  (This is used
   to implement reentrant APIs such as getpwnam_r(3).)  However, the
   convention in glibc for this case is to set errno = ERANGE and
   overload TRYAGAIN.  __nss_compat_result is updated to handle this
   case.

PR:		bin/60287
Reported by:	Lachlan O'Dea <odela01@ca.com>

include/nss.h

@@ -46,8 +46,9 @@ enum nss_status {
 	NSS_STATUS_RETURN
 };
 
-#define __nss_compat_result(rv)			\
-((rv == NSS_STATUS_TRYAGAIN) ? NS_TRYAGAIN :	\
+#define __nss_compat_result(rv, err)		\
+((rv == NSS_STATUS_TRYAGAIN && err == ERANGE) ? NS_RETURN : \
+ (rv == NSS_STATUS_TRYAGAIN) ? NS_TRYAGAIN :	\
  (rv == NSS_STATUS_UNAVAIL)  ? NS_UNAVAIL  :	\
  (rv == NSS_STATUS_NOTFOUND) ? NS_NOTFOUND :	\
  (rv == NSS_STATUS_SUCCESS)  ? NS_SUCCESS  :	\

lib/libc/net/nss_compat.c

@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
 #include <sys/param.h>
+#include <errno.h>
 #include <nss.h>
 #include <pthread.h>
 #include <pthread_np.h>
@@ -96,9 +97,10 @@ __nss_compat_getgrnam_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(name, grp, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct group **)retval = grp;
-	return (__nss_compat_result(status));
+	return (status);
 }
 
 
@@ -120,9 +122,10 @@ __nss_compat_getgrgid_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(gid, grp, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct group **)retval = grp;
-	return (__nss_compat_result(status));
+	return (status);
 }
 
 
@@ -144,11 +147,12 @@ __nss_compat_getgrent_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(grp, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct group **)retval = grp;
-	else
+	else if (status != NS_RETURN)
 		SET_TERMINATOR(group, &terminator);
-	return (__nss_compat_result(status));
+	return (status);
 }
 
 
@@ -194,9 +198,10 @@ __nss_compat_getpwnam_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(name, pwd, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct passwd **)retval = pwd;
-	return (__nss_compat_result(status));
+	return (status);
 }
 
 
@@ -218,9 +223,10 @@ __nss_compat_getpwuid_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(uid, pwd, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct passwd **)retval = pwd;
-	return (__nss_compat_result(status));
+	return (status);
 }
 
 
@@ -242,11 +248,12 @@ __nss_compat_getpwent_r(void *retval, void *mdata, va_list ap)
 	bufsize = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 	status = fn(pwd, buffer, bufsize, errnop);
+	status = __nss_compat_result(status, *errnop);
 	if (status == NS_SUCCESS)
 		*(struct passwd **)retval = pwd;
-	else
+	else if (status != NS_RETURN)
 		SET_TERMINATOR(passwd, &terminator);
-	return (__nss_compat_result(status));
+	return (status);
 }