freebsd-nq/lib/libc/stdlib/qsort.c
Stefan Eßer d106f982a5 qsort.c: prevent undefined behavior
Mark Milliard has detected a case of undefined behavior with the LLVM
UBSAN. The mandoc program called qsort with a==NULL and n==0, which is
allowed by the POSIX standard. The qsort() in FreeBSD did not attempt
to perform any accesses using the passed pointer for n==0, but it did
add an offset to the pointer value, which is undefined behavior in
case of a NULL pointer. This operation has no adverse effects on any
achitecture supported by FreeBSD, but could be caught in more strict
environments.

After some discussion in the freebsd-current mail list, it was
concluded that the case of a==NULL and n!=0 should still be caught by
UBSAN (or cause a program abort due to an illegal access) in order to
not hide errors in programs incorrectly invoking qsort().

Only the the case of a==NULL and n==0 should be fixed to not perform
the undefined operation on a NULL pointer.

This commit makes qsort() exit before reaching the point of
potentially undefined behvior for the case n==0, but does not test
the value of a, since the result will not depend on whether this
pointer is NULL or an actual pointer to an array if n==0.

The issue found by Mark Milliard in the whatis command has been
reported to the upstream (OpenBSD) and has already been patched
there.

MFC after:	1 week
2022-01-13 11:09:38 +01:00

251 lines
6.6 KiB
C

/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1992, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#if defined(LIBC_SCCS) && !defined(lint)
static char sccsid[] = "@(#)qsort.c 8.1 (Berkeley) 6/4/93";
#endif /* LIBC_SCCS and not lint */
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "libc_private.h"
#if defined(I_AM_QSORT_R)
typedef int cmp_t(void *, const void *, const void *);
#elif defined(I_AM_QSORT_S)
typedef int cmp_t(const void *, const void *, void *);
#else
typedef int cmp_t(const void *, const void *);
#endif
static inline char *med3(char *, char *, char *, cmp_t *, void *);
#define MIN(a, b) ((a) < (b) ? a : b)
/*
* Qsort routine from Bentley & McIlroy's "Engineering a Sort Function".
*/
static inline void
swapfunc(char *a, char *b, size_t es)
{
char t;
do {
t = *a;
*a++ = *b;
*b++ = t;
} while (--es > 0);
}
#define vecswap(a, b, n) \
if ((n) > 0) swapfunc(a, b, n)
#if defined(I_AM_QSORT_R)
#define CMP(t, x, y) (cmp((t), (x), (y)))
#elif defined(I_AM_QSORT_S)
#define CMP(t, x, y) (cmp((x), (y), (t)))
#else
#define CMP(t, x, y) (cmp((x), (y)))
#endif
static inline char *
med3(char *a, char *b, char *c, cmp_t *cmp, void *thunk
#if !defined(I_AM_QSORT_R) && !defined(I_AM_QSORT_S)
__unused
#endif
)
{
return CMP(thunk, a, b) < 0 ?
(CMP(thunk, b, c) < 0 ? b : (CMP(thunk, a, c) < 0 ? c : a ))
:(CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c ));
}
/*
* The actual qsort() implementation is static to avoid preemptible calls when
* recursing. Also give them different names for improved debugging.
*/
#if defined(I_AM_QSORT_R)
#define local_qsort local_qsort_r
#elif defined(I_AM_QSORT_S)
#define local_qsort local_qsort_s
#endif
static void
local_qsort(void *a, size_t n, size_t es, cmp_t *cmp, void *thunk)
{
char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
size_t d1, d2;
int cmp_result;
int swap_cnt;
if (__predict_false(n == 0))
return;
loop:
swap_cnt = 0;
if (n < 7) {
for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
for (pl = pm;
pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
pl -= es)
swapfunc(pl, pl - es, es);
return;
}
pm = (char *)a + (n / 2) * es;
if (n > 7) {
pl = a;
pn = (char *)a + (n - 1) * es;
if (n > 40) {
size_t d = (n / 8) * es;
pl = med3(pl, pl + d, pl + 2 * d, cmp, thunk);
pm = med3(pm - d, pm, pm + d, cmp, thunk);
pn = med3(pn - 2 * d, pn - d, pn, cmp, thunk);
}
pm = med3(pl, pm, pn, cmp, thunk);
}
swapfunc(a, pm, es);
pa = pb = (char *)a + es;
pc = pd = (char *)a + (n - 1) * es;
for (;;) {
while (pb <= pc && (cmp_result = CMP(thunk, pb, a)) <= 0) {
if (cmp_result == 0) {
swap_cnt = 1;
swapfunc(pa, pb, es);
pa += es;
}
pb += es;
}
while (pb <= pc && (cmp_result = CMP(thunk, pc, a)) >= 0) {
if (cmp_result == 0) {
swap_cnt = 1;
swapfunc(pc, pd, es);
pd -= es;
}
pc -= es;
}
if (pb > pc)
break;
swapfunc(pb, pc, es);
swap_cnt = 1;
pb += es;
pc -= es;
}
if (swap_cnt == 0) { /* Switch to insertion sort */
for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es)
for (pl = pm;
pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
pl -= es)
swapfunc(pl, pl - es, es);
return;
}
pn = (char *)a + n * es;
d1 = MIN(pa - (char *)a, pb - pa);
vecswap(a, pb - d1, d1);
/*
* Cast es to preserve signedness of right-hand side of MIN()
* expression, to avoid sign ambiguity in the implied comparison. es
* is safely within [0, SSIZE_MAX].
*/
d1 = MIN(pd - pc, pn - pd - (ssize_t)es);
vecswap(pb, pn - d1, d1);
d1 = pb - pa;
d2 = pd - pc;
if (d1 <= d2) {
/* Recurse on left partition, then iterate on right partition */
if (d1 > es) {
local_qsort(a, d1 / es, es, cmp, thunk);
}
if (d2 > es) {
/* Iterate rather than recurse to save stack space */
/* qsort(pn - d2, d2 / es, es, cmp); */
a = pn - d2;
n = d2 / es;
goto loop;
}
} else {
/* Recurse on right partition, then iterate on left partition */
if (d2 > es) {
local_qsort(pn - d2, d2 / es, es, cmp, thunk);
}
if (d1 > es) {
/* Iterate rather than recurse to save stack space */
/* qsort(a, d1 / es, es, cmp); */
n = d1 / es;
goto loop;
}
}
}
#if defined(I_AM_QSORT_R)
void
qsort_r(void *a, size_t n, size_t es, void *thunk, cmp_t *cmp)
{
local_qsort_r(a, n, es, cmp, thunk);
}
#elif defined(I_AM_QSORT_S)
errno_t
qsort_s(void *a, rsize_t n, rsize_t es, cmp_t *cmp, void *thunk)
{
if (n > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : n > RSIZE_MAX", EINVAL);
return (EINVAL);
} else if (es > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : es > RSIZE_MAX",
EINVAL);
return (EINVAL);
} else if (n != 0) {
if (a == NULL) {
__throw_constraint_handler_s("qsort_s : a == NULL",
EINVAL);
return (EINVAL);
} else if (cmp == NULL) {
__throw_constraint_handler_s("qsort_s : cmp == NULL",
EINVAL);
return (EINVAL);
}
}
local_qsort_s(a, n, es, cmp, thunk);
return (0);
}
#else
void
qsort(void *a, size_t n, size_t es, cmp_t *cmp)
{
local_qsort(a, n, es, cmp, NULL);
}
#endif