From b2e843161dc3b79777e873183447c92ed9c3703a Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Tue, 2 Nov 2021 11:31:17 +0000 Subject: [PATCH] Use a builtin where possible in msun Some of the functions in msun can be implemented using a compiler builtin function to generate a small number of instructions. Implement this support in fma, fmax, fmin, and sqrt on arm64. Care must be taken as the builtin can be implemented as a function call on some architectures that lack direct support. In these cases we need to use the original code path. As we don't set errno on failure build with -fno-math-errno so the toolchain doesn't convert a builtin into a function call when it detects a failure, e.g. gcc will add a call to sqrt when the input is negative leading to an infinite loop. Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D32801 --- lib/msun/Makefile | 5 +++++ lib/msun/aarch64/Makefile.inc | 12 ++++++++++++ lib/msun/src/e_sqrt.c | 18 +++++++++++++----- lib/msun/src/e_sqrtf.c | 8 ++++++++ lib/msun/src/s_fma.c | 8 ++++++++ lib/msun/src/s_fmaf.c | 8 ++++++++ lib/msun/src/s_fmax.c | 8 ++++++++ lib/msun/src/s_fmaxf.c | 8 ++++++++ lib/msun/src/s_fmin.c | 8 ++++++++ lib/msun/src/s_fminf.c | 8 ++++++++ 10 files changed, 86 insertions(+), 5 deletions(-) diff --git a/lib/msun/Makefile b/lib/msun/Makefile index d7c0e2f88358..6274c27c12c5 100644 --- a/lib/msun/Makefile +++ b/lib/msun/Makefile @@ -45,6 +45,11 @@ CFLAGS+= -I${.CURDIR}/${ARCH_SUBDIR} CFLAGS+= -ffp-exception-behavior=maytrap .endif +# Tell the compiler we don't set errno in any of the math functions. This +# stops gcc from trying to generate a function call to set errno when using +# a builtin in the implementation +CFLAGS+= -fno-math-errno + .PATH: ${.CURDIR}/bsdsrc .PATH: ${.CURDIR}/src .PATH: ${.CURDIR}/man diff --git a/lib/msun/aarch64/Makefile.inc b/lib/msun/aarch64/Makefile.inc index 286a60859591..b0e01488b0bb 100644 --- a/lib/msun/aarch64/Makefile.inc +++ b/lib/msun/aarch64/Makefile.inc @@ -2,3 +2,15 @@ LDBL_PREC = 113 +# Use a builtin when it generates the needed instruction +CFLAGS+=-DUSE_BUILTIN_FMAF +CFLAGS+=-DUSE_BUILTIN_FMA + +CFLAGS+=-DUSE_BUILTIN_FMAXF +CFLAGS+=-DUSE_BUILTIN_FMAX + +CFLAGS+=-DUSE_BUILTIN_FMINF +CFLAGS+=-DUSE_BUILTIN_FMIN + +CFLAGS+=-DUSE_BUILTIN_SQRTF +CFLAGS+=-DUSE_BUILTIN_SQRT diff --git a/lib/msun/src/e_sqrt.c b/lib/msun/src/e_sqrt.c index 12fb56e1340b..37351a425ce4 100644 --- a/lib/msun/src/e_sqrt.c +++ b/lib/msun/src/e_sqrt.c @@ -14,6 +14,18 @@ #include __FBSDID("$FreeBSD$"); +#include + +#include "math.h" +#include "math_private.h" + +#ifdef USE_BUILTIN_SQRT +double +__ieee754_sqrt(double x) +{ + return (__builtin_sqrt(x)); +} +#else /* __ieee754_sqrt(x) * Return correctly rounded sqrt. * ------------------------------------------ @@ -84,11 +96,6 @@ __FBSDID("$FreeBSD$"); *--------------- */ -#include - -#include "math.h" -#include "math_private.h" - static const double one = 1.0, tiny=1.0e-300; double @@ -187,6 +194,7 @@ __ieee754_sqrt(double x) INSERT_WORDS(z,ix0,ix1); return z; } +#endif #if (LDBL_MANT_DIG == 53) __weak_reference(sqrt, sqrtl); diff --git a/lib/msun/src/e_sqrtf.c b/lib/msun/src/e_sqrtf.c index 7eba4d07f92c..06e5d62fa5f9 100644 --- a/lib/msun/src/e_sqrtf.c +++ b/lib/msun/src/e_sqrtf.c @@ -20,6 +20,13 @@ static char rcsid[] = "$FreeBSD$"; #include "math.h" #include "math_private.h" +#ifdef USE_BUILTIN_SQRTF +float +__ieee754_sqrtf(float x) +{ + return (__builtin_sqrtf(x)); +} +#else static const float one = 1.0, tiny=1.0e-30; float @@ -87,3 +94,4 @@ __ieee754_sqrtf(float x) SET_FLOAT_WORD(z,ix); return z; } +#endif diff --git a/lib/msun/src/s_fma.c b/lib/msun/src/s_fma.c index 41a6424fdf7f..95cffd0ba36d 100644 --- a/lib/msun/src/s_fma.c +++ b/lib/msun/src/s_fma.c @@ -35,6 +35,13 @@ __FBSDID("$FreeBSD$"); #include "math_private.h" +#ifdef USE_BUILTIN_FMA +double +fma(double x, double y, double z) +{ + return (__builtin_fma(x, y, z)); +} +#else /* * A struct dd represents a floating-point number with twice the precision * of a double. We maintain the invariant that "hi" stores the 53 high-order @@ -284,6 +291,7 @@ fma(double x, double y, double z) else return (add_and_denormalize(r.hi, adj, spread)); } +#endif /* !USE_BUILTIN_FMA */ #if (LDBL_MANT_DIG == 53) __weak_reference(fma, fmal); diff --git a/lib/msun/src/s_fmaf.c b/lib/msun/src/s_fmaf.c index 389cf1b11bee..4591cc219bbb 100644 --- a/lib/msun/src/s_fmaf.c +++ b/lib/msun/src/s_fmaf.c @@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +#ifdef USE_BUILTIN_FMAF +float +fmaf(float x, float y, float z) +{ + return (__builtin_fmaf(x, y, z)); +} +#else /* * Fused multiply-add: Compute x * y + z with a single rounding error. * @@ -69,3 +76,4 @@ fmaf(float x, float y, float z) SET_LOW_WORD(adjusted_result, lr + 1); return (adjusted_result); } +#endif /* !USE_BUILTIN_FMAF */ diff --git a/lib/msun/src/s_fmax.c b/lib/msun/src/s_fmax.c index 0c234bcc5f7f..b53b1e64da04 100644 --- a/lib/msun/src/s_fmax.c +++ b/lib/msun/src/s_fmax.c @@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" +#ifdef USE_BUILTIN_FMAX +double +fmax(double x, double y) +{ + return (__builtin_fmax(x, y)); +} +#else double fmax(double x, double y) { @@ -54,6 +61,7 @@ fmax(double x, double y) return (x > y ? x : y); } +#endif #if (LDBL_MANT_DIG == 53) __weak_reference(fmax, fmaxl); diff --git a/lib/msun/src/s_fmaxf.c b/lib/msun/src/s_fmaxf.c index 8e9d1baf3e8e..8d3d14f4b52a 100644 --- a/lib/msun/src/s_fmaxf.c +++ b/lib/msun/src/s_fmaxf.c @@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" +#ifdef USE_BUILTIN_FMAXF +float +fmaxf(float x, float y) +{ + return (__builtin_fmaxf(x, y)); +} +#else float fmaxf(float x, float y) { @@ -53,3 +60,4 @@ fmaxf(float x, float y) return (x > y ? x : y); } +#endif diff --git a/lib/msun/src/s_fmin.c b/lib/msun/src/s_fmin.c index d7f24c121fb2..53f36c1c2d3d 100644 --- a/lib/msun/src/s_fmin.c +++ b/lib/msun/src/s_fmin.c @@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" +#ifdef USE_BUILTIN_FMIN +double +fmin(double x, double y) +{ + return (__builtin_fmin(x, y)); +} +#else double fmin(double x, double y) { @@ -54,6 +61,7 @@ fmin(double x, double y) return (x < y ? x : y); } +#endif #if (LDBL_MANT_DIG == 53) __weak_reference(fmin, fminl); diff --git a/lib/msun/src/s_fminf.c b/lib/msun/src/s_fminf.c index 258316725ddd..58b6a489b820 100644 --- a/lib/msun/src/s_fminf.c +++ b/lib/msun/src/s_fminf.c @@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" +#ifdef USE_BUILTIN_FMINF +float +fminf(float x, float y) +{ + return (__builtin_fminf(x, y)); +} +#else float fminf(float x, float y) { @@ -53,3 +60,4 @@ fminf(float x, float y) return (x < y ? x : y); } +#endif