Use a builtin where possible in msun

Some of the functions in msun can be implemented using a compiler
builtin function to generate a small number of instructions. Implement
this support in fma, fmax, fmin, and sqrt on arm64.

Care must be taken as the builtin can be implemented as a function
call on some architectures that lack direct support. In these cases
we need to use the original code path.

As we don't set errno on failure build with -fno-math-errno so the
toolchain doesn't convert a builtin into a function call when it
detects a failure, e.g. gcc will add a call to sqrt when the input
is negative leading to an infinite loop.

Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D32801

lib/msun/Makefile

@@ -45,6 +45,11 @@ CFLAGS+=	-I${.CURDIR}/${ARCH_SUBDIR}
 CFLAGS+=	-ffp-exception-behavior=maytrap
 .endif
 
+# Tell the compiler we don't set errno in any of the math functions. This
+# stops gcc from trying to generate a function call to set errno when using
+# a builtin in the implementation
+CFLAGS+=	-fno-math-errno
+
 .PATH:	${.CURDIR}/bsdsrc
 .PATH:	${.CURDIR}/src
 .PATH:	${.CURDIR}/man

lib/msun/aarch64/Makefile.inc

@@ -2,3 +2,15 @@
 
 LDBL_PREC = 113
 
+# Use a builtin when it generates the needed instruction
+CFLAGS+=-DUSE_BUILTIN_FMAF
+CFLAGS+=-DUSE_BUILTIN_FMA
+
+CFLAGS+=-DUSE_BUILTIN_FMAXF
+CFLAGS+=-DUSE_BUILTIN_FMAX
+
+CFLAGS+=-DUSE_BUILTIN_FMINF
+CFLAGS+=-DUSE_BUILTIN_FMIN
+
+CFLAGS+=-DUSE_BUILTIN_SQRTF
+CFLAGS+=-DUSE_BUILTIN_SQRT

lib/msun/src/e_sqrt.c

@@ -14,6 +14,18 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <float.h>
+
+#include "math.h"
+#include "math_private.h"
+
+#ifdef USE_BUILTIN_SQRT
+double
+__ieee754_sqrt(double x)
+{
+	return (__builtin_sqrt(x));
+}
+#else
 /* __ieee754_sqrt(x)
  * Return correctly rounded sqrt.
  *           ------------------------------------------
@@ -84,11 +96,6 @@ __FBSDID("$FreeBSD$");
  *---------------
  */
 
-#include <float.h>
-
-#include "math.h"
-#include "math_private.h"
-
 static	const double	one	= 1.0, tiny=1.0e-300;
 
 double
@@ -187,6 +194,7 @@ __ieee754_sqrt(double x)
 	INSERT_WORDS(z,ix0,ix1);
 	return z;
 }
+#endif
 
 #if (LDBL_MANT_DIG == 53)
 __weak_reference(sqrt, sqrtl);

lib/msun/src/e_sqrtf.c

@@ -20,6 +20,13 @@ static char rcsid[] = "$FreeBSD$";
 #include "math.h"
 #include "math_private.h"
 
+#ifdef USE_BUILTIN_SQRTF
+float
+__ieee754_sqrtf(float x)
+{
+	return (__builtin_sqrtf(x));
+}
+#else
 static	const float	one	= 1.0, tiny=1.0e-30;
 
 float
@@ -87,3 +94,4 @@ __ieee754_sqrtf(float x)
 	SET_FLOAT_WORD(z,ix);
 	return z;
 }
+#endif

lib/msun/src/s_fma.c

@@ -35,6 +35,13 @@ __FBSDID("$FreeBSD$");
 
 #include "math_private.h"
 
+#ifdef USE_BUILTIN_FMA
+double
+fma(double x, double y, double z)
+{
+	return (__builtin_fma(x, y, z));
+}
+#else
 /*
  * A struct dd represents a floating-point number with twice the precision
  * of a double.  We maintain the invariant that "hi" stores the 53 high-order
@@ -284,6 +291,7 @@ fma(double x, double y, double z)
 	else
 		return (add_and_denormalize(r.hi, adj, spread));
 }
+#endif /* !USE_BUILTIN_FMA */
 
 #if (LDBL_MANT_DIG == 53)
 __weak_reference(fma, fmal);

lib/msun/src/s_fmaf.c

@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
 #include "math.h"
 #include "math_private.h"
 
+#ifdef USE_BUILTIN_FMAF
+float
+fmaf(float x, float y, float z)
+{
+	return (__builtin_fmaf(x, y, z));
+}
+#else
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
@@ -69,3 +76,4 @@ fmaf(float x, float y, float z)
 		SET_LOW_WORD(adjusted_result, lr + 1);
 	return (adjusted_result);
 }
+#endif /* !USE_BUILTIN_FMAF */

lib/msun/src/s_fmax.c

@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
 
 #include "fpmath.h"
 
+#ifdef USE_BUILTIN_FMAX
+double
+fmax(double x, double y)
+{
+	return (__builtin_fmax(x, y));
+}
+#else
 double
 fmax(double x, double y)
 {
@@ -54,6 +61,7 @@ fmax(double x, double y)
 
 	return (x > y ? x : y);
 }
+#endif
 
 #if (LDBL_MANT_DIG == 53)
 __weak_reference(fmax, fmaxl);

lib/msun/src/s_fmaxf.c

@@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$");
 
 #include "fpmath.h"
 
+#ifdef USE_BUILTIN_FMAXF
+float
+fmaxf(float x, float y)
+{
+	return (__builtin_fmaxf(x, y));
+}
+#else
 float
 fmaxf(float x, float y)
 {
@@ -53,3 +60,4 @@ fmaxf(float x, float y)
 
 	return (x > y ? x : y);
 }
+#endif

lib/msun/src/s_fmin.c

@@ -34,6 +34,13 @@ __FBSDID("$FreeBSD$");
 
 #include "fpmath.h"
 
+#ifdef USE_BUILTIN_FMIN
+double
+fmin(double x, double y)
+{
+	return (__builtin_fmin(x, y));
+}
+#else
 double
 fmin(double x, double y)
 {
@@ -54,6 +61,7 @@ fmin(double x, double y)
 
 	return (x < y ? x : y);
 }
+#endif
 
 #if (LDBL_MANT_DIG == 53)
 __weak_reference(fmin, fminl);

lib/msun/src/s_fminf.c

@@ -33,6 +33,13 @@ __FBSDID("$FreeBSD$");
 
 #include "fpmath.h"
 
+#ifdef USE_BUILTIN_FMINF
+float
+fminf(float x, float y)
+{
+	return (__builtin_fminf(x, y));
+}
+#else
 float
 fminf(float x, float y)
 {
@@ -53,3 +60,4 @@ fminf(float x, float y)
 
 	return (x < y ? x : y);
 }
+#endif