Add WITH_LLVM_BINUTILS to install LLVM binutils instead of Elftoolchain

When WITH_LLVM_BINUTILS is set, we will install the LLVM binutils as
ar/ranlib/nm/objcopy/etc. instead of the elftoolchain ones.
Having the LLVM binutils instead of the elftoolchain ones allows us to use
features such as LTO that depend on binutils that understand LLVM IR.
Another benefit will be an improved user-experience when compiling with
AddressSanitizer, since ASAN does not symbolize backtraces correctly if
addr2line is elftoolchain addr2line instead of llvm-symbolizer.
See https://lists.freebsd.org/archives/freebsd-toolchain/2021-July/000062.html
for more details.

This is currently off by default but will be turned on by default at some
point in the near future.

Reviewed By:	emaste

Differential Revision: https://reviews.freebsd.org/D31060

Makefile.inc1

@@ -771,6 +771,7 @@ XMAKE=		${BMAKE} \
 		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		MK_CLANG_IS_CC=${MK_CLANG_BOOTSTRAP} \
 		MK_LLDB=no \
+		MK_LLVM_BINUTILS=no \
 		MK_TESTS=no
 
 # kernel-tools stage
@@ -2339,9 +2340,10 @@ _gensnmptree=	usr.sbin/bsnmpd/gensnmptree
 
 # We need to build tblgen when we're building clang or lld, either as
 # bootstrap tools, or as the part of the normal build.
+# llvm-tblgen is also needed for various llvm binutils (e.g. objcopy).
 .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_CLANG} != "no" || \
     ${MK_LLD_BOOTSTRAP} != "no" || ${MK_LLD} != "no" || \
-    ${MK_LLDB} != "no"
+    ${MK_LLDB} != "no" || ${MK_LLVM_BINUTILS} != "no"
 _clang_tblgen= \
 	lib/clang/libllvmminimal \
 	usr.bin/clang/llvm-tblgen

lib/Makefile

@@ -157,7 +157,10 @@ SUBDIR.${MK_BLUETOOTH}+=libbluetooth libsdp
 SUBDIR.${MK_BSNMP}+=	libbsnmp
 
 .if !defined(COMPAT_32BIT) && !defined(COMPAT_SOFTFP)
-SUBDIR.${MK_CLANG}+=	clang
+.if ${MK_CLANG} != "no" || ${MK_LLD} != "no" || \
+    ${MK_LLDB} != "no" || ${MK_LLVM_BINUTILS} != "no"
+SUBDIR+=	clang
+.endif
 .endif
 
 SUBDIR.${MK_CUSE}+= 	libcuse

lib/clang/Makefile

@@ -4,10 +4,12 @@
 
 # These have to be built in order.
 SUBDIR=		libllvm
+.if ${MK_CLANG} != "no"
 SUBDIR+=	libclang
+SUBDIR+=	headers
+.endif
 .if ${MK_LLDB} != "no"
 SUBDIR+=	liblldb
 .endif
-SUBDIR+=	headers
 
 .include <bsd.subdir.mk>

lib/clang/libllvm/Makefile

@@ -830,7 +830,7 @@ SRCS_MIN+=	Object/IRObjectFile.cpp
 SRCS_MIN+=	Object/IRSymtab.cpp
 SRCS_MIN+=	Object/MachOObjectFile.cpp
 SRCS_MIW+=	Object/MachOUniversal.cpp
-SRCS_EXT+=	Object/MachOUniversalWriter.cpp
+SRCS_MIW+=	Object/MachOUniversalWriter.cpp
 SRCS_MIW+=	Object/Minidump.cpp
 SRCS_MIN+=	Object/ModuleSymbolTable.cpp
 SRCS_EXT+=	Object/Object.cpp
@@ -920,7 +920,7 @@ SRCS_MIN+=	Support/Errno.cpp
 SRCS_MIN+=	Support/Error.cpp
 SRCS_MIN+=	Support/ErrorHandling.cpp
 SRCS_MIN+=	Support/FileCollector.cpp
-SRCS_EXL+=	Support/FileOutputBuffer.cpp
+SRCS_MIW+=	Support/FileOutputBuffer.cpp
 SRCS_MIN+=	Support/FileUtilities.cpp
 SRCS_MIN+=	Support/FoldingSet.cpp
 SRCS_MIN+=	Support/FormatVariadic.cpp
@@ -945,7 +945,7 @@ SRCS_MIN+=	Support/MD5.cpp
 SRCS_MIN+=	Support/ManagedStatic.cpp
 SRCS_MIN+=	Support/MathExtras.cpp
 SRCS_MIN+=	Support/MemAlloc.cpp
-SRCS_XDL+=	Support/Memory.cpp
+SRCS_MIW+=	Support/Memory.cpp
 SRCS_MIN+=	Support/MemoryBuffer.cpp
 SRCS_MIN+=	Support/MemoryBufferRef.cpp
 SRCS_MIN+=	Support/NativeFormatting.cpp

share/mk/src.opts.mk

@@ -203,6 +203,7 @@ __DEFAULT_NO_OPTIONS = \
     LOADER_FIREWIRE \
     LOADER_VERBOSE \
     LOADER_VERIEXEC_PASS_MANIFEST \
+    LLVM_BINUTILS \
     MALLOC_PRODUCTION \
     OFED_EXTRA \
     OPENLDAP \
@@ -459,6 +460,7 @@ MK_CLANG:=	no
 MK_INCLUDES:=	no
 MK_LLD:=	no
 MK_LLDB:=	no
+MK_LLVM_BINUTILS:=	no
 .endif
 
 .if ${MK_CLANG} == "no"
@@ -468,6 +470,12 @@ MK_CLANG_FULL:= no
 MK_LLVM_COV:= no
 .endif
 
+.if ${MK_LLVM_BINUTILS} == "yes"
+# MK_LLVM_CXXFILT is a subset of MK_LLVM_BINUTILS and should therefore be
+# enabled if MK_LLVM_BINUTILS is set.
+MK_LLVM_CXXFILT:=	yes
+.endif
+
 .if ${MK_LOADER_VERIEXEC} == "no"
 MK_LOADER_VERIEXEC_PASS_MANIFEST := no
 .endif

tools/build/options/WITH_LLVM_BINUTILS

@@ -0,0 +1,2 @@
+.\" $FreeBSD$
+Install LLVM's binutils (ar, addr2line, nm, etc.) instead of ELF Tool Chain's.

usr.bin/Makefile

@@ -195,7 +195,10 @@ SUBDIR.${MK_ATM}+=	atm
 SUBDIR.${MK_BLUETOOTH}+=	bluetooth
 SUBDIR.${MK_BSD_CPIO}+=	cpio
 SUBDIR.${MK_CALENDAR}+=	calendar
-SUBDIR.${MK_CLANG}+=	clang
+.if ${MK_CLANG} != "no" || ${MK_LLVM_BINUTILS} != "no" || \
+    ${MK_LLD} != "no" || ${MK_LLDB} != "no"
+SUBDIR+=	clang
+.endif
 SUBDIR.${MK_DIALOG}+=	dpv
 SUBDIR.${MK_EE}+=	ee
 SUBDIR.${MK_FILE}+=	file
@@ -251,25 +254,28 @@ SUBDIR.${MK_TESTS_SUPPORT}.${MK_CXX}+=	kyua
 SUBDIR.${MK_TESTS}+=	tests
 SUBDIR.${MK_TEXTPROC}+=	ul
 SUBDIR.${MK_TFTP}+=	tftp
+.if ${MK_LLVM_BINUTILS} == "no"
+# Only build the elftoolchain tools if we aren't using the LLVM ones.
 SUBDIR.${MK_TOOLCHAIN}+=	addr2line
 SUBDIR.${MK_TOOLCHAIN}+=	ar
+SUBDIR.${MK_TOOLCHAIN}+=	nm
+SUBDIR.${MK_TOOLCHAIN}+=	objcopy
+SUBDIR.${MK_TOOLCHAIN}+=	readelf
+SUBDIR.${MK_TOOLCHAIN}+=	size
+.endif
 SUBDIR.${MK_TOOLCHAIN}+=	c89
 SUBDIR.${MK_TOOLCHAIN}+=	c99
 SUBDIR.${MK_TOOLCHAIN}+=	ctags
 .if ${MK_LLVM_CXXFILT} == "no"
 SUBDIR.${MK_TOOLCHAIN}+=	cxxfilt
 .endif
-SUBDIR.${MK_TOOLCHAIN}+=	objcopy
 SUBDIR.${MK_TOOLCHAIN}+=	file2c
 SUBDIR.${MK_TOOLCHAIN}+=	gprof
 SUBDIR.${MK_TOOLCHAIN}+=	indent
 SUBDIR.${MK_TOOLCHAIN}+=	lex
 SUBDIR.${MK_TOOLCHAIN}+=	mkstr
-SUBDIR.${MK_TOOLCHAIN}+=	nm
-SUBDIR.${MK_TOOLCHAIN}+=	readelf
 SUBDIR.${MK_TOOLCHAIN}+=	rpcgen
 SUBDIR.${MK_TOOLCHAIN}+=	unifdef
-SUBDIR.${MK_TOOLCHAIN}+=	size
 SUBDIR.${MK_TOOLCHAIN}+=	xstr
 SUBDIR.${MK_TOOLCHAIN}+=	yacc
 SUBDIR.${MK_VI}+=	vi

usr.bin/clang/Makefile

@@ -7,12 +7,20 @@ SUBDIR+=	clang
 .endif
 
 .if !defined(TOOLS_PREFIX)
+# LLVM binutils are needed to support features such as LTO, so we build them
+# by default if clang is enabled. If MK_LLVM_BINUTILS is set, we also use them
+# as the default binutils (ar,nm,addr2line, etc.).
+.if ${MK_CLANG} != "no" || ${MK_LLVM_BINUTILS} != "no"
 SUBDIR+=	llvm-ar
 SUBDIR+=	llvm-nm
+SUBDIR+=	llvm-objcopy
 SUBDIR+=	llvm-objdump
+SUBDIR+=	llvm-size
+SUBDIR+=	llvm-strings
 SUBDIR+=	llvm-symbolizer
+.endif
 
-.if ${MK_CLANG_EXTRAS} != "no" || ${MK_LLVM_CXXFILT} != "no"
+.if ${MK_LLVM_BINUTILS} != "no" || ${MK_LLVM_CXXFILT} != "no"
 SUBDIR+=	llvm-cxxfilt
 .endif
 
@@ -34,11 +42,8 @@ SUBDIR+=	llvm-lto2
 SUBDIR+=	llvm-mc
 SUBDIR+=	llvm-mca
 SUBDIR+=	llvm-modextract
-SUBDIR+=	llvm-objcopy
 SUBDIR+=	llvm-pdbutil
 SUBDIR+=	llvm-rtdyld
-SUBDIR+=	llvm-size
-SUBDIR+=	llvm-strings
 SUBDIR+=	llvm-xray
 SUBDIR+=	opt
 .endif

usr.bin/clang/llvm-ar/Makefile

@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-ar
 MAN=		llvm-ar.1 llvm-ranlib.1
 
@@ -10,4 +12,9 @@ LIBADD+=	z
 
 LINKS+=		${BINDIR}/llvm-ar ${BINDIR}/llvm-ranlib
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-ar ${BINDIR}/ar ${BINDIR}/llvm-ar ${BINDIR}/ranlib
+MLINKS=		llvm-ar.1 ar.1 llvm-ar.1 ranlib.1
+.endif
+
 .include "../llvm.prog.mk"

usr.bin/clang/llvm-nm/Makefile

@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-nm
 
 SRCDIR=		llvm/tools/llvm-nm
@@ -7,4 +9,9 @@ SRCS+=		llvm-nm.cpp
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-nm ${BINDIR}/nm
+MLINKS=		llvm-nm.1 nm.1
+.endif
+
 .include "../llvm.prog.mk"

usr.bin/clang/llvm-objcopy/Makefile

@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-objcopy
 
 SRCDIR=		llvm/tools/llvm-objcopy
@@ -42,4 +44,9 @@ CLEANFILES+=	${TGHDRS} ${TGHDRS:C/$/.d/}
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-objcopy ${BINDIR}/objcopy
+MLINKS=		llvm-objcopy.1 objcopy.1
+.endif
+
 .include "../llvm.prog.mk"

usr.bin/clang/llvm-objdump/Makefile

@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-objdump
 
 SRCDIR=		llvm/tools/llvm-objdump
@@ -12,4 +14,9 @@ SRCS+=		llvm-objdump.cpp
 
 LIBADD+=	z
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-objdump ${BINDIR}/objdump
+MLINKS=		llvm-objdump.1 objdump.1
+.endif
+
 .include "../llvm.prog.mk"

usr.bin/clang/llvm-size/Makefile

@@ -1,8 +1,15 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-size
 
 SRCDIR=		llvm/tools/llvm-size
 SRCS+=		llvm-size.cpp
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-size ${BINDIR}/size
+MLINKS=		llvm-size.1 size.1
+.endif
+
 .include "../llvm.prog.mk"

usr.bin/clang/llvm-symbolizer/Makefile

@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG_CXX=	llvm-symbolizer
 MAN=		llvm-symbolizer.1 llvm-addr2line.1
 
@@ -26,4 +28,9 @@ LIBADD+=	z
 
 LINKS+=		${BINDIR}/llvm-symbolizer ${BINDIR}/llvm-addr2line
 
+.if ${MK_LLVM_BINUTILS} != "no"
+LINKS=		${BINDIR}/llvm-symbolizer ${BINDIR}/addr2line
+MLINKS=		llvm-addr2line.1 addr2line.1
+.endif
+
 .include "../llvm.prog.mk"