From 0e1e341b486cdf4769195ba1e5b3cb32e7387873 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Thu, 17 Sep 2020 15:07:25 +0000 Subject: [PATCH] Stop using lorder and ranlib when building libraries Use of ranlib or lorder is no longer necessary with current linkers (probably anything newer than ~1990) and ar's ability to create an object index and symbol table in the archive. Currently the build system uses lorder+tsort to sort the .o files in dependency order so that a single-pass linker can use them. However, we can use the -s flag to ar to add an index to the .a file which makes lorder unnecessary. Running ar -s is equivalent to running ranlib afterwards, so we can also skip the ranlib invocation. Similarly, we don't have to pass the .o files for shared libraries in dependency order since both ld.bfd and ld.lld will correctly resolve references between the .o files. This removes many fork()+execve calls for each library so should speed up builds a bit. Additionally lorder.sh uses a regular expression that is not supported by the macOS libc or glibc and results in many warnings when cross-building (see D25989). There is one functional change: lorder.sh removed duplicated .o files from the linker command line which now no longer happens. I fixed the duplicates in the base system in r364649. I also checked the ports tree for uses of bsd.lib.mk and found one duplicate source file which I fixed in r548168. Most ports use CMake/autotools rather than bsd.lib.mk but if this breaks any ports that I missed in my search please let me know. Avoiding the shell script actually speeds up the linking step noticeably: I measured how long it takes to rebuild the .a and .so files for lib/libc using a basic benchmark: `rm $LIBC_OBJDIR/*.so* $LIBC_OBJDIR/*.a* && /usr/bin/time make -DWITHOUT_TESTS -s > /dev/null` Without this change ~4.5 seconds and afterwards ~3.1 seconds. Looking at truss -cf output we can see that the number fork() system calls goes down from 27 to 12 (and the speedup while tracing is more noticeable: 81 seconds -> 65 seconds). See also https://www.gnu.org/software/coreutils/manual/html_node/tsort-background.html for some more background: This whole procedure has been obsolete since about 1980, because Unix archives now contain a symbol table (traditionally built by ranlib, now generally built by ar itself), and the Unix linker uses the symbol table to effectively make multiple passes over an archive file. Or alternatively https://www.unix.com/man-page/osf1/1/lorder/: The lorder command is essentially obsolete. Use the following command in its place: % ar -ts file.a Reviewed By: emaste, imp, dim Differential Revision: https://reviews.freebsd.org/D26044 --- share/mk/bsd.lib.mk | 15 +++------------ share/mk/sys.mk | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/share/mk/bsd.lib.mk b/share/mk/bsd.lib.mk index 484663b8d55d..f1bc3bf836d7 100644 --- a/share/mk/bsd.lib.mk +++ b/share/mk/bsd.lib.mk @@ -245,9 +245,7 @@ _LIBS= lib${LIB_PRIVATE}${LIB}.a lib${LIB_PRIVATE}${LIB}.a: ${OBJS} ${STATICOBJS} @${ECHO} building static ${LIB} library @rm -f ${.TARGET} - ${AR} ${ARFLAGS} ${.TARGET} `NM='${NM}' NMFLAGS='${NMFLAGS}' \ - ${LORDER} ${OBJS} ${STATICOBJS} | ${TSORT} ${TSORTFLAGS}` ${ARADD} - ${RANLIB} ${RANLIBFLAGS} ${.TARGET} + ${AR} ${ARFLAGS} ${.TARGET} ${OBJS} ${STATICOBJS} ${ARADD} .endif .if !defined(INTERNALLIB) @@ -261,9 +259,7 @@ CLEANFILES+= ${POBJS} lib${LIB_PRIVATE}${LIB}_p.a: ${POBJS} @${ECHO} building profiled ${LIB} library @rm -f ${.TARGET} - ${AR} ${ARFLAGS} ${.TARGET} `NM='${NM}' NMFLAGS='${NMFLAGS}' \ - ${LORDER} ${POBJS} | ${TSORT} ${TSORTFLAGS}` ${ARADD} - ${RANLIB} ${RANLIBFLAGS} ${.TARGET} + ${AR} ${ARFLAGS} ${.TARGET} ${POBJS} ${ARADD} .endif .if defined(LLVM_LINK) @@ -319,9 +315,7 @@ ${SHLIB_NAME_FULL}: ${SOBJS} @${INSTALL_LIBSYMLINK} ${TAG_ARGS:D${TAG_ARGS},dev} ${SHLIB_NAME} ${SHLIB_LINK} .endif ${_LD:N${CCACHE_BIN}} ${LDFLAGS} ${SSP_CFLAGS} ${SOLINKOPTS} \ - -o ${.TARGET} -Wl,-soname,${SONAME} \ - `NM='${NM}' NMFLAGS='${NMFLAGS}' ${LORDER} ${SOBJS} | \ - ${TSORT} ${TSORTFLAGS}` ${LDADD} + -o ${.TARGET} -Wl,-soname,${SONAME} ${SOBJS} ${LDADD} .if ${MK_CTF} != "no" ${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ${SOBJS} .endif @@ -347,7 +341,6 @@ lib${LIB_PRIVATE}${LIB}_pic.a: ${SOBJS} @${ECHO} building special pic ${LIB} library @rm -f ${.TARGET} ${AR} ${ARFLAGS} ${.TARGET} ${SOBJS} ${ARADD} - ${RANLIB} ${RANLIBFLAGS} ${.TARGET} .endif .if defined(BUILD_NOSSP_PIC_ARCHIVE) && defined(LIB) && !empty(LIB) @@ -360,7 +353,6 @@ lib${LIB_PRIVATE}${LIB}_nossp_pic.a: ${NOSSPSOBJS} @${ECHO} building special nossp pic ${LIB} library @rm -f ${.TARGET} ${AR} ${ARFLAGS} ${.TARGET} ${NOSSPSOBJS} ${ARADD} - ${RANLIB} ${RANLIBFLAGS} ${.TARGET} .endif .endif # !defined(INTERNALLIB) @@ -376,7 +368,6 @@ lib${LIB_PRIVATE}${LIB}_pie.a: ${PIEOBJS} @${ECHO} building pie ${LIB} library @rm -f ${.TARGET} ${AR} ${ARFLAGS} ${.TARGET} ${PIEOBJS} ${ARADD} - ${RANLIB} ${RANLIBFLAGS} ${.TARGET} .endif .if defined(_SKIP_BUILD) diff --git a/share/mk/sys.mk b/share/mk/sys.mk index 215b04563e23..f064c7eda41d 100644 --- a/share/mk/sys.mk +++ b/share/mk/sys.mk @@ -150,7 +150,7 @@ AR ?= ar .if defined(%POSIX) ARFLAGS ?= -rv .else -ARFLAGS ?= -crD +ARFLAGS ?= -crsD .endif RANLIB ?= ranlib .if !defined(%POSIX)