Simplify running the FreeBSD test suite

Replace `make regress` (legacy test make target) and `make test` (incomplete
test make target added with the FreeBSD test suite) with make check as it's
consistent with other open source projects.

`make check` defaults to running tests from `.OBJDIR`, but can be overridden
with the `CHECKDIR` variable.

Add `make checkworld` target to simplify running the FreeBSD test suite from
`TESTSBASE` (i.e. the top-level tests directory), similar to buildworld.

Document `make check` and `make checkworld` in build(7).

Other minor changes:

- Rename intermediate file (`Kyuafile.auto`) to `Kyuafile` to simplify
  `make check`.
- Remove terse warnings attached to `beforetest`/`aftertest`.
- Add kyua binary check to check target in suite.test.mk; error out if it's
  not found

The MFC is [partly] contingent on other build related changes being MFCed.

Differential Revision: https://reviews.freebsd.org/D4406
MFC after: 2 months
X-MFC to: stable/10
Relnotes: yes
Reviewed by: bdrewery, Evan Cramer <eccramer@gmail.com>
Sponsored by: EMC / Isilon Storage Division

Makefile

@@ -21,6 +21,7 @@
 # kernel-toolchains   - Build kernel-toolchain for all universe targets.
 # doxygen             - Build API documentation of the kernel, needs doxygen.
 # update              - Convenient way to update your source tree(s).
+# checkworld          - Run test suite on installed world.
 # check-old           - List obsolete directories/files/libraries.
 # check-old-dirs      - List obsolete directories.
 # check-old-files     - List obsolete files.
@@ -112,8 +113,8 @@
 .else
 
 TGTS=	all all-man buildenv buildenvvars buildkernel buildworld \
-	check-old check-old-dirs check-old-files check-old-libs \
+	check check-old check-old-dirs check-old-files check-old-libs \
-	checkdpadd clean cleandepend cleandir cleanworld \
+	checkdpadd checkworld clean cleandepend cleandir cleanworld \
 	delete-old delete-old-dirs delete-old-files delete-old-libs \
 	depend distribute distributekernel distributekernel.debug \
 	distributeworld distrib-dirs distribution doxygen \
@@ -121,7 +122,7 @@ TGTS=	all all-man buildenv buildenvvars buildkernel buildworld \
 	installkernel.debug packagekernel packageworld \
 	reinstallkernel reinstallkernel.debug \
 	installworld kernel-toolchain libraries lint maninstall \
-	obj objlink regress rerelease showconfig tags toolchain update \
+	obj objlink rerelease showconfig tags toolchain update \
 	_worldtmp _legacy _bootstrap-tools _cleanobj _obj \
 	_build-tools _cross-tools _includes _libraries _depend \
 	build32 builddtb distribute32 install32 xdev xdev-build xdev-install \
@@ -329,6 +330,10 @@ bmake: .PHONY
 		${MMAKE} all; \
 		${MMAKE} install DESTDIR=${MYMAKE:H} BINDIR=
 
+regress: .PHONY
+	@echo "'make regress' has been renamed 'make check'" | /usr/bin/fmt
+	@false
+
 tinderbox toolchains kernel-toolchains kernels worlds: upgrade_checks
 
 tinderbox:

Makefile.inc1

@@ -37,6 +37,7 @@
 # The intended user-driven targets are:
 # buildworld  - rebuild *everything*, including glue to help do upgrades
 # installworld- install everything built by "buildworld"
+# checkworld  - run test suite on installed world
 # doxygen     - build API documentation of the kernel
 # update      - convenient way to update your source tree (eg: svn/svnup)
 #
@@ -1322,6 +1323,20 @@ packagekernel:
 .endif
 .endif
 
+#
+#
+# checkworld
+#
+# Run test suite on installed world.
+#
+checkworld: .PHONY
+	@if [ ! -x ${LOCALBASE}/bin/kyua ]; then \
+		echo "You need kyua (devel/kyua) to run the test suite." | /usr/bin/fmt; \
+		exit 1; \
+	fi
+	${_+_}${LOCALBASE}/bin/kyua test -k ${TESTSBASE}/Kyuafile
+
+#
 #
 # doxygen
 #

share/man/man7/build.7

@@ -107,6 +107,16 @@ section below, and by the variables documented in
 The following list provides the names and actions for the targets
 supported by the build system:
 .Bl -tag -width ".Cm cleandepend"
+.It Cm check
+Run tests for a given subdirectory.
+The default directory used is
+.Pa ${.OBJDIR} ,
+but the check directory can be changed with
+.Pa ${CHECKDIR} .
+.It Cm checkworld
+Run the
+.Fx
+test suite on installed world.
 .It Cm clean
 Remove any files created during the build process.
 .It Cm cleandepend
@@ -653,6 +663,7 @@ make TARGET=sparc64 DESTDIR=/clients/sparc64 installworld
 .Xr mergemaster 8 ,
 .Xr portsnap 8 ,
 .Xr reboot 8 ,
-.Xr shutdown 8
+.Xr shutdown 8 ,
+.Xr tests 7
 .Sh AUTHORS
 .An Mike W. Meyer Aq Mt mwm@mired.org

share/mk/bsd.README

@@ -448,6 +448,17 @@ It has seven targets:
 
 	all:
 		build the test programs.
+	check:
+		runs the test programs from CHECKDIR with kyua test.
+
+		The beforecheck and aftercheck targets will be invoked, if
+		defined, to execute commands before and after the realcheck
+		target has been executed, respectively.
+
+		The devel/kyua package must be installed before invoking this
+		target.
+
+		See CHECKDIR for more details.
 	clean:
 		remove the test programs and any object files.
 	cleandir:
@@ -466,12 +477,6 @@ It has seven targets:
 		run lint on the source files.
 	tags:
 		create a tags file for the source files.
-	test:
-		runs the test programs from the object directory; if the
-		Makefile does not itself define the target test, the
-		targets beforetest and aftertest may also be used to
-		cause actions immediately before and after the test
-		target is executed.
 
 It sets/uses the following variables, among many others:
 
@@ -485,6 +490,10 @@ TESTSDIR	Path to the installed tests.  Must be a subdirectory of
 		${TESTSBASE}/${RELDIR:H} , e.g. /usr/tests/bin/ls when
 		included from bin/ls/tests .
 
+CHECKDIR	The directory that 'make check' executes tests from.
+
+		The value of CHECKDIR defaults to .OBJDIR.
+
 KYUAFILE	If 'auto' (the default), generate a Kyuafile out of the
 		test programs defined in the Makefile.  If 'yes', then a
 		manually-crafted Kyuafile must be supplied with the

share/mk/bsd.subdir.mk

@@ -43,11 +43,11 @@ SUBDIR_TARGETS+= \
 		checkdpadd clean cleandepend cleandir cleanilinks \
 		cleanobj depend distribute files includes installconfig \
 		installfiles installincludes realinstall lint maninstall \
-		manlint obj objlink regress tags \
+		manlint obj objlink tags \
 
 # Described above.
 STANDALONE_SUBDIR_TARGETS+= \
-		obj checkdpadd clean cleandepend cleandir \
+		obj check checkdpadd clean cleandepend cleandir \
 		cleanilinks cleanobj installconfig \
 
 .include <bsd.init.mk>

share/mk/bsd.sys.mk

@@ -178,11 +178,11 @@ CXXFLAGS+=	 ${CXXFLAGS.${COMPILER_TYPE}}
 # or expect to ever be up-to-date.
 PHONY_NOTMAIN = afterdepend afterinstall all beforedepend beforeinstall \
 		beforelinking build build-tools buildconfig buildfiles \
-		buildincludes checkdpadd clean cleandepend cleandir cleanobj \
+		buildincludes check checkdpadd clean cleandepend cleandir \
-		configure depend dependall distclean distribute exe \
+		cleanobj configure depend dependall distclean distribute exe \
 		files html includes install installconfig installfiles \
 		installincludes lint obj objlink objs objwarn realall \
-		realdepend realinstall regress subdir-all subdir-depend \
+		realdepend realinstall subdir-all subdir-depend \
 		subdir-install tags whereobj
 
 # we don't want ${PROG} to be PHONY

share/mk/bsd.test.mk

@@ -60,11 +60,15 @@ _TESTS=
 .include <plain.test.mk>
 .include <tap.test.mk>
 
+# kyua automatically descends directories; only run make check on the
+# top-level directory
+.if !make(check)
 .for ts in ${TESTS_SUBDIRS}
 .if empty(SUBDIR:M${ts})
 SUBDIR+= ${ts}
 .endif
 .endfor
+.endif
 
 # it is rare for test cases to have man pages
 .if !defined(MAN)
@@ -79,19 +83,14 @@ PROGS_TARGETS+= install
 .include <suite.test.mk>
 .endif
 
-.if !target(realtest)
+.if !target(realcheck)
-realtest: .PHONY
+realcheck: .PHONY
 	@echo "$@ not defined; skipping"
 .endif
 
-test: .PHONY
+beforecheck realcheck aftercheck check: .PHONY
-.ORDER: beforetest realtest
+.ORDER: beforecheck realcheck aftercheck
-test: beforetest realtest
+check: beforecheck realcheck aftercheck
-
-.if target(aftertest)
-.ORDER: realtest aftertest
-test: aftertest
-.endif
 
 .ifdef PROG
 # we came here via bsd.progs.mk below

share/mk/suite.test.mk

@@ -50,15 +50,12 @@ FILES+=	Kyuafile
 FILESDIR_Kyuafile= ${TESTSDIR}
 .endif
 
-.if ${KYUAFILE:tl} == "auto"
-CLEANFILES+= Kyuafile Kyuafile.tmp
-.endif
-
 .for _T in ${_TESTS}
 _TEST_METADATA.${_T}= ${TEST_METADATA} ${TEST_METADATA.${_T}}
 .endfor
 
 .if ${KYUAFILE:tl} == "auto"
+CLEANFILES+= Kyuafile Kyuafile.tmp
 Kyuafile: Makefile
 	@{ \
 	    echo '-- Automatically generated by bsd.test.mk.'; \
@@ -78,9 +75,11 @@ Kyuafile: Makefile
 	@mv ${.TARGET}.tmp ${.TARGET}
 .endif
 
+CHECKDIR?=	${DESTDIR}${TESTSDIR}
+
 KYUA= ${LOCALBASE}/bin/kyua
-.if exists(${KYUA})
+
-# Definition of the "make test" target and supporting variables.
+# Definition of the "make check" target and supporting variables.
 #
 # This target, by necessity, can only work for native builds (i.e. a FreeBSD
 # host building a release for the same system).  The target runs Kyua, which is
@@ -89,35 +88,15 @@ KYUA= ${LOCALBASE}/bin/kyua
 # Due to the dependencies of the binaries built by the source tree and how they
 # are used by tests, it is highly possible for a execution of "make test" to
 # report bogus results unless the new binaries are put in place.
-realtest: .PHONY
-	@echo "*** WARNING: make test is experimental"
-	@echo "***"
-	@echo "*** Using this test does not preclude you from running the tests"
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
-	@echo "*** positives and/or false negatives."
-	@echo
-	@${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \
-	result=0; \
-	echo; \
-	echo "*** Once again, note that "make test" is unsupported."; \
-	test $${result} -eq 0
-.endif
 
-beforetest: .PHONY
+realcheck: .PHONY
-.if defined(TESTSDIR)
+	@if [ ! -x ${KYUA} ]; then \
-.if ${TESTSDIR} == ${TESTSBASE}
+		echo; \
-# Forbid running from ${TESTSBASE}.  It can cause false positives/negatives and
+		echo "kyua binary not installed at expected location (${.TARGET})"; \
-# it does not cover all the tests (e.g. it misses testing software in external).
+		echo; \
-	@echo "*** Sorry, you cannot use make test from src/tests.  Install the"
+		echo "Please install via pkg install, or specify the path to the kyua"; \
-	@echo "*** tests into their final location and run them from ${TESTSBASE}"
+		echo "package via the \$${LOCALBASE} variable, e.g. "; \
-	@false
+		echo "LOCALBASE=\"${LOCALBASE}\""; \
-.else
+		false; \
-	@echo "*** Using this test does not preclude you from running the tests"
+	fi
-	@echo "*** installed in ${TESTSBASE}.  This test run may raise false"
+	@${KYUA} test -k ${CHECKDIR}/Kyuafile
-	@echo "*** positives and/or false negatives."
-.endif
-.else
-	@echo "*** No TESTSDIR defined; nothing to do."
-	@false
-.endif
-	@echo