Change the test infrastructure so that tests are easier to maintain and

so that make(1) will run in an almost clean environment and enhance the
description of the test infrastructure.

Add the ability to have multiple tests carried out per test script.

Give some tests more meaningful names.

Fix the usage message from the test scripts.

Make it possible to pass several commands to the test scripts like:
'sh test.t setup run compare clean'.

tools/regression/usr.bin/make/README

@@ -2,28 +2,68 @@ $FreeBSD$
 
 This directory contains regression tests for make(1).
 
-The tests are invoked via the test.sh script or prove(1) from p5-Test-Harness
+To invoke the tests install prove(1) from ports/devel/p5-Test-Harness and
+run 'prove -r'. Alternatively one can use 'sh ./all.sh test' and scan the
+output for '^not ok'.
 
-	% test.sh [-v] [-m path_to_make_binary] command
+----------------------------------------------------------------------------
 
-	clean	- Remove the results and the other temp files that
-		  are produced by running the test. This brings the
-		  test into its initial state.
+The rest of this file is intented for developers.
 
-	compare	- Check if results of the test match the expected
-		  output from stdout, stderr, and the status.
+The tests are invoked via the test.sh script or prove(1) from p5-Test-Harness.
+Tests are normally executed in a special test directory that is built under
+/tmp. The reason for this is, that make tests are generally influenced by
+all file in a directory, by files in one of make's obscure object directories
+as well as in other directories make happens to look into. Therefor the
+test scripts build a clean environment in the temp directory and the
+tests are executed by cd-ing into that directory and invoking make. The
+output of the make run (standard output, standard error and the exit status)
+are written into files that are created in another directory. So the layout
+for the shell/builtin test looks like:
 
-	desc	- print description of test
+	./shell/builtin/			- directory with test stuff
+	/tmp/make.${USER}/shell/builtin		- actual test directory
+	/tmp/make.${USER}/shell/builtin.OUTPUT	- output files
 
-	diff	- Output the diffs from the tests and the expected
-		  stdout, stderr, and the status files.
+So a full test consists of the following steps:
 
-	run	- Invoke test, compare, and clean in sequence.
+	setup	- Set up the test environment by creating the test directory
+		  and populating it with the needed files. If the test
+		  directory already exists an error is printed.
 
-	test	- Invoke the test code
+	run	- Run the test and produce the output into the output
+		  directory.
 
-	update	- Copy the output of the last test run as the expected
-		  output from stdout, stderr, and the status.
+	show	- Show the result files on the screen.
+
+	compare	- Compare the results in the output directory with those
+		  in the test source directory. This just prints whether
+		  the test was ok or not in the format used by prove(1).
+
+	diff	- Diff the output files and the expected output files.
+
+	reset	- Reset the test to its initial state.
+
+	clean	- Remove both the test directory and the output directory.
+
+Each of these steps can independently be invoked with the test script
+contained in each directory. These test scripts are called test.t.
+Additionally the scripts understand the following commands:
+
+	test	- Run setup, run and compare.
+
+	prove	- Run setup, run, compare and clean. This is identically
+		  to invoking the script without an argument.
+
+	desc	- Print a short test description.
+
+	update	- Update the expected results from the actual results.
+
+The test script has the following syntax:
+
+	% test.t [-v] [-m path_to_make_binary] command
+
+To invoke it via prove(1) use:
 
 	% [MAKE_PROG=path_to_make_binary] prove [options] [files/directories]
 
@@ -31,30 +71,102 @@ Example:
 	% sh test.t -m `pwd`/../obj/make run
 	% MAKE_PROG=/usr/obj/usr/src/usr.bin/make/make prove -r
 
-Variables
----------
-WORK_BASE	- base directory for working files
-SRC_BASE	- test source base directory
-SUBDIR		- subdirectory below WORK_BASE and SRC_BASE for current test
-WORK_DIR	- ${WORK_BASE}/${SUBDIR}
-SRC_DIR		- ${SRC_BASE}/${SUBDIR}
-MAKE_PROG	- path to the make program to test
+The test scripts use the following environment variables that can be set
+by the user in the test script's environment:
 
-WORK_BASE and MAKE_PROG are intented to be set by the user. All other
-variables are set by the script and can be used in scripts.
+	WORK_BASE
+		- Base directory for working files. If not set
+		  /tmp/make.${USER} is used.
 
-Directory layout
-----------------
-common.sh		- common code
-all.sh			- recursively call test scripts
+	MAKE_PROG
+		- Path to the make program to test. If not set
+		  /usr/bin/make is used.
 
-basic/
-      t0/test.t	- regression test
-      t1/test.t	- regression test
-      t2/test.t	- regression test
-variables/
-          t0/test.t	- regression test
-          t1/test.t	- regression test
+The following variables are available to test scripts:
+
+	SRC_BASE
+		- test source base directory. This is determined by
+		  repeatedly doing cd .. and checking for common.sh.
+		  Therefor this can fail if a test source directory is 
+		  actually a symbolic link and is physically not located
+		  below the directory containing common.sh.
+
+	SUBDIR	
+		- subdirectory below WORK_BASE and SRC_BASE for current test
+
+	WORK_DIR
+		- ${WORK_BASE}/${SUBDIR}
+
+	SRC_DIR
+		- ${SRC_BASE}/${SUBDIR}
+
+The following variables and functions may be defined by the test script.
+This also lists special filenames.
+
+	DESC
+		A one-line description of the test.
+
+	TEST_MAKE_DIRS
+		A list of pairs of directory names and modes. These
+		directories are created during setup and reset. When
+		the directory already exists (during reset) only the
+		mode change is applied.
+
+		TEST_MAKE_DIRS="subdir 775 subdir/sub 555"
+
+	TEST_COPY_FILES
+		A list of pairs of file names and modes. These files
+		are copied from the source to the working directory
+		during setup and reset. When the file already exists
+		(during reset) only the mode change is applied. Files
+		may be copied from/to sub-directories. The sub-directory
+		in the working directory must already exists (see
+		TEST_MAKE_DIRS).
+
+		TEST_COPY_FILES="libtest.a 444 subdir/libfoo.a 444"
+
+	TEST_TOUCH
+		List of pairs of file names and arguments to touch(1).
+		During setup and reset for each list element touch(1)
+		is executed.
+
+		TEST_TOUCH="file1 '-t 200501011257'"
+
+	TEST_LINK
+		List of pairs of filenames. Each pair is passed to ln(1).
+		All names are prefixed with the working directory.
+
+	Makefile
+		If a file with this name exists in the source directory
+		it is automatically copied to the working directory.
+
+	setup_test()
+		If this function exists it is executed at the end of the
+		setup.
+
+	reset_test()
+		If this function exists it is executed at the end of the
+		reset.
+
+	TEST_CLEAN_FILES
+		A list of file to be deleted when resetting.
+
+	TEST_N
+		Number of tests in this script. If not set this is assumed
+		to be 1.
+
+	TEST_<number>
+		Arguments to make for test number <number>. If not set
+		the default argument of test<number> is used. To run a test
+		without argument to make, set TEST_<number> to the empty string.
+
+	TEST_<number>_SKIP
+		To skip a test (for whatever reason) this should be set
+		to a string explaining the reason for skipping the test.
+
+	run_test()
+		Function to run a test. This function gets a single argument
+		which is the number of the test to executed. The default
+		function evaluates the variable TEST_<number> and calls
+		make with the arguments in this variable.
 
-Each test directory should contain at least a test.t script
-and the expected output files.

tools/regression/usr.bin/make/archives/fmt_44bsd/Makefile

@@ -2,18 +2,10 @@
 # This test checks the code reading archive files. The archive file
 # is a BSD4.4 file with __.SYMTAB and #1/N long file names.
 
-all:
-	-@make test_ok1 && echo "1 ok"
-	-@make test_ok2 && echo "2 ok"
-	-@make test_ok3 && echo "3 ok"
-	-@make test_ok4 && echo "4 ok"
-	-@make test_fail5 || echo "5 ok"
-	-@make test_fail6 || echo "6 ok"
-	-@make test_fail7 || echo "7 ok"
-
 #############################################################################
 
-test_ok1: libtest.a(short.o)
+# should be ok
+test1: libtest.a(short.o)
 	@:
 
 libtest.a(short.o): ood
@@ -21,7 +13,8 @@ libtest.a(short.o): ood
 
 #############################################################################
 
-test_ok2: libtest.a(exactly15char.o)
+# should be ok
+test2: libtest.a(exactly15char.o)
 	@:
 
 libtest.a(exactly15char.o): ood
@@ -29,7 +22,8 @@ libtest.a(exactly15char.o): ood
 
 #############################################################################
 
-test_ok3: libtest.a(exactly16chars.o)
+# should be ok
+test3: libtest.a(exactly16chars.o)
 	@:
 
 libtest.a(exactly16chars.o): ood
@@ -37,7 +31,8 @@ libtest.a(exactly16chars.o): ood
 
 #############################################################################
 
-test_ok4: libtest.a(verylongobjectname.o)
+# should be ok
+test4: libtest.a(verylongobjectname.o)
 	@:
 
 libtest.a(verylongobjectname.o): ood
@@ -46,18 +41,21 @@ libtest.a(verylongobjectname.o): ood
 #############################################################################
 
 # Truncated to 16 characters
-test_fail5: libtest.a(verylongobjectna)
+# should fail
+test5: libtest.a(verylongobjectna)
 	@:
 
 #############################################################################
 
 # Truncated to 15 characters
-test_fail6: libtest.a(verylongobjectn)
+# should fail
+test6: libtest.a(verylongobjectn)
 	@:
 
 #############################################################################
 
-test_fail7: libtest.a(\#1/20)
+# should fail
+test7: libtest.a(\#1/20)
 	@:
 
 ood:

tools/regression/usr.bin/make/archives/fmt_44bsd/expected.stderr.5

@@ -0,0 +1 @@
+make: don't know how to make verylongobjectna. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd/expected.stderr.6

@@ -1,2 +1 @@
 make: don't know how to make verylongobjectn. Stop
-make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd/expected.stderr.7

@@ -0,0 +1 @@
+make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd/test.t

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# $FreeBSD$
+
+cd `dirname $0`
+. ../../common.sh
+
+# Description
+DESC="Archive parsing (BSD4.4 format)."
+
+# Setup
+TEST_COPY_FILES="libtest.a 644"
+
+# Run
+TEST_N=7
+
+eval_cmd $*

tools/regression/usr.bin/make/archives/fmt_44bsd_mod/Makefile

@@ -5,18 +5,10 @@
 # The modification of the archive consists in slashes appended to short names.
 #
 
-all:
-	-@make test_ok1 && echo "1 ok"
-	-@make test_ok2 && echo "2 ok"
-	-@make test_ok3 && echo "3 ok"
-	-@make test_ok4 && echo "4 ok"
-	-@make test_fail5 || echo "5 ok"
-	-@make test_fail6 || echo "6 ok"
-	-@make test_fail7 || echo "7 ok"
-
 #############################################################################
 
-test_ok1: libtest.a(short.o)
+# must be ok
+test1: libtest.a(short.o)
 	@:
 
 libtest.a(short.o): ood
@@ -24,7 +16,8 @@ libtest.a(short.o): ood
 
 #############################################################################
 
-test_ok2: libtest.a(exactly15char.o)
+# must be ok
+test2: libtest.a(exactly15char.o)
 	@:
 
 libtest.a(exactly15char.o): ood
@@ -32,7 +25,8 @@ libtest.a(exactly15char.o): ood
 
 #############################################################################
 
-test_ok3: libtest.a(exactly16chars.o)
+# must be ok
+test3: libtest.a(exactly16chars.o)
 	@:
 
 libtest.a(exactly16chars.o): ood
@@ -40,7 +34,8 @@ libtest.a(exactly16chars.o): ood
 
 #############################################################################
 
-test_ok4: libtest.a(verylongobjectname.o)
+# must be ok
+test4: libtest.a(verylongobjectname.o)
 	@:
 
 libtest.a(verylongobjectname.o): ood
@@ -49,18 +44,21 @@ libtest.a(verylongobjectname.o): ood
 #############################################################################
 
 # Truncated to 16 characters
-test_fail5: libtest.a(verylongobjectna)
+# must fail
+test5: libtest.a(verylongobjectna)
 	@:
 
 #############################################################################
 
 # Truncated to 15 characters
-test_fail6: libtest.a(verylongobjectn)
+# must fail
+test6: libtest.a(verylongobjectn)
 	@:
 
 #############################################################################
 
-test_fail7: libtest.a(\#1/20)
+# must fail
+test7: libtest.a(\#1/20)
 	@:
 
 ood:

tools/regression/usr.bin/make/archives/fmt_44bsd_mod/expected.stderr.5

@@ -0,0 +1 @@
+make: don't know how to make verylongobjectna. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd_mod/expected.stderr.6

@@ -0,0 +1 @@
+make: don't know how to make verylongobjectn. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd_mod/expected.stderr.7

@@ -0,0 +1 @@
+make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/fmt_44bsd_mod/test.t

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# $FreeBSD$
+
+cd `dirname $0`
+. ../../common.sh
+
+# Description
+DESC="Archive parsing (modified BSD4.4 format)."
+
+# Setup
+TEST_COPY_FILES="libtest.a 644"
+
+# Run
+TEST_N=7
+
+eval_cmd $*

tools/regression/usr.bin/make/archives/fmt_oldbsd/Makefile

@@ -3,18 +3,10 @@
 # is an old BSD file with __.SYMTAB and no long file names. Filenames
 # are truncated to 16 characters.
 
-all:
-	-@make test_ok1 && echo "1 ok"
-	-@make test_ok2 && echo "2 ok"
-	-@make test_ok3 && echo "3 ok"
-	-@make test_ok4 && echo "4 ok"
-	-@make test_fail5 && echo "5 ok"
-	-@make test_fail6 || echo "6 ok"
-	-@make test_fail7 || echo "7 ok"
-
 #############################################################################
 
-test_ok1: libtest.a(short.o)
+# must be ok
+test1: libtest.a(short.o)
 	@:
 
 libtest.a(short.o): ood
@@ -22,7 +14,8 @@ libtest.a(short.o): ood
 
 #############################################################################
 
-test_ok2: libtest.a(exactly15char.o)
+# must be ok
+test2: libtest.a(exactly15char.o)
 	@:
 
 libtest.a(exactly15char.o): ood
@@ -30,7 +23,8 @@ libtest.a(exactly15char.o): ood
 
 #############################################################################
 
-test_ok3: libtest.a(exactly16chars.o)
+# must be ok
+test3: libtest.a(exactly16chars.o)
 	@:
 
 libtest.a(exactly16chars.o): ood
@@ -38,7 +32,8 @@ libtest.a(exactly16chars.o): ood
 
 #############################################################################
 
-test_ok4: libtest.a(verylongobjectname.o)
+# must be ok
+test4: libtest.a(verylongobjectname.o)
 	@:
 
 libtest.a(verylongobjectname.o): ood
@@ -47,18 +42,21 @@ libtest.a(verylongobjectname.o): ood
 #############################################################################
 
 # Truncated to 16 characters
-test_fail5: libtest.a(verylongobjectna)
+# must be ok
+test5: libtest.a(verylongobjectna)
 	@:
 
 #############################################################################
 
 # Truncated to 15 characters
-test_fail6: libtest.a(verylongobjectn)
+# must fail
+test6: libtest.a(verylongobjectn)
 	@:
 
 #############################################################################
 
-test_fail7: libtest.a(\#1/20)
+# must fail
+test7: libtest.a(\#1/20)
 	@:
 
 ood:

tools/regression/usr.bin/make/archives/fmt_oldbsd/expected.status.4

@@ -0,0 +1 @@
+0

tools/regression/usr.bin/make/archives/fmt_oldbsd/expected.status.5

@@ -0,0 +1 @@
+0

tools/regression/usr.bin/make/archives/fmt_oldbsd/expected.status.7

@@ -0,0 +1 @@
+2

tools/regression/usr.bin/make/archives/fmt_oldbsd/expected.stderr.6

@@ -0,0 +1 @@
+make: don't know how to make verylongobjectn. Stop

tools/regression/usr.bin/make/archives/fmt_oldbsd/expected.stderr.7

@@ -0,0 +1 @@
+make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/fmt_oldbsd/test.t

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# $FreeBSD$
+
+cd `dirname $0`
+. ../../common.sh
+
+# Description
+DESC="Archive parsing (old BSD format)."
+
+# Setup
+TEST_COPY_FILES="libtest.a 644"
+
+# Run
+TEST_N=7
+
+eval_cmd $*

tools/regression/usr.bin/make/archives/t0/expected.stderr

@@ -1,3 +0,0 @@
-make: don't know how to make verylongobjectna. Stop
-make: don't know how to make verylongobjectn. Stop
-make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/t0/expected.stdout

@@ -1,7 +0,0 @@
-1 ok
-2 ok
-3 ok
-4 ok
-5 ok
-6 ok
-7 ok

tools/regression/usr.bin/make/archives/t0/test.t

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# $FreeBSD$
-
-cd `dirname $0`
-. ../../common.sh
-
-setup_test()
-{
-	cp libtest.a ${WORK_DIR}
-}
-
-desc_test()
-{
-	echo "Archive parsing (BSD4.4 format)."
-}
-
-eval_cmd $1

tools/regression/usr.bin/make/archives/t1/expected.stderr

@@ -1,3 +0,0 @@
-make: don't know how to make verylongobjectna. Stop
-make: don't know how to make verylongobjectn. Stop
-make: don't know how to make #1/20. Stop

tools/regression/usr.bin/make/archives/t1/expected.stdout

@@ -1,7 +0,0 @@
-1 ok
-2 ok
-3 ok
-4 ok
-5 ok
-6 ok
-7 ok

tools/regression/usr.bin/make/archives/t1/test.t

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# $FreeBSD$
-
-cd `dirname $0`
-. ../../common.sh
-
-setup_test()
-{
-	cp libtest.a ${WORK_DIR}
-}
-
-desc_test()
-{
-	echo "Archive parsing (modified BSD4.4 format)."
-}
-
-eval_cmd $1

tools/regression/usr.bin/make/archives/t2/expected.stdout

@@ -1,7 +0,0 @@
-1 ok
-2 ok
-3 ok
-4 ok
-5 ok
-6 ok
-7 ok

tools/regression/usr.bin/make/archives/t2/test.t

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# $FreeBSD$
-
-cd `dirname $0`
-. ../../common.sh
-
-setup_test()
-{
-	cp libtest.a ${WORK_DIR}
-}
-
-desc_test()
-{
-	echo "Archive parsing (old BSD format)."
-}
-
-eval_cmd $1

tools/regression/usr.bin/make/basic/t0/expected.status.1

@@ -0,0 +1 @@
+2

tools/regression/usr.bin/make/basic/t0/test.t

@@ -5,14 +5,14 @@
 cd `dirname $0`
 . ../../common.sh
 
-setup_test()
-{
-	cp /dev/null ${WORK_DIR}/Makefile
-}
+# Description
+DESC="An empty Makefile file and no target given."
 
-desc_test()
-{
-	echo "An empty Makefile file."
-}
+# Setup
+TEST_TOUCH="Makefile ''"
 
-eval_cmd $1
+# Run
+TEST_N=1
+TEST_1=
+
+eval_cmd $*

tools/regression/usr.bin/make/basic/t1/Makefile

@@ -0,0 +1,5 @@
+# $FreeBSD$
+#
+# Just a target and nothing else. No target on command line.
+#
+all:

tools/regression/usr.bin/make/basic/t1/expected.status.1

@@ -0,0 +1 @@
+0

tools/regression/usr.bin/make/basic/t1/test.t

@@ -5,16 +5,11 @@
 cd `dirname $0`
 . ../../common.sh
 
-setup_test()
-{
-	cat > ${WORK_DIR}/Makefile << _EOF_
-all:
-_EOF_
-}
+# Description
+DESC="A Makefile file with only a 'all:' file dependency specification."
 
-desc_test()
-{
-	echo "A Makefile file with only a 'all:' file dependency specification."
-}
+# Run
+TEST_N=1
+TEST_1=
 
-eval_cmd $1
+eval_cmd $*

tools/regression/usr.bin/make/basic/t2/Makefile

@@ -0,0 +1,6 @@
+# $FreeBSD$
+#
+# Just a target and a command. No command line targets.
+#
+all:
+	echo hello

tools/regression/usr.bin/make/basic/t2/expected.status.1

@@ -0,0 +1 @@
+0

tools/regression/usr.bin/make/basic/t2/test.t

@@ -5,17 +5,11 @@
 cd `dirname $0`
 . ../../common.sh
 
-setup_test()
-{
-	cat > ${WORK_DIR}/Makefile << _EOF_
-all:
-	echo hello
-_EOF_
-}
+# Description
+DESC="A Makefile file with only a 'all:' file dependency specification, and shell command."
 
-desc_test()
-{
-	echo "A Makefile file with only a 'all:' file dependency specification, and shell command."
-}
+# Run
+TEST_N=1
+TEST_1=
 
-eval_cmd $1
+eval_cmd $*

tools/regression/usr.bin/make/basic/t3/expected.status.1

@@ -0,0 +1 @@
+2

tools/regression/usr.bin/make/basic/t3/test.t

@@ -5,14 +5,11 @@
 cd `dirname $0`
 . ../../common.sh
 
-setup_test()
-{
-	rm -f ${WORK_DIR}/Makefile
-}
+# Description
+DESC="No Makefile file, no command line target."
 
-desc_test()
-{
-	echo "No Makefile file."
-}
+# Run
+TEST_N=1
+TEST_1=
 
-eval_cmd $1
+eval_cmd $*

tools/regression/usr.bin/make/common.sh

@@ -4,103 +4,287 @@
 #
 # $FreeBSD$
 
+#
+# Output a message and exit with an error.
+#
+fatal()
+{
+	echo "fatal: $*" >/dev/stderr
+	exit 1
+}
+
+#
+# Check whether the working directory exists - it must.
+#
+ensure_workdir()
+{
+	if [ ! -d ${WORK_DIR} ] ; then
+		fatal "working directory ${WORK_DIR} does not exist."
+	fi
+}
+
+#
+# Make sure all tests have been run
+#
+ensure_run()
+{
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
+	fi
+
+	FAIL=
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			if [ ! -f ${OUTPUT_DIR}/status.${N} -o \
+			     ! -f ${OUTPUT_DIR}/stdout.${N} -o \
+			     ! -f ${OUTPUT_DIR}/stderr.${N} ] ; then
+				echo "Test ${SUBDIR}/${N} no yet run"
+				FAIL=yes
+			fi
+		fi
+		N=$((N + 1))
+	done
+
+	if [ ! -z "${FAIL}" ] ; then
+		exit 1
+	fi
+}
+
 #
 # Output usage messsage.
 #
 print_usage()
 {
-	echo "Usage: $0 command"
-	echo "	clean	- remove temp files (get initial state)"
-	echo "	compare	- compare result of test to expected"
-	echo "	desc	- print description of test"
-	echo "	diff	- print diffs between results and expected"
-	echo "	harness	- produce output suiteable for Test::Harness"
-	echo "	run	- run the {test, compare, clean}"
-	echo "	test	- run test case"
-	echo "	update	- update the expected with current results"
+	echo "Usage: sh -v -m <path> -w <dir> $0 command(s)"
+	echo " setup	- setup working directory"
+	echo " run	- run the tests"
+	echo " show	- show test results"
+	echo " compare	- compare actual and expected results"
+	echo " diff	- diff actual and expected results"
+	echo " reset	- reset the test to its initial state"
+	echo " clean	- delete working and output directory"
+	echo " test	- setup + run + compare"
+	echo " prove	- setup + run + compare + clean"
+	echo " desc	- print short description"
+	echo " update	- update the expected results with the current results"
+	echo " help	- show this information"
 }
 
 #
-# Check if the test result is the same as the expected result.
+# Common function for setup and reset.
 #
-# $1	Input file
-#
-hack_cmp()
+common_setup()
 {
-	local EXPECTED RESULT
-	EXPECTED="expected.$1"
-	RESULT=${WORK_DIR}/$1
-
-	if [ -f $EXPECTED ]; then
-		diff -q $EXPECTED $RESULT 1> /dev/null 2> /dev/null
-		return $?
-	else
-		return 1	# FAIL
+	#
+	# If a Makefile exists in the source directory - copy it over
+	#
+	if [ -e Makefile -a ! -e ${WORK_DIR}/Makefile ] ; then
+		cp Makefile ${WORK_DIR}/Makefile
 	fi
+
+	#
+	# If the TEST_MAKE_DIRS variable is set, create those directories
+	#
+	set -- ${TEST_MAKE_DIRS}
+	while [ $# -ne 0 ] ; do
+		if [ ! -d ${WORK_DIR}/${1} ] ; then
+			mkdir -p -m ${2} ${WORK_DIR}/${1}
+		else
+			chmod ${2} ${WORK_DIR}/${1}
+		fi
+		shift ; shift
+	done
+
+	#
+	# If the TEST_COPY_FILES variable is set, copy those files over to
+	# the working directory. The value is assumed to be pairs of
+	# filenames and modes.
+	#
+	set -- ${TEST_COPY_FILES}
+	while [ $# -ne 0 ] ; do
+		if [ ! -e ${WORK_DIR}/${1} ] ; then
+			cp ${1} ${WORK_DIR}/${1}
+		fi
+		chmod ${2} ${WORK_DIR}/${1}
+		shift ; shift
+	done
+
+	#
+	# If the TEST_TOUCH variable is set, it is taken to be a list
+	# of pairs of filenames and arguments to touch(1). The arguments
+	# to touch must be surrounded by single quotes if there are more
+	# than one argument.
+	#
+	eval set -- ${TEST_TOUCH}
+	while [ $# -ne 0 ] ; do
+		eval touch ${2} ${WORK_DIR}/${1}
+		shift ; shift
+	done
+
+	#
+	# Now create links
+	#
+	eval set -- ${TEST_LINKS}
+	while [ $# -ne 0 ] ; do
+		eval ln ${WORK_DIR}/${1} ${WORK_DIR}/${2}
+		shift ; shift
+	done
 }
 
 #
-# Check if the test result is the same as the expected result.
+# Setup the test. This creates the working and output directories and
+# populates it with files. If there is a setup_test() function - call it.
 #
-# $1	Input file
-#
-hack_diff()
+eval_setup()
 {
-	local EXPECTED RESULT
-	EXPECTED="expected.$1"
-	RESULT=${WORK_DIR}/$1
-
-	echo diff -u $EXPECTED $RESULT
-	if [ -f $EXPECTED ]; then
-		diff -u $EXPECTED $RESULT
-		return $?
-	else
-		return 1	# FAIL
+	#
+	# Check whether the working directory exists. If it does exit
+	# fatally so that we don't clobber a test the user is working on.
+	#
+	if [ -d ${WORK_DIR} ] ; then
+		fatal "working directory ${WORK_DIR} already exists."
 	fi
+
+	#
+	# Now create it and the output directory
+	#
+	mkdir -p ${WORK_DIR}
+	rm -rf ${OUTPUT_DIR}
+	mkdir -p ${OUTPUT_DIR}
+
+	#
+	# Common stuff
+	#
+	common_setup
+
+	#
+	# Now after all execute the user's setup function if it exists.
+	#
+	setup_test
 }
 
 #
-# Default setup_test() function.
-#
-# The default function just does nothing.
-#
-# Both the variables SRC_BASE WORK_BASE are available.
+# Default setup_test function does nothing. This may be overriden by
+# the test.
 #
 setup_test()
 {
 }
 
 #
-# Default run_test() function.  It can be replace by the
-# user specified regression test.
+# Reset the test. Here we need to rely on information from the test.
+# We executed the same steps as in the setup, by try not to clobber existing
+# files.
+# All files and directories that are listed on the TEST_CLEAN_FILES
+# variable are removed. Then the TEST_TOUCH list is executed and finally
+# the reset_test() function called if it exists.
 #
-# Both the variables SRC_BASE WORK_BASE are available.
-#
-# Note: this function executes from a subshell.
-#
-run_test()
+eval_reset()
 {
-	cd ${WORK_DIR}
-        $MAKE_PROG 1> stdout 2> stderr
-        echo $? > status
+	ensure_workdir
+
+	#
+	# Clean the output directory
+	#
+	rm -rf ${OUTPUT_DIR}/*
+
+	#
+	# Common stuff
+	#
+	common_setup
+
+	#
+	# Remove files.
+	#
+	for f in ${TEST_CLEAN_FILES} ; do
+		rm -rf ${WORK_DIR}/${f}
+	done
+
+	#
+	# Execute test's function
+	#
+	reset_test
 }
 
 #
-# Default clean routine
+# Default reset_test function does nothing. This may be overriden by
+# the test.
 #
-clean_test()
+reset_test()
 {
 }
 
 #
-# Clean working directory
+# Clean the test. This simply removes the working and output directories.
 #
 eval_clean()
 {
-	rm -f ${WORK_DIR}/stdout
-	rm -f ${WORK_DIR}/stderr
-	rm -f ${WORK_DIR}/status
-	clean_test
+	rm -rf ${WORK_DIR}
+	rm -rf ${OUTPUT_DIR}
+}
+
+#
+# Run the test.
+#
+eval_run()
+{
+	ensure_workdir
+
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
+	fi
+
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			( cd ${WORK_DIR} ;
+			  exec 1>${OUTPUT_DIR}/stdout.${N} 2>${OUTPUT_DIR}/stderr.${N}
+			  run_test ${N}
+			  echo $? >${OUTPUT_DIR}/status.${N}
+			)
+		fi
+		N=$((N + 1))
+	done
+}
+
+#
+# Default run_test() function.  It can be replaced by the
+# user specified regression test. The argument to this function is
+# the test number.
+#
+run_test()
+{
+	eval args=\${TEST_${1}-test${1}}
+        ${MAKE_PROG} $args
+}
+
+#
+# Show test results.
+#
+eval_show()
+{
+	ensure_workdir
+
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
+	fi
+
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			echo "=== Test ${N} Status =================="
+			cat ${OUTPUT_DIR}/status.${N}
+			echo ".......... Stdout .................."
+			cat ${OUTPUT_DIR}/stdout.${N}
+			echo ".......... Stderr .................."
+			cat ${OUTPUT_DIR}/stderr.${N}
+		fi
+		N=$((N + 1))
+	done
 }
 
 #
@@ -108,89 +292,97 @@ eval_clean()
 #
 eval_compare()
 {
-	hack_cmp stdout || FAIL="stdout $FAIL"
-	hack_cmp stderr || FAIL="stderr $FAIL"
-	hack_cmp status || FAIL="status $FAIL"
+	ensure_workdir
+	ensure_run
 
-	if [ ! -z "$FAIL" ]; then
-		FAIL=`echo $FAIL`
-		echo "$SUBDIR: Test failed {$FAIL}"
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
 	fi
+
+	echo "1..${TEST_N}"
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			FAIL=
+			do_compare stdout ${N} || FAIL="${FAIL}stdout "
+			do_compare stderr ${N} || FAIL="${FAIL}stderr "
+			do_compare status ${N} || FAIL="${FAIL}status "
+			if [ ! -z "$FAIL" ]; then
+				echo "not ok ${N} ${SUBDIR}/${N} # reason: ${FAIL}"
+			else
+				echo "ok ${N} ${SUBDIR}/${N}"
+			fi
+		else
+			echo "ok ${N} ${SUBDIR}/${N} # skip: ${skip}"
+		fi
+		N=$((N + 1))
+	done
 }
 
 #
-# Compare results with expected results for prove(1)
+# Check if the test result is the same as the expected result.
 #
-eval_hcompare()
+# $1	Input file
+# $2	Test number
+#
+do_compare()
 {
-	FAIL=
-	hack_cmp stdout || FAIL="stdout $FAIL"
-	hack_cmp stderr || FAIL="stderr $FAIL"
-	hack_cmp status || FAIL="status $FAIL"
+	local EXPECTED RESULT
+	EXPECTED="expected.$1.$2"
+	RESULT="${OUTPUT_DIR}/$1.$2"
 
-	if [ ! -z "$FAIL" ]; then
-		FAIL=`echo $FAIL`
-		echo "not ok 1 $SUBDIR # reason: {$FAIL}"
+	if [ -f $EXPECTED ]; then
+		diff -q $EXPECTED $RESULT 1>/dev/null 2>/dev/null
+		return $?
 	else
-		echo "ok 1 $SUBDIR"
+		return 1	# FAIL
 	fi
 }
 
-#
-# Print description
-#
-eval_desc()
-{
-	echo -n "$SUBDIR: "
-	desc_test
-}
-
-#
-# Prepare and run the test
-#
-eval_test()
-{
-	[ -d ${WORK_DIR} ] || mkdir -p ${WORK_DIR}
-	if [ -f Makefile ] ; then
-		cp Makefile ${WORK_DIR}
-	fi
-	setup_test
-	( run_test )
-}
-
 #
 # Diff current and expected results
 #
 eval_diff()
 {
-	eval_test
-	echo "------------------------"
-	echo "- $SUBDIR"
-	echo "------------------------"
-	hack_diff stdout
-	hack_diff stderr
-	hack_diff status
+	ensure_workdir
+	ensure_run
+
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
+	fi
+
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			FAIL=
+			do_diff stdout ${N}
+			do_diff stderr ${N}
+			do_diff status ${N}
+		fi
+		N=$((N + 1))
+	done
 }
 
 #
-# Run the test for prove(1)
+# Check if the test result is the same as the expected result.
 #
-eval_harness()
+# $1	Input file
+# $2	Test number
+#
+do_diff()
 {
-	echo 1..1
-	eval_test
-	eval_hcompare
-	eval_clean
-}
+	local EXPECTED RESULT
+	EXPECTED="expected.$1.$2"
+	RESULT="${OUTPUT_DIR}/$1.$2"
 
-#
-# Run the test
-#
-eval_run()
-{
-	eval_test
-	eval_compare
-	eval_clean
+	echo diff -u $EXPECTED $RESULT
+	if [ -f $EXPECTED ]; then
+		diff -u $EXPECTED $RESULT
+	else
+		echo "${EXPECTED} does not exist"
+	fi
 }
 
 #
@@ -198,32 +390,85 @@ eval_run()
 #
 eval_update()
 {
-	eval_test
-	cat ${WORK_DIR}/stdout > expected.stdout
-	cat ${WORK_DIR}/stderr > expected.stderr
-	cat ${WORK_DIR}/status > expected.status
+	ensure_workdir
+	ensure_run
+
+	if [ -z "${TEST_N}" ] ; then
+		TEST_N=1
+	fi
+
+	FAIL=
+	N=1
+	while [ ${N} -le ${TEST_N} ] ; do
+		eval skip=\${TEST_${N}_SKIP}
+		if [ -z "${skip}" ] ; then
+			cp ${OUTPUT_DIR}/stdout.${N} expected.stdout.${N}
+			cp ${OUTPUT_DIR}/stderr.${N} expected.stderr.${N}
+			cp ${OUTPUT_DIR}/status.${N} expected.status.${N}
+		fi
+		N=$((N + 1))
+	done
 }
 
 #
-# Note: Uses global variable $DIR which might be assigned by
-#	the script which sourced this file.
+# Print description
+#
+eval_desc()
+{
+	echo "${SUBDIR}: ${DESC}"
+}
+
+#
+# Run the test
+#
+eval_test()
+{
+	eval_setup
+	eval_run
+	eval_compare
+}
+
+#
+# Run the test for prove(1)
+#
+eval_prove()
+{
+	eval_setup
+	eval_run
+	eval_compare
+	eval_clean
+}
+
+#
+# Main function. Execute the command(s) on the command line.
 #
 eval_cmd()
 {
 	if [ $# -eq 0 ] ; then
-		set -- harness
+		# if no arguments given default to 'prove'
+		set -- prove
 	fi
 
-	case $1 in
-	clean|compare|hcompare|desc|diff|harness|run|test|update)
-		eval eval_$1
-		;;
-	*)
-		print_usage
-		;;
-	esac
+	for i
+	do
+		case $i in
+
+		setup | run | compare | diff | clean | reset | show | \
+		test | prove | desc | update)
+			eval eval_$i
+			;;
+		* | help)
+			print_usage
+			;;
+		esac
+	done
 }
 
+##############################################################################
+#
+# Main code
+#
+
 #
 # Parse command line arguments.
 #
@@ -263,14 +508,14 @@ SRC_DIR=`pwd`
 SRC_BASE=`while [ ! -f common.sh ] ; do cd .. ; done ; pwd`
 SUBDIR=`echo ${SRC_DIR} | sed "s@${SRC_BASE}/@@"`
 
+#
+# Construct working directory
+#
 WORK_BASE=${WORK_BASE:-"/tmp/$USER.make.test"}
 WORK_DIR=${WORK_BASE}/${SUBDIR}
-MAKE_PROG=${MAKE_PROG:-/usr/bin/make}
+OUTPUT_DIR=${WORK_DIR}.OUTPUT
 
-export MAKE_PROG
-export VERBOSE
-export SRC_BASE
-export WORK_BASE
-export SUBDIR
-export SRC_DIR
-export WORK_DIR
+#
+# Make to use
+#
+MAKE_PROG=${MAKE_PROG:-/usr/bin/make}