Convert bin/sh/tests to ATF

The new code uses a "test discovery mechanism" to determine
what tests are available for execution

The test shell can be specified via:

  kyua test -v test_suites.FreeBSD.bin.sh.test_shell=/path/to/test/sh

Sponsored by: EMC / Isilon Storage Division
Approved by: jmmv (mentor)
Reviewed by: jilles (maintainer)

bin/sh/tests/Makefile

@@ -4,15 +4,12 @@
 
 TESTSDIR=	${TESTSBASE}/bin/sh
 
-TAP_TESTS_SH=	legacy_test
-TAP_TESTS_SH_SED_legacy_test=	-e 's,__SH__,/bin/sh,g'
-# Some tests in here are silently not run when the tests are executed as
-# root.  Explicitly tell Kyua to drop privileges.
-#
-# TODO(jmmv): Kyua needs to do this by default, not only when explicitly
-# requested.  See https://code.google.com/p/kyua/issues/detail?id=6
-TEST_METADATA.legacy_test+= required_user="unprivileged"
-
-SUBDIR+=	builtins errors execution expansion parameters parser set-e
+TESTS_SUBDIRS+=	builtins
+TESTS_SUBDIRS+=	errors
+TESTS_SUBDIRS+=	execution
+TESTS_SUBDIRS+=	expansion
+TESTS_SUBDIRS+=	parameters
+TESTS_SUBDIRS+=	parser
+TESTS_SUBDIRS+=	set-e
 
 .include <bsd.test.mk>

bin/sh/tests/builtins/Makefile

@@ -1,9 +1,13 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
-FILESDIR=	${TESTSBASE}/bin/sh/builtins
-KYUAFILE=	no
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
+
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		alias.0 alias.0.stdout
 FILES+=		alias.1 alias.1.stderr

bin/sh/tests/errors/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/errors
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		assignment-error1.0
 FILES+=		assignment-error2.0

bin/sh/tests/errors/bad-parm-exp2.2.stderr

@@ -1 +1 @@
-./errors/bad-parm-exp2.2: ${}: Bad substitution
+./bad-parm-exp2.2: ${}: Bad substitution

bin/sh/tests/errors/bad-parm-exp3.2.stderr

@@ -1 +1 @@
-./errors/bad-parm-exp3.2: ${foo/}: Bad substitution
+./bad-parm-exp3.2: ${foo/}: Bad substitution

bin/sh/tests/errors/bad-parm-exp4.2.stderr

@@ -1 +1 @@
-./errors/bad-parm-exp4.2: ${foo:@...}: Bad substitution
+./bad-parm-exp4.2: ${foo:@...}: Bad substitution

bin/sh/tests/errors/bad-parm-exp5.2.stderr

@@ -1 +1 @@
-./errors/bad-parm-exp5.2: ${/}: Bad substitution
+./bad-parm-exp5.2: ${/}: Bad substitution

bin/sh/tests/errors/bad-parm-exp6.2.stderr

@@ -1 +1 @@
-./errors/bad-parm-exp6.2: ${foo...}: Bad substitution
+./bad-parm-exp6.2: ${foo...}: Bad substitution

bin/sh/tests/execution/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/execution
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		bg1.0
 FILES+=		bg2.0

bin/sh/tests/expansion/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/expansion
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		arith1.0
 FILES+=		arith2.0

bin/sh/tests/functional_test.sh

@@ -0,0 +1,78 @@
+#
+# Copyright 2014 EMC Corp.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# $FreeBSD$
+
+SRCDIR=$(atf_get_srcdir)
+
+check()
+{
+	local tc=${1}; shift
+
+	export SH=$(atf_config_get bin.sh.test_shell /bin/sh)
+
+	local err_file="${SRCDIR}/${tc}.stderr"
+	[ -f "${err_file}" ] && err_flag="-e file:${err_file}"
+	local out_file="${SRCDIR}/${tc}.stdout"
+	[ -f "${out_file}" ] && out_flag="-o file:${out_file}"
+
+	# We need to copy the testcase scenario file because some of the
+	# testcases hardcode relative paths in the stderr/stdout.
+	#
+	# TODO: we might be able to generate this path at build time
+	cp ${SRCDIR}/${tc} .
+
+	atf_check -s exit:${tc##*.} ${err_flag} ${out_flag} ${SH} "./${tc}"
+}
+
+add_testcase()
+{
+	local tc=${1}
+	local tc_escaped word
+
+	case "${tc%.*}" in
+	*-*)
+		local IFS="-"
+		for word in ${tc%.*}; do
+			tc_escaped="${tc_escaped:+${tc_escaped}_}${word}"
+		done
+		;;
+	*)
+		tc_escaped=${tc%.*}
+		;;
+	esac
+
+	atf_test_case ${tc_escaped}
+	eval "${tc_escaped}_body() { check ${tc}; }"
+	atf_add_test_case ${tc_escaped}
+}
+
+atf_init_test_cases()
+{
+	for path in $(find -Es "${SRCDIR}" -regex '.*\.[0-9]+$'); do
+		add_testcase ${path##*/}
+	done
+}

bin/sh/tests/legacy_test.sh

@@ -1,46 +0,0 @@
-# $FreeBSD$
-
-: ${SH:="__SH__"}
-export SH
-
-# TODO(jmmv): The Kyua TAP interface should be passing us the value of
-# "srcdir" as an environment variable, just as it does with the ATF
-# interface in the form of a configuration variable.  For now, just try
-# to guess this.
-: ${TESTS_DATA:=$(dirname ${0})}
-
-COUNTER=1
-
-do_test() {
-	c=${COUNTER}
-	COUNTER=$((COUNTER+1))
-	${SH} $1 > tmp.stdout 2> tmp.stderr
-	if [ $? -ne $2 ]; then
-		echo "not ok ${c} - ${1} # wrong exit status"
-		rm tmp.stdout tmp.stderr
-		return
-	fi
-	sed -I '' -e "s|^${TESTS_DATA}|.|" tmp.stderr
-	for i in stdout stderr; do
-		if [ -f ${1}.${i} ]; then
-			if ! cmp -s tmp.${i} ${1}.${i}; then
-				echo "not ok ${c} - ${1} # wrong output on ${i}"
-				rm tmp.stdout tmp.stderr
-				return
-			fi
-		elif [ -s tmp.${i} ]; then
-			echo "not ok ${c} - ${1} # wrong output on ${i}"
-			rm tmp.stdout tmp.stderr
-			return
-		fi
-	done
-	echo "ok ${c} - ${1}"
-	rm tmp.stdout tmp.stderr
-}
-
-TESTS=$(find -Es ${TESTS_DATA} -regex ".*\.[0-9]+")
-printf "1..%d\n" $(echo ${TESTS} | wc -w)
-
-for i in ${TESTS} ; do
-	do_test ${i} ${i##*.}
-done

bin/sh/tests/parameters/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/parameters
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		env1.0
 FILES+=		exitstatus1.0

bin/sh/tests/parser/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/parser
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		alias1.0
 FILES+=		alias2.0

bin/sh/tests/set-e/Makefile

@@ -1,9 +1,11 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+TESTSDIR=	${TESTSBASE}/bin/sh/${.CURDIR:T}
 
-FILESDIR=	${TESTSBASE}/bin/sh/set-e
-KYUAFILE=	no
+.PATH: ${.CURDIR:H}
+ATF_TESTS_SH=	functional_test
+
+FILESDIR=	${TESTSDIR}
 
 FILES=		and1.0
 FILES+=		and2.1

tools/build/mk/OptionalObsoleteFiles.inc

@@ -4168,6 +4168,7 @@ OLD_FILES+=usr/share/aclocal/atf-common.m4
 OLD_FILES+=usr/share/aclocal/atf-sh.m4
 OLD_DIRS+=usr/share/aclocal
 OLD_FILES+=usr/tests/bin/chown/units_basics
+OLD_FILES+=usr/tests/bin/sh/legacy_test
 OLD_FILES+=usr/tests/usr.bin/atf/Kyuafile
 OLD_FILES+=usr/tests/usr.bin/atf/atf-sh/Kyuafile
 OLD_FILES+=usr/tests/usr.bin/atf/atf-sh/atf_check_test