From 194d5628721f6a306aa94f7e33be909f1ab16b49 Mon Sep 17 00:00:00 2001
From: Li-Wen Hsu <lwhsu@FreeBSD.org>
Date: Tue, 6 Oct 2020 06:45:52 +0000
Subject: [PATCH] Make capsicum test cases fine-grained

Add a wrapping script to use ATF to run tests written with Googletest
one by one. This helps locating and tracking the failing case in CI easier.

This is a temporarily solution while Googletest support in Kyua is developing.
We will revert this once Kyua+Googletest integration is ready.

Reviewed by:	emaste
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D25896
---
 share/mk/bsd.test.mk             |  3 +-
 share/mk/googletest.test.mk      |  8 +++-
 tests/sys/capsicum/Makefile      |  3 +-
 tests/sys/capsicum/functional.sh | 68 ++++++++++++++++++++++++++++++++
 4 files changed, 79 insertions(+), 3 deletions(-)
 create mode 100755 tests/sys/capsicum/functional.sh

diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk
index 072c5612ae29..588cfbcb45ed 100644
--- a/share/mk/bsd.test.mk
+++ b/share/mk/bsd.test.mk
@@ -62,10 +62,11 @@ TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW:C/ +/:/g}
 _TESTS=
 
 # Pull in the definitions of all supported test interfaces.
-.include <atf.test.mk>
 .include <googletest.test.mk>
 .include <plain.test.mk>
 .include <tap.test.mk>
+# Include atf last to let other test framework use it
+.include <atf.test.mk>
 
 # Sort the tests alphabetically, so the results are deterministically formed
 # across runs.
diff --git a/share/mk/googletest.test.mk b/share/mk/googletest.test.mk
index b55f28d572f0..b2e57e8d6e0a 100644
--- a/share/mk/googletest.test.mk
+++ b/share/mk/googletest.test.mk
@@ -30,12 +30,18 @@ GTESTS?=
 .include <googletest.test.inc.mk>
 
 PROGS_CXX+= ${GTESTS}
-_TESTS+= ${GTESTS}
 .for _T in ${GTESTS}
 BINDIR.${_T}= ${TESTSDIR}
 CXXFLAGS.${_T}+= ${GTESTS_CXXFLAGS}
 MAN.${_T}?= # empty
 SRCS.${_T}?= ${_T}.cc
+.if !empty(GTESTS_WRAPPER_SH.${_T})
+# A stopgap/workaround to let kyua execute test case one by one
+ATF_TESTS_SH+= ${GTESTS_WRAPPER_SH.${_T}}
+.else
+_TESTS+= ${_T}
 TEST_INTERFACE.${_T}= plain
+.endif
+
 .endfor
 .endif
diff --git a/tests/sys/capsicum/Makefile b/tests/sys/capsicum/Makefile
index 1b46f9acb471..75dd0238895f 100644
--- a/tests/sys/capsicum/Makefile
+++ b/tests/sys/capsicum/Makefile
@@ -14,6 +14,7 @@ CFLAGS+=	-I${SRCTOP}/tests
 .PATH: ${SRCTOP}/contrib/capsicum-test
 
 GTESTS+=	capsicum-test
+GTESTS_WRAPPER_SH.capsicum-test=	functional
 
 SRCS.capsicum-test+=	\
 	capsicum-test-main.cc \
@@ -50,6 +51,6 @@ BINMODE.mini-me.setuid=	4555
 
 WARNS.capsicum-test=	3
 
-.endif
+.endif # MK_GOOGLETEST
 
 .include <bsd.test.mk>
diff --git a/tests/sys/capsicum/functional.sh b/tests/sys/capsicum/functional.sh
new file mode 100755
index 000000000000..3d0a5a9a4935
--- /dev/null
+++ b/tests/sys/capsicum/functional.sh
@@ -0,0 +1,68 @@
+#-
+# SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+#
+# Copyright (c) 2020 The FreeBSD Foundation
+#
+# This software was developed by Li-Wen Hsu <lwhsu@FreeBSD.org>
+# under sponsorship from the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+SRCDIR=$(atf_get_srcdir)
+CAPSICUM_TEST_BIN=capsicum-test
+
+check()
+{
+	local tc=${1}
+
+	atf_check -s exit:0 -o match:PASSED  -e ignore \
+		${SRCDIR}/${CAPSICUM_TEST_BIN} --gtest_filter=${tc}
+}
+
+add_testcase()
+{
+	local tc=${1}
+	local tc_escaped word
+
+	tc_escaped=$(echo ${tc} | sed -e 's/\./__/')
+
+	atf_test_case ${tc_escaped}
+	eval "${tc_escaped}_body() { check ${tc}; }"
+	atf_add_test_case ${tc_escaped}
+}
+
+list_tests()
+{
+	${SRCDIR}/${CAPSICUM_TEST_BIN} --gtest_list_tests | awk '
+		/^[^ ]/ { CAT=$0 }
+		/^[ ]/ { print CAT $1}'
+}
+
+atf_init_test_cases()
+{
+	local t
+	for t in `list_tests`; do
+		add_testcase $t
+	done
+}