diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000000..fd9b6e4a47df --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,26 @@ +env: + CIRRUS_CLONE_DEPTH: 1 + ARCH: amd64 + +task: + matrix: + - name: 13.0-CURRENT + freebsd_instance: + image_family: freebsd-13-0-snap + - name: 12.2-STABLE + freebsd_instance: + image_family: freebsd-12-2-snap + - name: 12.1-RELEASE + freebsd_instance: + image_family: freebsd-12-1 + install_script: + - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf + - ASSUME_ALWAYS_YES=yes pkg bootstrap -f + - pkg install -y autoconf automake libtool kyua + script: + - env JUNIT_OUTPUT=$(pwd)/test-results.xml ./admin/travis-build.sh + always: + junit_artifacts: + path: "test-results.xml" + type: text/xml + format: junit diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..396785ce2052 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +*.la +*.lo +*.o +*.pc +*_helper +*_helpers +*_test +.deps +.dirstamp +.libs + +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.h +config.h.in +config.h.in~ +config.log +config.status +configure +installcheck.log +libtool +stamp-h1 +testsuite.log diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..1949aae54468 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +language: cpp + +compiler: + - gcc + - clang + +before_install: + - ./admin/travis-install-deps.sh + +env: + - ARCH=amd64 AS_ROOT=no + - ARCH=amd64 AS_ROOT=yes + - ARCH=i386 AS_ROOT=no + +matrix: + exclude: + - compiler: clang + env: ARCH=i386 AS_ROOT=no + +script: + - ./admin/travis-build.sh + +notifications: + email: + - atf-log@googlegroups.com diff --git a/Makefile.am b/Makefile.am index 41c9a6bfb046..913910c274a3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,7 +39,6 @@ INSTALLCHECK_TARGETS = PHONY_TARGETS = ACLOCAL_AMFLAGS = -I m4 -AM_DISTCHECK_CONFIGURE_FLAGS = --enable-tools include admin/Makefile.am.inc include atf-c/Makefile.am.inc @@ -75,8 +74,19 @@ PHONY_TARGETS += installcheck-kyua if HAVE_KYUA INSTALLCHECK_TARGETS += installcheck-kyua installcheck-kyua: + set +e; cd $(pkgtestsdir) && $(TESTS_ENVIRONMENT) \ + $(KYUA) --config='$(KYUA_TEST_CONFIG_FILE)' test; \ + ret=$$?; \ + echo $$ret; \ + if [ -n "$$JUNIT_OUTPUT" ]; then \ + cd $(pkgtestsdir) && $(TESTS_ENVIRONMENT) \ + $(KYUA) --config='$(KYUA_TEST_CONFIG_FILE)' \ + report-junit --output="$$JUNIT_OUTPUT"; \ + sed -i '' 's/encoding="iso-8859-1"/econding="utf-8"/' "$$JUNIT_OUTPUT"; \ + fi; \ cd $(pkgtestsdir) && $(TESTS_ENVIRONMENT) \ - $(KYUA) --config='$(KYUA_TEST_CONFIG_FILE)' test + $(KYUA) --config='$(KYUA_TEST_CONFIG_FILE)' report --verbose; \ + exit $$ret endif installcheck-local: $(INSTALLCHECK_TARGETS) diff --git a/NEWS b/NEWS index 35b6e0c69238..548b069f3558 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,24 @@ Major changes between releases Automated Testing Framework =========================================================================== +Changes in version 0.22 +*********************** + +STILL UNDER DEVELOPMENT; NOT RELEASED YET. +DON'T FORGET TO BUMP THE -version-info PRE-RELEASE IF NECESSARY! + +* Issue #23: Fix double-free triggered by atf_map_insert in low memory + scenarios, caused by an overlook in the atf_list code. + +* Issue #29: Fixed various typos and formatting errors in manual pages. + +* Issue #31: Added require.progs metadata properties to the tests that + need a compiler to run. + +* Added the atf_check_not_equal function to atf-sh to check for + unequal values. + + Changes in version 0.21 *********************** diff --git a/README.md b/README.md new file mode 100644 index 000000000000..d245552f35c9 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# Welcome to the ATF project! + +ATF, or Automated Testing Framework, is a **collection of libraries** to +write test programs in **C, C++ and POSIX shell**. + +The ATF libraries offer a simple API. The API is orthogonal through the +various bindings, allowing developers to quickly learn how to write test +programs in different languages. + +ATF-based test programs offer a **consistent end-user command-line +interface** to allow both humans and automation to run the tests. + +ATF-based test programs **rely on an execution engine** to be run and +this execution engine is *not* shipped with ATF. +**[Kyua](https://github.com/jmmv/kyua/) is the engine of choice.** + +## Download + +Formal releases for source files are available for download from GitHub: + +* [atf 0.20](../../releases/tag/atf-0.20), released on February 7th, 2014. + +## Installation + +You are encouraged to install binary packages for your operating system +wherever available: + +* Fedora 20 and above: install the `atf` package with `yum install atf`. + +* FreeBSD 10.0 and above: install the `atf` package with `pkg install atf`. + +* NetBSD with pkgsrc: install the `pkgsrc/devel/atf` package. + +* OpenBSD: install the `devel/atf` package with `pkg_add atf`. + +Should you want to build and install ATF from the source tree provided +here, follow the instructions in the [INSTALL file](INSTALL). + +## Support + +Please use the +[atf-discuss mailing list](https://groups.google.com/forum/#!forum/atf-discuss) +for any support inquiries related to `atf-c`, `atf-c++` or `atf-sh`. + +If you have any questions on Kyua proper, please use the +[kyua-discuss mailing list](https://groups.google.com/forum/#!forum/kyua-discuss) +instead. diff --git a/admin/.gitignore b/admin/.gitignore new file mode 100644 index 000000000000..64f348c68ebf --- /dev/null +++ b/admin/.gitignore @@ -0,0 +1,8 @@ +ar-lib +compile +config.guess +config.sub +depcomp +install-sh +ltmain.sh +missing diff --git a/admin/clean-all.sh b/admin/clean-all.sh new file mode 100755 index 000000000000..e278d150d773 --- /dev/null +++ b/admin/clean-all.sh @@ -0,0 +1,84 @@ +#! /bin/sh +# Copyright (c) 2007 The NetBSD Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND +# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Prog_Name=${0##*/} + +if [ ! -f ./atf-c.h ]; then + echo "${Prog_Name}: must be run from atf source's top directory" 1>&2 + exit 1 +fi + +if [ ! -f configure ]; then + echo "${Prog_Name}: nothing to clean" 1>&2 + exit 1 +fi + +[ -f Makefile ] || ./configure +make distclean + +# Top-level directory. +rm -f .gdb_history +rm -f Makefile.in +rm -f aclocal.m4 +rm -rf autom4te.cache +rm -f config.h.in* +rm -f configure +rm -f mkinstalldirs +rm -f atf-*.tar.gz +rm -rf release-test + +# `admin' directory. +rm -f admin/compile +rm -f admin/config.guess +rm -f admin/config.sub +rm -f admin/depcomp +rm -f admin/install-sh +rm -f admin/ltmain.sh +rm -f admin/missing + +# `m4' directory. +rm -f m4/libtool.m4 +rm -f m4/lt*.m4 + +# `bootstrap' directory. +rm -f bootstrap/package.m4 +rm -f bootstrap/testsuite + +# Files and directories spread all around the tree. +find . -name '#*' | xargs rm -rf +find . -name '*~' | xargs rm -rf +find . -name .deps | xargs rm -rf +find . -name .gdb_history | xargs rm -rf +find . -name .libs | xargs rm -rf +find . -name .tmp | xargs rm -rf + +# Show remaining files +if [ -n "${GIT}" ]; then + echo ">>> untracked and ignored files" + "${GIT}" status --porcelain --ignored | grep -E '^(\?\?|!!)' +fi + +# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4 diff --git a/admin/travis-build.sh b/admin/travis-build.sh new file mode 100755 index 000000000000..2c5fe756886d --- /dev/null +++ b/admin/travis-build.sh @@ -0,0 +1,78 @@ +#! /bin/sh +# Copyright 2014 Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -e -x + +if [ -d /usr/local/share/aclocal ]; then + autoreconf -isv -I/usr/local/share/aclocal +else + autoreconf -isv +fi + +ret=0 +./configure || ret=${?} +if [ ${ret} -ne 0 ]; then + cat config.log || true + exit ${ret} +fi + +archflags= +[ "${ARCH?}" != i386 ] || archflags=-m32 + +f= + +if [ -n "${archflags}" ]; then + CC=${CC-"cc"} + CXX=${CXX-"c++"} + + f="${f} ATF_BUILD_CC='${CC} ${archflags}'" + f="${f} ATF_BUILD_CXX='${CXX} ${archflags}'" + f="${f} CFLAGS='${archflags}'" + f="${f} CXXFLAGS='${archflags}'" + f="${f} LDFLAGS='${archflags}'" +fi + +if [ "${AS_ROOT:-no}" = yes ]; then + cat >root-kyua.conf <&2 + exit 1 + ;; + esac + wget "http://dl.bintray.com/jmmv/kyua/${name}" || return 1 + sudo tar -xzvp -C / -f "${name}" + rm -f "${name}" +} + +install_deps +if ! install_from_bintray; then + install_from_github atf 0.21 + install_from_github lutok 0.4 + install_from_github kyua 0.12 +fi + +# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4 diff --git a/atf-c++/Makefile.am.inc b/atf-c++/Makefile.am.inc index b131e555a06f..9d982b0f8c8d 100644 --- a/atf-c++/Makefile.am.inc +++ b/atf-c++/Makefile.am.inc @@ -58,7 +58,7 @@ EXTRA_DIST += atf-c++/atf-c++.pc.in atf-c++/atf-c++.pc: $(srcdir)/atf-c++/atf-c++.pc.in Makefile $(AM_V_GEN)test -d atf-c++ || mkdir -p atf-c++; \ sed -e 's#__ATF_VERSION__#$(PACKAGE_VERSION)#g' \ - -e 's#__CXX__#$(CXX)#g' \ + -e 's#__CXX__#$(ATF_BUILD_CXX)#g' \ -e 's#__INCLUDEDIR__#$(includedir)#g' \ -e 's#__LIBDIR__#$(libdir)#g' \ <$(srcdir)/atf-c++/atf-c++.pc.in >atf-c++/atf-c++.pc.tmp; \ @@ -70,18 +70,29 @@ tests_atf_c___DATA = atf-c++/Kyuafile \ tests_atf_c__dir = $(pkgtestsdir)/atf-c++ EXTRA_DIST += $(tests_atf_c___DATA) +ATF_CXX_TEST_HELPERS_CPPFLAGS = "-DATF_BUILD_CXX=\"$(ATF_BUILD_CXX)\"" +ATF_CXX_TEST_HELPERS_LDADD = atf-c++/detail/libtest_helpers.la + tests_atf_c___PROGRAMS = atf-c++/atf_c++_test atf_c___atf_c___test_SOURCES = atf-c++/atf_c++_test.cpp -atf_c___atf_c___test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___atf_c___test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___atf_c___test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + tests_atf_c___PROGRAMS += atf-c++/build_test atf_c___build_test_SOURCES = atf-c++/build_test.cpp atf-c/h_build.h -atf_c___build_test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___build_test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___build_test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + tests_atf_c___PROGRAMS += atf-c++/check_test atf_c___check_test_SOURCES = atf-c++/check_test.cpp -atf_c___check_test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___check_test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___check_test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + tests_atf_c___PROGRAMS += atf-c++/macros_test atf_c___macros_test_SOURCES = atf-c++/macros_test.cpp -atf_c___macros_test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___macros_test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___macros_test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + tests_atf_c___SCRIPTS = atf-c++/pkg_config_test CLEANFILES += atf-c++/pkg_config_test EXTRA_DIST += atf-c++/pkg_config_test.sh @@ -91,10 +102,14 @@ atf-c++/pkg_config_test: $(srcdir)/atf-c++/pkg_config_test.sh tests_atf_c___PROGRAMS += atf-c++/tests_test atf_c___tests_test_SOURCES = atf-c++/tests_test.cpp -atf_c___tests_test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___tests_test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___tests_test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + tests_atf_c___PROGRAMS += atf-c++/utils_test atf_c___utils_test_SOURCES = atf-c++/utils_test.cpp -atf_c___utils_test_LDADD = atf-c++/detail/libtest_helpers.la $(ATF_CXX_LIBS) +atf_c___utils_test_CPPFLAGS = $(ATF_CXX_TEST_HELPERS_CPPFLAGS) +atf_c___utils_test_LDADD = $(ATF_CXX_TEST_HELPERS_LDADD) $(ATF_CXX_LIBS) + include atf-c++/detail/Makefile.am.inc # vim: syntax=make:noexpandtab:shiftwidth=8:softtabstop=8 diff --git a/atf-c++/atf-c++.3 b/atf-c++/atf-c++.3 index 984ec936446a..c39d905567b8 100644 --- a/atf-c++/atf-c++.3 +++ b/atf-c++/atf-c++.3 @@ -145,10 +145,10 @@ ATF provides a C++ programming interface to implement test programs. C++-based test programs follow this template: .Bd -literal -offset indent extern "C" { -.Ns ... C-specific includes go here ... +\&... C-specific includes go here ... } -.Ns ... C++-specific includes go here ... +\&... C++-specific includes go here ... #include @@ -182,7 +182,7 @@ ATF_TEST_CASE_BODY(tc3) ... third test case's body ... } -.Ns ... additional test cases ... +\&... additional test cases ... ATF_INIT_TEST_CASES(tcs) { @@ -202,7 +202,7 @@ To define test cases, one can use the .Fn ATF_TEST_CASE_WITH_CLEANUP or the .Fn ATF_TEST_CASE_WITHOUT_HEAD -macros, which take a single parameter specifiying the test case's +macros, which take a single parameter specifying the test case's name. .Fn ATF_TEST_CASE , requires to define a head and a body for the test case, @@ -232,7 +232,7 @@ opening and closing brackets. Additionally, the .Fn ATF_TEST_CASE_NAME macro can be used to obtain the name of the class corresponding to a -particular test case, as the name is internally manged by the library to +particular test case, as the name is internally managed by the library to prevent clashes with other user identifiers. Similarly, the .Fn ATF_TEST_CASE_USE @@ -403,8 +403,8 @@ in the collection. takes the name of an exception and a statement and raises a failure if the statement does not throw the specified exception. .Fn ATF_REQUIRE_THROW_RE -takes the name of an exception, a regular expresion and a statement and raises a -failure if the statement does not throw the specified exception and if the +takes the name of an exception, a regular expression and a statement, and raises +a failure if the statement does not throw the specified exception and if the message of the exception does not match the regular expression. .Pp .Fn ATF_CHECK_ERRNO diff --git a/atf-c++/detail/test_helpers.hpp b/atf-c++/detail/test_helpers.hpp index f166ee218a13..c1171801a3a7 100644 --- a/atf-c++/detail/test_helpers.hpp +++ b/atf-c++/detail/test_helpers.hpp @@ -36,6 +36,7 @@ #include +#include #include #define HEADER_TC(name, hdrname) \ @@ -44,6 +45,8 @@ { \ set_md_var("descr", "Tests that the " hdrname " file can be " \ "included on its own, without any prerequisites"); \ + const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \ + set_md_var("require.progs", cxx); \ } \ ATF_TEST_CASE_BODY(name) \ { \ @@ -55,6 +58,8 @@ ATF_TEST_CASE_HEAD(name) \ { \ set_md_var("descr", descr); \ + const std::string cxx = atf::env::get("ATF_BUILD_CXX", ATF_BUILD_CXX); \ + set_md_var("require.progs", cxx); \ } \ ATF_TEST_CASE_BODY(name) \ { \ diff --git a/atf-c++/tests.hpp b/atf-c++/tests.hpp index ce2fb1d165c8..a03cc852dcf8 100644 --- a/atf-c++/tests.hpp +++ b/atf-c++/tests.hpp @@ -73,7 +73,7 @@ class tc { tc(const tc&); tc& operator=(const tc&); - std::auto_ptr< tc_impl > pimpl; + std::unique_ptr< tc_impl > pimpl; protected: virtual void head(void); diff --git a/atf-c++/utils.cpp b/atf-c++/utils.cpp index a6ab08f0d770..995d78c6542e 100644 --- a/atf-c++/utils.cpp +++ b/atf-c++/utils.cpp @@ -70,6 +70,13 @@ atf::utils::fork(void) return atf_utils_fork(); } +void +atf::utils::reset_resultsfile(void) +{ + + atf_utils_reset_resultsfile(); +} + bool atf::utils::grep_file(const std::string& regex, const std::string& path) { diff --git a/atf-c++/utils.hpp b/atf-c++/utils.hpp index 8f5c5e337455..34d77a126df7 100644 --- a/atf-c++/utils.hpp +++ b/atf-c++/utils.hpp @@ -41,6 +41,7 @@ void copy_file(const std::string&, const std::string&); void create_file(const std::string&, const std::string&); bool file_exists(const std::string&); pid_t fork(void); +void reset_resultsfile(void); bool grep_file(const std::string&, const std::string&); bool grep_string(const std::string&, const std::string&); void redirect(const int, const std::string&); diff --git a/atf-c++/utils_test.cpp b/atf-c++/utils_test.cpp index 34e0709f580a..93e16652bac1 100644 --- a/atf-c++/utils_test.cpp +++ b/atf-c++/utils_test.cpp @@ -335,6 +335,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr) std::cerr << "Some error\n"; exit(123); } + atf::utils::reset_resultsfile(); atf::utils::wait(pid, exitstatus, expout, experr); exit(EXIT_SUCCESS); } diff --git a/atf-c/.gitignore b/atf-c/.gitignore new file mode 100644 index 000000000000..e7f0fb647c32 --- /dev/null +++ b/atf-c/.gitignore @@ -0,0 +1 @@ +defs.h diff --git a/atf-c/Makefile.am.inc b/atf-c/Makefile.am.inc index bcb06da3a149..ec0e9e5d41ec 100644 --- a/atf-c/Makefile.am.inc +++ b/atf-c/Makefile.am.inc @@ -71,7 +71,7 @@ EXTRA_DIST += atf-c/atf-c.pc.in atf-c/atf-c.pc: $(srcdir)/atf-c/atf-c.pc.in Makefile $(AM_V_GEN)test -d atf-c || mkdir -p atf-c; \ sed -e 's#__ATF_VERSION__#$(PACKAGE_VERSION)#g' \ - -e 's#__CC__#$(CC)#g' \ + -e 's#__CC__#$(ATF_BUILD_CC)#g' \ -e 's#__INCLUDEDIR__#$(includedir)#g' \ -e 's#__LIBDIR__#$(libdir)#g' \ <$(srcdir)/atf-c/atf-c.pc.in >atf-c/atf-c.pc.tmp; \ @@ -83,25 +83,33 @@ tests_atf_c_DATA = atf-c/Kyuafile \ tests_atf_cdir = $(pkgtestsdir)/atf-c EXTRA_DIST += $(tests_atf_c_DATA) +ATF_C_TEST_HELPERS_CPPFLAGS = "-DATF_BUILD_CC=\"$(ATF_BUILD_CC)\"" +ATF_C_TEST_HELPERS_LDADD = atf-c/detail/libtest_helpers.la + tests_atf_c_PROGRAMS = atf-c/atf_c_test atf_c_atf_c_test_SOURCES = atf-c/atf_c_test.c -atf_c_atf_c_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_atf_c_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_atf_c_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/build_test atf_c_build_test_SOURCES = atf-c/build_test.c atf-c/h_build.h -atf_c_build_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_build_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_build_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/check_test atf_c_check_test_SOURCES = atf-c/check_test.c -atf_c_check_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_check_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_check_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/error_test atf_c_error_test_SOURCES = atf-c/error_test.c -atf_c_error_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_error_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_error_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/macros_test atf_c_macros_test_SOURCES = atf-c/macros_test.c -atf_c_macros_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_macros_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_macros_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_SCRIPTS = atf-c/pkg_config_test CLEANFILES += atf-c/pkg_config_test @@ -112,15 +120,18 @@ atf-c/pkg_config_test: $(srcdir)/atf-c/pkg_config_test.sh tests_atf_c_PROGRAMS += atf-c/tc_test atf_c_tc_test_SOURCES = atf-c/tc_test.c -atf_c_tc_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_tc_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_tc_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/tp_test atf_c_tp_test_SOURCES = atf-c/tp_test.c -atf_c_tp_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_tp_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_tp_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la tests_atf_c_PROGRAMS += atf-c/utils_test atf_c_utils_test_SOURCES = atf-c/utils_test.c atf-c/h_build.h -atf_c_utils_test_LDADD = atf-c/detail/libtest_helpers.la libatf-c.la +atf_c_utils_test_CPPFLAGS = $(ATF_C_TEST_HELPERS_CPPFLAGS) +atf_c_utils_test_LDADD = $(ATF_C_TEST_HELPERS_LDADD) libatf-c.la include atf-c/detail/Makefile.am.inc diff --git a/atf-c/atf-c.3 b/atf-c/atf-c.3 index edb7207d7cda..2e0de174c9a9 100644 --- a/atf-c/atf-c.3 +++ b/atf-c/atf-c.3 @@ -203,7 +203,7 @@ ATF provides a C programming interface to implement test programs. C-based test programs follow this template: .Bd -literal -offset indent -.Ns ... C-specific includes go here ... +\&... C-specific includes go here ... #include @@ -237,7 +237,7 @@ ATF_TC_BODY(tc3, tc) ... third test case's body ... } -.Ns ... additional test cases ... +\&... additional test cases ... ATF_TP_ADD_TCS(tp) { @@ -259,7 +259,7 @@ To define test cases, one can use the .Fn ATF_TC_WITH_CLEANUP or the .Fn ATF_TC_WITHOUT_HEAD -macros, which take a single parameter specifiying the test case's name. +macros, which take a single parameter specifying the test case's name. .Fn ATF_TC , requires to define a head and a body for the test case, .Fn ATF_TC_WITH_CLEANUP @@ -299,7 +299,7 @@ library to do it for you. This is done by using the .Fn ATF_TP_ADD_TCS macro, which is passed the name of the object that will hold the test -cases; i.e. the test program instance. +cases, i.e., the test program instance. This name can be whatever you want as long as it is a valid variable identifier. .Pp diff --git a/atf-c/check.c b/atf-c/check.c index 38afdf3743a6..1aec01bcca6b 100644 --- a/atf-c/check.c +++ b/atf-c/check.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -106,7 +107,7 @@ static int const_execvp(const char *file, const char *const *argv) { -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) return execvp(file, UNCONST(argv)); #undef UNCONST } diff --git a/atf-c/detail/fs_test.c b/atf-c/detail/fs_test.c index 3dbc4d3ba7ef..7812be0334b8 100644 --- a/atf-c/detail/fs_test.c +++ b/atf-c/detail/fs_test.c @@ -779,7 +779,7 @@ ATF_TC_BODY(rmdir_enotempty, tc) atf_fs_path_fini(&p); } -ATF_TC(rmdir_eperm); +ATF_TC_WITH_CLEANUP(rmdir_eperm); ATF_TC_HEAD(rmdir_eperm, tc) { atf_tc_set_md_var(tc, "descr", "Tests the atf_fs_rmdir function"); @@ -808,6 +808,13 @@ ATF_TC_BODY(rmdir_eperm, tc) atf_fs_path_fini(&p); } +ATF_TC_CLEANUP(rmdir_eperm, tc) +{ + if (chmod("test-dir", 0755) == -1) { + fprintf(stderr, "Failed to unprotect test-dir; test directory " + "cleanup will fail\n"); + } +} ATF_TC(mkdtemp_ok); ATF_TC_HEAD(mkdtemp_ok, tc) diff --git a/atf-c/detail/list.c b/atf-c/detail/list.c index d14216eb409f..7ac9f1fc948b 100644 --- a/atf-c/detail/list.c +++ b/atf-c/detail/list.c @@ -74,7 +74,7 @@ new_entry(void *object, bool managed) le->m_prev = le->m_next = NULL; le->m_object = object; le->m_managed = managed; - } else + } else if (managed) free(object); return le; diff --git a/atf-c/detail/process.c b/atf-c/detail/process.c index 8e08b6c57466..a6189bf78e20 100644 --- a/atf-c/detail/process.c +++ b/atf-c/detail/process.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -552,7 +553,7 @@ static int const_execvp(const char *file, const char *const *argv) { -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) return execvp(file, UNCONST(argv)); #undef UNCONST } diff --git a/atf-c/detail/test_helpers.h b/atf-c/detail/test_helpers.h index a601c293ffe4..90841f803c59 100644 --- a/atf-c/detail/test_helpers.h +++ b/atf-c/detail/test_helpers.h @@ -33,6 +33,7 @@ #include +#include #include #include @@ -46,8 +47,11 @@ struct atf_fs_path; ATF_TC(name); \ ATF_TC_HEAD(name, tc) \ { \ + const char *cc; \ atf_tc_set_md_var(tc, "descr", "Tests that the " hdrname " file can " \ "be included on its own, without any prerequisites"); \ + cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \ + atf_tc_set_md_var(tc, "require.progs", cc); \ } \ ATF_TC_BODY(name, tc) \ { \ @@ -58,7 +62,10 @@ struct atf_fs_path; ATF_TC(name); \ ATF_TC_HEAD(name, tc) \ { \ + const char *cc; \ atf_tc_set_md_var(tc, "descr", descr); \ + cc = atf_env_get_with_default("ATF_BUILD_CC", ATF_BUILD_CC); \ + atf_tc_set_md_var(tc, "require.progs", cc); \ } \ ATF_TC_BODY(name, tc) \ { \ diff --git a/atf-c/tc.c b/atf-c/tc.c index 92c3e12c99b1..69b31123f3a3 100644 --- a/atf-c/tc.c +++ b/atf-c/tc.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,7 @@ enum expect_type { struct context { const atf_tc_t *tc; const char *resfile; + int resfilefd; size_t fail_count; enum expect_type expect; @@ -73,12 +75,14 @@ struct context { }; static void context_init(struct context *, const atf_tc_t *, const char *); +static void context_set_resfile(struct context *, const char *); +static void context_close_resfile(struct context *); static void check_fatal_error(atf_error_t); static void report_fatal_error(const char *, ...) ATF_DEFS_ATTRIBUTE_NORETURN; static atf_error_t write_resfile(const int, const char *, const int, const atf_dynstr_t *); -static void create_resfile(const char *, const char *, const int, +static void create_resfile(struct context *, const char *, const int, atf_dynstr_t *); static void error_in_expect(struct context *, const char *, ...) ATF_DEFS_ATTRIBUTE_NORETURN; @@ -102,11 +106,16 @@ static void errno_test(struct context *, const char *, const size_t, static atf_error_t check_prog_in_dir(const char *, void *); static atf_error_t check_prog(struct context *, const char *); +/* No prototype in header for this one, it's a little sketchy (internal). */ +void atf_tc_set_resultsfile(const char *); + static void context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile) { + ctx->tc = tc; - ctx->resfile = resfile; + ctx->resfilefd = -1; + context_set_resfile(ctx, resfile); ctx->fail_count = 0; ctx->expect = EXPECT_PASS; check_fatal_error(atf_dynstr_init(&ctx->expect_reason)); @@ -116,6 +125,41 @@ context_init(struct context *ctx, const atf_tc_t *tc, const char *resfile) ctx->expect_signo = 0; } +static void +context_set_resfile(struct context *ctx, const char *resfile) +{ + atf_error_t err; + + context_close_resfile(ctx); + ctx->resfile = resfile; + if (strcmp(resfile, "/dev/stdout") == 0) + ctx->resfilefd = STDOUT_FILENO; + else if (strcmp(resfile, "/dev/stderr") == 0) + ctx->resfilefd = STDERR_FILENO; + else + ctx->resfilefd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (ctx->resfilefd == -1) { + err = atf_libc_error(errno, + "Cannot create results file '%s'", resfile); + check_fatal_error(err); + } + + ctx->resfile = resfile; +} + +static void +context_close_resfile(struct context *ctx) +{ + + if (ctx->resfilefd == -1) + return; + if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO) + close(ctx->resfilefd); + ctx->resfilefd = -1; + ctx->resfile = NULL; +} + static void check_fatal_error(atf_error_t err) { @@ -162,7 +206,7 @@ write_resfile(const int fd, const char *result, const int arg, INV(arg == -1 || reason != NULL); -#define UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#define UNCONST(a) ((void *)(uintptr_t)(const void *)(a)) iov[count].iov_base = UNCONST(result); iov[count++].iov_len = strlen(result); @@ -202,26 +246,23 @@ write_resfile(const int fd, const char *result, const int arg, * not return any error code. */ static void -create_resfile(const char *resfile, const char *result, const int arg, +create_resfile(struct context *ctx, const char *result, const int arg, atf_dynstr_t *reason) { atf_error_t err; - if (strcmp("/dev/stdout", resfile) == 0) { - err = write_resfile(STDOUT_FILENO, result, arg, reason); - } else if (strcmp("/dev/stderr", resfile) == 0) { - err = write_resfile(STDERR_FILENO, result, arg, reason); - } else { - const int fd = open(resfile, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (fd == -1) { - err = atf_libc_error(errno, "Cannot create results file '%s'", - resfile); - } else { - err = write_resfile(fd, result, arg, reason); - close(fd); - } - } + /* + * We'll attempt to truncate the results file, but only if it's not pointed + * at stdout/stderr. We could just blindly ftruncate() here, but it may + * be that stdout/stderr have been redirected to a file that we want to + * validate expectations on, for example. Kyua will want the truncation, + * but it will also redirect the results directly to some file and we'll + * have no issue here. + */ + if (ctx->resfilefd != STDOUT_FILENO && ctx->resfilefd != STDERR_FILENO && + ftruncate(ctx->resfilefd, 0) != -1) + lseek(ctx->resfilefd, 0, SEEK_SET); + err = write_resfile(ctx->resfilefd, result, arg, reason); if (reason != NULL) atf_dynstr_fini(reason); @@ -280,7 +321,8 @@ expected_failure(struct context *ctx, atf_dynstr_t *reason) { check_fatal_error(atf_dynstr_prepend_fmt(reason, "%s: ", atf_dynstr_cstring(&ctx->expect_reason))); - create_resfile(ctx->resfile, "expected_failure", -1, reason); + create_resfile(ctx, "expected_failure", -1, reason); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } @@ -290,7 +332,8 @@ fail_requirement(struct context *ctx, atf_dynstr_t *reason) if (ctx->expect == EXPECT_FAIL) { expected_failure(ctx, reason); } else if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "failed", -1, reason); + create_resfile(ctx, "failed", -1, reason); + context_close_resfile(ctx); exit(EXIT_FAILURE); } else { error_in_expect(ctx, "Test case raised a failure but was not " @@ -325,7 +368,8 @@ pass(struct context *ctx) error_in_expect(ctx, "Test case was expecting a failure but got " "a pass instead"); } else if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "passed", -1, NULL); + create_resfile(ctx, "passed", -1, NULL); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } else { error_in_expect(ctx, "Test case asked to explicitly pass but was " @@ -338,7 +382,8 @@ static void skip(struct context *ctx, atf_dynstr_t *reason) { if (ctx->expect == EXPECT_PASS) { - create_resfile(ctx->resfile, "skipped", -1, reason); + create_resfile(ctx, "skipped", -1, reason); + context_close_resfile(ctx); exit(EXIT_SUCCESS); } else { error_in_expect(ctx, "Can only skip a test case when running in " @@ -942,7 +987,7 @@ _atf_tc_expect_exit(struct context *ctx, const int exitcode, const char *reason, check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_exit", exitcode, &formatted); + create_resfile(ctx, "expected_exit", exitcode, &formatted); } static void @@ -959,7 +1004,7 @@ _atf_tc_expect_signal(struct context *ctx, const int signo, const char *reason, check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_signal", signo, &formatted); + create_resfile(ctx, "expected_signal", signo, &formatted); } static void @@ -975,7 +1020,7 @@ _atf_tc_expect_death(struct context *ctx, const char *reason, va_list ap) check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_death", -1, &formatted); + create_resfile(ctx, "expected_death", -1, &formatted); } static void @@ -991,7 +1036,14 @@ _atf_tc_expect_timeout(struct context *ctx, const char *reason, va_list ap) check_fatal_error(atf_dynstr_init_ap(&formatted, reason, ap2)); va_end(ap2); - create_resfile(ctx->resfile, "expected_timeout", -1, &formatted); + create_resfile(ctx, "expected_timeout", -1, &formatted); +} + +static void +_atf_tc_set_resultsfile(struct context *ctx, const char *file) +{ + + context_set_resfile(ctx, file); } /* --------------------------------------------------------------------- @@ -1215,3 +1267,13 @@ atf_tc_expect_timeout(const char *reason, ...) _atf_tc_expect_timeout(&Current, reason, ap); va_end(ap); } + +/* Internal! */ +void +atf_tc_set_resultsfile(const char *file) +{ + + PRE(Current.tc != NULL); + + _atf_tc_set_resultsfile(&Current, file); +} diff --git a/atf-c/utils.c b/atf-c/utils.c index 1e2aac1ed3b6..d8355bc68936 100644 --- a/atf-c/utils.c +++ b/atf-c/utils.c @@ -41,6 +41,9 @@ #include "atf-c/detail/dynstr.h" +/* No prototype in header for this one, it's a little sketchy (internal). */ +void atf_tc_set_resultsfile(const char *); + /** Allocate a filename to be used by atf_utils_{fork,wait}. * * In case of a failure, marks the calling test as failed when in_parent is @@ -271,6 +274,13 @@ atf_utils_fork(void) return pid; } +void +atf_utils_reset_resultsfile(void) +{ + + atf_tc_set_resultsfile("/dev/null"); +} + /** Frees an dynamically-allocated "argv" array. * * \param argv A dynamically-allocated array of dynamically-allocated diff --git a/atf-c/utils.h b/atf-c/utils.h index e4162b215fe5..422186a31e76 100644 --- a/atf-c/utils.h +++ b/atf-c/utils.h @@ -46,5 +46,6 @@ bool atf_utils_grep_string(const char *, const char *, ...) char *atf_utils_readline(int); void atf_utils_redirect(const int, const char *); void atf_utils_wait(const pid_t, const int, const char *, const char *); +void atf_utils_reset_resultsfile(void); #endif /* !defined(ATF_C_UTILS_H) */ diff --git a/atf-c/utils_test.c b/atf-c/utils_test.c index fb81cd3a1d9e..9d8f69683b9a 100644 --- a/atf-c/utils_test.c +++ b/atf-c/utils_test.c @@ -395,6 +395,7 @@ fork_and_wait(const int exitstatus, const char* expout, const char* experr) fprintf(stderr, "Some error\n"); exit(123); } + atf_utils_reset_resultsfile(); atf_utils_wait(pid, exitstatus, expout, experr); exit(EXIT_SUCCESS); } diff --git a/atf-sh/.gitignore b/atf-sh/.gitignore new file mode 100644 index 000000000000..a29438f1a9b8 --- /dev/null +++ b/atf-sh/.gitignore @@ -0,0 +1,2 @@ +atf-check +atf-sh diff --git a/atf-sh/atf-check.1 b/atf-sh/atf-check.1 index a3bd379c7438..b03058e8442c 100644 --- a/atf-sh/atf-check.1 +++ b/atf-sh/atf-check.1 @@ -22,7 +22,7 @@ .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd October 5, 2014 +.Dd June 21, 2020 .Dt ATF-CHECK 1 .Os .Sh NAME @@ -40,10 +40,12 @@ executes a given command and analyzes its results, including exit code, stdout and stderr. .Pp -.Em Test cases must use -.Em Xr atf-sh 3 Ns ' Ns s -.Em Nm atf_check -.Em builtin function instead of calling this utility directly. +.Bf Em +Test cases must use +.Xr atf-sh 3 Ns ' Ns s +.Nm atf_check +builtin function instead of calling this utility directly. +.Ef .Pp In the first synopsis form, .Nm @@ -118,10 +120,15 @@ as a shell command line, executing it with the system shell defined by .Va ATF_SHELL . You should avoid using this flag if at all possible to prevent shell quoting issues. +.It Fl r Ar timeout[:interval] +Repeats failed checks until the +.Ar timeout +(in seconds) expires. +If unspecified, the default +.Ar interval +(in milliseconds) is 50 ms. +This can be used to wait for an expected update to the contents of a file. .El -.Sh EXIT STATUS -.Nm -exits 0 on success, and other (unspecified) value on failure. .Sh ENVIRONMENT .Bl -tag -width ATFXSHELLXX -compact .It Va ATF_SHELL @@ -129,6 +136,9 @@ Path to the system shell to be used when the .Fl x is given to run commands. .El +.Sh EXIT STATUS +.Nm +exits 0 on success, and other (unspecified) value on failure. .Sh EXAMPLES The following are sample invocations from within a test case. Note that we use the @@ -155,6 +165,11 @@ atf_check -s signal:sigsegv my_program # Combined checks atf_check -o match:foo -o not-match:bar echo foo baz + +# Wait 5 seconds for a line to show up in a file +( sleep 2 ; echo "testing 123" > $test_path ) & +atf-check -o ignore -e ignore -s exit:0 -r 5 \e + grep "testing 123" $test_path .Ed .Sh SEE ALSO .Xr atf-sh 1 diff --git a/atf-sh/atf-check.cpp b/atf-sh/atf-check.cpp index 866b7bb483a2..c0658935086a 100644 --- a/atf-sh/atf-check.cpp +++ b/atf-sh/atf-check.cpp @@ -29,6 +29,7 @@ extern "C" { #include #include +#include #include } @@ -53,6 +54,10 @@ extern "C" { #include "atf-c++/detail/sanity.hpp" #include "atf-c++/detail/text.hpp" +static const useconds_t seconds_in_useconds = (1000 * 1000); +static const useconds_t mseconds_in_useconds = 1000; +static const useconds_t useconds_in_nseconds = 1000; + // ------------------------------------------------------------------------ // Auxiliary functions. // ------------------------------------------------------------------------ @@ -162,6 +167,33 @@ class temp_file : public std::ostream { } // anonymous namespace +static useconds_t +get_monotonic_useconds(void) +{ + struct timespec ts; + useconds_t res; + int rc; + + rc = clock_gettime(CLOCK_MONOTONIC, &ts); + if (rc != 0) + throw std::runtime_error("clock_gettime: " + + std::string(strerror(errno))); + + res = ts.tv_sec * seconds_in_useconds; + res += ts.tv_nsec / useconds_in_nseconds; + return res; +} + +static bool +timo_expired(useconds_t timeout) +{ + + if (get_monotonic_useconds() >= timeout) + return true; + return false; +} + + static int parse_exit_code(const std::string& str) { @@ -216,7 +248,7 @@ parse_signal(const std::string& str) if (signo == INT_MIN) { try { return atf::text::to_type< int >(str); - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { throw atf::application::usage_error("Invalid signal name or number " "in -s option"); } @@ -306,6 +338,62 @@ parse_output_check_arg(const std::string& arg) return output_check(type, negated, arg.substr(delimiter + 1)); } +static void +parse_repeat_check_arg(const std::string& arg, useconds_t *m_timo, + useconds_t *m_interval) +{ + const std::string::size_type delimiter = arg.find(':'); + const bool has_interval = (delimiter != std::string::npos); + const std::string timo_str = arg.substr(0, delimiter); + + long l; + char *end; + + // There is no reason this couldn't be a non-integer number of seconds, + // this was just easy to do for now. + errno = 0; + l = strtol(timo_str.c_str(), &end, 10); + if (errno == ERANGE) + throw atf::application::usage_error("Bogus timeout in seconds"); + else if (errno != 0) + throw atf::application::usage_error("Timeout must be a number"); + + if (*end != 0) + throw atf::application::usage_error("Timeout must be a number"); + + *m_timo = get_monotonic_useconds() + (l * seconds_in_useconds); + // 50 milliseconds is chosen arbitrarily. There is a tradeoff between + // longer and shorter poll times. A shorter poll time makes for faster + // tests. A longer poll time makes for lower CPU overhead for the polled + // operation. 50ms is chosen with these tradeoffs in mind: on + // microcontrollers, the hope is that we can still avoid meaningful CPU use + // with a small test every 50ms. And on typical fast x86 hardware, our + // tests can be much more precise with time wasted than they typically are + // without this feature. + *m_interval = 50 * mseconds_in_useconds; + + if (!has_interval) + return; + + const std::string intv_str = arg.substr(delimiter + 1, std::string::npos); + + // Same -- this could be non-integer milliseconds. + errno = 0; + l = strtol(intv_str.c_str(), &end, 10); + if (errno == ERANGE) + throw atf::application::usage_error( + "Bogus repeat interval in milliseconds"); + else if (errno != 0) + throw atf::application::usage_error( + "Repeat interval must be a number"); + + if (*end != 0) + throw atf::application::usage_error( + "Repeat interval must be a number"); + + *m_interval = l * mseconds_in_useconds; +} + static std::string flatten_argv(char* const* argv) @@ -693,8 +781,12 @@ run_output_checks(const std::vector< output_check >& checks, namespace { class atf_check : public atf::application::app { + bool m_rflag; bool m_xflag; + useconds_t m_timo; + useconds_t m_interval; + std::vector< status_check > m_status_checks; std::vector< output_check > m_stdout_checks; std::vector< output_check > m_stderr_checks; @@ -721,6 +813,7 @@ const char* atf_check::m_description = atf_check::atf_check(void) : app(m_description, "atf-check(1)"), + m_rflag(false), m_xflag(false) { } @@ -764,6 +857,8 @@ atf_check::specific_options(void) opts.insert(option('e', "action:arg", "Handle stderr. Action must be " "one of: empty ignore file: inline: match:regexp " "save:")); + opts.insert(option('r', "timeout[:interval]", "Repeat failed check until " + "the timeout expires.")); opts.insert(option('x', "", "Execute command as a shell command")); return opts; @@ -785,6 +880,11 @@ atf_check::process_option(int ch, const char* arg) m_stderr_checks.push_back(parse_output_check_arg(arg)); break; + case 'r': + m_rflag = true; + parse_repeat_check_arg(arg, &m_timo, &m_interval); + break; + case 'x': m_xflag = true; break; @@ -802,9 +902,6 @@ atf_check::main(void) int status = EXIT_FAILURE; - std::auto_ptr< atf::check::check_result > r = - m_xflag ? execute_with_shell(m_argv) : execute(m_argv); - if (m_status_checks.empty()) m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS)); else if (m_status_checks.size() > 1) { @@ -817,12 +914,23 @@ atf_check::main(void) if (m_stderr_checks.empty()) m_stderr_checks.push_back(output_check(oc_empty, false, "")); - if ((run_status_checks(m_status_checks, *r) == false) || - (run_output_checks(*r, "stderr") == false) || - (run_output_checks(*r, "stdout") == false)) - status = EXIT_FAILURE; - else - status = EXIT_SUCCESS; + do { + std::auto_ptr< atf::check::check_result > r = + m_xflag ? execute_with_shell(m_argv) : execute(m_argv); + + if ((run_status_checks(m_status_checks, *r) == false) || + (run_output_checks(*r, "stderr") == false) || + (run_output_checks(*r, "stdout") == false)) + status = EXIT_FAILURE; + else + status = EXIT_SUCCESS; + + if (m_rflag && status == EXIT_FAILURE) { + if (timo_expired(m_timo)) + break; + usleep(m_interval); + } + } while (m_rflag && status == EXIT_FAILURE); return status; } diff --git a/atf-sh/atf-sh.1 b/atf-sh/atf-sh.1 index e7eaff8b676a..9450c5f7f7dc 100644 --- a/atf-sh/atf-sh.1 +++ b/atf-sh/atf-sh.1 @@ -27,10 +27,10 @@ .Os .Sh NAME .Nm atf-sh -.Op Fl s Ar shell .Nd interpreter for shell-based test programs .Sh SYNOPSIS .Nm +.Op Fl s Ar shell .Ar script .Sh DESCRIPTION .Nm diff --git a/atf-sh/atf-sh.3 b/atf-sh/atf-sh.3 index be56539d04ec..c27a8956b885 100644 --- a/atf-sh/atf-sh.3 +++ b/atf-sh/atf-sh.3 @@ -22,13 +22,14 @@ .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd October 13, 2014 +.Dd June 08, 2017 .Dt ATF-SH 3 .Os .Sh NAME .Nm atf_add_test_case , .Nm atf_check , .Nm atf_check_equal , +.Nm atf_check_not_equal , .Nm atf_config_get , .Nm atf_config_has , .Nm atf_expect_death , @@ -54,6 +55,9 @@ .Nm atf_check_equal .Qq expected_expression .Qq actual_expression +.Nm atf_check_not_equal +.Qq expected_expression +.Qq actual_expression .Nm atf_config_get .Qq var_name .Nm atf_config_has @@ -129,7 +133,7 @@ tc2_cleanup() { ... second test case's cleanup ... } -.Ns ... additional test cases ... +\&... additional test cases ... atf_init_test_cases() { atf_add_test_case tc1 @@ -144,7 +148,7 @@ described in .Xr atf-test-case 4 . To define test cases, one can use the .Nm atf_test_case -function, which takes a first parameter specifiying the test case's +function, which takes a first parameter specifying the test case's name and instructs the library to set things up to accept it as a valid test case. The second parameter is optional and, if provided, must be @@ -307,6 +311,11 @@ This function takes two expressions, evaluates them and, if their results differ, aborts the test case with an appropriate failure message. The common style is to put the expected value in the first parameter and the actual value in the second parameter. +.It Nm atf_check_not_equal Qo expected_expression Qc Qo actual_expression Qc +This function takes two expressions, evaluates them and, if their +results are equal, aborts the test case with an appropriate failure message. +The common style is to put the expected value in the first parameter and the +actual value in the second parameter. .El .Sh EXAMPLES The following shows a complete test program with a single test case that @@ -334,7 +343,7 @@ atf_init_test_cases() { This other example shows how to include a file with extra helper functions in the test program: .Bd -literal -offset indent -.Ns ... definition of test cases ... +\&... definition of test cases ... atf_init_test_cases() { . $(atf_get_srcdir)/helper_functions.sh diff --git a/atf-sh/atf_check_test.sh b/atf-sh/atf_check_test.sh index 9e3cfb955f68..6fe2bb775274 100644 --- a/atf-sh/atf_check_test.sh +++ b/atf-sh/atf_check_test.sh @@ -164,6 +164,30 @@ equal_body() grep '^failed: \${x} != \${y} (a != b)$' resfile } +atf_test_case not_equal +not_equal_head() +{ + atf_set "descr" "Verifies that atf_check_not_equal works" +} +not_equal_body() +{ + h="$(atf_get_srcdir)/misc_helpers -s $(atf_get_srcdir)" + + atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_ok" + + atf_check -s eq:1 -o ignore -e ignore -x \ + "${h} -r resfile atf_check_not_equal_fail" + atf_check -s eq:0 -o ignore -e empty grep '^failed: a == b (a == b)$' \ + resfile + + atf_check -s eq:0 -o ignore -e ignore -x "${h} atf_check_not_equal_eval_ok" + + atf_check -s eq:1 -o ignore -e ignore -x \ + "${h} -r resfile atf_check_not_equal_eval_fail" + atf_check -s eq:0 -o ignore -e empty \ + grep '^failed: \${x} == \${y} (a == b)$' resfile +} + atf_test_case flush_stdout_on_death flush_stdout_on_death_body() { diff --git a/atf-sh/libatf-sh.subr b/atf-sh/libatf-sh.subr index a078975400af..a2478b6a9be1 100644 --- a/atf-sh/libatf-sh.subr +++ b/atf-sh/libatf-sh.subr @@ -100,6 +100,23 @@ atf_check_equal() atf_fail "${1} != ${2} (${_val1} != ${_val2})" } +# +# atf_check_not_equal expected_expression actual_expression +# +# Checks that expected_expression's value does not match actual_expression's +# and, if it does, raises an error. Ideally expected_expression and +# actual_expression should be provided quoted (not expanded) so that +# the error message is helpful; otherwise it will only show the values, +# not the expressions themselves. +# +atf_check_not_equal() +{ + eval _val1=\"${1}\" + eval _val2=\"${2}\" + test "${_val1}" != "${_val2}" || \ + atf_fail "${1} == ${2} (${_val1} == ${_val2})" +} + # # atf_config_get varname [defvalue] # @@ -536,7 +553,18 @@ _atf_list_tcs() # _atf_normalize() { - echo ${1} | tr .- __ + # Check if the string contains any of the forbidden characters using + # POSIX parameter expansion (the ${var//} string substitution is + # unfortunately not supported in POSIX sh) and only use tr(1) then. + # tr(1) is generally not a builtin, so doing the substring check first + # avoids unnecessary fork()+execve() calls. As this function is called + # many times in each test script startup, those overheads add up + # (especially when running on emulated platforms such as QEMU). + if [ "${1#*[.-]}" != "$1" ]; then + echo "$1" | tr .- __ + else + echo "$1" + fi } # @@ -734,7 +762,7 @@ main() ;; esac done - shift `expr ${OPTIND} - 1` + shift $((OPTIND - 1)) case ${Source_Dir} in /*) diff --git a/atf-sh/misc_helpers.sh b/atf-sh/misc_helpers.sh index ca0f4650d99b..a2b2c0b53d73 100644 --- a/atf-sh/misc_helpers.sh +++ b/atf-sh/misc_helpers.sh @@ -139,6 +139,50 @@ atf_check_equal_eval_fail_body() atf_check_equal '${x}' '${y}' } +atf_test_case atf_check_not_equal_ok +atf_check_not_equal_ok_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_ok_body() +{ + atf_check_not_equal a b +} + +atf_test_case atf_check_not_equal_fail +atf_check_not_equal_fail_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_fail_body() +{ + atf_check_not_equal a a +} + +atf_test_case atf_check_not_equal_eval_ok +atf_check_not_equal_eval_ok_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_eval_ok_body() +{ + x=a + y=b + atf_check_not_equal '${x}' '${y}' +} + +atf_test_case atf_check_not_equal_eval_fail +atf_check_not_equal_eval_fail_head() +{ + atf_set "descr" "Helper test case for the t_atf_check test program" +} +atf_check_not_equal_eval_fail_body() +{ + x=a + y=a + atf_check_not_equal '${x}' '${y}' +} + atf_test_case atf_check_flush_stdout atf_check_flush_stdout_head() { @@ -285,6 +329,10 @@ atf_init_test_cases() atf_add_test_case atf_check_equal_fail atf_add_test_case atf_check_equal_eval_ok atf_add_test_case atf_check_equal_eval_fail + atf_add_test_case atf_check_not_equal_ok + atf_add_test_case atf_check_not_equal_fail + atf_add_test_case atf_check_not_equal_eval_ok + atf_add_test_case atf_check_not_equal_eval_fail atf_add_test_case atf_check_flush_stdout # Add helper tests for t_config. diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 000000000000..63a4854a4e3f --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1,11 @@ +atconfig +h_app_empty +h_app_opts_args +h_tp_atf_check_sh +h_tp_basic_c +h_tp_basic_cpp +h_tp_basic_sh +h_tp_fail +h_tp_pass +package.m4 +testsuite diff --git a/bootstrap/h_tp_atf_check_sh.sh b/bootstrap/h_tp_atf_check_sh.sh old mode 100755 new mode 100644 diff --git a/bootstrap/h_tp_basic_sh.sh b/bootstrap/h_tp_basic_sh.sh old mode 100755 new mode 100644 diff --git a/bootstrap/h_tp_fail.sh b/bootstrap/h_tp_fail.sh old mode 100755 new mode 100644 diff --git a/bootstrap/h_tp_pass.sh b/bootstrap/h_tp_pass.sh old mode 100755 new mode 100644 diff --git a/configure.ac b/configure.ac index 14511480c83a..7e6c0e92dc7a 100644 --- a/configure.ac +++ b/configure.ac @@ -27,7 +27,7 @@ dnl ----------------------------------------------------------------------- dnl Initialize the GNU build system. dnl ----------------------------------------------------------------------- -AC_INIT([Automated Testing Framework], [0.21], +AC_INIT([Automated Testing Framework], [0.22], [atf-discuss@googlegroups.com], [atf], [https://github.com/jmmv/atf/]) AC_PREREQ([2.65]) diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000000..7c3185645015 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +atf.7 diff --git a/doc/atf-test-case.4 b/doc/atf-test-case.4 index 30254118ce6d..52db09d5da78 100644 --- a/doc/atf-test-case.4 +++ b/doc/atf-test-case.4 @@ -22,7 +22,7 @@ .\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN .\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -.Dd October 5, 2014 +.Dd July 3, 2015 .Dt ATF-TEST-CASE 4 .Os .Sh NAME @@ -79,8 +79,8 @@ Upon termination, a test case reports a status and, optionally, a textual reason describing why the test reported such status. The caller must ensure that the test case really performed the task that its status describes, as the test program may be bogus and therefore providing a -misleading result (e.g. providing a result that indicates success but the -error code of the program says otherwise). +misleading result, e.g., providing a result that indicates success but the +error code of the program says otherwise. .Pp The possible exit status of a test case are one of the following: .Bl -tag -width expectedXfailureXX @@ -149,11 +149,7 @@ APIs to implement the test cases. The standard input of the test cases is unconditionally connected to .Sq /dev/zero . .Ss Meta-data -The following list describes all meta-data properties interpreted -internally by ATF. -You are free to define new properties in your test cases and use them as -you wish, but non-standard properties must be prefixed by -.Sq X- . +The following metadata properties can be exposed via the test case's head: .Bl -tag -width requireXmachineXX .It descr Type: textual. @@ -275,6 +271,17 @@ test program. Can optionally be set to zero, in which case the test case has no run-time limit. This is discouraged. +.It X- Ns Sq NAME +Type: textual. +Optional. +.Pp +A user-defined property named +.Sq NAME . +These properties are free form, have no special meaning within ATF, and can +be specified at will by the test case. +The runtime engine should propagate these properties from the test case to +the end user so that the end user can rely on custom properties for test case +tagging and classification. .El .Ss Environment Every time a test case is executed, several environment variables are diff --git a/m4/.gitignore b/m4/.gitignore new file mode 100644 index 000000000000..38066ddf7cad --- /dev/null +++ b/m4/.gitignore @@ -0,0 +1,5 @@ +libtool.m4 +ltoptions.m4 +ltsugar.m4 +ltversion.m4 +lt~obsolete.m4