b0d29bc47d
Having kyua in the base system will simplify automated testing in CI and eliminates bootstrapping issues on new platforms. The build of kyua is controlled by WITH(OUT)_TESTS_SUPPORT. Reviewed by: emaste Obtained from: CheriBSD Sponsored by: DARPA Differential Revision: https://reviews.freebsd.org/D24103
201 lines
6.3 KiB
Plaintext
201 lines
6.3 KiB
Plaintext
dnl Copyright 2010 The Kyua Authors.
|
|
dnl All rights reserved.
|
|
dnl
|
|
dnl Redistribution and use in source and binary forms, with or without
|
|
dnl modification, are permitted provided that the following conditions are
|
|
dnl met:
|
|
dnl
|
|
dnl * Redistributions of source code must retain the above copyright
|
|
dnl notice, this list of conditions and the following disclaimer.
|
|
dnl * Redistributions in binary form must reproduce the above copyright
|
|
dnl notice, this list of conditions and the following disclaimer in the
|
|
dnl documentation and/or other materials provided with the distribution.
|
|
dnl * Neither the name of Google Inc. nor the names of its contributors
|
|
dnl may be used to endorse or promote products derived from this software
|
|
dnl without specific prior written permission.
|
|
dnl
|
|
dnl THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
dnl "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
dnl LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
dnl A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
dnl OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
dnl SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
dnl LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
dnl DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
dnl THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
dnl (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
dnl OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
AT_INIT([bootstrapping tests])
|
|
|
|
|
|
m4_define([GUESS_TOPDIR], {
|
|
old=$(pwd)
|
|
cd "$(dirname ${as_myself})"
|
|
# We need to locate a build product, not a source file, because the
|
|
# test suite may be run outside of the source tree (think distcheck).
|
|
while test $(pwd) != '/' -a ! -e bootstrap/plain_helpers; do
|
|
cd ..
|
|
done
|
|
topdir=$(pwd)
|
|
cd ${old}
|
|
echo ${topdir}
|
|
})
|
|
|
|
|
|
m4_define([CREATE_ATF_HELPERS], [
|
|
AT_DATA([Kyuafile], [
|
|
syntax(2)
|
|
test_suite("bootstrap")
|
|
atf_test_program{name="atf_helpers"}
|
|
])
|
|
ln -s $(GUESS_TOPDIR)/bootstrap/atf_helpers atf_helpers
|
|
])
|
|
m4_define([RUN_ATF_HELPERS],
|
|
[HOME=$(pwd) TESTS="$1" kyua --config=none \
|
|
test --results-file=bootstrap.db $2])
|
|
|
|
|
|
m4_define([CREATE_PLAIN_HELPERS], [
|
|
AT_DATA([Kyuafile], [
|
|
syntax(2)
|
|
test_suite("bootstrap")
|
|
atf_test_program{name="plain_helpers"}
|
|
])
|
|
ln -s $(GUESS_TOPDIR)/bootstrap/plain_helpers plain_helpers
|
|
])
|
|
m4_define([RUN_PLAIN_HELPER],
|
|
[HOME=$(pwd) HELPER="$1" kyua --config=none \
|
|
test --results-file=bootstrap.db])
|
|
|
|
|
|
AT_SETUP([test program crashes in test list])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_PLAIN_HELPERS
|
|
AT_CHECK([RUN_PLAIN_HELPER([abort_test_cases_list])], [1], [stdout], [])
|
|
re='plain_helpers:__test_cases_list__.*broken.*Test program received signal'
|
|
AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([test program prints an empty test list])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_PLAIN_HELPERS
|
|
AT_CHECK([RUN_PLAIN_HELPER([empty_test_cases_list])], [1], [stdout], [])
|
|
re="plain_helpers:__test_cases_list__.*broken.*Invalid header.*got ''"
|
|
AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([test program with zero test cases])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_PLAIN_HELPERS
|
|
AT_CHECK([RUN_PLAIN_HELPER([zero_test_cases])], [1], [stdout], [])
|
|
re='plain_helpers:__test_cases_list__.*broken.*No test cases'
|
|
AT_CHECK([grep "${re}" stdout], [0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run test case that passes])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([passes])], [0], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run test case that fails])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([fails])], [1], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails.*failed.*Failed on purpose" stdout],
|
|
[0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:passes" stdout], [1], [], [])
|
|
AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run test case that skips])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([skips])], [0], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes" stdout], [1], [], [])
|
|
AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
|
|
[0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run two test cases, success])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([passes skips])], [0], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails" stdout], [1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
|
|
[0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run two test cases, failure])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([fails passes])], [1], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
|
|
[1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:skips" stdout], [1], [], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run mixed test cases])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([RUN_ATF_HELPERS([fails passes skips])], [1], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
|
|
[1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
|
|
[0], [ignore], [])
|
|
|
|
AT_CLEANUP
|
|
|
|
|
|
AT_SETUP([run tests from build directories])
|
|
AT_TESTED([kyua])
|
|
|
|
CREATE_ATF_HELPERS
|
|
AT_CHECK([mkdir src], [0], [], [])
|
|
AT_CHECK([mv Kyuafile src], [0], [], [])
|
|
AT_CHECK([mkdir obj], [0], [], [])
|
|
AT_CHECK([mv atf_helpers obj], [0], [], [])
|
|
AT_CHECK([RUN_ATF_HELPERS([fails passes skips],
|
|
[--kyuafile=src/Kyuafile --build-root=obj])],
|
|
[1], [stdout], [])
|
|
AT_CHECK([grep "atf_helpers:fails.*failure.*Failed on purpose" stdout],
|
|
[1], [], [])
|
|
AT_CHECK([grep "atf_helpers:passes.*passed" stdout], [0], [ignore], [])
|
|
AT_CHECK([grep "atf_helpers:skips.*skipped.*Skipped on purpose" stdout],
|
|
[0], [ignore], [])
|
|
|
|
AT_CLEANUP
|