8fee91db34
Experimental version released on February 7th, 2014. This is the last release to bundle the code for the deprecated tools. The next release will drop their code and will stop worrying about backwards compatibility between the ATF libraries and what the old tools may or may not support. If you still require the old tools for some reason, grab a copy of the 'tools' directory now. The code in this directory is standalone and does not depend on any internal details of atf-c++ any longer. * Various fixes and improvements to support running as part of the FreeBSD test suite. * Project hosting moved from Google Code (as a subproject of Kyua) to GitHub (as a first-class project). The main reason for the change is the suppression of binary downloads in Google Code on Jan 15th, 2014. See https://github.com/jmmv/atf/ * Removed builtin help from atf-sh(1) and atf-check(1) for simplicity reasons. In other words, their -h option is gone. * Moved the code of the deprecated tools into a 'tools' directory and completely decoupled their code from the internals of atf-c++. The reason for this is to painlessly allow a third-party to maintain a copy of these tools after we delete them because upcoming changes to atf-c++ would break the stale tools.
450 lines
14 KiB
Bash
Executable File
450 lines
14 KiB
Bash
Executable File
#
|
|
# Automated Testing Framework (atf)
|
|
#
|
|
# Copyright (c) 2007 The NetBSD Foundation, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
|
|
# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
|
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
|
|
# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
|
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
|
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
|
|
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
create_helpers()
|
|
{
|
|
mkdir dir1
|
|
cp $(atf_get_srcdir)/pass_helper dir1/tp1
|
|
cp $(atf_get_srcdir)/fail_helper dir1/tp2
|
|
cp $(atf_get_srcdir)/pass_helper tp3
|
|
cp $(atf_get_srcdir)/fail_helper tp4
|
|
|
|
cat >tp5 <<EOF
|
|
#! $(atf-config -t atf_shell)
|
|
echo foo
|
|
EOF
|
|
chmod +x tp5
|
|
|
|
cat >Atffile <<EOF
|
|
Content-Type: application/X-atf-atffile; version="1"
|
|
|
|
prop: test-suite = atf
|
|
|
|
tp: dir1
|
|
tp: tp3
|
|
tp: tp4
|
|
tp: tp5
|
|
EOF
|
|
|
|
cat >dir1/Atffile <<EOF
|
|
Content-Type: application/X-atf-atffile; version="1"
|
|
|
|
prop: test-suite = atf
|
|
|
|
tp: tp1
|
|
tp: tp2
|
|
EOF
|
|
}
|
|
|
|
run_helpers()
|
|
{
|
|
mkdir etc
|
|
cat >etc/atf-run.hooks <<EOF
|
|
#! $(atf-config -t atf_shell)
|
|
|
|
info_start_hook()
|
|
{
|
|
atf_tps_writer_info "startinfo" "A value"
|
|
}
|
|
|
|
info_end_hook()
|
|
{
|
|
atf_tps_writer_info "endinfo" "Another value"
|
|
}
|
|
EOF
|
|
echo "Using atf-run to run helpers"
|
|
ATF_CONFDIR=$(pwd)/etc atf-run >tps.out 2>/dev/null
|
|
rm -rf etc
|
|
}
|
|
|
|
atf_test_case default
|
|
default_head()
|
|
{
|
|
atf_set "descr" "Checks that the default output uses the ticker" \
|
|
"format"
|
|
}
|
|
default_body()
|
|
{
|
|
create_helpers
|
|
run_helpers
|
|
|
|
# Check that the default output uses the ticker format.
|
|
atf_check -s eq:0 -o match:'test cases' -o match:'Failed test cases' \
|
|
-o match:'Summary for' -e empty -x 'atf-report <tps.out'
|
|
}
|
|
|
|
# XXX The test for all expect_ values should be intermixed with the other
|
|
# tests. However, to do that, we need to migrate to using C helpers for
|
|
# simplicity in raising signals...
|
|
atf_test_case expect
|
|
expect_body()
|
|
{
|
|
ln -s "$(atf_get_srcdir)/expect_helpers" .
|
|
cat >Atffile <<EOF
|
|
Content-Type: application/X-atf-atffile; version="1"
|
|
|
|
prop: test-suite = atf
|
|
|
|
tp: expect_helpers
|
|
EOF
|
|
run_helpers
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
tc, #.#, expect_helpers, death_and_exit, expected_death, Exit case
|
|
tc, #.#, expect_helpers, death_and_signal, expected_death, Signal case
|
|
tc, #.#, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution
|
|
tc, #.#, expect_helpers, exit_any_and_exit, expected_exit, Call will exit
|
|
tc, #.#, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution
|
|
tc, #.#, expect_helpers, exit_code_and_exit, expected_exit, Call will exit
|
|
tc, #.#, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details
|
|
tc, #.#, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure
|
|
tc, #.#, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised
|
|
tc, #.#, expect_helpers, pass_and_pass, passed
|
|
tc, #.#, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details
|
|
tc, #.#, expect_helpers, pass_but_fail_requirement, failed, Some reason
|
|
tc, #.#, expect_helpers, signal_any_and_signal, expected_signal, Call will signal
|
|
tc, #.#, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution
|
|
tc, #.#, expect_helpers, signal_no_and_signal, expected_signal, Call will signal
|
|
tc, #.#, expect_helpers, timeout_and_hang, expected_timeout, Will overrun
|
|
tc, #.#, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution
|
|
tp, #.#, expect_helpers, failed
|
|
EOF
|
|
# NO_CHECK_STYLE_END
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o csv:- <tps.out | " \
|
|
"sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
expect_helpers (1/1): 17 test cases
|
|
death_and_exit: [#.#s] Expected failure: Exit case
|
|
death_and_signal: [#.#s] Expected failure: Signal case
|
|
death_but_pass: [#.#s] Failed: Test case was expected to terminate abruptly but it continued execution
|
|
exit_any_and_exit: [#.#s] Expected failure: Call will exit
|
|
exit_but_pass: [#.#s] Failed: Test case was expected to exit cleanly but it continued execution
|
|
exit_code_and_exit: [#.#s] Expected failure: Call will exit
|
|
fail_and_fail_check: [#.#s] Expected failure: And fail again: 2 checks failed as expected; see output for more details
|
|
fail_and_fail_requirement: [#.#s] Expected failure: Fail reason: The failure
|
|
fail_but_pass: [#.#s] Failed: Test case was expecting a failure but none were raised
|
|
pass_and_pass: [#.#s] Passed.
|
|
pass_but_fail_check: [#.#s] Failed: 1 checks failed; see output for more details
|
|
pass_but_fail_requirement: [#.#s] Failed: Some reason
|
|
signal_any_and_signal: [#.#s] Expected failure: Call will signal
|
|
signal_but_pass: [#.#s] Failed: Test case was expected to receive a termination signal but it continued execution
|
|
signal_no_and_signal: [#.#s] Expected failure: Call will signal
|
|
timeout_and_hang: [#.#s] Expected failure: Will overrun
|
|
timeout_but_pass: [#.#s] Failed: Test case was expected to hang but it continued execution
|
|
[#.#s]
|
|
|
|
Test cases for known bugs:
|
|
expect_helpers:death_and_exit: Exit case
|
|
expect_helpers:death_and_signal: Signal case
|
|
expect_helpers:exit_any_and_exit: Call will exit
|
|
expect_helpers:exit_code_and_exit: Call will exit
|
|
expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details
|
|
expect_helpers:fail_and_fail_requirement: Fail reason: The failure
|
|
expect_helpers:signal_any_and_signal: Call will signal
|
|
expect_helpers:signal_no_and_signal: Call will signal
|
|
expect_helpers:timeout_and_hang: Will overrun
|
|
|
|
Failed test cases:
|
|
expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass
|
|
|
|
Summary for 1 test programs:
|
|
1 passed test cases.
|
|
7 failed test cases.
|
|
9 expected failed test cases.
|
|
0 skipped test cases.
|
|
EOF
|
|
# NO_CHECK_STYLE_END
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o ticker:- <tps.out | " \
|
|
"sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
|
|
|
|
# Just ensure that this does not crash for now...
|
|
atf_check -s eq:0 -o ignore -e empty -x "atf-report -o xml:- <tps.out"
|
|
}
|
|
|
|
atf_test_case oflag
|
|
oflag_head()
|
|
{
|
|
atf_set "descr" "Checks that the -o flag works"
|
|
}
|
|
oflag_body()
|
|
{
|
|
create_helpers
|
|
run_helpers
|
|
|
|
# Get the default output.
|
|
atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report <tps.out'
|
|
mv stdout defout
|
|
|
|
# Check that changing the stdout output works.
|
|
atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report -o csv:- <tps.out'
|
|
atf_check -s eq:1 -o empty -e empty cmp -s defout stdout
|
|
cp stdout expcsv
|
|
|
|
# Check that sending the output to a file does not write to stdout.
|
|
atf_check -s eq:0 -o empty -e empty -x 'atf-report -o csv:fmt.out <tps.out'
|
|
atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
|
|
rm -f fmt.out
|
|
|
|
# Check that defining two outputs using the same format works.
|
|
atf_check -s eq:0 -o empty -e empty -x \
|
|
'atf-report -o csv:fmt.out -o csv:fmt2.out <tps.out'
|
|
atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
|
|
atf_check -s eq:0 -o empty -e empty cmp -s fmt.out fmt2.out
|
|
rm -f fmt.out fmt2.out
|
|
|
|
# Check that defining two outputs using different formats works.
|
|
atf_check -s eq:0 -o empty -e empty -x \
|
|
'atf-report -o csv:fmt.out -o ticker:fmt2.out <tps.out'
|
|
atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
|
|
atf_check -s eq:1 -o empty -e empty cmp -s fmt.out fmt2.out
|
|
atf_check -s eq:0 -o ignore -e empty grep "test cases" fmt2.out
|
|
atf_check -s eq:0 -o ignore -e empty grep "Failed test cases" fmt2.out
|
|
atf_check -s eq:0 -o ignore -e empty grep "Summary for" fmt2.out
|
|
rm -f fmt.out fmt2.out
|
|
|
|
# Check that defining two outputs over the same file does not work.
|
|
atf_check -s eq:1 -o empty -e match:'more than once' -x \
|
|
'atf-report -o csv:fmt.out -o ticker:fmt.out <tps.out'
|
|
rm -f fmt.out
|
|
|
|
# Check that defining two outputs over stdout (but using different
|
|
# paths) does not work.
|
|
atf_check -s eq:1 -o empty -e match:'more than once' -x \
|
|
'atf-report -o csv:- -o ticker:/dev/stdout <tps.out'
|
|
rm -f fmt.out
|
|
}
|
|
|
|
atf_test_case output_csv
|
|
output_csv_head()
|
|
{
|
|
atf_set "descr" "Checks the CSV output format"
|
|
}
|
|
output_csv_body()
|
|
{
|
|
create_helpers
|
|
run_helpers
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
tc, #.#, dir1/tp1, main, passed
|
|
tp, #.#, dir1/tp1, passed
|
|
tc, #.#, dir1/tp2, main, failed, This always fails
|
|
tp, #.#, dir1/tp2, failed
|
|
tc, #.#, tp3, main, passed
|
|
tp, #.#, tp3, passed
|
|
tc, #.#, tp4, main, failed, This always fails
|
|
tp, #.#, tp4, failed
|
|
tp, #.#, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'
|
|
EOF
|
|
# NO_CHECK_STYLE_END
|
|
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o csv:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
|
|
}
|
|
|
|
atf_test_case output_ticker
|
|
output_ticker_head()
|
|
{
|
|
atf_set "descr" "Checks the ticker output format"
|
|
}
|
|
output_ticker_body()
|
|
{
|
|
create_helpers
|
|
run_helpers
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
dir1/tp1 (1/5): 1 test cases
|
|
main: [#.#s] Passed.
|
|
[#.#s]
|
|
|
|
dir1/tp2 (2/5): 1 test cases
|
|
main: [#.#s] Failed: This always fails
|
|
[#.#s]
|
|
|
|
tp3 (3/5): 1 test cases
|
|
main: [#.#s] Passed.
|
|
[#.#s]
|
|
|
|
tp4 (4/5): 1 test cases
|
|
main: [#.#s] Failed: This always fails
|
|
[#.#s]
|
|
|
|
tp5 (5/5): 0 test cases
|
|
tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:''
|
|
[#.#s]
|
|
|
|
Failed (bogus) test programs:
|
|
tp5
|
|
|
|
Failed test cases:
|
|
dir1/tp2:main, tp4:main
|
|
|
|
Summary for 5 test programs:
|
|
2 passed test cases.
|
|
2 failed test cases.
|
|
0 expected failed test cases.
|
|
0 skipped test cases.
|
|
EOF
|
|
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o ticker:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
|
|
}
|
|
# NO_CHECK_STYLE_END
|
|
|
|
atf_test_case output_xml
|
|
output_xml_head()
|
|
{
|
|
atf_set "descr" "Checks the XML output format"
|
|
}
|
|
output_xml_body()
|
|
{
|
|
create_helpers
|
|
run_helpers
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
<?xml version="1.0" encoding="ISO-8859-1"?>
|
|
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
|
|
|
|
<tests-results>
|
|
<info class="startinfo">A value</info>
|
|
<tp id="dir1/tp1">
|
|
<tc id="main">
|
|
<passed />
|
|
<tc-time>#.#</tc-time></tc>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<tp id="dir1/tp2">
|
|
<tc id="main">
|
|
<failed>This always fails</failed>
|
|
<tc-time>#.#</tc-time></tc>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<tp id="tp3">
|
|
<tc id="main">
|
|
<passed />
|
|
<tc-time>#.#</tc-time></tc>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<tp id="tp4">
|
|
<tc id="main">
|
|
<failed>This always fails</failed>
|
|
<tc-time>#.#</tc-time></tc>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<tp id="tp5">
|
|
<failed>Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'</failed>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<info class="endinfo">Another value</info>
|
|
</tests-results>
|
|
EOF
|
|
# NO_CHECK_STYLE_END
|
|
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o xml:- < tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
|
|
}
|
|
|
|
atf_test_case output_xml_space
|
|
output_xml_space_head()
|
|
{
|
|
atf_set "descr" "Checks that the XML output format properly preserves" \
|
|
"leading and trailing whitespace in stdout and stderr" \
|
|
"lines"
|
|
}
|
|
output_xml_space_body()
|
|
{
|
|
export TESTCASE=diff
|
|
cp $(atf_get_srcdir)/misc_helpers .
|
|
cat >Atffile <<EOF
|
|
Content-Type: application/X-atf-atffile; version="1"
|
|
|
|
prop: test-suite = atf
|
|
|
|
tp: misc_helpers
|
|
EOF
|
|
|
|
# NO_CHECK_STYLE_BEGIN
|
|
cat >expout <<EOF
|
|
<?xml version="1.0" encoding="ISO-8859-1"?>
|
|
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
|
|
|
|
<tests-results>
|
|
<info class="startinfo">A value</info>
|
|
<tp id="misc_helpers">
|
|
<tc id="diff">
|
|
<so>--- a 2007-11-04 14:00:41.000000000 +0100</so>
|
|
<so>+++ b 2007-11-04 14:00:48.000000000 +0100</so>
|
|
<so>@@ -1,7 +1,7 @@</so>
|
|
<so> This test is meant to simulate a diff.</so>
|
|
<so> Blank space at beginning of context lines must be preserved.</so>
|
|
<so> </so>
|
|
<so>-First original line.</so>
|
|
<so>-Second original line.</so>
|
|
<so>+First modified line.</so>
|
|
<so>+Second modified line.</so>
|
|
<so> </so>
|
|
<so> EOF</so>
|
|
<passed />
|
|
<tc-time>#.#</tc-time></tc>
|
|
<tp-time>#.#</tp-time></tp>
|
|
<info class="endinfo">Another value</info>
|
|
</tests-results>
|
|
EOF
|
|
# NO_CHECK_STYLE_END
|
|
|
|
run_helpers
|
|
atf_check -s eq:0 -o file:expout -e empty -x \
|
|
"atf-report -o xml:- <tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
|
|
}
|
|
|
|
atf_test_case too_many_args
|
|
too_many_args_body()
|
|
{
|
|
cat >experr <<EOF
|
|
atf-report: ERROR: No arguments allowed
|
|
EOF
|
|
atf_check -s eq:1 -o empty -e file:experr atf-report foo
|
|
}
|
|
|
|
atf_init_test_cases()
|
|
{
|
|
atf_add_test_case default
|
|
atf_add_test_case expect
|
|
atf_add_test_case oflag
|
|
atf_add_test_case output_csv
|
|
atf_add_test_case output_ticker
|
|
atf_add_test_case output_xml
|
|
atf_add_test_case output_xml_space
|
|
atf_add_test_case too_many_args
|
|
}
|
|
|
|
# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4
|