c243e4902b
Provenance: http://code.google.com/p/kyua Discussed with: gibbs, gnn, keramida, mdf, mlaier, Discussed with: Garrett Cooper
421 lines
13 KiB
Groff
421 lines
13 KiB
Groff
.\"
|
|
.\" Automated Testing Framework (atf)
|
|
.\"
|
|
.\" Copyright (c) 2008 The NetBSD Foundation, Inc.
|
|
.\" All rights reserved.
|
|
.\"
|
|
.\" Redistribution and use in source and binary forms, with or without
|
|
.\" modification, are permitted provided that the following conditions
|
|
.\" are met:
|
|
.\" 1. Redistributions of source code must retain the above copyright
|
|
.\" notice, this list of conditions and the following disclaimer.
|
|
.\" 2. Redistributions in binary form must reproduce the above copyright
|
|
.\" notice, this list of conditions and the following disclaimer in the
|
|
.\" documentation and/or other materials provided with the distribution.
|
|
.\"
|
|
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
|
|
.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
|
.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
.\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
|
|
.\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
|
.\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
|
.\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
|
|
.\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
.\"
|
|
.Dd January 21, 2012
|
|
.Dt ATF-C++-API 3
|
|
.Os
|
|
.Sh NAME
|
|
.Nm ATF_ADD_TEST_CASE ,
|
|
.Nm ATF_CHECK_ERRNO ,
|
|
.Nm ATF_FAIL ,
|
|
.Nm ATF_INIT_TEST_CASES ,
|
|
.Nm ATF_PASS ,
|
|
.Nm ATF_REQUIRE ,
|
|
.Nm ATF_REQUIRE_EQ ,
|
|
.Nm ATF_REQUIRE_ERRNO ,
|
|
.Nm ATF_REQUIRE_IN ,
|
|
.Nm ATF_REQUIRE_MATCH ,
|
|
.Nm ATF_REQUIRE_NOT_IN ,
|
|
.Nm ATF_REQUIRE_THROW ,
|
|
.Nm ATF_REQUIRE_THROW_RE ,
|
|
.Nm ATF_SKIP ,
|
|
.Nm ATF_TEST_CASE ,
|
|
.Nm ATF_TEST_CASE_BODY ,
|
|
.Nm ATF_TEST_CASE_CLEANUP ,
|
|
.Nm ATF_TEST_CASE_HEAD ,
|
|
.Nm ATF_TEST_CASE_NAME ,
|
|
.Nm ATF_TEST_CASE_USE ,
|
|
.Nm ATF_TEST_CASE_WITH_CLEANUP ,
|
|
.Nm ATF_TEST_CASE_WITHOUT_HEAD ,
|
|
.Nd C++ API to write ATF-based test programs
|
|
.Sh SYNOPSIS
|
|
.In atf-c++.hpp
|
|
.Fn ATF_ADD_TEST_CASE "tcs" "name"
|
|
.Fn ATF_CHECK_ERRNO "exp_errno" "bool_expression"
|
|
.Fn ATF_FAIL "reason"
|
|
.Fn ATF_INIT_TEST_CASES "tcs"
|
|
.Fn ATF_PASS
|
|
.Fn ATF_REQUIRE "expression"
|
|
.Fn ATF_REQUIRE_EQ "expression_1" "expression_2"
|
|
.Fn ATF_REQUIRE_ERRNO "exp_errno" "bool_expression"
|
|
.Fn ATF_REQUIRE_IN "element" "collection"
|
|
.Fn ATF_REQUIRE_MATCH "regexp" "string_expression"
|
|
.Fn ATF_REQUIRE_NOT_IN "element" "collection"
|
|
.Fn ATF_REQUIRE_THROW "expected_exception" "statement"
|
|
.Fn ATF_REQUIRE_THROW_RE "expected_exception" "regexp" "statement"
|
|
.Fn ATF_SKIP "reason"
|
|
.Fn ATF_TEST_CASE "name"
|
|
.Fn ATF_TEST_CASE_BODY "name"
|
|
.Fn ATF_TEST_CASE_CLEANUP "name"
|
|
.Fn ATF_TEST_CASE_HEAD "name"
|
|
.Fn ATF_TEST_CASE_NAME "name"
|
|
.Fn ATF_TEST_CASE_USE "name"
|
|
.Fn ATF_TEST_CASE_WITH_CLEANUP "name"
|
|
.Fn ATF_TEST_CASE_WITHOUT_HEAD "name"
|
|
.Sh DESCRIPTION
|
|
ATF provides a mostly-macro-based programming interface to implement test
|
|
programs in C or C++.
|
|
This interface is backed by a C++ implementation, but this fact is
|
|
hidden from the developer as much as possible through the use of
|
|
macros to simplify programming.
|
|
However, the use of C++ is not hidden everywhere and while you can
|
|
implement test cases without knowing anything at all about the object model
|
|
underneath the provided calls, you might need some minimum notions of the
|
|
language in very specific circumstances.
|
|
.Pp
|
|
C++-based test programs always follow this template:
|
|
.Bd -literal -offset indent
|
|
extern "C" {
|
|
.Ns ... C-specific includes go here ...
|
|
}
|
|
|
|
.Ns ... C++-specific includes go here ...
|
|
|
|
#include <atf-c++.hpp>
|
|
|
|
ATF_TEST_CASE(tc1);
|
|
ATF_TEST_CASE_HEAD(tc1)
|
|
{
|
|
... first test case's header ...
|
|
}
|
|
ATF_TEST_CASE_BODY(tc1)
|
|
{
|
|
... first test case's body ...
|
|
}
|
|
|
|
ATF_TEST_CASE_WITH_CLEANUP(tc2);
|
|
ATF_TEST_CASE_HEAD(tc2)
|
|
{
|
|
... second test case's header ...
|
|
}
|
|
ATF_TEST_CASE_BODY(tc2)
|
|
{
|
|
... second test case's body ...
|
|
}
|
|
ATF_TEST_CASE_CLEANUP(tc2)
|
|
{
|
|
... second test case's cleanup ...
|
|
}
|
|
|
|
ATF_TEST_CASE(tc3);
|
|
ATF_TEST_CASE_BODY(tc3)
|
|
{
|
|
... third test case's body ...
|
|
}
|
|
|
|
.Ns ... additional test cases ...
|
|
|
|
ATF_INIT_TEST_CASES(tcs)
|
|
{
|
|
ATF_ADD_TEST_CASE(tcs, tc1);
|
|
ATF_ADD_TEST_CASE(tcs, tc2);
|
|
ATF_ADD_TEST_CASE(tcs, tc3);
|
|
... add additional test cases ...
|
|
}
|
|
.Ed
|
|
.Ss Definition of test cases
|
|
Test cases have an identifier and are composed of three different parts:
|
|
the header, the body and an optional cleanup routine, all of which are
|
|
described in
|
|
.Xr atf-test-case 4 .
|
|
To define test cases, one can use the
|
|
.Fn ATF_TEST_CASE ,
|
|
.Fn ATF_TEST_CASE_WITH_CLEANUP
|
|
or the
|
|
.Fn ATF_TEST_CASE_WITHOUT_HEAD
|
|
macros, which take a single parameter specifiying the test case's
|
|
name.
|
|
.Fn ATF_TEST_CASE ,
|
|
requires to define a head and a body for the test case,
|
|
.Fn ATF_TEST_CASE_WITH_CLEANUP
|
|
requires to define a head, a body and a cleanup for the test case and
|
|
.Fn ATF_TEST_CASE_WITHOUT_HEAD
|
|
requires only a body for the test case.
|
|
It is important to note that these
|
|
.Em do not
|
|
set the test case up for execution when the program is run.
|
|
In order to do so, a later registration is needed through the
|
|
.Fn ATF_ADD_TEST_CASE
|
|
macro detailed in
|
|
.Sx Program initialization .
|
|
.Pp
|
|
Later on, one must define the three parts of the body by means of three
|
|
functions.
|
|
Their headers are given by the
|
|
.Fn ATF_TEST_CASE_HEAD ,
|
|
.Fn ATF_TEST_CASE_BODY
|
|
and
|
|
.Fn ATF_TEST_CASE_CLEANUP
|
|
macros, all of which take the test case's name.
|
|
Following each of these, a block of code is expected, surrounded by the
|
|
opening and closing brackets.
|
|
.Pp
|
|
Additionally, the
|
|
.Fn ATF_TEST_CASE_NAME
|
|
macro can be used to obtain the name of the class corresponding to a
|
|
particular test case, as the name is internally manged by the library to
|
|
prevent clashes with other user identifiers.
|
|
Similarly, the
|
|
.Fn ATF_TEST_CASE_USE
|
|
macro can be executed on a particular test case to mark it as "used" and
|
|
thus prevent compiler warnings regarding unused symbols.
|
|
Note that
|
|
.Em you should never have to use these macros during regular operation.
|
|
.Ss Program initialization
|
|
The library provides a way to easily define the test program's
|
|
.Fn main
|
|
function.
|
|
You should never define one on your own, but rely on the
|
|
library to do it for you.
|
|
This is done by using the
|
|
.Fn ATF_INIT_TEST_CASES
|
|
macro, which is passed the name of the list that will hold the test cases.
|
|
This name can be whatever you want as long as it is a valid variable value.
|
|
.Pp
|
|
After the macro, you are supposed to provide the body of a function, which
|
|
should only use the
|
|
.Fn ATF_ADD_TEST_CASE
|
|
macro to register the test cases the test program will execute.
|
|
The first parameter of this macro matches the name you provided in the
|
|
former call.
|
|
.Ss Header definitions
|
|
The test case's header can define the meta-data by using the
|
|
.Fn set
|
|
method, which takes two parameters: the first one specifies the
|
|
meta-data variable to be set and the second one specifies its value.
|
|
Both of them are strings.
|
|
.Ss Configuration variables
|
|
The test case has read-only access to the current configuration variables
|
|
by means of the
|
|
.Ft bool
|
|
.Fn has_config_var
|
|
and the
|
|
.Ft std::string
|
|
.Fn get_config_var
|
|
methods, which can be called in any of the three parts of a test case.
|
|
.Ss Access to the source directory
|
|
It is possible to get the path to the test case's source directory from any
|
|
of its three components by querying the
|
|
.Sq srcdir
|
|
configuration variable.
|
|
.Ss Requiring programs
|
|
Aside from the
|
|
.Va require.progs
|
|
meta-data variable available in the header only, one can also check for
|
|
additional programs in the test case's body by using the
|
|
.Fn require_prog
|
|
function, which takes the base name or full path of a single binary.
|
|
Relative paths are forbidden.
|
|
If it is not found, the test case will be automatically skipped.
|
|
.Ss Test case finalization
|
|
The test case finalizes either when the body reaches its end, at which
|
|
point the test is assumed to have
|
|
.Em passed ,
|
|
or at any explicit call to
|
|
.Fn ATF_PASS ,
|
|
.Fn ATF_FAIL
|
|
or
|
|
.Fn ATF_SKIP .
|
|
These three macros terminate the execution of the test case immediately.
|
|
The cleanup routine will be processed afterwards in a completely automated
|
|
way, regardless of the test case's termination reason.
|
|
.Pp
|
|
.Fn ATF_PASS
|
|
does not take any parameters.
|
|
.Fn ATF_FAIL
|
|
and
|
|
.Fn ATF_SKIP
|
|
take a single string that describes why the test case failed or
|
|
was skipped, respectively.
|
|
It is very important to provide a clear error message in both cases so that
|
|
the user can quickly know why the test did not pass.
|
|
.Ss Expectations
|
|
Everything explained in the previous section changes when the test case
|
|
expectations are redefined by the programmer.
|
|
.Pp
|
|
Each test case has an internal state called
|
|
.Sq expect
|
|
that describes what the test case expectations are at any point in time.
|
|
The value of this property can change during execution by any of:
|
|
.Bl -tag -width indent
|
|
.It Fn expect_death "reason"
|
|
Expects the test case to exit prematurely regardless of the nature of the
|
|
exit.
|
|
.It Fn expect_exit "exitcode" "reason"
|
|
Expects the test case to exit cleanly.
|
|
If
|
|
.Va exitcode
|
|
is not
|
|
.Sq -1 ,
|
|
.Xr atf-run 1
|
|
will validate that the exit code of the test case matches the one provided
|
|
in this call.
|
|
Otherwise, the exact value will be ignored.
|
|
.It Fn expect_fail "reason"
|
|
Any failure (be it fatal or non-fatal) raised in this mode is recorded.
|
|
However, such failures do not report the test case as failed; instead, the
|
|
test case finalizes cleanly and is reported as
|
|
.Sq expected failure ;
|
|
this report includes the provided
|
|
.Fa reason
|
|
as part of it.
|
|
If no error is raised while running in this mode, then the test case is
|
|
reported as
|
|
.Sq failed .
|
|
.Pp
|
|
This mode is useful to reproduce actual known bugs in tests.
|
|
Whenever the developer fixes the bug later on, the test case will start
|
|
reporting a failure, signaling the developer that the test case must be
|
|
adjusted to the new conditions.
|
|
In this situation, it is useful, for example, to set
|
|
.Fa reason
|
|
as the bug number for tracking purposes.
|
|
.It Fn expect_pass
|
|
This is the normal mode of execution.
|
|
In this mode, any failure is reported as such to the user and the test case
|
|
is marked as
|
|
.Sq failed .
|
|
.It Fn expect_race "reason"
|
|
Any failure or timeout during the execution of the test case will be
|
|
considered as if a race condition has been triggered and reported as such.
|
|
If no problems arise, the test will continue execution as usual.
|
|
.It Fn expect_signal "signo" "reason"
|
|
Expects the test case to terminate due to the reception of a signal.
|
|
If
|
|
.Va signo
|
|
is not
|
|
.Sq -1 ,
|
|
.Xr atf-run 1
|
|
will validate that the signal that terminated the test case matches the one
|
|
provided in this call.
|
|
Otherwise, the exact value will be ignored.
|
|
.It Fn expect_timeout "reason"
|
|
Expects the test case to execute for longer than its timeout.
|
|
.El
|
|
.Ss Helper macros for common checks
|
|
The library provides several macros that are very handy in multiple
|
|
situations.
|
|
These basically check some condition after executing a given statement or
|
|
processing a given expression and, if the condition is not met, they
|
|
automatically call
|
|
.Fn ATF_FAIL
|
|
with an appropriate error message.
|
|
.Pp
|
|
.Fn ATF_REQUIRE
|
|
takes an expression and raises a failure if it evaluates to false.
|
|
.Pp
|
|
.Fn ATF_REQUIRE_EQ
|
|
takes two expressions and raises a failure if the two do not evaluate to
|
|
the same exact value.
|
|
.Pp
|
|
.Fn ATF_REQUIRE_IN
|
|
takes an element and a collection and validates that the element is present in
|
|
the collection.
|
|
.Pp
|
|
.Fn ATF_REQUIRE_MATCH
|
|
takes a regular expression and a string and raises a failure if the regular
|
|
expression does not match the string.
|
|
.Pp
|
|
.Fn ATF_REQUIRE_NOT_IN
|
|
takes an element and a collection and validates that the element is not present
|
|
in the collection.
|
|
.Pp
|
|
.Fn ATF_REQUIRE_THROW
|
|
takes the name of an exception and a statement and raises a failure if
|
|
the statement does not throw the specified exception.
|
|
.Fn ATF_REQUIRE_THROW_EQ
|
|
takes the name of an exception, a regular expresion and a statement and raises a
|
|
failure if the statement does not throw the specified exception and if the
|
|
message of the exception does not match the regular expression.
|
|
.Pp
|
|
.Fn ATF_CHECK_ERRNO
|
|
and
|
|
.Fn ATF_REQUIRE_ERRNO
|
|
take, first, the error code that the check is expecting to find in the
|
|
.Va errno
|
|
variable and, second, a boolean expression that, if evaluates to true,
|
|
means that a call failed and
|
|
.Va errno
|
|
has to be checked against the first value.
|
|
.Sh EXAMPLES
|
|
The following shows a complete test program with a single test case that
|
|
validates the addition operator:
|
|
.Bd -literal -offset indent
|
|
#include <atf-c++.hpp>
|
|
|
|
ATF_TEST_CASE(addition);
|
|
ATF_TEST_CASE_HEAD(addition)
|
|
{
|
|
set("descr", "Sample tests for the addition operator");
|
|
}
|
|
ATF_TEST_CASE_BODY(addition)
|
|
{
|
|
ATF_REQUIRE_EQ(0 + 0, 0);
|
|
ATF_REQUIRE_EQ(0 + 1, 1);
|
|
ATF_REQUIRE_EQ(1 + 0, 1);
|
|
|
|
ATF_REQUIRE_EQ(1 + 1, 2);
|
|
|
|
ATF_REQUIRE_EQ(100 + 200, 300);
|
|
}
|
|
|
|
ATF_TEST_CASE(open_failure);
|
|
ATF_TEST_CASE_HEAD(open_failure)
|
|
{
|
|
set("descr", "Sample tests for the open function");
|
|
}
|
|
ATF_TEST_CASE_BODY(open_failure)
|
|
{
|
|
ATF_REQUIRE_ERRNO(ENOENT, open("non-existent", O_RDONLY) == -1);
|
|
}
|
|
|
|
ATF_TEST_CASE(known_bug);
|
|
ATF_TEST_CASE_HEAD(known_bug)
|
|
{
|
|
set("descr", "Reproduces a known bug");
|
|
}
|
|
ATF_TEST_CASE_BODY(known_bug)
|
|
{
|
|
expect_fail("See bug number foo/bar");
|
|
ATF_REQUIRE_EQ(3, 1 + 1);
|
|
expect_pass();
|
|
ATF_REQUIRE_EQ(3, 1 + 2);
|
|
}
|
|
|
|
ATF_INIT_TEST_CASES(tcs)
|
|
{
|
|
ATF_ADD_TEST_CASE(tcs, addition);
|
|
ATF_ADD_TEST_CASE(tcs, open_failure);
|
|
ATF_ADD_TEST_CASE(tcs, known_bug);
|
|
}
|
|
.Ed
|
|
.Sh SEE ALSO
|
|
.Xr atf-test-program 1 ,
|
|
.Xr atf-test-case 4 ,
|
|
.Xr atf 7
|