generic-library/googletest
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

When --gtest_filter is specified, XML report now doesn't contain information about tests that are filtered out (issue 141).

Browse Source
This commit is contained in:
vladlosev 2013-04-25 17:58:52 +00:00
parent c84afbeaf1
commit c506784b08
7 changed files with 123 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES
View File

@ -12,6 +12,8 @@ Changes for 1.7.0:
* Improvement: failure summary in XML reports now includes file and
line information.
* Improvement: the <testsuites> XML element now has a timestamp attribute.
* Improvement: When --gtest_filter is specified, XML report now doesn't
contain information about tests that are filtered out.
* Fixed the bug where long --gtest_filter flag values are truncated in
death tests.
* Potentially breaking change: RUN_ALL_TESTS() is now implemented as a

37
include/gtest/gtest.h
View File

@ -646,9 +646,9 @@ class GTEST_API_ TestInfo {
return NULL;
}
// Returns true if this test should run, that is if the test is not disabled
// (or it is disabled but the also_run_disabled_tests flag has been specified)
// and its full name matches the user-specified filter.
// Returns true if this test should run, that is if the test is not
// disabled (or it is disabled but the also_run_disabled_tests flag has
// been specified) and its full name matches the user-specified filter.
//
// Google Test allows the user to filter the tests by their full names.
// The full name of a test Bar in test case Foo is defined as
@ -664,6 +664,14 @@ class GTEST_API_ TestInfo {
// contains the character 'A' or starts with "Foo.".
bool should_run() const { return should_run_; }
// Returns true iff this test will appear in the XML report.
bool is_reportable() const {
// For now, the XML report includes all tests matching the filter.
// In the future, we may trim tests that are excluded because of
// sharding.
return matches_filter_;
}
// Returns the result of the test.
const TestResult* result() const { return &result_; }
@ -776,9 +784,15 @@ class GTEST_API_ TestCase {
// Gets the number of failed tests in this test case.
int failed_test_count() const;
// Gets the number of disabled tests that will be reported in the XML report.
int reportable_disabled_test_count() const;
// Gets the number of disabled tests in this test case.
int disabled_test_count() const;
// Gets the number of tests to be printed in the XML report.
int reportable_test_count() const;
// Get the number of tests in this test case that should run.
int test_to_run_count() const;
@ -854,11 +868,22 @@ class GTEST_API_ TestCase {
return test_info->should_run() && test_info->result()->Failed();
}
// Returns true iff the test is disabled and will be reported in the XML
// report.
static bool TestReportableDisabled(const TestInfo* test_info) {
return test_info->is_reportable() && test_info->is_disabled_;
}
// Returns true iff test is disabled.
static bool TestDisabled(const TestInfo* test_info) {
return test_info->is_disabled_;
}
// Returns true iff this test will appear in the XML report.
static bool TestReportable(const TestInfo* test_info) {
return test_info->is_reportable();
}
// Returns true if the given test should run.
static bool ShouldRunTest(const TestInfo* test_info) {
return test_info->should_run();
@ -1151,9 +1176,15 @@ class GTEST_API_ UnitTest {
// Gets the number of failed tests.
int failed_test_count() const;
// Gets the number of disabled tests that will be reported in the XML report.
int reportable_disabled_test_count() const;
// Gets the number of disabled tests.
int disabled_test_count() const;
// Gets the number of tests to be printed in the XML report.
int reportable_test_count() const;
// Gets the number of all tests.
int total_test_count() const;

6
src/gtest-internal-inl.h
View File

@ -550,9 +550,15 @@ class GTEST_API_ UnitTestImpl {
// Gets the number of failed tests.
int failed_test_count() const;
// Gets the number of disabled tests that will be reported in the XML report.
int reportable_disabled_test_count() const;
// Gets the number of disabled tests.
int disabled_test_count() const;
// Gets the number of tests to be printed in the XML report.
int reportable_test_count() const;
// Gets the number of all tests.
int total_test_count() const;

59
src/gtest.cc
View File

@ -731,11 +731,22 @@ int UnitTestImpl::failed_test_count() const {
return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count);
}
// Gets the number of disabled tests that will be reported in the XML report.
int UnitTestImpl::reportable_disabled_test_count() const {
return SumOverTestCaseList(test_cases_,
&TestCase::reportable_disabled_test_count);
}
// Gets the number of disabled tests.
int UnitTestImpl::disabled_test_count() const {
return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count);
}
// Gets the number of tests to be printed in the XML report.
int UnitTestImpl::reportable_test_count() const {
return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count);
}
// Gets the number of all tests.
int UnitTestImpl::total_test_count() const {
return SumOverTestCaseList(test_cases_, &TestCase::total_test_count);
@ -2338,10 +2349,21 @@ int TestCase::failed_test_count() const {
return CountIf(test_info_list_, TestFailed);
}
// Gets the number of disabled tests that will be reported in the XML report.
int TestCase::reportable_disabled_test_count() const {
return CountIf(test_info_list_, TestReportableDisabled);
}
// Gets the number of disabled tests in this test case.
int TestCase::disabled_test_count() const {
return CountIf(test_info_list_, TestDisabled);
}
// Gets the number of tests to be printed in the XML report.
int TestCase::reportable_test_count() const {
return CountIf(test_info_list_, TestReportable);
}
// Get the number of tests in this test case that should run.
int TestCase::test_to_run_count() const {
return CountIf(test_info_list_, ShouldRunTest);
@ -2851,7 +2873,7 @@ void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test,
num_failures == 1 ? "TEST" : "TESTS");
}
int num_disabled = unit_test.disabled_test_count();
int num_disabled = unit_test.reportable_disabled_test_count();
if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) {
if (!num_failures) {
printf("\n"); // Add a spacer if no FAILURE banner is displayed.
@ -3310,19 +3332,22 @@ void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream,
*stream << " <" << kTestsuite;
OutputXmlAttribute(stream, kTestsuite, "name", test_case.name());
OutputXmlAttribute(stream, kTestsuite, "tests",
StreamableToString(test_case.total_test_count()));
StreamableToString(test_case.reportable_test_count()));
OutputXmlAttribute(stream, kTestsuite, "failures",
StreamableToString(test_case.failed_test_count()));
OutputXmlAttribute(stream, kTestsuite, "disabled",
StreamableToString(test_case.disabled_test_count()));
OutputXmlAttribute(
stream, kTestsuite, "disabled",
StreamableToString(test_case.reportable_disabled_test_count()));
OutputXmlAttribute(stream, kTestsuite, "errors", "0");
OutputXmlAttribute(stream, kTestsuite, "time",
FormatTimeInMillisAsSeconds(test_case.elapsed_time()));
*stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result())
<< ">\n";
for (int i = 0; i < test_case.total_test_count(); ++i)
OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i));
for (int i = 0; i < test_case.total_test_count(); ++i) {
if (test_case.GetTestInfo(i)->is_reportable())
OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i));
}
*stream << " </" << kTestsuite << ">\n";
}
@ -3335,11 +3360,12 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream,
*stream << "<" << kTestsuites;
OutputXmlAttribute(stream, kTestsuites, "tests",
StreamableToString(unit_test.total_test_count()));
StreamableToString(unit_test.reportable_test_count()));
OutputXmlAttribute(stream, kTestsuites, "failures",
StreamableToString(unit_test.failed_test_count()));
OutputXmlAttribute(stream, kTestsuites, "disabled",
StreamableToString(unit_test.disabled_test_count()));
OutputXmlAttribute(
stream, kTestsuites, "disabled",
StreamableToString(unit_test.reportable_disabled_test_count()));
OutputXmlAttribute(stream, kTestsuites, "errors", "0");
OutputXmlAttribute(
stream, kTestsuites, "timestamp",
@ -3357,9 +3383,9 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream,
OutputXmlAttribute(stream, kTestsuites, "name", "AllTests");
*stream << ">\n";
for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
PrintXmlTestCase(stream, *unit_test.GetTestCase(i));
if (unit_test.GetTestCase(i)->reportable_test_count() > 0)
PrintXmlTestCase(stream, *unit_test.GetTestCase(i));
}
*stream << "</" << kTestsuites << ">\n";
}
@ -3629,11 +3655,21 @@ int UnitTest::successful_test_count() const {
// Gets the number of failed tests.
int UnitTest::failed_test_count() const { return impl()->failed_test_count(); }
// Gets the number of disabled tests that will be reported in the XML report.
int UnitTest::reportable_disabled_test_count() const {
return impl()->reportable_disabled_test_count();
}
// Gets the number of disabled tests.
int UnitTest::disabled_test_count() const {
return impl()->disabled_test_count();
}
// Gets the number of tests to be printed in the XML report.
int UnitTest::reportable_test_count() const {
return impl()->reportable_test_count();
}
// Gets the number of all tests.
int UnitTest::total_test_count() const { return impl()->total_test_count(); }
@ -3929,7 +3965,6 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent)
start_timestamp_(0),
elapsed_time_(0),
#if GTEST_HAS_DEATH_TEST
internal_run_death_test_flag_(NULL),
death_test_factory_(new DefaultDeathTestFactory),
#endif
// Will be overridden by the flag before first use.

5
test/gtest_output_test_golden_lin.txt
View File

@ -699,8 +699,6 @@ Expected: (3) >= (a[i]), actual: 3 vs 6
[ FAILED ] LoggingTest.InterleavingLoggingAndAssertions
4 FAILED TESTS
YOU HAVE 1 DISABLED TEST
Note: Google Test filter = *DISABLED_*
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
@ -720,6 +718,3 @@ Note: This is test shard 2 of 2.
[----------] Global test environment tear-down
[==========] 1 test from 1 test case ran.
[ PASSED ] 1 test.
YOU HAVE 1 DISABLED TEST

35
test/gtest_xml_output_unittest.py
View File

@ -44,6 +44,7 @@ import gtest_test_utils
import gtest_xml_test_utils
GTEST_FILTER_FLAG = '--gtest_filter'
GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
GTEST_OUTPUT_FLAG = "--gtest_output"
GTEST_DEFAULT_OUTPUT_FILE = "test_detail.xml"
@ -128,9 +129,18 @@ Invalid characters in brackets []%(stack)s]]></failure>
</testsuite>
</testsuites>""" % {'stack': STACK_TRACE_TEMPLATE}
EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" disabled="0" errors="0" time="*"
timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0"
errors="0" time="*">
<testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/>
</testsuite>
</testsuites>"""
EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="0" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests">
<testsuites tests="0" failures="0" disabled="0" errors="0" time="*"
timestamp="*" name="AllTests">
</testsuites>"""
GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
@ -169,7 +179,7 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
Runs a test program that generates an empty XML output, and checks if
the timestamp attribute in the testsuites tag is valid.
"""
actual = self._GetXmlOutput('gtest_no_test_unittest', 0)
actual = self._GetXmlOutput('gtest_no_test_unittest', [], 0)
date_time_str = actual.documentElement.getAttributeNode('timestamp').value
# datetime.strptime() is only available in Python 2.5+ so we have to
# parse the expected datetime manually.
@ -239,7 +249,17 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
self.assert_(not os.path.isfile(xml_path))
def _GetXmlOutput(self, gtest_prog_name, expected_exit_code):
def testFilteredTestXmlOutput(self):
"""Verifies XML output when a filter is applied.
Runs a test program that executes only some tests and verifies that
non-selected tests do not show up in the XML output.
"""
self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0,
extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
def _GetXmlOutput(self, gtest_prog_name, extra_args, expected_exit_code):
"""
Returns the xml output generated by running the program gtest_prog_name.
Furthermore, the program's exit code must be expected_exit_code.
@ -248,7 +268,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
gtest_prog_name + 'out.xml')
gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
command = [gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)]
command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] +
extra_args)
p = gtest_test_utils.Subprocess(command)
if p.terminated_by_signal:
self.assert_(False,
@ -262,7 +283,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
actual = minidom.parse(xml_path)
return actual
def _TestXmlOutput(self, gtest_prog_name, expected_xml, expected_exit_code):
def _TestXmlOutput(self, gtest_prog_name, expected_xml,
expected_exit_code, extra_args=None):
"""
Asserts that the XML document generated by running the program
gtest_prog_name matches expected_xml, a string containing another
@ -270,7 +292,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
expected_exit_code.
"""
actual = self._GetXmlOutput(gtest_prog_name, expected_exit_code)
actual = self._GetXmlOutput(gtest_prog_name, extra_args or [],
expected_exit_code)
expected = minidom.parseString(expected_xml)
self.NormalizeXml(actual.documentElement)
self.AssertEquivalentNodes(expected.documentElement,

8
test/gtest_xml_test_utils.py
View File

@ -90,9 +90,11 @@ class GTestXMLTestCase(gtest_test_utils.TestCase):
actual_attr is not None,
'expected attribute %s not found in element %s' %
(expected_attr.name, actual_node.tagName))
self.assertEquals(expected_attr.value, actual_attr.value,
' values of attribute %s in element %s differ' %
(expected_attr.name, actual_node.tagName))
self.assertEquals(
expected_attr.value, actual_attr.value,
' values of attribute %s in element %s differ: %s vs %s' %
(expected_attr.name, actual_node.tagName,
expected_attr.value, actual_attr.value))
expected_children = self._GetChildren(expected_node)
actual_children = self._GetChildren(actual_node)