gitMirror/googletest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Googletest export

Browse Source 
gtest: Add a flag to only set up/tear down test environments once when repeating

Currently when running a test multiple times using `--gtest_repeat` the global
test environment(s) are set up and torn down for each iteration of the test.
When checking for flakes in tests that have expensive dependencies that are set
up in the test environment (subprocesses, external dependencies, etc) this can
become expensive.

To support finding flakes in tests that fit into this category, where the setup
phase is expensive but each test case is fast, allow callers to specify via
`--gtest_recreate_environments_when_repeating=false` that the test environments
should only be set up once, for the first iteration, and only torn down once, on
the last iteration. This makes running a test with `--gtest_repeat=1000` a much
faster and more pleasant experience.

PiperOrigin-RevId: 382748942
This commit is contained in:
Abseil Team
2021-07-02 11:58:28 -04:00
committed by Derek Mauro
parent 4ec4cd23f4
commit 4cfd14984f
5 changed files with 149 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
googletest/test/googletest-global-environment-unittest.py
View File

@@ -35,16 +35,17 @@ This script tests such functionality by invoking
googletest-global-environment-unittest_ (a program written with Google Test).
"""
import re
import gtest_test_utils
def RunAndReturnOutput():
def RunAndReturnOutput(args=None):
"""Runs the test program and returns its output."""
return gtest_test_utils.Subprocess([
gtest_test_utils.GetTestExecutablePath(
'googletest-global-environment-unittest_')
]).output
] + (args or [])).output
class GTestGlobalEnvironmentUnitTest(gtest_test_utils.TestCase):
@@ -67,6 +68,61 @@ class GTestGlobalEnvironmentUnitTest(gtest_test_utils.TestCase):
# The test case shouldn't have been run.
self.assertNotIn('Unexpected call', txt)
def testEnvironmentSetUpAndTornDownForEachRepeat(self):
"""Tests the behavior of test environments and gtest_repeat."""
txt = RunAndReturnOutput(['--gtest_repeat=2'])
# By default, with gtest_repeat=2, the global test environment should be set
# up and torn down for each iteration.
expected_pattern = ('(.|\n)*'
r'Repeating all tests \(iteration 1\)'
'(.|\n)*'
'Global test environment set-up.'
'(.|\n)*'
'SomeTest.DoesFoo'
'(.|\n)*'
'Global test environment tear-down'
'(.|\n)*'
r'Repeating all tests \(iteration 2\)'
'(.|\n)*'
'Global test environment set-up.'
'(.|\n)*'
'SomeTest.DoesFoo'
'(.|\n)*'
'Global test environment tear-down'
'(.|\n)*')
self.assertRegex(txt, expected_pattern)
def testEnvironmentSetUpAndTornDownOnce(self):
"""Tests environment and --gtest_recreate_environments_when_repeating."""
txt = RunAndReturnOutput([
'--gtest_repeat=2', '--gtest_recreate_environments_when_repeating=false'
])
# When --gtest_recreate_environments_when_repeating is false, the test
# environment should only be set up and torn down once, at the start and
# end of the test respectively.
expected_pattern = ('(.|\n)*'
r'Repeating all tests \(iteration 1\)'
'(.|\n)*'
'Global test environment set-up.'
'(.|\n)*'
'SomeTest.DoesFoo'
'(.|\n)*'
r'Repeating all tests \(iteration 2\)'
'(.|\n)*'
'SomeTest.DoesFoo'
'(.|\n)*'
'Global test environment tear-down'
'(.|\n)*')
self.assertRegex(txt, expected_pattern)
self.assertEqual(len(re.findall('Global test environment set-up', txt)), 1)
self.assertEqual(
len(re.findall('Global test environment tear-down', txt)), 1)
if __name__ == '__main__':
gtest_test_utils.Main()

34
googletest/test/gtest_unittest.cc
View File

@@ -48,6 +48,7 @@ TEST(CommandLineFlagsTest, CanBeAccessedInCodeOnceGTestHIsIncluded) {
testing::GTEST_FLAG(brief) || testing::GTEST_FLAG(print_time) ||
testing::GTEST_FLAG(random_seed) ||
testing::GTEST_FLAG(repeat) > 0 ||
testing::GTEST_FLAG(recreate_environments_when_repeating) ||
testing::GTEST_FLAG(show_internal_stack_frames) ||
testing::GTEST_FLAG(shuffle) ||
testing::GTEST_FLAG(stack_trace_depth) > 0 ||
@@ -212,6 +213,7 @@ using testing::GTEST_FLAG(brief);
using testing::GTEST_FLAG(print_time);
using testing::GTEST_FLAG(random_seed);
using testing::GTEST_FLAG(repeat);
using testing::GTEST_FLAG(recreate_environments_when_repeating);
using testing::GTEST_FLAG(show_internal_stack_frames);
using testing::GTEST_FLAG(shuffle);
using testing::GTEST_FLAG(stack_trace_depth);
@@ -1610,6 +1612,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(print_time) = true;
GTEST_FLAG(random_seed) = 0;
GTEST_FLAG(repeat) = 1;
GTEST_FLAG(recreate_environments_when_repeating) = true;
GTEST_FLAG(shuffle) = false;
GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
GTEST_FLAG(stream_result_to) = "";
@@ -1639,6 +1642,7 @@ class GTestFlagSaverTest : public Test {
EXPECT_TRUE(GTEST_FLAG(print_time));
EXPECT_EQ(0, GTEST_FLAG(random_seed));
EXPECT_EQ(1, GTEST_FLAG(repeat));
EXPECT_TRUE(GTEST_FLAG(recreate_environments_when_repeating));
EXPECT_FALSE(GTEST_FLAG(shuffle));
EXPECT_EQ(kMaxStackTraceDepth, GTEST_FLAG(stack_trace_depth));
EXPECT_STREQ("", GTEST_FLAG(stream_result_to).c_str());
@@ -1657,6 +1661,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(print_time) = false;
GTEST_FLAG(random_seed) = 1;
GTEST_FLAG(repeat) = 100;
GTEST_FLAG(recreate_environments_when_repeating) = false;
GTEST_FLAG(shuffle) = true;
GTEST_FLAG(stack_trace_depth) = 1;
GTEST_FLAG(stream_result_to) = "localhost:1234";
@@ -5580,6 +5585,7 @@ struct Flags {
print_time(true),
random_seed(0),
repeat(1),
recreate_environments_when_repeating(true),
shuffle(false),
stack_trace_depth(kMaxStackTraceDepth),
stream_result_to(""),
@@ -5683,6 +5689,16 @@ struct Flags {
return flags;
}
// Creates a Flags struct where the gtest_recreate_environments_when_repeating
// flag has the given value.
static Flags RecreateEnvironmentsWhenRepeating(
bool recreate_environments_when_repeating) {
Flags flags;
flags.recreate_environments_when_repeating =
recreate_environments_when_repeating;
return flags;
}
// Creates a Flags struct where the gtest_shuffle flag has the given
// value.
static Flags Shuffle(bool shuffle) {
@@ -5728,6 +5744,7 @@ struct Flags {
bool print_time;
int32_t random_seed;
int32_t repeat;
bool recreate_environments_when_repeating;
bool shuffle;
int32_t stack_trace_depth;
const char* stream_result_to;
@@ -5751,6 +5768,7 @@ class ParseFlagsTest : public Test {
GTEST_FLAG(print_time) = true;
GTEST_FLAG(random_seed) = 0;
GTEST_FLAG(repeat) = 1;
GTEST_FLAG(recreate_environments_when_repeating) = true;
GTEST_FLAG(shuffle) = false;
GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
GTEST_FLAG(stream_result_to) = "";
@@ -5783,6 +5801,8 @@ class ParseFlagsTest : public Test {
EXPECT_EQ(expected.print_time, GTEST_FLAG(print_time));
EXPECT_EQ(expected.random_seed, GTEST_FLAG(random_seed));
EXPECT_EQ(expected.repeat, GTEST_FLAG(repeat));
EXPECT_EQ(expected.recreate_environments_when_repeating,
GTEST_FLAG(recreate_environments_when_repeating));
EXPECT_EQ(expected.shuffle, GTEST_FLAG(shuffle));
EXPECT_EQ(expected.stack_trace_depth, GTEST_FLAG(stack_trace_depth));
EXPECT_STREQ(expected.stream_result_to,
@@ -6161,6 +6181,20 @@ TEST_F(ParseFlagsTest, Repeat) {
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Repeat(1000), false);
}
// Tests parsing --gtest_recreate_environments_when_repeating
TEST_F(ParseFlagsTest, RecreateEnvironmentsWhenRepeating) {
const char* argv[] = {
"foo.exe",
"--gtest_recreate_environments_when_repeating=0",
nullptr,
};
const char* argv2[] = {"foo.exe", nullptr};
GTEST_TEST_PARSING_FLAGS_(
argv, argv2, Flags::RecreateEnvironmentsWhenRepeating(false), false);
}
// Tests having a --gtest_also_run_disabled_tests flag
TEST_F(ParseFlagsTest, AlsoRunDisabledTestsFlag) {
const char* argv[] = {"foo.exe", "--gtest_also_run_disabled_tests", nullptr};