gitMirror/googletest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adds the command line flags needed for test shuffling. Most code by Josh Kelley.

Browse Source
This commit is contained in:
zhanyong.wan
2009-07-14 22:56:46 +00:00
parent 89080477ae
commit 8bdb31e054
6 changed files with 249 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/gtest-internal-inl.h
View File

@@ -87,9 +87,42 @@ const char kFilterFlag[] = "filter";
const char kListTestsFlag[] = "list_tests";
const char kOutputFlag[] = "output";
const char kPrintTimeFlag[] = "print_time";
const char kRandomSeedFlag[] = "random_seed";
const char kRepeatFlag[] = "repeat";
const char kShuffleFlag[] = "shuffle";
const char kThrowOnFailureFlag[] = "throw_on_failure";
// A valid random seed must be in [1, kMaxRandomSeed].
const unsigned int kMaxRandomSeed = 99999;
// Returns the current time in milliseconds.
TimeInMillis GetTimeInMillis();
// Returns a random seed in range [1, kMaxRandomSeed] based on the
// given --gtest_random_seed flag value.
inline int GetRandomSeedFromFlag(Int32 random_seed_flag) {
const unsigned int raw_seed = (random_seed_flag == 0) ?
static_cast<unsigned int>(GetTimeInMillis()) :
static_cast<unsigned int>(random_seed_flag);
// Normalizes the actual seed to range [1, kMaxRandomSeed] such that
// it's easy to type.
const int normalized_seed =
static_cast<int>((raw_seed - 1U) % kMaxRandomSeed) + 1;
return normalized_seed;
}
// Returns the first valid random seed after 'seed'. The behavior is
// undefined if 'seed' is invalid. The seed after kMaxRandomSeed is
// considered to be 1.
inline int GetNextRandomSeed(int seed) {
GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed)
<< "Invalid random seed " << seed << " - must be in [1, "
<< kMaxRandomSeed << "].";
const int next_seed = seed + 1;
return (next_seed > kMaxRandomSeed) ? 1 : next_seed;
}
// This class saves the values of all Google Test flags in its c'tor, and
// restores them in its d'tor.
class GTestFlagSaver {
@@ -107,7 +140,9 @@ class GTestFlagSaver {
list_tests_ = GTEST_FLAG(list_tests);
output_ = GTEST_FLAG(output);
print_time_ = GTEST_FLAG(print_time);
random_seed_ = GTEST_FLAG(random_seed);
repeat_ = GTEST_FLAG(repeat);
shuffle_ = GTEST_FLAG(shuffle);
throw_on_failure_ = GTEST_FLAG(throw_on_failure);
}
@@ -124,7 +159,9 @@ class GTestFlagSaver {
GTEST_FLAG(list_tests) = list_tests_;
GTEST_FLAG(output) = output_;
GTEST_FLAG(print_time) = print_time_;
GTEST_FLAG(random_seed) = random_seed_;
GTEST_FLAG(repeat) = repeat_;
GTEST_FLAG(shuffle) = shuffle_;
GTEST_FLAG(throw_on_failure) = throw_on_failure_;
}
private:
@@ -141,7 +178,9 @@ class GTestFlagSaver {
String output_;
bool print_time_;
bool pretty_;
internal::Int32 random_seed_;
internal::Int32 repeat_;
bool shuffle_;
bool throw_on_failure_;
} GTEST_ATTRIBUTE_UNUSED_;
@@ -884,6 +923,9 @@ class UnitTestImpl {
friend class ReplaceDeathTestFactory;
#endif // GTEST_HAS_DEATH_TEST
// Gets the random seed used at the start of the current test run.
int random_seed() const { return random_seed_; }
private:
friend class ::testing::UnitTest;
@@ -932,7 +974,7 @@ class UnitTestImpl {
// This points to the TestCase for the currently running test. It
// changes as Google Test goes through one test case after another.
// When no test is running, this is set to NULL and Google Test
// stores assertion results in ad_hoc_test_result_. Initally NULL.
// stores assertion results in ad_hoc_test_result_. Initially NULL.
TestCase* current_test_case_;
// This points to the TestInfo for the currently running test. It
@@ -963,6 +1005,9 @@ class UnitTestImpl {
// desired.
OsStackTraceGetterInterface* os_stack_trace_getter_;
// The random number seed used at the beginning of the test run.
int random_seed_;
// How long the test took to run, in milliseconds.
TimeInMillis elapsed_time_;

57
src/gtest.cc
View File

@@ -217,23 +217,35 @@ GTEST_DEFINE_bool_(
"True iff " GTEST_NAME_
" should display elapsed time in text output.");
GTEST_DEFINE_int32_(
random_seed,
internal::Int32FromGTestEnv("random_seed", 0),
"Random number seed to use when shuffling test orders. Must be in range "
"[1, 99999], or 0 to use a seed based on the current time.");
GTEST_DEFINE_int32_(
repeat,
internal::Int32FromGTestEnv("repeat", 1),
"How many times to repeat each test. Specify a negative number "
"for repeating forever. Useful for shaking out flaky tests.");
GTEST_DEFINE_bool_(
show_internal_stack_frames, false,
"True iff " GTEST_NAME_ " should include internal stack frames when "
"printing test failure stack traces.");
GTEST_DEFINE_bool_(
shuffle,
internal::BoolFromGTestEnv("shuffle", false),
"True iff " GTEST_NAME_
" should randomize tests' order on every run.");
GTEST_DEFINE_int32_(
stack_trace_depth,
internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth),
"The maximum number of stack frames to print when an "
"assertion fails. The valid range is 0 through 100, inclusive.");
GTEST_DEFINE_bool_(
show_internal_stack_frames, false,
"True iff " GTEST_NAME_ " should include internal stack frames when "
"printing test failure stack traces.");
GTEST_DEFINE_bool_(
throw_on_failure,
internal::BoolFromGTestEnv("throw_on_failure", false),
@@ -774,9 +786,10 @@ String UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) {
return String("");
}
static TimeInMillis GetTimeInMillis() {
// Returns the current time in milliseconds.
TimeInMillis GetTimeInMillis() {
#if defined(_WIN32_WCE) || defined(__BORLANDC__)
// Difference between 1970-01-01 and 1601-01-01 in miliseconds.
// Difference between 1970-01-01 and 1601-01-01 in milliseconds.
// http://analogous.blogspot.com/2005/04/epoch.html
const TimeInMillis kJavaEpochToWinFileTimeDelta =
static_cast<TimeInMillis>(116444736UL) * 100000UL;
@@ -2719,6 +2732,12 @@ void PrettyUnitTestResultPrinter::OnUnitTestStart(const UnitTest& unit_test) {
internal::posix::GetEnv(kTestTotalShards));
}
if (GTEST_FLAG(shuffle)) {
ColoredPrintf(COLOR_YELLOW,
"Note: Randomizing tests' orders with a seed of %d .\n",
unit_test.random_seed());
}
ColoredPrintf(COLOR_GREEN, "[==========] ");
printf("Running %s from %s.\n",
FormatTestCount(unit_test.test_to_run_count()).c_str(),
@@ -3193,6 +3212,9 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(FILE* out,
unit_test.failed_test_count(),
unit_test.disabled_test_count(),
internal::FormatTimeInMillisAsSeconds(unit_test.elapsed_time()));
if (GTEST_FLAG(shuffle)) {
fprintf(out, "random_seed=\"%d\" ", unit_test.random_seed());
}
fprintf(out, "name=\"AllTests\">\n");
for (int i = 0; i < unit_test.total_test_case_count(); ++i)
PrintXmlTestCase(out, *unit_test.GetTestCase(i));
@@ -3539,6 +3561,9 @@ const TestInfo* UnitTest::current_test_info() const {
return impl_->current_test_info();
}
// Returns the random seed used at the start of the current test run.
int UnitTest::random_seed() const { return impl_->random_seed(); }
#if GTEST_HAS_PARAM_TEST
// Returns ParameterizedTestCaseRegistry object used to keep track of
// value-parameterized tests and instantiate and register them.
@@ -3604,6 +3629,7 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent)
ad_hoc_test_result_(),
result_printer_(NULL),
os_stack_trace_getter_(NULL),
random_seed_(0),
#if GTEST_HAS_DEATH_TEST
elapsed_time_(0),
internal_run_death_test_flag_(NULL),
@@ -3747,6 +3773,9 @@ int UnitTestImpl::RunAllTests() {
return 0;
}
random_seed_ = GTEST_FLAG(shuffle) ?
GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0;
// True iff at least one test has failed.
bool failed = false;
@@ -3796,6 +3825,11 @@ int UnitTestImpl::RunAllTests() {
failed = true;
}
ClearResult();
if (GTEST_FLAG(shuffle)) {
// Picks a new random seed for each run.
random_seed_ = GetNextRandomSeed(random_seed_);
}
}
// Returns 0 if all tests passed, or 1 other wise.
@@ -4262,8 +4296,15 @@ static const char kColorEncodedHelpMessage[] =
" matches any substring; ':' separates two patterns.\n"
" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n"
" Run all disabled tests too.\n"
"\n"
"Test Execution:\n"
" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n"
" Run the tests repeatedly; use a negative count to repeat forever.\n"
" @G--" GTEST_FLAG_PREFIX_ "shuffle\n"
" Randomize tests' orders on every run. To be implemented.\n"
" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n"
" Random number seed to use for shuffling test orders (between 1 and\n"
" 99999, or 0 to use a seed based on the current time).\n"
"\n"
"Test Output:\n"
" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n"
@@ -4332,7 +4373,9 @@ void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) {
ParseBoolFlag(arg, kListTestsFlag, &GTEST_FLAG(list_tests)) ||
ParseStringFlag(arg, kOutputFlag, &GTEST_FLAG(output)) ||
ParseBoolFlag(arg, kPrintTimeFlag, &GTEST_FLAG(print_time)) ||
ParseInt32Flag(arg, kRandomSeedFlag, &GTEST_FLAG(random_seed)) ||
ParseInt32Flag(arg, kRepeatFlag, &GTEST_FLAG(repeat)) ||
ParseBoolFlag(arg, kShuffleFlag, &GTEST_FLAG(shuffle)) ||
ParseBoolFlag(arg, kThrowOnFailureFlag, &GTEST_FLAG(throw_on_failure))
) {
// Yes. Shift the remainder of the argv list left by one. Note