572 lines
20 KiB
Python
Executable File
572 lines
20 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright 2009 Google Inc. All Rights Reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following disclaimer
|
|
# in the documentation and/or other materials provided with the
|
|
# distribution.
|
|
# * Neither the name of Google Inc. nor the names of its
|
|
# contributors may be used to endorse or promote products derived from
|
|
# this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
"""Tests for run_tests.py test runner script."""
|
|
|
|
__author__ = 'vladl@google.com (Vlad Losev)'
|
|
|
|
import os
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), os.pardir))
|
|
import run_tests
|
|
|
|
|
|
class FakePath(object):
|
|
"""A fake os.path module for testing."""
|
|
|
|
def __init__(self, current_dir=os.getcwd(), known_paths=None):
|
|
self.current_dir = current_dir
|
|
self.tree = {}
|
|
self.path_separator = os.sep
|
|
|
|
# known_paths contains either absolute or relative paths. Relative paths
|
|
# are absolutized with self.current_dir.
|
|
if known_paths:
|
|
self._AddPaths(known_paths)
|
|
|
|
def _AddPath(self, path):
|
|
ends_with_slash = path.endswith('/')
|
|
path = self.abspath(path)
|
|
if ends_with_slash:
|
|
path += self.path_separator
|
|
name_list = path.split(self.path_separator)
|
|
tree = self.tree
|
|
for name in name_list[:-1]:
|
|
if not name:
|
|
continue
|
|
if name in tree:
|
|
tree = tree[name]
|
|
else:
|
|
tree[name] = {}
|
|
tree = tree[name]
|
|
|
|
name = name_list[-1]
|
|
if name:
|
|
if name in tree:
|
|
assert tree[name] == 1
|
|
else:
|
|
tree[name] = 1
|
|
|
|
def _AddPaths(self, paths):
|
|
for path in paths:
|
|
self._AddPath(path)
|
|
|
|
def PathElement(self, path):
|
|
"""Returns an internal representation of directory tree entry for path."""
|
|
tree = self.tree
|
|
name_list = self.abspath(path).split(self.path_separator)
|
|
for name in name_list:
|
|
if not name:
|
|
continue
|
|
tree = tree.get(name, None)
|
|
if tree is None:
|
|
break
|
|
|
|
return tree
|
|
|
|
def normpath(self, path):
|
|
return os.path.normpath(path)
|
|
|
|
def abspath(self, path):
|
|
return self.normpath(os.path.join(self.current_dir, path))
|
|
|
|
def isfile(self, path):
|
|
return self.PathElement(self.abspath(path)) == 1
|
|
|
|
def isdir(self, path):
|
|
return type(self.PathElement(self.abspath(path))) == type(dict())
|
|
|
|
def basename(self, path):
|
|
return os.path.basename(path)
|
|
|
|
def dirname(self, path):
|
|
return os.path.dirname(path)
|
|
|
|
def join(self, *kargs):
|
|
return os.path.join(*kargs)
|
|
|
|
|
|
class FakeOs(object):
|
|
"""A fake os module for testing."""
|
|
P_WAIT = os.P_WAIT
|
|
|
|
def __init__(self, fake_path_module):
|
|
self.path = fake_path_module
|
|
|
|
# Some methods/attributes are delegated to the real os module.
|
|
self.environ = os.environ
|
|
|
|
def listdir(self, path):
|
|
assert self.path.isdir(path)
|
|
return self.path.PathElement(path).iterkeys()
|
|
|
|
def spawnv(self, wait, executable, *kargs):
|
|
assert wait == FakeOs.P_WAIT
|
|
return self.spawn_impl(executable, kargs)
|
|
|
|
|
|
class GetTestsToRunTest(unittest.TestCase):
|
|
"""Exercises TestRunner.GetTestsToRun."""
|
|
|
|
def AssertResultsEqual(self, results, expected):
|
|
"""Asserts results returned by GetTestsToRun equal to expected results."""
|
|
|
|
def NormalizeResultPaths(paths):
|
|
"""Normalizes values returned by GetTestsToRun for comparison."""
|
|
|
|
def NormalizeResultPair(pair):
|
|
return (os.path.normpath(pair[0]), os.path.normpath(pair[1]))
|
|
|
|
return (sorted(map(NormalizeResultPair, paths[0])),
|
|
sorted(map(NormalizeResultPair, paths[1])))
|
|
|
|
self.assertEqual(NormalizeResultPaths(results),
|
|
NormalizeResultPaths(expected),
|
|
'Incorrect set of tests %s returned vs %s expected' %
|
|
(results, expected))
|
|
|
|
def setUp(self):
|
|
self.fake_os = FakeOs(FakePath(
|
|
current_dir=os.path.abspath(os.path.dirname(run_tests.__file__)),
|
|
known_paths=['scons/build/dbg/scons/gtest_unittest',
|
|
'scons/build/opt/scons/gtest_unittest',
|
|
'test/gtest_color_test.py']))
|
|
self.fake_configurations = ['dbg', 'opt']
|
|
self.test_runner = run_tests.TestRunner(injected_os=self.fake_os,
|
|
injected_subprocess=None,
|
|
injected_script_dir='.')
|
|
|
|
def testBinaryTestsOnly(self):
|
|
"""Exercises GetTestsToRun with parameters designating binary tests only."""
|
|
|
|
# A default build.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# An explicitly specified directory.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_unittest'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# A particular configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'other',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/other/scons',
|
|
'scons/build/other/scons/gtest_unittest')]))
|
|
|
|
# All available configurations
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'all',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest'),
|
|
('scons/build/opt/scons', 'scons/build/opt/scons/gtest_unittest')]))
|
|
|
|
# All built configurations (unbuilt don't cause failure).
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'',
|
|
True,
|
|
available_configurations=self.fake_configurations + ['unbuilt']),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest'),
|
|
('scons/build/opt/scons', 'scons/build/opt/scons/gtest_unittest')]))
|
|
|
|
# A combination of an explicit directory and a configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_unittest'],
|
|
'opt',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest'),
|
|
('scons/build/opt/scons', 'scons/build/opt/scons/gtest_unittest')]))
|
|
|
|
# Same test specified in an explicit directory and via a configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_unittest'],
|
|
'dbg',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# All built configurations + explicit directory + explicit configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_unittest'],
|
|
'opt',
|
|
True,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest'),
|
|
('scons/build/opt/scons', 'scons/build/opt/scons/gtest_unittest')]))
|
|
|
|
def testPythonTestsOnly(self):
|
|
"""Exercises GetTestsToRun with parameters designating Python tests only."""
|
|
|
|
# A default build.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_color_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# An explicitly specified directory.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'test/gtest_color_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# A particular configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_color_test.py'],
|
|
'other',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/other/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# All available configurations
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['test/gtest_color_test.py'],
|
|
'all',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py'),
|
|
('scons/build/opt/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# All built configurations (unbuilt don't cause failure).
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_color_test.py'],
|
|
'',
|
|
True,
|
|
available_configurations=self.fake_configurations + ['unbuilt']),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py'),
|
|
('scons/build/opt/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# A combination of an explicit directory and a configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_color_test.py'],
|
|
'opt',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py'),
|
|
('scons/build/opt/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# Same test specified in an explicit directory and via a configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_color_test.py'],
|
|
'dbg',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
# All built configurations + explicit directory + explicit configuration.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['scons/build/dbg/scons', 'gtest_color_test.py'],
|
|
'opt',
|
|
True,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py'),
|
|
('scons/build/opt/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
def testCombinationOfBinaryAndPythonTests(self):
|
|
"""Exercises GetTestsToRun with mixed binary/Python tests."""
|
|
|
|
# Use only default configuration for this test.
|
|
|
|
# Neither binary nor Python tests are specified so find all.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
[],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# Specifying both binary and Python tests.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest', 'gtest_color_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# Specifying binary tests suppresses Python tests.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# Specifying Python tests suppresses binary tests.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_color_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
def testIgnoresNonTestFiles(self):
|
|
"""Verifies that GetTestsToRun ignores non-test files in the filesystem."""
|
|
|
|
self.fake_os = FakeOs(FakePath(
|
|
current_dir=os.path.abspath(os.path.dirname(run_tests.__file__)),
|
|
known_paths=['scons/build/dbg/scons/gtest_nontest',
|
|
'scons/build/opt/scons/gtest_nontest.exe',
|
|
'test/']))
|
|
self.test_runner = run_tests.TestRunner(injected_os=self.fake_os,
|
|
injected_subprocess=None,
|
|
injected_script_dir='.')
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
[],
|
|
'',
|
|
True,
|
|
available_configurations=self.fake_configurations),
|
|
([], []))
|
|
|
|
def testWorksFromDifferentDir(self):
|
|
"""Exercises GetTestsToRun from a directory different from run_test.py's."""
|
|
|
|
# Here we simulate an test script in directory /d/ called from the
|
|
# directory /a/b/c/.
|
|
self.fake_os = FakeOs(FakePath(
|
|
current_dir=os.path.abspath('/a/b/c'),
|
|
known_paths=['/a/b/c/',
|
|
'/d/scons/build/dbg/scons/gtest_unittest',
|
|
'/d/scons/build/opt/scons/gtest_unittest',
|
|
'/d/test/gtest_color_test.py']))
|
|
self.fake_configurations = ['dbg', 'opt']
|
|
self.test_runner = run_tests.TestRunner(injected_os=self.fake_os,
|
|
injected_subprocess=None,
|
|
injected_script_dir='/d/')
|
|
# A binary test.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_unittest'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([],
|
|
[('/d/scons/build/dbg/scons',
|
|
'/d/scons/build/dbg/scons/gtest_unittest')]))
|
|
|
|
# A Python test.
|
|
self.AssertResultsEqual(
|
|
self.test_runner.GetTestsToRun(
|
|
['gtest_color_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations),
|
|
([('/d/scons/build/dbg/scons', '/d/test/gtest_color_test.py')],
|
|
[]))
|
|
|
|
|
|
def testNonTestBinary(self):
|
|
"""Exercises GetTestsToRun with a non-test parameter."""
|
|
|
|
self.assert_(
|
|
not self.test_runner.GetTestsToRun(
|
|
['gtest_unittest_not_really'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations))
|
|
|
|
def testNonExistingPythonTest(self):
|
|
"""Exercises GetTestsToRun with a non-existent Python test parameter."""
|
|
|
|
self.assert_(
|
|
not self.test_runner.GetTestsToRun(
|
|
['nonexistent_test.py'],
|
|
'',
|
|
False,
|
|
available_configurations=self.fake_configurations))
|
|
|
|
|
|
class RunTestsTest(unittest.TestCase):
|
|
"""Exercises TestRunner.RunTests."""
|
|
|
|
def SpawnSuccess(self, unused_executable, unused_argv):
|
|
"""Fakes test success by returning 0 as an exit code."""
|
|
|
|
self.num_spawn_calls += 1
|
|
return 0
|
|
|
|
def SpawnFailure(self, unused_executable, unused_argv):
|
|
"""Fakes test success by returning 1 as an exit code."""
|
|
|
|
self.num_spawn_calls += 1
|
|
return 1
|
|
|
|
def setUp(self):
|
|
self.fake_os = FakeOs(FakePath(
|
|
current_dir=os.path.abspath(os.path.dirname(run_tests.__file__)),
|
|
known_paths=['scons/build/dbg/scons/gtest_unittest',
|
|
'scons/build/opt/scons/gtest_unittest',
|
|
'test/gtest_color_test.py']))
|
|
self.fake_configurations = ['dbg', 'opt']
|
|
self.test_runner = run_tests.TestRunner(injected_os=self.fake_os,
|
|
injected_subprocess=None)
|
|
self.num_spawn_calls = 0 # A number of calls to spawn.
|
|
|
|
def testRunPythonTestSuccess(self):
|
|
"""Exercises RunTests to handle a Python test success."""
|
|
|
|
self.fake_os.spawn_impl = self.SpawnSuccess
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]),
|
|
0)
|
|
self.assertEqual(self.num_spawn_calls, 1)
|
|
|
|
def testRunBinaryTestSuccess(self):
|
|
"""Exercises RunTests to handle a binary test success."""
|
|
|
|
self.fake_os.spawn_impl = self.SpawnSuccess
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[],
|
|
[('scons/build/dbg/scons',
|
|
'scons/build/dbg/scons/gtest_unittest')]),
|
|
0)
|
|
self.assertEqual(self.num_spawn_calls, 1)
|
|
|
|
def testRunPythonTestFauilure(self):
|
|
"""Exercises RunTests to handle a Python test failure."""
|
|
|
|
self.fake_os.spawn_impl = self.SpawnFailure
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[('scons/build/dbg/scons', 'test/gtest_color_test.py')],
|
|
[]),
|
|
1)
|
|
self.assertEqual(self.num_spawn_calls, 1)
|
|
|
|
def testRunBinaryTestFailure(self):
|
|
"""Exercises RunTests to handle a binary test failure."""
|
|
|
|
self.fake_os.spawn_impl = self.SpawnFailure
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[],
|
|
[('scons/build/dbg/scons',
|
|
'scons/build/dbg/scons/gtest_unittest')]),
|
|
1)
|
|
self.assertEqual(self.num_spawn_calls, 1)
|
|
|
|
def testCombinedTestSuccess(self):
|
|
"""Exercises RunTests to handle a success of both Python and binary test."""
|
|
|
|
self.fake_os.spawn_impl = self.SpawnSuccess
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')],
|
|
[('scons/build/dbg/scons',
|
|
'scons/build/dbg/scons/gtest_unittest')]),
|
|
0)
|
|
self.assertEqual(self.num_spawn_calls, 2)
|
|
|
|
def testCombinedTestSuccessAndFailure(self):
|
|
"""Exercises RunTests to handle a success of both Python and binary test."""
|
|
|
|
def SpawnImpl(executable, argv):
|
|
self.num_spawn_calls += 1
|
|
# Simulates failure of a Python test and success of a binary test.
|
|
if '.py' in executable or '.py' in argv[0]:
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
self.fake_os.spawn_impl = SpawnImpl
|
|
self.assertEqual(
|
|
self.test_runner.RunTests(
|
|
[('scons/build/dbg/scons', 'scons/build/dbg/scons/gtest_unittest')],
|
|
[('scons/build/dbg/scons',
|
|
'scons/build/dbg/scons/gtest_unittest')]),
|
|
0)
|
|
self.assertEqual(self.num_spawn_calls, 2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|