|  | #!/usr/bin/env vpython3 | 
|  | # Copyright 2016 The Chromium Authors | 
|  | # Use of this source code is governed by a BSD-style license that can be | 
|  | # found in the LICENSE file. | 
|  |  | 
|  | """Unittests for test_runner.py.""" | 
|  |  | 
|  | import collections | 
|  | import logging | 
|  | import mock | 
|  | import os | 
|  | import tempfile | 
|  | import unittest | 
|  |  | 
|  | import constants | 
|  | import iossim_util | 
|  | import result_sink_util | 
|  | import test_apps | 
|  | from test_result_util import ResultCollection, TestResult, TestStatus | 
|  | import test_runner | 
|  | import xcode_util | 
|  |  | 
|  |  | 
|  | class TestCase(unittest.TestCase): | 
|  | """Test case which supports installing mocks. Uninstalls on tear down.""" | 
|  |  | 
|  | def __init__(self, *args, **kwargs): | 
|  | """Initializes a new instance of this class.""" | 
|  | super(TestCase, self).__init__(*args, **kwargs) | 
|  |  | 
|  | # Maps object to a dict which maps names of mocked members to their | 
|  | # original values. | 
|  | self._mocks = collections.OrderedDict() | 
|  |  | 
|  | def mock(self, obj, member, mock): | 
|  | """Installs mock in place of the named member of the given obj. | 
|  |  | 
|  | Args: | 
|  | obj: Any object. | 
|  | member: String naming the attribute of the object to mock. | 
|  | mock: The mock to install. | 
|  | """ | 
|  | self._mocks.setdefault(obj, collections.OrderedDict()).setdefault( | 
|  | member, getattr(obj, member)) | 
|  | setattr(obj, member, mock) | 
|  |  | 
|  | def unmock(self, obj, member): | 
|  | """Uninstalls the mock from the named member of given obj. | 
|  |  | 
|  | Args: | 
|  | obj: An obj who's member has been mocked | 
|  | member: String naming the attribute of the object to unmock | 
|  | """ | 
|  | if self._mocks[obj][member]: | 
|  | setattr(obj, member, self._mocks[obj][member]) | 
|  |  | 
|  | def tearDown(self, *args, **kwargs): | 
|  | """Uninstalls mocks.""" | 
|  | super(TestCase, self).tearDown(*args, **kwargs) | 
|  |  | 
|  | for obj in self._mocks: | 
|  | for member, original_value in self._mocks[obj].items(): | 
|  | setattr(obj, member, original_value) | 
|  |  | 
|  |  | 
|  | class SimulatorTestRunnerTest(TestCase): | 
|  | """Tests for test_runner.SimulatorTestRunner.""" | 
|  |  | 
|  | def setUp(self): | 
|  | super(SimulatorTestRunnerTest, self).setUp() | 
|  | self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID') | 
|  | self.mock( | 
|  | iossim_util, 'get_platform_type_by_platform', | 
|  | lambda platform: constants.IOSPlatformType.TVOS if platform.startswith( | 
|  | 'Apple TV') else constants.IOSPlatformType.IPHONEOS) | 
|  | self.mock(result_sink_util.ResultSinkClient, | 
|  | 'post', lambda *args, **kwargs: None) | 
|  |  | 
|  | self.mock(test_runner, 'get_current_xcode_info', lambda: { | 
|  | 'version': 'test version', 'build': 'test build', 'path': 'test/path'}) | 
|  | self.mock(test_apps, 'get_bundle_id', lambda _: 'fake-bundle-id') | 
|  | self.mock(xcode_util, 'xctest_path', lambda _: 'fake-path') | 
|  | self.mock(test_apps.plistlib, 'dump', lambda _1, _2: '') | 
|  | self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) | 
|  | self.mock(os.path, 'exists', lambda _: True) | 
|  | self.mock(test_runner.TestRunner, 'set_sigterm_handler', | 
|  | lambda self, handler: 0) | 
|  | self.mock(os, 'listdir', lambda _: []) | 
|  | self.mock(test_apps.GTestsApp, 'fill_xctest_run', | 
|  | lambda _, folder: '/abs/path/to/%s' % folder) | 
|  |  | 
|  | def test_app_not_found(self): | 
|  | """Ensures AppNotFoundError is raised.""" | 
|  |  | 
|  | self.mock(os.path, 'exists', lambda p: not p.endswith('fake-app')) | 
|  |  | 
|  | with self.assertRaises(test_runner.AppNotFoundError): | 
|  | test_runner.SimulatorTestRunner( | 
|  | 'fake-app', | 
|  | 'fake-iossim', | 
|  | 'platform', | 
|  | 'os', | 
|  | 'out-dir', | 
|  | ) | 
|  |  | 
|  | def test_iossim_not_found(self): | 
|  | """Ensures SimulatorNotFoundError is raised.""" | 
|  | self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim')) | 
|  |  | 
|  | with self.assertRaises(test_runner.SimulatorNotFoundError): | 
|  | test_runner.SimulatorTestRunner( | 
|  | 'fake-app', | 
|  | 'fake-iossim', | 
|  | 'iPhone X', | 
|  | '11.4', | 
|  | 'out-dir', | 
|  | ) | 
|  |  | 
|  | def test_init(self): | 
|  | """Ensures instance is created.""" | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', | 
|  | 'fake-iossim', | 
|  | 'iPhone X', | 
|  | '11.4', | 
|  | 'out-dir', | 
|  | ) | 
|  |  | 
|  | self.assertTrue(tr) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_startup_crash(self, mock_run, _1, _2): | 
|  | """Ensures test is relaunched once on startup crash.""" | 
|  | result = ResultCollection() | 
|  | result.crashed = True | 
|  | mock_run.return_value = result | 
|  |  | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', | 
|  | 'fake-iossim', | 
|  | 'iPhone X', | 
|  | '11.4', | 
|  | 'out-dir', | 
|  | xctest=True, | 
|  | ) | 
|  | with self.assertRaises(test_runner.AppLaunchError): | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 2) | 
|  |  | 
|  | def test_relaunch(self): | 
|  | """Ensures test is relaunched on test crash until tests complete.""" | 
|  | def set_up(self): | 
|  | return | 
|  |  | 
|  | @staticmethod | 
|  | def _run(cmd, clones=None): | 
|  | if not any('retry_after_crash' in cmd_arg for cmd_arg in cmd): | 
|  | # First run, has no test filter supplied. Mock a crash. | 
|  | result = ResultCollection( | 
|  | test_results=[TestResult('crash', TestStatus.CRASH)]) | 
|  | result.crashed = True | 
|  | result.add_test_result(TestResult('pass', TestStatus.PASS)) | 
|  | result.add_test_result( | 
|  | TestResult('fail', TestStatus.FAIL, test_log='some logs')) | 
|  | return result | 
|  | else: | 
|  | return ResultCollection( | 
|  | test_results=[TestResult('crash', TestStatus.PASS)]) | 
|  |  | 
|  | def tear_down(self): | 
|  | return | 
|  |  | 
|  | self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) | 
|  | self.mock(test_runner.TestRunner, '_run', _run) | 
|  | self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) | 
|  |  | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', | 
|  | 'fake-iossim', | 
|  | 'iPhone X', | 
|  | '11.4', | 
|  | 'out-dir', | 
|  | ) | 
|  | tr.launch() | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_failed_test_retry(self, mock_run, _1, _2): | 
|  | test1_fail_result = TestResult('test1', TestStatus.FAIL) | 
|  | test2_fail_result = TestResult('test2', TestStatus.FAIL) | 
|  | test1_pass_result = TestResult('test1', TestStatus.PASS) | 
|  | test2_pass_result = TestResult('test2', TestStatus.PASS) | 
|  | result1 = ResultCollection( | 
|  | test_results=[test1_fail_result, test2_fail_result]) | 
|  | retry_result1 = ResultCollection(test_results=[test1_pass_result]) | 
|  | retry_result2 = ResultCollection(test_results=[test2_pass_result]) | 
|  | mock_run.side_effect = [result1, retry_result1, retry_result2] | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3) | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 3) | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_crashed_if_crash_in_final_crash_retry(self, mock_run, _1, _2): | 
|  | test1_crash_result = TestResult('test1', TestStatus.CRASH) | 
|  | test2_crash_result = TestResult('test2', TestStatus.CRASH) | 
|  | test3_pass_result = TestResult('test3', TestStatus.PASS) | 
|  | test1_pass_result = TestResult('test1', TestStatus.PASS) | 
|  | test2_pass_result = TestResult('test2', TestStatus.PASS) | 
|  | initial_result = ResultCollection(test_results=[test1_crash_result]) | 
|  | initial_result.crashed = True | 
|  | crash_retry1_result = ResultCollection(test_results=[test2_crash_result]) | 
|  | crash_retry1_result.crashed = True | 
|  | crash_retry2_result = ResultCollection(test_results=[test3_pass_result]) | 
|  | crash_retry2_result.crashed = True | 
|  | test_retry1_result = ResultCollection(test_results=[test1_pass_result]) | 
|  | test_retry2_result = ResultCollection(test_results=[test2_pass_result]) | 
|  | mock_run.side_effect = [ | 
|  | initial_result, crash_retry1_result, crash_retry2_result, | 
|  | test_retry1_result, test_retry2_result | 
|  | ] | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3) | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 5) | 
|  | self.assertTrue(tr.test_results['interrupted']) | 
|  | self.assertIn('test suite crash', tr.logs) | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_not_crashed_if_no_crash_in_final_crash_retry(self, mock_run, _1, _2): | 
|  | test1_crash_result = TestResult('test1', TestStatus.CRASH) | 
|  | test2_crash_result = TestResult('test2', TestStatus.CRASH) | 
|  | test3_pass_result = TestResult('test3', TestStatus.PASS) | 
|  | test1_pass_result = TestResult('test1', TestStatus.PASS) | 
|  | test2_pass_result = TestResult('test2', TestStatus.PASS) | 
|  | initial_result = ResultCollection(test_results=[test1_crash_result]) | 
|  | initial_result.crashed = True | 
|  | crash_retry1_result = ResultCollection(test_results=[test2_crash_result]) | 
|  | crash_retry1_result.crashed = True | 
|  | crash_retry2_result = ResultCollection(test_results=[test3_pass_result]) | 
|  | test_retry1_result = ResultCollection(test_results=[test1_pass_result]) | 
|  | test_retry2_result = ResultCollection(test_results=[test2_pass_result]) | 
|  | mock_run.side_effect = [ | 
|  | initial_result, crash_retry1_result, crash_retry2_result, | 
|  | test_retry1_result, test_retry2_result | 
|  | ] | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3) | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 5) | 
|  | self.assertFalse(tr.test_results['interrupted']) | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_not_crashed_if_crashed_in_failed_test_retry(self, mock_run, _1, _2): | 
|  | test1_fail_result = TestResult('test1', TestStatus.FAIL) | 
|  | initial_result = ResultCollection(test_results=[test1_fail_result]) | 
|  | test1_retry1_result = ResultCollection(test_results=[test1_fail_result]) | 
|  | test1_retry2_result = ResultCollection(test_results=[test1_fail_result]) | 
|  | test1_retry3_result = ResultCollection() | 
|  | test1_retry3_result.crashed = True | 
|  |  | 
|  | mock_run.side_effect = [ | 
|  | initial_result, test1_retry1_result, test1_retry2_result, | 
|  | test1_retry3_result | 
|  | ] | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3) | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 4) | 
|  | self.assertFalse(tr.test_results['interrupted']) | 
|  | self.assertEqual(tr.test_results['tests']['test1']['actual'], | 
|  | 'FAIL FAIL FAIL SKIP') | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  | @mock.patch('test_runner.SimulatorTestRunner.tear_down') | 
|  | @mock.patch('test_runner.SimulatorTestRunner.set_up') | 
|  | @mock.patch('test_runner.TestRunner._run') | 
|  | def test_crashed_spawning_launcher_no_retry(self, mock_run, _1, _2): | 
|  | test1_crash_result = TestResult('test1', TestStatus.CRASH) | 
|  | initial_result = ResultCollection(test_results=[test1_crash_result]) | 
|  | initial_result.crashed = True | 
|  | initial_result.spawning_test_launcher = True | 
|  | mock_run.side_effect = [initial_result] | 
|  | tr = test_runner.SimulatorTestRunner( | 
|  | 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3) | 
|  | tr.launch() | 
|  | self.assertEqual(len(mock_run.mock_calls), 1) | 
|  | self.assertTrue(tr.test_results['interrupted']) | 
|  | self.assertIn('test suite crash', tr.logs) | 
|  | self.assertTrue(tr.logs) | 
|  |  | 
|  |  | 
|  | class DeviceTestRunnerTest(TestCase): | 
|  | def setUp(self): | 
|  | super(DeviceTestRunnerTest, self).setUp() | 
|  |  | 
|  | def install_xcode(build, mac_toolchain_cmd, xcode_app_path): | 
|  | return True | 
|  |  | 
|  | self.mock(result_sink_util.ResultSinkClient, | 
|  | 'post', lambda *args, **kwargs: None) | 
|  | self.mock(test_runner, 'get_current_xcode_info', lambda: { | 
|  | 'version': 'test version', 'build': 'test build', 'path': 'test/path'}) | 
|  | self.mock(test_runner, 'install_xcode', install_xcode) | 
|  | self.mock(test_runner.subprocess, | 
|  | 'check_output', lambda _: b'fake-bundle-id') | 
|  | self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) | 
|  | self.mock(os.path, 'exists', lambda _: True) | 
|  | self.mock(os, 'listdir', lambda _: []) | 
|  | self.mock(tempfile, 'mkstemp', lambda: '/tmp/tmp_file') | 
|  | self.tr = test_runner.DeviceTestRunner( | 
|  | 'fake-app', | 
|  | 'xcode-version', | 
|  | 'xcode-build', | 
|  | 'out-dir', | 
|  | ) | 
|  | self.tr.xctestrun_data = {'TestTargetName': {}} | 
|  |  | 
|  |  | 
|  | if __name__ == '__main__': | 
|  | logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s', | 
|  | level=logging.DEBUG, datefmt='%I:%M:%S') | 
|  | unittest.main() |