| #!/usr/bin/env vpython3 |
| # Copyright 2018 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unittests for xcodebuild_runner.py.""" |
| |
| import logging |
| import mock |
| import os |
| import unittest |
| import sys |
| |
| import mac_util |
| import iossim_util |
| import result_sink_util |
| import test_apps |
| from test_result_util import ResultCollection, TestResult, TestStatus |
| import test_runner |
| import test_runner_test |
| import xcode_log_parser |
| import xcode_util |
| import xcodebuild_runner |
| |
| # if the current directory is in scripts, then we need to add plugin |
| # path in order to import from that directory |
| if os.path.split(os.path.dirname(__file__))[1] != 'plugin': |
| sys.path.append( |
| os.path.join(os.path.abspath(os.path.dirname(__file__)), 'plugin')) |
| import test_plugin_service |
| |
| _ROOT_FOLDER_PATH = 'root/folder' |
| _XCODE_BUILD_VERSION = '10B61' |
| _DESTINATION = 'A4E66321-177A-450A-9BA1-488D85B7278E' |
| _OUT_DIR = 'out/dir' |
| _XTEST_RUN = '/tmp/temp_file.xctestrun' |
| _EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH |
| _ALL_EG_TEST_NAMES = [('Class1', 'passedTest1'), ('Class1', 'passedTest2')] |
| _FLAKY_EGTEST_APP_PATH = 'path/to/ios_chrome_flaky_eg2test_module.app' |
| |
| _ENUMERATE_TESTS_OUTPUT = """ |
| { |
| "errors" : [ |
| |
| ], |
| "values" : [ |
| { |
| "children" : [ |
| { |
| "children" : [ |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "BaseEarlGreyTestCase" |
| }, |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "ChromeTestCase" |
| }, |
| { |
| "children" : [ |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "test", |
| "name" : "passedTest1" |
| }, |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "test", |
| "name" : "passedTest2" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "Class1" |
| }, |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "WebHttpServerChromeTestCase" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "target", |
| "name" : "ios_chrome_ui_eg2tests_module-Runner_module" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "plan", |
| "name" : "" |
| } |
| ] |
| } |
| """ |
| |
| _ENUMERATE_DISABLED_TESTS_OUTPUT = """ |
| { |
| "errors" : [ |
| |
| ], |
| "values" : [ |
| { |
| "children" : [ |
| { |
| "children" : [ |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "BaseEarlGreyTestCase" |
| }, |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "ChromeTestCase" |
| }, |
| { |
| "children" : [ |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "test", |
| "name" : "disabled_test3" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "Class2" |
| }, |
| { |
| "children" : [ |
| |
| ], |
| "disabled" : false, |
| "kind" : "class", |
| "name" : "WebHttpServerChromeTestCase" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "target", |
| "name" : "ios_chrome_ui_eg2tests_module-Runner_module" |
| } |
| ], |
| "disabled" : false, |
| "kind" : "plan", |
| "name" : "" |
| } |
| ] |
| } |
| """ |
| |
| class XCodebuildRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.""" |
| |
| def setUp(self): |
| super(XCodebuildRunnerTest, self).setUp() |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock(os, 'listdir', lambda _: ['any_egtests.xctest']) |
| self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) |
| self.mock(result_sink_util.ResultSinkClient, |
| 'post', lambda *args, **kwargs: None) |
| self.mock(test_apps.EgtestsApp, 'get_all_tests', |
| lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) |
| self.mock(test_apps.EgtestsApp, 'fill_xctest_run', |
| lambda _1, _2: 'xctestrun') |
| self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID') |
| self.mock(test_apps, 'get_bundle_id', lambda _: "fake-bundle-id") |
| self.mock(test_apps, 'is_running_rosetta', lambda: False) |
| self.mock(test_apps.plistlib, 'dump', lambda _1, _2: '') |
| self.mock(test_runner.SimulatorTestRunner, 'tear_down', lambda _: None) |
| self.mock(test_runner.DeviceTestRunner, 'tear_down', lambda _: None) |
| self.mock(xcodebuild_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner, 'print_process_output', lambda _, timeout: []) |
| self.mock(xcode_util, 'xctest_path', lambda _: 'fake-path') |
| self.mock(os.path, 'isfile', lambda _: True) |
| self.mock(xcodebuild_runner.SimulatorParallelTestRunner, |
| '_create_xctest_run_enum_tests', |
| lambda _, include_disabled: 'fake-path') |
| |
| def tearDown(self): |
| super(XCodebuildRunnerTest, self).tearDown() |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH, _ALL_EG_TEST_NAMES) |
| crashed_collection = ResultCollection() |
| crashed_collection.crashed = True |
| mock_collect_results.side_effect = [ |
| crashed_collection, |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| ] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| overall_result = launch_command.launch() |
| self.assertFalse(overall_result.crashed) |
| self.assertEqual(len(overall_result.all_test_names()), 2) |
| self.assertEqual(overall_result.expected_tests(), |
| set(['Class1/passedTest1', 'Class1/passedTest2'])) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| def testLaunchCommand_notRestartPassedTest(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH, _ALL_EG_TEST_NAMES) |
| collection = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| mock_collect_results.side_effect = [collection] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| launch_command.launch() |
| xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| self.assertEqual(1, len(mock_collect_results.mock_calls)) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| def test_launch_command_restart_failed_attempt(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH, _ALL_EG_TEST_NAMES) |
| mock_collect_results.side_effect = [ |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.FAIL), |
| TestResult('Class1/passedTest2', TestStatus.FAIL) |
| ]), |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| ] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| overall_result = launch_command.launch() |
| self.assertEqual(len(overall_result.all_test_names()), 2) |
| self.assertEqual(overall_result.expected_tests(), |
| set(['Class1/passedTest1', 'Class1/passedTest2'])) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| def test_launch_command_not_restart_crashed_attempt(self, |
| mock_collect_results): |
| """Crashed first attempt of runtime select test suite won't be retried.""" |
| egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH, _ALL_EG_TEST_NAMES) |
| crashed_collection = ResultCollection() |
| crashed_collection.crashed = True |
| mock_collect_results.return_value = crashed_collection |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| overall_result = launch_command.launch() |
| self.assertEqual(len(overall_result.all_test_names()), 0) |
| self.assertEqual(overall_result.expected_tests(), set([])) |
| self.assertTrue(overall_result.crashed) |
| |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| def test_launch_command_reset_video_plugin_before_attempt( |
| self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH, _ALL_EG_TEST_NAMES) |
| collection = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| mock_collect_results.side_effect = [collection] |
| mock_plugin_service = mock.MagicMock() |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, |
| _DESTINATION, |
| clones=1, |
| retries=3, |
| readline_timeout=180, |
| test_plugin_service=mock_plugin_service) |
| launch_command.launch() |
| xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, clones=1, retries=3, readline_timeout=180) |
| self.assertEqual(1, len(mock_collect_results.mock_calls)) |
| mock_plugin_service.reset.assert_called_once_with() |
| |
| |
| class DeviceXcodeTestRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.DeviceXcodeTestRunner.""" |
| |
| def setUp(self): |
| super(DeviceXcodeTestRunnerTest, self).setUp() |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock(test_runner, 'get_current_xcode_info', lambda: { |
| 'version': 'test version', 'build': 'test build', 'path': 'test/path'}) |
| self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) |
| |
| self.mock(result_sink_util.ResultSinkClient, |
| 'post', lambda *args, **kwargs: None) |
| self.mock( |
| test_runner.subprocess, |
| 'check_output', |
| lambda _, stderr=None: b'fake-output') |
| self.mock(test_runner.subprocess, 'check_call', lambda _: b'fake-out') |
| self.mock(test_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner.TestRunner, 'set_sigterm_handler', |
| lambda self, handler: 0) |
| self.mock(os, 'listdir', lambda _: []) |
| self.mock(xcodebuild_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner, 'print_process_output', lambda _, timeout: []) |
| self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0) |
| self.mock(test_runner.DeviceTestRunner, 'get_installed_packages', |
| lambda self: []) |
| self.mock(test_runner.DeviceTestRunner, 'wipe_derived_data', lambda _: None) |
| self.mock(test_runner.TestRunner, 'retrieve_derived_data', lambda _: None) |
| self.mock(test_runner.TestRunner, 'process_xcresult_dir', lambda _: None) |
| self.mock(test_apps.EgtestsApp, |
| 'fill_xctest_run', lambda _1, _2: 'xctestrun') |
| self.mock(test_apps.EgtestsApp, 'get_all_tests', |
| lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) |
| self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) |
| self.mock(xcode_util, 'using_xcode_15_or_higher', lambda: True) |
| self.mock(mac_util, 'kill_usbmuxd', lambda: None) |
| self.mock(xcode_util, 'xctest_path', lambda _: 'fake-path') |
| self.mock(os.path, 'isfile', lambda _: True) |
| self.mock(xcodebuild_runner.SimulatorParallelTestRunner, |
| '_create_xctest_run_enum_tests', |
| lambda _, include_disabled: 'fake-path') |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_launch(self, _, mock_result): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", |
| "fake-host-app-path", |
| "fake-out-dir") |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_unexpected_skipped_crash_reported(self, _, mock_result): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", |
| "fake-host-app-path", |
| "fake-out-dir") |
| crashed_collection = ResultCollection( |
| test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) |
| crashed_collection.crashed = True |
| mock_result.return_value = crashed_collection |
| self.assertFalse(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP') |
| self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS') |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_unexpected_skipped_not_reported(self, _, mock_result): |
| """Unexpected skip not reported for these selecting tests at runtime.""" |
| crashed_collection = ResultCollection( |
| test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) |
| crashed_collection.crashed = True |
| mock_result.return_value = crashed_collection |
| tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH, |
| "fake-host-app-path", |
| "fake-out-dir") |
| self.assertFalse(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 1) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| # Class1/passedTest2 doesn't appear in test results. |
| |
| @mock.patch( |
| 'builtins.open', |
| new=mock.mock_open(read_data=_ENUMERATE_DISABLED_TESTS_OUTPUT)) |
| @mock.patch('xcodebuild_runner.isinstance', return_value=True) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('test_apps.EgtestsApp', autospec=True) |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_disabled_reported(self, _, mock_test_app, mock_result, __): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| test_app = mock_test_app.return_value |
| test_app.test_app_path = _EGTESTS_APP_PATH |
| test_app.get_all_tests.return_value = [ |
| 'Class1/passedTest1', 'Class1/passedTest2' |
| ] |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| tr = xcodebuild_runner.DeviceXcodeTestRunner( |
| "fake-app-path", |
| "fake-host-app-path", |
| "fake-out-dir", |
| output_disabled_tests=True) |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 3) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| self.assertEqual(tests['Class1/passedTest2']['actual'], 'PASS') |
| self.assertEqual(tests['Class2/disabled_test3']['actual'], 'SKIP') |
| self.assertEqual(tests['Class2/disabled_test3']['expected'], 'SKIP') |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| def test_tear_down(self): |
| tr = xcodebuild_runner.DeviceXcodeTestRunner( |
| "fake-app-path", "fake-host-app-path", "fake-out-dir") |
| tr.tear_down() |
| |
| |
| class SimulatorParallelTestRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.SimulatorParallelTestRunner""" |
| |
| def setUp(self): |
| super(SimulatorParallelTestRunnerTest, self).setUp() |
| self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID') |
| |
| def set_up(self): |
| return |
| |
| self.mock(xcodebuild_runner.SimulatorParallelTestRunner, 'set_up', set_up) |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock( |
| test_runner, 'get_current_xcode_info', lambda: { |
| 'version': 'test version', |
| 'build': 'test build', |
| 'path': 'test/path' |
| }) |
| self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) |
| |
| self.mock(result_sink_util.ResultSinkClient, |
| 'post', lambda *args, **kwargs: None) |
| self.mock( |
| test_runner.subprocess, |
| 'check_output', |
| lambda _, stderr=None: b'fake-output') |
| self.mock(test_runner.subprocess, 'check_call', lambda _: b'fake-out') |
| self.mock(test_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner.TestRunner, |
| 'set_sigterm_handler', lambda self, handler: 0) |
| self.mock(os, 'listdir', lambda _: []) |
| self.mock(xcodebuild_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner, 'print_process_output', lambda _, timeout: []) |
| self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0) |
| self.mock(test_runner.TestRunner, 'retrieve_derived_data', lambda _: None) |
| self.mock(test_runner.TestRunner, 'process_xcresult_dir', lambda _: None) |
| self.mock(test_apps.EgtestsApp, 'fill_xctest_run', |
| lambda _1, _2: 'xctestrun') |
| self.mock(test_apps.EgtestsApp, 'get_all_tests', |
| lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) |
| self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) |
| self.mock(xcode_util, 'xctest_path', lambda _: 'fake-path') |
| self.mock(os.path, 'isfile', lambda _: True) |
| self.mock(xcodebuild_runner.SimulatorParallelTestRunner, |
| '_create_xctest_run_enum_tests', |
| lambda _, include_disabled: 'fake-path') |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_launch_egtest(self, _, mock_result): |
| """Tests launch method in SimulatorParallelTestRunner""" |
| tr = xcodebuild_runner.SimulatorParallelTestRunner( |
| "fake-app-path", "fake-host-app-path", "fake-iossim_path", |
| "fake-version", "fake-platform", "fake-out-dir") |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| |
| @mock.patch( |
| 'builtins.open', new=mock.mock_open(read_data=_ENUMERATE_TESTS_OUTPUT)) |
| @mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results') |
| @mock.patch('xcodebuild_runner.TestPluginServicerWrapper') |
| @mock.patch('platform.system', return_value='Darwin') |
| def test_launch_egtest_with_plugin_service(self, _, mock_plugin_service, |
| mock_result): |
| """ Tests launch method in SimulatorParallelTestRunner |
| with plugin service running """ |
| tr = xcodebuild_runner.SimulatorParallelTestRunner( |
| "fake-app-path", |
| "fake-host-app-path", |
| "fake-iossim_path", |
| "fake-version", |
| "fake-platform", |
| "fake-out-dir", |
| video_plugin_option='failed_only') |
| self.assertTrue(tr.test_plugin_service != None) |
| tr.test_plugin_service = mock_plugin_service |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| mock_plugin_service.start_server.assert_called_once_with() |
| mock_plugin_service.reset.assert_called_once_with() |
| mock_plugin_service.tear_down.assert_called_once_with() |
| |
| |
| if __name__ == '__main__': |
| logging.basicConfig( |
| format='[%(asctime)s:%(levelname)s] %(message)s', |
| level=logging.DEBUG, |
| datefmt='%I:%M:%S') |
| unittest.main() |
