| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unittests for xcodebuild_runner.py.""" |
| |
| import logging |
| import mock |
| import os |
| import unittest |
| |
| import iossim_util |
| import result_sink_util |
| import test_apps |
| from test_result_util import ResultCollection, TestResult, TestStatus |
| import test_runner |
| import test_runner_test |
| import xcode_log_parser |
| import xcodebuild_runner |
| |
| |
| _ROOT_FOLDER_PATH = 'root/folder' |
| _XCODE_BUILD_VERSION = '10B61' |
| _DESTINATION = 'A4E66321-177A-450A-9BA1-488D85B7278E' |
| _OUT_DIR = 'out/dir' |
| _XTEST_RUN = '/tmp/temp_file.xctestrun' |
| _EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH |
| _FLAKY_EGTEST_APP_PATH = 'path/to/ios_chrome_flaky_eg2test_module.app' |
| |
| |
| class XCodebuildRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.""" |
| |
| def setUp(self): |
| super(XCodebuildRunnerTest, self).setUp() |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock(xcode_log_parser, |
| 'get_parser', lambda: xcode_log_parser.Xcode11LogParser()) |
| self.mock(os, 'listdir', lambda _: ['any_egtests.xctest']) |
| self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) |
| self.mock(result_sink_util.ResultSinkClient, |
| 'post', lambda *args, **kwargs: None) |
| self.mock( |
| test_apps.GTestsApp, |
| 'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) |
| self.mock(test_apps.EgtestsApp, |
| 'fill_xctest_run', lambda _1, _2: 'xctestrun') |
| self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID') |
| self.mock(test_apps, 'get_bundle_id', lambda _: "fake-bundle-id") |
| self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '') |
| self.mock(test_runner.SimulatorTestRunner, 'tear_down', lambda _: None) |
| self.mock(test_runner.DeviceTestRunner, 'tear_down', lambda _: None) |
| self.mock(xcodebuild_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner, 'print_process_output', lambda _: []) |
| |
| def tearDown(self): |
| super(XCodebuildRunnerTest, self).tearDown() |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) |
| crashed_collection = ResultCollection() |
| crashed_collection.crashed = True |
| mock_collect_results.side_effect = [ |
| crashed_collection, |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| ] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, shards=1, retries=3) |
| overall_result = launch_command.launch() |
| self.assertFalse(overall_result.crashed) |
| self.assertEqual(len(overall_result.all_test_names()), 2) |
| self.assertEqual(overall_result.expected_tests(), |
| set(['Class1/passedTest1', 'Class1/passedTest2'])) |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def testLaunchCommand_notRestartPassedTest(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) |
| collection = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| mock_collect_results.side_effect = [collection] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, shards=1, retries=3) |
| launch_command.launch() |
| xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) |
| self.assertEqual(1, len(mock_collect_results.mock_calls)) |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def test_launch_command_restart_failed_attempt(self, mock_collect_results): |
| egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) |
| mock_collect_results.side_effect = [ |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.FAIL), |
| TestResult('Class1/passedTest2', TestStatus.FAIL) |
| ]), |
| ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| ] |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, shards=1, retries=3) |
| overall_result = launch_command.launch() |
| self.assertEqual(len(overall_result.all_test_names()), 2) |
| self.assertEqual(overall_result.expected_tests(), |
| set(['Class1/passedTest1', 'Class1/passedTest2'])) |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def test_launch_command_not_restart_crashed_attempt(self, |
| mock_collect_results): |
| """Crashed first attempt of runtime select test suite won't be retried.""" |
| egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH) |
| crashed_collection = ResultCollection() |
| crashed_collection.crashed = True |
| mock_collect_results.return_value = crashed_collection |
| launch_command = xcodebuild_runner.LaunchCommand( |
| egtests, _DESTINATION, shards=1, retries=3) |
| overall_result = launch_command.launch() |
| self.assertEqual(len(overall_result.all_test_names()), 0) |
| self.assertEqual(overall_result.expected_tests(), set([])) |
| self.assertTrue(overall_result.crashed) |
| |
| |
| class DeviceXcodeTestRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.DeviceXcodeTestRunner.""" |
| |
| def setUp(self): |
| super(DeviceXcodeTestRunnerTest, self).setUp() |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock(test_runner, 'get_current_xcode_info', lambda: { |
| 'version': 'test version', 'build': 'test build', 'path': 'test/path'}) |
| self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) |
| |
| self.mock(result_sink_util.ResultSinkClient, |
| 'post', lambda *args, **kwargs: None) |
| self.mock(test_runner.subprocess, 'check_output', lambda _: 'fake-output') |
| self.mock(test_runner.subprocess, 'check_call', lambda _: 'fake-out') |
| self.mock(test_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner.TestRunner, 'set_sigterm_handler', |
| lambda self, handler: 0) |
| self.mock(os, 'listdir', lambda _: []) |
| self.mock(xcodebuild_runner.subprocess, |
| 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') |
| self.mock(test_runner, 'print_process_output', lambda _: []) |
| self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0) |
| self.mock(test_runner.DeviceTestRunner, 'get_installed_packages', |
| lambda self: []) |
| self.mock(test_runner.DeviceTestRunner, 'wipe_derived_data', lambda _: None) |
| self.mock(test_runner.TestRunner, 'retrieve_derived_data', lambda _: None) |
| self.mock(test_runner.TestRunner, 'process_xcresult_dir', lambda _: None) |
| self.mock(xcode_log_parser, |
| 'get_parser', lambda: xcode_log_parser.Xcode11LogParser()) |
| self.mock(test_apps.EgtestsApp, |
| 'fill_xctest_run', lambda _1, _2: 'xctestrun') |
| self.mock( |
| test_apps.GTestsApp, |
| 'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) |
| self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def test_launch(self, mock_result): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", |
| "fake-host-app-path", |
| "fake-out-dir") |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def test_unexpected_skipped_crash_reported(self, mock_result): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", |
| "fake-host-app-path", |
| "fake-out-dir") |
| crashed_collection = ResultCollection( |
| test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) |
| crashed_collection.crashed = True |
| mock_result.return_value = crashed_collection |
| self.assertFalse(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 3) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP') |
| self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS') |
| |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| def test_unexpected_skipped_not_reported(self, mock_result): |
| """Unexpected skip not reported for these selecting tests at runtime.""" |
| crashed_collection = ResultCollection( |
| test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) |
| crashed_collection.crashed = True |
| mock_result.return_value = crashed_collection |
| tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH, |
| "fake-host-app-path", |
| "fake-out-dir") |
| self.assertFalse(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 2) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| |
| @mock.patch('xcodebuild_runner.isinstance', return_value=True) |
| @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results') |
| @mock.patch('test_apps.EgtestsApp', autospec=True) |
| def test_disabled_reported(self, mock_test_app, mock_result, _): |
| """Tests launch method in DeviceXcodeTestRunner""" |
| test_app = mock_test_app.return_value |
| test_app.test_app_path = _EGTESTS_APP_PATH |
| test_app.disabled_tests = ['Class2/disabled_test3'] |
| test_app.get_all_tests.return_value = [ |
| 'Class1/passedTest1', 'Class1/passedTest2' |
| ] |
| mock_result.return_value = ResultCollection(test_results=[ |
| TestResult('Class1/passedTest1', TestStatus.PASS), |
| TestResult('Class1/passedTest2', TestStatus.PASS) |
| ]) |
| tr = xcodebuild_runner.DeviceXcodeTestRunner( |
| "fake-app-path", "fake-host-app-path", "fake-out-dir") |
| self.assertTrue(tr.launch()) |
| self.assertEqual(len(tr.test_results['tests']), 3) |
| tests = tr.test_results['tests'] |
| self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') |
| self.assertEqual(tests['Class1/passedTest2']['actual'], 'PASS') |
| self.assertEqual(tests['Class2/disabled_test3']['actual'], 'SKIP') |
| self.assertEqual(tests['Class2/disabled_test3']['expected'], 'SKIP') |
| |
| def test_tear_down(self): |
| tr = xcodebuild_runner.DeviceXcodeTestRunner( |
| "fake-app-path", "fake-host-app-path", "fake-out-dir") |
| tr.tear_down() |
| |
| |
| if __name__ == '__main__': |
| logging.basicConfig( |
| format='[%(asctime)s:%(levelname)s] %(message)s', |
| level=logging.DEBUG, |
| datefmt='%I:%M:%S') |
| unittest.main() |
