| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unittests for xcodebuild_runner.py.""" |
| |
| import mock |
| import os |
| |
| import plistlib |
| import shutil |
| import tempfile |
| import test_runner |
| import test_runner_test |
| import xcodebuild_runner |
| |
| |
| _ROOT_FOLDER_PATH = 'root/folder' |
| _XCODE_BUILD_VERSION = '10B61' |
| _DESTINATION = 'platform=iOS Simulator,OS=12.0,name=iPhone X' |
| _OUT_DIR = 'out/dir' |
| _XTEST_RUN = '/tmp/temp_file.xctestrun' |
| _EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH |
| |
| |
| class XCodebuildRunnerTest(test_runner_test.TestCase): |
| """Test case to test xcodebuild_runner.""" |
| |
| def setUp(self): |
| super(XCodebuildRunnerTest, self).setUp() |
| self.mock(os.path, 'exists', lambda _: True) |
| self.mock(xcodebuild_runner.LaunchCommand, '_copy_screenshots', |
| lambda _1, _2, _3: None) |
| self.mock(xcodebuild_runner.LaunchCommand, 'summary_log', lambda _: None) |
| self.tmpdir = tempfile.mkdtemp() |
| |
| def tearDown(self): |
| shutil.rmtree(self.tmpdir, ignore_errors=True) |
| super(XCodebuildRunnerTest, self).tearDown() |
| |
| def fake_launch_attempt(self, obj, statuses): |
| attempt = [0] |
| info_plist_statuses = { |
| 'not_started': { |
| 'Actions': [{'ActionResult': {}}], |
| 'TestsCount': 0, 'TestsFailedCount': 0 |
| }, |
| 'fail': { |
| 'Actions': [ |
| {'ActionResult': { |
| 'TestSummaryPath': '1_Test/TestSummaries.plist'} |
| } |
| ], |
| 'TestsCount': 99, |
| 'TestsFailedCount': 1}, |
| 'pass': { |
| 'Actions': [ |
| {'ActionResult': { |
| 'TestSummaryPath': '1_Test/TestSummaries.plist'} |
| } |
| ], |
| 'TestsCount': 100, |
| 'TestsFailedCount': 0} |
| } |
| |
| test_summary = { |
| 'TestableSummaries': [ |
| {'TargetName': 'egtests', |
| 'Tests': [ |
| {'Subtests': [ |
| {'Subtests': [ |
| {'Subtests': [ |
| {'TestIdentifier': 'passed_test', |
| 'TestStatus': 'Success' |
| } |
| ] |
| } |
| ] |
| } |
| ] |
| } |
| ] |
| } |
| ] |
| } |
| |
| def the_fake(cmd, attempt_outdir): |
| index = attempt[0] |
| attempt[0] += 1 |
| self.assertEqual(os.path.join(self.tmpdir, 'attempt_%d' % index), |
| attempt_outdir) |
| self.assertEqual(1, cmd.count(attempt_outdir)) |
| os.mkdir(attempt_outdir) |
| with open(os.path.join(attempt_outdir, 'Info.plist'), 'w') as f: |
| plistlib.writePlist(info_plist_statuses[statuses[index]], f) |
| summary_folder = os.path.join(attempt_outdir, '1_Test') |
| os.mkdir(summary_folder) |
| with open(os.path.join(summary_folder, 'TestSummaries.plist'), 'w') as f: |
| plistlib.writePlist(test_summary, f) |
| return (-6, 'Output for attempt_%d' % index) |
| |
| obj.launch_attempt = the_fake |
| |
| def testEgtests_not_found_egtests_app(self): |
| self.mock(os.path, 'exists', lambda _: False) |
| with self.assertRaises(test_runner.AppNotFoundError): |
| xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) |
| |
| def testEgtests_not_found_plugins(self): |
| egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) |
| self.mock(os.path, 'exists', lambda _: False) |
| with self.assertRaises(test_runner.PlugInsNotFoundError): |
| egtests._xctest_path() |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testEgtests_found_xctest(self, mock_listdir): |
| mock_listdir.return_value = ['any_egtests.xctest'] |
| self.assertEqual('/PlugIns/any_egtests.xctest', |
| xcodebuild_runner.EgtestsApp( |
| _EGTESTS_APP_PATH)._xctest_path()) |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testEgtests_not_found_xctest(self, mock_listdir): |
| mock_listdir.return_value = ['some_egtests.xctest'] |
| egtest = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) |
| with self.assertRaises(test_runner.XCTestPlugInNotFoundError): |
| egtest._xctest_path() |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testEgtests_xctestRunNode_without_filter(self, mock_listdir): |
| mock_listdir.return_value = ['any_egtests.xctest'] |
| egtest_node = xcodebuild_runner.EgtestsApp( |
| _EGTESTS_APP_PATH).xctestrun_node()['any_egtests_module'] |
| self.assertNotIn('OnlyTestIdentifiers', egtest_node) |
| self.assertNotIn('SkipTestIdentifiers', egtest_node) |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testEgtests_xctestRunNode_with_filter_only_identifiers(self, |
| mock_listdir): |
| mock_listdir.return_value = ['any_egtests.xctest'] |
| filtered_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2', |
| 'TestCase2/testMethod1', 'TestCase1/testMethod2'] |
| egtest_node = xcodebuild_runner.EgtestsApp( |
| _EGTESTS_APP_PATH, filtered_tests=filtered_tests).xctestrun_node()[ |
| 'any_egtests_module'] |
| self.assertEqual(filtered_tests, egtest_node['OnlyTestIdentifiers']) |
| self.assertNotIn('SkipTestIdentifiers', egtest_node) |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testEgtests_xctestRunNode_with_filter_skip_identifiers(self, |
| mock_listdir): |
| mock_listdir.return_value = ['any_egtests.xctest'] |
| skipped_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2', |
| 'TestCase2/testMethod1', 'TestCase1/testMethod2'] |
| egtest_node = xcodebuild_runner.EgtestsApp( |
| _EGTESTS_APP_PATH, filtered_tests=skipped_tests, |
| invert=True).xctestrun_node()['any_egtests_module'] |
| self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers']) |
| self.assertNotIn('OnlyTestIdentifiers', egtest_node) |
| |
| @mock.patch('xcodebuild_runner.LaunchCommand.fill_xctest_run', autospec=True) |
| def testLaunchCommand_command(self, mock_fill_xctestrun): |
| mock_fill_xctestrun.return_value = _XTEST_RUN |
| mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) |
| type(mock_egtest).egtests_path = mock.PropertyMock( |
| return_value=_EGTESTS_APP_PATH) |
| cmd = xcodebuild_runner.LaunchCommand( |
| mock_egtest, _DESTINATION, shards=3, retries=1, out_dir=_OUT_DIR) |
| self.assertEqual(['xcodebuild', 'test-without-building', |
| '-xctestrun', '/tmp/temp_file.xctestrun', |
| '-destination', |
| 'platform=iOS Simulator,OS=12.0,name=iPhone X', |
| '-resultBundlePath', 'out/dir', |
| '-parallel-testing-enabled', 'YES', |
| '-parallel-testing-worker-count', '3'], |
| cmd.command(egtests_app=mock_egtest, |
| out_dir=_OUT_DIR, |
| destination=_DESTINATION, |
| shards=3)) |
| |
| @mock.patch('plistlib.writePlist', autospec=True) |
| @mock.patch('os.path.join', autospec=True) |
| def testFill_xctest_run(self, mock_path_join, _): |
| self._mocks[xcodebuild_runner.LaunchCommand].pop('fill_xctest_run', None) |
| mock_path_join.return_value = _XTEST_RUN |
| mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) |
| launch_command = xcodebuild_runner.LaunchCommand( |
| mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR) |
| self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest)) |
| self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls) |
| |
| def testFill_xctest_run_exception(self): |
| with self.assertRaises(test_runner.AppNotFoundError): |
| xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1, |
| out_dir=_OUT_DIR).fill_xctest_run([]) |
| |
| @mock.patch('xcodebuild_runner.LaunchCommand.fill_xctest_run', autospec=True) |
| def testLaunchCommand_make_cmd_list_for_failed_tests(self, |
| fill_xctest_run_mock): |
| fill_xctest_run_mock.side_effect = [ |
| '/var/folders/tmpfile1' |
| ] |
| egtest_app = 'module_1_egtests.app' |
| egtest_app_path = '%s/%s' % (_ROOT_FOLDER_PATH, egtest_app) |
| failed_tests = { |
| egtest_app: [ |
| 'TestCase1_1/TestMethod1', |
| 'TestCase1_1/TestMethod2', |
| 'TestCase1_2/TestMethod1', |
| ] |
| } |
| expected_egtests = xcodebuild_runner.EgtestsApp( |
| egtest_app_path, filtered_tests=failed_tests[egtest_app]) |
| mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) |
| type(mock_egtest).egtests_path = mock.PropertyMock( |
| return_value=egtest_app_path) |
| cmd = xcodebuild_runner.LaunchCommand( |
| egtests_app=mock_egtest, |
| destination=_DESTINATION, |
| out_dir='out/dir/attempt_2/iPhone X 12.0', |
| shards=1, |
| retries=1 |
| ) |
| cmd._make_cmd_list_for_failed_tests( |
| failed_tests, os.path.join(_OUT_DIR, 'attempt_2')) |
| self.assertEqual(1, len(fill_xctest_run_mock.mock_calls)) |
| self.assertItemsEqual(expected_egtests.__dict__, |
| fill_xctest_run_mock.mock_calls[0][1][1].__dict__) |
| |
| @mock.patch('os.listdir', autospec=True) |
| def testLaunchCommand_restartFailed1stAttempt(self, mock_listdir): |
| mock_listdir.return_value = ['any_egtests.xctest'] |
| egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) |
| launch_command = xcodebuild_runner.LaunchCommand(egtests, |
| _DESTINATION, |
| shards=1, |
| retries=3, |
| out_dir=self.tmpdir) |
| self.fake_launch_attempt(launch_command, ['not_started', 'pass']) |
| launch_command.launch() |
| self.assertEqual(2, len(launch_command.test_results)) |
