| #!/usr/bin/env vpython3 |
| # Copyright 2020 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import collections |
| from mock import patch |
| import os |
| import unittest |
| |
| import shard_util |
| |
| DEBUG_APP_OTOOL_OUTPUT = '\n'.join([ |
| 'Meta Class', 'name 0x1064b8438 CacheTestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'imp 0x1075e6887 -[CacheTestCase testA]', 'types 0x1064cc3e1', |
| 'imp 0x1075e6887 -[CacheTestCase testB]', |
| 'imp 0x1075e6887 -[CacheTestCase testc]', 'name 0x1064b8438 TabUITestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'imp 0x1075e6887 -[TabUITestCase testD]', 'types 0x1064cc3e1 v16@0:8', |
| 'imp 0x1075e6887 -[TabUITestCase testE]', |
| 'name 0x1064b8438 KeyboardTestCase', |
| 'imp 0x1075e6887 -[KeyboardTestCase testF]', |
| 'name 0x1064b8438 PasswordManagerTestCase', |
| 'imp 0x1075e6887 -[PasswordManagerTestCase testG]', |
| 'name 0x1064b8438 ToolBarTestCase', |
| 'imp 0x1075e6887 -[ToolBarTestCase testH]', |
| 'imp 0x1075e6887 -[ToolBarTestCase DISABLED_testI]', |
| 'imp 0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0', |
| 'name 0x1075d8539 PasswordManagerPasswordCheckupDisabledTestCase', |
| 'imp 0x1075d8539 -[PasswordManagerPasswordCheckupDisabledTestCase testK]' |
| ]).encode('utf-8') |
| |
| # Debug app otool output format in Xcode 11.4 toolchain. |
| DEBUG_APP_OTOOL_OUTPUT_114 = '\n'.join([ |
| 'Meta Class', 'name 0x1064b8438 CacheTestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| ' imp 0x1075e6887 -[CacheTestCase testA]', ' types 0x1064cc3e1', |
| ' imp 0x1075e6887 -[CacheTestCase testB]', |
| ' imp 0x1075e6887 -[CacheTestCase testc]', |
| ' name 0x1064b8438 TabUITestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| ' imp 0x1075e6887 -[TabUITestCase testD]', |
| ' types 0x1064cc3e1 v16@0:8', |
| ' imp 0x1075e6887 -[TabUITestCase testE]', |
| ' name 0x1064b8438 KeyboardTestCase', |
| ' imp 0x1075e6887 -[KeyboardTestCase testF]', |
| ' name 0x1064b8438 PasswordsTestCase', |
| ' imp 0x1075e6887 -[PasswordsTestCase testG]', |
| ' name 0x1064b8438 ToolBarTestCase', |
| ' imp 0x1075e6887 -[ToolBarTestCase testH]', |
| ' imp 0x1075e6887 -[ToolBarTestCase DISABLED_testI]', |
| ' imp 0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0' |
| ]).encode('utf-8') |
| |
| RELEASE_APP_OTOOL_OUTPUT = '\n'.join([ |
| 'Meta Class', 'name 0x1064b8438 CacheTestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'name 0x1075e6887 testA', 'types 0x1064cc3e1', 'name 0x1075e6887 testB', |
| 'name 0x1075e6887 testc', 'baseProtocols 0x0', |
| 'name 0x1064b8438 CacheTestCase', 'unrelated line', 'Meta Class', |
| 'name 0x1064b8438 TabUITestCase', 'no test methods in this case', |
| 'name 0x1064b8438 TabUITestCase', 'unrelated line', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testD', |
| 'types 0x1064cc3e1 v16@0:8', 'name 0x1075e6887 testE', |
| 'name 0x1075e6887 testF', 'baseProtocols 0x0', |
| 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testUnrelatedG', |
| 'unrelated line', 'name 0x1064b8438 ChromeTestCase', |
| 'name 0x1064b8438 setUp', 'name 0x1064b8438 testPort', |
| 'name 0x5345ac561 testSomeUnrelatedUtil', 'baseProtocols 0x0', |
| 'name 0x1064b8438 ChromeTestCase', 'unrelated line', |
| 'name 0x1064b8438 invalidTestCase', 'name 0x1075e6887 testG', |
| 'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase', |
| 'name 0x1075e6887 testG', 'name 0x1075e6887 testH', |
| 'name 0x1075e6887 DISABLED_testI', 'name 0x1075e6887 FLAKY_testJ', |
| 'name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0' |
| ]).encode('utf-8') |
| |
| RELEASE_APP_OTOOL_OUTPUT_CLASS_NOT_IN_PAIRS = '\n'.join([ |
| 'Meta Class', 'name 0x1064b8438 CacheTestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'name 0x1075e6887 testA', 'types 0x1064cc3e1', 'name 0x1075e6887 testB', |
| 'name 0x1075e6887 testc', 'baseProtocols 0x0', |
| 'name 0x1064b8438 CacheTestCase', 'unrelated line', 'Meta Class', |
| 'name 0x1064b8438 TabUITestCase', 'no test methods in this case', |
| 'name 0x1064b8438 TabUITestCase', 'unrelated line', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testD', |
| 'types 0x1064cc3e1 v16@0:8', 'name 0x1075e6887 testE', |
| 'name 0x1075e6887 testF', 'baseProtocols 0x0', |
| 'name 0x1075e6887 testUnrelatedG', 'unrelated line', |
| 'name 0x1064b8438 ChromeTestCase', 'name 0x1064b8438 setUp', |
| 'name 0x1064b8438 testPort', 'name 0x5345ac561 testSomeUnrelatedUtil', |
| 'baseProtocols 0x0', 'name 0x1064b8438 ChromeTestCase', 'unrelated line', |
| 'name 0x1064b8438 invalidTestCase', 'name 0x1075e6887 testG', |
| 'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase', |
| 'name 0x1075e6887 testG', 'name 0x1075e6887 testH', |
| 'name 0x1075e6887 DISABLED_testI', 'name 0x1075e6887 FLAKY_testJ', |
| 'name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0' |
| ]).encode('utf-8') |
| |
| # Release app otool output format in Xcode 11.4 toolchain. |
| RELEASE_APP_OTOOL_OUTPUT_114 = '\n'.join([ |
| 'Meta Class', ' name 0x1064b8438 CacheTestCase', |
| 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| ' name 0x1075e6887 testA', ' types 0x1064cc3e1', |
| ' name 0x1075e6887 testB', ' name 0x1075e6887 testc', |
| 'baseProtocols 0x0', ' name 0x1064b8438 CacheTestCase', |
| 'unrelated line', 'Meta Class', ' name 0x1064b8438 TabUITestCase', |
| 'no test methods in this case', ' name 0x1064b8438 TabUITestCase', |
| 'unrelated line', 'baseMethods 0x1068586d8 (struct method_list_t *)', |
| ' name 0x1064b8438 KeyboardTest', ' name 0x1075e6887 testD', |
| ' types 0x1064cc3e1 v16@0:8', ' name 0x1075e6887 testE', |
| ' name 0x1075e6887 testF', 'baseProtocols 0x0', |
| ' name 0x1064b8438 KeyboardTest', |
| ' name 0x1075e6887 testUnrelatedG', 'unrelated line', |
| ' name 0x1064b8438 ChromeTestCase', ' name 0x1064b8438 setUp', |
| ' name 0x1064b8438 testPort', |
| ' name 0x5345ac561 testSomeUnrelatedUtil', 'baseProtocols 0x0', |
| ' name 0x1064b8438 ChromeTestCase', 'unrelated line', |
| ' name 0x1064b8438 invalidTestCase', ' name 0x1075e6887 testG', |
| 'baseProtocols 0x0', ' name 0x1064b8438 ToolBarTestCase', |
| ' name 0x1075e6887 testG', ' name 0x1075e6887 testH', |
| ' name 0x1075e6887 DISABLED_testI', |
| ' name 0x1075e6887 FLAKY_testJ', |
| ' name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0' |
| ]).encode('utf-8') |
| |
| |
| class TestShardUtil(unittest.TestCase): |
| """Test cases for shard_util.py""" |
| |
| @patch('shard_util.os.path.abspath') |
| def test_determine_path_non_eg2(self, mock_abspath): |
| mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py' |
| app = 'some_ios_test.app' |
| |
| actual_path = shard_util.determine_app_path(app) |
| expected_path = os.path.join('/b/s/w/ir', 'out/Debug', app, 'some_ios_test') |
| self.assertEqual(actual_path, expected_path) |
| |
| @patch('shard_util.os.path.abspath') |
| def test_determine_path_eg2(self, mock_abspath): |
| mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py' |
| app = 'some_ios_test-Runner.app' |
| host = 'some_host.app' |
| |
| actual_path = shard_util.determine_app_path(app, host) |
| expected_path = os.path.join('/b/s/w/ir', 'out/Debug', app, 'PlugIns', |
| 'some_ios_test.xctest', 'some_ios_test') |
| self.assertEqual(actual_path, expected_path) |
| |
| @patch('shard_util.os.path.abspath') |
| def test_determine_path_eg2_release(self, mock_abspath): |
| mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py' |
| app = 'some_ios_test-Runner.app' |
| host = 'some_host.app' |
| |
| actual_path = shard_util.determine_app_path(app, host, True) |
| expected_path = os.path.join('/b/s/w/ir', 'out/Release', app, 'PlugIns', |
| 'some_ios_test.xctest', 'some_ios_test') |
| self.assertEqual(actual_path, expected_path) |
| |
| def test_fetch_test_names_debug(self): |
| """Ensures that the debug output is formatted correctly""" |
| resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT) |
| self.assertEqual(len(resp), 11) |
| expected_test_names = [ |
| ('CacheTestCase', 'testA'), |
| ('CacheTestCase', 'testB'), |
| ('CacheTestCase', 'testc'), |
| ('TabUITestCase', 'testD'), |
| ('TabUITestCase', 'testE'), |
| ('KeyboardTestCase', 'testF'), |
| ('PasswordManagerTestCase', 'testG'), |
| ('ToolBarTestCase', 'testH'), |
| ('ToolBarTestCase', 'DISABLED_testI'), |
| ('ToolBarTestCase', 'FLAKY_testJ'), |
| ('PasswordManagerPasswordCheckupDisabledTestCase', 'testK'), |
| ] |
| for test_name in expected_test_names: |
| self.assertTrue(test_name in resp) |
| |
| test_cases = [test_case for (test_case, _) in resp] |
| # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordManagerTestCase': 1, |
| # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3, |
| # 'PasswordManagerPasswordCheckupDisabledTestCase': 1}) |
| counts = collections.Counter(test_cases).most_common() |
| name, _ = counts[0] |
| # CacheTestCase and ToolBarTestCase each have 3 entries. |
| # In case of ties, most_common() returns the first encountered at index 0. |
| self.assertEqual(name, 'CacheTestCase') |
| |
| def test_fetch_test_counts_release(self): |
| """Ensures that the release output is formatted correctly""" |
| resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT) |
| self.assertEqual(len(resp), 10) |
| |
| expected_test_names = [ |
| ('CacheTestCase', 'testA'), |
| ('CacheTestCase', 'testB'), |
| ('CacheTestCase', 'testc'), |
| ('KeyboardTest', 'testD'), |
| ('KeyboardTest', 'testE'), |
| ('KeyboardTest', 'testF'), |
| ('ToolBarTestCase', 'testG'), |
| ('ToolBarTestCase', 'testH'), |
| ('ToolBarTestCase', 'DISABLED_testI'), |
| ('ToolBarTestCase', 'FLAKY_testJ'), |
| ] |
| for test_name in expected_test_names: |
| self.assertTrue(test_name in resp) |
| |
| test_cases = [test_case for (test_case, _) in resp] |
| # ({'KeyboardTest': 3, 'CacheTestCase': 3, |
| # 'ToolBarTestCase': 4}) |
| counts = collections.Counter(test_cases).most_common() |
| name, _ = counts[0] |
| self.assertEqual(name, 'ToolBarTestCase') |
| |
| def test_fetch_test_error_release(self): |
| """Ensures that unexpected release output raises error.""" |
| with self.assertRaises(shard_util.ShardingError) as context: |
| shard_util.fetch_test_names_for_release( |
| RELEASE_APP_OTOOL_OUTPUT_CLASS_NOT_IN_PAIRS) |
| expected_message = ( |
| 'Incorrect otool output in which a test class name doesn\'t appear in ' |
| 'group of 2. Test class: KeyboardTest') |
| self.assertTrue(expected_message in str(context.exception)) |
| |
| def test_fetch_test_names_debug_114(self): |
| """Test the debug output from otool in Xcode 11.4""" |
| resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT_114) |
| self.assertEqual(len(resp), 10) |
| expected_test_names = [ |
| ('CacheTestCase', 'testA'), |
| ('CacheTestCase', 'testB'), |
| ('CacheTestCase', 'testc'), |
| ('TabUITestCase', 'testD'), |
| ('TabUITestCase', 'testE'), |
| ('KeyboardTestCase', 'testF'), |
| ('PasswordsTestCase', 'testG'), |
| ('ToolBarTestCase', 'testH'), |
| ('ToolBarTestCase', 'DISABLED_testI'), |
| ('ToolBarTestCase', 'FLAKY_testJ'), |
| ] |
| for test_name in expected_test_names: |
| self.assertTrue(test_name in resp) |
| |
| test_cases = [test_case for (test_case, _) in resp] |
| |
| # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1, |
| # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3}) |
| counts = collections.Counter(test_cases).most_common() |
| name, _ = counts[0] |
| # CacheTestCase and ToolBarTestCase each have 3 entries. |
| # In case of ties, most_common() returns the first encountered at index 0. |
| self.assertEqual(name, 'CacheTestCase') |
| |
| def test_fetch_test_counts_release_114(self): |
| """Test the release output from otool in Xcode 11.4""" |
| resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT_114) |
| self.assertEqual(len(resp), 10) |
| |
| expected_test_names = [ |
| ('CacheTestCase', 'testA'), |
| ('CacheTestCase', 'testB'), |
| ('CacheTestCase', 'testc'), |
| ('KeyboardTest', 'testD'), |
| ('KeyboardTest', 'testE'), |
| ('KeyboardTest', 'testF'), |
| ('ToolBarTestCase', 'testG'), |
| ('ToolBarTestCase', 'testH'), |
| ('ToolBarTestCase', 'DISABLED_testI'), |
| ('ToolBarTestCase', 'FLAKY_testJ'), |
| ] |
| for test_name in expected_test_names: |
| self.assertTrue(test_name in resp) |
| |
| test_cases = [test_case for (test_case, _) in resp] |
| # ({'KeyboardTest': 3, 'CacheTestCase': 3, |
| # 'ToolBarTestCase': 4}) |
| counts = collections.Counter(test_cases).most_common() |
| name, _ = counts[0] |
| self.assertEqual(name, 'ToolBarTestCase') |
| |
| def test_balance_into_sublists_debug(self): |
| """Ensure the balancing algorithm works""" |
| resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT) |
| test_cases = [test_case for (test_case, _) in resp] |
| test_counts = collections.Counter(test_cases) |
| |
| sublists_1 = shard_util.balance_into_sublists(test_counts, 1) |
| self.assertEqual(len(sublists_1), 1) |
| self.assertEqual(len(sublists_1[0]), 6) |
| |
| sublists_3 = shard_util.balance_into_sublists(test_counts, 3) |
| self.assertEqual(len(sublists_3), 3) |
| # CacheTestCase has 3, |
| # TabUITestCase has 2, ToolBarTestCase has 4 |
| # PasswordManagerTestCase has 1, KeyboardTestCase has 1 |
| # PasswordManagerPasswordCheckupDisabledTestCase has 2 (due to inheritance) |
| # They will be balanced into: |
| # [[ToolBarTestCase, KeyboardTestCase], |
| # [CacheTestCase, PasswordManagerTestCase], |
| # [PasswordManagerPasswordCheckupDisabledTestCase, TabUITestCase]] |
| self.assertEqual( |
| sorted([len(sublists_3[0]), |
| len(sublists_3[1]), |
| len(sublists_3[2])]), [2, 2, 2]) |
| |
| def test_balance_into_sublists_release(self): |
| """Ensure the balancing algorithm works""" |
| resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT) |
| test_cases = [test_case for (test_case, _) in resp] |
| test_counts = collections.Counter(test_cases) |
| |
| sublists_3 = shard_util.balance_into_sublists(test_counts, 3) |
| self.assertEqual(len(sublists_3), 3) |
| # KeyboardTest has 3 |
| # CacheTestCase has 3 |
| # ToolbarTest Case has 4 |
| # They will be balanced as one in each shard. |
| self.assertEqual(len(sublists_3[0]), 1) |
| self.assertEqual(len(sublists_3[1]), 1) |
| self.assertEqual(len(sublists_3[2]), 1) |
| |
| def test_balance_into_sublists_release(self): |
| """Ensure the balancing algorithm works""" |
| resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT) |
| test_cases = [test_case for (test_case, _) in resp] |
| test_counts = collections.Counter(test_cases) |
| |
| sublists_3 = shard_util.balance_into_sublists(test_counts, 3) |
| self.assertEqual(len(sublists_3), 3) |
| # KeyboardTest has 3 |
| # CacheTestCase has 3 |
| # ToolbarTest Case has 4 |
| # They will be balanced as one in each shard. |
| self.assertEqual(len(sublists_3[0]), 1) |
| self.assertEqual(len(sublists_3[1]), 1) |
| self.assertEqual(len(sublists_3[2]), 1) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
