| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Top-level presubmit script for the tools/build repo. |
| |
| See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for |
| details on the presubmit API built into git cl. |
| """ |
| |
| import re |
| |
| PRESUBMIT_VERSION = '2.0.0' |
| |
| USE_PYTHON3 = True |
| |
| _IGNORE_FREEZE_FOOTER = 'Ignore-Freeze' |
| |
| # The time module's handling of timezones is abysmal, so the boundaries are |
| # precomputed in UNIX time |
| _FREEZE_START = 1702627200 # 2023/12/15 00:00 -0800 |
| _FREEZE_END = 1704182400 # 2024/01/02 00:00 -0800 |
| |
| |
| def CheckFreeze(input_api, output_api): |
| if _FREEZE_START <= input_api.time.time() < _FREEZE_END: |
| footers = input_api.change.GitFootersFromDescription() |
| if _IGNORE_FREEZE_FOOTER not in footers: |
| |
| def convert(t): |
| ts = input_api.time.localtime(t) |
| return input_api.time.strftime('%Y/%m/%d %H:%M %z', ts) |
| |
| return [ |
| output_api.PresubmitError( |
| 'There is a prod freeze in effect from {} until {}'.format( |
| convert(_FREEZE_START), convert(_FREEZE_END) |
| ) |
| ) |
| ] |
| |
| return [] |
| |
| |
| def GetFilesToSkip(input_api): |
| return list(input_api.DEFAULT_FILES_TO_SKIP) + [ |
| # recipes.py and the .recipe_deps directory are created by the recipe |
| # engine, so should not be checked as part of this repo |
| r'^recipes/recipes\.py$', |
| r'^recipes/\.recipe_deps/.*', |
| # *_pb2.py files are generated from proto files and should not be linted |
| r'.*_pb2.py$', |
| ] |
| |
| |
| def join(input_api, *args): |
| return input_api.os_path.normpath( |
| input_api.os_path.join(input_api.PresubmitLocalPath(), *args) |
| ) |
| |
| |
| # The following files are executed using python3.8, so they should be analyzed |
| # using pylint 2.7 instead of the current pylint. |
| _PY3_8_FILES = ( |
| 'hook-scripts/remove_orphaned_pycs.py', |
| 'recipes/bot_utils.py', |
| 'recipes/build_directory.py', |
| 'recipes/cloudtail_utils.py', |
| 'recipes/crash_utils.py', |
| 'recipes/daemonizer.py', |
| 'recipes/extract_build.py', |
| 'recipes/goma_bq_utils.py', |
| 'recipes/goma_utils.py', |
| 'recipes/recipe_modules/adb/resources/list_devices.py', |
| 'recipes/recipe_modules/archive/resources/batch.py', |
| 'recipes/recipe_modules/archive/resources/filter_build_files.py', |
| 'recipes/recipe_modules/archive/resources/zip_archive.py', |
| 'recipes/recipe_modules/binary_size/resources/trybot_failed_expectations_checker.py', |
| 'recipes/recipe_modules/chromium_android/resources/archive_build_unittest.py', |
| 'recipes/recipe_modules/chromium_android/resources/archive_build.py', |
| 'recipes/recipe_modules/chromium_android/resources/authorize_adb_devices.py', |
| 'recipes/recipe_modules/chromium_android/resources/clean_local_files.py', |
| 'recipes/recipe_modules/chromium_swarming/resources/collect_task.py', |
| 'recipes/recipe_modules/chromium_swarming/resources/noop_merge.py', |
| 'recipes/recipe_modules/chromium_swarming/resources/wait_for_finished_task_set.py', |
| 'recipes/recipe_modules/chromium_swarming/unittests/collect_task_test.py', |
| 'recipes/recipe_modules/chromium_swarming/unittests/common_merge_script_tests.py', |
| 'recipes/recipe_modules/chromium_swarming/unittests/noop_merge_test.py', |
| 'recipes/recipe_modules/chromium_swarming/unittests/wait_for_finished_task_set_unittest.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/buildozer_wrapper.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/generate_groupings.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/migrate.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/tests/buildozer_wrapper_unit_test.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/tests/generate_groupings_integration_test.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/tests/generate_groupings_unit_test.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/tests/migrate_integration_test.py', |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/scripts/tests/migrate_unit_test.py', |
| 'recipes/recipe_modules/chromium_tests/resources/archive_layout_test_results.py', |
| 'recipes/recipe_modules/chromium_tests/resources/find_command_lines.py', |
| 'recipes/recipe_modules/chromium/resources/clang_revision.py', |
| 'recipes/recipe_modules/chromium/resources/export_tarball.py', |
| 'recipes/recipe_modules/chromium/resources/generate_hashes.py', |
| 'recipes/recipe_modules/chromium/resources/ninja_wrapper_test.py', |
| 'recipes/recipe_modules/chromium/resources/ninja_wrapper.py', |
| 'recipes/recipe_modules/code_coverage/resources/aggregation_util.py', |
| 'recipes/recipe_modules/code_coverage/resources/clean_up_java_coverage_files.py', |
| 'recipes/recipe_modules/code_coverage/resources/combine_jacoco_reports.py', |
| 'recipes/recipe_modules/code_coverage/resources/generate_coverage_metadata_for_java.py', |
| 'recipes/recipe_modules/code_coverage/resources/generate_coverage_metadata_for_javascript.py', |
| 'recipes/recipe_modules/code_coverage/resources/generate_coverage_metadata.py', |
| 'recipes/recipe_modules/code_coverage/resources/gerrit_util.py', |
| 'recipes/recipe_modules/code_coverage/resources/get_binaries_with_valid_coverage_data.py', |
| 'recipes/recipe_modules/code_coverage/resources/get_jacoco_and_jar_files_for_java.py', |
| 'recipes/recipe_modules/code_coverage/resources/get_native_libraries_for_android_target.py', |
| 'recipes/recipe_modules/code_coverage/resources/get_unstripped_paths.py', |
| 'recipes/recipe_modules/code_coverage/resources/make_report.py', |
| 'recipes/recipe_modules/code_coverage/resources/merge_metadata_files.py', |
| 'recipes/recipe_modules/code_coverage/resources/rebase_line_number_from_bot_to_gerrit.py', |
| 'recipes/recipe_modules/code_coverage/resources/repository_util.py', |
| 'recipes/recipe_modules/code_coverage/resources/write_paths_to_instrument.py', |
| 'recipes/recipe_modules/code_coverage/unittests/aggregation_util_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/blame_util_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/combine_jacoco_reports_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/constants_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/diff_util_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/generate_coverage_metadata_for_java_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/generate_coverage_metadata_for_javascript_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/generate_coverage_metadata_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/gerrit_util_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/get_native_libraries_for_android_target_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/get_unstripped_paths_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/make_report_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/merge_metadata_files_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/rebase_line_number_from_bot_to_gerrit_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/repository_util_test.py', |
| 'recipes/recipe_modules/code_coverage/unittests/write_paths_test.py', |
| 'recipes/recipe_modules/cronet/resources/generate_changelist.py', |
| 'recipes/recipe_modules/disk/resources/statvfs.py', |
| 'recipes/recipe_modules/flaky_reproducer/libs/strategies/parallel_strategy.py', |
| 'recipes/recipe_modules/flaky_reproducer/libs/test_binary/base_test_binary.py', |
| 'recipes/recipe_modules/flaky_reproducer/libs/test_binary/gtest_test_binary.py', |
| 'recipes/recipe_modules/flaky_reproducer/libs/test_binary/utils.py', |
| 'recipes/recipe_modules/flaky_reproducer/strategy_runner.py', |
| 'recipes/recipe_modules/flaky_reproducer/unittests/test_strategy_runner.py', |
| 'recipes/recipe_modules/flaky_reproducer/unittests/test_test_binary.py', |
| 'recipes/recipe_modules/goma/resources/cloudtail.py', |
| 'recipes/recipe_modules/perf_dashboard/resources/post_json.py', |
| 'recipes/recipe_modules/profiles/resources/load_merge_errors.py', |
| 'recipes/recipe_modules/reclient/resources/cloudtail_wrapper.py', |
| 'recipes/recipe_modules/reclient/resources/generate_rpl_gzip.py', |
| 'recipes/recipe_modules/reclient/resources/perform_health_check.py', |
| 'recipes/recipe_modules/symupload/resources/symupload_script.py', |
| 'recipes/recipe_modules/symupload/unittests/symupload_script_test.py', |
| 'recipes/recipe_modules/tar/resources/tar_test.py', |
| 'recipes/recipe_modules/tar/resources/tar.py', |
| 'recipes/recipe_modules/tar/resources/untar.py', |
| 'recipes/recipe_modules/test_utils/resources/archive_layout_test_results_summary.py', |
| 'recipes/recipe_modules/test_utils/resources/query_cq_flakes.py', |
| 'recipes/recipe_modules/test_utils/unittests/query_cq_flakes_test.py', |
| 'recipes/recipe_modules/tricium_clang_tidy/resources/tricium_clang_tidy_script.py', |
| 'recipes/recipe_modules/tricium_clang_tidy/resources/tricium_clang_tidy_test.py', |
| 'recipes/recipe_modules/v8_tests/resources/collect_v8_task_test.py', |
| 'recipes/recipe_modules/v8_tests/resources/collect_v8_task.py', |
| 'recipes/recipe_modules/v8/resources/build-dep-stats.py', |
| 'recipes/recipe_modules/webrtc/resources/binary_sizes.py', |
| 'recipes/recipes/android/sdk_packager.resources/parse_sdkmanager_list_test.py', |
| 'recipes/recipes/android/sdk_packager.resources/parse_sdkmanager_list.py', |
| 'recipes/recipes/chromium_rts/create_model.resources/integration_tests.py', |
| 'recipes/recipes/flakiness/generate_builder_test_data.resources/query_test.py', |
| 'recipes/recipes/flakiness/generate_builder_test_data.resources/query.py', |
| 'recipes/recipes/swarming/deterministic_build.resources/move.py', |
| 'recipes/runisolatedtest.py', |
| 'recipes/runtest.py', |
| 'recipes/tee.py', |
| 'recipes/unittests/bot_utils_test.py', |
| 'recipes/unittests/extract_build_test.py', |
| 'recipes/unittests/goma_utils_test.py', |
| 'recipes/unittests/recipe_test.py', |
| 'recipes/unittests/runisolatedtest_test.py', |
| 'recipes/unittests/zip_build_test.py', |
| 'recipes/upload_goma_logs.py', |
| 'recipes/xvfb.py', |
| 'recipes/zip_build.py', |
| 'scripts/common/chromium_utils.py', |
| 'scripts/common/gtest_utils.py', |
| 'scripts/common/unittests/chromium_utils_test.py', |
| 'scripts/common/unittests/gtest_utils_test.py', |
| ) |
| |
| |
| def CheckPylintOnCommit(input_api, output_api): |
| disabled_warnings = [ |
| 'C0321', # More than one statement on a single line |
| 'W0613', # Unused argument |
| 'C3001', # unnecessary-lambda-assignment |
| |
| # f-strings should generally be preferred since they are faster than other |
| # forms of string construction and often (but not always) easier to read. |
| # The performance difference is unlikely to matter unless many strings are |
| # being constructed in a tight loop, so addressing the many findings of |
| # this category is unlikely to provide much performance benefit and so |
| # shouldn't be a priority. It would be nice if we could enable this only |
| # for new lines... |
| 'C0209', # consider-using-f-string |
| |
| # dict literals are more performant than using a dict call, but a little |
| # more noisy visually and the performance difference is unlikely to matter |
| # unless the dict is being constructed in a tight loop |
| 'R1735', # use-dict-literal |
| |
| # Below warnings are disabled for pylint version update, but better to |
| # remove if possible. |
| |
| # There are a number of long lines in the repo already, some of them in |
| # code that the formatter does not play nicely with. There is also a |
| # warning presubmit check that flags lines that are too long only in |
| # modified files, so it's not vital to have pylint cause errors for this. |
| 'C0301', # line-too-long |
| |
| # This will require adding many additional return statements and in turn |
| # will require test cases to be added/modified to get coverage on the new |
| # return statements |
| 'R1710', # inconsistent-return-statements |
| ] |
| extra_paths_list = [ |
| join(input_api, 'recipes'), |
| join(input_api, 'scripts'), |
| # Initially, a separate run was done for unit tests but now that |
| # pylint is fetched in memory with setuptools, it seems it caches |
| # sys.path so modifications to sys.path aren't kept. |
| join(input_api, 'recipes', 'unittests'), |
| ] |
| py3_8_files = [f'{re.escape(x)}$' for x in _PY3_8_FILES] |
| lints = input_api.canned_checks.RunPylint( |
| input_api, |
| output_api, |
| files_to_skip=GetFilesToSkip(input_api) + py3_8_files, |
| disabled_warnings=disabled_warnings, |
| extra_paths_list=extra_paths_list, |
| version='2.17', |
| ) |
| lints.extend( |
| input_api.canned_checks.RunPylint( |
| input_api, |
| output_api, |
| files_to_check=py3_8_files, |
| disabled_warnings=disabled_warnings, |
| extra_paths_list=extra_paths_list, |
| version='2.7', |
| ) |
| ) |
| return lints |
| |
| |
| def _GetTests(input_api, output_api, test_files): |
| return input_api.canned_checks.GetUnitTests( |
| input_api, |
| output_api, |
| sorted(test_files), |
| run_on_python3=True, |
| run_on_python2=False, |
| skip_shebang_check=True, |
| ) |
| |
| |
| # The following tests invoke recipes.py which isn't safe in parallel, so they'll |
| # be run in a separate check function that runs them sequentially |
| RECIPES_PY_TESTS = [ |
| 'recipes/unittests/recipe_test.py', |
| ( |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/' |
| 'scripts/tests/generate_groupings_integration_test.py' |
| ), |
| ( |
| 'recipes/recipe_modules/chromium_tests_builder_config/migration/' |
| 'scripts/tests/migrate_integration_test.py' |
| ), |
| ] |
| |
| |
| def _RecipesPyTestFiles(input_api): |
| return [join(input_api, f) for f in RECIPES_PY_TESTS] |
| |
| |
| def CheckRecipesPyTestsOnCommit(input_api, output_api): |
| tests = _GetTests(input_api, output_api, _RecipesPyTestFiles(input_api)) |
| return input_api.RunTests(tests, parallel=False) |
| |
| |
| def CheckTestsOnCommit(input_api, output_api): |
| excluded_test_files = set(_RecipesPyTestFiles(input_api)) |
| |
| test_files = [] |
| for dir_glob in ( |
| ('recipes', 'unittests'), |
| ('recipes', 'recipe_modules', '*', 'unittests'), |
| ('recipes', 'recipe_modules', '*', 'resources'), |
| ('recipes', 'recipe_modules', 'chromium_tests_builder_config', |
| 'migration', 'scripts', 'tests'), |
| ('recipes', 'recipes', '*.resources'), |
| ('recipes', 'recipes', '*', '*.resources'), |
| ('scripts', 'common', 'unittests'), |
| ): |
| glob = dir_glob + ('*_test.py',) |
| test_files.extend( |
| x for x in input_api.glob(join(input_api, *glob)) |
| if x not in excluded_test_files |
| ) |
| |
| tests = _GetTests(input_api, output_api, test_files) |
| return input_api.RunTests(tests) |
| |
| |
| def CheckPanProjectChecksOnCommit(input_api, output_api): |
| return input_api.canned_checks.PanProjectChecks( |
| input_api, |
| output_api, |
| excluded_paths=GetFilesToSkip(input_api), |
| owners_check=False |
| ) |
| |
| |
| def CheckConfigFilesParse(input_api, output_api): |
| file_filter = lambda x: x.LocalPath() == 'infra/config/recipes.cfg' |
| return input_api.canned_checks.CheckJsonParses( |
| input_api, output_api, file_filter=file_filter |
| ) |
| |
| |
| def CheckPatchFormatted(input_api, output_api): |
| # TODO(https://crbug.com/979330) If clang-format is fixed for non-chromium |
| # repos, remove check_clang_format=False so that proto files can be formatted |
| return input_api.canned_checks.CheckPatchFormatted( |
| input_api, |
| output_api, |
| check_clang_format=False, |
| result_factory=output_api.PresubmitError, |
| ) |
