| #!/usr/bin/env python3 |
| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Generate BUILD.gn to define all data required to run telemetry test. |
| |
| If a file/folder of large size is added to catapult but is not needed to run |
| telemetry test, then it should be added to the EXCLUDED_PATHS below and rerun |
| this script. |
| |
| This script can also run with --check to see if it needs to rerun to update |
| BUILD.gn. |
| |
| This script can also run with --chromium and rewrite the chromium file |
| //tools/perf/chrome_telemetry_build/BUILD.gn |
| This is for the purpose of running try jobs in chromium. |
| |
| """ |
| |
| import difflib |
| import logging |
| import os |
| import optparse |
| import sys |
| import subprocess |
| import textwrap |
| |
| LICENSE = textwrap.dedent( |
| """\ |
| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """) |
| |
| DO_NOT_EDIT_WARNING = textwrap.dedent( |
| """\ |
| # This file is auto-generated from |
| # //third_party/catapult/generated_telemetry_build.py |
| # DO NOT EDIT! |
| |
| """) |
| |
| TELEMETRY_SUPPORT_GROUP_NAME = 'telemetry_chrome_test_support' |
| |
| EXCLUDED_PATHS = { |
| 'BUILD.gn', |
| 'TEMP.gn', |
| 'common/node_runner/', |
| 'docs/', |
| 'experimental/', |
| 'generate_telemetry_build.py', |
| 'third_party/Paste/', |
| 'third_party/google-endpoints/', |
| 'third_party/polymer2/', |
| 'tracing/test_data/', |
| } |
| |
| |
| SEPARATE_TARGETS = { |
| 'devil': 'devil', |
| 'telemetry': 'telemetry', |
| 'third_party/gsutil': 'third_party/gsutil', |
| 'third_party/typ': 'third_party/typ', |
| 'third_party/vinn': 'third_party/vinn', |
| } |
| |
| |
| def GetUntrackedPaths(): |
| """Return directories/files in catapult/ that are not tracked by git.""" |
| output = subprocess.check_output([ |
| 'git', 'ls-files', '--others', '--exclude-standard', '--directory', |
| '--full-name'], text=True) |
| paths = output.split('\n') |
| return [os.path.abspath(p) for p in paths if p] |
| |
| |
| def WriteLists(data, data_deps, build_file, path_prefix): |
| if data: |
| build_file.write(' data += [\n') |
| for path in data: |
| if path_prefix: |
| path = path_prefix + path |
| build_file.write(' "%s",\n' % path) |
| build_file.write(' ]\n\n') |
| |
| if data_deps: |
| build_file.write(' data_deps += [\n') |
| for data_dep in data_deps: |
| build_file.write(' "%s",\n' % data_dep) |
| build_file.write(' ]\n\n') |
| |
| |
| def ProcessDir(root_path, path, build_file, path_prefix): |
| # Write all dirs and files directly under |path| unless they are excluded |
| # or need to be processed further because some of their children are excluded. |
| # Return a list of dirs that needs to processed further. |
| logging.debug('GenerateList for ' + path) |
| entries = os.listdir(path) |
| entries.sort() |
| files = [] |
| dirs = [] |
| data_deps = [] |
| dirs_to_expand = [] |
| untracked_paths = GetUntrackedPaths() |
| for entry in entries: |
| full_path = os.path.join(path, entry) |
| rel_path = os.path.relpath(full_path, root_path).replace('\\', '/') |
| if (any(full_path.startswith(p) for p in untracked_paths) or |
| entry.startswith('.') or entry.endswith('~') or |
| entry.endswith('.pyc') or entry.endswith('#')): |
| logging.debug('ignored %s', rel_path) |
| continue |
| if rel_path in SEPARATE_TARGETS: |
| data_deps.append(SEPARATE_TARGETS[rel_path]) |
| elif os.path.isfile(full_path): |
| if rel_path in EXCLUDED_PATHS: |
| logging.debug('excluded %s', rel_path) |
| else: |
| files.append(rel_path) |
| elif os.path.isdir(full_path): |
| rel_path = rel_path + '/' |
| if rel_path in EXCLUDED_PATHS: |
| logging.debug('excluded %s', rel_path) |
| elif any(e.startswith(rel_path) for e in EXCLUDED_PATHS): |
| dirs_to_expand.append(full_path) |
| else: |
| dirs.append(rel_path) |
| else: |
| assert False |
| WriteLists(files + dirs, |
| data_deps, |
| build_file, path_prefix) |
| return dirs_to_expand |
| |
| def WriteBuildFileHeader(build_file): |
| build_file.write(LICENSE) |
| build_file.write(DO_NOT_EDIT_WARNING) |
| build_file.write('import("//build/config/compiler/compiler.gni")\n\n') |
| |
| def WriteBuildFileBody(build_file, root_path, path_prefix): |
| build_file.write(textwrap.dedent( |
| """\ |
| group("%s") { |
| testonly = true |
| data = [] |
| data_deps = [] |
| |
| """ % TELEMETRY_SUPPORT_GROUP_NAME)) |
| |
| candidates = [root_path] |
| while len(candidates) > 0: |
| candidate = candidates.pop(0) |
| more = ProcessDir(root_path, candidate, build_file, path_prefix) |
| candidates.extend(more) |
| |
| build_file.write("}") |
| |
| def GenerateBuildFile(root_path, output_path, chromium): |
| CHROMIUM_GROUP = 'group("telemetry_chrome_test_without_chrome")' |
| CATAPULT_PREFIX = '//third_party/catapult' |
| CATAPULT_GROUP_NAME = CATAPULT_PREFIX + ':' + TELEMETRY_SUPPORT_GROUP_NAME |
| TELEMETRY_SUPPORT_GROUP = 'group("%s")' % TELEMETRY_SUPPORT_GROUP_NAME |
| if chromium: |
| build_file = open(output_path, 'r+') |
| contents = build_file.readlines() |
| build_file.seek(0) |
| remove_telemetry_support_group = False |
| for line in contents: |
| if TELEMETRY_SUPPORT_GROUP in line: |
| # --chromium has already run once, so remove the previously inserted |
| # TELEMETRY_SUPPORT_GROUP so we could add an updated one. |
| remove_telemetry_support_group = True |
| continue |
| if remove_telemetry_support_group: |
| if line == '}\n': |
| remove_telemetry_support_group = False |
| continue |
| if CHROMIUM_GROUP in line: |
| WriteBuildFileBody(build_file, root_path, CATAPULT_PREFIX + '/') |
| build_file.write('\n') |
| elif CATAPULT_GROUP_NAME in line: |
| line = line.replace(CATAPULT_GROUP_NAME, |
| ':' + TELEMETRY_SUPPORT_GROUP_NAME) |
| build_file.write(line) |
| build_file.close() |
| else: |
| build_file = open(output_path, 'w') |
| WriteBuildFileHeader(build_file) |
| WriteBuildFileBody(build_file, root_path, None) |
| build_file.close() |
| |
| def CheckForChanges(): |
| # Return 0 if no changes are detected; return 1 otherwise. |
| root_path = os.path.dirname(os.path.realpath(__file__)) |
| temp_path = os.path.join(root_path, "TEMP.gn") |
| GenerateBuildFile(root_path, temp_path, chromium=False) |
| |
| ref_path = os.path.join(root_path, "BUILD.gn") |
| if not os.path.exists(ref_path): |
| logging.error("Can't localte BUILD.gn!") |
| return 1 |
| |
| temp_file = open(temp_path, 'r') |
| temp_content = temp_file.readlines() |
| temp_file.close() |
| os.remove(temp_path) |
| ref_file = open(ref_path, 'r') |
| ref_content = ref_file.readlines() |
| ref_file.close() |
| |
| diff = difflib.unified_diff(temp_content, ref_content, fromfile=temp_path, |
| tofile=ref_path, lineterm='') |
| diff_data = [] |
| for line in diff: |
| diff_data.append(line) |
| if len(diff_data) > 0: |
| logging.error('Diff found. Please rerun generate_telemetry_build.py.') |
| logging.debug('\n' + ''.join(diff_data)) |
| return 1 |
| logging.debug('No diff found. Everything is good.') |
| return 0 |
| |
| |
| def main(argv): |
| parser = optparse.OptionParser() |
| parser.add_option("-v", "--verbose", action="store_true", default=False, |
| help="print out debug information") |
| parser.add_option("-c", "--check", action="store_true", default=False, |
| help="generate a temporary build file and compare if it " |
| "is the same as the current BUILD.gn") |
| parser.add_option("--chromium", action="store_true", default=False, |
| help="generate the build file into chromium workspace. " |
| "This is for the purpose of running try jobs in Chrome.") |
| (options, _) = parser.parse_args(args=argv) |
| if options.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| |
| if options.check: |
| return CheckForChanges() |
| if options.chromium: |
| root_path = os.path.dirname(os.path.realpath(__file__)) |
| output_path = os.path.join( |
| root_path, "../../tools/perf/chrome_telemetry_build/BUILD.gn") |
| GenerateBuildFile(root_path, output_path, chromium=True) |
| else: |
| root_path = os.path.dirname(os.path.realpath(__file__)) |
| output_path = os.path.join(root_path, "BUILD.gn") |
| GenerateBuildFile(root_path, output_path, chromium=False) |
| return 0 |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |