| #!/usr/bin/env python |
| # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| """Logic to generate lists of DEPS used by various parts of |
| the android_webview continuous integration (buildbot) infrastructure. |
| |
| Note: The root Chromium project (which is not explicitly listed here) |
| has a couple of third_party libraries checked in directly into it. This means |
| that the list of third parties present in this file is not a comprehensive |
| list of third party android_webview dependencies. |
| """ |
| |
| import argparse |
| import json |
| import logging |
| import os |
| import sys |
| |
| # Add android_webview/tools to path to get at known_issues. |
| sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools')) |
| import known_issues |
| |
| |
| class DepsWhitelist(object): |
| def __init__(self): |
| # If a new DEPS entry is needed for the AOSP bot to compile please add it |
| # here first. |
| # This is a staging area for deps that are accepted by the android_webview |
| # team and are in the process of having the required branches being created |
| # in the Android tree. |
| self._compile_but_not_snapshot_dependencies = [ |
| ] |
| |
| # Dependencies that need to be merged into the Android tree. |
| self._snapshot_into_android_dependencies = [ |
| 'sdch/open-vcdiff', |
| 'testing/gtest', |
| 'third_party/WebKit', |
| 'third_party/angle', |
| 'third_party/boringssl/src', |
| 'third_party/brotli/src', |
| ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/' |
| 'braille'), |
| 'third_party/freetype', |
| 'third_party/icu', |
| 'third_party/leveldatabase/src', |
| 'third_party/libaddressinput/src', |
| 'third_party/libjingle/source/talk', |
| 'third_party/libjpeg_turbo', |
| 'third_party/libphonenumber/src/phonenumbers', |
| 'third_party/libphonenumber/src/resources', |
| 'third_party/libsrtp', |
| 'third_party/libvpx', |
| 'third_party/libyuv', |
| 'third_party/mesa/src', |
| 'third_party/openmax_dl', |
| 'third_party/opus/src', |
| 'third_party/ots', |
| 'third_party/sfntly/cpp/src', |
| 'third_party/skia', |
| 'third_party/smhasher/src', |
| 'third_party/usrsctp/usrsctplib', |
| 'third_party/webrtc', |
| 'third_party/yasm/source/patched-yasm', |
| 'tools/grit', |
| 'tools/gyp', |
| 'v8', |
| ] |
| |
| # We can save some time by not rsyncing code we don't use. |
| self._prune_from_rsync_build = [ |
| 'third_party/WebKit/LayoutTests', |
| ] |
| |
| # Dependencies required to build android_webview. |
| self._compile_dependencies = (self._snapshot_into_android_dependencies + |
| self._compile_but_not_snapshot_dependencies) |
| |
| # Dependencies required to run android_webview tests but not required to |
| # compile. |
| self._test_data_dependencies = [ |
| 'chrome/test/data/perf/third_party/octane', |
| ] |
| |
| @staticmethod |
| def _read_deps_file(deps_file_path): |
| class FileImplStub(object): |
| """Stub for the File syntax.""" |
| def __init__(self, file_location): |
| pass |
| |
| @staticmethod |
| def GetPath(): |
| return '' |
| |
| @staticmethod |
| def GetFilename(): |
| return '' |
| |
| @staticmethod |
| def GetRevision(): |
| return None |
| |
| def from_stub(__, _=None): |
| """Stub for the From syntax.""" |
| return '' |
| |
| class VarImpl(object): |
| def __init__(self, custom_vars, local_scope): |
| self._custom_vars = custom_vars |
| self._local_scope = local_scope |
| |
| def Lookup(self, var_name): |
| """Implements the Var syntax.""" |
| if var_name in self._custom_vars: |
| return self._custom_vars[var_name] |
| elif var_name in self._local_scope.get("vars", {}): |
| return self._local_scope["vars"][var_name] |
| raise Exception("Var is not defined: %s" % var_name) |
| |
| local_scope = {} |
| var = VarImpl({}, local_scope) |
| global_scope = { |
| 'File': FileImplStub, |
| 'From': from_stub, |
| 'Var': var.Lookup, |
| 'deps_os': {}, |
| } |
| execfile(deps_file_path, global_scope, local_scope) |
| deps = local_scope.get('deps', {}) |
| deps_os = local_scope.get('deps_os', {}) |
| for os_specific_deps in deps_os.itervalues(): |
| deps.update(os_specific_deps) |
| return deps.keys() |
| |
| def _get_known_issues(self): |
| issues = [] |
| for root, paths in known_issues.KNOWN_INCOMPATIBLE.items(): |
| for path in paths: |
| issues.append(os.path.normpath(os.path.join(root, path))) |
| return issues |
| |
| def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps): |
| """Calculates the list of deps that need to be excluded from the deps_file |
| so that the only deps left are the one in the whitelist.""" |
| all_deps = self._read_deps_file(deps_file_path) |
| # The list of deps read from the DEPS file are prefixed with the source |
| # tree root, which is 'src' for Chromium. |
| def prepend_root(path): |
| return os.path.join('src', path) |
| whitelisted_deps = map(prepend_root, whitelisted_deps) |
| deps_blacklist = set(all_deps).difference(set(whitelisted_deps)) |
| return dict(map(lambda(x): (x, None), deps_blacklist)) |
| |
| def _make_blacklist(self, deps_file_path, whitelisted_deps): |
| """Calculates the list of paths we should exclude """ |
| all_deps = self._read_deps_file(deps_file_path) |
| def remove_src_prefix(path): |
| return path.replace('src/', '', 1) |
| all_deps = map(remove_src_prefix, all_deps) |
| # Ignore all deps except those whitelisted. |
| blacklist = set(all_deps).difference(whitelisted_deps) |
| # Ignore the 'known issues'. Typically these are the licence incompatible |
| # things checked directly into Chromium. |
| blacklist = blacklist.union(self._get_known_issues()) |
| # Ignore any other non-deps, non-licence paths we don't like. |
| blacklist = blacklist.union(self._prune_from_rsync_build) |
| return list(blacklist) |
| |
| def get_deps_for_android_build(self, deps_file_path): |
| """This is used to calculate the custom_deps list for the Android bot. |
| """ |
| if not deps_file_path: |
| raise Exception('You need to specify a DEPS file path.') |
| return self._make_gclient_blacklist(deps_file_path, |
| self._compile_dependencies) |
| |
| def get_deps_for_android_build_and_test(self, deps_file_path): |
| """This is used to calculate the custom_deps list for the Android perf bot. |
| """ |
| if not deps_file_path: |
| raise Exception('You need to specify a DEPS file path.') |
| return self._make_gclient_blacklist(deps_file_path, |
| self._compile_dependencies + |
| self._test_data_dependencies) |
| |
| def get_blacklist_for_android_rsync_build(self, deps_file_path): |
| """Calculates the list of paths we should exclude when building Android |
| either because of license compatibility or because they are large and |
| uneeded. |
| """ |
| if not deps_file_path: |
| raise Exception('You need to specify a DEPS file path.') |
| return self._make_blacklist(deps_file_path, self._compile_dependencies) |
| |
| def get_deps_for_android_merge(self, _): |
| """Calculates the list of deps that need to be merged into the Android tree |
| in order to build the C++ and Java android_webview code.""" |
| return self._snapshot_into_android_dependencies |
| |
| def get_deps_for_license_check(self, _): |
| """Calculates the list of deps that need to be checked for Android license |
| compatibility""" |
| return self._compile_dependencies |
| |
| |
| def execute_method(self, method_name, deps_file_path): |
| methods = { |
| 'android_build': self.get_deps_for_android_build, |
| 'android_build_and_test': |
| self.get_deps_for_android_build_and_test, |
| 'android_merge': self.get_deps_for_android_merge, |
| 'license_check': self.get_deps_for_license_check, |
| 'android_rsync_build': self.get_blacklist_for_android_rsync_build, |
| } |
| if not method_name in methods: |
| raise Exception('Method name %s is not valid. Valid choices are %s' % |
| (method_name, methods.keys())) |
| return methods[method_name](deps_file_path) |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--method', help='Method to use to fetch from whitelist.', |
| required=True) |
| parser.add_argument('--path-to-deps', help='Path to DEPS file.') |
| parser.add_argument('--output-json', help='Name of file to write output to.') |
| parser.add_argument('verbose', action='store_true', default=False) |
| opts = parser.parse_args() |
| |
| logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN) |
| |
| deps_whitelist = DepsWhitelist() |
| blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps) |
| |
| if (opts.output_json): |
| output_dict = { |
| 'blacklist' : blacklist |
| } |
| with open(opts.output_json, 'w') as output_json_file: |
| json.dump(output_dict, output_json_file) |
| else: |
| print blacklist |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
