| #!src/build/run_python |
| |
| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import ast |
| import distutils.spawn |
| import errno |
| import os |
| import pipes |
| import shutil |
| import subprocess |
| import sys |
| |
| from src.build import build_common |
| from src.build import config_runner |
| from src.build import download_arc_welder_deps |
| from src.build import download_cts_files |
| from src.build import download_sdk_and_ndk |
| from src.build import open_source |
| from src.build import staging |
| from src.build import sync_adb |
| from src.build import sync_gdb_multiarch |
| from src.build import sync_nacl_sdk |
| from src.build import toolchain |
| from src.build.build_options import OPTIONS |
| from src.build.util import file_util |
| |
| |
| def _set_up_git_hooks(): |
| # These git hooks do not make sense for the open source repo because they: |
| # 1) lint the source, but that was already done when committed internally, |
| # and we will run 'ninja all' as a test step before committing to open |
| # source. |
| # 2) add fields to the commit message for the internal dev workflow. |
| if open_source.is_open_source_repo(): |
| return |
| script_dir = os.path.dirname(__file__) |
| hooks = { |
| 'pre-push': os.path.join(script_dir, 'git_pre_push.py'), |
| 'prepare-commit-msg': os.path.join(script_dir, 'git_prepare_commit.py'), |
| 'commit-msg': 'third_party/gerrit/commit-msg', |
| } |
| obsolete_hooks = ['pre-commit'] # Replaced by pre-push hook. |
| |
| git_hooks_dir = os.path.join(build_common.get_arc_root(), '.git', 'hooks') |
| for git_hook, source_path in hooks.iteritems(): |
| symlink_path = os.path.join(git_hooks_dir, git_hook) |
| file_util.create_link(symlink_path, source_path, overwrite=True) |
| |
| for git_hook in obsolete_hooks: |
| symlink_path = os.path.join(git_hooks_dir, git_hook) |
| if os.path.lexists(symlink_path): |
| os.unlink(symlink_path) |
| |
| |
| def _check_javac_version(): |
| # Stamp file should keep the last modified time of the java binary. |
| javac_path = distutils.spawn.find_executable( |
| toolchain.get_tool('java', 'javac')) |
| stamp_file = build_common.StampFile( |
| '%s %f' % (javac_path, os.path.getmtime(javac_path)), |
| build_common.get_javac_revision_file()) |
| if stamp_file.is_up_to_date(): |
| return |
| |
| want_version = '1.7.' |
| javac_version = subprocess.check_output( |
| [javac_path, '-version'], stderr=subprocess.STDOUT) |
| if want_version not in javac_version: |
| print '\nWARNING: You are not using Java 7.', |
| print 'Installed version:', javac_version.strip() |
| print 'See docs/getting-java.md.\n' |
| else: |
| stamp_file.update() |
| |
| |
| def _cleanup_orphaned_pyc_files(): |
| # Watch out for .pyc files without a corresponding .py file |
| for base_path in ('src/', 'mods/'): |
| for root, dirs, files in os.walk(base_path): |
| for name in files: |
| fullpath = os.path.join(root, name) |
| base, ext = os.path.splitext(fullpath) |
| if ext == '.pyc' and not os.path.exists(base + '.py'): |
| print ('\nWARNING: %s appears to be a compiled python file without ' |
| 'any associated python code. It has been removed.') % fullpath |
| os.unlink(fullpath) |
| |
| |
| def _gclient_sync_third_party(): |
| gclient_filename = 'third_party/.gclient' |
| |
| # For whatever reason gclient wants to take revisions from the command line |
| # and does not read them from the .gclient file -- they used to be part of the |
| # url. To work around this, we look for a new revision key for each dictionary |
| # in the .gclient solution array, and use that to pass the revision |
| # information on the command line. |
| # TODO(lpique): Modify gclient to have it look for 'revision' in the .gclient |
| # file itself, which will make this block of code unnecessary. |
| with open(gclient_filename) as f: |
| # Read the gclient file ourselves to extract some extra information from it |
| gclient_content = f.read() |
| # Make sure it appears to have an expected beginning, so we can quickly |
| # parse it. |
| assert gclient_content.startswith('solutions = [') |
| # Use ast.literal_eval on the array, which works to evaluate simple python |
| # constants. Using the built-in eval is potentially unsafe as it can execute |
| # arbitrary code. |
| # We start with the first array bracket, ignoring anything before it. |
| gclient_contents = ast.literal_eval( |
| gclient_content[gclient_content.find('['):]) |
| |
| with open(gclient_filename, 'r') as f: |
| stamp_file = build_common.StampFile( |
| gclient_content, build_common.get_thirdparty_gclient_revision_file()) |
| if stamp_file.is_up_to_date(): |
| return |
| |
| cmd = ['gclient', 'sync', '--gclientfile', os.path.basename(gclient_filename)] |
| |
| # TODO(lpique): Modify gclient to have it look for 'revision' in the .gclient |
| # file itself, which will make this block of code unnecessary. |
| for entry in gclient_contents: |
| if 'name' in entry and 'revision' in entry: |
| cmd.extend(['--revision', pipes.quote('%(name)s@%(revision)s' % entry)]) |
| |
| try: |
| subprocess.check_output(cmd, cwd=os.path.dirname(gclient_filename)) |
| stamp_file.update() |
| except subprocess.CalledProcessError as e: |
| sys.stderr.write(e.output) |
| sys.exit('Error running "%s"' % ' '.join(cmd)) |
| |
| |
| def _ensure_downloads_up_to_date(): |
| cache_path = OPTIONS.download_cache_path() |
| cache_size = OPTIONS.download_cache_size() |
| |
| sync_nacl_sdk.check_and_perform_updates(cache_path, cache_size) |
| download_sdk_and_ndk.check_and_perform_updates(cache_path, cache_size) |
| download_cts_files.check_and_perform_updates(cache_path, cache_size) |
| download_arc_welder_deps.check_and_perform_updates(cache_path, cache_size) |
| |
| if sync_gdb_multiarch.main(): |
| sys.exit(1) |
| |
| # The open source repository does not have download_internal_apks.py. |
| if (not open_source.is_open_source_repo() and |
| OPTIONS.internal_apks_source() == 'prebuilt'): |
| import download_internal_apks |
| download_internal_apks.check_and_perform_updates(cache_path, cache_size) |
| |
| if not open_source.is_open_source_repo(): |
| import download_third_party_apks |
| download_third_party_apks.check_and_perform_updates(cache_path, cache_size) |
| |
| |
| def _configure_build_options(): |
| if OPTIONS.parse(sys.argv[1:]): |
| print 'Args error' |
| return False |
| |
| # Write out the configure file early so all other scripts can use |
| # the options passed into configure. (e.g., sync_chrome). |
| OPTIONS.write_configure_file() |
| |
| # Target directory is replaced. If an old directory, out/target/<target>, |
| # exists, move it to the new place, out/target/<target>_<opt>. |
| old_path = os.path.join('out/target', OPTIONS.target()) |
| new_path = build_common.get_build_dir() |
| if os.path.lexists(old_path): |
| if os.path.isdir(old_path) and not os.path.islink(old_path): |
| if os.path.exists(new_path): |
| file_util.rmtree(old_path) |
| else: |
| shutil.move(old_path, new_path) |
| else: |
| os.remove(old_path) |
| |
| # Create an empty directory as a placeholder if necessary. |
| file_util.makedirs_safely(new_path) |
| |
| # Create a symlink from new place to old place to keep as compatible as |
| # possible. |
| os.symlink(os.path.basename(new_path), old_path) |
| |
| # Write out the configure file to a target specific location, which can be |
| # queried later to find out what the config for a target was. |
| OPTIONS.write_configure_file(build_common.get_target_configure_options_file()) |
| |
| OPTIONS.set_up_goma() |
| return True |
| |
| |
| def _set_up_chromium_org_submodules(): |
| CHROMIUM_ORG_ROOT = 'third_party/android/external/chromium_org' |
| # android/external/chromium_org contains these required submodules. It is not |
| # posible to have submodules within a submodule path (i.e., chromium_org) |
| # using git submodules. This is the list of subdirectories relative to |
| # chromium_org that we need to symlink to the appropriate submodules. |
| submodules = [ |
| 'sdch/open-vcdiff', |
| 'testing/gtest', |
| 'third_party/WebKit', |
| 'third_party/angle', |
| 'third_party/brotli/src', |
| ('third_party/eyesfree/src/android/java/' |
| 'src/com/googlecode/eyesfree/braille'), |
| 'third_party/freetype', |
| 'third_party/icu', |
| 'third_party/leveldatabase/src', |
| 'third_party/libjingle/source/talk', |
| 'third_party/libphonenumber/src/phonenumbers', |
| 'third_party/libphonenumber/src/resources', |
| 'third_party/libsrtp', |
| 'third_party/libvpx', |
| 'third_party/libyuv', |
| 'third_party/mesa/src', |
| 'third_party/openmax_dl', |
| 'third_party/openssl', |
| 'third_party/opus/src', |
| 'third_party/ots', |
| 'third_party/sfntly/cpp/src', |
| 'third_party/skia', |
| 'third_party/smhasher/src', |
| 'third_party/usrsctp/usrsctplib', |
| 'third_party/webrtc', |
| 'third_party/yasm/source/patched-yasm', |
| 'v8'] |
| |
| # First remove all existing symlinks to make sure no stale links exist. |
| for dirpath, dirs, fnames in os.walk(CHROMIUM_ORG_ROOT): |
| # We only create symlinks for directories. |
| for name in dirs: |
| directory = os.path.join(dirpath, name) |
| if os.path.islink(directory): |
| os.unlink(directory) |
| |
| for s in submodules: |
| target = os.path.join(CHROMIUM_ORG_ROOT, s) |
| # As an example, this maps 'sdch/open-vcdiff' to |
| # 'android/external/chromium_org__sdch_open-vcdiff', which is the true |
| # location of the submodule checkout. |
| source = 'third_party/android/external/chromium_org__' + s.replace('/', '_') |
| if not os.path.exists(source): |
| print 'ERROR: path "%s" does not exist.' % source |
| print 'ERROR: Did you forget to run git submodules update --init?' |
| sys.exit(1) |
| |
| # Remove existing symlink for transition from previous version. |
| # The previous configuration script creates symlinks to the top of |
| # chromium.org submodules, and now it tries to symlink them one-level |
| # deeper. |
| # |
| # TODO(tzik): Remove this clean up code after the transition is no longer |
| # needed. |
| if os.path.islink(target): |
| os.remove(target) |
| |
| target_contents = set() |
| try: |
| target_contents = set(os.listdir(target)) |
| except OSError as e: |
| if e.errno != errno.ENOENT: |
| raise |
| |
| source_contents = set(os.listdir(source)) |
| for content in source_contents: |
| link_source = os.path.join(source, content) |
| link_target = os.path.join(target, content) |
| # If a real directory exists, remove it explicitly. |overwrite| |
| # flag does not care for real directories and files, but old symlinks. |
| if os.path.exists(link_target) and not os.path.islink(link_target): |
| file_util.rmtree(link_target) |
| |
| file_util.create_link(link_target, link_source, overwrite=True) |
| |
| for removed_item in target_contents.difference(source_contents): |
| os.unlink(os.path.join(target, removed_item)) |
| |
| |
| def _update_arc_version_file(): |
| stamp = build_common.StampFile(build_common.get_build_version(), |
| build_common.get_build_version_path()) |
| if not stamp.is_up_to_date(): |
| stamp.update() |
| |
| |
| def _set_up_internal_repo(): |
| if OPTIONS.internal_apks_source_is_internal(): |
| # Checkout internal/ repository if needed, and sync internal/third_party/* |
| # to internal/build/DEPS.*. The files needs to be re-staged and is done |
| # in staging.create_staging. |
| subprocess.check_call('src/build/sync_arc_int.py') |
| |
| # Create a symlink to the integration_test definition directory, either in the |
| # internal repository checkout or in the downloaded archive. |
| # It is used to determine the definitions loaded in run_integration_tests.py |
| # without relying on OPTIONS. Otherwise, run_integration_tests_test may fail |
| # due to the mismatch between the expectations and the actual test apk since |
| # it always runs under the default OPTIONS. |
| if OPTIONS.internal_apks_source_is_internal(): |
| test_dir = 'internal/integration_tests' |
| else: |
| test_dir = 'out/internal-apks/integration_tests' |
| |
| symlink_location = 'out/internal-apks-integration-tests' |
| if not os.path.islink(symlink_location) and os.path.isdir(symlink_location): |
| # TODO(crbug.com/517306): This is a short-term fix for making bots green. |
| # Remove once we implement a more proper solution (like, isolating the |
| # test bundle extraction directory from the checkout, or stop running |
| # ./configure on test-only builders.) |
| # |
| # 'Test only' bots extracts a directory structure from test-bundle zip, so |
| # in that case we don't need to create a symlink. |
| print 'WARNING: A directory exists at %s.' % symlink_location |
| print 'WARNING: If not on test-only bot, gms test expectation may be stale.' |
| else: |
| # Otherwise, (re)create the link. |
| file_util.create_link(symlink_location, test_dir, overwrite=True) |
| |
| |
| def main(): |
| # Disable line buffering |
| sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
| |
| if not _configure_build_options(): |
| return -1 |
| |
| _update_arc_version_file() |
| |
| _ensure_downloads_up_to_date() |
| |
| if not open_source.is_open_source_repo(): |
| import sync_chrome |
| sync_chrome.run() |
| |
| adb_target = 'linux-arm' if OPTIONS.is_arm() else 'linux-x86_64' |
| sync_adb.run(adb_target) |
| |
| _set_up_internal_repo() |
| |
| _gclient_sync_third_party() |
| _check_javac_version() |
| _cleanup_orphaned_pyc_files() |
| |
| _set_up_git_hooks() |
| |
| _set_up_chromium_org_submodules() |
| |
| # Make sure the staging directory is up to date whenever configure |
| # runs to make it easy to generate rules by scanning directories. |
| staging.create_staging() |
| |
| config_runner.generate_ninjas() |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |