| # Copyright 2023 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """This helper provides a build context that handles |
| the reclient lifecycle safely. It will automatically start |
| reproxy before running ninja and stop reproxy when build stops |
| for any reason e.g. build completion, keyboard interrupt etc.""" |
| |
| import atexit |
| import contextlib |
| import datetime |
| import hashlib |
| import os |
| import shutil |
| import socket |
| import subprocess |
| import sys |
| import time |
| import uuid |
| |
| import gclient_paths |
| import reclient_metrics |
| |
| |
| THIS_DIR = os.path.dirname(__file__) |
| RECLIENT_LOG_CLEANUP = os.path.join(THIS_DIR, 'reclient_log_cleanup.py') |
| |
| |
| def find_reclient_bin_dir(): |
| tools_path = gclient_paths.GetBuildtoolsPath() |
| if not tools_path: |
| return None |
| |
| reclient_bin_dir = os.path.join(tools_path, 'reclient') |
| if os.path.isdir(reclient_bin_dir): |
| return reclient_bin_dir |
| return None |
| |
| |
| def find_reclient_cfg(): |
| tools_path = gclient_paths.GetBuildtoolsPath() |
| if not tools_path: |
| return None |
| |
| reclient_cfg = os.path.join(tools_path, 'reclient_cfgs', 'reproxy.cfg') |
| if os.path.isfile(reclient_cfg): |
| return reclient_cfg |
| return None |
| |
| |
| def run(cmd_args): |
| if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': |
| print(' '.join(cmd_args)) |
| return subprocess.call(cmd_args) |
| |
| |
| def start_reproxy(reclient_cfg, reclient_bin_dir): |
| return run([ |
| os.path.join(reclient_bin_dir, |
| 'bootstrap' + gclient_paths.GetExeSuffix()), |
| '--re_proxy=' + os.path.join(reclient_bin_dir, |
| 'reproxy' + gclient_paths.GetExeSuffix()), |
| '--cfg=' + reclient_cfg |
| ]) |
| |
| |
| def stop_reproxy(reclient_cfg, reclient_bin_dir): |
| return run([ |
| os.path.join(reclient_bin_dir, |
| 'bootstrap' + gclient_paths.GetExeSuffix()), '--shutdown', |
| '--cfg=' + reclient_cfg |
| ]) |
| |
| |
| def find_ninja_out_dir(args): |
| # Ninja uses getopt_long, which allows to intermix non-option arguments. |
| # To leave non supported parameters untouched, we do not use getopt. |
| for index, arg in enumerate(args[1:]): |
| if arg == '-C': |
| # + 1 to get the next argument and +1 because we trimmed off args[0] |
| return args[index + 2] |
| if arg.startswith('-C'): |
| # Support -Cout/Default |
| return arg[2:] |
| return '.' |
| |
| |
| def find_cache_dir(tmp_dir): |
| """Helper to find the correct cache directory for a build. |
| |
| tmp_dir should be a build specific temp directory within the out directory. |
| |
| If this is called from within a gclient checkout, the cache dir will be: |
| <gclient_root>/.reproxy_cache/md5(tmp_dir)/ |
| If this is not called from within a gclient checkout, the cache dir will be: |
| tmp_dir/cache |
| """ |
| gclient_root = gclient_paths.FindGclientRoot(os.getcwd()) |
| if gclient_root: |
| return os.path.join(gclient_root, '.reproxy_cache', |
| hashlib.md5(tmp_dir.encode()).hexdigest()) |
| return os.path.join(tmp_dir, 'cache') |
| |
| |
| def auth_cache_status(): |
| cred_file = os.path.join(os.environ["RBE_cache_dir"], "reproxy.creds") |
| if not os.path.isfile(cred_file): |
| return "missing", "UNSPECIFIED" |
| try: |
| with open(cred_file) as f: |
| status = "valid" |
| mechanism = "UNSPECIFIED" |
| for line in f.readlines(): |
| if "seconds:" in line: |
| exp = int(line.strip()[len("seconds:"):].strip()) |
| if exp < (time.time() + 5 * 60): |
| status = "expired" |
| elif "mechanism:" in line: |
| mechanism = line.strip()[len("mechanism:"):].strip() |
| return status, mechanism |
| except OSError: |
| return "missing", "UNSPECIFIED" |
| |
| |
| def get_hostname(): |
| hostname = socket.gethostname() |
| # In case that returned an address, make a best effort attempt to get |
| # the hostname and ignore any errors. |
| try: |
| return socket.gethostbyaddr(hostname)[0] |
| except Exception: |
| return hostname |
| |
| |
| def set_reproxy_metrics_flags(tool): |
| """Helper to setup metrics collection flags for reproxy. |
| |
| The following env vars are set if not already set: |
| RBE_metrics_project=chromium-reclient-metrics |
| RBE_invocation_id=$AUTONINJA_BUILD_ID |
| RBE_metrics_table=rbe_metrics.builds |
| RBE_metrics_labels=source=developer,tool={tool} |
| RBE_metrics_prefix=go.chromium.org |
| """ |
| autoninja_id = os.environ.get("AUTONINJA_BUILD_ID") |
| if autoninja_id is not None: |
| os.environ.setdefault("RBE_invocation_id", |
| "%s/%s" % (get_hostname(), autoninja_id)) |
| os.environ.setdefault("RBE_metrics_project", "chromium-reclient-metrics") |
| os.environ.setdefault("RBE_metrics_table", "rbe_metrics.builds") |
| labels = "source=developer,tool=" + tool |
| auth_status, auth_mechanism = auth_cache_status() |
| labels += ",creds_cache_status=" + auth_status |
| labels += ",creds_cache_mechanism=" + auth_mechanism |
| os.environ.setdefault("RBE_metrics_labels", labels) |
| os.environ.setdefault("RBE_metrics_prefix", "go.chromium.org") |
| |
| |
| def remove_mdproxy_from_path(): |
| os.environ["PATH"] = os.pathsep.join( |
| d for d in os.environ.get("PATH", "").split(os.pathsep) |
| if "mdproxy" not in d) |
| |
| |
| # Mockable datetime.datetime.utcnow for testing. |
| def datetime_now(): |
| return datetime.datetime.utcnow() |
| |
| |
| # Deletes the tree at dir if it exists. |
| def rmtree_if_exists(rm_dir): |
| if os.path.exists(rm_dir) and os.path.isdir(rm_dir): |
| shutil.rmtree(rm_dir, ignore_errors=True) |
| |
| |
| def set_reproxy_path_flags(out_dir, make_dirs=True): |
| """Helper to setup the logs and cache directories for reclient. |
| |
| Creates the following directory structure if make_dirs is true: |
| If in a gclient checkout |
| out_dir/ |
| .reproxy_tmp/ |
| logs/ |
| <gclient_root> |
| .reproxy_cache/ |
| md5(out_dir/.reproxy_tmp)/ |
| |
| If not in a gclient checkout |
| out_dir/ |
| .reproxy_tmp/ |
| logs/ |
| cache/ |
| |
| The following env vars are set if not already set: |
| RBE_output_dir=out_dir/.reproxy_tmp/logs |
| RBE_proxy_log_dir=out_dir/.reproxy_tmp/logs |
| RBE_log_dir=out_dir/.reproxy_tmp/logs |
| RBE_cache_dir=out_dir/.reproxy_tmp/cache |
| *Nix Only: |
| RBE_server_address=unix://out_dir/.reproxy_tmp/reproxy.sock |
| Windows Only: |
| RBE_server_address=pipe://md5(out_dir/.reproxy_tmp)/reproxy.pipe |
| """ |
| os.environ.setdefault("AUTONINJA_BUILD_ID", str(uuid.uuid4())) |
| run_sub_dir = datetime_now().strftime( |
| '%Y%m%dT%H%M%S.%f') + "_" + os.environ["AUTONINJA_BUILD_ID"] |
| tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) |
| log_dir = os.path.join(tmp_dir, 'logs') |
| run_log_dir = os.path.join(log_dir, run_sub_dir) |
| racing_dir = os.path.join(tmp_dir, 'racing') |
| run_racing_dir = os.path.join(racing_dir, run_sub_dir) |
| cache_dir = find_cache_dir(tmp_dir) |
| |
| atexit.register(rmtree_if_exists, run_racing_dir) |
| |
| if make_dirs: |
| if os.path.isfile(os.path.join(log_dir, "rbe_metrics.txt")): |
| try: |
| # Delete entire log dir if it is in the old format |
| # which had no subdirectories for each build. |
| shutil.rmtree(log_dir) |
| except OSError: |
| print( |
| "Couldn't clear logs because reproxy did " |
| "not shutdown after the last build", |
| file=sys.stderr) |
| os.makedirs(tmp_dir, exist_ok=True) |
| os.makedirs(log_dir, exist_ok=True) |
| os.makedirs(run_log_dir, exist_ok=True) |
| os.makedirs(cache_dir, exist_ok=True) |
| os.makedirs(racing_dir, exist_ok=True) |
| os.makedirs(run_racing_dir, exist_ok=True) |
| |
| old_log_dirs = [ |
| d for d in os.listdir(log_dir) |
| if os.path.isdir(os.path.join(log_dir, d)) |
| ] |
| |
| if len(old_log_dirs) > 5: |
| old_log_dirs.sort(key=lambda dir: dir.split("_"), reverse=True) |
| for d in old_log_dirs[5:]: |
| shutil.rmtree(os.path.join(log_dir, d)) |
| |
| os.environ.setdefault("RBE_output_dir", run_log_dir) |
| os.environ.setdefault("RBE_proxy_log_dir", run_log_dir) |
| os.environ.setdefault("RBE_log_dir", run_log_dir) |
| os.environ.setdefault("RBE_cache_dir", cache_dir) |
| os.environ.setdefault("RBE_racing_tmp_dir", run_racing_dir) |
| if sys.platform.startswith('win'): |
| pipe_dir = hashlib.sha256(run_log_dir.encode()).hexdigest() |
| os.environ.setdefault("RBE_server_address", |
| "pipe://%s/reproxy.pipe" % pipe_dir) |
| else: |
| # unix domain socket has path length limit, so use fixed size path here. |
| # ref: https://www.man7.org/linux/man-pages/man7/unix.7.html |
| os.environ.setdefault( |
| "RBE_server_address", "unix:///tmp/reproxy_%s.sock" % |
| hashlib.sha256(run_log_dir.encode()).hexdigest()) |
| |
| |
| def set_racing_defaults(): |
| os.environ.setdefault("RBE_local_resource_fraction", "0.2") |
| os.environ.setdefault("RBE_racing_bias", "0.95") |
| |
| |
| def set_mac_defaults(): |
| # Reduce the cas concurrency on macs. Lower value doesn't impact |
| # performance when on high-speed connection, but does show improvements |
| # on easily congested networks. |
| os.environ.setdefault("RBE_cas_concurrency", "100") |
| # Enable the deps cache on macs. Mac needs a larger deps cache as it |
| # seems to have larger dependency sets per action. |
| os.environ.setdefault("RBE_enable_deps_cache", "true") |
| os.environ.setdefault("RBE_deps_cache_max_mb", "1024") |
| |
| |
| def set_win_defaults(): |
| # Enable the deps cache on windows. This makes a notable improvement |
| # in performance at the cost of a ~200MB cache file. |
| os.environ.setdefault("RBE_enable_deps_cache", "true") |
| # Reduce local resource fraction used to do local compile actions on |
| # windows, to try and prevent machine saturation. |
| os.environ.setdefault("RBE_local_resource_fraction", "0.05") |
| # Set execution strategy to remote_local_fallback while racing performance |
| # on windows is addressed. |
| os.environ.setdefault("RBE_exec_strategy", "remote_local_fallback") |
| # Turn off creds caching for windows, as luci-auth as credshelper shouldn't |
| # use it. |
| os.environ.setdefault("RBE_enable_creds_cache", "false") |
| # Extend timeouts on windows |
| os.environ.setdefault("RBE_exec_timeout","4m") |
| os.environ.setdefault("RBE_reclient_timeout","8m") |
| |
| |
| def workspace_is_cog(): |
| return sys.platform == "linux" and os.path.realpath( |
| os.getcwd()).startswith("/google/cog") |
| |
| |
| # pylint: disable=line-too-long |
| def reclient_setup_docs_url(): |
| if sys.platform == "darwin": |
| return "https://chromium.googlesource.com/chromium/src/+/main/docs/mac_build_instructions.md#use-reclient" |
| if sys.platform.startswith("win"): |
| return "https://chromium.googlesource.com/chromium/src/+/main/docs/windows_build_instructions.md#use-reclient" |
| return "https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md#use-reclient" |
| |
| |
| @contextlib.contextmanager |
| def build_context(argv, tool): |
| # If use_remoteexec is set, but the reclient binaries or configs don't |
| # exist, display an error message and stop. Otherwise, the build will |
| # attempt to run with rewrapper wrapping actions, but will fail with |
| # possible non-obvious problems. |
| reclient_bin_dir = find_reclient_bin_dir() |
| reclient_cfg = find_reclient_cfg() |
| if reclient_bin_dir is None or reclient_cfg is None: |
| print( |
| 'Build is configured to use reclient but necessary binaries ' |
| "or config files can't be found.\n" |
| 'Please check if `"download_remoteexec_cfg": True` custom var is ' |
| 'set in `.gclient`, and run `gclient sync`.', |
| file=sys.stderr) |
| yield 1 |
| return |
| |
| ninja_out = find_ninja_out_dir(argv) |
| |
| try: |
| set_reproxy_path_flags(ninja_out) |
| except OSError as e: |
| print(f"Error creating reproxy_tmp in output dir: {e}", file=sys.stderr) |
| yield 1 |
| return |
| |
| if reclient_metrics.check_status(ninja_out): |
| set_reproxy_metrics_flags(tool) |
| |
| if os.environ.get('RBE_instance', None): |
| print('WARNING: Using RBE_instance=%s\n' % |
| os.environ.get('RBE_instance', '')) |
| |
| remote_disabled = os.environ.get('RBE_remote_disabled') |
| if remote_disabled not in ('1', 't', 'T', 'true', 'TRUE', 'True'): |
| # If we are building inside a Cog workspace, racing is likely not a |
| # performance improvement, so we disable it by default. |
| if workspace_is_cog(): |
| os.environ.setdefault("RBE_exec_strategy", "remote_local_fallback") |
| set_racing_defaults() |
| if sys.platform == "darwin": |
| set_mac_defaults() |
| if sys.platform.startswith("win"): |
| set_win_defaults() |
| |
| # TODO(b/292523514) remove this once a fix is landed in reproxy |
| remove_mdproxy_from_path() |
| |
| start = time.time() |
| reproxy_ret_code = start_reproxy(reclient_cfg, reclient_bin_dir) |
| if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': |
| elapsed = time.time() - start |
| print('%1.3f s to start reproxy' % elapsed) |
| if reproxy_ret_code != 0: |
| print(f'''Failed to start reproxy! |
| See above error message for details. |
| Ensure you have completed the reproxy setup instructions: |
| {reclient_setup_docs_url()}''', |
| file=sys.stderr) |
| yield reproxy_ret_code |
| return |
| try: |
| yield |
| finally: |
| start = time.time() |
| stop_reproxy(reclient_cfg, reclient_bin_dir) |
| if os.environ.get('NINJA_SUMMARIZE_BUILD') == '1': |
| elapsed = time.time() - start |
| print('%1.3f s to stop reproxy' % elapsed) |
