blob: 01ae5adf5e42ced86be82e41c8d6ecb88eb4ba5e [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2012 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""Front end tool to operate on .isolate files.
This includes creating, merging or compiling them to generate a .isolated file.
See more information at
# Run ./ --help for more detailed information.
__version__ = '0.4.5'
import datetime
import itertools
import logging
import optparse
import os
import re
import subprocess
import sys
import auth
import isolate_format
import isolate_storage
import isolated_format
import isolateserver
import run_isolated
from utils import tools
# third_party/
import colorama
from depot_tools import fix_encoding
from depot_tools import subcommand
# pylint: disable=ungrouped-imports
from utils import logging_utils
from utils import file_path
from utils import fs
from utils import subprocess42
# Exit code of 'archive' and 'batcharchive' if the command fails due to an error
# in *.isolate file (format error, or some referenced files are missing, etc.)
# Exit code of 'archive' and 'batcharchive' if the command fails due to
# a network or server issue. It is an infrastructure failure.
# Supported version of *.isolated.gen.json files consumed by CMDbatcharchive.
# Warning printed when variables are used. This is meant to reduce the use of
# variables.
WARNING: --config-variables, --path-variables and --extra-variables
will be unsupported soon. Please contact the LUCI team.
class ExecutionError(Exception):
"""A generic error occurred."""
def __str__(self):
return self.args[0]
### Path handling code.
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):'Creating %s' % outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == fs.stat(outfile).st_size:
logging.warn('Overwriting %s' % metadata['h'])
outfile = os.path.join(outdir, relfile)
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
fs.symlink(pointed, outfile) # pylint: disable=E1101
file_path.link_file(outfile, infile, action)
### Variable stuff.
def _normalize_path_variable(cwd, relative_base_dir, key, value):
"""Normalizes a path variable into a relative directory.
# Variables could contain / or \ on windows. Always normalize to
# os.path.sep.
x = os.path.join(cwd, value.strip().replace('/', os.path.sep))
normalized = file_path.get_native_path_case(os.path.normpath(x))
if not os.path.isdir(normalized):
raise ExecutionError('%s=%s is not a directory' % (key, normalized))
# All variables are relative to the .isolate file.
normalized = os.path.relpath(normalized, relative_base_dir)
'Translated variable %s from %s to %s', key, value, normalized)
return normalized
def normalize_path_variables(cwd, path_variables, relative_base_dir):
"""Processes path variables as a special case and returns a copy of the dict.
For each 'path' variable: first normalizes it based on |cwd|, verifies it
exists then sets it as relative to relative_base_dir.
'normalize_path_variables(%s, %s, %s)', cwd, path_variables,
assert isinstance(cwd, unicode), cwd
assert isinstance(relative_base_dir, unicode), relative_base_dir
relative_base_dir = file_path.get_native_path_case(relative_base_dir)
return dict(
(k, _normalize_path_variable(cwd, relative_base_dir, k, v))
for k, v in path_variables.iteritems())
### Internal state files.
def isolatedfile_to_state(filename):
"""For a '.isolate' file, returns the path to the saved '.state' file."""
return filename + '.state'
def chromium_save_isolated(isolated, data, path_variables, algo):
"""Writes one or many .isolated files.
This slightly increases the cold cache cost but greatly reduce the warm cache
cost by splitting low-churn files off the master .isolated file. It also
reduces overall isolateserver memcache consumption.
slaves = []
def extract_into_included_isolated(prefix):
new_slave = {
'algo': data['algo'],
'files': {},
'version': data['version'],
for f in data['files'].keys():
if f.startswith(prefix):
new_slave['files'][f] = data['files'].pop(f)
if new_slave['files']:
# Split test/data/ in its own .isolated file.
extract_into_included_isolated(os.path.join('test', 'data', ''))
# Split everything out of PRODUCT_DIR in its own .isolated file.
if path_variables.get('PRODUCT_DIR'):
files = []
for index, f in enumerate(slaves):
slavepath = isolated[:-len('.isolated')] + '.%d.isolated' % index
tools.write_json(slavepath, f, True)
data.setdefault('includes', []).append(
isolated_format.hash_file(slavepath, algo))
isolated_format.save_isolated(isolated, data)
return files
class Flattenable(object):
"""Represents data that can be represented as a json file."""
def flatten(self):
"""Returns a json-serializable version of itself.
Skips None entries.
items = ((member, getattr(self, member)) for member in self.MEMBERS)
return dict((member, value) for member, value in items if value is not None)
def load(cls, data, *args, **kwargs):
"""Loads a flattened version."""
data = data.copy()
out = cls(*args, **kwargs)
for member in out.MEMBERS:
if member in data:
# Access to a protected member XXX of a client class
# pylint: disable=W0212
out._load_member(member, data.pop(member))
if data:
raise ValueError(
'Found unexpected entry %s while constructing an object %s' %
(data, cls.__name__), data, cls.__name__)
return out
def _load_member(self, member, value):
"""Loads a member into self."""
setattr(self, member, value)
def load_file(cls, filename, *args, **kwargs):
"""Loads the data from a file or return an empty instance."""
out = cls.load(tools.read_json(filename), *args, **kwargs)
logging.debug('Loaded %s(%s)', cls.__name__, filename)
except (IOError, ValueError) as e:
# On failure, loads the default instance.
out = cls(*args, **kwargs)
logging.warn('Failed to load %s: %s', filename, e)
return out
class SavedState(Flattenable):
"""Describes the content of a .state file.
This file caches the items calculated by this script and is used to increase
the performance of the script. This file is not loaded by
This file can always be safely removed.
It is important to note that the 'files' dict keys are using native OS path
separator instead of '/' used in .isolate file.
# Value of sys.platform so that the file is rejected if loaded from a
# different OS. While this should never happen in practice, users are ...
# "creative".
# Algorithm used to generate the hash. The only supported value is at the
# time of writing 'sha-1'.
# List of included .isolated files. Used to support/remember 'slave'
# .isolated files. Relative path to isolated_basedir.
# Cache of the processed command. This value is saved because .isolated
# files are never loaded by so it's the only way to load the
# command safely.
# GYP variables that are used to generate conditions. The most frequent
# example is 'OS'.
# GYP variables that will be replaced in 'command' and paths but will not be
# considered a relative directory.
# Cache of the files found so the next run can skip hash calculation.
# Path of the original .isolate file. Relative path to isolated_basedir.
# GYP variables used to generate the .isolated files paths based on path
# variables. Frequent examples are DEPTH and PRODUCT_DIR.
# If the generated directory tree should be read-only. Defaults to 1.
# Relative cwd to use to start the command.
# Root directory the files are mapped from.
# Version of the saved state file format. Any breaking change must update
# the value.
# Bump this version whenever the saved state changes. It is also keyed on the
# .isolated file version so any change in the generator will invalidate .state
# files.
def __init__(self, algo_name, isolated_basedir):
"""Creates an empty SavedState.
algo_name: name of the algorithm used, 'sha-1', 'sha-256' or 'sha-512'.
isolated_basedir: the directory where the .isolated and .isolated.state
files are saved.
super(SavedState, self).__init__()
assert os.path.isabs(isolated_basedir), isolated_basedir
assert os.path.isdir(isolated_basedir), isolated_basedir
self.isolated_basedir = isolated_basedir
# The default algorithm used.
self.OS = sys.platform
self.algo_name = algo_name
assert algo_name in isolated_format.SUPPORTED_ALGOS, algo_name
self.algo = isolated_format.SUPPORTED_ALGOS[algo_name]
self.child_isolated_files = []
self.command = []
self.config_variables = {}
self.extra_variables = {}
self.files = {}
self.isolate_file = None
self.path_variables = {}
# Defaults to 1 when compiling to .isolated.
self.read_only = None
self.relative_cwd = None
self.root_dir = None
self.version = self.EXPECTED_VERSION
def update_config(self, config_variables):
"""Updates the saved state with only config variables."""
def update(self, isolate_file, path_variables, extra_variables):
"""Updates the saved state with new data to keep GYP variables and internal
reference to the original .isolate file.
assert os.path.isabs(isolate_file)
# Convert back to a relative path. On Windows, if the isolate and
# isolated files are on different drives, isolate_file will stay an absolute
# path.
isolate_file = file_path.safe_relpath(isolate_file, self.isolated_basedir)
# The same .isolate file should always be used to generate the .isolated and
# .isolated.state.
assert isolate_file == self.isolate_file or not self.isolate_file, (
isolate_file, self.isolate_file)
self.isolate_file = isolate_file
def update_isolated(self, command, infiles, read_only, relative_cwd):
"""Updates the saved state with data necessary to generate a .isolated file.
The new files in |infiles| are added to self.files dict but their hash is
not calculated here.
self.command = command
# Add new files.
for f in infiles:
self.files.setdefault(f, {})
# Prune extraneous files that are not a dependency anymore.
for f in set(self.files).difference(set(infiles)):
del self.files[f]
if read_only is not None:
self.read_only = read_only
self.relative_cwd = relative_cwd
def to_isolated(self):
"""Creates a .isolated dictionary out of the saved state.
def strip(data):
"""Returns a 'files' entry with only the whitelisted keys."""
return dict((k, data[k]) for k in ('h', 'l', 'm', 's') if k in data)
out = {
'algo': self.algo_name,
'files': dict(
(filepath, strip(data)) for filepath, data in self.files.iteritems()),
# The version of the .state file is different than the one of the
# .isolated file.
'version': isolated_format.ISOLATED_FILE_VERSION,
out['read_only'] = self.read_only if self.read_only is not None else 1
if self.command:
out['command'] = self.command
if self.relative_cwd:
# Only set relative_cwd if a command was also specified. This reduce the
# noise for Swarming tasks where the command is specified as part of the
# Swarming task request and not through the isolated file.
out['relative_cwd'] = self.relative_cwd
return out
def isolate_filepath(self):
"""Returns the absolute path of self.isolate_file."""
return os.path.normpath(
os.path.join(self.isolated_basedir, self.isolate_file))
# Arguments number differs from overridden method
def load(cls, data, algo_name, isolated_basedir): # pylint: disable=W0221
"""Special case loading to disallow different OS.
It is not possible to load a .isolated.state files from a different OS, this
file is saved in OS-specific format.
out = super(SavedState, cls).load(data, algo_name, isolated_basedir)
if data.get('OS') != sys.platform:
raise isolated_format.IsolatedError('Unexpected OS %s' % data.get('OS'))
# pylint: disable=no-member
# Converts human readable form back into the proper class type.
algo = data.get('algo')
if not algo in isolated_format.SUPPORTED_ALGOS:
raise isolated_format.IsolatedError('Unknown algo \'%s\'' % out.algo)
if algo_name != algo:
# pylint: disable=no-member
raise isolated_format.IsolatedError(
'Unexpected algo \'%s\', expected %s' % (out.algo, algo_name))
out.algo = isolated_format.SUPPORTED_ALGOS[algo]
# Refuse the load non-exact version, even minor difference. This is unlike
# isolateserver.load_isolated(). This is because .isolated.state could have
# changed significantly even in minor version difference.
if out.version != cls.EXPECTED_VERSION:
raise isolated_format.IsolatedError(
'Unsupported version \'%s\'' % out.version)
# The .isolate file must be valid. If it is not present anymore, zap the
# value as if it was not noted, so .isolate_file can safely be overridden
# later.
if out.isolate_file and not fs.isfile(out.isolate_filepath):
out.isolate_file = None
if out.isolate_file:
# It could be absolute on Windows if the drive containing the .isolate and
# the drive containing the .isolated files differ, .e.g .isolate is on
# C:\\ and .isolated is on D:\\ .
assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
assert fs.isfile(out.isolate_filepath), out.isolate_filepath
return out
def flatten(self):
"""Makes sure 'algo' is in human readable form."""
out = super(SavedState, self).flatten()
out['algo'] = isolated_format.SUPPORTED_ALGOS_REVERSE[out['algo']]
return out
def __str__(self):
def dict_to_str(d):
return ''.join('\n %s=%s' % (k, d[k]) for k in sorted(d))
out = '%s(\n' % self.__class__.__name__
out += ' command: %s\n' % self.command
out += ' files: %d\n' % len(self.files)
out += ' isolate_file: %s\n' % self.isolate_file
out += ' read_only: %s\n' % self.read_only
out += ' relative_cwd: %s\n' % self.relative_cwd
out += ' child_isolated_files: %s\n' % self.child_isolated_files
out += ' path_variables: %s\n' % dict_to_str(self.path_variables)
out += ' config_variables: %s\n' % dict_to_str(self.config_variables)
out += ' extra_variables: %s\n' % dict_to_str(self.extra_variables)
return out
class CompleteState(object):
"""Contains all the state to run the task at hand."""
def __init__(self, isolated_filepath, saved_state):
super(CompleteState, self).__init__()
assert isolated_filepath is None or os.path.isabs(isolated_filepath)
self.isolated_filepath = isolated_filepath
# Contains the data to ease developer's use-case but that is not strictly
# necessary.
self.saved_state = saved_state
def load_files(cls, algo_name, isolated_filepath):
"""Loads state from disk."""
assert os.path.isabs(isolated_filepath), isolated_filepath
isolated_basedir = os.path.dirname(isolated_filepath)
s = SavedState.load_file(
isolatedfile_to_state(isolated_filepath), algo_name, isolated_basedir)
return cls(isolated_filepath, s)
def load_isolate(
self, cwd, isolate_file, path_variables, config_variables,
extra_variables, blacklist, ignore_broken_items, collapse_symlinks):
"""Updates self.isolated and self.saved_state with information loaded from a
.isolate file.
Processes the loaded data, deduce root_dir, relative_cwd.
# Make sure to not depend on os.getcwd().
assert os.path.isabs(isolate_file), isolate_file
isolate_file = file_path.get_native_path_case(isolate_file)
'CompleteState.load_isolate(%s, %s, %s, %s, %s, %s, %s)',
cwd, isolate_file, path_variables, config_variables, extra_variables,
ignore_broken_items, collapse_symlinks)
# Config variables are not affected by the paths and must be used to
# retrieve the paths, so update them first.
with, 'r') as f:
# At that point, variables are not replaced yet in command and infiles.
# infiles may contain directory entries and is in posix style.
command, infiles, read_only, isolate_cmd_dir = (
# Processes the variables with the new found relative root. Note that 'cwd'
# is used when path variables are used.
path_variables = normalize_path_variables(
cwd, path_variables, isolate_cmd_dir)
# Update the rest of the saved state.
self.saved_state.update(isolate_file, path_variables, extra_variables)
total_variables = self.saved_state.path_variables.copy()
command = [
isolate_format.eval_variables(i, total_variables) for i in command
total_variables = self.saved_state.path_variables.copy()
infiles = [
isolate_format.eval_variables(f, total_variables) for f in infiles
# root_dir is automatically determined by the deepest root accessed with the
# form '../../foo/bar'. Note that path variables must be taken in account
# too, add them as if they were input files.
self.saved_state.root_dir = isolate_format.determine_root_dir(
isolate_cmd_dir, infiles + self.saved_state.path_variables.values())
# The relative directory is automatically determined by the relative path
# between root_dir and the directory containing the .isolate file,
# isolate_base_dir.
relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
# Now that we know where the root is, check that the path_variables point
# inside it.
for k, v in self.saved_state.path_variables.iteritems():
dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
if not file_path.path_starts_with(self.saved_state.root_dir, dest):
raise isolated_format.MappingError(
'Path variable %s=%r points outside the inferred root directory '
'%s; %s'
% (k, v, self.saved_state.root_dir, dest))
# Normalize the files based to self.saved_state.root_dir. It is important to
# keep the trailing os.path.sep at that step.
infiles = [
file_path.normpath(os.path.join(isolate_cmd_dir, f)),
for f in infiles
follow_symlinks = False
if not collapse_symlinks:
follow_symlinks = sys.platform != 'win32'
# Expand the directories by listing each file inside. Up to now, trailing
# os.path.sep must be kept.
infiles = _expand_directories_and_symlinks(
# Finally, update the new data to be able to generate the foo.isolated file,
# the file that is used by
self.saved_state.update_isolated(command, infiles, read_only, relative_cwd)
def files_to_metadata(self, subdir, collapse_symlinks):
"""Updates self.saved_state.files with the files' mode and hash.
If |subdir| is specified, filters to a subdirectory. The resulting .isolated
file is tainted.
See isolated_format.file_to_metadata() for more information.
for infile in sorted(self.saved_state.files):
if subdir and not infile.startswith(subdir):
filepath = os.path.join(self.root_dir, infile)
# This code used to try to reuse the previous data if possible in
# saved_state. This performance optimization is not done anymore. This
# code is going away soon and shouldn't be used in new code.
meta = isolated_format.file_to_metadata(
if 'l' not in meta:
meta['h'] = isolated_format.hash_file(filepath, self.saved_state.algo)
self.saved_state.files[infile] = meta
def save_files(self):
"""Saves self.saved_state and creates a .isolated file."""
logging.debug('Dumping to %s' % self.isolated_filepath)
self.saved_state.child_isolated_files = chromium_save_isolated(
total_bytes = sum(
i.get('s', 0) for i in self.saved_state.files.itervalues())
if total_bytes:
# TODO(maruel): Stats are missing the .isolated files.
logging.debug('Total size: %d bytes' % total_bytes)
saved_state_file = isolatedfile_to_state(self.isolated_filepath)
logging.debug('Dumping to %s' % saved_state_file)
tools.write_json(saved_state_file, self.saved_state.flatten(), True)
def root_dir(self):
return self.saved_state.root_dir
def __str__(self):
def indent(data, indent_length):
"""Indents text."""
spacing = ' ' * indent_length
return ''.join(spacing + l for l in str(data).splitlines(True))
out = '%s(\n' % self.__class__.__name__
out += ' root_dir: %s\n' % self.root_dir
out += ' saved_state: %s)' % indent(self.saved_state, 2)
return out
def load_complete_state(options, cwd, subdir, skip_update):
"""Loads a CompleteState.
This includes data from .isolate and .isolated.state files. Never reads the
.isolated file.
options: Options instance generated with process_isolate_options. For either
options.isolate and options.isolated, if the value is set, it is an
absolute path.
cwd: base directory to be used when loading the .isolate file.
subdir: optional argument to only process file in the subdirectory, relative
to CompleteState.root_dir.
skip_update: Skip trying to load the .isolate file and processing the
dependencies. It is useful when not needed, like when tracing.
assert not options.isolate or os.path.isabs(options.isolate)
assert not options.isolated or os.path.isabs(options.isolated)
cwd = file_path.get_native_path_case(unicode(cwd))
# maruel: This is incorrect but it's not worth fixing.
namespace = getattr(options, 'namespace', 'default')
algo_name = isolate_storage.ServerRef('', namespace).hash_algo_name
if options.isolated:
# Load the previous state if it was present. Namely, "foo.isolated.state".
# Note: this call doesn't load the .isolate file.
complete_state = CompleteState.load_files(algo_name, options.isolated)
# Constructs a dummy object that cannot be saved. Useful for temporary
# commands like 'run'. There is no directory containing a .isolated file so
# specify the current working directory as a valid directory.
complete_state = CompleteState(None, SavedState(algo_name, os.getcwd()))
if not options.isolate:
if not complete_state.saved_state.isolate_file:
if not skip_update:
raise ExecutionError('A .isolate file is required.')
isolate = None
isolate = complete_state.saved_state.isolate_filepath
isolate = options.isolate
if complete_state.saved_state.isolate_file:
rel_isolate = file_path.safe_relpath(
options.isolate, complete_state.saved_state.isolated_basedir)
if rel_isolate != complete_state.saved_state.isolate_file:
# This happens if the .isolate file was moved for example. In this case,
# discard the saved state.
'--isolated %s != %s as saved in %s. Discarding saved state',
complete_state = CompleteState(
SavedState(algo_name, complete_state.saved_state.isolated_basedir))
if not skip_update:
# Then load the .isolate and expands directories.
cwd, isolate, options.path_variables, options.config_variables,
options.extra_variables, options.blacklist, options.ignore_broken_items,
# Regenerate complete_state.saved_state.files.
if subdir:
subdir = unicode(subdir)
# This is tricky here. If it is a path, take it from the root_dir. If
# it is a variable, it must be keyed from the directory containing the
# .isolate file. So translate all variables first.
translated_path_variables = dict(
for k, v in complete_state.saved_state.path_variables.iteritems())
subdir = isolate_format.eval_variables(subdir, translated_path_variables)
subdir = subdir.replace('/', os.path.sep)
if not skip_update:
complete_state.files_to_metadata(subdir, options.collapse_symlinks)
return complete_state
def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
"""Creates a isolated tree usable for test execution.
Returns the current working directory where the isolated command should be
started in.
# Forcibly copy when the tree has to be read only. Otherwise the inode is
# modified, and this cause real problems because the user's source tree
# becomes read only. On the other hand, the cost of doing file copy is huge.
if read_only not in (0, None):
action = file_path.COPY
action = file_path.HARDLINK_WITH_FALLBACK
cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
# cwd may not exist when no files are mapped from the directory containing the
# .isolate file. But the directory must exist to be the current working
# directory.
run_isolated.change_tree_read_only(outdir, read_only)
return cwd
def prepare_for_archival(options, cwd):
"""Loads the isolated file and create |infiles| for archival."""
complete_state = load_complete_state(
options, cwd, options.subdir, False)
# Make sure that complete_state isn't modified until save_files() is
# called, because any changes made to it here will propagate to the files
# created (which is probably not intended).
infiles = complete_state.saved_state.files
# Add all the .isolated files.
isolated_hash = []
isolated_files = [
] + complete_state.saved_state.child_isolated_files
for item in isolated_files:
item_path = os.path.join(
os.path.dirname(complete_state.isolated_filepath), item)
# Do not use isolated_format.hash_file() here because the file is
# likely smallish (under 500kb) and its file size is needed.
with, 'rb') as f:
content =
isolated_metadata = {
'h': isolated_hash[-1],
's': len(content),
'priority': '0'
infiles[item_path] = isolated_metadata
return complete_state, infiles, isolated_hash
def _process_infiles(infiles):
"""Converts |infiles| into FileItem ready to upload.
Convert |infiles| into a list of FileItem objects, skip duplicates.
infiles: iterable of pairs (absolute path, metadata dict) of files.
isolateserver.FileItem() items.
# Filter out symlinks, since they are not represented by items on isolate
# server side.
seen = set()
skipped = 0
for filepath, metadata in infiles:
if 'l' not in metadata and filepath not in seen:
yield isolateserver.FileItem(
high_priority=metadata.get('priority') == '0')
skipped += 1'Skipped %d duplicated entries', skipped)
def _expand_directories_and_symlinks(
indir, infiles, blacklist, follow_symlinks, ignore_broken_items):
"""Expands the directories and the symlinks, applies the blacklist and
verifies files exist.
Files are specified in os native path separator.
list of relative file path of each file inside every directory specified.
# The calling code is not smart enough to leverage the generator, so
# materialize the list.
# Since we want to get rid of this tool, it's not a big deal that it's not
# optimal.
out = []
for relfile in infiles:
# Ignore the symlink hint, this code will be eventually deleted so it is
# not worth optimizing.
relpath for relpath, _is_symlink
in isolated_format.expand_directory_and_symlink(
indir, relfile, blacklist, follow_symlinks))
except isolated_format.MappingError as e:
if not ignore_broken_items:
raise'warning: %s', e)
return out
def isolate_and_archive(trees, server_ref):
"""Isolates and uploads a bunch of isolated trees.
trees: list of pairs (Options, working directory) that describe what tree
to isolate. Options are processed by 'process_isolate_options'.
server_ref: isolate_storage.ServerRef instance.
Returns a dict {target name -> isolate hash or None}, where target name is
a name of *.isolated file without an extension (e.g. 'base_unittests').
Have multiple failure modes:
* If the upload fails due to server or network error returns None.
* If some *.isolate file is incorrect (but rest of them are fine and were
successfully uploaded), returns a dict where the value of the entry
corresponding to invalid *.isolate file is None.
if not trees:
return {}
# Helper generator to avoid materializing the full (huge) list of files until
# the very end (in upload_items()).
def emit_files(root_dir, files):
for path, meta in files.iteritems():
yield (os.path.join(root_dir, path), meta)
# Process all *.isolate files, it involves parsing, file system traversal and
# hashing. The result is a list of generators that produce files to upload
# and the mapping {target name -> hash of *.isolated file} to return from
# this function.
files_generators = []
isolated_hashes = {}
with tools.Profiler('Isolate'):
for opts, cwd in trees:
target_name = os.path.splitext(os.path.basename(opts.isolated))[0]
complete_state, files, isolated_hash = prepare_for_archival(opts, cwd)
files_generators.append(emit_files(complete_state.root_dir, files))
isolated_hashes[target_name] = isolated_hash[0]
print('%s %s' % (isolated_hash[0], target_name))
except Exception:
logging.exception('Exception when isolating %s', target_name)
isolated_hashes[target_name] = None
# All bad? Nothing to upload.
if all(v is None for v in isolated_hashes.itervalues()):
return isolated_hashes
# Now upload all necessary files at once.
with tools.Profiler('Upload'):
items = _process_infiles(itertools.chain(*files_generators))
with isolateserver.get_storage(server_ref) as storage:
except Exception:
logging.exception('Exception while uploading files')
return None
return isolated_hashes
def parse_archive_command_line(args, cwd):
"""Given list of arguments for 'archive' command returns parsed options.
Used by CMDbatcharchive to parse options passed via JSON. See also CMDarchive.
parser = optparse.OptionParser()
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
process_isolate_options(parser, options, cwd)
return options
### Commands.
def CMDarchive(parser, args):
"""Creates a .isolated file and uploads the tree to an isolate server.
All the files listed in the .isolated file are put in the isolate server
cache via
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
process_isolate_options(parser, options)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True, True)
server_ref = isolate_storage.ServerRef(
options.isolate_server, options.namespace)
result = isolate_and_archive([(options, unicode(os.getcwd()))], server_ref)
if result is None:
assert len(result) == 1, result
if result.values()[0] is None:
return 0
@subcommand.usage('-- GEN_JSON_1 GEN_JSON_2 ...')
def CMDbatcharchive(parser, args):
"""Archives multiple isolated trees at once.
Using single command instead of multiple sequential invocations allows to cut
redundant work when isolated trees share common files (e.g. file hashes are
checked only once, their presence on the server is checked only once, and
so on).
Takes a list of paths to *.isolated.gen.json files that describe what trees to
isolate. Format of files is:
"version": 1,
"dir": <absolute path to a directory all other paths are relative to>,
"args": [list of command line arguments for single 'archive' command]
help='Write isolated hashes of archived trees to this file as JSON')
options, args = parser.parse_args(args)
auth.process_auth_options(parser, options)
isolateserver.process_isolate_server_options(parser, options, True, True)
# Validate all incoming options, prepare what needs to be archived as a list
# of tuples (archival options, working directory).
work_units = []
for gen_json_path in args:
# Validate JSON format of a *.isolated.gen.json file.
data = tools.read_json(gen_json_path)
except IOError as e:
parser.error('Failed to open %s: %s' % (gen_json_path, e))
if data.get('version') != ISOLATED_GEN_JSON_VERSION:
parser.error('Invalid version in %s' % gen_json_path)
cwd = data.get('dir')
if not isinstance(cwd, unicode) or not fs.isdir(cwd):
parser.error('Invalid dir in %s' % gen_json_path)
args = data.get('args')
if (not isinstance(args, list) or
not all(isinstance(x, unicode) for x in args)):
parser.error('Invalid args in %s' % gen_json_path)
# Convert command line (embedded in JSON) to Options object.
work_units.append((parse_archive_command_line(args, cwd), cwd))
# Perform the archival, all at once.
server_ref = isolate_storage.ServerRef(
options.isolate_server, options.namespace)
isolated_hashes = isolate_and_archive(work_units, server_ref)
# TODO(vadimsh): isolate_and_archive returns None on upload failure, there's
# no way currently to figure out what *.isolated file from a batch were
# successfully uploaded, so consider them all failed (and emit empty dict
# as JSON result).
if options.dump_json:
tools.write_json(options.dump_json, isolated_hashes or {}, False)
if isolated_hashes is None:
# isolated_hashes[x] is None if 'x.isolate' contains a error.
if not all(isolated_hashes.itervalues()):
return 0
def CMDcheck(parser, args):
"""Checks that all the inputs are present and generates .isolated."""
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
process_isolate_options(parser, options)
complete_state = load_complete_state(
options, os.getcwd(), options.subdir, False)
# Nothing is done specifically. Just store the result and state.
return 0
def CMDremap(parser, args):
"""Creates a directory with all the dependencies mapped into it.
Useful to test manually why a test is failing. The target executable is not
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
cwd = os.getcwd()
process_isolate_options(parser, options, cwd, require_isolated=False)
process_outdir_options(parser, options, cwd)
complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
print('Remapping into %s' % options.outdir)
if fs.listdir(options.outdir):
raise ExecutionError('Can\'t remap in a non-empty directory')
options.outdir, complete_state.root_dir, complete_state.saved_state.files,
if complete_state.isolated_filepath:
return 0
@subcommand.usage('-- [extra arguments]')
def CMDrun(parser, args):
"""Runs the test executable in an isolated (temporary) directory.
All the dependencies are mapped into the temporary directory and the
directory is cleaned up after the target exits.
Argument processing stops at -- and these arguments are appended to the
command line of the target to run. For example, use: run --isolated foo.isolated -- --gtest_filter=Foo.Bar
options, args = parser.parse_args(args)
process_isolate_options(parser, options, require_isolated=False)
complete_state = load_complete_state(
options, os.getcwd(), None, options.skip_refresh)
cmd = complete_state.saved_state.command + args
if not cmd:
raise ExecutionError('No command to run.')
cmd = tools.fix_python_cmd(cmd)
outdir = run_isolated.make_temp_dir(
u'isolate-%s' %,
# TODO(maruel): Use run_isolated.run_tha_test().
cwd = create_isolate_tree(
outdir, complete_state.root_dir, complete_state.saved_state.files,
file_path.ensure_command_has_abs_path(cmd, cwd)'Running %s, cwd=%s' % (cmd, cwd))
result =, cwd=cwd)
except OSError:
'Failed to executed the command; executable is missing, maybe you\n'
'forgot to map it in the .isolate file?\n %s\n in %s\n' %
(' '.join(cmd), cwd))
result = 1
if complete_state.isolated_filepath:
return result
def _process_variable_arg(option, opt, _value, parser):
"""Called by OptionParser to process a --<foo>-variable argument."""
if not parser.rargs:
raise optparse.OptionValueError(
'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
k = parser.rargs.pop(0)
variables = getattr(parser.values, option.dest)
if '=' in k:
k, v = k.split('=', 1)
if not parser.rargs:
raise optparse.OptionValueError(
'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
v = parser.rargs.pop(0)
if not re.match('^' + isolate_format.VALID_VARIABLE + '$', k):
raise optparse.OptionValueError(
'Variable \'%s\' doesn\'t respect format \'%s\'' %
(k, isolate_format.VALID_VARIABLE))
variables.append((k, v.decode('utf-8')))
def add_variable_option(parser):
"""Adds --isolated and --<foo>-variable to an OptionParser."""
'-s', '--isolated',
help='.isolated file to generate or read')
# Keep for compatibility. TODO(maruel): Remove once not used anymore.
'-r', '--result',
is_win = sys.platform in ('win32', 'cygwin')
# There is really 3 kind of variables:
# - path variables, like DEPTH or PRODUCT_DIR that should be
# replaced opportunistically when tracing tests.
# - extraneous things like EXECUTABE_SUFFIX.
# - configuration variables that are to be used in deducing the matrix to
# reduce.
# - unrelated variables that are used as command flags for example.
metavar='FOO BAR',
help='Config variables are used to determine which conditions should be '
'matched when loading a .isolate file, default: %default. '
'All 3 kinds of variables are persistent accross calls, they are '
'saved inside <.isolated>.state')
metavar='FOO BAR',
help='Path variables are used to replace file paths when loading a '
'.isolate file, default: %default')
default=[('EXECUTABLE_SUFFIX', '.exe' if is_win else '')],
metavar='FOO BAR',
help='Extraneous variables are replaced on the \'command\' entry and on '
'paths in the .isolate file but are not considered relative paths.')
def add_isolate_options(parser):
"""Adds --isolate, --isolated, --out and --<foo>-variable options."""
group = optparse.OptionGroup(parser, 'Common options')
'-i', '--isolate',
help='.isolate file to load the dependency data from')
'--ignore_broken_items', action='store_true',
help='Indicates that invalid entries in the isolated file to be '
'only be logged and not stop processing. Defaults to True if '
'-L', '--collapse_symlinks', action='store_true',
help='Treat any symlinks as if they were the normal underlying file')
def add_subdir_option(parser):
help='Filters to a subdirectory. Its behavior changes depending if it '
'is a relative path as a string or as a path variable. Path '
'variables are always keyed from the directory containing the '
'.isolate file. Anything else is keyed on the root directory.')
def add_skip_refresh_option(parser):
'--skip-refresh', action='store_true',
help='Skip reading .isolate file and do not refresh the hash of '
def add_outdir_options(parser):
"""Adds --outdir, which is orthogonal to --isolate-server.
Note: On upload, separate commands are used between 'archive' and 'hashtable'.
On 'download', the same command can download from either an isolate server or
a file system.
'-o', '--outdir', metavar='DIR',
help='Directory used to recreate the tree.')
def process_outdir_options(parser, options, cwd):
if not options.outdir:
parser.error('--outdir is required.')
if file_path.is_url(options.outdir):
parser.error('Can\'t use an URL for --outdir.')
options.outdir = unicode(options.outdir).replace('/', os.path.sep)
# outdir doesn't need native path case since tracing is never done from there.
options.outdir = os.path.abspath(
os.path.normpath(os.path.join(cwd, options.outdir)))
# In theory, we'd create the directory outdir right away. Defer doing it in
# case there's errors in the command line.
def process_isolate_options(parser, options, cwd=None, require_isolated=True):
"""Handles options added with 'add_isolate_options'.
Mutates |options| in place, by normalizing path to isolate file, values of
variables, etc.
cwd = file_path.get_native_path_case(unicode(cwd or os.getcwd()))
# Parse --isolated option.
if options.isolated:
options.isolated = os.path.abspath(
os.path.join(cwd, unicode(options.isolated).replace('/', os.path.sep)))
if require_isolated and not options.isolated:
parser.error('--isolated is required.')
if options.isolated and not options.isolated.endswith('.isolated'):
parser.error('--isolated value must end with \'.isolated\'')
# Processes all the --<foo>-variable flags.
def try_make_int(s):
"""Converts a value to int if possible, converts to unicode otherwise."""
return int(s)
except ValueError:
return s.decode('utf-8')
options.config_variables = dict(
(k, try_make_int(v)) for k, v in options.config_variables)
options.path_variables = dict(options.path_variables)
options.extra_variables = dict(options.extra_variables)
# Account for default EXECUTABLE_SUFFIX.
if (options.config_variables or options.path_variables or
len(options.extra_variables) > 1):
# Normalize the path in --isolate.
if options.isolate:
# TODO(maruel): Work with non-ASCII.
# The path must be in native path case for tracing purposes.
options.isolate = unicode(options.isolate).replace('/', os.path.sep)
options.isolate = os.path.abspath(os.path.join(cwd, options.isolate))
options.isolate = file_path.get_native_path_case(options.isolate)
def main(argv):
dispatcher = subcommand.CommandDispatcher(__name__)
parser = logging_utils.OptionParserWithLogging(
version=__version__, verbose=int(os.environ.get('ISOLATE_DEBUG', 0)))
return dispatcher.execute(parser, argv)
except isolated_format.MappingError as e:
print >> sys.stderr, 'Failed to find an input file: %s' % e
return 1
except ExecutionError as e:
print >> sys.stderr, 'Execution failure: %s' % e
return 1
if __name__ == '__main__':