| # Copyright 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import re |
| import sys |
| |
| # pylint: disable=relative-import |
| import manual_bisect_files |
| from recipe_engine import recipe_api |
| |
| |
| # Regular expression to identify a Git hash. |
| GIT_COMMIT_HASH_RE = re.compile(r'[a-zA-Z0-9]{40}') |
| # The Google Storage metadata key for the full commit position. |
| GS_COMMIT_POSITION_KEY = 'Cr-Commit-Position' |
| # The Google Storage metadata key for the commit position number. |
| GS_COMMIT_POSITION_NUMBER_KEY = 'Cr-Commit-Position-Number' |
| # The Google Storage metadata key for the Git commit hash. |
| GS_GIT_COMMIT_KEY = 'Cr-Git-Commit' |
| |
| |
| class ArchiveApi(recipe_api.RecipeApi): |
| """Chromium specific module for zipping, uploading and downloading build |
| artifacts implemented as a wrapper around zip_build.py script. |
| |
| If you need to upload or download build artifacts (or any other files) for |
| something other than Chromium flavor, consider using 'zip' + 'gsutil' or |
| 'isolate' modules instead. |
| """ |
| def zip_and_upload_build( |
| self, step_name, target, build_url=None, src_dir=None, |
| build_revision=None, cros_board=None, package_dsym_files=False, |
| exclude_files=None, exclude_perf_test_files=False, |
| update_properties=None, store_by_hash=True, |
| platform=None, **kwargs): |
| """Returns a step invoking zip_build.py to zip up a Chromium build. |
| If build_url is specified, also uploads the build.""" |
| if not src_dir: |
| src_dir = self.m.path['checkout'] |
| args = [ |
| '--target', target, |
| '--gsutil-py-path', self.m.depot_tools.gsutil_py_path, |
| '--staging-dir', self.m.path['cache'].join('chrome_staging'), |
| '--src-dir', src_dir, |
| ] |
| args += self.m.build.slave_utils_args |
| if 'build_archive_url' in self.m.properties: |
| args.extend(['--use-build-url-name', '--build-url', |
| self.m.properties['build_archive_url']]) |
| elif build_url: |
| args.extend(['--build-url', build_url]) |
| if build_revision: |
| args.extend(['--build_revision', build_revision]) |
| if cros_board: |
| args.extend(['--cros-board', cros_board]) |
| if package_dsym_files: |
| args.append('--package-dsym-files') |
| if exclude_files: |
| args.extend(['--exclude-files', exclude_files]) |
| if 'gs_acl' in self.m.properties: |
| args.extend(['--gs-acl', self.m.properties['gs_acl']]) |
| if exclude_perf_test_files and platform: |
| include_bisect_file_list = ( |
| manual_bisect_files.CHROME_REQUIRED_FILES.get(platform)) |
| include_bisect_strip_list = ( |
| manual_bisect_files.CHROME_STRIP_LIST.get(platform)) |
| include_bisect_whitelist = ( |
| manual_bisect_files.CHROME_WHITELIST_FILES.get(platform)) |
| if include_bisect_file_list: |
| inclusions = ','.join(include_bisect_file_list) |
| args.extend(['--include-files', inclusions]) |
| if include_bisect_strip_list: |
| strip_files = ','.join(include_bisect_strip_list) |
| args.extend(['--strip-files', strip_files]) |
| if include_bisect_whitelist: |
| args.extend(['--whitelist', include_bisect_whitelist]) |
| args.extend(['--exclude-extra']) |
| |
| # If update_properties is passed in and store_by_hash is False, |
| # we store it with commit position number instead of a hash |
| if update_properties and not store_by_hash: |
| commit_position = self._get_commit_position( |
| update_properties, None) |
| _, cp_number = self.m.commit_position.parse(commit_position) |
| args.extend(['--build_revision', cp_number]) |
| |
| properties_json = self.m.json.dumps(self.m.properties.legacy()) |
| args.extend(['--factory-properties', properties_json, |
| '--build-properties', properties_json]) |
| |
| kwargs['allow_subannotations'] = True |
| return self.m.build.python( |
| step_name, |
| self.repo_resource('scripts', 'slave', 'zip_build.py'), |
| args, |
| infra_step=True, |
| **kwargs |
| ) |
| |
| def _get_commit_position(self, update_properties, primary_project): |
| """Returns the commit position of the project (or the specified primary |
| project). |
| """ |
| if primary_project: |
| key = 'got_%s_revision_cp' % primary_project |
| else: |
| key = 'got_revision_cp' |
| return update_properties.get(key, update_properties.get( |
| 'got_src_revision_cp')) |
| |
| def _get_git_commit(self, update_properties, primary_project): |
| """Returns: (str/None) the git commit hash for a given project. |
| |
| Attempts to identify the git commit hash for a given project. If |
| 'primary_project' is None, or if there is no git commit hash for the |
| specified primary project, the checkout-wide commit hash will be used. |
| |
| If none of the candidate configurations are present, the value None will be |
| returned. |
| """ |
| if primary_project: |
| commit = update_properties.get('got_%s_revision' % primary_project) |
| if commit: |
| assert GIT_COMMIT_HASH_RE.match(commit), commit |
| return commit |
| |
| commit = update_properties.get('got_revision') |
| assert GIT_COMMIT_HASH_RE.match(commit), commit |
| return commit |
| |
| def _get_comparable_upload_path_for_sort_key(self, branch, number): |
| """Returns a sortable string corresponding to the commit position.""" |
| if branch and branch != 'refs/heads/master': |
| branch = branch.replace('/', '_') |
| return '%s-%s' % (branch, number) |
| return str(number) |
| |
| def clusterfuzz_archive( |
| self, build_dir, update_properties, gs_bucket, |
| archive_prefix, archive_subdir_suffix='', gs_acl=None, |
| revision_dir=None, primary_project=None, bitness=None, use_legacy=True, |
| **kwargs): |
| # TODO(machenbach): Merge revision_dir and primary_project. The |
| # revision_dir is only used for building the archive name while the |
| # primary_project is authoritative for the commit position. |
| """Archives and uploads a build to google storage. |
| |
| The build is filtered by a list of file exclusions and then zipped. It is |
| uploaded to google storage with some metadata about the commit position |
| and revision attached. The zip file follows the naming pattern used by |
| clusterfuzz. The file pattern is: |
| <archive name>-<platform>-<target><optional component>-<sort-key>.zip |
| |
| If the build is experimental, -experimental is appended in the name. |
| |
| Example: cool-project-linux-release-refs_heads_b1-12345.zip |
| The archive name is "cool-project" and there's no component build. The |
| commit is on a branch called b1 at commit position number 12345. |
| |
| Example: cool-project-mac-debug-x10-component-234.zip |
| The archive name is "cool-project" and the component's name is "x10". The |
| component is checked out in branch master with commit position number 234. |
| |
| Args: |
| build_dir: The absolute path to the build output directory, e.g. |
| [slave-build]/src/out/Release |
| update_properties: The properties from the bot_update step (containing |
| commit information) |
| gs_bucket: Name of the google storage bucket to upload to |
| archive_prefix: Prefix of the archive zip file |
| archive_subdir_suffix: Optional suffix to the google storage subdirectory |
| name that contains the archive files |
| gs_acl: ACL used for the file on google storage |
| revision_dir: Optional component name if the main revision for this |
| archive is a component revision |
| primary_project: Optional project name for specifying the revision of the |
| checkout |
| bitness: The bitness of the build (32 or 64) to distinguish archive |
| names. |
| use_legacy: Specify if legacy paths and archive names should be used. Set |
| to false for new builders. |
| """ |
| # We should distinguish build archives also by bitness on new bots, so that |
| # 32 and 64 bit bots can coexist. We don't change old bots to not confuse |
| # clusterfuzz bisect jobs. |
| assert use_legacy or bitness, 'Must specify bitness for new builders.' |
| target = self.m.path.split(build_dir)[-1] |
| commit_position = self._get_commit_position( |
| update_properties, primary_project) |
| cp_branch, cp_number = self.m.commit_position.parse(commit_position) |
| build_git_commit = self._get_git_commit(update_properties, primary_project) |
| staging_dir = self.m.path['cleanup'].join('chrome_staging') |
| self.m.file.ensure_directory('create staging_dir', staging_dir) |
| |
| llvm_tools_to_copy = ['llvm-symbolizer', 'sancov'] |
| llvm_bin_dir = self.m.path['checkout'].join('third_party', 'llvm-build', |
| 'Release+Asserts', 'bin') |
| ext = '.exe' if self.m.platform.is_win else '' |
| |
| for tool in llvm_tools_to_copy: |
| tool_src = self.m.path.join(llvm_bin_dir, tool + ext) |
| tool_dst = self.m.path.join(build_dir, tool + ext) |
| |
| if not self.m.path.exists(tool_src): |
| continue |
| |
| try: |
| self.m.file.copy('Copy ' + tool, tool_src, tool_dst) |
| except self.m.step.StepFailure: # pragma: no cover |
| # On some builds, it appears that a soft/hard link of llvm-symbolizer |
| # exists in the build directory, which causes shutil.copy to raise an |
| # exception. Either way, this shouldn't cause the whole build to fail. |
| pass |
| |
| # Build the list of files to archive. |
| filter_result = self.m.python( |
| 'filter build_dir', |
| self.resource('filter_build_files.py'), |
| [ |
| '--dir', build_dir, |
| '--platform', self.m.platform.name, |
| '--output', self.m.json.output(), |
| ], |
| infra_step=True, |
| step_test_data=lambda: self.m.json.test_api.output(['file1', 'file2']), |
| **kwargs |
| ) |
| |
| zip_file_list = filter_result.json.output |
| |
| # Use the legacy platform name if specified as Clusterfuzz has some |
| # expectations on this (it only affects Windows, where it replace 'win' |
| # by 'win32'). |
| if use_legacy: |
| platform_name = self.legacy_platform_name() |
| target_name = target.lower() |
| else: |
| # Always qualify platform with bitness on new bots. E.g. linux32 or win64. |
| platform_name = self.m.platform.name + str(bitness) |
| # Split off redundant _x64 suffix on windows. The bitness is part of the |
| # platform. |
| target_name = target.lower().split('_')[0] |
| |
| pieces = [platform_name, target_name] |
| if archive_subdir_suffix: |
| pieces.append(archive_subdir_suffix) |
| subdir = '-'.join(pieces) |
| |
| # Components like v8 get a <name>-v8-component-<revision> infix. |
| component = '' |
| if revision_dir: |
| component = '-%s-component' % revision_dir |
| |
| sortkey_path = self._get_comparable_upload_path_for_sort_key( |
| cp_branch, cp_number) |
| zip_file_base_name = '%s-%s-%s%s-%s' % (archive_prefix, |
| platform_name, |
| target_name, |
| component, |
| sortkey_path) |
| if self.m.runtime.is_experimental: |
| zip_file_base_name += ('-experimental') |
| zip_file_name = '%s.zip' % zip_file_base_name |
| |
| self.m.build.python( |
| 'zipping', |
| self.resource('zip_archive.py'), |
| [ |
| staging_dir, |
| zip_file_base_name, |
| self.m.json.input(zip_file_list), |
| build_dir, |
| ], |
| infra_step=True, |
| **kwargs |
| ) |
| |
| zip_file = staging_dir.join(zip_file_name) |
| |
| gs_metadata = { |
| GS_COMMIT_POSITION_NUMBER_KEY: cp_number, |
| } |
| if commit_position: |
| gs_metadata[GS_COMMIT_POSITION_KEY] = commit_position |
| if build_git_commit: |
| gs_metadata[GS_GIT_COMMIT_KEY] = build_git_commit |
| |
| gs_args = [] |
| if gs_acl: |
| gs_args.extend(['-a', gs_acl]) |
| self.m.gsutil.upload( |
| zip_file, |
| gs_bucket, |
| "/".join([subdir, zip_file_name]), |
| args=gs_args, |
| metadata=gs_metadata, |
| use_retry_wrapper=False, |
| ) |
| self.m.file.remove(zip_file_name, zip_file) |
| |
| def download_and_unzip_build( |
| self, step_name, target, build_url, src_dir=None, |
| build_revision=None, build_archive_url=None, **kwargs): |
| """Returns a step invoking extract_build.py to download and unzip |
| a Chromium build.""" |
| if not src_dir: |
| src_dir = self.m.path['checkout'] |
| args = [ |
| '--gsutil-py-path', self.m.depot_tools.gsutil_py_path, |
| '--target', target, |
| '--src-dir', src_dir, |
| ] |
| args += self.m.build.slave_utils_args |
| if build_archive_url: |
| args.extend(['--build-archive-url', build_archive_url]) |
| else: |
| args.extend(['--build-url', build_url]) |
| if build_revision: |
| args.extend(['--build_revision', build_revision]) |
| |
| properties = ( |
| ('mastername', '--master-name'), |
| ('parent_builddir', '--parent-build-dir'), |
| ('parentname', '--parent-builder-name'), |
| ('parentslavename', '--parent-slave-name'), |
| ('webkit_dir', '--webkit-dir'), |
| ('revision_dir', '--revision-dir'), |
| ) |
| for property_name, switch_name in properties: |
| if self.m.properties.get(property_name): |
| args.extend([switch_name, self.m.properties[property_name]]) |
| |
| if self.m.properties.get('parent_buildnumber'): |
| args.extend([ |
| '--parent-build-number', |
| int(self.m.properties.get('parent_buildnumber')), |
| ]) |
| args.extend(['--build-number', self.m.buildbucket.build.number]) |
| |
| self.m.build.python( |
| step_name, |
| self.repo_resource('scripts', 'slave', 'extract_build.py'), |
| args, |
| infra_step=True, |
| **kwargs |
| ) |
| |
| # FIXME(machenbach): This is currently used by win64 builders as well, which |
| # have win32 in their archive names, which is confusing. |
| def legacy_platform_name(self): |
| """Replicates the behavior of PlatformName() in chromium_utils.py.""" |
| if self.m.platform.is_win: |
| return 'win32' |
| return self.m.platform.name |
| |
| def _legacy_url(self, is_download, gs_bucket_name, extra_url_components): |
| """Computes a build_url suitable for uploading a zipped Chromium |
| build to Google Storage. |
| |
| The reason this is named 'legacy' is that there are a large number |
| of dependencies on the exact form of this URL. The combination of |
| zip_build.py, extract_build.py, slave_utils.py, and runtest.py |
| require that: |
| |
| * The platform name be exactly one of 'win32', 'mac', or 'linux' |
| * The upload URL only name the directory on GS into which the |
| build goes (zip_build.py computes the name of the file) |
| * The download URL contain the unversioned name of the zip archive |
| * The revision on the builder and tester machines be exactly the |
| same |
| |
| There were too many dependencies to tease apart initially, so this |
| function simply emulates the form of the URL computed by the |
| underlying scripts. |
| |
| extra_url_components, if specified, should be a string without a |
| trailing '/' which is inserted in the middle of the URL. |
| |
| The builder_name, or parent_buildername, is always automatically |
| inserted into the URL. |
| |
| If build is running in experimental mode (see recipe_engine.runtime module), |
| then 'experimental/' is prepended to path inside bucket automatically. This |
| protects production builds from intererence from experimentation. |
| """ |
| |
| result = ('gs://' + gs_bucket_name) |
| if self.m.runtime.is_experimental: |
| result += ('/experimental') |
| if extra_url_components: |
| result += ('/' + extra_url_components) |
| if is_download: |
| result += ('/' + self.m.properties['parent_buildername'] + '/' + |
| 'full-build-' + self.legacy_platform_name() + |
| '.zip') |
| else: |
| result += '/' + self.m.buildbucket.builder_name |
| return result |
| |
| def legacy_upload_url(self, gs_bucket_name, extra_url_components=None): |
| """Returns a url suitable for uploading a Chromium build to Google |
| Storage. |
| |
| extra_url_components, if specified, should be a string without a |
| trailing '/' which is inserted in the middle of the URL. |
| |
| The builder_name, or parent_buildername, is always automatically |
| inserted into the URL.""" |
| return self._legacy_url(False, gs_bucket_name, extra_url_components) |
| |
| def legacy_download_url(self, gs_bucket_name, extra_url_components=None): |
| """Returns a url suitable for downloading a Chromium build from |
| Google Storage. |
| |
| extra_url_components, if specified, should be a string without a |
| trailing '/' which is inserted in the middle of the URL. |
| |
| The builder_name, or parent_buildername, is always automatically |
| inserted into the URL.""" |
| return self._legacy_url(True, gs_bucket_name, extra_url_components) |
