blob: 92fec5c5e0f936ea05f1ba79c9fa5b34373dd2b9 [file] [log] [blame]
# Copyright 2017 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""API for interacting with the buildbucket service.
Requires `buildbucket` command in `$PATH`:
https://godoc.org/go.chromium.org/luci/buildbucket/client/cmd/buildbucket
`url_title_fn` parameter used in this module is a function that accepts a
`build_pb2.Build` and returns a link title.
If it returns `None`, the link is not reported. Default link title is build id.
"""
from contextlib import contextmanager
from google import protobuf
from google.protobuf import field_mask_pb2
from google.protobuf import json_format
from recipe_engine import recipe_api
from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2
from PB.go.chromium.org.luci.buildbucket.proto \
import builds_service as builds_service_pb2
from . import util
class BuildbucketApi(recipe_api.RecipeApi):
"""A module for interacting with buildbucket."""
HOST_PROD = 'cr-buildbucket.appspot.com'
HOST_DEV = 'cr-buildbucket-dev.appspot.com'
# The Build message fields that will be requested by default in buildbucket
# rpc requests.
DEFAULT_FIELDS = frozenset({
'builder',
'create_time',
'created_by',
'critical',
'end_time',
'id',
'input',
'number',
'output',
'start_time',
'status',
'update_time',
})
# Sentinel to indicate that a child build launched by `schedule_request()`
# should use the same value as its parent for a specific attribute.
INHERIT = object()
def __init__(self, props, glob_props, *args, **kwargs):
super(BuildbucketApi, self).__init__(*args, **kwargs)
self._service_account_key = None
self._host = props.build.infra.buildbucket.hostname or self.HOST_PROD
self._runtime_tags = {}
self._build = build_pb2.Build()
if props.HasField('build'):
self._build = props.build
self._bucket_v1 = 'luci.%s.%s' % (
self._build.builder.project, self._build.builder.bucket)
else:
# Legacy mode.
self._bucket_v1 = None
self.build.number = int(glob_props.buildnumber or 0)
self.build.created_by = ''
_legacy_builder_id(
glob_props.mastername, glob_props.buildername, self._build.builder)
_legacy_input_gerrit_changes(
self._build.input.gerrit_changes, glob_props.patch_storage,
glob_props.patch_gerrit_url, glob_props.patch_project,
glob_props.patch_issue or glob_props.issue,
glob_props.patch_set)
_legacy_input_gitiles_commit(
self._build.input.gitiles_commit,
glob_props.revision or glob_props.parent_got_revision,
glob_props.branch)
self._next_test_build_id = 8922054662172514000
@property
def host(self):
"""Hostname of buildbucket to use in API calls.
Defaults to the hostname that the current build is originating from.
"""
return self._host
@host.setter
def host(self, value):
self._host = value
def set_buildbucket_host(self, host):
"""DEPRECATED. Use host property."""
self.host = host
@contextmanager
def with_host(self, host):
"""Set the buildbucket host while in context, then reverts it."""
previous_host = self.host
try:
self.host = host
yield
finally:
self.host = previous_host
def use_service_account_key(self, key_path):
"""Tells this module to start using given service account key for auth.
Otherwise the module is using the default account (when running on LUCI or
locally), or no auth at all (when running on Buildbot).
Exists mostly to support Buildbot environment. Recipe for LUCI environment
should not use this.
Args:
* key_path (str): a path to JSON file with service account credentials.
"""
self._service_account_key = key_path
@property
def build(self):
"""Returns current build as a `buildbucket.v2.Build` protobuf message.
For value format, see `Build` message in
[build.proto](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto).
DO NOT MODIFY the returned value.
Do not implement conditional logic on returned tags; they are for indexing.
Use returned `build.input` instead.
Pure Buildbot support: to simplify transition to buildbucket, returns a
message even if the current build is not a buildbucket build. Provides as
much information as possible. Some fields may be left empty, violating
the rules described in the .proto files.
If the current build is not a buildbucket build, returned `build.id` is 0.
"""
return self._build
@property
def builder_name(self):
"""Returns builder name. Shortcut for `.build.builder.builder`."""
return self.build.builder.builder
@property
def builder_realm(self):
"""Returns the LUCI realm name of the current build.
Raises `InfraFailure` if the build proto doesn't have `project` or `bucket`
set. This can happen in tests that don't properly mock build proto.
"""
if not self._build.builder.project:
raise self.m.step.InfraFailure('The build has no project')
if not self._build.builder.bucket: # pragma: no cover
raise self.m.step.InfraFailure('The build has no bucket')
return '%s:%s' % (self._build.builder.project, self._build.builder.bucket)
def build_url(self, host=None, build_id=None):
"""Returns url to a build. Defaults to current build."""
return 'https://%s/build/%s' % (
host or self._host, build_id or self._build.id)
@property
def gitiles_commit(self):
"""Returns input gitiles commit. Shortcut for `.build.input.gitiles_commit`.
For value format, see
[`GitilesCommit` message](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto).
Never returns None, but sub-fields may be empty.
"""
return self.build.input.gitiles_commit
def is_critical(self, build=None):
"""Returns True if the build is critical. Build defaults to the current one.
"""
build = build or self.build
return build.critical in (common_pb2.UNSET, common_pb2.YES)
@property
def tags_for_child_build(self):
"""A dict of tags (key -> value) derived from current (parent) build for a
child build."""
original_tags = {t.key: t.value for t in self.build.tags}
new_tags = {'user_agent': 'recipe'}
# TODO(nodir): switch to ScheduleBuild API where we don't have to convert
# build input back to tags.
# This function returns a dict, so there can be only one buildset, although
# we can have multiple sources.
# Priority: CL buildset, commit buildset, custom buildset.
commit = self.build.input.gitiles_commit
if self.build.input.gerrit_changes:
cl = self.build.input.gerrit_changes[0]
new_tags['buildset'] = 'patch/gerrit/%s/%d/%d' % (
cl.host, cl.change, cl.patchset)
# Note: an input gitiles commit with ref without id is valid
# but such commit cannot be used to construct a valid commit buildset.
elif commit.host and commit.project and commit.id:
new_tags['buildset'] = (
'commit/gitiles/%s/%s/+/%s' % (
commit.host, commit.project, commit.id))
if commit.ref:
new_tags['gitiles_ref'] = commit.ref
else:
buildset = original_tags.get('buildset')
if buildset:
new_tags['buildset'] = buildset
if self.build.number:
new_tags['parent_buildnumber'] = str(self.build.number)
if self.build.builder.builder:
new_tags['parent_buildername'] = str(self.build.builder.builder)
return new_tags
def set_output_gitiles_commit(self, gitiles_commit):
"""Sets `buildbucket.v2.Build.output.gitiles_commit` field.
This will tell other systems, consuming the build, what version of the code
was actually used in this build and what is the position of this build
relative to other builds of the same builder.
Args:
* gitiles_commit(buildbucket.common_pb2.GitilesCommit): the commit that was
actually checked out. Must have host, project and id.
ID must match r'^[0-9a-f]{40}$' (git revision).
If position is present, the build can be ordered along commits.
Position requires ref.
Ref, if not empty, must start with `refs/`.
Can be called at most once per build.
"""
# Validate commit object.
c = gitiles_commit
assert isinstance(c, common_pb2.GitilesCommit), c
assert c.host
assert '/' not in c.host, c.host
assert c.project
assert not c.project.startswith('/'), c.project
assert not c.project.startswith('a/'), c.project
assert not c.project.endswith('/'), c.project
assert c.ref.startswith('refs/'), c.ref
assert not c.ref.endswith('/'), c.ref
# We allow non-sha1 commits in test mode because it's convenient to set
# commits like "branchname-HEAD-SHA" rather than inventing something which
# looks like a git commit.
if not self._test_data.enabled: # pragma: no cover
assert util.is_sha1_hex(c.id), c.id
# position is uint32
# Does not need extra validation.
self._build.output.gitiles_commit.CopyFrom(c)
# The fact that it sets a property value is an implementation detail.
res = self.m.step('set_output_gitiles_commit', cmd=None)
prop_name = '$recipe_engine/buildbucket/output_gitiles_commit'
res.presentation.properties[prop_name] = json_format.MessageToDict(
gitiles_commit)
@staticmethod
def tags(**tags):
"""Alias for tags in util.py. See doc there."""
return util.tags(**tags)
def add_tags_to_current_build(self, tags):
"""Adds arbitrary tags during the runtime of a build.
Args:
* tags(list of common_pb2.StringPair): tags to add. May contain duplicates.
Empty tag values won't remove existing tags with matching keys, since tags
can only be added.
"""
assert isinstance(tags, list), (
'Expected type for tags is list; got %s' % type(tags))
assert all(isinstance(tag, common_pb2.StringPair) for tag in tags), map(
type, tags)
# Multiple values for the same key are allowed in tags.
for tag in tags:
self._runtime_tags.setdefault(tag.key, []).append(tag.value)
res = self.m.step('buildbucket.add_tags_to_current_build', cmd=None)
res.presentation.properties['$recipe_engine/buildbucket/runtime-tags'] = (
self._runtime_tags)
def hide_current_build_in_gerrit(self):
"""Hides the build in UI"""
self.add_tags_to_current_build(self.tags(**{'hide-in-gerrit': 'pointless'}))
@property
def builder_cache_path(self):
"""Path to the builder cache directory.
Such directory can be used to cache builder-specific data.
It remains on the bot from build to build.
See "Builder cache" in
https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/project_config.proto
"""
return self.m.path['cache'].join('builder')
# RPCs.
def _make_field_mask(self, paths=DEFAULT_FIELDS, path_prefix=''):
"""Returns a FieldMask message to use in requests."""
paths = set(paths)
if 'id' not in paths:
paths.add('id')
return field_mask_pb2.FieldMask(
paths=[path_prefix + p for p in sorted(paths)])
def run(
self, schedule_build_requests, collect_interval=None, timeout=None,
url_title_fn=None, step_name=None, raise_if_unsuccessful=False):
"""Runs builds and returns results.
A shortcut for schedule() and collect_builds().
See their docstrings.
Returns:
A list of completed
[Builds](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto)
in the same order as schedule_build_requests.
"""
with self.m.step.nest(step_name or 'buildbucket.run'):
builds = self.schedule(
schedule_build_requests, step_name='schedule',
url_title_fn=url_title_fn)
build_dict = self.collect_builds(
[b.id for b in builds],
interval=collect_interval,
timeout=timeout,
step_name='collect',
raise_if_unsuccessful=raise_if_unsuccessful,
# Do not print links. self.schedule printed them already.
url_title_fn=lambda b: None,
)
return [build_dict[b.id] for b in builds]
def schedule_request(
self,
builder,
project=INHERIT,
bucket=INHERIT,
properties=None,
experimental=INHERIT,
experiments=None,
gitiles_commit=INHERIT,
gerrit_changes=INHERIT,
tags=None,
inherit_buildsets=True,
swarming_parent_run_id=None,
dimensions=None,
priority=INHERIT,
critical=INHERIT,
exe_cipd_version=INHERIT,
fields=DEFAULT_FIELDS,
):
"""Creates a new `ScheduleBuildRequest` message with reasonable defaults.
This is a convenient function to create a `ScheduleBuildRequest` message.
Among args, messages can be passed as dicts of the same structure.
Example:
request = api.buildbucket.schedule_request(
builder='linux',
tags=api.buildbucket.tags(a='b'),
)
build = api.buildbucket.schedule([request])[0]
Args:
* builder (str): name of the destination builder.
* project (str|INHERIT): project containing the destinaiton builder.
Defaults to the project of the current build.
* bucket (str|INHERIT): bucket containing the destination builder.
Defaults to the bucket of the current build.
* properties (dict): input properties for the new build.
* experimental (common_pb2.Trinary|INHERIT): whether the build is allowed
to affect prod.
Defaults to the value of the current build.
Read more about
[`experimental` field](https://cs.chromium.org/chromium/infra/go/src/go.chromium.org/luci/buildbucket/proto/build.proto?q="bool experimental").
* experiments (Dict[str, bool]|None): enabled and disabled experiments
for the new build. Overrides the result computed from experiments defined
in builder config.
* gitiles_commit (common_pb2.GitilesCommit|INHERIT): input commit.
Defaults to the input commit of the current build.
Read more about
[`gitiles_commit`](https://cs.chromium.org/chromium/infra/go/src/go.chromium.org/luci/buildbucket/proto/build.proto?q=Input.gitiles_commit).
* gerrit_changes (list of common_pb2.GerritChange|INHERIT): list of input
CLs.
Defaults to gerrit changes of the current build.
Read more about
[`gerrit_changes`](https://cs.chromium.org/chromium/infra/go/src/go.chromium.org/luci/buildbucket/proto/build.proto?q=Input.gerrit_changes).
* tags (list of common_pb2.StringPair): tags for the new build.
* inherit_buildsets (bool): if `True` (default), the returned request will
include buildset tags from the current build.
* swarming_parent_run_id (str|NoneType): associate the new build as child of
the given swarming run id.
Defaults to `None` meaning no association.
If passed, must be a valid swarming *run* id (specific execution of a
task) for the swarming instance on which build will execute. Typically,
you'd want to set it to
[`api.swarming.task_id`](https://cs.chromium.org/chromium/infra/recipes-py/recipe_modules/swarming/api.py?type=cs&q=recipe_modules/swarming/api.py+%22def+task_id%22&sq=package:chromium&g=0&l=924).
Read more about
[`parent_run_id`](https://cs.chromium.org/chromium/infra/go/src/go.chromium.org/luci/buildbucket/proto/rpc.proto?type=cs&q="string+parent_run_id").
* dimensions (list of common_pb2.RequestedDimension): override dimensions
defined on the server.
* priority (int|NoneType|INHERIT): Swarming task priority.
The lower the more important. Valid values are `[20..255]`.
Defaults to the value of the current build.
Pass `None` to use the priority of the destination builder.
* critical (bool|common_pb2.Trinary|INHERIT): whether the build status
should not be used to assess correctness of the commit/CL.
Defaults to .build.critical.
See also Build.critical in
https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto
* exe_cipd_version (str|INHERIT): CIPD version of the LUCI Executable (e.g.
recipe) to use instead of the server-configured one.
* fields (list of strs): a list of fields to include in the response, names
relative to `build_pb2.Build` (e.g. ["tags", "infra.swarming"]).
"""
def as_msg(value, typ):
assert isinstance(value, (dict, protobuf.message.Message)), type(value)
if isinstance(value, dict):
value = typ(**value)
return value
def copy_msg(src, dest):
dest.CopyFrom(as_msg(src, type(dest)))
def as_trinary(value):
assert isinstance(value, (bool, int))
if isinstance(value, bool):
value = common_pb2.YES if value else common_pb2.NO
return value
def if_inherit(value, parent_value):
if value is self.INHERIT:
return parent_value
return value
b = self.build
req = builds_service_pb2.ScheduleBuildRequest(
request_id='%d-%s' % (b.id, self.m.uuid.random()),
builder=dict(
project=if_inherit(project, b.builder.project),
bucket=if_inherit(bucket, b.builder.bucket),
builder=builder,
),
priority=if_inherit(priority, b.infra.swarming.priority),
critical=as_trinary(if_inherit(critical, b.critical)),
# If not `INHERIT`, `experimental` must be trinary already, so only
# pass the parent (boolean) value through `as_trinary`.
experimental=if_inherit(experimental, as_trinary(b.input.experimental)),
experiments=dict(experiments) if experiments else None,
fields=self._make_field_mask(paths=fields),
)
if swarming_parent_run_id:
req.swarming.parent_run_id = swarming_parent_run_id
exe_cipd_version = if_inherit(exe_cipd_version, b.exe.cipd_version)
if exe_cipd_version:
req.exe.cipd_version = exe_cipd_version
# The Buildbucket server rejects requests that have the `gitiles_commit`
# field populated, but with all empty sub-fields. So only populate it if
# the parent build has the field.
gitiles_commit = if_inherit(
gitiles_commit,
b.input.gitiles_commit if b.input.HasField('gitiles_commit') else None)
if gitiles_commit:
copy_msg(gitiles_commit, req.gitiles_commit)
for c in if_inherit(gerrit_changes, b.input.gerrit_changes):
copy_msg(c, req.gerrit_changes.add())
req.properties.update(properties or {})
# Populate tags.
tag_set = {('user_agent', 'recipe')}
for t in tags or []:
t = as_msg(t, common_pb2.StringPair)
tag_set.add((t.key, t.value))
if inherit_buildsets:
for t in b.tags:
if t.key == 'buildset':
tag_set.add((t.key, t.value))
# TODO(tandrii, nodir): find better way to communicate cq_experimental
# status to Gerrit Buildbucket plugin.
for t in b.tags:
if t.key == 'cq_experimental':
tag_set.add((t.key, t.value))
for k, v in sorted(tag_set):
req.tags.add(key=k, value=v)
for d in dimensions or []:
copy_msg(d, req.dimensions.add())
return req
def schedule(
self, schedule_build_requests, url_title_fn=None, step_name=None):
"""Schedules a batch of builds.
Example:
```python
req = api.buildbucket.schedule_request(builder='linux')
api.buildbucket.schedule([req])
```
Hint: when scheduling builds for CQ, let CQ know about them:
```python
api.cq.record_triggered_builds(*api.buildbucket.schedule([req1, req2]))
```
Args:
* schedule_build_requests: a list of `buildbucket.v2.ScheduleBuildRequest`
protobuf messages. Create one by calling `schedule_request` method.
* url_title_fn: generates a build URL title. See module docstring.
* step_name: name for this step.
Returns:
A list of
[`Build`](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto)
messages in the same order as requests.
Raises:
`InfraFailure` if any of the requests fail.
"""
assert isinstance(schedule_build_requests, list), schedule_build_requests
for r in schedule_build_requests:
assert isinstance(r, builds_service_pb2.ScheduleBuildRequest), r
if not schedule_build_requests:
return []
batch_req = builds_service_pb2.BatchRequest(
requests=[dict(schedule_build=r) for r in schedule_build_requests]
)
test_res = builds_service_pb2.BatchResponse()
for r in schedule_build_requests:
test_res.responses.add(
schedule_build=dict(
id=self._next_test_build_id,
builder=r.builder,
)
)
self._next_test_build_id += 1
step_res, batch_res, has_errors = self._batch_request(
step_name or 'buildbucket.schedule', batch_req, test_res)
# Append build links regardless of errors.
for r in batch_res.responses:
if not r.HasField('error'):
self._report_build_maybe(
step_res, r.schedule_build, url_title_fn=url_title_fn)
if has_errors:
raise self.m.step.InfraFailure('Build creation failed')
# Return Build messages.
return [r.schedule_build for r in batch_res.responses]
def _report_build_maybe(self, step_result, build, url_title_fn=None):
"""Reports a build in the step presentation.
url_title_fn is a function that accepts a `build_pb2.Build` and returns a
link title. If returns None, the link is not reported.
Default link title is build id.
"""
build_title = url_title_fn(build) if url_title_fn else build.id
if build_title is not None:
pres = step_result.presentation
pres.links[str(build_title)] = self.build_url(build_id=build.id)
def put(self, builds, **kwargs):
"""Puts a batch of builds.
DEPRECATED. Use `schedule()` instead.
Args:
* builds (list): A list of dicts, where keys are:
* 'bucket': (required) name of the bucket for the request.
* 'parameters' (dict): (required) arbitrary json-able parameters that a
build system would be able to interpret.
* 'experimental': (optional) a bool indicating whether build is
experimental. If not provided, the value will be determined by whether
the currently running build is experimental.
* 'tags': (optional) a dict(str->str) of tags for the build. These will
be added to those generated by this method and override them if
appropriate. If you need to remove a tag set by default, set its value
to `None` (for example, `tags={'buildset': None}` will ensure build is
triggered without `buildset` tag).
Returns:
A step that as its `.stdout` property contains the response object as
returned by buildbucket.
"""
build_specs = []
for build in builds:
build_specs.append(self.m.json.dumps({
'bucket': build['bucket'],
'parameters_json': self.m.json.dumps(build['parameters']),
'tags': self._tags_for_build(build['bucket'], build['parameters'],
build.get('tags')),
'experimental': build.get('experimental',
self.m.runtime.is_experimental),
}))
return self._run_buildbucket('put', build_specs, **kwargs)
def search(self,
predicate,
limit=None,
url_title_fn=None,
report_build=True,
step_name=None,
fields=DEFAULT_FIELDS):
"""Searches for builds.
Example: find all builds of the current CL.
```python
from PB.go.chromium.org.luci.buildbucket.proto import rpc as builds_service_pb2
related_builds = api.buildbucket.search(builds_service_pb2.BuildPredicate(
gerrit_changes=list(api.buildbucket.build.input.gerrit_changes),
))
```
Args:
* predicate: a `builds_service_pb2.BuildPredicate` object or a list thereof.
If a list, the predicates are connected with logical OR.
* limit: max number of builds to return. Defaults to 1000.
* url_title_fn: generates a build URL title. See module docstring.
* report_build: whether to report build search results in step
presentation. Defaults to True.
* fields: a list of fields to include in the response, names relative
to `build_pb2.Build` (e.g. ["tags", "infra.swarming"]).
Returns:
A list of builds ordered newest-to-oldest.
"""
assert isinstance(predicate, (list, builds_service_pb2.BuildPredicate)), predicate
if not isinstance(predicate, list):
predicate = [predicate]
assert all(isinstance(p, builds_service_pb2.BuildPredicate) for p in predicate)
assert isinstance(limit, (type(None), int))
assert limit is None or limit >= 0
limit = limit or 1000
args = [
'-json',
'-nopage',
'-n', limit,
'-fields', ','.join(sorted(set(fields)))]
for p in predicate:
args.append('-predicate')
# Note: json.dumps produces compact JSON to reduce argument size
args.append(self.m.json.dumps(json_format.MessageToDict(p)))
step_result = self._run_bb(
subcommand='ls',
step_name=step_name or 'buildbucket.search',
args=args,
stdout=self.m.raw_io.output_text(add_output_log=True))
ret = []
# Every line is a build serialized in JSON format
for line in step_result.stdout.splitlines():
build = json_format.Parse(
line, build_pb2.Build(),
# Do not fail because recipe's proto copy is stale.
ignore_unknown_fields=True)
if report_build:
self._report_build_maybe(step_result, build, url_title_fn=url_title_fn)
ret.append(build)
assert len(ret) <= limit, (
'bb ls returns %d builds when limit set to %d' % (len(ret), limit))
return ret
def cancel_build(self, build_id, reason=' ', step_name=None):
"""Cancel the build associated with the provided build id.
Args:
* `build_id` (int|str): a buildbucket build ID.
It should be either an integer(e.g. 123456789 or '123456789')
or the numeric value in string format.
* `reason` (str): reason for canceling the given build.
Can't be None or Empty. Markdown is supported.
Returns:
None if build is successfully canceled. Otherwise, an InfraFailure will
be raised
"""
self._check_build_id(build_id)
cancel_req = builds_service_pb2.BatchRequest(
requests=[
dict(cancel_build=dict(
# Expecting id to be of type int64 according to the proto definition
id=int(build_id),
summary_markdown=str(reason)
))])
test_res = builds_service_pb2.BatchResponse(
responses=[
dict(cancel_build=dict(
id=int(build_id),
status=common_pb2.CANCELED
))])
_, batch_res, has_errors = self._batch_request(
step_name or 'buildbucket.cancel', cancel_req, test_res)
if has_errors:
raise self.m.step.InfraFailure(
'Failed to cancel build [%s]. Message: %s' %(
build_id, batch_res.responses[0].error.message))
return None
def get_multi(self, build_ids, url_title_fn=None, step_name=None,
fields=DEFAULT_FIELDS):
"""Gets multiple builds.
Args:
* `build_ids`: a list of build IDs.
* `url_title_fn`: generates build URL title. See module docstring.
* `step_name`: name for this step.
* `fields`: a list of fields to include in the response, names relative
to `build_pb2.Build` (e.g. ["tags", "infra.swarming"]).
Returns:
A dict {build_id: build_pb2.Build}.
"""
return self._get_multi(build_ids, url_title_fn, step_name, fields)[1]
def _get_multi(self, build_ids, url_title_fn, step_name, fields):
"""Implements get_multi, but also returns StepResult."""
batch_req = builds_service_pb2.BatchRequest(
requests=[
dict(get_build=dict(id=id, fields=self._make_field_mask(
paths=fields)))
for id in build_ids
],
)
test_res = builds_service_pb2.BatchResponse(
responses=[
dict(get_build=dict(id=id, status=common_pb2.SUCCESS))
for id in build_ids
]
)
step_res, batch_res, has_errors = self._batch_request(
step_name or 'buildbucket.get_multi', batch_req, test_res)
ret = {}
for res in batch_res.responses:
if res.HasField('get_build'):
b = res.get_build
self._report_build_maybe(step_res, b, url_title_fn=url_title_fn)
ret[b.id] = b
if has_errors:
raise self.m.step.InfraFailure('Getting builds failed')
return step_res, ret
def get(self, build_id, url_title_fn=None, step_name=None,
fields=DEFAULT_FIELDS):
"""Gets a build.
Args:
* `build_id`: a buildbucket build ID.
* `url_title_fn`: generates build URL title. See module docstring.
* `step_name`: name for this step.
* `fields`: a list of fields to include in the response, names relative
to `build_pb2.Build` (e.g. ["tags", "infra.swarming"]).
Returns:
A build_pb2.Build.
"""
builds = self.get_multi(
[build_id],
url_title_fn=url_title_fn,
step_name=step_name or 'buildbucket.get',
fields=fields)
return builds[build_id]
def get_build(self, build_id, **kwargs):
"""DEPRECATED. Use get()."""
return self._run_buildbucket('get', [build_id], **kwargs)
def collect_build(self, build_id, **kwargs):
"""Shorthand for `collect_builds` below, but for a single build only.
Args:
* build_id: Integer ID of the build to wait for.
Returns:
[Build](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto).
for the ended build.
"""
assert isinstance(build_id, int)
return self.collect_builds([build_id], **kwargs)[build_id]
def collect_builds(
self, build_ids, interval=None, timeout=None, step_name=None,
raise_if_unsuccessful=False, url_title_fn=None,
mirror_status=False, fields=DEFAULT_FIELDS,
):
"""Waits for a set of builds to end and returns their details.
Args:
* `build_ids`: List of build IDs to wait for.
* `interval`: Delay (in secs) between requests while waiting for build to
end. Defaults to 1m.
* `timeout`: Maximum time to wait for builds to end. Defaults to 1h.
* `step_name`: Custom name for the generated step.
* `raise_if_unsuccessful`: if any build being collected did not succeed,
raise an exception.
* `url_title_fn`: generates build URL title. See module docstring.
* `mirror_status`: mark the step as failed/infra-failed if any of the builds
did not succeed. Ignored if raise_if_unsuccessful is True.
* `fields`: a list of fields to include in the response, names relative
to `build_pb2.Build` (e.g. ["tags", "infra.swarming"]).
Returns:
A map from integer build IDs to the corresponding
[Build](https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/build.proto)
for all specified builds.
"""
if not build_ids:
return {}
interval = interval or 60
timeout = timeout or 3600
with self.m.step.nest(step_name or 'buildbucket.collect'):
# Wait for the builds to finish.
self._run_bb(
step_name='wait',
subcommand='collect',
args=['-interval', '%ds' % interval] + build_ids,
timeout=timeout,
)
# Fetch build details.
if raise_if_unsuccessful or mirror_status:
if fields and 'status' not in fields:
fields = fields[:]
fields.append('status')
step_res, builds = self._get_multi(
build_ids, url_title_fn=url_title_fn, step_name='get', fields=fields)
if raise_if_unsuccessful:
unsuccessful_builds = sorted(
b.id for b in builds.itervalues()
if b.status != common_pb2.SUCCESS
)
if unsuccessful_builds:
step_res.presentation.status = self.m.step.FAILURE
step_res.presentation.logs['unsuccessful_builds'] = map(
str, unsuccessful_builds)
raise self.m.step.InfraFailure(
'Triggered build(s) did not succeed, unexpectedly')
elif mirror_status:
bs = builds.values()
if any(b.status == common_pb2.INFRA_FAILURE for b in bs):
step_res.presentation.status = self.m.step.EXCEPTION
elif any(b.status == common_pb2.FAILURE for b in bs):
step_res.presentation.status = self.m.step.FAILURE
return builds
# Internal.
def _batch_request(self, step_name, request, test_response):
"""Makes a Builds.Batch request.
Returns (StepResult, builds_service_pb2.BatchResponse, has_errors) tuple.
"""
request_dict = json_format.MessageToDict(request)
try:
self._run_bb(
step_name=step_name,
subcommand='batch',
stdin=self.m.json.input(request_dict),
stdout=self.m.json.output(),
step_test_data=lambda: self.m.json.test_api.output_stream(
json_format.MessageToDict(test_response)
),
)
except self.m.step.StepFailure: # pragma: no cover
# Ignore the exit code and parse the response as BatchResponse.
# Fail if parsing fails.
pass
step_res = self.m.step.active_result
# Log the request.
step_res.presentation.logs['request'] = self.m.json.dumps(
request_dict, indent=2, sort_keys=True).splitlines()
# Parse the response.
if step_res.stdout is None:
raise self.m.step.InfraFailure('Buildbucket Internal Error')
batch_res = builds_service_pb2.BatchResponse()
json_format.ParseDict(
step_res.stdout, batch_res,
# Do not fail the build because recipe's proto copy is stale.
ignore_unknown_fields=True)
# Print response errors in step text.
step_text = []
has_errors = False
for i, r in enumerate(batch_res.responses):
if r.HasField('error'):
has_errors = True
step_text.extend([
'Request #%d' % i,
'Status code: %s' % r.error.code,
'Message: %s' % r.error.message,
'', # Blank line.
])
step_res.presentation.step_text = '<br>'.join(step_text)
return (step_res, batch_res, has_errors)
def _run_bb(
self, subcommand, step_name=None, args=None, stdin=None, stdout=None,
step_test_data=None, timeout=None):
cmdline = [
'bb', subcommand,
'-host', self._host,
]
# Do not pass -service-account-json. It is not needed on LUCI.
# TODO(nodir): change api.runtime.is_luci default to True and assert
# it is true here.
cmdline += args or []
return self.m.step(
step_name or ('bb ' + subcommand),
cmdline,
infra_step=True,
stdin=stdin,
stdout=stdout,
step_test_data=step_test_data,
timeout=timeout,
)
# TODO(nodir): remove in favor of _run_bb
def _run_buildbucket(
self, subcommand, args=None, json_stdout=True, name=None, **kwargs):
step_name = name or ('buildbucket.' + subcommand)
args = args or []
if self._service_account_key:
args = ['-service-account-json', self._service_account_key] + args
args = ['buildbucket', subcommand, '-host', self._host] + args
kwargs.setdefault('infra_step', True)
stdout = self.m.json.output() if json_stdout else None
return self.m.step(step_name, args, stdout=stdout, **kwargs)
def _tags_for_build(self, bucket, parameters, override_tags=None):
new_tags = self.tags_for_child_build
builder_name = parameters.get('builder_name')
if builder_name:
new_tags['builder'] = builder_name
new_tags.update(override_tags or {})
return sorted(
'%s:%s' % (k, v)
for k, v in new_tags.iteritems()
if v is not None)
def _check_build_id(self, build_id):
"""Raise ValueError if the given build id is not a number or a string
that represents numeric value.
"""
is_int = isinstance(build_id, (int, long))
is_str_num = isinstance(build_id, str) and build_id.isdigit()
if not (is_int or is_str_num):
raise ValueError('Expected a numeric build id, got %s' %(build_id,))
@property
def bucket_v1(self):
"""Returns bucket name in v1 format.
Mostly useful for scheduling new builds using V1 API.
"""
return self._bucket_v1
# DEPRECATED API.
@property
def build_id(self): # pragma: no cover
"""DEPRECATED, use build.id instead."""
return self.build.id or None
@property
def build_input(self): # pragma: no cover
"""DEPRECATED, use build.input instead."""
return self.build.input
@property
def builder_id(self): # pragma: no cover
"""Deprecated. Use build.builder instead."""
return self.build.builder
# Legacy support.
def _legacy_input_gerrit_changes(
dest_repeated,
patch_storage, patch_gerrit_url, patch_project, patch_issue, patch_set):
if patch_storage == 'gerrit' and patch_project:
host, path = util.parse_http_host_and_path(patch_gerrit_url)
if host and (not path or path == '/'):
try:
patch_issue = int(patch_issue or 0)
patch_set = int(patch_set or 0)
except ValueError: # pragma: no cover
pass
else:
if patch_issue and patch_set:
dest_repeated.add(
host=host,
project=patch_project,
change=patch_issue,
patchset=patch_set)
return
def _legacy_input_gitiles_commit(dest, revision, branch):
if util.is_sha1_hex(revision):
dest.id = revision
if branch:
dest.ref = 'refs/heads/%s' % branch
def _legacy_builder_id(mastername, buildername, builder_id):
if mastername:
builder_id.bucket = 'master.%s' % mastername
builder_id.builder = buildername or ''