blob: 35f32f1cb268209b7564bbe0b83dac35a7a1de3d [file] [log] [blame]
# Copyright 2019 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
import collections
import os
import traceback
import json
from PB.recipe_engine import result as result_pb2
from recipe_engine.third_party import subprocess42
from .. import recipe_api
from .. import types
from .. import util
from .exceptions import RecipeUsageError
from .recipe_deps import Recipe
# TODO(dnj): Replace "properties" with a generic runtime instance. This instance
# will be used to seed recipe clients and expanded to include managed runtime
# entities.
def run_steps(recipe_deps, properties, stream_engine, step_runner,
"""Runs a recipe (given by the 'recipe' property) for real.
* recipe_deps (RecipeDeps) - The loaded recipe repo dependencies.
* properties: a dictionary of properties to pass to the recipe. The
'recipe' property defines which recipe to actually run.
* stream_engine: the StreamEngine to use to create individual step streams.
* step_runner: The StepRunner to use to 'actually run' the steps.
* emit_initial_properties (bool): If True, write the initial recipe engine
properties in the "setup_build" step.
Returns: result_pb2.Result
with stream_engine.make_step_stream('setup_build') as s:
if emit_initial_properties:
for key in sorted(properties.iterkeys()):
s.set_build_property(key, json.dumps(properties[key], sort_keys=True))
engine = RecipeEngine(
recipe_deps, step_runner, properties, os.environ)
assert 'recipe' in properties
recipe = properties['recipe']
run_recipe_help_lines = [
'To repro this locally, run the following line from the root of a %r'
' checkout:' % (,
'%s run --properties-file - %s <<EOF' % (
'.', recipe_deps.main_repo.simple_cfg.recipes_path, ''),
json.dumps(properties, indent=2).splitlines())
run_recipe_help_lines += [
'To run on Windows, you can put the JSON in a file and redirect the',
'contents of the file into, with the < operator.',
with s.new_log_stream('run_recipe') as l:
for line in run_recipe_help_lines:
# Find and load the recipe to run.
# This does all loading and importing of the recipe script.
recipe_obj =[recipe]
# Make sure `global_symbols` (which is a cached execfile of the recipe
# python file) executes here so that we can correctly catch any
# RecipeUsageError exceptions which exec'ing it may cause.
# TODO(iannucci): rethink how all this exception reporting stuff should
# work.
_ = recipe_obj.global_symbols
s.write_line('Running recipe with %s' % (properties,))
s.add_step_text('running recipe: "%s"' % recipe)
except (RecipeUsageError, ImportError, AssertionError) as e:
for line in str(e).splitlines():
return result_pb2.Result(failure=result_pb2.Failure(human_reason=str(e)))
# The engine will use step_runner to run the steps, and the step_runner in
# turn uses stream_engine internally to build steam steps IO.
class RecipeEngine(object):
Knows how to execute steps emitted by a recipe, holds global state such as
step history and build properties. Each recipe module API has a reference to
this object.
Recipe modules that are aware of the engine:
* properties - uses
* step - uses engine.create_step(...), and previous_step_result.
ActiveStep = collections.namedtuple('ActiveStep', (
'config', 'step_result', 'open_step'))
def __init__(self, recipe_deps, step_runner, properties, environ):
"""See run_steps() for parameter meanings."""
self._recipe_deps = recipe_deps
self._step_runner = step_runner
self._properties = properties
self._environ = environ.copy()
self._clients = {client.IDENT: client for client in (
recipe_api.SourceManifestClient(self, properties),
# A stack of ActiveStep objects, holding the most recently executed step at
# each nest level (objects deeper in the stack have lower nest levels).
# When we pop from this stack, we close the corresponding step stream.
self._step_stack = []
def properties(self):
return self._properties
def environ(self):
return self._environ
def resolve_requirement(self, req):
"""Resolves a requirement or raises ValueError if it cannot be resolved.
* req (_UnresolvedRequirement): The requirement to resolve.
Returns the resolved requirement.
Raises ValueError if the requirement cannot be satisfied.
assert isinstance(req, recipe_api._UnresolvedRequirement)
if req._typ == 'client':
return self._clients.get(req._name)
raise ValueError('Unknown requirement type [%s]' % (req._typ,))
def initialize_path_client_HACK(self, root_api):
"""This is a hack; the "PathsClient" currently works to provide a reverse
string->Path lookup by walking down the recipe's `api` object and calling
the various 'root' path methods (like .resource(), etc.).
However, we would like to:
* Make the 'api' wholly internal to the `RecipeScript.run_steps` function.
* Eventually simplify the 'paths' system, whose whole complexity exists to
facilitate 'pure-data' processing, which is also going to be
deprecated in favor of protos and removal of the config subsystem.
* root_api (RecipeScriptApi): The root `api` object which would be passed
to the recipe's RunSteps function.
def _close_until_ns(self, namespace):
"""Close all open steps until we close all of them or until we find one
that's a parent of of namespace.
open_steps = [
name=('namespace', 'subspace'),
name=('namespace', 'subspace', 'step'),
# if the new step is ('namespace', 'subspace', 'new_step') we call:
_close_until_ns(('namespace', 'subspace'))
# Closes ('namespace', 'subspace', 'step')
# if the new step is ('namespace', 'new_subspace') we call:
# Closes ('namespace', 'subspace', 'step')
# Closes ('namespace', 'subspace')
# if the new step is ('bob',) we call:
# Closes ('namespace', 'subspace', 'step')
# Closes ('namespace', 'subspace')
# Closes ('namespace')
namespace (Tuple[basestring]): the namespace we're looking to get back to.
while self._step_stack:
if self._step_stack[-1].config.name_tokens == namespace:
cur = self._step_stack.pop()
if cur.step_result:
def active_step(self):
"""Returns the current ActiveStep (if there is one) or None."""
if self._step_stack:
return self._step_stack[-1]
return None
def run_step(self, step_config):
Runs a step.
step_config (StepConfig): The step configuration to run.
A StepData object containing the result of running the step.
with util.raises((recipe_api.StepFailure, OSError),
step_result = None
open_step = self._step_runner.open_step(step_config)
step_result =
self._step_stack[-1] = (
if step_result.presentation.status == 'SUCCESS':
return step_result
exc = recipe_api.StepFailure
if step_result.presentation.status == 'EXCEPTION':
exc = recipe_api.InfraFailure
if step_result.retcode <= -100:
# Windows error codes such as 0xC0000005 and 0xC0000409 are much
# easier to recognize and differentiate in hex. In order to print them
# as unsigned hex we need to add 4 Gig to them.
error_number = "0x%08X" % (step_result.retcode + (1 << 32))
error_number = "%d" % step_result.retcode
'step returned non-zero exit code: %s' % error_number)
raise exc(, step_result)
def run(self, recipe, test_data=None):
"""Run a recipe.
This function blocks until recipe finishes.
It mainly executes the recipe, and has some exception handling logic.
* recipe (Recipe): The recipe to run.
* test_data (None|TestData): The test data for this recipe run.
result_pb2.Result which has return value or status code and exception.
assert isinstance(recipe, Recipe), type(recipe)
result = result_pb2.Result()
with self._step_runner.run_context():
raw_result = recipe.run_steps(self, test_data)
result.json_result = json.dumps(raw_result)
except recipe_api.InfraFailure as ex:
result.failure.human_reason = ex.reason
except recipe_api.AggregatedStepFailure as ex:
result.failure.human_reason = ex.reason
if not ex.result.contains_infra_failure:
except recipe_api.StepFailure as ex:
result.failure.human_reason = ex.reason
except Exception as ex:
result.failure.human_reason = "Uncaught Exception: %r" % ex
# Let the step runner run_context decide what to do.
return result