Revert "[luci_context] Move writing of LUCI_CONTEXT files to the engine core."
This reverts commit 6126bc2f1092567bb3b0d96f0381330ba01ead75.
Reason for revert: Some steps are wiping out temp_dir which also accidentally deleted LUCI_CONTEXT file. See: https://crbug.com/1135137
Original change's description:
> [luci_context] Move writing of LUCI_CONTEXT files to the engine core.
>
> This removes an odd file-write from 'userspace' recipe code, but also
> enables compuation of 'timeout' for the step much closer to the actual
> step execution. Specifically, this avoids having 'timeout' also cover
> the time that we block for resource contention when running multiple
> greenlets in the recipe.
>
> R=​vadimsh, yiwzhang
>
> Bug: 1127089
> Change-Id: Iff59e7698e9556737a6144508c00cb3342811641
> Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/recipes-py/+/2441885
> Reviewed-by: Yiwei Zhang <yiwzhang@google.com>
> Commit-Queue: Robbie Iannucci <iannucci@chromium.org>
TBR=iannucci@chromium.org,vadimsh@chromium.org,yiwzhang@google.com,infra-scoped@luci-project-accounts.iam.gserviceaccount.com
# Not skipping CQ checks because original CL landed > 1 day ago.
Bug: 1127089
Change-Id: I9c3dec72e64238397d88293d24508b44c057491c
Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/recipes-py/+/2451447
Reviewed-by: Yiwei Zhang <yiwzhang@google.com>
Commit-Queue: Yiwei Zhang <yiwzhang@google.com>
diff --git a/README.recipes.md b/README.recipes.md
index f5ed9e7..eddae46 100644
--- a/README.recipes.md
+++ b/README.recipes.md
@@ -165,7 +165,7 @@
[DEPS](/recipe_modules/archive/__init__.py#5): [json](#recipe_modules-json), [path](#recipe_modules-path), [platform](#recipe_modules-platform), [python](#recipe_modules-python), [step](#recipe_modules-step)
-#### **class [ArchiveApi](/recipe_modules/archive/api.py#8)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [ArchiveApi](/recipe_modules/archive/api.py#8)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Provides steps to manipulate archive files (tar, zip, etc.).
@@ -225,7 +225,7 @@
Package object.
### *recipe_modules* / [assertions](/recipe_modules/assertions)
-#### **class [AssertionsApi](/recipe_modules/assertions/api.py#56)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [AssertionsApi](/recipe_modules/assertions/api.py#56)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Provides access to the assertion methods of the python unittest module.
@@ -283,7 +283,7 @@
`build_pb2.Build` and returns a link title.
If it returns `None`, the link is not reported. Default link title is build id.
-#### **class [BuildbucketApi](/recipe_modules/buildbucket/api.py#29)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [BuildbucketApi](/recipe_modules/buildbucket/api.py#29)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
A module for interacting with buildbucket.
@@ -667,7 +667,7 @@
API for interacting with cas client.
-#### **class [CasApi](/recipe_modules/cas/api.py#13)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [CasApi](/recipe_modules/cas/api.py#13)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
A module for interacting with cas client.
@@ -709,7 +709,7 @@
Depends on 'cipd' binary available in PATH:
https://godoc.org/go.chromium.org/luci/cipd/client/cmd/cipd
-#### **class [CIPDApi](/recipe_modules/cipd/api.py#202)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [CIPDApi](/recipe_modules/cipd/api.py#202)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
CIPDApi provides basic support for CIPD.
@@ -977,7 +977,7 @@
Returns the CIPDApi.Pin instance.
### *recipe_modules* / [commit\_position](/recipe_modules/commit_position)
-#### **class [CommitPositionApi](/recipe_modules/commit_position/api.py#10)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [CommitPositionApi](/recipe_modules/commit_position/api.py#10)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Recipe module providing commit position parsing and formatting.
@@ -1017,7 +1017,7 @@
api.step("cat subdir/foo", ['cat', './foo'])
```
-#### **class [ContextApi](/recipe_modules/context/api.py#78)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [ContextApi](/recipe_modules/context/api.py#78)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
  **@contextmanager**<br>— **def [\_\_call\_\_](/recipe_modules/context/api.py#108)(self, cwd=None, env_prefixes=None, env_suffixes=None, env=None, infra_steps=None, luciexe=None, realm=None):**
@@ -1067,7 +1067,7 @@
Look at the examples in "examples/" for examples of context module usage.
-  **@property**<br>— **def [cwd](/recipe_modules/context/api.py#234)(self):**
+  **@property**<br>— **def [cwd](/recipe_modules/context/api.py#242)(self):**
Returns the current working directory that steps will run in.
@@ -1075,7 +1075,7 @@
equivalent to api.path['start_dir'], though only occurs if no cwd has been
set (e.g. in the outermost context of RunSteps).
-  **@property**<br>— **def [env](/recipe_modules/context/api.py#244)(self):**
+  **@property**<br>— **def [env](/recipe_modules/context/api.py#252)(self):**
Returns modifications to the environment.
@@ -1086,7 +1086,7 @@
**Returns (dict)** - The env-key -> value mapping of current environment
modifications.
-  **@property**<br>— **def [env\_prefixes](/recipe_modules/context/api.py#259)(self):**
+  **@property**<br>— **def [env\_prefixes](/recipe_modules/context/api.py#267)(self):**
Returns Path prefix modifications to the environment.
@@ -1096,7 +1096,7 @@
**Returns (dict)** - The env-key -> value(Path) mapping of current
environment prefix modifications.
-  **@property**<br>— **def [env\_suffixes](/recipe_modules/context/api.py#273)(self):**
+  **@property**<br>— **def [env\_suffixes](/recipe_modules/context/api.py#281)(self):**
Returns Path suffix modifications to the environment.
@@ -1106,7 +1106,7 @@
**Returns (dict)** - The env-key -> value(Path) mapping of current
environment suffix modifications.
-  **@property**<br>— **def [infra\_step](/recipe_modules/context/api.py#287)(self):**
+  **@property**<br>— **def [infra\_step](/recipe_modules/context/api.py#295)(self):**
Returns the current value of the infra_step setting.
@@ -1114,19 +1114,19 @@
— **def [initialize](/recipe_modules/context/api.py#88)(self):**
-  **@property**<br>— **def [luci\_context](/recipe_modules/context/api.py#295)(self):**
+  **@property**<br>— **def [luci\_context](/recipe_modules/context/api.py#303)(self):**
Returns the currently tracked LUCI_CONTEXT sections as a dict of proto
messages.
Only contains `luciexe`, `realm`, and `deadline`.
-  **@property**<br>— **def [luciexe](/recipe_modules/context/api.py#307)(self):**
+  **@property**<br>— **def [luciexe](/recipe_modules/context/api.py#315)(self):**
Returns the current value (sections_pb2.LUCIExe) of luciexe section in
the current LUCI_CONTEXT. Returns None if luciexe is not defined.
-  **@property**<br>— **def [realm](/recipe_modules/context/api.py#317)(self):**
+  **@property**<br>— **def [realm](/recipe_modules/context/api.py#325)(self):**
Returns the LUCI realm of the current context.
@@ -1136,7 +1136,7 @@
[DEPS](/recipe_modules/cq/__init__.py#7): [buildbucket](#recipe_modules-buildbucket), [properties](#recipe_modules-properties), [step](#recipe_modules-step)
-#### **class [CQApi](/recipe_modules/cq/api.py#14)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [CQApi](/recipe_modules/cq/api.py#14)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
This module provides recipe API of LUCI CQ, aka pre-commit testing system.
@@ -1266,7 +1266,7 @@
File manipulation (read/write/delete/glob) methods.
-#### **class [FileApi](/recipe_modules/file/api.py#83)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [FileApi](/recipe_modules/file/api.py#83)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [compute\_hash](/recipe_modules/file/api.py#182)(self, name, paths, base_path, test_data=''):**
@@ -1656,7 +1656,7 @@
Implements in-recipe concurrency via green threads.
-#### **class [FuturesApi](/recipe_modules/futures/api.py#42)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [FuturesApi](/recipe_modules/futures/api.py#42)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Provides access to the Recipe concurrency primitives.
@@ -1805,7 +1805,7 @@
another repo. It is not recommended to use this, and it will be removed in the
near future.
-#### **class [GeneratorScriptApi](/recipe_modules/generator_script/api.py#16)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [GeneratorScriptApi](/recipe_modules/generator_script/api.py#16)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [\_\_call\_\_](/recipe_modules/generator_script/api.py#44)(self, path_to_script, \*args):**
@@ -1846,7 +1846,7 @@
[DEPS](/recipe_modules/isolated/__init__.py#1): [cipd](#recipe_modules-cipd), [context](#recipe_modules-context), [json](#recipe_modules-json), [path](#recipe_modules-path), [properties](#recipe_modules-properties), [raw\_io](#recipe_modules-raw_io), [runtime](#recipe_modules-runtime), [step](#recipe_modules-step)
-#### **class [IsolatedApi](/recipe_modules/isolated/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [IsolatedApi](/recipe_modules/isolated/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
API for interacting with isolated.
@@ -1905,7 +1905,7 @@
Methods for producing and consuming JSON.
-#### **class [JsonApi](/recipe_modules/json/api.py#95)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [JsonApi](/recipe_modules/json/api.py#95)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
  **@staticmethod**<br>— **def [dumps](/recipe_modules/json/api.py#96)(\*args, \*\*kwargs):**
@@ -1946,7 +1946,7 @@
[DEPS](/recipe_modules/led/__init__.py#5): [cipd](#recipe_modules-cipd), [context](#recipe_modules-context), [json](#recipe_modules-json), [path](#recipe_modules-path), [proto](#recipe_modules-proto), [step](#recipe_modules-step)
-#### **class [LedApi](/recipe_modules/led/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [LedApi](/recipe_modules/led/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Interface to the led tool.
@@ -2014,7 +2014,7 @@
build (using the Merge Step feature from luciexe protocol). This is the
replacement for allow_subannotation feature in the legacy annotate mode.
-#### **class [LegacyAnnotationApi](/recipe_modules/legacy_annotation/api.py#24)([RecipeApiPlain](/recipe_engine/recipe_api.py#708)):**
+#### **class [LegacyAnnotationApi](/recipe_modules/legacy_annotation/api.py#24)([RecipeApiPlain](/recipe_engine/recipe_api.py#729)):**
— **def [\_\_call\_\_](/recipe_modules/legacy_annotation/api.py#28)(self, name, cmd, timeout=None, step_test_data=None, cost=_ResourceCost()):**
@@ -2029,7 +2029,7 @@
API for specifying Milo behavior.
-#### **class [MiloApi](/recipe_modules/milo/api.py#17)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [MiloApi](/recipe_modules/milo/api.py#17)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
A module for interacting with Milo.
@@ -2080,7 +2080,7 @@
`depot_tools/infra_paths` module). Refer to those modules for additional
documentation.
-#### **class [PathApi](/recipe_modules/path/api.py#206)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [PathApi](/recipe_modules/path/api.py#206)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [\_\_getitem\_\_](/recipe_modules/path/api.py#438)(self, name):**
@@ -2246,7 +2246,7 @@
Mockable system platform identity functions.
-#### **class [PlatformApi](/recipe_modules/platform/api.py#24)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [PlatformApi](/recipe_modules/platform/api.py#24)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
Provides host-platform-detection properties.
@@ -2331,7 +2331,7 @@
intentionally no API to write property values (lest they become a kind of
random-access global variable).
-#### **class [PropertiesApi](/recipe_modules/properties/api.py#28)([RecipeApiPlain](/recipe_engine/recipe_api.py#708), collections.Mapping):**
+#### **class [PropertiesApi](/recipe_modules/properties/api.py#28)([RecipeApiPlain](/recipe_engine/recipe_api.py#729), collections.Mapping):**
PropertiesApi implements all the standard Mapping functions, so you
can use it like a read-only dict.
@@ -2358,7 +2358,7 @@
Methods for producing and consuming protobuf data to/from steps and the
filesystem.
-#### **class [ProtoApi](/recipe_modules/proto/api.py#75)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [ProtoApi](/recipe_modules/proto/api.py#75)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
  **@staticmethod**<br>— **def [decode](/recipe_modules/proto/api.py#151)(data, msg_class, codec, \*\*decoding_kwargs):**
@@ -2435,7 +2435,7 @@
correctly for bots (e.g. ensuring that python is working on Windows, passing the
unbuffered flag, etc.)
-#### **class [PythonApi](/recipe_modules/python/api.py#17)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [PythonApi](/recipe_modules/python/api.py#17)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [\_\_call\_\_](/recipe_modules/python/api.py#18)(self, name, script, args=None, unbuffered=True, venv=None, \*\*kwargs):**
@@ -2519,7 +2519,7 @@
api.random.shuffle(my_list)
# my_list is now random!
-#### **class [RandomApi](/recipe_modules/random/api.py#31)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [RandomApi](/recipe_modules/random/api.py#31)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [\_\_getattr\_\_](/recipe_modules/random/api.py#38)(self, name):**
@@ -2530,7 +2530,7 @@
Provides objects for reading and writing raw data to and from steps.
-#### **class [RawIOApi](/recipe_modules/raw_io/api.py#297)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [RawIOApi](/recipe_modules/raw_io/api.py#297)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
  **@[returns\_placeholder](/recipe_engine/util.py#151)**<br>  **@staticmethod**<br>— **def [input](/recipe_modules/raw_io/api.py#298)(data, suffix='', name=None):**
@@ -2625,7 +2625,7 @@
Requires `rdb` command in `$PATH`:
https://godoc.org/go.chromium.org/luci/resultdb/cmd/rdb
-#### **class [ResultDBAPI](/recipe_modules/resultdb/api.py#19)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [ResultDBAPI](/recipe_modules/resultdb/api.py#19)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
A module for interacting with ResultDB.
@@ -2747,7 +2747,7 @@
may be repeated.
### *recipe_modules* / [runtime](/recipe_modules/runtime)
-#### **class [RuntimeApi](/recipe_modules/runtime/api.py#8)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [RuntimeApi](/recipe_modules/runtime/api.py#8)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
This module assists in experimenting with production recipes.
@@ -2776,7 +2776,7 @@
RPCExplorer available at
https://luci-scheduler.appspot.com/rpcexplorer/services/scheduler.Scheduler
-#### **class [SchedulerApi](/recipe_modules/scheduler/api.py#26)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [SchedulerApi](/recipe_modules/scheduler/api.py#26)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
A module for interacting with LUCI Scheduler service.
@@ -2832,7 +2832,7 @@
Depends on luci-auth to be in PATH.
-#### **class [ServiceAccountApi](/recipe_modules/service_account/api.py#16)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [ServiceAccountApi](/recipe_modules/service_account/api.py#16)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [default](/recipe_modules/service_account/api.py#57)(self):**
@@ -2858,7 +2858,7 @@
Step is the primary API for running steps (external programs, scripts,
etc.).
-#### **class [StepApi](/recipe_modules/step/api.py#26)([RecipeApiPlain](/recipe_engine/recipe_api.py#708)):**
+#### **class [StepApi](/recipe_modules/step/api.py#26)([RecipeApiPlain](/recipe_engine/recipe_api.py#729)):**
  **@property**<br>— **def [InfraFailure](/recipe_modules/step/api.py#138)(self):**
@@ -3174,7 +3174,7 @@
[DEPS](/recipe_modules/swarming/__init__.py#8): [buildbucket](#recipe_modules-buildbucket), [cas](#recipe_modules-cas), [cipd](#recipe_modules-cipd), [context](#recipe_modules-context), [isolated](#recipe_modules-isolated), [json](#recipe_modules-json), [path](#recipe_modules-path), [properties](#recipe_modules-properties), [raw\_io](#recipe_modules-raw_io), [runtime](#recipe_modules-runtime), [step](#recipe_modules-step)
-#### **class [SwarmingApi](/recipe_modules/swarming/api.py#1209)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [SwarmingApi](/recipe_modules/swarming/api.py#1209)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
API for interacting with swarming.
@@ -3276,7 +3276,7 @@
Allows mockable access to the current time.
-#### **class [TimeApi](/recipe_modules/time/api.py#12)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [TimeApi](/recipe_modules/time/api.py#12)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [ms\_since\_epoch](/recipe_modules/time/api.py#49)(self):**
@@ -3304,7 +3304,7 @@
API for Tricium analyzers to use.
-#### **class [TriciumApi](/recipe_modules/tricium/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [TriciumApi](/recipe_modules/tricium/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
TriciumApi provides basic support for Tricium.
@@ -3326,7 +3326,7 @@
Methods for interacting with HTTP(s) URLs.
-#### **class [UrlApi](/recipe_modules/url/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [UrlApi](/recipe_modules/url/api.py#15)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [get\_file](/recipe_modules/url/api.py#131)(self, url, path, step_name=None, headers=None, transient_retry=True, strip_prefix=None, timeout=None):**
@@ -3431,7 +3431,7 @@
Allows test-repeatable access to a random UUID.
-#### **class [UuidApi](/recipe_modules/uuid/api.py#11)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [UuidApi](/recipe_modules/uuid/api.py#11)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
— **def [random](/recipe_modules/uuid/api.py#20)(self):**
@@ -3440,7 +3440,7 @@
Thin API for parsing semver strings into comparable object.
-#### **class [VersionApi](/recipe_modules/version/api.py#12)([RecipeApi](/recipe_engine/recipe_api.py#856)):**
+#### **class [VersionApi](/recipe_modules/version/api.py#12)([RecipeApi](/recipe_engine/recipe_api.py#877)):**
  **@staticmethod**<br>— **def [parse](/recipe_modules/version/api.py#14)(version):**
@@ -3456,7 +3456,7 @@
Allows recipe modules to issue warnings in simulation test.
-#### **class [WarningApi](/recipe_modules/warning/api.py#12)([RecipeApiPlain](/recipe_engine/recipe_api.py#708)):**
+#### **class [WarningApi](/recipe_modules/warning/api.py#12)([RecipeApiPlain](/recipe_engine/recipe_api.py#729)):**
  **@recipe_api.escape_all_warnings**<br>— **def [issue](/recipe_modules/warning/api.py#15)(self, name):**
diff --git a/recipe_engine/internal/engine.py b/recipe_engine/internal/engine.py
index a0c4417..3f3c01c 100644
--- a/recipe_engine/internal/engine.py
+++ b/recipe_engine/internal/engine.py
@@ -15,7 +15,6 @@
import attr
import gevent
import gevent.local
-import six
from google.protobuf import json_format as jsonpb
from pympler import summary, tracker
@@ -29,7 +28,6 @@
from ..step_data import StepData, ExecutionResult
from ..types import StepPresentation, thaw
from ..types import PerGreenletState, PerGreentletStateRegistry
-from ..third_party import luci_context
from .engine_env import merge_envs
from .exceptions import RecipeUsageError, CrashEngine
@@ -117,8 +115,8 @@
"""
def __init__(self, recipe_deps, step_runner, stream_engine, warning_recorder,
- properties, environ, start_dir, initial_luci_context,
- num_logical_cores, memory_mb):
+ properties, environ, start_dir, luci_context, num_logical_cores,
+ memory_mb):
"""See run_steps() for parameter meanings."""
self._recipe_deps = recipe_deps
self._step_runner = step_runner
@@ -131,7 +129,7 @@
recipe_api.ConcurrencyClient(
stream_engine.supports_concurrency,
self.spawn_greenlet),
- recipe_api.LUCIContextClient(initial_luci_context),
+ recipe_api.LUCIContextClient(luci_context),
recipe_api.PathsClient(start_dir),
recipe_api.PropertiesClient(properties),
recipe_api.StepClient(self),
@@ -474,7 +472,7 @@
@classmethod
def run_steps(cls, recipe_deps, properties, stream_engine, step_runner,
- warning_recorder, environ, cwd, initial_luci_context,
+ warning_recorder, environ, cwd, luci_context,
num_logical_cores, memory_mb,
emit_initial_properties=False, test_data=None,
skip_setup_build=False):
@@ -493,8 +491,8 @@
* environ: The mapping object representing the environment in which
recipe runs. Generally obtained via `os.environ`.
* cwd (str): The current working directory to run the recipe.
- * initial_luci_context (Dict[str, Dict]): The content of LUCI_CONTEXT to
- pass to the recipe.
+ * luci_context (Dict[str, Dict]): The content of LUCI_CONTEXT to pass
+ to the recipe.
* num_logical_cores (int): The number of logical CPU cores to assume the
machine has.
* memory_mb (int): The amount of memory to assume the machine has, in MiB.
@@ -526,8 +524,7 @@
engine = cls(
recipe_deps, step_runner, stream_engine, warning_recorder,
- properties, environ, cwd, initial_luci_context, num_logical_cores,
- memory_mb)
+ properties, environ, cwd, luci_context, num_logical_cores, memory_mb)
api = recipe_obj.mk_api(engine, test_data)
engine.initialize_path_client_HACK(api)
except (RecipeUsageError, ImportError, AssertionError) as ex:
@@ -768,25 +765,16 @@
pathsep)
env.update(step_stream.env_vars)
- if step_config.luci_context:
- debug.write_line('writing LUCI_CONTEXT file')
- lctx_file = step_runner.write_luci_context({
- key: jsonpb.MessageToDict(pb_val) if pb_val is not None else None
- for key, pb_val in six.iteritems(step_config.luci_context)
- })
- debug.write_line(' done: %r' % (lctx_file,))
- env[luci_context.ENV_KEY] = lctx_file
-
debug.write_line('checking cwd: %r' % (step_config.cwd,))
cwd = step_config.cwd or start_dir
if not step_runner.isabs(name_tokens, cwd):
- debug.write_line(' not absolute: %r' % (cwd,))
+ debug.write_line(' not absolute: %r' % (cwd))
return None
if not step_runner.isdir(name_tokens, cwd):
- debug.write_line(' not a directory: %r' % (cwd,))
+ debug.write_line(' not a directory: %r' % (cwd))
return None
if not step_runner.access(name_tokens, cwd, os.R_OK):
- debug.write_line(' no read perms: %r' % (cwd,))
+ debug.write_line(' no read perms: %r' % (cwd))
return None
path = env.get('PATH', '').split(pathsep)
diff --git a/recipe_engine/internal/step_runner/__init__.py b/recipe_engine/internal/step_runner/__init__.py
index 862e844..7cfa151 100644
--- a/recipe_engine/internal/step_runner/__init__.py
+++ b/recipe_engine/internal/step_runner/__init__.py
@@ -135,15 +135,6 @@
"""
return cmd0
- def write_luci_context(self, section_values):
- """Writes a mapping of str->dict to disk (as a temp file), returning that
- path.
-
- `section_values` represents the 'diff' against LUCI_CONTEXT for the
- recipe engine process. The standard LUCI_CONTEXT merge rules should apply.
- """
- raise NotImplementedError()
-
def run(self, name_tokens, debug_log, step):
"""Runs the step defined by step_config.
diff --git a/recipe_engine/internal/step_runner/sim.py b/recipe_engine/internal/step_runner/sim.py
index af42b49..3f3564a 100644
--- a/recipe_engine/internal/step_runner/sim.py
+++ b/recipe_engine/internal/step_runner/sim.py
@@ -104,10 +104,6 @@
self._used_steps[dot_name], handle_name)
return self._used_placeholders[key]
- def write_luci_context(self, section_pb_values):
- # We ignore this environment variable anyway.
- return ""
-
def run(self, name_tokens, debug_log, step):
del debug_log # unused
diff --git a/recipe_engine/internal/step_runner/subproc.py b/recipe_engine/internal/step_runner/subproc.py
index be86d0c..1f085be 100644
--- a/recipe_engine/internal/step_runner/subproc.py
+++ b/recipe_engine/internal/step_runner/subproc.py
@@ -6,14 +6,12 @@
import signal
import sys
-from gevent import subprocess
import attr
import gevent
+from gevent import subprocess
from ...step_data import ExecutionResult
-from ...third_party import luci_context
-
from . import StepRunner
_MSWINDOWS = sys.platform == 'win32'
@@ -155,10 +153,6 @@
return None
- def write_luci_context(self, section_values):
- with luci_context.stage(_leak=True, **section_values) as file_path:
- return file_path
-
def run(self, name_tokens, debug_log, step):
proc, gid, pipes = self._mk_proc(step, debug_log)
@@ -307,8 +301,8 @@
'finished waiting for process, retcode %r' % ret.retcode)
# TODO(iannucci): Make leaking subprocesses explicit (e.g. goma compiler
- # daemon). Better, change deamons to be owned by a gevent Greenlet (so
- # that we don't need to leak processes ever).
+ # daemon). Better, change deamons to be owned by a gevent Greenlet (so that
+ # we don't need to leak processes ever).
#
# _kill(proc, gid) # In case of leaked subprocesses or timeout.
if ret.retcode is None:
diff --git a/recipe_engine/recipe_api.py b/recipe_engine/recipe_api.py
index f249824..d2b0681 100644
--- a/recipe_engine/recipe_api.py
+++ b/recipe_engine/recipe_api.py
@@ -20,6 +20,7 @@
from six import iteritems
from google.protobuf import message
+from google.protobuf import json_format as jsonpb
from .config_types import Path
from .internal import engine_step
@@ -108,6 +109,26 @@
initial_context = attr.ib(validator=attr_dict_type(str, (dict, FrozenDict)),
factory=dict, converter=freeze)
+ def new_context(self, **section_pb_values):
+ """Creates a new LUCI_CONTEXT file with the provided section values, all
+ unmentioned sections in the current context will be copied over. The
+ environment variable will NOT not be switched to the newly created context.
+
+ Args:
+ * section_pb_values (Dict[str, message.Message]) - A mapping of
+ section_key to the new message value for that section. If the value
+ is None, the corresponding section will be removed from the context.
+
+ Returns the path (str) to the newly created LUCI_CONTEXT file. Returns None
+ if section_pb_values is empty (i.e. No change to current context).
+ """
+ section_values = {
+ key: jsonpb.MessageToDict(pb_val) if pb_val is not None else None
+ for key, pb_val in iteritems(section_pb_values)
+ }
+ with luci_context.stage(_leak=True, **section_values) as file_path:
+ return file_path
+
class PathsClient(object):
"""A recipe engine client which exposes all known base paths.
diff --git a/recipe_modules/context/api.py b/recipe_modules/context/api.py
index 5e7be93..2715f69 100644
--- a/recipe_modules/context/api.py
+++ b/recipe_modules/context/api.py
@@ -219,6 +219,14 @@
sections_pb2.Realm(name=realm) if realm else None)
if section_pb_values:
_add_to_context('luci_context', section_pb_values, _override)
+ env = {} if env is None else dict(env)
+ if self._test_data.enabled:
+ self._test_counter += 1
+ env[self._lucictx_client.ENV_KEY] = (
+ '/path/to/lucictx_%d.json' % self._test_counter)
+ else: # pragma: no cover
+ env[self._lucictx_client.ENV_KEY] = (
+ self._lucictx_client.new_context(**section_pb_values))
_add_to_context('env_prefixes', env_prefixes, _as_env_prefixes)
