[futures] Add ability to kill greenlets.

This will allow us to implement patterns like the Goma Compiler Proxy
cleanly.

R=tandrii@chromium.org, vadimsh@chromium.org

Bug: 910369
Recipe-Nontrivial-Roll: skia
Change-Id: Iddd32a3339f114fe5fe7f86dc9d359a737f285aa
Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/recipes-py/+/1641963
Commit-Queue: Robbie Iannucci <iannucci@chromium.org>
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
diff --git a/README.recipes.md b/README.recipes.md
index 9213613..01d2ee6 100644
--- a/README.recipes.md
+++ b/README.recipes.md
@@ -1328,7 +1328,7 @@
 
 Provides access to the Recipe concurrency primitives.
 
-&emsp; **@staticmethod**<br>&mdash; **def [iwait](/recipe_modules/futures/api.py#202)(futures, timeout=None, count=None):**
+&emsp; **@staticmethod**<br>&mdash; **def [iwait](/recipe_modules/futures/api.py#215)(futures, timeout=None, count=None):**
 
 Iteratively yield up to `count` Futures as they become done.
 
@@ -1371,7 +1371,7 @@
 timeout or count. May also be used with a context manager to avoid
 leaking resources if you don't plan on consuming the entire iterable.
 
-&mdash; **def [make\_channel](/recipe_modules/futures/api.py#94)(self):**
+&mdash; **def [make\_channel](/recipe_modules/futures/api.py#107)(self):**
 
 Returns a single-slot communication device for passing data and control
 between concurrent functions.
@@ -1391,7 +1391,7 @@
 
 Channels will raise ValueError if used with @@@annotation@@@ mode.
 
-&mdash; **def [spawn](/recipe_modules/futures/api.py#119)(self, func, \*args, \*\*kwargs):**
+&mdash; **def [spawn](/recipe_modules/futures/api.py#132)(self, func, \*args, \*\*kwargs):**
 
 Prepares a Future to run `func(*args, **kwargs)` concurrently.
 
@@ -1424,7 +1424,7 @@
 
 Returns a Future of `func`'s result.
 
-&mdash; **def [spawn\_immediate](/recipe_modules/futures/api.py#158)(self, func, \*args, \*\*kwargs):**
+&mdash; **def [spawn\_immediate](/recipe_modules/futures/api.py#171)(self, func, \*args, \*\*kwargs):**
 
 Returns a Future to the concurrently running `func(*args, **kwargs)`.
 
@@ -1441,7 +1441,7 @@
 
 Returns a Future of `func`'s result.
 
-&emsp; **@staticmethod**<br>&mdash; **def [wait](/recipe_modules/futures/api.py#183)(futures, timeout=None, count=None):**
+&emsp; **@staticmethod**<br>&mdash; **def [wait](/recipe_modules/futures/api.py#196)(futures, timeout=None, count=None):**
 
 Blocks until `count` `futures` are done (or timeout occurs) then
 returns the list of done futures.
@@ -2925,11 +2925,11 @@
 
 [DEPS](/recipe_modules/futures/examples/background_helper.py#8): [futures](#recipe_modules-futures), [json](#recipe_modules-json), [path](#recipe_modules-path), [python](#recipe_modules-python), [raw\_io](#recipe_modules-raw_io), [step](#recipe_modules-step)
 
-&mdash; **def [RunSteps](/recipe_modules/futures/examples/background_helper.py#91)(api):**
+&mdash; **def [RunSteps](/recipe_modules/futures/examples/background_helper.py#78)(api):**
 
 &mdash; **def [manage\_helper](/recipe_modules/futures/examples/background_helper.py#21)(api, chn):**
 
-&emsp; **@contextmanager**<br>&mdash; **def [run\_helper](/recipe_modules/futures/examples/background_helper.py#64)(api):**
+&emsp; **@contextmanager**<br>&mdash; **def [run\_helper](/recipe_modules/futures/examples/background_helper.py#51)(api):**
 
 Runs the background helper.
 
diff --git a/recipe_engine/internal/engine.py b/recipe_engine/internal/engine.py
index 5657b3d..05d7106 100644
--- a/recipe_engine/internal/engine.py
+++ b/recipe_engine/internal/engine.py
@@ -308,7 +308,7 @@
 
       self._step_stack.append(_ActiveStep(ret, step_stream, False))
 
-      # _run_step should never raise an exception
+      # _run_step should never raise an exception, except for GreenletExit
       caught = _run_step(
           debug_log, ret, step_stream, self._step_runner, self._resource,
           step_config, self._environ, self._start_dir)
@@ -326,6 +326,10 @@
         # TODO(iannucci): Python3 incompatible.
         raise caught[0], caught[1], caught[2]
 
+      # If the step was cancelled, raise GreenletExit.
+      if ret.exc_result.was_cancelled:
+        raise gevent.GreenletExit()
+
       if ret.presentation.status == 'SUCCESS':
         return ret
 
@@ -501,6 +505,8 @@
     presentation.status = 'EXCEPTION'
     return
 
+  # TODO(iannucci): Add a real status for CANCELED?
+
   if (step_config.ok_ret is step_config.ALL_OK or
       exc_result.retcode in step_config.ok_ret):
     presentation.status = 'SUCCESS'
@@ -680,9 +686,13 @@
       def _blocked_on(amount):
         debug_log.write_line(
             '  waiting for %d millicores to be available' % amount)
-      with resource.cpu(step_config.cpu, _blocked_on):
-        step_data.exc_result = step_runner.run(
-            step_data.name_tokens, debug_log, rendered_step)
+      try:
+        with resource.cpu(step_config.cpu, _blocked_on):
+          step_data.exc_result = step_runner.run(
+              step_data.name_tokens, debug_log, rendered_step)
+      except gevent.GreenletExit:
+        # Greenlet was killed while waiting for CPU resource
+        step_data.exc_result = ExecutionResult(was_cancelled=True)
       if step_data.exc_result.retcode is not None:
         # Windows error codes such as 0xC0000005 and 0xC0000409 are much
         # easier to recognize and differentiate in hex. In order to print them
@@ -716,6 +726,8 @@
     exc_details.write_line('Step timed out.')
   if step_data.exc_result.had_exception:
     exc_details.write_line('Step had exception.')
+  if step_data.exc_result.was_cancelled:
+    exc_details.write_line('Step was cancelled.')
   exc_details.close()
 
   # Re-render the presentation status; If one of the output placeholders blew up
diff --git a/recipe_engine/internal/step_runner/subproc.py b/recipe_engine/internal/step_runner/subproc.py
index b067bd5..9d73358 100644
--- a/recipe_engine/internal/step_runner/subproc.py
+++ b/recipe_engine/internal/step_runner/subproc.py
@@ -6,6 +6,7 @@
 import signal
 import sys
 
+import attr
 import gevent
 from gevent import subprocess
 
@@ -282,28 +283,40 @@
 
     Should not raise an exception.
     """
-    # TODO(iannucci): This API changes in python3 to raise an exception on
-    # timeout.
-    proc.wait(timeout)
-    retcode = proc.poll()
-    debug_log.write_line('finished waiting for process, retcode %r' % retcode)
+    ret = ExecutionResult()
 
-    # TODO(iannucci): Make leaking subprocesses explicit (e.g. goma compiler
-    # daemon). Better, change deamons to be owned by a gevent Greenlet (so that
-    # we don't need to leak processes ever).
+    # We're about to do gevent-blocking operations (waiting on the subprocess)
+    # and so another greenlet could kill us; we guard all of these operations
+    # with a `try/except GreenletExit` to handle this and return an
+    # ExecutionResult(was_cancelled=True) in that case.
     #
-    # _kill(proc, gid)  # In case of leaked subprocesses or timeout.
-    if retcode is None:
-      debug_log.write_line('timeout! killing process group %r' % gid)
-      # Process timed out, kill it. Currently all uses of non-None timeout
-      # intend to actually kill the subprocess when the timeout pops.
-      _kill(proc, gid)
-      proc.wait()
+    # The engine will raise a new GreenletExit exception after processing this
+    # step.
+    try:
+      # TODO(iannucci): This API changes in python3 to raise an exception on
+      # timeout.
+      proc.wait(timeout)
+      ret = attr.evolve(ret, retcode=proc.poll())
+      debug_log.write_line(
+          'finished waiting for process, retcode %r' % ret.retcode)
 
-    if retcode is not None:
-      return ExecutionResult(retcode=proc.returncode)
+      # TODO(iannucci): Make leaking subprocesses explicit (e.g. goma compiler
+      # daemon). Better, change deamons to be owned by a gevent Greenlet (so that
+      # we don't need to leak processes ever).
+      #
+      # _kill(proc, gid)  # In case of leaked subprocesses or timeout.
+      if ret.retcode:
+        debug_log.write_line('timeout! killing process group %r' % gid)
+        # Process timed out, kill it. Currently all uses of non-None timeout
+        # intend to actually kill the subprocess when the timeout pops.
+        ret = attr.evolve(ret, retcode=_kill(proc, gid), had_timeout=True)
 
-    return ExecutionResult(had_timeout=True)
+    except gevent.GreenletExit:
+      debug_log.write_line(
+          'caught GreenletExit, killing process group %r' % (gid,))
+      ret = attr.evolve(ret, retcode=_kill(proc, gid), was_cancelled=True)
+
+    return ret
 
   @staticmethod
   def _reap_workers(workers, to_close, debug_log):
diff --git a/recipe_engine/step_data.py b/recipe_engine/step_data.py
index e840b4e..7db7773 100644
--- a/recipe_engine/step_data.py
+++ b/recipe_engine/step_data.py
@@ -37,6 +37,7 @@
   retcode = attr.ib(validator=attr_type((int, type(None))), default=None)
   had_timeout = attr.ib(validator=attr_type(bool), default=False)
   had_exception = attr.ib(validator=attr_type(bool), default=False)
+  was_cancelled = attr.ib(validator=attr_type(bool), default=False)
 
 
 @attr.s
diff --git a/recipe_modules/cipd/examples/full.expected/pkg_bad_file.json b/recipe_modules/cipd/examples/full.expected/pkg_bad_file.json
index 6c2ba1e..476b573 100644
--- a/recipe_modules/cipd/examples/full.expected/pkg_bad_file.json
+++ b/recipe_modules/cipd/examples/full.expected/pkg_bad_file.json
@@ -466,7 +466,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/cipd/examples/full.expected/pkg_bad_mode.json b/recipe_modules/cipd/examples/full.expected/pkg_bad_mode.json
index a71240c..972ac60 100644
--- a/recipe_modules/cipd/examples/full.expected/pkg_bad_mode.json
+++ b/recipe_modules/cipd/examples/full.expected/pkg_bad_mode.json
@@ -466,7 +466,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/cipd/examples/full.expected/pkg_bad_verfile.json b/recipe_modules/cipd/examples/full.expected/pkg_bad_verfile.json
index 7c7704f..035655c 100644
--- a/recipe_modules/cipd/examples/full.expected/pkg_bad_verfile.json
+++ b/recipe_modules/cipd/examples/full.expected/pkg_bad_verfile.json
@@ -466,7 +466,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/futures/api.py b/recipe_modules/futures/api.py
index 02e898c..614b428 100644
--- a/recipe_modules/futures/api.py
+++ b/recipe_modules/futures/api.py
@@ -48,7 +48,7 @@
     """Represents a unit of concurrent work.
 
     Modeled after Python 3's `concurrent.futures.Future`. We can expand this
-    API carefully as we need it (e.g. potentially adding `cancel`).
+    API carefully as we need it.
     """
 
     _greenlet = attr.ib(
@@ -74,6 +74,17 @@
       """Returns True iff this Future is no longer running."""
       return self._greenlet.dead
 
+    def cancel(self):
+      """Raises GreenletExit in the underlying greenlet.
+
+      If the greenlet is waiting on a subprocess (step), the subprocess will be
+      killed, and the step's ExecutionResult will have `was_cancelled=True`.
+
+      Does not block on the death of the greenlet.
+      Does not switch away from the current greenlet.
+      """
+      self._greenlet.kill()
+
     def exception(self, timeout=None):
       """Blocks until this Future is done, then returns (not raises) this
       Future's exception (or None if there was no exception).
diff --git a/recipe_modules/futures/examples/background_helper.expected/basic.json b/recipe_modules/futures/examples/background_helper.expected/basic.json
index 17edab9..ce5844c 100644
--- a/recipe_modules/futures/examples/background_helper.expected/basic.json
+++ b/recipe_modules/futures/examples/background_helper.expected/basic.json
@@ -43,19 +43,6 @@
     "name": "do something with live helper"
   },
   {
-    "cmd": [
-      "python",
-      "-u",
-      "RECIPE[recipe_engine::futures:examples/background_helper].resources/kill_helper.py",
-      "12345"
-    ],
-    "cpu": 0,
-    "name": "helper.kill helper",
-    "~followup_annotations": [
-      "@@@STEP_NEST_LEVEL@1@@@"
-    ]
-  },
-  {
     "name": "$result"
   }
 ]
\ No newline at end of file
diff --git a/recipe_modules/futures/examples/background_helper.py b/recipe_modules/futures/examples/background_helper.py
index 3b61495..4d6584d 100644
--- a/recipe_modules/futures/examples/background_helper.py
+++ b/recipe_modules/futures/examples/background_helper.py
@@ -34,17 +34,8 @@
           cpu=0, # always run the checker.
       ).json.output
     except api.step.StepFailure:
-      # Probably check StepFailure to see if it had a timeout, or was just
-      # a regular failure.
-      #
-      # TODO(iannucci): Improve this to not rely on exceptions.
-      #
-      # TODO(iannucci): if/when we add 'cancel' functionality to futures API,
-      # cancel the helper_future here. The recipe engine knows its pid and so
-      # could send it a signal without having to wait for `contact helper` to
-      # succeed.
-      #
-      # helper_future.cancel()
+      helper_future.cancel()
+      helper_future.result()
       chn.put(HELPER_TIMEOUT)
       return  # pragma: no cover
 
@@ -53,12 +44,8 @@
     # Now we wait on the channel to see when to shut down.
     chn.get()
 
-    # Again, when we add cancel functionality, we could use
-    # helper_future.cancel() here.
-    api.python(
-        'kill helper', api.resource('kill_helper.py'), [proc_data['pid']],
-        cpu=0,
-    )
+    helper_future.cancel()
+    helper_future.result()
 
 
 @contextmanager
diff --git a/recipe_modules/futures/examples/background_helper.resources/helper.py b/recipe_modules/futures/examples/background_helper.resources/helper.py
index 96562d5..8721500 100644
--- a/recipe_modules/futures/examples/background_helper.resources/helper.py
+++ b/recipe_modules/futures/examples/background_helper.resources/helper.py
@@ -6,6 +6,15 @@
 import os
 import sys
 import time
+import signal
+
+signal.signal(
+    (
+      signal.SIGBREAK  # pylint: disable=no-member
+      if sys.platform.startswith('win') else
+      signal.SIGTERM
+    ),
+    lambda _signum, _frame: sys.exit(0))
 
 with open(sys.argv[1], 'wb') as pid_file:
   json.dump({
diff --git a/recipe_modules/futures/examples/background_helper.resources/kill_helper.py b/recipe_modules/futures/examples/background_helper.resources/kill_helper.py
deleted file mode 100644
index c22de19..0000000
--- a/recipe_modules/futures/examples/background_helper.resources/kill_helper.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2019 The LUCI Authors. All rights reserved.
-# Use of this source code is governed under the Apache License, Version 2.0
-# that can be found in the LICENSE file.
-
-import os
-import sys
-import signal
-
-pid = int(sys.argv[1])
-if sys.platform.startswith('win'):
-  os.kill(pid, signal.CTRL_BREAK_EVENT)  # pylint: disable=no-member
-else:
-  os.kill(pid, signal.SIGTERM)
diff --git a/recipe_modules/properties/examples/full.expected/prop_wrong_type.json b/recipe_modules/properties/examples/full.expected/prop_wrong_type.json
index 0c31395..adaa082 100644
--- a/recipe_modules/properties/examples/full.expected/prop_wrong_type.json
+++ b/recipe_modules/properties/examples/full.expected/prop_wrong_type.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 691, in run_steps",
       "    ignore_unknown_fields=True))",
diff --git a/recipe_modules/step/examples/full.expected/deep_invalid_access.json b/recipe_modules/step/examples/full.expected/deep_invalid_access.json
index e808d81..e05e5cb 100644
--- a/recipe_modules/step/examples/full.expected/deep_invalid_access.json
+++ b/recipe_modules/step/examples/full.expected/deep_invalid_access.json
@@ -158,7 +158,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/step/examples/full.expected/exceptional.json b/recipe_modules/step/examples/full.expected/exceptional.json
index 139d6df..2591639 100644
--- a/recipe_modules/step/examples/full.expected/exceptional.json
+++ b/recipe_modules/step/examples/full.expected/exceptional.json
@@ -106,7 +106,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/step/examples/full.expected/extra_junk.json b/recipe_modules/step/examples/full.expected/extra_junk.json
index e58ef4b..cc118a8 100644
--- a/recipe_modules/step/examples/full.expected/extra_junk.json
+++ b/recipe_modules/step/examples/full.expected/extra_junk.json
@@ -155,7 +155,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
@@ -165,7 +165,7 @@
       "    return callable_obj(*props, **additional_args)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_modules/step/examples/full.py\", line 130, in RunSteps",
       "    result.json = \"hi\"",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/step_data.py\", line 330, in __setattr__",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/step_data.py\", line 331, in __setattr__",
       "    (name, self.name))",
       "ValueError: Cannot assign to 'json' on finalized StepData from step 'no-op'"
     ]
diff --git a/recipe_modules/step/examples/full.expected/invalid_access.json b/recipe_modules/step/examples/full.expected/invalid_access.json
index 5c3ac28..7659f44 100644
--- a/recipe_modules/step/examples/full.expected/invalid_access.json
+++ b/recipe_modules/step/examples/full.expected/invalid_access.json
@@ -155,7 +155,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
@@ -165,7 +165,7 @@
       "    return callable_obj(*props, **additional_args)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_modules/step/examples/full.py\", line 120, in RunSteps",
       "    _ = result.json.output",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/step_data.py\", line 338, in __getattr__",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/step_data.py\", line 339, in __getattr__",
       "    'StepData from step %r has no attribute %r.' % (self.name, name))",
       "AttributeError: StepData from step 'no-op' has no attribute 'json'."
     ]
diff --git a/recipe_modules/url/tests/validate_url.expected/invalid_scheme.json b/recipe_modules/url/tests/validate_url.expected/invalid_scheme.json
index 1e8e989..eb8b96c 100644
--- a/recipe_modules/url/tests/validate_url.expected/invalid_scheme.json
+++ b/recipe_modules/url/tests/validate_url.expected/invalid_scheme.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/url/tests/validate_url.expected/no_host.json b/recipe_modules/url/tests/validate_url.expected/no_host.json
index 75ec0b2..000eca1 100644
--- a/recipe_modules/url/tests/validate_url.expected/no_host.json
+++ b/recipe_modules/url/tests/validate_url.expected/no_host.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipe_modules/url/tests/validate_url.expected/no_scheme.json b/recipe_modules/url/tests/validate_url.expected/no_scheme.json
index 1e8e989..eb8b96c 100644
--- a/recipe_modules/url/tests/validate_url.expected/no_scheme.json
+++ b/recipe_modules/url/tests/validate_url.expected/no_scheme.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipes/engine_tests/expect_exception.expected/basic.json b/recipes/engine_tests/expect_exception.expected/basic.json
index ce89214..3deb38e 100644
--- a/recipes/engine_tests/expect_exception.expected/basic.json
+++ b/recipes/engine_tests/expect_exception.expected/basic.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipes/engine_tests/placeholder_exception.expected/basic.json b/recipes/engine_tests/placeholder_exception.expected/basic.json
index 7b00f9a..0c72a91 100644
--- a/recipes/engine_tests/placeholder_exception.expected/basic.json
+++ b/recipes/engine_tests/placeholder_exception.expected/basic.json
@@ -17,9 +17,9 @@
       "@@@STEP_LOG_LINE@$debug@  <BadPlaceholder>@@@",
       "@@@STEP_LOG_LINE@$debug@Unhandled exception:@@@",
       "@@@STEP_LOG_LINE@$debug@Traceback (most recent call last):@@@",
-      "@@@STEP_LOG_LINE@$debug@  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 673, in _run_step@@@",
+      "@@@STEP_LOG_LINE@$debug@  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 679, in _run_step@@@",
       "@@@STEP_LOG_LINE@$debug@    base_environ, start_dir)@@@",
-      "@@@STEP_LOG_LINE@$debug@  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 569, in _render_config@@@",
+      "@@@STEP_LOG_LINE@$debug@  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 575, in _render_config@@@",
       "@@@STEP_LOG_LINE@$debug@    cmd.extend(itm.render(step_runner.placeholder(name_tokens, itm)))@@@",
       "@@@STEP_LOG_LINE@$debug@  File \"RECIPE_REPO[recipe_engine]/recipes/engine_tests/placeholder_exception.py\", line 16, in render@@@",
       "@@@STEP_LOG_LINE@$debug@    raise Exception(\"EXPLOSION\")@@@",
@@ -36,7 +36,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
@@ -52,9 +52,9 @@
       "    step_test_data=step_test_data,",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", line 235, in run_step",
       "    return self._engine.run_step(step)",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 673, in _run_step",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 679, in _run_step",
       "    base_environ, start_dir)",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 569, in _render_config",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 575, in _render_config",
       "    cmd.extend(itm.render(step_runner.placeholder(name_tokens, itm)))",
       "  File \"RECIPE_REPO[recipe_engine]/recipes/engine_tests/placeholder_exception.py\", line 16, in render",
       "    raise Exception(\"EXPLOSION\")",
diff --git a/recipes/engine_tests/undeclared_method.expected/attribute.json b/recipes/engine_tests/undeclared_method.expected/attribute.json
index 2d57a57..75668ea 100644
--- a/recipes/engine_tests/undeclared_method.expected/attribute.json
+++ b/recipes/engine_tests/undeclared_method.expected/attribute.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipes/engine_tests/undeclared_method.expected/from_recipe.json b/recipes/engine_tests/undeclared_method.expected/from_recipe.json
index 20e318a..f5bebf8 100644
--- a/recipes/engine_tests/undeclared_method.expected/from_recipe.json
+++ b/recipes/engine_tests/undeclared_method.expected/from_recipe.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",
diff --git a/recipes/engine_tests/undeclared_method.expected/module.json b/recipes/engine_tests/undeclared_method.expected/module.json
index 3978655..e67f11d 100644
--- a/recipes/engine_tests/undeclared_method.expected/module.json
+++ b/recipes/engine_tests/undeclared_method.expected/module.json
@@ -7,7 +7,7 @@
       "The recipe has crashed at point 'Uncaught exception'!",
       "",
       "Traceback (most recent call last):",
-      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 450, in run_steps",
+      "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", line 454, in run_steps",
       "    raw_result = recipe_obj.run_steps(api, engine)",
       "  File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", line 706, in run_steps",
       "    properties_def, api=api)",