commit 2a047c3e77c1dc47e236e284554fd39c01226d51
author Yiwei Zhang <yiwzhang@google.com> Fri Oct 30 19:15:27 2020
committer LUCI CQ <infra-scoped@luci-project-accounts.iam.gserviceaccount.com> Fri Oct 30 19:15:27 2020
tree 2fabd745b46aab1ddcda959c42e155bc88b4d2c8
parent ed160e8309749854f21123705a3bd5af592e5d49

[terminate] Handle GreenletExit exception

R=vadimsh

Bug: 1144073
Change-Id: I5161c3ddb7da19b3e656c903f65c1c24b137a51e
Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/recipes-py/+/2508692
Reviewed-by: Vadim Shtayura <vadimsh@chromium.org>
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
Auto-Submit: Yiwei Zhang <yiwzhang@google.com>

recipe_engine/internal/engine.py

diff --git a/recipe_engine/internal/engine.py b/recipe_engine/internal/engine.py
index 62bc1fc..84f40c6 100644
--- a/recipe_engine/internal/engine.py
+++ b/recipe_engine/internal/engine.py
@@ -37,7 +37,7 @@
 from .exceptions import RecipeUsageError, CrashEngine
 from .step_runner import Step
 from .resource_semaphore import ResourceWaiter
-from .global_shutdown import GLOBAL_SHUTDOWN
+from .global_shutdown import GLOBAL_SHUTDOWN, GLOBAL_TIMEOUT
 
 
 @attr.s(frozen=True, slots=True, repr=False)
@@ -579,6 +579,12 @@
         result.status = common_pb2.INFRA_FAILURE if (
           is_infra_failure) else common_pb2.FAILURE
         result.summary_markdown = ex.reason
+      except gevent.GreenletExit:
+        result.status = common_pb2.INFRA_FAILURE
+        if GLOBAL_TIMEOUT.ready():
+          result.summary_markdown = 'Recipe timed out'
+        else:
+          result.summary_markdown = 'Recipe was interrupted'
 
     # All other exceptions are reported to the user and are fatal.
     except Exception as ex:  # pylint: disable=broad-except

recipe_engine/internal/global_shutdown.py

diff --git a/recipe_engine/internal/global_shutdown.py b/recipe_engine/internal/global_shutdown.py
index 15792d1..f16d754 100644
--- a/recipe_engine/internal/global_shutdown.py
+++ b/recipe_engine/internal/global_shutdown.py
@@ -45,6 +45,12 @@
 # for test mode.
 GLOBAL_QUITQUITQUIT = gevent.event.Event()
 
+# GLOBAL_TIMEOUT is set if recipe terminates because of timeout.
+#
+# This event is only installed for real runs of the recipe; It blocks forever
+# for test mode.
+GLOBAL_TIMEOUT = gevent.event.Event()
+
 # UNKILLED_PGIDS is a global set of process groups which haven't been SIGKILL'd
 # yet.
 #
@@ -88,8 +94,11 @@
     if d.soft_deadline:
       now = time.time()
       if d.soft_deadline > now:
-        gevent.wait([GLOBAL_SHUTDOWN], timeout=(d.soft_deadline - now))
-        GLOBAL_SHUTDOWN.set()
+        ready = gevent.wait([GLOBAL_SHUTDOWN], timeout=(d.soft_deadline - now))
+        if not ready:
+          # wait ends because of hitting timeout
+          GLOBAL_TIMEOUT.set()
+          GLOBAL_SHUTDOWN.set()
     else:
       GLOBAL_SHUTDOWN.wait()
     LOG.info('Initiating GLOBAL_SHUTDOWN')