Attempt to avoid timing out after successful iOS test runs by catching SIGTERM.
The test runner is sent SIGTERM typically 30s before the hard timeout
of a test run; knowing that, this CL attempts to catch that SIGTERM
and force kill a hung test process, allowing it to exit more gracefully.
(See:
https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_bot/bot_code/bot_main.py?l=891&rcl=599029f4b1abebfa812f3b2b9df9a5c6c7acbca4)
This should not affect the success/failure of a run, as that is
determined by "TEST EXECUTION SUCCEEDED" appearing in the stdout
for a test run. Return code is noted but not used for success.
I also add some minor logging in this CL for clarity.
Change-Id: I5ec882862db6693f09f69f6e191261f2c6ba43f9
Bug: 898549
Reviewed-on: https://chromium-review.googlesource.com/c/1448684
Commit-Queue: ericale <ericale@chromium.org>
Reviewed-by: Sergey Berezin <sergeyberezin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#628935}diff --git a/ios/build/bots/scripts/test_runner.py b/ios/build/bots/scripts/test_runner.py
index 09052c9..88307b1f 100644
--- a/ios/build/bots/scripts/test_runner.py
+++ b/ios/build/bots/scripts/test_runner.py
@@ -139,7 +139,8 @@
class ShardingDisabledError(TestRunnerError):
"""Temporary error indicating that sharding is not yet implemented."""
def __init__(self):
- super(ShardingDisabledError, self).__init__('Sharding has not been implemented!')
+ super(ShardingDisabledError, self).__init__(
+ 'Sharding has not been implemented!')
def get_kif_test_filter(tests, invert=False):
@@ -456,6 +457,31 @@
"""
raise NotImplementedError
+ def set_sigterm_handler(self, handler):
+ """Sets the SIGTERM handler for the test runner.
+
+ This is its own separate function so it can be mocked in tests.
+
+ Args:
+ handler: The handler to be called when a SIGTERM is caught
+
+ Returns:
+ The previous SIGTERM handler for the test runner.
+ """
+ return signal.signal(signal.SIGTERM, handler)
+
+ def handle_sigterm(self, proc):
+ """Handles a SIGTERM sent while a test command is executing.
+
+ Will SIGKILL the currently executing test process, then
+ attempt to exit gracefully.
+
+ Args:
+ proc: The currently executing test process.
+ """
+ print "Sigterm caught during test run. Killing test process."
+ proc.kill()
+
def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
@@ -497,6 +523,9 @@
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
+ old_handler = self.set_sigterm_handler(
+ lambda _signum, _frame: self.handle_sigterm(proc))
+
while True:
line = proc.stdout.readline()
if not line:
@@ -506,7 +535,10 @@
print line
sys.stdout.flush()
+ print "Waiting for test process to terminate."
proc.wait()
+ print "Test process terminated."
+ self.set_sigterm_handler(old_handler)
sys.stdout.flush()
returncode = proc.returncode
@@ -1119,6 +1151,8 @@
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
+ old_handler = self.set_sigterm_handler(
+ lambda _signum, _frame: self.handle_sigterm(proc))
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
@@ -1135,6 +1169,7 @@
sys.stdout.flush()
proc.wait()
+ self.set_sigterm_handler(old_handler)
sys.stdout.flush()
self.wprgo_stop()
diff --git a/ios/build/bots/scripts/test_runner_test.py b/ios/build/bots/scripts/test_runner_test.py
index 446c43fc..1da6292 100755
--- a/ios/build/bots/scripts/test_runner_test.py
+++ b/ios/build/bots/scripts/test_runner_test.py
@@ -127,6 +127,8 @@
lambda _: 'fake-bundle-id')
self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
self.mock(os.path, 'exists', lambda _: True)
+ self.mock(test_runner.TestRunner, 'set_sigterm_handler',
+ lambda self, handler: 0)
def test_app_not_found(self):
"""Ensures AppNotFoundError is raised."""
@@ -361,9 +363,14 @@
lambda _: 'fake-bundle-id')
self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
self.mock(os.path, 'exists', lambda _: True)
- self.mock(test_runner.SimulatorTestRunner, 'getSimulator', lambda _: 'fake-id')
- self.mock(test_runner.SimulatorTestRunner, 'deleteSimulator', lambda a, b: True)
- self.mock(test_runner.WprProxySimulatorTestRunner, 'copy_trusted_certificate', lambda a, b: True)
+ self.mock(test_runner.TestRunner, 'set_sigterm_handler',
+ lambda self, handler: 0)
+ self.mock(test_runner.SimulatorTestRunner, 'getSimulator',
+ lambda _: 'fake-id')
+ self.mock(test_runner.SimulatorTestRunner, 'deleteSimulator',
+ lambda a, b: True)
+ self.mock(test_runner.WprProxySimulatorTestRunner,
+ 'copy_trusted_certificate', lambda a, b: True)
def test_replay_path_not_found(self):
"""Ensures ReplayPathNotFoundError is raised."""
@@ -466,8 +473,10 @@
'xcode-build',
'out-dir',
)
- self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_start', lambda a,b: None)
- self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_stop', lambda _: None)
+ self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_start',
+ lambda a,b: None)
+ self.mock(test_runner.WprProxySimulatorTestRunner, 'wprgo_stop',
+ lambda _: None)
self.mock(os.path, 'isfile', lambda _: True)
self.mock(glob, 'glob', lambda _: ["file1", "file2"])
