Enforce a single result per subtest for pytest results
Previously, if we got a test failure and an error during teardown, we'd end up with
multiple results for the same test. This just picks the final result for the test.
Differential Revision: https://phabricator.services.mozilla.com/D151007
bugzilla-url: https://bugzilla.mozilla.org/show_bug.cgi?id=1778083
gecko-commit: 228851107e0f9f76f38d2a9558cb377ce50ec52e
gecko-reviewers: webdriver-reviewers, jdescottes
diff --git a/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py b/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py
index 2da0813..f22d948 100644
--- a/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py
+++ b/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py
@@ -16,6 +16,7 @@
import os
import shutil
import tempfile
+from collections import OrderedDict
pytest = None
@@ -79,7 +80,8 @@
finally:
os.environ = old_environ
- return (harness.outcome, subtests.results)
+ subtests_results = [(key,) + value for (key, value) in subtests.results.items()]
+ return (harness.outcome, subtests_results)
class HarnessResultRecorder:
@@ -100,7 +102,7 @@
class SubtestResultRecorder:
def __init__(self):
- self.results = []
+ self.results = OrderedDict()
def pytest_runtest_logreport(self, report):
if report.passed and report.when == "call":
@@ -144,8 +146,15 @@
def record(self, test, status, message=None, stack=None):
if stack is not None:
stack = str(stack)
- new_result = (test.split("::")[-1], status, message, stack)
- self.results.append(new_result)
+ # Ensure we get a single result per subtest; pytest will sometimes
+ # call pytest_runtest_logreport more than once per test e.g. if
+ # it fails and then there's an error during teardown.
+ subtest_id = test.split("::")[-1]
+ if subtest_id in self.results and status == "PASS":
+ # This shouldn't happen, but never overwrite an existing result with PASS
+ return
+ new_result = (status, message, stack)
+ self.results[subtest_id] = new_result
class TemporaryDirectory:
