Support expected-fail-message per RFC 165 (#42406)
Added a function to read "expected-fail-message" from metadata, and
reset expected FAIL/PRECONDITION_FAILED to unexpected when a mismatch
in the output message is found.
Bug: Chromium: 1481730
diff --git a/tools/wptrunner/wptrunner/testrunner.py b/tools/wptrunner/wptrunner/testrunner.py
index 05e0436..bea76e1 100644
--- a/tools/wptrunner/wptrunner/testrunner.py
+++ b/tools/wptrunner/wptrunner/testrunner.py
@@ -682,6 +682,14 @@
is_unexpected = expected != result.status and result.status not in known_intermittent
is_expected_notrun = (expected == "NOTRUN" or "NOTRUN" in known_intermittent)
+ if not is_unexpected and result.status in ["FAIL", "PRECONDITION_FAILED"]:
+ # subtest is expected FAIL or expected PRECONDITION_FAILED,
+ # change result to unexpected if expected_fail_message does not
+ # match
+ expected_fail_message = test.expected_fail_message(result.name)
+ if expected_fail_message is not None and result.message != expected_fail_message:
+ is_unexpected = True
+
if is_unexpected:
subtest_unexpected = True
diff --git a/tools/wptrunner/wptrunner/wpttest.py b/tools/wptrunner/wptrunner/wpttest.py
index ba038a2..6df1c4c 100644
--- a/tools/wptrunner/wptrunner/wpttest.py
+++ b/tools/wptrunner/wptrunner/wpttest.py
@@ -392,6 +392,19 @@
prefs.update(meta_prefs)
return prefs
+ def expected_fail_message(self, subtest):
+ if subtest is None:
+ return None
+
+ metadata = self._get_metadata(subtest)
+ if metadata is None:
+ return None
+
+ try:
+ return metadata.get("expected-fail-message")
+ except KeyError:
+ return None
+
def expected(self, subtest=None):
if subtest is None:
default = self.result_cls.default_expected
