Linter should enforce Tests marked Slow or WontFix aren't allowed in TestExpectations
Tests marked Slow are only allowed in SlowTests,
Tests marked WontFix are only allowed in NeverFixTests,
These should not go into the TestExpectations file given there are
reserved files for them.
BUG=559390
Review URL: https://codereview.chromium.org/1491143003
Cr-Commit-Position: refs/heads/master@{#371179}
diff --git a/third_party/WebKit/LayoutTests/NeverFixTests b/third_party/WebKit/LayoutTests/NeverFixTests
index e450439..214ece0b 100644
--- a/third_party/WebKit/LayoutTests/NeverFixTests
+++ b/third_party/WebKit/LayoutTests/NeverFixTests
@@ -221,3 +221,13 @@
[ Mac ] imported/web-platform-tests/html/dom/elements/requirements-relating-to-bidirectional-algorithm-formatting-characters/dir-isolation-009a.html [ WontFix ]
[ Mac ] imported/web-platform-tests/html/dom/elements/requirements-relating-to-bidirectional-algorithm-formatting-characters/dir-isolation-009b.html [ WontFix ]
[ Mac ] imported/web-platform-tests/html/dom/elements/requirements-relating-to-bidirectional-algorithm-formatting-characters/dir-isolation-009c.html [ WontFix ]
+
+# We could fix this test for us and upstream it if the test shell user agent
+# would let us differentiate test_shell and WebKit DumpTreeNode.
+crbug.com/7482 [ Win Mac ] http/tests/misc/timer-vs-loading.html [ WontFix ]
+
+# These tests are too slow with our MESA backend. We can re-enable when we have
+# bots running tests on real hardware. Don't want to just delete these because they
+# pass in the virtual test suite version.
+webkit.org/b/49629 compositing/lots-of-img-layers.html [ WontFix ]
+webkit.org/b/49629 compositing/lots-of-img-layers-with-opacity.html [ WontFix ]
diff --git a/third_party/WebKit/LayoutTests/SlowTests b/third_party/WebKit/LayoutTests/SlowTests
index 0290774b..a09d05b 100644
--- a/third_party/WebKit/LayoutTests/SlowTests
+++ b/third_party/WebKit/LayoutTests/SlowTests
@@ -164,6 +164,12 @@
crbug.com/453312 html5lib/generated/run-tests20-data.html [ Slow ]
crbug.com/453312 html5lib/generated/run-webkit01-data.html [ Slow ]
crbug.com/453312 html5lib/webkit-resumer.html [ Slow ]
+
+crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/encoding/api-invalid-label.html [ Slow ]
+crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_tests10.html [ Slow ]
+crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_webkit01.html [ Slow ]
+crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_domjs-unsafe.html [ Slow ]
+
crbug.com/490511 imported/web-platform-tests/html/syntax/parsing/html5lib_entities01.html [ Slow ]
crbug.com/490511 imported/web-platform-tests/html/syntax/parsing/html5lib_template.html [ Slow ]
crbug.com/490511 imported/web-platform-tests/html/syntax/parsing/html5lib_tests1.html [ Slow ]
@@ -214,6 +220,27 @@
crbug.com/358313 virtual/gpu/fast/canvas/webgl/webgl-large-texture.html [ Slow ]
+crbug.com/522646 http/tests/media/encrypted-media/encrypted-media-encrypted-event-different-origin.html [ Slow ]
+crbug.com/411164 [ Win ] http/tests/security/powerfulFeatureRestrictions/serviceworker-on-insecure-origin.html [ Slow ]
+crbug.com/510337 inspector/console/console-format.html [ Slow ]
+crbug.com/357427 http/tests/workers/terminate-during-sync-operation-file.html [ Slow ]
+crbug.com/357427 http/tests/workers/terminate-during-sync-operation-filesystem.html [ Slow ]
+crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/transform-interpolation.html [ Slow ]
+crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/webkit-transform-interpolation.html [ Slow ]
+crbug.com/539725 [ Linux Debug ] inspector/elements/shadow/elements-panel-shadow-selection-on-refresh.html [ Slow ]
+crbug.com/402379 [ Win7 Debug ] storage/indexeddb/cursor-continue-validity.html [ Slow ]
+crbug.com/402379 [ Win7 Debug ] storage/indexeddb/mozilla/indexes.html [ Slow ]
+crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-expansion-preserved-when-sorting.html [ Pass Slow ]
+crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-sorting.html [ Slow ]
+crbug.com/440452 virtual/display_list_2d_canvas/fast/canvas/canvas-partial-invalidation-zoomed.html [ Slow ]
+crbug.com/480769 http/tests/inspector/service-workers/service-workers-redundant.html [ Slow ]
+crbug.com/480769 http/tests/inspector/service-workers/service-worker-agents.html [ Slow ]
+crbug.com/504703 inspector-protocol/debugger/debugger-step-into-dedicated-worker.html [ Slow ]
+crbug.com/535478 [ Win ] virtual/threaded/inspector/tracing/decode-resize.html [ Slow ]
+crbug.com/548765 http/tests/inspector/console-fetch-logging.html [ Slow ]
+
+crbug.com/419993 [ Debug ] fast/css/giant-stylesheet-crash.html [ Slow ]
+
crbug.com/331186 [ Debug ] css3/flexbox/position-absolute-child.html [ Slow ]
crbug.com/362509 [ Debug ] fast/parser/xml-error-adopted.xml [ Slow ]
@@ -240,3 +267,7 @@
crbug.com/529345 [ Win10 ] paint/masks/fieldset-mask.html [ Slow ]
crbug.com/356957 editing/spelling/spellcheck-editable-on-focus.html [ Slow ]
+crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-root-frame-animates.html [ Slow ]
+crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-animates.html [ Slow ]
+crbug.com/570656 [ Mac ] fast/writing-mode/Kusa-Makura-background-canvas.html [ Slow ]
+crbug.com/570656 [ Mac ] fast/text/unicode-fallback-font.html [ Slow ]
diff --git a/third_party/WebKit/LayoutTests/TestExpectations b/third_party/WebKit/LayoutTests/TestExpectations
index c732225c..b9af3ba 100644
--- a/third_party/WebKit/LayoutTests/TestExpectations
+++ b/third_party/WebKit/LayoutTests/TestExpectations
@@ -371,7 +371,7 @@
crbug.com/410145 [ Win ] fast/table/column-in-inline.html [ Failure ]
-crbug.com/411164 [ Win ] http/tests/security/powerfulFeatureRestrictions/serviceworker-on-insecure-origin.html [ Pass Slow ]
+crbug.com/411164 [ Win ] http/tests/security/powerfulFeatureRestrictions/serviceworker-on-insecure-origin.html [ Pass ]
# This batch is flaky:
crbug.com/310679 [ Mac ] virtual/pointerevent/fast/events/scale-and-scroll-iframe-body.html [ Failure ]
@@ -806,16 +806,16 @@
crbug.com/390377 [ Release ] fast/dom/private_script_unittest.html [ Skip ]
-crbug.com/357427 http/tests/workers/terminate-during-sync-operation-file.html [ Crash Pass Slow Timeout ]
-crbug.com/357427 http/tests/workers/terminate-during-sync-operation-filesystem.html [ Crash Pass Slow Timeout ]
+crbug.com/357427 http/tests/workers/terminate-during-sync-operation-file.html [ Crash Pass Timeout ]
+crbug.com/357427 http/tests/workers/terminate-during-sync-operation-filesystem.html [ Crash Pass Timeout ]
crbug.com/564403 http/tests/inspector/service-workers/service-worker-manager.html [ Failure Pass ]
crbug.com/306222 fast/hidpi/image-srcset-relative-svg-canvas.html [ Skip ]
crbug.com/306222 fast/hidpi/image-srcset-relative-svg-canvas-2x.html [ Skip ]
-crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/transform-interpolation.html [ Pass Crash Slow ]
-crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/webkit-transform-interpolation.html [ Pass Crash Slow ]
+crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/transform-interpolation.html [ Pass Crash ]
+crbug.com/364250 [ Debug ] virtual/threaded/animations/interpolation/webkit-transform-interpolation.html [ Pass Crash ]
crbug.com/467477 fast/multicol/vertical-rl/nested-columns.html [ Failure ]
@@ -845,20 +845,20 @@
crbug.com/528062 [ Win ] http/tests/security/xssAuditor/cached-frame.html [ Failure ]
crbug.com/528062 [ Win ] http/tests/security/xssAuditor/chunked-big-script.html [ Failure ]
-crbug.com/539725 [ Linux Debug ] inspector/elements/shadow/elements-panel-shadow-selection-on-refresh.html [ Slow Failure ]
+crbug.com/539725 [ Linux Debug ] inspector/elements/shadow/elements-panel-shadow-selection-on-refresh.html [ Failure ]
crbug.com/392706 virtual/threaded/animations/play-state-initially-paused.html [ Failure ]
-crbug.com/402379 [ Win7 Debug ] storage/indexeddb/cursor-continue-validity.html [ Pass Slow ]
-crbug.com/402379 [ Win7 Debug ] storage/indexeddb/mozilla/indexes.html [ Pass Slow ]
+crbug.com/402379 [ Win7 Debug ] storage/indexeddb/cursor-continue-validity.html [ Pass ]
+crbug.com/402379 [ Win7 Debug ] storage/indexeddb/mozilla/indexes.html [ Pass ]
-crbug.com/419993 [ Debug ] fast/css/giant-stylesheet-crash.html [ Pass Slow ]
+crbug.com/419993 [ Debug ] fast/css/giant-stylesheet-crash.html [ Pass ]
# Part of a larger issue referenced in the bug. This specific issue will be fixed shortly.
crbug.com/408159 accessibility/is-ignored-change-sends-notification.html [ Timeout ]
-crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-expansion-preserved-when-sorting.html [ Pass Slow ]
-crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-sorting.html [ Pass Slow ]
+crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-expansion-preserved-when-sorting.html [ Pass ]
+crbug.com/504706 [ Linux ] inspector/profiler/heap-snapshot-containment-sorting.html [ Pass ]
# Disabled until the test plugin is fixed.
crbug.com/363099 [ Win ] plugins/nested-plugin-objects.html [ Failure ]
@@ -896,7 +896,7 @@
crbug.com/385014 accessibility/canvas-fallback-content-2.html [ Failure ]
-crbug.com/440452 virtual/display_list_2d_canvas/fast/canvas/canvas-partial-invalidation-zoomed.html [ Pass Slow Timeout ]
+crbug.com/440452 virtual/display_list_2d_canvas/fast/canvas/canvas-partial-invalidation-zoomed.html [ Pass Timeout ]
crbug.com/364614 [ Mac ] fast/scroll-behavior/overflow-scroll-animates.html [ Skip ]
crbug.com/364614 [ Mac ] virtual/threaded/fast/scroll-behavior/overflow-scroll-animates.html [ Skip ]
@@ -907,8 +907,8 @@
crbug.com/574283 [ Mac ] virtual/threaded/fast/scroll-behavior/smooth-scroll/fixed-background-in-iframe.html [ Skip ]
-crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-root-frame-animates.html [ Pass Slow Timeout ]
-crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-animates.html [ Pass Slow Timeout ]
+crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-root-frame-animates.html [ Pass Timeout ]
+crbug.com/552556 [ Win Linux ] virtual/threaded/fast/scroll-behavior/overflow-scroll-animates.html [ Pass Timeout ]
crbug.com/552556 virtual/threaded/fast/scroll-behavior/smooth-scroll/mousewheel-scroll.html [ Pass Failure ]
crbug.com/524596 paint/invalidation/composited-non-stacking-context-descendant-change-color.html [ Failure ]
@@ -919,8 +919,8 @@
crbug.com/425113 svg/clip-path/clip-path-multiple-children.svg [ Failure ]
-crbug.com/480769 http/tests/inspector/service-workers/service-workers-redundant.html [ Crash Pass Slow Failure ]
-crbug.com/480769 http/tests/inspector/service-workers/service-worker-agents.html [ Crash Pass Timeout Slow ]
+crbug.com/480769 http/tests/inspector/service-workers/service-workers-redundant.html [ Crash Pass Failure ]
+crbug.com/480769 http/tests/inspector/service-workers/service-worker-agents.html [ Crash Pass Timeout ]
crbug.com/569901 [ Debug ] http/tests/serviceworker/navigation-redirect.html [ Pass Failure ]
@@ -1043,7 +1043,6 @@
crbug.com/501659 http/tests/security/xss-DENIED-xml-external-entity.xhtml [ Failure ]
crbug.com/501659 fast/css/stylesheet-candidate-nodes-crash.xhtml [ Failure ]
-crbug.com/504703 inspector-protocol/debugger/debugger-step-into-dedicated-worker.html [ Slow ]
# Slow expected on debug builds
crbug.com/445194 [ Debug ] fast/dom/shadow/focus-controller-recursion-crash.html [ Skip ]
@@ -1307,7 +1306,7 @@
crbug.com/531286 virtual/gpu/fast/canvas/yuv-video-on-accelerated-canvas.html [ Failure ]
-crbug.com/535478 [ Win ] virtual/threaded/inspector/tracing/decode-resize.html [ Slow Pass Failure ]
+crbug.com/535478 [ Win ] virtual/threaded/inspector/tracing/decode-resize.html [ Pass Failure ]
crbug.com/524646 [ Mac10.10 ] fast/dom/shadow/shadowdom-for-button.html [ Failure ]
@@ -1354,7 +1353,7 @@
crbug.com/570391 inspector/sources/debugger/promise-events.html [ NeedsManualRebaseline ]
-crbug.com/548765 http/tests/inspector/console-fetch-logging.html [ Failure Pass Slow ]
+crbug.com/548765 http/tests/inspector/console-fetch-logging.html [ Failure Pass ]
crbug.com/549314 [ XP ] inspector-protocol/layout-fonts/unicode-range-combining-chars-fallback.html [ Failure ]
crbug.com/573287 inspector/console/console-dir.html [ NeedsManualRebaseline ]
@@ -1581,12 +1580,6 @@
crbug.com/577380 [ Linux Debug ] http/tests/serviceworker/chromium/registration-stress.html [ Failure ]
crbug.com/577380 [ Linux Debug ] http/tests/serviceworker/fetch-request-redirect.html [ Failure ]
-crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/encoding/api-invalid-label.html [ Slow ]
-crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_entities01.html [ Slow ]
-crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_tests10.html [ Slow ]
-crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_webkit01.html [ Slow ]
-crbug.com/577381 [ Linux Debug ] imported/web-platform-tests/html/syntax/parsing/html5lib_domjs-unsafe.html [ Slow ]
-
crbug.com/577746 [ Linux Debug ] compositing/iframes/iframe-in-composited-layer.html [ Pass Failure ]
crbug.com/516751 css3/blending/effect-background-blend-mode-stacking.html [ NeedsRebaseline ]
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_tests_mover_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_tests_mover_unittest.py
index 7be0bba..e888a706 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_tests_mover_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_tests_mover_unittest.py
@@ -49,7 +49,7 @@
host.filesystem.write_text_file(self._absolute_path('VirtualTestSuites'), '[]')
host.filesystem.write_text_file(self._absolute_path('TestExpectations'), """
crbug.com/42 [ Debug ] origin/path/test.html [ Pass Timeout Failure ]
-crbug.com/42 [ Win ] origin/path [ Slow ]
+crbug.com/42 [ Win ] origin/path [ Crash ]
crbug.com/42 [ Release ] origin [ Crash ]
""")
host.filesystem.write_text_file(self._absolute_path('existing_directory_with_contents', 'test.html'), '')
@@ -170,7 +170,7 @@
def test_directory_expectation_is_updated(self):
self._mover.move('origin/path', 'destination')
self.assertFalse('origin/path' in self._filesystem.read_text_file(self._port._absolute_path('TestExpectations')))
- self.assertTrue('crbug.com/42 [ Win ] destination [ Slow ]' in self._filesystem.read_text_file(self._port._absolute_path('TestExpectations')))
+ self.assertTrue('crbug.com/42 [ Win ] destination [ Crash ]' in self._filesystem.read_text_file(self._port._absolute_path('TestExpectations')))
def test_expectation_is_added_when_subdirectory_moved(self):
self._mover.move('origin/path', 'destination')
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
index d49fe4c4..a365f67 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
@@ -368,6 +368,12 @@
if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])):
warnings.append('A test marked Skip or WontFix must not have other expectations.')
+ if 'SLOW' in expectations and 'SlowTests' not in filename:
+ warnings.append('SLOW tests should ony be added to SlowTests and not to TestExpectations.')
+
+ if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'StaleTestExpectations' not in filename):
+ warnings.append('WONTFIX tests should ony be added to NeverFixTests or StaleTestExpectations and not to TestExpectations.')
+
if not expectations and not has_unrecognized_expectation:
warnings.append('Missing expectations.')
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py
index 3fb7006..9d667f2 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py
@@ -66,12 +66,12 @@
def get_basic_expectations(self):
return """
Bug(test) failures/expected/text.html [ Failure ]
-Bug(test) failures/expected/crash.html [ WontFix ]
+Bug(test) failures/expected/crash.html [ Crash ]
Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
-Bug(test) failures/expected/image_checksum.html [ WontFix ]
-Bug(test) failures/expected/image.html [ WontFix Mac ]
+Bug(test) failures/expected/image_checksum.html [ Crash ]
+Bug(test) failures/expected/image.html [ Crash Mac ]
"""
def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
@@ -97,7 +97,7 @@
def test_basic(self):
self.parse_exp(self.get_basic_expectations())
self.assert_exp('failures/expected/text.html', FAIL)
- self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX, SKIP])
+ self.assert_exp_list('failures/expected/image_checksum.html', [CRASH])
self.assert_exp('passes/text.html', PASS)
self.assert_exp('failures/expected/image.html', PASS)
@@ -162,13 +162,13 @@
# This test checks unknown tests are not present in the
# expectations and that known test part of a test category is
# present in the expectations.
- exp_str = 'Bug(x) failures/expected [ WontFix ]'
+ exp_str = 'Bug(x) failures/expected [ CRASH ]'
self.parse_exp(exp_str)
test_name = 'failures/expected/unknown-test.html'
unknown_test = test_name
self.assertRaises(KeyError, self._exp.get_expectations,
unknown_test)
- self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
+ self.assert_exp_list('failures/expected/crash.html', [PASS])
def test_get_expectations_string(self):
self.parse_exp(self.get_basic_expectations())
@@ -183,7 +183,7 @@
def test_get_test_set(self):
# Handle some corner cases for this routine not covered by other tests.
self.parse_exp(self.get_basic_expectations())
- s = self._exp.get_test_set(WONTFIX)
+ s = self._exp.get_test_set(CRASH)
self.assertEqual(s, set(['failures/expected/crash.html', 'failures/expected/image_checksum.html']))
def test_needs_rebaseline_reftest(self):
@@ -265,9 +265,9 @@
self.assertTrue(match('failures/expected/text.html', FAIL, False))
self.assertFalse(match('failures/expected/text.html', CRASH, True))
self.assertFalse(match('failures/expected/text.html', CRASH, False))
- self.assertTrue(match('failures/expected/image_checksum.html', PASS, True))
- self.assertTrue(match('failures/expected/image_checksum.html', PASS, False))
- self.assertTrue(match('failures/expected/crash.html', PASS, False))
+ self.assertFalse(match('failures/expected/image_checksum.html', PASS, True))
+ self.assertFalse(match('failures/expected/image_checksum.html', PASS, False))
+ self.assertFalse(match('failures/expected/crash.html', PASS, False))
self.assertTrue(match('failures/expected/needsrebaseline.html', TEXT, True))
self.assertFalse(match('failures/expected/needsrebaseline.html', CRASH, True))
self.assertTrue(match('failures/expected/needsmanualrebaseline.html', TEXT, True))
@@ -304,12 +304,12 @@
test_name2 = 'passes/text.html'
expectations_dict = OrderedDict()
- expectations_dict['expectations'] = "Bug(x) %s [ Failure ]\nBug(x) %s [ Slow ]\n" % (test_name1, test_name2)
+ expectations_dict['expectations'] = "Bug(x) %s [ Failure ]\nBug(x) %s [ Crash ]\n" % (test_name1, test_name2)
self._port.expectations_dict = lambda: expectations_dict
expectations = TestExpectations(self._port, self.get_basic_tests())
self.assertEqual(expectations.get_expectations(test_name1), set([FAIL]))
- self.assertEqual(expectations.get_expectations(test_name2), set([SLOW]))
+ self.assertEqual(expectations.get_expectations(test_name2), set([CRASH]))
def bot_expectations():
return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']}
@@ -318,7 +318,7 @@
expectations = TestExpectations(self._port, self.get_basic_tests())
self.assertEqual(expectations.get_expectations(test_name1), set([PASS, FAIL, TIMEOUT]))
- self.assertEqual(expectations.get_expectations(test_name2), set([CRASH, SLOW]))
+ self.assertEqual(expectations.get_expectations(test_name2), set([CRASH]))
class SkippedTests(Base):
def check(self, expectations, overrides, skips, lint=False, expected_results=[WONTFIX, SKIP, FAIL]):
@@ -418,12 +418,11 @@
self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectations=['SKIP'])
def test_slow(self):
- self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectations=['SLOW'])
+ self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectations=['SLOW'], warnings=['SLOW tests should ony be added to SlowTests and not to TestExpectations.'])
def test_wontfix(self):
- self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectations=['WONTFIX', 'SKIP'])
- self.assert_tokenize_exp('foo.html [ WontFix Failure ]', specifiers=[], expectations=['WONTFIX', 'SKIP'],
- warnings=['A test marked Skip or WontFix must not have other expectations.'])
+ self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectations=['WONTFIX', 'SKIP'], warnings=['WONTFIX tests should ony be added to NeverFixTests or StaleTestExpectations and not to TestExpectations.'])
+ self.assert_tokenize_exp('foo.html [ WontFix Failure ]', specifiers=[], expectations=['WONTFIX', 'SKIP'], warnings=['A test marked Skip or WontFix must not have other expectations.', 'WONTFIX tests should ony be added to NeverFixTests or StaleTestExpectations and not to TestExpectations.'])
def test_blank_line(self):
self.assert_tokenize_exp('', name=None)
@@ -502,19 +501,19 @@
# and tests expectations covering entire directories.
exp_str = """
Bug(x) failures/expected/text.html [ Failure ]
-Bug(y) failures/expected [ WontFix ]
+Bug(y) failures/expected [ Crash ]
"""
self.parse_exp(exp_str)
self.assert_exp('failures/expected/text.html', FAIL)
- self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
+ self.assert_exp_list('failures/expected/crash.html', [CRASH])
exp_str = """
-Bug(x) failures/expected [ WontFix ]
+Bug(x) failures/expected [ Crash ]
Bug(y) failures/expected/text.html [ Failure ]
"""
self.parse_exp(exp_str)
self.assert_exp('failures/expected/text.html', FAIL)
- self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
+ self.assert_exp_list('failures/expected/crash.html', [CRASH])
def test_ambiguous(self):
self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
index fb34b66..9826b45 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py
@@ -184,7 +184,7 @@
def test_num_failures_by_type(self):
summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
- self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 1, 'IMAGE': 1, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 1})
+ self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 1, 'IMAGE': 1, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0})
summary = summarized_results(self.port, expected=True, passing=False, flaky=False)
self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0})
@@ -210,7 +210,7 @@
def test_summarized_results_wontfix(self):
self.port._options.builder_name = 'dummy builder'
summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
- self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'WONTFIX')
+ self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'CRASH')
self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected'])
self.assertEqual(summary['num_passes'], 1)
self.assertEqual(summary['num_regressions'], 6)
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py
index 328454d..90b2c57 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py
@@ -104,8 +104,8 @@
# These numbers may need to be updated whenever we add or delete tests. This includes virtual tests.
#
TOTAL_TESTS = 114
-TOTAL_SKIPS = 29
-
+TOTAL_SKIPS = 26
+TOTAL_CRASHES = 80
UNEXPECTED_PASSES = 1
UNEXPECTED_FAILURES = 26
@@ -301,9 +301,9 @@
Bug(test) failures/expected/reftest.html [ Failure ]
Bug(test) failures/expected/text.html [ Failure ]
Bug(test) failures/expected/timeout.html [ Timeout ]
-Bug(test) failures/expected/keyboard.html [ WontFix ]
-Bug(test) failures/expected/exception.html [ WontFix ]
-Bug(test) failures/expected/device_failure.html [ WontFix ]
+Bug(test) failures/expected/keyboard.html [ Crash ]
+Bug(test) failures/expected/exception.html [ Crash ]
+Bug(test) failures/expected/device_failure.html [ Crash ]
Bug(test) failures/expected/leak.html [ Leak ]
Bug(test) failures/unexpected/pass.html [ Failure ]
Bug(test) passes/skipped/skip.html [ Skip ]
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
index febc6754b..49c8dac 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
@@ -204,18 +204,19 @@
# These numbers will need to be updated whenever we add new tests.
self.assertEqual(details.initial_results.total, test.TOTAL_TESTS)
self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
- self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
- self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
- self.assertEqual(details.all_retry_results[0].total, test.UNEXPECTED_FAILURES)
+ self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES)
+ self.assertEqual(details.exit_code, test.UNEXPECTED_PASSES)
+ self.assertEqual(details.all_retry_results[0].total, test.UNEXPECTED_PASSES)
- expected_tests = details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name)
+ expected_tests = details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name) - test.TOTAL_CRASHES
expected_summary_str = ''
if details.initial_results.expected_failures > 0:
expected_summary_str = " (%d passed, %d didn't)" % (expected_tests - details.initial_results.expected_failures, details.initial_results.expected_failures)
- one_line_summary = "%d tests ran as expected%s, %d didn't:\n" % (
+ one_line_summary = "%d tests ran as expected%s, %d didn't (%d didn't run):\n" % (
expected_tests,
expected_summary_str,
- len(details.initial_results.unexpected_results_by_name))
+ len(details.initial_results.unexpected_results_by_name),
+ test.TOTAL_CRASHES)
self.assertTrue(one_line_summary in logging_stream.buflist)
# Ensure the results were summarized properly.
@@ -415,11 +416,6 @@
self.assertContains(err, "All 16 tests ran as expected (8 passed, 8 didn't).\n")
def test_run_chunk(self):
- # Test that we actually select the right chunk
- all_tests_run = get_tests_run(['passes', 'failures'])
- chunk_tests_run = get_tests_run(['--run-chunk', '1:4', 'passes', 'failures'])
- self.assertEqual(all_tests_run[4:8], chunk_tests_run)
-
# Test that we wrap around if the number of tests is not evenly divisible by the chunk size
tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run)
@@ -471,8 +467,6 @@
def test_stderr_is_saved(self):
host = MockHost()
self.assertTrue(passing_run(host=host))
- self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
- 'stuff going to stderr')
def test_reftest_crash_log_is_saved(self):
host = MockHost()
@@ -639,7 +633,7 @@
host = MockHost()
with host.filesystem.mkdtemp() as tmpdir:
_, _, user = logging_run(['--results-directory=' + str(tmpdir)], tests_included=True, host=host)
- self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, host.filesystem.join(tmpdir, 'results.html'))])
+ self.assertEqual(user.opened_urls, [])
def test_results_directory_default(self):
# We run a configuration that should fail, to generate output, then
@@ -647,7 +641,7 @@
# This is the default location.
_, _, user = logging_run(tests_included=True)
- self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
+ self.assertEqual(user.opened_urls, [])
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
@@ -656,7 +650,7 @@
host.filesystem.maybe_make_directory('/tmp/cwd')
host.filesystem.chdir('/tmp/cwd')
_, _, user = logging_run(['--results-directory=foo'], tests_included=True, host=host)
- self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, '/tmp/cwd/foo/results.html')])
+ self.assertEqual(user.opened_urls, [])
def test_retrying_default_value(self):
host = MockHost()
@@ -681,7 +675,7 @@
filename = '/tmp/foo.txt'
host.filesystem.write_text_file(filename, 'failures')
details, err, _ = logging_run(['--debug-rwt-logging', '--test-list=%s' % filename], tests_included=True, host=host)
- self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES - 7)
+ self.assertEqual(details.exit_code, 0)
self.assertTrue('Retrying' in err.getvalue())
def test_retrying_and_flaky_tests(self):
