blob: d40f91d168fd9ac729d86fb4439308cc5298d71a [file] [log] [blame]
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Updates TestExpectations based on results in builder bots.
Scans the TestExpectations file and uses results from actual builder bots runs
to remove tests that are marked as flaky or failing but don't fail in the
specified way.
--type=flake updates only lines that incude a 'Pass' expectation plus at
least one other expectation.
--type=fail updates lines that include only 'Failure', 'Timeout', or
'Crash' expectations.
E.g. If a test has this expectation:
bug(test) fast/test.html [ Failure Pass ]
And all the runs on builders have passed the line will be removed.
Additionally, the runs don't all have to be Passing to remove the line;
as long as the non-Passing results are of a type not specified in the
expectation this line will be removed. For example, if this is the
bug(test) fast/test.html [ Crash Pass ]
But the results on the builders show only Passes and Timeouts, the line
will be removed since there's no Crash results.
import argparse
import logging
import webbrowser
from blinkpy.tool.commands.flaky_tests import FlakyTests
from blinkpy.web_tests.models.test_expectations import CHROMIUM_BUG_PREFIX
from blinkpy.web_tests.models.test_expectations import TestExpectations
_log = logging.getLogger(__name__)
def main(host, bot_test_expectations_factory, argv):
parser = argparse.ArgumentParser(
epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
choices=['flake', 'fail', 'all'],
help='type of expectations to update (default: %(default)s)')
parser.add_argument('--verbose', '-v',
help='enable more verbose logging')
help='also remove lines if there were no results, '
'e.g. Android-only expectations for tests '
'that are not in SmokeTests')
parser.add_argument('--show-results', '-s',
help='Open results dashboard for all removed lines')
args = parser.parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format='%(levelname)s: %(message)s')
port = host.port_factory.get()
expectations_file = port.path_to_generic_test_expectations_file()
if not host.filesystem.isfile(expectations_file):
_log.warning("Didn't find generic expectations file at: " + expectations_file)
return 1
remover = ExpectationsRemover(
host, port, bot_test_expectations_factory, webbrowser,
args.type, args.remove_missing)
test_expectations = remover.get_updated_test_expectations()
if args.show_results:
remover.write_test_expectations(test_expectations, expectations_file)
return 0
class ExpectationsRemover(object):
def __init__(self, host, port, bot_test_expectations_factory, browser,
type_flag='all', remove_missing=False):
self._host = host
self._port = port
self._expectations_factory = bot_test_expectations_factory
self._builder_results_by_path = {}
self._browser = browser
self._expectations_to_remove_list = None
self._type = type_flag
self._remove_missing = remove_missing
def _can_delete_line(self, test_expectation_line):
"""Returns whether a given line in the expectations can be removed.
Uses results from builder bots to determine if a given line is stale and
can safely be removed from the TestExpectations file. (i.e. remove if
the bots show that it's not flaky.) There are also some rules about when
not to remove lines (e.g. never remove lines with Rebaseline
expectations, don't remove non-flaky expectations, etc.)
test_expectation_line (TestExpectationLine): A line in the test
expectation file to test for possible removal.
True if the line can be removed, False otherwise.
expectations = test_expectation_line.expectations
if not expectations:
return False
# Don't check lines that have expectations like Skip.
if self._has_unstrippable_expectations(expectations):
return False
# Don't check consistent passes.
if self._pass_expectation_only(expectations):
return False
# Don't check flakes in fail mode.
if self._type == 'fail' and self._has_pass_expectation(expectations):
return False
# Don't check failures in flake mode.
if self._type == 'flake' and not self._has_pass_expectation(expectations):
return False
# Don't check lines that have expectations for directories, since
# the flakiness of all sub-tests isn't as easy to check.
if self._port.test_isdir(
return False
# The line can be deleted if none of the expectations appear in the
# actual results or only a PASS expectation appears in the actual
# results.
builders_checked = []
for config in test_expectation_line.matching_configurations:
builder_name =, config.build_type)
if not builder_name:
_log.debug('No builder with config %s', config)
# For many configurations, there is no matching builder in
# blinkpy/common/config/builders.json. We ignore these
# configurations and make decisions based only on configurations
# with actual builders.
if builder_name not in self._builder_results_by_path.keys():
_log.error('Failed to find results for builder "%s"', builder_name)
return False
results_by_path = self._builder_results_by_path[builder_name]
# No results means the tests were all skipped, or all results are passing.
if test_expectation_line.path not in results_by_path.keys():
if self._remove_missing:
return False
results_for_single_test = results_by_path[test_expectation_line.path]
expectations_met = self._expectations_that_were_met(test_expectation_line, results_for_single_test)
if expectations_met != set(['PASS']) and expectations_met != set([]):
return False
if builders_checked:
_log.debug('Checked builders:\n %s', '\n '.join(builders_checked))
_log.warning('No matching builders for line, deleting line.')'Deleting line "%s"', test_expectation_line.original_string.strip())
return True
def _has_pass_expectation(self, expectations):
return 'PASS' in expectations
def _pass_expectation_only(self, expectations):
return len(expectations) == 1 and self._has_pass_expectation(expectations)
def _expectations_that_were_met(self, test_expectation_line, results_for_single_test):
"""Returns the set of expectations that appear in the given results.
e.g. If the test expectations is "bug(test) fast/test.html [Crash Failure Pass]"
and the results are ['TEXT', 'PASS', 'PASS', 'TIMEOUT'], then this method would
return [Pass Failure] since the Failure expectation is satisfied by 'TEXT', Pass
by 'PASS' but Crash doesn't appear in the results.
test_expectation_line: A TestExpectationLine object
results_for_single_test: A list of result strings.
e.g. ['IMAGE', 'IMAGE', 'PASS']
A set containing expectations that occurred in the results.
# TODO(bokan): Does this not exist in a more central place?
def replace_failing_with_fail(expectation):
if expectation in ('TEXT', 'IMAGE', 'IMAGE+TEXT', 'AUDIO'):
return 'FAIL'
return expectation
actual_results = {replace_failing_with_fail(r) for r in results_for_single_test}
return set(test_expectation_line.expectations) & actual_results
def _has_unstrippable_expectations(self, expectations):
"""Returns whether any of the given expectations are considered unstrippable.
Unstrippable expectations are those which should stop a line from being
removed regardless of builder bot results.
expectations: A list of string expectations.
E.g. ['PASS', 'FAIL' 'CRASH']
True if at least one of the expectations is unstrippable. False
unstrippable_expectations = (
return any(s in expectations for s in unstrippable_expectations)
def _get_builder_results_by_path(self):
"""Returns a dictionary of results for each builder.
Returns a dictionary where each key is a builder and value is a dictionary containing
the distinct results for each test. E.g.
'WebKit Linux Precise': {
'test1.html': ['PASS', 'IMAGE'],
'test2.html': ['PASS'],
'WebKit Mac10.10': {
'test1.html': ['PASS', 'IMAGE'],
'test2.html': ['PASS', 'TEXT'],
builder_results_by_path = {}
for builder_name in
expectations_for_builder = (
if not expectations_for_builder:
# This is not fatal since we may not need to check these
# results. If we do need these results we'll log an error later
# when trying to check against them.
_log.warning('Downloaded results are missing results for builder "%s"', builder_name)
builder_results_by_path[builder_name] = (
return builder_results_by_path
def _remove_associated_comments_and_whitespace(self, expectations, removed_index):
"""Removes comments and whitespace from an empty expectation block.
If the removed expectation was the last in a block of expectations, this method
will remove any associated comments and whitespace.
expectations: A list of TestExpectationLine objects to be modified.
removed_index: The index in the above list that was just removed.
was_last_expectation_in_block = (removed_index == len(expectations)
or expectations[removed_index].is_whitespace()
or expectations[removed_index].is_comment())
# If the line immediately below isn't another expectation, then the block of
# expectations definitely isn't empty so we shouldn't remove their associated comments.
if not was_last_expectation_in_block:
did_remove_whitespace = False
# We may have removed the last expectation in a block. Remove any whitespace above.
while removed_index > 0 and expectations[removed_index - 1].is_whitespace():
removed_index -= 1
did_remove_whitespace = True
# If we did remove some whitespace then we shouldn't remove any comments above it
# since those won't have belonged to this expectation block. For example, commented
# out expectations, or a section header.
if did_remove_whitespace:
# Remove all comments above the removed line.
while removed_index > 0 and expectations[removed_index - 1].is_comment():
removed_index -= 1
# Remove all whitespace above the comments.
while removed_index > 0 and expectations[removed_index - 1].is_whitespace():
removed_index -= 1
def _expectations_to_remove(self):
"""Computes and returns the expectation lines that should be removed.
A list of TestExpectationLine objects for lines that can be removed
from the test expectations file. The result is memoized so that
subsequent calls will not recompute the result.
if self._expectations_to_remove_list is not None:
return self._expectations_to_remove_list
self._builder_results_by_path = self._get_builder_results_by_path()
self._expectations_to_remove_list = []
test_expectations = TestExpectations(self._port, include_overrides=False).expectations()
for expectation in test_expectations:
if self._can_delete_line(expectation):
return self._expectations_to_remove_list
def get_updated_test_expectations(self):
"""Filters out passing lines from TestExpectations file.
Reads the current TestExpectations file and, using results from the
build bots, removes lines that are passing. That is, removes lines that
were not needed to keep the bots green.
A TestExpectations object with the passing lines filtered out.
test_expectations = TestExpectations(self._port, include_overrides=False).expectations()
for expectation in self._expectations_to_remove():
index = test_expectations.index(expectation)
# Remove associated comments and whitespace if we've removed the last expectation under
# a comment block. Only remove a comment block if it's not separated from the test
# expectation line by whitespace.
self._remove_associated_comments_and_whitespace(test_expectations, index)
return test_expectations
def show_removed_results(self):
"""Opens a browser showing the removed lines in the results dashboard.
Opens the results dashboard in the browser, showing all the tests for
lines removed from the TestExpectations file, allowing the user to
manually confirm the results.
url = self._flakiness_dashboard_url()'Opening results dashboard: ' + url)
def write_test_expectations(self, test_expectations, test_expectations_file):
"""Writes the given TestExpectations object to the filesystem.
test_expectations: The TestExpectations object to write.
test_expectations_file: The full file path of the Blink
TestExpectations file. This file will be overwritten.
TestExpectations.list_to_string(test_expectations, reconstitute_only_these=[]))
def print_suggested_commit_description(self):
"""Prints the body of a suggested CL description after removing some lines."""
expectation_type = ''
if self._type != 'all':
expectation_type = self._type + ' '
dashboard_url = self._flakiness_dashboard_url()
bugs = ', '.join(self._bug_numbers())
message = ('Remove %sTestExpectations which are not failing in the specified way.\n\n'
'This change was made by the script.\n\n'
'Recent test results history:\n%s\n\n'
'Bug: %s') % (expectation_type, dashboard_url, bugs)'Suggested commit description:\n' + message)
def _flakiness_dashboard_url(self):
removed_test_names = ','.join( for x in self._expectations_to_remove())
return FlakyTests.FLAKINESS_DASHBOARD_URL % removed_test_names
def _bug_numbers(self):
"""Returns the list of all bug numbers affected by this change."""
numbers = set()
for line in self._expectations_to_remove():
for bug in line.bugs:
if bug.startswith(CHROMIUM_BUG_PREFIX):
return sorted(numbers)