| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """This file contains printing-related functionality of the bisect.""" |
| |
| import datetime |
| import re |
| |
| from bisect_results import BisectResults |
| import bisect_utils |
| import source_control |
| |
| |
| # The perf dashboard looks for a string like "Estimated Confidence: 95%" |
| # to decide whether or not to cc the author(s). If you change this, please |
| # update the perf dashboard as well. |
| RESULTS_BANNER = """ |
| ===== BISECT JOB RESULTS ===== |
| Status: %(status)s |
| |
| Test Command: %(command)s |
| Test Metric: %(metric)s |
| Relative Change: %(change)s |
| Estimated Confidence: %(confidence).02f%% |
| Retested CL with revert: %(retest)s""" |
| |
| # When the bisect was aborted without a bisect failure the following template |
| # is used. |
| ABORT_REASON_TEMPLATE = """ |
| ===== BISECTION ABORTED ===== |
| The bisect was aborted because %(abort_reason)s |
| Please contact the the team (see below) if you believe this is in error. |
| |
| Bug ID: %(bug_id)s |
| |
| Test Command: %(command)s |
| Test Metric: %(metric)s |
| Good revision: %(good_revision)s |
| Bad revision: %(bad_revision)s """ |
| |
| # The perf dashboard specifically looks for the string |
| # "Author : " to parse out who to cc on a bug. If you change the |
| # formatting here, please update the perf dashboard as well. |
| RESULTS_REVISION_INFO = """ |
| ===== SUSPECTED CL(s) ===== |
| Subject : %(subject)s |
| Author : %(author)s%(commit_info)s |
| Commit : %(cl)s |
| Date : %(cl_date)s""" |
| |
| RESULTS_THANKYOU = """ |
| | O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq |
| | X | for more information addressing perf regression bugs. For feedback, |
| | / \\ | file a bug with label Cr-Tests-AutoBisect. Thank you!""" |
| |
| |
| class BisectPrinter(object): |
| |
| def __init__(self, opts, depot_registry=None): |
| self.opts = opts |
| self.depot_registry = depot_registry |
| |
| def FormatAndPrintResults(self, bisect_results): |
| """Prints the results from a bisection run in a readable format. |
| |
| Also prints annotations creating buildbot step "Results". |
| |
| Args: |
| bisect_results: BisectResult object containing results to be printed. |
| """ |
| if bisect_results.abort_reason: |
| self._PrintAbortResults(bisect_results.abort_reason) |
| return |
| |
| if self.opts.output_buildbot_annotations: |
| bisect_utils.OutputAnnotationStepStart('Build Status Per Revision') |
| |
| print |
| print 'Full results of bisection:' |
| for revision_state in bisect_results.state.GetRevisionStates(): |
| build_status = revision_state.passed |
| |
| if type(build_status) is bool: |
| if build_status: |
| build_status = 'Good' |
| else: |
| build_status = 'Bad' |
| |
| print ' %20s %40s %s' % (revision_state.depot, |
| revision_state.revision, |
| build_status) |
| print |
| |
| if self.opts.output_buildbot_annotations: |
| bisect_utils.OutputAnnotationStepClosed() |
| # The perf dashboard scrapes the "results" step in order to comment on |
| # bugs. If you change this, please update the perf dashboard as well. |
| bisect_utils.OutputAnnotationStepStart('Results') |
| |
| self._PrintBanner(bisect_results) |
| self._PrintWarnings(bisect_results.warnings) |
| |
| if bisect_results.culprit_revisions and bisect_results.confidence: |
| for culprit in bisect_results.culprit_revisions: |
| cl, info, depot = culprit |
| self._PrintRevisionInfo(cl, info, depot) |
| self._PrintRetestResults(bisect_results) |
| self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(), |
| bisect_results.first_working_revision, |
| bisect_results.last_broken_revision, |
| bisect_results.confidence, |
| final_step=True) |
| self._PrintStepTime(bisect_results.state.GetRevisionStates()) |
| self._PrintThankYou() |
| if self.opts.output_buildbot_annotations: |
| bisect_utils.OutputAnnotationStepClosed() |
| |
| def PrintPartialResults(self, bisect_state): |
| revision_states = bisect_state.GetRevisionStates() |
| first_working_rev, last_broken_rev = BisectResults.FindBreakingRevRange( |
| revision_states) |
| self._PrintTestedCommitsTable(revision_states, first_working_rev, |
| last_broken_rev, 100, final_step=False) |
| |
| def _PrintAbortResults(self, abort_reason): |
| if self.opts.output_buildbot_annotations: |
| bisect_utils.OutputAnnotationStepStart('Results') |
| |
| # Metric string in config is not split in case of return code mode. |
| if (self.opts.metric and |
| self.opts.bisect_mode != bisect_utils.BISECT_MODE_RETURN_CODE): |
| metric = '/'.join(self.opts.metric) |
| else: |
| metric = self.opts.metric |
| |
| print ABORT_REASON_TEMPLATE % { |
| 'abort_reason': abort_reason, |
| 'bug_id': self.opts.bug_id or 'NOT SPECIFIED', |
| 'command': self.opts.command, |
| 'metric': metric, |
| 'good_revision': self.opts.good_revision, |
| 'bad_revision': self.opts.bad_revision, |
| } |
| self._PrintThankYou() |
| if self.opts.output_buildbot_annotations: |
| bisect_utils.OutputAnnotationStepClosed() |
| |
| @staticmethod |
| def _PrintThankYou(): |
| print RESULTS_THANKYOU |
| |
| @staticmethod |
| def _PrintStepTime(revision_states): |
| """Prints information about how long various steps took. |
| |
| Args: |
| revision_states: Ordered list of revision states.""" |
| step_perf_time_avg = 0.0 |
| step_build_time_avg = 0.0 |
| step_count = 0.0 |
| for revision_state in revision_states: |
| if revision_state.value: |
| step_perf_time_avg += revision_state.perf_time |
| step_build_time_avg += revision_state.build_time |
| step_count += 1 |
| if step_count: |
| step_perf_time_avg = step_perf_time_avg / step_count |
| step_build_time_avg = step_build_time_avg / step_count |
| print |
| print 'Average build time : %s' % datetime.timedelta( |
| seconds=int(step_build_time_avg)) |
| print 'Average test time : %s' % datetime.timedelta( |
| seconds=int(step_perf_time_avg)) |
| |
| @staticmethod |
| def _GetViewVCLinkFromDepotAndHash(git_revision, depot): |
| """Gets link to the repository browser.""" |
| if depot and 'viewvc' in bisect_utils.DEPOT_DEPS_NAME[depot]: |
| return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision |
| return '' |
| |
| def _PrintRevisionInfo(self, cl, info, depot=None): |
| commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot) |
| if commit_link: |
| commit_link = '\nLink : %s' % commit_link |
| else: |
| commit_link = ('\Description:\n%s' % info['body']) |
| print RESULTS_REVISION_INFO % { |
| 'subject': info['subject'], |
| 'author': info['email'], |
| 'commit_info': commit_link, |
| 'cl': cl, |
| 'cl_date': info['date'] |
| } |
| |
| @staticmethod |
| def _PrintTableRow(column_widths, row_data): |
| """Prints out a row in a formatted table that has columns aligned. |
| |
| Args: |
| column_widths: A list of column width numbers. |
| row_data: A list of items for each column in this row. |
| """ |
| assert len(column_widths) == len(row_data) |
| text = '' |
| for i in xrange(len(column_widths)): |
| current_row_data = row_data[i].center(column_widths[i], ' ') |
| text += ('%%%ds' % column_widths[i]) % current_row_data |
| print text |
| |
| def _PrintTestedCommitsHeader(self): |
| if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN: |
| self._PrintTableRow( |
| [20, 12, 70, 14, 12, 13], |
| ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State']) |
| elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV: |
| self._PrintTableRow( |
| [20, 12, 70, 14, 12, 13], |
| ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State']) |
| elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE: |
| self._PrintTableRow( |
| [20, 12, 70, 14, 13], |
| ['Depot', 'Position', 'SHA', 'Return Code', 'State']) |
| else: |
| assert False, 'Invalid bisect_mode specified.' |
| |
| def _PrintTestedCommitsEntry(self, revision_state, commit_position, cl_link, |
| state_str): |
| if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN: |
| std_error = '+-%.02f' % revision_state.value['std_err'] |
| mean = '%.02f' % revision_state.value['mean'] |
| self._PrintTableRow( |
| [20, 12, 70, 12, 14, 13], |
| [revision_state.depot, commit_position, cl_link, mean, std_error, |
| state_str]) |
| elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV: |
| std_error = '+-%.02f' % revision_state.value['std_err'] |
| mean = '%.02f' % revision_state.value['mean'] |
| self._PrintTableRow( |
| [20, 12, 70, 12, 14, 13], |
| [revision_state.depot, commit_position, cl_link, std_error, mean, |
| state_str]) |
| elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE: |
| mean = '%d' % revision_state.value['mean'] |
| self._PrintTableRow( |
| [20, 12, 70, 14, 13], |
| [revision_state.depot, commit_position, cl_link, mean, |
| state_str]) |
| |
| def _PrintTestedCommitsTable( |
| self, revision_states, first_working_revision, last_broken_revision, |
| confidence, final_step=True): |
| print |
| if final_step: |
| print '===== TESTED COMMITS =====' |
| else: |
| print '===== PARTIAL RESULTS =====' |
| self._PrintTestedCommitsHeader() |
| state = 0 |
| for revision_state in revision_states: |
| if revision_state.value: |
| if (revision_state == last_broken_revision or |
| revision_state == first_working_revision): |
| # If confidence is too low, don't add this empty line since it's |
| # used to put focus on a suspected CL. |
| if confidence and final_step: |
| print |
| state += 1 |
| if state == 2 and not final_step: |
| # Just want a separation between "bad" and "good" cl's. |
| print |
| |
| state_str = 'Bad' |
| if state == 1 and final_step: |
| state_str = 'Suspected CL' |
| elif state == 2: |
| state_str = 'Good' |
| |
| # If confidence is too low, don't bother outputting good/bad. |
| if not confidence: |
| state_str = '' |
| state_str = state_str.center(13, ' ') |
| commit_position = source_control.GetCommitPosition( |
| revision_state.revision, |
| self.depot_registry.GetDepotDir(revision_state.depot)) |
| display_commit_pos = '' |
| if commit_position: |
| display_commit_pos = str(commit_position) |
| self._PrintTestedCommitsEntry(revision_state, |
| display_commit_pos, |
| revision_state.revision, |
| state_str) |
| |
| def _PrintRetestResults(self, bisect_results): |
| if (not bisect_results.retest_results_tot or |
| not bisect_results.retest_results_reverted): |
| return |
| print |
| print '===== RETEST RESULTS =====' |
| self._PrintTestedCommitsEntry( |
| bisect_results.retest_results_tot, '', '', '') |
| self._PrintTestedCommitsEntry( |
| bisect_results.retest_results_reverted, '', '', '') |
| |
| def _PrintBanner(self, bisect_results): |
| if self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE: |
| metric = 'N/A' |
| change = 'Yes' |
| else: |
| metric = '/'.join(self.opts.metric) |
| change = '%.02f%% (+/-%.02f%%)' % ( |
| bisect_results.regression_size, bisect_results.regression_std_err) |
| if not bisect_results.culprit_revisions: |
| change = 'No significant change reproduced.' |
| |
| print RESULTS_BANNER % { |
| 'status': self._StatusMessage(bisect_results), |
| 'command': self.opts.command, |
| 'metric': metric, |
| 'change': change, |
| 'confidence': bisect_results.confidence, |
| 'retest': 'Yes' if bisect_results.retest_results_tot else 'No', |
| } |
| |
| @staticmethod |
| def _StatusMessage(bisect_results): |
| if bisect_results.confidence >= bisect_utils.HIGH_CONFIDENCE: |
| return 'Positive: Reproduced a change.' |
| elif bisect_results.culprit_revisions: |
| return 'Negative: Found possible suspect(s), but with low confidence.' |
| return 'Negative: Did not reproduce a change.' |
| |
| @staticmethod |
| def _PrintWarnings(warnings): |
| """Prints a list of warning strings if there are any.""" |
| if not warnings: |
| return |
| print |
| print 'WARNINGS:' |
| for w in set(warnings): |
| print ' ! %s' % w |
