| #!/usr/bin/python |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Views to support the urls defined in urls.py. |
| |
| Modeled after the Django views.py file but with the benefits of bottlepy. |
| """ |
| |
| import cgi, collections |
| |
| import bottle |
| import whining_errors |
| import wmutils |
| |
| from models import named_filters |
| from models import pivot_table |
| from models import whining_model |
| |
| from src import settings |
| |
| |
| def catch_whining_errors(fn): |
| """@Decorator to catch & handle frequent errors (e.g. bad user input).""" |
| def wrapped(*args, **kwargs): |
| try: |
| return fn(*args, **kwargs) |
| except whining_errors.WhiningError as e: |
| def dummy_query_string(add={}, remove=[], list_qps=False): |
| if list_qps: |
| return [] |
| return '' |
| |
| exception_message = str(e) |
| return bottle.template('error_view', tpl_vars=locals(), |
| query_string=dummy_query_string) |
| return wrapped |
| |
| |
| def static_file(file, root): |
| """Serve a static file from /static.""" |
| return bottle.static_file(file, root) |
| |
| |
| @catch_whining_errors |
| def render_builds(): |
| """Serve a lookup view which is a list of known builds.""" |
| db = whining_model.model_factory(bottle.request) |
| builds = db.get_builds(hyperlinks=True) |
| filter_tag = 'lookups' |
| return bottle.template('builds_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_comments(): |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_comments() |
| return bottle.template('comment_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def post_comment(): |
| comment = bottle.request.forms.get('comment') |
| test_id = bottle.request.forms.get('test_id') |
| db = whining_model.model_factory(bottle.request) |
| db.run_query_cmd("CALL add_triage_comment('%s', '%s')" % (test_id, comment)) |
| return |
| |
| |
| @catch_whining_errors |
| def render_dashboard(filter_tag): |
| """Serve our main aggregate view of summary + failures.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| return bottle.template('dash_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| def render_debug(): |
| """Serve debug data for troubleshooting.""" |
| db = whining_model.model_factory(bottle.request) |
| # Insert anything here to display it and debug issues. |
| test_results = {} |
| return bottle.template('debug_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_summary(filter_tag): |
| """Serve a summary view of recent test failures.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| data = db.get_summary(filter_tag) |
| return bottle.template('summary_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_filters(): |
| """Serve filter data to aid filter understanding.""" |
| db = whining_model.model_factory(bottle.request) |
| filters = named_filters.NAMED_FILTERS |
| return bottle.template('filter_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_help(): |
| """Serve a list of valid urls to aid users.""" |
| db = whining_model.model_factory(bottle.request) |
| filter_tag = named_filters.default_filter() |
| known_qvars = ', '.join(sorted(named_filters.KNOWN_QVARS)) |
| filter_names = sorted(named_filters.NAMED_FILTERS.keys()) |
| return bottle.template('urlhelp', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_jobs(filter_tag): |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_jobs(filter_tag) |
| return bottle.template('jobs_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_modems(): |
| """Serve a modems view for 3G tests.""" |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_modems() |
| filter_tag = '3g' |
| return bottle.template('modems_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_platform(filter_tag): |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| data = db.get_platform_summary(filter_tag) |
| return bottle.template('platform_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_platforms(): |
| """Serve a lookup view which is a list of known platforms.""" |
| db = whining_model.model_factory(bottle.request) |
| platforms = db.get_platforms(hyperlinks=True) |
| filter_tag = 'lookups' |
| return bottle.template('platforms_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_releases(filter_tag): |
| """Serve a lookup view which is a list of known builds.""" |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_releases(hyperlinks=True) |
| return bottle.template('releases_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_suites(): |
| """Serve a lookup view which is a list of known suites.""" |
| db = whining_model.model_factory(bottle.request) |
| suites = db.get_suites(hyperlinks=True) |
| filter_tag = 'lookups' |
| return bottle.template('suites_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_suite_stats(filter_tag): |
| """Serve a lookup view which is a table of suite statistics.""" |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_suite_stats(filter_tag) |
| return bottle.template('suitestats_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_matrix(filter_tag): |
| """Serve a summary matrix of recent test results.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| data = db.get_matrix(filter_tag) |
| return bottle.template('matrix_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_temp_filter(): |
| """Serve a page that can create a temporary filter (url). |
| |
| Produces a custom URL. Does not add any filtes to named_filters |
| at this time. |
| """ |
| def _make_list_string(list_obj): |
| return "['%s']" % "', '".join(str(x) for x in list_obj) |
| db = whining_model.model_factory(bottle.request) |
| builds = _make_list_string(db.get_builds()) |
| platforms = _make_list_string(db.get_platforms()) |
| releases = _make_list_string([str(r) for r in db.get_releases()]) |
| suites = _make_list_string(db.get_suites()) |
| # The angular js code for handling object queries gets overwhelmed |
| # by the large number of tests in each of these suites. We do not |
| # show them separately in the searchable criteria. |
| #| SUITE | #tests | |
| #| wificell | 143 | |
| #| pyauto | 192 | |
| #| sync | 261 | |
| #| browsertests | 3033 | |
| tests = _make_list_string(db.get_tests( |
| exclude_suites=['browsertests', 'pyauto', 'sync', 'wificell'])) |
| return bottle.template('temp_filter_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_tests(): |
| """Serve a lookup view which is a list of known tests.""" |
| db = whining_model.model_factory(bottle.request) |
| tests = db.get_tests(hyperlinks=True) |
| filter_tag = 'lookups' |
| return bottle.template('tests_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_testrun(filter_tag): |
| """Serve a view with full details of a test run.""" |
| db = whining_model.model_factory(bottle.request) |
| test_runs = db.get_testrun(filter_tag) |
| status_classes = collections.defaultdict(lambda:'warning_summary') |
| status_classes.update({'GOOD': 'success', 'FAIL': 'failure'}) |
| host = settings.settings.autotest_host |
| user = settings.settings.autotest_user |
| return bottle.template('testrun_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_failures(filter_tag): |
| """Serve a list of recent failures.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| data = db.get_failures(filter_tag) |
| return bottle.template('failures_view', tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| @catch_whining_errors |
| def redirect_to_testrun_log(filter_tag): |
| """Redirect to the link of a test run's log.""" |
| db = whining_model.model_factory(bottle.request) |
| |
| # A test job may have multiple runs, this function redirect to the log url |
| # of the test job, not individual test runs. |
| test_runs = db.get_testrun(filter_tag) |
| log_url = ('%s/tko/retrieve_logs.cgi?job=/results/%s-%s' % |
| (settings.settings.autotest_host, test_runs[0]['afe_job_id'], |
| settings.settings.autotest_user)) |
| # Redirect to the test log if there is only 1 test run. |
| if len(test_runs) == 1: |
| log_url += ('/%s/%s' % |
| (test_runs[0]['hostname'], test_runs[0]['test_name'])) |
| return bottle.redirect(log_url) |
| |
| |
| @catch_whining_errors |
| def render_suitehealth(): |
| """A suite health matrix - failure rates per suite per platform.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| filter_tag = '' |
| pt = db.get_suitehealth() |
| |
| # Sort suites and platforms by failure rates. |
| suites = pt.keylists['suite'] |
| suites.sort(reverse=True, key=lambda s: pt.data['suite'][s].rate) |
| platforms = pt.keylists['platform'] |
| platforms.sort() |
| |
| # Put all the data into an HTML table generator. |
| tbl = pivot_table.HtmlTable(suites, platforms, pt.data['sp']) |
| qs_add = {} |
| qs_add['days_back'] = pt.days_back |
| qs = db.query_string(add=qs_add).strip('?') |
| rh_tpl = ('<th>' |
| '<a href="%s/unfiltered?%s&suites={0}">matrix</a>, ' |
| '<a href="%s/testhealth/unfiltered?%s&suites={0}">' |
| 'tests</a> ● {0}</th>' |
| ) % (settings.settings.relative_root, qs, |
| settings.settings.relative_root, qs) |
| |
| tbl.format_row_header = rh_tpl.format |
| tbl.row_headers.append(pt.data['suite']) |
| tbl.col_headers.append(pt.data['platform']) |
| totals = pt.data['totals'][tuple()] |
| tbl.title_class = 'centered headeritem' |
| days_str = ' in the last %d days' % pt.days_back |
| title = ('Failure rates per suite. (Global failure rate: {} - ' |
| '{:,} out of {:,} runs{})' |
| ) |
| tbl.title = title.format(totals.str_rate, |
| totals.bad_count, |
| totals.all_count, |
| days_str, |
| ) |
| |
| # Secondary headers - 2nd row and 2nd column of the table. |
| header_template = ( |
| '<td>\n' |
| ' <div title="{bad_count}/{all_count}"> {str_rate} </div>\n' |
| '</td>\n' |
| ) |
| for cell in pt.cell_iterator(['suite', 'platform']): |
| cell.template = header_template |
| |
| # Format inner cells with links and color heatmap. |
| for cell in pt.data['sp'].values(): |
| color = wmutils.logcolor(cell.rate) |
| platform_href = db.make_href('platform', |
| 'unfiltered', |
| suites=cell.suite, |
| platforms=cell.platform, |
| days_back=pt.days_back, |
| ) |
| cell.html = ( |
| '<td style="background:%s"><a href="%s">%s</a></td>' |
| % (color, platform_href, cell.str_rate) |
| ) |
| |
| return bottle.template('generic_table_view', |
| view='suitehealth', |
| title='Suite Health', |
| tpl_vars=locals(), |
| query_string=db.query_string, |
| ) |
| |
| |
| @catch_whining_errors |
| def render_testhealth(filter_tag): |
| """Show most frequently failing tests.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| pt = db.get_testhealth(filter_tag) |
| |
| # Sort suites and platforms by failure rates. |
| tests = pt.keylists['test'] |
| tests.sort(reverse=True, key=lambda t: pt.data['test'][t].rate) |
| platforms = pt.keylists['platform'] |
| platforms.sort() |
| |
| # HTML templates for table cells. |
| header_template = ( |
| '<td>\n' |
| ' <div title="{bad_count}/{all_count}"> {str_rate} </div>\n' |
| '</td>\n' |
| ) |
| |
| for cell in pt.cell_iterator(['test', 'platform']): |
| cell.template = header_template |
| |
| inner_cell_template = ''.join([ |
| '\n<td style="background:{color}">\n', |
| ' <div title="{bad_count}/{all_count}"> {str_rate} </div>\n', |
| '</td>\n', |
| ]) |
| |
| # Calculate colors based on failure rate. |
| for cell in pt.data['tp'].values(): |
| cell.template = inner_cell_template |
| cell.color = wmutils.logcolor(cell.rate) |
| |
| # Function for HTML formatting cells that are row headers as: |
| # "matrix, reasons * suite/test_name" with links matrix and reasons views |
| def format_test_name(test): |
| suite, test_name = test.split('/', 1) |
| href_matrix = db.make_href('matrix', filter_tag, |
| tests=test_name, suites=suite) |
| href_reasons = db.make_href('reasons', |
| tests=test_name, suites=suite) |
| extra = ' class="experimental"' if pt.data['test'][test].is_exp else '' |
| td = ( |
| '<th%s><a href="%s">matrix</a>, <a href="%s">reasons</a> ' |
| '● %s</th>' |
| ) % (extra, href_matrix, href_reasons, test) |
| return td |
| |
| # Put all the data into an HTML table generator. |
| tbl = pivot_table.HtmlTable(tests, platforms, pt.data['tp']) |
| tbl.format_row_header = format_test_name |
| tbl.row_headers.append(pt.data['test']) |
| tbl.col_headers.append(pt.data['platform']) |
| tbl.title_class = 'centered headeritem' |
| tbl.title = 'Test failure rates for the last %s days' % pt.days_back |
| |
| return bottle.template('generic_table_view', |
| view='testhealth', |
| title='Test Failure Rates', |
| tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_reasons(): |
| """Frequencies of reason strings per platform for a given test.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| filter_tag = '' |
| pt = db.get_reasons() |
| test_name = pt.test_name |
| |
| cell_tpl = '<td><div>{str_percent} ({count})</div></td>' |
| for cell in pt.cell_iterator(['rsn', 'rp']): |
| cell.template = cell_tpl |
| for cell in pt.data['platform'].values(): |
| cell.value = cell.count # Use default template with 'value' |
| |
| # Sorting |
| reasons = pt.keylists['rsn'] |
| reasons.sort(reverse=True, key=lambda r: pt.data['rsn'][r].count) |
| platforms = pt.keylists['platform'] |
| platforms.sort() |
| |
| matrix_href = db.make_href('matrix', 'unfiltered', hide_missing=True, |
| tests=test_name) |
| matrix_link = '<a href="%s">matrix</a>' % matrix_href |
| |
| # Put all the data into an HTML table generator. |
| tbl = pivot_table.HtmlTable(reasons, platforms, pt.data['rp']) |
| tbl.title = 'Reason string frequency for the last %d days - %s - links: %s' |
| tbl.title %= (pt.days_back, test_name, matrix_link) |
| tbl.title_class = 'centered headeritem' |
| |
| def format_platform(platform): |
| href = db.make_href('failures', 'unfiltered', platforms=platform) |
| return ( |
| '<th class="centered"><a href="%s">%s</a></th>' |
| ) % (href, platform) |
| |
| # There are some manipulations for grouping similar reasons |
| def format_reason(rsn): |
| th = '<th>' |
| cell = pt.data['rsn'][rsn] |
| reasons = set(d['reason'].strip() for d in cell.lst) |
| |
| # If several similar reasons, add a note as tooltip on an asterisk |
| if len(reasons) > 1: |
| msg = 'Several reason strings grouped by omitting all numbers' |
| th += '<span title="%s" style="color: red">*</span>' % msg |
| |
| reason = max(reasons).strip() |
| # Trim reason string to max_len characters for reasonably looking UI. |
| max_len = 120 |
| tooltip = '' |
| if len(reason) > (max_len + 1): |
| tooltip = reason |
| reason = reason[:max_len] + '...' |
| reason = cgi.escape(reason) |
| if tooltip: |
| th += '<span title="%s">%s</span>' % (tooltip, reason) |
| else: |
| th += '<span>%s</span>' % reason |
| return th + '</th>' |
| |
| tbl.format_row_header = format_reason |
| tbl.format_col_header = format_platform |
| tbl.row_headers.append(pt.data['rsn']) |
| tbl.col_headers.append(pt.data['platform']) |
| |
| return bottle.template('generic_table_view', |
| view='reasons', |
| title='Test Failure Reasons', |
| tpl_vars=locals(), |
| query_string=db.query_string, |
| ) |
| |
| |
| @catch_whining_errors |
| def render_retry_teststats(filter_tag): |
| """Show retry pass rate of tests.""" |
| db = whining_model.model_factory(bottle.request) |
| releases = db.get_releases() |
| pt = db.get_retry_teststats(filter_tag) |
| |
| # Sort suites and platforms by retry pass rates. |
| tests = pt.keylists['test'] |
| tests.sort(reverse=True, key=lambda t: pt.data['test'][t].rate) |
| platforms = pt.keylists['platform'] |
| platforms.sort() |
| |
| # HTML templates for table cells. |
| header_template = ( |
| '<td>\n' |
| ' <div title="pass/total: {good_count}/{all_count}"> ' |
| '{str_rate} ({all_count}) </div>\n' |
| '</td>\n' |
| ) |
| |
| for cell in pt.cell_iterator(['test', 'platform']): |
| cell.template = header_template |
| |
| inner_cell_template = ''.join([ |
| '\n<td style="background:{color}">\n', |
| ' <div title="pass/total: {good_count}/{all_count}, ' |
| 'avg retry counts: {str_avg_retry_count}"> ' |
| '{str_rate} <a href="' + settings.settings.relative_root + |
| '/testrun/?test_ids={test_list}">' |
| '({good_count}/{all_count})</a> </div>\n', |
| '</td>\n', |
| ]) |
| |
| # Calculate colors based on failure rate. |
| for cell in pt.data['tp'].values(): |
| cell.template = inner_cell_template |
| cell.color = wmutils.logcolor(cell.rate) |
| |
| # Function for HTML formatting cells that are row headers as: |
| # "matrix, reasons * suite/test_name" with links matrix and reasons views |
| def format_test_name(test): |
| suite, test_name = test.split('/', 1) |
| extra = ' class="experimental"' if pt.data['test'][test].is_exp else '' |
| return "<th%s>%s</th>" % (extra, test) |
| |
| # Put all the data into an HTML table generator. |
| tbl = pivot_table.HtmlTable(tests, platforms, pt.data['tp']) |
| tbl.format_row_header = format_test_name |
| tbl.row_headers.append(pt.data['test']) |
| tbl.col_headers.append(pt.data['platform']) |
| tbl.title_class = 'centered headeritem' |
| tbl.title = ('Test retry stats for the last %s days - ' |
| 'retry pass rate (total retries)') % pt.days_back |
| |
| return bottle.template('retry_teststats_view', |
| view='retry_teststats', |
| title='Test retry stats', |
| tpl_vars=locals(), |
| query_string=db.query_string) |
| |
| |
| @catch_whining_errors |
| def render_retry_hostinfo(filter_tag): |
| """Show top flaky reasons per host""" |
| db = whining_model.model_factory(bottle.request) |
| data = db.get_retry_hostinfo(filter_tag) |
| title = 'Top 10 flaky reasons for host(s)' |
| return bottle.template( |
| 'retry_hostinfo_view', |
| view='retry_hostinfo', |
| tpl_vars=locals(), |
| query_string=db.query_string, |
| title=title) |
| |
| |
| @catch_whining_errors |
| def render_retry_hoststats(filter_tag): |
| """Show retry pass rate of hosts""" |
| db = whining_model.model_factory(bottle.request) |
| pt = db.get_retry_hoststats(filter_tag) |
| |
| host = pt.keylists['host'] |
| platforms = pt.keylists['platform'] |
| platforms.sort() |
| |
| def list_format(*args, **kargs): |
| """Override the cell's format function.""" |
| html = ['<td>'] |
| for host_info in kargs['lst']: |
| qs_add = {} |
| qs_add['hostnames'] = host_info['hostname'] |
| qs = db.query_string(add=qs_add) |
| |
| tpl = '\n'.join([ |
| '<div title="pass/total: {good_count}/{all_count}">', |
| ' <a href="{root}/retry_hostinfo/{qs}"> {hostname} </a>', |
| ' ({rate}, {good_count}/{all_count})', |
| '</div>', |
| ]) |
| tpl = tpl.format(good_count=host_info['good_count'], |
| all_count=host_info['all_count'], |
| filter_tag=filter_tag, |
| qs=qs, |
| hostname=host_info['hostname'], |
| rate=wmutils.format_rate(host_info['rate']), |
| root=settings.settings.relative_root) |
| html.append(tpl) |
| html.append('</td>') |
| return '\n'.join(html) |
| |
| for cell in pt.data['ph'].values(): |
| cell.format = list_format |
| |
| # Put all the data into an HTML table generator. |
| tbl = pivot_table.HtmlTable(platforms, host, pt.data['ph']) |
| tbl.title_class = 'centered headeritem' |
| tbl.title = 'Per host retry pass rate (most flaky hosts)' |
| if pt.days_back: |
| tbl.title += ' for the last %s days' % pt.days_back |
| return bottle.template('retry_hoststats_view', |
| view='retry_hoststats', |
| title='Per host retry pass rate', |
| tpl_vars=locals(), |
| query_string=db.query_string) |
