blob: ae2a09f947921732f99d7fbf8298282e41aa2266 [file] [log] [blame]
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import re
import time
from urllib import urlencode
from urlparse import parse_qs, urlsplit, urlunsplit
from buildbot.status.builder import FAILURE, SUCCESS
from master import build_utils
from master.chromium_notifier import ChromiumNotifier
from master.failures_history import FailuresHistory
from twisted.internet import defer
from twisted.python import log
# Create a block to work around evil sys.modules manipulation in
# email/ that triggers pylint false positives.
# pylint has issues importing it.
# pylint: disable=E0611,F0401
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
from email.Utils import formatdate
except ImportError:
# The history of results expire every day.
_EXPIRATION_TIME = 24 * 3600
# Perf results key words used in test result step.
# Key to last time an email is sent per builder
def PerfLog(msg):
log.msg('[PerfCountNotifier] %s' % msg)
class PerfCountNotifier(ChromiumNotifier):
"""This is a status notifier that only alerts on consecutive perf changes.
The notifier only notifies when a number of consecutive REGRESS or IMPROVE
perf results are recorded.
See builder.interfaces.IStatusReceiver for more information about
parameters type.
def __init__(self, step_names, minimum_count=5, combine_results=True,
"""Initializes the PerfCountNotifier on tests starting with test_name.
step_names: List of perf steps names. This is needed to know perf steps
from other steps especially when the step is successful.
minimum_count: The number of minimum consecutive (REGRESS|IMPROVE) needed
to notify.
combine_results: Combine summary results email for all builders in one.
# Set defaults.
ChromiumNotifier.__init__(self, **kwargs)
self.minimum_count = minimum_count
self.combine_results = combine_results
self.step_names = step_names
self.recent_results = None
self.error_email = False
self.new_email_results = {}
self.recent_results = FailuresHistory(expiration_time=_EXPIRATION_TIME,
def AddNewEmailResult(self, result):
"""Stores an email result for a builder.
result: A tuple of the form ('REGRESS|IMPROVE', 'value_name', 'builder').
builder_name = result[2]
build_results = self.GetEmailResults(builder_name)
if not result[1] in build_results[result[0]]:
PerfLog('(%s) email result has already been stored.' % ', '.join(result))
def GetEmailResults(self, builder_name):
"""Returns the email results for a builder."""
if not builder_name in self.new_email_results:
self.new_email_results[builder_name] = GetNewBuilderResult()
return self.new_email_results[builder_name]
def _UpdateResults(self, builder_name, results):
"""Updates the results by adding/removing from the history.
builder_name: Builder name the results belong to.
results: List of result tuples, each tuple is of the form
('REGRESS|IMPROVE', 'value_name', 'builder').
new_results_ids = [' '.join(result) for result in results]
# Delete the old results if the new results do not have them.
to_delete = [old_id for old_id in self.recent_results.failures
if (old_id not in new_results_ids and
for old_id in to_delete:
# Update the new results history
for new_id in results:
def _StoreResult(self, result):
"""Stores the result value and removes counter results.
Example: if this is a REGRESS result then it is stored and its counter
IMPROVE result, if any, is reset.
result: A tuple of the form ('REGRESS|IMPROVE', 'value_name', 'builder').
self.recent_results.Put(' '.join(result))
if result[0] == REGRESS:
counter_id = IMPROVE + ' '.join(result[1:])
counter_id = REGRESS + ' '.join(result[1:])
# Reset counter_id count since this breaks the consecutive count of it.
def _DeleteResult(self, result_id):
"""Removes the history of results identified by result_id.
result_id: The id of the history entry (see _StoreResult() for details).
num_results = self.recent_results.GetCount(result_id)
if num_results > 0:
# This is a hack into FailuresHistory since it does not allow to delete
# entries in its history unless they are expired.
# FailuresHistory.failures_count is the total number of entries in the
# history limitted by FailuresHistory.size_limit.
del self.recent_results.failures[result_id]
self.recent_results.failures_count -= num_results
def _DeleteAllForBuild(self, builder_name):
"""Deletes all test results related to a builder."""
to_delete = [result for result in self.recent_results.failures
if result.endswith(builder_name)]
for result in to_delete:
def _ResetResults(self, builder_name):
"""Reset pending email results for builder."""
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
self.new_email_results[builder_name] = GetNewBuilderResult()
self.new_email_results[builder_name][EMAIL_TIME] = time.time()
def _IsPerfStep(self, step_status):
"""Checks if the step name is one of the defined perf tests names."""
return self.getName(step_status) in self.step_names
def isInterestingStep(self, build_status, step_status, results):
"""Ignore the step if it is not one of the perf results steps.
True: - if a REGRESS|IMPROVE happens consecutive minimum number of times.
- if it is not a SUCCESS step and neither REGRESS|IMPROVE.
False: - if it is a SUCCESS step.
- if it is a notification which has already been notified.
self.error_email = False
step_text = ' '.join(step_status.getText())
PerfLog('Analyzing failure text: %s.' % step_text)
if (not self._IsPerfStep(step_status) or
not self.isInterestingBuilder(build_status.getBuilder())):
return False
# In case of exceptions, sometimes results output is empty.
if not results:
results = [FAILURE]
builder_name = build_status.getBuilder().getName()
self.SetBuilderGraphURL(self.getName(step_status), build_status)
# If it is a success step, i.e. not interesting, then reset counters.
if results[0] == SUCCESS:
return False
# step_text is similar to:
# media_tests_av_perf <div class="BuildResultInfo"> PERF_REGRESS:
# time/t (89.07%) PERF_IMPROVE: fps/video (5.40%) </div>
# regex would return tuples of the form:
# ('REGRESS', 'time/t', 'linux-rel')
# ('IMPROVE', 'fps/video', 'win-debug')
# It is important to put the builder name as the last element in the tuple
# since it is used to check tests that belong to same builder.
step_text = ' '.join(step_status.getText())
PerfLog('Analyzing failure text: %s.' % step_text)
perf_regress = perf_improve = ''
perf_results = []
if PERF_REGRESS in step_text:
perf_regress = step_text[step_text.find(PERF_REGRESS) + len(PERF_REGRESS)
+ 1: step_text.find(PERF_IMPROVE)]
perf_results.extend([(REGRESS, test_name, builder_name) for test_name in
re.findall(r'(\S+) (?=\(.+\))', perf_regress)])
if PERF_IMPROVE in step_text:
# Based on log_parser/ PerformanceChangesAsText() function,
# we assume that PERF_REGRESS (if any) appears before PERF_IMPROVE.
perf_improve = step_text[step_text.find(PERF_IMPROVE) + len(PERF_IMPROVE)
+ 1:]
perf_results.extend([(IMPROVE, test_name, builder_name) for test_name in
re.findall(r'(\S+) (?=\(.+\))', perf_improve)])
# If there is no regress or improve then this could be warning or exception.
if not perf_results:
if not self.recent_results.GetCount(step_text):
PerfLog('Unrecognized step status. Reporting status as interesting.')
# Force the build box to show in email
self.error_email = True
return True
PerfLog('This problem has already been notified.')
return False
update_list = []
for result in perf_results:
if len(result) != 3:
# We expect a tuple similar to ('REGRESS', 'time/t', 'linux-rel')
result_id = ' '.join(result)
PerfLog('Result: %s happened %d times in a row.' %
(result_id, self.recent_results.GetCount(result_id) + 1))
if self.recent_results.GetCount(result_id) >= self.minimum_count - 1:
# This is an interesting result! We got the minimum consecutive count of
# this result. Store it in email results.
PerfLog('Result: %s happened enough consecutive times to be reported.'
% result_id)
self._UpdateResults(builder_name, update_list)
# Final decision is made based on whether there are any notifications to
# email based on this and older build results.
return self.ShouldSendEmail(builder_name)
def buildMessage(self, builder_name, build_status, results, step_name):
"""Send an email about this interesting step.
Add the perf regressions/improvements that resulted in this email if any.
PerfLog('About to send an email.')
email_subject = self.GetEmailSubject(builder_name, build_status, results,
email_body = self.GetEmailBody(builder_name, build_status, results,
html_content = (
'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">'
'<html xmlns=""><body>%s</body></html>' %
defered_object = self.BuildEmailObject(email_subject, html_content,
builder_name, build_status,
return defered_object
def ShouldSendEmail(self, builder_name):
"""Returns if we should send a summary email at this moment.
True if it has been at least minimum_delay_between_alert since the
last email sent. False otherwise.
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
if self._ShouldSendEmail(builder_name):
return True
return False
def _ShouldSendEmail(self, builder_name):
results = self.GetEmailResults(builder_name)
last_time_mail_sent = results[EMAIL_TIME]
if (last_time_mail_sent and last_time_mail_sent >
time.time() - self.minimum_delay_between_alert):
# Rate limit tree alerts.
PerfLog('Time since last email is too short. Should not send email.')
return False
# Return True if there are any builder results to email about.
return results and (results[REGRESS] or results[IMPROVE])
def GetEmailSubject(self, builder_name, build_status, results, step_name):
"""Returns the subject of for an email based on perf results."""
project_name = self.master_status.getTitle()
latest_revision = build_utils.getLatestRevision(build_status)
result = 'changes'
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
return ('%s %s on %s, revision %s' %
(project_name, result, ', '.join(builders), str(latest_revision)))
def GetEmailHeader(self, builder_name, build_status, results, step_name):
"""Returns a header message in an email.
Used for backward compatibility with chromium_notifier. It allows the
users to add text to every email from the master.cfg setup.
status_text = self.status_header % {
'builder': builder_name,
'steps': step_name,
'results': results
return status_text
def GetEmailBody(self, builder_name, build_status, results, step_name):
"""Returns the main email body content."""
email_body = ''
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
email_body += '%s%s\n' % (
self.GetEmailHeader(builder_name, build_status, results, step_name),
# Latest build box is not relevant with multiple builder results combined.
if not self.combine_results or self.error_email:
email_body += ('\n\nLatest build results:%s' %
self.GenStepBox(builder_name, build_status, step_name))
PerfLog('Perf email body: %s' % email_body)
return email_body.replace('\n', '<br>')
def GetPerfEmailBody(self, builder_name):
builder_results = self.GetEmailResults(builder_name)
graph_url = builder_results[GRAPH_URL]
msg = ''
# Add regression HTML links.
if builder_results[REGRESS]:
test_urls = CreateHTMLTestURLList(graph_url, builder_results[REGRESS])
msg += '<strong>%s</strong>: %s.\n' % (PERF_REGRESS, ', '.join(test_urls))
# Add improvement HTML links.
if builder_results[IMPROVE]:
test_urls = CreateHTMLTestURLList(graph_url,
msg += '<strong>%s</strong>: %s.\n' % (PERF_IMPROVE, ', '.join(test_urls))
return msg or 'No perf results.\n'
def BuildEmailObject(self, email_subject, html_content, builder_name,
build_status, step_name):
"""Creates an email object ready to be sent."""
m = MIMEMultipart('alternative')
m.attach(MIMEText(html_content, 'html', 'iso-8859-1'))
m['Date'] = formatdate(localtime=True)
m['Subject'] = email_subject
m['From'] = self.fromaddr
if self.reply_to:
m['Reply-To'] = self.reply_to
recipients = list(self.extraRecipients[:])
dl = []
if self.sendToInterestedUsers and self.lookup:
for u in build_status.getInterestedUsers():
d = defer.maybeDeferred(self.lookup.getAddress, u)
defered_object = defer.DeferredList(dl)
defered_object.addCallback(self._gotRecipients, recipients, m)
defered_object.addCallback(self.getFinishedMessage, builder_name,
build_status, step_name)
return defered_object
def GenStepBox(self, builder_name, build_status, step_name):
"""Generates a HTML styled summary box for one step."""
waterfall_url = self.master_status.getBuildbotURL()
styles = dict(build_utils.DEFAULT_STYLES)
builder_results = self.GetEmailResults(builder_name)
if builder_results[IMPROVE] and not builder_results[REGRESS]:
styles['warnings'] = styles['success']
return build_utils.EmailableBuildTable_bb8(build_status, waterfall_url,
def SetBuilderGraphURL(self, step_name, build_status):
"""Stores the graph URL used in emails for this builder."""
builder_name = build_status.getBuilder().getName()
builder_results = self.GetEmailResults(builder_name)
graph_url = GetGraphURL(step_name, build_status)
latest_revision = build_utils.getLatestRevision(build_status)
if latest_revision:
graph_url = SetQueryParameter(graph_url, 'rev', latest_revision)
builder_results[GRAPH_URL] = graph_url
def GetStepByName(build_status, step_name):
"""Returns the build step with step_name."""
for step in build_status.getSteps():
if step.getName() == step_name:
return step
return None
def GetGraphURL(step_name, build_status):
"""Returns the graph result's URL from the step with step_name.
step_name: The name of the step to get the URL from.
build_status: The build status containing all steps in this build.
A string URL for the results graph page in the status step.
step = GetStepByName(build_status, step_name)
if step and step.getURLs():
# Find the URL for results page
for name, target in step.getURLs().iteritems():
if 'report.html' in target:
PerfLog('Found graph URL %s %s ' % (name, target))
return SetQueryParameter(target, 'history', 150)
PerfLog('Could not find graph URL, step_name: %s.' % step_name)
return None
def CreateHTMLTestURLList(graph_url, test_names):
"""Creates a list of href HTML graph links for each test name result.
graph_url: The main result page URL.
test_names: A list of test names that should be included in the email.
A list of strings, each containing HTML href links to specific test
results with graph and trace parameters set. Example:
graph_url = '\
test_name = ['audio_latency/latency']
Return value = ['<a href="\
def CreateTraceURL(test_name):
names = test_name.split('/')
url = SetQueryParameter(graph_url, 'graph', names[0])
if len(names) > 1:
url = SetQueryParameter(url, 'trace', names[1], append_param=True)
return url
urls = []
for name in test_names:
urls.append('<a href="%s">%s</a>' % (CreateTraceURL(name), name))
return urls
def GetNewBuilderResult():
return {
def SetQueryParameter(url, param_name, param_value, append_param=False):
"""Returns a url with the parameter value pair updated or added.
If append_param=True then the URL will append a new param value to the URL.
if not url:
return '%s=%s' % (param_name, param_value)
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
if append_param and param_name in query_params:
query_params[param_name] = [param_value]
new_query_string = urlencode(query_params, doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))