| #!/usr/bin/env python |
| |
| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| import datetime |
| import getpass |
| import json |
| import os |
| import smtplib |
| import sys |
| import time |
| import urllib |
| import urllib2 |
| |
| class Emailer: |
| DEFAULT_EMAIL_PASSWORD_FILE = '.email_password' |
| GMAIL_SMTP_SERVER = 'smtp.gmail.com:587' |
| SUBJECT = 'Chrome GPU Bots Notification' |
| |
| def __init__(self, email_from, email_to, email_password_file): |
| self.email_from = email_from |
| self.email_to = email_to |
| self.email_password = Emailer._getEmailPassword(email_password_file) |
| |
| @staticmethod |
| def format_email_body(time_str, offline_str, failed_str, noteworthy_str): |
| return '%s%s%s%s' % (time_str, offline_str, failed_str, noteworthy_str) |
| |
| def send_email(self, body): |
| message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (self.email_from, |
| ','.join(self.email_to), Emailer.SUBJECT, body) |
| |
| try: |
| server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER) |
| server.starttls() |
| server.login(self.email_from, self.email_password) |
| server.sendmail(self.email_from, self.email_to, message) |
| server.quit() |
| except Exception as e: |
| print 'Error sending email: %s' % str(e) |
| |
| def testEmailLogin(self): |
| server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER) |
| server.starttls() |
| server.login(self.email_from, self.email_password) |
| server.quit() |
| |
| @staticmethod |
| def _getEmailPassword(email_password_file): |
| password = '' |
| |
| password_file = (email_password_file if email_password_file is not None |
| else Emailer.DEFAULT_EMAIL_PASSWORD_FILE) |
| |
| if os.path.isfile(password_file): |
| with open(password_file, 'r') as f: |
| password = f.read().strip() |
| else: |
| password = getpass.getpass( |
| 'Please enter email password for source email account: ') |
| |
| return password |
| |
| class GpuBot: |
| def __init__(self, waterfall_name, bot_name, bot_data): |
| self.waterfall_name = waterfall_name |
| self.bot_name = bot_name |
| self.bot_data = bot_data |
| self._end_time = None |
| self._hours_since_last_run = None |
| self.failure_string = None |
| self.bot_url = None |
| self.build_url = None |
| |
| def getEndTime(self): |
| return self._end_time |
| |
| def setEndTime(self, end_time): |
| self._end_time = end_time |
| self._hours_since_last_run = \ |
| roughTimeDiffInHours(end_time, time.localtime()) |
| |
| def getHoursSinceLastRun(self): |
| return self._hours_since_last_run |
| |
| def toDict(self): |
| dict = {'waterfall_name': self.waterfall_name, 'bot_name': self.bot_name} |
| |
| if self._end_time is not None: |
| dict['end_time'] = serialTime(self._end_time) |
| dict['hours_since_last_run'] = self._hours_since_last_run |
| |
| if self.failure_string is not None: |
| dict['failure_string'] = self.failure_string |
| |
| if self.bot_url is not None: |
| dict['bot_url'] = self.bot_url |
| |
| if self.build_url is not None: |
| dict['build_url'] = self.build_url |
| |
| return dict |
| |
| @staticmethod |
| def fromDict(dict): |
| gpu_bot = GpuBot(dict['waterfall_name'], dict['bot_name'], None) |
| |
| if 'end_time' in dict: |
| gpu_bot._end_time = unserializeTime(dict['end_time']) |
| |
| if 'hours_since_last_run' in dict: |
| gpu_bot._hours_since_last_run = dict['hours_since_last_run'] |
| |
| if 'failure_string' in dict: |
| gpu_bot.failure_string = dict['failure_string'] |
| |
| if 'bot_url' in dict: |
| gpu_bot.bot_url = dict['bot_url'] |
| |
| if 'build_url' in dict: |
| gpu_bot.build_url = dict['build_url'] |
| |
| return gpu_bot |
| |
| def errorNoMostRecentBuild(waterfall_name, bot_name): |
| print 'No most recent build available: %s::%s' % (waterfall_name, bot_name) |
| |
| class Waterfall: |
| BASE_URL = 'http://build.chromium.org/p/' |
| BASE_BUILD_URL = BASE_URL + '%s/builders/%s' |
| SPECIFIC_BUILD_URL = BASE_URL + '%s/builders/%s/builds/%s' |
| BASE_JSON_BUILDERS_URL = BASE_URL + '%s/json/builders' |
| BASE_JSON_BUILDS_URL = BASE_URL + '%s/json/builders/%s/builds' |
| REGULAR_WATERFALLS = ['chromium.gpu', |
| 'tryserver.chromium.gpu', |
| 'chromium.gpu.fyi'] |
| WEBKIT_GPU_BOTS = ['GPU Win Builder', |
| 'GPU Win Builder (dbg)', |
| 'GPU Win7 (NVIDIA)', |
| 'GPU Win7 (dbg) (NVIDIA)', |
| 'GPU Mac Builder', |
| 'GPU Mac Builder (dbg)', |
| 'GPU Mac10.7', |
| 'GPU Mac10.7 (dbg)', |
| 'GPU Linux Builder', |
| 'GPU Linux Builder (dbg)', |
| 'GPU Linux (NVIDIA)', |
| 'GPU Linux (dbg) (NVIDIA)'] |
| FILTERED_WATERFALLS = [('chromium.webkit', WEBKIT_GPU_BOTS)] |
| |
| @staticmethod |
| def getJsonFromUrl(url): |
| conn = urllib2.urlopen(url) |
| result = conn.read() |
| conn.close() |
| return json.loads(result) |
| |
| @staticmethod |
| def getBuildersJsonForWaterfall(waterfall): |
| querystring = '?filter' |
| return (Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s') |
| % (waterfall, querystring))) |
| |
| @staticmethod |
| def getLastNBuildsForBuilder(n, waterfall, builder): |
| if n <= 0: |
| return {} |
| |
| querystring = '?' |
| |
| for i in range(n): |
| querystring += 'select=-%d&' % (i + 1) |
| |
| querystring += 'filter' |
| |
| return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDS_URL + '%s') % |
| (waterfall, urllib.quote(builder), querystring)) |
| |
| @staticmethod |
| def getFilteredBuildersJsonForWaterfall(waterfall, filter): |
| querystring = '?' |
| |
| for bot_name in filter: |
| querystring += 'select=%s&' % urllib.quote(bot_name) |
| |
| querystring += 'filter' |
| |
| return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s') |
| % (waterfall, querystring)) |
| |
| @staticmethod |
| def getAllGpuBots(): |
| allbots = {k: Waterfall.getBuildersJsonForWaterfall(k) |
| for k in Waterfall.REGULAR_WATERFALLS} |
| |
| filteredbots = {k[0]: |
| Waterfall.getFilteredBuildersJsonForWaterfall(k[0], k[1]) |
| for k in Waterfall.FILTERED_WATERFALLS} |
| |
| allbots.update(filteredbots) |
| |
| return allbots |
| |
| @staticmethod |
| def getOfflineBots(bots): |
| offline_bots = [] |
| |
| for waterfall_name in bots: |
| waterfall = bots[waterfall_name] |
| |
| for bot_name in waterfall: |
| bot = waterfall[bot_name] |
| |
| if bot['state'] != 'offline': |
| continue |
| |
| gpu_bot = GpuBot(waterfall_name, bot_name, bot) |
| gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name, |
| urllib.quote(bot_name)) |
| |
| most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot( |
| gpu_bot) |
| |
| if (most_recent_build and 'times' in most_recent_build and |
| most_recent_build['times']): |
| gpu_bot.setEndTime(time.localtime(most_recent_build['times'][1])) |
| else: |
| errorNoMostRecentBuild(waterfall_name, bot_name) |
| |
| offline_bots.append(gpu_bot) |
| |
| return offline_bots |
| |
| @staticmethod |
| def getMostRecentlyCompletedBuildForBot(bot): |
| if bot.bot_data is not None and 'most_recent_build' in bot.bot_data: |
| return bot.bot_data['most_recent_build'] |
| |
| # Unfortunately, the JSON API doesn't provide a "most recent completed |
| # build" call. We just have to get some number of the most recent (including |
| # current, in-progress builds) and give up if that's not enough. |
| NUM_BUILDS = 10 |
| builds = Waterfall.getLastNBuildsForBuilder(NUM_BUILDS, bot.waterfall_name, |
| bot.bot_name) |
| |
| for i in range(NUM_BUILDS): |
| current_build_name = '-%d' % (i + 1) |
| current_build = builds[current_build_name] |
| |
| if 'results' in current_build and current_build['results'] is not None: |
| if bot.bot_data is not None: |
| bot.bot_data['most_recent_build'] = current_build |
| |
| return current_build |
| |
| return None |
| |
| @staticmethod |
| def getFailedBots(bots): |
| failed_bots = [] |
| |
| for waterfall_name in bots: |
| waterfall = bots[waterfall_name] |
| |
| for bot_name in waterfall: |
| bot = waterfall[bot_name] |
| gpu_bot = GpuBot(waterfall_name, bot_name, bot) |
| gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name, |
| urllib.quote(bot_name)) |
| |
| most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot( |
| gpu_bot) |
| |
| if (most_recent_build and 'text' in most_recent_build and |
| 'failed' in most_recent_build['text']): |
| gpu_bot.failure_string = ' '.join(most_recent_build['text']) |
| gpu_bot.build_url = Waterfall.SPECIFIC_BUILD_URL % (waterfall_name, |
| urllib.quote(bot_name), most_recent_build['number']) |
| failed_bots.append(gpu_bot) |
| elif not most_recent_build: |
| errorNoMostRecentBuild(waterfall_name, bot_name) |
| |
| return failed_bots |
| |
| def formatTime(t): |
| return time.strftime("%a, %d %b %Y %H:%M:%S", t) |
| |
| def roughTimeDiffInHours(t1, t2): |
| datetimes = [] |
| |
| for t in [t1, t2]: |
| datetimes.append(datetime.datetime(t.tm_year, t.tm_mon, t.tm_mday, |
| t.tm_hour, t.tm_min, t.tm_sec)) |
| |
| datetime_diff = datetimes[0] - datetimes[1] |
| |
| hours = float(datetime_diff.total_seconds()) / 3600.0 |
| |
| return abs(hours) |
| |
| def getBotStr(bot): |
| s = ' %s::%s\n' % (bot.waterfall_name, bot.bot_name) |
| |
| if bot.failure_string is not None: |
| s += ' failure: %s\n' % bot.failure_string |
| |
| if bot.getEndTime() is not None: |
| s += (' last build end time: %s (roughly %f hours ago)\n' % |
| (formatTime(bot.getEndTime()), bot.getHoursSinceLastRun())) |
| |
| if bot.bot_url is not None: |
| s += ' bot url: %s\n' % bot.bot_url |
| |
| if bot.build_url is not None: |
| s += ' build url: %s\n' % bot.build_url |
| |
| s += '\n' |
| return s |
| |
| def getBotsStr(bots): |
| s = '' |
| |
| for bot in bots: |
| s += getBotStr(bot) |
| |
| s += '\n' |
| return s |
| |
| def getOfflineBotsStr(offline_bots): |
| return 'Offline bots:\n%s' % getBotsStr(offline_bots) |
| |
| def getFailedBotsStr(failed_bots): |
| return 'Failed bots:\n%s' % getBotsStr(failed_bots) |
| |
| def getBotDicts(bots): |
| dicts = [] |
| |
| for bot in bots: |
| dicts.append(bot.toDict()) |
| |
| return dicts |
| |
| def unserializeTime(t): |
| return time.struct_time((t['year'], t['mon'], t['day'], t['hour'], t['min'], |
| t['sec'], 0, 0, 0)) |
| |
| def serialTime(t): |
| return {'year': t.tm_year, 'mon': t.tm_mon, 'day': t.tm_mday, |
| 'hour': t.tm_hour, 'min': t.tm_min, 'sec': t.tm_sec} |
| |
| def getSummary(offline_bots, failed_bots): |
| offline_bot_dict = getBotDicts(offline_bots) |
| failed_bot_dict = getBotDicts(failed_bots) |
| return {'offline': offline_bot_dict, 'failed': failed_bot_dict} |
| |
| def findBot(name, lst): |
| for bot in lst: |
| if bot.bot_name == name: |
| return bot |
| |
| return None |
| |
| def getNoteworthyEvents(offline_bots, failed_bots, previous_results): |
| CRITICAL_NUM_HOURS = 1.0 |
| |
| previous_offline = (previous_results['offline'] if 'offline' |
| in previous_results else []) |
| |
| previous_failures = (previous_results['failed'] if 'failed' |
| in previous_results else []) |
| |
| noteworthy_offline = [] |
| for bot in offline_bots: |
| if bot.getHoursSinceLastRun() >= CRITICAL_NUM_HOURS: |
| previous_bot = findBot(bot.bot_name, previous_offline) |
| |
| if (previous_bot is None or |
| previous_bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS): |
| noteworthy_offline.append(bot) |
| |
| noteworthy_new_failures = [] |
| for bot in failed_bots: |
| previous_bot = findBot(bot.bot_name, previous_failures) |
| |
| if previous_bot is None: |
| noteworthy_new_failures.append(bot) |
| |
| noteworthy_new_offline_recoveries = [] |
| for bot in previous_offline: |
| if bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS: |
| continue |
| |
| current_bot = findBot(bot.bot_name, offline_bots) |
| if current_bot is None: |
| noteworthy_new_offline_recoveries.append(bot) |
| |
| noteworthy_new_failure_recoveries = [] |
| for bot in previous_failures: |
| current_bot = findBot(bot.bot_name, failed_bots) |
| |
| if current_bot is None: |
| noteworthy_new_failure_recoveries.append(bot) |
| |
| return {'offline': noteworthy_offline, 'failed': noteworthy_new_failures, |
| 'recovered_failures': noteworthy_new_failure_recoveries, |
| 'recovered_offline': noteworthy_new_offline_recoveries} |
| |
| def getNoteworthyStr(noteworthy_events): |
| s = '' |
| |
| if noteworthy_events['offline']: |
| s += 'IMPORTANT bots newly offline for over an hour:\n' |
| |
| for bot in noteworthy_events['offline']: |
| s += getBotStr(bot) |
| |
| s += '\n' |
| |
| if noteworthy_events['failed']: |
| s += 'IMPORTANT new failing bots:\n' |
| |
| for bot in noteworthy_events['failed']: |
| s += getBotStr(bot) |
| |
| s += '\n' |
| |
| if noteworthy_events['recovered_offline']: |
| s += 'IMPORTANT newly recovered previously offline bots:\n' |
| |
| for bot in noteworthy_events['recovered_offline']: |
| s += getBotStr(bot) |
| |
| s += '\n' |
| |
| if noteworthy_events['recovered_failures']: |
| s += 'IMPORTANT newly recovered failing bots:\n' |
| |
| for bot in noteworthy_events['recovered_failures']: |
| s += getBotStr(bot) |
| |
| s += '\n' |
| |
| return s |
| |
| def dictsToBots(bots): |
| offline_bots = [] |
| for bot in bots['offline']: |
| offline_bots.append(GpuBot.fromDict(bot)) |
| |
| failed_bots = [] |
| for bot in bots['failed']: |
| failed_bots.append(GpuBot.fromDict(bot)) |
| |
| return {'offline': offline_bots, 'failed': failed_bots} |
| |
| class GpuBotPoller: |
| DEFAULT_PREVIOUS_RESULTS_FILE = '.check_gpu_bots_previous_results' |
| |
| def __init__(self, emailer, send_email_for_recovered_offline_bots, |
| send_email_for_recovered_failing_bots, send_email_on_error, |
| previous_results_file): |
| self.emailer = emailer |
| |
| self.send_email_for_recovered_offline_bots = \ |
| send_email_for_recovered_offline_bots |
| |
| self.send_email_for_recovered_failing_bots = \ |
| send_email_for_recovered_failing_bots |
| |
| self.send_email_on_error = send_email_on_error |
| self.previous_results_file = previous_results_file |
| |
| def shouldEmail(self, noteworthy_events): |
| if noteworthy_events['offline'] or noteworthy_events['failed']: |
| return True |
| |
| if (self.send_email_for_recovered_offline_bots and |
| noteworthy_events['recovered_offline']): |
| return True |
| |
| if (self.send_email_for_recovered_failing_bots and |
| noteworthy_events['recovered_failures']): |
| return True |
| |
| return False |
| |
| def writeResults(self, summary): |
| results_file = (self.previous_results_file |
| if self.previous_results_file is not None |
| else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE) |
| |
| with open(results_file, 'w') as f: |
| f.write(json.dumps(summary)) |
| |
| def getPreviousResults(self): |
| previous_results_file = (self.previous_results_file |
| if self.previous_results_file is not None |
| else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE) |
| |
| previous_results = {} |
| if os.path.isfile(previous_results_file): |
| with open(previous_results_file, 'r') as f: |
| previous_results = dictsToBots(json.loads(f.read())) |
| |
| return previous_results |
| |
| def checkBots(self): |
| time_str = 'Current time: %s\n\n' % (formatTime(time.localtime())) |
| print time_str |
| |
| try: |
| bots = Waterfall.getAllGpuBots() |
| |
| offline_bots = Waterfall.getOfflineBots(bots) |
| offline_str = getOfflineBotsStr(offline_bots) |
| print offline_str |
| |
| failed_bots = Waterfall.getFailedBots(bots) |
| failed_str = getFailedBotsStr(failed_bots) |
| print failed_str |
| |
| previous_results = self.getPreviousResults() |
| noteworthy_events = getNoteworthyEvents(offline_bots, failed_bots, |
| previous_results) |
| |
| noteworthy_str = getNoteworthyStr(noteworthy_events) |
| print noteworthy_str |
| |
| summary = getSummary(offline_bots, failed_bots) |
| self.writeResults(summary) |
| |
| if (self.emailer is not None and self.shouldEmail(noteworthy_events)): |
| self.emailer.send_email(Emailer.format_email_body(time_str, offline_str, |
| failed_str, noteworthy_str)) |
| except Exception as e: |
| error_str = 'Error: %s' % str(e) |
| print error_str |
| |
| if self.send_email_on_error: |
| self.emailer.send_email(error_str) |
| |
| def parseArgs(sys_args): |
| parser = argparse.ArgumentParser(prog=sys_args[0], |
| description='Query the Chromium GPU Bots Waterfall, output ' + |
| 'potential problems, and optionally repeat automatically and/or ' + |
| 'email notifications of results.') |
| |
| parser.add_argument('--repeat-delay', type=int, dest='repeat_delay', |
| required=False, |
| help='How often to automatically re-run the script, in minutes.') |
| |
| parser.add_argument('--email-from', type=str, dest='email_from', |
| required=False, |
| help='Email address to send from. Requires also specifying ' + |
| '\'--email-to\'.') |
| |
| parser.add_argument('--email-to', type=str, dest='email_to', required=False, |
| nargs='+', |
| help='Email address(es) to send to. Requires also specifying ' + |
| '\'--email-from\'') |
| |
| parser.add_argument('--send-email-for-recovered-offline-bots', |
| dest='send_email_for_recovered_offline_bots', action='store_true', |
| default=False, |
| help='Send an email out when a bot which has been offline for more ' + |
| 'than 1 hour goes back online.') |
| |
| parser.add_argument('--send-email-for-recovered-failing-bots', |
| dest='send_email_for_recovered_failing_bots', |
| action='store_true', default=False, |
| help='Send an email when a failing bot recovers.') |
| |
| parser.add_argument('--send-email-on-error', |
| dest='send_email_on_error', |
| action='store_true', default=False, |
| help='Send an email when the script has an error. For example, if ' + |
| 'the server is unreachable.') |
| |
| parser.add_argument('--email-password-file', |
| dest='email_password_file', |
| required=False, |
| help=(('File containing the plaintext password of the source email ' + |
| 'account. By default, \'%s\' will be tried. If it does not exist, ' + |
| 'you will be prompted. If you opt to store your password on disk ' + |
| 'in plaintext, use of a dummy account is strongly recommended.') |
| % Emailer.DEFAULT_EMAIL_PASSWORD_FILE)) |
| |
| parser.add_argument('--previous-results-file', |
| dest='previous_results_file', |
| required=False, |
| help=(('File to store the results of the previous invocation of ' + |
| 'this script. By default, \'%s\' will be used.') |
| % GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)) |
| |
| args = parser.parse_args(sys_args[1:]) |
| |
| if args.email_from is not None and args.email_to is None: |
| parser.error('--email-from requires --email-to.') |
| elif args.email_to is not None and args.email_from is None: |
| parser.error('--email-to requires --email-from.') |
| elif args.email_from is None and args.send_email_for_recovered_offline_bots: |
| parser.error('--send-email-for-recovered-offline-bots requires ' + |
| '--email-to and --email-from.') |
| elif (args.email_from is None and args.send_email_for_recovered_failing_bots): |
| parser.error('--send-email-for-recovered-failing-bots ' + |
| 'requires --email-to and --email-from.') |
| elif (args.email_from is None and args.send_email_on_error): |
| parser.error('--send-email-on-error ' + |
| 'requires --email-to and --email-from.') |
| elif (args.email_password_file and |
| not os.path.isfile(args.email_password_file)): |
| parser.error('File does not exist: %s' % args.email_password_file) |
| |
| return args |
| |
| def main(sys_args): |
| args = parseArgs(sys_args) |
| |
| emailer = None |
| if args.email_from is not None and args.email_to is not None: |
| emailer = Emailer(args.email_from, args.email_to, args.email_password_file) |
| |
| try: |
| emailer.testEmailLogin() |
| except Exception as e: |
| print 'Error logging into email account: %s' % str(e) |
| return 1 |
| |
| poller = GpuBotPoller(emailer, |
| args.send_email_for_recovered_offline_bots, |
| args.send_email_for_recovered_failing_bots, |
| args.send_email_on_error, |
| args.previous_results_file) |
| |
| while True: |
| poller.checkBots() |
| |
| if args.repeat_delay is None: |
| break |
| |
| print 'Will run again in %d minutes...\n' % args.repeat_delay |
| time.sleep(args.repeat_delay * 60) |
| |
| return 0 |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv)) |