gpu/tools/check_gpu_bots.py - chromium/src - Git at Google

 #!/usr/bin/env python

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 import argparse
 import datetime
 import getpass
 import json
 import os
 import smtplib
 import sys
 import time
 import urllib
 import urllib2

 class Emailer:
   DEFAULT_EMAIL_PASSWORD_FILE = '.email_password'
   GMAIL_SMTP_SERVER = 'smtp.gmail.com:587'
   SUBJECT = 'Chrome GPU Bots Notification'

   def __init__(self, email_from, email_to, email_password_file):
     self.email_from = email_from
     self.email_to = email_to
     self.email_password = Emailer._getEmailPassword(email_password_file)

   @staticmethod
   def format_email_body(time_str, offline_str, failed_str, noteworthy_str):
     return '%s%s%s%s' % (time_str, offline_str, failed_str, noteworthy_str)

   def send_email(self, body):
     message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (self.email_from,
             ','.join(self.email_to), Emailer.SUBJECT, body)

     try:
       server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
       server.starttls()
       server.login(self.email_from, self.email_password)
       server.sendmail(self.email_from, self.email_to, message)
       server.quit()
     except Exception as e:
       print 'Error sending email: %s' % str(e)

   def testEmailLogin(self):
     server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
     server.starttls()
     server.login(self.email_from, self.email_password)
     server.quit()

   @staticmethod
   def _getEmailPassword(email_password_file):
     password = ''

     password_file = (email_password_file if email_password_file is not None
             else Emailer.DEFAULT_EMAIL_PASSWORD_FILE)

     if os.path.isfile(password_file):
       with open(password_file, 'r') as f:
         password = f.read().strip()
     else:
       password = getpass.getpass(
               'Please enter email password for source email account: ')

     return password

 class GpuBot:
   def __init__(self, waterfall_name, bot_name, bot_data):
     self.waterfall_name = waterfall_name
     self.bot_name = bot_name
     self.bot_data = bot_data
     self._end_time = None
     self._hours_since_last_run = None
     self.failure_string = None
     self.bot_url = None
     self.build_url = None

   def getEndTime(self):
     return self._end_time

   def setEndTime(self, end_time):
     self._end_time = end_time
     self._hours_since_last_run = \
             roughTimeDiffInHours(end_time, time.localtime())

   def getHoursSinceLastRun(self):
     return self._hours_since_last_run

   def toDict(self):
     dict = {'waterfall_name': self.waterfall_name, 'bot_name': self.bot_name}

     if self._end_time is not None:
       dict['end_time'] = serialTime(self._end_time)
       dict['hours_since_last_run'] = self._hours_since_last_run

     if self.failure_string is not None:
       dict['failure_string'] = self.failure_string

     if self.bot_url is not None:
       dict['bot_url'] = self.bot_url

     if self.build_url is not None:
       dict['build_url'] = self.build_url

     return dict

   @staticmethod
   def fromDict(dict):
     gpu_bot = GpuBot(dict['waterfall_name'], dict['bot_name'], None)

     if 'end_time' in dict:
       gpu_bot._end_time = unserializeTime(dict['end_time'])

     if 'hours_since_last_run' in dict:
       gpu_bot._hours_since_last_run = dict['hours_since_last_run']

     if 'failure_string' in dict:
       gpu_bot.failure_string = dict['failure_string']

     if 'bot_url' in dict:
       gpu_bot.bot_url = dict['bot_url']

     if 'build_url' in dict:
       gpu_bot.build_url = dict['build_url']

     return gpu_bot

 def errorNoMostRecentBuild(waterfall_name, bot_name):
   print 'No most recent build available: %s::%s' % (waterfall_name, bot_name)

 class Waterfall:
   BASE_URL = 'http://build.chromium.org/p/'
   BASE_BUILD_URL = BASE_URL + '%s/builders/%s'
   SPECIFIC_BUILD_URL = BASE_URL + '%s/builders/%s/builds/%s'
   BASE_JSON_BUILDERS_URL = BASE_URL + '%s/json/builders'
   BASE_JSON_BUILDS_URL = BASE_URL + '%s/json/builders/%s/builds'
   REGULAR_WATERFALLS = ['chromium.gpu', 'chromium.gpu.fyi']
   WEBKIT_GPU_BOTS = ['GPU Win Builder',
           'GPU Win Builder (dbg)',
           'GPU Win7 (NVIDIA)',
           'GPU Win7 (dbg) (NVIDIA)',
           'GPU Mac Builder',
           'GPU Mac Builder (dbg)',
           'GPU Mac10.7',
           'GPU Mac10.7 (dbg)',
           'GPU Linux Builder',
           'GPU Linux Builder (dbg)',
           'GPU Linux (NVIDIA)',
           'GPU Linux (dbg) (NVIDIA)']
   FILTERED_WATERFALLS = [('chromium.webkit', WEBKIT_GPU_BOTS)]

   @staticmethod
   def getJsonFromUrl(url):
     conn = urllib2.urlopen(url)
     result = conn.read()
     conn.close()
     return json.loads(result)

   @staticmethod
   def getBuildersJsonForWaterfall(waterfall):
     querystring = '?filter'
     return (Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
         % (waterfall, querystring)))

   @staticmethod
   def getLastNBuildsForBuilder(n, waterfall, builder):
     if n <= 0:
       return {}

     querystring = '?'

     for i in range(n):
       querystring += 'select=-%d&' % (i + 1)

     querystring += 'filter'

     return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDS_URL + '%s') %
             (waterfall, urllib.quote(builder), querystring))

   @staticmethod
   def getFilteredBuildersJsonForWaterfall(waterfall, filter):
     querystring = '?'

     for bot_name in filter:
       querystring += 'select=%s&' % urllib.quote(bot_name)

     querystring += 'filter'

     return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
             % (waterfall, querystring))

   @staticmethod
   def getAllGpuBots():
     allbots = {k: Waterfall.getBuildersJsonForWaterfall(k)
             for k in Waterfall.REGULAR_WATERFALLS}

     filteredbots = {k[0]:
             Waterfall.getFilteredBuildersJsonForWaterfall(k[0], k[1])
             for k in Waterfall.FILTERED_WATERFALLS}

     allbots.update(filteredbots)

     return allbots

   @staticmethod
   def getOfflineBots(bots):
     offline_bots = []

     for waterfall_name in bots:
       waterfall = bots[waterfall_name]

       for bot_name in waterfall:
         bot = waterfall[bot_name]

         if bot['state'] != 'offline':
           continue

         gpu_bot = GpuBot(waterfall_name, bot_name, bot)
         gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
                 urllib.quote(bot_name))

         most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
                 gpu_bot)

         if (most_recent_build and 'times' in most_recent_build and
                 most_recent_build['times']):
           gpu_bot.setEndTime(time.localtime(most_recent_build['times'][1]))
         else:
           errorNoMostRecentBuild(waterfall_name, bot_name)

         offline_bots.append(gpu_bot)

     return offline_bots

   @staticmethod
   def getMostRecentlyCompletedBuildForBot(bot):
     if bot.bot_data is not None and 'most_recent_build' in bot.bot_data:
       return bot.bot_data['most_recent_build']

     # Unfortunately, the JSON API doesn't provide a "most recent completed
     # build" call. We just have to get some number of the most recent (including
     # current, in-progress builds) and give up if that's not enough.
     NUM_BUILDS = 10
     builds = Waterfall.getLastNBuildsForBuilder(NUM_BUILDS, bot.waterfall_name,
             bot.bot_name)

     for i in range(NUM_BUILDS):
       current_build_name = '-%d' % (i + 1)
       current_build = builds[current_build_name]

       if 'results' in current_build and current_build['results'] is not None:
         if bot.bot_data is not None:
           bot.bot_data['most_recent_build'] = current_build

         return current_build

     return None

   @staticmethod
   def getFailedBots(bots):
     failed_bots = []

     for waterfall_name in bots:
       waterfall = bots[waterfall_name]

       for bot_name in waterfall:
         bot = waterfall[bot_name]
         gpu_bot = GpuBot(waterfall_name, bot_name, bot)
         gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
                 urllib.quote(bot_name))

         most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
                 gpu_bot)

         if (most_recent_build and 'text' in most_recent_build and
                 'failed' in most_recent_build['text']):
           gpu_bot.failure_string = ' '.join(most_recent_build['text'])
           gpu_bot.build_url = Waterfall.SPECIFIC_BUILD_URL % (waterfall_name,
                   urllib.quote(bot_name), most_recent_build['number'])
           failed_bots.append(gpu_bot)
         elif not most_recent_build:
           errorNoMostRecentBuild(waterfall_name, bot_name)

     return failed_bots

 def formatTime(t):
   return time.strftime("%a, %d %b %Y %H:%M:%S", t)

 def roughTimeDiffInHours(t1, t2):
   datetimes = []

   for t in [t1, t2]:
     datetimes.append(datetime.datetime(t.tm_year, t.tm_mon, t.tm_mday,
         t.tm_hour, t.tm_min, t.tm_sec))

   datetime_diff = datetimes[0] - datetimes[1]

   hours = float(datetime_diff.total_seconds()) / 3600.0

   return abs(hours)

 def getBotStr(bot):
   s = '  %s::%s\n' % (bot.waterfall_name, bot.bot_name)

   if bot.failure_string is not None:
     s += '  failure: %s\n' % bot.failure_string

   if bot.getEndTime() is not None:
     s += ('  last build end time: %s (roughly %f hours ago)\n' %
     (formatTime(bot.getEndTime()), bot.getHoursSinceLastRun()))

   if bot.bot_url is not None:
     s += '  bot url: %s\n' % bot.bot_url

   if bot.build_url is not None:
     s += '  build url: %s\n' % bot.build_url

   s += '\n'
   return s

 def getBotsStr(bots):
   s = ''

   for bot in bots:
     s += getBotStr(bot)

   s += '\n'
   return s

 def getOfflineBotsStr(offline_bots):
   return 'Offline bots:\n%s' % getBotsStr(offline_bots)

 def getFailedBotsStr(failed_bots):
   return 'Failed bots:\n%s' % getBotsStr(failed_bots)

 def getBotDicts(bots):
   dicts = []

   for bot in bots:
     dicts.append(bot.toDict())

   return dicts

 def unserializeTime(t):
   return time.struct_time((t['year'], t['mon'], t['day'], t['hour'], t['min'],
       t['sec'], 0, 0, 0))

 def serialTime(t):
   return {'year': t.tm_year, 'mon': t.tm_mon, 'day': t.tm_mday,
           'hour': t.tm_hour, 'min': t.tm_min, 'sec': t.tm_sec}

 def getSummary(offline_bots, failed_bots):
   offline_bot_dict = getBotDicts(offline_bots)
   failed_bot_dict = getBotDicts(failed_bots)
   return {'offline': offline_bot_dict, 'failed': failed_bot_dict}

 def findBot(name, lst):
   for bot in lst:
     if bot.bot_name == name:
       return bot

   return None

 def getNoteworthyEvents(offline_bots, failed_bots, previous_results):
   CRITICAL_NUM_HOURS = 1.0

   previous_offline = (previous_results['offline'] if 'offline'
           in previous_results else [])

   previous_failures = (previous_results['failed'] if 'failed'
           in previous_results else [])

   noteworthy_offline = []
   for bot in offline_bots:
     if bot.getHoursSinceLastRun() >= CRITICAL_NUM_HOURS:
       previous_bot = findBot(bot.bot_name, previous_offline)

       if (previous_bot is None or
               previous_bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS):
         noteworthy_offline.append(bot)

   noteworthy_new_failures = []
   for bot in failed_bots:
     previous_bot = findBot(bot.bot_name, previous_failures)

     if previous_bot is None:
       noteworthy_new_failures.append(bot)

   noteworthy_new_offline_recoveries = []
   for bot in previous_offline:
     if bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS:
       continue

     current_bot = findBot(bot.bot_name, offline_bots)
     if current_bot is None:
       noteworthy_new_offline_recoveries.append(bot)

   noteworthy_new_failure_recoveries = []
   for bot in previous_failures:
     current_bot = findBot(bot.bot_name, failed_bots)

     if current_bot is None:
       noteworthy_new_failure_recoveries.append(bot)

   return {'offline': noteworthy_offline, 'failed': noteworthy_new_failures,
           'recovered_failures': noteworthy_new_failure_recoveries,
           'recovered_offline': noteworthy_new_offline_recoveries}

 def getNoteworthyStr(noteworthy_events):
   s = ''

   if noteworthy_events['offline']:
     s += 'IMPORTANT bots newly offline for over an hour:\n'

     for bot in noteworthy_events['offline']:
       s += getBotStr(bot)

     s += '\n'

   if noteworthy_events['failed']:
     s += 'IMPORTANT new failing bots:\n'

     for bot in noteworthy_events['failed']:
       s += getBotStr(bot)

     s += '\n'

   if noteworthy_events['recovered_offline']:
     s += 'IMPORTANT newly recovered previously offline bots:\n'

     for bot in noteworthy_events['recovered_offline']:
       s += getBotStr(bot)

     s += '\n'

   if noteworthy_events['recovered_failures']:
     s += 'IMPORTANT newly recovered failing bots:\n'

     for bot in noteworthy_events['recovered_failures']:
       s += getBotStr(bot)

     s += '\n'

   return s

 def dictsToBots(bots):
   offline_bots = []
   for bot in bots['offline']:
     offline_bots.append(GpuBot.fromDict(bot))

   failed_bots = []
   for bot in bots['failed']:
     failed_bots.append(GpuBot.fromDict(bot))

   return {'offline': offline_bots, 'failed': failed_bots}

 class GpuBotPoller:
   DEFAULT_PREVIOUS_RESULTS_FILE = '.check_gpu_bots_previous_results'

   def __init__(self, emailer, send_email_for_recovered_offline_bots,
           send_email_for_recovered_failing_bots, send_email_on_error,
           previous_results_file):
     self.emailer = emailer

     self.send_email_for_recovered_offline_bots = \
             send_email_for_recovered_offline_bots

     self.send_email_for_recovered_failing_bots = \
             send_email_for_recovered_failing_bots

     self.send_email_on_error = send_email_on_error
     self.previous_results_file = previous_results_file

   def shouldEmail(self, noteworthy_events):
     if noteworthy_events['offline'] or noteworthy_events['failed']:
       return True

     if (self.send_email_for_recovered_offline_bots and
             noteworthy_events['recovered_offline']):
       return True

     if (self.send_email_for_recovered_failing_bots and
           noteworthy_events['recovered_failures']):
       return True

     return False

   def writeResults(self, summary):
     results_file = (self.previous_results_file
             if self.previous_results_file is not None
             else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

     with open(results_file, 'w') as f:
       f.write(json.dumps(summary))

   def getPreviousResults(self):
     previous_results_file = (self.previous_results_file
             if self.previous_results_file is not None
             else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

     previous_results = {}
     if os.path.isfile(previous_results_file):
       with open(previous_results_file, 'r') as f:
         previous_results = dictsToBots(json.loads(f.read()))

     return previous_results

   def checkBots(self):
     time_str = 'Current time: %s\n\n' % (formatTime(time.localtime()))
     print time_str

     try:
       bots = Waterfall.getAllGpuBots()

       offline_bots = Waterfall.getOfflineBots(bots)
       offline_str = getOfflineBotsStr(offline_bots)
       print offline_str

       failed_bots = Waterfall.getFailedBots(bots)
       failed_str = getFailedBotsStr(failed_bots)
       print failed_str

       previous_results = self.getPreviousResults()
       noteworthy_events = getNoteworthyEvents(offline_bots, failed_bots,
               previous_results)

       noteworthy_str = getNoteworthyStr(noteworthy_events)
       print noteworthy_str

       summary = getSummary(offline_bots, failed_bots)
       self.writeResults(summary)

       if (self.emailer is not None and self.shouldEmail(noteworthy_events)):
         self.emailer.send_email(Emailer.format_email_body(time_str, offline_str,
             failed_str, noteworthy_str))
     except Exception as e:
       error_str = 'Error: %s' % str(e)
       print error_str

       if self.send_email_on_error:
         self.emailer.send_email(error_str)

 def parseArgs(sys_args):
   parser = argparse.ArgumentParser(prog=sys_args[0],
           description='Query the Chromium GPU Bots Waterfall, output ' +
           'potential problems, and optionally repeat automatically and/or ' +
           'email notifications of results.')

   parser.add_argument('--repeat-delay', type=int, dest='repeat_delay',
           required=False,
           help='How often to automatically re-run the script, in minutes.')

   parser.add_argument('--email-from', type=str, dest='email_from',
           required=False,
           help='Email address to send from. Requires also specifying ' +
           '\'--email-to\'.')

   parser.add_argument('--email-to', type=str, dest='email_to', required=False,
           nargs='+',
           help='Email address(es) to send to. Requires also specifying ' +
           '\'--email-from\'')

   parser.add_argument('--send-email-for-recovered-offline-bots',
           dest='send_email_for_recovered_offline_bots', action='store_true',
           default=False,
           help='Send an email out when a bot which has been offline for more ' +
           'than 1 hour goes back online.')

   parser.add_argument('--send-email-for-recovered-failing-bots',
           dest='send_email_for_recovered_failing_bots',
           action='store_true', default=False,
           help='Send an email when a failing bot recovers.')

   parser.add_argument('--send-email-on-error',
           dest='send_email_on_error',
           action='store_true', default=False,
           help='Send an email when the script has an error. For example, if ' +
           'the server is unreachable.')

   parser.add_argument('--email-password-file',
           dest='email_password_file',
           required=False,
           help=(('File containing the plaintext password of the source email ' +
           'account. By default, \'%s\' will be tried. If it does not exist, ' +
           'you will be prompted. If you opt to store your password on disk ' +
           'in plaintext, use of a dummy account is strongly recommended.')
           % Emailer.DEFAULT_EMAIL_PASSWORD_FILE))

   parser.add_argument('--previous-results-file',
           dest='previous_results_file',
           required=False,
           help=(('File to store the results of the previous invocation of ' +
               'this script. By default, \'%s\' will be used.')
               % GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE))

   args = parser.parse_args(sys_args[1:])

   if args.email_from is not None and args.email_to is None:
     parser.error('--email-from requires --email-to.')
   elif args.email_to is not None and args.email_from is None:
     parser.error('--email-to requires --email-from.')
   elif args.email_from is None and args.send_email_for_recovered_offline_bots:
     parser.error('--send-email-for-recovered-offline-bots requires ' +
             '--email-to and --email-from.')
   elif (args.email_from is None and args.send_email_for_recovered_failing_bots):
     parser.error('--send-email-for-recovered-failing-bots ' +
             'requires --email-to and --email-from.')
   elif (args.email_from is None and args.send_email_on_error):
     parser.error('--send-email-on-error ' +
             'requires --email-to and --email-from.')
   elif (args.email_password_file and
           not os.path.isfile(args.email_password_file)):
     parser.error('File does not exist: %s' % args.email_password_file)

   return args

 def main(sys_args):
   args = parseArgs(sys_args)

   emailer = None
   if args.email_from is not None and args.email_to is not None:
     emailer = Emailer(args.email_from, args.email_to, args.email_password_file)

     try:
       emailer.testEmailLogin()
     except Exception as e:
       print 'Error logging into email account: %s' % str(e)
       return 1

   poller = GpuBotPoller(emailer,
           args.send_email_for_recovered_offline_bots,
           args.send_email_for_recovered_failing_bots,
           args.send_email_on_error,
           args.previous_results_file)

   while True:
     poller.checkBots()

     if args.repeat_delay is None:
       break

     print 'Will run again in %d minutes...\n' % args.repeat_delay
     time.sleep(args.repeat_delay * 60)

   return 0

 if __name__ == '__main__':
   sys.exit(main(sys.argv))
	#!/usr/bin/env python

	# Copyright 2014 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	import argparse
	import datetime
	import getpass
	import json
	import os
	import smtplib
	import sys
	import time
	import urllib
	import urllib2

	class Emailer:
	DEFAULT_EMAIL_PASSWORD_FILE = '.email_password'
	GMAIL_SMTP_SERVER = 'smtp.gmail.com:587'
	SUBJECT = 'Chrome GPU Bots Notification'

	def __init__(self, email_from, email_to, email_password_file):
	self.email_from = email_from
	self.email_to = email_to
	self.email_password = Emailer._getEmailPassword(email_password_file)

	@staticmethod
	def format_email_body(time_str, offline_str, failed_str, noteworthy_str):
	return '%s%s%s%s' % (time_str, offline_str, failed_str, noteworthy_str)

	def send_email(self, body):
	message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (self.email_from,
	','.join(self.email_to), Emailer.SUBJECT, body)

	try:
	server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
	server.starttls()
	server.login(self.email_from, self.email_password)
	server.sendmail(self.email_from, self.email_to, message)
	server.quit()
	except Exception as e:
	print 'Error sending email: %s' % str(e)

	def testEmailLogin(self):
	server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
	server.starttls()
	server.login(self.email_from, self.email_password)
	server.quit()

	@staticmethod
	def _getEmailPassword(email_password_file):
	password = ''

	password_file = (email_password_file if email_password_file is not None
	else Emailer.DEFAULT_EMAIL_PASSWORD_FILE)

	if os.path.isfile(password_file):
	with open(password_file, 'r') as f:
	password = f.read().strip()
	else:
	password = getpass.getpass(
	'Please enter email password for source email account: ')

	return password

	class GpuBot:
	def __init__(self, waterfall_name, bot_name, bot_data):
	self.waterfall_name = waterfall_name
	self.bot_name = bot_name
	self.bot_data = bot_data
	self._end_time = None
	self._hours_since_last_run = None
	self.failure_string = None
	self.bot_url = None
	self.build_url = None

	def getEndTime(self):
	return self._end_time

	def setEndTime(self, end_time):
	self._end_time = end_time
	self._hours_since_last_run = \
	roughTimeDiffInHours(end_time, time.localtime())

	def getHoursSinceLastRun(self):
	return self._hours_since_last_run

	def toDict(self):
	dict = {'waterfall_name': self.waterfall_name, 'bot_name': self.bot_name}

	if self._end_time is not None:
	dict['end_time'] = serialTime(self._end_time)
	dict['hours_since_last_run'] = self._hours_since_last_run

	if self.failure_string is not None:
	dict['failure_string'] = self.failure_string

	if self.bot_url is not None:
	dict['bot_url'] = self.bot_url

	if self.build_url is not None:
	dict['build_url'] = self.build_url

	return dict

	@staticmethod
	def fromDict(dict):
	gpu_bot = GpuBot(dict['waterfall_name'], dict['bot_name'], None)

	if 'end_time' in dict:
	gpu_bot._end_time = unserializeTime(dict['end_time'])

	if 'hours_since_last_run' in dict:
	gpu_bot._hours_since_last_run = dict['hours_since_last_run']

	if 'failure_string' in dict:
	gpu_bot.failure_string = dict['failure_string']

	if 'bot_url' in dict:
	gpu_bot.bot_url = dict['bot_url']

	if 'build_url' in dict:
	gpu_bot.build_url = dict['build_url']

	return gpu_bot

	def errorNoMostRecentBuild(waterfall_name, bot_name):
	print 'No most recent build available: %s::%s' % (waterfall_name, bot_name)

	class Waterfall:
	BASE_URL = 'http://build.chromium.org/p/'
	BASE_BUILD_URL = BASE_URL + '%s/builders/%s'
	SPECIFIC_BUILD_URL = BASE_URL + '%s/builders/%s/builds/%s'
	BASE_JSON_BUILDERS_URL = BASE_URL + '%s/json/builders'
	BASE_JSON_BUILDS_URL = BASE_URL + '%s/json/builders/%s/builds'
	REGULAR_WATERFALLS = ['chromium.gpu', 'chromium.gpu.fyi']
	WEBKIT_GPU_BOTS = ['GPU Win Builder',
	'GPU Win Builder (dbg)',
	'GPU Win7 (NVIDIA)',
	'GPU Win7 (dbg) (NVIDIA)',
	'GPU Mac Builder',
	'GPU Mac Builder (dbg)',
	'GPU Mac10.7',
	'GPU Mac10.7 (dbg)',
	'GPU Linux Builder',
	'GPU Linux Builder (dbg)',
	'GPU Linux (NVIDIA)',
	'GPU Linux (dbg) (NVIDIA)']
	FILTERED_WATERFALLS = [('chromium.webkit', WEBKIT_GPU_BOTS)]

	@staticmethod
	def getJsonFromUrl(url):
	conn = urllib2.urlopen(url)
	result = conn.read()
	conn.close()
	return json.loads(result)

	@staticmethod
	def getBuildersJsonForWaterfall(waterfall):
	querystring = '?filter'
	return (Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
	% (waterfall, querystring)))

	@staticmethod
	def getLastNBuildsForBuilder(n, waterfall, builder):
	if n <= 0:
	return {}

	querystring = '?'

	for i in range(n):
	querystring += 'select=-%d&' % (i + 1)

	querystring += 'filter'

	return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDS_URL + '%s') %
	(waterfall, urllib.quote(builder), querystring))

	@staticmethod
	def getFilteredBuildersJsonForWaterfall(waterfall, filter):
	querystring = '?'

	for bot_name in filter:
	querystring += 'select=%s&' % urllib.quote(bot_name)

	querystring += 'filter'

	return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
	% (waterfall, querystring))

	@staticmethod
	def getAllGpuBots():
	allbots = {k: Waterfall.getBuildersJsonForWaterfall(k)
	for k in Waterfall.REGULAR_WATERFALLS}

	filteredbots = {k[0]:
	Waterfall.getFilteredBuildersJsonForWaterfall(k[0], k[1])
	for k in Waterfall.FILTERED_WATERFALLS}

	allbots.update(filteredbots)

	return allbots

	@staticmethod
	def getOfflineBots(bots):
	offline_bots = []

	for waterfall_name in bots:
	waterfall = bots[waterfall_name]

	for bot_name in waterfall:
	bot = waterfall[bot_name]

	if bot['state'] != 'offline':
	continue

	gpu_bot = GpuBot(waterfall_name, bot_name, bot)
	gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
	urllib.quote(bot_name))

	most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
	gpu_bot)

	if (most_recent_build and 'times' in most_recent_build and
	most_recent_build['times']):
	gpu_bot.setEndTime(time.localtime(most_recent_build['times'][1]))
	else:
	errorNoMostRecentBuild(waterfall_name, bot_name)

	offline_bots.append(gpu_bot)

	return offline_bots

	@staticmethod
	def getMostRecentlyCompletedBuildForBot(bot):
	if bot.bot_data is not None and 'most_recent_build' in bot.bot_data:
	return bot.bot_data['most_recent_build']

	# Unfortunately, the JSON API doesn't provide a "most recent completed
	# build" call. We just have to get some number of the most recent (including
	# current, in-progress builds) and give up if that's not enough.
	NUM_BUILDS = 10
	builds = Waterfall.getLastNBuildsForBuilder(NUM_BUILDS, bot.waterfall_name,
	bot.bot_name)

	for i in range(NUM_BUILDS):
	current_build_name = '-%d' % (i + 1)
	current_build = builds[current_build_name]

	if 'results' in current_build and current_build['results'] is not None:
	if bot.bot_data is not None:
	bot.bot_data['most_recent_build'] = current_build

	return current_build

	return None

	@staticmethod
	def getFailedBots(bots):
	failed_bots = []

	for waterfall_name in bots:
	waterfall = bots[waterfall_name]

	for bot_name in waterfall:
	bot = waterfall[bot_name]
	gpu_bot = GpuBot(waterfall_name, bot_name, bot)
	gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
	urllib.quote(bot_name))

	most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
	gpu_bot)

	if (most_recent_build and 'text' in most_recent_build and
	'failed' in most_recent_build['text']):
	gpu_bot.failure_string = ' '.join(most_recent_build['text'])
	gpu_bot.build_url = Waterfall.SPECIFIC_BUILD_URL % (waterfall_name,
	urllib.quote(bot_name), most_recent_build['number'])
	failed_bots.append(gpu_bot)
	elif not most_recent_build:
	errorNoMostRecentBuild(waterfall_name, bot_name)

	return failed_bots

	def formatTime(t):
	return time.strftime("%a, %d %b %Y %H:%M:%S", t)

	def roughTimeDiffInHours(t1, t2):
	datetimes = []

	for t in [t1, t2]:
	datetimes.append(datetime.datetime(t.tm_year, t.tm_mon, t.tm_mday,
	t.tm_hour, t.tm_min, t.tm_sec))

	datetime_diff = datetimes[0] - datetimes[1]

	hours = float(datetime_diff.total_seconds()) / 3600.0

	return abs(hours)

	def getBotStr(bot):
	s = ' %s::%s\n' % (bot.waterfall_name, bot.bot_name)

	if bot.failure_string is not None:
	s += ' failure: %s\n' % bot.failure_string

	if bot.getEndTime() is not None:
	s += (' last build end time: %s (roughly %f hours ago)\n' %
	(formatTime(bot.getEndTime()), bot.getHoursSinceLastRun()))

	if bot.bot_url is not None:
	s += ' bot url: %s\n' % bot.bot_url

	if bot.build_url is not None:
	s += ' build url: %s\n' % bot.build_url

	s += '\n'
	return s

	def getBotsStr(bots):
	s = ''

	for bot in bots:
	s += getBotStr(bot)

	s += '\n'
	return s

	def getOfflineBotsStr(offline_bots):
	return 'Offline bots:\n%s' % getBotsStr(offline_bots)

	def getFailedBotsStr(failed_bots):
	return 'Failed bots:\n%s' % getBotsStr(failed_bots)

	def getBotDicts(bots):
	dicts = []

	for bot in bots:
	dicts.append(bot.toDict())

	return dicts

	def unserializeTime(t):
	return time.struct_time((t['year'], t['mon'], t['day'], t['hour'], t['min'],
	t['sec'], 0, 0, 0))

	def serialTime(t):
	return {'year': t.tm_year, 'mon': t.tm_mon, 'day': t.tm_mday,
	'hour': t.tm_hour, 'min': t.tm_min, 'sec': t.tm_sec}

	def getSummary(offline_bots, failed_bots):
	offline_bot_dict = getBotDicts(offline_bots)
	failed_bot_dict = getBotDicts(failed_bots)
	return {'offline': offline_bot_dict, 'failed': failed_bot_dict}

	def findBot(name, lst):
	for bot in lst:
	if bot.bot_name == name:
	return bot

	return None

	def getNoteworthyEvents(offline_bots, failed_bots, previous_results):
	CRITICAL_NUM_HOURS = 1.0

	previous_offline = (previous_results['offline'] if 'offline'
	in previous_results else [])

	previous_failures = (previous_results['failed'] if 'failed'
	in previous_results else [])

	noteworthy_offline = []
	for bot in offline_bots:
	if bot.getHoursSinceLastRun() >= CRITICAL_NUM_HOURS:
	previous_bot = findBot(bot.bot_name, previous_offline)

	if (previous_bot is None or
	previous_bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS):
	noteworthy_offline.append(bot)

	noteworthy_new_failures = []
	for bot in failed_bots:
	previous_bot = findBot(bot.bot_name, previous_failures)

	if previous_bot is None:
	noteworthy_new_failures.append(bot)

	noteworthy_new_offline_recoveries = []
	for bot in previous_offline:
	if bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS:
	continue

	current_bot = findBot(bot.bot_name, offline_bots)
	if current_bot is None:
	noteworthy_new_offline_recoveries.append(bot)

	noteworthy_new_failure_recoveries = []
	for bot in previous_failures:
	current_bot = findBot(bot.bot_name, failed_bots)

	if current_bot is None:
	noteworthy_new_failure_recoveries.append(bot)

	return {'offline': noteworthy_offline, 'failed': noteworthy_new_failures,
	'recovered_failures': noteworthy_new_failure_recoveries,
	'recovered_offline': noteworthy_new_offline_recoveries}

	def getNoteworthyStr(noteworthy_events):
	s = ''

	if noteworthy_events['offline']:
	s += 'IMPORTANT bots newly offline for over an hour:\n'

	for bot in noteworthy_events['offline']:
	s += getBotStr(bot)

	s += '\n'

	if noteworthy_events['failed']:
	s += 'IMPORTANT new failing bots:\n'

	for bot in noteworthy_events['failed']:
	s += getBotStr(bot)

	s += '\n'

	if noteworthy_events['recovered_offline']:
	s += 'IMPORTANT newly recovered previously offline bots:\n'

	for bot in noteworthy_events['recovered_offline']:
	s += getBotStr(bot)

	s += '\n'

	if noteworthy_events['recovered_failures']:
	s += 'IMPORTANT newly recovered failing bots:\n'

	for bot in noteworthy_events['recovered_failures']:
	s += getBotStr(bot)

	s += '\n'

	return s

	def dictsToBots(bots):
	offline_bots = []
	for bot in bots['offline']:
	offline_bots.append(GpuBot.fromDict(bot))

	failed_bots = []
	for bot in bots['failed']:
	failed_bots.append(GpuBot.fromDict(bot))

	return {'offline': offline_bots, 'failed': failed_bots}

	class GpuBotPoller:
	DEFAULT_PREVIOUS_RESULTS_FILE = '.check_gpu_bots_previous_results'

	def __init__(self, emailer, send_email_for_recovered_offline_bots,
	send_email_for_recovered_failing_bots, send_email_on_error,
	previous_results_file):
	self.emailer = emailer

	self.send_email_for_recovered_offline_bots = \
	send_email_for_recovered_offline_bots

	self.send_email_for_recovered_failing_bots = \
	send_email_for_recovered_failing_bots

	self.send_email_on_error = send_email_on_error
	self.previous_results_file = previous_results_file

	def shouldEmail(self, noteworthy_events):
	if noteworthy_events['offline'] or noteworthy_events['failed']:
	return True

	if (self.send_email_for_recovered_offline_bots and
	noteworthy_events['recovered_offline']):
	return True

	if (self.send_email_for_recovered_failing_bots and
	noteworthy_events['recovered_failures']):
	return True

	return False

	def writeResults(self, summary):
	results_file = (self.previous_results_file
	if self.previous_results_file is not None
	else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

	with open(results_file, 'w') as f:
	f.write(json.dumps(summary))

	def getPreviousResults(self):
	previous_results_file = (self.previous_results_file
	if self.previous_results_file is not None
	else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

	previous_results = {}
	if os.path.isfile(previous_results_file):
	with open(previous_results_file, 'r') as f:
	previous_results = dictsToBots(json.loads(f.read()))

	return previous_results

	def checkBots(self):
	time_str = 'Current time: %s\n\n' % (formatTime(time.localtime()))
	print time_str

	try:
	bots = Waterfall.getAllGpuBots()

	offline_bots = Waterfall.getOfflineBots(bots)
	offline_str = getOfflineBotsStr(offline_bots)
	print offline_str

	failed_bots = Waterfall.getFailedBots(bots)
	failed_str = getFailedBotsStr(failed_bots)
	print failed_str

	previous_results = self.getPreviousResults()
	noteworthy_events = getNoteworthyEvents(offline_bots, failed_bots,
	previous_results)

	noteworthy_str = getNoteworthyStr(noteworthy_events)
	print noteworthy_str

	summary = getSummary(offline_bots, failed_bots)
	self.writeResults(summary)

	if (self.emailer is not None and self.shouldEmail(noteworthy_events)):
	self.emailer.send_email(Emailer.format_email_body(time_str, offline_str,
	failed_str, noteworthy_str))
	except Exception as e:
	error_str = 'Error: %s' % str(e)
	print error_str

	if self.send_email_on_error:
	self.emailer.send_email(error_str)

	def parseArgs(sys_args):
	parser = argparse.ArgumentParser(prog=sys_args[0],
	description='Query the Chromium GPU Bots Waterfall, output ' +
	'potential problems, and optionally repeat automatically and/or ' +
	'email notifications of results.')

	parser.add_argument('--repeat-delay', type=int, dest='repeat_delay',
	required=False,
	help='How often to automatically re-run the script, in minutes.')

	parser.add_argument('--email-from', type=str, dest='email_from',
	required=False,
	help='Email address to send from. Requires also specifying ' +
	'\'--email-to\'.')

	parser.add_argument('--email-to', type=str, dest='email_to', required=False,
	nargs='+',
	help='Email address(es) to send to. Requires also specifying ' +
	'\'--email-from\'')

	parser.add_argument('--send-email-for-recovered-offline-bots',
	dest='send_email_for_recovered_offline_bots', action='store_true',
	default=False,
	help='Send an email out when a bot which has been offline for more ' +
	'than 1 hour goes back online.')

	parser.add_argument('--send-email-for-recovered-failing-bots',
	dest='send_email_for_recovered_failing_bots',
	action='store_true', default=False,
	help='Send an email when a failing bot recovers.')

	parser.add_argument('--send-email-on-error',
	dest='send_email_on_error',
	action='store_true', default=False,
	help='Send an email when the script has an error. For example, if ' +
	'the server is unreachable.')

	parser.add_argument('--email-password-file',
	dest='email_password_file',
	required=False,
	help=(('File containing the plaintext password of the source email ' +
	'account. By default, \'%s\' will be tried. If it does not exist, ' +
	'you will be prompted. If you opt to store your password on disk ' +
	'in plaintext, use of a dummy account is strongly recommended.')
	% Emailer.DEFAULT_EMAIL_PASSWORD_FILE))

	parser.add_argument('--previous-results-file',
	dest='previous_results_file',
	required=False,
	help=(('File to store the results of the previous invocation of ' +
	'this script. By default, \'%s\' will be used.')
	% GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE))

	args = parser.parse_args(sys_args[1:])

	if args.email_from is not None and args.email_to is None:
	parser.error('--email-from requires --email-to.')
	elif args.email_to is not None and args.email_from is None:
	parser.error('--email-to requires --email-from.')
	elif args.email_from is None and args.send_email_for_recovered_offline_bots:
	parser.error('--send-email-for-recovered-offline-bots requires ' +
	'--email-to and --email-from.')
	elif (args.email_from is None and args.send_email_for_recovered_failing_bots):
	parser.error('--send-email-for-recovered-failing-bots ' +
	'requires --email-to and --email-from.')
	elif (args.email_from is None and args.send_email_on_error):
	parser.error('--send-email-on-error ' +
	'requires --email-to and --email-from.')
	elif (args.email_password_file and
	not os.path.isfile(args.email_password_file)):
	parser.error('File does not exist: %s' % args.email_password_file)

	return args

	def main(sys_args):
	args = parseArgs(sys_args)

	emailer = None
	if args.email_from is not None and args.email_to is not None:
	emailer = Emailer(args.email_from, args.email_to, args.email_password_file)

	try:
	emailer.testEmailLogin()
	except Exception as e:
	print 'Error logging into email account: %s' % str(e)
	return 1

	poller = GpuBotPoller(emailer,
	args.send_email_for_recovered_offline_bots,
	args.send_email_for_recovered_failing_bots,
	args.send_email_on_error,
	args.previous_results_file)

	while True:
	poller.checkBots()

	if args.repeat_delay is None:
	break

	print 'Will run again in %d minutes...\n' % args.repeat_delay
	time.sleep(args.repeat_delay * 60)

	return 0

	if __name__ == '__main__':
	sys.exit(main(sys.argv))