GoogleGit

blob: 9f8c9e739b138388bd5e306020fd02f40b10048a [file] [log] [blame]
  1. #!/usr/bin/env python
  2. # Copyright 2013 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Closes tree if configured masters have failed tree-closing steps.
  6. Given a list of masters, gatekeeper_ng will get a list of the latest builds from
  7. the specified masters. It then checks if any tree-closing steps have failed, and
  8. if so closes the tree and emails appropriate parties. Configuration for which
  9. steps to close and which parties to notify are in a local gatekeeper.json file.
  10. """
  11. from collections import defaultdict
  12. from contextlib import closing, contextmanager
  13. import fnmatch
  14. import getpass
  15. import hashlib
  16. import hmac
  17. import itertools
  18. import json
  19. import logging
  20. import operator
  21. import optparse
  22. import os
  23. import random
  24. import re
  25. import sys
  26. import time
  27. import urllib
  28. import urllib2
  29. from slave import build_scan
  30. from slave import build_scan_db
  31. from slave import gatekeeper_ng_config
  32. DATA_DIR = os.path.dirname(os.path.abspath(__file__))
  33. # Buildbot status enum.
  34. SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6)
  35. def get_pwd(password_file):
  36. if os.path.isfile(password_file):
  37. return open(password_file, 'r').read().strip()
  38. return getpass.getpass()
  39. def in_glob_list(value, glob_list):
  40. """Returns True if 'value' matches any glob in 'glob_list'.
  41. Args:
  42. value: (str) The value to search for.
  43. glob_list: (list) A list of glob strings to test.
  44. """
  45. return any(fnmatch.fnmatch(value, glob)
  46. for glob in glob_list)
  47. def logging_urlopen(url, *args, **kwargs):
  48. try:
  49. return urllib2.urlopen(url, *args, **kwargs)
  50. except urllib2.HTTPError as e:
  51. logging.debug('error accessing url %s: %s' % (url, e))
  52. raise
  53. def update_status(tree_message, status_url_root, username, password):
  54. """Connects to chromium-status and closes the tree."""
  55. #TODO(xusydoc): append status if status is already closed.
  56. if isinstance(tree_message, unicode):
  57. tree_message = tree_message.encode('utf8')
  58. elif isinstance(tree_message, str):
  59. tree_message = tree_message.decode('utf8')
  60. params = urllib.urlencode({
  61. 'message': tree_message,
  62. 'username': username,
  63. 'password': password
  64. })
  65. # Standard urllib doesn't raise an exception on 403, urllib2 does.
  66. status_url = status_url_root + "/status"
  67. f = logging_urlopen(status_url, params)
  68. f.close()
  69. logging.info('success')
  70. def get_tree_status(status_url_root, username, password):
  71. status_url = status_url_root + "/current?format=json"
  72. data = logging_urlopen(status_url).read()
  73. try:
  74. return json.loads(data)
  75. except ValueError:
  76. # Failed due to authentication error?
  77. if 'login' not in data:
  78. raise
  79. # Try using bot password to authenticate.
  80. params = urllib.urlencode({
  81. 'username': username,
  82. 'password': password
  83. })
  84. try:
  85. data = logging_urlopen(status_url, params).read()
  86. except urllib2.HTTPError, e:
  87. if e.code == 405:
  88. logging.warn("update your chromium_status app.")
  89. raise
  90. return json.loads(data)
  91. def get_builder_section(gatekeeper_section, builder):
  92. """Returns the applicable gatekeeper config for the builder.
  93. If the builder isn't present or is excluded, return None.
  94. """
  95. if builder in gatekeeper_section:
  96. builder_section = gatekeeper_section[builder]
  97. elif '*' in gatekeeper_section:
  98. builder_section = gatekeeper_section['*']
  99. else:
  100. return None
  101. if not in_glob_list(builder, builder_section.get('excluded_builders', ())):
  102. return builder_section
  103. return None
  104. def check_builds(master_builds, master_jsons, gatekeeper_config):
  105. """Given a gatekeeper configuration, see which builds have failed."""
  106. succeeded_builds = []
  107. failed_builds = []
  108. # Sort by buildnumber, highest first.
  109. sorted_builds = sorted(master_builds, key=lambda x: x[3], reverse=True)
  110. successful_builder_steps = defaultdict(lambda: defaultdict(set))
  111. current_builds_successful = True
  112. for build_json, master_url, builder, buildnum in sorted_builds:
  113. gatekeeper_sections = gatekeeper_config.get(master_url, [])
  114. for gatekeeper_section in gatekeeper_sections:
  115. section_hash = gatekeeper_ng_config.gatekeeper_section_hash(
  116. gatekeeper_section)
  117. gatekeeper = get_builder_section(
  118. gatekeeper_section, build_json['builderName'])
  119. if not gatekeeper:
  120. succeeded_builds.append((master_url, builder, buildnum))
  121. continue
  122. steps = build_json['steps']
  123. excluded_steps = set(gatekeeper.get('excluded_steps', []))
  124. forgiving = set(gatekeeper.get('forgiving_steps', [])) - excluded_steps
  125. forgiving_optional = (
  126. set(gatekeeper.get('forgiving_optional', [])) - excluded_steps)
  127. closing_steps = (
  128. set(gatekeeper.get('closing_steps', [])) | forgiving) - excluded_steps
  129. closing_optional = (
  130. (set(gatekeeper.get('closing_optional', [])) | forgiving_optional) -
  131. excluded_steps
  132. )
  133. tree_notify = set(gatekeeper.get('tree_notify', []))
  134. sheriff_classes = set(gatekeeper.get('sheriff_classes', []))
  135. status_template = gatekeeper.get(
  136. 'status_template', gatekeeper_ng_config.DEFAULTS['status_template'])
  137. subject_template = gatekeeper.get(
  138. 'subject_template', gatekeeper_ng_config.DEFAULTS[
  139. 'subject_template'])
  140. finished = [s for s in steps if s.get('isFinished')]
  141. close_tree = gatekeeper.get('close_tree', True)
  142. respect_build_status = gatekeeper.get('respect_build_status', False)
  143. # We ignore EXCEPTION and RETRY here since those are usually
  144. # infrastructure-related instead of actual test errors.
  145. successful_steps = set(s['name'] for s in finished
  146. if s.get('results', [FAILURE])[0] != FAILURE)
  147. successful_builder_steps[master_url][builder].update(successful_steps)
  148. finished_steps = set(s['name'] for s in finished)
  149. if '*' in forgiving_optional:
  150. forgiving_optional = (finished_steps - excluded_steps)
  151. if '*' in closing_optional:
  152. closing_optional = (finished_steps - excluded_steps)
  153. unsatisfied_steps = closing_steps - successful_steps
  154. failed_steps = finished_steps - successful_steps
  155. failed_optional_steps = failed_steps & closing_optional
  156. unsatisfied_steps |= failed_optional_steps
  157. # Build is not yet finished, don't penalize on unstarted/unfinished steps.
  158. if build_json.get('results', None) is None:
  159. unsatisfied_steps &= finished_steps
  160. # If the entire build failed.
  161. if (not unsatisfied_steps and 'results' in build_json and
  162. build_json['results'] == FAILURE and respect_build_status):
  163. unsatisfied_steps.add('[overall build status]')
  164. buildbot_url = master_jsons[master_url]['project']['buildbotURL']
  165. project_name = master_jsons[master_url]['project']['title']
  166. if unsatisfied_steps:
  167. failed_builds.append(({'base_url': buildbot_url,
  168. 'build': build_json,
  169. 'close_tree': close_tree,
  170. 'forgiving_steps': (
  171. forgiving | forgiving_optional),
  172. 'project_name': project_name,
  173. 'sheriff_classes': sheriff_classes,
  174. 'subject_template': subject_template,
  175. 'status_template': status_template,
  176. 'tree_notify': tree_notify,
  177. 'unsatisfied': unsatisfied_steps,
  178. },
  179. master_url,
  180. builder,
  181. buildnum,
  182. section_hash))
  183. # If there is a failing step that a newer builder hasn't succeeded on,
  184. # don't open the tree.
  185. still_failing_steps = (
  186. unsatisfied_steps - successful_builder_steps[master_url][builder])
  187. if still_failing_steps and close_tree:
  188. logging.debug('%s failed on %s, not yet resolved.',
  189. ','.join(still_failing_steps),
  190. generate_build_url(failed_builds[-1][0]))
  191. current_builds_successful = False
  192. else:
  193. succeeded_builds.append((master_url, builder, buildnum))
  194. return (list(reversed(failed_builds)), list(reversed(succeeded_builds)),
  195. successful_builder_steps, current_builds_successful)
  196. def propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db):
  197. """Write back to build_db which finished steps failed or succeeded."""
  198. for _, master_url, builder, buildnum, _ in failure_tuples:
  199. builder_dict = build_db.masters[master_url][builder]
  200. if builder_dict[buildnum].finished:
  201. # pylint: disable=W0212
  202. builder_dict[buildnum] = builder_dict[buildnum]._replace(
  203. succeeded=False)
  204. for master_url, builder, buildnum in success_tuples:
  205. builder_dict = build_db.masters[master_url][builder]
  206. if builder_dict[buildnum].finished:
  207. # pylint: disable=W0212
  208. builder_dict[buildnum] = builder_dict[buildnum]._replace(
  209. succeeded=True)
  210. def get_build_properties(build_json, properties):
  211. """Obtains multiple build_properties from a build.
  212. Sets a property to None if it's not in the build.
  213. """
  214. properties = set(properties)
  215. result = dict.fromkeys(properties) # Populates dict with {key: None}.
  216. for p in build_json.get('properties', []):
  217. if p[0] in properties:
  218. result[p[0]] = p[1]
  219. return result
  220. @contextmanager
  221. def log_section(url, builder, buildnum, section_hash=None):
  222. """Wraps a code block with information about a build it operates on."""
  223. logging.debug('%sbuilders/%s/builds/%d ----', url, builder, buildnum)
  224. if section_hash:
  225. logging.debug(' section hash: %s', section_hash)
  226. yield
  227. logging.debug('----')
  228. COMMIT_POSITION_REGEX = re.compile(r'(.*)@{#(\d+)}')
  229. def parse_commit_position(prop):
  230. """Determine if the revision is a SVN revision or a git commit position.
  231. If the revision is a git commit position, return just the numeric part.
  232. """
  233. if not isinstance(prop, basestring):
  234. return prop
  235. match = COMMIT_POSITION_REGEX.match(prop)
  236. if not match:
  237. return prop
  238. else:
  239. return int(match.group(2))
  240. def convert_revisions_to_positions(property_dict):
  241. """Given a dictionary of revisions, return a dict of parsed revisions."""
  242. result = {}
  243. for k, v in property_dict.iteritems():
  244. result[k] = parse_commit_position(v)
  245. return result
  246. def reject_old_revisions(failed_builds, build_db):
  247. """Ignore builds which triggered on revisions older than the current.
  248. triggered_revisions has the format: {'revision': 500,
  249. 'got_webkit_revision': 15,
  250. }
  251. Each key is a buildproperty that was previously triggered on, and each value
  252. was the value of that key. Note that all keys present in triggered_revisions
  253. are used for the comparison. Only builds where at least one number is greater
  254. than and all numbers are greater than or equal are considered 'new' and are
  255. not rejected by this function. Any change in the set of keys triggers a full
  256. reset of the recorded data. In the common case, triggered_revisions only has
  257. one key ('revision') and rejects all builds where revision is less than or
  258. equal to the last triggered revision.
  259. """
  260. triggered_revisions = build_db.aux.get('triggered_revisions', {})
  261. if not triggered_revisions:
  262. # There was no previous revision information, so by default keep all
  263. # failing builds.
  264. logging.debug('no previous revision tracking information, '
  265. 'keeping all failures.')
  266. return failed_builds
  267. def build_start_time(build):
  268. """Sorting key that returns a build's build start time.
  269. By using reversed start time, we sort such that the latest builds come
  270. first. This gives us a crude approximation of revision order, which means
  271. we can update triggered_revisions with the highest revision first. Note that
  272. this isn't perfect, but the likelihood of multiple failures occurring in the
  273. same minute is low and multi-revision sorting is potentially error-prone. An
  274. action-log based approach would obviate this hack.
  275. """
  276. return build['build'].get('times', [None])[0]
  277. kept_builds = []
  278. for build in sorted(failed_builds, key=build_start_time, reverse=True):
  279. builder = build['build']['builderName']
  280. buildnum = build['build']['number']
  281. with log_section(build['base_url'], builder, buildnum):
  282. # get_build_properties will return a dict with all the keys given to it.
  283. # Since we're giving it triggered_revisions.keys(), revisions is
  284. # guaranteed to have the same keys as triggered_revisions.
  285. revisions = convert_revisions_to_positions(get_build_properties(
  286. build['build'], triggered_revisions.keys()))
  287. logging.debug('previous revision information: %s',
  288. str(triggered_revisions))
  289. logging.debug('current revision information: %s', str(revisions))
  290. if any(x is None for x in revisions.itervalues()):
  291. # The revisions aren't in this build, err on the side of noisy.
  292. logging.debug('Nones detected in revision tracking information, '
  293. 'keeping build.')
  294. triggered_revisions = revisions
  295. kept_builds.append(build)
  296. continue
  297. paired = []
  298. for k in revisions:
  299. paired.append((triggered_revisions[k], revisions[k]))
  300. if all(l <= r for l, r in paired) and any(l < r for l, r in paired):
  301. # At least one revision is greater and all the others are >=, so let
  302. # this revision through.
  303. # TODO(stip): evaluate the greatest revision if we see a stream of
  304. # failures at once.
  305. logging.debug('keeping build')
  306. kept_builds.append(build)
  307. triggered_revisions = revisions
  308. continue
  309. logging.debug('rejecting build')
  310. build_db.aux['triggered_revisions'] = triggered_revisions
  311. return kept_builds
  312. def debounce_failures(failed_builds, current_builds_successful, build_db):
  313. """Using trigger information in build_db, make sure we don't double-fire."""
  314. @contextmanager
  315. def save_build_failures(master_url, builder, buildnum, section_hash,
  316. unsatisfied):
  317. yield
  318. build_db.masters[master_url][builder][buildnum].triggered[
  319. section_hash] = unsatisfied
  320. if failed_builds and current_builds_successful:
  321. logging.debug(
  322. 'All failing steps succeeded in later runs, not closing tree.')
  323. return []
  324. true_failed_builds = []
  325. for build, master_url, builder, buildnum, section_hash in failed_builds:
  326. with log_section(build['base_url'], builder, buildnum, section_hash):
  327. with save_build_failures(master_url, builder, buildnum, section_hash,
  328. build['unsatisfied']):
  329. build_db_builder = build_db.masters[master_url][builder]
  330. # Determine what the current and previous failing steps are.
  331. prev_triggered = []
  332. if buildnum-1 in build_db_builder:
  333. prev_triggered = build_db_builder[buildnum-1].triggered.get(
  334. section_hash, [])
  335. logging.debug(' previous failing tests: %s', ','.join(
  336. sorted(prev_triggered)))
  337. logging.debug(' current failing tests: %s', ','.join(
  338. sorted(build['unsatisfied'])))
  339. # Skip build if we already fired (or if the failing tests aren't new).
  340. if section_hash in build_db_builder[buildnum].triggered:
  341. logging.debug(' section has already been triggered for this build, '
  342. 'skipping...')
  343. continue
  344. new_tests = set(build['unsatisfied']) - set(prev_triggered)
  345. if not new_tests:
  346. logging.debug(' no new steps failed since previous build %d',
  347. buildnum-1)
  348. continue
  349. logging.debug(' new failing steps since build %d: %s', buildnum-1,
  350. ','.join(sorted(new_tests)))
  351. # If we're here it's a legit failing build.
  352. true_failed_builds.append(build)
  353. logging.debug(' build steps: %s', ', '.join(
  354. s['name'] for s in build['build']['steps']))
  355. logging.debug(' build complete: %s', bool(
  356. build['build'].get('results', None) is not None))
  357. logging.debug(' set to close tree: %s', build['close_tree'])
  358. logging.debug(' build failed: %s', bool(build['unsatisfied']))
  359. return true_failed_builds
  360. def parse_sheriff_file(url):
  361. """Given a sheriff url, download and parse the appropirate sheriff list."""
  362. with closing(logging_urlopen(url)) as f:
  363. line = f.readline()
  364. usernames_matcher_ = re.compile(r'document.write\(\'([\w, ]+)\'\)')
  365. usernames_match = usernames_matcher_.match(line)
  366. sheriffs = set()
  367. if usernames_match:
  368. usernames_str = usernames_match.group(1)
  369. if usernames_str != 'None (channel is sheriff)':
  370. for sheriff in usernames_str.split(', '):
  371. if sheriff.count('@') == 0:
  372. sheriff += '@google.com'
  373. sheriffs.add(sheriff)
  374. return sheriffs
  375. def get_sheriffs(classes, base_url):
  376. """Given a list of sheriff classes, download and combine sheriff emails."""
  377. sheriff_sets = (parse_sheriff_file(base_url % cls) for cls in classes)
  378. return reduce(operator.or_, sheriff_sets, set())
  379. def hash_message(message, url, secret):
  380. utc_now = time.time()
  381. salt = random.getrandbits(32)
  382. hasher = hmac.new(secret, message, hashlib.sha256)
  383. hasher.update(str(utc_now))
  384. hasher.update(str(salt))
  385. client_hash = hasher.hexdigest()
  386. return {'message': message,
  387. 'time': utc_now,
  388. 'salt': salt,
  389. 'url': url,
  390. 'hmac-sha256': client_hash,
  391. }
  392. def submit_email(email_app, build_data, secret):
  393. """Submit json to a mailer app which sends out the alert email."""
  394. url = email_app + '/email'
  395. data = hash_message(json.dumps(build_data, sort_keys=True), url, secret)
  396. req = urllib2.Request(url, urllib.urlencode({'json': json.dumps(data)}))
  397. with closing(logging_urlopen(req)) as f:
  398. code = f.getcode()
  399. if code != 200:
  400. response = f.read()
  401. raise Exception('error connecting to email app: code %d %s' % (
  402. code, response))
  403. def open_tree_if_possible(build_db, master_jsons, successful_builder_steps,
  404. current_builds_successful, username, password, status_url_root,
  405. set_status, emoji):
  406. if not current_builds_successful:
  407. logging.debug('Not opening tree because failing steps were detected.')
  408. return
  409. previously_failed_builds = []
  410. for master_url, master in master_jsons.iteritems():
  411. for builder in master['builders']:
  412. builder_dict = build_db.masters.get(master_url, {}).get(builder, {})
  413. for buildnum, build in builder_dict.iteritems():
  414. if build.finished:
  415. if not build.succeeded:
  416. if build.triggered:
  417. # See crbug.com/389740 for why the 0 is there.
  418. failing_steps = set(build.triggered.values()[0])
  419. else:
  420. failing_steps = set()
  421. still_failing_steps = (
  422. failing_steps - successful_builder_steps[master_url][builder])
  423. if still_failing_steps:
  424. previously_failed_builds.append(
  425. '%s on %s %s/builders/%s/builds/%d' % (
  426. ','.join(still_failing_steps), builder, master_url,
  427. urllib.quote(builder), buildnum))
  428. if previously_failed_builds:
  429. logging.debug(
  430. 'Not opening tree because previous builds weren\'t successful:')
  431. for build in previously_failed_builds:
  432. logging.debug(' %s' % build)
  433. return
  434. status = get_tree_status(status_url_root, username, password)
  435. # Don't change the status unless the tree is currently closed.
  436. if status['general_state'] != 'closed':
  437. return
  438. # Don't override human closures.
  439. closed_tree_key = 'closed_tree-%s' % status_url_root
  440. last_gatekeeper_closure = build_db.aux.get(closed_tree_key)
  441. if last_gatekeeper_closure:
  442. if last_gatekeeper_closure['message'] != status['message']:
  443. return
  444. else:
  445. # Backwards compatability hack.
  446. if not re.search(r"automatic", status['message'], re.IGNORECASE):
  447. return
  448. logging.info('All builders are green, opening the tree...')
  449. tree_status = 'Tree is open (Automatic)'
  450. if emoji:
  451. random_emoji = random.choice(emoji)
  452. if random_emoji.endswith(')'):
  453. random_emoji += ' '
  454. tree_status = 'Tree is open (Automatic: %s)' % random_emoji
  455. logging.info('Opening tree with message: \'%s\'' % tree_status)
  456. build_db.aux[closed_tree_key] = {}
  457. if set_status:
  458. update_status(tree_status, status_url_root, username, password)
  459. else:
  460. logging.info('set-status not set, not connecting to chromium-status!')
  461. def generate_build_url(build):
  462. """Creates a URL to reference the build."""
  463. return '%s/builders/%s/builds/%d' % (
  464. build['base_url'].rstrip('/'),
  465. urllib.quote(build['build']['builderName']),
  466. build['build']['number']
  467. )
  468. def get_results_string(result_value):
  469. """Returns a string for a BuildBot result value (SUCCESS, FAILURE, etc.)."""
  470. return {
  471. SUCCESS: 'success',
  472. WARNINGS: 'warnings',
  473. FAILURE: 'failure',
  474. SKIPPED: 'skipped',
  475. EXCEPTION: 'exception',
  476. RETRY: 'retry',
  477. }.get(result_value, 'unknown')
  478. def close_tree_if_necessary(build_db, failed_builds, username, password,
  479. status_url_root, set_status, revision_properties):
  480. """Given a list of failed builds, close the tree if necessary."""
  481. closing_builds = [b for b in failed_builds if b['close_tree']]
  482. if not closing_builds:
  483. logging.info('no tree-closing failures!')
  484. return
  485. status = get_tree_status(status_url_root, username, password)
  486. # Don't change the status unless the tree is currently open.
  487. if status['general_state'] != 'open':
  488. return
  489. logging.info('%d failed builds found, closing the tree...' %
  490. len(closing_builds))
  491. template_build = closing_builds[0]
  492. template_vars = {
  493. 'blamelist': ','.join(template_build['build']['blame']),
  494. 'build_url': generate_build_url(template_build),
  495. 'builder_name': template_build['build']['builderName'],
  496. 'project_name': template_build['project_name'],
  497. 'unsatisfied': ','.join(template_build['unsatisfied']),
  498. 'result': get_results_string(template_build['build'].get('results')),
  499. }
  500. # First populate un-transformed build properties
  501. revision_props = get_build_properties(template_build['build'],
  502. ['revision', 'got_revision', 'buildnumber',])
  503. # Second add in transformed specified revision_properties.
  504. revision_props.update(convert_revisions_to_positions(
  505. get_build_properties(template_build['build'], revision_properties)))
  506. template_vars.update(revision_props)
  507. # Close on first failure seen.
  508. tree_status = template_build['status_template'] % template_vars
  509. logging.info('closing the tree with message: \'%s\'' % tree_status)
  510. if set_status:
  511. update_status(tree_status, status_url_root, username, password)
  512. closed_tree_key = 'closed_tree-%s' % status_url_root
  513. build_db.aux[closed_tree_key] = {
  514. 'message': tree_status,
  515. }
  516. else:
  517. logging.info('set-status not set, not connecting to chromium-status!')
  518. def notify_failures(failed_builds, sheriff_url, default_from_email,
  519. email_app_url, secret, domain, filter_domain,
  520. disable_domain_filter):
  521. # Email everyone that should be notified.
  522. emails_to_send = []
  523. for failed_build in failed_builds:
  524. waterfall_url = failed_build['base_url'].rstrip('/')
  525. build_url = generate_build_url(failed_build)
  526. project_name = failed_build['project_name']
  527. fromaddr = failed_build['build'].get('fromAddr', default_from_email)
  528. tree_notify = failed_build['tree_notify']
  529. if failed_build['unsatisfied'] <= failed_build['forgiving_steps']:
  530. blamelist = set()
  531. else:
  532. blamelist = set(failed_build['build']['blame'])
  533. sheriffs = get_sheriffs(failed_build['sheriff_classes'], sheriff_url)
  534. watchers = list(tree_notify | blamelist | sheriffs)
  535. build_data = {
  536. 'build_url': build_url,
  537. 'from_addr': fromaddr,
  538. 'project_name': project_name,
  539. 'subject_template': failed_build['subject_template'],
  540. 'steps': [],
  541. 'unsatisfied': list(failed_build['unsatisfied']),
  542. 'waterfall_url': waterfall_url,
  543. }
  544. for field in ['builderName', 'number', 'reason']:
  545. build_data[field] = failed_build['build'][field]
  546. # The default value here is 2. In the case of failing on an unfinished
  547. # build, the build won't have a result yet. As of now, chromium-build treats
  548. # anything as 'not failure' as warning. Since we can't get into
  549. # notify_failures without a failure, it makes sense to have the default
  550. # value be failure (2) here.
  551. build_data['result'] = failed_build['build'].get('results', 2)
  552. build_data['blamelist'] = failed_build['build']['blame']
  553. build_data['changes'] = failed_build['build'].get('sourceStamp', {}).get(
  554. 'changes', [])
  555. build_data['revisions'] = [x['revision'] for x in build_data['changes']]
  556. for step in failed_build['build']['steps']:
  557. new_step = {}
  558. for field in ['text', 'name', 'logs']:
  559. new_step[field] = step[field]
  560. new_step['started'] = step.get('isStarted', False)
  561. new_step['urls'] = step.get('urls', [])
  562. new_step['results'] = step.get('results', [0, None])[0]
  563. build_data['steps'].append(new_step)
  564. if email_app_url and watchers:
  565. emails_to_send.append((watchers, json.dumps(build_data, sort_keys=True)))
  566. buildnum = failed_build['build']['number']
  567. steps = failed_build['unsatisfied']
  568. builder = failed_build['build']['builderName']
  569. logging.info(
  570. 'to %s: failure in %s build %s: %s' % (', '.join(watchers),
  571. builder, buildnum,
  572. list(steps)))
  573. if not email_app_url:
  574. logging.warn('no email_app_url specified, no email sent!')
  575. filtered_emails_to_send = []
  576. for email in emails_to_send:
  577. new_watchers = [x if '@' in x else (x + '@' + domain) for x in email[0]]
  578. if not disable_domain_filter:
  579. new_watchers = [x for x in new_watchers if x.split('@')[-1] in
  580. filter_domain]
  581. if new_watchers:
  582. filtered_emails_to_send.append((new_watchers, email[1]))
  583. # Deduplicate emails.
  584. keyfunc = lambda x: x[1]
  585. for k, g in itertools.groupby(sorted(filtered_emails_to_send, key=keyfunc),
  586. keyfunc):
  587. watchers = list(reduce(operator.or_, [set(e[0]) for e in g], set()))
  588. build_data = json.loads(k)
  589. build_data['recipients'] = watchers
  590. submit_email(email_app_url, build_data, secret)
  591. def get_options():
  592. prog_desc = 'Closes the tree if annotated builds fail.'
  593. usage = '%prog [options] <one or more master urls>'
  594. parser = optparse.OptionParser(usage=(usage + '\n\n' + prog_desc))
  595. parser.add_option('--build-db', default='build_db.json',
  596. help='records the last-seen build for each builder')
  597. parser.add_option('--clear-build-db', action='store_true',
  598. help='reset build_db to be empty')
  599. parser.add_option('--sync-build-db', action='store_true',
  600. help='don\'t process any builds, but update build_db '
  601. 'to the latest build numbers')
  602. parser.add_option('--skip-build-db-update', action='store_true',
  603. help='don\' write to the build_db, overridden by sync and'
  604. ' clear db options')
  605. parser.add_option('--password-file', default='.status_password',
  606. help='password file to update chromium-status')
  607. parser.add_option('-s', '--set-status', action='store_true',
  608. help='close the tree by connecting to chromium-status')
  609. parser.add_option('--open-tree', action='store_true',
  610. help='open the tree by connecting to chromium-status')
  611. parser.add_option('--status-url',
  612. default='https://chromium-status.appspot.com',
  613. help='URL for root of the status app')
  614. parser.add_option('--track-revisions', action='store_true',
  615. help='only close on increasing revisions')
  616. parser.add_option('--revision-properties', default='revision',
  617. help='comma-separated list of buildproperties to compare '
  618. 'revision on.')
  619. parser.add_option('--status-user', default='buildbot@chromium.org',
  620. help='username for the status app')
  621. parser.add_option('--disable-domain-filter', action='store_true',
  622. help='allow emailing any domain')
  623. parser.add_option('--filter-domain', default='chromium.org,google.com',
  624. help='only email users in these comma separated domains')
  625. parser.add_option('--email-domain', default='google.com',
  626. help='default email domain to add to users without one')
  627. parser.add_option('--sheriff-url',
  628. default='http://build.chromium.org/p/chromium/%s.js',
  629. help='URL pattern for the current sheriff list')
  630. parser.add_option('--parallelism', default=16,
  631. help='up to this many builds can be queried simultaneously')
  632. parser.add_option('--default-from-email',
  633. default='buildbot@chromium.org',
  634. help='default email address to send from')
  635. parser.add_option('--email-app-url',
  636. default='https://chromium-build.appspot.com/mailer',
  637. help='URL of the application to send email from')
  638. parser.add_option('--email-app-secret-file',
  639. default='.mailer_password',
  640. help='file containing secret used in email app auth')
  641. parser.add_option('--no-email-app', action='store_true',
  642. help='don\'t send emails')
  643. parser.add_option('--json', default=os.path.join(DATA_DIR, 'gatekeeper.json'),
  644. help='location of gatekeeper configuration file')
  645. parser.add_option('--emoji',
  646. default=os.path.join(DATA_DIR, 'gatekeeper_emoji.json'),
  647. help='location of gatekeeper configuration file (None to'
  648. 'turn off)')
  649. parser.add_option('--verify', action='store_true',
  650. help='verify that the gatekeeper config file is correct')
  651. parser.add_option('--flatten-json', action='store_true',
  652. help='display flattened gatekeeper.json for debugging')
  653. parser.add_option('--no-hashes', action='store_true',
  654. help='don\'t insert gatekeeper section hashes')
  655. parser.add_option('-v', '--verbose', action='store_true',
  656. help='turn on extra debugging information')
  657. options, args = parser.parse_args()
  658. options.email_app_secret = None
  659. options.password = None
  660. if options.no_hashes and not options.flatten_json:
  661. parser.error('specifying --no-hashes doesn\'t make sense without '
  662. '--flatten-json')
  663. if options.verify or options.flatten_json:
  664. return options, args
  665. if not args:
  666. parser.error('you need to specify at least one master URL')
  667. if options.no_email_app:
  668. options.email_app_url = None
  669. if options.email_app_url:
  670. if os.path.exists(options.email_app_secret_file):
  671. with open(options.email_app_secret_file) as f:
  672. options.email_app_secret = f.read().strip()
  673. else:
  674. parser.error('Must provide email app auth with %s.' % (
  675. options.email_app_secret_file))
  676. options.filter_domain = options.filter_domain.split(',')
  677. args = [url.rstrip('/') for url in args]
  678. return options, args
  679. def main():
  680. options, args = get_options()
  681. logging.basicConfig(level=logging.DEBUG if options.verbose else logging.INFO)
  682. gatekeeper_config = gatekeeper_ng_config.load_gatekeeper_config(options.json)
  683. if options.verify:
  684. return 0
  685. if options.flatten_json:
  686. if not options.no_hashes:
  687. gatekeeper_config = gatekeeper_ng_config.inject_hashes(gatekeeper_config)
  688. gatekeeper_ng_config.flatten_to_json(gatekeeper_config, sys.stdout)
  689. print
  690. return 0
  691. if options.set_status:
  692. options.password = get_pwd(options.password_file)
  693. masters = set(args)
  694. if not masters <= set(gatekeeper_config):
  695. print 'The following masters are not present in the gatekeeper config:'
  696. for m in masters - set(gatekeeper_config):
  697. print ' ' + m
  698. return 1
  699. emoji = []
  700. if options.emoji != 'None':
  701. try:
  702. with open(options.emoji) as f:
  703. emoji = json.load(f)
  704. except (IOError, ValueError) as e:
  705. logging.warning('Could not load emoji file %s: %s', options.emoji, e)
  706. if options.clear_build_db:
  707. build_db = {}
  708. build_scan_db.save_build_db(build_db, gatekeeper_config,
  709. options.build_db)
  710. else:
  711. build_db = build_scan_db.get_build_db(options.build_db)
  712. master_jsons, build_jsons = build_scan.get_updated_builds(
  713. masters, build_db, options.parallelism)
  714. if options.sync_build_db:
  715. build_scan_db.save_build_db(build_db, gatekeeper_config,
  716. options.build_db)
  717. return 0
  718. (failure_tuples, success_tuples, successful_builder_steps,
  719. current_builds_successful) = check_builds(
  720. build_jsons, master_jsons, gatekeeper_config)
  721. # Write failure / success information back to the build_db.
  722. propagate_build_status_back_to_db(failure_tuples, success_tuples, build_db)
  723. # opening is an option, mostly to keep the unittests working which
  724. # assume that any setting of status is negative.
  725. if options.open_tree:
  726. open_tree_if_possible(build_db, master_jsons, successful_builder_steps,
  727. current_builds_successful, options.status_user, options.password,
  728. options.status_url, options.set_status, emoji)
  729. # debounce_failures does 3 things:
  730. # 1. Groups logging by builder
  731. # 2. Selects out the "build" part from the failure tuple.
  732. # 3. Rejects builds we've already warned about (and logs).
  733. new_failures = debounce_failures(failure_tuples,
  734. current_builds_successful, build_db)
  735. if options.track_revisions:
  736. # Only close the tree if it's a newer revision than before.
  737. properties = options.revision_properties.split(',')
  738. triggered_revisions = build_db.aux.get('triggered_revisions', {})
  739. if not triggered_revisions or (
  740. sorted(triggered_revisions) != sorted(properties)):
  741. logging.info('revision properties have changed from %s to %s. '
  742. 'clearing previous data.', triggered_revisions, properties)
  743. build_db.aux['triggered_revisions'] = dict.fromkeys(properties)
  744. new_failures = reject_old_revisions(new_failures, build_db)
  745. close_tree_if_necessary(build_db, new_failures,
  746. options.status_user, options.password,
  747. options.status_url, options.set_status,
  748. options.revision_properties.split(','))
  749. notify_failures(new_failures, options.sheriff_url, options.default_from_email,
  750. options.email_app_url, options.email_app_secret,
  751. options.email_domain, options.filter_domain,
  752. options.disable_domain_filter)
  753. if not options.skip_build_db_update:
  754. build_scan_db.save_build_db(build_db, gatekeeper_config,
  755. options.build_db)
  756. return 0
  757. if __name__ == '__main__':
  758. sys.exit(main())