blob: 359acb6ae8bee76aff244310929d025c3c5d9576 [file] [log] [blame]
# coding=utf8
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Sends patches to the Try server and reads back results.
- RietveldTryJobs contains RietveldTryJob, one per try job on a builder.
- TryRunnerRietveld uses Rietveld to signal and poll job results.
import collections
import errno
import logging
import re
import socket
import time
import urllib2
import buildbot_json
import model
from verification import base
from verification import try_job_steps
# A build running for longer than this is considered to be timed out.
TIMED_OUT = 12 * 60 * 60
def is_job_expired(now, revision, timestamp, checkout):
"""Returns False if the job result is still somewhat valid.
A job that occured more than 4 days ago or more than 200 commits behind
is 'expired'.
if timestamp < (now - 4*24*60*60):
return True
if checkout.revisions(revision, None) >= 200:
return True
return False
TryJobProperties = collections.namedtuple(
['key', 'parent_key', 'builder', 'build', 'buildnumber', 'properties'])
def filter_jobs(try_job_results, watched_builders, current_irrelevant_keys,
"""For each try jobs results, query the Try Server for updated status and
returns details about each job in a TryJobProperties.
Returns a list of namedtuple describing the updated results and and the new
list of irrelevant keys.
It adds the build to the ignored list if the build doesn't exist on the Try
Server anymore (usually it's too old) or if the try job was not triggered by
the Commit Queue itself.
irrelevant = set(current_irrelevant_keys)
try_jobs_with_props = []
for result in try_job_results:
key = result['key']
assert key
if key in current_irrelevant_keys:
builder = result['builder']
buildnumber = int(result['buildnumber'])
except (TypeError, ValueError):
if buildnumber < 0:
logging.debug('Ignoring %s/%d; invalid', builder, buildnumber)
if builder not in watched_builders:
logging.debug('Ignoring %s/%d; no step verifier is examining it', builder,
# Constructing the object itself doesn't throw an exception, it's reading
# its properties that throws.
build =[builder].builds[buildnumber]
props = build.properties_as_dict
except IOError:
'Build %s/%s is not on the try server anymore',
builder, buildnumber)
parent_key = props.get('parent_try_job_key')
if parent_key:
# Triggered build
key = '%s/%d_triggered_%s' % (builder, buildnumber, parent_key)
elif props.get('try_job_key') != key:
# not triggered, not valid
'Ignoring %s/%d; not from rietveld', builder, buildnumber)
TryJobProperties(key, parent_key, builder, build, buildnumber, props))
# Sort the non-triggered builds first so triggered jobs
# can expect their parent to be added to self.try_jobs
try_jobs_with_props.sort(key=lambda tup: tup.parent_key)
return try_jobs_with_props, list(irrelevant)
def _is_skip_try_job(pending):
"""Returns True if a description contains NOTRY=true."""
match ='^NOTRY=(.*)$', pending.description, re.MULTILINE)
return match and == 'true'
class RietveldTryJobPending(model.PersistentMixIn):
"""Represents a pending try job for a pending commit that we care about.
It is immutable.
builder = unicode
revision = (None, unicode, int)
requested_steps = list
clobber = bool
# Number of retries for this configuration. Initial try is 1.
tries = int
init_time = float
def __init__(self, **kwargs):
required = set(self._persistent_members())
actual = set(kwargs)
assert required == actual, (required - actual, required, actual)
super(RietveldTryJobPending, self).__init__(**kwargs)
# Then mark it read-only.
self._read_only = True
class RietveldTryJob(model.PersistentMixIn):
"""Represents a try job for a pending commit that we care about.
This data can be regenerated by parsing all the try job names but it is a bit
hard on the try server.
It is immutable.
builder = unicode
build = int
revision = (None, unicode, int)
requested_steps = list
# The timestamp when the build started. buildbot_json returns int.
started = int
steps_passed = list
steps_failed = list
clobber = bool
completed = bool
# Number of retries for this configuration. Initial try is 1.
tries = int
parent_key = (None, unicode)
init_time = float
def __init__(self, **kwargs):
required = set(self._persistent_members())
actual = set(kwargs)
assert required == actual, (required - actual, required, actual)
super(RietveldTryJob, self).__init__(**kwargs)
# Then mark it read-only.
self._read_only = True
def result(self):
if self.steps_failed:
return buildbot_json.FAILURE
if self.completed:
return buildbot_json.SUCCESS
return None
class RietveldTryJobs(base.IVerifierStatus):
"""A set of try jobs that were sent for a specific patch.
Multiple concurrent try jobs can be sent on a single builder. For example, a
previous valid try job could have been triggered by the user but was not
completed so another was sent with the missing tests.
Also, a try job is sent as soon as a test failure is detected.
# An dict of RietveldTryJob objects per key.
try_jobs = dict
# The try job keys we ignore because they can't be used to give a good
# signal: either they are too old (old revision) or they were not triggerd
# by Rietveld, so we don't know if the diff is 100% good.
irrelevant = list
# When NOTRY=true is specified.
skipped = bool
# List of test verifiers. All the logic to decide when they are
# and what bots they trigger is hidden inside.
step_verifiers = list
# Jobs that have been sent but are not found yet. Likely a builder is fully
# utilized or the try server hasn't polled Rietveld yet. list of
# RietveldTryJobPending() instances.
pendings = list
def get_state(self):
"""Returns the state of this verified.
Failure can be from:
- For each entry in self.step_verifiers:
- A Try Job in self.try_jobs has been retried too often.
In particular, there is no need to wait for every Try Job to complete.
if self.error_message:
return base.FAILED
if not self.tests_waiting_for_result():
return base.SUCCEEDED
return base.PROCESSING
def tests_need_to_be_run(self, now):
"""Returns which tests need to be run.
These are the tests that are not pending on any try job, either running or
in the pending list.
# Skipped or failed, nothing to do.
if self.skipped or self.error_message:
return {}
# What originally needed to be run.
# All_tests is {builder_name: set(test_name*)}
all_tests = {}
for verifier in self.step_verifiers:
(builder, tests) = verifier.need_to_trigger(self.try_jobs, now)
if tests:
all_tests.setdefault(builder, set()).update(tests)
# Removes what is queued to be run but hasn't started yet.
for try_job in self.pendings:
if try_job.builder in all_tests:
all_tests[try_job.builder] -= set(try_job.requested_steps)
return dict(
(builder, sorted(tests)) for builder, tests in all_tests.iteritems()
if tests)
def tests_waiting_for_result(self):
"""Returns the tests that we are waiting for results on pending or running
# Skipped or failed, nothing to do.
if self.skipped or self.error_message:
return {}
# What originally needed to be run.
all_tests = {}
for verification in self.step_verifiers:
(builder, tests) = verification.waiting_for(self.try_jobs)
if tests:
all_tests.setdefault(builder, set()).update(tests)
# Removes what was run.
for try_job in self.try_jobs.itervalues():
if try_job.builder in all_tests:
all_tests[try_job.builder] -= set(try_job.steps_passed)
return dict(
(builder, list(tests)) for builder, tests in all_tests.iteritems()
if tests)
def watched_builders(self):
"""Marks all the jobs that the step_verifiers don't examine as
# Generate the list of builders to keep.
watched_builders = set()
for step_verifier in self.step_verifiers:
if isinstance(step_verifier, try_job_steps.TryJobTriggeredSteps):
return watched_builders
def update_jobs_from_rietveld(
self, data, status, checkout, now):
"""Retrieves the jobs statuses from rietveld and updates its state.
owner: Owner of the CL.
data: Patchset properties as returned from Rietveld.
status: A buildbot_json.Buildbot instance.
checkout: A depot_tools' Checkout instance.
now: epoch time of what should be considered to be 'now'.
Keys which were updated.
updated = []
try_job_results = data.get('try_job_results', [])
logging.debug('Found %d entries', len(try_job_results))
try_jobs_with_props, self.irrelevant = filter_jobs(
try_job_results, self.watched_builders() , self.irrelevant, status)
# Ensure that all irrelevant jobs have been removed from the set of valid
# try jobs.
for irrelevant_key in self.irrelevant:
if irrelevant_key in self.try_jobs:
del self.try_jobs[irrelevant_key]
if irrelevant_key + '_old' in self.try_jobs:
del self.try_jobs[irrelevant_key + '_old']
for i in try_jobs_with_props:
if self._update_try_job_status(checkout, i, now):
return updated
def _update_try_job_status(self, checkout, try_job_properties, now):
"""Updates status of a specific RietveldTryJob.
try_job_property is an instance of TryJobProperties.
Returns True if it was updated.
key = try_job_properties.key
builder = try_job_properties.builder
buildnumber = try_job_properties.buildnumber
if key in self.irrelevant:
logging.debug('Ignoring %s/%d; irrelevant', builder, buildnumber)
return False
if (try_job_properties.parent_key and
try_job_properties.parent_key not in self.try_jobs):
logging.debug('Ignoring %s, parent unknown', key)
return False
requested_steps = []
# Set it to 0 as the default value since when the job is new and previous
# try jobs are found, we don't want to count them as tries.
tries = 0
job = self.try_jobs.get(key)
build =
if job:
if job.completed:
logging.debug('Ignoring %s/%d; completed', builder, buildnumber)
return False
if now - job.started > TIMED_OUT:
# Flush it and start over.
del self.try_jobs[key]
return False
requested_steps = job.requested_steps
tries = job.tries
init_time = job.init_time
# This try job is new. See if we triggered it previously by
# looking in self.pendings.
for index, pending_job in enumerate(self.pendings):
if pending_job.builder == builder:
# Reuse its item.
requested_steps = pending_job.requested_steps
tries = pending_job.tries
# Is this a good build? It must not be too old and triggered by
# rietveld.
if is_job_expired(now, build.revision, build.start_time, checkout):
logging.debug('Ignoring %s/%d; expired', builder, buildnumber)
return False
init_time = now
passed = [ for s in build.steps if s.simplified_result]
failed = [ for s in build.steps if s.simplified_result is False]
# The steps in neither passed or failed were skipped.
new_job = RietveldTryJob(
if job and and and !=
# It's tricky because 'key' is the same for both. The trick is to create
# a fake key for the old build and mark it as completed. Note that
# Rietveld is confused by it too.
'Try Server was restarted and restarted builds with the same keys. '
'I\'m confused. %s: %d != %d', job.builder,,
# Resave the old try job and mark it as completed.
self.try_jobs[key + '_old'] = RietveldTryJob(
if not job or not model.is_equivalent(new_job, job):
'Job update: %s: %s/%d','issue'),
self.try_jobs[key] = new_job
return key
def signal_as_failed_if_needed(self, job, url, now):
"""Detects if the RietveldTryJob instance is in a state where it is
impossible to make progress.
If so, mark ourself as failed by setting self.error_message and return True.
if self.skipped or self.error_message:
return False
# Figure out steps that should be retried for this builder.
missing_tests = self.tests_need_to_be_run(now).get(job.builder, [])
if not missing_tests:
return False
if job.tries > 2:
self.error_message = (
'Retried try job too often on %s for step(s) %s\n%s' %
(job.builder, ', '.join(missing_tests), url))
return True
return False
def why_not(self):
# Skipped or failed, nothing to do.
if self.skipped or self.error_message:
return None
waiting = self.tests_waiting_for_result()
if waiting:
out = 'Waiting for the following jobs:\n'
for builder in sorted(waiting):
out += ' %s: %s\n' % (builder, ','.join(waiting[builder]))
return out
class TryRunnerRietveld(base.VerifierCheckout):
"""Stateless communication with a try server.
Uses Rietveld to trigger the try job and reads try job status with the json
Analysis goes as following:
- compile step itself is not flaky. already takes care of most
flakiness and clobber build is done by default. If compile step fails, try
again with clobber=True
- test steps are flaky and can be retried as necessary.
1. For each existing try jobs from rietveld.
1. Fetch result from try server.
2. If try job was generated from rietveld;
1. If not is_job_expired();
1. Skip any scheduled test that succeeded on this builder.
2. For each builder with tests scheduled;
1. If no step waiting to be triggered, skip this builder completely.
2. For each non succeeded job;
1. Send try jobs to rietveld.
Note: It needs rietveld, hence it uses VerifierCheckout, but it doesn't need a
name = 'try job rietveld'
# Only updates a job status once every 60 seconds.
update_latency = 60
def __init__(
self, context_obj, try_server_url, commit_user, step_verifiers,
ignored_steps, solution):
super(TryRunnerRietveld, self).__init__(context_obj)
self.try_server_url = try_server_url.rstrip('/')
self.commit_user = commit_user
# TODO(maruel): Have it be overridden by presubmit_support.DoGetTrySlaves.
self.step_verifiers = step_verifiers
self.ignored_steps = set(ignored_steps)
# Time to poll the Try Server, and not Rietveld.
self.last_update = time.time() - self.update_latency
self.solution = solution
def verify(self, pending):
"""Sends a try job to the try server and returns a RietveldTryJob list.
This function is called synchronously.
jobs = pending.verifications.setdefault(, RietveldTryJobs())
if _is_skip_try_job(pending):
# Do not run try job for it.
jobs.skipped = True
# Overridde any previous list from the last restart.
jobs.step_verifiers = []
for step in self.step_verifiers:
if isinstance(step, try_job_steps.TryJobTriggeredOrNormalSteps):
# Since the steps are immutable, create a new step so that swarm
# can be enabled.
# First, update the status of the current try jobs on Rietveld.
now = time.time()
self._update_jobs_from_rietveld(pending, jobs, False, now)
# Add anything that is missing.
self._send_jobs(pending, jobs, now)
# Slightly postpone next check.
self.last_update = min(now, self.last_update + (self.update_latency / 4))
def update_status(self, queue):
"""Grabs the current status of all try jobs and update self.queue.
Note: it would be more efficient to be event based.
if not queue:
logging.debug('The list is empty, nothing to do')
# Hard code 'now' to the value before querying and sending them. This will
# cause some issues when querying state or sending the jobs takes a
# non-trivial amount of time but in general it will be fine.
now = time.time()
if now - self.last_update < self.update_latency:
logging.debug('TS: Throttling updates')
self.last_update = now
# Update the status of the current pending CLs on Rietveld.
for pending, jobs in self.loop(queue, RietveldTryJobs, True):
# Update 'now' since querying the try jobs may take a significant amount
# of time.
now = time.time()
if self._update_jobs_from_rietveld(pending, jobs, True, now):
# Send any necessary job. Noop if not needed.
self._send_jobs(pending, jobs, now)
def _add_pending_job_and_send_if_needed(self, builder, steps, jobs,
send_job, pending, now):
# Find if there was a previous try.
previous_jobs = [
job for job in jobs.try_jobs.itervalues() if job.builder == builder
if previous_jobs:
tries = max(job.tries for job in previous_jobs)
clobber = max(
(job.clobber or 'compile' in job.steps_failed)
for job in previous_jobs)
tries = 0
clobber = False
if tries > 4:
# Fail safe.
jobs.error_message = (
( 'The commit queue went berserk retrying too often for a\n'
'seemingly flaky test on builder %s:\n%s') %
( builder,
'\n'.join(self._build_status_url(j) for j in previous_jobs)))
return False
# Don't always send the job (triggered bots don't need to send there own
# request).
if send_job:
'Sending job %s for %s: %s', pending.issue, builder, ','.join(steps))
pending.issue, pending.patchset, 'CQ', clobber, 'HEAD',
{builder: steps})
except urllib2.HTTPError as e:
if e.code == 400:
# This probably mean a new patchset was uploaded since the last poll,
# so it's better to drop the CL.
jobs.error_message = 'Failed to trigger a try job on %s\n%s' % (
builder, e)
return False
# Set the status of this pending job here and on the CQ page.
tries=tries + 1))
# Update the status on the AppEngine status to signal a new try job was
# sent.
info = {
'builder': builder,
'clobber': clobber,
'job_name': 'CQ',
'revision': None, #revision,
self.send_status(pending, info)
return True
def _get_triggered_bots(self, builder, steps):
"""Returns a dict of all the (builder, steps) pairs of bots that will get
triggered by the given builder steps combination."""
triggered_bots = {}
for verifier in self.step_verifiers:
builder, steps = verifier.get_triggered_steps(builder, steps)
if steps:
triggered_bots[builder] = steps
return triggered_bots
def _send_jobs(self, pending, jobs, now):
"""Prepares the RietveldTryJobs instance |jobs| to send try jobs to the try
if jobs.error_message:
# Too late.
remaining = jobs.tests_need_to_be_run(now)
if not remaining:
# Send them in order to simplify testing.
for builder in sorted(remaining):
tests = remaining[builder]
if not self._add_pending_job_and_send_if_needed(builder, tests, jobs,
True, pending, now):
# If the main job wasn't sent, we can't skip the triggered jobs since
# they won't get triggered.
# Add any pending bots that will be triggered from this build.
triggered_bots = self._get_triggered_bots(builder, tests)
for builder, steps in triggered_bots.iteritems():
self._add_pending_job_and_send_if_needed(builder, steps, jobs, False,
pending, now)
def _build_status_url(self, job):
"""Html url for this try job."""
assert is not None, str(job)
return '%s/buildstatus?builder=%s&number=%s' % (
self.try_server_url, job.builder,
def _update_dashboard(self, pending, job):
"""Updates the CQ dashboard with the current Try Job state as known to the
logging.debug('_update_dashboard(%s/%s)', job.builder,
info = {
'builder': job.builder,
'job_name': 'CQ',
'result': job.result,
'revision': job.revision,
'url': self._build_status_url(job),
self.send_status(pending, info)
def _update_jobs_from_rietveld(self, pending, jobs, handle, now):
"""Grabs data from Rietveld and pass it to
Returns True on success.
status = buildbot_json.Buildbot(self.try_server_url)
data = self.context.rietveld.get_patchset_properties(
pending.issue, pending.patchset)
except urllib2.HTTPError as e:
if e.code == 404:
# TODO(phajdan.jr): Maybe generate a random id to correlate the user's
# error message and exception in the logs.
# Don't put exception traceback in the user-visible message to avoid
# leaking sensitive CQ data (passwords etc).
jobs.error_message = ('Failed to get patchset properties (patchset '
'not found?)')
return False
# Update the RietvedTryJobs object.
keys = jobs.update_jobs_from_rietveld(
except urllib2.HTTPError as e:
if e.code in (500, 502, 503):
# Temporary AppEngine hiccup. Just log it and return failure.
logging.warning('%s while accessing %s. Ignoring error.' % (
str(e), e.url))
return False
except urllib2.URLError as e:
if 'timed out' in e.reason:
# Handle timeouts gracefully.
logging.warning('%s while updating tryserver status for '
'rietveld issue %s', e, pending.issue)
return False
except socket.error as e:
# Temporary AppEngine hiccup. Just log it and return failure.
if e.errno == errno.ECONNRESET:
'%s while updating tryserver status for rietveld issue %s.' % (
str(e), str(pending.issue)))
return False
except IOError as e:
# Temporary AppEngine hiccup. Just log it and return failure.
if e.errno == 'socket error':
'%s while updating tryserver status for rietveld issue %s.' % (
str(e), str(pending.issue)))
return False
if handle:
for updated_key in keys:
job = jobs.try_jobs[updated_key]
self._update_dashboard(pending, job)
jobs.signal_as_failed_if_needed(job, self._build_status_url(job), now)
return True