blob: 7b13e7ab93b89db768e9aae6c0eabf7f4ca6cbc5 [file] [log] [blame]
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Returns a sequence of new innocent CLs which failed pre-cq.
Queries CIDB for sanity Pre-CQ build failures.
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
import datetime
import itertools
from chromite.lib import constants
from chromite.lib import cidb
from chromite.lib import clactions
from chromite.lib import parallel
from chromite.lib import cros_logging as logging
from import datastore # pylint: disable=E0611,import-error
from infra_libs import ts_mon
from exonerator import checkpointlib
from exonerator import innocent_cls_cq
from exonerator import innocent_cls_precq_flake
_SANITY_BUILD_REASON = 'sanity-pre-cq'
# Forgive failed builds up to 3 days before a sanity failure, and 1 day
# afterward.
# Only kick off one build per tick during peak hours
True: 1,
# Kick off pre-CQ builds faster during non-peak hours
False: 5,
description=('The current number of failed sanity runs being examined by '
'CL Exonerator.'),
def NewInnocentCLs(conn, on_peak, checkpoint=True):
"""Finds new innocent CLs since the last run, with checkpointing.
Looping over the results with a for loop will result in a checkpoint for
each CL as it is processed. Do not convert the results to a list unless you
don't care that this will checkpoint immediately during the list construction.
conn: The CIDBConnection to use.
on_peak: Whether it's business hours (peak load) or off-peak.
checkpoint: Whether to save progress after every buildMessage processed.
lists of ChangeWithBuild objects, representing possibly innocent CLs that
have Pre-CQ faliures which were associtaed with a recent sanity build
limit = _PRE_CQ_CL_LIMITS[on_peak]
innocents = _InnocentCLsFromSanityBuildFailuresCheckpointed(
conn, on_peak, limit, checkpoint)
# TODO( add some logic here to find all the CLs which
# participated in the build. Put them together in a batch to yield.
# Also change docstring of limit to be "number of BUILDS to process."
# Verify that this build was the latest run. We don't want to exonerate a
# stale pre-cq run when there is a newer run.
filtered = [
innocent_cls_cq.ChangeWithBuild(change, build_id)
for change, build_id in innocents
if _ShouldExonerate(conn, change, build_id)]
return [filtered] if filtered else []
def _InnocentCLsFromSanityBuildFailuresCheckpointed(conn, on_peak, limit,
"""Finds Pre-CQ failures to forgive.
Checkpoints processed builds with a PreCQForgiven row. Specifically,
x = next(fetch)
# x is not checkpointed yet
y = next(fetch) # This checkpoints that x is done
conn: The CIDBConnection to use.
on_peak: Whether it's business hours (peak load) or off-peak.
limit: The maximum number of innocent CLs to yield.
checkpoint: Whether to save progress after every build yielded.
tuples of (change, pre-cq build id) to forgive.
# The first priority is to forgive pre-CQ builds associated with a sanity
# failure. Afterwards, we can consider pre-CQ failures due to flakey pre-CQ
# configs.
seq = itertools.chain(
innocent_cls_precq_flake.InnocentPreCQsFromFlake(conn, on_peak))
if not checkpoint:
return itertools.islice(seq, limit)
ds = datastore.Client()
def _AlreadyProcessed(item):
change, build_id = item
return _AlreadyForgiven(ds, change, build_id)
def _Save(item):
change, build_id = item
entity = ds.get(key=ds.key(PRECQ_PROCESSED_KEY, build_id))
if not entity:'Inserting Pre-CQ checkpoint row %s.', build_id)
entity = datastore.Entity(key=ds.key(PRECQ_PROCESSED_KEY, build_id))
else:'Updating Pre-CQ checkpoint row %s.', build_id)
# A build may include more than one CL, so add a property to the entity for
# each CL of the build that was processed.
entity.update({str(change.gerrit_number): True})
return checkpointlib.CheckpointSequence(
_AlreadyProcessed, _Save, seq, limit=limit)
def _InnocentCLsFromSanityBuildFailures(conn):
"""Finds Pre-CQ failures to forgive.
conn: The CIDBConnection to use.
Tuples of (change, pre-cq build) to forgive.
sanity_runs = conn.GetLatestBuildRequestsForReason(_SANITY_BUILD_REASON)
for sanity_run in _FailedSanityRuns(conn, sanity_runs):
for change, build_id in _InnocentPreCQsFromSanity(conn, sanity_run):
yield change, build_id
def _FailedSanityRuns(conn, sanity_runs):
"""Filters the latest sanity runs to only the failed sanity runs.
conn: The CIDBConnection to use.
sanity_runs: An iterable of sanity builds (has a .build_id).
The sanity runs that failed.
# TODO(phobbs) add a JOIN buildTable in GetLatestBuildRequestsForReason
# to avoid creating / destroying a bunch of cidb connections.
statuses = parallel.RunTasksInProcessPool(
[[run.build_id] for run in sanity_runs],
faileds = [status['status'] == constants.BUILDER_STATUS_FAILED
for status in statuses]
_SANITY_FAILURES_EXAMINED.set(sum(map(int, faileds)))
for sanity_run, failed in itertools.izip(sanity_runs, faileds):
if failed:
# TODO( yield build ids from here.
yield sanity_run
def _InnocentPreCQsFromSanity(conn, sanity_failure):
"""Finds forgiveable Pre-CQ failures from a sanity failure
conn: A CIDBConnection
sanity_failure: A BuildRequest corresponding to a sanity build failure.
(GerritChangeTuple, build_id) corresponding to pre-cq failures to forgive
lower_limit = (
sanity_failure.timestamp - datetime.timedelta(days=_DAYS_LOWER_LIMIT))
upper_limit = (
sanity_failure.timestamp + datetime.timedelta(days=_DAYS_UPPER_LIMIT))
builds = conn.GetBuildHistory(
for build in builds:
actions = conn.GetActionsForBuild(build['id'])
if not actions:
if any(action.status == constants.BUILDER_STATUS_FAILED
for action in actions):
action = actions[0]
change = clactions.GerritPatchTuple(
action.change_number, action.patch_number,
action.change_source == 'internal')
# TODO( yield build ids from here.
yield change, action.build_id
def _AlreadyForgiven(ds, change, build_id):
"""Whether the given Pre-CQ build was forgiven already.
ds: The cloud Datastore client.
change: A GerritPatchTuple instance.
build_id: The CIDB build id for the pre-cq run.
entity = ds.get(key=ds.key(PRECQ_PROCESSED_KEY, build_id))
# Each entity stores the CLs that were processed by exonerator for that
# build. For simplicity, assume gerrit_number:change is a one-to-one mapping.
# Technically, this is false, but it's extremely unlikely to affect the
# behavior.
return (entity is not None
and entity.get(str(change.gerrit_number), False))
def _ShouldExonerate(conn, change, build_id):
"""Whether we should exonerate the patch.
conn: The cidb.CIDBConnection
change: A GerritPatchTuple to possibly exonerate
build_id: The threshold build_id.
actions = conn.GetActionsForChanges([change])
return (
not _ExistsNewerPreCQForCL(actions, build_id)
and not _PatchAlreadyExonerated(actions, change))
def _ExistsNewerPreCQForCL(actions, build_id):
"""Whether there is a newer action for a given CL of a certain type.
actions: The CLActions for the change
build_id: The threshold build_id.
A boolean indicating whether there are any new CLActions of the given
actions_for_build = [a for a in actions if a.build_id == build_id]
build_config = actions_for_build[0].build_config
except IndexError:
return False
newer_pickup_actions_for_build_config = [
a.build_id > build_id
for a in actions
if (a.build_config == build_config
and a.action == constants.CL_ACTION_PICKED_UP)]
return any(newer_pickup_actions_for_build_config)
def _PatchAlreadyExonerated(actions, change):
"""Whether the patch was already exonerated once.
This prevents infinitely retrying pre-cq on a single patch.
TODO: consider retrying more times (2 or 3?) if there is low pre-cq load.
actions: The CLActions for the change
change: A GerritPatchTuple to find actions for.
exonerations_for_patch = [
a for a in actions
if (a.action == constants.CL_ACTION_EXONERATED
and a.patch_number == change.patch_number)]
return bool(exonerations_for_patch)