blob: 3318b6d8868997dcdccc318588874aea67f3a6bd [file] [log] [blame]
// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"time"
"cloud.google.com/go/bigquery"
"github.com/golang/protobuf/ptypes"
"google.golang.org/api/iterator"
evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)
var terminalRejectionFragment = &evalpb.RejectionFragment{
Terminal: true,
}
// rejections calls the callback for each found CQ rejection.
func (r *presubmitHistoryRun) rejections(ctx context.Context, callback func(*evalpb.RejectionFragment) error) error {
q, err := r.bqQuery(ctx, rejectedPatchSetsSQL)
if err != nil {
return err
}
q.Parameters = append(q.Parameters, bigquery.QueryParameter{
Name: "minCLFlakes",
Value: r.minCLFlakes,
})
it, err := q.Read(ctx)
if err != nil {
return err
}
// The result rows are ordered by change and patchset numbers,
// and so are the rejection fragments.
// Keep track of the current patchset to detect patchset boundaries.
var curChange, curPS int
curPSIsDeleted := false
maybeFinalizeCurrent := func() error {
if curChange == 0 || curPSIsDeleted {
return nil
}
return callback(terminalRejectionFragment)
}
// Iterate over results and call the callback.
// One fragment represents at most one BigQuery row.
for {
// Read the next row.
var row rejectionRow
switch err := it.Next(&row); {
case err == iterator.Done:
return maybeFinalizeCurrent()
case err != nil:
return err
}
rej := &evalpb.Rejection{
FailedTestVariants: []*evalpb.TestVariant{row.FailedTestVariant.proto()},
}
// Finalize the current rejection if this patchset is different.
if curChange != row.Change || curPS != row.Patchset {
if err := maybeFinalizeCurrent(); err != nil {
return err
}
// Include the patchset-level info only in the first fragment.
row.populatePatchsetInfo(rej)
switch err := r.populateChangedFiles(ctx, rej.Patchsets[0]); {
case err == errPatchsetDeleted:
curPSIsDeleted = true
case err != nil:
return err
default:
curPSIsDeleted = false
}
}
if !curPSIsDeleted {
if err := callback(&evalpb.RejectionFragment{Rejection: rej}); err != nil {
return err
}
}
curChange = row.Change
curPS = row.Patchset
}
}
type rejectionRow struct {
Change int
Patchset int
Timestamp time.Time
FailedTestVariant testVariantRow
}
func (r *rejectionRow) populatePatchsetInfo(rej *evalpb.Rejection) {
rej.Patchsets = []*evalpb.GerritPatchset{{
Change: &evalpb.GerritChange{
// Assume all Chromium source code is in
// https://chromium.googlesource.com/chromium/src
// TOOD(nodir): make it fail if it is not.
Host: "chromium-review.googlesource.com",
Project: "chromium/src",
Number: int64(r.Change),
},
Patchset: int64(r.Patchset),
}}
rej.Timestamp, _ = ptypes.TimestampProto(r.Timestamp)
}
type testVariantRow struct {
ID string
FileName string
Variant []struct {
Key string
Value string
}
}
func (t *testVariantRow) proto() *evalpb.TestVariant {
ret := &evalpb.TestVariant{
Id: t.ID,
FileName: t.FileName,
Variant: make(map[string]string, len(t.Variant)),
}
for _, kv := range t.Variant {
ret.Variant[kv.Key] = kv.Value
}
return ret
}
// rejectedPatchSetsSQL is a BigQuery query that returns patchsets with test
// failures. Excludes flaky tests.
//
// This query intentionally does not use ARRAY_AGG for failed tests because
// otherwise a result row size may exceed the API limitation of 10MB.
const rejectedPatchSetsSQL = `
WITH
-- Select all tryjobs that participated in CQ in the given time window,
-- along with their patchsets.
--
-- Use earliest_equivalent_patchset field instead of patchset, such that
-- trivial rebases are treated as the same patchset, and to ignore
-- CL description edits.
tryjobs AS (
SELECT
b.id as build_id,
ps.change,
ps.earliest_equivalent_patchset as patchset,
partition_time as ps_approx_timestamp,
FROM commit-queue.chromium.attempts a, a.gerrit_changes ps, a.builds b
WHERE partition_time BETWEEN @startTime AND @endTime
),
-- Select 'try' test results. All early filtering is done here.
test_results AS (
SELECT
-- select the build_id to join with tryjobs.
CAST(REGEXP_EXTRACT(exported.id, r'build-(\d+)') as INT64) as build_id,
test_id,
variant_hash,
variant,
expected,
-- Replace NULLs with '' to keep Go code simple.
IFNULL(test_location.file_name, '') as file_name,
FROM luci-resultdb.chromium.try_test_results tr
-- Read prev-day and next-day results too to ensure that we have ALL
-- results of a given CQ attempt.
WHERE partition_time BETWEEN TIMESTAMP_SUB(@startTime, INTERVAL 1 DAY) and TIMESTAMP_ADD(@endTime, INTERVAL 1 DAY)
AND not exonerated
AND status != 'SKIP' -- not needed for RTS purposes
AND (@test_id_regexp = '' OR REGEXP_CONTAINS(test_id, @test_id_regexp))
AND (@builder_regexp = '' OR EXISTS (SELECT 0 FROM tr.variant WHERE key='builder' AND REGEXP_CONTAINS(value, @builder_regexp)))
-- Exclude broken test locations.
-- TODO(nodir): remove this after crbug.com/1130425 is fixed.
AND REGEXP_CONTAINS(IFNULL(test_location.file_name, ''), r'(?i)^(|.*\.(cc|html|m|c|cpp|js))$')
-- Exclude broken prefixes.
-- TODO(nodir): remove after crbug.com/1017288 is fixed.
AND (test_id NOT LIKE 'ninja://:blink_web_tests/%' OR test_location.file_name LIKE '//third_party/%')
),
-- Test results annotated with patchset.
-- Join the two queries above.
ps_test_results AS (
SELECT change, patchset, ps_approx_timestamp, r.*
FROM test_results r
JOIN tryjobs USING (build_id)
),
-- The following two sub-queries detect flaky tests.
-- It is important to exclude them from analysis because they represent
-- noise.
--
-- A test variant is considered flaky if it has mixed results in >=N
-- separate CLs. N=1 is too small because otherwise the query is vulnerable
-- to a single bad patchset that introduces flakiness and never lands.
--
-- The first sub-query finds test variants with mixed results per CL.
-- Note that GROUP BY includes patchset, but SELECT doesn't and uses
-- DISTINCT.
-- The second sub-query filters out test variant candidates that have mixed
-- results win fewer than @minCLFlakes CLs.
flaky_test_variants_per_cl AS (
SELECT DISTINCT change, test_id, variant_hash
FROM ps_test_results
GROUP BY change, patchset, test_id, variant_hash
HAVING LOGICAL_OR(expected) AND LOGICAL_OR(NOT expected)
),
flaky_test_variants AS (
SELECT test_id, variant_hash
FROM flaky_test_variants_per_cl
GROUP BY test_id, variant_hash
HAVING COUNT(change) >= @minCLFlakes
),
-- Select test variants with unexpected results for each patchset.
-- Does not filter out flaky test variants just yet.
failed_test_variants_per_ps AS (
SELECT
change,
patchset,
test_id,
variant_hash,
MIN(ps_approx_timestamp) as ps_approx_timestamp,
ANY_VALUE(variant) as variant,
ANY_VALUE(file_name) as file_name
FROM ps_test_results
GROUP BY change, patchset, test_id, variant_hash
HAVING LOGICAL_AND(NOT expected)
)
-- Exclude flaky tests from failed_test_variants_per_ps.
-- Prepare the results for consumption by Go code (fields, order).
SELECT
change as Change,
patchset as Patchset,
ps_approx_timestamp as Timestamp,
STRUCT(
test_id as ID,
variant as Variant,
file_name as FileName
) as FailedTestVariant
FROM failed_test_variants_per_ps
LEFT JOIN flaky_test_variants flaky USING (test_id, variant_hash)
-- flaky.test_id is NULL if LEFT JOIN did not find a flaky_test_variants row
-- for this test variant, i.e. the test is not flaky
WHERE flaky.test_id IS NULL
ORDER BY change, patchset
`