blob: a21fd21d55898265aadd16c235ed2bfd18c221ac [file] [log] [blame]
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import {
BuildbucketV2Client,
makeBuildRequests,
RETRY_FAILED_TAG,
} from './buildbucket-client.js';
import {
compareBuildIds,
getNewOperationId,
isCQExperimental,
getFailedBuilders,
splitPatchsetGroups,
} from './buildbucket-utils.js';
import {ResultDbV1Client} from './resultdb-client.js';
import {openTryjobPicker} from './cr-tryjob-picker.js';
/**
* Heads up! Everything in this file is still in flux. The reboot checks API
* is still in development. So everything in this file can change. And it is
* expected that comments and tests are limited for the time being.
*/
// The field is used to store arbitrary data in a CheckResult.
export const DATA_SYMBOL = Symbol('chromeData');
// Tags used to label CheckResults.
export const CQ_EXPERIMENTAL_TAG = 'CQ_EXPERIMENTAL';
export const INFRA_FAILURE_TAG = 'INFRA_FAILURE';
// Neither encodeURI() nor encodeURIComponent() encode parentheses. escape()
// does encode parentheses, but its usage is discouraged. URL_ESCAPES is used
// as an alternative to all the above.
const URL_ESCAPES = {
'(': '%28',
')': '%29',
};
const URL_ESCAPE_REGEX = new RegExp(
`[${Object.keys(URL_ESCAPES).map(c => `\\${c}`).join('')}]`, 'g');
// An ordered list of TestResult statuses from most to least important. Used to
// sort TestResults.
const RESULT_STATUSES = [
'FAIL',
'CRASH',
'ABORT',
'SKIP',
'PASS',
];
export class ChecksFetcher {
constructor(plugin, gerritInstance, buildbucketHost, maxVariantsPerBuild) {
this.plugin = plugin;
this.gerritHost = `${gerritInstance}-review.googlesource.com`;
this.buildbucketHost = buildbucketHost;
this.bbClient = new BuildbucketV2Client(this.buildbucketHost);
this.maxVariantsPerBuild = maxVariantsPerBuild;
this.includeExperiments = false;
this.controller = null;
}
async fetch(changeData) {
// Abort any unfinished fetch requests. If this is the first fetch, don't
// abort. Otherwise, abort using old controller.
if (this.controller !== null) {
this.controller.abort();
}
this.controller = new AbortController();
const {changeNumber, patchsetNumber, repo} = changeData;
// When testing locally, changeInfo may not be included in changeData.
// Fetch if this is the case.
let {changeInfo} = changeData;
if (!changeInfo) {
changeInfo = await this.plugin.restApi().get(
`/changes/${changeNumber}/?o=ALL_REVISIONS`);
}
// Query BuildBucket for build info.
const builds = await this.fetchDisplayedBuilds(
changeNumber, patchsetNumber, repo, changeInfo);
builds.sort((a, b) => -compareBuildIds(a.id, b.id));
// Query ResultDB for unexpected TestVariants and artifacts.
const rdbResults = await Promise.all(builds.map(build => {
const rdbInfo = build.infra.resultdb;
const rdbClient = new ResultDbV1Client(rdbInfo.hostname);
return Promise.all([
this.fetchUnexpectedTestVariants(rdbClient, rdbInfo.invocation)
.catch(e => {
console.error(e);
return [];
}),
this.fetchArtifacts(rdbClient, rdbInfo.invocation)
.catch(e => {
console.error(e);
return [];
}),
]);
}));
const runs = [];
const attempts = this.createAttemptList(builds);
builds.forEach((build, i) => {
const [variantsResult, artifactsResult] = rdbResults[i];
const run = this.convertBuildToRun(
build,
patchsetNumber,
attempts,
variantsResult,
artifactsResult,
repo,
);
if (run) {
runs.push(run);
}
});
const topLevelActions = [{
name: 'Choose Tryjobs',
callback: (change, patchset) =>
this.chooseTryjobsCallback(change, patchset, repo),
}];
const failedBuilders = getFailedBuilders(builds);
if (failedBuilders.length > 0) {
topLevelActions.push({
name: 'Retry Failed Builds',
callback: (change, patchset) => this.retryFailedBuildsCallback(
failedBuilders, change, patchset, repo),
});
}
topLevelActions.push({
name: `${this.includeExperiments ? 'Hide' : 'Show'} ` +
'Experimental Results',
callback: () => this.filterExperimentsCallback(),
});
return {
responseCode: 'OK',
actions: topLevelActions,
runs,
};
}
/**
* Returns Buildbucket builds to be displayed in the UI.
*
* Compared to fetchBuilds, this method handles filtering experimental builds
* and fetching equivalent patchsets.
*/
async fetchDisplayedBuilds(change, patchset, project, changeInfo) {
let patchsets = [];
const groups = splitPatchsetGroups(changeInfo);
for (const group of groups) {
if (!group.has(patchset)) {
continue;
}
// Builds for the patchsets greater than patchNum are not included,
// even if they're in the same equivalent group. Why? This matches
// the historical behavior of the buildbucket plugin.
patchsets = Array.from(group).filter(n => (n <= patchset)).sort();
break;
}
if (patchsets.length === 0) {
return [];
}
// Get builds for the equivalent patchsets.
let builds = await this.fetchBuilds(change, patchsets, project);
if (!this.includeExperiments) {
builds = builds.filter(build => !isCQExperimental(build));
}
if (builds.length > 0 || changeInfo.status !== 'MERGED') {
return builds;
}
// If the submit strategy is "Rebase Always" (such as for Chromium), then
// the last patchset of the merged CL is autogenerated, and it may be
// labeled as "REWORK" even though it's actually a trivial rebase; in
// this case we try again with the previous group of patchsets.
if (groups.length >= 2 && groups[0].size === 1 &&
patchsets.length === 1 && groups[0].has(patchsets[0])) {
const previousPatchsets = Array.from(groups[1]);
return await this.fetchBuilds(change, previousPatchsets, project);
}
// If we reach this point, we're in a merged change with no builds to
// display, but the last patchset doesn't appear to be a special
// autogenerated patchset.
return [];
}
/**
* Returns all Buildbucket builds for the given change and patchsets.
*/
async fetchBuilds(change, patchsets, project) {
const builds = [];
const fields = [
'id',
'builder',
'number',
'tags',
'status',
'critical',
'createTime',
'startTime',
'endTime',
'summaryMarkdown',
'infra.resultdb',
'infra.swarming',
].map(f => `builds.*.${f}`).join(',');
let requests = patchsets.map(patchset => ({
searchBuilds: {
pageSize: 1000,
predicate: {
gerritChanges: [{
host: this.gerritHost,
project,
change,
patchset,
}],
},
fields,
},
}));
while (requests.length > 0 && !this.controller?.signal.aborted) {
const {responses} =
await this.bbClient.batch({requests}, this.controller?.signal);
const newRequests = [];
responses.forEach((response, i) => {
const search = response.searchBuilds;
builds.push(...(search.builds || []));
// If there are more builds to fetch, add the request to be re-fetched.
if (search.nextPageToken) {
requests[i].searchBuilds.pageToken = search.nextPageToken;
newRequests.push(requests[i]);
}
});
requests = newRequests;
}
return builds;
}
/**
* Returns TestVariants for the given invocation whose status is not EXPECTED.
*/
async fetchUnexpectedTestVariants(client, invocation) {
const variants = [];
if (!invocation) {
return variants;
}
const pageSize = 1000;
let pageToken = '';
// queryTestVariants returns all UNEXPECTED, FLAKY, and EXONERATED
// TestVariants before returning EXPECTED TestVariants.
while (true) {
const response = await client.queryTestVariants(
{
invocations: [invocation],
pageSize,
pageToken,
},
this.controller?.signal,
);
// Exit if there are no unexpected TestVariants.
if (!response.testVariants || response.testVariants.length === 0) {
break;
}
if (response.testVariants[0].status === 'EXPECTED') {
break;
}
variants.push(...response.testVariants);
// Exit if there are no more unexpected TestVariants to fetch.
if (response.testVariants.length < pageSize || !response.nextPageToken) {
break;
}
pageToken = response.nextPageToken;
}
return variants;
}
/**
* Returns artifacts for variants with unexpected results under the given
* invocation.
*
* TODO(gavinmak): Support invocation-level artifacts.
*/
async fetchArtifacts(client, invocation) {
const artifacts = [];
if (!invocation) {
return artifacts;
}
let pageToken = '';
while (true) {
const response = await client.queryArtifacts({
invocations: [invocation],
predicate: {
testResultPredicate: {
expectancy: 'VARIANTS_WITH_UNEXPECTED_RESULTS',
},
followEdges: {
testResults: true,
},
},
pageSize: 1000,
pageToken,
});
artifacts.push(...(response.artifacts || []));
// Exit if there are no more artifacts to fetch.
if (!response.nextPageToken) {
break;
}
pageToken = response.nextPageToken;
}
return artifacts;
}
/**
* Converts a Buildbucket `Build` object to a Reboot Checks API `Run` object.
*/
convertBuildToRun(
build, patchset, attempts, unexpectedTestVariants, artifacts, project) {
if (!build || !build.builder || !build.builder.builder) return undefined;
const buildLink = this.createBuildLink(this.buildbucketHost, build.id);
const buildResourceName =
this.buildbucketResourceName(this.buildbucketHost, build.id);
const run = {
patchset,
attempt: attempts[build.builder.builder].indexOf(build.id) + 1,
externalId: buildResourceName,
checkName: build.builder.builder,
checkLink: this.createBuilderLink(build),
status: this.getRunStatusFromBuild(build),
statusDescription: build.status,
statusLink: buildLink,
actions: this.createRunActions(build, project),
scheduledTimestamp: this.getDateFromTimestamp(build.createTime),
startedTimestamp: this.getDateFromTimestamp(build.startTime),
finishedTimestamp: this.getDateFromTimestamp(build.endTime),
results: [],
};
// There are no CheckResults for SCHEDULED builds.
if (build.status === 'SCHEDULED') {
return run;
}
const resultActions = this.createResultActions(project);
const buildLinks = [{
url: buildLink,
tooltip: 'Build',
primary: true,
}];
if (build.status !== 'SUCCESS') {
const stdoutLink = this.createCompileStdoutLink(build);
stdoutLink && buildLinks.push({
url: stdoutLink,
tooltip: 'Step Stdout',
primary: true,
});
}
const commonTags = [];
const buildIsExperimental = isCQExperimental(build);
buildIsExperimental && commonTags.push({name: CQ_EXPERIMENTAL_TAG});
if (build.status === 'INFRA_FAILURE') {
// TODO(gavinmak): Add TagColor once PURPLE is available.
commonTags.push({name: INFRA_FAILURE_TAG});
}
if (unexpectedTestVariants) {
// Create a CheckResult for each TestVariant. Limit the number created to
// this.maxVariantsPerBuild.
const numDisplayed =
Math.min(unexpectedTestVariants.length, this.maxVariantsPerBuild);
// Create a map of Artifacts to use for all TestVariants.
const artifactMap = this.createArtifactMap(artifacts);
unexpectedTestVariants.slice(0, numDisplayed).forEach(variant => {
// TODO(gavinmak): Differentiate links with LinkIcons.
const variantLinks = [{
url: this.createVariantLink(variant, build),
tooltip: 'Test Results',
primary: true,
}];
// Link to Web Test Results if available.
const layoutLink = this.createLayoutResultLink(variant, build);
if (layoutLink) {
variantLinks.push({
url: layoutLink,
tooltip: 'Web Test Results',
primary: false,
});
}
// Sort results by status from most to least important.
variant.results.sort((a, b) => {
return RESULT_STATUSES.indexOf(a.result.status) -
RESULT_STATUSES.indexOf(b.result.status);
});
// If the status is FLAKY, process all results. Otherwise, use only the
// first result to create artifact links and the CheckResult message.
const numResults =
variant.status === 'FLAKY' ? variant.results.length : 1;
// Create a data object to store information used when rendering a
// CheckResult row. This is currently only used for results from
// test variants.
const data = {variantData: []};
const messages = [];
for (let i = 0; i < numResults; i++) {
const {result} = variant.results[i];
const summaryHtml = result.summaryHtml || '';
const resultArtifacts = artifactMap[result.name] || [];
data.variantData.push({result, resultArtifacts});
// Remove unnecessary HTML tags from summaryHtml when displaying
// message. When rendered in a CheckResult row, message should be a
// plain text summary of results.
messages.push(
`Run #${i+1}: ${result.status}`,
summaryHtml.replace(/<[^>]*>/g, ''),
);
// Link to Artifacts if available.
resultArtifacts.forEach(artifact => {
variantLinks.push({
url: artifact.fetchUrl,
tooltip: `${artifact.artifactId} Artifact of Result ${i+1}`,
primary: false,
});
});
}
// Link to the swarming tasks.
const taskLinks = [...new Set(
variant.results.map(r => this.createTestTaskLink(r.result)))];
if (taskLinks.length === 1) {
variantLinks.push({
url: taskLinks[0],
tooltip: 'Swarming Task',
primary: false,
});
} else {
taskLinks.forEach((link, i) => {
variantLinks.push({
url: link,
tooltip: `Swarming Task (${i+1} of ${taskLinks.length})`,
primary: false,
});
});
}
// Gerrit doesn't define any 'data' field in the CheckResult object.
// To store the data we need in checks-result.js, we use DATA_SYMBOL.
// Unlike with a string attribute, values stored with symbols are
// guaranteed not to be overwritten by Gerrit along the way to
// checks-result.js.
run.results.push({
category: this.getResultCategoryFromVariant(variant, build),
summary: variant.testMetadata?.name || variant.testId,
tags: commonTags,
links: variantLinks,
actions: resultActions,
message: messages.join(' '),
[DATA_SYMBOL]: data,
});
});
// Create a CheckResult to direct to additional failures.
if (unexpectedTestVariants.length > this.maxVariantsPerBuild) {
const numExtra =
unexpectedTestVariants.length - this.maxVariantsPerBuild;
run.results.push({
category: buildIsExperimental ? 'INFO' : 'ERROR',
summary: build.builder.builder,
message: `${numExtra} more test${numExtra > 1 ? 's' : ''} failed`,
tags: commonTags,
links: buildLinks,
actions: resultActions,
});
}
}
// Create a CheckResult for the Buildbucket build after the unexpected test
// variants.
//
// To minimize clutter for the user, SUCCESSful builds should not contain
// the Buildbucket CheckResult. Relevant information should be put in the
// CheckRun object.
if (build.status !== 'SUCCESS') {
run.results.push({
externalId: buildResourceName,
category: this.getResultCategoryFromBuild(build),
summary: `Build ${build.status}`,
message: build.summaryMarkdown || '',
tags: commonTags,
links: buildLinks,
actions: resultActions,
});
}
return run;
}
/**
* Returns a CheckRun status from a Buildbucket build.
*/
getRunStatusFromBuild(build) {
switch (build.status) {
case 'SCHEDULED':
case 'STARTED':
return 'RUNNING';
case 'SUCCESS':
case 'FAILURE':
case 'CANCELED':
case 'INFRA_FAILURE':
return 'COMPLETED';
default:
return 'RUNNABLE';
}
}
/**
* Returns a CheckResult category from a Buildbucket build.
*/
getResultCategoryFromBuild(build) {
if (['SCHEDULED', 'CANCELED', 'STARTED', 'SUCCESS']
.includes(build.status) || isCQExperimental(build)) {
return 'INFO';
} else if (build.critical === 'NO') {
return 'WARNING';
} else {
return 'ERROR';
}
}
/**
* Returns a CheckResult category from a TestVariant and Buildbucket build.
*/
getResultCategoryFromVariant(variant, build) {
const status = variant.status;
if (status === 'EXPECTED' || isCQExperimental(build)) {
return 'INFO';
} else if (['EXONERATED', 'FLAKY'].includes(status)) {
return 'WARNING';
} else {
return 'ERROR';
}
}
/**
* Returns a Date object from a Timestamp string defined in:
* https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
*/
getDateFromTimestamp(timestamp) {
const m = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d{1,9})?Z$/
.exec(timestamp);
if (!m) { return null; }
// m[7] matches the fractional seconds of timestamp and takes the format
// ".[0-9]{1-9}". This gets an integer number of milliseconds given m[7].
return new Date(m[1], m[2], m[3], m[4], m[5], m[6],
Math.round(Number(m[7] || '0') * 1000));
}
/**
* Returns a link to the Swarming test task.
*/
createTestTaskLink(testResult) {
const match =
/^invocations\/task-([a-zA-Z\-\.]+\.com)-(\w+)/.exec(testResult.name);
return match && encodeURI(`https://${match[1]}/task?id=${match[2]}`);
}
/**
* Returns a deep link to a specific TestVariant.
*/
createVariantLink(variant, build) {
const builder = build.builder;
return encodeURI(`https://ci.chromium.org/ui/p/${builder.project}/` +
`builders/${builder.bucket}/` +
`${encodeURIComponent(builder.builder)}/`) +
`b${build.id}/test-results?q=ExactID%3A` +
`${encodeURIComponent(variant.testId)}+VHash%3A` +
`${encodeURIComponent(variant.variantHash)}&clean=`;
}
/**
* Returns a link to the Web Test Result viewer.
*/
createLayoutResultLink(variant, build) {
if (!/^ninja:\/\/:blink_web_tests\//.test(variant.testId) ||
!build.number) {
return '';
}
let stepName = variant.results[0].result
.tags?.find(t => t.key === 'step_name')?.value;
if (!stepName) {
return '';
}
// Remove everything past the first ')' and encode special characters, ex:
// 'test (patch) on some OS' becomes 'test%20%28patch%29'
stepName = /^([^\)]*\)?)/.exec(stepName)[1];
stepName = encodeURIComponent(stepName);
stepName = stepName.replaceAll(URL_ESCAPE_REGEX, m => URL_ESCAPES[m]);
return `https://test-results.appspot.com/data/layout_results/` +
`${encodeURIComponent(build.builder.builder)}/${build.number}/` +
`${stepName}/layout-test-results/results.html`;
}
/**
* Returns a link to the stdout of a failed compile step.
*/
createCompileStdoutLink(build) {
if (build.builder.project !== 'chromium' || !build.summaryMarkdown ||
build.status !== 'FAILURE') {
return '';
}
// Check if the build failed on a compile step.
const match = /Step _([^_]+)_/.exec(build.summaryMarkdown);
const stepName = match && match[1];
if (!stepName?.includes('compile')) {
return '';
}
return `https://${this.buildbucketHost}/log/${build.id}/` +
encodeURIComponent(stepName);
}
/**
* Returns a link to the builder page for the given build.
*/
createBuilderLink(build) {
const builder = build.builder;
return encodeURI(`https://ci.chromium.org/p/${builder.project}/builders/` +
`${builder.bucket}/${builder.builder}`);
}
/**
* Returns a link to the build page for a given Buildbucket host and buildId.
*/
createBuildLink(host, buildId) {
return encodeURI('https:' + this.buildbucketResourceName(host, buildId));
}
/**
* Returns the full resource name for a given Buildbucket host and buildId.
*/
buildbucketResourceName(host, buildId) {
return `//${host}/build/${buildId}`;
}
/**
* Returns a dictionary mapping a TestResult's name to a list of the
* TestResult's artifacts.
*/
createArtifactMap(artifacts) {
const artifactMap = {};
for (const artifact of artifacts) {
const key = this.extractResultName(artifact);
if (key in artifactMap) {
artifactMap[key].push(artifact)
} else {
artifactMap[key] = [artifact];
}
}
return artifactMap;
}
/**
* Returns a dictionary mapping builders to an ordered list of build ids.
*/
createAttemptList(builds) {
const dct = {};
builds.forEach(build => {
const name = build.builder.builder;
if (!(name in dct)) {
dct[name] = [build.id];
} else {
dct[name].push(build.id);
}
});
Object.values(dct).forEach(arr => arr.sort(compareBuildIds));
return dct;
}
/**
* Returns a list of actions used in a Reboot Checks API `Run` object.
*/
createRunActions(build, project) {
const actions = [
{
name: 'Run',
tooltip: 'Start a new builder run and cancel an existing run if it ' +
'exists',
primary: false,
callback: (change, patchset, attempt, extId) =>
this.startRunCallback(change, patchset, attempt, extId, project),
},
];
if (['SCHEDULED', 'STARTED'].includes(build.status)) {
actions.push({
name: 'Cancel',
tooltip: 'Cancel existing runs',
primary: false,
callback:(_change, _patchset, _attempt, extId) =>
this.cancelRunCallback(extId),
});
}
return actions;
}
/**
* Returns a list of actions used in a Reboot Checks API `Result` object.
*/
createResultActions(project) {
return [
{
name: 'Run',
tooltip: 'Start a new builder run and cancel an existing run if it ' +
'exists',
primary: false,
callback: (change, patchset, attempt, extId) =>
this.startRunCallback(change, patchset, attempt, extId, project),
},
];
}
/**
* ActionCallback that cancels an existing run and starts a new builder run
* via Buildbucket.
*/
async startRunCallback(change, patchset, attempt, externalId, project) {
await this.cancelRunCallback(change, patchset, attempt, externalId);
const gerritChange =
{
host: this.gerritHost,
project,
change,
patchset,
};
await this.bbClient.scheduleBuild({
// TODO(gavinmak): Add request ID.
templateBuildId: this.extractBuildbucketId(externalId),
gerritChanges: [gerritChange],
});
this.plugin.checks().announceUpdate();
return {};
}
/**
* ActionCallback that cancels an existing build. If the build is in an end
* state, this is a no-op.
*/
async cancelRunCallback(_change, _patchset, _attempt, externalId) {
await this.bbClient.cancelBuild({
id: this.extractBuildbucketId(externalId),
summaryMarkdown: 'Cancel build',
});
this.plugin.checks().announceUpdate();
return {};
}
/**
* Callback that retries all failedBuilders for the given change and patchset.
*/
async retryFailedBuildsCallback(failedBuilders, change, patchset, project) {
if (failedBuilders.length === 0) {
return {errorMessage: 'Retry failed triggered with no failed builds.'};
}
const gerritChanges = [{
host: this.gerritHost,
project,
change,
patchset,
}];
const requests = makeBuildRequests(
failedBuilders, gerritChanges, getNewOperationId(), [RETRY_FAILED_TAG]);
const result = await this.bbClient.batch(requests);
(result.responses || []).forEach(r => {
if (r.error) {
console.error('Failed to trigger build:', r.error.message);
}
});
this.plugin.checks().announceUpdate();
return {};
}
/**
* Top-Level ActionCallback that opens the tryjob picker popup.
*/
async chooseTryjobsCallback(change, patchset, project) {
const pluginName = encodeURIComponent(this.plugin.getPluginName());
const config = await this.plugin.restApi()
.get(`/projects/${encodeURIComponent(project)}/${pluginName}~config`);
await openTryjobPicker(this.plugin, config, this.buildbucketHost, project,
change, patchset);
this.plugin.checks().announceUpdate();
return {};
}
/**
* Top-Level ActionCallback that toggles the display and fetching of
* experimental builds, CheckRuns, and CheckResults.
*/
async filterExperimentsCallback() {
this.includeExperiments = !this.includeExperiments;
this.plugin.checks().announceUpdate();
return {};
}
/**
* Returns the ID to use for buildbucket RPCs. Assumes externalId is a full
* resource name which takes the format "//${host}/build/${buildId}"
*/
extractBuildbucketId(externalId) {
const match = /build\/(\d+)$/.exec(externalId);
return match && match[1];
}
/**
* Returns the TestResult name corresponding to this artifact. Assumes that
* the artifact name takes the format "${result_name}/artifacts/..."
*/
extractResultName(artifact) {
const match = /^(.*?)\/artifacts\//.exec(artifact.name);
return match && match[1];
}
}