blob: 4e37eaa7a4ca2a5e5375587198daffe4b88da662 [file]
<!DOCTYPE html>
<!--
Copyright 2020 The Chromium Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
-->
<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
<title>checks-fetcher tests</title>
<script src="../node_modules/@webcomponents/webcomponentsjs/webcomponents-loader.js"></script>
<script src="../node_modules/web-component-tester/browser.js"></script>
<script type="module">
import './common-test-setup.js';
import {
BuildbucketV2Client,
BuildStatus,
RETRY_FAILED_TAG,
} from '../src/main/resources/static/buildbucket-client.js';
import {
LRUStorageCacheObject,
} from '../src/main/resources/static/lru-storage-cache-object.js';
import {
ResultDbV1Client,
TestVariantStatus,
TestStatus,
} from '../src/main/resources/static/resultdb-client.js';
import {
ChecksFetcher,
Category,
LinkIcon,
RunStatus,
} from '../src/main/resources/static/checks-fetcher.js';
suite('checks-fetcher tests', () => {
let fetcher;
let sandbox;
let plugin;
let client;
let changeObject;
setup(() => {
sandbox = sinon.sandbox.create();
sandbox.stub(BuildbucketV2Client.prototype, '_getAccessToken')
.returns('accessToken');
sandbox.stub(ResultDbV1Client.prototype, '_getAccessToken')
.returns('accessToken');
sandbox.stub(ChecksFetcher.prototype, '_getAccessToken')
.returns('accessToken');
plugin = {
restApi: () => ({
get: () => ({
buckets: ['foo.bar.baz'],
hideRetryButton: false,
}),
}),
getPluginName: () => 'buildbucket',
};
client = new ResultDbV1Client('resultdb.example.com');
fetcher = new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
fetcher.retryEnabled = true;
changeObject = {
changeNumber: 123456,
patchsetNumber: 1,
repo: 'foo/bar',
changeInfo: {
status: 'NEW',
revisions: [{kind: 'REWORK', _number: 1}],
},
};
});
teardown(() => {
localStorage.clear();
sandbox.restore();
});
function stubFetch(response) {
sandbox.stub(window, 'fetch').returns(Promise.resolve({
ok: true,
text: async () => `)]}'${JSON.stringify(response)}`,
}));
}
test('fetchTestVariants gets an empty list for no TestVariants',
async () => {
stubFetch({nextPageToken: 'foo'});
const build = {
infra: {
resultdb: {
hostname: 'resultdb.example.com',
invocation: '123456',
},
},
};
assert.deepEqual(
await fetcher.fetchTestVariants(build, false),
[],
);
});
test('fetchTestVariants can query successful builds',
async () => {
const variant = {testId: 'foo'};
stubFetch({testVariants: [variant]});
const build = {
status: BuildStatus.SUCCESS,
infra: {
resultdb: {
hostname: 'resultdb.example.com',
invocation: '123456',
},
},
};
assert.deepEqual(
await fetcher.fetchTestVariants(build, false),
[],
);
assert.deepEqual(
await fetcher.fetchTestVariants(build, true),
[variant],
);
});
test('fetchTestVariants queries unexpected TestVariants',
async () => {
stubFetch({
nextPageToken: 'foo',
testVariants: [
{status: TestVariantStatus.UNEXPECTED},
{status: TestVariantStatus.UNEXPECTED},
],
});
const build = {infra: {resultdb: {hostname: 'host', invocation: 'inv'}}};
assert.deepEqual(
await fetcher.fetchTestVariants(build, false),
[
{status: TestVariantStatus.UNEXPECTED},
{status: TestVariantStatus.UNEXPECTED},
],
);
sinon.assert.calledOnce(fetch);
});
test('fetchBuilds fetches builds', async () => {
const firstResponse = {
responses: [
{
searchBuilds: {
builds: [{id: 1, tags:[]}],
nextPageToken: 'foo',
},
},
{searchBuilds: {}},
{
searchBuilds: {
builds: [{id: 3, tags:[]}],
nextPageToken: 'bar',
},
},
],
};
const secondResponse = {
responses: [
{
searchBuilds: {
builds: [{id: 2, tags:[]}],
nextPageToken: 'baz',
},
},
{searchBuilds: {}},
],
};
const thirdResponse = {responses: [{searchBuilds: {}}]};
const fetch = sandbox.stub(window, 'fetch');
fetch.onCall(0).returns(Promise.resolve({
ok: true,
text: async () => `)]}'${JSON.stringify(firstResponse)}`,
}));
fetch.onCall(1).returns(Promise.resolve({
ok: true,
text: async () => `)]}'${JSON.stringify(secondResponse)}`,
}));
fetch.onCall(2).returns(Promise.resolve({
ok: true,
text: async () => `)]}'${JSON.stringify(thirdResponse)}`,
}));
assert.deepEqual(
await fetcher.fetchBuilds(123456, [1, 2, 3]),
[{id: 1, tags:[]}, {id: 3, tags:[]}, {id: 2, tags:[]}],
);
sinon.assert.calledThrice(fetch);
});
test('fetchBuilds filters experimental builds', async () => {
const expTag = {key: 'cq_experimental', value: 'true'};
const nonExpTag = {key: 'cq_experimental', value: 'false'};
stubFetch({
responses: [
{
searchBuilds: {
builds: [
{id: 1, tags: [nonExpTag]},
]
},
},
{
searchBuilds: {
builds: [
{id: 2, tags: [expTag]},
{id: 3, tags: [nonExpTag]},
]
},
},
],
});
assert.deepEqual(
await fetcher.fetchBuilds(123456, [1, 2], 'project', false),
[
{id: 1, tags: [nonExpTag]},
{id: 3, tags: [nonExpTag]},
],
);
assert.deepEqual(
await fetcher.fetchBuilds(123456, [1, 2], 'project', true),
[
{id: 1, tags: [nonExpTag]},
{id: 2, tags: [expTag]},
{id: 3, tags: [nonExpTag]},
],
);
});
test('fetchBuilds hides builds tagged hide-in-gerrit', async () => {
const hiddenTag = {key: 'hide-in-gerrit', value: 'pointless'};
const nonHiddenTag = {key: 'hide-in-gerrit', value: 'false'};
stubFetch({
responses: [
{
searchBuilds: {
builds: [
{id: 1, tags: [nonHiddenTag]},
]
},
},
{
searchBuilds: {
builds: [
{id: 2, tags: [hiddenTag]},
{id: 3, tags: [nonHiddenTag]},
]
},
},
{
searchBuilds: {
builds: [
{id: 4, tags: [hiddenTag]},
]
},
},
],
});
assert.deepEqual(
await fetcher.fetchBuilds(123456, [1, 2], 'project'),
[
{id: 1, tags: [nonHiddenTag]},
{id: 3, tags: [nonHiddenTag]},
],
);
});
test('fetchDisplayedBuilds handles equivalent patchsets', async () => {
const stub = sandbox.stub(ChecksFetcher.prototype, 'fetchBuilds');
const build5 = {id: 5, tags: [{key: 'cq_experimental', value: 'false'}]};
const build4 = {id: 4, tags: [{key: 'cq_experimental', value: 'false'}]};
const build3 = {id: 3, tags: [{key: 'cq_experimental', value: 'false'}]};
const build2 = {id: 2, tags: [{key: 'cq_experimental', value: 'false'}]};
stub.withArgs(123, [6], 'project').returns([]);
stub.withArgs(123, [5], 'project').returns([build5]);
stub.withArgs(123, [2, 3], 'project').returns([build2, build3]);
stub.withArgs(123, [2], 'project').returns([build2]);
const changeInfo = {
revisions: [
{kind: 'REWORK', _number: 6},
{kind: 'REWORK', _number: 5},
{kind: 'TRIVIAL_REBASE', _number: 4},
{kind: 'NO_CHANGE', _number: 3},
{kind: 'REWORK', _number: 2},
{kind: 'REWORK', _number: 1},
],
status: 'MERGED',
};
fetcher = new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
// Fetching PS 3 gets builds 2 and 3.
assert.deepEqual(
await fetcher.fetchDisplayedBuilds(123, 3, 'project', changeInfo),
[build2, build3],
)
// Fetching PS 2 gets build 2.
assert.deepEqual(
await fetcher.fetchDisplayedBuilds(123, 2, 'project', changeInfo),
[build2],
)
// Fetching PS 6 gets build 5.
assert.deepEqual(
await fetcher.fetchDisplayedBuilds(123, 6, 'project', changeInfo),
[build5],
)
});
test('convertBuildToRun creates no results for SCHEDULED or STARTED builds',
() => {
const scheduledBuild = {
builder: {builder: 'foo'},
id: '12345',
status: BuildStatus.SCHEDULED,
};
assert.deepEqual(
fetcher.convertBuildToRun(
scheduledBuild, 1, 1, [{id: '123', status: BuildStatus.SCHEDULED}]).results,
[]);
const startedBuild = {
builder: {builder: 'foo'},
id: '12345',
status: BuildStatus.STARTED,
};
assert.deepEqual(
fetcher.convertBuildToRun(
startedBuild, 1, 1, [{id: '123', status: BuildStatus.STARTED}]).results,
[]);
});
test('convertBuildToRun creates statuses for SCHEDULED and STARTED builds',
() => {
const scheduledBuild = {
builder: {builder: 'foo'},
id: '12345',
status: BuildStatus.SCHEDULED,
};
assert.deepEqual(
fetcher.convertBuildToRun(
scheduledBuild,
1,
1,
[{id: '123', status: BuildStatus.SCHEDULED}],
).status,
RunStatus.SCHEDULED);
const startedBuild = {
builder: {builder: 'foo'},
id: '12345',
status: BuildStatus.STARTED,
};
assert.deepEqual(
fetcher.convertBuildToRun(
startedBuild,
1,
1,
[{id: '123', status: BuildStatus.STARTED}],
).status,
RunStatus.RUNNING);
});
test('fetch returns only variant results for SUCCESSful builds',
async () => {
sandbox.stub(ChecksFetcher.prototype, 'fetchDisplayedBuilds').returns([{
builder: {builder: 'foo'},
id: '123456',
endTime: '2017-12-15T01:30:15.05Z',
status: BuildStatus.SUCCESS,
infra: {resultdb: {hostname: 'host'}},
}]);
sandbox.stub(ChecksFetcher.prototype, 'fetchTestVariants')
.returns(Promise.resolve([
{testMetadata: {name: 'test'}, results: [{result: {}}]}
]));
const res = await fetcher.fetch(changeObject);
const {results} = res.runs[0];
assert.strictEqual(results.length, 1);
assert.strictEqual(results[0].summary,
'Test test failed but the build succeeded.',
);
});
test('fetch does not fetch variant results for builds tagged with ' +
'hide-test-results-in-gerrit', async () => {
sandbox.stub(ChecksFetcher.prototype, 'fetchDisplayedBuilds').returns([{
builder: {builder: 'foo'},
id: '123456',
endTime: '2017-12-15T01:30:15.05Z',
status: BuildStatus.FAILURE,
infra: {resultdb: {hostname: 'host'}},
tags: [{key: 'hide-test-results-in-gerrit', value: 'true'}],
}]);
const variantSpy = sandbox.spy(
ResultDbV1Client.prototype, 'fetchTestVariants');
const res = await fetcher.fetch(changeObject);
const {results} = res.runs[0];
assert.strictEqual(results.length, 1);
assert.strictEqual(results[0].summary,
'Build foo failed.',
);
sinon.assert.notCalled(variantSpy);
});
test('convertBuildToRun gets Buildbucket CheckResults', () => {
const build = {
builder: {builder: 'foo'},
infra: {},
id: '123456',
status: BuildStatus.FAILURE,
};
const run = fetcher.convertBuildToRun(
build, 1, 1, [{id: '123456', status: BuildStatus.FAILURE}]);
assert.strictEqual(run.results.length, 1);
assert.strictEqual(run.results[0].summary, 'Build foo failed.');
});
test('fetch orders Buildbucket CheckResults first', async () => {
sandbox.stub(ChecksFetcher.prototype, 'fetchDisplayedBuilds').returns([{
builder: {builder: 'foo'},
id: '123456',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
status: BuildStatus.FAILURE,
}]);
sandbox.stub(ChecksFetcher.prototype, 'fetchTestVariants')
.returns(Promise.resolve([
{
testMetadata: {name: 'test0'},
results: [{result: {}}],
},
{
testMetadata: {name: 'test1'},
results: [{result: {}}],
},
]));
const res = await fetcher.fetch(changeObject);
const {results} = res.runs[0];
assert.strictEqual(results.length, 4);
assert.strictEqual(results[0].summary, 'Build foo failed.');
assert.strictEqual(results[3].message,
'More tests may have failed. View all results on the build page.');
});
test('convertBuildToRun handles non-critical builds', () => {
const build = {
builder: {builder: 'foo'},
critical: 'NO',
infra: {},
id: '123456',
status: BuildStatus.FAILURE,
};
const run = fetcher.convertBuildToRun(
build, 1, 1, [{id: '123456', status: BuildStatus.FAILURE, build}]);
assert.strictEqual(run.results.length, 1);
assert.strictEqual(run.results[0].category, Category.WARNING);
assert.strictEqual(
run.results[0].summary,
'Non-critical build foo failed.',
);
});
test('convertBuildToRun sets CheckName based on latest attempt', () => {
const build = {
builder: {builder: 'foo'},
critical: 'NO',
infra: {},
id: '123456',
status: BuildStatus.FAILURE,
};
const startedRun = fetcher.convertBuildToRun(build, 1, 1, [
{id: '1', status: BuildStatus.FAILURE},
{id: '2', status: BuildStatus.FAILURE},
{id: '3', status: BuildStatus.STARTED},
]);
assert.strictEqual(startedRun.checkName, 'foo');
const infraFailedRun = fetcher.convertBuildToRun(build, 2, 2, [
{id: '1', status: BuildStatus.FAILURE},
{id: '2', status: BuildStatus.INFRA_FAILURE},
{id: '3', status: BuildStatus.FAILURE},
]);
assert.strictEqual(infraFailedRun.checkName, 'foo');
});
test('convertBuildToRun shows previous attempts', () => {
fetcher.showPreviousAttempts = true;
const attempts = [
{id: '3', build: {id: '3', status: BuildStatus.INFRA_FAILURE}},
{id: '2', build: {id: '2', status: BuildStatus.SUCCESS}},
{id: '1', build: {id: '1', status: BuildStatus.FAILURE}},
];
let run = fetcher.convertBuildToRun({
builder: {builder: 'foo'},
critical: 'NO',
infra: {},
id: '1',
status: BuildStatus.FAILURE,
}, 1, 1, attempts);
assert.strictEqual(run.results.length, 3);
// Order is reversed. Show latest results first.
assert.strictEqual(run.results[0].summary, 'Attempt 3 failed.');
assert.strictEqual(run.results[1].summary, 'Attempt 2 succeeded.');
assert.strictEqual(run.results[2].summary, 'Attempt 1 infra failed.');
// Attempts shown should be <= current build.
run = fetcher.convertBuildToRun({
builder: {builder: 'foo'},
critical: 'NO',
infra: {},
id: '3',
status: BuildStatus.INFRA_FAILURE,
}, 1, 1, attempts);
assert.strictEqual(run.results.length, 1);
assert.strictEqual(run.results[0].summary, 'Attempt 1 infra failed.');
});
test('convertBuildToRun uses max status if showing previous attempts',
() => {
fetcher.showPreviousAttempts = true;
// Any critical failure should set category to ERROR.
let run = fetcher.convertBuildToRun(
{
builder: {builder: 'foo'},
infra: {},
id: '1',
status: BuildStatus.SUCCESS,
},
1,
1,
[
{
id: '3',
status: BuildStatus.FAILURE,
build: {id: '3', status: BuildStatus.FAILURE},
},
{
id: '2',
status: BuildStatus.CANCELED,
build: {id: '2', status: BuildStatus.CANCELED},
},
{
id: '1',
status: BuildStatus.SUCCESS,
build: {id: '1', status: BuildStatus.SUCCESS},
},
],
);
assert.strictEqual(run.results.length, 3);
for (const res of run.results) {
assert.strictEqual(res.category, Category.ERROR);
}
// A non-critical failure can set category to warning.
run = fetcher.convertBuildToRun(
{
builder: {builder: 'foo'},
infra: {},
id: '1',
status: BuildStatus.SUCCESS,
},
1,
1,
[
{
id: '3',
status: BuildStatus.FAILURE,
build: {id: '3', status: BuildStatus.FAILURE, critical: 'NO'}},
{
id: '2',
status: BuildStatus.CANCELED,
build: {id: '2', status: BuildStatus.CANCELED},
},
{
id: '1',
status: BuildStatus.SUCCESS,
build: {id: '1', status: BuildStatus.SUCCESS},
},
],
);
assert.strictEqual(run.results.length, 3);
for (const res of run.results) {
assert.strictEqual(res.category, Category.WARNING);
}
// A CANCELED build can set category to INFO.
run = fetcher.convertBuildToRun(
{
builder: {builder: 'foo'},
infra: {},
id: '1',
status: BuildStatus.SUCCESS,
},
1,
1,
[
{
id: '3',
status: BuildStatus.SUCCESS,
build: {id: '3', status: BuildStatus.SUCCESS},
},
{
id: '2',
status: BuildStatus.CANCELED,
build: {id: '2', status: BuildStatus.CANCELED},
},
{
id: '1',
status: BuildStatus.SUCCESS,
build: {id: '1', status: BuildStatus.SUCCESS},
},
],
);
assert.strictEqual(run.results.length, 3);
for (const res of run.results) {
assert.strictEqual(res.category, Category.INFO);
}
// Build where all attempts succeed should not return results.
run = fetcher.convertBuildToRun(
{
builder: {builder: 'foo'},
infra: {},
id: '1',
status: BuildStatus.SUCCESS,
},
1,
1,
[
{
id: '3',
status: BuildStatus.SUCCESS,
build: {id: '3', status: BuildStatus.SUCCESS},
},
{
id: '2',
status: BuildStatus.SUCCESS,
build: {id: '2', status: BuildStatus.SUCCESS},
},
{
id: '1',
status: BuildStatus.SUCCESS,
build: {id: '1', status: BuildStatus.SUCCESS},
},
],
);
assert.strictEqual(run.results.length, 0);
});
test('convertBuildToRun handles RTS builds', () => {
const build = {
builder: {builder: 'foo'},
critical: 'NO',
infra: {},
id: '1',
status: BuildStatus.INFRA_FAILURE,
output: {
properties: {
rts_was_used: true,
}
}
};
let run = fetcher.convertBuildToRun(build, 1, 1, [
{id: '1', status: BuildStatus.INFRA_FAILURE, isQuickRun: true},
]);
assert.strictEqual(run.checkName, '🟪\u2005\u2005🚀\u2005\u2005foo');
assert.isTrue(run.statusDescription.includes('Quick Run'));
assert.isTrue(run.results[0].tags.map(t => t.name)
.includes('🚀\u2005\u2005Quick Run'));
// The last attempt should be used for check run naming.
run = fetcher.convertBuildToRun(build, 1, 1, [
{id: '2', status: BuildStatus.INFRA_FAILURE, isQuickRun: true},
{id: '1', status: BuildStatus.INFRA_FAILURE, isQuickRun: false},
]);
assert.strictEqual(run.checkName, '🟪\u2005\u2005foo');
});
test('convertBuildToRun tags CheckResults for experimental builds', () => {
const baseBuild = {
builder: {builder: 'foo'},
id: '123456',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
status: BuildStatus.FAILURE,
};
const attempts = [{id: '123456', status: BuildStatus.FAILURE}];
const expBuild = Object.assign(baseBuild,
{tags: [{key: 'cq_experimental', value: 'true'}]});
const expRun = fetcher.convertBuildToRun(expBuild, 1, 1, attempts);
expRun.results.forEach((result) => {
assert.isTrue(
result.tags.map(t => t.name).includes('Experimental'));
});
const nonExpBuild = Object.assign(baseBuild,
{tags: [{key: 'cq_experimental', value: 'false'}]});
const nonExpRun = fetcher.convertBuildToRun(nonExpBuild, 1, 1, attempts);
nonExpRun.results.forEach((result) => {
assert.isFalse(
result.tags.map(t => t.name).includes('Experimental'));
});
});
test('convertBuildToRun tags infra failures', () => {
const baseBuild = {
builder: {builder: 'foo'},
id: '123456',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const attempts = [{id: '123456', status: BuildStatus.INFRA_FAILURE}];
const infraFailedBuild = Object.assign(
baseBuild, {status: BuildStatus.INFRA_FAILURE});
const infraRun = fetcher.convertBuildToRun(
infraFailedBuild, 1, 1, attempts);
assert.strictEqual(infraRun.checkName, '🟪\u2005\u2005foo');
infraRun.results.forEach((r) => {
assert.isTrue(r.tags.map(t => t.name).includes('Infra Failure'));
});
const failedBuild = Object.assign(baseBuild, {status: BuildStatus.FAILURE});
const nonInfraRun = fetcher.convertBuildToRun(
failedBuild, 1, 1, attempts);
nonInfraRun.results.forEach((r) => {
assert.isFalse(r.tags.map(t => t.name).includes('Infra Failure'));
});
});
test('convertBuildToRun categorizes failed test results as WARNING if ' +
'build succeeded', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
endTime: '2017-12-15T01:30:15.05Z',
status: BuildStatus.SUCCESS,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [{
testMetadata: {name: 'test'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL, summaryHtml: '<p>foo</p>'}}],
}];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
assert.strictEqual(results.length, 1);
const [result] = results;
assert.strictEqual(result.category, Category.WARNING);
assert.strictEqual(
result.summary,
'Test test failed but the build succeeded.',
);
assert.strictEqual(result.message, 'Run #1: unexpectedly failed.');
});
test('convertVariantsToCheckResults handles flaky results', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
status: BuildStatus.FAILURE,
endTime: '2017-12-15T01:30:15.05Z',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [{
testMetadata: {name: 'foo-test'},
status: TestVariantStatus.FLAKY,
results: [
{
result: {
status: TestStatus.FAIL,
},
},
{
result: {
status: TestStatus.PASS,
expected: true,
},
},
],
}];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
const checkResult = results[0];
assert.strictEqual(checkResult.category, Category.WARNING);
assert.strictEqual(checkResult.summary, 'Test foo-test flaked.');
assert.strictEqual(checkResult.message, 'Run #1: unexpectedly failed.');
// If includeAdditionalResults is true, show all results.
fetcher.includeAdditionalResults = true;
const additionalResults = await fetcher.convertVariantsToCheckResults(build, variants);
const additionalCheckResult = additionalResults[0];
assert.strictEqual(additionalCheckResult.category, Category.WARNING);
assert.strictEqual(additionalCheckResult.summary, 'Test foo-test flaked.');
assert.strictEqual(
additionalCheckResult.message,
'Run #1: unexpectedly failed. Run #2: expectedly passed.',
);
});
test('convertVariantsToCheckResults handles exonerated results', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
status: BuildStatus.FAILURE,
endTime: '2017-12-15T01:30:15.05Z',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [{
testMetadata: {name: 'foo-test'},
status: TestVariantStatus.EXONERATED,
results: [{result: {status: TestStatus.PASS}}],
}];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
const checkResult = results[0];
assert.strictEqual(checkResult.category, Category.WARNING);
assert.strictEqual(checkResult.summary, 'Test foo-test was exonerated.');
assert.strictEqual(checkResult.message, 'Run #1: unexpectedly passed.');
});
test('convertVariantsToCheckResults handles unexpectedly skipped results', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
status: BuildStatus.FAILURE,
endTime: '2017-12-15T01:30:15.05Z',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [{
testMetadata: {name: 'foo-test'},
status: TestVariantStatus.UNEXPECTEDLY_SKIPPED,
results: [{result: {status: TestStatus.SKIP}}],
}];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
const checkResult = results[0];
assert.strictEqual(checkResult.category, Category.WARNING);
assert.strictEqual(checkResult.summary, 'Test foo-test was unexpectedly skipped.');
assert.strictEqual(checkResult.message,'Run #1: unexpectedly skipped.');
});
test('convertVariantsToCheckResults categorizes preliminary results', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
status: BuildStatus.STARTED,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [
{
testMetadata: {name: 'test0'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL, summaryHtml: '<p>foo</p>'}}],
},
{
testMetadata: {name: 'test1'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL, summaryHtml: '<p>foo</p>'}}],
},
];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
assert.strictEqual(results.length, 3);
const [checkResult, _, extraResult] = results;
assert.strictEqual(checkResult.category, Category.WARNING);
assert.strictEqual(
checkResult.summary,
'Test test0 failed.',
);
assert.isTrue(checkResult.tags
.map(t => t.name).includes('Preliminary'));
assert.strictEqual(extraResult.category, Category.INFO);
assert.isTrue(extraResult.tags.map(t => t.name).includes('Preliminary'));
});
test('convertVariantsToCheckResults tags results with variant def', async () => {
const build = {
builder: {builder: 'foo'},
id: '123456',
endTime: '2017-12-15T01:30:15.05Z',
status: BuildStatus.FAILURE,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [
{
variant: {def: {test_suite: 'suite_foo', gpu: 'gpu_foo'}},
testMetadata: {name: 'test2'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL}}],
},
{
variant: {def: {test_suite: 'suite_foo', build_target: 'build_foo'}},
testMetadata: {name: 'test1'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL}}],
},
{
variant: {def: {test_suite: 'suite_bar', gpu: 'gpu_bar'}},
testMetadata: {name: 'test1'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL}}],
},
{
variant: {def: {}},
testMetadata: {name: 'test0'},
status: TestVariantStatus.UNEXPECTED,
results: [{result: {status: TestStatus.FAIL}}],
},
];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
assert.isTrue(results[0].summary.includes('test0'));
assert.strictEqual(results[0].tags.length, 0);
assert.isTrue(results[1].summary.includes('test1'));
assert.strictEqual(results[1].tags.length, 2);
assert.isTrue(results[1].tags[0].tooltip.includes('suite_bar'));
assert.isTrue(results[1].tags[1].tooltip.includes('gpu_bar'));
assert.isTrue(results[2].summary.includes('test1'));
assert.strictEqual(results[2].tags.length, 2);
assert.isTrue(results[2].tags[0].tooltip.includes('suite_foo'));
assert.isTrue(results[2].tags[1].tooltip.includes('build_foo'));
assert.isTrue(results[3].summary.includes('test2'));
assert.strictEqual(results[3].tags.length, 2);
assert.isTrue(results[3].tags[0].tooltip.includes('suite_foo'));
assert.isTrue(results[3].tags[1].tooltip.includes('gpu_foo'));
});
test('convertBuildToRun returns correct actions', () => {
const startedBuild = {
builder: {builder: 'foo'},
id: '123',
status: BuildStatus.STARTED,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const startedRun = fetcher.convertBuildToRun(
startedBuild, 1, 1, [{id: '123', status: BuildStatus.STARTED}]);
assert.strictEqual(startedRun.checkName, 'foo');
assert.strictEqual(startedRun.actions.length, 1);
assert.strictEqual(startedRun.actions[0].name, 'Cancel');
const noRetryBuild = {
builder: {builder: 'foo'},
id: '123456',
status: BuildStatus.FAILURE,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
tags: [{key: 'skip-retry-in-gerrit', value: 'true'}],
};
const noRetryRun = fetcher.convertBuildToRun(
noRetryBuild, 1, 1, [{id: '123', status: BuildStatus.FAILURE}]);
assert.strictEqual(noRetryRun.checkName, 'foo');
assert.strictEqual(noRetryRun.actions.length, 0);
});
test('convertBuildToRun returns correct build links', () => {
const successBuild = {
builder: {builder: 'bar', project: 'chromium', bucket: 'baz'},
id: '123456',
summaryMarkdown: 'Build passed',
number: 987,
status: BuildStatus.SUCCESS,
};
const failedBuild = {
builder: {builder: 'foo', project: 'chromium', bucket: 'baz'},
id: '123456',
summaryMarkdown: 'Build failed',
number: 987,
status: BuildStatus.FAILURE,
};
const successRun = fetcher.convertBuildToRun(
successBuild, 1, 1, [{id: '123456', status: BuildStatus.SUCCESS}], []);
assert.deepEqual(successRun.results, []);
assert.strictEqual(
successRun.statusDescription,
'Build bar succeeded.',
);
const failedRun = fetcher.convertBuildToRun(
failedBuild, 1, 1, [{id: '123456', status: BuildStatus.FAILURE}], []);
assert.strictEqual(failedRun.results[0].summary, 'Build foo failed.');
assert.strictEqual(failedRun.results[0].message, 'Build failed');
assert.deepEqual(
failedRun.results[0].links,
[{
url: 'https://buildbucket.example.com/build/123456',
tooltip: 'Build',
primary: true,
icon: LinkIcon.EXTERNAL,
}],
);
});
test('convertVariantsToCheckResults returns correct links', async () => {
// Build is SUCCESSful and should only contain TestVariant CheckResults.
const build = {
builder: {builder: 'foo', project: 'bar', bucket: 'baz'},
id: '123456',
number: 987,
status: BuildStatus.SUCCESS,
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
};
const variants = [{
testMetadata: {name: 'test0'},
status: TestVariantStatus.UNEXPECTED,
variantHash: 'deadbeef',
testId: 'ninja://:blink_web_tests/bar',
results: [
{
result: {
name: 'invocations/task-foo.com-123',
tags: [
{key: 'step_name', value: 'foo (with bar) on baz'},
],
},
},
{result: {name: 'this is a bad name'}},
],
}];
const results = await fetcher.convertVariantsToCheckResults(build, variants);
assert.deepEqual(
results[0].links,
[
{
url: 'https://ci.chromium.org/ui/p/bar/builders/baz/foo/b123456/' +
'test-results?q=ExactID%3Aninja%3A%2F%2F%3Ablink_web_tests%2F' +
'bar+VHash%3Adeadbeef&clean=',
tooltip: 'Test Results',
primary: true,
icon: LinkIcon.EXTERNAL,
},
{
url: 'https://test-results.appspot.com/data/layout_results/foo/' +
'987/foo%20%28with%20bar%29/layout-test-results/results.html',
tooltip: 'Web Test Results',
primary: false,
icon: LinkIcon.EXTERNAL,
},
{
url: 'https://foo.com/task?id=123',
tooltip: 'Swarming Task',
primary: false,
icon: LinkIcon.EXTERNAL,
},
],
);
});
test('fetch experimental builds', async () => {
stubFetch({
responses: [{
searchBuilds: {
builds: [
{
builder: {builder: 'bar'},
id: '123456',
tags: [{key: 'cq_experimental', value: 'true'}],
infra: {resultdb: {}, swarming: {}},
status: BuildStatus.FAILURE,
},
{
builder: {builder: 'foo'},
id: '654321',
tags: [{key: 'cq_experimental', value: 'false'}],
infra: {resultdb: {}, swarming: {}},
status: BuildStatus.FAILURE,
},
],
},
}],
});
sandbox.stub(ChecksFetcher.prototype, 'fetchTestVariants')
.returns(Promise.resolve([]));
const update = sandbox.stub().returns({});
const checks = sandbox.stub().returns({announceUpdate: update});
const plugin = {
checks,
restApi: () => ({
get: () => ({
buckets: ['foo.bar.baz'],
hideRetryButton: false,
}),
}),
getPluginName: () => 'buildbucket',
};
// CheckResult should not contain experimental runs.
const stubbedFetcher =
new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
const nonExpResult = await stubbedFetcher.fetch(changeObject);
assert.strictEqual(nonExpResult.runs[0].results.length, 1);
assert.isFalse(nonExpResult.runs[0].results[0].tags
.map(t => t.name).includes('Experimental'));
assert.strictEqual(
nonExpResult.actions[2].name, 'Show Additional Results');
// Simulate "Show/Hide Additional" click.
await nonExpResult.actions[2].callback(1, 1, 1, 1, 'name', 'action');
sinon.assert.calledOnce(update);
// CheckResult should contain experimental runs.
const expResult = await stubbedFetcher.fetch(changeObject);
assert.strictEqual(expResult.runs.length, 2);
assert.isTrue(expResult.runs[0].results[0].tags
.map(t => t.name).includes('Experimental'));
assert.isFalse(expResult.runs[1].results[0].tags
.map(t => t.name).includes('Experimental'));
assert.strictEqual(
expResult.actions[2].name, 'Hide Additional Results');
});
test('fetch handles ResultDB failures', async () => {
const builds = [{
builder: {builder: 'bar'},
id: '421532',
infra: {resultdb: {hostname: 'host', invocation: 'inv'}},
status: BuildStatus.FAILURE,
}];
sandbox.stub(ChecksFetcher.prototype, 'fetchDisplayedBuilds')
.returns(builds);
sandbox.stub(ChecksFetcher.prototype, 'fetchTestVariants')
.returns(Promise.reject('ResultDB failed!'));
// A notice run should be generated.
const res = await fetcher.fetch(changeObject);
assert.strictEqual(res.runs.length, 2);
assert.strictEqual(res.runs[1].checkName, 'Notice');
assert.strictEqual(res.runs[1].results[0].summary,
'Test results may be missing.');
});
test('retry failed without retriable builds', async () => {
stubFetch({
responses: [
{
searchBuilds: {
builds: [],
},
},
{
searchBuilds: {
builds: [],
},
},
{
searchBuilds: {
builds: [],
},
},
],
});
assert.deepEqual(await fetcher.retryFailedBuildsCallback(50, 3, 'repo'), {
message: 'No failed builds to retry',
});
});
test('retry failed with retriable builds', async () => {
stubFetch({
responses: [
{
searchBuilds: {
builds: [
{
id: "10",
builder: {builder: 'foo'},
status: BuildStatus.FAILURE,
},
]
},
},
{
searchBuilds: {
builds: [
{
id: "9",
builder: {builder: 'bar'},
status: BuildStatus.FAILURE,
},
{
id: "8",
builder: {builder: 'baz'},
status: BuildStatus.FAILURE,
},
]
},
},
{
searchBuilds: {
builds: [
{
id: "7",
builder: {builder: 'foo'},
status: BuildStatus.SUCCESS,
},
{
id: "6",
builder: {builder: 'baz'},
status: BuildStatus.STARTED,
},
]
},
},
],
});
const stub = sandbox.stub(
ChecksFetcher.prototype, 'scheduleBuildsCallback');
await fetcher.retryFailedBuildsCallback(50, 3, 'repo');
sinon.assert.calledWith(
stub, 50, 3, 'repo', [{ builder: 'bar' }], [RETRY_FAILED_TAG]);
});
test('run action returns an error message if failed', async () => {
const plugin = {
checks: () => {
return {announceUpdate: sandbox.stub()};
},
};
const stubbedFetcher =
new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
stubbedFetcher.retryEnabled = true;
const batchStub = sandbox.stub(BuildbucketV2Client.prototype, 'batch');
batchStub.onCall(0).returns(
Promise.resolve({responses: [{error: 'bad cancel'}]}));
const runAction = stubbedFetcher.createRunActions(
{
id: 987,
builder: {builder: 'foo'},
endTime: '2017-12-15T01:30:15.05Z',
status: BuildStatus.FAILURE,
},
'project',
1,
[{
id: 987,
status: BuildStatus.FAILURE,
endTime: '2017-12-15T01:30:15.05Z'
}],
)[0].callback;
let res = await runAction(123456);
assert.strictEqual(
res.message, '1 of 1 cancel requests failed. See console logs.');
batchStub.onCall(1).returns(Promise.resolve({responses: []}));
batchStub.onCall(2).returns(
Promise.resolve({responses: [{error: 'bad schedule'}]}));
res = await runAction(123456);
assert.strictEqual(
res.message, '1 of 1 schedule requests failed. See console logs.');
});
test('chooseTryjobsCallback opens a tryjob picker', async () => {
// TODO(gavinmak): Rework test and move to cr-tryjobs-picker_test.html.
stubFetch({responses: [{searchBuilds: {}}]});
sandbox.stub(ChecksFetcher.prototype, 'fetchTestVariants')
.returns(Promise.resolve([]));
const element = sandbox.mock({
buildbucketHost: 'oldhost',
pluginConfig: {config: 'old'},
change: {_number: 678, project: 'old/project'},
revision: {_number: 9},
});
const picker = {
close: sandbox.stub(),
_getElement: function() {
return {querySelector: sandbox.stub().returns(element)};
},
};
const config = {
buckets: ['foo.bar.baz'],
hideRetryButton: false,
};
const plugin = {
popup: sandbox.stub().returns(picker),
restApi: () => {
return {get: sandbox.stub().returns(config)};
},
getPluginName: () => 'buildbucket',
checks: () => {
return {announceUpdate: sandbox.stub()};
},
};
const fetcherWithPlugin = new ChecksFetcher(plugin, 'newhost', 2);
const result = await fetcherWithPlugin.fetch(changeObject);
assert.strictEqual(result.actions[0].name, 'Choose Tryjobs');
await result.actions[0].callback(123456, 321);
sinon.assert.calledOnce(plugin.popup);
assert.strictEqual(element.buildbucketHost, 'newhost');
assert.deepEqual(element.pluginConfig, config);
assert.deepEqual(element.change._number, 123456);
assert.deepEqual(element.change.project, 'foo/bar');
assert.deepEqual(element.revision, {_number: 1});
});
test('choose tryjobs is hidden if buckets are empty', async () => {
stubFetch({responses: [{searchBuilds: {}}]});
const plugin = {
restApi: () => {
return {get: sandbox.stub().returns({buckets: []})};
},
getPluginName: () => 'buildbucket',
checks: () => {
return {announceUpdate: sandbox.stub()};
},
};
const fetcherWithPlugin = new ChecksFetcher(plugin, 'newhost', 2);
const result = await fetcherWithPlugin.fetch(changeObject);
assert.isFalse(result.actions.some(a => a.name === 'Choose Tryjobs'));
});
test('retry failed is hidden if disabled in config', async () => {
stubFetch({
responses: [{
searchBuilds: {
builds: [
{
id: '10',
builder: {builder: 'foo'},
status: BuildStatus.FAILURE,
},
]
},
}]
});
const plugin = {
restApi: () => {
return {get: sandbox.stub().returns({hideRetryButton: true})};
},
getPluginName: () => 'buildbucket',
checks: () => {
return {announceUpdate: sandbox.stub()};
},
};
const fetcherWithPlugin = new ChecksFetcher(plugin, 'newhost', 2);
const result = await fetcherWithPlugin.fetch(changeObject);
assert.isFalse(result.actions.some(a => a.name === 'Retry Failed Builds'));
assert.strictEqual(result.runs[0].actions.length, 0);
});
test('toggle additional res action is primary if only action', async () => {
stubFetch({responses: [{searchBuilds: {}}]});
const plugin = {
restApi: () => {
return {get: sandbox.stub().returns({
buckets: [],
hideRetryButton: true,
})};
},
getPluginName: () => 'buildbucket',
checks: () => {
return {announceUpdate: sandbox.stub()};
},
};
const fetcherWithPlugin = new ChecksFetcher(plugin, 'newhost', 2);
const result = await fetcherWithPlugin.fetch(changeObject);
assert.strictEqual(result.actions[0].name, 'Show Additional Results');
assert.isTrue(result.actions[0].primary);
});
test('fetch aborts pending fetch requests', async () => {
sandbox.stub(window, 'fetch').returns(
new Promise((resolve, reject) => {
const wait = setTimeout(() => {
clearTimeout(wait);
resolve('foo');
}, 9999999);
})
);
// First set of fetches should not be aborted.
const controllerKey = fetcher.getChangeDataId(changeObject);
assert.strictEqual(fetcher.controllers.get(controllerKey), undefined);
fetcher.fetch(changeObject);
await new Promise(r => setTimeout(r, 100));
const firstController = fetcher.controllers.get(controllerKey);
assert.strictEqual(firstController.signal.aborted, false);
// Following fetches should be aborted.
fetcher.fetch(changeObject);
await new Promise(r => setTimeout(r, 100));
const secondController = fetcher.controllers.get(controllerKey);
assert.notStrictEqual(firstController, secondController);
assert.strictEqual(firstController.signal.aborted, true);
assert.strictEqual(secondController.signal.aborted, false);
fetcher.fetch(changeObject)
await new Promise(r => setTimeout(r, 100));
assert.strictEqual(secondController.signal.aborted, true);
});
test('fetch aborts based on repo, change, and patchset', async () => {
sandbox.stub(window, 'fetch').returns(
new Promise((resolve, reject) => {
const wait = setTimeout(() => {
clearTimeout(wait);
resolve('foo');
}, 9999999);
})
);
const firstChangeObject = {
changeNumber: 123456,
patchsetNumber: 1,
repo: 'foo/bar',
changeInfo: {
status: 'NEW',
revisions: [{kind: 'REWORK', _number: 1}],
},
};
const secondChangeObject = {
changeNumber: 123456,
patchsetNumber: 2,
repo: 'foo/bar',
changeInfo: {
status: 'NEW',
revisions: [{kind: 'REWORK', _number: 1}],
},
};
// Fetch from firstChangeObject.
const firstControllerKey = fetcher.getChangeDataId(firstChangeObject);
assert.strictEqual(fetcher.controllers.get(firstControllerKey), undefined);
fetcher.fetch(firstChangeObject);
await new Promise(r => setTimeout(r, 100));
const firstController = fetcher.controllers.get(firstControllerKey);
assert.strictEqual(firstController.signal.aborted, false);
// A fetch from secondChangeObject should not abort firstController.
fetcher.fetch(secondChangeObject);
await new Promise(r => setTimeout(r, 100));
const secondControllerKey = fetcher.getChangeDataId(secondChangeObject);
const secondController = fetcher.controllers.get(secondControllerKey);
assert.notStrictEqual(firstController, secondController);
assert.strictEqual(firstController.signal.aborted, false);
assert.strictEqual(secondController.signal.aborted, false);
// Aborting secondChangeObject should not abort firstController.
fetcher.fetch(secondChangeObject)
await new Promise(r => setTimeout(r, 100));
assert.strictEqual(firstController.signal.aborted, false);
assert.strictEqual(secondController.signal.aborted, true);
});
});
</script>