test/checks-fetcher_test.html - infra/gerrit-plugins/buildbucket - Git at Google

 <!DOCTYPE html>
 <!--
 Copyright 2020 The Chromium Authors. All rights reserved.
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 -->
 <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
 <title>checks-fetcher tests</title>

 <script src="../bower_components/webcomponentsjs/webcomponents-lite.js"></script>
 <script src="../bower_components/web-component-tester/browser.js"></script>

 <script type="module">
   import './common-test-setup.js';
   import {BuildbucketV2Client} from '../src/main/resources/static/buildbucket-client.js';
   import {ResultDbV1Client} from '../src/main/resources/static/resultdb-client.js';
   import {
       ChecksFetcher,
       CQ_EXPERIMENTAL_TAG,
     } from '../src/main/resources/static/checks-fetcher.js';

   suite('checks-fetcher tests', () => {
     let fetcher;
     let sandbox;
     let client;

     setup(() => {
       sandbox = sinon.sandbox.create();
       sandbox.stub(BuildbucketV2Client.prototype, '_getAccessToken').returns('accessToken');
       sandbox.stub(ResultDbV1Client.prototype, '_getAccessToken').returns('accessToken');
       sandbox.stub(ChecksFetcher.prototype, 'extractProjectFromUrl').returns('foo/bar');
       sandbox.stub(ChecksFetcher.prototype, 'extractHostFromUrl').returns('host.example.com');
       fetcher = new ChecksFetcher(null, 'buildbucket.example.com', 2);
     });

     teardown(() => {
       sandbox.restore();
     });

     test('fetchUnexpectedTestVariants returns an empty list for no TestVariants', async () => {
       const response = {
         nextPageToken: 'foo',
       };
       sandbox.stub(window, 'fetch').returns(Promise.resolve({
         ok: true,
         text: async () => `)]}'${JSON.stringify(response)}`,
       }));

       const build = {
         infra: {
           resultdb: {
             hostname: 'resultdb.example.com',
             invocation: '123456',
           },
         },
       };
       assert.deepEqual(await fetcher.fetchUnexpectedTestVariants(build), []);
     });

     test('fetchUnexpectedTestVariants queries all unexpected TestVariants', async () => {
       const fetch = sandbox.stub(window, 'fetch');

       const firstUnexpectedPage = {
         nextPageToken: 'foo',
         testVariants: Array.from({length: 1000}, () => ({status: 'UNEXPECTED'})),
       };
       fetch.onCall(0).returns(Promise.resolve({
         ok: true,
         text: async () => `)]}'${JSON.stringify(firstUnexpectedPage)}`,
       }));

       const secondUnexpectedPage = {
         nextPageToken: 'bar',
         testVariants: Array.from({length: 1000}, () => ({status: 'FLAKY'})),
       };
       fetch.onCall(1).returns(Promise.resolve({
         ok: true,
         text: async () => `)]}'${JSON.stringify(secondUnexpectedPage)}`,
       }));

       const expectedPage = {
         nextPageToken: 'baz',
         testVariants: Array.from({length: 1000}, () => ({status: 'EXPECTED'})),
       };
       fetch.returns(Promise.resolve({
         ok: true,
         text: async () => `)]}'${JSON.stringify(expectedPage)}`,
       }));

       const build = {
         infra: {
           resultdb: {
             hostname: 'resultdb.example.com',
             invocation: '123456',
           },
         },
       };
       assert.deepEqual(
         await fetcher.fetchUnexpectedTestVariants(build),
         [...firstUnexpectedPage.testVariants, ...secondUnexpectedPage.testVariants],
       );
       sinon.assert.calledThrice(fetch);
     });

     test('convertBuildToRun creates no CheckResults for SCHEDULED builds', () => {
       const build = {
         builder: {builder: 'foo'},
         id: '123456',
         status: 'SCHEDULED',
       };
       const variants = [{testId: 1}];
       assert.deepEqual(
         fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants).results, []);
     });

     test('convertBuildToRun returns only TestVariant CheckResults for SUCCESSful builds', () => {
       const build = {
         builder: {builder: 'foo'},
         id: '123456',
         tags: [],
         status: 'SUCCESS',
       };
       const variantId = '532142';
       const run = fetcher.convertBuildToRun(
         build,
         1,
         {foo: ['123456']},
         [{
           testId: variantId,
           results: [{result: {}}],
         }],
       );
       assert.strictEqual(run.results.length, 1);
       assert.strictEqual(run.results[0].summary, variantId);
     });

     test('convertBuildToRun gets a Buildbucket CheckResult without unexpectedTestVariants', () => {
       const build = {
         builder: {builder: 'foo'},
         infra: {},
         id: '123456',
         tags: [],
         status: 'FAILURE',
       };
       const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, []);
       assert.strictEqual(run.results.length, 1);
       assert.strictEqual(run.results[0].summary, 'Build FAILURE');
     });

     test('convertBuildToRun tags CheckResults for experimental builds', () => {
       const baseBuild = {
         builder: {builder: 'foo'},
         id: '123456',
         tags: [],
         infra: {},
         status: 'FAILED',
       };
       const attempts = {foo: ['123456']};
       // Create one extra TestVariant for the 'more test failed' CheckResult.
       const variants = [
         {
           testId: 0,
           results: [{result: {}}],
         },
         {
           testId: 1,
           results: [{result: {}}],
         },
         {
           testId: 2,
           results: [{result: {}}],
         },
       ];

       const expBuild = Object.assign(baseBuild,
         {tags: [{key: 'cq_experimental', value: 'true'}]});
       const expRun = fetcher.convertBuildToRun(expBuild, 1, attempts, variants);
       expRun.results.forEach((result) => {
         assert.isTrue(result.tags.includes(CQ_EXPERIMENTAL_TAG));
       });
       assert.strictEqual(expRun.results[expRun.results.length - 1].category, 'INFO');

       const nonExpBuild = Object.assign(baseBuild,
         {tags: [{key: 'cq_experimental', value: 'false'}]});
       const nonExpRun = fetcher.convertBuildToRun(nonExpBuild, 1, attempts, variants);
       nonExpRun.results.forEach((result) => {
         assert.isFalse(result.tags.includes(CQ_EXPERIMENTAL_TAG));
       });
       assert.strictEqual(nonExpRun.results[nonExpRun.results.length - 1].category, 'ERROR');
     });

     test('convertBuildToRun limits returned CheckResults', () => {
       // Build is SUCCESSful and should only contain TestVariant CheckResults.
       const build = {
         builder: {builder: 'foo'},
         id: '123456',
         tags: [],
         status: 'SUCCESS',
       };
       const variants = [
         {
           testId: 0,
           results: [{result: {}}],
         },
         {
           testId: 1,
           results: [{result: {}}],
         },
         {
           testId: 2,
           results: [{result: {}}],
         },
         {
           testId: 3,
           results: [{result: {}}],
         },
       ];

       const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants);
       assert.strictEqual(run.results.length, 3);
       assert.strictEqual(
         run.results[2].message,
         `2 more tests failed`,
       );
     });

     test('convertBuildToRun sorts FLAKY TestVariants', () => {
       // Build is SUCCESSful and should only contain TestVariant CheckResults.
       const build = {
         builder: {builder: 'foo'},
         id: '123456',
         tags: [],
         status: 'SUCCESS',
       };

       const variants = [
         {
           testId: 0,
           status: 'EXONERATED',
           results: [
             {result: {status: 'FAIL', summaryHtml: 'FAIL'}},
             {result: {status: 'CRASH', summaryHtml: 'CRASH'}},
           ],
         },
         {
           testId: 1,
           status: 'FLAKY',
           results: [
             {result: {status: 'CRASH', summaryHtml: 'CRASH'}},
             {result: {status: 'FAIL', summaryHtml: 'FAIL'}},
             {result: {status: 'PASS', summaryHtml: 'PASS'}},
             {result: {status: 'FAIL', summaryHtml: 'FAIL'}},
             {result: {status: 'PASS', summaryHtml: 'PASS'}},
           ],
         },
       ];

       const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants);

       const exoneratedResult = run.results[0];
       assert.strictEqual(exoneratedResult.summary, 0);
       assert.strictEqual(exoneratedResult.message, 'FAIL');

       const flakyResult = run.results[1];
       assert.strictEqual(flakyResult.summary, 1);
       assert.strictEqual(flakyResult.message, 'FAIL\nCRASH\nPASS');
     });

     test('fetch experimental builds', async () => {
       const buildResponse = {
         responses: [{
           searchBuilds: {
             builds: [
               {
                 builder: {builder: 'foo'},
                 id: '123456',
                 tags: [{key: 'cq_experimental', value: 'true'}],
                 infra: {resultdb: {}, swarming: {}},
                 status: 'FAILURE',
               },
               {
                 builder: {builder: 'bar'},
                 id: '654321',
                 tags: [{key: 'cq_experimental', value: 'false'}],
                 infra: {resultdb: {}, swarming: {}},
                 status: 'FAILURE',
               },
             ],
           },
         }],
       };
       sandbox.stub(window, 'fetch').returns(Promise.resolve({
         ok: true,
         text: async () => `)]}'${JSON.stringify(buildResponse)}`,
       }));
       sandbox.stub(ChecksFetcher.prototype, 'fetchUnexpectedTestVariants').returns([]);
       const update = sandbox.stub().returns({});
       const checks = sandbox.stub().returns({announceUpdate: update});
       const plugin = {checks};

       // CheckResult should not contain experimental runs.
       const fetcherWithPlugin = new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
       const nonExpResult = await fetcherWithPlugin.fetch(123456, 1);
       assert.strictEqual(nonExpResult.runs[0].results.length, 1);
       assert.isFalse(nonExpResult.runs[0].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
       assert.strictEqual(nonExpResult.actions[1].name, 'Show Experimental Results');

       // Simulate "Show/Hide Experimental" click.
       nonExpResult.actions[1].callback(1, 1, 1, 1, 'name', 'action');
       sinon.assert.calledOnce(update);

       // CheckResult should contain experimental runs.
       const expResult = await fetcherWithPlugin.fetch(123456, 1);
       assert.strictEqual(expResult.runs.length, 2);
       assert.isTrue(expResult.runs[0].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
       assert.isFalse(expResult.runs[1].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
       assert.strictEqual(expResult.actions[1].name, 'Hide Experimental Results');
     });
   });
 </script>
	<!DOCTYPE html>
	<!--
	Copyright 2020 The Chromium Authors. All rights reserved.
	Use of this source code is governed by a BSD-style license that can be
	found in the LICENSE file.
	-->
	<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
	<title>checks-fetcher tests</title>

	<script src="../bower_components/webcomponentsjs/webcomponents-lite.js"></script>
	<script src="../bower_components/web-component-tester/browser.js"></script>

	<script type="module">
	import './common-test-setup.js';
	import {BuildbucketV2Client} from '../src/main/resources/static/buildbucket-client.js';
	import {ResultDbV1Client} from '../src/main/resources/static/resultdb-client.js';
	import {
	ChecksFetcher,
	CQ_EXPERIMENTAL_TAG,
	} from '../src/main/resources/static/checks-fetcher.js';

	suite('checks-fetcher tests', () => {
	let fetcher;
	let sandbox;
	let client;

	setup(() => {
	sandbox = sinon.sandbox.create();
	sandbox.stub(BuildbucketV2Client.prototype, '_getAccessToken').returns('accessToken');
	sandbox.stub(ResultDbV1Client.prototype, '_getAccessToken').returns('accessToken');
	sandbox.stub(ChecksFetcher.prototype, 'extractProjectFromUrl').returns('foo/bar');
	sandbox.stub(ChecksFetcher.prototype, 'extractHostFromUrl').returns('host.example.com');
	fetcher = new ChecksFetcher(null, 'buildbucket.example.com', 2);
	});

	teardown(() => {
	sandbox.restore();
	});

	test('fetchUnexpectedTestVariants returns an empty list for no TestVariants', async () => {
	const response = {
	nextPageToken: 'foo',
	};
	sandbox.stub(window, 'fetch').returns(Promise.resolve({
	ok: true,
	text: async () => `)]}'${JSON.stringify(response)}`,
	}));

	const build = {
	infra: {
	resultdb: {
	hostname: 'resultdb.example.com',
	invocation: '123456',
	},
	},
	};
	assert.deepEqual(await fetcher.fetchUnexpectedTestVariants(build), []);
	});

	test('fetchUnexpectedTestVariants queries all unexpected TestVariants', async () => {
	const fetch = sandbox.stub(window, 'fetch');

	const firstUnexpectedPage = {
	nextPageToken: 'foo',
	testVariants: Array.from({length: 1000}, () => ({status: 'UNEXPECTED'})),
	};
	fetch.onCall(0).returns(Promise.resolve({
	ok: true,
	text: async () => `)]}'${JSON.stringify(firstUnexpectedPage)}`,
	}));

	const secondUnexpectedPage = {
	nextPageToken: 'bar',
	testVariants: Array.from({length: 1000}, () => ({status: 'FLAKY'})),
	};
	fetch.onCall(1).returns(Promise.resolve({
	ok: true,
	text: async () => `)]}'${JSON.stringify(secondUnexpectedPage)}`,
	}));

	const expectedPage = {
	nextPageToken: 'baz',
	testVariants: Array.from({length: 1000}, () => ({status: 'EXPECTED'})),
	};
	fetch.returns(Promise.resolve({
	ok: true,
	text: async () => `)]}'${JSON.stringify(expectedPage)}`,
	}));

	const build = {
	infra: {
	resultdb: {
	hostname: 'resultdb.example.com',
	invocation: '123456',
	},
	},
	};
	assert.deepEqual(
	await fetcher.fetchUnexpectedTestVariants(build),
	[...firstUnexpectedPage.testVariants, ...secondUnexpectedPage.testVariants],
	);
	sinon.assert.calledThrice(fetch);
	});

	test('convertBuildToRun creates no CheckResults for SCHEDULED builds', () => {
	const build = {
	builder: {builder: 'foo'},
	id: '123456',
	status: 'SCHEDULED',
	};
	const variants = [{testId: 1}];
	assert.deepEqual(
	fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants).results, []);
	});

	test('convertBuildToRun returns only TestVariant CheckResults for SUCCESSful builds', () => {
	const build = {
	builder: {builder: 'foo'},
	id: '123456',
	tags: [],
	status: 'SUCCESS',
	};
	const variantId = '532142';
	const run = fetcher.convertBuildToRun(
	build,
	1,
	{foo: ['123456']},
	[{
	testId: variantId,
	results: [{result: {}}],
	}],
	);
	assert.strictEqual(run.results.length, 1);
	assert.strictEqual(run.results[0].summary, variantId);
	});

	test('convertBuildToRun gets a Buildbucket CheckResult without unexpectedTestVariants', () => {
	const build = {
	builder: {builder: 'foo'},
	infra: {},
	id: '123456',
	tags: [],
	status: 'FAILURE',
	};
	const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, []);
	assert.strictEqual(run.results.length, 1);
	assert.strictEqual(run.results[0].summary, 'Build FAILURE');
	});

	test('convertBuildToRun tags CheckResults for experimental builds', () => {
	const baseBuild = {
	builder: {builder: 'foo'},
	id: '123456',
	tags: [],
	infra: {},
	status: 'FAILED',
	};
	const attempts = {foo: ['123456']};
	// Create one extra TestVariant for the 'more test failed' CheckResult.
	const variants = [
	{
	testId: 0,
	results: [{result: {}}],
	},
	{
	testId: 1,
	results: [{result: {}}],
	},
	{
	testId: 2,
	results: [{result: {}}],
	},
	];

	const expBuild = Object.assign(baseBuild,
	{tags: [{key: 'cq_experimental', value: 'true'}]});
	const expRun = fetcher.convertBuildToRun(expBuild, 1, attempts, variants);
	expRun.results.forEach((result) => {
	assert.isTrue(result.tags.includes(CQ_EXPERIMENTAL_TAG));
	});
	assert.strictEqual(expRun.results[expRun.results.length - 1].category, 'INFO');

	const nonExpBuild = Object.assign(baseBuild,
	{tags: [{key: 'cq_experimental', value: 'false'}]});
	const nonExpRun = fetcher.convertBuildToRun(nonExpBuild, 1, attempts, variants);
	nonExpRun.results.forEach((result) => {
	assert.isFalse(result.tags.includes(CQ_EXPERIMENTAL_TAG));
	});
	assert.strictEqual(nonExpRun.results[nonExpRun.results.length - 1].category, 'ERROR');
	});

	test('convertBuildToRun limits returned CheckResults', () => {
	// Build is SUCCESSful and should only contain TestVariant CheckResults.
	const build = {
	builder: {builder: 'foo'},
	id: '123456',
	tags: [],
	status: 'SUCCESS',
	};
	const variants = [
	{
	testId: 0,
	results: [{result: {}}],
	},
	{
	testId: 1,
	results: [{result: {}}],
	},
	{
	testId: 2,
	results: [{result: {}}],
	},
	{
	testId: 3,
	results: [{result: {}}],
	},
	];

	const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants);
	assert.strictEqual(run.results.length, 3);
	assert.strictEqual(
	run.results[2].message,
	`2 more tests failed`,
	);
	});

	test('convertBuildToRun sorts FLAKY TestVariants', () => {
	// Build is SUCCESSful and should only contain TestVariant CheckResults.
	const build = {
	builder: {builder: 'foo'},
	id: '123456',
	tags: [],
	status: 'SUCCESS',
	};

	const variants = [
	{
	testId: 0,
	status: 'EXONERATED',
	results: [
	{result: {status: 'FAIL', summaryHtml: 'FAIL'}},
	{result: {status: 'CRASH', summaryHtml: 'CRASH'}},
	],
	},
	{
	testId: 1,
	status: 'FLAKY',
	results: [
	{result: {status: 'CRASH', summaryHtml: 'CRASH'}},
	{result: {status: 'FAIL', summaryHtml: 'FAIL'}},
	{result: {status: 'PASS', summaryHtml: 'PASS'}},
	{result: {status: 'FAIL', summaryHtml: 'FAIL'}},
	{result: {status: 'PASS', summaryHtml: 'PASS'}},
	],
	},
	];

	const run = fetcher.convertBuildToRun(build, 1, {foo: ['123456']}, variants);

	const exoneratedResult = run.results[0];
	assert.strictEqual(exoneratedResult.summary, 0);
	assert.strictEqual(exoneratedResult.message, 'FAIL');

	const flakyResult = run.results[1];
	assert.strictEqual(flakyResult.summary, 1);
	assert.strictEqual(flakyResult.message, 'FAIL\nCRASH\nPASS');
	});

	test('fetch experimental builds', async () => {
	const buildResponse = {
	responses: [{
	searchBuilds: {
	builds: [
	{
	builder: {builder: 'foo'},
	id: '123456',
	tags: [{key: 'cq_experimental', value: 'true'}],
	infra: {resultdb: {}, swarming: {}},
	status: 'FAILURE',
	},
	{
	builder: {builder: 'bar'},
	id: '654321',
	tags: [{key: 'cq_experimental', value: 'false'}],
	infra: {resultdb: {}, swarming: {}},
	status: 'FAILURE',
	},
	],
	},
	}],
	};
	sandbox.stub(window, 'fetch').returns(Promise.resolve({
	ok: true,
	text: async () => `)]}'${JSON.stringify(buildResponse)}`,
	}));
	sandbox.stub(ChecksFetcher.prototype, 'fetchUnexpectedTestVariants').returns([]);
	const update = sandbox.stub().returns({});
	const checks = sandbox.stub().returns({announceUpdate: update});
	const plugin = {checks};

	// CheckResult should not contain experimental runs.
	const fetcherWithPlugin = new ChecksFetcher(plugin, 'buildbucket.example.com', 2);
	const nonExpResult = await fetcherWithPlugin.fetch(123456, 1);
	assert.strictEqual(nonExpResult.runs[0].results.length, 1);
	assert.isFalse(nonExpResult.runs[0].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
	assert.strictEqual(nonExpResult.actions[1].name, 'Show Experimental Results');

	// Simulate "Show/Hide Experimental" click.
	nonExpResult.actions[1].callback(1, 1, 1, 1, 'name', 'action');
	sinon.assert.calledOnce(update);

	// CheckResult should contain experimental runs.
	const expResult = await fetcherWithPlugin.fetch(123456, 1);
	assert.strictEqual(expResult.runs.length, 2);
	assert.isTrue(expResult.runs[0].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
	assert.isFalse(expResult.runs[1].results[0].tags.includes(CQ_EXPERIMENTAL_TAG));
	assert.strictEqual(expResult.actions[1].name, 'Hide Experimental Results');
	});
	});
	</script>