web/failure-analysis.ts - infra/gerrit-plugins/buildbucket - Git at Google

 /**
  * @license
  * Copyright 2023 The Chromium Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 import {css, html, LitElement, TemplateResult} from 'lit';
 import {customElement, property} from 'lit/decorators';
 import {ChecksDB} from './checks-db';
 import {Build} from './checks-fetcher';
 import {BuildStatus} from './buildbucket-client';
 import {compareBuildIds} from './buildbucket-utils';

 export enum FaultAttribute {
   MATCHING_FAILURE_FOUND = 'MATCHING_FAILURE_FOUND',
   DIFFERING_FAILURE_FOUND = 'DIFFERING_FAILURE_FOUND',
   SUCCESS_FOUND = 'SUCCESS_FOUND',
   NO_COMPARISON = 'NO_COMPARISON',
 }

 export declare interface CqFaultAttribution {
   testFailureAttributions: FaultAttributedBuildTarget[];
   comparedSnapshots?: any[];
 }

 export declare interface FaultAttributedBuildTarget {
   buildTarget: string;
   faultAttributes: FaultAttributeProperties[];
   model?: string;
 }

 export declare interface ComparisonSnapshot {
   sourceBuildId: string;
   sourceStartedUnixTimestamp: string;
   sourceCompletedUnixTimestamp?: string;
 }

 export declare interface FaultAttributeProperties {
   testName: string;
   snapshotComparisonFaultAttribution: FaultAttribute;
   attempt?: number;
   likelyFlaky?: boolean;
   comparisonSnapshot?: ComparisonSnapshot;
 }

 interface FailureChip {
   class: string;
   tooltip: string;
   key: FaultAttribute;
   chipText: TemplateResult;
 }

 export class FailureAnalysis {
   cache: ChecksDB;

   intervalId: any | null = null;

   constructor(cache: ChecksDB) {
     this.cache = cache;
   }

   async register(element: FailureAnalysisView) {
     const registerData = async () => {
       await this.loadBuilds(element);
     };

     await registerData();

     if (!element.testFaultAttributionToCount) {
       this.intervalId = setInterval(registerData, 1000);
     }
   }

   async loadBuilds(element: FailureAnalysisView) {
     const builds = await this.cache.getBuilds(
       element.change.project,
       element.change._number,
       Object.values(element.change.revisions).length,
       /* includeAdditionalResults= */ false
     );

     if (!builds) {
       return;
     }

     if (this.intervalId) {
       clearTimeout(this.intervalId!);
     }

     const latestOrch = builds
       .filter(build => build.builder.builder === 'cq-orchestrator')
       .sort((a, b) => -compareBuildIds(a.id, b.id))[0];

     if (!latestOrch?.endTime) {
       return;
     }

     const shouldShowFailureAnalysisRow =
       ![BuildStatus.FAILURE, BuildStatus.INFRA_FAILURE].includes(
         latestOrch.status
       ) || latestOrch.output?.properties?.cq_fault_attributions;
     if (shouldShowFailureAnalysisRow) {
       const faultAttributeCount = this.getFaultAttributeCounts(latestOrch);
       element.testFaultAttributionToCount = faultAttributeCount['testFailures'];
     }
   }

   /**
    * getFaultAttributeCounts returns the corresponding count for the latest
    * attempt of each fault attribute.
    * @param latestOrch Latest, terminal CQ orchestrator build.
    */
   private getFaultAttributeCounts(latestOrch: Build) {
     const faultAttributeCount: any = {
       // TODO(b/273345354): Include 'buildFailures'.
       testFailures: {
         [FaultAttribute.MATCHING_FAILURE_FOUND]: 0,
         [FaultAttribute.DIFFERING_FAILURE_FOUND]: 0,
         [FaultAttribute.SUCCESS_FOUND]: 0,
         [FaultAttribute.NO_COMPARISON]: 0,
       },
     };
     latestOrch.output?.properties?.cq_fault_attributions?.testFailureAttributions.forEach(
       (build_target: any) => {
         const testFaultAttributionMap: Map<
           string,
           {attempt: number; faultAttribute: string}
         > = new Map();
         build_target.faultAttributes.forEach((faultAttribute: any) => {
           const faultAttributeProperties = {
             attempt: faultAttribute.attempt ?? 0,
             faultAttribute: faultAttribute.snapshotComparisonFaultAttribution,
           };
           if (testFaultAttributionMap.has(faultAttribute.testName)) {
             const fa = testFaultAttributionMap.get(faultAttribute.testName);
             if (faultAttribute.attempt > fa!.attempt) {
               testFaultAttributionMap.set(
                 faultAttribute.testName,
                 faultAttributeProperties
               );
             }
           } else {
             testFaultAttributionMap.set(
               faultAttribute.testName,
               faultAttributeProperties
             );
           }
         });

         [...testFaultAttributionMap.values()].forEach(faultAttribute => {
           faultAttributeCount['testFailures'][
             faultAttribute.faultAttribute
           ] += 1;
         });
       }
     );

     return faultAttributeCount;
   }
 }

 /**
  * Failure analysis view for fault attributed build and test failures in the
  * latest terminal orchestrator run.
  */
 @customElement('failure-analysis-view')
 export class FailureAnalysisView extends LitElement {
   change: any;

   @property()
   testFaultAttributionToCount: {[key: string]: number} | null = null;

   static override styles = css`
     gr-icon {
       font-size: var(--line-height-small);
     }
     #main {
       margin-left: var(--spacing-s);
       max-width: 40em;
       display: flex;
       padding-bottom: 8px;
     }
     #key {
       line-height: var(--line-height-small);
       color: var(--deemphasized-text-color);
       padding: var(--spacing-xs) var(--spacing-s) var(--spacing-xs)
         var(--spacing-s);
       white-space: nowrap;
     }
     .failure-analysis-feedback-parent {
       align-items: center;
       display: flex;
       z-index: 10;
       justify-content: center;
     }
     .failure-analysis-feedback-parent a {
       align-items: center;
       display: flex;
       justify-content: center;
     }
     .failure-analysis-feedback {
       cursor: pointer;
       margin-left: var(--spacing-m);
     }
     .no-failures {
       font-style: italic;
       color: #8e8e8e;
     }
     .value {
       align-items: center;
       border-radius: 12px;
       cursor: pointer;
       display: flex;
       font-size: var(--font-size-small);
       line-height: var(--line-height-small);
       margin-right: var(--spacing-s);
       padding: var(--spacing-xxs) var(--spacing-m) var(--spacing-xxs)
         var(--spacing-m);
       width: fit-content;
       word-break: break-word;
     }
     .value.ignorable {
       background-color: var(--disabled-background);
       border: 1px solid var(--disabled-foreground);
     }
     .value.needs-attention {
       background-color: var(--error-background);
       border: 1px solid var(--error-foreground);
     }
     .value.no-comparison {
       background-color: var(--info-background);
       border: 1px solid var(--info-foreground);
     }
     .value.warning {
       background-color: var(--yellow-50);
       border: 1px solid var(--yellow-tonal);
     }
     .value.ignorable span,
     .value.no-comparison span {
       color: black;
     }
     .value.needs-attention span {
       color: var(--error-foreground);
     }
     .value.warning span {
       color: var(--yellow-tonal);
     }
     .value:hover {
       box-shadow: var(--elevation-level-1);
     }
     .value-container {
       display: flex;
       flex: none;
     }
     .value-container a {
       text-decoration: none;
     }
     .value-container .value.hidden {
       display: none;
     }
   `;

   override render() {
     if (!this.testFaultAttributionToCount) {
       return;
     }

     return html`<div id="main">
       <div id="key">
         Failure Analysis
         <gr-tooltip-content
           has-tooltip
           title="CQ failure analysis uses snapshot build results to categorize test failures."
         >
           <gr-icon icon="help_outline"></gr-icon>
         </gr-tooltip-content>
       </div>
       ${this.getChipElements()}
     </div>`;
   }

   override disconnectedCallback() {
     super.disconnectedCallback();
     if (this.testFaultAttributionToCount) {
       this.testFaultAttributionToCount = null;
     }
   }

   private getVisibility(key: FaultAttribute) {
     return this.testFaultAttributionToCount?.[key] === 0 ? 'hidden' : '';
   }

   private getPlural(key: FaultAttribute) {
     if (!this.testFaultAttributionToCount?.[key]) {
       return '';
     }

     return this.testFaultAttributionToCount[key] > 1 ? 's' : '';
   }

   private isAnyFailurePresent() {
     if (!this.testFaultAttributionToCount) {
       return false;
     }

     return Object.values(this.testFaultAttributionToCount).some(
       (val: any) => val > 0
     );
   }

   private isAnyTriagedFailurePresent() {
     if (!this.testFaultAttributionToCount) {
       return false;
     }

     return Object.keys(this.testFaultAttributionToCount)
       .filter((key: any) => key !== FaultAttribute.NO_COMPARISON)
       .some((key: any) => this.testFaultAttributionToCount![key] > 0);
   }

   /**
    * getChipElements returns the HTML content for the fault analysis chips based
    * on counts in testFaultAttributionToCount
    */
   private getChipElements() {
     if (!this.isAnyFailurePresent()) {
       return html`<div class="no-failures">No test failures</div>`;
     }

     const chips: FailureChip[] = [
       {
         class: 'needs-attention',
         tooltip:
           'These failures were absent in the snapshot builds this run was compared to and may need further debugging.',
         key: FaultAttribute.SUCCESS_FOUND,
         chipText: html`${this.testFaultAttributionToCount?.[
           FaultAttribute.SUCCESS_FOUND
         ]}
         new test${this.getPlural(FaultAttribute.SUCCESS_FOUND)}`,
       },
       {
         class: 'warning',
         tooltip:
           'These failures were present in the snapshot build this run was compared to, but with a different failure reason.',
         key: FaultAttribute.DIFFERING_FAILURE_FOUND,
         chipText: html`${this.testFaultAttributionToCount?.[
           FaultAttribute.DIFFERING_FAILURE_FOUND
         ]}
         pre-existing
         test${this.getPlural(FaultAttribute.DIFFERING_FAILURE_FOUND)} | diff
         cause`,
       },
       {
         class: 'ignorable',
         tooltip:
           'These failures were present in the snapshot build this run was compared to, with the same failure reason.',
         key: FaultAttribute.MATCHING_FAILURE_FOUND,
         chipText: html`${this.testFaultAttributionToCount?.[
           FaultAttribute.MATCHING_FAILURE_FOUND
         ]}
         pre-existing
         test${this.getPlural(FaultAttribute.MATCHING_FAILURE_FOUND)} | same
         cause`,
       },
     ];

     const missingComparisonChipTooltip =
       'These failures do not have sufficient recent test history in snapshots and therefore, have not been analyzed.';
     let missingComparisonChipText = html`${this.testFaultAttributionToCount?.[
       FaultAttribute.NO_COMPARISON
     ]}
     test${this.getPlural(FaultAttribute.NO_COMPARISON)} missing analysis`;

     if (this.isAnyTriagedFailurePresent()) {
       missingComparisonChipText = html`+
       ${this.testFaultAttributionToCount?.[FaultAttribute.NO_COMPARISON]}`;
     }
     // TODO (b/294747944): Preserve other param values and only set "tab".
     const missingComparisonChip = html`<a href="?tab=checks">
       <div
         class="value no-comparison ${this.getVisibility(
           FaultAttribute.NO_COMPARISON
         )}"
       >
         <gr-tooltip-content has-tooltip title="${missingComparisonChipTooltip}">
           <span> ${missingComparisonChipText} </span>
         </gr-tooltip-content>
       </div>
     </a>`;

     return html`<div class="value-container">
       ${chips.map(
         element =>
           // TODO (b/294747944): Preserve other param values and only set "tab".
           html`<a href="?tab=checks">
             <div
               class="value ${element.class} ${this.getVisibility(element.key)}"
             >
               <gr-tooltip-content has-tooltip title="${element.tooltip}">
                 <span>${element.chipText}</span>
               </gr-tooltip-content>
             </div>
           </a>`
       )}
       ${missingComparisonChip}
       <div class="failure-analysis-feedback-parent">
         <a
           target="_blank"
           href="
 https://buganizer.corp.google.com/issues/new?component=1315651&template=1849636"
         >
           <gr-icon class="failure-analysis-feedback" icon="feedback"></gr-icon>
         </a>
       </div>
     </div>`;
   }
 }
	/**
	* @license
	* Copyright 2023 The Chromium Authors. All rights reserved.
	* Use of this source code is governed by a BSD-style license that can be
	* found in the LICENSE file.
	*/
	import {css, html, LitElement, TemplateResult} from 'lit';
	import {customElement, property} from 'lit/decorators';
	import {ChecksDB} from './checks-db';
	import {Build} from './checks-fetcher';
	import {BuildStatus} from './buildbucket-client';
	import {compareBuildIds} from './buildbucket-utils';

	export enum FaultAttribute {
	MATCHING_FAILURE_FOUND = 'MATCHING_FAILURE_FOUND',
	DIFFERING_FAILURE_FOUND = 'DIFFERING_FAILURE_FOUND',
	SUCCESS_FOUND = 'SUCCESS_FOUND',
	NO_COMPARISON = 'NO_COMPARISON',
	}

	export declare interface CqFaultAttribution {
	testFailureAttributions: FaultAttributedBuildTarget[];
	comparedSnapshots?: any[];
	}

	export declare interface FaultAttributedBuildTarget {
	buildTarget: string;
	faultAttributes: FaultAttributeProperties[];
	model?: string;
	}

	export declare interface ComparisonSnapshot {
	sourceBuildId: string;
	sourceStartedUnixTimestamp: string;
	sourceCompletedUnixTimestamp?: string;
	}

	export declare interface FaultAttributeProperties {
	testName: string;
	snapshotComparisonFaultAttribution: FaultAttribute;
	attempt?: number;
	likelyFlaky?: boolean;
	comparisonSnapshot?: ComparisonSnapshot;
	}

	interface FailureChip {
	class: string;
	tooltip: string;
	key: FaultAttribute;
	chipText: TemplateResult;
	}

	export class FailureAnalysis {
	cache: ChecksDB;

	intervalId: any \| null = null;

	constructor(cache: ChecksDB) {
	this.cache = cache;
	}

	async register(element: FailureAnalysisView) {
	const registerData = async () => {
	await this.loadBuilds(element);
	};

	await registerData();

	if (!element.testFaultAttributionToCount) {
	this.intervalId = setInterval(registerData, 1000);
	}
	}

	async loadBuilds(element: FailureAnalysisView) {
	const builds = await this.cache.getBuilds(
	element.change.project,
	element.change._number,
	Object.values(element.change.revisions).length,
	/* includeAdditionalResults= */ false
	);

	if (!builds) {
	return;
	}

	if (this.intervalId) {
	clearTimeout(this.intervalId!);
	}

	const latestOrch = builds
	.filter(build => build.builder.builder === 'cq-orchestrator')
	.sort((a, b) => -compareBuildIds(a.id, b.id))[0];

	if (!latestOrch?.endTime) {
	return;
	}

	const shouldShowFailureAnalysisRow =
	![BuildStatus.FAILURE, BuildStatus.INFRA_FAILURE].includes(
	latestOrch.status
	) \|\| latestOrch.output?.properties?.cq_fault_attributions;
	if (shouldShowFailureAnalysisRow) {
	const faultAttributeCount = this.getFaultAttributeCounts(latestOrch);
	element.testFaultAttributionToCount = faultAttributeCount['testFailures'];
	}
	}

	/**
	* getFaultAttributeCounts returns the corresponding count for the latest
	* attempt of each fault attribute.
	* @param latestOrch Latest, terminal CQ orchestrator build.
	*/
	private getFaultAttributeCounts(latestOrch: Build) {
	const faultAttributeCount: any = {
	// TODO(b/273345354): Include 'buildFailures'.
	testFailures: {
	[FaultAttribute.MATCHING_FAILURE_FOUND]: 0,
	[FaultAttribute.DIFFERING_FAILURE_FOUND]: 0,
	[FaultAttribute.SUCCESS_FOUND]: 0,
	[FaultAttribute.NO_COMPARISON]: 0,
	},
	};
	latestOrch.output?.properties?.cq_fault_attributions?.testFailureAttributions.forEach(
	(build_target: any) => {
	const testFaultAttributionMap: Map<
	string,
	{attempt: number; faultAttribute: string}
	> = new Map();
	build_target.faultAttributes.forEach((faultAttribute: any) => {
	const faultAttributeProperties = {
	attempt: faultAttribute.attempt ?? 0,
	faultAttribute: faultAttribute.snapshotComparisonFaultAttribution,
	};
	if (testFaultAttributionMap.has(faultAttribute.testName)) {
	const fa = testFaultAttributionMap.get(faultAttribute.testName);
	if (faultAttribute.attempt > fa!.attempt) {
	testFaultAttributionMap.set(
	faultAttribute.testName,
	faultAttributeProperties
	);
	}
	} else {
	testFaultAttributionMap.set(
	faultAttribute.testName,
	faultAttributeProperties
	);
	}
	});

	[...testFaultAttributionMap.values()].forEach(faultAttribute => {
	faultAttributeCount['testFailures'][
	faultAttribute.faultAttribute
	] += 1;
	});
	}
	);

	return faultAttributeCount;
	}
	}

	/**
	* Failure analysis view for fault attributed build and test failures in the
	* latest terminal orchestrator run.
	*/
	@customElement('failure-analysis-view')
	export class FailureAnalysisView extends LitElement {
	change: any;

	@property()
	testFaultAttributionToCount: {[key: string]: number} \| null = null;

	static override styles = css`
	gr-icon {
	font-size: var(--line-height-small);
	}
	#main {
	margin-left: var(--spacing-s);
	max-width: 40em;
	display: flex;
	padding-bottom: 8px;
	}
	#key {
	line-height: var(--line-height-small);
	color: var(--deemphasized-text-color);
	padding: var(--spacing-xs) var(--spacing-s) var(--spacing-xs)
	var(--spacing-s);
	white-space: nowrap;
	}
	.failure-analysis-feedback-parent {
	align-items: center;
	display: flex;
	z-index: 10;
	justify-content: center;
	}
	.failure-analysis-feedback-parent a {
	align-items: center;
	display: flex;
	justify-content: center;
	}
	.failure-analysis-feedback {
	cursor: pointer;
	margin-left: var(--spacing-m);
	}
	.no-failures {
	font-style: italic;
	color: #8e8e8e;
	}
	.value {
	align-items: center;
	border-radius: 12px;
	cursor: pointer;
	display: flex;
	font-size: var(--font-size-small);
	line-height: var(--line-height-small);
	margin-right: var(--spacing-s);
	padding: var(--spacing-xxs) var(--spacing-m) var(--spacing-xxs)
	var(--spacing-m);
	width: fit-content;
	word-break: break-word;
	}
	.value.ignorable {
	background-color: var(--disabled-background);
	border: 1px solid var(--disabled-foreground);
	}
	.value.needs-attention {
	background-color: var(--error-background);
	border: 1px solid var(--error-foreground);
	}
	.value.no-comparison {
	background-color: var(--info-background);
	border: 1px solid var(--info-foreground);
	}
	.value.warning {
	background-color: var(--yellow-50);
	border: 1px solid var(--yellow-tonal);
	}
	.value.ignorable span,
	.value.no-comparison span {
	color: black;
	}
	.value.needs-attention span {
	color: var(--error-foreground);
	}
	.value.warning span {
	color: var(--yellow-tonal);
	}
	.value:hover {
	box-shadow: var(--elevation-level-1);
	}
	.value-container {
	display: flex;
	flex: none;
	}
	.value-container a {
	text-decoration: none;
	}
	.value-container .value.hidden {
	display: none;
	}
	`;

	override render() {
	if (!this.testFaultAttributionToCount) {
	return;
	}

	return html`<div id="main">
	<div id="key">
	Failure Analysis
	<gr-tooltip-content
	has-tooltip
	title="CQ failure analysis uses snapshot build results to categorize test failures."
	>
	<gr-icon icon="help_outline"></gr-icon>
	</gr-tooltip-content>
	</div>
	${this.getChipElements()}
	</div>`;
	}

	override disconnectedCallback() {
	super.disconnectedCallback();
	if (this.testFaultAttributionToCount) {
	this.testFaultAttributionToCount = null;
	}
	}

	private getVisibility(key: FaultAttribute) {
	return this.testFaultAttributionToCount?.[key] === 0 ? 'hidden' : '';
	}

	private getPlural(key: FaultAttribute) {
	if (!this.testFaultAttributionToCount?.[key]) {
	return '';
	}

	return this.testFaultAttributionToCount[key] > 1 ? 's' : '';
	}

	private isAnyFailurePresent() {
	if (!this.testFaultAttributionToCount) {
	return false;
	}

	return Object.values(this.testFaultAttributionToCount).some(
	(val: any) => val > 0
	);
	}

	private isAnyTriagedFailurePresent() {
	if (!this.testFaultAttributionToCount) {
	return false;
	}

	return Object.keys(this.testFaultAttributionToCount)
	.filter((key: any) => key !== FaultAttribute.NO_COMPARISON)
	.some((key: any) => this.testFaultAttributionToCount![key] > 0);
	}

	/**
	* getChipElements returns the HTML content for the fault analysis chips based
	* on counts in testFaultAttributionToCount
	*/
	private getChipElements() {
	if (!this.isAnyFailurePresent()) {
	return html`<div class="no-failures">No test failures</div>`;
	}

	const chips: FailureChip[] = [
	{
	class: 'needs-attention',
	tooltip:
	'These failures were absent in the snapshot builds this run was compared to and may need further debugging.',
	key: FaultAttribute.SUCCESS_FOUND,
	chipText: html`${this.testFaultAttributionToCount?.[
	FaultAttribute.SUCCESS_FOUND
	]}
	new test${this.getPlural(FaultAttribute.SUCCESS_FOUND)}`,
	},
	{
	class: 'warning',
	tooltip:
	'These failures were present in the snapshot build this run was compared to, but with a different failure reason.',
	key: FaultAttribute.DIFFERING_FAILURE_FOUND,
	chipText: html`${this.testFaultAttributionToCount?.[
	FaultAttribute.DIFFERING_FAILURE_FOUND
	]}
	pre-existing
	test${this.getPlural(FaultAttribute.DIFFERING_FAILURE_FOUND)} \| diff
	cause`,
	},
	{
	class: 'ignorable',
	tooltip:
	'These failures were present in the snapshot build this run was compared to, with the same failure reason.',
	key: FaultAttribute.MATCHING_FAILURE_FOUND,
	chipText: html`${this.testFaultAttributionToCount?.[
	FaultAttribute.MATCHING_FAILURE_FOUND
	]}
	pre-existing
	test${this.getPlural(FaultAttribute.MATCHING_FAILURE_FOUND)} \| same
	cause`,
	},
	];

	const missingComparisonChipTooltip =
	'These failures do not have sufficient recent test history in snapshots and therefore, have not been analyzed.';
	let missingComparisonChipText = html`${this.testFaultAttributionToCount?.[
	FaultAttribute.NO_COMPARISON
	]}
	test${this.getPlural(FaultAttribute.NO_COMPARISON)} missing analysis`;

	if (this.isAnyTriagedFailurePresent()) {
	missingComparisonChipText = html`+
	${this.testFaultAttributionToCount?.[FaultAttribute.NO_COMPARISON]}`;
	}
	// TODO (b/294747944): Preserve other param values and only set "tab".
	const missingComparisonChip = html`<a href="?tab=checks">
	<div
	class="value no-comparison ${this.getVisibility(
	FaultAttribute.NO_COMPARISON
	)}"
	>
	<gr-tooltip-content has-tooltip title="${missingComparisonChipTooltip}">
	<span> ${missingComparisonChipText} </span>
	</gr-tooltip-content>
	</div>
	</a>`;

	return html`<div class="value-container">
	${chips.map(
	element =>
	// TODO (b/294747944): Preserve other param values and only set "tab".
	html`<a href="?tab=checks">
	<div
	class="value ${element.class} ${this.getVisibility(element.key)}"
	>
	<gr-tooltip-content has-tooltip title="${element.tooltip}">
	<span>${element.chipText}</span>
	</gr-tooltip-content>
	</div>
	</a>`
	)}
	${missingComparisonChip}
	<div class="failure-analysis-feedback-parent">
	<a
	target="_blank"
	href="
	https://buganizer.corp.google.com/issues/new?component=1315651&template=1849636"
	>
	<gr-icon class="failure-analysis-feedback" icon="feedback"></gr-icon>
	</a>
	</div>
	</div>`;
	}
	}