web/test/failure-analysis-utils_test.ts

/**
 * @license
 * Copyright 2023 The Chromium Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
import {TagColor} from '@gerritcodereview/typescript-api/checks';
import {FaultAttribute, FaultAttributeProperties} from '../failure-analysis';
import {
  assignFaultAttributesToVariants,
  getFaultAttributeEntryKey,
  getFaultAttributeTagColor,
  getFaultAttributeTagDisplayName,
  getFaultAttributeTagTooltip,
} from '../failure-analysis-utils';
import {assert} from '@open-wc/testing';
import {TestVariant, TestVariantStatus} from '../resultdb-client';
import {Build} from '../checks-fetcher';
import {BuildStatus} from '../buildbucket-client';

suite('failure analysis utils', () => {
  test('getFaultAttributeEntryKey returns a hyphened combination of test variant properties in the correct order', () => {
    const buildId = '12345';
    const buildTarget = 'buildTarget';
    const modelName = 'model';
    const testName = 'some.test';

    const boardAndModelKey = getFaultAttributeEntryKey(
      buildId,
      buildTarget,
      modelName,
      testName
    );
    assert.equal(
      boardAndModelKey,
      `${buildId}-${buildTarget}-${modelName}-${testName}`
    );

    const boardKey = getFaultAttributeEntryKey(
      buildId,
      buildTarget,
      '',
      testName
    );
    assert.equal(boardKey, `${buildId}-${buildTarget}-${testName}`);
  });

  test('assignFaultAttributesToVariants correctly assigns fault attributes to test variants', () => {
    // Set up test data.
    const testVariant1ForBuild1FaultAttribute = {
      comparisonSnapshot: {
        sourceBuildId: '8775275012462334081',
        sourceStartedUnixTimestamp: '1689668613',
      },
      snapshotComparisonFaultAttribution: FaultAttribute.SUCCESS_FOUND,
      testName: 'test1',
      likelyFlaky: true,
    } as FaultAttributeProperties;
    const testVariant2ForBuild1FaultAttribute = {
      comparisonSnapshot: {
        sourceBuildId: '8775275012462334081',
        sourceStartedUnixTimestamp: '1689668613',
      },
      snapshotComparisonFaultAttribution: FaultAttribute.MATCHING_FAILURE_FOUND,
      testName: 'test2',
    } as FaultAttributeProperties;
    const orchBuild1 = {
      id: '1',
      builder: {
        id: {
          builder: 'cq-orchestrator',
        },
        builder: 'cq-orchestrator',
      },
      status: BuildStatus.SUCCESS,
      createTime: '1/1/2023',
      startTime: '1/1/2023',
      endTime: '1/1/2023',
      output: {
        properties: {
          rts_was_used: false,
          cq_fault_attributions: {
            testFailureAttributions: [
              {
                buildTarget: 'target',
                faultAttributes: [
                  testVariant1ForBuild1FaultAttribute,
                  testVariant2ForBuild1FaultAttribute,
                ],
              },
            ],
          },
        },
      },
    } as Build;
    const testVariant1ForBuild2FaultAttribute = {
      comparisonSnapshot: {
        sourceBuildId: '8775275012462334081',
        sourceStartedUnixTimestamp: '1689668613',
      },
      snapshotComparisonFaultAttribution:
        FaultAttribute.DIFFERING_FAILURE_FOUND,
      testName: 'test1',
    } as FaultAttributeProperties;
    const orchBuild2 = {
      id: '2',
      builder: {
        id: {
          builder: 'cq-orchestrator',
        },
        builder: 'cq-orchestrator',
      },
      status: BuildStatus.SUCCESS,
      createTime: '1/1/2023',
      startTime: '1/1/2023',
      endTime: '1/1/2023',
      output: {
        properties: {
          rts_was_used: false,
          cq_fault_attributions: {
            testFailureAttributions: [
              {
                buildTarget: 'target',
                faultAttributes: [testVariant1ForBuild2FaultAttribute],
              },
            ],
          },
        },
      },
    } as Build;
    const testVariant1ForNonOrchBuildFaultAttribute = {
      comparisonSnapshot: {
        sourceBuildId: '8775275012462334081',
        sourceStartedUnixTimestamp: '1689668613',
      },
      snapshotComparisonFaultAttribution: FaultAttribute.MATCHING_FAILURE_FOUND,
      testName: 'test1',
    };
    const nonOrchBuild = {
      id: '3',
      builder: {
        id: {
          builder: 'cq-orchestrator',
        },
        builder: 'cq-orchestrator',
      },
      status: BuildStatus.SUCCESS,
      createTime: '1/1/2023',
      startTime: '1/1/2023',
      endTime: '1/1/2023',
      output: {
        properties: {
          rts_was_used: false,
          cq_fault_attributions: {
            testFailureAttributions: [
              {
                buildTarget: 'target',
                faultAttributes: [testVariant1ForNonOrchBuildFaultAttribute],
              },
            ],
          },
        },
      },
    } as Build;
    const testVariant1ForBuild1 = {
      testId: 'test1',
      variant: {
        def: {
          build_target: 'target',
        },
        results: [],
        exonerations: [],
        status: TestVariantStatus.UNEXPECTED,
      },
    } as TestVariant;
    const testVariant2ForBuild1 = {
      testId: 'test2',
      variant: {
        def: {
          build_target: 'target',
        },
        results: [],
        exonerations: [],
        status: TestVariantStatus.UNEXPECTED,
      },
    } as TestVariant;
    const testVariant1ForBuild2 = {
      testId: 'test1',
      variant: {
        def: {
          build_target: 'target',
        },
        results: [],
        exonerations: [],
        status: TestVariantStatus.UNEXPECTED,
      },
    } as TestVariant;
    const testVariant1ForNonOrchBuild = {
      testId: 'test1',
      variant: {
        def: {
          build_target: 'target',
        },
        results: [],
        exonerations: [],
        status: TestVariantStatus.UNEXPECTED,
      },
    } as TestVariant;
    const tvsByBuilds = {
      '1': {
        variants: [testVariant1ForBuild1, testVariant2ForBuild1],
        errorMessage: '',
      },
      '2': {variants: [testVariant1ForBuild2], errorMessage: ''},
      '3': {variants: [testVariant1ForNonOrchBuild], errorMessage: ''},
    };
    const attemptsByBuilder = {
      'cq-orchestrator': [
        {isQuickRun: false, build: orchBuild1},
        {isQuickRun: false, build: orchBuild2},
      ],
      'non-orch-build': [{isQuickRun: false, build: nonOrchBuild}],
    };

    // Run operation.
    assignFaultAttributesToVariants(tvsByBuilds, attemptsByBuilder);

    // Verify expectations.
    assert.equal(
      testVariant1ForBuild1.faultAttribute,
      testVariant1ForBuild1FaultAttribute
    );
  });

  suite('failure analysis tag properties', () => {
    const flakyFaultAttribute = {
      testName: 'some.test',
      snapshotComparisonFaultAttribution:
        FaultAttribute.DIFFERING_FAILURE_FOUND,
      likelyFlaky: true,
    } as FaultAttributeProperties;
    const newFailureFaultAttribute = {
      testName: 'some.test',
      snapshotComparisonFaultAttribution: FaultAttribute.SUCCESS_FOUND,
    } as FaultAttributeProperties;
    const preexistingDiffFailureFaultAttribute = {
      testName: 'some.test',
      snapshotComparisonFaultAttribution:
        FaultAttribute.DIFFERING_FAILURE_FOUND,
    } as FaultAttributeProperties;
    const preexistingSameFailureFaultAttribute = {
      testName: 'some.test',
      snapshotComparisonFaultAttribution: FaultAttribute.MATCHING_FAILURE_FOUND,
    } as FaultAttributeProperties;
    const anyOtherFaultAttribute = {
      testName: 'some.test',
      snapshotComparisonFaultAttribution: FaultAttribute.NO_COMPARISON,
    } as FaultAttributeProperties;

    test('getFaultAttributeTagDisplayName returns expected display names', () => {
      assert.equal(
        getFaultAttributeTagDisplayName(flakyFaultAttribute),
        'Likely flaky'
      );
      assert.equal(
        getFaultAttributeTagDisplayName(newFailureFaultAttribute),
        'New failure'
      );
      assert.equal(
        getFaultAttributeTagDisplayName(preexistingDiffFailureFaultAttribute),
        'Pre-existing differing failure'
      );
      assert.equal(
        getFaultAttributeTagDisplayName(preexistingSameFailureFaultAttribute),
        'Pre-existing identical failure'
      );
      assert.isNull(getFaultAttributeTagDisplayName(anyOtherFaultAttribute));
    });

    test('getFaultAttributeTagColor returns expected tag colors', () => {
      assert.equal(
        getFaultAttributeTagColor(flakyFaultAttribute),
        TagColor.YELLOW
      );
      assert.equal(
        getFaultAttributeTagColor(newFailureFaultAttribute),
        TagColor.PINK
      );
      assert.equal(
        getFaultAttributeTagColor(preexistingDiffFailureFaultAttribute),
        TagColor.YELLOW
      );
      assert.equal(
        getFaultAttributeTagColor(preexistingSameFailureFaultAttribute),
        TagColor.GRAY
      );
      assert.isNull(getFaultAttributeTagColor(anyOtherFaultAttribute));
    });

    test('getFaultAttributeTagTooltip returns expected tooltip', () => {
      assert.equal(
        getFaultAttributeTagTooltip(flakyFaultAttribute),
        'This test failure is likely flaky'
      );
      assert.equal(
        getFaultAttributeTagTooltip(newFailureFaultAttribute),
        'This test failure is not present in ToT and may need some additional debugging'
      );
      assert.equal(
        getFaultAttributeTagTooltip(preexistingDiffFailureFaultAttribute),
        'This test failure occurred in an earlier snapshot build with a different failure reason'
      );
      assert.equal(
        getFaultAttributeTagTooltip(preexistingSameFailureFaultAttribute),
        'This test failure occurred in an earlier snapshot build with an identical failure reason'
      );
      assert.isNull(getFaultAttributeTagTooltip(anyOtherFaultAttribute));
    });
  });
});