web/checks-fetcher.ts

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import {AuthorizationHeader, getAuthorizationHeader} from './auth';
import {
  BuildbucketV2Client,
  makeBuildRequests,
  retry,
  BuildStatus,
  DEFAULT_UPDATE_INTERVAL_MS,
  MAX_UPDATE_INTERVAL_MS,
  RETRY_FAILED_TAG,
  Builder,
  BuildRequests,
  IndividualBatchResponse,
  RequestStatus,
  SearchBuildsRequest,
  SearchBuildsResponse,
} from './buildbucket-client';
import {
  createBuilderLink,
  createBuildUrl,
  createBuildResourceName,
  compareBuildIds,
  getDateFromTimestamp,
  getNewOperationId,
  hasHideInGerritTag,
  hasHideTestResultsInGerritTag,
  isExperimental,
  getRetryBuilders,
  splitPatchsetGroups,
  shouldSkipRetry,
  showChooseTryjobs,
  showRetryFailed,
} from './buildbucket-utils';
import {ChecksDB} from './checks-db';
import {CqFaultAttribution, FaultAttribute} from './failure-analysis';
import {
  assignFaultAttributesToVariants,
  getFaultAttributeTagColor,
  getFaultAttributeTagDisplayName,
  getFaultAttributeTagTooltip,
} from './failure-analysis-utils';
import {
  Artifact,
  ResultDbV1Client,
  TestStatus,
  TestVariant,
  TestVariantStatus,
} from './resultdb-client';
import {
  createArtifactLink,
  createLayoutResultLink,
  createTestTaskLink,
  createVariantName,
  createVariantLink,
} from './resultdb-utils';
import {openTryjobPicker} from './cr-tryjob-picker';
import {PluginApi} from '@gerritcodereview/typescript-api/plugin';
import {
  Action,
  Category,
  ChangeData,
  CheckResult,
  Link,
  LinkIcon,
  ResponseCode,
  Tag,
  TagColor,
  RunStatus,
  CheckRun,
} from '@gerritcodereview/typescript-api/checks';
import {ChangeInfo} from '@gerritcodereview/typescript-api/rest-api';

// The field is used to store arbitrary data in a CheckResult.
export const DATA_SYMBOL = Symbol('chromeData');

// An mapping of Buildbucket build statuses to adjectives. Used to create
// CheckResult summaries.
const BUILD_STATUS_TO_ADJECTIVE = {
  [BuildStatus.SCHEDULED]: 'scheduled',
  [BuildStatus.STARTED]: 'started',
  [BuildStatus.SUCCESS]: 'succeeded',
  [BuildStatus.FAILURE]: 'failed',
  [BuildStatus.INFRA_FAILURE]: 'infra failed',
  [BuildStatus.CANCELED]: 'was canceled',
};

const INFRA_FAILURE_ICON = '🟪';
const QUICK_RUN_ICON = '🚀';

// The space to use between the icons above and regular text. This is equivalent
// to two four-per-em spaces and works well between Chrome on Linux and Mac.
const ICON_SPACE = '\u2005\u2005';

export declare interface Build {
  id: string;
  builder: Builder;
  number?: number;
  tags?: {
    key: string;
    value: string;
  }[];
  status: BuildStatus;
  critical?: string;
  createTime: string;
  startTime: string;
  endTime: string;
  summaryMarkdown?: string;
  infra?: {
    resultdb?: {
      invocation: string;
      hostname: string;
    };
  };
  input?: {experiments: string[]};
  output?: {
    properties?: {
      rts_was_used: boolean;
      cq_fault_attributions?: CqFaultAttribution;
    };
  };
  ancestorIds?: string[];
  retriable?: string;
}

export declare interface Config {
  buckets?: {
    name: string;
    builders?: string[];
  }[];
  hideRetryButton?: boolean;
  gerritHost?: string;
  gitHost?: string;
}

export declare interface Attempt {
  isQuickRun: boolean;
  build: Build;
}

export class ChecksFetcher {
  plugin: PluginApi;

  config: Config | null;

  buildbucketHost: string;

  maxVariantsPerBuild: number;

  includeAdditionalResults: boolean;

  retryEnabled: boolean;

  controllers: Map<string, AbortController>;

  fetchBuildsStarted: boolean;

  cache: ChecksDB;

  changeData: ChangeData | null;

  refreshFrequency: number;

  constructor(
    plugin: PluginApi,
    buildbucketHost: string,
    maxVariantsPerBuild: number,
    cache: ChecksDB,
    refreshFrequency: number
  ) {
    this.plugin = plugin;
    this.config = null;
    this.buildbucketHost = buildbucketHost;
    this.maxVariantsPerBuild = maxVariantsPerBuild;
    this.includeAdditionalResults = false;
    this.retryEnabled = false;
    this.fetchBuildsStarted = false;
    this.cache = cache;

    // Maps a change and patchset to an AbortController for aborting pending
    // requests.
    this.controllers = new Map();

    this.changeData = null;

    // How often (in seconds) we should poll for for udpated data from
    // buildbucket and resultdb.
    this.refreshFrequency = refreshFrequency;
  }

  async remoteFetch(refresh: boolean) {
    if (this.changeData == null) {
      throw new Error('changeData must be set before remoteFetch() is called');
    }
    this.fetchBuildsStarted = true;
    const changeId = this.getChangeDataId(this.changeData);

    // Abort any unfinished requests for this changeData. If this is the
    // first fetch, don't abort. Otherwise, abort using old controller.
    if (this.controllers.has(changeId)) {
      this.controllers.get(changeId)?.abort();
    }

    const controller = new AbortController();
    this.controllers.set(changeId, controller);

    const {changeNumber, patchsetNumber, repo, changeInfo} = this.changeData;
    // Query BuildBucket for build info.
    const builds = await this.fetchDisplayedBuilds(
      changeNumber,
      patchsetNumber,
      repo,
      changeInfo,
      controller
    );
    if (!controller?.signal.aborted) {
      builds.sort((a, b) => -compareBuildIds(a.id, b.id));

      // Query ResultDB for TestVariants.
      const rdbResults = await Promise.allSettled(
        builds.map(build =>
          this.fetchTestVariants(
            build,
            this.includeAdditionalResults,
            changeNumber
          )
        )
      );
      await this.cache.storeBuilds(
        builds,
        repo,
        changeNumber,
        patchsetNumber,
        this.includeAdditionalResults
      );
      for (const [i, build] of builds.entries()) {
        const variant = rdbResults[i];
        if (variant.status === 'rejected') {
          console.warn(
            `Failed to query ResultDB for build ${build.id}: ${variant.reason}`
          );
        }
        await this.cache.storeTestVariants(
          (variant as PromiseFulfilledResult<TestVariant[]>).value,
          build.id,
          this.includeAdditionalResults,
          variant.status == 'rejected' ? variant.reason : ''
        );
      }
    }

    // Tell plugin to update with latest builds stored in IndexedDB and
    // set timeout for when we should next fetch for latest results.
    this.plugin.checks().announceUpdate();
    if (refresh) {
      setTimeout(() => {
        this.remoteFetch(refresh);
      }, this.refreshFrequency * 1000);
    }
  }

  async fetch(changeData: ChangeData) {
    const {changeNumber, patchsetNumber, repo, changeInfo} = changeData;

    if (!this.config) {
      const pluginName = encodeURIComponent(this.plugin.getPluginName());
      const config: Config = await this.plugin
        .restApi()
        .get(`/projects/${encodeURIComponent(repo)}/${pluginName}~config`);
      if (!config) {
        console.info('Buildbucket plugin not configured for this project.');
        return {responseCode: ResponseCode.OK};
      }
      this.config = config;
    }

    // Start remote fetching of builds in background if it has not already started for this
    // fetcher.
    this.changeData = changeData;
    if (!this.fetchBuildsStarted) {
      this.remoteFetch(true);
    }

    const builds = await this.cache.getBuilds(
      repo,
      changeNumber,
      patchsetNumber,
      this.includeAdditionalResults
    );

    const latestPatchset = Object.keys(changeInfo.revisions as {}).length;
    let runs: CheckRun[] = [];
    if (builds === undefined) {
      runs.push({
        change: changeNumber,
        patchset: patchsetNumber,
        checkName: 'Fetching results...',
        status: RunStatus.COMPLETED,
        results: [
          {
            category: Category.WARNING,
            summary: 'Fetching results...',
          },
        ],
      });
    } else {
      const allAttempts = this.createAttemptList(builds);
      runs = await this.convertAttemptsToRuns(
        patchsetNumber,
        changeNumber,
        repo,
        latestPatchset,
        allAttempts
      );
    }

    runs.sort((a, b) => {
      const aName = this.sortName(a.checkName);
      const bName = this.sortName(b.checkName);
      if (aName < bName) {
        return -1;
      }
      if (aName > bName) {
        return 1;
      }
      return 0;
    });

    const accessToken = await this.getAuthorizationHeader(changeNumber);
    const loggedIn = Object.keys(accessToken).length !== 0;
    this.retryEnabled = showRetryFailed(this.config, loggedIn, changeInfo);

    const actions = [];
    if (showChooseTryjobs(this.config, loggedIn, changeInfo)) {
      actions.push({
        name: 'Choose Tryjobs',
        primary: true,
        summary: true,
        tooltip: 'Select specific builders to run on the latest patchset',
        callback: (change: number) =>
          this.chooseTryjobsCallback(change, latestPatchset, repo),
      });
    }

    if (this.retryEnabled) {
      // We delay checking failed build length since checks 'retry failed'
      // queries all builds from all patchsets and could significantly impact
      // loading time.
      actions.push({
        name: 'Retry Failed Builds',
        primary: true,
        summary: false,
        tooltip: 'Retry any failed builds from all patchsets',
        callback: (change: number) =>
          this.retryFailedBuildsCallback(change, latestPatchset, repo),
      });
    }

    // Set primary for 'Additional Results' and 'Give Feedback' if none of
    // 'Choose Tryjobs' or 'Retry Failed' are added as buttons.
    const primary = actions.length === 0;
    actions.push(
      {
        name:
          `${this.includeAdditionalResults ? 'Hide' : 'Show'} ` +
          'Additional Results',
        primary,
        summary: false,
        tooltip: 'Display experimental, flaky, etc. results',
        callback: () => this.filterAddlResultsCallback(),
      },
      {
        name: 'Give Feedback',
        primary,
        summary: false,
        tooltip: 'Tell us what you think about Checks!',
        // WARNING: window.open is safe only if the arguments are hard-coded
        // and unchanged.
        callback: () => window.open('https://forms.gle/1wP6uGSdyKiiWHY38'),
      }
    );
    return {
      responseCode: ResponseCode.OK,
      actions,
      runs,
      links: [
        {
          url:
            'https://bugs.chromium.org/p/chromium/issues/entry' +
            '?components=Infra%3ELUCI%3EBuildService%3EPreSubmit%3EGerrit',
          tooltip: 'File a bug',
          primary: true,
          icon: LinkIcon.REPORT_BUG,
        },
      ],
    };
  }

  /**
   * Returns Buildbucket builds to be displayed in the UI.
   *
   * Compared to fetchBuilds, this method handles fetching equivalent patchsets.
   */
  async fetchDisplayedBuilds(
    change: number,
    patchset: number,
    project: string,
    changeInfo: ChangeInfo,
    controller: AbortController
  ): Promise<Build[]> {
    let patchsets: number[] = [];
    const groups = splitPatchsetGroups(changeInfo);

    for (const group of groups) {
      if (!group.has(patchset)) {
        continue;
      }
      // Builds for the patchsets greater than patchNum are not included,
      // even if they're in the same equivalent group. Why? This matches
      // the historical behavior of the buildbucket plugin.
      patchsets = Array.from(group)
        .filter(n => n <= patchset)
        .sort();
      break;
    }

    if (patchsets.length === 0) {
      return [];
    }

    // Get builds for the equivalent patchsets.
    const builds = await this.fetchBuilds(
      change,
      patchsets,
      project,
      this.includeAdditionalResults,
      controller
    );
    if (builds.length > 0 || changeInfo.status !== 'MERGED') {
      return builds;
    }

    // If the submit strategy is "Rebase Always" (such as for Chromium), then
    // the last patchset of the merged CL is autogenerated, and it may be
    // labeled as "REWORK" even though it's actually a trivial rebase; in
    // this case we try again with the previous group of patchsets.
    if (
      groups.length >= 2 &&
      groups[0].size === 1 &&
      patchsets.length === 1 &&
      groups[0].has(patchsets[0])
    ) {
      const previousPatchsets = Array.from(groups[1]);
      return await this.fetchBuilds(
        change,
        previousPatchsets,
        project,
        this.includeAdditionalResults,
        controller
      );
    }

    // If we reach this point, we're in a merged change with no builds to
    // display, but the last patchset doesn't appear to be a special
    // autogenerated patchset.
    return [];
  }

  /**
   * Returns Buildbucket builds for the given change and patchsets.
   */
  async fetchBuilds(
    change: number,
    patchsets: number[],
    project: string,
    inclExp: boolean,
    controller: AbortController
  ): Promise<Build[]> {
    const builds: Build[] = [];
    const fields = [
      'id',
      'builder',
      'number',
      'tags',
      'status',
      'critical',
      'createTime',
      'startTime',
      'endTime',
      'summaryMarkdown',
      'infra.resultdb',
      'ancestorIds',
      'retriable',
    ].join(',');

    let requests: SearchBuildsRequest[] = patchsets.map(patchset => {
      return {
        searchBuilds: {
          pageSize: 1000,
          predicate: {
            includeExperimental: inclExp,
            gerritChanges: [
              {
                host: this.config?.gerritHost,
                project,
                change,
                patchset,
              },
            ],
          },
          mask: {
            fields,
            outputProperties: [
              {path: ['rts_was_used']},
              {path: ['cq_fault_attributions']},
            ],
          },
        },
      };
    });

    const bbClient = new BuildbucketV2Client(
      this.buildbucketHost,
      String(change)
    );
    while (requests.length > 0 && !controller?.signal.aborted) {
      const batch = async () =>
        await bbClient.batch({requests}, controller?.signal);
      const {responses} = await retry(
        batch,
        DEFAULT_UPDATE_INTERVAL_MS,
        MAX_UPDATE_INTERVAL_MS,
        3,
        2
      );

      const newRequests: SearchBuildsRequest[] = [];
      (responses || []).forEach((response: IndividualBatchResponse, i: number) => {
        if (response.hasOwnProperty('error')) {
          const error = response['error'] as RequestStatus;
          console.error(
            'Buildbucket request failed with error code ' +
              `${error.code}: ${error.message}`
          );
          return;
        }

        const searchBuilds = response['searchBuilds'] as SearchBuildsResponse;
        builds.push(...(searchBuilds.builds || []));

        // If there are more builds to fetch, add the request to be re-fetched.
        if (searchBuilds.nextPageToken) {
          requests[i].searchBuilds.pageToken = searchBuilds.nextPageToken;
          newRequests.push(requests[i]);
        }
      });
      requests = newRequests;
    }

    const visibleBuilds = builds.filter(build => !hasHideInGerritTag(build));
    if (inclExp) {
      return visibleBuilds;
    }
    return visibleBuilds.filter(build => !isExperimental(build));
  }

  /**
   * Returns TestVariants for the given build. If inclAdditional is true, also
   * fetch results for SUCCESSful builds and exonerated, flaky, or skipped
   * results.
   *
   * If a build is tagged with 'hide-test-results-in-gerrit', ResultDB will not
   * be queried.
   */
  async fetchTestVariants(
    build: Build,
    inclAdditional: boolean,
    changeId: number
  ): Promise<TestVariant[]> {
    // Don't query ResultDB for successful builds.
    if (
      (build.status === BuildStatus.SUCCESS && !inclAdditional) ||
      !build?.infra?.resultdb?.invocation ||
      hasHideTestResultsInGerritTag(build)
    ) {
      return [];
    }

    const client = new ResultDbV1Client(
      build.infra.resultdb.hostname,
      String(changeId)
    );
    return await client.fetchTestVariants(
      build,
      this.maxVariantsPerBuild,
      inclAdditional
    );
  }

  /**
   * Returns an array of attempts normalized against ancestors.
   *
   * If builds have ancestors, we want to normalize their attempts
   * with their ancestor attempts.
   * Child attempts [c1, c2] may map to to a1, a3 of ancestor attempts
   * [a1, a2, a3] and should look like [c1, undefined, c2].
   */
  normalizeAttempts(
    buildsByID: Map<string, Build>,
    attemptsByBuilder: {[key: string]: Attempt[]},
    attempts: Attempt[]
  ): (Attempt | undefined)[] {
    const firstAttempt = attempts[0];
    if (!firstAttempt.build.ancestorIds?.length) {
      return attempts;
    }

    let normalizeComplete = true;

    const ancestorId: string = firstAttempt.build.ancestorIds[0].toString();
    // A build can have several ancestorIds. All ancestors of any build should be
    // associated with the same change and patchset and therefore present in buildsByID.
    // `ancestorIds` is ordered top-to-bottom, making the first value the build's
    // root ancestor.
    // https://source.chromium.org/chromium/infra/infra/+/main:recipes-py/recipe_proto/go.chromium.org/luci/buildbucket/proto/build.proto;l=321;drc=428eeaebf2e1de58ed1c7cae50daba3384d730cd
    const ancestorBuilder = buildsByID.get(ancestorId)?.builder.builder;
    const ancestorAttempts: Attempt[] | undefined =
      attemptsByBuilder[ancestorBuilder as string];

    // If the user has read access in the realm of a child build but not its
    // parent, then ancestorAttempts is undefined.
    if (
      ancestorAttempts === undefined ||
      ancestorAttempts.length === attempts.length
    ) {
      return attempts;
    }

    const ancestorAttemptIds = ancestorAttempts.map(
      attempt => attempt.build.id
    );
    const normalizedAttempts: (Attempt | undefined)[] = ancestorAttemptIds.map(
      () => undefined
    );

    let attemptsSlice: undefined | number;
    let slicedAttempts = attempts;
    for (const [i, attempt] of attempts.entries()) {
      const currentAncestorId: string = attempt.build
        .ancestorIds?.[0] as string;

      // If currentAncestorId is not in buildsByID, then it was removed
      // because there are earlier attempts we want to show that are more relevant.
      // Any child builds of this ancestor should also not be shown. Do not add
      // them to the `normalizedAttempts` and remove them from `attempts`.
      if (!buildsByID.has(currentAncestorId)) {
        attemptsSlice = i;
        break;
      }

      // We expect all build attempts for any builder have the same ancestor builders.
      // But config changes might have occured and ancestors may have changes.
      if (
        buildsByID.get(currentAncestorId)?.builder.builder !== ancestorBuilder
      ) {
        normalizeComplete = false;
        break;
      }

      const ancestorI = ancestorAttemptIds.indexOf(currentAncestorId);
      normalizedAttempts[ancestorI] = attempt;
    }
    if (attemptsSlice !== undefined) {
      slicedAttempts = attempts.slice(0, attemptsSlice);
    }

    if (!normalizeComplete) {
      return attempts;
    }
    // If we have no results for the latest attempt, the ancestor build is in progress
    // and has either:
    // (1) not kicked off an attempt for this builder yet OR
    // (2) will NEVER kick off a an attempt for this builder because it decided this
    // builder does not matter anymore.
    // For both cases if the latest actual status we have of the builder is NOT a success,
    // we want to hide the result in the checks UI summary view by making the latest
    // attempt 'undefined'/NOT_RUN.

    // If the latest actual status we have for this builder is a success we do not want
    // to hide it, so we clear all the 'undefined' attempts after it.

    if (
      normalizedAttempts[normalizedAttempts.length - 1] === undefined &&
      slicedAttempts[slicedAttempts.length - 1].build.status ===
        BuildStatus.SUCCESS
    ) {
      let i = normalizedAttempts.length - 1;
      while (normalizedAttempts[i] === undefined) {
        i--;
      }
      return normalizedAttempts.slice(0, i + 1);
    }
    return normalizedAttempts;
  }

  /**
   * Converts a Buildbucket `Build` object to a Reboot Checks API `Run` object.
   */
  async convertAttemptsToRuns(
    currPatchset: number,
    changeNumber: number,
    repo: string,
    latestPatchset: number,
    attemptsByBuilder: {[key: string]: Attempt[]}
  ): Promise<CheckRun[]> {
    const runs: CheckRun[] = [];
    let incompleteTestVariantResults = false;

    const hiddenRootBuilders: {[key: string]: Attempt[]} = {};
    for (const [builder, builderAttempts] of Object.entries(
      attemptsByBuilder
    )) {
      let i = builderAttempts.length - 1;

      // For root builders we want to throw away scheduled/started if there are
      // earlier attempts still running.
      if (
        !builderAttempts[i].build.ancestorIds &&
        [BuildStatus.SCHEDULED, BuildStatus.STARTED].includes(
          builderAttempts[i].build.status
        )
      ) {
        // Find the index of first attempt that is still started/scheduled.
        // Rare edge case: for builders that have completed attempts (b), between two
        // started/scheduled attempts (a) and (c), this will return the index of (c).
        // (b) is the latest complete run, so we want to show that in gerrit.
        // The result is (c) will be what is visible in the Checks summary, but if it is blocked on
        // (a), it may not show any updates for awhile.
        while (
          i > 0 &&
          [BuildStatus.SCHEDULED, BuildStatus.STARTED].includes(
            builderAttempts[i - 1].build.status
          )
        ) {
          i--;
        }
        attemptsByBuilder[builder] = builderAttempts.slice(0, i + 1);
        if (builderAttempts.slice(i + 1).length) {
          // Track builds we have hidden, so we can add their links to
          // a run result and still give users access to them.
          hiddenRootBuilders[builder] = builderAttempts.slice(i + 1);
        }
      }
    }

    const buildsByID: Map<string, Build> = new Map();
    for (const builderAttempts of Object.values(attemptsByBuilder)) {
      for (const attempt of builderAttempts.values()) {
        buildsByID.set(attempt.build.id, attempt.build);
      }
    }
    const tvsByBuilds = await this.cache.getAllTestVariants(
      Array.from(buildsByID.keys()),
      this.includeAdditionalResults
    );

    assignFaultAttributesToVariants(tvsByBuilds, attemptsByBuilder);

    for (const [builder, builderAttempts] of Object.entries(
      attemptsByBuilder
    )) {
      const finalBuilderAttempts: (Attempt | undefined)[] =
        this.normalizeAttempts(buildsByID, attemptsByBuilder, builderAttempts);

      const checkName = this.getRunNameFromAttempts(builder, builderAttempts);
      for (const [i, attempt] of finalBuilderAttempts.entries()) {
        const currAttemptNum = i + 1;
        if (attempt === undefined) {
          runs.push({
            patchset: currPatchset,
            attempt: currAttemptNum,
            checkName,
            results: [],
            status: RunStatus.RUNNABLE,
          });
          continue;
        }
        const buildResourceName = createBuildResourceName(
          this.buildbucketHost,
          attempt.build.id
        );

        const run: CheckRun = {
          patchset: currPatchset,
          attempt: currAttemptNum,
          externalId: buildResourceName,
          checkName,
          checkLink: createBuilderLink(attempt.build),
          status: this.getRunStatusFromBuild(attempt.build),
          statusDescription: this.getRunStatusDescFromBuild(attempt.build),
          statusLink: createBuildUrl(this.buildbucketHost, attempt.build.id),
          actions: [this.copyNameAction(attempt.build.builder.builder!)],
          scheduledTimestamp: getDateFromTimestamp(attempt.build.createTime),
          startedTimestamp: getDateFromTimestamp(attempt.build.startTime),
          finishedTimestamp: getDateFromTimestamp(attempt.build.endTime),
          results: [],
        };

        const resultCategory = this.getResultCategoryFromStatus(
          attempt.build.status,
          attempt.build
        );

        run.results?.push({
          externalId: buildResourceName,
          category: resultCategory,
          summary: this.getResultSummaryFromBuild(attempt.build),
          message: attempt.build.summaryMarkdown || '',
          tags: this.createResultTags(attempt.build),
          links: this.createBuildResultLinks(attempt.build),
        });

        const builderName = attempt.build.builder.builder;
        if (
          currAttemptNum === finalBuilderAttempts.length &&
          hiddenRootBuilders[builderName!]
        ) {
          const links: Link[] = [];
          hiddenRootBuilders[builderName!].forEach(hiddenAttempt => {
            const link = this.createBuildResultLinks(hiddenAttempt.build);
            link[0].primary = false;
            links.push(link[0]);
          });
          run.results?.push({
            externalId: `${buildResourceName}-hidden`,
            category: Category.INFO,
            summary: 'Other builds',
            message:
              'Other scheduled builds blocked on current build or triggered by other changes.',
            links,
          });
        }

        const scheduleActions = this.createScheduleActions(
          changeNumber,
          attempt.build,
          repo,
          latestPatchset,
          builderAttempts
        );
        run.actions?.push(...scheduleActions);

        const {variants, errorMessage} = tvsByBuilds[attempt.build.id] || {
          variants: [],
          errorMessage: '',
        };
        if (errorMessage) {
          incompleteTestVariantResults = true;
        }

        const checkResults = await this.convertVariantsToCheckResults(
          attempt.build,
          variants,
          changeNumber
        );
        for (const r of checkResults) {
          r.actions?.push(...scheduleActions);
        }
        if (!run.results) {
          run.results = [];
        }
        run.results.push(...checkResults);
        runs.push(run);
      }
    }

    // Create a fake run notifying the user that ResultDB results are missing.
    if (incompleteTestVariantResults) {
      runs.push({
        change: changeNumber,
        patchset: currPatchset,
        checkName: 'Notice',
        status: RunStatus.COMPLETED,
        results: [
          {
            category: Category.WARNING,
            summary: 'Test results may be missing.',
            message: 'View console logs for more information.',
          },
        ],
      });
    }
    return runs;
  }

  /**
   * Converts ResultDB test variants into CheckResults. For more information
   * about TestVariants, see:
   *   https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/internal/proto/ui/ui.proto
   */
  async convertVariantsToCheckResults(
    build: Build,
    testVariants: TestVariant[],
    changeId: number
  ): Promise<CheckResult[]> {
    if (!testVariants?.length) return [];
    const commonTags = this.createResultTags(build);
    if (!build.endTime) {
      commonTags.push({
        name: 'Preliminary',
        color: TagColor.YELLOW,
        tooltip: 'This result may not match the final result.',
      });
    }

    const checkResults: CheckResult[] = [];
    for (const variant of testVariants) {
      const variantTags = [...commonTags];

      // Tag test results based on variant definition.
      const def = variant.variant?.def;
      const suite = def?.test_suite || '';
      if (suite) {
        variantTags.push({
          name: `suite: ${suite}`,
          color: TagColor.GRAY,
          tooltip: `This test belongs to the ${suite} test suite.`,
        });
      }

      const gpu = def?.gpu || '';
      if (gpu) {
        variantTags.push({
          name: `gpu: ${gpu}`,
          color: TagColor.GRAY,
          tooltip: `This test was run with ${gpu} GPU.`,
        });
      }

      const buildTarget = def?.build_target || '';
      if (buildTarget) {
        variantTags.push({
          name: `build_target: ${buildTarget}`,
          color: TagColor.GRAY,
          tooltip: `This test was run on ${buildTarget} build target.`,
        });
      }

      const faultAttribute = variant.faultAttribute;
      if (
        faultAttribute &&
        faultAttribute.snapshotComparisonFaultAttribution !==
          FaultAttribute.NO_COMPARISON
      ) {
        variantTags.push({
          name: getFaultAttributeTagDisplayName(faultAttribute)!,
          color: getFaultAttributeTagColor(faultAttribute)!,
          tooltip: getFaultAttributeTagTooltip(faultAttribute)!,
        });
      }
      // Create a data object to store information used when rendering a
      // CheckResult row.
      const numTestResults = this.includeAdditionalResults
        ? variant.results.length
        : 1;
      const variantName = createVariantName(variant);
      // TODO(gavinmak): Simplify data.
      const rdbHost = build.infra?.resultdb?.hostname || '';
      const data = {
        plugin: this.plugin,
        variant,
        variantName,
        variantTestSuite: suite,
        testResults: variant.results.slice(0, numTestResults),
        testExonerations: variant.exonerations || [],
        rdbHost,
        addedArtifactLinks: false,
        changeId,
      };

      const messages: string[] = [];
      const allInvArtifacts: Artifact[] = [];
      const resArtifactLinks: Link[] = [];
      const variantLinks = this.createTestResultLinks(variant, build);
      const client = new ResultDbV1Client(rdbHost, String(changeId));
      for (let i = 0; i < numTestResults; i++) {
        const {result: testResult} = variant.results[i];
        // Link to artifacts if previously fetched and available in cache.
        const {fromCache, resArtifacts, invArtifacts} =
          await client.fetchArtifacts(testResult.name, false);
        data.addedArtifactLinks = fromCache;
        const runNumber = numTestResults > 1 ? i + 1 : null;
        const artifactLinks = await Promise.all(
          resArtifacts.map(async a => createArtifactLink(a, false, runNumber))
        );
        resArtifactLinks.push(...artifactLinks);
        allInvArtifacts.push(...invArtifacts);

        // TODO(gavinmak): Once multiple results are processed, create a better
        // summary.
        messages.push(
          `Run #${i + 1}: ${createTestResultSummary(
            testResult,
            variant,
            /* includeSnapshotlink= */ false
          )}.`
        );
      }

      // Result artifacts are named 'Run #${runNumber}: ${tooltip}'.
      // @ts-ignore
      resArtifactLinks.sort((a, b) => a.tooltip.localeCompare(b.tooltip));

      // Dedupe invocation-level artifacts.
      const invLevelArtifacts = [
        ...new Map(allInvArtifacts.map(a => [a.name, a])).values(),
      ];
      const invArtifactLinks = await Promise.all(
        invLevelArtifacts.map(async a => createArtifactLink(a, true, null))
      );

      // Gerrit doesn't define any 'data' field in the CheckResult object.
      // To store the data we need in checks-result.js, we use DATA_SYMBOL.
      // Unlike with a string attribute, values stored with symbols are
      // guaranteed not to be overwritten by Gerrit along the way to
      // checks-result.js.
      checkResults.push({
        externalId: `${variantName}:${variant.variantHash}`,
        category: this.getResultCategoryFromVariant(variant, build),
        summary: this.getResultSummaryFromVariant(variant, build),
        tags: variantTags,
        links: [...variantLinks, ...resArtifactLinks, ...invArtifactLinks],
        message: messages.join(' '),
        actions: [this.copyNameAction(variantName)],
        [DATA_SYMBOL]: data,
      } as any);
    }

    // Sort and group tests by test suite.
    function sortResults(element: string) {
      checkResults.sort((a: any, b: any) => {
        const aName = a[DATA_SYMBOL][element]?.toUpperCase();
        const bName = b[DATA_SYMBOL][element]?.toUpperCase();
        if (aName < bName) {
          return -1;
        }
        if (aName > bName) {
          return 1;
        }
        return 0;
      });
    }
    sortResults('variantName');
    sortResults('variantTestSuite');

    // Create a CheckResult to direct to additional failures.
    if (testVariants.length >= this.maxVariantsPerBuild) {
      checkResults.push({
        externalId:
          createBuildResourceName(this.buildbucketHost, build.id) +
          '/extra-failures',
        category: this.getResultCategoryFromStatus(build.status, build),
        summary: this.getResultSummaryFromBuild(build),
        message:
          'More tests may have failed. View all results on the ' +
          'build page.',
        tags: commonTags,
        links: this.createBuildResultLinks(build),
      });
    }
    return checkResults;
  }

  /**
   * Returns a CheckResult name from a builder's list of runs.
   */
  getRunNameFromAttempts(builder: string, attempts: Attempt[]) {
    const icons = [];

    // Use the latest attempt's status to create the name, otherwise Checks will
    // not collate all builds into one run.
    const latestAttempt = attempts[attempts.length - 1];
    if (latestAttempt.build.status === BuildStatus.INFRA_FAILURE) {
      icons.push(INFRA_FAILURE_ICON);
    }
    if (latestAttempt.isQuickRun) {
      icons.push(QUICK_RUN_ICON);
    }
    return icons.map(i => i + ICON_SPACE).join('') + builder;
  }

  /**
   * Returns a CheckRun status from a Buildbucket build.
   */
  getRunStatusFromBuild(build: Build): RunStatus {
    switch (build.status) {
      case BuildStatus.SCHEDULED:
        return RunStatus.SCHEDULED;
      case BuildStatus.STARTED:
        return RunStatus.RUNNING;
      case BuildStatus.SUCCESS:
      case BuildStatus.FAILURE:
      case BuildStatus.CANCELED:
      case BuildStatus.INFRA_FAILURE:
        return RunStatus.COMPLETED;
      default:
        return RunStatus.RUNNABLE;
    }
  }

  /**
   * Returns a CheckRun statusDescription from a Buildbucket build.
   */
  getRunStatusDescFromBuild(build: Build): string {
    let desc = '';
    if (build.status === BuildStatus.INFRA_FAILURE) {
      desc =
        ` The ${INFRA_FAILURE_ICON}${ICON_SPACE}icon is used for builds ` +
        'that may have ended unsuccessfully from a failure independent of ' +
        'this change and patchset.';
    }
    if (build.output?.properties?.rts_was_used) {
      desc +=
        ` The ${QUICK_RUN_ICON}${ICON_SPACE}icon indicates that this ` +
        'build used Quick Run. Unlike dry runs, these cannot be reused ' +
        'when submitting.';
    }
    return this.getResultSummaryFromBuild(build) + desc;
  }

  /**
   * Returns a CheckResult summary from a Buildbucket build.
   */
  getResultSummaryFromBuild(build: Build): string {
    return `${build.builder.builder} ${
      BUILD_STATUS_TO_ADJECTIVE[build.status]
    }.`;
  }

  /**
   * Returns a CheckResult summary from a Buildbucket build and attempt number.
   */
  getResultSummaryFromBuildAttempt(build: Build, attemptNum: number): string {
    return `Attempt ${attemptNum} ${BUILD_STATUS_TO_ADJECTIVE[build.status]}.`;
  }

  /**
   * Returns a CheckResult category from a Buildbucket status.
   */
  getResultCategoryFromStatus(status: BuildStatus, build: Build): Category {
    if (status === BuildStatus.SUCCESS) {
      return Category.SUCCESS;
    } else if (
      [
        BuildStatus.SCHEDULED,
        BuildStatus.CANCELED,
        BuildStatus.STARTED,
      ].includes(status)
    ) {
      return Category.INFO;
    } else if (build?.critical === 'NO') {
      return Category.WARNING;
    } else {
      return Category.ERROR;
    }
  }

  /**
   * Returns a CheckResult summary from a TestVariant.
   */
  getResultSummaryFromVariant(variant: TestVariant, build: Build): string {
    const summary = `Test ${createVariantName(variant)} `;

    // Build has terminated.
    if (variant.status === TestVariantStatus.FLAKY) {
      return summary + 'flaked.';
    }
    if (variant.status === TestVariantStatus.EXONERATED) {
      return summary + 'was exonerated.';
    }
    if (variant.status === TestVariantStatus.UNEXPECTEDLY_SKIPPED) {
      return summary + 'was unexpectedly skipped.';
    }

    // Variant status is UNEXPECTED here.
    if (build.status === BuildStatus.SUCCESS) {
      return summary + 'failed but the build succeeded.';
    }
    return summary + 'failed.';
  }

  /**
   * Returns a CheckResult category for a TestVariant CheckResult.
   */
  getResultCategoryFromVariant(variant: TestVariant, build: Build): Category {
    const warningStatuses = [
      TestVariantStatus.UNEXPECTEDLY_SKIPPED,
      TestVariantStatus.FLAKY,
      TestVariantStatus.EXONERATED,
    ];
    if (
      build.status === BuildStatus.SUCCESS ||
      !build.endTime ||
      warningStatuses.includes(variant.status)
    ) {
      return Category.WARNING;
    } else {
      return Category.ERROR;
    }
  }

  /**
   * Returns an array of common Checks tags for CheckResults.
   */
  createResultTags(build: Build): Tag[] {
    const tags: Tag[] = [];
    if (build.output?.properties?.rts_was_used) {
      tags.push({
        name: `${QUICK_RUN_ICON}${ICON_SPACE}Quick Run`,
        color: TagColor.PINK,
        tooltip:
          'This result was produced by a build using the Quick Run CQ ' +
          'mode.',
      });
    }
    if (build.critical === 'NO') {
      tags.push({
        name: `Non-critical`,
        color: TagColor.YELLOW,
      });
    }
    if (isExperimental(build)) {
      tags.push({name: 'Experimental', color: TagColor.CYAN});
    }
    if (build.status === BuildStatus.INFRA_FAILURE) {
      tags.push({name: 'Infra Failure', color: TagColor.PURPLE});
    }
    return tags;
  }

  /**
   * Returns an array of Checks links for test variant CheckResults.
   */
  createTestResultLinks(variant: TestVariant, build: Build): Link[] {
    const links = [
      {
        url: createVariantLink(variant, build),
        tooltip: 'Test Results',
        primary: true,
        icon: LinkIcon.EXTERNAL,
      },
    ];

    // Link to Web Test Results if available.
    const layoutLink = createLayoutResultLink(variant, build);
    if (layoutLink) {
      links.push({
        url: layoutLink,
        tooltip: 'Web Test Results',
        primary: false,
        icon: LinkIcon.EXTERNAL,
      });
    }

    // Link to the swarming tasks.
    const taskLinks = [
      ...new Set(variant.results.map(r => createTestTaskLink(r.result))),
    ].filter(l => l !== '');
    for (const [i, link] of taskLinks.entries()) {
      links.push({
        url: link,
        tooltip:
          'Swarming Task' +
          (taskLinks.length > 1 ? ` (${i + 1} of ${taskLinks.length})` : ''),
        primary: false,
        icon: LinkIcon.EXTERNAL,
      });
    }

    return links;
  }

  /**
   * Returns an array of Checks links for Buildbucket CheckResults.
   */
  createBuildResultLinks(build: Build): Link[] {
    return [
      {
        url: createBuildUrl(this.buildbucketHost, build.id),
        tooltip: 'Build',
        primary: true,
        icon: LinkIcon.EXTERNAL,
      },
    ];
  }

  /**
   * Returns a dictionary mapping builders to an ordered list of build ids,
   * build statuses, and quick run statuses.
   *
   * TODO(gavinmak): Refactor to only include build and isQuickRun.
   */
  createAttemptList(builds: Build[]): {[key: string]: Attempt[]} {
    const dct: {[key: string]: Attempt[]} = {};
    builds.forEach(build => {
      const isQuickRun = !!build.output?.properties?.rts_was_used;
      const info: Attempt = {isQuickRun, build};
      const name = build.builder.builder!;
      if (!(name in dct)) {
        dct[name] = [info];
      } else {
        dct[name].push(info);
      }
    });
    Object.values(dct).forEach(arr => {
      arr.sort((a, b) => compareBuildIds(a.build.id, b.build.id));
    });
    return dct;
  }

  /**
   * Returns a list of ChecksAPI Action that control scheduling of builds, e.g.
   * running and cancelling.
   */
  createScheduleActions(
    change: number,
    build: Build,
    project: string,
    patchset: number,
    attempts: Attempt[]
  ): Action[] {
    const actions: Action[] = [];
    if (!this.retryEnabled) {
      return actions;
    }

    const cancellableBuilds = attempts.filter(a =>
      [BuildStatus.SCHEDULED, BuildStatus.STARTED].includes(a.build.status)
    );

    // Successful or unfinished builds should not have a run action.
    if (
      build.status !== BuildStatus.SUCCESS &&
      build.endTime &&
      !shouldSkipRetry(build)
    ) {
      actions.push({
        name: 'Run',
        tooltip:
          'Start a new builder run on the latest patchset. This ' +
          'cancels any existing runs if they exist.',
        primary: false,
        summary: false,
        callback: async change => {
          const cancelRes = await this.cancelRunsCallback(
            change,
            cancellableBuilds
          );
          if (cancelRes.message) {
            return cancelRes;
          }
          return await this.startRunCallback(change, patchset, project, build);
        },
      });
    }

    if (cancellableBuilds.length) {
      actions.push({
        name: 'Cancel',
        tooltip: 'Cancel any existing runs.',
        primary: false,
        summary: false,
        callback: () => this.cancelRunsCallback(change, cancellableBuilds),
      });
    }

    return actions;
  }

  /**
   * ActionCallback that starts a new builder run via Buildbucket.
   */
  async startRunCallback(
    change: number,
    patchset: number,
    project: string,
    build: Build
  ): Promise<{message: string}> {
    const tags = [];
    if (
      [BuildStatus.FAILURE, BuildStatus.INFRA_FAILURE].includes(build.status)
    ) {
      tags.push(RETRY_FAILED_TAG);
    }
    return await this.scheduleBuildsCallback(
      change,
      patchset,
      project,
      [build.builder],
      tags
    );
  }

  /**
   * A helper callback that uses Buildbucket's batch operation.
   */
  async batchCallback(
    change: number,
    requests: BuildRequests,
    operation: string
  ): Promise<{message: string}> {
    const bbClient = new BuildbucketV2Client(
      this.buildbucketHost,
      String(change)
    );
    const batch = async () => await bbClient.batch(requests);
    const {responses} = await retry(
      batch,
      DEFAULT_UPDATE_INTERVAL_MS,
      MAX_UPDATE_INTERVAL_MS,
      3,
      2
    );
    let numError = 0;
    (responses || []).forEach((r: {error: any}) => {
      if (r.error) {
        numError++;
        console.warn('Batch request failed:', r.error);
      }
    });
    this.remoteFetch(false);
    return {
      message: numError
        ? `${numError} of ${responses.length} ${operation} requests ` +
          'failed. See console logs.'
        : '',
    };
  }

  /**
   * ActionCallback that cancels any existing builds. If a build is in an end
   * state, this is a no-op.
   */
  async cancelRunsCallback(
    change: number,
    attempts: Attempt[]
  ): Promise<{message: string}> {
    const requests = attempts.map(attempt => {
      return {
        cancelBuild: {
          id: attempt.build.id,
          summaryMarkdown: 'Cancel build',
        },
      };
    });
    return await this.batchCallback(change, {requests}, 'cancel');
  }

  /**
   * Callback that schedules all builders for the given change and patchset.
   */
  async scheduleBuildsCallback(
    change: number,
    patchset: number,
    project: string,
    builders: Builder[],
    tags: any[]
  ): Promise<{message: string}> {
    const requests = makeBuildRequests(
      builders,
      [
        {
          host: this.config?.gerritHost,
          project,
          change,
          patchset,
        },
      ],
      getNewOperationId(),
      tags || []
    );
    return await this.batchCallback(change, requests, 'schedule');
  }

  /**
   * Top-Level ActionCallback that opens the tryjob picker popup.
   */
  async chooseTryjobsCallback(
    change: number,
    patchset: number,
    project: string
  ): Promise<{message: string}> {
    try {
      await openTryjobPicker(
        this.plugin,
        this.config,
        this.buildbucketHost,
        project,
        change,
        patchset
      );
    } catch (e) {
      return {message: `Failed to open tryjobs panel: ${e}`};
    }
    this.plugin.checks().announceUpdate();
    return {message: ''};
  }

  /**
   * Top-Level ActionCallback that retries any builders whose latest build on
   * any patchset has failed.
   */
  async retryFailedBuildsCallback(
    change: number,
    patchset: number,
    project: string
  ): Promise<{message: string}> {
    const controllerKey = `${project}-{change}-{patchset}-retry`;
    if (this.controllers.has(controllerKey)) {
      this.controllers.get(controllerKey)?.abort();
    }
    const controller = new AbortController();
    this.controllers.set(controllerKey, controller);

    const allPs = Array.from(Array(patchset), (_, x) => x + 1);
    const allBuilds = await this.fetchBuilds(
      change,
      allPs,
      project,
      false,
      controller
    );
    const retryBuilders = getRetryBuilders(allBuilds);
    if (!retryBuilders?.length) {
      return {message: 'No failed builds to retry'};
    }

    return await this.scheduleBuildsCallback(
      change,
      patchset,
      project,
      retryBuilders,
      [RETRY_FAILED_TAG]
    );
  }

  // TODO(b/241165277): remoteFetch should fetch all results and save
  // everything to indexedDB. Then additional results filtering should
  // occur in fetch() and we should set `shouldReload` to true in returned
  // object.
  /**
   * Top-Level ActionCallback that toggles the display and fetching of
   * experimental, flaky, etc. builds, CheckRuns, and CheckResults.
   */
  async filterAddlResultsCallback(): Promise<{message: string}> {
    this.includeAdditionalResults = !this.includeAdditionalResults;
    this.remoteFetch(false);
    return {message: ''};
  }

  /**
   * ActionCallback that copies a given string to the clipboard.
   */
  copyNameAction(txt: string): Action {
    return {
      name: 'Copy Name',
      primary: false,
      summary: false,
      callback: async () => {
        try {
          await navigator.clipboard.writeText(txt);
          return {message: ''};
        } catch (e) {
          const message = `Failed to copy "${txt}" to the clipboard: ${e}`;
          console.warn(message);
          return {message};
        }
      },
    };
  }

  /**
   * Cleans up a string to be used for sorting results.
   */
  sortName(s: string) {
    return (s || '')
      .replace(/[^\x00-\x7F]+/gm, '')
      .trim()
      .toUpperCase();
  }

  /**
   * Creates a unique identifier for a given changeObject. Corresponds to a
   * single repo, change, and patchset.
   */
  getChangeDataId(changeObject: ChangeData): string {
    const {repo, changeNumber, patchsetNumber} = changeObject;
    return `${repo}:${changeNumber}:${patchsetNumber}${
      this.includeAdditionalResults ? ':additional' : ''
    }`;
  }

  /**
   * Mockable equivalent of window.buildbucket.getAuthorizationHeader.
   *
   * @returns authorization header to use in requests.
   */
  getAuthorizationHeader(changeId: number): Promise<AuthorizationHeader> {
    return getAuthorizationHeader(String(changeId));
  }
}

export function createTestResultSummary(
  result: any,
  variant: TestVariant | undefined,
  includeSnapshotlink = false
): string | null {
  const {status, expected} = result;
  let prefix = expected ? 'expectedly ' : 'unexpectedly ';

  switch (status) {
    case TestStatus.PASS:
      prefix += 'passed';
      break;
    case TestStatus.FAIL:
      prefix += 'failed';
      break;
    case TestStatus.CRASH:
      prefix += 'crashed';
      break;
    case TestStatus.ABORT:
      prefix += 'aborted';
      break;
    case TestStatus.SKIP:
      prefix += 'skipped';
      break;
    default:
      return null;
  }

  const faultAttribute = variant?.faultAttribute;
  const comparisonSnapshot = faultAttribute?.comparisonSnapshot;
  if (
    comparisonSnapshot &&
    faultAttribute?.snapshotComparisonFaultAttribution !==
      FaultAttribute.SUCCESS_FOUND
  ) {
    const dateString = comparisonSnapshot.sourceStartedUnixTimestamp
      ? new Date(
          Number(comparisonSnapshot.sourceStartedUnixTimestamp) * 1000
        ).toLocaleString() //Date constructor requires ms, hence * 1000.
      : `b${comparisonSnapshot.sourceBuildId}`;
    if (includeSnapshotlink) {
      let miloLink = `${comparisonSnapshot.sourceBuildId}/test-results?q=ExactID:${faultAttribute.testName}`;
      // If the same model was used for the fault attribution comparison,
      // specify the variant hash.
      if (!faultAttribute.diffModelUsed) {
        miloLink += `+VHash:${variant!.variantHash}`;
      }

      return (
        prefix +
        ` as of <a href="https://ci.chromium.org/ui/b/${miloLink}">${dateString}</a>`
      );
    }
    return prefix + ` as of ${dateString}`;
  } else {
    return prefix;
  }
}