web/checks-fetcher.ts

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import {AuthorizationHeader, getAuthorizationHeader} from './auth';
import {
    BuildbucketV2Client,
    makeBuildRequests,
    retry,
    BuildStatus,
    DEFAULT_UPDATE_INTERVAL_MS,
    MAX_UPDATE_INTERVAL_MS,
    RETRY_FAILED_TAG,
    Builder,
    BuildRequests,
    SearchBuildsRequest,
    SearchBuildsResponse,
  } from './buildbucket-client';
import {
    createBuilderLink,
    createBuildUrl,
    createBuildResourceName,
    compareBuildIds,
    getDateFromTimestamp,
    getNewOperationId,
    hasHideInGerritTag,
    hasHideTestResultsInGerritTag,
    isExperimental,
    getFailedBuilders,
    splitPatchsetGroups,
    shouldSkipRetry,
    showChooseTryjobs,
    showRetryFailed,
  } from './buildbucket-utils';
import {
    Artifact,
    ResultDbV1Client,
    TestStatus,
    TestVariant,
    TestVariantStatus,
  } from './resultdb-client';
import {
  createLayoutResultLink,
  createTestTaskLink,
  createVariantName,
  createVariantLink,
} from './resultdb-utils';
import {openTryjobPicker} from './cr-tryjob-picker';
import {PluginApi} from '@gerritcodereview/typescript-api/plugin';
import {
  Action,
  Category,
  ChangeData,
  CheckResult,
  Link,
  LinkIcon,
  ResponseCode,
  Tag,
  TagColor
} from '@gerritcodereview/typescript-api/checks';
import {ChangeInfo} from '@gerritcodereview/typescript-api/rest-api';

// TODO(aravindvasudev): Update to @gerritcodereview/typescript-api/checks implementation once Gerrit updates their API.
export enum RunStatus {
  RUNNABLE = 'RUNNABLE',
  RUNNING = 'RUNNING',
  SCHEDULED = 'SCHEDULED',
  COMPLETED = 'COMPLETED',
}

// Categories sorted by decreasing severity.
const CATEGORY_ORDER = [
  Category.ERROR,
  Category.WARNING,
  Category.INFO,
  Category.SUCCESS,
];

// The field is used to store arbitrary data in a CheckResult.
export const DATA_SYMBOL = Symbol('chromeData');

// An mapping of Buildbucket build statuses to adjectives. Used to create
// CheckResult summaries.
const BUILD_STATUS_TO_ADJECTIVE = {
  [BuildStatus.SCHEDULED]: 'scheduled',
  [BuildStatus.STARTED]: 'started',
  [BuildStatus.SUCCESS]: 'succeeded',
  [BuildStatus.FAILURE]: 'failed',
  [BuildStatus.INFRA_FAILURE]: 'infra failed',
  [BuildStatus.CANCELED]: 'was canceled',
};

const INFRA_FAILURE_ICON = '🟪';
const QUICK_RUN_ICON = '🚀';

// The space to use between the icons above and regular text. This is equivalent
// to two four-per-em spaces and works well between Chrome on Linux and Mac.
const ICON_SPACE = '\u2005\u2005';

export declare interface Build {
  id: string;
  builder: Builder;
  number?: number;
  tags?: {
    key: string;
    value: string;
  }[];
  status: BuildStatus;
  critical?: string;
  createTime: string;
  startTime: string;
  endTime: string;
  summaryMarkdown?: string;
  infra?: {
    resultdb?: {
      invocation: string;
      hostname: string;
    }
  };
  input?: {experiments: string[]};
  output?: {
    properties?: {
      rts_was_used: boolean;
    }
  };
}

export declare interface Config {
  buckets?: {
    name: string;
    builders?: string[];
  }[];
  hideRetryButton?: boolean;
  gerritHost?: string;
  gitHost?: string;
}

export declare interface Attempt {
  id: string;
  status: BuildStatus;
  isQuickRun: boolean;
  build: Build;
}

// TODO(aravindvasudev): Update to @gerritcodereview/typescript-api/checks implementation once Gerrit updates their API.
export declare interface CheckRun {
  change?: number;
  patchset?: number;
  attempt?: number;
  externalId?: string;
  checkName: string;
  checkDescription?: string;
  checkLink?: string;
  status: RunStatus;
  statusDescription?: string;
  statusLink?: string;
  labelName?: string;
  actions?: Action[];
  scheduledTimestamp?: Date;
  startedTimestamp?: Date;
  finishedTimestamp?: Date;
  results?: CheckResult[];
}

export class ChecksFetcher {
  plugin: PluginApi;
  config: Config | null;
  buildbucketHost: string;
  maxVariantsPerBuild: number;
  includeAdditionalResults: boolean;
  showPreviousAttempts: boolean;
  retryEnabled: boolean;
  controllers: Map<string, AbortController>;

  constructor(plugin: PluginApi, buildbucketHost: string, maxVariantsPerBuild: number) {
    this.plugin = plugin;
    this.config = null;
    this.buildbucketHost = buildbucketHost;
    this.maxVariantsPerBuild = maxVariantsPerBuild;
    this.includeAdditionalResults = false;
    this.showPreviousAttempts = false;
    this.retryEnabled = false;

    // Maps a change and patchset to an AbortController for aborting pending
    // requests.
    this.controllers = new Map();
  }

  async fetch(changeData: ChangeData) {
    // Abort any unfinished fetch requests for this changeData. If this is the
    // first fetch, don't abort. Otherwise, abort using old controller.
    const controllerKey = this.getChangeDataId(changeData);
    if (this.controllers.has(controllerKey)) {
      this.controllers.get(controllerKey)?.abort();
    }
    const controller = new AbortController();
    this.controllers.set(controllerKey, controller);

    const {changeNumber, patchsetNumber, repo, changeInfo} = changeData;

    if (!this.config) {
      const pluginName = encodeURIComponent(this.plugin.getPluginName());
      const config = await this.plugin.restApi()
          .get(`/projects/${encodeURIComponent(repo)}/${pluginName}~config`) as Config;
      if (!config) {
        console.info('Buildbucket plugin not configured for this project.');
        return {responseCode: ResponseCode.OK};
      }
      this.config = config;
    }

    const accessToken = await this.getAuthorizationHeader(changeNumber);
    const loggedIn = Object.keys(accessToken).length !== 0;

    this.retryEnabled = showRetryFailed(this.config, loggedIn, changeInfo);

    // Query BuildBucket for build info.
    const builds = await this.fetchDisplayedBuilds(
        changeNumber, patchsetNumber, repo, changeInfo, controller);
    builds.sort((a, b) => -compareBuildIds(a.id, b.id));

    // Query ResultDB for TestVariants.
    const rdbResults = await Promise.allSettled(builds.map(build =>
        this.fetchTestVariants(
          build, this.includeAdditionalResults, changeNumber)));

    let complete = true;
    const runs = [];
    const allAttempts = this.createAttemptList(builds);
    const latestPatchset = Object.keys(changeInfo.revisions as {}).length;

    for (const [i, build] of builds.entries()) {
      const run = this.convertBuildToRun(
          changeNumber,
          build,
          patchsetNumber,
          latestPatchset,
          allAttempts[build.builder.builder!],
          repo);
      if (!run) {
        continue;
      }
      const result = rdbResults[i];
      if (result.status === 'rejected') {
        console.warn(
            `Failed to query ResultDB for build ${build.id}: ${result.reason}`);
        complete = false;
      }
      const checkResults = await this.convertVariantsToCheckResults(
          build, (result as PromiseFulfilledResult<TestVariant[]>).value, changeNumber);

      if (!run.results) {
        run.results = [];
      }

      run.results.push(...checkResults);
      runs.push(run);
    };

    // Create a fake run notifying the user that ResultDB results are missing.
    if (!complete) {
      runs.push({
        change: changeNumber,
        patchset: patchsetNumber,
        checkName: 'Notice',
        status: RunStatus.COMPLETED,
        results: [{
          category: Category.WARNING,
          summary: 'Test results may be missing.',
          message: 'View console logs for more information.',
        }],
      });
    }

    runs.sort((a, b) => {
      const aName = this.sortName(a.checkName);
      const bName = this.sortName(b.checkName);
      if (aName < bName) {
        return -1;
      }
      if (aName > bName) {
        return 1;
      }
      return 0;
    });

    const actions = [];
    if (showChooseTryjobs(this.config, loggedIn, changeInfo)) {
      actions.push({
        name: 'Choose Tryjobs',
        primary: true,
        summary: true,
        tooltip: 'Select specific builders to run on the latest patchset',
        callback: (change: number) =>
            this.chooseTryjobsCallback(change, latestPatchset, repo),
      });
    }

    if (this.retryEnabled) {
      // We delay checking failed build length since checks 'retry failed'
      // queries all builds from all patchsets and could significantly impact
      // loading time.
      actions.push({
        name: 'Retry Failed Builds',
        primary: true,
        summary: false,
        tooltip: 'Retry any failed builds from all patchsets',
        callback: (change: number) =>
            this.retryFailedBuildsCallback(
                change, latestPatchset, repo, controller),
      });
    }

    // Set primary for 'Additional Results' and 'Give Feedback' if none of
    // 'Choose Tryjobs' or 'Retry Failed' are added as buttons.
    const primary = actions.length === 0;
    actions.push(
      {
        name: `${this.includeAdditionalResults ? 'Hide' : 'Show'} ` +
            'Additional Results',
        primary,
        summary: false,
        tooltip: 'Display experimental, flaky, etc. results',
        callback: () => this.filterAddlResultsCallback(),
      },
      {
        name: `${this.showPreviousAttempts ? 'Hide' : 'Show'} ` +
            'Previous Attempts',
        primary: false,
        summary: false,
        tooltip: 'Display previous builder attempts on this patchset as ' +
            'individual results',
        callback: () => this.showPreviousAttemptsCallback(),
      },
      {
        name: 'Give Feedback',
        primary,
        summary: false,
        tooltip: 'Tell us what you think about Checks!',
        // WARNING: window.open is safe only if the arguments are hard-coded
        // and unchanged.
        callback: () => window.open('https://forms.gle/1wP6uGSdyKiiWHY38'),
      },
    );

    return {
      responseCode: ResponseCode.OK,
      actions,
      runs,
      links: [{
        url: 'https://bugs.chromium.org/p/chromium/issues/entry' +
            '?components=Infra%3ELUCI%3EBuildService%3EPreSubmit%3EGerrit',
        tooltip: 'File a bug',
        primary: true,
        icon: LinkIcon.REPORT_BUG,
      }],
    };
  }

  /**
   * Returns Buildbucket builds to be displayed in the UI.
   *
   * Compared to fetchBuilds, this method handles fetching equivalent patchsets.
   */
  async fetchDisplayedBuilds(change: number, patchset: number, project: string, changeInfo: ChangeInfo, controller: AbortController): Promise<Build[]> {
    let patchsets: number[] = [];
    const groups = splitPatchsetGroups(changeInfo);

    for (const group of groups) {
      if (!group.has(patchset)) {
        continue;
      }
      // Builds for the patchsets greater than patchNum are not included,
      // even if they're in the same equivalent group. Why? This matches
      // the historical behavior of the buildbucket plugin.
      patchsets = Array.from(group).filter(n => (n <= patchset)).sort();
      break;
    }

    if (patchsets.length === 0) {
      return [];
    }

    // Get builds for the equivalent patchsets.
    const builds = await this.fetchBuilds(
        change, patchsets, project, this.includeAdditionalResults, controller);
    if (builds.length > 0 || changeInfo.status !== 'MERGED') {
      return builds;
    }

    // If the submit strategy is "Rebase Always" (such as for Chromium), then
    // the last patchset of the merged CL is autogenerated, and it may be
    // labeled as "REWORK" even though it's actually a trivial rebase; in
    // this case we try again with the previous group of patchsets.
    if (groups.length >= 2 && groups[0].size === 1 &&
        patchsets.length === 1 && groups[0].has(patchsets[0])) {
      const previousPatchsets = Array.from(groups[1]);
      return await this.fetchBuilds(
          change, previousPatchsets, project, this.includeAdditionalResults, controller);
    }

    // If we reach this point, we're in a merged change with no builds to
    // display, but the last patchset doesn't appear to be a special
    // autogenerated patchset.
    return [];
  }

  /**
   * Returns Buildbucket builds for the given change and patchsets.
   */
  async fetchBuilds(change: number, patchsets: number[], project: string, inclExp: boolean, controller: AbortController): Promise<Build[]> {
    const builds: Build[] = [];
    const fields = [
      'id',
      'builder',
      'number',
      'tags',
      'status',
      'critical',
      'createTime',
      'startTime',
      'endTime',
      'summaryMarkdown',
      'infra.resultdb',
    ].join(',');

    let requests: SearchBuildsRequest[] = patchsets.map(patchset => ({
      searchBuilds: {
        pageSize: 1000,
        predicate: {
          includeExperimental: inclExp,
          gerritChanges: [{
            host: this.config?.gerritHost,
            project,
            change,
            patchset,
          }],
        },
        mask: {
          fields,
          outputProperties: [{path: ['rts_was_used']}],
        },
      },
    }));

    const bbClient = new BuildbucketV2Client(this.buildbucketHost, String(change));
    while (requests.length > 0 && !controller?.signal.aborted) {
      const batch = async () => await bbClient.batch(
        {requests}, controller?.signal);
      const {responses} = await retry(
          batch, DEFAULT_UPDATE_INTERVAL_MS, MAX_UPDATE_INTERVAL_MS, 3, 2);

      const newRequests: SearchBuildsRequest[] = [];
      (responses || []).forEach((response: SearchBuildsResponse, i: number) => {
        if (response.error) {
          console.error('Buildbucket request failed with error code ' +
              `${response.error.code}: ${response.error.message}`);
          return;
        }
        builds.push(...(response.searchBuilds?.builds || []));

        // If there are more builds to fetch, add the request to be re-fetched.
        if (response.searchBuilds?.nextPageToken) {
          requests[i].searchBuilds.pageToken = response.searchBuilds.nextPageToken;
          newRequests.push(requests[i]);
        }
      });
      requests = newRequests;
    }

    const visibleBuilds = builds.filter(build => !hasHideInGerritTag(build));
    if (inclExp) {
      return visibleBuilds;
    }
    return visibleBuilds.filter(build => !isExperimental(build));
  }

  /**
   * Returns TestVariants for the given build. If inclAdditional is true, also
   * fetch results for SUCCESSful builds and exonerated, flaky, or skipped
   * results.
   *
   * If a build is tagged with 'hide-test-results-in-gerrit', ResultDB will not
   * be queried.
   */
  async fetchTestVariants(build: Build, inclAdditional: boolean, changeId: number): Promise<TestVariant[]> {
    // Don't query ResultDB for successful builds.
    if (build.status === BuildStatus.SUCCESS && !inclAdditional ||
        !build?.infra?.resultdb?.invocation ||
        hasHideTestResultsInGerritTag(build)) {
      return [];
    }

    const client = new ResultDbV1Client(
      build.infra.resultdb.hostname, String(changeId));
    return await client.fetchTestVariants(
        build, this.maxVariantsPerBuild, inclAdditional);
  }

  /**
   * Converts a Buildbucket `Build` object to a Reboot Checks API `Run` object.
   *
   * TODO(gavinmak): Once attempts contains only builds, remove build arg and
   * rely only on attempts.
   */
  convertBuildToRun(change: number, build: Build, currPatchset: number, latestPatchset: number, attempts: Attempt[], project: string): CheckRun {
    const buildResourceName =
        createBuildResourceName(this.buildbucketHost, build.id);
    const currAttemptNum = attempts.map(a => a.id).indexOf(build.id) + 1;
    const run: CheckRun = {
      patchset: currPatchset,
      attempt: currAttemptNum,
      externalId: buildResourceName,
      checkName: this.getRunNameFromBuild(build, attempts),
      checkLink: createBuilderLink(build),
      status: this.getRunStatusFromBuild(build),
      statusDescription: this.getRunStatusDescFromBuild(build),
      statusLink: createBuildUrl(this.buildbucketHost, build.id),
      actions: this.createRunActions(change, build, project, latestPatchset, attempts),
      scheduledTimestamp: getDateFromTimestamp(build.createTime),
      startedTimestamp: getDateFromTimestamp(build.startTime),
      finishedTimestamp: getDateFromTimestamp(build.endTime),
      results: [],
    };

    // Find the most severe category for this set of attempts.
    let maxCategory = this.getResultCategoryFromStatus(build.status, build);
    if (this.showPreviousAttempts) {
      let maxCategoryIdx = CATEGORY_ORDER.indexOf(maxCategory);
      for (const attempt of attempts) {
        const category =
            this.getResultCategoryFromStatus(attempt.status, attempt.build);
        const idx = CATEGORY_ORDER.indexOf(category);
        // Lower index means higher severity.
        if (idx < maxCategoryIdx) {
          maxCategory = category;
          maxCategoryIdx = idx;
        }
      }
    }

    // Successful results are noisy.
    if (maxCategory === Category.SUCCESS) {
      return run;
    }

    if (this.showPreviousAttempts) {
      // Display most recent attempts first.
      for (let attemptNum = currAttemptNum; attemptNum > 0; attemptNum--) {
        const currBuild = attempts[attemptNum - 1].build;

        // Using the same result category for each previous attempt result means
        // attempts are grouped together.
        run.results?.push({
          externalId: createBuildResourceName(this.buildbucketHost, currBuild.id),
          category: maxCategory,
          summary: this.getResultSummaryFromBuildAttempt(currBuild, attemptNum),
          message: currBuild.summaryMarkdown || '',
          tags: this.createResultTags(currBuild),
          links: this.createBuildResultLinks(currBuild),
        });
      }
    } else if (![BuildStatus.SCHEDULED, BuildStatus.STARTED]
        .includes(build.status)) {
      // To minimize clutter for the user, SUCCESSful and STARTED builds should
      // not contain the Buildbucket CheckResult. Relevant information should be
      // put in the CheckRun object.
      run.results?.push({
        externalId: buildResourceName,
        category: maxCategory,
        summary: this.getResultSummaryFromBuild(build),
        message: build.summaryMarkdown || '',
        tags: this.createResultTags(build),
        links: this.createBuildResultLinks(build),
      });
    }
    return run;
  }

  /**
   * Converts ResultDB test variants into CheckResults. For more information
   * about TestVariants, see:
   *   https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/internal/proto/ui/ui.proto
   */
  async convertVariantsToCheckResults(build: Build, testVariants: TestVariant[], changeId: number): Promise<CheckResult[]> {
    if (!testVariants?.length) return [];
    const commonTags = this.createResultTags(build);
    if (!build.endTime) {
      commonTags.push({
        name: 'Preliminary',
        color: TagColor.YELLOW,
        tooltip: 'This result may not match the final result.'
      });
    }

    const checkResults: CheckResult[] = [];
    for (const variant of testVariants) {
      const variantTags = [...commonTags];

      // Tag test results based on variant definition.
      const def = variant.variant?.def;
      const suite = def?.test_suite || '';
      if (suite) {
        variantTags.push({
          name: `suite: ${suite}`,
          color: TagColor.GRAY,
          tooltip: `This test belongs to the ${suite} test suite.`,
        });
      }

      const gpu = def?.gpu || '';
      if (gpu) {
        variantTags.push({
          name: `gpu: ${gpu}`,
          color: TagColor.GRAY,
          tooltip: `This test was run with ${gpu} GPU.`,
        });
      }

      const buildTarget = def?.build_target || '';
      if (buildTarget) {
        variantTags.push({
          name: `build_target: ${buildTarget}`,
          color: TagColor.GRAY,
          tooltip: `This test was run on ${buildTarget} build target.`,
        });
      }

      // Create a data object to store information used when rendering a
      // CheckResult row.
      const numTestResults =
          this.includeAdditionalResults ? variant.results.length : 1;
      const variantName = createVariantName(variant);
      // TODO(gavinmak): Simplify data.
      const rdbHost = build.infra?.resultdb?.hostname || '';
      const data = {
        plugin: this.plugin,
        variant: {variant: variant.variant},
        variantName,
        variantTestSuite: suite,
        testResults: variant.results.slice(0, numTestResults),
        testExonerations: variant.exonerations || [],
        rdbHost,
        addedArtifactLinks: false,
        changeId,
      };

      const messages: string[] = [];
      const allInvArtifacts: Artifact[] = [];
      const resArtifactLinks: Link[] = [];
      const variantLinks = this.createTestResultLinks(variant, build);
      const client = new ResultDbV1Client(rdbHost, String(changeId));
      for (let i = 0; i < numTestResults; i++) {
        const {result: testResult} = variant.results[i];
        // Link to artifacts if previously fetched and available in cache.
        const {fromCache, resArtifacts, invArtifacts} =
            await client.fetchArtifacts(testResult.name, false);
        data.addedArtifactLinks = fromCache;
        const runNumber = numTestResults > 1 ? i + 1 : null;
        resArtifactLinks.push(
            ...resArtifacts.map(a => createArtifactLink(a, false, runNumber)));
        allInvArtifacts.push(...invArtifacts);

        // TODO(gavinmak): Once multiple results are processed, create a better
        // summary.
        messages.push(`Run #${i + 1}: ${createTestResultSummary(testResult)}.`);
      }

      // Result artifacts are named 'Run #${runNumber}: ${tooltip}'.
      // @ts-ignore
      resArtifactLinks.sort((a, b) => a.tooltip.localeCompare(b.tooltip));

      // Dedupe invocation-level artifacts.
      const invArtifactLinks = [
        ...new Map(allInvArtifacts.map(a => [a.name, a])).values(),
      ].map(a => createArtifactLink(a, true, null));

      // Gerrit doesn't define any 'data' field in the CheckResult object.
      // To store the data we need in checks-result.js, we use DATA_SYMBOL.
      // Unlike with a string attribute, values stored with symbols are
      // guaranteed not to be overwritten by Gerrit along the way to
      // checks-result.js.
      checkResults.push({
        externalId: variantName,
        category: this.getResultCategoryFromVariant(variant, build),
        summary: this.getResultSummaryFromVariant(variant, build),
        tags: variantTags,
        links: [
          ...variantLinks,
          ...resArtifactLinks,
          ...invArtifactLinks,
        ],
        message: messages.join(' '),
        [DATA_SYMBOL]: data,
      } as any);
    }

    // Sort and group tests by test suite.
    function sortResults(element: string) {
      checkResults.sort((a: any, b: any) => {
        const aName = a[DATA_SYMBOL][element]?.toUpperCase();
        const bName = b[DATA_SYMBOL][element]?.toUpperCase();
        if (aName < bName) {
          return -1;
        }
        if (aName > bName) {
          return 1;
        }
        return 0;
      });
    }
    sortResults('variantName');
    sortResults('variantTestSuite');

    // Create a CheckResult to direct to additional failures.
    if (testVariants.length >= this.maxVariantsPerBuild) {
      checkResults.push({
        externalId:
            createBuildResourceName(this.buildbucketHost, build.id) +
            '/extra-failures',
        category: this.getResultCategoryFromStatus(build.status, build),
        summary: this.getResultSummaryFromBuild(build),
        message: 'More tests may have failed. View all results on the ' +
            'build page.',
        tags: commonTags,
        links: this.createBuildResultLinks(build),
      });
    }
    return checkResults;
  }

  /**
   * Returns a CheckResult name from a Buildbucket build and a list of runs.
   */
  getRunNameFromBuild(build: Build, attempts: Attempt[]) {
    const icons = []

    // Use the latest attempt's status to create the name, otherwise Checks will
    // not collate all builds into one run.
    const {status, isQuickRun} = attempts[attempts.length - 1];
    if (status === BuildStatus.INFRA_FAILURE) {
      icons.push(INFRA_FAILURE_ICON);
    }
    if (isQuickRun) {
      icons.push(QUICK_RUN_ICON);
    }
    return icons.map(i => i + ICON_SPACE).join('') + build.builder.builder;
  }

  /**
   * Returns a CheckRun status from a Buildbucket build.
   */
  getRunStatusFromBuild(build: Build): RunStatus {
    switch (build.status) {
      case BuildStatus.SCHEDULED:
        return RunStatus.SCHEDULED;
      case BuildStatus.STARTED:
        return RunStatus.RUNNING;
      case BuildStatus.SUCCESS:
      case BuildStatus.FAILURE:
      case BuildStatus.CANCELED:
      case BuildStatus.INFRA_FAILURE:
        return RunStatus.COMPLETED;
      default:
        return RunStatus.RUNNABLE;
    }
  }

  /**
   * Returns a CheckRun statusDescription from a Buildbucket build.
   */
  getRunStatusDescFromBuild(build: Build): string {
    let desc = '';
    if (build.status === BuildStatus.INFRA_FAILURE) {
      desc = ` The ${INFRA_FAILURE_ICON}${ICON_SPACE}icon is used for builds ` +
          'that may have ended unsuccessfully from a failure independent of ' +
          'this change and patchset.';
    }
    if (build.output?.properties?.rts_was_used) {
      desc += ` The ${QUICK_RUN_ICON}${ICON_SPACE}icon indicates that this ` +
          'build used Quick Run. Unlike dry runs, these cannot be reused ' +
          'when submitting.';
    }
    return this.getResultSummaryFromBuild(build) + desc;
  }

  /**
   * Returns a CheckResult summary from a Buildbucket build.
   */
  getResultSummaryFromBuild(build: Build): string {
    const summary = build?.critical  === 'NO' ? 'Non-critical build': 'Build';
    return `${summary} ${build.builder.builder} ` +
        `${BUILD_STATUS_TO_ADJECTIVE[build.status]}.`;
  }

  /**
   * Returns a CheckResult summary from a Buildbucket build and attempt number.
   */
  getResultSummaryFromBuildAttempt(build: Build, attemptNum: number): string {
    return `Attempt ${attemptNum} ${BUILD_STATUS_TO_ADJECTIVE[build.status]}.`;
  }

  /**
   * Returns a CheckResult category from a Buildbucket status.
   */
  getResultCategoryFromStatus(status: BuildStatus, build: Build): Category {
    if (status === BuildStatus.SUCCESS) {
      return Category.SUCCESS;
    } else if ([BuildStatus.SCHEDULED, BuildStatus.CANCELED,
        BuildStatus.STARTED].includes(status)) {
      return Category.INFO;
    } else if (build?.critical === 'NO') {
      return Category.WARNING;
    } else {
      return Category.ERROR;
    }
  }

  /**
   * Returns a CheckResult summary from a TestVariant.
   */
  getResultSummaryFromVariant(variant: TestVariant, build: Build): string {
    const summary = `Test ${createVariantName(variant)} `;

    // Build has terminated.
    if (variant.status === TestVariantStatus.FLAKY) {
      return summary + 'flaked.';
    }
    if (variant.status === TestVariantStatus.EXONERATED) {
      return summary + 'was exonerated.';
    }
    if (variant.status === TestVariantStatus.UNEXPECTEDLY_SKIPPED) {
      return summary + 'was unexpectedly skipped.';
    }

    // Variant status is UNEXPECTED here.
    if (build.status === BuildStatus.SUCCESS) {
      return summary + 'failed but the build succeeded.';
    }
    return summary + 'failed.';
  }

  /**
   * Returns a CheckResult category for a TestVariant CheckResult.
   */
  getResultCategoryFromVariant(variant: TestVariant, build: Build): Category {
    const warningStatuses = [
      TestVariantStatus.UNEXPECTEDLY_SKIPPED,
      TestVariantStatus.FLAKY,
      TestVariantStatus.EXONERATED,
    ];
    if (build.status === BuildStatus.SUCCESS || !build.endTime ||
        warningStatuses.includes(variant.status)) {
      return Category.WARNING;
    } else {
      return Category.ERROR;
    }
  }

  /**
   * Returns an array of common Checks tags for CheckResults.
   */
  createResultTags(build: Build): Tag[] {
    const tags: Tag[] = [];
    if (build.output?.properties?.rts_was_used) {
      tags.push({
        name: `${QUICK_RUN_ICON}${ICON_SPACE}Quick Run`,
        color: TagColor.PINK,
        tooltip: 'This result was produced by a build using the Quick Run CQ ' +
            'mode.',
      });
    }
    if (isExperimental(build)) {
      tags.push({name: 'Experimental', color: TagColor.CYAN});
    }
    if (build.status === BuildStatus.INFRA_FAILURE) {
      tags.push({name: 'Infra Failure', color: TagColor.PURPLE});
    }
    return tags;
  }

  /**
   * Returns an array of Checks links for test variant CheckResults.
   */
  createTestResultLinks(variant: TestVariant, build: Build): Link[] {
    const links = [{
      url: createVariantLink(variant, build),
      tooltip: 'Test Results',
      primary: true,
      icon: LinkIcon.EXTERNAL,
    }];

    // Link to Web Test Results if available.
    const layoutLink = createLayoutResultLink(variant, build);
    if (layoutLink) {
      links.push({
        url: layoutLink,
        tooltip: 'Web Test Results',
        primary: false,
        icon: LinkIcon.EXTERNAL,
      });
    }

    // Link to the swarming tasks.
    const taskLinks = [...new Set(
      variant.results.map(r => createTestTaskLink(r.result)))
    ].filter(l => l !== '');
    for (const [i, link] of taskLinks.entries()) {
      links.push({
        url: link,
        tooltip: 'Swarming Task' +
            (taskLinks.length > 1 ? ` (${i+1} of ${taskLinks.length})` : ''),
        primary: false,
        icon: LinkIcon.EXTERNAL,
      });
    }

    return links;
  }

  /**
   * Returns an array of Checks links for Buildbucket CheckResults.
   */
  createBuildResultLinks(build: Build): Link[] {
    return [{
      url: createBuildUrl(this.buildbucketHost, build.id),
      tooltip: 'Build',
      primary: true,
      icon: LinkIcon.EXTERNAL,
    }];
  }

  /**
   * Returns a dictionary mapping builders to an ordered list of build ids,
   * build statuses, and quick run statuses.
   *
   * TODO(gavinmak): Refactor to only include build and isQuickRun.
   */
  createAttemptList(builds: Build[]): {[key: string]: Attempt[]} {
    const dct: {[key: string]: Attempt[]} = {};
    builds.forEach(build => {
      const {id, status} = build;
      const isQuickRun = !!build.output?.properties?.rts_was_used;
      const info: Attempt = {id, status, isQuickRun, build};
      const name = build.builder.builder!;
      if (!(name in dct)) {
        dct[name] = [info];
      } else {
        dct[name].push(info);
      }
    });
    Object.values(dct).forEach(
        arr => {(arr as Attempt[]).sort((a, b) => compareBuildIds(a.id, b.id))});
    return dct;
  }

  /**
   * Returns a list of actions used in a Reboot Checks API `Run` object.
   */
  createRunActions(change: number, build: Build, project: string, patchset: number, attempts: Attempt[]): Action[] {
    const actions: Action[] = [];
    if (!this.retryEnabled) {
      return actions;
    }

    const cancellableBuilds = attempts.filter(
        a => [BuildStatus.SCHEDULED, BuildStatus.STARTED].includes(a.status));

    // Successful or unfinished builds should not have a run action.
    if (build.status !== BuildStatus.SUCCESS && build.endTime &&
        !shouldSkipRetry(build)) {
      actions.push({
        name: 'Run',
        tooltip: 'Start a new builder run on the latest patchset. This ' +
            'cancels any existing runs if they exist.',
        primary: false,
        summary: false,
        callback: async (change) => {
          const cancelRes = await this.cancelRunsCallback(change, cancellableBuilds);
          if (cancelRes.message) {
            return cancelRes;
          }
          return await this.startRunCallback(change, patchset, project, build);
        },
      });
    }

    if (cancellableBuilds.length) {
      actions.push({
        name: 'Cancel',
        tooltip: 'Cancel any existing runs.',
        primary: false,
        summary: false,
        callback: () => this.cancelRunsCallback(change, cancellableBuilds),
      });
    }

    return actions;
  }

  /**
   * ActionCallback that starts a new builder run via Buildbucket.
   */
  async startRunCallback(change: number, patchset: number, project: string, build: Build): Promise<{message: string}> {
    const tags = [];
    if ([BuildStatus.FAILURE, BuildStatus.INFRA_FAILURE]
        .includes(build.status)) {
      tags.push(RETRY_FAILED_TAG);
    }
    return await this.scheduleBuildsCallback(
        change, patchset, project, [build.builder], tags);
  }

  /**
   * A helper callback that uses Buildbucket's batch operation.
   */
  async batchCallback(change: number, requests: BuildRequests, operation: string): Promise<{message: string}> {
    const bbClient = new BuildbucketV2Client(this.buildbucketHost, String(change));
    const batch = async () =>  await bbClient.batch(requests);
    const {responses} = await retry(
        batch, DEFAULT_UPDATE_INTERVAL_MS, MAX_UPDATE_INTERVAL_MS, 3, 2);
    let numError = 0;
    (responses || []).forEach((r: {error: any}) => {
      if (r.error) {
        numError++;
        console.warn('Batch request failed:', r.error);
      }
    });
    this.plugin.checks().announceUpdate();
    return {
      message: numError
          ? `${numError} of ${responses.length} ${operation} requests ` +
              'failed. See console logs.'
          : '',
    };
  }

  /**
   * ActionCallback that cancels any existing builds. If a build is in an end
   * state, this is a no-op.
   */
  async cancelRunsCallback(change: number, builds: Attempt[]): Promise<{message: string}> {
    const requests = builds.map(build => ({
      cancelBuild: {
        id: build.id,
        summaryMarkdown: 'Cancel build',
      },
    }));
    return await this.batchCallback(change, {requests}, 'cancel');
  }

  /**
   * Callback that schedules all builders for the given change and patchset.
   */
  async scheduleBuildsCallback(change: number, patchset: number, project: string, builders: Builder[], tags: any[]): Promise<{message: string}> {
    const requests = makeBuildRequests(
      builders,
      [{
        host: this.config?.gerritHost,
        project,
        change,
        patchset,
      }],
      getNewOperationId(),
      tags || [],
    );
    return await this.batchCallback(change, requests, 'schedule');
  }

  /**
   * Top-Level ActionCallback that opens the tryjob picker popup.
   */
  async chooseTryjobsCallback(change: number, patchset: number, project: string): Promise<{message: string}> {
    try {
      await openTryjobPicker(this.plugin, this.config, this.buildbucketHost,
          project, change, patchset);
    } catch (e) {
      return {message: `Failed to open tryjobs panel: ${e}`};
    }
    this.plugin.checks().announceUpdate();
    return {message: ''};
  }

  /**
   * Top-Level ActionCallback that retries any builders whose latest build on
   * any patchset has failed.
   */
  async retryFailedBuildsCallback(change: number, patchset: number, project: string, controller: AbortController): Promise<{message: string}> {
    const allPs = Array.from(Array(patchset), (_, x) => x + 1);
    const allBuilds = await this.fetchBuilds(
        change, allPs, project, false, controller);
    const failedBuilders = getFailedBuilders(allBuilds);
    if (!failedBuilders?.length) {
      return {message: 'No failed builds to retry'};
    }
    return await this.scheduleBuildsCallback(
        change, patchset, project, failedBuilders, [RETRY_FAILED_TAG])
   }

  /**
   * Top-Level ActionCallback that toggles the display and fetching of
   * experimental, flaky, etc. builds, CheckRuns, and CheckResults.
   */
  async filterAddlResultsCallback(): Promise<{message: string}> {
    this.includeAdditionalResults = !this.includeAdditionalResults;
    this.plugin.checks().announceUpdate();
    return {message: ''};
  }

  /**
   * Top-Level ActionCallback that toggles the display of previous attempts of
   * builders as individual results.
   */
  async showPreviousAttemptsCallback(): Promise<{message: string}> {
    this.showPreviousAttempts = !this.showPreviousAttempts;
    this.plugin.checks().announceUpdate();
    return {message: ''};
  }

  /**
   * Cleans up a string to be used for sorting results.
   */
  sortName(s: string) {
    return (s || '').replace(/[^\x00-\x7F]+/gm, '').trim().toUpperCase();
  }

  /**
   * Creates a unique identifier for a given changeObject. Corresponds to a
   * single repo, change, and patchset.
   */
  getChangeDataId(changeObject: ChangeData): string {
    const {repo, changeNumber, patchsetNumber} = changeObject;
    return `${repo}:${changeNumber}:${patchsetNumber}`;
  }

  /**
   * Mockable equivalent of window.buildbucket.getAuthorizationHeader.
   *
   * @returns authorization header to use in requests.
   */
  getAuthorizationHeader(changeId: number): Promise<AuthorizationHeader> {
    return getAuthorizationHeader(String(changeId));
  }
}

/**
 * Returns a Checks link object for a given artifact and parent.
 *
 * @param artifact A ResultDB artifact.
 * @param isInvLevel Whether artifact is invocation-level.
 * @param number The numbered run this belongs to. If this is null or
 *   undefined, the link is not labeled with a run number.
 */
export function createArtifactLink(artifact: any, isInvLevel: boolean, number: number | null): Link {
  let tooltip = number != null ? `Run ${number}: ` : '';
  tooltip += artifact.artifactId + (isInvLevel ? ' of Parent Invocation' : '');
  return {
    url: artifact.fetchUrl,
    tooltip,
    primary: false,
    icon: LinkIcon.FILE_PRESENT,
  };
}

export function createTestResultSummary(result: any): string | null {
  const {status, expected} = result;
  const prefix = expected ? 'expectedly ' : 'unexpectedly ';
  if (status === TestStatus.PASS) {
    return prefix + 'passed';
  }
  if (status === TestStatus.FAIL) {
    return prefix + 'failed';
  }
  if (status === TestStatus.CRASH) {
    return prefix + 'crashed';
  }
  if (status === TestStatus.ABORT) {
    return prefix + 'aborted';
  }
  if (status === TestStatus.SKIP) {
    return prefix + 'skipped';
  }

  return null;
}