web/resultdb-utils.ts

// Copyright 2021 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import {getDateFromTimestamp} from './buildbucket-utils';
import {Build} from './checks-fetcher';
import {LRUMemoryCacheObject} from './lru-memory-cache-object';
import {Artifact, TestResult, TestVariant} from './resultdb-client';
import {Link, LinkIcon} from '@gerritcodereview/typescript-api/checks';

// The maximum number of bytes to fetch when fetching text artifacts.
export const MAX_ARTIFACT_SIZE = 50000; // 50 KB

// Stores the last 256 fetch promises for link artifacts.
const linkArtifactPromiseCache = new LRUMemoryCacheObject(256);

// Allowlist of hosts, used to validate URLs specified in the contents of link
// artifacts. If the URL specified in a link artifact is not in this allowlist,
// the original fetch URL for the artifact will be returned instead.
const LINK_ARTIFACT_HOST_ALLOWLIST = [
  'cros-test-analytics.appspot.com', // Testhaus logs
  'stainless.corp.google.com', // Stainless logs
];

// TODO(gavinmak): Make a resultdb-utils test.

// Neither encodeURI() nor encodeURIComponent() encode parentheses. escape()
// does encode parentheses, but its usage is discouraged. URL_ESCAPES is used
// as an alternative to all the above.
const URL_ESCAPES: {[key: string]: string} = {
  '(': '%28',
  ')': '%29',
};
const URL_ESCAPE_REGEX = new RegExp(
  `[${Object.keys(URL_ESCAPES)
    .map(c => `\\${c}`)
    .join('')}]`,
  'g'
);

/**
 * Returns a Checks link object for a given artifact and parent.
 *
 * @param artifact A ResultDB artifact.
 * @param isInvLevel Whether artifact is invocation-level.
 * @param run The numbered run this belongs to. If this is null or
 *   undefined, the link is not labeled with a run number.
 */
export async function createArtifactLink(
  artifact: any,
  isInvLevel: boolean,
  run: number | null | undefined
): Promise<Link> {
  const invLevelDescription = isInvLevel ? ' of Parent Invocation' : '';
  let tooltip = run !== null && run !== undefined ? `Run ${run}: ` : '';
  tooltip += `${artifact.artifactId}${invLevelDescription}`;

  if (artifact.contentType !== 'text/x-uri') {
    // Return a link using the artifact's fetch URL.
    return {
      url: artifact.fetchUrl,
      tooltip,
      primary: false,
      icon: LinkIcon.FILE_PRESENT,
    };
  }

  // The artifact is a link artifact; its contents is solely a URL, so we can
  // make a direct link instead.
  const targetURL = await fetchLinkArtifactURL(artifact);
  const linkIcon =
    targetURL === artifact.fetchUrl ? LinkIcon.FILE_PRESENT : LinkIcon.EXTERNAL;
  return {
    url: targetURL,
    tooltip,
    primary: false,
    icon: linkIcon,
  };
}

/**
 * Returns a link to the Web Test Result viewer.
 *
 * @param variant Test variant.
 * @param build Buildbucket build.
 */
export function createLayoutResultLink(
  variant: TestVariant,
  build: Build
): string {
  if (
    !/^ninja:\/\/:[^\/]*blink_(web|wpt)_tests.*\//.test(variant.testId!) ||
    !build.number
  ) {
    return '';
  }

  // Convert the builder name following the same rule as
  // _archive_lyaout_test_results() in
  // https://chromium.googlesource.com/chromium/tools/build/+/main/recipes/recipe_modules/chromium_tests/steps.py.
  let builderName = build.builder.builder!;
  builderName = builderName.replace(/[.()]/, '_');

  let stepName = variant.results[0].result.tags?.find(
    t => t['key'] === 'step_name'
  )?.['value'];
  if (!stepName) {
    return '';
  }

  // Remove everything between the first ' ' and the first '(' (or the end) and
  // everything past the first ')', and encode special characters, ex:
  //   'test on some GPU (patch) on some OS' becomes 'test%20%28patch%29', and
  //   'test on some OS' becomes 'test'.
  stepName = stepName.replace(/^([^ \(]* )[^\(]*/, '$1').trim();
  stepName = /^([^\)]*\)?)/.exec(stepName)![1];
  stepName = encodeURIComponent(stepName);
  stepName = stepName.replaceAll(URL_ESCAPE_REGEX, m => URL_ESCAPES[m]);

  return (
    'https://chromium-layout-test-archives.storage.googleapis.com/results.html?json=' +
    `${encodeURIComponent(builderName)}/${build.number}/` +
    `${stepName}/full_results_jsonp.js`
  );
}

/**
 * Returns a deep link to a specific TestVariant.
 *
 * @param variant Test variant.
 * @param build Buildbucket build.
 */
export function createVariantLink(variant: TestVariant, build: Build): string {
  const builder = build.builder;
  return (
    encodeURI(
      `https://ci.chromium.org/ui/p/${builder.project}/` +
        `builders/${builder.bucket}/` +
        `${encodeURIComponent(builder.builder!)}/`
    ) +
    `b${build.id}/test-results?q=ExactID%3A` +
    `${encodeURIComponent(variant.testId!)}+VHash%3A` +
    `${encodeURIComponent(variant.variantHash!)}&clean=`
  );
}

/**
 * Returns a link to the Swarming test task.
 *
 * @param testResult Test result.
 */
export function createTestTaskLink(testResult: TestResult): string {
  const match = /^invocations\/task-([a-zA-Z\-\.]+\.com)-(\w+)/.exec(
    testResult.name
  );
  if (match) {
    return encodeURI(`https://${match[1]}/task?id=${match[2]}`);
  }

  // If the regex doesn't match, check tags for the swarming task ID.
  const taskId = testResult.tags?.find(t => t['key'] === 'swarming_task_id')?.[
    'value'
  ];
  if (taskId) {
    return `https://chromium-swarm.appspot.com/task?id=${taskId}`;
  }
  return '';
}

/**
 * Returns an appropriate human-readable variant name.
 *
 * @param variant Test variant.
 */
export function createVariantName(variant: TestVariant): string {
  return variant.testMetadata?.name || variant.testId;
}

/**
 * Returns the contents of the given ResultDB artifact, which must be exactly
 * one complete URL.
 *
 * @param artifact ResultDB artifact.
 * @returns Promise<string>: the artifact's target URL stored in its contents.
 */
async function fetchLinkArtifactURL(artifact: Artifact): Promise<string> {
  if (!artifact.sizeBytes) {
    console.warn(`Link artifact ${artifact.name} is empty.`);
    return artifact.fetchUrl;
  }
  if (artifact.sizeBytes > MAX_ARTIFACT_SIZE) {
    console.warn(
      `Link artifact ${artifact.name} is large - returning the original ` +
        'fetch URL for the artifact instead.'
    );
    return artifact.fetchUrl;
  }

  let linkArtifactPromise = linkArtifactPromiseCache.read(artifact.name);
  if (!linkArtifactPromise) {
    const urlObj = new URL(artifact.fetchUrl);
    urlObj.searchParams.set('n', String(MAX_ARTIFACT_SIZE));
    linkArtifactPromise = fetch(urlObj.toString()).then(async res => {
      const text = await res.text();
      if (!res.ok) {
        const message = text || res.statusText;
        console.warn(
          `Failed to fetch ${artifact.name} artifact from` +
            ` ${artifact.fetchUrl}. ResultDB responded with status code ` +
            `${res.status}: ${message}`
        );
        throw new Error(message);
      }

      const url = new URL(text);
      const allowedProtocol = ['http:', 'https:'].includes(url.protocol);
      const allowedHost = LINK_ARTIFACT_HOST_ALLOWLIST.includes(url.host);

      if (allowedProtocol && allowedHost) {
        // URL in the artifact contents is valid
        return text;
      }

      console.warn(
        `Invalid target URL for link artifact ${artifact.name} - ` +
          'returning the original fetch URL for the artifact instead.'
      );
      return artifact.fetchUrl;
    });
    linkArtifactPromiseCache.write(artifact.name, linkArtifactPromise);
  }

  try {
    const targetURL = await linkArtifactPromise;
    return targetURL;
  } catch (e: unknown) {
    // Evict the rejected promise from the cache.
    if (linkArtifactPromiseCache.read(artifact.name) === linkArtifactPromise) {
      linkArtifactPromiseCache.delete(artifact.name);
    }
  }

  return artifact.fetchUrl;
}

/**
 * Returns the parent invocation from a TestResult's name.
 *
 * @param resultName The result name.
 */
export function getParentInv(resultName: string): string {
  const match = /^(invocations\/.+?)\//.exec(resultName);
  return (match && match[1]) ?? '';
}

/**
 * Returns the earliest fetchUrlExpiration in a list of artifacts. The
 * returned fetchUrlExpiration is represented as the number of milliseconds
 * since the Unix Epoch.
 */
export function getEarliestArtifactExpiration(artifacts: Artifact[]): number {
  return Math.min(
    ...artifacts.map(a => getDateFromTimestamp(a.fetchUrlExpiration)!.getTime())
  );
}