Eval: use the GCP bucket and store examples there This CL uses the newly created GCP bucket in two ways: 1. `gclient sync` will now fetch the contents of the bucket. 2. The `upload_to_gcp.ts` script will upload the latest contents of the directory to GCP, and will then update the `DEPs` file with the latest revision. It first generates a hash of all the files in the directory to ensure that we do not end up uploading a folder that is identical to the existing one. R=alexrudenko@chromium.org Bug: 436223818 Change-Id: I28641b7eee64945c78a5fbdb3378f3633fc6a0ea Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6862825 Commit-Queue: Jack Franklin <jacktfranklin@chromium.org> Reviewed-by: Alex Rudenko <alexrudenko@chromium.org>
diff --git a/.gitignore b/.gitignore index 8209b92..21aa428 100644 --- a/.gitignore +++ b/.gitignore
@@ -28,7 +28,8 @@ /scripts/ai_assistance/auto-run/data /scripts/ai_assistance/performance-trace-downloads /scripts/ai_assistance/auto-run/performance-trace-downloads -/scripts/ai_assistance/suite/outputs/**/*.json +# This folder is populated from the GCP bucket and managed by gclient sync. +/scripts/ai_assistance/suite/outputs /build /buildtools
diff --git a/DEPS b/DEPS index 6e7631c..111e43f 100644 --- a/DEPS +++ b/DEPS
@@ -216,6 +216,19 @@ }, ], }, + "scripts/ai_assistance/suite/outputs": { + "dep_type": "gcs", + 'condition': 'checkout_ai_evals == True', + "bucket": "chrome-devtools-ai-evals", + "objects": [ + { + "object_name": "f0e8e7b99dc61f7a943bfdf284552982c63bdf8d6217091f5260bc8ebd84ca9f", + "sha256sum": "af579f30f2384089e1bece67db9afb71b902aa6ff99cb9749d4694ce53783670", + "size_bytes": 3582, + "generation": 1755705853621054 + } + ] + }, 'third_party/node/win': { 'dep_type': 'gcs', 'condition': 'host_os == "win" and build_with_chromium == False and non_git_source',
diff --git a/scripts/ai_assistance/suite/README.md b/scripts/ai_assistance/suite/README.md new file mode 100644 index 0000000..e3f6518 --- /dev/null +++ b/scripts/ai_assistance/suite/README.md
@@ -0,0 +1,57 @@ +# DevTools AI Evals Suite + +This folder contains the scripts required to execute the DevTools eval suite. + +At this time, this is being heavily iterated on and may change rapidly. Chat to jacktfranklin@ if you have any questions or feedback. + +## Getting started + +### 1: get the outputs from GCP + +The actual output files you need to run the suite are hosted in a GCP bucket. The contents are fetched for you by `gclient sync` but only if you set the `checkout_ai_evals` arg in your `.gclient` config: + +``` +solutions = [ + { + "name": "devtools-frontend", + "url": "https://chromium.googlesource.com/devtools/devtools-frontend.git", + "deps_file": "DEPS", + "managed": False, + "custom_deps": {}, + "custom_vars": { + "checkout_ai_evals": True, + } + }, +] +``` + +Note that you have to be a Google employee to access this. + +### 2: set your Gemini API key. + +Evaluations that engage an LLM will expect to find a `GEMINI_API_KEY` environment variable defined. + +## Running the suite + +Run `cd scripts/ai_assistance && npm run eval-suite` to execute the suite. + +## Adding new outputs + +Once you have new outputs you want to put into the set, move them into the right place in the `suite/outputs/outputs` folder.: + +The structure of files in this folder is like so: `outputs/type/YYYY-MM-DD/label-XYZ.json`. + +- Type here should map roughly to an agent, e.g. "performance". +- For actual outputs, store them as JSON files within a `YYYY-MM-DD` folder. +- The file name must start with a label - e.g. `lcp-breakdown`. After that, it can have any other content you like. You can have one JSON file with multiple outputs, or you can have multiple files with outputs. If there are multiple files, they will be merged accordingly when the eval suite is run. + +Then, run (from the DevTools root directory in this case, but it doesn't matter): + +``` +node scripts/ai_assistance/suite/upload_to_gcp.ts +``` + +This will upload the changes to the GCP bucket and update the `DEPS` file for you, which you should ensure you commit in a CL. + +If you get any authorisation errors, run `gsutil.py config` to refresh your authentication status. +
diff --git a/scripts/ai_assistance/suite/helpers/outputs.ts b/scripts/ai_assistance/suite/helpers/outputs.ts index d86b759..8043ac9 100644 --- a/scripts/ai_assistance/suite/helpers/outputs.ts +++ b/scripts/ai_assistance/suite/helpers/outputs.ts
@@ -8,7 +8,7 @@ import type {Conversation, EvalFileOutput} from '../types'; -const BASE_DIR = path.join(path.dirname(import.meta.filename), '..', 'outputs'); +const BASE_DIR = path.join(path.dirname(import.meta.filename), '..', 'outputs', 'outputs'); export function getMarkdownConversation(example: Conversation): string { let markdown = '';
diff --git a/scripts/ai_assistance/suite/outputs/README.md b/scripts/ai_assistance/suite/outputs/README.md deleted file mode 100644 index 9f05a58..0000000 --- a/scripts/ai_assistance/suite/outputs/README.md +++ /dev/null
@@ -1,5 +0,0 @@ -## Eval Outputs - -This folder contains outputs from running DevTools AI Agents with examples. - -Currently, the architecture is a work in progress - talk to @jacktfranklin if you have any questions. These files will be fetched from a GCP bucket via `gclient sync` and put into this directory.
diff --git a/scripts/ai_assistance/suite/upload_to_gcp.ts b/scripts/ai_assistance/suite/upload_to_gcp.ts new file mode 100644 index 0000000..9c14ca6 --- /dev/null +++ b/scripts/ai_assistance/suite/upload_to_gcp.ts
@@ -0,0 +1,104 @@ +// Copyright 2025 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {execSync} from 'node:child_process'; +import crypto from 'node:crypto'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const filename = fileURLToPath(import.meta.url); +const dirname = path.dirname(filename); + +// the upload_to_google_storage_first_class.py script only lets you upload +// folders in the immediate directory, so we run it from the `outputs` +// directory. Remember that the outputs directory is nested and looks like +// this: +// scripts/ai_assistance/suite/outputs/outputs +// We need to run the script from the first outputs directory, uploading the second. +const CWD_TO_UPLOAD_FROM = path.join(dirname, 'outputs'); + +const DEVTOOLS_ROOT = path.join(dirname, '..', '..', '..'); + +const hash = await getDirectoryHash(path.join(CWD_TO_UPLOAD_FROM, 'outputs')); + +const output = execSync( + `upload_to_google_storage_first_class.py --bucket chrome-devtools-ai-evals --object-name ${hash} outputs`, + {cwd: CWD_TO_UPLOAD_FROM, encoding: 'utf8'}); + +// The output contains some logs and then a JSON object that we want to use to +// update our DEPS with the newest data. +const jsonRegex = /{.*}/s; +const jsonMatch = output.match(jsonRegex); +if (!jsonMatch) { + console.error('ERROR: could not parse DEPs JSON out of upload response.'); + process.exit(1); +} + +console.log('Data uploaded to GCP bucket.'); + +const data = JSON.parse(jsonMatch[0]) as { + path: { + // eslint-disable-next-line @typescript-eslint/naming-convention + dep_type: 'gcs', + bucket: 'chrome-devtools-ai-evals', + objects: [{ + // eslint-disable-next-line @typescript-eslint/naming-convention + object_name: string, + sha256sum: string, + // eslint-disable-next-line @typescript-eslint/naming-convention + size_bytes: number, + generation: number, + }], + }, +}; + +// We now need to update the DEPs entry. We can use `gclient setdeps` which +// expects a particular format (taken from the --help output): +// If it is a GCS dependency, dep must be of the form path@object_name,sha256sum,size_bytes,generation +const DEPS_PATH = 'scripts/ai_assistance/suite/outputs'; +const setDepInput = [...Object.values(data.path.objects[0])].join(','); + +execSync(`gclient setdep -r ${DEPS_PATH}@${setDepInput}`, {cwd: DEVTOOLS_ROOT, encoding: 'utf8', stdio: 'ignore'}); + +console.log('DEPS file updated. You should commit this change.'); + +/** + * Walks a directory and creates a hash of it based on the contents of all the + * files. Used to generate the name of the TAR file that gets put onto GCP. We + * do it like this to ensure we don't upload the same contents again. + */ +async function getDirectoryHash(directoryPath: string): Promise<string> { + const files: string[] = []; + + async function walk(currentPath: string) { + const items = await fs.readdir(currentPath, {withFileTypes: true}); + + for (const item of items) { + const fullPath = path.join(currentPath, item.name); + if (item.isDirectory()) { + await walk(fullPath); + } else if (item.name.endsWith('.json')) { + files.push(fullPath); + } + } + } + + await walk(directoryPath); + // Sort paths for a consistent hash + files.sort(); + + const hasher = crypto.createHash('sha256'); + for (const filePath of files) { + // Get a hash of the file's content + const fileContent = await fs.readFile(filePath); + const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex'); + + // Combine the relative path and file hash + const relativePath = path.relative(directoryPath, filePath); + hasher.update(`${relativePath}:${fileHash}`); + } + + return hasher.digest('hex'); +}