Add stagecraft-live elements AI eval tests Bug: 578789249 Change-Id: I36c22b0fd11e58852fdc3170d78fa9fcf0c0eca1 Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/8689907 Presubmit-Verified: devtools-frontend-scoped@luci-project-accounts.iam.gserviceaccount.com <devtools-frontend-scoped@luci-project-accounts.iam.gserviceaccount.com> Reviewed-by: Victor Porof <victorporof@chromium.org> Auto-Submit: Finnur Thorarinsson <finnur@chromium.org> Commit-Queue: Victor Porof <victorporof@chromium.org>
diff --git a/test/ai_evals/elements/BUILD.gn b/test/ai_evals/elements/BUILD.gn index 137cd5b..2f52483 100644 --- a/test/ai_evals/elements/BUILD.gn +++ b/test/ai_evals/elements/BUILD.gn
@@ -14,6 +14,18 @@ "extraction-parent-computed-styles.test.ts", "extraction-site-color-palette.test.ts", "extraction-text-size-spacing.test.ts", + "layout-horizontal-scrollbar.test.ts", + "layout-positioning-center-element.test.ts", + "layout-positioning-reorder-section.test.ts", + "layout-positioning-rotate-image.test.ts", + "layout-scroll-body-overflow-hidden.test.ts", + "layout-scroll-container-height.test.ts", + "layout-scroll-invisible-overlay.test.ts", + "layout-scroll-js-prevent-default.test.ts", + "layout-visibility-css-hidden.test.ts", + "layout-visibility-not-in-dom.test.ts", + "layout-visibility-offscreen.test.ts", + "layout-visibility-zindex-covered.test.ts", "themes-background-transparency.test.ts", "themes-chat-bubble-border-radius.test.ts", "themes-dark-mode-emulation.test.ts",
diff --git a/test/ai_evals/elements/layout-horizontal-scrollbar.test.ts b/test/ai_evals/elements/layout-horizontal-scrollbar.test.ts new file mode 100644 index 0000000..798f20f --- /dev/null +++ b/test/ai_evals/elements/layout-horizontal-scrollbar.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-horizontal-scrollbar', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-horizontal-scrollbar', + taskId: 'layout-horizontal-scrollbar', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-horizontal-scrollbar'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['How do I remove the horizontal scroll bar on this page?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-positioning-center-element.test.ts b/test/ai_evals/elements/layout-positioning-center-element.test.ts new file mode 100644 index 0000000..7369319 --- /dev/null +++ b/test/ai_evals/elements/layout-positioning-center-element.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-positioning-center-element', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-positioning-center-element', + taskId: 'layout-positioning-center-element', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-positioning-center-element'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['How do I center the danger alert inside the card body?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-positioning-reorder-section.test.ts b/test/ai_evals/elements/layout-positioning-reorder-section.test.ts new file mode 100644 index 0000000..cf25004 --- /dev/null +++ b/test/ai_evals/elements/layout-positioning-reorder-section.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-positioning-reorder-section', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-positioning-reorder-section', + taskId: 'layout-positioning-reorder-section', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-positioning-reorder-section'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['How do I move the upcoming tour section below the featured artists section?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-positioning-rotate-image.test.ts b/test/ai_evals/elements/layout-positioning-rotate-image.test.ts new file mode 100644 index 0000000..69afd23 --- /dev/null +++ b/test/ai_evals/elements/layout-positioning-rotate-image.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-positioning-rotate-image', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-positioning-rotate-image', + taskId: 'layout-positioning-rotate-image', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-positioning-rotate-image'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['How do I rotate the stage pass image 90 degrees clockwise?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-scroll-body-overflow-hidden.test.ts b/test/ai_evals/elements/layout-scroll-body-overflow-hidden.test.ts new file mode 100644 index 0000000..4048bf1 --- /dev/null +++ b/test/ai_evals/elements/layout-scroll-body-overflow-hidden.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-scroll-body-overflow-hidden', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-scroll-body-overflow-hidden', + taskId: 'layout-scroll-body-overflow-hidden', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-scroll-body-overflow-hidden'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why can\'t I scroll down the festival schedule page?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-scroll-container-height.test.ts b/test/ai_evals/elements/layout-scroll-container-height.test.ts new file mode 100644 index 0000000..cc29d4b --- /dev/null +++ b/test/ai_evals/elements/layout-scroll-container-height.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-scroll-container-height', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-scroll-container-height', + taskId: 'layout-scroll-container-height', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-scroll-container-height'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why won\'t the seating roster panel scroll?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-scroll-invisible-overlay.test.ts b/test/ai_evals/elements/layout-scroll-invisible-overlay.test.ts new file mode 100644 index 0000000..2fc890f --- /dev/null +++ b/test/ai_evals/elements/layout-scroll-invisible-overlay.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-scroll-invisible-overlay', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-scroll-invisible-overlay', + taskId: 'layout-scroll-invisible-overlay', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-scroll-invisible-overlay'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Help me make the workshops page interactable, I can\'t scroll.'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-scroll-js-prevent-default.test.ts b/test/ai_evals/elements/layout-scroll-js-prevent-default.test.ts new file mode 100644 index 0000000..83a0ed2 --- /dev/null +++ b/test/ai_evals/elements/layout-scroll-js-prevent-default.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-scroll-js-prevent-default', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-scroll-js-prevent-default', + taskId: 'layout-scroll-js-prevent-default', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-scroll-js-prevent-default'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why won\'t the afterparty lineup page scroll when I use the mouse wheel?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-visibility-css-hidden.test.ts b/test/ai_evals/elements/layout-visibility-css-hidden.test.ts new file mode 100644 index 0000000..1e96c28 --- /dev/null +++ b/test/ai_evals/elements/layout-visibility-css-hidden.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-visibility-css-hidden', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-visibility-css-hidden', + taskId: 'layout-visibility-css-hidden', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-visibility-css-hidden'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why isn\'t the VIP backstage upgrade banner visible?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-visibility-not-in-dom.test.ts b/test/ai_evals/elements/layout-visibility-not-in-dom.test.ts new file mode 100644 index 0000000..ab79902 --- /dev/null +++ b/test/ai_evals/elements/layout-visibility-not-in-dom.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-visibility-not-in-dom', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-visibility-not-in-dom', + taskId: 'layout-visibility-not-in-dom', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-visibility-not-in-dom'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why isn\'t the loyalty discount widget visible on checkout?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-visibility-offscreen.test.ts b/test/ai_evals/elements/layout-visibility-offscreen.test.ts new file mode 100644 index 0000000..a0b7e66 --- /dev/null +++ b/test/ai_evals/elements/layout-visibility-offscreen.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-visibility-offscreen', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-visibility-offscreen', + taskId: 'layout-visibility-offscreen', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-visibility-offscreen'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why isn\'t the presale alert visible on the screen?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});
diff --git a/test/ai_evals/elements/layout-visibility-zindex-covered.test.ts b/test/ai_evals/elements/layout-visibility-zindex-covered.test.ts new file mode 100644 index 0000000..1f7cc4a --- /dev/null +++ b/test/ai_evals/elements/layout-visibility-zindex-covered.test.ts
@@ -0,0 +1,44 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import {assert} from 'chai'; + +import { + setupAndRunEvalApp, + stopEvalApp, +} from '../helpers/ai_eval-helpers.js'; +import {describe, it, setup} from '../mocha-interface.js'; + +describe('Elements AI Eval: layout-visibility-zindex-covered', function() { + this.timeout(180_000); + + setup({ + dockingMode: 'undocked', + enabledFeatures: ['DevToolsAiV2Architecture'], + devToolsSettings: { + 'ai-assistance-enabled': true, + 'ai-assistance-v2-opt-in-change-dialog-seen': true, + }, + evalConfig: { + type: 'elements', + label: 'layout-visibility-zindex-covered', + taskId: 'layout-visibility-zindex-covered', + }, + }); + + it('launches app, navigates to initial URL, and executes prompt', async ({inspectedPage, evalTask}) => { + const app = await setupAndRunEvalApp('', 'layout-visibility-zindex-covered'); + try { + assert.isDefined(app.config.initialUrl, 'Expected initial_url to be defined in task.textproto'); + + await inspectedPage.page.goto(app.config.initialUrl, {waitUntil: 'networkidle0'}); + + await evalTask.executePrompt({ + prompts: ['Why isn\'t the stage status card visible on the page?'], + }); + } finally { + await stopEvalApp(app); + } + }); +});