RPP: add auto-annotation label generation
Fixed: 405061899
Change-Id: Idf06927221664dfa45a1ba3215fb5f2a6ad99084
Also-by: alinavarkki@chromium.org
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6381853
Reviewed-by: Ergün Erdoğmuş <ergunsh@chromium.org>
Commit-Queue: Jack Franklin <jacktfranklin@chromium.org>
Auto-Submit: Jack Franklin <jacktfranklin@chromium.org>
diff --git a/front_end/models/ai_assistance/BUILD.gn b/front_end/models/ai_assistance/BUILD.gn
index 43abab4..b1f77bb 100644
--- a/front_end/models/ai_assistance/BUILD.gn
+++ b/front_end/models/ai_assistance/BUILD.gn
@@ -56,6 +56,7 @@
":*",
"../../entrypoints/main/*",
"../../panels/ai_assistance/*",
+ "../../panels/timeline/*",
]
visibility += devtools_models_visibility
diff --git a/front_end/models/ai_assistance/agents/PerformanceAgent.test.ts b/front_end/models/ai_assistance/agents/PerformanceAgent.test.ts
index 542ae14..613d0d5 100644
--- a/front_end/models/ai_assistance/agents/PerformanceAgent.test.ts
+++ b/front_end/models/ai_assistance/agents/PerformanceAgent.test.ts
@@ -226,4 +226,22 @@
assert.isFalse(enhancedQuery3.includes(mockAiCallTree.serialize()));
});
});
+
+ describe('generating an AI entry label', () => {
+ it('generates a label from the final answer and trims newlines', async function() {
+ const agent = new PerformanceAgent({
+ aidaClient: mockAidaClient([[{
+ explanation: 'hello world\n',
+ }]]),
+ });
+ const {parsedTrace} = await TraceLoader.traceEngine(this, 'web-dev-with-commit.json.gz');
+ const evalScriptEvent = parsedTrace.Renderer.allTraceEntries.find(
+ event => event.name === Trace.Types.Events.Name.EVALUATE_SCRIPT && event.ts === 122411195649);
+ assert.exists(evalScriptEvent);
+ const aiCallTree = TimelineUtils.AICallTree.AICallTree.fromEvent(evalScriptEvent, parsedTrace);
+ assert.isOk(aiCallTree);
+ const label = await agent.generateAIEntryLabel(aiCallTree);
+ assert.strictEqual(label, 'hello world');
+ });
+ });
});
diff --git a/front_end/models/ai_assistance/agents/PerformanceAgent.ts b/front_end/models/ai_assistance/agents/PerformanceAgent.ts
index ea82708..a2aa873 100644
--- a/front_end/models/ai_assistance/agents/PerformanceAgent.ts
+++ b/front_end/models/ai_assistance/agents/PerformanceAgent.ts
@@ -245,4 +245,22 @@
const perfEnhancementQuery = treeStr ? `${treeStr}\n\n# User request\n\n` : '';
return `${perfEnhancementQuery}${query}`;
}
+
+ /**
+ * Used in the Performance panel to automatically generate a label for a selected entry.
+ */
+ async generateAIEntryLabel(callTree: TimelineUtils.AICallTree.AICallTree): Promise<string> {
+ const context = new CallTreeContext(callTree);
+ const response = await Array.fromAsync(this.run(AI_LABEL_GENERATION_PROMPT, {selected: context}));
+ const lastResponse = response.at(-1);
+ if (lastResponse && lastResponse.type === ResponseType.ANSWER && lastResponse.complete === true) {
+ return lastResponse.text.trim();
+ }
+ throw new Error('Failed to generate AI entry label');
+ }
}
+
+const AI_LABEL_GENERATION_PROMPT =
+ `Generate a very short label for the selected callframe of only a few words describing what the callframe is broadly doing, but provide the most important information for debugging performance.
+
+Important: Describe selected callframe in just 1 sentence under 80 characters without line breaks. We will use your response for this callframe annotation so start with the sentence directly with what the callframe is doing and do not return any other text.`;
diff --git a/front_end/panels/settings/AISettingsTab.ts b/front_end/panels/settings/AISettingsTab.ts
index 2972dbc..6712b59 100644
--- a/front_end/panels/settings/AISettingsTab.ts
+++ b/front_end/panels/settings/AISettingsTab.ts
@@ -342,7 +342,7 @@
text: noLogging ? i18nString(UIStrings.generatedAiAnnotationsSendDataNoLogging) :
i18nString(UIStrings.generatedAiAnnotationsSendData)
}],
- // TODO: Add a relevant link
+ // TODO(b/405316456): Add a relevant link here once we have written the documentation.
learnMoreLink: {url: '', linkJSLogContext: 'learn-more.ai-annotations'},
settingExpandState: {
isSettingExpanded: false,
diff --git a/front_end/panels/timeline/TimelineFlameChartView.ts b/front_end/panels/timeline/TimelineFlameChartView.ts
index 192f8d0..80c5ea9 100644
--- a/front_end/panels/timeline/TimelineFlameChartView.ts
+++ b/front_end/panels/timeline/TimelineFlameChartView.ts
@@ -273,6 +273,9 @@
networkProvider: this.networkDataProvider,
},
entryQueries: {
+ parsedTrace: () => {
+ return this.#parsedTrace;
+ },
isEntryCollapsedByUser: (entry: Trace.Types.Events.Event): boolean => {
return ModificationsManager.activeManager()?.getEntriesFilter().entryIsInvisible(entry) ?? false;
},
diff --git a/front_end/panels/timeline/overlays/BUILD.gn b/front_end/panels/timeline/overlays/BUILD.gn
index bd7a578..e14fadb 100644
--- a/front_end/panels/timeline/overlays/BUILD.gn
+++ b/front_end/panels/timeline/overlays/BUILD.gn
@@ -10,12 +10,14 @@
sources = [ "OverlaysImpl.ts" ]
deps = [
+ "../../../core/common:bundle",
+ "../../../core/i18n:bundle",
"../../../core/platform:bundle",
"../../../models/trace:bundle",
"../../../services/trace_bounds:bundle",
"../../../ui/legacy/components/perf_ui:bundle",
- "../../../ui/lit:bundle",
- "../../timeline/utils:bundle",
+ "../../../ui/visual_logging:bundle",
+ "../utils:bundle",
"./components:bundle",
]
}
diff --git a/front_end/panels/timeline/overlays/OverlaysImpl.test.ts b/front_end/panels/timeline/overlays/OverlaysImpl.test.ts
index c3dacae..72c1164 100644
--- a/front_end/panels/timeline/overlays/OverlaysImpl.test.ts
+++ b/front_end/panels/timeline/overlays/OverlaysImpl.test.ts
@@ -3,7 +3,9 @@
// found in the LICENSE file.
import * as Common from '../../../core/common/common.js';
+import * as AiAssistanceModels from '../../../models/ai_assistance/ai_assistance.js';
import * as Trace from '../../../models/trace/trace.js';
+import {mockAidaClient} from '../../../testing/AiAssistanceHelpers.js';
import {cleanTextContent, dispatchClickEvent} from '../../../testing/DOMHelpers.js';
import {describeWithEnvironment, updateHostConfig} from '../../../testing/EnvironmentHelpers.js';
import {
@@ -22,6 +24,9 @@
import * as Overlays from './overlays.js';
const FAKE_OVERLAY_ENTRY_QUERIES: Overlays.Overlays.OverlayEntryQueries = {
+ parsedTrace() {
+ return null;
+ },
isEntryCollapsedByUser() {
return false;
},
@@ -277,7 +282,12 @@
network: networkFlameChartsContainer,
},
charts,
- entryQueries: FAKE_OVERLAY_ENTRY_QUERIES,
+ entryQueries: {
+ ...FAKE_OVERLAY_ENTRY_QUERIES,
+ parsedTrace() {
+ return parsedTrace;
+ },
+ },
});
const currManager = Timeline.ModificationsManager.ModificationsManager.activeManager();
// The Annotations Overlays are added through the ModificationsManager listener
@@ -322,6 +332,7 @@
inputField: HTMLElement,
overlays: Overlays.Overlays.Overlays,
event: Trace.Types.Events.Event,
+ component: Components.EntryLabelOverlay.EntryLabelOverlay,
}> {
updateHostConfig({
devToolsAiGeneratedTimelineLabels: {
@@ -341,6 +352,7 @@
label: label ?? '',
});
await overlays.update();
+ await RenderCoordinator.done();
// Ensure that the overlay was created.
const overlayDOM = container.querySelector<HTMLElement>('.overlay-type-ENTRY_LABEL');
@@ -353,7 +365,7 @@
const inputField = elementsWrapper.querySelector<HTMLElement>('.input-field');
assert.isOk(inputField);
- return {elementsWrapper, inputField, overlays, event};
+ return {elementsWrapper, inputField, overlays, event, component};
}
it('can render an entry selected overlay', async function() {
@@ -483,6 +495,7 @@
// Double click on the label box to make it editable and focus on it
inputField.dispatchEvent(new FocusEvent('dblclick', {bubbles: true}));
+ await RenderCoordinator.done();
const aiLabelButtonWrapper =
elementsWrapper.querySelector<HTMLElement>('.ai-label-button-wrapper') as HTMLSpanElement;
@@ -493,6 +506,7 @@
// This dialog should not be visible unless the `generate annotation` button is clicked
assert.isFalse(showFreDialogStub.called, 'Expected FreDialog to be not shown but it\'s shown');
aiButton.dispatchEvent(new FocusEvent('click', {bubbles: true}));
+ await RenderCoordinator.done();
// This dialog should be visible
assert.isTrue(showFreDialogStub.called, 'Expected FreDialog to be shown but it\'s not shown');
@@ -558,6 +572,35 @@
assert.strictEqual(inputField?.innerText, 'entry label');
});
+ it('generates a label when the user clicks "Generate" if the setting is enabled', async function() {
+ const {elementsWrapper, inputField, component} = await createAnnotationsLabelElement(this, 'web-dev.json.gz', 50);
+ Common.Settings.moduleSetting('ai-annotations-enabled').set(true);
+
+ const generateButton = elementsWrapper.querySelector<HTMLElement>('.ai-label-button');
+ assert.isOk(generateButton, 'could not find "Generate label" button');
+ assert.isTrue(generateButton.classList.contains('enabled'));
+ const agent = new AiAssistanceModels.PerformanceAgent({
+ aidaClient: mockAidaClient([[{
+ explanation: 'This is an interesting entry',
+ metadata: {
+ rpcGlobalId: 123,
+ }
+ }]])
+ });
+ component.overrideAIAgentForTest(agent);
+
+ // The Agent call is async, so wait for the change event on the label to ensure the UI is updated.
+ const changeEvent = new Promise<void>(resolve => {
+ component.addEventListener(
+ Components.EntryLabelOverlay.EntryLabelChangeEvent.eventName, () => resolve(), {once: true});
+ });
+ dispatchClickEvent(generateButton);
+ await RenderCoordinator.done();
+ await changeEvent;
+
+ assert.strictEqual(inputField.innerHTML, 'This is an interesting entry');
+ });
+
it('Correct security tooltip on the `generate ai label` info icon hover for the users with logging enabled',
async function() {
const {elementsWrapper, inputField} =
diff --git a/front_end/panels/timeline/overlays/OverlaysImpl.ts b/front_end/panels/timeline/overlays/OverlaysImpl.ts
index 5910085..ea40ad6 100644
--- a/front_end/panels/timeline/overlays/OverlaysImpl.ts
+++ b/front_end/panels/timeline/overlays/OverlaysImpl.ts
@@ -7,7 +7,7 @@
import * as Trace from '../../../models/trace/trace.js';
import type * as PerfUI from '../../../ui/legacy/components/perf_ui/perf_ui.js';
import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js';
-import {EntryStyles} from '../../timeline/utils/utils.js';
+import * as Utils from '../utils/utils.js';
import * as Components from './components/components.js';
@@ -386,6 +386,7 @@
}
export interface OverlayEntryQueries {
+ parsedTrace: () => Trace.Handlers.Types.ParsedTrace | null;
isEntryCollapsedByUser: (entry: Trace.Types.Events.Event) => boolean;
firstVisibleParentForEntry: (entry: Trace.Types.Events.Event) => Trace.Types.Events.Event | null;
}
@@ -1528,6 +1529,11 @@
case 'ENTRY_LABEL': {
const shouldDrawLabelBelowEntry = Trace.Types.Events.isLegacyTimelineFrame(overlay.entry);
const component = new Components.EntryLabelOverlay.EntryLabelOverlay(overlay.label, shouldDrawLabelBelowEntry);
+ // Generate the AI Call Tree for the AI Auto-Annotation feature.
+ const parsedTrace = this.#queries.parsedTrace();
+ const callTree = parsedTrace ? Utils.AICallTree.AICallTree.fromEvent(overlay.entry, parsedTrace) : null;
+ component.callTree = callTree;
+
component.addEventListener(Components.EntryLabelOverlay.EmptyEntryLabelRemoveEvent.eventName, () => {
this.dispatchEvent(new AnnotationOverlayActionEvent(overlay, 'Remove'));
});
@@ -1602,7 +1608,7 @@
return overlayElement;
}
case 'TIMINGS_MARKER': {
- const {color} = EntryStyles.markerDetailsForEvent(overlay.entries[0]);
+ const {color} = Utils.EntryStyles.markerDetailsForEvent(overlay.entries[0]);
const markersComponent = this.#createTimingsMarkerElement(overlay);
overlayElement.appendChild(markersComponent);
overlayElement.style.backgroundColor = color;
@@ -1671,7 +1677,7 @@
const markers = document.createElement('div');
markers.classList.add('markers');
for (const entry of overlay.entries) {
- const {color, title} = EntryStyles.markerDetailsForEvent(entry);
+ const {color, title} = Utils.EntryStyles.markerDetailsForEvent(entry);
const marker = document.createElement('div');
marker.classList.add('marker-title');
marker.textContent = title;
diff --git a/front_end/panels/timeline/overlays/components/BUILD.gn b/front_end/panels/timeline/overlays/components/BUILD.gn
index defb9efc..c7056ff 100644
--- a/front_end/panels/timeline/overlays/components/BUILD.gn
+++ b/front_end/panels/timeline/overlays/components/BUILD.gn
@@ -25,14 +25,21 @@
]
deps = [
+ "../../../../core/host:bundle",
"../../../../core/i18n:bundle",
+ "../../../../core/platform:bundle",
+ "../../../../core/root:bundle",
+ "../../../../models/ai_assistance:bundle",
"../../../../models/trace:bundle",
"../../../../panels/common:bundle",
"../../../../ui/components/helpers:bundle",
"../../../../ui/components/icon_button:bundle",
"../../../../ui/components/tooltips:bundle",
+ "../../../../ui/legacy/theme_support:bundle",
"../../../../ui/lit:bundle",
"../../../../ui/visual_logging:bundle",
+ "../../../common:bundle",
+ "../../utils:bundle",
]
}
diff --git a/front_end/panels/timeline/overlays/components/EntryLabelOverlay.ts b/front_end/panels/timeline/overlays/components/EntryLabelOverlay.ts
index 2e346bb..0a5597c 100644
--- a/front_end/panels/timeline/overlays/components/EntryLabelOverlay.ts
+++ b/front_end/panels/timeline/overlays/components/EntryLabelOverlay.ts
@@ -6,9 +6,11 @@
import '../../../../ui/components/tooltips/tooltips.js';
import * as Common from '../../../../core/common/common.js';
+import * as Host from '../../../../core/host/host.js';
import * as i18n from '../../../../core/i18n/i18n.js';
import * as Platform from '../../../../core/platform/platform.js';
import * as Root from '../../../../core/root/root.js';
+import * as AiAssistanceModels from '../../../../models/ai_assistance/ai_assistance.js';
import * as Buttons from '../../../../ui/components/buttons/buttons.js';
import * as ComponentHelpers from '../../../../ui/components/helpers/helpers.js';
import * as UI from '../../../../ui/legacy/legacy.js';
@@ -16,6 +18,7 @@
import * as Lit from '../../../../ui/lit/lit.js';
import * as VisualLogging from '../../../../ui/visual_logging/visual_logging.js';
import * as PanelCommon from '../../../common/common.js';
+import type * as Utils from '../../utils/utils.js';
import stylesRaw from './entryLabelOverlay.css.js';
@@ -92,6 +95,10 @@
const i18nString = i18n.i18n.getLocalizedString.bind(undefined, str_);
const lockedString = i18n.i18n.lockedString;
+function isAiAssistanceServerSideLoggingEnabled(): boolean {
+ return !Root.Runtime.hostConfig.aidaAvailability?.disallowLogging;
+}
+
export class EmptyEntryLabelRemoveEvent extends Event {
static readonly eventName = 'emptyentrylabelremoveevent';
@@ -143,6 +150,17 @@
#label: string;
#shouldDrawBelowEntry: boolean;
#richTooltip: Lit.Directives.Ref<HTMLElement> = Directives.createRef();
+
+ /**
+ * Required to generate a label with AI.
+ */
+ #callTree: Utils.AICallTree.AICallTree|null = null;
+ // Creates or gets the setting if it exists.
+ #aiAnnotationsEnabledSetting = Common.Settings.Settings.instance().createSetting('ai-annotations-enabled', false);
+ #performanceAgent = new AiAssistanceModels.PerformanceAgent({
+ aidaClient: new Host.AidaClient.AidaClient(),
+ serverSideLoggingEnabled: isAiAssistanceServerSideLoggingEnabled(),
+ });
/**
* The entry label overlay consists of 3 parts - the label part with the label string inside,
* the line connecting the label to the entry, and a black box around an entry to highlight the entry with a label.
@@ -185,6 +203,13 @@
this.#drawConnector();
}
+ /**
+ * So we can provide a mocked agent in tests. Do not call this method outside of a test!
+ */
+ overrideAIAgentForTest(agent: AiAssistanceModels.PerformanceAgent): void {
+ this.#performanceAgent = agent;
+ }
+
connectedCallback(): void {
this.#shadow.adoptedStyleSheets = [styles];
}
@@ -400,6 +425,10 @@
}
}
+ set callTree(callTree: Utils.AICallTree.AICallTree|null) {
+ this.#callTree = callTree;
+ }
+
// Generate the AI label suggestion if:
// 1. the user has already already seen the fre dialog and confirmed the feature usage
// or
@@ -407,20 +436,25 @@
//
// Otherwise, show the fre dialog with a 'Got it' button that turns the setting on.
async #handleAiButtonClick(): Promise<void> {
- // Creates or gets the setting if it exists.
- const onboardingCompleteSetting =
- Common.Settings.Settings.instance().createSetting('ai-annotations-enabled', false);
-
- if (onboardingCompleteSetting.get()) {
- // TODO: Actually generate the ai label
- if (this.#inputField) {
- this.#label = 'ai generated label';
+ if (this.#aiAnnotationsEnabledSetting.get()) {
+ // TODO(b/405354543): handle the loading state here.
+ if (!this.#callTree || !this.#inputField) {
+ // Shouldn't happen as we only show the Generate UI when we have this, but this satisfies TS.
+ return;
+ }
+ try {
+ this.#label = await this.#performanceAgent.generateAIEntryLabel(this.#callTree);
this.dispatchEvent(new EntryLabelChangeEvent(this.#label));
this.#inputField.innerText = this.#label;
+ } catch {
+ // TODO(b/405354265): handle the error state
}
- return;
+ } else {
+ await this.#showUserAiFirstRunDialog();
}
+ }
+ async #showUserAiFirstRunDialog(): Promise<void> {
const userConsented = await PanelCommon.FreDialog.show({
header: {iconName: 'pen-spark', text: lockedString(UIStringsNotTranslate.freDisclaimerHeader)},
reminderItems: [
@@ -449,20 +483,23 @@
// clang-format on
}
],
- // TODO: Update this to handle learn more click.
- onLearnMoreClick: () => {}
+ onLearnMoreClick: () => {
+ void UI.ViewManager.ViewManager.instance().showView('chrome-ai');
+ }
});
if (userConsented) {
- onboardingCompleteSetting.set(true);
+ this.#aiAnnotationsEnabledSetting.set(true);
}
}
// Check if the user is logged in, over 18, in a supported location and offline.
// If the user is not logged in, `blockedByAge` will return true.
#isAiAvailable(): boolean|undefined {
- return !Root.Runtime.hostConfig.aidaAvailability?.blockedByAge &&
+ const aiAvailable = !Root.Runtime.hostConfig.aidaAvailability?.blockedByAge &&
!Root.Runtime.hostConfig.aidaAvailability?.blockedByGeo && !navigator.onLine === false;
+ const dataToGenerateLabelAvailable = this.#callTree !== null;
+ return aiAvailable && dataToGenerateLabelAvailable;
}
#renderAITooltip(opts: {textContent: string, includeSettingsButton: boolean}): Lit.TemplateResult {
diff --git a/front_end/ui/visual_logging/KnownContextValues.ts b/front_end/ui/visual_logging/KnownContextValues.ts
index 5e7e6a2..15fe906 100644
--- a/front_end/ui/visual_logging/KnownContextValues.ts
+++ b/front_end/ui/visual_logging/KnownContextValues.ts
@@ -1988,6 +1988,7 @@
'layout-count',
'layout-shifts',
'learn-more',
+ 'learn-more.ai-annotations',
'learn-more.ai-assistance',
'learn-more.console-insights',
'learn-more.coop-coep',
