| // Copyright 2024 The Chromium Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "chrome/browser/ui/lens/lens_overlay_controller.h" |
| |
| #include <memory> |
| #include <optional> |
| #include <set> |
| #include <sstream> |
| #include <string> |
| #include <vector> |
| |
| #include "base/containers/contains.h" |
| #include "base/functional/bind.h" |
| #include "base/json/json_reader.h" |
| #include "base/metrics/histogram_functions.h" |
| #include "base/no_destructor.h" |
| #include "base/numerics/checked_math.h" |
| #include "base/process/kill.h" |
| #include "base/strings/string_split.h" |
| #include "base/strings/string_util.h" |
| #include "base/strings/utf_string_conversions.h" |
| #include "base/system/sys_info.h" |
| #include "base/task/bind_post_task.h" |
| #include "base/task/sequenced_task_runner.h" |
| #include "base/task/thread_pool.h" |
| #include "chrome/browser/feedback/show_feedback_page.h" |
| #include "chrome/browser/lens/core/mojom/geometry.mojom.h" |
| #include "chrome/browser/lens/core/mojom/lens_side_panel.mojom.h" |
| #include "chrome/browser/lens/core/mojom/overlay_object.mojom.h" |
| #include "chrome/browser/lens/core/mojom/text.mojom.h" |
| #include "chrome/browser/profiles/profile.h" |
| #include "chrome/browser/search/search.h" |
| #include "chrome/browser/task_manager/web_contents_tags.h" |
| #include "chrome/browser/themes/theme_service.h" |
| #include "chrome/browser/ui/browser_element_identifiers.h" |
| #include "chrome/browser/ui/browser_window/public/browser_window_features.h" |
| #include "chrome/browser/ui/browser_window/public/browser_window_interface.h" |
| #include "chrome/browser/ui/exclusive_access/exclusive_access_manager.h" |
| #include "chrome/browser/ui/hats/hats_service.h" |
| #include "chrome/browser/ui/hats/hats_service_factory.h" |
| #include "chrome/browser/ui/lens/lens_overlay_entry_point_controller.h" |
| #include "chrome/browser/ui/lens/lens_overlay_event_handler.h" |
| #include "chrome/browser/ui/lens/lens_overlay_image_helper.h" |
| #include "chrome/browser/ui/lens/lens_overlay_languages_controller.h" |
| #include "chrome/browser/ui/lens/lens_overlay_proto_converter.h" |
| #include "chrome/browser/ui/lens/lens_overlay_query_controller.h" |
| #include "chrome/browser/ui/lens/lens_overlay_side_panel_coordinator.h" |
| #include "chrome/browser/ui/lens/lens_overlay_theme_utils.h" |
| #include "chrome/browser/ui/lens/lens_overlay_untrusted_ui.h" |
| #include "chrome/browser/ui/lens/lens_overlay_url_builder.h" |
| #include "chrome/browser/ui/lens/lens_preselection_bubble.h" |
| #include "chrome/browser/ui/lens/lens_search_contextualization_controller.h" |
| #include "chrome/browser/ui/lens/lens_search_controller.h" |
| #include "chrome/browser/ui/lens/lens_searchbox_controller.h" |
| #include "chrome/browser/ui/lens/page_content_type_conversions.h" |
| #include "chrome/browser/ui/search/omnibox_utils.h" |
| #include "chrome/browser/ui/tabs/public/tab_features.h" |
| #include "chrome/browser/ui/ui_features.h" |
| #include "chrome/browser/ui/user_education/browser_user_education_interface.h" |
| #include "chrome/browser/ui/views/side_panel/side_panel.h" |
| #include "chrome/browser/ui/views/side_panel/side_panel_coordinator.h" |
| #include "chrome/browser/ui/views/side_panel/side_panel_enums.h" |
| #include "chrome/browser/ui/views/side_panel/side_panel_ui.h" |
| #include "chrome/browser/ui/views/side_panel/side_panel_util.h" |
| #include "chrome/browser/ui/webui/webui_embedding_context.h" |
| #include "chrome/common/chrome_render_frame.mojom.h" |
| #include "chrome/common/pref_names.h" |
| #include "chrome/common/webui_url_constants.h" |
| #include "chrome/grit/branded_strings.h" |
| #include "components/feature_engagement/public/feature_constants.h" |
| #include "components/find_in_page/find_tab_helper.h" |
| #include "components/lens/lens_features.h" |
| #include "components/lens/lens_overlay_metrics.h" |
| #include "components/lens/lens_overlay_mime_type.h" |
| #include "components/lens/lens_overlay_permission_utils.h" |
| #include "components/omnibox/browser/lens_suggest_inputs_utils.h" |
| #include "components/optimization_guide/content/browser/page_content_proto_provider.h" |
| #include "components/optimization_guide/content/browser/page_context_eligibility.h" |
| #include "components/permissions/permission_request_manager.h" |
| #include "components/signin/public/identity_manager/identity_manager.h" |
| #include "components/tabs/public/tab_interface.h" |
| #include "components/viz/common/frame_timing_details.h" |
| #include "components/zoom/zoom_controller.h" |
| #include "content/public/browser/child_process_termination_info.h" |
| #include "content/public/browser/download_manager.h" |
| #include "content/public/browser/download_request_utils.h" |
| #include "content/public/browser/render_frame_host.h" |
| #include "content/public/browser/render_process_host.h" |
| #include "content/public/browser/render_view_host.h" |
| #include "content/public/browser/render_widget_host.h" |
| #include "content/public/browser/render_widget_host_view.h" |
| #include "content/public/browser/web_contents_user_data.h" |
| #include "content/public/browser/web_ui.h" |
| #include "net/base/network_change_notifier.h" |
| #include "pdf/buildflags.h" |
| #include "services/metrics/public/cpp/ukm_builders.h" |
| #include "services/metrics/public/cpp/ukm_source_id.h" |
| #include "third_party/blink/public/common/associated_interfaces/associated_interface_provider.h" |
| #include "third_party/lens_server_proto/lens_overlay_selection_type.pb.h" |
| #include "third_party/lens_server_proto/lens_overlay_service_deps.pb.h" |
| #include "ui/base/clipboard/scoped_clipboard_writer.h" |
| #include "ui/base/interaction/element_tracker.h" |
| #include "ui/base/l10n/l10n_util.h" |
| #include "ui/base/webui/web_ui_util.h" |
| #include "ui/base/window_open_disposition_utils.h" |
| #include "ui/compositor/compositor.h" |
| #include "ui/compositor/layer.h" |
| #include "ui/gfx/geometry/rounded_corners_f.h" |
| #include "ui/native_theme/native_theme.h" |
| #include "ui/views/bubble/bubble_dialog_delegate_view.h" |
| #include "ui/views/controls/webview/web_contents_set_background_color.h" |
| #include "ui/views/controls/webview/webview.h" |
| #include "ui/views/view_class_properties.h" |
| #include "ui/views/widget/native_widget.h" |
| |
| #if BUILDFLAG(ENABLE_PDF) |
| #include "components/pdf/browser/pdf_document_helper.h" |
| #include "pdf/mojom/pdf.mojom.h" |
| #endif // BUILDFLAG(ENABLE_PDF) |
| |
| void* kLensOverlayPreselectionWidgetIdentifier = |
| &kLensOverlayPreselectionWidgetIdentifier; |
| |
| namespace { |
| |
| // Timeout for the fadeout animation. This is purposely set to be twice the |
| // duration of the fade out animation on the WebUI JS because there is a delay |
| // between us notifying the WebUI, and the WebUI receiving our event. |
| constexpr base::TimeDelta kFadeoutAnimationTimeout = base::Milliseconds(300); |
| |
| // The amount of time to wait for a reflow after closing the side panel before |
| // taking a screenshot. |
| constexpr base::TimeDelta kReflowWaitTimeout = base::Milliseconds(200); |
| |
| // The amount of change in bytes that is considered a significant change and |
| // should trigger a page content update request. This provides tolerance in |
| // case there is slight variation in the retrievied bytes in between calls. |
| constexpr float kByteChangeTolerancePercent = 0.01; |
| |
| // The maximum length of the DOM text to consider for OCR similarity. |
| // Currently 50 MB |
| constexpr int kMaxDomTextLengthForOcrSimilarity = 50 * 1000 * 1000; |
| |
| // Copy the objects of a vector into another without transferring |
| // ownership. |
| std::vector<lens::mojom::OverlayObjectPtr> CopyObjects( |
| const std::vector<lens::mojom::OverlayObjectPtr>& objects) { |
| std::vector<lens::mojom::OverlayObjectPtr> objects_copy(objects.size()); |
| std::transform( |
| objects.begin(), objects.end(), objects_copy.begin(), |
| [](const lens::mojom::OverlayObjectPtr& obj) { return obj->Clone(); }); |
| return objects_copy; |
| } |
| |
| // Given a BGR bitmap, converts into a RGB bitmap instead. Returns empty bitmap |
| // if creation fails. |
| SkBitmap CreateRgbBitmap(const SkBitmap& bgr_bitmap) { |
| // Convert bitmap from color type `kBGRA_8888_SkColorType` into a new Bitmap |
| // with color type `kRGBA_8888_SkColorType` which will allow the bitmap to |
| // render properly in the WebUI. |
| sk_sp<SkColorSpace> srgb_color_space = |
| bgr_bitmap.colorSpace()->makeSRGBGamma(); |
| SkImageInfo rgb_info = bgr_bitmap.info() |
| .makeColorType(kRGBA_8888_SkColorType) |
| .makeColorSpace(SkColorSpace::MakeSRGB()); |
| SkBitmap rgb_bitmap; |
| rgb_bitmap.setInfo(rgb_info); |
| rgb_bitmap.allocPixels(rgb_info); |
| if (rgb_bitmap.writePixels(bgr_bitmap.pixmap())) { |
| return rgb_bitmap; |
| } |
| |
| // Bitmap creation failed. |
| return SkBitmap(); |
| } |
| |
| // Returns a new string with all non-alphanumeric characters removed from the |
| // ends of the string. |
| std::string TrimNonAlphaNumeric(const std::string& text) { |
| if (text.empty()) { |
| return text; |
| } |
| |
| // Find the first alphanumeric character from the beginning. |
| size_t first_alphanum_index = |
| std::find_if(text.begin(), text.end(), ::isalnum) - text.begin(); |
| |
| // If no alphanumeric character is found in the entire string, return an empty |
| // string. |
| if (first_alphanum_index == text.length()) { |
| return ""; |
| } |
| |
| // Find the index of the last alphanumeric character from the end. |
| size_t last_alphanum_index = |
| std::find_if(text.rbegin(), text.rend(), ::isalnum) - text.rbegin(); |
| // `last_alphanumeric` is the count from the end of the string, so convert to |
| // index from the beginning. |
| last_alphanum_index = text.length() - 1 - last_alphanum_index; |
| |
| // Extract the substring containing only the alphanumeric characters and those |
| // in between. |
| return text.substr(first_alphanum_index, |
| last_alphanum_index - first_alphanum_index + 1); |
| } |
| |
| // Returns the percentage of words in the OCR text that are also in the DOM |
| // text. |
| double CalculateWordOverlapSimilarity(std::string dom_text, |
| lens::mojom::TextPtr ocr_text) { |
| // Split dom_text into possible words. |
| std::vector<std::string> dom_words = base::SplitString( |
| dom_text, " \t\r\n<>", base::TRIM_WHITESPACE, base::SPLIT_WANT_NONEMPTY); |
| |
| // Convert dom_text to lowercase, alphanumeric only map for comparison. The |
| // map value is the number of times the word appears in the dom text. |
| std::map<std::string, int> dom_words_map; |
| for (std::string& word : dom_words) { |
| std::string processed_word = TrimNonAlphaNumeric(base::ToLowerASCII(word)); |
| if (!processed_word.empty()) { |
| dom_words_map[processed_word]++; |
| } |
| } |
| |
| // Count the number of words in ocr_text that are also in the dom text. |
| double overlap_count = 0; |
| double total_ocr_words = 0; |
| if (ocr_text && ocr_text->text_layout && |
| ocr_text->text_layout->paragraphs.size() > 0) { |
| for (const auto& paragraph : ocr_text->text_layout->paragraphs) { |
| if (paragraph && paragraph->lines.size() > 0) { |
| for (const auto& line : paragraph->lines) { |
| if (line && line->words.size() > 0) { |
| for (const auto& word : line->words) { |
| if (word) { |
| std::string processed_word = |
| TrimNonAlphaNumeric(base::ToLowerASCII(word->plain_text)); |
| if (processed_word.empty()) { |
| continue; |
| } |
| |
| // Find the process word in the dom words. |
| auto word_iterator = dom_words_map.find(processed_word); |
| if (word_iterator != dom_words_map.end() && |
| word_iterator->second > 0) { |
| // The word is in the dom text. |
| overlap_count++; |
| |
| // Decrement the count in the map so if there are multiple of |
| // this word in the DOM, we only count it for each instance. |
| word_iterator->second--; |
| } |
| total_ocr_words++; |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| // Avoid divide by zero. Return the percentage of words in the OCR text that |
| // are also in the DOM text. |
| return total_ocr_words == 0 ? 0.0 : overlap_count / total_ocr_words; |
| } |
| |
| // Converts a JSON string array to a vector. |
| std::vector<std::string> JSONArrayToVector(const std::string& json_array) { |
| std::optional<base::Value> json_value = base::JSONReader::Read(json_array); |
| |
| if (!json_value) { |
| return {}; |
| } |
| |
| base::Value::List* entries = json_value->GetIfList(); |
| if (!entries) { |
| return {}; |
| } |
| |
| std::vector<std::string> result; |
| result.reserve(entries->size()); |
| for (const base::Value& entry : *entries) { |
| const std::string* filter = entry.GetIfString(); |
| if (filter) { |
| result.emplace_back(*filter); |
| } |
| } |
| return result; |
| } |
| |
| LensOverlayController* GetLensOverlayControllerFromTabInterface( |
| tabs::TabInterface* tab_interface) { |
| return tab_interface |
| ? tab_interface->GetTabFeatures()->lens_overlay_controller() |
| : nullptr; |
| } |
| |
| bool IsPageContextEligible( |
| const GURL& main_frame_url, |
| std::vector<optimization_guide::FrameMetadata> frame_metadata, |
| optimization_guide::PageContextEligibility* page_context_eligibility) { |
| if (!page_context_eligibility || |
| !lens::features::IsLensSearchProtectedPageEnabled() || |
| !lens::features::IsLensOverlayContextualSearchboxEnabled() || |
| !lens::features::UseApcAsContext()) { |
| return true; |
| } |
| return page_context_eligibility->api().IsPageContextEligible( |
| main_frame_url.host(), main_frame_url.path(), std::move(frame_metadata)); |
| } |
| |
| } // namespace |
| |
| LensOverlayController::LensOverlayController( |
| tabs::TabInterface* tab, |
| LensSearchController* lens_search_controller, |
| variations::VariationsClient* variations_client, |
| signin::IdentityManager* identity_manager, |
| PrefService* pref_service, |
| syncer::SyncService* sync_service, |
| ThemeService* theme_service) |
| : tab_(tab), |
| lens_search_controller_(lens_search_controller), |
| variations_client_(variations_client), |
| identity_manager_(identity_manager), |
| pref_service_(pref_service), |
| sync_service_(sync_service), |
| theme_service_(theme_service), |
| gen204_controller_( |
| std::make_unique<lens::LensOverlayGen204Controller>()) { |
| InitializeTutorialIPHUrlMatcher(); |
| |
| // Listen to WebContents events |
| tab_contents_observer_ = std::make_unique<UnderlyingWebContentsObserver>( |
| tab_->GetContents(), this); |
| } |
| |
| LensOverlayController::~LensOverlayController() { |
| tab_contents_observer_.reset(); |
| state_ = State::kOff; |
| } |
| |
| DEFINE_CLASS_ELEMENT_IDENTIFIER_VALUE(LensOverlayController, kOverlayId); |
| DEFINE_CLASS_ELEMENT_IDENTIFIER_VALUE(LensOverlayController, |
| kOverlaySidePanelWebViewId); |
| |
| // static. |
| LensOverlayController* LensOverlayController::FromWebUIWebContents( |
| content::WebContents* webui_web_contents) { |
| return GetLensOverlayControllerFromTabInterface( |
| webui::GetTabInterface(webui_web_contents)); |
| } |
| |
| // static. |
| LensOverlayController* LensOverlayController::FromTabWebContents( |
| content::WebContents* tab_web_contents) { |
| return GetLensOverlayControllerFromTabInterface( |
| tabs::TabInterface::GetFromContents(tab_web_contents)); |
| } |
| |
| void LensOverlayController::TriggerOverlayCloseAnimation( |
| base::OnceClosure callback) { |
| if (state_ == State::kOff || IsOverlayClosing()) { |
| return; |
| } |
| |
| // Notify the overlay so it can do any animations or cleanup. The page_ is not |
| // guaranteed to exist if CloseUIAsync is called during the setup process. |
| if (page_) { |
| page_->NotifyOverlayClosing(); |
| } |
| |
| // Set a short 200ms timeout to give the fade out time to transition. |
| base::SequencedTaskRunner::GetCurrentDefault()->PostDelayedTask( |
| FROM_HERE, std::move(callback), kFadeoutAnimationTimeout); |
| } |
| |
| void LensOverlayController::CloseUI( |
| lens::LensOverlayDismissalSource dismissal_source) { |
| if (state_ == State::kOff) { |
| return; |
| } |
| |
| state_ = State::kClosing; |
| |
| // Closes preselection toast if it exists. |
| ClosePreselectionBubble(); |
| |
| // Notify the query controller to loose references to this classes data before |
| // it gets cleaned up to prevent dangling ptrs. |
| lens_overlay_query_controller_->ResetPageContentData(); |
| lens_overlay_query_controller_ = nullptr; |
| |
| // A permission prompt may be suspended if the overlay was showing when the |
| // permission was queued. Restore the suspended prompt if possible. |
| // TODO(crbug.com/331940245): Refactor to be decoupled from |
| // PermissionPromptFactory |
| content::WebContents* contents = tab_->GetContents(); |
| CHECK(contents); |
| auto* permission_request_manager = |
| permissions::PermissionRequestManager::FromWebContents(contents); |
| if (permission_request_manager && |
| permission_request_manager->CanRestorePrompt()) { |
| permission_request_manager->RestorePrompt(); |
| } |
| |
| results_side_panel_coordinator_ = nullptr; |
| side_panel_in_use_.reset(); |
| pre_initialization_suggest_inputs_.reset(); |
| pre_initialization_objects_.reset(); |
| pre_initialization_text_.reset(); |
| |
| side_panel_shown_subscription_ = base::CallbackListSubscription(); |
| side_panel_coordinator_ = nullptr; |
| |
| // Re-enable mouse and keyboard events to the tab contents web view. |
| auto* contents_web_view = tab_->GetBrowserWindowInterface()->GetWebView(); |
| CHECK(contents_web_view); |
| contents_web_view->SetEnabled(true); |
| |
| if (overlay_web_view_) { |
| // Remove render frame observer. |
| overlay_web_view_->GetWebContents() |
| ->GetPrimaryMainFrame() |
| ->GetProcess() |
| ->RemoveObserver(this); |
| } |
| |
| initialization_data_.reset(); |
| |
| tab_contents_view_observer_.Reset(); |
| omnibox_tab_helper_observer_.Reset(); |
| find_tab_observer_.Reset(); |
| receiver_.reset(); |
| page_.reset(); |
| languages_controller_.reset(); |
| scoped_tab_modal_ui_.reset(); |
| pending_region_.reset(); |
| fullscreen_observation_.Reset(); |
| immersive_mode_observer_.Reset(); |
| lens_overlay_blur_layer_delegate_.reset(); |
| last_navigation_time_.reset(); |
| #if BUILDFLAG(IS_MAC) |
| pref_change_registrar_.Reset(); |
| #endif // BUILDFLAG(IS_MAC) |
| |
| // Notify the searchbox controller to reset its handlers before the overlay |
| // is cleaned up. This is needed to prevent a dangling ptr. |
| GetLensSearchboxController()->ResetOverlaySearchboxHandler(); |
| |
| // Cleanup all of the lens overlay related views. The overlay view is owned by |
| // the browser view and is reused for each Lens overlay session. Clean it up |
| // so it is ready for the next invocation. |
| if (overlay_view_) { |
| overlay_view_->RemoveChildViewT( |
| std::exchange(preselection_widget_anchor_, nullptr)); |
| overlay_view_->RemoveChildViewT(std::exchange(overlay_web_view_, nullptr)); |
| MaybeHideSharedOverlayView(); |
| overlay_view_ = nullptr; |
| } |
| |
| lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; |
| should_show_overlay_ = true; |
| is_page_context_eligible_ = true; |
| should_send_screenshot_on_init_ = false; |
| |
| state_ = State::kOff; |
| |
| // Update the entrypoints now that the controller is closed. |
| UpdateEntryPointsState(); |
| |
| RecordEndOfSessionMetrics(dismissal_source); |
| } |
| |
| // static |
| const std::u16string LensOverlayController::GetFilenameForURL(const GURL& url) { |
| if (!url.has_host() || url.HostIsIPAddress()) { |
| return u"screenshot.png"; |
| } |
| |
| return base::ASCIIToUTF16(base::StrCat({"screenshot_", url.host(), ".png"})); |
| } |
| |
| void LensOverlayController::BindOverlay( |
| mojo::PendingReceiver<lens::mojom::LensPageHandler> receiver, |
| mojo::PendingRemote<lens::mojom::LensPage> page) { |
| if (state_ != State::kStartingWebUI) { |
| return; |
| } |
| receiver_.Bind(std::move(receiver)); |
| page_.Bind(std::move(page)); |
| |
| InitializeOverlay(/*initialization_data=*/nullptr); |
| } |
| |
| void LensOverlayController::BindOverlayGhostLoader( |
| mojo::PendingRemote<lens::mojom::LensGhostLoaderPage> page) { |
| overlay_ghost_loader_page_.reset(); |
| overlay_ghost_loader_page_.Bind(std::move(page)); |
| } |
| |
| void LensOverlayController::BindSidePanelGhostLoader( |
| mojo::PendingRemote<lens::mojom::LensGhostLoaderPage> page) { |
| side_panel_ghost_loader_page_.reset(); |
| side_panel_ghost_loader_page_.Bind(std::move(page)); |
| } |
| |
| uint64_t LensOverlayController::GetInvocationTimeSinceEpoch() { |
| return invocation_time_since_epoch_.InMillisecondsSinceUnixEpoch(); |
| } |
| |
| views::View* LensOverlayController::GetOverlayViewForTesting() { |
| return overlay_view_.get(); |
| } |
| |
| views::WebView* LensOverlayController::GetOverlayWebViewForTesting() { |
| return overlay_web_view_.get(); |
| } |
| |
| void LensOverlayController::SendText(lens::mojom::TextPtr text) { |
| if (!page_) { |
| // Store the text to send once the page is bound. |
| pre_initialization_text_ = std::move(text); |
| return; |
| } |
| page_->TextReceived(std::move(text)); |
| } |
| |
| lens::mojom::OverlayThemePtr LensOverlayController::CreateTheme( |
| lens::PaletteId palette_id) { |
| CHECK(base::Contains(lens::kPaletteColors, palette_id)); |
| const auto& palette = lens::kPaletteColors.at(palette_id); |
| auto theme = lens::mojom::OverlayTheme::New(); |
| theme->primary = palette.at(lens::ColorId::kPrimary); |
| theme->shader_layer_1 = palette.at(lens::ColorId::kShaderLayer1); |
| theme->shader_layer_2 = palette.at(lens::ColorId::kShaderLayer2); |
| theme->shader_layer_3 = palette.at(lens::ColorId::kShaderLayer3); |
| theme->shader_layer_4 = palette.at(lens::ColorId::kShaderLayer4); |
| theme->shader_layer_5 = palette.at(lens::ColorId::kShaderLayer5); |
| theme->scrim = palette.at(lens::ColorId::kScrim); |
| theme->surface_container_highest_light = |
| palette.at(lens::ColorId::kSurfaceContainerHighestLight); |
| theme->surface_container_highest_dark = |
| palette.at(lens::ColorId::kSurfaceContainerHighestDark); |
| theme->selection_element = palette.at(lens::ColorId::kSelectionElement); |
| return theme; |
| } |
| |
| void LensOverlayController::SendObjects( |
| std::vector<lens::mojom::OverlayObjectPtr> objects) { |
| if (!page_) { |
| // Store the objects to send once the page is bound. |
| pre_initialization_objects_ = std::move(objects); |
| return; |
| } |
| page_->ObjectsReceived(std::move(objects)); |
| } |
| |
| void LensOverlayController::NotifyResultsPanelOpened() { |
| page_->NotifyResultsPanelOpened(); |
| } |
| |
| void LensOverlayController::TriggerCopy() { |
| // This prevents a race condition where the overlay is closed as a keyboard |
| // event is being processed. |
| if (!page_) { |
| return; |
| } |
| page_->OnCopyCommand(); |
| } |
| |
| bool LensOverlayController::IsOverlayShowing() const { |
| return state_ == State::kStartingWebUI || state_ == State::kOverlay || |
| state_ == State::kOverlayAndResults; |
| } |
| |
| bool LensOverlayController::IsOverlayActive() const { |
| return IsOverlayShowing() || state_ == State::kLivePageAndResults; |
| } |
| |
| bool LensOverlayController::IsOverlayInitializing() { |
| return state_ == State::kStartingWebUI || state_ == State::kScreenshot || |
| state_ == State::kClosingOpenedSidePanel; |
| } |
| |
| bool LensOverlayController::IsOverlayClosing() { |
| return state_ == State::kClosing; |
| } |
| |
| bool LensOverlayController::IsScreenshotPossible( |
| content::RenderWidgetHostView* view) { |
| return view && view->IsSurfaceAvailableForCopy(); |
| } |
| |
| tabs::TabInterface* LensOverlayController::GetTabInterface() { |
| return tab_; |
| } |
| |
| void LensOverlayController::IssueLensRegionRequestForTesting( |
| lens::mojom::CenterRotatedBoxPtr region, |
| bool is_click) { |
| IssueLensRegionRequest(std::move(region), is_click); |
| } |
| |
| void LensOverlayController::IssueTextSelectionRequestForTesting( |
| const std::string& text_query, |
| int selection_start_index, |
| int selection_end_index, |
| bool is_translate) { |
| IssueTextSelectionRequest(text_query, selection_start_index, |
| selection_end_index, is_translate); |
| } |
| |
| void LensOverlayController:: |
| RecordUkmAndTaskCompletionForLensOverlayInteractionForTesting( |
| lens::mojom::UserAction user_action) { |
| RecordUkmAndTaskCompletionForLensOverlayInteraction(user_action); |
| } |
| |
| void LensOverlayController::RecordSemanticEventForTesting( |
| lens::mojom::SemanticEvent event) { |
| RecordLensOverlaySemanticEvent(event); |
| } |
| |
| void LensOverlayController::IssueSearchBoxRequestForTesting( |
| const std::string& search_box_text, |
| AutocompleteMatchType::Type match_type, |
| bool is_zero_prefix_suggestion, |
| std::map<std::string, std::string> additional_query_params) { |
| IssueSearchBoxRequest(search_box_text, match_type, is_zero_prefix_suggestion, |
| additional_query_params); |
| } |
| |
| void LensOverlayController::IssueTranslateSelectionRequestForTesting( |
| const std::string& text_query, |
| const std::string& content_language, |
| int selection_start_index, |
| int selection_end_index) { |
| IssueTranslateSelectionRequest(text_query, content_language, |
| selection_start_index, selection_end_index); |
| } |
| |
| void LensOverlayController::IssueMathSelectionRequestForTesting( |
| const std::string& query, |
| const std::string& formula, |
| int selection_start_index, |
| int selection_end_index) { |
| IssueMathSelectionRequest(query, formula, selection_start_index, |
| selection_end_index); |
| } |
| |
| void LensOverlayController::IssueTranslateFullPageRequestForTesting( |
| const std::string& source_language, |
| const std::string& target_language) { |
| IssueTranslateFullPageRequest(source_language, target_language); |
| } |
| |
| void LensOverlayController::IssueEndTranslateModeRequestForTesting() { |
| IssueEndTranslateModeRequest(); |
| } |
| |
| void LensOverlayController::IssueTranslateFullPageRequest( |
| const std::string& source_language, |
| const std::string& target_language) { |
| // Remove the selection thumbnail, if it exists. |
| GetLensSearchboxController()->SetSearchboxThumbnail(std::string()); |
| ClearRegionSelection(); |
| // Set the coachmark text. |
| if (preselection_widget_) { |
| // This cast is safe since we know the widget delegate will always be a |
| // `lens::LensPreselectionBubble`. |
| auto* bubble_view = static_cast<lens::LensPreselectionBubble*>( |
| preselection_widget_->widget_delegate()); |
| bubble_view->SetLabelText( |
| IDS_LENS_OVERLAY_INITIAL_TOAST_MESSAGE_SELECT_TEXT); |
| } |
| // Set the translate options on initialization data in case we need to |
| // re-enable translate mode later. |
| initialization_data_->translate_options_ = |
| lens::TranslateOptions(source_language, target_language); |
| |
| lens_overlay_query_controller_->SendFullPageTranslateQuery(source_language, |
| target_language); |
| MaybeLaunchSurvey(); |
| } |
| |
| void LensOverlayController::IssueEndTranslateModeRequest() { |
| // Reset the coachmark text back to default. |
| if (preselection_widget_) { |
| // This cast is safe since we know the widget delegate will always be a |
| // `lens::LensPreselectionBubble`. |
| auto* bubble_view = static_cast<lens::LensPreselectionBubble*>( |
| preselection_widget_->widget_delegate()); |
| bubble_view->SetLabelText(IDS_LENS_OVERLAY_INITIAL_TOAST_MESSAGE); |
| } |
| lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; |
| initialization_data_->selected_text_.reset(); |
| initialization_data_->translate_options_.reset(); |
| lens_overlay_query_controller_->SendEndTranslateModeQuery(); |
| } |
| |
| void LensOverlayController::NotifyOverlayInitialized() { |
| // Now that the overlay is actually showing, it is safe to start doing a Lens |
| // request without showing the page reflowing. |
| if (pending_region_) { |
| // If there is a pending region (i.e. for image right click) |
| // use INJECTED_IMAGE as the selection type. |
| IssueLensRequest(std::move(pending_region_), lens::INJECTED_IMAGE, |
| pending_region_bitmap_); |
| pending_region_bitmap_.reset(); |
| } |
| } |
| |
| void LensOverlayController::CopyText(const std::string& text) { |
| ui::ScopedClipboardWriter clipboard_writer(ui::ClipboardBuffer::kCopyPaste); |
| clipboard_writer.WriteText(base::UTF8ToUTF16(text)); |
| } |
| |
| void LensOverlayController::CopyImage(lens::mojom::CenterRotatedBoxPtr region) { |
| if (initialization_data_->initial_screenshot_.drawsNothing()) { |
| return; |
| } |
| |
| SkBitmap cropped = lens::CropBitmapToRegion( |
| initialization_data_->initial_screenshot_, std::move(region)); |
| ui::ScopedClipboardWriter clipboard_writer(ui::ClipboardBuffer::kCopyPaste); |
| clipboard_writer.WriteImage(cropped); |
| } |
| |
| void LensOverlayController::RecordUkmAndTaskCompletionForLensOverlayInteraction( |
| lens::mojom::UserAction user_action) { |
| ukm::SourceId source_id = |
| tab_->GetContents()->GetPrimaryMainFrame()->GetPageUkmSourceId(); |
| ukm::builders::Lens_Overlay_Overlay_UserAction(source_id) |
| .SetUserAction(static_cast<int64_t>(user_action)) |
| .Record(ukm::UkmRecorder::Get()); |
| lens_overlay_query_controller_->SendTaskCompletionGen204IfEnabled( |
| user_action); |
| } |
| |
| void LensOverlayController::RecordLensOverlaySemanticEvent( |
| lens::mojom::SemanticEvent event) { |
| lens_overlay_query_controller_->SendSemanticEventGen204IfEnabled(event); |
| } |
| |
| void LensOverlayController::SaveAsImage( |
| lens::mojom::CenterRotatedBoxPtr region) { |
| SkBitmap cropped = lens::CropBitmapToRegion( |
| initialization_data_->initial_screenshot_, std::move(region)); |
| const GURL data_url = GURL(webui::GetBitmapDataUrl(cropped)); |
| content::DownloadManager* download_manager = |
| tab_->GetBrowserWindowInterface()->GetProfile()->GetDownloadManager(); |
| net::NetworkTrafficAnnotationTag traffic_annotation = |
| net::DefineNetworkTrafficAnnotation("lens_overlay_save", R"( |
| semantics { |
| sender: "Lens Overlay" |
| description: |
| "The user may capture a selection of the current screenshot in the " |
| "Lens overlay via a button in the overlay. The resulting image is " |
| "saved from a data URL to the disk on the local client." |
| trigger: "User clicks 'Save as image' in the Lens Overlay after " |
| "activating the Lens Overlay and making a selection on the " |
| "screenshot." |
| data: "A capture of a portion of a screenshot of the current page." |
| destination: LOCAL |
| last_reviewed: "2024-08-23" |
| user_data { |
| type: WEB_CONTENT |
| } |
| internal { |
| contacts { |
| owners: "//chrome/browser/ui/lens/OWNERS" |
| } |
| } |
| } |
| policy { |
| cookies_allowed: NO |
| setting: |
| "No user-visible setting for this feature. Configured via Finch." |
| policy_exception_justification: |
| "This is not a network request." |
| })"); |
| std::unique_ptr<download::DownloadUrlParameters> params = |
| content::DownloadRequestUtils::CreateDownloadForWebContentsMainFrame( |
| overlay_web_view_->GetWebContents(), data_url, traffic_annotation); |
| params->set_prompt(true); |
| params->set_suggested_name( |
| GetFilenameForURL(tab_->GetContents()->GetLastCommittedURL())); |
| download_manager->DownloadUrl(std::move(params)); |
| } |
| |
| void LensOverlayController::MaybeShowTranslateFeaturePromo() { |
| auto* tracker = ui::ElementTracker::GetElementTracker(); |
| translate_button_shown_subscription_ = |
| tracker->AddElementShownInAnyContextCallback( |
| kLensOverlayTranslateButtonElementId, |
| base::BindRepeating( |
| &LensOverlayController::TryShowTranslateFeaturePromo, |
| weak_factory_.GetWeakPtr())); |
| } |
| |
| void LensOverlayController::MaybeCloseTranslateFeaturePromo( |
| bool feature_engaged) { |
| if (auto* const interface = |
| BrowserUserEducationInterface::MaybeGetForWebContentsInTab( |
| tab_->GetContents())) { |
| if (!interface->IsFeaturePromoActive( |
| feature_engagement::kIPHLensOverlayTranslateButtonFeature)) { |
| // Do nothing if feature promo is not active. |
| return; |
| } |
| |
| if (feature_engaged) { |
| interface->NotifyFeaturePromoFeatureUsed( |
| feature_engagement::kIPHLensOverlayTranslateButtonFeature, |
| FeaturePromoFeatureUsedAction::kClosePromoIfPresent); |
| } else { |
| interface->AbortFeaturePromo( |
| feature_engagement::kIPHLensOverlayTranslateButtonFeature); |
| } |
| } |
| } |
| |
| void LensOverlayController::FetchSupportedLanguages( |
| FetchSupportedLanguagesCallback callback) { |
| CHECK(languages_controller_); |
| languages_controller_->SendGetSupportedLanguagesRequest(std::move(callback)); |
| } |
| |
| void LensOverlayController::TryShowTranslateFeaturePromo( |
| ui::TrackedElement* element) { |
| if (!element) { |
| return; |
| } |
| |
| if (auto* const interface = |
| BrowserUserEducationInterface::MaybeGetForWebContentsInTab( |
| tab_->GetContents())) { |
| interface->MaybeShowFeaturePromo( |
| feature_engagement::kIPHLensOverlayTranslateButtonFeature); |
| } |
| } |
| |
| std::string LensOverlayController::GetInvocationSourceString() { |
| return lens::InvocationSourceToString(invocation_source_); |
| } |
| |
| content::WebContents* |
| LensOverlayController::GetSidePanelWebContentsForTesting() { |
| if (!results_side_panel_coordinator_) { |
| return nullptr; |
| } |
| return results_side_panel_coordinator_->GetSidePanelWebContents(); |
| } |
| |
| const GURL& LensOverlayController::GetPageURLForTesting() { |
| return lens_search_controller_->GetPageURL(); |
| } |
| |
| SessionID LensOverlayController::GetTabIdForTesting() { |
| return GetLensSearchboxController()->GetTabId(); |
| } |
| |
| metrics::OmniboxEventProto::PageClassification |
| LensOverlayController::GetPageClassificationForTesting() { |
| return GetPageClassification(); |
| } |
| |
| const std::string& LensOverlayController::GetThumbnailForTesting() { |
| return GetLensSearchboxController()->GetThumbnail(); |
| } |
| |
| void LensOverlayController::OnTextModifiedForTesting() { |
| GetLensSearchboxController()->OnTextModified(); |
| } |
| |
| void LensOverlayController::OnThumbnailRemovedForTesting() { |
| GetLensSearchboxController()->OnThumbnailRemoved(); |
| } |
| |
| void LensOverlayController::OnFocusChangedForTesting(bool focused) { |
| GetLensSearchboxController()->OnFocusChanged(focused); |
| } |
| |
| void LensOverlayController::OnZeroSuggestShownForTesting() { |
| OnZeroSuggestShown(); |
| } |
| |
| void LensOverlayController::OpenSidePanelForTesting() { |
| MaybeOpenSidePanel(); |
| } |
| |
| const lens::proto::LensOverlaySuggestInputs& |
| LensOverlayController::GetLensSuggestInputsForTesting() { |
| return GetLensSuggestInputs(); |
| } |
| |
| bool LensOverlayController::IsUrlEligibleForTutorialIPHForTesting( |
| const GURL& url) { |
| return IsUrlEligibleForTutorialIPH(url); |
| } |
| |
| void LensOverlayController::ShowUI( |
| lens::LensOverlayInvocationSource invocation_source, |
| lens::LensOverlayQueryController* lens_overlay_query_controller) { |
| // If UI is already showing or in the process of showing, do nothing. |
| if (state_ != State::kOff) { |
| return; |
| } |
| |
| // The UI should only show if the tab is in the foreground or if the tab web |
| // contents is not in a crash state. |
| if (!tab_->IsActivated() || tab_->GetContents()->IsCrashed()) { |
| return; |
| } |
| |
| // If a different tab-modal is showing, do nothing. |
| if (!tab_->CanShowModalUI()) { |
| return; |
| } |
| |
| // Increment the counter for the number of times the Lens Overlay has been |
| // started. |
| int lens_overlay_start_count = |
| pref_service_->GetInteger(prefs::kLensOverlayStartCount); |
| pref_service_->SetInteger(prefs::kLensOverlayStartCount, |
| lens_overlay_start_count + 1); |
| |
| // Store reference for later use. |
| invocation_source_ = invocation_source; |
| lens_overlay_query_controller_ = lens_overlay_query_controller; |
| |
| // Grab reference to the side panel coordinator it not already done so. |
| if (!results_side_panel_coordinator_) { |
| results_side_panel_coordinator_ = |
| lens_search_controller_->lens_overlay_side_panel_coordinator(); |
| } |
| |
| Profile* profile = |
| Profile::FromBrowserContext(tab_->GetContents()->GetBrowserContext()); |
| side_panel_coordinator_ = |
| tab_->GetBrowserWindowInterface()->GetFeatures().side_panel_coordinator(); |
| CHECK(side_panel_coordinator_); |
| |
| // Create the languages controller. |
| languages_controller_ = |
| std::make_unique<lens::LensOverlayLanguagesController>(profile); |
| |
| // Setup observer to be notified of side panel opens and closes. |
| side_panel_shown_subscription_ = |
| side_panel_coordinator_->RegisterSidePanelShown( |
| base::BindRepeating(&LensOverlayController::OnSidePanelDidOpen, |
| weak_factory_.GetWeakPtr())); |
| |
| if (find_in_page::FindTabHelper* const find_tab_helper = |
| find_in_page::FindTabHelper::FromWebContents(tab_->GetContents())) { |
| find_tab_observer_.Observe(find_tab_helper); |
| } |
| |
| if (!omnibox_tab_helper_observer_.IsObserving()) { |
| if (auto* helper = OmniboxTabHelper::FromWebContents(tab_->GetContents())) { |
| omnibox_tab_helper_observer_.Observe(helper); |
| } |
| } |
| |
| // This is safe because we checked if another modal was showing above. |
| scoped_tab_modal_ui_ = tab_->ShowModalUI(); |
| fullscreen_observation_.Observe(tab_->GetBrowserWindowInterface() |
| ->GetExclusiveAccessManager() |
| ->fullscreen_controller()); |
| |
| // The preselection widget can cover top Chrome in immersive fullscreen. |
| // Observer the reveal state to hide the widget when top Chrome is shown. |
| immersive_mode_observer_.Observe( |
| tab_->GetBrowserWindowInterface()->GetImmersiveModeController()); |
| |
| #if BUILDFLAG(IS_MAC) |
| // Add observer to listen for changes in the always show toolbar state, |
| // since that requires the preselection bubble to rerender to show properly. |
| pref_change_registrar_.Init(pref_service_); |
| pref_change_registrar_.Add( |
| prefs::kShowFullscreenToolbar, |
| base::BindRepeating( |
| &LensOverlayController::CloseAndReshowPreselectionBubble, |
| base::Unretained(this))); |
| #endif // BUILDFLAG(IS_MAC) |
| |
| NotifyUserEducationAboutOverlayUsed(); |
| |
| // Establish data required for session metrics. |
| search_performed_in_session_ = false; |
| invocation_time_ = base::TimeTicks::Now(); |
| invocation_time_since_epoch_ = base::Time::Now(); |
| hats_triggered_in_session_ = false; |
| ocr_dom_similarity_recorded_in_session_ = false; |
| |
| // This should be the last thing called in ShowUI, so if something goes wrong |
| // in capturing the screenshot, the state gets cleaned up correctly. |
| if (side_panel_coordinator_->IsSidePanelShowing()) { |
| // Close the currently opened side panel synchronously. Postpone the |
| // screenshot for a fixed time to allow reflow. |
| state_ = State::kClosingOpenedSidePanel; |
| side_panel_coordinator_->Close(/*suppress_animations=*/true); |
| base::SingleThreadTaskRunner::GetCurrentDefault() |
| ->PostNonNestableDelayedTask( |
| FROM_HERE, |
| base::BindOnce(&LensOverlayController::FinishedWaitingForReflow, |
| weak_factory_.GetWeakPtr()), |
| kReflowWaitTimeout); |
| } else { |
| CaptureScreenshot(); |
| } |
| } |
| |
| void LensOverlayController::IssueContextualSearchRequest( |
| const GURL& destination_url, |
| lens::LensOverlayQueryController* lens_overlay_query_controller, |
| AutocompleteMatchType::Type match_type, |
| bool is_zero_prefix_suggestion, |
| lens::LensOverlayInvocationSource invocation_source) { |
| // Ignore the request if the overlay is off or closing. |
| if (IsOverlayClosing()) { |
| return; |
| } |
| |
| // If the overlay is off, turn it on so the request can be fulfilled. |
| if (state_ == State::kOff) { |
| // TODO(crbug.com/403573362): This is a temporary fix to unblock |
| // prototyping. Since this flow goes straight to the side panel results with |
| // not overlay UI, this flow does a lot of unnecessary work. There should be |
| // a new flow that can contextualize without the overlay UI being |
| // initialized. |
| StartContextualizationWithoutOverlay(invocation_source, |
| lens_overlay_query_controller); |
| } |
| |
| // Hold the request until the overlay has finished initializing. |
| if (IsOverlayInitializing()) { |
| pending_contextual_search_request_ = |
| base::BindOnce(&LensOverlayController::IssueContextualSearchRequest, |
| weak_factory_.GetWeakPtr(), destination_url, |
| lens_overlay_query_controller, match_type, |
| is_zero_prefix_suggestion, invocation_source); |
| return; |
| } |
| |
| // TODO(crbug.com/401583049): Revisit if this should go through the |
| // OnSuggestionAccepted flow or if there should be a more direct contextual |
| // search flow. |
| GetLensSearchboxController()->OnSuggestionAccepted( |
| destination_url, match_type, is_zero_prefix_suggestion); |
| } |
| |
| void LensOverlayController::StartContextualizationWithoutOverlay( |
| lens::LensOverlayInvocationSource invocation_source, |
| lens::LensOverlayQueryController* lens_overlay_query_controller) { |
| should_show_overlay_ = false; |
| ShowUI(invocation_source, lens_overlay_query_controller); |
| } |
| |
| void LensOverlayController::ShowUIWithPendingRegion( |
| lens::LensOverlayQueryController* lens_overlay_query_controller, |
| lens::LensOverlayInvocationSource invocation_source, |
| lens::mojom::CenterRotatedBoxPtr region, |
| const SkBitmap& region_bitmap) { |
| pending_region_ = std::move(region); |
| pending_region_bitmap_ = region_bitmap; |
| ShowUI(invocation_source, lens_overlay_query_controller); |
| // Overrides value set in ShowUI since invoking lens overlay with a pending |
| // region is considered a search. |
| search_performed_in_session_ = true; |
| } |
| |
| std::string LensOverlayController::GetVsridForNewTab() { |
| return lens_overlay_query_controller_->GetVsridForNewTab(); |
| } |
| |
| void LensOverlayController::SetTranslateMode( |
| std::optional<lens::TranslateOptions> translate_options) { |
| if (translate_options.has_value()) { |
| page_->SetTranslateMode(translate_options->source_language, |
| translate_options->target_language); |
| } else { |
| // If the overlay was previously in translate mode, send a |
| // request to end translate mode so the WebUI can update its state. |
| if (initialization_data_->translate_options_.has_value()) { |
| IssueEndTranslateModeRequest(); |
| results_side_panel_coordinator_->SetSidePanelIsLoadingResults(true); |
| } |
| // Disable translate mode by setting source and target languages to empty |
| // strings. This is a no-op if translate mode is already disabled. |
| page_->SetTranslateMode(std::string(), std::string()); |
| } |
| // Store the latest translate options. |
| initialization_data_->translate_options_ = translate_options; |
| } |
| |
| void LensOverlayController::SetTextSelection(int32_t selection_start_index, |
| int32_t selection_end_index) { |
| page_->SetTextSelection(selection_start_index, selection_end_index); |
| initialization_data_->selected_text_ = |
| std::make_pair(selection_start_index, selection_end_index); |
| } |
| |
| void LensOverlayController::SetPostRegionSelection( |
| lens::mojom::CenterRotatedBoxPtr box) { |
| page_->SetPostRegionSelection(box->Clone()); |
| initialization_data_->selected_region_ = std::move(box); |
| } |
| |
| void LensOverlayController::SetAdditionalSearchQueryParams( |
| std::map<std::string, std::string> additional_search_query_params) { |
| initialization_data_->additional_search_query_params_ = |
| additional_search_query_params; |
| } |
| |
| void LensOverlayController::ClearTextSelection() { |
| if(!IsOverlayShowing()) { |
| return; |
| } |
| if (initialization_data_->selected_text_.has_value()) { |
| initialization_data_->selected_text_.reset(); |
| page_->ClearTextSelection(); |
| } |
| } |
| |
| void LensOverlayController::ClearRegionSelection() { |
| if(!IsOverlayShowing()) { |
| return; |
| } |
| GetLensSearchboxController()->SetSearchboxThumbnail(""); |
| lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; |
| initialization_data_->selected_region_.reset(); |
| initialization_data_->selected_region_bitmap_.reset(); |
| page_->ClearRegionSelection(); |
| } |
| |
| void LensOverlayController::ClearAllSelections() { |
| page_->ClearAllSelections(); |
| initialization_data_->selected_region_.reset(); |
| initialization_data_->selected_region_bitmap_.reset(); |
| initialization_data_->selected_text_.reset(); |
| if (!IsContextualSearchbox()) { |
| lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; |
| } |
| } |
| |
| void LensOverlayController::OnSearchboxFocusChanged(bool focused) { |
| if (!focused) { |
| return; |
| } |
| |
| if (IsContextualSearchbox()) { |
| if (!csb_session_end_metrics_.searchbox_focused_) { |
| // This is the first time the searchbox is focused in this session. |
| // Record the time between the overlay being invoked and the searchbox |
| // being focused. |
| lens::RecordContextualSearchboxTimeToFirstFocus( |
| base::TimeTicks::Now() - invocation_time_, |
| initialization_data_->primary_content_type_); |
| } else { |
| RecordContextualSearchboxTimeToFocusAfterNavigation(); |
| } |
| csb_session_end_metrics_.searchbox_focused_ = true; |
| |
| if (state() == State::kLivePageAndResults) { |
| // If the live page is showing and the searchbox becomes focused, showing |
| // intent to issue a new query, upload the new page content for |
| // contextualization. |
| TryUpdatePageContextualization(); |
| } |
| } |
| } |
| |
| void LensOverlayController::ShowGhostLoaderErrorState() { |
| if (!IsContextualSearchbox()) { |
| return; |
| } |
| if (overlay_ghost_loader_page_) { |
| overlay_ghost_loader_page_->ShowErrorState(); |
| } |
| if (side_panel_ghost_loader_page_) { |
| side_panel_ghost_loader_page_->ShowErrorState(); |
| } |
| } |
| |
| void LensOverlayController::OnZeroSuggestShown() { |
| if (!IsContextualSearchbox()) { |
| return; |
| } |
| |
| if (state() == State::kOverlay) { |
| csb_session_end_metrics_.zps_shown_on_initial_query_ = true; |
| } else { |
| csb_session_end_metrics_.zps_shown_on_follow_up_query_ = true; |
| } |
| } |
| |
| void LensOverlayController::IssueLensRequest( |
| lens::mojom::CenterRotatedBoxPtr region, |
| lens::LensOverlaySelectionType selection_type, |
| std::optional<SkBitmap> region_bytes) { |
| CHECK(initialization_data_); |
| CHECK(region); |
| GetLensSearchboxController()->SetSearchboxInputText(std::string()); |
| initialization_data_->selected_region_ = region.Clone(); |
| initialization_data_->selected_text_.reset(); |
| initialization_data_->additional_search_query_params_.clear(); |
| lens_selection_type_ = selection_type; |
| if (region_bytes) { |
| initialization_data_->selected_region_bitmap_ = region_bytes.value(); |
| } else { |
| initialization_data_->selected_region_bitmap_.reset(); |
| } |
| |
| if (is_page_context_eligible_) { |
| lens_overlay_query_controller_->SendRegionSearch( |
| region.Clone(), selection_type, |
| initialization_data_->additional_search_query_params_, region_bytes); |
| } |
| MaybeOpenSidePanel(); |
| RecordTimeToFirstInteraction( |
| lens::LensOverlayFirstInteractionType::kRegionSelect); |
| state_ = State::kOverlayAndResults; |
| MaybeLaunchSurvey(); |
| } |
| |
| void LensOverlayController::IssueMultimodalRequest( |
| lens::mojom::CenterRotatedBoxPtr region, |
| const std::string& text_query, |
| lens::LensOverlaySelectionType selection_type, |
| std::optional<SkBitmap> region_bitmap) { |
| if (is_page_context_eligible_) { |
| lens_overlay_query_controller_->SendMultimodalRequest( |
| std::move(region), text_query, selection_type, |
| initialization_data_->additional_search_query_params_, region_bitmap); |
| } |
| } |
| |
| void LensOverlayController::IssueSearchBoxRequest( |
| const std::string& search_box_text, |
| AutocompleteMatchType::Type match_type, |
| bool is_zero_prefix_suggestion, |
| std::map<std::string, std::string> additional_query_params) { |
| // Log the interaction time here so the time to fetch new page bytes is not |
| // intcluded. |
| RecordContextualSearchboxTimeToInteractionAfterNavigation(); |
| RecordTimeToFirstInteraction( |
| lens::LensOverlayFirstInteractionType::kSearchbox); |
| |
| // Do not attempt to contextualize if CSB is disabled, if recontextualization |
| // on each query is disabled, if the live page is not being displayed, or if |
| // the user is not in the contextual search flow (aka, issues an image request |
| // already). |
| if (!lens::features::IsLensOverlayContextualSearchboxEnabled() || |
| !lens::features::ShouldLensOverlayRecontextualizeOnQuery() || |
| state() != State::kLivePageAndResults || !IsContextualSearchbox()) { |
| IssueSearchBoxRequestPart2(search_box_text, match_type, |
| is_zero_prefix_suggestion, |
| additional_query_params); |
| return; |
| } |
| |
| // If contextual searchbox is enabled, make sure the page bytes are current |
| // prior to issuing the search box request. |
| GetContextualizationController()->GetPageContextualization( |
| base::BindOnce(&LensOverlayController::UpdatePageContextualization, |
| weak_factory_.GetWeakPtr()) |
| .Then(base::BindOnce( |
| &LensOverlayController::IssueSearchBoxRequestPart2, |
| weak_factory_.GetWeakPtr(), search_box_text, match_type, |
| is_zero_prefix_suggestion, additional_query_params))); |
| } |
| |
| void LensOverlayController::IssueContextualTextRequest( |
| const std::string& text_query, |
| lens::LensOverlaySelectionType selection_type) { |
| if (is_page_context_eligible_) { |
| lens_overlay_query_controller_->SendContextualTextQuery( |
| text_query, selection_type, |
| initialization_data_->additional_search_query_params_); |
| } |
| } |
| |
| void LensOverlayController::AddOverlayStateToSearchQuery( |
| lens::SearchQuery& search_query) { |
| // In the case where a query was triggered by a selection on the overlay or |
| // use of the searchbox, initialization_data_ and |
| // additional_search_query_params_ will have already been set. Record that |
| // state in a search query struct. |
| if (initialization_data_->selected_region_) { |
| search_query.selected_region_ = |
| initialization_data_->selected_region_->Clone(); |
| } |
| if (!initialization_data_->selected_region_bitmap_.drawsNothing()) { |
| search_query.selected_region_bitmap_ = |
| initialization_data_->selected_region_bitmap_; |
| } |
| if (initialization_data_->selected_text_.has_value()) { |
| search_query.selected_text_ = initialization_data_->selected_text_.value(); |
| } |
| if (initialization_data_->translate_options_.has_value()) { |
| search_query.translate_options_ = |
| initialization_data_->translate_options_.value(); |
| } |
| search_query.lens_selection_type_ = lens_selection_type_; |
| search_query.additional_search_query_params_ = |
| initialization_data_->additional_search_query_params_; |
| } |
| |
| LensOverlayController::OverlayInitializationData::OverlayInitializationData( |
| const SkBitmap& screenshot, |
| SkBitmap rgb_screenshot, |
| lens::PaletteId color_palette, |
| GURL page_url, |
| std::optional<std::string> page_title) |
| : initial_screenshot_(screenshot), |
| initial_rgb_screenshot_(std::move(rgb_screenshot)), |
| updated_screenshot_(screenshot), |
| color_palette_(color_palette), |
| page_url_(page_url), |
| page_title_(page_title) {} |
| LensOverlayController::OverlayInitializationData::~OverlayInitializationData() = |
| default; |
| |
| class LensOverlayController::UnderlyingWebContentsObserver |
| : public content::WebContentsObserver { |
| public: |
| UnderlyingWebContentsObserver(content::WebContents* web_contents, |
| LensOverlayController* lens_overlay_controller) |
| : content::WebContentsObserver(web_contents), |
| lens_overlay_controller_(lens_overlay_controller) {} |
| |
| ~UnderlyingWebContentsObserver() override = default; |
| |
| UnderlyingWebContentsObserver(const UnderlyingWebContentsObserver&) = delete; |
| UnderlyingWebContentsObserver& operator=( |
| const UnderlyingWebContentsObserver&) = delete; |
| |
| // content::WebContentsObserver |
| void DidFinishNavigation( |
| content::NavigationHandle* navigation_handle) override { |
| // If the overlay is off, check if we should display IPH. |
| if (lens_overlay_controller_->state() == State::kOff) { |
| // Only check IPH eligibility if the navigation changed the primary page. |
| if (base::FeatureList::IsEnabled( |
| feature_engagement::kIPHLensOverlayFeature) && |
| navigation_handle->IsInPrimaryMainFrame() && |
| !navigation_handle->IsSameDocument() && |
| navigation_handle->HasCommitted()) { |
| lens_overlay_controller_->MaybeShowDelayedTutorialIPH( |
| navigation_handle->GetURL()); |
| } |
| return; |
| } |
| |
| // If the overlay is open, check if we should close it. |
| bool is_user_reload = |
| navigation_handle->GetReloadType() != content::ReloadType::NONE && |
| !navigation_handle->IsRendererInitiated(); |
| // We don't need to close if: |
| // 1) The navigation is not for the main page. |
| // 2) The navigation hasn't been committed yet. |
| // 3) The URL did not change and the navigation wasn't the user reloading |
| // the page. |
| if (!navigation_handle->IsInPrimaryMainFrame() || |
| !navigation_handle->HasCommitted() || |
| (navigation_handle->GetPreviousPrimaryMainFrameURL() == |
| navigation_handle->GetURL() && |
| !is_user_reload)) { |
| return; |
| } |
| if (lens_overlay_controller_->state() == State::kLivePageAndResults) { |
| lens_overlay_controller_->UpdateNavigationMetrics(); |
| lens_overlay_controller_->NotifyPageContentUpdated(); |
| return; |
| } |
| lens_overlay_controller_->lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kPageChanged); |
| } |
| |
| void PrimaryMainFrameRenderProcessGone( |
| base::TerminationStatus status) override { |
| // Exit early if the overlay is off or already closing. |
| if (lens_overlay_controller_->state() == State::kOff || |
| lens_overlay_controller_->IsOverlayClosing()) { |
| return; |
| } |
| |
| lens_overlay_controller_->lens_search_controller_->CloseLensSync( |
| status == base::TERMINATION_STATUS_NORMAL_TERMINATION |
| ? lens::LensOverlayDismissalSource::kPageRendererClosedNormally |
| : lens::LensOverlayDismissalSource:: |
| kPageRendererClosedUnexpectedly); |
| } |
| |
| private: |
| raw_ptr<LensOverlayController> lens_overlay_controller_; |
| }; |
| |
| void LensOverlayController::CaptureScreenshot() { |
| state_ = State::kScreenshot; |
| |
| // Begin the process of grabbing a screenshot. |
| content::RenderWidgetHostView* view = tab_->GetContents() |
| ->GetPrimaryMainFrame() |
| ->GetRenderViewHost() |
| ->GetWidget() |
| ->GetView(); |
| |
| // During initialization and shutdown a capture may not be possible. |
| if (!IsScreenshotPossible(view)) { |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kErrorScreenshotCreationFailed); |
| return; |
| } |
| |
| // Side panel is now full closed, take screenshot and open overlay. |
| view->CopyFromSurface( |
| /*src_rect=*/gfx::Rect(), /*output_size=*/gfx::Size(), |
| base::BindPostTask( |
| base::SequencedTaskRunner::GetCurrentDefault(), |
| base::BindOnce( |
| &LensOverlayController::FetchViewportImageBoundingBoxes, |
| weak_factory_.GetWeakPtr()))); |
| } |
| |
| void LensOverlayController::FetchViewportImageBoundingBoxes( |
| const SkBitmap& bitmap) { |
| content::RenderFrameHost* render_frame_host = |
| tab_->GetContents()->GetPrimaryMainFrame(); |
| mojo::AssociatedRemote<chrome::mojom::ChromeRenderFrame> chrome_render_frame; |
| render_frame_host->GetRemoteAssociatedInterfaces()->GetInterface( |
| &chrome_render_frame); |
| // Bind the InterfacePtr into the callback so that it's kept alive until |
| // there's either a connection error or a response. |
| auto* frame = chrome_render_frame.get(); |
| |
| frame->RequestBoundsHintForAllImages(base::BindOnce( |
| &LensOverlayController::GetPdfCurrentPage, weak_factory_.GetWeakPtr(), |
| std::move(chrome_render_frame), ++screenshot_attempt_id_, bitmap)); |
| } |
| |
| void LensOverlayController::GetPdfCurrentPage( |
| mojo::AssociatedRemote<chrome::mojom::ChromeRenderFrame> |
| chrome_render_frame, |
| int attempt_id, |
| const SkBitmap& bitmap, |
| const std::vector<gfx::Rect>& bounds) { |
| #if BUILDFLAG(ENABLE_PDF) |
| if (lens::features::SendPdfCurrentPageEnabled()) { |
| pdf::PDFDocumentHelper* pdf_helper = |
| pdf::PDFDocumentHelper::MaybeGetForWebContents(tab_->GetContents()); |
| if (pdf_helper) { |
| pdf_helper->GetMostVisiblePageIndex(base::BindOnce( |
| &LensOverlayController::DidCaptureScreenshot, |
| weak_factory_.GetWeakPtr(), std::move(chrome_render_frame), |
| attempt_id, bitmap, bounds)); |
| return; |
| } |
| } |
| #endif // BUILDFLAG(ENABLE_PDF) |
| |
| DidCaptureScreenshot(std::move(chrome_render_frame), attempt_id, bitmap, |
| bounds, /*pdf_current_page=*/std::nullopt); |
| } |
| |
| void LensOverlayController::DidCaptureScreenshot( |
| mojo::AssociatedRemote<chrome::mojom::ChromeRenderFrame> |
| chrome_render_frame, |
| int attempt_id, |
| const SkBitmap& bitmap, |
| const std::vector<gfx::Rect>& all_bounds, |
| std::optional<uint32_t> pdf_current_page) { |
| // While capturing a screenshot the overlay was cancelled. Do nothing. |
| if (state_ == State::kOff || IsOverlayClosing()) { |
| return; |
| } |
| |
| // An id mismatch implies this is not the most recent screenshot attempt. |
| if (screenshot_attempt_id_ != attempt_id) { |
| return; |
| } |
| |
| // The documentation for CopyFromSurface claims that the copy can fail, but |
| // without providing information about how this can happen. |
| // Supposedly IsSurfaceAvailableForCopy() should guard against this case, but |
| // this is a multi-process, multi-threaded environment so there may be a |
| // TOCTTOU race condition. |
| if (bitmap.drawsNothing()) { |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kErrorScreenshotCreationFailed); |
| return; |
| } |
| |
| if (lens::features::IsLensOverlayEarlyStartQueryFlowOptimizationEnabled()) { |
| // Start the query as soon as the image is ready since it is the only |
| // critical asynchronous flow. This optimization parallelizes the query flow |
| // with other async startup processes. |
| const auto& tab_url = tab_->GetContents()->GetLastCommittedURL(); |
| |
| auto bitmap_to_send = bitmap; |
| auto page_url = lens_search_controller_->GetPageURL(); |
| auto page_title = GetPageTitle(); |
| if (!IsPageContextEligible( |
| tab_url, {}, lens_search_controller_->page_context_eligibility())) { |
| is_page_context_eligible_ = false; |
| bitmap_to_send = SkBitmap(); |
| page_url = GURL(); |
| page_title = ""; |
| } |
| |
| lens_overlay_query_controller_->StartQueryFlow( |
| bitmap_to_send, page_url, page_title, |
| ConvertSignificantRegionBoxes(all_bounds), |
| std::vector<lens::PageContent>(), lens::MimeType::kUnknown, |
| pdf_current_page, GetUiScaleFactor(), invocation_time_); |
| } |
| |
| // The following two methods happen async to parallelize the two bottlenecks |
| // in our invocation flow. |
| CreateInitializationData(bitmap, all_bounds, pdf_current_page); |
| ShowOverlay(); |
| |
| state_ = State::kStartingWebUI; |
| } |
| |
| void LensOverlayController::CreateInitializationData( |
| const SkBitmap& screenshot, |
| const std::vector<gfx::Rect>& all_bounds, |
| std::optional<uint32_t> pdf_current_page) { |
| // Create the new RGB bitmap async to prevent the main thread from blocking on |
| // the encoding. |
| base::ThreadPool::PostTaskAndReplyWithResult( |
| FROM_HERE, {base::TaskPriority::USER_BLOCKING}, |
| base::BindOnce(&CreateRgbBitmap, screenshot), |
| base::BindOnce(&LensOverlayController::ContinueCreateInitializationData, |
| weak_factory_.GetWeakPtr(), screenshot, all_bounds, |
| pdf_current_page)); |
| } |
| |
| void LensOverlayController::ContinueCreateInitializationData( |
| const SkBitmap& screenshot, |
| const std::vector<gfx::Rect>& all_bounds, |
| std::optional<uint32_t> pdf_current_page, |
| SkBitmap rgb_screenshot) { |
| if (state_ != State::kStartingWebUI || rgb_screenshot.drawsNothing()) { |
| // TODO(b/334185985): Handle case when screenshot RGB encoding fails. |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kErrorScreenshotEncodingFailed); |
| return; |
| } |
| |
| // Resolve the color palette based on the vibrant screenshot color. |
| lens::PaletteId color_palette = lens::PaletteId::kFallback; |
| if (lens::features::IsDynamicThemeDetectionEnabled()) { |
| std::vector<SkColor> colors; |
| for (const auto& pair : lens::kPalettes) { |
| colors.emplace_back(pair.first); |
| } |
| SkColor screenshot_color = lens::ExtractVibrantOrDominantColorFromImage( |
| screenshot, lens::features::DynamicThemeMinPopulationPct()); |
| SkColor theme_color = lens::FindBestMatchedColorOrTransparent( |
| colors, screenshot_color, lens::features::DynamicThemeMinChroma()); |
| if (theme_color != SK_ColorTRANSPARENT) { |
| color_palette = lens::kPalettes.at(theme_color); |
| } |
| } |
| |
| auto initialization_data = std::make_unique<OverlayInitializationData>( |
| screenshot, std::move(rgb_screenshot), color_palette, |
| lens_search_controller_->GetPageURL(), GetPageTitle()); |
| initialization_data->significant_region_boxes_ = |
| ConvertSignificantRegionBoxes(all_bounds); |
| initialization_data->last_retrieved_most_visible_page_ = pdf_current_page; |
| |
| GetContextualizationController()->GetPageContextualization(base::BindOnce( |
| &LensOverlayController::StorePageContentAndContinueInitialization, |
| weak_factory_.GetWeakPtr(), std::move(initialization_data))); |
| } |
| |
| void LensOverlayController::StorePageContentAndContinueInitialization( |
| std::unique_ptr<OverlayInitializationData> initialization_data, |
| std::vector<lens::PageContent> page_contents, |
| lens::MimeType primary_content_type, |
| std::optional<uint32_t> page_count) { |
| initialization_data->page_contents_ = page_contents; |
| initialization_data->primary_content_type_ = primary_content_type; |
| initialization_data->pdf_page_count_ = page_count; |
| InitializeOverlay(std::move(initialization_data)); |
| |
| RecordDocumentMetrics(page_count); |
| } |
| |
| std::vector<lens::mojom::CenterRotatedBoxPtr> |
| LensOverlayController::ConvertSignificantRegionBoxes( |
| const std::vector<gfx::Rect>& all_bounds) { |
| std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes; |
| int max_regions = lens::features::GetLensOverlayMaxSignificantRegions(); |
| if (max_regions == 0) { |
| return significant_region_boxes; |
| } |
| content::RenderFrameHost* render_frame_host = |
| tab_->GetContents()->GetPrimaryMainFrame(); |
| auto view_bounds = render_frame_host->GetView()->GetViewBounds(); |
| for (auto& image_bounds : all_bounds) { |
| // Check the original area of the images against the minimum area. |
| if (image_bounds.width() * image_bounds.height() >= |
| lens::features::GetLensOverlaySignificantRegionMinArea()) { |
| // We only have bounds for images in the main frame of the tab (i.e. not |
| // in iframes), so view bounds are identical to tab bounds and can be |
| // used for both parameters. |
| significant_region_boxes.emplace_back( |
| lens::GetCenterRotatedBoxFromTabViewAndImageBounds( |
| view_bounds, view_bounds, image_bounds)); |
| } |
| } |
| // If an image is outside the viewpoint, the box will have zero area. |
| std::erase_if(significant_region_boxes, [](const auto& box) { |
| return box->box.height() == 0 || box->box.width() == 0; |
| }); |
| // Sort by descending area. |
| std::sort(significant_region_boxes.begin(), significant_region_boxes.end(), |
| [](const auto& box1, const auto& box2) { |
| return box1->box.height() * box1->box.width() > |
| box2->box.height() * box2->box.width(); |
| }); |
| // Treat negative values of max_regions as no limit. |
| if (max_regions > 0 && |
| significant_region_boxes.size() > (unsigned long)max_regions) { |
| significant_region_boxes.resize(max_regions); |
| } |
| |
| return significant_region_boxes; |
| } |
| |
| void LensOverlayController::TryUpdatePageContextualization() { |
| // If there is already an upload, do not send another request. |
| // TODO(crbug.com/399154548): Ideally, there could be two uploads in progress |
| // at a time, however, the current query controller implementation does not |
| // support this. |
| if (lens_overlay_query_controller_->IsPageContentUploadInProgress()) { |
| return; |
| } |
| |
| GetContextualizationController()->GetPageContextualization( |
| base::BindOnce(&LensOverlayController::UpdatePageContextualization, |
| weak_factory_.GetWeakPtr())); |
| } |
| |
| void LensOverlayController::UpdatePageContextualization( |
| std::vector<lens::PageContent> page_contents, |
| lens::MimeType primary_content_type, |
| std::optional<uint32_t> page_count) { |
| if (!lens::features::IsLensOverlayContextualSearchboxEnabled()) { |
| return; |
| } |
| |
| // If the protected page is showing, then return early as none of the content |
| // will be sent. |
| if (results_side_panel_coordinator_->IsShowingProtectedErrorPage()) { |
| return; |
| } |
| |
| // Do not capture a new screenshot if the feature param is not enabled or if |
| // the user is not viewing the live page, meaning the viewport cannot have |
| // changed. |
| if (!lens::features::UpdateViewportEachQueryEnabled() || |
| state_ != State::kLivePageAndResults) { |
| UpdatePageContextualizationPart2(page_contents, primary_content_type, |
| page_count, SkBitmap()); |
| return; |
| } |
| |
| // Begin the process of grabbing a screenshot. |
| content::RenderWidgetHostView* view = tab_->GetContents() |
| ->GetPrimaryMainFrame() |
| ->GetRenderViewHost() |
| ->GetWidget() |
| ->GetView(); |
| if (!IsScreenshotPossible(view)) { |
| UpdatePageContextualizationPart2(page_contents, primary_content_type, |
| page_count, SkBitmap()); |
| return; |
| } |
| view->CopyFromSurface( |
| /*src_rect=*/gfx::Rect(), /*output_size=*/gfx::Size(), |
| base::BindPostTask( |
| base::SequencedTaskRunner::GetCurrentDefault(), |
| base::BindOnce( |
| &LensOverlayController::UpdatePageContextualizationPart2, |
| weak_factory_.GetWeakPtr(), page_contents, primary_content_type, |
| page_count))); |
| } |
| |
| void LensOverlayController::UpdatePageContextualizationPart2( |
| std::vector<lens::PageContent> page_contents, |
| lens::MimeType primary_content_type, |
| std::optional<uint32_t> page_count, |
| const SkBitmap& bitmap) { |
| #if BUILDFLAG(ENABLE_PDF) |
| if (lens::features::SendPdfCurrentPageEnabled()) { |
| pdf::PDFDocumentHelper* pdf_helper = |
| pdf::PDFDocumentHelper::MaybeGetForWebContents(tab_->GetContents()); |
| if (pdf_helper) { |
| pdf_helper->GetMostVisiblePageIndex(base::BindOnce( |
| &LensOverlayController::UpdatePageContextualizationPart3, |
| weak_factory_.GetWeakPtr(), page_contents, primary_content_type, |
| page_count, bitmap)); |
| return; |
| } |
| } |
| #endif // BUILDFLAG(ENABLE_PDF) |
| |
| UpdatePageContextualizationPart3(page_contents, primary_content_type, |
| page_count, bitmap, |
| /*most_visible_page=*/std::nullopt); |
| } |
| |
| void LensOverlayController::UpdatePageContextualizationPart3( |
| std::vector<lens::PageContent> page_contents, |
| lens::MimeType primary_content_type, |
| std::optional<uint32_t> page_count, |
| const SkBitmap& bitmap, |
| std::optional<uint32_t> most_visible_page) { |
| bool sending_bitmap = false; |
| if (!bitmap.drawsNothing() && |
| (initialization_data_->updated_screenshot_.drawsNothing() || |
| !lens::AreBitmapsEqual(initialization_data_->updated_screenshot_, |
| bitmap))) { |
| initialization_data_->updated_screenshot_ = bitmap; |
| sending_bitmap = true; |
| } |
| initialization_data_->last_retrieved_most_visible_page_ = most_visible_page; |
| |
| // TODO(crbug.com/399215935): Ideally, this check should ensure that any of |
| // the content date has not changed. For now, we only check if the |
| // primary_content_type bytes have changed. |
| auto old_page_content_it = std::ranges::find_if( |
| initialization_data_->page_contents_, |
| [&primary_content_type](const auto& page_content) { |
| return page_content.content_type_ == primary_content_type; |
| }); |
| auto new_page_content_it = std::ranges::find_if( |
| page_contents, [&primary_content_type](const auto& page_content) { |
| return page_content.content_type_ == primary_content_type; |
| }); |
| const lens::PageContent* old_page_content = |
| old_page_content_it != initialization_data_->page_contents_.end() |
| ? &(*old_page_content_it) |
| : nullptr; |
| const lens::PageContent* new_page_content = |
| new_page_content_it != page_contents.end() ? &(*new_page_content_it) |
| : nullptr; |
| |
| if (initialization_data_->primary_content_type_ == primary_content_type && |
| old_page_content && new_page_content) { |
| const float old_size = old_page_content->bytes_.size(); |
| const float new_size = new_page_content->bytes_.size(); |
| const float percent_changed = abs((new_size - old_size) / old_size); |
| if (percent_changed < kByteChangeTolerancePercent) { |
| if (!sending_bitmap) { |
| // If the bytes have not changed more than our threshold and the |
| // screenshot has not changed, exit early. Notify the query controller |
| // that the user may be issuing a search request, and therefore the |
| // query should be restarted if TTL expired. If the bytes did change, |
| // this will happen automatically as a result of the |
| // SendUpdatedPageContent call below. |
| lens_overlay_query_controller_->MaybeRestartQueryFlow(); |
| return; |
| } |
| |
| // If the screenshot has changed but the bytes have not, send only the |
| // screenshot. |
| lens_overlay_query_controller_->SendUpdatedPageContent( |
| std::nullopt, std::nullopt, std::nullopt, std::nullopt, |
| initialization_data_->last_retrieved_most_visible_page_, |
| sending_bitmap ? bitmap : SkBitmap()); |
| return; |
| } |
| } |
| |
| // Since the page content has changed, let the query controller know to avoid |
| // dangling pointers. |
| lens_overlay_query_controller_->ResetPageContentData(); |
| |
| initialization_data_->page_contents_ = page_contents; |
| initialization_data_->primary_content_type_ = primary_content_type; |
| |
| // If no bytes were retrieved from the page, the query won't be able to be |
| // contextualized. Notify the side panel so the ghost loader isn't shown. No |
| // need to update update the overlay as this update only happens on navigation |
| // where the side panel will already be open. |
| if (!new_page_content || new_page_content->bytes_.empty()) { |
| SuppressGhostLoader(); |
| } |
| |
| #if BUILDFLAG(ENABLE_PDF) |
| // If the new page is a PDF, fetch the text from the page to be used as early |
| // suggest signals. |
| if (new_page_content && |
| new_page_content->content_type_ == lens::MimeType::kPdf) { |
| GetContextualizationController()->FetchVisiblePageIndexAndGetPartialPdfText( |
| lens_overlay_query_controller_, page_count.value_or(0), |
| base::BindOnce(&LensOverlayController::OnPdfPartialPageTextRetrieved, |
| weak_factory_.GetWeakPtr())); |
| } |
| #endif |
| |
| is_upload_progress_bar_shown_ = true; |
| is_first_upload_handler_event_ = true; |
| lens_overlay_query_controller_->SendUpdatedPageContent( |
| initialization_data_->page_contents_, |
| initialization_data_->primary_content_type_, |
| lens_search_controller_->GetPageURL(), GetPageTitle(), |
| initialization_data_->last_retrieved_most_visible_page_, |
| sending_bitmap ? bitmap : SkBitmap()); |
| |
| RecordDocumentMetrics(page_count); |
| } |
| |
| void LensOverlayController::SuppressGhostLoader() { |
| if (page_) { |
| page_->SuppressGhostLoader(); |
| } |
| results_side_panel_coordinator_->SuppressGhostLoader(); |
| } |
| |
| void LensOverlayController::SetLiveBlur(bool enabled) { |
| if (!lens_overlay_blur_layer_delegate_) { |
| return; |
| } |
| |
| if (enabled) { |
| lens_overlay_blur_layer_delegate_->StartBackgroundImageCapture(); |
| return; |
| } |
| |
| lens_overlay_blur_layer_delegate_->StopBackgroundImageCapture(); |
| } |
| |
| void LensOverlayController::ShowOverlay() { |
| // Grab the tab contents web view and disable mouse and keyboard inputs to it. |
| auto* contents_web_view = tab_->GetBrowserWindowInterface()->GetWebView(); |
| CHECK(contents_web_view); |
| if (should_show_overlay_) { |
| contents_web_view->SetEnabled(false); |
| } |
| |
| // If the view already exists, we just need to reshow it. |
| if (overlay_view_) { |
| // Exit early to avoid reshowing the overlay if it should not be shown. |
| if (!should_show_overlay_) { |
| return; |
| } |
| // Restore the state to show the overlay. |
| overlay_view_->SetVisible(true); |
| preselection_widget_anchor_->SetVisible(true); |
| overlay_web_view_->SetVisible(true); |
| |
| // Restart the live blur since the view is visible again. |
| SetLiveBlur(true); |
| |
| // The overlay needs to be focused on show to immediately begin |
| // receiving key events. |
| overlay_web_view_->RequestFocus(); |
| return; |
| } |
| |
| // Create the views that will house our UI. The overlay view might not |
| // actually be shown, as dictated by `should_show_overlay_`. It still needs to |
| // be created so the initialization process completes. |
| overlay_view_ = CreateViewForOverlay(); |
| overlay_view_->SetVisible(should_show_overlay_); |
| |
| // Sanity check that the overlay view is above the contents web view. |
| auto* parent_view = overlay_view_->parent(); |
| views::View* child_contents_view = contents_web_view; |
| // TODO(crbug.com/406794005): Remove this block if overlay_view_ ends up |
| // getting reparented such that it always shares a parent with |
| // contents_web_view. |
| if (base::FeatureList::IsEnabled(features::kSideBySide)) { |
| // When split view is enabled, there are two additional layers of |
| // hierarchy: |
| // BrowserView->MultiContentsView->ContentsContainerView->ContentsWebView |
| // vs. |
| // BrowserView->ContentsWebView |
| // Since the overlay view is parented by BrowserView, to properly pass the |
| // check below, we should only compare direct children of BrowserView. |
| child_contents_view = child_contents_view->parent()->parent(); |
| } |
| CHECK(parent_view->GetIndexOf(overlay_view_) > |
| parent_view->GetIndexOf(child_contents_view)); |
| |
| // Observe the overlay view to handle resizing the background blur layer. |
| tab_contents_view_observer_.Observe(overlay_view_); |
| |
| // The overlay needs to be focused on show to immediately begin |
| // receiving key events. |
| if (should_show_overlay_) { |
| CHECK(overlay_web_view_); |
| overlay_web_view_->RequestFocus(); |
| } |
| |
| // Listen to the render process housing out overlay. |
| overlay_web_view_->GetWebContents() |
| ->GetPrimaryMainFrame() |
| ->GetProcess() |
| ->AddObserver(this); |
| } |
| |
| void LensOverlayController::HideOverlay() { |
| // Hide the overlay view, but keep the web view attached to the overlay view |
| // so that the overlay can be re-shown without creating a new web view. |
| preselection_widget_anchor_->SetVisible(false); |
| overlay_web_view_->SetVisible(false); |
| MaybeHideSharedOverlayView(); |
| |
| SetLiveBlur(false); |
| HidePreselectionBubble(); |
| // Re-enable mouse and keyboard events to the tab contents web view. |
| auto* contents_web_view = tab_->GetBrowserWindowInterface()->GetWebView(); |
| CHECK(contents_web_view); |
| contents_web_view->SetEnabled(true); |
| } |
| |
| void LensOverlayController::MaybeHideSharedOverlayView() { |
| if (!overlay_view_) { |
| return; |
| } |
| for (views::View* child : overlay_view_->children()) { |
| if (child->GetVisible()) { |
| // If any child is visible, it is being used by another tab so do not hide |
| // the overlay view. |
| return; |
| } |
| } |
| overlay_view_->SetVisible(false); |
| } |
| |
| void LensOverlayController::MaybeOpenSidePanel() { |
| if (side_panel_in_use_) { |
| // Exit early if this class has already requested access to the side panel. |
| return; |
| } |
| side_panel_in_use_ = results_side_panel_coordinator_->RegisterEntryAndShow(); |
| } |
| |
| void LensOverlayController::InitializeOverlay( |
| std::unique_ptr<OverlayInitializationData> initialization_data) { |
| // Initialization data is ready. |
| if (initialization_data) { |
| // Confirm initialization_data has not already been assigned. |
| CHECK(!initialization_data_); |
| initialization_data_ = std::move(initialization_data); |
| |
| // If suggest inputs were updated before the initialization data was ready, |
| // attach them to the initialization data now. |
| if (pre_initialization_suggest_inputs_.has_value()) { |
| initialization_data_->suggest_inputs_ = |
| pre_initialization_suggest_inputs_.value(); |
| pre_initialization_suggest_inputs_.reset(); |
| } |
| } |
| |
| // We can only continue once both the WebUI is bound and the initialization |
| // data is processed and ready. If either of those conditions aren't met, we |
| // exit early and wait for the other condition to call this method again. |
| if (!page_ || !initialization_data_) { |
| return; |
| } |
| |
| // Move the data that was stored prior to initialization into |
| // initialization_data_. |
| if (pre_initialization_objects_.has_value()) { |
| initialization_data_->objects_ = |
| std::move(pre_initialization_objects_.value()); |
| pre_initialization_objects_.reset(); |
| } |
| if (pre_initialization_text_.has_value()) { |
| initialization_data_->text_ = std::move(pre_initialization_text_.value()); |
| pre_initialization_text_.reset(); |
| } |
| |
| InitializeOverlayUI(*initialization_data_); |
| base::UmaHistogramBoolean("Lens.Overlay.Shown", true); |
| |
| #if BUILDFLAG(ENABLE_PDF) |
| // If PDF content was extracted from the page, fetch the text from the PDF to |
| // be used as early suggest signals. |
| if (!initialization_data_->page_contents_.empty() && |
| initialization_data_->page_contents_.front().content_type_ == |
| lens::MimeType::kPdf) { |
| CHECK(initialization_data_->pdf_page_count_.has_value()); |
| GetContextualizationController()->FetchVisiblePageIndexAndGetPartialPdfText( |
| lens_overlay_query_controller_, |
| initialization_data_->pdf_page_count_.value(), |
| base::BindOnce(&LensOverlayController::OnPdfPartialPageTextRetrieved, |
| weak_factory_.GetWeakPtr())); |
| } |
| #endif |
| |
| // If the StartQueryFlow optimization is enabled, the page contents will not |
| // be sent with the initial image request, so we need to send it here. |
| if (lens::features::IsLensOverlayContextualSearchboxEnabled() && |
| lens::features::IsLensOverlayEarlyStartQueryFlowOptimizationEnabled() && |
| is_page_context_eligible_) { |
| // The screenshot is not sent here unless forced by |
| // `should_send_screenshot_on_init_` as it should have been sent in the |
| // original StartQueryFlow call. |
| lens_overlay_query_controller_->SendUpdatedPageContent( |
| initialization_data_->page_contents_, |
| initialization_data_->primary_content_type_, |
| lens_search_controller_->GetPageURL(), GetPageTitle(), |
| initialization_data_->last_retrieved_most_visible_page_, |
| should_send_screenshot_on_init_ |
| ? initialization_data_->initial_screenshot_ |
| : SkBitmap()); |
| } |
| |
| // Show the preselection overlay now that the overlay is initialized and ready |
| // to be shown. |
| if (!pending_region_ && should_show_overlay_) { |
| ShowPreselectionBubble(); |
| } |
| |
| // Create the blur delegate so it is ready to blur once the view is visible. |
| if (lens::features::GetLensOverlayUseBlur()) { |
| content::RenderWidgetHost* live_page_widget_host = |
| tab_->GetContents() |
| ->GetPrimaryMainFrame() |
| ->GetRenderViewHost() |
| ->GetWidget(); |
| lens_overlay_blur_layer_delegate_ = |
| std::make_unique<lens::LensOverlayBlurLayerDelegate>( |
| live_page_widget_host); |
| } |
| |
| state_ = State::kOverlay; |
| lens_search_controller_->NotifyOverlayOpened(); |
| |
| // Update the entry points state to ensure that the entry points are disabled |
| // now that the overlay is showing. |
| UpdateEntryPointsState(); |
| |
| // Only start the query flow again if we don't already have a full image |
| // response, unless the early start query flow optimization is enabled. |
| if (!initialization_data_->has_full_image_response() && |
| !lens::features::IsLensOverlayEarlyStartQueryFlowOptimizationEnabled()) { |
| if (!is_page_context_eligible_) { |
| initialization_data_->initial_screenshot_ = SkBitmap(); |
| initialization_data_->page_url_ = GURL(); |
| initialization_data_->page_title_ = ""; |
| should_send_screenshot_on_init_ = true; |
| } |
| |
| lens_overlay_query_controller_->StartQueryFlow( |
| initialization_data_->initial_screenshot_, |
| initialization_data_->page_url_, initialization_data_->page_title_, |
| std::move(initialization_data_->significant_region_boxes_), |
| initialization_data_->page_contents_, |
| initialization_data_->primary_content_type_, |
| initialization_data_->last_retrieved_most_visible_page_, |
| GetUiScaleFactor(), invocation_time_); |
| } |
| |
| // If there is a pending contextual search request, issue it now that the |
| // overlay is initialized. |
| if (pending_contextual_search_request_) { |
| std::move(pending_contextual_search_request_).Run(); |
| } |
| |
| // TODO(b/352622136): We should not start the lens request until the overlay |
| // is open to prevent the side panel from opening while the overlay UI is |
| // rendering. |
| if (pending_region_) { |
| // If there is a pending region (i.e. for image right click) |
| // use INJECTED_IMAGE as the selection type. |
| IssueLensRequest(std::move(pending_region_), lens::INJECTED_IMAGE, |
| pending_region_bitmap_); |
| pending_region_bitmap_.reset(); |
| } |
| } |
| |
| void LensOverlayController::InitializeOverlayUI( |
| const OverlayInitializationData& init_data) { |
| // This should only contain LensPage mojo calls and should not affect |
| // `state_`. |
| CHECK(page_); |
| // TODO(b/371593619), it would be more efficent to send all initialization |
| // data to the overlay web UI in a single message. |
| page_->ThemeReceived(CreateTheme(init_data.color_palette_)); |
| |
| bool should_show_csb = !init_data.page_contents_.empty() && |
| !init_data.page_contents_.front().bytes_.empty(); |
| if (should_show_csb) { |
| // Reset metric booleans in case they were set to true previously and the |
| // overlay was reopened. |
| csb_session_end_metrics_ = lens::ContextualSearchboxSessionEndMetrics(); |
| csb_session_end_metrics_.searchbox_shown_ = true; |
| } |
| initial_page_content_type_ = |
| init_data.page_contents_.empty() |
| ? lens::MimeType::kUnknown |
| : init_data.page_contents_.front().content_type_; |
| initial_document_type_ = lens::StringMimeTypeToDocumentType( |
| tab_->GetContents()->GetContentsMimeType()); |
| page_->ShouldShowContextualSearchBox(should_show_csb); |
| |
| // Send the initial document type to the overlay web UI. |
| NotifyPageContentUpdated(); |
| |
| page_->ScreenshotDataReceived(init_data.initial_rgb_screenshot_); |
| if (!init_data.objects_.empty()) { |
| SendObjects(CopyObjects(init_data.objects_)); |
| } |
| if (init_data.text_) { |
| SendText(init_data.text_->Clone()); |
| } |
| if (pending_region_) { |
| page_->SetPostRegionSelection(pending_region_->Clone()); |
| } |
| if (IsHandshakeComplete()) { |
| // Notify the overlay that it is safe to query autocomplete. |
| page_->NotifyHandshakeComplete(); |
| } |
| |
| // Record the UMA for lens overlay invocation. |
| lens::RecordInvocation(invocation_source_, initial_document_type_); |
| } |
| |
| bool LensOverlayController::IsContextualSearchbox() { |
| return lens_search_controller_->lens_searchbox_controller() |
| ->IsContextualSearchbox(); |
| } |
| |
| raw_ptr<views::View> LensOverlayController::CreateViewForOverlay() { |
| // Grab the host view for the overlay which is owned by the browser view. |
| auto* host_view = tab_->GetBrowserWindowInterface()->LensOverlayView(); |
| CHECK(host_view); |
| |
| // Setup a preselection anchor view. Usually bubbles are anchored to top |
| // chrome, but top chrome is not always visible when our overlay is visible. |
| // Instead of anchroing to top chrome, we anchor to this view because 1) it |
| // always exists when the overlay exists and 2) it is before the WebView in |
| // the view hierarchy and therefore will receive focus first when tabbing from |
| // top chrome. |
| std::unique_ptr<views::View> anchor_view = std::make_unique<views::View>(); |
| anchor_view->SetFocusBehavior(views::View::FocusBehavior::NEVER); |
| preselection_widget_anchor_ = host_view->AddChildView(std::move(anchor_view)); |
| |
| // Create the web view. |
| std::unique_ptr<views::WebView> web_view = std::make_unique<views::WebView>( |
| tab_->GetContents()->GetBrowserContext()); |
| content::WebContents* web_view_contents = web_view->GetWebContents(); |
| web_view->SetProperty(views::kElementIdentifierKey, kOverlayId); |
| views::WebContentsSetBackgroundColor::CreateForWebContentsWithColor( |
| web_view_contents, SK_ColorTRANSPARENT); |
| |
| // Set the label for the renderer process in Chrome Task Manager. |
| task_manager::WebContentsTags::CreateForToolContents( |
| web_view_contents, IDS_LENS_OVERLAY_RENDERER_LABEL); |
| |
| // As the embedder for the lens overlay WebUI content we must set the |
| // appropriate tab interface here. |
| webui::SetTabInterface(web_view_contents, GetTabInterface()); |
| |
| // Set the web contents delegate to this controller so we can handle keyboard |
| // events. Allow accelerators (e.g. hotkeys) to work on this web view. |
| web_view->set_allow_accelerators(true); |
| web_view->GetWebContents()->SetDelegate(this); |
| |
| // Load the untrusted WebUI into the web view. |
| web_view->LoadInitialURL(GURL(chrome::kChromeUILensOverlayUntrustedURL)); |
| |
| overlay_web_view_ = host_view->AddChildView(std::move(web_view)); |
| return host_view; |
| } |
| |
| bool LensOverlayController::HandleContextMenu( |
| content::RenderFrameHost& render_frame_host, |
| const content::ContextMenuParams& params) { |
| // We do not want to show the browser context menu on the overlay unless we |
| // are in debugging mode. Returning true is equivalent to not showing the |
| // context menu. |
| return !lens::features::IsLensOverlayDebuggingEnabled(); |
| } |
| |
| bool LensOverlayController::HandleKeyboardEvent( |
| content::WebContents* source, |
| const input::NativeWebKeyboardEvent& event) { |
| // This can be called before the overlay web view is attached to the overlay |
| // view. In that case, the focus manager could be null. |
| if (!overlay_web_view_ || !overlay_web_view_->GetFocusManager()) { |
| return false; |
| } |
| return lens_search_controller_->lens_overlay_event_handler() |
| ->HandleKeyboardEvent(source, event, |
| overlay_web_view_->GetFocusManager()); |
| } |
| |
| void LensOverlayController::OnFullscreenStateChanged() { |
| // Flag is enabled to allow Lens Overlay in fullscreen no matter what so we |
| // can exit early. |
| if (lens::features::GetLensOverlayEnableInFullscreen()) { |
| return; |
| } |
| // If there is top chrome we can keep the overlay open. |
| if (tab_->GetBrowserWindowInterface()->IsTabStripVisible()) { |
| return; |
| } |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kFullscreened); |
| } |
| |
| void LensOverlayController::OnViewBoundsChanged(views::View* observed_view) { |
| CHECK(observed_view == overlay_view_); |
| |
| // We now want to start the live blur since the screenshot has resized to |
| // allow the blur to peek through. |
| if (IsOverlayShowing()) { |
| SetLiveBlur(true); |
| } |
| |
| // Set our view to the same bounds as the contents web view so it always |
| // covers the tab contents. |
| if (lens_overlay_blur_layer_delegate_) { |
| // Set the blur to have the same bounds as our view, but since it is in our |
| // views local coordinate system, the blur should be positioned at (0,0). |
| lens_overlay_blur_layer_delegate_->layer()->SetBounds( |
| overlay_view_->GetLocalBounds()); |
| } |
| } |
| |
| #if BUILDFLAG(IS_MAC) |
| void LensOverlayController::OnWidgetActivationChanged(views::Widget* widget, |
| bool active) { |
| if (active && preselection_widget_) { |
| // On Mac, traversing out of the preselection widget into the browser causes |
| // the browser to restore its focus to the wrong place. Thus, when entering |
| // the preselection widget, make sure to clear out the browser's native |
| // focus. This causes the preselection widget to lose activation, so |
| // reactivate it manually. |
| tab_->GetBrowserWindowInterface() |
| ->TopContainer() |
| ->GetWidget() |
| ->GetFocusManager() |
| ->ClearNativeFocus(); |
| preselection_widget_->Activate(); |
| } |
| } |
| #endif |
| |
| void LensOverlayController::OnWidgetDestroying(views::Widget* widget) { |
| preselection_widget_ = nullptr; |
| preselection_widget_observer_.Reset(); |
| } |
| |
| void LensOverlayController::OnOmniboxFocusChanged( |
| OmniboxFocusState state, |
| OmniboxFocusChangeReason reason) { |
| if (state_ == LensOverlayController::State::kOverlay) { |
| if (state == OMNIBOX_FOCUS_NONE) { |
| ShowPreselectionBubble(); |
| } else { |
| HidePreselectionBubble(); |
| } |
| } |
| } |
| |
| void LensOverlayController::OnFindEmptyText( |
| content::WebContents* web_contents) { |
| if (state_ == State::kLivePageAndResults) { |
| return; |
| } |
| lens_search_controller_->CloseLensAsync( |
| lens::LensOverlayDismissalSource::kFindInPageInvoked); |
| } |
| |
| void LensOverlayController::OnFindResultAvailable( |
| content::WebContents* web_contents) { |
| if (state_ == State::kLivePageAndResults) { |
| return; |
| } |
| lens_search_controller_->CloseLensAsync( |
| lens::LensOverlayDismissalSource::kFindInPageInvoked); |
| } |
| |
| void LensOverlayController::OnImmersiveRevealStarted() { |
| // The toolbar has began to reveal. If the overlay is showing, hide the |
| // preselection bubble to ensure it doesn't cover with the toolbar UI. |
| if (IsOverlayShowing()) { |
| HidePreselectionBubble(); |
| } |
| } |
| |
| void LensOverlayController::OnImmersiveRevealEnded() { |
| // The toolbar is no longer revealed. If the overlay is showing, reshow the |
| // preselection bubble to ensure it doesn't cover with the toolbar UI. |
| if (IsOverlayShowing()) { |
| ShowPreselectionBubble(); |
| } |
| } |
| |
| void LensOverlayController::OnImmersiveFullscreenEntered() { |
| // The browser entered immersive fullscreen. If the overlay is showing, call |
| // close and reopen the preselection bubble to ensure it respositions |
| // correctly. |
| if (IsOverlayShowing()) { |
| CloseAndReshowPreselectionBubble(); |
| } |
| } |
| |
| void LensOverlayController::OnImmersiveFullscreenExited() { |
| // The browser exited immersive fullscreen. If the overlay is showing, call |
| // close and reopen the preselection bubble to ensure it respositions |
| // correctly. |
| if (IsOverlayShowing()) { |
| CloseAndReshowPreselectionBubble(); |
| } |
| } |
| |
| void LensOverlayController::OnHandshakeComplete() { |
| CHECK(IsHandshakeComplete()); |
| // Notify the overlay that the handshake is complete if its initialized. |
| if (page_) { |
| page_->NotifyHandshakeComplete(); |
| } |
| |
| // Send the suggest inputs to any pending callbacks. |
| pending_suggest_inputs_callbacks_.Notify(GetLensSuggestInputs()); |
| } |
| |
| metrics::OmniboxEventProto::PageClassification |
| LensOverlayController::GetPageClassification() const { |
| return lens_search_controller_->lens_searchbox_controller() |
| ->GetPageClassification(); |
| } |
| |
| const lens::proto::LensOverlaySuggestInputs& |
| LensOverlayController::GetLensSuggestInputs() const { |
| if (!initialization_data_ && !pre_initialization_suggest_inputs_) { |
| return lens::proto::LensOverlaySuggestInputs().default_instance(); |
| } |
| return initialization_data_ ? initialization_data_->suggest_inputs_ |
| : pre_initialization_suggest_inputs_.value(); |
| } |
| |
| base::CallbackListSubscription |
| LensOverlayController::GetLensSuggestInputsWhenReady( |
| LensOverlaySuggestInputsCallback callback) { |
| // Exit early if the overlay is either off or going to soon be off. |
| if (state() == State::kOff || IsOverlayClosing()) { |
| std::move(callback).Run(std::nullopt); |
| return {}; |
| } |
| |
| // If the handshake is complete, return the Lens suggest inputs immediately. |
| if (IsHandshakeComplete()) { |
| std::move(callback).Run(initialization_data_->suggest_inputs_); |
| return {}; |
| } |
| return pending_suggest_inputs_callbacks_.Add(std::move(callback)); |
| } |
| |
| std::optional<std::string> LensOverlayController::GetPageTitle() { |
| std::optional<std::string> page_title; |
| content::WebContents* active_web_contents = tab_->GetContents(); |
| if (lens::CanSharePageTitleWithLensOverlay(sync_service_, pref_service_)) { |
| page_title = std::make_optional<std::string>( |
| base::UTF16ToUTF8(active_web_contents->GetTitle())); |
| } |
| return page_title; |
| } |
| |
| float LensOverlayController::GetUiScaleFactor() { |
| int device_scale_factor = |
| tab_->GetContents()->GetRenderWidgetHostView()->GetDeviceScaleFactor(); |
| float page_scale_factor = |
| zoom::ZoomController::FromWebContents(tab_->GetContents()) |
| ->GetZoomPercent() / |
| 100.0f; |
| return device_scale_factor * page_scale_factor; |
| } |
| |
| void LensOverlayController::OnSidePanelDidOpen() { |
| // If a side panel opens that is not ours, we must close the overlay. |
| if (side_panel_coordinator_->GetCurrentEntryId() != |
| SidePanelEntry::Id::kLensOverlayResults) { |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kUnexpectedSidePanelOpen); |
| } |
| } |
| |
| void LensOverlayController::FinishedWaitingForReflow() { |
| if (state_ == State::kClosingOpenedSidePanel) { |
| // This path is invoked after the user invokes the overlay, but we needed |
| // to close the side panel before taking a screenshot. The Side panel is |
| // now closed so we can now take the screenshot of the page. |
| CaptureScreenshot(); |
| } |
| } |
| |
| void LensOverlayController::RenderProcessExited( |
| content::RenderProcessHost* host, |
| const content::ChildProcessTerminationInfo& info) { |
| // Exit early if the overlay is already closing. |
| if (IsOverlayClosing()) { |
| return; |
| } |
| |
| // The overlay's primary main frame process has exited, either cleanly or |
| // unexpectedly. Close the overlay so that the user does not get into a broken |
| // state where the overlay cannot be dismissed. Note that RenderProcessExited |
| // can be called during the destruction of a frame in the overlay, so it is |
| // important to post a task to close the overlay to avoid double-freeing the |
| // overlay's frames. See https://crbug.com/371643466. |
| base::SequencedTaskRunner::GetCurrentDefault()->PostTask( |
| FROM_HERE, |
| base::BindOnce( |
| &LensSearchController::CloseLensSync, |
| lens_search_controller_->GetWeakPtr(), |
| info.status == base::TERMINATION_STATUS_NORMAL_TERMINATION |
| ? lens::LensOverlayDismissalSource::kOverlayRendererClosedNormally |
| : lens::LensOverlayDismissalSource:: |
| kOverlayRendererClosedUnexpectedly)); |
| } |
| |
| void LensOverlayController::TabForegrounded(tabs::TabInterface* tab) { |
| // Ignore the event if the overlay is not backgrounded. |
| if (state_ != State::kBackground) { |
| // TODO(crbug.com/404941800): This is a temporary DCHECK. This should be a |
| // CHECK and the if statement above should be removed once the root cause |
| // causing the CHECK(state_ == State::kBackground) to fail is found and |
| // fixed. |
| DCHECK(state_ == State::kBackground) |
| << "State should be kBackground but is instead " |
| << static_cast<int>(state_); |
| return; |
| } |
| |
| // If the overlay was backgrounded, restore the previous state. |
| if (backgrounded_state_ != State::kLivePageAndResults) { |
| ShowOverlay(); |
| } |
| if (backgrounded_state_ != State::kOverlayAndResults && |
| backgrounded_state_ != State::kLivePageAndResults) { |
| ShowPreselectionBubble(); |
| } |
| if (lens::features::IsLensOverlayContextualSearchboxEnabled()) { |
| SuppressGhostLoader(); |
| } |
| |
| state_ = backgrounded_state_; |
| UpdateEntryPointsState(); |
| } |
| |
| void LensOverlayController::TabWillEnterBackground(tabs::TabInterface* tab) { |
| // If the current tab was already backgrounded, do nothing. |
| if (state_ == State::kBackground) { |
| DCHECK(state_ != State::kBackground) << "State should not be kBackground."; |
| return; |
| } |
| |
| // If the overlay is active, background it. |
| if (IsOverlayActive()) { |
| // If the overlay is currently showing, then we should hide the UI. |
| if (IsOverlayShowing()) { |
| HideOverlay(); |
| } |
| |
| backgrounded_state_ = state_; |
| state_ = State::kBackground; |
| UpdateEntryPointsState(); |
| |
| // TODO(crbug.com/335516480): Schedule the UI to be suspended. |
| return; |
| } |
| |
| // This is still possible when the controller is in state kScreenshot and the |
| // tab was backgrounded. We should close the UI as the overlay has not been |
| // created yet. |
| lens_search_controller_->CloseLensSync( |
| lens::LensOverlayDismissalSource::kTabBackgroundedWhileScreenshotting); |
| } |
| |
| void LensOverlayController::ActivityRequestedByOverlay( |
| ui::mojom::ClickModifiersPtr click_modifiers) { |
| // The tab is expected to be in the foreground. |
| if (!tab_->IsActivated()) { |
| return; |
| } |
| tab_->GetBrowserWindowInterface()->OpenGURL( |
| GURL(lens::features::GetLensOverlayActivityURL()), |
| ui::DispositionFromClick( |
| click_modifiers->middle_button, click_modifiers->alt_key, |
| click_modifiers->ctrl_key, click_modifiers->meta_key, |
| click_modifiers->shift_key, |
| WindowOpenDisposition::NEW_FOREGROUND_TAB)); |
| } |
| |
| void LensOverlayController::ActivityRequestedByEvent(int event_flags) { |
| // The tab is expected to be in the foreground. |
| if (!tab_->IsActivated()) { |
| return; |
| } |
| tab_->GetBrowserWindowInterface()->OpenGURL( |
| GURL(lens::features::GetLensOverlayActivityURL()), |
| ui::DispositionFromEventFlags(event_flags, |
| WindowOpenDisposition::NEW_FOREGROUND_TAB)); |
| } |
| |
| void LensOverlayController::AddBackgroundBlur() { |
| // We do not blur unless the overlay is currently active and the blur delegate |
| // was created. |
| if (!lens_overlay_blur_layer_delegate_ || |
| (state_ != State::kOverlay && state_ != State::kOverlayAndResults)) { |
| return; |
| } |
| |
| // Add our blur layer to the view. |
| overlay_web_view_->SetPaintToLayer(); |
| overlay_web_view_->layer()->Add(lens_overlay_blur_layer_delegate_->layer()); |
| overlay_web_view_->layer()->StackAtBottom( |
| lens_overlay_blur_layer_delegate_->layer()); |
| lens_overlay_blur_layer_delegate_->layer()->SetBounds( |
| overlay_web_view_->GetLocalBounds()); |
| } |
| |
| void LensOverlayController::CloseRequestedByOverlayCloseButton() { |
| lens_search_controller_->CloseLensAsync( |
| lens::LensOverlayDismissalSource::kOverlayCloseButton); |
| } |
| |
| void LensOverlayController::CloseRequestedByOverlayBackgroundClick() { |
| lens_search_controller_->CloseLensAsync( |
| lens::LensOverlayDismissalSource::kOverlayBackgroundClick); |
| } |
| |
| void LensOverlayController::FeedbackRequestedByOverlay() { |
| chrome::ShowFeedbackPage( |
| tab_->GetContents()->GetLastCommittedURL(), |
| tab_->GetBrowserWindowInterface()->GetProfile(), |
| feedback::kFeedbackSourceLensOverlay, |
| /*description_template=*/std::string(), |
| /*description_placeholder_text=*/ |
| l10n_util::GetStringUTF8(IDS_LENS_SEND_FEEDBACK_PLACEHOLDER), |
| /*category_tag=*/"lens_overlay", |
| /*extra_diagnostics=*/std::string()); |
| } |
| |
| void LensOverlayController::FeedbackRequestedByEvent(int event_flags) { |
| FeedbackRequestedByOverlay(); |
| } |
| |
| void LensOverlayController::GetOverlayInvocationSource( |
| GetOverlayInvocationSourceCallback callback) { |
| std::move(callback).Run(GetInvocationSourceString()); |
| } |
| |
| void LensOverlayController::InfoRequestedByOverlay( |
| ui::mojom::ClickModifiersPtr click_modifiers) { |
| // The tab is expected to be in the foreground. |
| if (!tab_->IsActivated()) { |
| return; |
| } |
| tab_->GetBrowserWindowInterface()->OpenGURL( |
| GURL(lens::features::GetLensOverlayHelpCenterURL()), |
| ui::DispositionFromClick( |
| click_modifiers->middle_button, click_modifiers->alt_key, |
| click_modifiers->ctrl_key, click_modifiers->meta_key, |
| click_modifiers->shift_key, |
| WindowOpenDisposition::NEW_FOREGROUND_TAB)); |
| } |
| |
| void LensOverlayController::InfoRequestedByEvent(int event_flags) { |
| // The tab is expected to be in the foreground. |
| if (!tab_->IsActivated()) { |
| return; |
| } |
| tab_->GetBrowserWindowInterface()->OpenGURL( |
| GURL(lens::features::GetLensOverlayHelpCenterURL()), |
| ui::DispositionFromEventFlags(event_flags, |
| WindowOpenDisposition::NEW_FOREGROUND_TAB)); |
| } |
| |
| void LensOverlayController::IssueLensRegionRequest( |
| lens::mojom::CenterRotatedBoxPtr region, |
| bool is_click) { |
| IssueLensRequest(std::move(region), |
| is_click ? lens::TAP_ON_EMPTY : lens::REGION_SEARCH, |
| std::nullopt); |
| } |
| |
| void LensOverlayController::IssueLensObjectRequest( |
| lens::mojom::CenterRotatedBoxPtr region, |
| bool is_mask_click) { |
| IssueLensRequest( |
| std::move(region), |
| is_mask_click ? lens::TAP_ON_REGION_GLEAM : lens::TAP_ON_OBJECT, |
| std::nullopt); |
| } |
| |
| void LensOverlayController::IssueTextSelectionRequest(const std::string& query, |
| int selection_start_index, |
| int selection_end_index, |
| bool is_translate) { |
| initialization_data_->additional_search_query_params_.clear(); |
| lens_selection_type_ = |
| is_translate ? lens::SELECT_TRANSLATED_TEXT : lens::SELECT_TEXT_HIGHLIGHT; |
| |
| IssueTextSelectionRequestInner(query, selection_start_index, |
| selection_end_index); |
| } |
| |
| void LensOverlayController::IssueTranslateSelectionRequest( |
| const std::string& query, |
| const std::string& content_language, |
| int selection_start_index, |
| int selection_end_index) { |
| initialization_data_->additional_search_query_params_.clear(); |
| lens::AppendTranslateParamsToMap( |
| initialization_data_->additional_search_query_params_, query, "auto"); |
| lens_selection_type_ = lens::TRANSLATE_CHIP; |
| |
| IssueTextSelectionRequestInner(query, selection_start_index, |
| selection_end_index); |
| } |
| |
| void LensOverlayController::IssueMathSelectionRequest( |
| const std::string& query, |
| const std::string& formula, |
| int selection_start_index, |
| int selection_end_index) { |
| initialization_data_->additional_search_query_params_.clear(); |
| lens::AppendStickinessSignalForFormula( |
| initialization_data_->additional_search_query_params_, formula); |
| lens_selection_type_ = lens::SYMBOLIC_MATH_OBJECT; |
| |
| IssueTextSelectionRequestInner(query, selection_start_index, |
| selection_end_index); |
| } |
| |
| void LensOverlayController::IssueTextSelectionRequestInner( |
| const std::string& query, |
| int selection_start_index, |
| int selection_end_index) { |
| initialization_data_->selected_region_.reset(); |
| initialization_data_->selected_region_bitmap_.reset(); |
| initialization_data_->selected_text_ = |
| std::make_pair(selection_start_index, selection_end_index); |
| |
| GetLensSearchboxController()->SetSearchboxInputText(query); |
| GetLensSearchboxController()->SetSearchboxThumbnail(std::string()); |
| |
| lens_overlay_query_controller_->SendTextOnlyQuery( |
| query, lens_selection_type_, |
| initialization_data_->additional_search_query_params_); |
| MaybeOpenSidePanel(); |
| RecordTimeToFirstInteraction( |
| lens::LensOverlayFirstInteractionType::kTextSelect); |
| state_ = State::kOverlayAndResults; |
| MaybeLaunchSurvey(); |
| } |
| |
| void LensOverlayController::ClosePreselectionBubble() { |
| if (preselection_widget_) { |
| preselection_widget_->Close(); |
| preselection_widget_ = nullptr; |
| preselection_widget_observer_.Reset(); |
| } |
| } |
| |
| void LensOverlayController::ShowPreselectionBubble() { |
| // Don't show the preselection bubble if the overlay is not being shown. |
| if (!should_show_overlay_ || state() == State::kOverlayAndResults) { |
| return; |
| } |
| |
| #if BUILDFLAG(IS_MAC) |
| // On Mac, the kShowFullscreenToolbar pref is used to determine whether the |
| // toolbar is always shown. This causes the toolbar to never unreveal, meaning |
| // the preselection bubble will never be shown. Check for this case and show |
| // the preselection bubble if needed. |
| const bool always_show_toolbar = |
| pref_service_->GetBoolean(prefs::kShowFullscreenToolbar); |
| #else |
| const bool always_show_toolbar = false; |
| #endif // BUILDFLAG(IS_MAC) |
| |
| if (!always_show_toolbar && tab_->GetBrowserWindowInterface() |
| ->GetImmersiveModeController() |
| ->IsRevealed()) { |
| // If the immersive mode controller is revealing top chrome, do not show |
| // the preselection bubble. The bubble will be shown once the reveal |
| // finishes. |
| return; |
| } |
| |
| if (!preselection_widget_) { |
| CHECK(preselection_widget_anchor_); |
| // Setup the preselection widget. |
| preselection_widget_ = views::BubbleDialogDelegateView::CreateBubble( |
| std::make_unique<lens::LensPreselectionBubble>( |
| weak_factory_.GetWeakPtr(), preselection_widget_anchor_, |
| net::NetworkChangeNotifier::IsOffline(), |
| /*exit_clicked_callback=*/ |
| base::BindRepeating( |
| &LensSearchController::CloseLensSync, |
| lens_search_controller_->GetWeakPtr(), |
| lens::LensOverlayDismissalSource::kPreselectionToastExitButton), |
| /*on_cancel_callback=*/ |
| base::BindOnce(&LensSearchController::CloseLensSync, |
| lens_search_controller_->GetWeakPtr(), |
| lens::LensOverlayDismissalSource:: |
| kPreselectionToastEscapeKeyPress))); |
| preselection_widget_->SetNativeWindowProperty( |
| views::kWidgetIdentifierKey, |
| const_cast<void*>(kLensOverlayPreselectionWidgetIdentifier)); |
| preselection_widget_observer_.Observe(preselection_widget_); |
| // Setting the parent allows focus traversal out of the preselection widget. |
| preselection_widget_->SetFocusTraversableParent( |
| preselection_widget_anchor_->GetWidget()->GetFocusTraversable()); |
| preselection_widget_->SetFocusTraversableParentView( |
| preselection_widget_anchor_); |
| } |
| |
| // When in fullscreen, top Chrome may cover this widget on Mac. Set the |
| // z-order to floating UI element to ensure the widget is above the top |
| // Chrome. Only do this if immersive mode is enabled to avoid issues with |
| // the preselection widget covering other windows. |
| if (tab_->GetBrowserWindowInterface() |
| ->GetImmersiveModeController() |
| ->IsEnabled()) { |
| preselection_widget_->SetZOrderLevel(ui::ZOrderLevel::kFloatingUIElement); |
| } else { |
| preselection_widget_->SetZOrderLevel(ui::ZOrderLevel::kNormal); |
| } |
| |
| auto* bubble_view = static_cast<lens::LensPreselectionBubble*>( |
| preselection_widget_->widget_delegate()); |
| bubble_view->SetCanActivate(true); |
| |
| // The bubble position is dependent on if top chrome is showing. Resize the |
| // bubble to ensure the correct position is used. |
| bubble_view->SizeToContents(); |
| // Show inactive so that the overlay remains active. |
| preselection_widget_->ShowInactive(); |
| } |
| |
| void LensOverlayController::CloseAndReshowPreselectionBubble() { |
| // If the preselection bubble is already closed, do not reshow it. |
| if (!preselection_widget_) { |
| return; |
| } |
| ClosePreselectionBubble(); |
| ShowPreselectionBubble(); |
| } |
| |
| void LensOverlayController::HidePreselectionBubble() { |
| if (preselection_widget_) { |
| // The preselection bubble remains in the browser's focus order even when it |
| // is hidden, for example, when another browser tab is active. This means it |
| // remains possible for the bubble to be activated by keyboard input i.e. |
| // tabbing into the bubble, which unhides the bubble even on a browser tab |
| // where the overlay is not being shown. Prevent this by setting the bubble |
| // to non-activatable while it is hidden. |
| auto* bubble_view = static_cast<lens::LensPreselectionBubble*>( |
| preselection_widget_->widget_delegate()); |
| bubble_view->SetCanActivate(false); |
| |
| preselection_widget_->Hide(); |
| } |
| } |
| |
| void LensOverlayController::IssueSearchBoxRequestPart2( |
| const std::string& search_box_text, |
| AutocompleteMatchType::Type match_type, |
| bool is_zero_prefix_suggestion, |
| std::map<std::string, std::string> additional_query_params) { |
| // If the overlay is closing or is off, do not attempt to issue the query. |
| if (IsOverlayClosing() || state() == State::kOff) { |
| return; |
| } |
| initialization_data_->additional_search_query_params_ = |
| additional_query_params; |
| |
| if (initialization_data_->selected_region_.is_null() && |
| GetPageClassification() == |
| metrics::OmniboxEventProto::SEARCH_SIDE_PANEL_SEARCHBOX) { |
| // Non-Lens and non-contextual searches should not have a selection type. |
| lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; |
| } else if (is_zero_prefix_suggestion) { |
| lens_selection_type_ = lens::MULTIMODAL_SUGGEST_ZERO_PREFIX; |
| } else if (match_type == AutocompleteMatchType::Type::SEARCH_WHAT_YOU_TYPED) { |
| lens_selection_type_ = lens::MULTIMODAL_SEARCH; |
| } else { |
| lens_selection_type_ = lens::MULTIMODAL_SUGGEST_TYPEAHEAD; |
| } |
| |
| if (!is_page_context_eligible_) { |
| // Do not send any requests if the page is not context eligible. |
| } else if (initialization_data_->selected_region_.is_null() && |
| IsContextualSearchbox()) { |
| lens_overlay_query_controller_->SendContextualTextQuery( |
| search_box_text, lens_selection_type_, |
| initialization_data_->additional_search_query_params_); |
| csb_session_end_metrics_.zps_used_ = |
| csb_session_end_metrics_.zps_used_ || is_zero_prefix_suggestion; |
| csb_session_end_metrics_.query_issued_ = true; |
| if (state() == State::kLivePageAndResults) { |
| csb_session_end_metrics_.follow_up_query_issued_ = true; |
| } |
| if (state() == State::kOverlay && |
| !csb_session_end_metrics_.zps_shown_on_initial_query_) { |
| // If the query was made in the initial state, and the ZPS has not been |
| // shown, mark the query as issued before ZPS shown. |
| csb_session_end_metrics_.initial_query_issued_before_zps_shown_ = true; |
| } else if (state() == State::kLivePageAndResults && |
| !csb_session_end_metrics_.zps_shown_on_follow_up_query_) { |
| // If a follow up query was made, and the ZPS has not been |
| // shown for the follow up query, mark the query as issued before ZPS |
| // shown. |
| csb_session_end_metrics_.follow_up_query_issued_before_zps_shown_ = true; |
| } |
| } else if (initialization_data_->selected_region_.is_null()) { |
| lens_overlay_query_controller_->SendTextOnlyQuery( |
| search_box_text, lens_selection_type_, |
| initialization_data_->additional_search_query_params_); |
| } else { |
| std::optional<SkBitmap> selected_region_bitmap = |
| initialization_data_->selected_region_bitmap_.drawsNothing() |
| ? std::nullopt |
| : std::make_optional<SkBitmap>( |
| initialization_data_->selected_region_bitmap_); |
| lens_overlay_query_controller_->SendMultimodalRequest( |
| initialization_data_->selected_region_.Clone(), search_box_text, |
| lens_selection_type_, |
| initialization_data_->additional_search_query_params_, |
| selected_region_bitmap); |
| } |
| |
| // If we are in the zero state, this request must have come from CSB. In that |
| // case, hide the overlay to allow live page to show through. |
| if (state_ == State::kOverlay) { |
| HideOverlay(); |
| } |
| |
| // If this a search query from the side panel search box with the overlay |
| // showing, keep the state as kOverlayAndResults. Else, we are in our |
| // contextual flow and the state needs to stay as State::kLivePageAndResults. |
| state_ = state_ == State::kOverlayAndResults ? State::kOverlayAndResults |
| : State::kLivePageAndResults; |
| |
| // The searchbox text is set once the URL loads in the results frame, however, |
| // adding it here allows the user to see the text query in the searchbox while |
| // a long query loads. |
| GetLensSearchboxController()->SetSearchboxInputText(search_box_text); |
| |
| MaybeOpenSidePanel(); |
| // Only set the side panel to loading if the page is context eligible because |
| // otherwise there will be no results to load. |
| results_side_panel_coordinator_->SetSidePanelIsLoadingResults( |
| is_page_context_eligible_); |
| MaybeLaunchSurvey(); |
| |
| // After the searchbox request is sent, mark the follow up zps as not shown so |
| // it is false for the next follow up query. |
| csb_session_end_metrics_.zps_shown_on_follow_up_query_ = false; |
| } |
| |
| void LensOverlayController::HandleStartQueryResponse( |
| std::vector<lens::mojom::OverlayObjectPtr> objects, |
| lens::mojom::TextPtr text, |
| bool is_error) { |
| // If the side panel is open, then the error page state can change depending |
| // on whether the query succeeded or not. If the side panel is not open, the |
| // error page state can only change if the query failed since the first side |
| // panel navigation will take care of recording whether the result was shown |
| const bool is_side_panel_open = |
| results_side_panel_coordinator_->IsSidePanelBound(); |
| if (is_side_panel_open) { |
| results_side_panel_coordinator_->MaybeSetSidePanelShowErrorPage( |
| is_error, |
| is_error |
| ? lens::mojom::SidePanelResultStatus::kErrorPageShownStartQueryError |
| : lens::mojom::SidePanelResultStatus::kResultShown); |
| } else if (!is_side_panel_open && is_error) { |
| results_side_panel_coordinator_->MaybeSetSidePanelShowErrorPage( |
| /*should_show_error_page=*/true, |
| lens::mojom::SidePanelResultStatus::kErrorPageShownStartQueryError); |
| } |
| |
| if (!objects.empty()) { |
| SendObjects(std::move(objects)); |
| } |
| |
| // Text can be null if there was no text within the server response. |
| if (!text.is_null()) { |
| // If the initialization data is not yet ready, SendText will store the text |
| // to be attached when ready. |
| if (initialization_data_) { |
| initialization_data_->text_ = text.Clone(); |
| } |
| |
| SendText(std::move(text)); |
| |
| // Try and record the OCR DOM similarity since the OCR text is now |
| // available. |
| TryCalculateAndRecordOcrDomSimilarity(); |
| } |
| } |
| |
| void LensOverlayController::HandleInteractionURLResponse( |
| lens::proto::LensOverlayUrlResponse response) { |
| MaybeOpenSidePanel(); |
| if (lens::features::IsLensSearchSidePanelScrollToAPIEnabled()) { |
| results_side_panel_coordinator_->SetLatestPageUrlWithResponse( |
| GURL(response.page_url())); |
| } |
| results_side_panel_coordinator_->LoadURLInResultsFrame(GURL(response.url())); |
| } |
| |
| void LensOverlayController::HandleInteractionResponse( |
| lens::mojom::TextPtr text) { |
| SendText(std::move(text)); |
| } |
| |
| void LensOverlayController::HandleSuggestInputsResponse( |
| lens::proto::LensOverlaySuggestInputs suggest_inputs) { |
| if (!initialization_data_) { |
| // If the initialization data is not ready, store the suggest inputs to be |
| // attached to the initialization data when it is ready. |
| pre_initialization_suggest_inputs_ = std::make_optional(suggest_inputs); |
| return; |
| } |
| |
| // If the handshake was already complete, without the new suggest inputs, |
| // exit early so that we do not notify OnHandshakeComplete() multiple times. |
| if (IsHandshakeComplete()) { |
| initialization_data_->suggest_inputs_ = suggest_inputs; |
| return; |
| } |
| |
| // Check if the handshake with the server has been completed with the new |
| // inputs. If so, this is the first time we are receiving the suggest inputs, |
| // so we need to notify OnHandshakeComplete() to allow the searchbox to query |
| // autocomplete. |
| initialization_data_->suggest_inputs_ = suggest_inputs; |
| if (IsHandshakeComplete()) { |
| // Notify the overlay that it is now safe to query autocomplete. |
| OnHandshakeComplete(); |
| } |
| } |
| |
| void LensOverlayController::HandlePageContentUploadProgress(uint64_t position, |
| uint64_t total) { |
| // If the progress bar is disabled, do not show it. |
| if (!lens::features::ShouldShowUploadProgressBar() || |
| !is_upload_progress_bar_shown_ || !IsContextualSearchbox()) { |
| return; |
| } |
| |
| float progress = total > 0 ? static_cast<float>(position) / total : 1.0f; |
| |
| // For the first upload handler event received, check if the progress is above |
| // the heuristic threshold. If so, do not show the progress bar because it is |
| // assumed that the upload will finish quickly, and showing the progress bar |
| // would be distracting. |
| if (is_first_upload_handler_event_) { |
| is_first_upload_handler_event_ = false; |
| if (total > 0 && |
| progress > lens::features::GetUploadProgressBarShowHeuristic()) { |
| is_upload_progress_bar_shown_ = false; |
| return; |
| } |
| } |
| |
| results_side_panel_coordinator_->SetPageContentUploadProgress( |
| total > 0 ? static_cast<float>(position) / total : 1.0f); |
| } |
| |
| void LensOverlayController::RecordTimeToFirstInteraction( |
| lens::LensOverlayFirstInteractionType interaction_type) { |
| if (search_performed_in_session_) { |
| return; |
| } |
| DCHECK(!invocation_time_.is_null()); |
| base::TimeDelta time_to_first_interaction = |
| base::TimeTicks::Now() - invocation_time_; |
| ukm::SourceId source_id = |
| tab_->GetContents()->GetPrimaryMainFrame()->GetPageUkmSourceId(); |
| // UMA and UKM TimeToFirstInteraction. |
| lens::RecordTimeToFirstInteraction(invocation_source_, |
| time_to_first_interaction, |
| interaction_type, source_id); |
| search_performed_in_session_ = true; |
| } |
| |
| void LensOverlayController:: |
| RecordContextualSearchboxTimeToFocusAfterNavigation() { |
| if (!last_navigation_time_.has_value() || |
| contextual_searchbox_focused_after_navigation_) { |
| return; |
| } |
| base::TimeDelta time_to_focus = |
| base::TimeTicks::Now() - last_navigation_time_.value(); |
| lens::RecordContextualSearchboxTimeToFocusAfterNavigation( |
| time_to_focus, initialization_data_->primary_content_type_); |
| contextual_searchbox_focused_after_navigation_ = true; |
| } |
| |
| void LensOverlayController:: |
| RecordContextualSearchboxTimeToInteractionAfterNavigation() { |
| if (!last_navigation_time_.has_value()) { |
| return; |
| } |
| base::TimeDelta time_to_interaction = |
| base::TimeTicks::Now() - last_navigation_time_.value(); |
| lens::RecordContextualSearchboxTimeToInteractionAfterNavigation( |
| time_to_interaction, initialization_data_->primary_content_type_); |
| last_navigation_time_.reset(); |
| } |
| |
| void LensOverlayController::RecordEndOfSessionMetrics( |
| lens::LensOverlayDismissalSource dismissal_source) { |
| // UMA unsliced Dismissed. |
| lens::RecordDismissal(dismissal_source); |
| |
| // UMA InvocationResultedInSearch. |
| lens::RecordInvocationResultedInSearch(invocation_source_, |
| search_performed_in_session_); |
| |
| // UMA session duration. |
| DCHECK(!invocation_time_.is_null()); |
| base::TimeDelta session_duration = base::TimeTicks::Now() - invocation_time_; |
| lens::RecordSessionDuration(invocation_source_, session_duration); |
| |
| // UKM session end metrics. Includes invocation source, whether the |
| // session resulted in a search, invocation document type and session |
| // duration. |
| ukm::SourceId source_id = |
| tab_->GetContents()->GetPrimaryMainFrame()->GetPageUkmSourceId(); |
| lens::RecordUKMSessionEndMetrics(source_id, invocation_source_, |
| search_performed_in_session_, |
| session_duration, initial_document_type_); |
| |
| // UMA and UKM end of session metrics for the CSB. Only recorded if CSB is |
| // shown in session. |
| lens::RecordContextualSearchboxSessionEndMetrics( |
| source_id, csb_session_end_metrics_, initial_page_content_type_, |
| initial_document_type_); |
| } |
| |
| void LensOverlayController::RecordDocumentMetrics( |
| std::optional<uint32_t> page_count) { |
| // Use the first lens::PageContent as it is the most important PDF/Webpage. |
| // TODO(crbug.com/398304347): Ideally all lens::PageContent sizes should be |
| // recorded. |
| auto content_type = |
| initialization_data_->page_contents_.empty() |
| ? lens::MimeType::kUnknown |
| : initialization_data_->page_contents_.front().content_type_; |
| auto page_content_bytes_size = |
| initialization_data_->page_contents_.empty() |
| ? 0 |
| : initialization_data_->page_contents_.front().bytes_.size(); |
| lens::RecordDocumentSizeBytes(content_type, page_content_bytes_size); |
| |
| if (page_count.has_value() && content_type == lens::MimeType::kPdf) { |
| lens::RecordPdfPageCount(page_count.value()); |
| return; |
| } |
| |
| // Fetch and record the other content type for representing the webpage. |
| // TODO(crbug.com/398304347): Remove these once both the innerHtml and |
| // innerText metrics are recorded as part of the content data. |
| auto* render_frame_host = tab_->GetContents()->GetPrimaryMainFrame(); |
| if (content_type == lens::MimeType::kHtml) { |
| // Fetch the innerText to log the size. |
| content_extraction::GetInnerText( |
| *render_frame_host, /*node_id=*/std::nullopt, |
| base::BindOnce(&LensOverlayController::RecordInnerTextSize, |
| weak_factory_.GetWeakPtr())); |
| } else if (content_type == lens::MimeType::kPlainText) { |
| // Fetch the innerHtml bytes to log the size. |
| content_extraction::GetInnerHtml( |
| *render_frame_host, |
| base::BindOnce(&LensOverlayController::RecordInnerHtmlSize, |
| weak_factory_.GetWeakPtr())); |
| } |
| |
| // Try and record the OCR DOM similarity since the page content is now |
| // available. |
| TryCalculateAndRecordOcrDomSimilarity(); |
| } |
| |
| void LensOverlayController::TryCalculateAndRecordOcrDomSimilarity() { |
| // Only record the similarity once per session. |
| if (ocr_dom_similarity_recorded_in_session_) { |
| return; |
| } |
| |
| // Exit early if we do not have all the data needed to calculate the |
| // similarity. |
| if (!initialization_data_ || initialization_data_->text_.is_null() || |
| initialization_data_->page_contents_.empty()) { |
| return; |
| } |
| |
| const auto& page_content_bytes = |
| initialization_data_->page_contents_.front().bytes_; |
| |
| const auto primary_content_type = initialization_data_->primary_content_type_; |
| bool is_dom = primary_content_type == lens::MimeType::kHtml || |
| primary_content_type == lens::MimeType::kPlainText || |
| primary_content_type == lens::MimeType::kAnnotatedPageContent; |
| bool is_dom_too_large = |
| page_content_bytes.size() > kMaxDomTextLengthForOcrSimilarity; |
| bool is_english = initialization_data_->text_->content_language == "en"; |
| |
| // Exit early if the page content is not from the DOM, the DOM is very large |
| // and might bog down the thread, or the page is not in English since the |
| // score is not reliable for other languages. |
| if (!is_dom || is_dom_too_large || !is_english) { |
| // If the page content is not from the HTML DOM, we cannot calculate the |
| // similarity, so mark this as true to avoid trying again. |
| ocr_dom_similarity_recorded_in_session_ = true; |
| return; |
| } |
| |
| // Post to a background thread to calculate the similarity to avoid slowing |
| // down the main thread. |
| base::ThreadPool::PostTaskAndReplyWithResult( |
| FROM_HERE, {base::TaskPriority::BEST_EFFORT}, |
| base::BindOnce( |
| &CalculateWordOverlapSimilarity, |
| std::string(page_content_bytes.begin(), page_content_bytes.end()), |
| initialization_data_->text_.Clone()), |
| base::BindOnce(&lens::RecordOcrDomSimilarity)); |
| ocr_dom_similarity_recorded_in_session_ = true; |
| } |
| |
| void LensOverlayController::RecordInnerTextSize( |
| std::unique_ptr<content_extraction::InnerTextResult> result) { |
| if (!result) { |
| return; |
| } |
| lens::RecordDocumentSizeBytes(lens::MimeType::kPlainText, |
| result->inner_text.size()); |
| } |
| |
| void LensOverlayController::RecordInnerHtmlSize( |
| const std::optional<std::string>& result) { |
| if (!result) { |
| return; |
| } |
| lens::RecordDocumentSizeBytes(lens::MimeType::kHtml, result->size()); |
| } |
| |
| void LensOverlayController::MaybeLaunchSurvey() { |
| if (!base::FeatureList::IsEnabled(lens::features::kLensOverlaySurvey)) { |
| return; |
| } |
| if (hats_triggered_in_session_) { |
| return; |
| } |
| HatsService* hats_service = HatsServiceFactory::GetForProfile( |
| tab_->GetBrowserWindowInterface()->GetProfile(), |
| /*create_if_necessary=*/true); |
| if (!hats_service) { |
| // HaTS may not be available in e.g. guest profile |
| return; |
| } |
| hats_triggered_in_session_ = true; |
| hats_service->LaunchDelayedSurveyForWebContents( |
| kHatsSurveyTriggerLensOverlayResults, tab_->GetContents(), |
| lens::features::GetLensOverlaySurveyResultsTime().InMilliseconds(), |
| /*product_specific_bits_data=*/{}, |
| /*product_specific_string_data=*/ |
| {{"ID that's tied to your Google Lens session", |
| base::NumberToString(lens_overlay_query_controller_->gen204_id())}}); |
| } |
| |
| void LensOverlayController::InitializeTutorialIPHUrlMatcher() { |
| if (!base::FeatureList::IsEnabled( |
| feature_engagement::kIPHLensOverlayFeature)) { |
| return; |
| } |
| |
| tutorial_iph_url_matcher_ = std::make_unique<url_matcher::URLMatcher>(); |
| base::MatcherStringPattern::ID id(0); |
| url_matcher::util::AddFiltersWithLimit( |
| tutorial_iph_url_matcher_.get(), true, &id, |
| JSONArrayToVector( |
| feature_engagement::kIPHLensOverlayUrlAllowFilters.Get()), |
| &iph_url_filters_); |
| url_matcher::util::AddFiltersWithLimit( |
| tutorial_iph_url_matcher_.get(), false, &id, |
| JSONArrayToVector( |
| feature_engagement::kIPHLensOverlayUrlBlockFilters.Get()), |
| &iph_url_filters_); |
| |
| auto force_allow_url_strings = JSONArrayToVector( |
| feature_engagement::kIPHLensOverlayUrlForceAllowedUrlMatchPatterns.Get()); |
| std::vector<base::MatcherStringPattern> force_allow_url_patterns; |
| std::vector<const base::MatcherStringPattern*> force_allow_url_pointers; |
| force_allow_url_patterns.reserve(force_allow_url_strings.size()); |
| force_allow_url_pointers.reserve(force_allow_url_strings.size()); |
| for (const std::string& entry : force_allow_url_strings) { |
| force_allow_url_patterns.emplace_back(entry, ++id); |
| force_allow_url_pointers.push_back(&force_allow_url_patterns.back()); |
| } |
| forced_url_matcher_ = std::make_unique<url_matcher::RegexSetMatcher>(); |
| // Pointers will not be referenced after AddPatterns() completes. |
| forced_url_matcher_->AddPatterns(force_allow_url_pointers); |
| |
| auto allow_strings = JSONArrayToVector( |
| feature_engagement::kIPHLensOverlayUrlPathMatchAllowPatterns.Get()); |
| std::vector<base::MatcherStringPattern> allow_patterns; |
| std::vector<const base::MatcherStringPattern*> allow_pointers; |
| allow_patterns.reserve(allow_strings.size()); |
| allow_pointers.reserve(allow_strings.size()); |
| for (const std::string& entry : allow_strings) { |
| allow_patterns.emplace_back(entry, ++id); |
| allow_pointers.push_back(&allow_patterns.back()); |
| } |
| page_path_allow_matcher_ = std::make_unique<url_matcher::RegexSetMatcher>(); |
| // Pointers will not be referenced after AddPatterns() completes. |
| page_path_allow_matcher_->AddPatterns(allow_pointers); |
| |
| auto block_strings = JSONArrayToVector( |
| feature_engagement::kIPHLensOverlayUrlPathMatchBlockPatterns.Get()); |
| std::vector<base::MatcherStringPattern> block_patterns; |
| std::vector<const base::MatcherStringPattern*> block_pointers; |
| block_patterns.reserve(block_strings.size()); |
| block_pointers.reserve(block_strings.size()); |
| for (const std::string& entry : block_strings) { |
| block_patterns.emplace_back(entry, ++id); |
| block_pointers.push_back(&block_patterns.back()); |
| } |
| page_path_block_matcher_ = std::make_unique<url_matcher::RegexSetMatcher>(); |
| // Pointers will not be referenced after AddPatterns() completes. |
| page_path_block_matcher_->AddPatterns(block_pointers); |
| } |
| |
| void LensOverlayController::MaybeShowDelayedTutorialIPH(const GURL& url) { |
| auto* entry_point_controller = tab_->GetBrowserWindowInterface() |
| ->GetFeatures() |
| .lens_overlay_entry_point_controller(); |
| if (!entry_point_controller || !entry_point_controller->IsEnabled()) { |
| return; |
| } |
| |
| // If a tutorial IPH was already queued, cancel it. |
| tutorial_iph_timer_.Stop(); |
| |
| if (IsUrlEligibleForTutorialIPH(url)) { |
| tutorial_iph_timer_.Start( |
| FROM_HERE, feature_engagement::kIPHLensOverlayDelayTime.Get(), |
| base::BindOnce(&LensOverlayController::ShowTutorialIPH, |
| weak_factory_.GetWeakPtr())); |
| } |
| } |
| |
| void LensOverlayController::UpdateNavigationMetrics() { |
| last_navigation_time_ = base::TimeTicks::Now(); |
| contextual_searchbox_focused_after_navigation_ = false; |
| } |
| |
| bool LensOverlayController::IsHandshakeComplete() { |
| if (!initialization_data_ && !pre_initialization_suggest_inputs_) { |
| return false; |
| } |
| const auto& suggest_inputs = initialization_data_ |
| ? initialization_data_->suggest_inputs_ |
| : pre_initialization_suggest_inputs_; |
| return AreLensSuggestInputsReady(suggest_inputs); |
| } |
| |
| bool LensOverlayController::IsUrlEligibleForTutorialIPH(const GURL& url) { |
| if (!tutorial_iph_url_matcher_) { |
| return false; |
| } |
| |
| // Check if the URL matches any of the allow filters. If it does not, |
| // return false immediately as this should not be a shown match. |
| auto matches = tutorial_iph_url_matcher_.get()->MatchURL(url); |
| if (!matches.size()) { |
| return false; |
| } |
| |
| // Now that the URL is allowed, check if it matches any of the block filters. |
| // If it does, return false as to block this URL from showing the IPH. |
| for (auto match : matches) { |
| // Blocks take precedence over allows. |
| if (!iph_url_filters_[match].allow) { |
| return false; |
| } |
| } |
| |
| // Now that the URL is an allowed URL, verify the match is not blocked by |
| // the block matcher. If it does contain blocked words in its path, return |
| // false to prevent the IPH from being shown. |
| if (page_path_block_matcher_ && !page_path_block_matcher_->IsEmpty() && |
| page_path_block_matcher_->Match(url.path(), &matches)) { |
| return false; |
| } |
| |
| // Check if the URL matches any of the forced allowed URLs. If it does, return |
| // true as this should be a shown match even if the path does not contain an |
| // allowlisted pattern (below). |
| if (forced_url_matcher_ && !forced_url_matcher_->IsEmpty() && |
| forced_url_matcher_->Match(url.spec(), &matches)) { |
| return true; |
| } |
| |
| // Finally, check if the URL matches any of the allowed patterns. If it |
| // doesn't, return false to prevent the IPH from being shown. |
| if (page_path_allow_matcher_ && !page_path_allow_matcher_->IsEmpty() && |
| !page_path_allow_matcher_->Match(url.path(), &matches)) { |
| return false; |
| } |
| |
| // Finally if all checks pass, this must be a valid match. I.e.: |
| // 1. The URL matches at least one of the allowed URLs. |
| // 2. The URL does not match any of the blocked URLs. |
| // 3. The URL does not match any of the block path patterns. |
| // 4. The URL matches at least one of the allowed path patterns. |
| return true; |
| } |
| |
| void LensOverlayController::ShowTutorialIPH() { |
| if (auto* user_ed = |
| tab_->GetBrowserWindowInterface()->GetUserEducationInterface()) { |
| user_ed->MaybeShowFeaturePromo(feature_engagement::kIPHLensOverlayFeature); |
| } |
| } |
| |
| void LensOverlayController::NotifyUserEducationAboutOverlayUsed() { |
| if (auto* user_ed = |
| tab_->GetBrowserWindowInterface()->GetUserEducationInterface()) { |
| user_ed->NotifyFeaturePromoFeatureUsed( |
| feature_engagement::kIPHLensOverlayFeature, |
| FeaturePromoFeatureUsedAction::kClosePromoIfPresent); |
| } |
| } |
| |
| void LensOverlayController::NotifyPageContentUpdated() { |
| auto page_content_type = lens::StringMimeTypeToMojoPageContentType( |
| tab_->GetContents()->GetContentsMimeType()); |
| if (page_) { |
| page_->PageContentTypeChanged(page_content_type); |
| } |
| results_side_panel_coordinator_->NotifyPageContentUpdated(); |
| } |
| |
| void LensOverlayController::UpdateEntryPointsState() { |
| tab_->GetBrowserWindowInterface() |
| ->GetFeatures() |
| .lens_overlay_entry_point_controller() |
| ->UpdateEntryPointsState( |
| /*hide_toolbar_entrypoint=*/false); |
| } |
| |
| void LensOverlayController::OnPdfPartialPageTextRetrieved( |
| std::vector<std::u16string> pdf_pages_text) { |
| initialization_data_->pdf_pages_text_ = std::move(pdf_pages_text); |
| } |
| |
| lens::LensSearchboxController* |
| LensOverlayController::GetLensSearchboxController() { |
| return lens_search_controller_->lens_searchbox_controller(); |
| } |
| |
| lens::LensSearchContextualizationController* |
| LensOverlayController::GetContextualizationController() { |
| return lens_search_controller_->lens_search_contextualization_controller(); |
| } |