blob: 39a8a3e0c031ae3d604b14d8896a1c764c384a65 [file] [log] [blame]
// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/ash/mahi/mahi_manager_impl.h"
#include <stdint.h>
#include <algorithm>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>
#include "ash/constants/ash_pref_names.h"
#include "ash/shell.h"
#include "ash/system/mahi/mahi_nudge_controller.h"
#include "ash/system/mahi/mahi_ui_controller.h"
#include "ash/webui/settings/public/constants/routes.mojom.h"
#include "ash/webui/settings/public/constants/setting.mojom.h"
#include "base/check.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/memory/weak_ptr.h"
#include "base/metrics/histogram_functions.h"
#include "base/scoped_observation.h"
#include "base/strings/strcat.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "base/unguessable_token.h"
#include "base/values.h"
#include "chrome/browser/ash/magic_boost/magic_boost_controller_ash.h"
#include "chrome/browser/ash/mahi/mahi_availability.h"
#include "chrome/browser/ash/mahi/mahi_cache_manager.h"
#include "chrome/browser/feedback/show_feedback_page.h"
#include "chrome/browser/history/history_service_factory.h"
#include "chrome/browser/manta/manta_service_factory.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/browser/ui/browser_finder.h"
#include "chrome/browser/ui/settings_window_manager_chromeos.h"
#include "chromeos/components/magic_boost/public/cpp/magic_boost_state.h"
#include "chromeos/components/mahi/public/cpp/mahi_manager.h"
#include "chromeos/components/mahi/public/cpp/mahi_media_app_content_manager.h"
#include "chromeos/components/mahi/public/cpp/mahi_web_contents_manager.h"
#include "chromeos/constants/chromeos_features.h"
#include "chromeos/crosapi/mojom/magic_boost.mojom.h"
#include "chromeos/crosapi/mojom/mahi.mojom.h"
#include "chromeos/strings/grit/chromeos_strings.h"
#include "components/feedback/feedback_constants.h"
#include "components/manta/manta_service.h"
#include "ui/base/l10n/l10n_util.h"
#include "ui/gfx/image/image_skia.h"
#include "ui/views/widget/unique_widget_ptr.h"
namespace {
// Aliases ---------------------------------------------------------------------
using chromeos::MahiGetContentResponseStatus;
using chromeos::MahiResponseStatus;
using crosapi::mojom::MahiContextMenuActionType;
// Constants -------------------------------------------------------------------
const char kMahiCacheHit[] = "ChromeOS.Mahi.CacheStateOnAccess";
const char kMahiResponseStatus[] = "ChromeOS.Mahi.ResponseStatusOnRequest";
const char kMahiProviderCreationStatus[] =
"ChromeOS.Mahi.ProviderCreationStatus";
const char kMediaAppPDFUrlPrefix[] = "file:///media-app/";
// The following enum classes are persisted to logs. Entries should not be
// renumbered and numeric values should never be reused.
// CacheHit --------------------------------------------------------------------
enum class CacheHit {
kNoHit = 0,
kSummary = 1,
kContent = 2,
kMaxValue = kContent,
};
// Provider creation -----------------------------------------------------------
enum class ProviderCreationStatus {
kOk = 0,
// kMantaServiceDisabled = 1,
kProfileUnavailable = 2,
kMantaServiceIsNull = 3,
kMantaServiceFailedToCreate = 4,
kMaxValue = kMantaServiceFailedToCreate,
};
void LogProviderCreationStatus(ProviderCreationStatus status) {
base::UmaHistogramEnumeration(kMahiProviderCreationStatus, status);
}
std::optional<std::string> MaybeGetUrl(
const crosapi::mojom::MahiPageInfoPtr& mahi_page_info) {
// Do not send the fake URL of media app PDF files.
return chromeos::features::IsMahiSendingUrl() &&
!mahi_page_info->url.spec().starts_with(kMediaAppPDFUrlPrefix)
? std::make_optional(mahi_page_info->url.spec())
: std::nullopt;
}
// OnConsentStateUpdateClosureRunner -------------------------------------------
// Runs the specified closures when the consent state becomes approved or
// declined. NOTE: This class should be used only when the magic boost feature
// is enabled.
class OnConsentStateUpdateClosureRunner
: public chromeos::MagicBoostState::Observer {
public:
OnConsentStateUpdateClosureRunner(base::OnceClosure on_approved_closure,
base::OnceClosure on_declined_closure)
: on_approved_closure_(std::move(on_approved_closure)),
on_declined_closure_(std::move(on_declined_closure)) {
CHECK(chromeos::MagicBoostState::Get()->IsUserEligibleForGenAIFeatures());
magic_boost_state_observation_.Observe(chromeos::MagicBoostState::Get());
}
OnConsentStateUpdateClosureRunner(const OnConsentStateUpdateClosureRunner&) =
delete;
OnConsentStateUpdateClosureRunner& operator=(
const OnConsentStateUpdateClosureRunner&) = delete;
~OnConsentStateUpdateClosureRunner() override = default;
private:
// chromeos::MagicBoostState::Observer:
void OnHMRConsentStatusUpdated(chromeos::HMRConsentStatus status) override {
switch (status) {
case chromeos::HMRConsentStatus::kApproved:
magic_boost_state_observation_.Reset();
std::move(on_approved_closure_).Run();
return;
case chromeos::HMRConsentStatus::kDeclined:
magic_boost_state_observation_.Reset();
std::move(on_declined_closure_).Run();
return;
case chromeos::HMRConsentStatus::kPendingDisclaimer:
case chromeos::HMRConsentStatus::kUnset:
return;
}
}
void OnIsDeleting() override { magic_boost_state_observation_.Reset(); }
// The closure that runs when the consent status becomes approved.
base::OnceClosure on_approved_closure_;
// The closure that runs when the consent status becomes declined.
// NOTE: `on_declined_closure_` could destroy this observer.
base::OnceClosure on_declined_closure_;
base::ScopedObservation<chromeos::MagicBoostState,
chromeos::MagicBoostState::Observer>
magic_boost_state_observation_{this};
};
MahiResponseStatus GetMahiResponseStatusFromMantaStatus(
manta::MantaStatusCode code) {
switch (code) {
case manta::MantaStatusCode::kOk:
return MahiResponseStatus::kSuccess;
case manta::MantaStatusCode::kGenericError:
case manta::MantaStatusCode::kBackendFailure:
case manta::MantaStatusCode::kNoInternetConnection:
case manta::MantaStatusCode::kNoIdentityManager:
return MahiResponseStatus::kUnknownError;
case manta::MantaStatusCode::kRestrictedCountry:
return MahiResponseStatus::kRestrictedCountry;
case manta::MantaStatusCode::kUnsupportedLanguage:
return MahiResponseStatus::kUnsupportedLanguage;
case manta::MantaStatusCode::kBlockedOutputs:
return MahiResponseStatus::kInappropriate;
case manta::MantaStatusCode::kResourceExhausted:
return MahiResponseStatus::kResourceExhausted;
case manta::MantaStatusCode::kPerUserQuotaExceeded:
return MahiResponseStatus::kQuotaLimitHit;
default:
return MahiResponseStatus::kUnknownError;
}
}
std::unique_ptr<manta::MahiProvider> CreateProvider() {
Profile* profile = ProfileManager::GetActiveUserProfile();
if (!profile) {
LogProviderCreationStatus(ProviderCreationStatus::kProfileUnavailable);
return nullptr;
}
if (manta::MantaService* service =
manta::MantaServiceFactory::GetForProfile(profile)) {
auto provider = service->CreateMahiProvider();
if (!provider) {
LogProviderCreationStatus(
ProviderCreationStatus::kMantaServiceFailedToCreate);
return nullptr;
}
LogProviderCreationStatus(ProviderCreationStatus::kOk);
return provider;
}
LogProviderCreationStatus(ProviderCreationStatus::kMantaServiceIsNull);
return nullptr;
}
// Returns true if the Mahi feature has been approved to be used.
bool IsMahiApproved() {
auto* magic_boost_state = chromeos::MagicBoostState::Get();
CHECK(magic_boost_state->IsUserEligibleForGenAIFeatures())
<< "GenAI feature surfaced to non-eligible user.";
return magic_boost_state->hmr_consent_status() ==
chromeos::HMRConsentStatus::kApproved;
}
} // namespace
namespace ash {
MahiManagerImpl::MahiManagerImpl()
: cache_manager_(std::make_unique<MahiCacheManager>()),
mahi_nudge_controller_(std::make_unique<MahiNudgeController>()) {
magic_boost_state_observation_.Observe(chromeos::MagicBoostState::Get());
}
MahiManagerImpl::~MahiManagerImpl() {
mahi_provider_.reset();
mahi_web_contents_manager_ = nullptr;
}
std::u16string MahiManagerImpl::GetContentTitle() {
return current_page_info_->title;
}
gfx::ImageSkia MahiManagerImpl::GetContentIcon() {
return current_page_info_->favicon_image;
}
GURL MahiManagerImpl::GetContentUrl() {
return current_page_info_->url;
}
std::u16string MahiManagerImpl::GetSelectedText() {
return current_selected_text_.value_or(std::u16string());
}
void MahiManagerImpl::GetContent(MahiContentCallback callback) {
if (!MaybeInitializeAndDiscardPendingRequests()) {
std::move(callback).Run(u"", MahiGetContentResponseStatus::kUnknownError);
LOG(ERROR) << "Initialized unsuccessfully.";
return;
}
// Uses page content if it is already in the cache.
const auto cached_content =
cache_manager_->GetPageContentForUrl(current_page_info_->url.spec());
if (!cached_content.empty()) {
OnGetPageContent(current_page_info_->Clone(), std::move(callback),
crosapi::mojom::MahiPageContent::New(
/*client_id=*/base::UnguessableToken(),
/*page_id=*/base::UnguessableToken(), cached_content));
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kContent);
return;
}
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kNoHit);
auto get_content_done_callback =
base::BindOnce(&MahiManagerImpl::OnGetPageContent,
weak_ptr_factory_for_requests_.GetWeakPtr(),
current_page_info_->Clone(), std::move(callback));
if (media_app_pdf_focused_) {
chromeos::MahiMediaAppContentManager::Get()->GetContent(
media_app_client_id_, std::move(get_content_done_callback));
} else {
mahi_web_contents_manager_->RequestContent(
current_page_info_->page_id, std::move(get_content_done_callback));
}
}
void MahiManagerImpl::GetSummary(MahiSummaryCallback callback) {
// Resets latest_elucidation_ to avoid messing up the feedback body.
latest_elucidation_ = std::u16string();
if (!MaybeInitializeAndDiscardPendingRequests()) {
latest_response_status_ = MahiResponseStatus::kUnknownError;
std::move(callback).Run(u"", latest_response_status_);
LOG(ERROR) << "Initialized unsuccessfully.";
return;
}
current_panel_info_ = current_page_info_->Clone();
const auto cached_content =
cache_manager_->GetPageContentForUrl(current_panel_info_->url.spec());
// Uses the cached summary only if the request is for the whole page
// (`current_selected_text_` is nullopt).
const auto cached_summary =
cache_manager_->GetSummaryForUrl(current_panel_info_->url.spec());
if (current_selected_text_ == std::nullopt && cached_summary.has_value()) {
current_panel_content_ = crosapi::mojom::MahiPageContent::New(
/*client_id=*/base::UnguessableToken(),
/*page_id=*/base::UnguessableToken(), cached_content);
current_panel_qa_.clear();
// TODO(b:338140794): consider loading the QA cache here as well.
latest_summary_ = cached_summary.value();
latest_response_status_ = MahiResponseStatus::kSuccess;
std::move(callback).Run(cached_summary.value(),
MahiResponseStatus::kSuccess);
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kSummary);
return;
}
// Uses page content if it is already in the cache.
if (!cached_content.empty()) {
OnGetPageContentForSummary(
current_panel_info_->Clone(), std::move(callback),
crosapi::mojom::MahiPageContent::New(
/*client_id=*/base::UnguessableToken(),
/*page_id=*/base::UnguessableToken(), cached_content));
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kContent);
return;
}
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kNoHit);
auto get_content_done_callback =
base::BindOnce(&MahiManagerImpl::OnGetPageContentForSummary,
weak_ptr_factory_for_requests_.GetWeakPtr(),
current_panel_info_->Clone(), std::move(callback));
if (media_app_pdf_focused_) {
chromeos::MahiMediaAppContentManager::Get()->GetContent(
media_app_client_id_, std::move(get_content_done_callback));
} else {
mahi_web_contents_manager_->RequestContent(
current_page_info_->page_id, std::move(get_content_done_callback));
}
}
void MahiManagerImpl::GetElucidation(MahiElucidationCallback callback) {
// Resets latest_summary_ to avoid messing up feedback.
latest_summary_ = std::u16string();
if (!MaybeInitializeAndDiscardPendingRequests()) {
latest_response_status_ = MahiResponseStatus::kUnknownError;
std::move(callback).Run(u"", latest_response_status_);
LOG(ERROR) << "Initialized unsuccessfully.";
return;
}
current_panel_info_ = current_page_info_->Clone();
// Do not CHECK and crash here. It's true that Elucidation button should only
// show when the selected text passed the eligiblity check, but this may also
// be called by clicking `retry` link when error happens, and because of
// crbug.com/375292907, the current_selected_text may change and not eligible
// anymore (e.g. becomes empty). In such cases we returns an error.
if (current_selected_text_->empty()) {
std::move(callback).Run(u"", MahiResponseStatus::kInappropriate);
return;
}
const auto cached_content =
cache_manager_->GetPageContentForUrl(current_panel_info_->url.spec());
// Uses page content if it is already in the cache.
if (!cached_content.empty()) {
OnGetPageContentForElucidation(
current_selected_text_.value(), current_panel_info_->Clone(),
std::move(callback),
crosapi::mojom::MahiPageContent::New(
/*client_id=*/base::UnguessableToken(),
/*page_id=*/base::UnguessableToken(), cached_content));
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kContent);
return;
}
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kNoHit);
auto get_content_done_callback =
base::BindOnce(&MahiManagerImpl::OnGetPageContentForElucidation,
weak_ptr_factory_for_requests_.GetWeakPtr(),
current_selected_text_.value(),
current_panel_info_->Clone(), std::move(callback));
if (media_app_pdf_focused_) {
chromeos::MahiMediaAppContentManager::Get()->GetContent(
media_app_client_id_, std::move(get_content_done_callback));
} else {
mahi_web_contents_manager_->RequestContent(
current_page_info_->page_id, std::move(get_content_done_callback));
}
}
void MahiManagerImpl::GetOutlines(MahiOutlinesCallback callback) {
std::vector<chromeos::MahiOutline> outlines;
for (int i = 0; i < 5; i++) {
outlines.emplace_back(
chromeos::MahiOutline(i, u"Outline " + base::NumberToString16(i)));
}
std::move(callback).Run(outlines, MahiResponseStatus::kSuccess);
}
void MahiManagerImpl::GoToOutlineContent(int outline_id) {}
void MahiManagerImpl::AnswerQuestion(const std::u16string& question,
bool current_panel_content,
MahiAnswerQuestionCallback callback) {
if (!MaybeInitializeAndDiscardPendingRequests()) {
latest_response_status_ = MahiResponseStatus::kUnknownError;
std::move(callback).Run(u"", latest_response_status_);
LOG(ERROR) << "Initialized unsuccessfully.";
return;
}
if (current_panel_content) {
mahi_provider_->QuestionAndAnswer(
base::UTF16ToUTF8(current_panel_content_->page_content),
base::UTF16ToUTF8(current_panel_info_->title),
MaybeGetUrl(current_page_info_), current_panel_qa_,
base::UTF16ToUTF8(question),
base::BindOnce(&MahiManagerImpl::OnMahiProviderQAResponse,
weak_ptr_factory_for_requests_.GetWeakPtr(),
current_panel_info_->Clone(), question,
std::move(callback)));
return;
}
current_panel_info_ = current_page_info_->Clone();
// Uses page content if it is already in the cache.
const auto cached_content =
cache_manager_->GetPageContentForUrl(current_panel_info_->url.spec());
if (!cached_content.empty()) {
OnGetPageContentForQA(
current_panel_info_->Clone(), question, std::move(callback),
crosapi::mojom::MahiPageContent::New(
/*client_id=*/base::UnguessableToken(),
/*page_id=*/base::UnguessableToken(), cached_content));
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kContent);
return;
}
base::UmaHistogramEnumeration(kMahiCacheHit, CacheHit::kNoHit);
auto get_content_done_callback = base::BindOnce(
&MahiManagerImpl::OnGetPageContentForQA,
weak_ptr_factory_for_requests_.GetWeakPtr(), current_panel_info_->Clone(),
question, std::move(callback));
if (media_app_pdf_focused_) {
chromeos::MahiMediaAppContentManager::Get()->GetContent(
media_app_client_id_, std::move(get_content_done_callback));
} else {
mahi_web_contents_manager_->RequestContent(
current_panel_info_->page_id, std::move(get_content_done_callback));
}
}
void MahiManagerImpl::GetSuggestedQuestion(
MahiGetSuggestedQuestionCallback callback) {
std::move(callback).Run(u"test suggested question",
MahiResponseStatus::kSuccess);
}
void MahiManagerImpl::SetCurrentFocusedPageInfo(
crosapi::mojom::MahiPageInfoPtr info) {
media_app_pdf_focused_ = false;
// TODO(crbug.com/318565610): consider adding default icon when there is no
// icon available.
current_page_info_ = std::move(info);
const bool availability =
current_page_info_->IsDistillable.value_or(false) &&
!current_panel_info_->url.EqualsIgnoringRef(current_page_info_->url);
NotifyRefreshAvailability(/*available=*/availability);
}
void MahiManagerImpl::OnContextMenuClicked(
crosapi::mojom::MahiContextMenuRequestPtr context_menu_request) {
const MahiContextMenuActionType action_type =
context_menu_request->action_type;
// Show a disclaimer view before fulfilling `context_menu_request` if:
// 1. Mahi is not approved by user; AND
// 2. `context_menu_request` is NOT related to the Mahi settings. User
// is allowed to access the Mahi settings before approval.
if (!IsMahiApproved() &&
action_type != MahiContextMenuActionType::kSettings) {
InterrputRequestHandlingWithDisclaimerView(std::move(context_menu_request));
return;
}
switch (action_type) {
case MahiContextMenuActionType::kElucidation:
// Retrieves selected text from corresponding content manager, so that the
// result panel ui can get this info from Mahi Manager directly instead of
// contacting content managers.
UpdateCurrentSelectedText();
OpenMahiPanelForElucidation(
context_menu_request->display_id,
context_menu_request->mahi_menu_bounds.has_value()
? context_menu_request->mahi_menu_bounds.value()
: gfx::Rect());
return;
case MahiContextMenuActionType::kSummary:
current_selected_text_ = std::nullopt;
OpenMahiPanel(context_menu_request->display_id,
context_menu_request->mahi_menu_bounds.has_value()
? context_menu_request->mahi_menu_bounds.value()
: gfx::Rect());
return;
case MahiContextMenuActionType::kSummaryOfSelection:
UpdateCurrentSelectedText();
OpenMahiPanel(context_menu_request->display_id,
context_menu_request->mahi_menu_bounds.has_value()
? context_menu_request->mahi_menu_bounds.value()
: gfx::Rect());
return;
case MahiContextMenuActionType::kQA:
current_selected_text_ = std::nullopt;
OpenMahiPanel(context_menu_request->display_id,
context_menu_request->mahi_menu_bounds.has_value()
? context_menu_request->mahi_menu_bounds.value()
: gfx::Rect());
// Ask question.
// TODO(b/331837721): `MahiManagerImpl` should own an instance of
// `MahiUiController` and use it to answer question here. This
// functionality shouldn't need to be routed through the widget. We also
// need to add unit test logic for this after the refactor.
if (!context_menu_request->question) {
return;
}
if (!ui_controller_.IsMahiPanelOpen()) {
return;
}
// Because we call `MahiUiController::SendQuestion` right after
// opening the panel here, `SendQuestion` will cancel the call to get
// summary due to `MahiUiController::InvalidatePendingRequests()`. Thus,
// we need to update the summary after answering the question to make sure
// that user gets summary when navigating back to the summary UI
// (b/345621992).
// When the user sends a question from the context menu, we treat it as
// the start of a new journey, so we set `current_panel_content` false.
ui_controller_.SendQuestion(
context_menu_request->question.value(),
/*current_panel_content=*/false,
MahiUiController::QuestionSource::kMenuView,
/*update_summary_after_answer_question=*/true);
return;
case MahiContextMenuActionType::kSettings:
chrome::SettingsWindowManager::GetInstance()->ShowOSSettings(
ProfileManager::GetActiveUserProfile(),
chromeos::settings::mojom::kSystemPreferencesSectionPath,
chromeos::settings::mojom::Setting::kMahiOnOff);
return;
case MahiContextMenuActionType::kNone:
case MahiContextMenuActionType::kOutline:
// TODO(b/318565610): Update the behaviour of kOutline.
return;
}
}
void MahiManagerImpl::OpenFeedbackDialog() {
std::string description_template = base::StringPrintf(
"#Mahi user feedback:\n\n-----------\nlatest status code: %d",
static_cast<int>(latest_response_status_));
if (!latest_summary_.empty()) {
base::StringAppendF(&description_template, "\nlatest summary: %s",
base::UTF16ToUTF8(latest_summary_).c_str());
if (current_selected_text_ != std::nullopt) {
base::StringAppendF(
&description_template, "\n\nfor the selected text: %s",
base::UTF16ToUTF8(current_selected_text_.value()).c_str());
}
if (!current_panel_qa_.empty()) {
base::StringAppendF(&description_template, "\n\nQA history:");
for (const auto& [question, answer] : current_panel_qa_) {
base::StringAppendF(&description_template, "\nQ:%s\nA:%s\n",
question.c_str(), answer.c_str());
}
}
} else if (!latest_elucidation_.empty()) {
base::StringAppendF(
&description_template,
"\nlatest simplified text: %s\n\nfor the selected text: %s\n",
base::UTF16ToUTF8(latest_elucidation_).c_str(),
base::UTF16ToUTF8(current_selected_text_.value()).c_str());
}
base::Value::Dict ai_metadata;
ai_metadata.Set(feedback::kMahiMetadataKey, "true");
chrome::ShowFeedbackPage(
/*browser=*/chrome::FindBrowserWithProfile(
ProfileManager::GetActiveUserProfile()),
/*source=*/feedback::kFeedbackSourceAI, description_template,
/*description_placeholder_text=*/
base::UTF16ToUTF8(
l10n_util::GetStringUTF16(IDS_MAHI_FEEDBACK_PLACEHOLDER)),
/*category_tag=*/"mahi",
/*extra_diagnostics=*/std::string(),
/*autofill_metadata=*/base::Value::Dict(), std::move(ai_metadata));
}
void MahiManagerImpl::OpenMahiPanel(int64_t display_id,
const gfx::Rect& mahi_menu_bounds) {
ui_controller_.OpenMahiPanel(display_id, mahi_menu_bounds,
/*elucidation_in_use=*/false);
}
void MahiManagerImpl::OpenMahiPanelForElucidation(
int64_t display_id,
const gfx::Rect& mahi_menu_bounds) {
ui_controller_.OpenMahiPanel(display_id, mahi_menu_bounds,
/*elucidation_in_use=*/true);
}
bool MahiManagerImpl::IsEnabled() {
return (mahi_availability::IsMahiAvailable().has_value() &&
mahi_availability::IsMahiAvailable().value()) &&
chromeos::MagicBoostState::Get()->hmr_enabled().value_or(false);
}
void MahiManagerImpl::SetMediaAppPDFFocused() {
chromeos::MahiMediaAppContentManager* media_app_content_manager =
chromeos::MahiMediaAppContentManager::Get();
CHECK(media_app_content_manager);
bool old_media_app_pdf_focused = media_app_pdf_focused_;
base::UnguessableToken old_media_app_client_id = media_app_client_id_;
const std::u16string old_title = current_page_info_->title;
media_app_client_id_ = media_app_content_manager->active_client_id();
media_app_pdf_focused_ = true;
std::optional<std::string> file_name =
media_app_content_manager->GetFileName(media_app_client_id_);
CHECK(file_name.has_value());
// Fits the media app page info into a MahiPageInfoPtr.
// Particularly, makes up a GURL with the file name.
// TODO(b:338140794): Two file with the same name can hit the same cache.
// Need to find a way to fix this.
current_page_info_ = crosapi::mojom::MahiPageInfo::New(
media_app_client_id_,
/*page_id=*/media_app_client_id_,
GURL{base::StrCat({kMediaAppPDFUrlPrefix, file_name.value()})},
/*title=*/base::UTF8ToUTF16(file_name.value()), gfx::ImageSkia(),
/*distillable=*/true, /*is_incognito=*/false);
// To avoid refresh banner flicker. This could happen when a new PDF file is
// opened from file picker dialog in media app.
if (old_media_app_pdf_focused &&
old_media_app_client_id == media_app_client_id_ &&
current_page_info_->title == old_title) {
return;
}
const bool availability =
!current_panel_info_->url.EqualsIgnoringRef(current_page_info_->url);
NotifyRefreshAvailability(/*available=*/availability);
}
void MahiManagerImpl::MediaAppPDFClosed(
const base::UnguessableToken media_app_client_id) {
if (media_app_pdf_focused_ && media_app_client_id_ == media_app_client_id &&
current_page_info_->client_id == media_app_client_id) {
// In this case if there's a refresh banner, it must be targeted to
// the destroying media app PDF. Hides it by a false notification.
NotifyRefreshAvailability(/*available=*/false);
current_page_info_ = crosapi::mojom::MahiPageInfo::New();
}
media_app_pdf_focused_ = false;
media_app_client_id_ = base::UnguessableToken::Null();
}
std::optional<base::UnguessableToken> MahiManagerImpl::GetMediaAppPDFClientId()
const {
if (media_app_pdf_focused_) {
return media_app_client_id_;
}
return std::nullopt;
}
void MahiManagerImpl::ClearCache() {
cache_manager_->ClearCache();
}
void MahiManagerImpl::NotifyRefreshAvailability(bool available) {
// Do not notify if the result on the panel is based on the user selected
// text, because clicking the refresh banner will update the panel with
// summary of the new whole document, which is not consistent with the current
// purpose of the panel.
if (ui_controller_.IsMahiPanelOpen() &&
current_selected_text_ == std::nullopt) {
ui_controller_.NotifyRefreshAvailabilityChanged(available);
}
// Attempt showing an educational nudge when users visit eligible content.
if (available) {
mahi_nudge_controller_->MaybeShowNudge();
}
}
void MahiManagerImpl::OnHistoryDeletions(
history::HistoryService* history_service,
const history::DeletionInfo& deletion_info) {
// If IsAllHistory() returns true, all URLs are deleted and `deleted_rows()`
// and `favicon_urls()` are undefined.
if (deletion_info.IsAllHistory()) {
cache_manager_->ClearCache();
} else {
for (const auto& row : deletion_info.deleted_rows()) {
cache_manager_->DeleteCacheForUrl(row.url().spec());
}
}
}
void MahiManagerImpl::OnHMREnabledUpdated(bool enabled) {
if (enabled) {
return;
}
ui_controller_.CloseMahiPanel();
cache_manager_->ClearCache();
}
void MahiManagerImpl::OnIsDeleting() {
magic_boost_state_observation_.Reset();
}
bool MahiManagerImpl::MaybeInitializeAndDiscardPendingRequests() {
if (!mahi_provider_) {
mahi_provider_ = CreateProvider();
}
if (!mahi_web_contents_manager_) {
mahi_web_contents_manager_ = chromeos::MahiWebContentsManager::Get();
}
if (weak_ptr_factory_for_requests_.HasWeakPtrs()) {
weak_ptr_factory_for_requests_.InvalidateWeakPtrs();
}
MaybeObserveHistoryService();
return mahi_provider_ != nullptr && mahi_web_contents_manager_ != nullptr;
}
void MahiManagerImpl::MaybeObserveHistoryService() {
Profile* profile = ProfileManager::GetActiveUserProfile();
if (!profile) {
return;
}
history::HistoryService* service =
HistoryServiceFactory::GetForProfileWithoutCreating(profile);
if (service && !scoped_history_service_observer_.IsObserving()) {
scoped_history_service_observer_.Observe(service);
}
}
void MahiManagerImpl::InterrputRequestHandlingWithDisclaimerView(
crosapi::mojom::MahiContextMenuRequestPtr context_menu_request) {
CHECK(chromeos::MagicBoostState::Get()->IsUserEligibleForGenAIFeatures());
// Cache the display id before moving `context_menu_request`.
const int64_t display_id = context_menu_request->display_id;
// Invalidate the closures of the existing closure runner, if any.
weak_ptr_factory_for_closure_runner_.InvalidateWeakPtrs();
// The closure that resets `on_consent_state_update_closure_runner_`.
base::RepeatingClosure reset_observer_closure = base::BindRepeating(
[](const base::WeakPtr<MahiManagerImpl>& weak_ptr) {
if (weak_ptr) {
weak_ptr->on_consent_state_update_closure_runner_.reset();
}
},
weak_ptr_factory_for_closure_runner_.GetWeakPtr());
on_consent_state_update_closure_runner_ =
std::make_unique<OnConsentStateUpdateClosureRunner>(
/*on_approved_closure=*/
base::BindOnce(&MahiManagerImpl::OnContextMenuClicked,
weak_ptr_factory_for_closure_runner_.GetWeakPtr(),
std::move(context_menu_request))
.Then(reset_observer_closure),
/*on_declined_closure=*/reset_observer_closure);
ash::MagicBoostControllerAsh::Get()->ShowDisclaimerUi(
display_id,
crosapi::mojom::MagicBoostController::TransitionAction::kDoNothing,
chromeos::MagicBoostState::Get()->ShouldIncludeOrcaInOptInSync()
? OptInFeatures::kOrcaAndHmr
: OptInFeatures::kHmrOnly);
}
void MahiManagerImpl::OnGetPageContent(
crosapi::mojom::MahiPageInfoPtr request_page_info,
MahiContentCallback callback,
crosapi::mojom::MahiPageContentPtr mahi_content_ptr) {
if (!mahi_content_ptr || mahi_content_ptr->page_content.empty()) {
std::move(callback).Run(
u"", MahiGetContentResponseStatus::kContentExtractionError);
// TODO(b:371080356) add histogram metrics.
return;
}
// Cache current panel content.
CacheCurrentPanelContent(*request_page_info, *mahi_content_ptr);
std::move(callback).Run(mahi_content_ptr->page_content,
MahiGetContentResponseStatus::kSuccess);
}
void MahiManagerImpl::OnGetPageContentForSummary(
crosapi::mojom::MahiPageInfoPtr request_page_info,
MahiSummaryCallback callback,
crosapi::mojom::MahiPageContentPtr mahi_content_ptr) {
if (!mahi_content_ptr || mahi_content_ptr->page_content.empty()) {
latest_response_status_ = MahiResponseStatus::kContentExtractionError;
std::move(callback).Run(u"", latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
// Assign current panel content and clear the current panel QA
current_panel_content_ = std::move(mahi_content_ptr);
current_panel_qa_.clear();
CacheCurrentPanelContent(*request_page_info, *current_panel_content_);
// Summarizes the `current_selected_text_` if it's not null, otherwise
// summarizes the whole page content.
// Note: we get the page content even if the `current_selected_text_` is not
// null. The page content is not used for the summary request in such cases,
// but it's necessary to be present because questions sent from the result
// panel relies on it.
const std::u16string text_to_summary =
current_selected_text_.value_or(current_panel_content_->page_content);
if (text_to_summary.empty()) {
latest_response_status_ = MahiResponseStatus::kInappropriate;
std::move(callback).Run(u"", latest_response_status_);
return;
}
std::optional<std::string> context = std::nullopt;
if (current_selected_text_ != std::nullopt) {
context = base::UTF16ToUTF8(current_panel_content_->page_content);
}
CHECK(mahi_provider_);
mahi_provider_->Summarize(
base::UTF16ToUTF8(text_to_summary),
base::UTF16ToUTF8(request_page_info->title), context,
MaybeGetUrl(request_page_info),
base::BindOnce(&MahiManagerImpl::OnMahiProviderSummaryResponse,
weak_ptr_factory_for_requests_.GetWeakPtr(),
std::move(request_page_info), std::move(callback)));
}
void MahiManagerImpl::OnGetPageContentForElucidation(
const std::u16string& selected_text,
crosapi::mojom::MahiPageInfoPtr request_page_info,
MahiElucidationCallback callback,
crosapi::mojom::MahiPageContentPtr mahi_content_ptr) {
if (!mahi_content_ptr || mahi_content_ptr->page_content.empty()) {
latest_response_status_ = MahiResponseStatus::kContentExtractionError;
std::move(callback).Run(u"", latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
// Assign current panel content and clear the current panel QA
current_panel_content_ = std::move(mahi_content_ptr);
current_panel_qa_.clear();
CacheCurrentPanelContent(*request_page_info, *current_panel_content_);
CHECK(mahi_provider_);
mahi_provider_->Elucidate(
base::UTF16ToUTF8(selected_text),
base::UTF16ToUTF8(current_panel_content_->page_content),
base::UTF16ToUTF8(request_page_info->title),
MaybeGetUrl(request_page_info),
base::BindOnce(&MahiManagerImpl::OnMahiProviderElucidationResponse,
weak_ptr_factory_for_requests_.GetWeakPtr(),
std::move(request_page_info), selected_text,
std::move(callback)));
}
void MahiManagerImpl::OnGetPageContentForQA(
crosapi::mojom::MahiPageInfoPtr request_page_info,
const std::u16string& question,
MahiAnswerQuestionCallback callback,
crosapi::mojom::MahiPageContentPtr mahi_content_ptr) {
if (!mahi_content_ptr || mahi_content_ptr->page_content.empty()) {
latest_response_status_ = MahiResponseStatus::kContentExtractionError;
std::move(callback).Run(std::nullopt, latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
// Assign current panel content and clear the current panel QA
current_panel_content_ = std::move(mahi_content_ptr);
current_panel_qa_.clear();
CacheCurrentPanelContent(*request_page_info, *current_panel_content_);
mahi_provider_->QuestionAndAnswer(
base::UTF16ToUTF8(current_panel_content_->page_content),
base::UTF16ToUTF8(request_page_info->title),
MaybeGetUrl(request_page_info), current_panel_qa_,
base::UTF16ToUTF8(question),
base::BindOnce(&MahiManagerImpl::OnMahiProviderQAResponse,
weak_ptr_factory_for_requests_.GetWeakPtr(),
std::move(request_page_info), question,
std::move(callback)));
}
void MahiManagerImpl::OnMahiProviderSummaryResponse(
crosapi::mojom::MahiPageInfoPtr request_page_info,
MahiSummaryCallback summary_callback,
base::Value::Dict dict,
manta::MantaStatus status) {
latest_summary_ = u"...";
if (status.status_code != manta::MantaStatusCode::kOk) {
latest_response_status_ =
GetMahiResponseStatusFromMantaStatus(status.status_code);
std::move(summary_callback)
.Run(u"Couldn't get summary", latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
if (auto* text = dict.FindString("outputData")) {
latest_response_status_ = MahiResponseStatus::kSuccess;
latest_summary_ = base::UTF8ToUTF16(*text);
// Caches the summary if it is not for the selected text.
if (current_selected_text_ == std::nullopt) {
cache_manager_->TryToUpdateSummaryForUrl(request_page_info->url.spec(),
latest_summary_);
}
std::move(summary_callback).Run(latest_summary_, latest_response_status_);
} else {
latest_response_status_ = MahiResponseStatus::kCantFindOutputData;
std::move(summary_callback)
.Run(u"Cannot find output data", latest_response_status_);
}
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
}
void MahiManagerImpl::OnMahiProviderElucidationResponse(
crosapi::mojom::MahiPageInfoPtr request_page_info,
const std::u16string& selected_text,
MahiElucidationCallback elucidation_callback,
base::Value::Dict dict,
manta::MantaStatus status) {
CHECK(current_selected_text_.value_or(u"") == selected_text);
latest_elucidation_ = u"...";
if (status.status_code != manta::MantaStatusCode::kOk) {
latest_response_status_ =
GetMahiResponseStatusFromMantaStatus(status.status_code);
std::move(elucidation_callback)
.Run(u"Couldn't get elucidation", latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
if (auto* text = dict.FindString("outputData")) {
latest_response_status_ = MahiResponseStatus::kSuccess;
latest_elucidation_ = base::UTF8ToUTF16(*text);
std::move(elucidation_callback)
.Run(latest_elucidation_, latest_response_status_);
} else {
latest_response_status_ = MahiResponseStatus::kCantFindOutputData;
std::move(elucidation_callback)
.Run(u"Cannot find output data", latest_response_status_);
}
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
}
void MahiManagerImpl::OnMahiProviderQAResponse(
crosapi::mojom::MahiPageInfoPtr request_page_info,
const std::u16string& question,
MahiAnswerQuestionCallback callback,
base::Value::Dict dict,
manta::MantaStatus status) {
if (status.status_code != manta::MantaStatusCode::kOk) {
latest_response_status_ =
GetMahiResponseStatusFromMantaStatus(status.status_code);
current_panel_qa_.emplace_back(base::UTF16ToUTF8(question), "");
std::move(callback).Run(std::nullopt, latest_response_status_);
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
return;
}
if (auto* text = dict.FindString("outputData")) {
latest_response_status_ = MahiResponseStatus::kSuccess;
current_panel_qa_.emplace_back(base::UTF16ToUTF8(question), *text);
std::move(callback).Run(base::UTF8ToUTF16(*text), latest_response_status_);
} else {
latest_response_status_ = MahiResponseStatus::kCantFindOutputData;
std::move(callback).Run(std::nullopt, latest_response_status_);
}
base::UmaHistogramEnumeration(kMahiResponseStatus, latest_response_status_);
}
void MahiManagerImpl::CacheCurrentPanelContent(
crosapi::mojom::MahiPageInfo request_page_info,
crosapi::mojom::MahiPageContent mahi_content) {
// Add page content to the cache.
// TODO(b:338140794): consider adding the QA to the cache.
if (!request_page_info.is_incognito) {
cache_manager_->AddCacheForUrl(
request_page_info.url.spec(),
MahiCacheManager::MahiData(
request_page_info.url.spec(), request_page_info.title,
mahi_content.page_content, request_page_info.favicon_image,
/*summary=*/std::nullopt,
/*previous_qa=*/{}));
}
}
void MahiManagerImpl::UpdateCurrentSelectedText() {
if (media_app_pdf_focused_) {
current_selected_text_ = base::UTF8ToUTF16(
chromeos::MahiMediaAppContentManager::Get()->GetSelectedText());
} else {
current_selected_text_ =
chromeos::MahiWebContentsManager::Get()->GetSelectedText();
}
}
// Repeating answers are not allowed for Mahi as all questions must only return
// one answer.
bool MahiManagerImpl::AllowRepeatingAnswers() {
return false;
}
// This function will never be called as consecutive answers are not allowed for
// Mahi.
void MahiManagerImpl::AnswerQuestionRepeating(
const std::u16string& question,
bool current_panel_content,
MahiAnswerQuestionCallbackRepeating callback) {}
} // namespace ash