blob: e7d783219d14afb8ab3f0083e0b2d065e5e98bf9 [file] [log] [blame]
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_tab_helper.h"
#include <string>
#include "base/barrier_closure.h"
#include "base/bind.h"
#include "base/callback_helpers.h"
#include "base/command_line.h"
#include "base/containers/contains.h"
#include "base/feature_list.h"
#include "base/metrics/histogram.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/rand_util.h"
#include "base/time/time.h"
#include "chrome/browser/chrome_content_browser_client.h"
#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service_factory.h"
#include "chrome/browser/net/prediction_options.h"
#include "chrome/browser/prefetch/no_state_prefetch/no_state_prefetch_manager_factory.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_features.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_network_context_client.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_origin_decider.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_params.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_prefetch_metrics_collector.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_proxy_configurator.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_service.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_service_factory.h"
#include "chrome/browser/prefetch/prefetch_proxy/prefetch_proxy_subresource_manager.h"
#include "chrome/browser/profiles/profile.h"
#include "components/data_reduction_proxy/core/browser/data_reduction_proxy_settings.h"
#include "components/google/core/common/google_util.h"
#include "components/language/core/browser/pref_names.h"
#include "components/no_state_prefetch/browser/no_state_prefetch_manager.h"
#include "components/page_load_metrics/browser/metrics_web_contents_observer.h"
#include "components/prefs/pref_service.h"
#include "components/search_engines/template_url_service.h"
#include "components/version_info/version_info.h"
#include "content/public/browser/browser_context.h"
#include "content/public/browser/navigation_handle.h"
#include "content/public/browser/network_service_instance.h"
#include "content/public/browser/render_frame_host.h"
#include "content/public/browser/render_process_host.h"
#include "content/public/browser/service_worker_context.h"
#include "content/public/browser/storage_partition.h"
#include "content/public/browser/web_contents.h"
#include "content/public/common/content_constants.h"
#include "content/public/common/content_switches.h"
#include "content/public/common/user_agent.h"
#include "mojo/public/cpp/bindings/self_owned_receiver.h"
#include "net/base/isolation_info.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/cookies/cookie_store.h"
#include "net/http/http_status_code.h"
#include "net/http/http_util.h"
#include "net/traffic_annotation/network_traffic_annotation.h"
#include "services/cert_verifier/public/mojom/cert_verifier_service_factory.mojom.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "services/network/public/cpp/simple_url_loader.h"
#include "services/network/public/cpp/wrapper_shared_url_loader_factory.h"
#include "services/network/public/mojom/cookie_manager.mojom.h"
#include "services/network/public/mojom/network_context.mojom.h"
#include "services/network/public/mojom/network_service.mojom.h"
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/blink/public/common/storage_key/storage_key.h"
#include "url/origin.h"
namespace {
absl::optional<base::TimeDelta> GetTotalPrefetchTime(
network::mojom::URLResponseHead* head) {
DCHECK(head);
base::Time start = head->request_time;
base::Time end = head->response_time;
if (start.is_null() || end.is_null())
return absl::nullopt;
return end - start;
}
absl::optional<base::TimeDelta> GetPrefetchConnectTime(
network::mojom::URLResponseHead* head) {
DCHECK(head);
base::TimeTicks start = head->load_timing.connect_timing.connect_start;
base::TimeTicks end = head->load_timing.connect_timing.connect_end;
if (start.is_null() || end.is_null())
return absl::nullopt;
return end - start;
}
void InformPLMOfLikelyPrefetching(content::WebContents* web_contents) {
page_load_metrics::MetricsWebContentsObserver* metrics_web_contents_observer =
page_load_metrics::MetricsWebContentsObserver::FromWebContents(
web_contents);
if (!metrics_web_contents_observer)
return;
metrics_web_contents_observer->OnPrefetchLikely();
}
void OnGotCookieList(
const GURL& url,
PrefetchProxyTabHelper::OnEligibilityResultCallback result_callback,
const net::CookieAccessResultList& cookie_list,
const net::CookieAccessResultList& excluded_cookies) {
if (!cookie_list.empty()) {
std::move(result_callback)
.Run(url, false,
PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasCookies);
return;
}
// Cookies are tricky because cookies for different paths or a higher level
// domain (e.g.: m.foo.com and foo.com) may not show up in |cookie_list|, but
// they will show up in |excluded_cookies|. To check for any cookies for a
// domain, compare the domains of the prefetched |url| and the domains of all
// the returned cookies.
bool excluded_cookie_has_tld = false;
for (const auto& cookie_result : excluded_cookies) {
if (cookie_result.cookie.IsExpired(base::Time::Now())) {
// Expired cookies don't count.
continue;
}
if (url.DomainIs(cookie_result.cookie.DomainWithoutDot())) {
excluded_cookie_has_tld = true;
break;
}
}
if (excluded_cookie_has_tld) {
std::move(result_callback)
.Run(url, false,
PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasCookies);
return;
}
std::move(result_callback).Run(url, true, absl::nullopt);
}
void CookieSetHelper(base::RepeatingClosure run_me,
net::CookieAccessResult access_result) {
run_me.Run();
}
bool ShouldStartSpareRenderer() {
if (!PrefetchProxyStartsSpareRenderer()) {
return false;
}
for (content::RenderProcessHost::iterator iter(
content::RenderProcessHost::AllHostsIterator());
!iter.IsAtEnd(); iter.Advance()) {
if (iter.GetCurrentValue()->IsUnused()) {
// There is already a spare renderer.
return false;
}
}
return true;
}
bool ShouldConsiderDecoyRequestForStatus(PrefetchProxyPrefetchStatus status) {
switch (status) {
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasCookies:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasServiceWorker:
// If the prefetch is not eligible because of cookie or a service worker,
// then maybe send a decoy.
return true;
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleGoogleDomain:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleSchemeIsNotHttps:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleHostIsIPAddress:
case PrefetchProxyPrefetchStatus::
kPrefetchNotEligibleNonDefaultStoragePartition:
case PrefetchProxyPrefetchStatus::kPrefetchPositionIneligible:
case PrefetchProxyPrefetchStatus::kPrefetchIneligibleRetryAfter:
case PrefetchProxyPrefetchStatus::kPrefetchProxyNotAvailable:
// These statuses don't relate to any user state, so don't send a decoy
// request.
return false;
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbe:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccess:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailed:
case PrefetchProxyPrefetchStatus::kPrefetchNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchNotFinishedInTime:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNetError:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNon2XX:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNotHTML:
case PrefetchProxyPrefetchStatus::kPrefetchSuccessful:
case PrefetchProxyPrefetchStatus::kNavigatedToLinkNotOnSRP:
case PrefetchProxyPrefetchStatus::kSubresourceThrottled:
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailedWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::
kPrefetchNotUsedProbeFailedNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailedNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchIsPrivacyDecoy:
case PrefetchProxyPrefetchStatus::kPrefetchIsStale:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPNotStarted:
// These statuses should not be returned by the eligibility checks, and
// thus not be passed in here.
NOTREACHED();
return false;
}
}
} // namespace
PrefetchProxyTabHelper::PrefetchMetrics::PrefetchMetrics() = default;
PrefetchProxyTabHelper::PrefetchMetrics::~PrefetchMetrics() = default;
PrefetchProxyTabHelper::AfterSRPMetrics::AfterSRPMetrics() = default;
PrefetchProxyTabHelper::AfterSRPMetrics::AfterSRPMetrics(
const AfterSRPMetrics& other) = default;
PrefetchProxyTabHelper::AfterSRPMetrics::~AfterSRPMetrics() = default;
PrefetchProxyTabHelper::CurrentPageLoad::CurrentPageLoad(
content::NavigationHandle* handle)
: profile_(handle ? Profile::FromBrowserContext(
handle->GetWebContents()->GetBrowserContext())
: nullptr),
navigation_start_(handle ? handle->NavigationStart() : base::TimeTicks()),
srp_metrics_(
base::MakeRefCounted<PrefetchProxyTabHelper::PrefetchMetrics>()) {}
PrefetchProxyTabHelper::CurrentPageLoad::~CurrentPageLoad() {
if (PrefetchProxyStartsSpareRenderer()) {
UMA_HISTOGRAM_COUNTS_100("PrefetchProxy.SpareRenderer.CountStartedOnSRP",
number_of_spare_renderers_started_);
}
if (!profile_)
return;
PrefetchProxyService* service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
if (!service) {
return;
}
for (const GURL& url : no_state_prefetched_urls_) {
service->DestroySubresourceManagerForURL(url);
}
for (const GURL& url : urls_to_no_state_prefetch_) {
service->DestroySubresourceManagerForURL(url);
}
}
static content::ServiceWorkerContext* g_service_worker_context_for_test =
nullptr;
// static
void PrefetchProxyTabHelper::SetServiceWorkerContextForTest(
content::ServiceWorkerContext* context) {
g_service_worker_context_for_test = context;
}
PrefetchProxyTabHelper::PrefetchProxyTabHelper(
content::WebContents* web_contents)
: content::WebContentsObserver(web_contents) {
page_ = std::make_unique<CurrentPageLoad>(nullptr);
profile_ = Profile::FromBrowserContext(web_contents->GetBrowserContext());
NavigationPredictorKeyedService* navigation_predictor_service =
NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
if (navigation_predictor_service) {
navigation_predictor_service->AddObserver(this);
}
// Make sure the global service is up and running so that the service worker
// registrations can be queried before the first navigation prediction.
PrefetchProxyServiceFactory::GetForProfile(profile_);
}
PrefetchProxyTabHelper::~PrefetchProxyTabHelper() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NavigationPredictorKeyedService* navigation_predictor_service =
NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
if (navigation_predictor_service) {
navigation_predictor_service->RemoveObserver(this);
}
}
void PrefetchProxyTabHelper::AddObserverForTesting(Observer* observer) {
observer_list_.AddObserver(observer);
}
void PrefetchProxyTabHelper::RemoveObserverForTesting(Observer* observer) {
observer_list_.RemoveObserver(observer);
}
network::mojom::NetworkContext*
PrefetchProxyTabHelper::GetIsolatedContextForTesting() const {
return page_->isolated_network_context_.get();
}
absl::optional<PrefetchProxyTabHelper::AfterSRPMetrics>
PrefetchProxyTabHelper::after_srp_metrics() const {
if (page_->after_srp_metrics_) {
return *(page_->after_srp_metrics_);
}
return absl::nullopt;
}
// static
bool PrefetchProxyTabHelper::IsProfileEligible(Profile* profile) {
if (profile->IsOffTheRecord()) {
return false;
}
if (PrefetchProxyOnlyForLiteMode()) {
return data_reduction_proxy::DataReductionProxySettings::
IsDataSaverEnabledByUser(profile->IsOffTheRecord(),
profile->GetPrefs());
}
return true;
}
bool PrefetchProxyTabHelper::IsProfileEligible() const {
return IsProfileEligible(profile_);
}
void PrefetchProxyTabHelper::DidStartNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!navigation_handle->IsInPrimaryMainFrame()) {
return;
}
// This check is only relevant for detecting AMP pages. For this feature, AMP
// pages won't get sped up any so just ignore them.
if (navigation_handle->IsSameDocument()) {
return;
}
// Don't take any actions during a prerender since it was probably triggered
// by another instance of this class and we don't want to interfere.
prerender::NoStatePrefetchManager* no_state_prefetch_manager =
prerender::NoStatePrefetchManagerFactory::GetForBrowserContext(profile_);
if (no_state_prefetch_manager &&
no_state_prefetch_manager->IsWebContentsPrerendering(web_contents())) {
return;
}
const GURL& url = navigation_handle->GetURL();
// TODO(https://crbug.com/1238926): At this point in the navigation it's not
// guaranteed that we serve the prefetch, so consider moving the cookies to
// the interception path for robustness.
if (page_->prefetched_responses_.find(url) !=
page_->prefetched_responses_.end()) {
// Content older than 5 minutes should not be served.
auto time_it = page_->prefetch_start_times_.find(url);
if (time_it != page_->prefetch_start_times_.end() &&
base::TimeTicks::Now() <
time_it->second + PrefetchProxyCacheableDuration()) {
// Start copying any needed cookies over to the main profile if this page
// was prefetched.
CopyIsolatedCookiesOnAfterSRPClick(url);
}
}
// User is navigating, don't bother prefetching further.
page_->url_loaders_.clear();
if (page_->srp_metrics_->prefetch_attempted_count_ > 0) {
UMA_HISTOGRAM_COUNTS_100(
"PrefetchProxy.Prefetch.Mainframe.TotalRedirects",
page_->srp_metrics_->prefetch_total_redirect_count_);
}
// Notify the subresource manager (if applicable) that its page is being
// navigated to so that the prefetched subresources can be used from cache.
PrefetchProxyService* service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
if (!service)
return;
PrefetchProxySubresourceManager* subresource_manager =
service->GetSubresourceManagerForURL(navigation_handle->GetURL());
if (!subresource_manager)
return;
subresource_manager->NotifyPageNavigatedToAfterSRP();
}
void PrefetchProxyTabHelper::NotifyPrefetchProbeLatency(
base::TimeDelta probe_latency) {
page_->probe_latency_ = probe_latency;
}
void PrefetchProxyTabHelper::ReportProbeResult(
const GURL& url,
PrefetchProxyProbeResult result) {
if (!page_->prefetch_metrics_collector_) {
return;
}
page_->prefetch_metrics_collector_->OnMainframeNavigationProbeResult(url,
result);
}
void PrefetchProxyTabHelper::OnPrefetchStatusUpdate(
const GURL& url,
PrefetchProxyPrefetchStatus usage) {
page_->prefetch_status_by_url_[url] = usage;
}
PrefetchProxyPrefetchStatus
PrefetchProxyTabHelper::MaybeUpdatePrefetchStatusWithNSPContext(
const GURL& url,
PrefetchProxyPrefetchStatus status) const {
switch (status) {
// These are the statuses we want to update.
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbe:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccess:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailed:
case PrefetchProxyPrefetchStatus::kPrefetchIsStale:
break;
// These statuses are not applicable since the prefetch was not used after
// the click.
case PrefetchProxyPrefetchStatus::kPrefetchNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleGoogleDomain:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasCookies:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleUserHasServiceWorker:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleSchemeIsNotHttps:
case PrefetchProxyPrefetchStatus::kPrefetchNotEligibleHostIsIPAddress:
case PrefetchProxyPrefetchStatus::
kPrefetchNotEligibleNonDefaultStoragePartition:
case PrefetchProxyPrefetchStatus::kPrefetchNotFinishedInTime:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNetError:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNon2XX:
case PrefetchProxyPrefetchStatus::kPrefetchFailedNotHTML:
case PrefetchProxyPrefetchStatus::kPrefetchSuccessful:
case PrefetchProxyPrefetchStatus::kNavigatedToLinkNotOnSRP:
case PrefetchProxyPrefetchStatus::kSubresourceThrottled:
case PrefetchProxyPrefetchStatus::kPrefetchPositionIneligible:
case PrefetchProxyPrefetchStatus::kPrefetchIneligibleRetryAfter:
case PrefetchProxyPrefetchStatus::kPrefetchProxyNotAvailable:
case PrefetchProxyPrefetchStatus::kPrefetchIsPrivacyDecoy:
return status;
// These statuses we are going to update to, and this is the only place that
// they are set so they are not expected to be passed in.
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailedWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::
kPrefetchNotUsedProbeFailedNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailedNSPNotStarted:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleWithNSP:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPAttemptDenied:
case PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPNotStarted:
NOTREACHED();
return status;
}
bool no_state_prefetch_not_started =
base::Contains(page_->urls_to_no_state_prefetch_, url);
bool no_state_prefetch_complete =
base::Contains(page_->no_state_prefetched_urls_, url);
bool no_state_prefetch_failed =
base::Contains(page_->failed_no_state_prefetch_urls_, url);
if (!no_state_prefetch_not_started && !no_state_prefetch_complete &&
!no_state_prefetch_failed) {
return status;
}
// At most one of those bools should be true.
DCHECK(no_state_prefetch_not_started ^ no_state_prefetch_complete ^
no_state_prefetch_failed);
if (no_state_prefetch_complete) {
switch (status) {
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbe:
return PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeWithNSP;
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccess:
return PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccessWithNSP;
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailed:
return PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailedWithNSP;
case PrefetchProxyPrefetchStatus::kPrefetchIsStale:
return PrefetchProxyPrefetchStatus::kPrefetchIsStaleWithNSP;
default:
break;
}
}
if (no_state_prefetch_failed) {
switch (status) {
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbe:
return PrefetchProxyPrefetchStatus::
kPrefetchUsedNoProbeNSPAttemptDenied;
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccess:
return PrefetchProxyPrefetchStatus::
kPrefetchUsedProbeSuccessNSPAttemptDenied;
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailed:
return PrefetchProxyPrefetchStatus::
kPrefetchNotUsedProbeFailedNSPAttemptDenied;
case PrefetchProxyPrefetchStatus::kPrefetchIsStale:
return PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPAttemptDenied;
default:
break;
}
}
if (no_state_prefetch_not_started) {
switch (status) {
case PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbe:
return PrefetchProxyPrefetchStatus::kPrefetchUsedNoProbeNSPNotStarted;
case PrefetchProxyPrefetchStatus::kPrefetchUsedProbeSuccess:
return PrefetchProxyPrefetchStatus::
kPrefetchUsedProbeSuccessNSPNotStarted;
case PrefetchProxyPrefetchStatus::kPrefetchNotUsedProbeFailed:
return PrefetchProxyPrefetchStatus::
kPrefetchNotUsedProbeFailedNSPNotStarted;
case PrefetchProxyPrefetchStatus::kPrefetchIsStale:
return PrefetchProxyPrefetchStatus::kPrefetchIsStaleNSPNotStarted;
default:
break;
}
}
NOTREACHED();
return status;
}
std::unique_ptr<PrefetchProxyTabHelper::AfterSRPMetrics>
PrefetchProxyTabHelper::ComputeAfterSRPMetricsBeforeCommit(
content::NavigationHandle* handle) const {
if (page_->srp_metrics_->predicted_urls_count_ <= 0) {
return nullptr;
}
auto metrics = std::make_unique<AfterSRPMetrics>();
metrics->url_ = handle->GetURL();
metrics->prefetch_eligible_count_ =
page_->srp_metrics_->prefetch_eligible_count_;
metrics->probe_latency_ = page_->probe_latency_;
// Check every url in the redirect chain for a status, starting at the end
// and working backwards. Note: When a redirect chain is eligible all the
// way to the end, the status is already propagated. But if a redirect was
// not eligible then this will find its last known status.
DCHECK(!handle->GetRedirectChain().empty());
absl::optional<PrefetchProxyPrefetchStatus> status;
absl::optional<size_t> prediction_position;
for (auto back_iter = handle->GetRedirectChain().rbegin();
back_iter != handle->GetRedirectChain().rend(); ++back_iter) {
GURL chain_url = *back_iter;
auto status_iter = page_->prefetch_status_by_url_.find(chain_url);
if (!status && status_iter != page_->prefetch_status_by_url_.end()) {
status = MaybeUpdatePrefetchStatusWithNSPContext(chain_url,
status_iter->second);
}
// Same check for the original prediction ordering.
auto position_iter = page_->original_prediction_ordering_.find(chain_url);
if (!prediction_position &&
position_iter != page_->original_prediction_ordering_.end()) {
prediction_position = position_iter->second;
}
}
if (status) {
metrics->prefetch_status_ = *status;
} else {
metrics->prefetch_status_ =
PrefetchProxyPrefetchStatus::kNavigatedToLinkNotOnSRP;
}
metrics->clicked_link_srp_position_ = prediction_position;
return metrics;
}
void PrefetchProxyTabHelper::DidFinishNavigation(
content::NavigationHandle* navigation_handle) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!navigation_handle->IsInPrimaryMainFrame()) {
return;
}
// This check is only relevant for detecting AMP pages. For this feature, AMP
// pages won't get sped up any so just ignore them.
if (navigation_handle->IsSameDocument()) {
return;
}
if (!navigation_handle->HasCommitted()) {
return;
}
// Don't take any actions during a prerender since it was probably triggered
// by another instance of this class and we don't want to interfere.
prerender::NoStatePrefetchManager* no_state_prefetch_manager =
prerender::NoStatePrefetchManagerFactory::GetForBrowserContext(profile_);
if (no_state_prefetch_manager &&
no_state_prefetch_manager->IsWebContentsPrerendering(web_contents())) {
return;
}
// Ensure there's no ongoing prefetches.
page_->url_loaders_.clear();
GURL url = navigation_handle->GetURL();
std::unique_ptr<CurrentPageLoad> new_page =
std::make_unique<CurrentPageLoad>(navigation_handle);
if (page_->srp_metrics_->predicted_urls_count_ > 0) {
page_->prefetch_metrics_collector_->OnMainframeNavigatedTo(url);
// If the previous page load was a Google SRP, the AfterSRPMetrics class
// needs to be created now from the SRP's |page_| and then set on the new
// one when we set it at the end of this method.
new_page->after_srp_metrics_ =
ComputeAfterSRPMetricsBeforeCommit(navigation_handle);
// See if the page being navigated to was prerendered. If so, copy over its
// subresource manager and networking pipes.
PrefetchProxyService* service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
std::unique_ptr<PrefetchProxySubresourceManager> manager =
service->TakeSubresourceManagerForURL(url);
if (manager) {
new_page->subresource_manager_ = std::move(manager);
new_page->isolated_cookie_manager_ =
std::move(page_->isolated_cookie_manager_);
new_page->isolated_url_loader_factory_ =
std::move(page_->isolated_url_loader_factory_);
new_page->isolated_network_context_ =
std::move(page_->isolated_network_context_);
}
}
// |page_| is reset on commit so that any available cached prefetches that
// result from a redirect get used.
page_ = std::move(new_page);
}
void PrefetchProxyTabHelper::OnVisibilityChanged(
content::Visibility visibility) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!PrefetchProxyIsEnabled()) {
return;
}
// Start prefetching if the tab has become visible and prefetching is
// inactive. Hidden and occluded visibility is ignored here so that pending
// prefetches can finish.
if (visibility == content::Visibility::VISIBLE && !PrefetchingActive())
Prefetch();
}
std::unique_ptr<PrefetchedMainframeResponseContainer>
PrefetchProxyTabHelper::TakePrefetchResponse(const GURL& url) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
auto it = page_->prefetched_responses_.find(url);
if (it == page_->prefetched_responses_.end())
return nullptr;
// Content older than 5 minutes should not be served.
auto time_it = page_->prefetch_start_times_.find(url);
if (time_it == page_->prefetch_start_times_.end() ||
base::TimeTicks::Now() >
time_it->second + PrefetchProxyCacheableDuration()) {
OnPrefetchStatusUpdate(url, PrefetchProxyPrefetchStatus::kPrefetchIsStale);
return nullptr;
}
std::unique_ptr<PrefetchedMainframeResponseContainer> response =
std::move(it->second);
page_->prefetched_responses_.erase(it);
page_->prefetch_start_times_.erase(time_it);
return response;
}
std::unique_ptr<PrefetchedMainframeResponseContainer>
PrefetchProxyTabHelper::CopyPrefetchResponseForNSP(const GURL& url) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
auto it = page_->prefetched_responses_.find(url);
if (it == page_->prefetched_responses_.end())
return nullptr;
return it->second->Clone();
}
bool PrefetchProxyTabHelper::PrefetchingActive() const {
return page_ && !page_->url_loaders_.empty();
}
void PrefetchProxyTabHelper::Prefetch() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(PrefetchProxyIsEnabled());
if (!page_->srp_metrics_->navigation_to_prefetch_start_.has_value()) {
page_->srp_metrics_->navigation_to_prefetch_start_ =
base::TimeTicks::Now() - page_->navigation_start_;
DCHECK_GT(page_->srp_metrics_->navigation_to_prefetch_start_.value(),
base::TimeDelta());
}
if (PrefetchProxyCloseIdleSockets() && page_->isolated_network_context_) {
page_->isolated_network_context_->CloseIdleConnections(base::DoNothing());
}
if (web_contents()->GetVisibility() != content::Visibility::VISIBLE) {
// |OnVisibilityChanged| will restart prefetching when the tab becomes
// visible again.
return;
}
DCHECK_GT(PrefetchProxyMaximumNumberOfConcurrentPrefetches(), 0U);
while (
// Checks that the total number of prefetches has not been met.
!(PrefetchProxyMaximumNumberOfPrefetches().has_value() &&
page_->decoy_requests_attempted_ +
page_->srp_metrics_->prefetch_attempted_count_ >=
PrefetchProxyMaximumNumberOfPrefetches().value()) &&
// Checks that there are still urls to prefetch.
!page_->urls_to_prefetch_.empty() &&
// Checks that the max number of concurrent prefetches has not been met.
page_->url_loaders_.size() <
PrefetchProxyMaximumNumberOfConcurrentPrefetches()) {
StartSinglePrefetch();
}
}
void PrefetchProxyTabHelper::StartSinglePrefetch() {
DCHECK(!page_->urls_to_prefetch_.empty());
DCHECK(!(PrefetchProxyMaximumNumberOfPrefetches().has_value() &&
page_->decoy_requests_attempted_ +
page_->srp_metrics_->prefetch_attempted_count_ >=
PrefetchProxyMaximumNumberOfPrefetches().value()));
DCHECK(page_->url_loaders_.size() <
PrefetchProxyMaximumNumberOfConcurrentPrefetches());
GURL url = page_->urls_to_prefetch_[0];
page_->urls_to_prefetch_.erase(page_->urls_to_prefetch_.begin());
// Only update these metrics on normal prefetches.
if (page_->decoy_urls_.find(url) == page_->decoy_urls_.end()) {
page_->srp_metrics_->prefetch_attempted_count_++;
// The status is updated to be successful or failed when it finishes.
OnPrefetchStatusUpdate(
url, PrefetchProxyPrefetchStatus::kPrefetchNotFinishedInTime);
} else {
page_->decoy_requests_attempted_++;
OnPrefetchStatusUpdate(
url, PrefetchProxyPrefetchStatus::kPrefetchIsPrivacyDecoy);
}
url::Origin origin = url::Origin::Create(url);
net::IsolationInfo isolation_info = net::IsolationInfo::Create(
net::IsolationInfo::RequestType::kMainFrame, origin, origin,
net::SiteForCookies::FromOrigin(origin));
network::ResourceRequest::TrustedParams trusted_params;
trusted_params.isolation_info = isolation_info;
std::unique_ptr<network::ResourceRequest> request =
std::make_unique<network::ResourceRequest>();
request->url = url;
request->method = "GET";
request->enable_load_timing = true;
request->load_flags = net::LOAD_DISABLE_CACHE | net::LOAD_PREFETCH;
request->credentials_mode = network::mojom::CredentialsMode::kInclude;
request->headers.SetHeader(content::kCorsExemptPurposeHeaderName, "prefetch");
// Remove the user agent header if it was set so that the network context's
// default is used.
request->headers.RemoveHeader("User-Agent");
request->trusted_params = trusted_params;
request->site_for_cookies = trusted_params.isolation_info.site_for_cookies();
net::NetworkTrafficAnnotationTag traffic_annotation =
net::DefineNetworkTrafficAnnotation("navigation_predictor_srp_prefetch",
R"(
semantics {
sender: "Navigation Predictor SRP Prefetch Loader"
description:
"Prefetches the mainframe HTML of a page linked from a Google "
"Search Result Page (SRP). This is done out-of-band of normal "
"prefetches to allow total isolation of this request from the "
"rest of browser traffic and user state like cookies and cache."
trigger:
"Used for sites off of Google SRPs (Search Result Pages) only "
"for Lite mode users when the feature is enabled."
data: "None."
destination: WEBSITE
}
policy {
cookies_allowed: NO
setting:
"Users can control Lite mode on Android via the settings menu. "
"Lite mode is not available on iOS, and on desktop only for "
"developer testing."
policy_exception_justification: "Not implemented."
})");
std::unique_ptr<network::SimpleURLLoader> loader =
network::SimpleURLLoader::Create(std::move(request), traffic_annotation);
// base::Unretained is safe because |loader| is owned by |this|.
loader->SetOnRedirectCallback(
base::BindRepeating(&PrefetchProxyTabHelper::OnPrefetchRedirect,
base::Unretained(this), loader.get(), url));
loader->SetAllowHttpErrorResults(true);
loader->SetTimeoutDuration(PrefetchProxyTimeoutDuration());
loader->DownloadToString(
GetURLLoaderFactory(),
base::BindOnce(&PrefetchProxyTabHelper::OnPrefetchComplete,
base::Unretained(this), loader.get(), url, isolation_info),
PrefetchProxyMainframeBodyLengthLimit());
page_->url_loaders_.emplace(std::move(loader));
// Start a spare renderer now so that it will be ready by the time it is
// useful to have.
if (ShouldStartSpareRenderer()) {
StartSpareRenderer();
}
}
void PrefetchProxyTabHelper::OnPrefetchRedirect(
network::SimpleURLLoader* loader,
const GURL& original_url,
const net::RedirectInfo& redirect_info,
const network::mojom::URLResponseHead& response_head,
std::vector<std::string>* removed_headers) {
DCHECK(PrefetchingActive());
// Copy the position ordering when there is a redirect so the metrics don't
// miss out on redirects.
auto position_iter = page_->original_prediction_ordering_.find(original_url);
if (position_iter != page_->original_prediction_ordering_.end()) {
page_->original_prediction_ordering_.emplace(redirect_info.new_url,
position_iter->second);
}
if (page_->decoy_urls_.find(original_url) != page_->decoy_urls_.end()) {
// Check whether the next url is eligible (without considering user data) to
// be prefetched as a decoy.
auto result =
CheckEligibilityOfURLSansUserData(profile_, redirect_info.new_url);
if (result.first && PrefetchProxySendDecoyRequestForIneligiblePrefetch()) {
page_->decoy_urls_.emplace(redirect_info.new_url);
page_->urls_to_prefetch_.push_back(redirect_info.new_url);
}
// Cancels the current request.
DCHECK(page_->url_loaders_.find(loader) != page_->url_loaders_.end());
page_->url_loaders_.erase(page_->url_loaders_.find(loader));
Prefetch();
return;
}
page_->srp_metrics_->prefetch_total_redirect_count_++;
// Run the new URL through all the eligibility checks. In the mean time,
// continue on with other Prefetches.
CheckEligibilityOfURL(
profile_, redirect_info.new_url,
base::BindOnce(&PrefetchProxyTabHelper::OnGotEligibilityResult,
weak_factory_.GetWeakPtr()));
// Cancels the current request.
DCHECK(page_->url_loaders_.find(loader) != page_->url_loaders_.end());
page_->url_loaders_.erase(page_->url_loaders_.find(loader));
Prefetch();
}
void PrefetchProxyTabHelper::OnPrefetchComplete(
network::SimpleURLLoader* loader,
const GURL& url,
const net::IsolationInfo& isolation_info,
std::unique_ptr<std::string> body) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(PrefetchingActive());
if (page_->decoy_urls_.find(url) != page_->decoy_urls_.end()) {
if (loader->CompletionStatus()) {
page_->prefetch_metrics_collector_->OnDecoyPrefetchComplete(
url, page_->original_prediction_ordering_.find(url)->second,
loader->ResponseInfo() ? loader->ResponseInfo()->Clone() : nullptr,
loader->CompletionStatus().value());
}
for (auto& observer : observer_list_) {
observer.OnDecoyPrefetchCompleted(url);
}
// Do nothing with the response, i.e.: don't cache it.
return;
}
base::UmaHistogramSparse("PrefetchProxy.Prefetch.Mainframe.NetError",
std::abs(loader->NetError()));
if (loader->CompletionStatus()) {
page_->prefetch_metrics_collector_->OnMainframeResourcePrefetched(
url, page_->original_prediction_ordering_.find(url)->second,
loader->ResponseInfo() ? loader->ResponseInfo()->Clone() : nullptr,
loader->CompletionStatus().value());
}
if (loader->NetError() != net::OK) {
OnPrefetchStatusUpdate(
url, PrefetchProxyPrefetchStatus::kPrefetchFailedNetError);
for (auto& observer : observer_list_) {
observer.OnPrefetchCompletedWithError(url, loader->NetError());
}
}
if (loader->NetError() == net::OK && body && loader->ResponseInfo()) {
network::mojom::URLResponseHeadPtr head = loader->ResponseInfo()->Clone();
DCHECK(!head->proxy_server.is_direct());
HandlePrefetchResponse(url, isolation_info, std::move(head),
std::move(body));
}
DCHECK(page_->url_loaders_.find(loader) != page_->url_loaders_.end());
page_->url_loaders_.erase(page_->url_loaders_.find(loader));
Prefetch();
}
void PrefetchProxyTabHelper::HandlePrefetchResponse(
const GURL& url,
const net::IsolationInfo& isolation_info,
network::mojom::URLResponseHeadPtr head,
std::unique_ptr<std::string> body) {
DCHECK(!head->was_fetched_via_cache);
if (!head->headers)
return;
UMA_HISTOGRAM_COUNTS_10M("PrefetchProxy.Prefetch.Mainframe.BodyLength",
body->size());
absl::optional<base::TimeDelta> total_time = GetTotalPrefetchTime(head.get());
if (total_time) {
UMA_HISTOGRAM_CUSTOM_TIMES("PrefetchProxy.Prefetch.Mainframe.TotalTime",
*total_time, base::Milliseconds(10),
base::Seconds(30), 100);
}
absl::optional<base::TimeDelta> connect_time =
GetPrefetchConnectTime(head.get());
if (connect_time) {
UMA_HISTOGRAM_TIMES("PrefetchProxy.Prefetch.Mainframe.ConnectTime",
*connect_time);
}
int response_code = head->headers->response_code();
base::UmaHistogramSparse("PrefetchProxy.Prefetch.Mainframe.RespCode",
response_code);
if (response_code < 200 || response_code >= 300) {
OnPrefetchStatusUpdate(url,
PrefetchProxyPrefetchStatus::kPrefetchFailedNon2XX);
for (auto& observer : observer_list_) {
observer.OnPrefetchCompletedWithError(url, response_code);
}
if (response_code == net::HTTP_SERVICE_UNAVAILABLE) {
base::TimeDelta retry_after;
std::string retry_after_string;
if (head->headers->EnumerateHeader(nullptr, "Retry-After",
&retry_after_string) &&
net::HttpUtil::ParseRetryAfterHeader(
retry_after_string, base::Time::Now(), &retry_after)) {
PrefetchProxyService* service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
service->origin_decider()->ReportOriginRetryAfter(url, retry_after);
}
}
return;
}
if (head->mime_type != "text/html") {
OnPrefetchStatusUpdate(url,
PrefetchProxyPrefetchStatus::kPrefetchFailedNotHTML);
return;
}
std::unique_ptr<PrefetchedMainframeResponseContainer> response =
std::make_unique<PrefetchedMainframeResponseContainer>(
isolation_info, std::move(head), std::move(body));
page_->prefetched_responses_.emplace(url, std::move(response));
page_->prefetch_start_times_.emplace(url, base::TimeTicks::Now());
page_->srp_metrics_->prefetch_successful_count_++;
OnPrefetchStatusUpdate(url, PrefetchProxyPrefetchStatus::kPrefetchSuccessful);
MaybeDoNoStatePrefetch(url);
for (auto& observer : observer_list_) {
observer.OnPrefetchCompletedSuccessfully(url);
}
}
void PrefetchProxyTabHelper::MaybeDoNoStatePrefetch(const GURL& url) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!PrefetchProxyNoStatePrefetchSubresources()) {
return;
}
// Not all prefetches are eligible for NSP, which fetches subresources.
if (!base::Contains(page_->allowed_to_prefetch_subresources_, url))
return;
page_->urls_to_no_state_prefetch_.push_back(url);
DoNoStatePrefetch();
}
void PrefetchProxyTabHelper::DoNoStatePrefetch() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (page_->urls_to_no_state_prefetch_.empty()) {
return;
}
// Ensure there is not an active navigation.
if (web_contents()->GetController().GetPendingEntry()) {
return;
}
absl::optional<size_t> max_attempts =
PrefetchProxyMaximumNumberOfNoStatePrefetchAttempts();
if (max_attempts.has_value() &&
page_->number_of_no_state_prefetch_attempts_ >= max_attempts.value()) {
return;
}
prerender::NoStatePrefetchManager* no_state_prefetch_manager =
prerender::NoStatePrefetchManagerFactory::GetForBrowserContext(profile_);
if (!no_state_prefetch_manager) {
return;
}
PrefetchProxyService* service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
if (!service) {
return;
}
GURL url = page_->urls_to_no_state_prefetch_[0];
// Don't start another NSP until the previous one finishes.
{
PrefetchProxySubresourceManager* manager =
service->GetSubresourceManagerForURL(url);
if (manager && manager->has_nsp_handle()) {
return;
}
}
// The manager must be created here so that the mainframe response can be
// given to the URLLoaderInterceptor in this call stack, but may be destroyed
// before the end of the method if the handle is not created.
PrefetchProxySubresourceManager* manager =
service->OnAboutToNoStatePrefetch(url, CopyPrefetchResponseForNSP(url));
DCHECK_EQ(manager, service->GetSubresourceManagerForURL(url));
manager->SetPrefetchMetricsCollector(page_->prefetch_metrics_collector_);
manager->SetCreateIsolatedLoaderFactoryCallback(
base::BindRepeating(&PrefetchProxyTabHelper::CreateNewURLLoaderFactory,
weak_factory_.GetWeakPtr()));
content::SessionStorageNamespace* session_storage_namespace =
web_contents()->GetController().GetDefaultSessionStorageNamespace();
gfx::Size size = web_contents()->GetContainerBounds().size();
std::unique_ptr<prerender::NoStatePrefetchHandle> handle =
no_state_prefetch_manager->AddIsolatedPrerender(
url, session_storage_namespace, size);
if (!handle) {
// Clean up the prefetch response in |service| since it wasn't used.
service->DestroySubresourceManagerForURL(url);
// Don't use |manager| again!
page_->failed_no_state_prefetch_urls_.push_back(url);
// Try the next URL.
page_->urls_to_no_state_prefetch_.erase(
page_->urls_to_no_state_prefetch_.begin());
DoNoStatePrefetch();
return;
}
page_->number_of_no_state_prefetch_attempts_++;
// It is possible for the manager to be destroyed during the NoStatePrefetch
// navigation. If this happens, abort the NSP and try again.
manager = service->GetSubresourceManagerForURL(url);
if (!manager) {
handle->OnCancel();
handle.reset();
page_->failed_no_state_prefetch_urls_.push_back(url);
// Try the next URL.
page_->urls_to_no_state_prefetch_.erase(
page_->urls_to_no_state_prefetch_.begin());
DoNoStatePrefetch();
return;
}
manager->ManageNoStatePrefetch(
std::move(handle),
base::BindOnce(&PrefetchProxyTabHelper::OnPrerenderDone,
weak_factory_.GetWeakPtr(), url));
}
void PrefetchProxyTabHelper::OnPrerenderDone(const GURL& url) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// The completed NSP probably consumed a previously started spare renderer, so
// kick off another one if needed.
if (ShouldStartSpareRenderer()) {
StartSpareRenderer();
}
// It is possible that this is run as a callback after a navigation has
// already happened and |page_| is now a different instance than when the
// prerender was started. In this case, just return.
if (page_->urls_to_no_state_prefetch_.empty() ||
url != page_->urls_to_no_state_prefetch_[0]) {
return;
}
page_->no_state_prefetched_urls_.push_back(
page_->urls_to_no_state_prefetch_[0]);
for (auto& observer : observer_list_) {
observer.OnNoStatePrefetchFinished();
}
page_->urls_to_no_state_prefetch_.erase(
page_->urls_to_no_state_prefetch_.begin());
DoNoStatePrefetch();
}
void PrefetchProxyTabHelper::StartSpareRenderer() {
page_->number_of_spare_renderers_started_++;
content::RenderProcessHost::WarmupSpareRenderProcessHost(profile_);
}
void PrefetchProxyTabHelper::PrefetchSpeculationCandidates(
const std::vector<GURL>& private_prefetches_with_subresources,
const std::vector<GURL>& private_prefetches,
const GURL& source_document_url) {
// Use navigation predictor by default.
if (!PrefetchProxyUseSpeculationRules())
return;
// For IP-private prefetches, using the Google proxy needs to be restricted to
// first party sites until we understand the benefit and determine interest
// from other sites.
if (!PrefetchProxyAllowAllDomains() &&
!IsGoogleDomainUrl(source_document_url, google_util::ALLOW_SUBDOMAIN,
google_util::ALLOW_NON_STANDARD_PORTS)) {
return;
}
std::vector<GURL> prefetches = private_prefetches;
std::set<GURL> allowed_to_prefetch_subresources;
for (auto url : private_prefetches_with_subresources) {
prefetches.push_back(url);
allowed_to_prefetch_subresources.insert(url);
}
PrefetchUrls(prefetches, allowed_to_prefetch_subresources);
}
void PrefetchProxyTabHelper::OnPredictionUpdated(
const absl::optional<NavigationPredictorKeyedService::Prediction>
prediction) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// Use speculation rules API instead of navigation predictor.
if (PrefetchProxyUseSpeculationRules())
return;
if (!prediction.has_value()) {
return;
}
if (prediction->prediction_source() !=
NavigationPredictorKeyedService::PredictionSource::
kAnchorElementsParsedFromWebPage) {
return;
}
if (prediction.value().web_contents() != web_contents()) {
// We only care about predictions in this tab.
return;
}
const absl::optional<GURL>& source_document_url =
prediction->source_document_url();
if (!source_document_url || source_document_url->is_empty())
return;
if (!google_util::IsGoogleSearchUrl(source_document_url.value())) {
return;
}
// For the navigation predictor approach, we assume all predicted URLs are
// eligible for NSP.
std::set<GURL> allowed_to_prefetch_subresources(
prediction.value().sorted_predicted_urls().begin(),
prediction.value().sorted_predicted_urls().end());
PrefetchUrls(prediction.value().sorted_predicted_urls(),
allowed_to_prefetch_subresources);
}
void PrefetchProxyTabHelper::PrefetchUrls(
const std::vector<GURL>& prefetch_targets,
const std::set<GURL>& allowed_to_prefetch_subresources) {
if (!PrefetchProxyIsEnabled()) {
return;
}
if (!IsProfileEligible()) {
return;
}
// This checks whether the user has enabled pre* actions in the settings UI.
if (!chrome_browser_net::CanPreresolveAndPreconnectUI(profile_->GetPrefs())) {
return;
}
if (!page_->prefetch_metrics_collector_) {
page_->prefetch_metrics_collector_ =
base::MakeRefCounted<PrefetchProxyPrefetchMetricsCollector>(
page_->navigation_start_,
web_contents()->GetMainFrame()->GetPageUkmSourceId());
}
// Remove duplicate prefetches, but allow |allowed_to_prefetch_subresources|
// to be set for any upgraded prefetches.
std::vector<GURL> new_targets;
for (const auto& prefetch : prefetch_targets) {
if (page_->original_prediction_ordering_.find(prefetch) ==
page_->original_prediction_ordering_.end()) {
new_targets.push_back(prefetch);
}
}
// It's very likely we'll prefetch something at this point, so inform PLM to
// start tracking metrics.
InformPLMOfLikelyPrefetching(web_contents());
page_->srp_metrics_->predicted_urls_count_ += new_targets.size();
// It is possible, since it is not stipulated by the API contract, that the
// navigation predictor will issue multiple predictions during a single page
// load. Additional predictions should be treated as appending to the ordering
// of previous predictions.
size_t original_prediction_ordering_starting_size =
page_->original_prediction_ordering_.size();
page_->allowed_to_prefetch_subresources_.insert(
allowed_to_prefetch_subresources.begin(),
allowed_to_prefetch_subresources.end());
for (size_t i = 0; i < new_targets.size(); ++i) {
GURL url = new_targets[i];
size_t url_index = original_prediction_ordering_starting_size + i;
page_->original_prediction_ordering_.emplace(url, url_index);
CheckEligibilityOfURL(
profile_, url,
base::BindOnce(&PrefetchProxyTabHelper::OnGotEligibilityResult,
weak_factory_.GetWeakPtr()));
}
}
// static
content::ServiceWorkerContext* PrefetchProxyTabHelper::GetServiceWorkerContext(
Profile* profile) {
if (g_service_worker_context_for_test)
return g_service_worker_context_for_test;
return profile->GetDefaultStoragePartition()->GetServiceWorkerContext();
}
// static
std::pair<bool, absl::optional<PrefetchProxyPrefetchStatus>>
PrefetchProxyTabHelper::CheckEligibilityOfURLSansUserData(Profile* profile,
const GURL& url) {
if (!IsProfileEligible(profile)) {
return std::make_pair(false, absl::nullopt);
}
if (!PrefetchProxyUseSpeculationRules() &&
google_util::IsGoogleAssociatedDomainUrl(url)) {
return std::make_pair(
false, PrefetchProxyPrefetchStatus::kPrefetchNotEligibleGoogleDomain);
}
if (url.HostIsIPAddress()) {
return std::make_pair(
false,
PrefetchProxyPrefetchStatus::kPrefetchNotEligibleHostIsIPAddress);
}
if (!url.SchemeIs(url::kHttpsScheme)) {
return std::make_pair(
false,
PrefetchProxyPrefetchStatus::kPrefetchNotEligibleSchemeIsNotHttps);
}
PrefetchProxyService* prefetch_proxy_service =
PrefetchProxyServiceFactory::GetForProfile(profile);
if (!prefetch_proxy_service) {
return std::make_pair(false, absl::nullopt);
}
if (!prefetch_proxy_service->proxy_configurator()
->IsPrefetchProxyAvailable()) {
return std::make_pair(
false, PrefetchProxyPrefetchStatus::kPrefetchProxyNotAvailable);
}
return std::make_pair(true, absl::nullopt);
}
// static
void PrefetchProxyTabHelper::CheckEligibilityOfURL(
Profile* profile,
const GURL& url,
OnEligibilityResultCallback result_callback) {
auto no_user_data_check = CheckEligibilityOfURLSansUserData(profile, url);
if (!no_user_data_check.first) {
std::move(result_callback).Run(url, false, no_user_data_check.second);
return;
}
content::StoragePartition* default_storage_partition =
profile->GetDefaultStoragePartition();
// Only the default storage partition is supported since that is the only
// place where service workers are observed by
// |PrefetchProxyServiceWorkersObserver|.
if (default_storage_partition !=
profile->GetStoragePartitionForUrl(url,
/*can_create=*/false)) {
std::move(result_callback)
.Run(url, false,
PrefetchProxyPrefetchStatus::
kPrefetchNotEligibleNonDefaultStoragePartition);
return;
}
PrefetchProxyService* prefetch_proxy_service =
PrefetchProxyServiceFactory::GetForProfile(profile);
if (!prefetch_proxy_service) {
std::move(result_callback).Run(url, false, absl::nullopt);
return;
}
if (!prefetch_proxy_service->origin_decider()
->IsOriginOutsideRetryAfterWindow(url)) {
std::move(result_callback)
.Run(url, false,
PrefetchProxyPrefetchStatus::kPrefetchIneligibleRetryAfter);
return;
}
content::ServiceWorkerContext* service_worker_context_ =
GetServiceWorkerContext(profile);
bool site_has_service_worker =
service_worker_context_->MaybeHasRegistrationForOrigin(
url::Origin::Create(url));
if (site_has_service_worker) {
std::move(result_callback)
.Run(url, false,
PrefetchProxyPrefetchStatus::
kPrefetchNotEligibleUserHasServiceWorker);
return;
}
net::CookieOptions options = net::CookieOptions::MakeAllInclusive();
options.set_return_excluded_cookies();
default_storage_partition->GetCookieManagerForBrowserProcess()->GetCookieList(
url, options,
base::BindOnce(&OnGotCookieList, url, std::move(result_callback)));
}
void PrefetchProxyTabHelper::OnGotEligibilityResult(
const GURL& url,
bool eligible,
absl::optional<PrefetchProxyPrefetchStatus> status) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// It is possible that this callback is being run late. That is, after the
// user has navigated away from the origin SRP. To detect this, check if the
// url exists in the set of predicted urls. If it doesn't, do nothing.
if (page_->original_prediction_ordering_.find(url) ==
page_->original_prediction_ordering_.end()) {
return;
}
if (!eligible) {
if (status) {
OnPrefetchStatusUpdate(url, *status);
if (page_->prefetch_metrics_collector_) {
page_->prefetch_metrics_collector_->OnMainframeResourceNotEligible(
url, page_->original_prediction_ordering_.find(url)->second,
*status);
}
// Consider whether to send a decoy request to mask any user state (i.e.:
// cookies), and if so randomly decide whether to send a decoy request.
if (ShouldConsiderDecoyRequestForStatus(*status) &&
PrefetchProxySendDecoyRequestForIneligiblePrefetch()) {
page_->decoy_urls_.emplace(url);
page_->urls_to_prefetch_.push_back(url);
OnPrefetchStatusUpdate(
url, PrefetchProxyPrefetchStatus::kPrefetchIsPrivacyDecoy);
Prefetch();
}
}
return;
}
// TODO(robertogden): Consider adding redirect URLs to the front of the list.
page_->urls_to_prefetch_.push_back(url);
page_->srp_metrics_->prefetch_eligible_count_++;
OnPrefetchStatusUpdate(url, PrefetchProxyPrefetchStatus::kPrefetchNotStarted);
if (page_->original_prediction_ordering_.find(url) !=
page_->original_prediction_ordering_.end()) {
size_t original_prediction_index =
page_->original_prediction_ordering_.find(url)->second;
// Check that we won't go above the allowable size.
if (original_prediction_index <
sizeof(page_->srp_metrics_->ordered_eligible_pages_bitmask_) * 8) {
page_->srp_metrics_->ordered_eligible_pages_bitmask_ |=
1 << original_prediction_index;
}
if (!PrefetchProxyShouldPrefetchPosition(original_prediction_index)) {
OnPrefetchStatusUpdate(
url, PrefetchProxyPrefetchStatus::kPrefetchPositionIneligible);
return;
}
}
Prefetch();
for (auto& observer : observer_list_) {
observer.OnNewEligiblePrefetchStarted();
}
}
bool PrefetchProxyTabHelper::IsWaitingForAfterSRPCookiesCopy() const {
switch (page_->cookie_copy_status_) {
case CookieCopyStatus::kNoNavigation:
case CookieCopyStatus::kCopyComplete:
return false;
case CookieCopyStatus::kWaitingForCopy:
return true;
}
}
void PrefetchProxyTabHelper::SetOnAfterSRPCookieCopyCompleteCallback(
base::OnceClosure callback) {
// We don't expect a callback unless there's something to wait on.
DCHECK(IsWaitingForAfterSRPCookiesCopy());
page_->on_after_srp_cookie_copy_complete_ = std::move(callback);
}
void PrefetchProxyTabHelper::CopyIsolatedCookiesOnAfterSRPClick(
const GURL& url) {
if (!page_->isolated_network_context_) {
// Not set in unit tests.
return;
}
page_->cookie_copy_status_ = CookieCopyStatus::kWaitingForCopy;
if (!page_->isolated_cookie_manager_) {
page_->isolated_network_context_->GetCookieManager(
page_->isolated_cookie_manager_.BindNewPipeAndPassReceiver());
}
net::CookieOptions options = net::CookieOptions::MakeAllInclusive();
page_->isolated_cookie_manager_->GetCookieList(
url, options,
base::BindOnce(
&PrefetchProxyTabHelper::OnGotIsolatedCookiesToCopyAfterSRPClick,
weak_factory_.GetWeakPtr(), url));
}
void PrefetchProxyTabHelper::OnGotIsolatedCookiesToCopyAfterSRPClick(
const GURL& url,
const net::CookieAccessResultList& cookie_list,
const net::CookieAccessResultList& excluded_cookies) {
DCHECK(IsWaitingForAfterSRPCookiesCopy());
UMA_HISTOGRAM_COUNTS_100("PrefetchProxy.Prefetch.Mainframe.CookiesToCopy",
cookie_list.size());
if (cookie_list.empty()) {
OnCopiedIsolatedCookiesAfterSRPClick();
return;
}
// When |barrier| is run |cookie_list.size()| times, it will run
// |OnCopiedIsolatedCookiesAfterSRPClick|.
base::RepeatingClosure barrier = base::BarrierClosure(
cookie_list.size(),
base::BindOnce(
&PrefetchProxyTabHelper::OnCopiedIsolatedCookiesAfterSRPClick,
weak_factory_.GetWeakPtr()));
content::StoragePartition* default_storage_partition =
profile_->GetDefaultStoragePartition();
net::CookieOptions options = net::CookieOptions::MakeAllInclusive();
for (const net::CookieWithAccessResult& cookie : cookie_list) {
default_storage_partition->GetCookieManagerForBrowserProcess()
->SetCanonicalCookie(cookie.cookie, url, options,
base::BindOnce(&CookieSetHelper, barrier));
}
}
void PrefetchProxyTabHelper::OnCopiedIsolatedCookiesAfterSRPClick() {
DCHECK(IsWaitingForAfterSRPCookiesCopy());
page_->cookie_copy_status_ = CookieCopyStatus::kCopyComplete;
if (page_->on_after_srp_cookie_copy_complete_) {
std::move(page_->on_after_srp_cookie_copy_complete_).Run();
}
}
network::mojom::URLLoaderFactory*
PrefetchProxyTabHelper::GetURLLoaderFactory() {
if (!page_->isolated_url_loader_factory_) {
CreateIsolatedURLLoaderFactory();
}
DCHECK(page_->isolated_url_loader_factory_);
return page_->isolated_url_loader_factory_.get();
}
void PrefetchProxyTabHelper::CreateNewURLLoaderFactory(
mojo::PendingReceiver<network::mojom::URLLoaderFactory> pending_receiver,
absl::optional<net::IsolationInfo> isolation_info) {
DCHECK(page_->isolated_network_context_);
auto factory_params = network::mojom::URLLoaderFactoryParams::New();
factory_params->process_id = network::mojom::kBrowserProcessId;
factory_params->is_trusted = true;
factory_params->is_corb_enabled = false;
if (isolation_info) {
factory_params->isolation_info = *isolation_info;
}
page_->isolated_network_context_->CreateURLLoaderFactory(
std::move(pending_receiver), std::move(factory_params));
}
void PrefetchProxyTabHelper::CreateIsolatedURLLoaderFactory() {
page_->isolated_network_context_.reset();
page_->isolated_url_loader_factory_.reset();
PrefetchProxyService* prefetch_proxy_service =
PrefetchProxyServiceFactory::GetForProfile(profile_);
auto context_params = network::mojom::NetworkContextParams::New();
context_params->user_agent = content::GetReducedUserAgent(
base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kUseMobileUserAgent),
version_info::GetMajorVersionNumber());
context_params->accept_language = net::HttpUtil::GenerateAcceptLanguageHeader(
profile_->GetPrefs()->GetString(language::prefs::kAcceptLanguages));
context_params->initial_custom_proxy_config =
prefetch_proxy_service->proxy_configurator()->CreateCustomProxyConfig();
context_params->custom_proxy_connection_observer_remote =
prefetch_proxy_service->proxy_configurator()
->NewProxyConnectionObserverRemote();
context_params->cert_verifier_params = content::GetCertVerifierParams(
cert_verifier::mojom::CertVerifierCreationParams::New());
context_params->cors_exempt_header_list = {
content::kCorsExemptPurposeHeaderName};
context_params->cookie_manager_params =
network::mojom::CookieManagerParams::New();
context_params->http_cache_enabled = true;
DCHECK(!context_params->http_cache_path);
// Also register a client config receiver so that updates to the set of proxy
// hosts or proxy headers will be updated.
mojo::Remote<network::mojom::CustomProxyConfigClient> config_client;
context_params->custom_proxy_config_client_receiver =
config_client.BindNewPipeAndPassReceiver();
prefetch_proxy_service->proxy_configurator()->AddCustomProxyConfigClient(
std::move(config_client));
// Explicitly disallow network service features which could cause a privacy
// leak.
context_params->enable_certificate_reporting = false;
context_params->enable_expect_ct_reporting = false;
context_params->enable_domain_reliability = false;
content::CreateNetworkContextInNetworkService(
page_->isolated_network_context_.BindNewPipeAndPassReceiver(),
std::move(context_params));
// Configure a context client to ensure Web Reports and other privacy leak
// surfaces won't be enabled.
mojo::PendingRemote<network::mojom::NetworkContextClient> client_remote;
mojo::MakeSelfOwnedReceiver(
std::make_unique<PrefetchProxyNetworkContextClient>(),
client_remote.InitWithNewPipeAndPassReceiver());
page_->isolated_network_context_->SetClient(std::move(client_remote));
mojo::PendingRemote<network::mojom::URLLoaderFactory> isolated_factory_remote;
CreateNewURLLoaderFactory(
isolated_factory_remote.InitWithNewPipeAndPassReceiver(), absl::nullopt);
page_->isolated_url_loader_factory_ = network::SharedURLLoaderFactory::Create(
std::make_unique<network::WrapperPendingSharedURLLoaderFactory>(
std::move(isolated_factory_remote)));
}
WEB_CONTENTS_USER_DATA_KEY_IMPL(PrefetchProxyTabHelper)