Google Git
Sign in
chromium / chromium / src / 8c86fa01f664444e097c792b16c0995bb56ee46c / . / content / browser / appcache / appcache_update_job.cc
blob: 1cabcfbb68775287477ac4b782f32795abe2fba0 [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/browser/appcache/appcache_update_job.h"
#include "base/bind.h"
#include "base/bind_helpers.h"
#include "base/compiler_specific.h"
#include "base/location.h"
#include "base/metrics/histogram_functions.h"
#include "base/single_thread_task_runner.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/threading/thread_task_runner_handle.h"
#include "content/browser/appcache/appcache_disk_cache_ops.h"
#include "content/browser/appcache/appcache_group.h"
#include "content/browser/appcache/appcache_histograms.h"
#include "content/browser/appcache/appcache_response_info.h"
#include "content/browser/appcache/appcache_update_job_cache_copier.h"
#include "content/browser/appcache/appcache_update_url_fetcher.h"
#include "content/browser/appcache/appcache_update_url_loader_request.h"
#include "content/public/browser/browser_thread.h"
#include "net/base/io_buffer.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/base/request_priority.h"
#include "storage/browser/quota/padding_key.h"
#include "third_party/blink/public/mojom/appcache/appcache.mojom.h"
#include "third_party/blink/public/mojom/devtools/console_message.mojom.h"
#include "url/origin.h"
namespace content {
namespace {
const int kAppCacheFetchBufferSize = 32768;
const size_t kMaxConcurrentUrlFetches = 2;
enum class ResourceCheck {
kValid,
kInvalid,
};
std::string FormatUrlErrorMessage(
const char* format, const GURL& url,
AppCacheUpdateJob::ResultType error,
int response_code) {
// Show the net response code if we have one.
int code = response_code;
if (error != AppCacheUpdateJob::SERVER_ERROR)
code = static_cast<int>(error);
return base::StringPrintf(format, code, url.spec().c_str());
}
bool IsEvictableError(AppCacheUpdateJob::ResultType result,
const blink::mojom::AppCacheErrorDetails& details) {
switch (result) {
case AppCacheUpdateJob::DB_ERROR:
case AppCacheUpdateJob::DISKCACHE_ERROR:
case AppCacheUpdateJob::QUOTA_ERROR:
case AppCacheUpdateJob::NETWORK_ERROR:
case AppCacheUpdateJob::CANCELLED_ERROR:
return false;
case AppCacheUpdateJob::REDIRECT_ERROR:
case AppCacheUpdateJob::SERVER_ERROR:
case AppCacheUpdateJob::SECURITY_ERROR:
return true;
case AppCacheUpdateJob::MANIFEST_ERROR:
return details.reason ==
blink::mojom::AppCacheErrorReason::APPCACHE_SIGNATURE_ERROR;
default:
NOTREACHED();
return true;
}
}
ResourceCheck CanUseExistingResource(
const net::HttpResponseInfo* http_info,
AppCacheUpdateMetricsRecorder& update_metrics) {
update_metrics.IncrementExistingResourceCheck();
if (!http_info->headers)
return ResourceCheck::kInvalid;
base::Time request_time = http_info->request_time;
base::Time response_time = http_info->response_time;
// The logic below works around the following confluence of problems.
//
// 1) If a cached response contains a Last-Modified header,
// AppCacheUpdateJob::URLFetcher::AddConditionalHeaders() adds an
// If-Modified-Since header, so the server may return an HTTP 304 Not Modified
// response. AppCacheUpdateJob::HandleResourceFetchCompleted() reuses the
// existing cache entry when a 304 is received, even though the HTTP
// specification mandates updating the cached headers with the headers in the
// 304 response.
//
// This deviation from the HTTP specification is Web-observable when AppCache
// resources are served with Last-Modified and Cache-Control: max-age headers.
// Specifically, if a server returns a 304 with a Cache-Control: max-age
// header, the response stored in AppCache should be updated to reflect the
// new cache expiration time. Instead, Chrome ignores all the headers in the
// 304 response, so the Cache-Control: max-age directive is discarded.
//
// In other words, once a cached resource's lifetime expires, 304 responses
// won't refresh its lifetime. Chrome gets stuck in a cycle where it sends
// If-Modified-Since requests, the server responds with 304, and the response
// headers are discarded.
//
// 2) The implementation of
// AppCacheUpdateJob::UpdateURLLoaderRequest::OnReceiveResponse() introduced
// in https://crrev.com/c/599359 did not populate |request_time| and
// |response_time|. When the Network Service was enabled, caches got populated
// with the default value of base::Time, which is the Windows epoch. So,
// cached entries with max-age values below ~40 years will require
// re-validation. https://crrev.com/c/1636266 fixed the cache population bug,
// but did not address the incorrect times that have already been written to
// users' disks.
//
// The 1st problem, on its own, hasn't had a large impact. This is likely
// because we have been advising sites to set max-age=31536000 (~1 year) for
// immutable resources, and most AppCache caches have been getting evicted
// before the entries' max-age expired. However, the 2nd problem caused us to
// create a large number of expired cache entries, and the unnecessary
// If-Modified-Since requests are causing noticeable levels of traffic.
//
// The logic below is a workaround while a longer-term fix gets developed and
// deployed. We'll consider all cache entries with invalid times to have been
// created on Sun, Jun 16 2019.
//
// TODO(cmp): Add timeline info here.
bool found_corruption = false;
static constexpr base::Time::Exploded kInvalidTimePlaceholderExploded = {
2019, 7, 0, 7, 0, 0, 0, 0};
if (request_time.is_null()) {
bool conversion_succeeded = base::Time::FromUTCExploded(
kInvalidTimePlaceholderExploded, &request_time);
DCHECK(conversion_succeeded);
found_corruption = true;
}
if (response_time.is_null()) {
bool conversion_succeeded = base::Time::FromUTCExploded(
kInvalidTimePlaceholderExploded, &response_time);
DCHECK(conversion_succeeded);
found_corruption = true;
}
if (found_corruption) {
update_metrics.IncrementExistingResourceCorrupt();
} else {
update_metrics.IncrementExistingResourceNotCorrupt();
}
// Record the max age / expiry value on this entry in days.
net::HttpResponseHeaders::FreshnessLifetimes lifetimes =
http_info->headers->GetFreshnessLifetimes(response_time);
base::UmaHistogramCounts10000("appcache.UpdateJobResourceFreshness",
lifetimes.freshness.InDays());
// Check HTTP caching semantics based on max-age and expiration headers.
if (http_info->headers->RequiresValidation(request_time, response_time,
base::Time::Now())) {
return ResourceCheck::kInvalid;
}
// Responses with a "vary" header generally get treated as expired,
// but we special case the "Origin" header since we know it's invariant.
// Also, content decoding is handled by the network library, the appcache
// stores decoded response bodies, so we can safely ignore varying on
// the "Accept-Encoding" header.
std::string value;
size_t iter = 0;
while (http_info->headers->EnumerateHeader(&iter, "vary", &value)) {
if (!base::EqualsCaseInsensitiveASCII(value, "Accept-Encoding") &&
!base::EqualsCaseInsensitiveASCII(value, "Origin")) {
return ResourceCheck::kInvalid;
}
}
update_metrics.IncrementExistingResourceReused();
return ResourceCheck::kValid;
}
void EmptyCompletionCallback(int result) {}
int64_t ComputeAppCacheResponsePadding(const GURL& response_url,
const GURL& manifest_url) {
// All cross-origin resources should have their size padded in response to
// queries regarding quota usage.
if (response_url.GetOrigin() == manifest_url.GetOrigin())
return 0;
return storage::ComputeResponsePadding(response_url.spec(),
storage::GetDefaultPaddingKey(),
/*has_metadata=*/false,
/*loaded_with_credentials=*/false);
}
} // namespace
const base::Feature kAppCacheUpdateResourceOn304Feature{
"AppCacheUpdateResourceOn304", base::FEATURE_DISABLED_BY_DEFAULT};
// Helper class for collecting hosts per frontend when sending notifications
// so that only one notification is sent for all hosts using the same frontend.
class HostNotifier {
public:
// Caller is responsible for ensuring there will be no duplicate hosts.
void AddHost(AppCacheHost* host) {
hosts_to_notify_.insert(host->frontend());
}
void AddHosts(const std::set<AppCacheHost*>& hosts) {
for (AppCacheHost* host : hosts)
AddHost(host);
}
void SendNotifications(blink::mojom::AppCacheEventID event_id) {
for (auto* frontend : hosts_to_notify_)
frontend->EventRaised(event_id);
}
void SendProgressNotifications(const GURL& url,
int num_total,
int num_complete) {
for (auto* frontend : hosts_to_notify_)
frontend->ProgressEventRaised(url, num_total, num_complete);
}
void SendErrorNotifications(
const blink::mojom::AppCacheErrorDetails& details) {
DCHECK(!details.message.empty());
for (auto* frontend : hosts_to_notify_)
frontend->ErrorEventRaised(details.Clone());
}
void SendLogMessage(const std::string& message) {
for (auto* frontend : hosts_to_notify_)
frontend->LogMessage(blink::mojom::ConsoleMessageLevel::kWarning,
message);
}
private:
std::set<blink::mojom::AppCacheFrontend*> hosts_to_notify_;
};
AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
bool checked,
AppCacheResponseInfo* info)
: url(url),
storage_checked(checked),
existing_response_info(info) {
}
AppCacheUpdateJob::UrlToFetch::UrlToFetch(const UrlToFetch& other) = default;
AppCacheUpdateJob::UrlToFetch::~UrlToFetch() = default;
AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service,
AppCacheGroup* group)
: service_(service),
manifest_url_(group->manifest_url()),
cached_manifest_parser_version_(-1),
fetched_manifest_parser_version_(-1),
cached_manifest_scope_(""),
fetched_manifest_scope_(""),
refetched_manifest_scope_(""),
update_resource_on_304_enabled_(
base::FeatureList::IsEnabled(kAppCacheUpdateResourceOn304Feature)),
group_(group),
update_type_(UNKNOWN_TYPE),
internal_state_(AppCacheUpdateJobState::FETCH_MANIFEST),
doing_full_update_check_(false),
master_entries_completed_(0),
url_fetches_completed_(0),
manifest_fetcher_(nullptr),
manifest_has_valid_mime_type_(false),
stored_state_(UNSTORED),
storage_(service->storage()) {
service_->AddObserver(this);
}
AppCacheUpdateJob::~AppCacheUpdateJob() {
update_metrics_.RecordFinalInternalState(internal_state_);
if (service_)
service_->RemoveObserver(this);
if (internal_state_ != AppCacheUpdateJobState::COMPLETED)
Cancel();
DCHECK(!inprogress_cache_.get());
DCHECK(pending_master_entries_.empty());
// No fetcher may outlive the job.
CHECK(!manifest_fetcher_);
CHECK(pending_url_fetches_.empty());
CHECK(master_entry_fetches_.empty());
if (group_)
group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
// Upload accumulated update job metrics to UMA. We expect at this point the
// update job has finalized its work and no external references exist back to
// it that may trigger more metrics to be logged. Especially,
// SetUpdateAppCacheStatus() causes the cache group's update job reference to
// be set to nullptr.
update_metrics_.UploadMetrics();
}
void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
const GURL& new_master_resource) {
DCHECK_EQ(group_->update_job(), this);
DCHECK(!group_->is_obsolete());
bool is_new_pending_master_entry = false;
if (!new_master_resource.is_empty()) {
DCHECK_EQ(new_master_resource, host->pending_master_entry_url());
DCHECK(!new_master_resource.has_ref());
DCHECK_EQ(new_master_resource.GetOrigin(), manifest_url_.GetOrigin());
if (base::Contains(failed_master_entries_, new_master_resource))
return;
// Cannot add more to this update if already terminating.
if (IsTerminating()) {
group_->QueueUpdate(host, new_master_resource);
return;
}
auto emplace_result = pending_master_entries_.emplace(
new_master_resource, std::vector<AppCacheHost*>());
is_new_pending_master_entry = emplace_result.second;
emplace_result.first->second.push_back(host);
host->AddObserver(this);
}
// Notify host (if any) if already checking or downloading.
AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status();
if (update_status == AppCacheGroup::CHECKING ||
update_status == AppCacheGroup::DOWNLOADING) {
if (host) {
NotifySingleHost(host,
blink::mojom::AppCacheEventID::APPCACHE_CHECKING_EVENT);
if (update_status == AppCacheGroup::DOWNLOADING)
NotifySingleHost(
host, blink::mojom::AppCacheEventID::APPCACHE_DOWNLOADING_EVENT);
// Add to fetch list or an existing entry if already fetched.
if (!new_master_resource.is_empty()) {
AddMasterEntryToFetchList(host, new_master_resource,
is_new_pending_master_entry);
}
}
return;
}
// Begin update process for the group.
MadeProgress();
group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING);
if (group_->HasCache()) {
base::TimeDelta kFullUpdateInterval = base::TimeDelta::FromHours(24);
update_type_ = UPGRADE_ATTEMPT;
AppCache* cache = group_->newest_complete_cache();
cached_manifest_parser_version_ = cache->manifest_parser_version();
cached_manifest_scope_ = cache->manifest_scope();
base::TimeDelta time_since_last_check =
base::Time::Now() - group_->last_full_update_check_time();
doing_full_update_check_ = time_since_last_check > kFullUpdateInterval;
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_CHECKING_EVENT);
} else {
update_type_ = CACHE_ATTEMPT;
doing_full_update_check_ = true;
DCHECK(host);
NotifySingleHost(host,
blink::mojom::AppCacheEventID::APPCACHE_CHECKING_EVENT);
}
if (!new_master_resource.is_empty()) {
AddMasterEntryToFetchList(host, new_master_resource,
is_new_pending_master_entry);
}
BrowserThread::PostBestEffortTask(
FROM_HERE, base::ThreadTaskRunnerHandle::Get(),
base::BindOnce(&AppCacheUpdateJob::FetchManifest,
weak_factory_.GetWeakPtr()));
}
std::unique_ptr<AppCacheResponseWriter>
AppCacheUpdateJob::CreateResponseWriter() {
std::unique_ptr<AppCacheResponseWriter> writer =
storage_->CreateResponseWriter(manifest_url_);
stored_response_ids_.push_back(writer->response_id());
return writer;
}
void AppCacheUpdateJob::HandleCacheFailure(
const blink::mojom::AppCacheErrorDetails& error_details,
ResultType result,
const GURL& failed_resource_url) {
// 7.9.4 cache failure steps 2-8.
DCHECK(internal_state_ != AppCacheUpdateJobState::CACHE_FAILURE);
DCHECK(!error_details.message.empty());
DCHECK(result != UPDATE_OK);
internal_state_ = AppCacheUpdateJobState::CACHE_FAILURE;
CancelAllUrlFetches();
CancelAllMasterEntryFetches(error_details);
NotifyAllError(error_details);
DiscardInprogressCache();
internal_state_ = AppCacheUpdateJobState::COMPLETED;
if (update_type_ == CACHE_ATTEMPT ||
!IsEvictableError(result, error_details) ||
service_->storage() != storage_) {
DeleteSoon();
return;
}
if (group_->first_evictable_error_time().is_null()) {
group_->set_first_evictable_error_time(base::Time::Now());
storage_->StoreEvictionTimes(group_);
DeleteSoon();
return;
}
base::TimeDelta kMaxEvictableErrorDuration = base::TimeDelta::FromDays(14);
base::TimeDelta error_duration =
base::Time::Now() - group_->first_evictable_error_time();
if (error_duration > kMaxEvictableErrorDuration) {
// Break the connection with the group prior to calling
// DeleteAppCacheGroup, otherwise that method would delete |this|
// and we need the stack to unwind prior to deletion.
group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
group_ = nullptr;
service_->DeleteAppCacheGroup(manifest_url_,
base::BindOnce(EmptyCompletionCallback));
}
DeleteSoon(); // To unwind the stack prior to deletion.
}
void AppCacheUpdateJob::FetchManifest() {
DCHECK(!manifest_fetcher_);
manifest_fetcher_ = std::make_unique<URLFetcher>(
manifest_url_, URLFetcher::FetchType::kManifest, this,
kAppCacheFetchBufferSize);
// Maybe load the cached headers to make a conditional request.
AppCacheEntry* entry =
(update_type_ == UPGRADE_ATTEMPT)
? group_->newest_complete_cache()->GetEntry(manifest_url_)
: nullptr;
if (entry && !doing_full_update_check_) {
// Asynchronously load response info for manifest from newest cache.
storage_->LoadResponseInfo(manifest_url_, entry->response_id(), this);
return;
}
manifest_fetcher_->Start();
return;
}
void AppCacheUpdateJob::RefetchManifest() {
DCHECK(!manifest_fetcher_);
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::REFETCH_MANIFEST);
DCHECK(manifest_response_info_.get());
manifest_fetcher_ = std::make_unique<URLFetcher>(
manifest_url_, URLFetcher::FetchType::kManifestRefetch, this,
kAppCacheFetchBufferSize);
manifest_fetcher_->set_existing_response_headers(
manifest_response_info_->headers.get());
manifest_fetcher_->Start();
}
void AppCacheUpdateJob::HandleManifestFetchCompleted(URLFetcher* url_fetcher,
int net_error) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
DCHECK_EQ(manifest_fetcher_.get(), url_fetcher);
std::unique_ptr<URLFetcher> manifest_fetcher = std::move(manifest_fetcher_);
UpdateURLLoaderRequest* request = manifest_fetcher->request();
auto token_expires = storage_->GetOriginTrialExpiration(
request->GetURL(), request->GetResponseHeaders(), base::Time::Now());
int response_code = -1;
bool is_valid_response_code = false;
std::string optional_manifest_scope;
if (net_error == net::OK) {
response_code = request->GetResponseCode();
is_valid_response_code = (response_code / 100 == 2);
std::string mime_type = request->GetMimeType();
manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest");
optional_manifest_scope = request->GetAppCacheAllowedHeader();
}
fetched_manifest_scope_ =
AppCache::GetManifestScope(manifest_url_, optional_manifest_scope);
if (is_valid_response_code) {
manifest_data_ = manifest_fetcher->manifest_data();
manifest_response_info_ =
std::make_unique<net::HttpResponseInfo>(request->GetResponseInfo());
if (update_type_ == UPGRADE_ATTEMPT) {
CheckIfManifestChanged(token_expires); // continues asynchronously
} else {
HandleFetchedManifestChanged(token_expires);
}
return;
}
if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
if (fetched_manifest_scope_ == cached_manifest_scope_) {
HandleFetchedManifestIsUnchanged();
} else {
// We don't check if |cached_manifest_parser_version_| is 0 here since in
// that case we didn't add conditional headers and don't expect a 304
// response.
ReadManifestFromCacheAndContinue();
}
return;
}
if ((response_code == 404 || response_code == 410) &&
update_type_ == UPGRADE_ATTEMPT) {
storage_->MakeGroupObsolete(group_, this, response_code); // async
return;
}
const char kFormatString[] = "Manifest fetch failed (%d) %s";
std::string message = FormatUrlErrorMessage(
kFormatString, manifest_url_, manifest_fetcher->result(), response_code);
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message, blink::mojom::AppCacheErrorReason::APPCACHE_MANIFEST_ERROR,
manifest_url_, response_code, false /*is_cross_origin*/),
manifest_fetcher->result(), GURL());
}
void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
bool success,
int response_code) {
DCHECK(master_entry_fetches_.empty());
CancelAllMasterEntryFetches(blink::mojom::AppCacheErrorDetails(
"The cache has been made obsolete, "
"the manifest file returned 404 or 410",
blink::mojom::AppCacheErrorReason::APPCACHE_MANIFEST_ERROR, GURL(),
response_code, false /*is_cross_origin*/));
if (success) {
DCHECK(group->is_obsolete());
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_OBSOLETE_EVENT);
internal_state_ = AppCacheUpdateJobState::COMPLETED;
MaybeCompleteUpdate();
} else {
// Treat failure to mark group obsolete as a cache failure.
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Failed to mark the cache as obsolete",
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR, GURL(),
0, false /*is_cross_origin*/),
DB_ERROR, GURL());
}
}
void AppCacheUpdateJob::HandleFetchedManifestIsUnchanged() {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
DCHECK_EQ(update_type_, UPGRADE_ATTEMPT);
internal_state_ = AppCacheUpdateJobState::NO_UPDATE;
// We should only ever allow AppCaches to remain unchanged if their parser
// version is 1 or higher.
DCHECK_GE(cached_manifest_parser_version_, 1);
// No manifest update is planned. Set the fetched manifest parser version
// and scope to match their initial values.
fetched_manifest_parser_version_ = cached_manifest_parser_version_;
fetched_manifest_scope_ = cached_manifest_scope_;
// Set |refetched_manifest_scope_| to match |fetched_manifest_scope_| so
// StoreGroupAndCache() can verify the overall state of the
// AppCacheUpdateJob is correct.
refetched_manifest_scope_ = fetched_manifest_scope_;
// Wait for pending master entries to download.
FetchMasterEntries();
MaybeCompleteUpdate(); // if not done, run async 7.9.4 step 7 substeps
}
void AppCacheUpdateJob::HandleFetchedManifestChanged(base::Time token_expires) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
AppCacheManifest manifest;
if (!ParseManifest(manifest_url_, fetched_manifest_scope_,
manifest_data_.data(), manifest_data_.length(),
manifest_has_valid_mime_type_
? PARSE_MANIFEST_ALLOWING_DANGEROUS_FEATURES
: PARSE_MANIFEST_PER_STANDARD,
manifest)) {
const char kFormatString[] = "Failed to parse manifest %s";
const std::string message = base::StringPrintf(kFormatString,
manifest_url_.spec().c_str());
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_SIGNATURE_ERROR, GURL(),
0, false /*is_cross_origin*/),
MANIFEST_ERROR, GURL());
VLOG(1) << message;
return;
}
// Ensure the manifest parser version matches what we configured.
DCHECK_EQ(manifest.parser_version, 1);
fetched_manifest_parser_version_ = manifest.parser_version;
// Ensure the manifest scope matches what we configured.
DCHECK_EQ(manifest.scope, fetched_manifest_scope_);
// Proceed with update process. Section 7.9.4 steps 8-20.
internal_state_ = AppCacheUpdateJobState::DOWNLOADING;
inprogress_cache_ =
base::MakeRefCounted<AppCache>(storage_, storage_->NewCacheId());
BuildUrlFileList(manifest);
inprogress_cache_->InitializeWithManifest(&manifest, token_expires);
// Associate all pending master hosts with the newly created cache.
for (const auto& pair : pending_master_entries_) {
const std::vector<AppCacheHost*>& hosts = pair.second;
for (AppCacheHost* host : hosts) {
host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
}
}
// Warn about dangerous features being ignored due to the wrong content-type
// Must be done after associating all pending master hosts.
if (manifest.did_ignore_intercept_namespaces) {
std::string message(
"Ignoring the INTERCEPT section of the application cache manifest "
"because the content type is not text/cache-manifest");
LogConsoleMessageToAll(message);
}
if (manifest.did_ignore_fallback_namespaces) {
std::string message(
"Ignoring out of scope FALLBACK entries of the application cache "
"manifest because the content-type is not text/cache-manifest");
LogConsoleMessageToAll(message);
}
group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING);
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_DOWNLOADING_EVENT);
FetchUrls();
FetchMasterEntries();
MaybeCompleteUpdate(); // if not done, continues when async fetches complete
}
void AppCacheUpdateJob::HandleResourceFetchCompleted(URLFetcher* url_fetcher,
int net_error) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::DOWNLOADING);
UpdateURLLoaderRequest* request = url_fetcher->request();
const GURL& url = request->GetURL();
auto it = pending_url_fetches_.find(url);
if (it == pending_url_fetches_.end()) {
NOTREACHED() << "Entry URL not found in pending_url_fetches_";
return;
}
DCHECK_EQ(it->second.get(), url_fetcher);
std::unique_ptr<URLFetcher> entry_fetcher = std::move(it->second);
pending_url_fetches_.erase(it);
// URLFetcher should only trigger this for resources, even if those entries
// happen to be manifest entries.
DCHECK_EQ(entry_fetcher->fetch_type(), URLFetcher::FetchType::kResource);
int response_code = net_error == net::OK
? request->GetResponseCode()
: entry_fetcher->redirect_response_code();
AppCacheEntry& entry = url_file_list_.find(url)->second;
if (update_resource_on_304_enabled_ && response_code == 304 &&
(entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept() ||
entry.IsMaster())) {
// If response code is 304, then we must have issued a conditional request,
// which means that we must have an existing entry and on that we must
// have a response id.
DCHECK(entry_fetcher->existing_entry().has_response_id());
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
auto cache_copier = std::make_unique<CacheCopier>(this, url, manifest_url_,
std::move(entry_fetcher),
CreateResponseWriter());
CacheCopier* cache_copier_ptr = cache_copier.get();
cache_copier_by_url_.emplace(url, std::move(cache_copier));
cache_copier_ptr->Run();
// Async continues in |ContinueHandleResourceFetchCompleted|.
return;
}
NotifyAllProgress(url);
++url_fetches_completed_;
if (response_code / 100 == 2) {
// Associate storage with the new entry.
DCHECK(entry_fetcher->response_writer());
entry.set_response_id(entry_fetcher->response_writer()->response_id());
entry.SetResponseAndPaddingSizes(
entry_fetcher->response_writer()->amount_written(),
ComputeAppCacheResponsePadding(url, manifest_url_));
entry.set_token_expires(storage_->GetOriginTrialExpiration(
url, entry_fetcher->request()->GetResponseHeaders(),
base::Time::Now()));
if (!inprogress_cache_->AddOrModifyEntry(url, entry))
duplicate_response_ids_.push_back(entry.response_id());
// TODO(michaeln): Check for <html manifest=xxx>
// See http://code.google.com/p/chromium/issues/detail?id=97930
// if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
// if (!manifestAttribute) skip it
// Foreign entries will be detected during cache selection.
// Note: 7.9.4, step 17.9 possible optimization: if resource is HTML or XML
// file whose root element is an html element with a manifest attribute
// whose value doesn't match the manifest url of the application cache
// being processed, mark the entry as being foreign.
} else if (!update_resource_on_304_enabled_ &&
(entry.IsExplicit() || entry.IsFallback() ||
entry.IsIntercept()) &&
response_code == 304 &&
entry_fetcher->existing_entry().has_response_id()) {
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
// Keep the existing response.
entry.set_response_id(entry_fetcher->existing_entry().response_id());
entry.SetResponseAndPaddingSizes(
entry_fetcher->existing_entry().response_size(),
entry_fetcher->existing_entry().padding_size());
inprogress_cache_->AddOrModifyEntry(url, entry);
} else if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
const char kFormatString[] = "Resource fetch failed (%d) %s";
std::string message = FormatUrlErrorMessage(
kFormatString, url, entry_fetcher->result(), response_code);
ResultType result = entry_fetcher->result();
bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin();
switch (result) {
case DISKCACHE_ERROR:
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR,
GURL(), 0, is_cross_origin),
result, url);
break;
case NETWORK_ERROR:
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_RESOURCE_ERROR, url,
0, is_cross_origin),
result, url);
break;
default:
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_RESOURCE_ERROR, url,
response_code, is_cross_origin),
result, url);
break;
}
return;
} else if (response_code == 404 || response_code == 410) {
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
// Entry is skipped. They are dropped from the cache.
} else if (update_type_ == UPGRADE_ATTEMPT &&
entry_fetcher->existing_entry().has_response_id()) {
if (update_resource_on_304_enabled_) {
// We check above for response code 304 for the following entry types, so
// if we end up here with a 304 response code, ensure that it's for an
// entry of a different type.
DCHECK_NE(response_code == 304,
entry.IsExplicit() || entry.IsFallback() ||
entry.IsIntercept() || entry.IsMaster());
}
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
// Keep the existing response.
// TODO(michaeln): Not sure this is a good idea. This is spec compliant
// but the old resource may or may not be compatible with the new contents
// of the cache. Impossible to know one way or the other.
entry.set_response_id(entry_fetcher->existing_entry().response_id());
entry.SetResponseAndPaddingSizes(
entry_fetcher->existing_entry().response_size(),
entry_fetcher->existing_entry().padding_size());
entry.set_token_expires(storage_->GetOriginTrialExpiration(
url, entry_fetcher->request()->GetResponseHeaders(),
base::Time::Now()));
inprogress_cache_->AddOrModifyEntry(url, entry);
}
// Fetch another URL now that one request has completed.
DCHECK(internal_state_ != AppCacheUpdateJobState::CACHE_FAILURE);
FetchUrls();
MaybeCompleteUpdate();
}
void AppCacheUpdateJob::ContinueHandleResourceFetchCompleted(
const GURL& url,
URLFetcher* entry_fetcher) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::DOWNLOADING);
auto it = cache_copier_by_url_.find(url);
DCHECK(it != cache_copier_by_url_.end());
std::unique_ptr<CacheCopier> cache_copier = std::move(it->second);
DCHECK_EQ(entry_fetcher->fetch_type(), URLFetcher::FetchType::kResource);
DCHECK_EQ(entry_fetcher->request()->GetResponseCode(), 304);
DCHECK_EQ(entry_fetcher->request()->GetURL(), url);
AppCacheEntry& entry = url_file_list_.find(url)->second;
DCHECK(!entry.has_response_id());
entry.set_response_id(cache_copier->response_writer()->response_id());
entry.SetResponseAndPaddingSizes(
cache_copier->response_writer()->amount_written(),
ComputeAppCacheResponsePadding(url, manifest_url_));
entry.set_token_expires(storage_->GetOriginTrialExpiration(
url, entry_fetcher->request()->GetResponseHeaders(), base::Time::Now()));
inprogress_cache_->AddOrModifyEntry(url, entry);
cache_copier.reset();
cache_copier_by_url_.erase(url);
NotifyAllProgress(url);
++url_fetches_completed_;
// Fetch another URL now that one request has completed.
FetchUrls();
MaybeCompleteUpdate();
}
void AppCacheUpdateJob::HandleNewMasterEntryFetchCompleted(
URLFetcher* url_fetcher,
int net_error) {
DCHECK(internal_state_ == AppCacheUpdateJobState::NO_UPDATE ||
internal_state_ == AppCacheUpdateJobState::DOWNLOADING);
// TODO(jennb): Handle downloads completing during cache failure when update
// no longer fetches master entries directly. For now, we cancel all pending
// master entry fetches when entering cache failure state so this will never
// be called in CACHE_FAILURE state.
UpdateURLLoaderRequest* request = url_fetcher->request();
const GURL& url = request->GetURL();
auto it = master_entry_fetches_.find(url);
if (it == master_entry_fetches_.end()) {
NOTREACHED() << "Entry URL not found in master_entry_fetches_";
return;
}
DCHECK_EQ(it->second.get(), url_fetcher);
std::unique_ptr<URLFetcher> entry_fetcher = std::move(it->second);
master_entry_fetches_.erase(it);
// URLFetcher triggers this function, so we verify here that the fetch type
// in URLFetcher is what we expect: kNewMasterEntry.
DCHECK_EQ(entry_fetcher->fetch_type(),
URLFetcher::FetchType::kNewMasterEntry);
++master_entries_completed_;
int response_code = net_error == net::OK ? request->GetResponseCode() : -1;
auto found = pending_master_entries_.find(url);
DCHECK(found != pending_master_entries_.end());
std::vector<AppCacheHost*>& hosts = found->second;
// Section 7.9.4. No update case: step 7.3, else step 22.
if (response_code / 100 == 2) {
// Add fetched master entry to the appropriate cache.
AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
: group_->newest_complete_cache();
DCHECK(entry_fetcher->response_writer());
// Master entries cannot be cross-origin by definition, so they do not
// require padding.
base::Time token_expires = storage_->GetOriginTrialExpiration(
url, entry_fetcher->request()->GetResponseHeaders(), base::Time::Now());
AppCacheEntry master_entry(
AppCacheEntry::MASTER, entry_fetcher->response_writer()->response_id(),
entry_fetcher->response_writer()->amount_written(),
/*padding_size=*/0, token_expires);
if (cache->AddOrModifyEntry(url, master_entry))
added_master_entries_.push_back(url);
else
duplicate_response_ids_.push_back(master_entry.response_id());
// In no-update case, associate host with the newest cache.
if (!inprogress_cache_.get()) {
// TODO(michaeln): defer until the updated cache has been stored
DCHECK_EQ(cache, group_->newest_complete_cache());
for (AppCacheHost* host : hosts)
host->AssociateCompleteCache(cache);
}
} else {
HostNotifier host_notifier;
for (AppCacheHost* host : hosts) {
host_notifier.AddHost(host);
// In downloading case, disassociate host from inprogress cache.
if (inprogress_cache_.get())
host->AssociateNoCache(GURL());
host->RemoveObserver(this);
}
hosts.clear();
failed_master_entries_.insert(url);
const char kFormatString[] = "Manifest fetch failed (%d) %s";
std::string message =
FormatUrlErrorMessage(kFormatString, request->GetURL(),
entry_fetcher->result(), response_code);
host_notifier.SendErrorNotifications(blink::mojom::AppCacheErrorDetails(
message, blink::mojom::AppCacheErrorReason::APPCACHE_MANIFEST_ERROR,
request->GetURL(), response_code, false /*is_cross_origin*/));
// In downloading case, update result is different if all master entries
// failed vs. only some failing.
if (inprogress_cache_.get()) {
// Only count successful downloads to know if all master entries failed.
pending_master_entries_.erase(found);
--master_entries_completed_;
// Section 7.9.4, step 22.3.
if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_MANIFEST_ERROR,
request->GetURL(), response_code, false /*is_cross_origin*/),
entry_fetcher->result(), GURL());
return;
}
}
}
DCHECK(internal_state_ != AppCacheUpdateJobState::CACHE_FAILURE);
FetchMasterEntries();
MaybeCompleteUpdate();
}
void AppCacheUpdateJob::HandleManifestRefetchCompleted(URLFetcher* url_fetcher,
int net_error) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::REFETCH_MANIFEST);
DCHECK_EQ(manifest_fetcher_.get(), url_fetcher);
std::unique_ptr<URLFetcher> manifest_fetcher = std::move(manifest_fetcher_);
UpdateURLLoaderRequest* request = manifest_fetcher->request();
int response_code = -1;
std::string optional_manifest_scope;
if (net_error == net::OK) {
response_code = request->GetResponseCode();
optional_manifest_scope = request->GetAppCacheAllowedHeader();
}
refetched_manifest_scope_ =
AppCache::GetManifestScope(manifest_url_, optional_manifest_scope);
if ((response_code == 304 &&
fetched_manifest_scope_ == refetched_manifest_scope_) ||
(manifest_data_ == manifest_fetcher->manifest_data())) {
// Only need to store response in storage if manifest is not already an
// entry in the cache.
AppCacheEntry* entry = nullptr;
if (inprogress_cache_)
entry = inprogress_cache_->GetEntry(manifest_url_);
if (entry) {
entry->add_types(AppCacheEntry::MANIFEST);
StoreGroupAndCache();
} else {
manifest_response_writer_ = CreateResponseWriter();
scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
base::MakeRefCounted<HttpResponseInfoIOBuffer>(
std::move(manifest_response_info_));
base::Time token_expires = storage_->GetOriginTrialExpiration(
manifest_url_, manifest_fetcher->request()->GetResponseHeaders(),
base::Time::Now());
manifest_response_writer_->WriteInfo(
io_buffer.get(),
base::BindOnce(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
base::Unretained(this), token_expires));
}
} else {
VLOG(1) << "Request error: " << net_error
<< " response code: " << response_code;
ScheduleUpdateRetry(kRerunDelayMs);
if (response_code == 200) {
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Manifest changed during update",
blink::mojom::AppCacheErrorReason::APPCACHE_CHANGED_ERROR, GURL(),
0, false /*is_cross_origin*/),
MANIFEST_ERROR, GURL());
} else {
const char kFormatString[] = "Manifest re-fetch failed (%d) %s";
std::string message =
FormatUrlErrorMessage(kFormatString, manifest_url_,
manifest_fetcher->result(), response_code);
ResultType result = manifest_fetcher->result();
if (result == UPDATE_OK) {
// URLFetcher considers any 2xx response a success, however in this
// particular case we want to treat any non 200 responses as failures.
result = SERVER_ERROR;
}
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
message,
blink::mojom::AppCacheErrorReason::APPCACHE_MANIFEST_ERROR,
GURL(), response_code, false /*is_cross_origin*/),
result, GURL());
}
}
}
void AppCacheUpdateJob::OnManifestInfoWriteComplete(base::Time token_expires,
int result) {
if (result > 0) {
scoped_refptr<net::StringIOBuffer> io_buffer =
base::MakeRefCounted<net::StringIOBuffer>(manifest_data_);
manifest_response_writer_->WriteData(
io_buffer.get(), manifest_data_.length(),
base::BindOnce(&AppCacheUpdateJob::OnManifestDataWriteComplete,
base::Unretained(this), token_expires));
} else {
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Failed to write the manifest headers to storage",
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR, GURL(),
0, false /*is_cross_origin*/),
DISKCACHE_ERROR, GURL());
}
}
void AppCacheUpdateJob::OnManifestDataWriteComplete(base::Time token_expires,
int result) {
if (result > 0) {
// The manifest determines the cache's origin, so the manifest entry is
// always same-origin, and thus does not require padding.
AppCacheEntry entry(AppCacheEntry::MANIFEST,
manifest_response_writer_->response_id(),
manifest_response_writer_->amount_written(),
/*padding_size=*/0, token_expires);
if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
duplicate_response_ids_.push_back(entry.response_id());
// Although the manifest url entry has the right token_expires,
// push this to the cache as well.
// TODO(pwnall): Figure out if we want to overwrite or max the two entries.
inprogress_cache_->set_token_expires(
std::max(token_expires, inprogress_cache_->token_expires()));
StoreGroupAndCache();
} else {
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Failed to write the manifest data to storage",
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR, GURL(),
0, false /*is_cross_origin*/),
DISKCACHE_ERROR, GURL());
}
}
void AppCacheUpdateJob::StoreGroupAndCache() {
DCHECK_EQ(stored_state_, UNSTORED);
stored_state_ = STORING;
scoped_refptr<AppCache> newest_cache;
if (inprogress_cache_.get())
newest_cache.swap(inprogress_cache_);
else
newest_cache = group_->newest_complete_cache();
newest_cache->set_update_time(base::Time::Now());
// Verify that cache contains the associated manifest parser version and
// scope values.
DCHECK_EQ(fetched_manifest_parser_version_,
newest_cache->manifest_parser_version());
DCHECK_EQ(fetched_manifest_scope_, newest_cache->manifest_scope());
// Verify fetched manifest parser version and scope:
// 1. Values must be initialized and valid:
// - For parser version, the version must not be -1.
// - For scope, the the value must not be the empty string.
DCHECK_NE(fetched_manifest_parser_version_, -1);
DCHECK_NE(fetched_manifest_scope_, "");
// 2. Check that the UpdateJob value state is correct:
// - For parser version, the newly fetched parser version must be greater
// than or equal to the version we began with.
// - For scope, the fetched manifest scope must be valid.
DCHECK_GE(fetched_manifest_parser_version_, cached_manifest_parser_version_);
DCHECK_EQ(fetched_manifest_scope_, refetched_manifest_scope_);
DCHECK(AppCache::CheckValidManifestScope(manifest_url_,
fetched_manifest_scope_));
group_->set_first_evictable_error_time(base::Time());
if (doing_full_update_check_)
group_->set_last_full_update_check_time(base::Time::Now());
// TODO(pwnall): Figure out if we want to overwrite or max the two entries.
group_->set_token_expires(
std::max(group_->token_expires(), newest_cache->token_expires()));
storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
}
void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
AppCache* newest_cache,
bool success,
bool would_exceed_quota) {
DCHECK_EQ(stored_state_, STORING);
if (success) {
stored_state_ = STORED;
MaybeCompleteUpdate(); // will definitely complete
return;
}
stored_state_ = UNSTORED;
// Restore inprogress_cache_ to get the proper events delivered
// and the proper cleanup to occur.
if (newest_cache != group->newest_complete_cache())
inprogress_cache_ = newest_cache;
ResultType result = DB_ERROR;
blink::mojom::AppCacheErrorReason reason =
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR;
std::string message("Failed to commit new cache to storage");
if (would_exceed_quota) {
message.append(", would exceed quota");
result = QUOTA_ERROR;
reason = blink::mojom::AppCacheErrorReason::APPCACHE_QUOTA_ERROR;
}
HandleCacheFailure(blink::mojom::AppCacheErrorDetails(
message, reason, GURL(), 0, false /*is_cross_origin*/),
result, GURL());
}
void AppCacheUpdateJob::NotifySingleHost(
AppCacheHost* host,
blink::mojom::AppCacheEventID event_id) {
host->frontend()->EventRaised(event_id);
}
void AppCacheUpdateJob::NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID event_id) {
HostNotifier host_notifier;
AddAllAssociatedHostsToNotifier(&host_notifier);
host_notifier.SendNotifications(event_id);
}
void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
HostNotifier host_notifier;
AddAllAssociatedHostsToNotifier(&host_notifier);
host_notifier.SendProgressNotifications(
url, url_file_list_.size(), url_fetches_completed_);
}
void AppCacheUpdateJob::NotifyAllFinalProgress() {
DCHECK_EQ(url_file_list_.size(), url_fetches_completed_);
NotifyAllProgress(GURL());
}
void AppCacheUpdateJob::NotifyAllError(
const blink::mojom::AppCacheErrorDetails& details) {
HostNotifier host_notifier;
AddAllAssociatedHostsToNotifier(&host_notifier);
host_notifier.SendErrorNotifications(details);
}
void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) {
HostNotifier host_notifier;
AddAllAssociatedHostsToNotifier(&host_notifier);
host_notifier.SendLogMessage(message);
}
void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
HostNotifier* host_notifier) {
// Collect hosts so we only send one notification per frontend.
// A host can only be associated with a single cache so no need to worry
// about duplicate hosts being added to the notifier.
if (inprogress_cache_.get()) {
DCHECK(internal_state_ == AppCacheUpdateJobState::DOWNLOADING ||
internal_state_ == AppCacheUpdateJobState::CACHE_FAILURE);
host_notifier->AddHosts(inprogress_cache_->associated_hosts());
}
for (AppCache* cache : group_->old_caches())
host_notifier->AddHosts(cache->associated_hosts());
AppCache* newest_cache = group_->newest_complete_cache();
if (newest_cache)
host_notifier->AddHosts(newest_cache->associated_hosts());
}
void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
// The host is about to be deleted; remove from our collection.
auto found = pending_master_entries_.find(host->pending_master_entry_url());
CHECK(found != pending_master_entries_.end());
std::vector<AppCacheHost*>& hosts = found->second;
auto it = std::find(hosts.begin(), hosts.end(), host);
CHECK(it != hosts.end());
hosts.erase(it);
}
void AppCacheUpdateJob::OnServiceReinitialized(
AppCacheStorageReference* old_storage_ref) {
// We continue to use the disabled instance, but arrange for its
// deletion when its no longer needed.
if (old_storage_ref->storage() == storage_)
disabled_storage_reference_ = old_storage_ref;
}
void AppCacheUpdateJob::CheckIfManifestChanged(base::Time token_expires) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
DCHECK_EQ(update_type_, UPGRADE_ATTEMPT);
AppCacheEntry* entry = nullptr;
if (group_->newest_complete_cache())
entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
if (!entry) {
// TODO(pwnall): Old documentation said this avoided the crash at
// https://crbug.com/95101. A removed histogram shows that
// this path is hit very rarely.
if (service_->storage() == storage_) {
// Use a local variable because service_ is reset in HandleCacheFailure.
AppCacheServiceImpl* service = service_;
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Manifest entry not found in existing cache",
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR, GURL(),
0, false /*is_cross_origin*/),
DB_ERROR, GURL());
service->DeleteAppCacheGroup(manifest_url_,
net::CompletionOnceCallback());
}
return;
}
if (fetched_manifest_scope_ != cached_manifest_scope_) {
HandleFetchedManifestChanged(token_expires);
return;
}
if (cached_manifest_parser_version_ < 1) {
HandleFetchedManifestChanged(token_expires);
return;
}
// Load manifest data from storage to compare against fetched manifest.
manifest_response_reader_ =
storage_->CreateResponseReader(manifest_url_, entry->response_id());
read_manifest_buffer_ =
base::MakeRefCounted<net::IOBuffer>(kAppCacheFetchBufferSize);
manifest_response_reader_->ReadData(
read_manifest_buffer_.get(), kAppCacheFetchBufferSize,
base::BindOnce(&AppCacheUpdateJob::OnManifestDataReadComplete,
base::Unretained(this), token_expires)); // async read
}
void AppCacheUpdateJob::OnManifestDataReadComplete(base::Time token_expires,
int result) {
DCHECK_GE(cached_manifest_parser_version_, 1);
DCHECK_EQ(fetched_manifest_scope_, cached_manifest_scope_);
if (result > 0) {
loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
manifest_response_reader_->ReadData(
read_manifest_buffer_.get(), kAppCacheFetchBufferSize,
base::BindOnce(&AppCacheUpdateJob::OnManifestDataReadComplete,
base::Unretained(this), token_expires)); // read more
} else {
read_manifest_buffer_ = nullptr;
manifest_response_reader_.reset();
if (result < 0 || manifest_data_ != loaded_manifest_data_ ||
read_manifest_token_expires_ != token_expires) {
HandleFetchedManifestChanged(token_expires);
} else {
HandleFetchedManifestIsUnchanged();
}
}
}
void AppCacheUpdateJob::ReadManifestFromCacheAndContinue() {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
DCHECK_EQ(update_type_, UPGRADE_ATTEMPT);
DCHECK_NE(fetched_manifest_scope_, cached_manifest_scope_);
// |manifest_response_info_| should have been saved in OnResponseInfoLoaded(),
// we'll reuse it later in ContinueHandleManifestFetchCompleted() so make sure
// it's still there.
DCHECK(manifest_response_info_.get());
AppCacheEntry* entry = nullptr;
if (group_->newest_complete_cache())
entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
if (!entry) {
// TODO(pwnall): Old documentation said this avoided the crash at
// https://crbug.com/95101. A removed histogram shows that
// this path is hit very rarely.
if (service_->storage() == storage_) {
// Use a local variable because service_ is reset in HandleCacheFailure.
AppCacheServiceImpl* service = service_;
HandleCacheFailure(
blink::mojom::AppCacheErrorDetails(
"Manifest entry not found in existing cache",
blink::mojom::AppCacheErrorReason::APPCACHE_UNKNOWN_ERROR, GURL(),
0, false /*is_cross_origin*/),
DB_ERROR, GURL());
service->DeleteAppCacheGroup(manifest_url_,
net::CompletionOnceCallback());
}
return;
}
// Load manifest data from storage so we can continue parsing using the new
// scope.
manifest_response_reader_ =
storage_->CreateResponseReader(manifest_url_, entry->response_id());
read_manifest_buffer_ =
base::MakeRefCounted<net::IOBuffer>(kAppCacheFetchBufferSize);
manifest_response_reader_->ReadData(
read_manifest_buffer_.get(), kAppCacheFetchBufferSize,
base::BindOnce(&AppCacheUpdateJob::OnManifestFromCacheDataReadComplete,
base::Unretained(this))); // async read
read_manifest_token_expires_ = entry->token_expires();
}
void AppCacheUpdateJob::OnManifestFromCacheDataReadComplete(int result) {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::FETCH_MANIFEST);
DCHECK_EQ(update_type_, UPGRADE_ATTEMPT);
if (result > 0) {
loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
manifest_response_reader_->ReadData(
read_manifest_buffer_.get(), kAppCacheFetchBufferSize,
base::BindOnce(&AppCacheUpdateJob::OnManifestFromCacheDataReadComplete,
base::Unretained(this))); // read more
} else {
manifest_data_ = loaded_manifest_data_;
read_manifest_buffer_ = nullptr;
manifest_response_reader_.reset();
// This path is only hit via a 304 response, where we are not expected
// to update the token expires, so pretend it hasn't been set.
base::Time token_expires;
HandleFetchedManifestChanged(token_expires);
}
}
void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) {
for (const std::string& explicit_url : manifest.explicit_urls)
AddUrlToFileList(GURL(explicit_url), AppCacheEntry::EXPLICIT);
for (const auto& intercept : manifest.intercept_namespaces)
AddUrlToFileList(intercept.target_url, AppCacheEntry::INTERCEPT);
for (const auto& fallback : manifest.fallback_namespaces)
AddUrlToFileList(fallback.target_url, AppCacheEntry::FALLBACK);
// Add all master entries from newest complete cache.
if (update_type_ == UPGRADE_ATTEMPT) {
for (const auto& pair : group_->newest_complete_cache()->entries()) {
const AppCacheEntry& entry = pair.second;
if (entry.IsMaster())
AddUrlToFileList(pair.first, AppCacheEntry::MASTER);
}
}
}
void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
auto emplace_result = url_file_list_.emplace(url, AppCacheEntry(type));
if (emplace_result.second) {
urls_to_fetch_.emplace_back(url, false, nullptr);
} else {
// URL already exists. Merge types.
emplace_result.first->second.add_types(type);
}
}
void AppCacheUpdateJob::FetchUrls() {
DCHECK_EQ(internal_state_, AppCacheUpdateJobState::DOWNLOADING);
// Fetch each URL in the list according to section 7.9.4 step 18.1-18.3.
// Fetch up to the concurrent limit. Other fetches will be triggered as each
// each fetch completes.
while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
!urls_to_fetch_.empty()) {
UrlToFetch url_to_fetch = urls_to_fetch_.front();
urls_to_fetch_.pop_front();
auto it = url_file_list_.find(url_to_fetch.url);
DCHECK(it != url_file_list_.end());
AppCacheEntry& entry = it->second;
if (ShouldSkipUrlFetch(entry)) {
NotifyAllProgress(url_to_fetch.url);
++url_fetches_completed_;
} else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
NotifyAllProgress(url_to_fetch.url);
++url_fetches_completed_; // saved a URL request
} else if (!url_to_fetch.storage_checked &&
MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
// Continues asynchronously after data is loaded from newest cache.
} else {
auto fetcher = std::make_unique<URLFetcher>(
url_to_fetch.url, URLFetcher::FetchType::kResource, this,
kAppCacheFetchBufferSize);
if (url_to_fetch.existing_response_info.get() &&
group_->newest_complete_cache()) {
AppCacheEntry* existing_entry =
group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
DCHECK(existing_entry);
DCHECK_EQ(existing_entry->response_id(),
url_to_fetch.existing_response_info->response_id());
fetcher->set_existing_response_headers(
url_to_fetch.existing_response_info->http_response_info()
.headers.get());
fetcher->set_existing_entry(*existing_entry);
}
fetcher->Start();
pending_url_fetches_.emplace(url_to_fetch.url, std::move(fetcher));
}
}
}
void AppCacheUpdateJob::CancelAllUrlFetches() {
// Cancel any pending URL requests.
url_fetches_completed_ +=
pending_url_fetches_.size() + urls_to_fetch_.size();
pending_url_fetches_.clear();
urls_to_fetch_.clear();
}
bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
// 6.6.4 Step 17
// If the resource URL being processed was flagged as neither an
// "explicit entry" nor or a "fallback entry", then the user agent
// may skip this URL.
if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
return false;
// TODO(jennb): decide if entry should be skipped to expire it from cache
return false;
}
bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
int entry_type) {
DCHECK(internal_state_ == AppCacheUpdateJobState::DOWNLOADING ||
internal_state_ == AppCacheUpdateJobState::NO_UPDATE);
AppCacheEntry* existing =
inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
: group_->newest_complete_cache()->GetEntry(url);
if (existing) {
existing->add_types(entry_type);
return true;
}
return false;
}
void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
const GURL& url,
bool is_new) {
DCHECK(!IsTerminating());
if (internal_state_ == AppCacheUpdateJobState::DOWNLOADING ||
internal_state_ == AppCacheUpdateJobState::NO_UPDATE) {
AppCache* cache;
if (inprogress_cache_.get()) {
// always associate
host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
cache = inprogress_cache_.get();
} else {
cache = group_->newest_complete_cache();
}
// Update existing entry if it has already been fetched.
AppCacheEntry* entry = cache->GetEntry(url);
if (entry) {
entry->add_types(AppCacheEntry::MASTER);
if (internal_state_ == AppCacheUpdateJobState::NO_UPDATE &&
!inprogress_cache_.get()) {
// only associate if have entry
host->AssociateCompleteCache(cache);
}
if (is_new)
++master_entries_completed_; // pretend fetching completed
return;
}
}
// Add to fetch list if not already fetching.
if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
master_entries_to_fetch_.insert(url);
if (internal_state_ == AppCacheUpdateJobState::DOWNLOADING ||
internal_state_ == AppCacheUpdateJobState::NO_UPDATE)
FetchMasterEntries();
}
}
void AppCacheUpdateJob::FetchMasterEntries() {
DCHECK(internal_state_ == AppCacheUpdateJobState::NO_UPDATE ||
internal_state_ == AppCacheUpdateJobState::DOWNLOADING);
// Fetch each master entry in the list, up to the concurrent limit.
// Additional fetches will be triggered as each fetch completes.
while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
!master_entries_to_fetch_.empty()) {
const GURL& url = *master_entries_to_fetch_.begin();
if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
++master_entries_completed_; // saved a URL request
// In no update case, associate hosts to newest cache in group
// now that master entry has been "successfully downloaded".
if (internal_state_ == AppCacheUpdateJobState::NO_UPDATE) {
// TODO(michaeln): defer until the updated cache has been stored.
DCHECK(!inprogress_cache_.get());
AppCache* cache = group_->newest_complete_cache();
auto found = pending_master_entries_.find(url);
DCHECK(found != pending_master_entries_.end());
std::vector<AppCacheHost*>& hosts = found->second;
for (AppCacheHost* host : hosts)
host->AssociateCompleteCache(cache);
}
} else {
auto fetcher = std::make_unique<URLFetcher>(
url, URLFetcher::FetchType::kNewMasterEntry, this,
kAppCacheFetchBufferSize);
fetcher->Start();
master_entry_fetches_.emplace(url, std::move(fetcher));
}
master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
}
}
void AppCacheUpdateJob::CancelAllMasterEntryFetches(
const blink::mojom::AppCacheErrorDetails& error_details) {
// For now, cancel all in-progress fetches for master entries and pretend
// all master entries fetches have completed.
// TODO(jennb): Delete this when update no longer fetches master entries
// directly.
// Cancel all in-progress fetches.
for (auto& pair : master_entry_fetches_) {
// Move URLs back to the unfetched list.
master_entries_to_fetch_.emplace(std::move(pair.first));
}
master_entry_fetches_.clear();
master_entries_completed_ += master_entries_to_fetch_.size();
// Cache failure steps, step 2.
// Pretend all master entries that have not yet been fetched have completed
// downloading. Unassociate hosts from any appcache and send ERROR event.
HostNotifier host_notifier;
while (!master_entries_to_fetch_.empty()) {
const GURL& url = *master_entries_to_fetch_.begin();
auto found = pending_master_entries_.find(url);
DCHECK(found != pending_master_entries_.end());
std::vector<AppCacheHost*>& hosts = found->second;
for (AppCacheHost* host : hosts) {
host->AssociateNoCache(GURL());
host_notifier.AddHost(host);
host->RemoveObserver(this);
}
hosts.clear();
master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
}
host_notifier.SendErrorNotifications(error_details);
}
bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
AppCacheEntry& entry) {
if (update_type_ != UPGRADE_ATTEMPT)
return false;
AppCache* newest = group_->newest_complete_cache();
AppCacheEntry* copy_me = newest->GetEntry(url);
if (!copy_me || !copy_me->has_response_id())
return false;
// Load HTTP headers for entry from newest cache.
loading_responses_.emplace(copy_me->response_id(), url);
storage_->LoadResponseInfo(manifest_url_, copy_me->response_id(), this);
// Async: wait for OnResponseInfoLoaded to complete.
return true;
}
void AppCacheUpdateJob::OnResponseInfoLoaded(
AppCacheResponseInfo* response_info,
int64_t response_id) {
const net::HttpResponseInfo* http_info =
response_info ? &response_info->http_response_info() : nullptr;
// Needed response info for a manifest fetch request.
if (internal_state_ == AppCacheUpdateJobState::FETCH_MANIFEST) {
if (http_info) {
// Save a copy of the HttpResponseInfo in case we need it later. We would
// use it if we attach conditional headers and the server replies with a
// 304. In that case, we would use these same headers again to refetch the
// manifest. In the case that the server replies with 200 OK, this
// manifest_response_info_ will be overwritten with that response's
// HttpResponseInfo and since it's a unique_ptr this HttpResponseInfo will
// be deleted.
manifest_response_info_ =
std::make_unique<net::HttpResponseInfo>(*http_info);
if (cached_manifest_parser_version_ >= 1) {
manifest_fetcher_->set_existing_response_headers(
http_info->headers.get());
}
}
manifest_fetcher_->Start();
return;
}
auto found = loading_responses_.find(response_id);
DCHECK(found != loading_responses_.end());
const GURL& url = found->second;
if (!http_info) {
LoadFromNewestCacheFailed(url, nullptr); // no response found
} else {
ResourceCheck result = CanUseExistingResource(http_info, update_metrics_);
if (result == ResourceCheck::kInvalid) {
// An invalid resource was found, but we may want to add conditional
// headers that could result in a 304 NOT MODIFIED response.
LoadFromNewestCacheFailed(url, response_info);
} else {
DCHECK(result == ResourceCheck::kValid);
DCHECK(group_->newest_complete_cache());
AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
DCHECK(copy_me);
DCHECK_EQ(copy_me->response_id(), response_id);
auto it = url_file_list_.find(url);
DCHECK(it != url_file_list_.end());
AppCacheEntry& entry = it->second;
entry.set_response_id(response_id);
entry.SetResponseAndPaddingSizes(copy_me->response_size(),
copy_me->padding_size());
entry.set_token_expires(copy_me->token_expires());
inprogress_cache_->AddOrModifyEntry(url, entry);
NotifyAllProgress(url);
++url_fetches_completed_;
}
}
loading_responses_.erase(found);
MaybeCompleteUpdate();
}
void AppCacheUpdateJob::LoadFromNewestCacheFailed(
const GURL& url, AppCacheResponseInfo* response_info) {
if (internal_state_ == AppCacheUpdateJobState::CACHE_FAILURE)
return;
// Re-insert url at front of fetch list. Indicate storage has been checked.
urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
FetchUrls();
}
void AppCacheUpdateJob::MaybeCompleteUpdate() {
DCHECK(internal_state_ != AppCacheUpdateJobState::CACHE_FAILURE);
// Must wait for any pending master entries or url fetches to complete.
if (master_entries_completed_ != pending_master_entries_.size() ||
url_fetches_completed_ != url_file_list_.size()) {
DCHECK(internal_state_ != AppCacheUpdateJobState::COMPLETED);
return;
}
switch (internal_state_) {
case AppCacheUpdateJobState::NO_UPDATE:
if (master_entries_completed_ > 0) {
switch (stored_state_) {
case UNSTORED:
StoreGroupAndCache();
return;
case STORING:
return;
case STORED:
break;
}
} else {
bool times_changed = false;
if (!group_->first_evictable_error_time().is_null()) {
group_->set_first_evictable_error_time(base::Time());
times_changed = true;
}
if (doing_full_update_check_) {
group_->set_last_full_update_check_time(base::Time::Now());
times_changed = true;
}
if (times_changed)
storage_->StoreEvictionTimes(group_);
}
group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
// 7.9.4 steps 7.3-7.7.
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_NO_UPDATE_EVENT);
DiscardDuplicateResponses();
internal_state_ = AppCacheUpdateJobState::COMPLETED;
break;
case AppCacheUpdateJobState::DOWNLOADING:
internal_state_ = AppCacheUpdateJobState::REFETCH_MANIFEST;
RefetchManifest();
break;
case AppCacheUpdateJobState::REFETCH_MANIFEST:
DCHECK_EQ(stored_state_, STORED);
NotifyAllFinalProgress();
group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
if (update_type_ == CACHE_ATTEMPT)
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_CACHED_EVENT);
else
NotifyAllAssociatedHosts(
blink::mojom::AppCacheEventID::APPCACHE_UPDATE_READY_EVENT);
DiscardDuplicateResponses();
internal_state_ = AppCacheUpdateJobState::COMPLETED;
break;
case AppCacheUpdateJobState::CACHE_FAILURE:
NOTREACHED(); // See HandleCacheFailure
break;
default:
break;
}
// Let the stack unwind before deletion to make it less risky as this
// method is called from multiple places in this file.
if (internal_state_ == AppCacheUpdateJobState::COMPLETED)
DeleteSoon();
}
void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
// TODO(jennb): post a delayed task with the "same parameters" as this job
// to retry the update at a later time. Need group, URLs of pending master
// entries and their hosts.
}
void AppCacheUpdateJob::Cancel() {
update_metrics_.RecordCanceled();
internal_state_ = AppCacheUpdateJobState::CANCELLED;
manifest_fetcher_.reset();
pending_url_fetches_.clear();
master_entry_fetches_.clear();
ClearPendingMasterEntries();
DiscardInprogressCache();
// Delete response writer to avoid any callbacks.
if (manifest_response_writer_)
manifest_response_writer_.reset();
storage_->CancelDelegateCallbacks(this);
}
void AppCacheUpdateJob::ClearPendingMasterEntries() {
for (auto& pair : pending_master_entries_) {
std::vector<AppCacheHost*>& hosts = pair.second;
for (AppCacheHost* host : hosts)
host->RemoveObserver(this);
}
pending_master_entries_.clear();
}
void AppCacheUpdateJob::DiscardInprogressCache() {
if (stored_state_ == STORING) {
// We can make no assumptions about whether the StoreGroupAndCacheTask
// actually completed or not. This condition should only be reachable
// during shutdown. Free things up and return to do no harm.
inprogress_cache_ = nullptr;
added_master_entries_.clear();
return;
}
storage_->DoomResponses(manifest_url_, stored_response_ids_);
if (!inprogress_cache_.get()) {
// We have to undo the changes we made, if any, to the existing cache.
if (group_ && group_->newest_complete_cache()) {
for (auto& url : added_master_entries_)
group_->newest_complete_cache()->RemoveEntry(url);
}
added_master_entries_.clear();
return;
}
AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
while (!hosts.empty())
(*hosts.begin())->AssociateNoCache(GURL());
inprogress_cache_ = nullptr;
added_master_entries_.clear();
}
void AppCacheUpdateJob::DiscardDuplicateResponses() {
storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
}
void AppCacheUpdateJob::DeleteSoon() {
ClearPendingMasterEntries();
manifest_response_writer_.reset();
storage_->CancelDelegateCallbacks(this);
service_->RemoveObserver(this);
service_ = nullptr;
// Break the connection with the group so the group cannot call delete
// on this object after we've posted a task to delete ourselves.
if (group_) {
group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE);
group_ = nullptr;
}
base::ThreadTaskRunnerHandle::Get()->DeleteSoon(FROM_HERE, this);
}
bool AppCacheUpdateJob::IsFinished() const {
return (internal_state_ == AppCacheUpdateJobState::CACHE_FAILURE ||
internal_state_ == AppCacheUpdateJobState::CANCELLED ||
internal_state_ == AppCacheUpdateJobState::COMPLETED);
}
} // namespace content