blob: 5067deca2f88d2a646fd30782d1ec2978c73caa3 [file] [log] [blame]
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/browser/dom_storage/local_storage_context_mojo.h"
#include <inttypes.h>
#include <algorithm>
#include <cctype> // for std::isalnum
#include <set>
#include <string>
#include <utility>
#include "base/barrier_closure.h"
#include "base/bind.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/single_thread_task_runner.h"
#include "base/stl_util.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/stringprintf.h"
#include "base/system/sys_info.h"
#include "base/trace_event/memory_dump_manager.h"
#include "build/build_config.h"
#include "components/services/leveldb/public/cpp/util.h"
#include "components/services/leveldb/public/interfaces/leveldb.mojom.h"
#include "content/browser/dom_storage/dom_storage_area.h"
#include "content/browser/dom_storage/dom_storage_database.h"
#include "content/browser/dom_storage/dom_storage_task_runner.h"
#include "content/browser/dom_storage/local_storage_database.pb.h"
#include "content/browser/dom_storage/storage_area_impl.h"
#include "content/common/dom_storage/dom_storage_types.h"
#include "content/public/browser/storage_usage_info.h"
#include "services/file/public/mojom/constants.mojom.h"
#include "services/service_manager/public/cpp/connector.h"
#include "sql/database.h"
#include "storage/browser/quota/special_storage_policy.h"
#include "third_party/leveldatabase/env_chromium.h"
#include "third_party/leveldatabase/leveldb_chrome.h"
namespace content {
// LevelDB database schema
// =======================
//
// Version 1 (in sorted order):
// key: "VERSION"
// value: "1"
//
// key: "META:" + <url::Origin 'origin'>
// value: <LocalStorageOriginMetaData serialized as a string>
//
// key: "_" + <url::Origin> 'origin'> + '\x00' + <script controlled key>
// value: <script controlled value>
namespace {
const char kVersionKey[] = "VERSION";
const char kOriginSeparator = '\x00';
const char kDataPrefix[] = "_";
const uint8_t kMetaPrefix[] = {'M', 'E', 'T', 'A', ':'};
const int64_t kMinSchemaVersion = 1;
const int64_t kCurrentLocalStorageSchemaVersion = 1;
// After this many consecutive commit errors we'll throw away the entire
// database.
const int kCommitErrorThreshold = 8;
// Limits on the cache size and number of areas in memory, over which the areas
// are purged.
#if defined(OS_ANDROID)
const unsigned kMaxLocalStorageAreaCount = 10;
const size_t kMaxLocalStorageCacheSize = 2 * 1024 * 1024;
#else
const unsigned kMaxLocalStorageAreaCount = 50;
const size_t kMaxLocalStorageCacheSize = 20 * 1024 * 1024;
#endif
static const uint8_t kUTF16Format = 0;
static const uint8_t kLatin1Format = 1;
std::vector<uint8_t> CreateMetaDataKey(const url::Origin& origin) {
auto serialized_origin = leveldb::StdStringToUint8Vector(origin.Serialize());
std::vector<uint8_t> key;
key.reserve(base::size(kMetaPrefix) + serialized_origin.size());
key.insert(key.end(), kMetaPrefix, kMetaPrefix + base::size(kMetaPrefix));
key.insert(key.end(), serialized_origin.begin(), serialized_origin.end());
return key;
}
void SuccessResponse(base::OnceClosure callback, bool success) {
std::move(callback).Run();
}
void DatabaseErrorResponse(base::OnceClosure callback,
leveldb::mojom::DatabaseError error) {
std::move(callback).Run();
}
void MigrateStorageHelper(
base::FilePath db_path,
const scoped_refptr<base::SingleThreadTaskRunner> reply_task_runner,
base::Callback<void(std::unique_ptr<StorageAreaImpl::ValueMap>)> callback) {
DOMStorageDatabase db(db_path);
DOMStorageValuesMap map;
db.ReadAllValues(&map);
auto values = std::make_unique<StorageAreaImpl::ValueMap>();
for (const auto& it : map) {
(*values)[LocalStorageContextMojo::MigrateString(it.first)] =
LocalStorageContextMojo::MigrateString(it.second.string());
}
reply_task_runner->PostTask(
FROM_HERE, base::BindOnce(std::move(callback), std::move(values)));
}
// Helper to convert from OnceCallback to Callback.
void CallMigrationCalback(StorageAreaImpl::ValueMapCallback callback,
std::unique_ptr<StorageAreaImpl::ValueMap> data) {
std::move(callback).Run(std::move(data));
}
void AddDeleteOriginOperations(
std::vector<leveldb::mojom::BatchedOperationPtr>* operations,
const url::Origin& origin) {
leveldb::mojom::BatchedOperationPtr item =
leveldb::mojom::BatchedOperation::New();
item->type = leveldb::mojom::BatchOperationType::DELETE_PREFIXED_KEY;
item->key = leveldb::StdStringToUint8Vector(kDataPrefix + origin.Serialize() +
kOriginSeparator);
operations->push_back(std::move(item));
item = leveldb::mojom::BatchedOperation::New();
item->type = leveldb::mojom::BatchOperationType::DELETE_KEY;
item->key = CreateMetaDataKey(origin);
operations->push_back(std::move(item));
}
enum class CachePurgeReason {
NotNeeded,
SizeLimitExceeded,
AreaCountLimitExceeded,
InactiveOnLowEndDevice,
AggressivePurgeTriggered
};
void RecordCachePurgedHistogram(CachePurgeReason reason,
size_t purged_size_kib) {
UMA_HISTOGRAM_COUNTS_100000("LocalStorageContext.CachePurgedInKB",
purged_size_kib);
switch (reason) {
case CachePurgeReason::SizeLimitExceeded:
UMA_HISTOGRAM_COUNTS_100000(
"LocalStorageContext.CachePurgedInKB.SizeLimitExceeded",
purged_size_kib);
break;
case CachePurgeReason::AreaCountLimitExceeded:
UMA_HISTOGRAM_COUNTS_100000(
"LocalStorageContext.CachePurgedInKB.AreaCountLimitExceeded",
purged_size_kib);
break;
case CachePurgeReason::InactiveOnLowEndDevice:
UMA_HISTOGRAM_COUNTS_100000(
"LocalStorageContext.CachePurgedInKB.InactiveOnLowEndDevice",
purged_size_kib);
break;
case CachePurgeReason::AggressivePurgeTriggered:
UMA_HISTOGRAM_COUNTS_100000(
"LocalStorageContext.CachePurgedInKB.AggressivePurgeTriggered",
purged_size_kib);
break;
case CachePurgeReason::NotNeeded:
NOTREACHED();
break;
}
}
} // namespace
class LocalStorageContextMojo::StorageAreaHolder final
: public StorageAreaImpl::Delegate {
public:
StorageAreaHolder(LocalStorageContextMojo* context, const url::Origin& origin)
: context_(context), origin_(origin) {
// Delay for a moment after a value is set in anticipation
// of other values being set, so changes are batched.
static constexpr base::TimeDelta kCommitDefaultDelaySecs =
base::TimeDelta::FromSeconds(5);
// To avoid excessive IO we apply limits to the amount of data being written
// and the frequency of writes.
static constexpr int kMaxBytesPerHour = kPerStorageAreaQuota;
static constexpr int kMaxCommitsPerHour = 60;
StorageAreaImpl::Options options;
options.max_size = kPerStorageAreaQuota + kPerStorageAreaOverQuotaAllowance;
options.default_commit_delay = kCommitDefaultDelaySecs;
options.max_bytes_per_hour = kMaxBytesPerHour;
options.max_commits_per_hour = kMaxCommitsPerHour;
#if defined(OS_ANDROID)
options.cache_mode = StorageAreaImpl::CacheMode::KEYS_ONLY_WHEN_POSSIBLE;
#else
options.cache_mode = StorageAreaImpl::CacheMode::KEYS_AND_VALUES;
if (base::SysInfo::IsLowEndDevice()) {
options.cache_mode = StorageAreaImpl::CacheMode::KEYS_ONLY_WHEN_POSSIBLE;
}
#endif
area_ = std::make_unique<StorageAreaImpl>(
context_->database_.get(),
kDataPrefix + origin_.Serialize() + kOriginSeparator, this, options);
area_ptr_ = area_.get();
}
StorageAreaImpl* storage_area() { return area_ptr_; }
void OnNoBindings() override {
has_bindings_ = false;
// Don't delete ourselves, but do schedule an immediate commit. Possible
// deletion will happen under memory pressure or when another localstorage
// area is opened.
storage_area()->ScheduleImmediateCommit();
}
std::vector<leveldb::mojom::BatchedOperationPtr> PrepareToCommit() override {
std::vector<leveldb::mojom::BatchedOperationPtr> operations;
// Write schema version if not already done so before.
if (!context_->database_initialized_) {
leveldb::mojom::BatchedOperationPtr item =
leveldb::mojom::BatchedOperation::New();
item->type = leveldb::mojom::BatchOperationType::PUT_KEY;
item->key = leveldb::StdStringToUint8Vector(kVersionKey);
item->value = leveldb::StdStringToUint8Vector(
base::NumberToString(kCurrentLocalStorageSchemaVersion));
operations.push_back(std::move(item));
context_->database_initialized_ = true;
}
leveldb::mojom::BatchedOperationPtr item =
leveldb::mojom::BatchedOperation::New();
item->type = leveldb::mojom::BatchOperationType::PUT_KEY;
item->key = CreateMetaDataKey(origin_);
if (storage_area()->empty()) {
item->type = leveldb::mojom::BatchOperationType::DELETE_KEY;
} else {
item->type = leveldb::mojom::BatchOperationType::PUT_KEY;
LocalStorageOriginMetaData data;
data.set_last_modified(base::Time::Now().ToInternalValue());
data.set_size_bytes(storage_area()->storage_used());
item->value = leveldb::StdStringToUint8Vector(data.SerializeAsString());
}
operations.push_back(std::move(item));
return operations;
}
void DidCommit(leveldb::mojom::DatabaseError error) override {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.CommitResult",
leveldb::GetLevelDBStatusUMAValue(error),
leveldb_env::LEVELDB_STATUS_MAX);
// Delete any old database that might still exist if we successfully wrote
// data to LevelDB, and our LevelDB is actually disk backed.
if (error == leveldb::mojom::DatabaseError::OK && !deleted_old_data_ &&
!context_->subdirectory_.empty() && context_->task_runner_ &&
!context_->old_localstorage_path_.empty()) {
deleted_old_data_ = true;
context_->task_runner_->PostShutdownBlockingTask(
FROM_HERE, DOMStorageTaskRunner::PRIMARY_SEQUENCE,
base::BindOnce(base::IgnoreResult(&sql::Database::Delete),
sql_db_path()));
}
context_->OnCommitResult(error);
}
void MigrateData(StorageAreaImpl::ValueMapCallback callback) override {
if (context_->task_runner_ && !context_->old_localstorage_path_.empty()) {
context_->task_runner_->PostShutdownBlockingTask(
FROM_HERE, DOMStorageTaskRunner::PRIMARY_SEQUENCE,
base::BindOnce(
&MigrateStorageHelper, sql_db_path(),
base::ThreadTaskRunnerHandle::Get(),
base::Bind(&CallMigrationCalback, base::Passed(&callback))));
return;
}
std::move(callback).Run(nullptr);
}
std::vector<StorageAreaImpl::Change> FixUpData(
const StorageAreaImpl::ValueMap& data) override {
std::vector<StorageAreaImpl::Change> changes;
// Chrome M61/M62 had a bug where keys that should have been encoded as
// Latin1 were instead encoded as UTF16. Fix this by finding any 8-bit only
// keys, and re-encode those. If two encodings of the key exist, the Latin1
// encoded value should take precedence.
size_t fix_count = 0;
for (const auto& it : data) {
// Skip over any Latin1 encoded keys, or unknown encodings/corrupted data.
if (it.first.empty() || it.first[0] != kUTF16Format)
continue;
// Check if key is actually 8-bit safe.
bool is_8bit = true;
for (size_t i = 1; i < it.first.size(); i += sizeof(base::char16)) {
// Don't just cast to char16* as that could be undefined behavior.
// Instead use memcpy for the conversion, which compilers will generally
// optimize away anyway.
base::char16 char_val;
memcpy(&char_val, it.first.data() + i, sizeof(base::char16));
if (char_val & 0xff00) {
is_8bit = false;
break;
}
}
if (!is_8bit)
continue;
// Found a key that should have been encoded differently. Decode and
// re-encode.
std::vector<uint8_t> key(1 + (it.first.size() - 1) / 2);
key[0] = kLatin1Format;
for (size_t in = 1, out = 1; in < it.first.size();
in += sizeof(base::char16), out++) {
base::char16 char_val;
memcpy(&char_val, it.first.data() + in, sizeof(base::char16));
key[out] = char_val;
}
// Delete incorrect key.
changes.push_back(std::make_pair(it.first, base::nullopt));
fix_count++;
// Check if correct key already exists in data.
auto new_it = data.find(key);
if (new_it != data.end())
continue;
// Update value for correct key.
changes.push_back(std::make_pair(key, it.second));
}
UMA_HISTOGRAM_BOOLEAN("LocalStorageContext.MigrationFixUpNeeded",
fix_count != 0);
return changes;
}
void OnMapLoaded(leveldb::mojom::DatabaseError error) override {
if (error != leveldb::mojom::DatabaseError::OK)
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.MapLoadError",
leveldb::GetLevelDBStatusUMAValue(error),
leveldb_env::LEVELDB_STATUS_MAX);
}
void Bind(blink::mojom::StorageAreaRequest request) {
has_bindings_ = true;
storage_area()->Bind(std::move(request));
}
bool has_bindings() const { return has_bindings_; }
private:
base::FilePath sql_db_path() const {
if (context_->old_localstorage_path_.empty())
return base::FilePath();
return context_->old_localstorage_path_.Append(
DOMStorageArea::DatabaseFileNameFromOrigin(origin_));
}
LocalStorageContextMojo* context_;
url::Origin origin_;
std::unique_ptr<StorageAreaImpl> area_;
// Holds the same value as |area_|. The reason for this is that
// during destruction of the StorageAreaImpl instance we might still get
// called and need access to the StorageAreaImpl instance. The unique_ptr
// could already be null, but this field should still be valid.
StorageAreaImpl* area_ptr_;
bool deleted_old_data_ = false;
bool has_bindings_ = false;
};
LocalStorageContextMojo::LocalStorageContextMojo(
scoped_refptr<base::SequencedTaskRunner> task_runner,
service_manager::Connector* connector,
scoped_refptr<DOMStorageTaskRunner> legacy_task_runner,
const base::FilePath& old_localstorage_path,
const base::FilePath& subdirectory,
scoped_refptr<storage::SpecialStoragePolicy> special_storage_policy)
: connector_(connector ? connector->Clone() : nullptr),
subdirectory_(subdirectory),
special_storage_policy_(std::move(special_storage_policy)),
memory_dump_id_(base::StringPrintf("LocalStorage/0x%" PRIXPTR,
reinterpret_cast<uintptr_t>(this))),
task_runner_(std::move(legacy_task_runner)),
old_localstorage_path_(old_localstorage_path),
is_low_end_device_(base::SysInfo::IsLowEndDevice()),
weak_ptr_factory_(this) {
base::trace_event::MemoryDumpManager::GetInstance()
->RegisterDumpProviderWithSequencedTaskRunner(
this, "LocalStorage", task_runner, MemoryDumpProvider::Options());
}
void LocalStorageContextMojo::OpenLocalStorage(
const url::Origin& origin,
blink::mojom::StorageAreaRequest request) {
RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::BindLocalStorage,
weak_ptr_factory_.GetWeakPtr(), origin,
std::move(request)));
}
void LocalStorageContextMojo::GetStorageUsage(
GetStorageUsageCallback callback) {
RunWhenConnected(
base::BindOnce(&LocalStorageContextMojo::RetrieveStorageUsage,
weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
}
void LocalStorageContextMojo::DeleteStorage(const url::Origin& origin,
base::OnceClosure callback) {
if (connection_state_ != CONNECTION_FINISHED) {
RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::DeleteStorage,
weak_ptr_factory_.GetWeakPtr(), origin,
std::move(callback)));
return;
}
auto found = areas_.find(origin);
if (found != areas_.end()) {
// Renderer process expects |source| to always be two newline separated
// strings.
found->second->storage_area()->DeleteAll(
"\n", base::BindOnce(&SuccessResponse, std::move(callback)));
found->second->storage_area()->ScheduleImmediateCommit();
} else if (database_) {
std::vector<leveldb::mojom::BatchedOperationPtr> operations;
AddDeleteOriginOperations(&operations, origin);
database_->Write(
std::move(operations),
base::BindOnce(&DatabaseErrorResponse, std::move(callback)));
} else {
std::move(callback).Run();
}
}
void LocalStorageContextMojo::PerformStorageCleanup(
base::OnceClosure callback) {
if (connection_state_ != CONNECTION_FINISHED) {
RunWhenConnected(
base::BindOnce(&LocalStorageContextMojo::PerformStorageCleanup,
weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
return;
}
if (database_) {
// Try to commit all changes before rewriting the database. If
// an area is not ready to commit its changes, nothing breaks but the
// rewrite doesn't remove all traces of old data.
Flush();
database_->RewriteDB(
base::BindOnce(&DatabaseErrorResponse, std::move(callback)));
} else {
std::move(callback).Run();
}
}
void LocalStorageContextMojo::Flush() {
if (connection_state_ != CONNECTION_FINISHED) {
RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::Flush,
weak_ptr_factory_.GetWeakPtr()));
return;
}
for (const auto& it : areas_)
it.second->storage_area()->ScheduleImmediateCommit();
}
void LocalStorageContextMojo::FlushOriginForTesting(const url::Origin& origin) {
if (connection_state_ != CONNECTION_FINISHED)
return;
const auto& it = areas_.find(origin);
if (it == areas_.end())
return;
it->second->storage_area()->ScheduleImmediateCommit();
}
void LocalStorageContextMojo::ShutdownAndDelete() {
DCHECK_NE(connection_state_, CONNECTION_SHUTDOWN);
// Nothing to do if no connection to the database was ever finished.
if (connection_state_ != CONNECTION_FINISHED) {
connection_state_ = CONNECTION_SHUTDOWN;
OnShutdownComplete(leveldb::mojom::DatabaseError::OK);
return;
}
connection_state_ = CONNECTION_SHUTDOWN;
// Flush any uncommitted data.
for (const auto& it : areas_) {
auto* area = it.second->storage_area();
LOCAL_HISTOGRAM_BOOLEAN(
"LocalStorageContext.ShutdownAndDelete.MaybeDroppedChanges",
area->has_pending_load_tasks());
area->ScheduleImmediateCommit();
// TODO(dmurph): Monitor the above histogram, and if dropping changes is
// common then handle that here.
area->CancelAllPendingRequests();
}
// Respect the content policy settings about what to
// keep and what to discard.
if (force_keep_session_state_) {
OnShutdownComplete(leveldb::mojom::DatabaseError::OK);
return; // Keep everything.
}
bool has_session_only_origins =
special_storage_policy_.get() &&
special_storage_policy_->HasSessionOnlyOrigins();
if (has_session_only_origins) {
RetrieveStorageUsage(
base::BindOnce(&LocalStorageContextMojo::OnGotStorageUsageForShutdown,
base::Unretained(this)));
} else {
OnShutdownComplete(leveldb::mojom::DatabaseError::OK);
}
}
void LocalStorageContextMojo::PurgeMemory() {
size_t total_cache_size, unused_area_count;
GetStatistics(&total_cache_size, &unused_area_count);
for (auto it = areas_.begin(); it != areas_.end();) {
if (it->second->has_bindings()) {
it->second->storage_area()->PurgeMemory();
++it;
} else {
it = areas_.erase(it);
}
}
// Track the size of cache purged.
size_t final_total_cache_size;
GetStatistics(&final_total_cache_size, &unused_area_count);
size_t purged_size_kib = (total_cache_size - final_total_cache_size) / 1024;
RecordCachePurgedHistogram(CachePurgeReason::AggressivePurgeTriggered,
purged_size_kib);
}
void LocalStorageContextMojo::PurgeUnusedAreasIfNeeded() {
size_t total_cache_size, unused_area_count;
GetStatistics(&total_cache_size, &unused_area_count);
// Nothing to purge.
if (!unused_area_count)
return;
CachePurgeReason purge_reason = CachePurgeReason::NotNeeded;
if (total_cache_size > kMaxLocalStorageCacheSize)
purge_reason = CachePurgeReason::SizeLimitExceeded;
else if (areas_.size() > kMaxLocalStorageAreaCount)
purge_reason = CachePurgeReason::AreaCountLimitExceeded;
else if (is_low_end_device_)
purge_reason = CachePurgeReason::InactiveOnLowEndDevice;
if (purge_reason == CachePurgeReason::NotNeeded)
return;
for (auto it = areas_.begin(); it != areas_.end();) {
if (it->second->has_bindings())
++it;
else
it = areas_.erase(it);
}
// Track the size of cache purged.
size_t final_total_cache_size;
GetStatistics(&final_total_cache_size, &unused_area_count);
size_t purged_size_kib = (total_cache_size - final_total_cache_size) / 1024;
RecordCachePurgedHistogram(purge_reason, purged_size_kib);
}
void LocalStorageContextMojo::SetDatabaseForTesting(
leveldb::mojom::LevelDBDatabaseAssociatedPtr database) {
DCHECK_EQ(connection_state_, NO_CONNECTION);
connection_state_ = CONNECTION_IN_PROGRESS;
database_ = std::move(database);
OnDatabaseOpened(true, leveldb::mojom::DatabaseError::OK);
}
bool LocalStorageContextMojo::OnMemoryDump(
const base::trace_event::MemoryDumpArgs& args,
base::trace_event::ProcessMemoryDump* pmd) {
if (connection_state_ != CONNECTION_FINISHED)
return true;
std::string context_name =
base::StringPrintf("site_storage/localstorage/0x%" PRIXPTR,
reinterpret_cast<uintptr_t>(this));
// Account for leveldb memory usage, which actually lives in the file service.
auto* global_dump = pmd->CreateSharedGlobalAllocatorDump(memory_dump_id_);
// The size of the leveldb dump will be added by the leveldb service.
auto* leveldb_mad = pmd->CreateAllocatorDump(context_name + "/leveldb");
// Specifies that the current context is responsible for keeping memory alive.
int kImportance = 2;
pmd->AddOwnershipEdge(leveldb_mad->guid(), global_dump->guid(), kImportance);
if (args.level_of_detail ==
base::trace_event::MemoryDumpLevelOfDetail::BACKGROUND) {
size_t total_cache_size, unused_area_count;
GetStatistics(&total_cache_size, &unused_area_count);
auto* mad = pmd->CreateAllocatorDump(context_name + "/cache_size");
mad->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
total_cache_size);
mad->AddScalar("total_areas",
base::trace_event::MemoryAllocatorDump::kUnitsObjects,
areas_.size());
return true;
}
for (const auto& it : areas_) {
// Limit the url length to 50 and strip special characters.
std::string url = it.first.Serialize().substr(0, 50);
for (size_t index = 0; index < url.size(); ++index) {
if (!std::isalnum(url[index]))
url[index] = '_';
}
std::string area_dump_name = base::StringPrintf(
"%s/%s/0x%" PRIXPTR, context_name.c_str(), url.c_str(),
reinterpret_cast<uintptr_t>(it.second->storage_area()));
it.second->storage_area()->OnMemoryDump(area_dump_name, pmd);
}
return true;
}
// static
std::vector<uint8_t> LocalStorageContextMojo::MigrateString(
const base::string16& input) {
// TODO(mek): Deduplicate this somehow with the code in
// LocalStorageCachedArea::String16ToUint8Vector.
bool is_8bit = true;
for (const auto& c : input) {
if (c & 0xff00) {
is_8bit = false;
break;
}
}
if (is_8bit) {
std::vector<uint8_t> result(input.size() + 1);
result[0] = kLatin1Format;
std::copy(input.begin(), input.end(), result.begin() + 1);
return result;
}
const uint8_t* data = reinterpret_cast<const uint8_t*>(input.data());
std::vector<uint8_t> result;
result.reserve(input.size() * sizeof(base::char16) + 1);
result.push_back(kUTF16Format);
result.insert(result.end(), data, data + input.size() * sizeof(base::char16));
return result;
}
LocalStorageContextMojo::~LocalStorageContextMojo() {
DCHECK_EQ(connection_state_, CONNECTION_SHUTDOWN);
base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
this);
}
void LocalStorageContextMojo::RunWhenConnected(base::OnceClosure callback) {
DCHECK_NE(connection_state_, CONNECTION_SHUTDOWN);
// If we don't have a filesystem_connection_, we'll need to establish one.
if (connection_state_ == NO_CONNECTION) {
connection_state_ = CONNECTION_IN_PROGRESS;
InitiateConnection();
}
if (connection_state_ == CONNECTION_IN_PROGRESS) {
// Queue this OpenLocalStorage call for when we have a level db pointer.
on_database_opened_callbacks_.push_back(std::move(callback));
return;
}
std::move(callback).Run();
}
void LocalStorageContextMojo::InitiateConnection(bool in_memory_only) {
DCHECK_EQ(connection_state_, CONNECTION_IN_PROGRESS);
// Unit tests might not always have a Connector, use in-memory only if that
// happens.
if (!connector_) {
OnDatabaseOpened(false, leveldb::mojom::DatabaseError::OK);
return;
}
if (!subdirectory_.empty() && !in_memory_only) {
// We were given a subdirectory to write to. Get it and use a disk backed
// database.
connector_->BindInterface(file::mojom::kServiceName, &file_system_);
file_system_->GetSubDirectory(
subdirectory_.AsUTF8Unsafe(), MakeRequest(&directory_),
base::BindOnce(&LocalStorageContextMojo::OnDirectoryOpened,
weak_ptr_factory_.GetWeakPtr()));
} else {
// We were not given a subdirectory. Use a memory backed database.
connector_->BindInterface(file::mojom::kServiceName, &leveldb_service_);
leveldb_service_->OpenInMemory(
memory_dump_id_, "local-storage", MakeRequest(&database_),
base::BindOnce(&LocalStorageContextMojo::OnDatabaseOpened,
weak_ptr_factory_.GetWeakPtr(), true));
}
}
void LocalStorageContextMojo::OnMojoConnectionDestroyed() {
UMA_HISTOGRAM_BOOLEAN("LocalStorageContext.OnConnectionDestroyed", true);
LOG(ERROR) << "Lost connection to database";
// We're about to set database_ to null, so delete the StorageAreaImpls
// that might still be using the old database.
for (const auto& it : areas_)
it.second->storage_area()->CancelAllPendingRequests();
areas_.clear();
database_ = nullptr;
// TODO(dullweber): Should we try to recover? E.g. try to reopen and if this
// fails, call DeleteAndRecreateDatabase().
}
void LocalStorageContextMojo::OnDirectoryOpened(base::File::Error err) {
if (err != base::File::Error::FILE_OK) {
// We failed to open the directory; continue with startup so that we create
// the |areas_|.
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DirectoryOpenError", -err,
-base::File::FILE_ERROR_MAX);
LogDatabaseOpenResult(OpenResult::DIRECTORY_OPEN_FAILED);
OnDatabaseOpened(false, leveldb::mojom::DatabaseError::OK);
return;
}
// Now that we have a directory, connect to the LevelDB service and get our
// database.
connector_->BindInterface(file::mojom::kServiceName, &leveldb_service_);
// We might still need to use the directory, so create a clone.
filesystem::mojom::DirectoryPtr directory_clone;
directory_->Clone(MakeRequest(&directory_clone));
leveldb_env::Options options;
options.create_if_missing = true;
options.max_open_files = 0; // use minimum
// Default write_buffer_size is 4 MB but that might leave a 3.999
// memory allocation in RAM from a log file recovery.
options.write_buffer_size = 64 * 1024;
options.block_cache = leveldb_chrome::GetSharedWebBlockCache();
leveldb_service_->OpenWithOptions(
std::move(options), std::move(directory_clone), "leveldb",
memory_dump_id_, MakeRequest(&database_),
base::BindOnce(&LocalStorageContextMojo::OnDatabaseOpened,
weak_ptr_factory_.GetWeakPtr(), false));
}
void LocalStorageContextMojo::OnDatabaseOpened(
bool in_memory,
leveldb::mojom::DatabaseError status) {
if (status != leveldb::mojom::DatabaseError::OK) {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError",
leveldb::GetLevelDBStatusUMAValue(status),
leveldb_env::LEVELDB_STATUS_MAX);
if (in_memory) {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError.Memory",
leveldb::GetLevelDBStatusUMAValue(status),
leveldb_env::LEVELDB_STATUS_MAX);
} else {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError.Disk",
leveldb::GetLevelDBStatusUMAValue(status),
leveldb_env::LEVELDB_STATUS_MAX);
}
LogDatabaseOpenResult(OpenResult::DATABASE_OPEN_FAILED);
// If we failed to open the database, try to delete and recreate the
// database, or ultimately fallback to an in-memory database.
DeleteAndRecreateDatabase("LocalStorageContext.OpenResultAfterOpenFailed");
return;
}
// Verify DB schema version.
if (database_) {
database_.set_connection_error_handler(
base::BindOnce(&LocalStorageContextMojo::OnMojoConnectionDestroyed,
weak_ptr_factory_.GetWeakPtr()));
database_->Get(
leveldb::StdStringToUint8Vector(kVersionKey),
base::BindOnce(&LocalStorageContextMojo::OnGotDatabaseVersion,
weak_ptr_factory_.GetWeakPtr()));
return;
}
OnConnectionFinished();
}
void LocalStorageContextMojo::OnGotDatabaseVersion(
leveldb::mojom::DatabaseError status,
const std::vector<uint8_t>& value) {
if (status == leveldb::mojom::DatabaseError::NOT_FOUND) {
// New database, nothing more to do. Current version will get written
// when first data is committed.
} else if (status == leveldb::mojom::DatabaseError::OK) {
// Existing database, check if version number matches current schema
// version.
int64_t db_version;
if (!base::StringToInt64(leveldb::Uint8VectorToStdString(value),
&db_version) ||
db_version < kMinSchemaVersion ||
db_version > kCurrentLocalStorageSchemaVersion) {
LogDatabaseOpenResult(OpenResult::INVALID_VERSION);
DeleteAndRecreateDatabase(
"LocalStorageContext.OpenResultAfterInvalidVersion");
return;
}
database_initialized_ = true;
} else {
// Other read error. Possibly database corruption.
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.ReadVersionError",
leveldb::GetLevelDBStatusUMAValue(status),
leveldb_env::LEVELDB_STATUS_MAX);
LogDatabaseOpenResult(OpenResult::VERSION_READ_ERROR);
DeleteAndRecreateDatabase(
"LocalStorageContext.OpenResultAfterReadVersionError");
return;
}
OnConnectionFinished();
}
void LocalStorageContextMojo::OnConnectionFinished() {
DCHECK_EQ(connection_state_, CONNECTION_IN_PROGRESS);
if (!database_) {
directory_.reset();
file_system_.reset();
leveldb_service_.reset();
}
// If connection was opened successfully, reset tried_to_recreate_during_open_
// to enable recreating the database on future errors.
if (database_)
tried_to_recreate_during_open_ = false;
LogDatabaseOpenResult(OpenResult::SUCCESS);
open_result_histogram_ = nullptr;
// |database_| should be known to either be valid or invalid by now. Run our
// delayed bindings.
connection_state_ = CONNECTION_FINISHED;
for (size_t i = 0; i < on_database_opened_callbacks_.size(); ++i)
std::move(on_database_opened_callbacks_[i]).Run();
on_database_opened_callbacks_.clear();
}
void LocalStorageContextMojo::DeleteAndRecreateDatabase(
const char* histogram_name) {
// We're about to set database_ to null, so delete the StorageAreaImpls
// that might still be using the old database.
for (const auto& it : areas_)
it.second->storage_area()->CancelAllPendingRequests();
areas_.clear();
// Reset state to be in process of connecting. This will cause requests for
// StorageAreas to be queued until the connection is complete.
connection_state_ = CONNECTION_IN_PROGRESS;
commit_error_count_ = 0;
database_ = nullptr;
open_result_histogram_ = histogram_name;
bool recreate_in_memory = false;
// If tried to recreate database on disk already, try again but this time
// in memory.
if (tried_to_recreate_during_open_ && !subdirectory_.empty()) {
recreate_in_memory = true;
} else if (tried_to_recreate_during_open_) {
// Give up completely, run without any database.
OnConnectionFinished();
return;
}
tried_to_recreate_during_open_ = true;
// Unit tests might not have a bound file_service_, in which case there is
// nothing to retry.
if (!file_system_.is_bound()) {
OnConnectionFinished();
return;
}
// Destroy database, and try again.
if (directory_.is_bound()) {
leveldb_service_->Destroy(
std::move(directory_), "leveldb",
base::BindOnce(&LocalStorageContextMojo::OnDBDestroyed,
weak_ptr_factory_.GetWeakPtr(), recreate_in_memory));
} else {
// No directory, so nothing to destroy. Retrying to recreate will probably
// fail, but try anyway.
InitiateConnection(recreate_in_memory);
}
}
void LocalStorageContextMojo::OnDBDestroyed(
bool recreate_in_memory,
leveldb::mojom::DatabaseError status) {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DestroyDBResult",
leveldb::GetLevelDBStatusUMAValue(status),
leveldb_env::LEVELDB_STATUS_MAX);
// We're essentially ignoring the status here. Even if destroying failed we
// still want to go ahead and try to recreate.
InitiateConnection(recreate_in_memory);
}
// The (possibly delayed) implementation of OpenLocalStorage(). Can be called
// directly from that function, or through |on_database_open_callbacks_|.
void LocalStorageContextMojo::BindLocalStorage(
const url::Origin& origin,
blink::mojom::StorageAreaRequest request) {
GetOrCreateStorageArea(origin)->Bind(std::move(request));
}
LocalStorageContextMojo::StorageAreaHolder*
LocalStorageContextMojo::GetOrCreateStorageArea(const url::Origin& origin) {
DCHECK_EQ(connection_state_, CONNECTION_FINISHED);
auto found = areas_.find(origin);
if (found != areas_.end()) {
return found->second.get();
}
size_t total_cache_size, unused_area_count;
GetStatistics(&total_cache_size, &unused_area_count);
// Track the total localStorage cache size.
UMA_HISTOGRAM_COUNTS_100000("LocalStorageContext.CacheSizeInKB",
total_cache_size / 1024);
PurgeUnusedAreasIfNeeded();
auto holder = std::make_unique<StorageAreaHolder>(this, origin);
StorageAreaHolder* holder_ptr = holder.get();
areas_[origin] = std::move(holder);
return holder_ptr;
}
void LocalStorageContextMojo::RetrieveStorageUsage(
GetStorageUsageCallback callback) {
if (!database_) {
// If for whatever reason no leveldb database is available, no storage is
// used, so return an array only containing the current areas.
std::vector<StorageUsageInfo> result;
base::Time now = base::Time::Now();
for (const auto& it : areas_)
result.emplace_back(it.first, 0, now);
std::move(callback).Run(std::move(result));
return;
}
database_->GetPrefixed(
std::vector<uint8_t>(kMetaPrefix, kMetaPrefix + base::size(kMetaPrefix)),
base::BindOnce(&LocalStorageContextMojo::OnGotMetaData,
weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
}
void LocalStorageContextMojo::OnGotMetaData(
GetStorageUsageCallback callback,
leveldb::mojom::DatabaseError status,
std::vector<leveldb::mojom::KeyValuePtr> data) {
std::vector<StorageUsageInfo> result;
std::set<url::Origin> origins;
for (const auto& row : data) {
DCHECK_GT(row->key.size(), base::size(kMetaPrefix));
GURL origin(leveldb::Uint8VectorToStdString(row->key).substr(
base::size(kMetaPrefix)));
origins.insert(url::Origin::Create(origin));
if (!origin.is_valid()) {
// TODO(mek): Deal with database corruption.
continue;
}
LocalStorageOriginMetaData row_data;
if (!row_data.ParseFromArray(row->value.data(), row->value.size())) {
// TODO(mek): Deal with database corruption.
continue;
}
result.emplace_back(
url::Origin::Create(origin), row_data.size_bytes(),
base::Time::FromInternalValue(row_data.last_modified()));
}
// Add any origins for which StorageAreas exist, but which haven't
// committed any data to disk yet.
base::Time now = base::Time::Now();
for (const auto& it : areas_) {
if (origins.find(it.first) != origins.end())
continue;
// Skip any origins that definitely don't have any data.
if (!it.second->storage_area()->has_pending_load_tasks() &&
it.second->storage_area()->empty()) {
continue;
}
result.emplace_back(it.first, 0, now);
}
std::move(callback).Run(std::move(result));
}
void LocalStorageContextMojo::OnGotStorageUsageForShutdown(
std::vector<StorageUsageInfo> usage) {
std::vector<leveldb::mojom::BatchedOperationPtr> operations;
for (const auto& info : usage) {
if (special_storage_policy_->IsStorageProtected(info.origin.GetURL()))
continue;
if (!special_storage_policy_->IsStorageSessionOnly(info.origin.GetURL()))
continue;
AddDeleteOriginOperations(&operations, info.origin);
}
if (!operations.empty()) {
database_->Write(
std::move(operations),
base::BindOnce(&LocalStorageContextMojo::OnShutdownComplete,
base::Unretained(this)));
} else {
OnShutdownComplete(leveldb::mojom::DatabaseError::OK);
}
}
void LocalStorageContextMojo::OnShutdownComplete(
leveldb::mojom::DatabaseError error) {
delete this;
}
void LocalStorageContextMojo::GetStatistics(size_t* total_cache_size,
size_t* unused_area_count) {
*total_cache_size = 0;
*unused_area_count = 0;
for (const auto& it : areas_) {
*total_cache_size += it.second->storage_area()->memory_used();
if (!it.second->has_bindings())
(*unused_area_count)++;
}
}
void LocalStorageContextMojo::OnCommitResult(
leveldb::mojom::DatabaseError error) {
DCHECK(connection_state_ == CONNECTION_FINISHED ||
connection_state_ == CONNECTION_SHUTDOWN)
<< connection_state_;
if (error == leveldb::mojom::DatabaseError::OK) {
commit_error_count_ = 0;
return;
}
commit_error_count_++;
if (commit_error_count_ > kCommitErrorThreshold &&
connection_state_ != CONNECTION_SHUTDOWN) {
if (tried_to_recover_from_commit_errors_) {
// We already tried to recover from a high commit error rate before, but
// are still having problems: there isn't really anything left to try, so
// just ignore errors.
return;
}
tried_to_recover_from_commit_errors_ = true;
// Deleting StorageAreas in here could cause more commits (and commit
// errors), but those commits won't reach OnCommitResult because the area
// will have been deleted before the commit finishes.
DeleteAndRecreateDatabase(
"LocalStorageContext.OpenResultAfterCommitErrors");
}
}
void LocalStorageContextMojo::LogDatabaseOpenResult(OpenResult result) {
if (result != OpenResult::SUCCESS) {
UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.OpenError", result,
OpenResult::MAX);
}
if (open_result_histogram_) {
base::UmaHistogramEnumeration(open_result_histogram_, result,
OpenResult::MAX);
}
}
} // namespace content