| // Copyright 2016 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "content/browser/dom_storage/local_storage_context_mojo.h" |
| |
| #include <inttypes.h> |
| |
| #include <algorithm> |
| #include <cctype> // for std::isalnum |
| #include <set> |
| #include <string> |
| #include <utility> |
| |
| #include "base/barrier_closure.h" |
| #include "base/metrics/histogram_functions.h" |
| #include "base/metrics/histogram_macros.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/stl_util.h" |
| #include "base/strings/string_number_conversions.h" |
| #include "base/strings/stringprintf.h" |
| #include "base/system/sys_info.h" |
| #include "base/trace_event/memory_dump_manager.h" |
| #include "build/build_config.h" |
| #include "components/services/leveldb/public/cpp/util.h" |
| #include "components/services/leveldb/public/interfaces/leveldb.mojom.h" |
| #include "content/browser/dom_storage/dom_storage_area.h" |
| #include "content/browser/dom_storage/dom_storage_database.h" |
| #include "content/browser/dom_storage/dom_storage_task_runner.h" |
| #include "content/browser/dom_storage/local_storage_database.pb.h" |
| #include "content/browser/dom_storage/storage_area_impl.h" |
| #include "content/common/dom_storage/dom_storage_types.h" |
| #include "content/public/browser/storage_usage_info.h" |
| #include "services/file/public/mojom/constants.mojom.h" |
| #include "services/service_manager/public/cpp/connector.h" |
| #include "sql/database.h" |
| #include "storage/browser/quota/special_storage_policy.h" |
| #include "third_party/leveldatabase/env_chromium.h" |
| #include "third_party/leveldatabase/leveldb_chrome.h" |
| |
| namespace content { |
| |
| // LevelDB database schema |
| // ======================= |
| // |
| // Version 1 (in sorted order): |
| // key: "VERSION" |
| // value: "1" |
| // |
| // key: "META:" + <url::Origin 'origin'> |
| // value: <LocalStorageOriginMetaData serialized as a string> |
| // |
| // key: "_" + <url::Origin> 'origin'> + '\x00' + <script controlled key> |
| // value: <script controlled value> |
| |
| namespace { |
| |
| const char kVersionKey[] = "VERSION"; |
| const char kOriginSeparator = '\x00'; |
| const char kDataPrefix[] = "_"; |
| const uint8_t kMetaPrefix[] = {'M', 'E', 'T', 'A', ':'}; |
| const int64_t kMinSchemaVersion = 1; |
| const int64_t kCurrentLocalStorageSchemaVersion = 1; |
| |
| // After this many consecutive commit errors we'll throw away the entire |
| // database. |
| const int kCommitErrorThreshold = 8; |
| |
| // Limits on the cache size and number of areas in memory, over which the areas |
| // are purged. |
| #if defined(OS_ANDROID) |
| const unsigned kMaxLocalStorageAreaCount = 10; |
| const size_t kMaxLocalStorageCacheSize = 2 * 1024 * 1024; |
| #else |
| const unsigned kMaxLocalStorageAreaCount = 50; |
| const size_t kMaxLocalStorageCacheSize = 20 * 1024 * 1024; |
| #endif |
| |
| static const uint8_t kUTF16Format = 0; |
| static const uint8_t kLatin1Format = 1; |
| |
| std::vector<uint8_t> CreateMetaDataKey(const url::Origin& origin) { |
| auto serialized_origin = leveldb::StdStringToUint8Vector(origin.Serialize()); |
| std::vector<uint8_t> key; |
| key.reserve(base::size(kMetaPrefix) + serialized_origin.size()); |
| key.insert(key.end(), kMetaPrefix, kMetaPrefix + base::size(kMetaPrefix)); |
| key.insert(key.end(), serialized_origin.begin(), serialized_origin.end()); |
| return key; |
| } |
| |
| void SuccessResponse(base::OnceClosure callback, bool success) { |
| std::move(callback).Run(); |
| } |
| void DatabaseErrorResponse(base::OnceClosure callback, |
| leveldb::mojom::DatabaseError error) { |
| std::move(callback).Run(); |
| } |
| |
| void MigrateStorageHelper( |
| base::FilePath db_path, |
| const scoped_refptr<base::SingleThreadTaskRunner> reply_task_runner, |
| base::Callback<void(std::unique_ptr<StorageAreaImpl::ValueMap>)> callback) { |
| DOMStorageDatabase db(db_path); |
| DOMStorageValuesMap map; |
| db.ReadAllValues(&map); |
| auto values = std::make_unique<StorageAreaImpl::ValueMap>(); |
| for (const auto& it : map) { |
| (*values)[LocalStorageContextMojo::MigrateString(it.first)] = |
| LocalStorageContextMojo::MigrateString(it.second.string()); |
| } |
| reply_task_runner->PostTask( |
| FROM_HERE, base::BindOnce(std::move(callback), std::move(values))); |
| } |
| |
| // Helper to convert from OnceCallback to Callback. |
| void CallMigrationCalback(StorageAreaImpl::ValueMapCallback callback, |
| std::unique_ptr<StorageAreaImpl::ValueMap> data) { |
| std::move(callback).Run(std::move(data)); |
| } |
| |
| void AddDeleteOriginOperations( |
| std::vector<leveldb::mojom::BatchedOperationPtr>* operations, |
| const url::Origin& origin) { |
| leveldb::mojom::BatchedOperationPtr item = |
| leveldb::mojom::BatchedOperation::New(); |
| item->type = leveldb::mojom::BatchOperationType::DELETE_PREFIXED_KEY; |
| item->key = leveldb::StdStringToUint8Vector(kDataPrefix + origin.Serialize() + |
| kOriginSeparator); |
| operations->push_back(std::move(item)); |
| |
| item = leveldb::mojom::BatchedOperation::New(); |
| item->type = leveldb::mojom::BatchOperationType::DELETE_KEY; |
| item->key = CreateMetaDataKey(origin); |
| operations->push_back(std::move(item)); |
| } |
| |
| enum class CachePurgeReason { |
| NotNeeded, |
| SizeLimitExceeded, |
| AreaCountLimitExceeded, |
| InactiveOnLowEndDevice, |
| AggressivePurgeTriggered |
| }; |
| |
| void RecordCachePurgedHistogram(CachePurgeReason reason, |
| size_t purged_size_kib) { |
| UMA_HISTOGRAM_COUNTS_100000("LocalStorageContext.CachePurgedInKB", |
| purged_size_kib); |
| switch (reason) { |
| case CachePurgeReason::SizeLimitExceeded: |
| UMA_HISTOGRAM_COUNTS_100000( |
| "LocalStorageContext.CachePurgedInKB.SizeLimitExceeded", |
| purged_size_kib); |
| break; |
| case CachePurgeReason::AreaCountLimitExceeded: |
| UMA_HISTOGRAM_COUNTS_100000( |
| "LocalStorageContext.CachePurgedInKB.AreaCountLimitExceeded", |
| purged_size_kib); |
| break; |
| case CachePurgeReason::InactiveOnLowEndDevice: |
| UMA_HISTOGRAM_COUNTS_100000( |
| "LocalStorageContext.CachePurgedInKB.InactiveOnLowEndDevice", |
| purged_size_kib); |
| break; |
| case CachePurgeReason::AggressivePurgeTriggered: |
| UMA_HISTOGRAM_COUNTS_100000( |
| "LocalStorageContext.CachePurgedInKB.AggressivePurgeTriggered", |
| purged_size_kib); |
| break; |
| case CachePurgeReason::NotNeeded: |
| NOTREACHED(); |
| break; |
| } |
| } |
| |
| } // namespace |
| |
| class LocalStorageContextMojo::StorageAreaHolder final |
| : public StorageAreaImpl::Delegate { |
| public: |
| StorageAreaHolder(LocalStorageContextMojo* context, const url::Origin& origin) |
| : context_(context), origin_(origin) { |
| // Delay for a moment after a value is set in anticipation |
| // of other values being set, so changes are batched. |
| static constexpr base::TimeDelta kCommitDefaultDelaySecs = |
| base::TimeDelta::FromSeconds(5); |
| |
| // To avoid excessive IO we apply limits to the amount of data being written |
| // and the frequency of writes. |
| static constexpr int kMaxBytesPerHour = kPerStorageAreaQuota; |
| static constexpr int kMaxCommitsPerHour = 60; |
| |
| StorageAreaImpl::Options options; |
| options.max_size = kPerStorageAreaQuota + kPerStorageAreaOverQuotaAllowance; |
| options.default_commit_delay = kCommitDefaultDelaySecs; |
| options.max_bytes_per_hour = kMaxBytesPerHour; |
| options.max_commits_per_hour = kMaxCommitsPerHour; |
| #if defined(OS_ANDROID) |
| options.cache_mode = StorageAreaImpl::CacheMode::KEYS_ONLY_WHEN_POSSIBLE; |
| #else |
| options.cache_mode = StorageAreaImpl::CacheMode::KEYS_AND_VALUES; |
| if (base::SysInfo::IsLowEndDevice()) { |
| options.cache_mode = StorageAreaImpl::CacheMode::KEYS_ONLY_WHEN_POSSIBLE; |
| } |
| #endif |
| area_ = std::make_unique<StorageAreaImpl>( |
| context_->database_.get(), |
| kDataPrefix + origin_.Serialize() + kOriginSeparator, this, options); |
| area_ptr_ = area_.get(); |
| } |
| |
| StorageAreaImpl* storage_area() { return area_ptr_; } |
| |
| void OnNoBindings() override { |
| has_bindings_ = false; |
| // Don't delete ourselves, but do schedule an immediate commit. Possible |
| // deletion will happen under memory pressure or when another localstorage |
| // area is opened. |
| storage_area()->ScheduleImmediateCommit(); |
| } |
| |
| std::vector<leveldb::mojom::BatchedOperationPtr> PrepareToCommit() override { |
| std::vector<leveldb::mojom::BatchedOperationPtr> operations; |
| |
| // Write schema version if not already done so before. |
| if (!context_->database_initialized_) { |
| leveldb::mojom::BatchedOperationPtr item = |
| leveldb::mojom::BatchedOperation::New(); |
| item->type = leveldb::mojom::BatchOperationType::PUT_KEY; |
| item->key = leveldb::StdStringToUint8Vector(kVersionKey); |
| item->value = leveldb::StdStringToUint8Vector( |
| base::Int64ToString(kCurrentLocalStorageSchemaVersion)); |
| operations.push_back(std::move(item)); |
| context_->database_initialized_ = true; |
| } |
| |
| leveldb::mojom::BatchedOperationPtr item = |
| leveldb::mojom::BatchedOperation::New(); |
| item->type = leveldb::mojom::BatchOperationType::PUT_KEY; |
| item->key = CreateMetaDataKey(origin_); |
| if (storage_area()->empty()) { |
| item->type = leveldb::mojom::BatchOperationType::DELETE_KEY; |
| } else { |
| item->type = leveldb::mojom::BatchOperationType::PUT_KEY; |
| LocalStorageOriginMetaData data; |
| data.set_last_modified(base::Time::Now().ToInternalValue()); |
| data.set_size_bytes(storage_area()->storage_used()); |
| item->value = leveldb::StdStringToUint8Vector(data.SerializeAsString()); |
| } |
| operations.push_back(std::move(item)); |
| |
| return operations; |
| } |
| |
| void DidCommit(leveldb::mojom::DatabaseError error) override { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.CommitResult", |
| leveldb::GetLevelDBStatusUMAValue(error), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| |
| // Delete any old database that might still exist if we successfully wrote |
| // data to LevelDB, and our LevelDB is actually disk backed. |
| if (error == leveldb::mojom::DatabaseError::OK && !deleted_old_data_ && |
| !context_->subdirectory_.empty() && context_->task_runner_ && |
| !context_->old_localstorage_path_.empty()) { |
| deleted_old_data_ = true; |
| context_->task_runner_->PostShutdownBlockingTask( |
| FROM_HERE, DOMStorageTaskRunner::PRIMARY_SEQUENCE, |
| base::BindOnce(base::IgnoreResult(&sql::Database::Delete), |
| sql_db_path())); |
| } |
| |
| context_->OnCommitResult(error); |
| } |
| |
| void MigrateData(StorageAreaImpl::ValueMapCallback callback) override { |
| if (context_->task_runner_ && !context_->old_localstorage_path_.empty()) { |
| context_->task_runner_->PostShutdownBlockingTask( |
| FROM_HERE, DOMStorageTaskRunner::PRIMARY_SEQUENCE, |
| base::BindOnce( |
| &MigrateStorageHelper, sql_db_path(), |
| base::ThreadTaskRunnerHandle::Get(), |
| base::Bind(&CallMigrationCalback, base::Passed(&callback)))); |
| return; |
| } |
| std::move(callback).Run(nullptr); |
| } |
| |
| std::vector<StorageAreaImpl::Change> FixUpData( |
| const StorageAreaImpl::ValueMap& data) override { |
| std::vector<StorageAreaImpl::Change> changes; |
| // Chrome M61/M62 had a bug where keys that should have been encoded as |
| // Latin1 were instead encoded as UTF16. Fix this by finding any 8-bit only |
| // keys, and re-encode those. If two encodings of the key exist, the Latin1 |
| // encoded value should take precedence. |
| size_t fix_count = 0; |
| for (const auto& it : data) { |
| // Skip over any Latin1 encoded keys, or unknown encodings/corrupted data. |
| if (it.first.empty() || it.first[0] != kUTF16Format) |
| continue; |
| // Check if key is actually 8-bit safe. |
| bool is_8bit = true; |
| for (size_t i = 1; i < it.first.size(); i += sizeof(base::char16)) { |
| // Don't just cast to char16* as that could be undefined behavior. |
| // Instead use memcpy for the conversion, which compilers will generally |
| // optimize away anyway. |
| base::char16 char_val; |
| memcpy(&char_val, it.first.data() + i, sizeof(base::char16)); |
| if (char_val & 0xff00) { |
| is_8bit = false; |
| break; |
| } |
| } |
| if (!is_8bit) |
| continue; |
| // Found a key that should have been encoded differently. Decode and |
| // re-encode. |
| std::vector<uint8_t> key(1 + (it.first.size() - 1) / 2); |
| key[0] = kLatin1Format; |
| for (size_t in = 1, out = 1; in < it.first.size(); |
| in += sizeof(base::char16), out++) { |
| base::char16 char_val; |
| memcpy(&char_val, it.first.data() + in, sizeof(base::char16)); |
| key[out] = char_val; |
| } |
| // Delete incorrect key. |
| changes.push_back(std::make_pair(it.first, base::nullopt)); |
| fix_count++; |
| // Check if correct key already exists in data. |
| auto new_it = data.find(key); |
| if (new_it != data.end()) |
| continue; |
| // Update value for correct key. |
| changes.push_back(std::make_pair(key, it.second)); |
| } |
| UMA_HISTOGRAM_BOOLEAN("LocalStorageContext.MigrationFixUpNeeded", |
| fix_count != 0); |
| return changes; |
| } |
| |
| void OnMapLoaded(leveldb::mojom::DatabaseError error) override { |
| if (error != leveldb::mojom::DatabaseError::OK) |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.MapLoadError", |
| leveldb::GetLevelDBStatusUMAValue(error), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| } |
| |
| void Bind(blink::mojom::StorageAreaRequest request) { |
| has_bindings_ = true; |
| storage_area()->Bind(std::move(request)); |
| } |
| |
| bool has_bindings() const { return has_bindings_; } |
| |
| private: |
| base::FilePath sql_db_path() const { |
| if (context_->old_localstorage_path_.empty()) |
| return base::FilePath(); |
| return context_->old_localstorage_path_.Append( |
| DOMStorageArea::DatabaseFileNameFromOrigin(origin_)); |
| } |
| |
| LocalStorageContextMojo* context_; |
| url::Origin origin_; |
| std::unique_ptr<StorageAreaImpl> area_; |
| // Holds the same value as |area_|. The reason for this is that |
| // during destruction of the StorageAreaImpl instance we might still get |
| // called and need access to the StorageAreaImpl instance. The unique_ptr |
| // could already be null, but this field should still be valid. |
| StorageAreaImpl* area_ptr_; |
| bool deleted_old_data_ = false; |
| bool has_bindings_ = false; |
| }; |
| |
| LocalStorageContextMojo::LocalStorageContextMojo( |
| scoped_refptr<base::SequencedTaskRunner> task_runner, |
| service_manager::Connector* connector, |
| scoped_refptr<DOMStorageTaskRunner> legacy_task_runner, |
| const base::FilePath& old_localstorage_path, |
| const base::FilePath& subdirectory, |
| scoped_refptr<storage::SpecialStoragePolicy> special_storage_policy) |
| : connector_(connector ? connector->Clone() : nullptr), |
| subdirectory_(subdirectory), |
| special_storage_policy_(std::move(special_storage_policy)), |
| memory_dump_id_(base::StringPrintf("LocalStorage/0x%" PRIXPTR, |
| reinterpret_cast<uintptr_t>(this))), |
| task_runner_(std::move(legacy_task_runner)), |
| old_localstorage_path_(old_localstorage_path), |
| is_low_end_device_(base::SysInfo::IsLowEndDevice()), |
| weak_ptr_factory_(this) { |
| base::trace_event::MemoryDumpManager::GetInstance() |
| ->RegisterDumpProviderWithSequencedTaskRunner( |
| this, "LocalStorage", task_runner, MemoryDumpProvider::Options()); |
| } |
| |
| void LocalStorageContextMojo::OpenLocalStorage( |
| const url::Origin& origin, |
| blink::mojom::StorageAreaRequest request) { |
| RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::BindLocalStorage, |
| weak_ptr_factory_.GetWeakPtr(), origin, |
| std::move(request))); |
| } |
| |
| void LocalStorageContextMojo::GetStorageUsage( |
| GetStorageUsageCallback callback) { |
| RunWhenConnected( |
| base::BindOnce(&LocalStorageContextMojo::RetrieveStorageUsage, |
| weak_ptr_factory_.GetWeakPtr(), std::move(callback))); |
| } |
| |
| void LocalStorageContextMojo::DeleteStorage(const url::Origin& origin, |
| base::OnceClosure callback) { |
| if (connection_state_ != CONNECTION_FINISHED) { |
| RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::DeleteStorage, |
| weak_ptr_factory_.GetWeakPtr(), origin, |
| std::move(callback))); |
| return; |
| } |
| |
| auto found = areas_.find(origin); |
| if (found != areas_.end()) { |
| // Renderer process expects |source| to always be two newline separated |
| // strings. |
| found->second->storage_area()->DeleteAll( |
| "\n", base::BindOnce(&SuccessResponse, std::move(callback))); |
| found->second->storage_area()->ScheduleImmediateCommit(); |
| } else if (database_) { |
| std::vector<leveldb::mojom::BatchedOperationPtr> operations; |
| AddDeleteOriginOperations(&operations, origin); |
| database_->Write( |
| std::move(operations), |
| base::BindOnce(&DatabaseErrorResponse, std::move(callback))); |
| } else { |
| std::move(callback).Run(); |
| } |
| } |
| |
| void LocalStorageContextMojo::PerformStorageCleanup( |
| base::OnceClosure callback) { |
| if (connection_state_ != CONNECTION_FINISHED) { |
| RunWhenConnected( |
| base::BindOnce(&LocalStorageContextMojo::PerformStorageCleanup, |
| weak_ptr_factory_.GetWeakPtr(), std::move(callback))); |
| return; |
| } |
| if (database_) { |
| // Try to commit all changes before rewriting the database. If |
| // an area is not ready to commit its changes, nothing breaks but the |
| // rewrite doesn't remove all traces of old data. |
| Flush(); |
| database_->RewriteDB( |
| base::BindOnce(&DatabaseErrorResponse, std::move(callback))); |
| } else { |
| std::move(callback).Run(); |
| } |
| } |
| |
| void LocalStorageContextMojo::Flush() { |
| if (connection_state_ != CONNECTION_FINISHED) { |
| RunWhenConnected(base::BindOnce(&LocalStorageContextMojo::Flush, |
| weak_ptr_factory_.GetWeakPtr())); |
| return; |
| } |
| for (const auto& it : areas_) |
| it.second->storage_area()->ScheduleImmediateCommit(); |
| } |
| |
| void LocalStorageContextMojo::FlushOriginForTesting(const url::Origin& origin) { |
| if (connection_state_ != CONNECTION_FINISHED) |
| return; |
| const auto& it = areas_.find(origin); |
| if (it == areas_.end()) |
| return; |
| it->second->storage_area()->ScheduleImmediateCommit(); |
| } |
| |
| void LocalStorageContextMojo::ShutdownAndDelete() { |
| DCHECK_NE(connection_state_, CONNECTION_SHUTDOWN); |
| |
| // Nothing to do if no connection to the database was ever finished. |
| if (connection_state_ != CONNECTION_FINISHED) { |
| connection_state_ = CONNECTION_SHUTDOWN; |
| OnShutdownComplete(leveldb::mojom::DatabaseError::OK); |
| return; |
| } |
| |
| connection_state_ = CONNECTION_SHUTDOWN; |
| |
| // Flush any uncommitted data. |
| for (const auto& it : areas_) { |
| auto* area = it.second->storage_area(); |
| LOCAL_HISTOGRAM_BOOLEAN( |
| "LocalStorageContext.ShutdownAndDelete.MaybeDroppedChanges", |
| area->has_pending_load_tasks()); |
| area->ScheduleImmediateCommit(); |
| // TODO(dmurph): Monitor the above histogram, and if dropping changes is |
| // common then handle that here. |
| area->CancelAllPendingRequests(); |
| } |
| |
| // Respect the content policy settings about what to |
| // keep and what to discard. |
| if (force_keep_session_state_) { |
| OnShutdownComplete(leveldb::mojom::DatabaseError::OK); |
| return; // Keep everything. |
| } |
| |
| bool has_session_only_origins = |
| special_storage_policy_.get() && |
| special_storage_policy_->HasSessionOnlyOrigins(); |
| |
| if (has_session_only_origins) { |
| RetrieveStorageUsage( |
| base::BindOnce(&LocalStorageContextMojo::OnGotStorageUsageForShutdown, |
| base::Unretained(this))); |
| } else { |
| OnShutdownComplete(leveldb::mojom::DatabaseError::OK); |
| } |
| } |
| |
| void LocalStorageContextMojo::PurgeMemory() { |
| size_t total_cache_size, unused_area_count; |
| GetStatistics(&total_cache_size, &unused_area_count); |
| |
| for (auto it = areas_.begin(); it != areas_.end();) { |
| if (it->second->has_bindings()) { |
| it->second->storage_area()->PurgeMemory(); |
| ++it; |
| } else { |
| it = areas_.erase(it); |
| } |
| } |
| |
| // Track the size of cache purged. |
| size_t final_total_cache_size; |
| GetStatistics(&final_total_cache_size, &unused_area_count); |
| size_t purged_size_kib = (total_cache_size - final_total_cache_size) / 1024; |
| RecordCachePurgedHistogram(CachePurgeReason::AggressivePurgeTriggered, |
| purged_size_kib); |
| } |
| |
| void LocalStorageContextMojo::PurgeUnusedAreasIfNeeded() { |
| size_t total_cache_size, unused_area_count; |
| GetStatistics(&total_cache_size, &unused_area_count); |
| |
| // Nothing to purge. |
| if (!unused_area_count) |
| return; |
| |
| CachePurgeReason purge_reason = CachePurgeReason::NotNeeded; |
| |
| if (total_cache_size > kMaxLocalStorageCacheSize) |
| purge_reason = CachePurgeReason::SizeLimitExceeded; |
| else if (areas_.size() > kMaxLocalStorageAreaCount) |
| purge_reason = CachePurgeReason::AreaCountLimitExceeded; |
| else if (is_low_end_device_) |
| purge_reason = CachePurgeReason::InactiveOnLowEndDevice; |
| |
| if (purge_reason == CachePurgeReason::NotNeeded) |
| return; |
| |
| for (auto it = areas_.begin(); it != areas_.end();) { |
| if (it->second->has_bindings()) |
| ++it; |
| else |
| it = areas_.erase(it); |
| } |
| |
| // Track the size of cache purged. |
| size_t final_total_cache_size; |
| GetStatistics(&final_total_cache_size, &unused_area_count); |
| size_t purged_size_kib = (total_cache_size - final_total_cache_size) / 1024; |
| RecordCachePurgedHistogram(purge_reason, purged_size_kib); |
| } |
| |
| void LocalStorageContextMojo::SetDatabaseForTesting( |
| leveldb::mojom::LevelDBDatabaseAssociatedPtr database) { |
| DCHECK_EQ(connection_state_, NO_CONNECTION); |
| connection_state_ = CONNECTION_IN_PROGRESS; |
| database_ = std::move(database); |
| OnDatabaseOpened(true, leveldb::mojom::DatabaseError::OK); |
| } |
| |
| bool LocalStorageContextMojo::OnMemoryDump( |
| const base::trace_event::MemoryDumpArgs& args, |
| base::trace_event::ProcessMemoryDump* pmd) { |
| if (connection_state_ != CONNECTION_FINISHED) |
| return true; |
| |
| std::string context_name = |
| base::StringPrintf("site_storage/localstorage/0x%" PRIXPTR, |
| reinterpret_cast<uintptr_t>(this)); |
| |
| // Account for leveldb memory usage, which actually lives in the file service. |
| auto* global_dump = pmd->CreateSharedGlobalAllocatorDump(memory_dump_id_); |
| // The size of the leveldb dump will be added by the leveldb service. |
| auto* leveldb_mad = pmd->CreateAllocatorDump(context_name + "/leveldb"); |
| // Specifies that the current context is responsible for keeping memory alive. |
| int kImportance = 2; |
| pmd->AddOwnershipEdge(leveldb_mad->guid(), global_dump->guid(), kImportance); |
| |
| if (args.level_of_detail == |
| base::trace_event::MemoryDumpLevelOfDetail::BACKGROUND) { |
| size_t total_cache_size, unused_area_count; |
| GetStatistics(&total_cache_size, &unused_area_count); |
| auto* mad = pmd->CreateAllocatorDump(context_name + "/cache_size"); |
| mad->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize, |
| base::trace_event::MemoryAllocatorDump::kUnitsBytes, |
| total_cache_size); |
| mad->AddScalar("total_areas", |
| base::trace_event::MemoryAllocatorDump::kUnitsObjects, |
| areas_.size()); |
| return true; |
| } |
| for (const auto& it : areas_) { |
| // Limit the url length to 50 and strip special characters. |
| std::string url = it.first.Serialize().substr(0, 50); |
| for (size_t index = 0; index < url.size(); ++index) { |
| if (!std::isalnum(url[index])) |
| url[index] = '_'; |
| } |
| std::string area_dump_name = base::StringPrintf( |
| "%s/%s/0x%" PRIXPTR, context_name.c_str(), url.c_str(), |
| reinterpret_cast<uintptr_t>(it.second->storage_area())); |
| it.second->storage_area()->OnMemoryDump(area_dump_name, pmd); |
| } |
| return true; |
| } |
| |
| // static |
| std::vector<uint8_t> LocalStorageContextMojo::MigrateString( |
| const base::string16& input) { |
| // TODO(mek): Deduplicate this somehow with the code in |
| // LocalStorageCachedArea::String16ToUint8Vector. |
| bool is_8bit = true; |
| for (const auto& c : input) { |
| if (c & 0xff00) { |
| is_8bit = false; |
| break; |
| } |
| } |
| if (is_8bit) { |
| std::vector<uint8_t> result(input.size() + 1); |
| result[0] = kLatin1Format; |
| std::copy(input.begin(), input.end(), result.begin() + 1); |
| return result; |
| } |
| const uint8_t* data = reinterpret_cast<const uint8_t*>(input.data()); |
| std::vector<uint8_t> result; |
| result.reserve(input.size() * sizeof(base::char16) + 1); |
| result.push_back(kUTF16Format); |
| result.insert(result.end(), data, data + input.size() * sizeof(base::char16)); |
| return result; |
| } |
| |
| LocalStorageContextMojo::~LocalStorageContextMojo() { |
| DCHECK_EQ(connection_state_, CONNECTION_SHUTDOWN); |
| base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider( |
| this); |
| } |
| |
| void LocalStorageContextMojo::RunWhenConnected(base::OnceClosure callback) { |
| DCHECK_NE(connection_state_, CONNECTION_SHUTDOWN); |
| |
| // If we don't have a filesystem_connection_, we'll need to establish one. |
| if (connection_state_ == NO_CONNECTION) { |
| connection_state_ = CONNECTION_IN_PROGRESS; |
| InitiateConnection(); |
| } |
| |
| if (connection_state_ == CONNECTION_IN_PROGRESS) { |
| // Queue this OpenLocalStorage call for when we have a level db pointer. |
| on_database_opened_callbacks_.push_back(std::move(callback)); |
| return; |
| } |
| |
| std::move(callback).Run(); |
| } |
| |
| void LocalStorageContextMojo::InitiateConnection(bool in_memory_only) { |
| DCHECK_EQ(connection_state_, CONNECTION_IN_PROGRESS); |
| // Unit tests might not always have a Connector, use in-memory only if that |
| // happens. |
| if (!connector_) { |
| OnDatabaseOpened(false, leveldb::mojom::DatabaseError::OK); |
| return; |
| } |
| |
| if (!subdirectory_.empty() && !in_memory_only) { |
| // We were given a subdirectory to write to. Get it and use a disk backed |
| // database. |
| connector_->BindInterface(file::mojom::kServiceName, &file_system_); |
| file_system_->GetSubDirectory( |
| subdirectory_.AsUTF8Unsafe(), MakeRequest(&directory_), |
| base::BindOnce(&LocalStorageContextMojo::OnDirectoryOpened, |
| weak_ptr_factory_.GetWeakPtr())); |
| } else { |
| // We were not given a subdirectory. Use a memory backed database. |
| connector_->BindInterface(file::mojom::kServiceName, &leveldb_service_); |
| leveldb_service_->OpenInMemory( |
| memory_dump_id_, "local-storage", MakeRequest(&database_), |
| base::BindOnce(&LocalStorageContextMojo::OnDatabaseOpened, |
| weak_ptr_factory_.GetWeakPtr(), true)); |
| } |
| } |
| |
| void LocalStorageContextMojo::OnMojoConnectionDestroyed() { |
| UMA_HISTOGRAM_BOOLEAN("LocalStorageContext.OnConnectionDestroyed", true); |
| LOG(ERROR) << "Lost connection to database"; |
| // We're about to set database_ to null, so delete the StorageAreaImpls |
| // that might still be using the old database. |
| for (const auto& it : areas_) |
| it.second->storage_area()->CancelAllPendingRequests(); |
| areas_.clear(); |
| database_ = nullptr; |
| // TODO(dullweber): Should we try to recover? E.g. try to reopen and if this |
| // fails, call DeleteAndRecreateDatabase(). |
| } |
| |
| void LocalStorageContextMojo::OnDirectoryOpened(base::File::Error err) { |
| if (err != base::File::Error::FILE_OK) { |
| // We failed to open the directory; continue with startup so that we create |
| // the |areas_|. |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DirectoryOpenError", -err, |
| -base::File::FILE_ERROR_MAX); |
| LogDatabaseOpenResult(OpenResult::DIRECTORY_OPEN_FAILED); |
| OnDatabaseOpened(false, leveldb::mojom::DatabaseError::OK); |
| return; |
| } |
| |
| // Now that we have a directory, connect to the LevelDB service and get our |
| // database. |
| connector_->BindInterface(file::mojom::kServiceName, &leveldb_service_); |
| |
| // We might still need to use the directory, so create a clone. |
| filesystem::mojom::DirectoryPtr directory_clone; |
| directory_->Clone(MakeRequest(&directory_clone)); |
| |
| leveldb_env::Options options; |
| options.create_if_missing = true; |
| options.max_open_files = 0; // use minimum |
| // Default write_buffer_size is 4 MB but that might leave a 3.999 |
| // memory allocation in RAM from a log file recovery. |
| options.write_buffer_size = 64 * 1024; |
| options.block_cache = leveldb_chrome::GetSharedWebBlockCache(); |
| leveldb_service_->OpenWithOptions( |
| std::move(options), std::move(directory_clone), "leveldb", |
| memory_dump_id_, MakeRequest(&database_), |
| base::BindOnce(&LocalStorageContextMojo::OnDatabaseOpened, |
| weak_ptr_factory_.GetWeakPtr(), false)); |
| } |
| |
| void LocalStorageContextMojo::OnDatabaseOpened( |
| bool in_memory, |
| leveldb::mojom::DatabaseError status) { |
| if (status != leveldb::mojom::DatabaseError::OK) { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError", |
| leveldb::GetLevelDBStatusUMAValue(status), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| if (in_memory) { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError.Memory", |
| leveldb::GetLevelDBStatusUMAValue(status), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| } else { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DatabaseOpenError.Disk", |
| leveldb::GetLevelDBStatusUMAValue(status), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| } |
| LogDatabaseOpenResult(OpenResult::DATABASE_OPEN_FAILED); |
| // If we failed to open the database, try to delete and recreate the |
| // database, or ultimately fallback to an in-memory database. |
| DeleteAndRecreateDatabase("LocalStorageContext.OpenResultAfterOpenFailed"); |
| return; |
| } |
| |
| // Verify DB schema version. |
| if (database_) { |
| database_.set_connection_error_handler( |
| base::BindOnce(&LocalStorageContextMojo::OnMojoConnectionDestroyed, |
| weak_ptr_factory_.GetWeakPtr())); |
| database_->Get( |
| leveldb::StdStringToUint8Vector(kVersionKey), |
| base::BindOnce(&LocalStorageContextMojo::OnGotDatabaseVersion, |
| weak_ptr_factory_.GetWeakPtr())); |
| return; |
| } |
| |
| OnConnectionFinished(); |
| } |
| |
| void LocalStorageContextMojo::OnGotDatabaseVersion( |
| leveldb::mojom::DatabaseError status, |
| const std::vector<uint8_t>& value) { |
| if (status == leveldb::mojom::DatabaseError::NOT_FOUND) { |
| // New database, nothing more to do. Current version will get written |
| // when first data is committed. |
| } else if (status == leveldb::mojom::DatabaseError::OK) { |
| // Existing database, check if version number matches current schema |
| // version. |
| int64_t db_version; |
| if (!base::StringToInt64(leveldb::Uint8VectorToStdString(value), |
| &db_version) || |
| db_version < kMinSchemaVersion || |
| db_version > kCurrentLocalStorageSchemaVersion) { |
| LogDatabaseOpenResult(OpenResult::INVALID_VERSION); |
| DeleteAndRecreateDatabase( |
| "LocalStorageContext.OpenResultAfterInvalidVersion"); |
| return; |
| } |
| |
| database_initialized_ = true; |
| } else { |
| // Other read error. Possibly database corruption. |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.ReadVersionError", |
| leveldb::GetLevelDBStatusUMAValue(status), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| LogDatabaseOpenResult(OpenResult::VERSION_READ_ERROR); |
| DeleteAndRecreateDatabase( |
| "LocalStorageContext.OpenResultAfterReadVersionError"); |
| return; |
| } |
| |
| OnConnectionFinished(); |
| } |
| |
| void LocalStorageContextMojo::OnConnectionFinished() { |
| DCHECK_EQ(connection_state_, CONNECTION_IN_PROGRESS); |
| if (!database_) { |
| directory_.reset(); |
| file_system_.reset(); |
| leveldb_service_.reset(); |
| } |
| |
| // If connection was opened successfully, reset tried_to_recreate_during_open_ |
| // to enable recreating the database on future errors. |
| if (database_) |
| tried_to_recreate_during_open_ = false; |
| |
| LogDatabaseOpenResult(OpenResult::SUCCESS); |
| open_result_histogram_ = nullptr; |
| |
| // |database_| should be known to either be valid or invalid by now. Run our |
| // delayed bindings. |
| connection_state_ = CONNECTION_FINISHED; |
| for (size_t i = 0; i < on_database_opened_callbacks_.size(); ++i) |
| std::move(on_database_opened_callbacks_[i]).Run(); |
| on_database_opened_callbacks_.clear(); |
| } |
| |
| void LocalStorageContextMojo::DeleteAndRecreateDatabase( |
| const char* histogram_name) { |
| // We're about to set database_ to null, so delete the StorageAreaImpls |
| // that might still be using the old database. |
| for (const auto& it : areas_) |
| it.second->storage_area()->CancelAllPendingRequests(); |
| areas_.clear(); |
| |
| // Reset state to be in process of connecting. This will cause requests for |
| // StorageAreas to be queued until the connection is complete. |
| connection_state_ = CONNECTION_IN_PROGRESS; |
| commit_error_count_ = 0; |
| database_ = nullptr; |
| open_result_histogram_ = histogram_name; |
| |
| bool recreate_in_memory = false; |
| |
| // If tried to recreate database on disk already, try again but this time |
| // in memory. |
| if (tried_to_recreate_during_open_ && !subdirectory_.empty()) { |
| recreate_in_memory = true; |
| } else if (tried_to_recreate_during_open_) { |
| // Give up completely, run without any database. |
| OnConnectionFinished(); |
| return; |
| } |
| |
| tried_to_recreate_during_open_ = true; |
| |
| // Unit tests might not have a bound file_service_, in which case there is |
| // nothing to retry. |
| if (!file_system_.is_bound()) { |
| OnConnectionFinished(); |
| return; |
| } |
| |
| // Destroy database, and try again. |
| if (directory_.is_bound()) { |
| leveldb_service_->Destroy( |
| std::move(directory_), "leveldb", |
| base::BindOnce(&LocalStorageContextMojo::OnDBDestroyed, |
| weak_ptr_factory_.GetWeakPtr(), recreate_in_memory)); |
| } else { |
| // No directory, so nothing to destroy. Retrying to recreate will probably |
| // fail, but try anyway. |
| InitiateConnection(recreate_in_memory); |
| } |
| } |
| |
| void LocalStorageContextMojo::OnDBDestroyed( |
| bool recreate_in_memory, |
| leveldb::mojom::DatabaseError status) { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.DestroyDBResult", |
| leveldb::GetLevelDBStatusUMAValue(status), |
| leveldb_env::LEVELDB_STATUS_MAX); |
| // We're essentially ignoring the status here. Even if destroying failed we |
| // still want to go ahead and try to recreate. |
| InitiateConnection(recreate_in_memory); |
| } |
| |
| // The (possibly delayed) implementation of OpenLocalStorage(). Can be called |
| // directly from that function, or through |on_database_open_callbacks_|. |
| void LocalStorageContextMojo::BindLocalStorage( |
| const url::Origin& origin, |
| blink::mojom::StorageAreaRequest request) { |
| GetOrCreateStorageArea(origin)->Bind(std::move(request)); |
| } |
| |
| LocalStorageContextMojo::StorageAreaHolder* |
| LocalStorageContextMojo::GetOrCreateStorageArea(const url::Origin& origin) { |
| DCHECK_EQ(connection_state_, CONNECTION_FINISHED); |
| auto found = areas_.find(origin); |
| if (found != areas_.end()) { |
| return found->second.get(); |
| } |
| |
| size_t total_cache_size, unused_area_count; |
| GetStatistics(&total_cache_size, &unused_area_count); |
| |
| // Track the total localStorage cache size. |
| UMA_HISTOGRAM_COUNTS_100000("LocalStorageContext.CacheSizeInKB", |
| total_cache_size / 1024); |
| |
| PurgeUnusedAreasIfNeeded(); |
| |
| auto holder = std::make_unique<StorageAreaHolder>(this, origin); |
| StorageAreaHolder* holder_ptr = holder.get(); |
| areas_[origin] = std::move(holder); |
| return holder_ptr; |
| } |
| |
| void LocalStorageContextMojo::RetrieveStorageUsage( |
| GetStorageUsageCallback callback) { |
| if (!database_) { |
| // If for whatever reason no leveldb database is available, no storage is |
| // used, so return an array only containing the current areas. |
| std::vector<StorageUsageInfo> result; |
| base::Time now = base::Time::Now(); |
| for (const auto& it : areas_) |
| result.emplace_back(it.first, 0, now); |
| std::move(callback).Run(std::move(result)); |
| return; |
| } |
| |
| database_->GetPrefixed( |
| std::vector<uint8_t>(kMetaPrefix, kMetaPrefix + base::size(kMetaPrefix)), |
| base::BindOnce(&LocalStorageContextMojo::OnGotMetaData, |
| weak_ptr_factory_.GetWeakPtr(), std::move(callback))); |
| } |
| |
| void LocalStorageContextMojo::OnGotMetaData( |
| GetStorageUsageCallback callback, |
| leveldb::mojom::DatabaseError status, |
| std::vector<leveldb::mojom::KeyValuePtr> data) { |
| std::vector<StorageUsageInfo> result; |
| std::set<url::Origin> origins; |
| for (const auto& row : data) { |
| DCHECK_GT(row->key.size(), base::size(kMetaPrefix)); |
| GURL origin(leveldb::Uint8VectorToStdString(row->key).substr( |
| base::size(kMetaPrefix))); |
| |
| origins.insert(url::Origin::Create(origin)); |
| if (!origin.is_valid()) { |
| // TODO(mek): Deal with database corruption. |
| continue; |
| } |
| |
| LocalStorageOriginMetaData row_data; |
| if (!row_data.ParseFromArray(row->value.data(), row->value.size())) { |
| // TODO(mek): Deal with database corruption. |
| continue; |
| } |
| result.emplace_back( |
| url::Origin::Create(origin), row_data.size_bytes(), |
| base::Time::FromInternalValue(row_data.last_modified())); |
| } |
| // Add any origins for which StorageAreas exist, but which haven't |
| // committed any data to disk yet. |
| base::Time now = base::Time::Now(); |
| for (const auto& it : areas_) { |
| if (origins.find(it.first) != origins.end()) |
| continue; |
| // Skip any origins that definitely don't have any data. |
| if (!it.second->storage_area()->has_pending_load_tasks() && |
| it.second->storage_area()->empty()) { |
| continue; |
| } |
| result.emplace_back(it.first, 0, now); |
| } |
| std::move(callback).Run(std::move(result)); |
| } |
| |
| void LocalStorageContextMojo::OnGotStorageUsageForShutdown( |
| std::vector<StorageUsageInfo> usage) { |
| std::vector<leveldb::mojom::BatchedOperationPtr> operations; |
| for (const auto& info : usage) { |
| if (special_storage_policy_->IsStorageProtected(info.origin.GetURL())) |
| continue; |
| if (!special_storage_policy_->IsStorageSessionOnly(info.origin.GetURL())) |
| continue; |
| |
| AddDeleteOriginOperations(&operations, info.origin); |
| } |
| |
| if (!operations.empty()) { |
| database_->Write( |
| std::move(operations), |
| base::BindOnce(&LocalStorageContextMojo::OnShutdownComplete, |
| base::Unretained(this))); |
| } else { |
| OnShutdownComplete(leveldb::mojom::DatabaseError::OK); |
| } |
| } |
| |
| void LocalStorageContextMojo::OnShutdownComplete( |
| leveldb::mojom::DatabaseError error) { |
| delete this; |
| } |
| |
| void LocalStorageContextMojo::GetStatistics(size_t* total_cache_size, |
| size_t* unused_area_count) { |
| *total_cache_size = 0; |
| *unused_area_count = 0; |
| for (const auto& it : areas_) { |
| *total_cache_size += it.second->storage_area()->memory_used(); |
| if (!it.second->has_bindings()) |
| (*unused_area_count)++; |
| } |
| } |
| |
| void LocalStorageContextMojo::OnCommitResult( |
| leveldb::mojom::DatabaseError error) { |
| DCHECK(connection_state_ == CONNECTION_FINISHED || |
| connection_state_ == CONNECTION_SHUTDOWN) |
| << connection_state_; |
| if (error == leveldb::mojom::DatabaseError::OK) { |
| commit_error_count_ = 0; |
| return; |
| } |
| |
| commit_error_count_++; |
| if (commit_error_count_ > kCommitErrorThreshold && |
| connection_state_ != CONNECTION_SHUTDOWN) { |
| if (tried_to_recover_from_commit_errors_) { |
| // We already tried to recover from a high commit error rate before, but |
| // are still having problems: there isn't really anything left to try, so |
| // just ignore errors. |
| return; |
| } |
| tried_to_recover_from_commit_errors_ = true; |
| |
| // Deleting StorageAreas in here could cause more commits (and commit |
| // errors), but those commits won't reach OnCommitResult because the area |
| // will have been deleted before the commit finishes. |
| DeleteAndRecreateDatabase( |
| "LocalStorageContext.OpenResultAfterCommitErrors"); |
| } |
| } |
| |
| void LocalStorageContextMojo::LogDatabaseOpenResult(OpenResult result) { |
| if (result != OpenResult::SUCCESS) { |
| UMA_HISTOGRAM_ENUMERATION("LocalStorageContext.OpenError", result, |
| OpenResult::MAX); |
| } |
| if (open_result_histogram_) { |
| base::UmaHistogramEnumeration(open_result_histogram_, result, |
| OpenResult::MAX); |
| } |
| } |
| |
| } // namespace content |