blob: 0f1eafdf58a7dda73fb50dd8b8ac8b4ff2098a5a [file] [log] [blame]
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/log/bounded_file_net_log_observer.h"
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "base/bind.h"
#include "base/files/file_util.h"
#include "base/json/json_writer.h"
#include "base/logging.h"
#include "base/strings/string_number_conversions.h"
#include "base/synchronization/lock.h"
#include "base/threading/thread.h"
#include "base/values.h"
#include "net/log/net_log_util.h"
#include "net/url_request/url_request_context.h"
namespace {
// Number of events that can build up in |write_queue_| before file thread
// is triggered to drain the queue.
const int kNumWriteQueueEvents = 15;
} // namespace
namespace net {
// Used to store events to be written to file.
using EventQueue = std::queue<std::unique_ptr<std::string>>;
// WriteQueue receives events from BoundedFileNetLogObserver on the main
// thread and holds them in a queue until they are drained from the queue
// and written to file on the file thread.
// WriteQueue contains the resources shared between the main thread and the
// file thread. |lock_| must be acquired to read or write to |queue_| and
// |memory_|.
// WriteQueue is refcounted and should be destroyed once all events on the
// file thread have finished executing.
class BoundedFileNetLogObserver::WriteQueue
: public base::RefCountedThreadSafe<WriteQueue> {
// |memory_max| indicates the maximum amount of memory that the virtual write
// queue can use. If |memory_| exceeds |memory_max_|, the |queue_| of events
// is overwritten.
WriteQueue(size_t memory_max);
// Adds |event| to |queue_|. Also manages the size of |memory_|; if it
// exceeds |memory_max_|, then old events are dropped from |queue_| without
// being written to file.
// Returns the number of events in the |queue_|.
size_t AddEntryToQueue(std::unique_ptr<std::string> event);
// Swaps |queue_| with |local_queue|. |local_queue| should be empty, so that
// |queue_| is emptied. Resets |memory_| to 0.
void SwapQueue(EventQueue* local_queue);
friend class base::RefCountedThreadSafe<WriteQueue>;
// Queue of events to be written shared between main thread and file thread.
// Main thread adds events to the queue and the file thread drains them and
// writes the events to file.
// |lock_| must be acquired to read or write to this.
EventQueue queue_;
// Tracks how much memory is being used by the virtual write queue.
// Incremented in AddEntryToQueue() when events are added to the
// buffer, and decremented when SwapQueue() is called and the file thread's
// local queue is swapped with the shared write queue.
// |lock_| must be acquired to read or write to this.
size_t memory_;
// Indicates the maximum amount of memory that the |queue_| is allowed to
// use.
const size_t memory_max_;
// Protects access to |queue_| and |memory_|.
// A lock is necessary because |queue_| and |memory_| are shared between the
// file thread and the main thread. NetLog's lock protects OnAddEntry(),
// which calls AddEntryToQueue(), but it does not protect access to the
// observer's member variables. Thus, a race condition exists if a thread is
// calling OnAddEntry() at the same time that the file thread is accessing
// |memory_| and |queue_| to write events to file. The |queue_| and |memory_|
// counter are necessary to bound the amount of memory that is used for the
// queue in the event that the file thread lags significantly behind the main
// thread in writing events to file.
base::Lock lock_;
// FileWriter drains events from WriteQueue and writes them to file.
// Owned by BoundedFileNetLogObserver. FileWriter can be constructed on any
// thread, and afterwards is only accessed on the file thread.
class BoundedFileNetLogObserver::FileWriter {
FileWriter(const base::FilePath& path,
size_t max_file_size,
size_t total_num_files,
scoped_refptr<base::SingleThreadTaskRunner> task_runner);
// Writes |constants_value| to constants.json, and opens the
// events array (closed in Stop()).
void Initialize(std::unique_ptr<base::Value> constants_value);
// Closes the events array opened in Initialize() and writes |tab_info| to
// end_netlog.json. If |tab_info| cannot be converted to proper JSON, then it
// is ignored.
void Stop(std::unique_ptr<base::Value> tab_info);
// Drains |queue_| from WriteQueue into a local file queue and writes the
// events in the queue to file.
void Flush(scoped_refptr<WriteQueue> write_queue);
// Deletes all netlog files, including constants.json and end_netlog.json.
// It is not valid to call any method of BoundedFileNetLogObserver after
// DeleteAllFiles().
void DeleteAllFiles();
// Increments |current_file_idx_|, and handles the clearing and openining of
// the new current file. Also sets |event_files_[current_file_idx_]| to point
// to the new current file.
void IncrementCurrentFile();
// Each ScopedFILE points to a netlog event file with the file name
// "event_file_<index>.json".
std::vector<base::ScopedFILE> event_files_;
// The directory where the netlog files are created.
const base::FilePath directory_;
// Indicates the total number of netlog event files, which does not include
// the constants file (constants.json), or closing file (end_netlog.json).
const size_t total_num_files_;
// Indicates the index of the file in |event_files_| currently being written
// into.
size_t current_file_idx_;
// Indicates the maximum size of each individual netlogging file, excluding
// the constant file.
const size_t max_file_size_;
// Task runner from the file thread.
const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
scoped_refptr<base::SingleThreadTaskRunner> task_runner)
: capture_mode_(NetLogCaptureMode::Default()), task_runner_(task_runner) {}
BoundedFileNetLogObserver::~BoundedFileNetLogObserver() {
if (net_log()) {
// StopObserving was not called.
task_runner_->DeleteSoon(FROM_HERE, file_writer_);
void BoundedFileNetLogObserver::set_capture_mode(
NetLogCaptureMode capture_mode) {
capture_mode_ = capture_mode;
void BoundedFileNetLogObserver::StartObserving(
NetLog* net_log,
const base::FilePath& filepath,
base::Value* constants,
URLRequestContext* url_request_context,
size_t max_total_size,
size_t total_num_files) {
DCHECK_GT(total_num_files, 0u);
file_writer_ = new FileWriter(filepath, max_total_size / total_num_files,
total_num_files, task_runner_);
// The |file_writer_| uses a soft limit to write events to file that allows
// the size of the file to exceed the limit, but the |write_queue_| uses a
// hard limit which the size of the |queue_| cannot exceed. Thus, the
// |file_writer_| may write more events to file than can be contained by the
// |write_queue_| if they have the same size limit. The maximum size of the
// |write_queue_| is doubled to allow the |queue_| to hold enough events for
// the |file_writer_| to fill all files. As long as all events have sizes <=
// the size of an individual event file, the discrepancy between the hard
// limit and the soft limit will not cause an issue.
// TODO(dconnol): Handle the case when the |write_queue_| still doesn't
// contain enough events to fill all files, because of very large events
// relative to file size.
write_queue_ = make_scoped_refptr(new WriteQueue(max_total_size * 2));
FROM_HERE, base::Bind(&BoundedFileNetLogObserver::FileWriter::Initialize,
base::Passed(constants ? constants->CreateDeepCopy()
: GetNetConstants())));
if (url_request_context) {
std::set<URLRequestContext*> contexts;
CreateNetLogEntriesForActiveObjects(contexts, this);
net_log->DeprecatedAddObserver(this, capture_mode_);
void BoundedFileNetLogObserver::StopObserving(
URLRequestContext* url_request_context,
const base::Closure& callback) {
FROM_HERE, base::Bind(&BoundedFileNetLogObserver::FileWriter::Flush,
base::Unretained(file_writer_), write_queue_));
FROM_HERE, base::Bind(&BoundedFileNetLogObserver::FileWriter::Stop,
? GetNetInfo(url_request_context,
: nullptr)),
void BoundedFileNetLogObserver::OnAddEntry(const NetLog::Entry& entry) {
std::unique_ptr<std::string> json(new std::string);
// If |entry| cannot be converted to proper JSON, ignore it.
if (!base::JSONWriter::Write(*entry.ToValue(), json.get()))
size_t queue_size = write_queue_->AddEntryToQueue(std::move(json));
// If events build up in |write_queue_|, trigger the file thread to drain
// the queue.
if (queue_size >= kNumWriteQueueEvents) {
FROM_HERE, base::Bind(&BoundedFileNetLogObserver::FileWriter::Flush,
base::Unretained(file_writer_), write_queue_));
BoundedFileNetLogObserver::WriteQueue::WriteQueue(size_t memory_max)
: memory_(0), memory_max_(memory_max) {}
size_t BoundedFileNetLogObserver::WriteQueue::AddEntryToQueue(
std::unique_ptr<std::string> event) {
base::AutoLock lock(lock_);
memory_ += event->size();
while (memory_ > memory_max_ && !queue_.empty()) {
// Delete oldest events in the queue.
memory_ -= queue_.front()->size();
return queue_.size();
void BoundedFileNetLogObserver::WriteQueue::SwapQueue(EventQueue* local_queue) {
base::AutoLock lock(lock_);
memory_ = 0;
BoundedFileNetLogObserver::WriteQueue::~WriteQueue() {}
const base::FilePath& path,
size_t max_file_size,
size_t total_num_files,
scoped_refptr<base::SingleThreadTaskRunner> task_runner)
: directory_(path),
task_runner_(task_runner) {
BoundedFileNetLogObserver::FileWriter::~FileWriter() {}
void BoundedFileNetLogObserver::FileWriter::Initialize(
std::unique_ptr<base::Value> constants_value) {
event_files_[current_file_idx_] = base::ScopedFILE(
base::OpenFile(directory_.AppendASCII("event_file_0.json"), "w"));
base::ScopedFILE constants_file(
base::OpenFile(directory_.AppendASCII("constants.json"), "w"));
// Print constants to file and open events array.
std::string json;
// It should always be possible to convert constants to JSON.
if (!base::JSONWriter::Write(*constants_value, &json))
fprintf(constants_file.get(), "{\"constants\":%s,\n\"events\": [\n",
void BoundedFileNetLogObserver::FileWriter::Stop(
std::unique_ptr<base::Value> tab_info) {
base::ScopedFILE closing_file(
base::OpenFile(directory_.AppendASCII("end_netlog.json"), "w"));
std::string json;
if (tab_info)
base::JSONWriter::Write(*tab_info, &json);
fprintf(closing_file.get(), "]%s}",
json.empty() ? "" : (",\"tabInfo\": " + json + "\n").c_str());
// Flush all fprintfs to disk so that files can be safely accessed on
// callback.
void BoundedFileNetLogObserver::FileWriter::IncrementCurrentFile() {
current_file_idx_ %= total_num_files_;
event_files_[current_file_idx_] = base::ScopedFILE(base::OpenFile(
directory_.AppendASCII("event_file_" +
base::SizeTToString(current_file_idx_) + ".json"),
void BoundedFileNetLogObserver::FileWriter::Flush(
scoped_refptr<BoundedFileNetLogObserver::WriteQueue> write_queue) {
EventQueue local_file_queue;
std::string to_print;
size_t file_size = ftell(event_files_[current_file_idx_].get());
size_t memory_freed = 0;
while (!local_file_queue.empty()) {
if (file_size >= max_file_size_) {
// The current file is full. Start a new current file.
file_size = 0;
fprintf(event_files_[current_file_idx_].get(), "%s,\n",
file_size += local_file_queue.front()->size();
memory_freed += local_file_queue.front()->size();
void BoundedFileNetLogObserver::FileWriter::DeleteAllFiles() {
// Reset |event_files_| to release all file handles so base::DeleteFile can
// safely access files.
base::DeleteFile(directory_.AppendASCII("constants.json"), false);
base::DeleteFile(directory_.AppendASCII("end_netlog.json"), false);
for (size_t i = 0; i < total_num_files_; i++) {
base::DeleteFile(directory_.AppendASCII("event_file_" +
base::SizeTToString(i) + ".json"),
} // namespace net