blob: 8ea46b0552b573f52b4ded3c21fc3a477f74465b [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/url_request/url_request_throttler_manager.h"
#include "base/memory/scoped_ptr.h"
#include "base/metrics/histogram.h"
#include "base/pickle.h"
#include "base/stringprintf.h"
#include "base/string_number_conversions.h"
#include "base/time.h"
#include "net/base/load_flags.h"
#include "net/base/test_completion_callback.h"
#include "net/url_request/url_request_context.h"
#include "net/url_request/url_request_throttler_header_interface.h"
#include "net/url_request/url_request_throttler_test_support.h"
#include "testing/gtest/include/gtest/gtest.h"
using base::TimeDelta;
using base::TimeTicks;
namespace net {
namespace {
using base::Histogram;
using base::StatisticsRecorder;
class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
public:
explicit MockURLRequestThrottlerEntry(
net::URLRequestThrottlerManager* manager)
: net::URLRequestThrottlerEntry(manager, ""),
mock_backoff_entry_(&backoff_policy_) {
InitPolicy();
}
MockURLRequestThrottlerEntry(
net::URLRequestThrottlerManager* manager,
const TimeTicks& exponential_backoff_release_time,
const TimeTicks& sliding_window_release_time,
const TimeTicks& fake_now)
: net::URLRequestThrottlerEntry(manager, ""),
fake_time_now_(fake_now),
mock_backoff_entry_(&backoff_policy_) {
InitPolicy();
mock_backoff_entry_.set_fake_now(fake_now);
set_exponential_backoff_release_time(exponential_backoff_release_time);
set_sliding_window_release_time(sliding_window_release_time);
}
virtual ~MockURLRequestThrottlerEntry() {}
void InitPolicy() {
// Some tests become flaky if we have jitter.
backoff_policy_.jitter_factor = 0.0;
// This lets us avoid having to make multiple failures initially (this
// logic is already tested in the BackoffEntry unit tests).
backoff_policy_.num_errors_to_ignore = 0;
}
const BackoffEntry* GetBackoffEntry() const {
return &mock_backoff_entry_;
}
BackoffEntry* GetBackoffEntry() {
return &mock_backoff_entry_;
}
static bool ExplicitUserRequest(int load_flags) {
return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
}
void ResetToBlank(const TimeTicks& time_now) {
fake_time_now_ = time_now;
mock_backoff_entry_.set_fake_now(time_now);
GetBackoffEntry()->Reset();
GetBackoffEntry()->SetCustomReleaseTime(time_now);
set_sliding_window_release_time(time_now);
}
// Overridden for tests.
virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
void set_exponential_backoff_release_time(
const base::TimeTicks& release_time) {
GetBackoffEntry()->SetCustomReleaseTime(release_time);
}
base::TimeTicks sliding_window_release_time() const {
return URLRequestThrottlerEntry::sliding_window_release_time();
}
void set_sliding_window_release_time(
const base::TimeTicks& release_time) {
URLRequestThrottlerEntry::set_sliding_window_release_time(
release_time);
}
TimeTicks fake_time_now_;
MockBackoffEntry mock_backoff_entry_;
};
class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
public:
MockURLRequestThrottlerManager() : create_entry_index_(0) {}
// Method to process the URL using URLRequestThrottlerManager protected
// method.
std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
// Method to use the garbage collecting method of URLRequestThrottlerManager.
void DoGarbageCollectEntries() { GarbageCollectEntries(); }
// Returns the number of entries in the map.
int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
void CreateEntry(bool is_outdated) {
TimeTicks time = TimeTicks::Now();
if (is_outdated) {
time -= TimeDelta::FromMilliseconds(
MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
}
std::string fake_url_string("http://www.fakeurl.com/");
fake_url_string.append(base::IntToString(create_entry_index_++));
GURL fake_url(fake_url_string);
OverrideEntryForTests(
fake_url,
new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
TimeTicks::Now()));
}
private:
int create_entry_index_;
};
struct TimeAndBool {
TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
time = time_value;
result = expected;
line = line_num;
}
TimeTicks time;
bool result;
int line;
};
struct GurlAndString {
GurlAndString(const GURL& url_value,
const std::string& expected,
int line_num) {
url = url_value;
result = expected;
line = line_num;
}
GURL url;
std::string result;
int line;
};
} // namespace
class URLRequestThrottlerEntryTest : public testing::Test {
protected:
virtual void SetUp();
// After calling this function, histogram snapshots in |samples_| contain
// only the delta caused by the test case currently running.
void CalculateHistogramDeltas();
TimeTicks now_;
MockURLRequestThrottlerManager manager_; // Dummy object, not used.
scoped_refptr<MockURLRequestThrottlerEntry> entry_;
std::map<std::string, Histogram::SampleSet> original_samples_;
std::map<std::string, Histogram::SampleSet> samples_;
};
// List of all histograms we care about in these unit tests.
const char* kHistogramNames[] = {
"Throttling.CustomRetryAfterMs",
"Throttling.FailureCountAtSuccess",
"Throttling.PerceivedDowntime",
"Throttling.RequestThrottled",
"Throttling.SiteOptedOut",
};
void URLRequestThrottlerEntryTest::SetUp() {
now_ = TimeTicks::Now();
entry_ = new MockURLRequestThrottlerEntry(&manager_);
entry_->ResetToBlank(now_);
for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
// Must retrieve original samples for each histogram for comparison
// as other tests may affect them.
const char* name = kHistogramNames[i];
Histogram::SampleSet& original = original_samples_[name];
Histogram* histogram;
if (StatisticsRecorder::FindHistogram(name, &histogram)) {
histogram->SnapshotSample(&original);
}
}
}
void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
const char* name = kHistogramNames[i];
Histogram::SampleSet& original = original_samples_[name];
Histogram::SampleSet& sample = samples_[name];
Histogram* histogram;
if (StatisticsRecorder::FindHistogram(name, &histogram)) {
ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag, histogram->flags());
histogram->SnapshotSample(&sample);
// Ensure |original| size is same as |sample|, then subtract original
// values.
original.Resize(*histogram);
sample.Subtract(original);
}
}
// Ensure we don't accidentally use the originals in our tests.
original_samples_.clear();
}
std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
return out << time.ToInternalValue();
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
entry_->set_exponential_backoff_release_time(
entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
EXPECT_TRUE(entry_->ShouldRejectRequest(0));
// Also end-to-end test the load flags exceptions.
EXPECT_FALSE(entry_->ShouldRejectRequest(LOAD_MAYBE_USER_GESTURE));
CalculateHistogramDeltas();
ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(0));
ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(1));
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
EXPECT_FALSE(entry_->ShouldRejectRequest(0));
entry_->set_exponential_backoff_release_time(
entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
EXPECT_FALSE(entry_->ShouldRejectRequest(0));
CalculateHistogramDeltas();
ASSERT_EQ(2, samples_["Throttling.RequestThrottled"].counts(0));
ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(1));
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
// If the response we received has a retry-after field,
// the request should be delayed.
MockURLRequestThrottlerHeaderAdapter header_w_delay_header("5.5", "", 200);
entry_->UpdateWithResponse("", &header_w_delay_header);
EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
<< "When the server put a positive value in retry-after we should "
"increase release_time";
entry_->ResetToBlank(now_);
MockURLRequestThrottlerHeaderAdapter header_w_negative_header(
"-5.5", "", 200);
entry_->UpdateWithResponse("", &header_w_negative_header);
EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
<< "When given a negative value, it should not change the release_time";
CalculateHistogramDeltas();
ASSERT_EQ(1, samples_["Throttling.CustomRetryAfterMs"].TotalCount());
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
MockURLRequestThrottlerHeaderAdapter failure_response(503);
entry_->UpdateWithResponse("", &failure_response);
EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
<< "A failure should increase the release_time";
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
MockURLRequestThrottlerHeaderAdapter success_response(200);
entry_->UpdateWithResponse("", &success_response);
EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
<< "A success should not add any delay";
}
TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
MockURLRequestThrottlerHeaderAdapter failure_response(503);
MockURLRequestThrottlerHeaderAdapter success_response(200);
entry_->UpdateWithResponse("", &success_response);
entry_->UpdateWithResponse("", &failure_response);
EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
<< "This scenario should add delay";
entry_->UpdateWithResponse("", &success_response);
CalculateHistogramDeltas();
ASSERT_EQ(1, samples_["Throttling.FailureCountAtSuccess"].counts(1));
ASSERT_EQ(1, samples_["Throttling.PerceivedDowntime"].TotalCount());
}
TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
TimeDelta lifetime = TimeDelta::FromMilliseconds(
MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
TimeAndBool test_values[] = {
TimeAndBool(now_, false, __LINE__),
TimeAndBool(now_ - kFiveMs, false, __LINE__),
TimeAndBool(now_ + kFiveMs, false, __LINE__),
TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
TimeAndBool(now_ - lifetime, true, __LINE__),
TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
for (unsigned int i = 0; i < arraysize(test_values); ++i) {
entry_->set_exponential_backoff_release_time(test_values[i].time);
EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
"Test case #" << i << " line " << test_values[i].line << " failed";
}
}
TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
for (int i = 0; i < 30; ++i) {
MockURLRequestThrottlerHeaderAdapter response_adapter(503);
entry_->UpdateWithResponse("", &response_adapter);
}
TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
EXPECT_EQ(delay.InMilliseconds(),
MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
}
TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
MockURLRequestThrottlerHeaderAdapter response_adapter(503);
for (int i = 0; i < 5; ++i)
entry_->UpdateWithResponse("", &response_adapter);
TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
// Inform the entry that a response body was malformed, which is supposed to
// increase the back-off time. Note that we also submit a successful
// UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
// is what happens in practice (if a body is received, then a non-500
// response must also have been received).
entry_->ReceivedContentWasMalformed(200);
MockURLRequestThrottlerHeaderAdapter success_adapter(200);
entry_->UpdateWithResponse("", &success_adapter);
EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
}
TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
int sliding_window =
URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
TimeTicks time_1 = entry_->fake_time_now_ +
TimeDelta::FromMilliseconds(sliding_window / 3);
TimeTicks time_2 = entry_->fake_time_now_ +
TimeDelta::FromMilliseconds(2 * sliding_window / 3);
TimeTicks time_3 = entry_->fake_time_now_ +
TimeDelta::FromMilliseconds(sliding_window);
TimeTicks time_4 = entry_->fake_time_now_ +
TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
entry_->set_exponential_backoff_release_time(time_1);
for (int i = 0; i < max_send / 2; ++i) {
EXPECT_EQ(2 * sliding_window / 3,
entry_->ReserveSendingTimeForNextRequest(time_2));
}
EXPECT_EQ(time_2, entry_->sliding_window_release_time());
entry_->fake_time_now_ = time_3;
for (int i = 0; i < (max_send + 1) / 2; ++i)
EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
EXPECT_EQ(time_4, entry_->sliding_window_release_time());
}
TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
LOAD_MAYBE_USER_GESTURE));
ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
~LOAD_MAYBE_USER_GESTURE));
}
TEST(URLRequestThrottlerManager, IsUrlStandardised) {
MockURLRequestThrottlerManager manager;
GurlAndString test_values[] = {
GurlAndString(GURL("http://www.example.com"),
std::string("http://www.example.com/"),
__LINE__),
GurlAndString(GURL("http://www.Example.com"),
std::string("http://www.example.com/"),
__LINE__),
GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
std::string("http://www.ex4mple.com/pr4c71c41"),
__LINE__),
GurlAndString(GURL("http://www.example.com/0/token/false"),
std::string("http://www.example.com/0/token/false"),
__LINE__),
GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
std::string("http://www.example.com/index.php"),
__LINE__),
GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
std::string("http://www.example.com/index.php"),
__LINE__),
GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
std::string("http://www.example.com/index.php"),
__LINE__),
GurlAndString(GURL("http://www.example.com:1234/"),
std::string("http://www.example.com:1234/"),
__LINE__)};
for (unsigned int i = 0; i < arraysize(test_values); ++i) {
std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
EXPECT_EQ(temp, test_values[i].result) <<
"Test case #" << i << " line " << test_values[i].line << " failed";
}
}
TEST(URLRequestThrottlerManager, AreEntriesBeingCollected) {
MockURLRequestThrottlerManager manager;
manager.CreateEntry(true); // true = Entry is outdated.
manager.CreateEntry(true);
manager.CreateEntry(true);
manager.DoGarbageCollectEntries();
EXPECT_EQ(0, manager.GetNumberOfEntries());
manager.CreateEntry(false);
manager.CreateEntry(false);
manager.CreateEntry(false);
manager.CreateEntry(true);
manager.DoGarbageCollectEntries();
EXPECT_EQ(3, manager.GetNumberOfEntries());
}
TEST(URLRequestThrottlerManager, IsHostBeingRegistered) {
MockURLRequestThrottlerManager manager;
manager.RegisterRequestUrl(GURL("http://www.example.com/"));
manager.RegisterRequestUrl(GURL("http://www.google.com/"));
manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
EXPECT_EQ(3, manager.GetNumberOfEntries());
}
void ExpectEntryAllowsAllOnErrorIfOptedOut(
net::URLRequestThrottlerEntryInterface* entry,
bool opted_out) {
EXPECT_FALSE(entry->ShouldRejectRequest(0));
MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
for (int i = 0; i < 10; ++i) {
// Host doesn't really matter in this scenario so we skip it.
entry->UpdateWithResponse("", &failure_adapter);
}
EXPECT_NE(opted_out, entry->ShouldRejectRequest(0));
if (opted_out) {
// We're not mocking out GetTimeNow() in this scenario
// so add a 100 ms buffer to avoid flakiness (that should always
// give enough time to get from the TimeTicks::Now() call here
// to the TimeTicks::Now() call in the entry class).
EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
entry->GetExponentialBackoffReleaseTime());
} else {
// As above, add 100 ms.
EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
entry->GetExponentialBackoffReleaseTime());
}
}
TEST(URLRequestThrottlerManager, OptOutHeader) {
MockURLRequestThrottlerManager manager;
scoped_refptr<net::URLRequestThrottlerEntryInterface> entry =
manager.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
// Fake a response with the opt-out header.
MockURLRequestThrottlerHeaderAdapter response_adapter(
"",
MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue,
200);
entry->UpdateWithResponse("www.google.com", &response_adapter);
// Ensure that the same entry on error always allows everything.
ExpectEntryAllowsAllOnErrorIfOptedOut(entry, true);
// Ensure that a freshly created entry (for a different URL on an
// already opted-out host) also gets "always allow" behavior.
scoped_refptr<net::URLRequestThrottlerEntryInterface> other_entry =
manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry, true);
// Fake a response with the opt-out header incorrectly specified.
scoped_refptr<net::URLRequestThrottlerEntryInterface> no_opt_out_entry =
manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
MockURLRequestThrottlerHeaderAdapter wrong_adapter("", "yesplease", 200);
no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry, false);
// A localhost entry should always be opted out.
scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry =
manager.RegisterRequestUrl(GURL("http://localhost/hello"));
ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true);
}
TEST(URLRequestThrottlerManager, ClearOnNetworkChange) {
for (int i = 0; i < 3; ++i) {
MockURLRequestThrottlerManager manager;
scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before =
manager.RegisterRequestUrl(GURL("http://www.example.com/"));
MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
for (int j = 0; j < 10; ++j) {
// Host doesn't really matter in this scenario so we skip it.
entry_before->UpdateWithResponse("", &failure_adapter);
}
EXPECT_TRUE(entry_before->ShouldRejectRequest(0));
switch (i) {
case 0:
manager.OnIPAddressChanged();
break;
case 1:
manager.OnOnlineStateChanged(true);
break;
case 2:
manager.OnOnlineStateChanged(false);
break;
default:
FAIL();
}
scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after =
manager.RegisterRequestUrl(GURL("http://www.example.com/"));
EXPECT_FALSE(entry_after->ShouldRejectRequest(0));
}
}
} // namespace net