blob: 4c0e127bd63e4522f2d26a810e7020c235bed179 [file] [log] [blame]
// Copyright (c) 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "google_apis/gcm/engine/connection_factory_impl.h"
#include <cmath>
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "base/test/simple_test_tick_clock.h"
#include "google_apis/gcm/base/mcs_util.h"
#include "google_apis/gcm/engine/fake_connection_handler.h"
#include "google_apis/gcm/monitoring/fake_gcm_stats_recorder.h"
#include "net/base/backoff_entry.h"
#include "net/http/http_network_session.h"
#include "testing/gtest/include/gtest/gtest.h"
class Policy;
namespace gcm {
namespace {
const char kMCSEndpoint[] = "http://my.server";
const char kMCSEndpoint2[] = "http://my.alt.server";
const int kBackoffDelayMs = 1;
const int kBackoffMultiplier = 2;
// A backoff policy with small enough delays that tests aren't burdened.
const net::BackoffEntry::Policy kTestBackoffPolicy = {
// Number of initial errors (in sequence) to ignore before applying
// exponential back-off rules.
0,
// Initial delay for exponential back-off in ms.
kBackoffDelayMs,
// Factor by which the waiting time will be multiplied.
kBackoffMultiplier,
// Fuzzing percentage. ex: 10% will spread requests randomly
// between 90%-100% of the calculated time.
0,
// Maximum amount of time we are willing to delay our request in ms.
10,
// Time to keep an entry from being discarded even when it
// has no significant state, -1 to never discard.
-1,
// Don't use initial delay unless the last request was an error.
false,
};
std::vector<GURL> BuildEndpoints() {
std::vector<GURL> endpoints;
endpoints.push_back(GURL(kMCSEndpoint));
endpoints.push_back(GURL(kMCSEndpoint2));
return endpoints;
}
// Helper for calculating total expected exponential backoff delay given an
// arbitrary number of failed attempts. See BackoffEntry::CalculateReleaseTime.
double CalculateBackoff(int num_attempts) {
double delay = kBackoffDelayMs;
for (int i = 1; i < num_attempts; ++i) {
delay += kBackoffDelayMs * pow(static_cast<double>(kBackoffMultiplier),
i - 1);
}
DVLOG(1) << "Expected backoff " << delay << " milliseconds.";
return delay;
}
void ReadContinuation(
scoped_ptr<google::protobuf::MessageLite> message) {
}
void WriteContinuation() {
}
// A connection factory that stubs out network requests and overrides the
// backoff policy.
class TestConnectionFactoryImpl : public ConnectionFactoryImpl {
public:
TestConnectionFactoryImpl(const base::Closure& finished_callback);
~TestConnectionFactoryImpl() override;
void InitializeFactory();
// Overridden stubs.
void ConnectImpl() override;
void InitHandler() override;
scoped_ptr<net::BackoffEntry> CreateBackoffEntry(
const net::BackoffEntry::Policy* const policy) override;
scoped_ptr<ConnectionHandler> CreateConnectionHandler(
base::TimeDelta read_timeout,
const ConnectionHandler::ProtoReceivedCallback& read_callback,
const ConnectionHandler::ProtoSentCallback& write_callback,
const ConnectionHandler::ConnectionChangedCallback& connection_callback)
override;
base::TimeTicks NowTicks() override;
// Helpers for verifying connection attempts are made. Connection results
// must be consumed.
void SetConnectResult(int connect_result);
void SetMultipleConnectResults(int connect_result, int num_expected_attempts);
// Force a login handshake to be delayed.
void SetDelayLogin(bool delay_login);
// Simulate a socket error.
void SetSocketError();
base::SimpleTestTickClock* tick_clock() { return &tick_clock_; }
private:
// Clock for controlling delay.
base::SimpleTestTickClock tick_clock_;
// The result to return on the next connect attempt.
int connect_result_;
// The number of expected connection attempts;
int num_expected_attempts_;
// Whether all expected connection attempts have been fulfilled since an
// expectation was last set.
bool connections_fulfilled_;
// Whether to delay a login handshake completion or not.
bool delay_login_;
// Callback to invoke when all connection attempts have been made.
base::Closure finished_callback_;
// A temporary scoped pointer to make sure we don't leak the handler in the
// cases it's never consumed by the ConnectionFactory.
scoped_ptr<FakeConnectionHandler> scoped_handler_;
// The current fake connection handler..
FakeConnectionHandler* fake_handler_;
// Dummy GCM Stats recorder.
FakeGCMStatsRecorder dummy_recorder_;
};
TestConnectionFactoryImpl::TestConnectionFactoryImpl(
const base::Closure& finished_callback)
: ConnectionFactoryImpl(BuildEndpoints(),
net::BackoffEntry::Policy(),
NULL,
NULL,
NULL,
&dummy_recorder_),
connect_result_(net::ERR_UNEXPECTED),
num_expected_attempts_(0),
connections_fulfilled_(true),
delay_login_(false),
finished_callback_(finished_callback),
scoped_handler_(
new FakeConnectionHandler(base::Bind(&ReadContinuation),
base::Bind(&WriteContinuation))),
fake_handler_(scoped_handler_.get()) {
// Set a non-null time.
tick_clock_.Advance(base::TimeDelta::FromMilliseconds(1));
}
TestConnectionFactoryImpl::~TestConnectionFactoryImpl() {
EXPECT_EQ(0, num_expected_attempts_);
}
void TestConnectionFactoryImpl::ConnectImpl() {
ASSERT_GT(num_expected_attempts_, 0);
ASSERT_FALSE(GetConnectionHandler()->CanSendMessage());
scoped_ptr<mcs_proto::LoginRequest> request(BuildLoginRequest(0, 0, ""));
GetConnectionHandler()->Init(*request, NULL);
OnConnectDone(connect_result_);
if (!NextRetryAttempt().is_null()) {
// Advance the time to the next retry time.
base::TimeDelta time_till_retry =
NextRetryAttempt() - tick_clock_.NowTicks();
tick_clock_.Advance(time_till_retry);
}
--num_expected_attempts_;
if (num_expected_attempts_ == 0) {
connect_result_ = net::ERR_UNEXPECTED;
connections_fulfilled_ = true;
finished_callback_.Run();
}
}
void TestConnectionFactoryImpl::InitHandler() {
EXPECT_NE(connect_result_, net::ERR_UNEXPECTED);
if (!delay_login_)
ConnectionHandlerCallback(net::OK);
}
scoped_ptr<net::BackoffEntry> TestConnectionFactoryImpl::CreateBackoffEntry(
const net::BackoffEntry::Policy* const policy) {
return make_scoped_ptr(new net::BackoffEntry(&kTestBackoffPolicy,
&tick_clock_));
}
scoped_ptr<ConnectionHandler>
TestConnectionFactoryImpl::CreateConnectionHandler(
base::TimeDelta read_timeout,
const ConnectionHandler::ProtoReceivedCallback& read_callback,
const ConnectionHandler::ProtoSentCallback& write_callback,
const ConnectionHandler::ConnectionChangedCallback& connection_callback) {
return scoped_handler_.Pass();
}
base::TimeTicks TestConnectionFactoryImpl::NowTicks() {
return tick_clock_.NowTicks();
}
void TestConnectionFactoryImpl::SetConnectResult(int connect_result) {
DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
ASSERT_EQ(0, num_expected_attempts_);
connections_fulfilled_ = false;
connect_result_ = connect_result;
num_expected_attempts_ = 1;
fake_handler_->ExpectOutgoingMessage(
MCSMessage(kLoginRequestTag, BuildLoginRequest(0, 0, "")));
}
void TestConnectionFactoryImpl::SetMultipleConnectResults(
int connect_result,
int num_expected_attempts) {
DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
DCHECK_GT(num_expected_attempts, 0);
ASSERT_EQ(0, num_expected_attempts_);
connections_fulfilled_ = false;
connect_result_ = connect_result;
num_expected_attempts_ = num_expected_attempts;
for (int i = 0 ; i < num_expected_attempts; ++i) {
fake_handler_->ExpectOutgoingMessage(
MCSMessage(kLoginRequestTag, BuildLoginRequest(0, 0, "")));
}
}
void TestConnectionFactoryImpl::SetDelayLogin(bool delay_login) {
delay_login_ = delay_login;
fake_handler_->set_fail_login(delay_login_);
}
void TestConnectionFactoryImpl::SetSocketError() {
fake_handler_->set_had_error(true);
}
} // namespace
class ConnectionFactoryImplTest
: public testing::Test,
public ConnectionFactory::ConnectionListener {
public:
ConnectionFactoryImplTest();
~ConnectionFactoryImplTest() override;
TestConnectionFactoryImpl* factory() { return &factory_; }
GURL& connected_server() { return connected_server_; }
void WaitForConnections();
// ConnectionFactory::ConnectionListener
void OnConnected(const GURL& current_server,
const net::IPEndPoint& ip_endpoint) override;
void OnDisconnected() override;
private:
void ConnectionsComplete();
TestConnectionFactoryImpl factory_;
base::MessageLoop message_loop_;
scoped_ptr<base::RunLoop> run_loop_;
GURL connected_server_;
};
ConnectionFactoryImplTest::ConnectionFactoryImplTest()
: factory_(base::Bind(&ConnectionFactoryImplTest::ConnectionsComplete,
base::Unretained(this))),
run_loop_(new base::RunLoop()) {
factory()->SetConnectionListener(this);
factory()->Initialize(
ConnectionFactory::BuildLoginRequestCallback(),
ConnectionHandler::ProtoReceivedCallback(),
ConnectionHandler::ProtoSentCallback());
}
ConnectionFactoryImplTest::~ConnectionFactoryImplTest() {}
void ConnectionFactoryImplTest::WaitForConnections() {
run_loop_->Run();
run_loop_.reset(new base::RunLoop());
}
void ConnectionFactoryImplTest::ConnectionsComplete() {
if (!run_loop_)
return;
run_loop_->Quit();
}
void ConnectionFactoryImplTest::OnConnected(
const GURL& current_server,
const net::IPEndPoint& ip_endpoint) {
connected_server_ = current_server;
}
void ConnectionFactoryImplTest::OnDisconnected() {
connected_server_ = GURL();
}
// Verify building a connection handler works.
TEST_F(ConnectionFactoryImplTest, Initialize) {
ASSERT_FALSE(factory()->GetConnectionHandler());
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
}
// An initial successful connection should not result in backoff.
TEST_F(ConnectionFactoryImplTest, ConnectSuccess) {
factory()->SetConnectResult(net::OK);
factory()->Connect();
ASSERT_TRUE(factory()->GetConnectionHandler());
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[0]);
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
}
// A connection failure should result in backoff, and attempting the fallback
// endpoint next.
TEST_F(ConnectionFactoryImplTest, ConnectFail) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
factory()->Connect();
EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[1]);
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
}
// A connection success after a failure should reset backoff.
TEST_F(ConnectionFactoryImplTest, FailThenSucceed) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
factory()->Connect();
WaitForConnections();
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
base::TimeTicks retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
EXPECT_GE((retry_time - connect_time).InMilliseconds(), CalculateBackoff(1));
factory()->SetConnectResult(net::OK);
WaitForConnections();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
}
// Multiple connection failures should retry with an exponentially increasing
// backoff, then reset on success.
TEST_F(ConnectionFactoryImplTest, MultipleFailuresThenSucceed) {
const int kNumAttempts = 5;
factory()->SetMultipleConnectResults(net::ERR_CONNECTION_FAILED,
kNumAttempts);
base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
factory()->Connect();
WaitForConnections();
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
base::TimeTicks retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
EXPECT_GE((retry_time - connect_time).InMilliseconds(),
CalculateBackoff(kNumAttempts));
factory()->SetConnectResult(net::OK);
WaitForConnections();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
}
// Network change events should trigger canary connections.
TEST_F(ConnectionFactoryImplTest, FailThenNetworkChangeEvent) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
factory()->Connect();
WaitForConnections();
base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
EXPECT_FALSE(initial_backoff.is_null());
factory()->SetConnectResult(net::ERR_FAILED);
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
WaitForConnections();
// Backoff should increase.
base::TimeTicks next_backoff = factory()->NextRetryAttempt();
EXPECT_GT(next_backoff, initial_backoff);
EXPECT_FALSE(factory()->IsEndpointReachable());
}
// Verify that we reconnect even if a canary succeeded then disconnected while
// a backoff was pending.
TEST_F(ConnectionFactoryImplTest, CanarySucceedsThenDisconnects) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
factory()->Connect();
WaitForConnections();
base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
EXPECT_FALSE(initial_backoff.is_null());
factory()->SetConnectResult(net::OK);
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_ETHERNET);
WaitForConnections();
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
factory()->SetConnectResult(net::OK);
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
WaitForConnections();
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
}
// Verify that if a canary connects, but hasn't finished the handshake, a
// pending backoff attempt doesn't interrupt the connection.
TEST_F(ConnectionFactoryImplTest, CanarySucceedsRetryDuringLogin) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
factory()->Connect();
WaitForConnections();
base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
EXPECT_FALSE(initial_backoff.is_null());
factory()->SetDelayLogin(true);
factory()->SetConnectResult(net::OK);
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
WaitForConnections();
EXPECT_FALSE(factory()->IsEndpointReachable());
// Pump the loop, to ensure the pending backoff retry has no effect.
base::MessageLoop::current()->PostDelayedTask(
FROM_HERE,
base::MessageLoop::QuitClosure(),
base::TimeDelta::FromMilliseconds(1));
WaitForConnections();
}
// Fail after successful connection via signal reset.
TEST_F(ConnectionFactoryImplTest, FailViaSignalReset) {
factory()->SetConnectResult(net::OK);
factory()->Connect();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
EXPECT_FALSE(factory()->IsEndpointReachable());
}
TEST_F(ConnectionFactoryImplTest, IgnoreResetWhileConnecting) {
factory()->SetConnectResult(net::OK);
factory()->Connect();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
base::TimeTicks retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
EXPECT_FALSE(factory()->IsEndpointReachable());
const int kNumAttempts = 5;
for (int i = 0; i < kNumAttempts; ++i)
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
EXPECT_EQ(retry_time, factory()->NextRetryAttempt());
EXPECT_FALSE(factory()->IsEndpointReachable());
}
// Go into backoff due to connection failure. On successful connection, receive
// a signal reset. The original backoff should be restored and extended, rather
// than a new backoff starting from scratch.
TEST_F(ConnectionFactoryImplTest, SignalResetRestoresBackoff) {
factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
factory()->Connect();
WaitForConnections();
base::TimeTicks retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
factory()->SetConnectResult(net::OK);
connect_time = factory()->tick_clock()->NowTicks();
WaitForConnections();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
EXPECT_NE(retry_time, factory()->NextRetryAttempt());
retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
EXPECT_GE((retry_time - connect_time).InMilliseconds(),
CalculateBackoff(2));
factory()->SetConnectResult(net::OK);
connect_time = factory()->tick_clock()->NowTicks();
factory()->tick_clock()->Advance(
factory()->NextRetryAttempt() - connect_time);
WaitForConnections();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(connected_server().is_valid());
factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
EXPECT_NE(retry_time, factory()->NextRetryAttempt());
retry_time = factory()->NextRetryAttempt();
EXPECT_FALSE(retry_time.is_null());
EXPECT_GE((retry_time - connect_time).InMilliseconds(),
CalculateBackoff(3));
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_FALSE(connected_server().is_valid());
}
// When the network is disconnected, close the socket and suppress further
// connection attempts until the network returns.
// Disabled while crbug.com/396687 is being investigated.
TEST_F(ConnectionFactoryImplTest, DISABLED_SuppressConnectWhenNoNetwork) {
factory()->SetConnectResult(net::OK);
factory()->Connect();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_TRUE(factory()->IsEndpointReachable());
// Advance clock so the login window reset isn't encountered.
factory()->tick_clock()->Advance(base::TimeDelta::FromSeconds(11));
// Will trigger reset, but will not attempt a new connection.
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_NONE);
EXPECT_FALSE(factory()->IsEndpointReachable());
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
// When the network returns, attempt to connect.
factory()->SetConnectResult(net::OK);
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G);
WaitForConnections();
EXPECT_TRUE(factory()->IsEndpointReachable());
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
}
// Receiving a network change event before the initial connection should have
// no effect.
TEST_F(ConnectionFactoryImplTest, NetworkChangeBeforeFirstConnection) {
factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G);
factory()->SetConnectResult(net::OK);
factory()->Connect();
EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
EXPECT_TRUE(factory()->IsEndpointReachable());
}
// Test that if the client attempts to reconnect while a connection is already
// open, we don't crash.
TEST_F(ConnectionFactoryImplTest, ConnectionResetRace) {
// Initial successful connection.
factory()->SetConnectResult(net::OK);
factory()->Connect();
WaitForConnections();
EXPECT_TRUE(factory()->IsEndpointReachable());
// Trigger a connection error under the hood.
factory()->SetSocketError();
EXPECT_FALSE(factory()->IsEndpointReachable());
// Now trigger force a re-connection.
factory()->SetConnectResult(net::OK);
factory()->Connect();
WaitForConnections();
// Re-connection should succeed.
EXPECT_TRUE(factory()->IsEndpointReachable());
}
} // namespace gcm