1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "google_apis/gcm/engine/connection_factory_impl.h"
6
7#include <cmath>
8
9#include "base/message_loop/message_loop.h"
10#include "base/run_loop.h"
11#include "base/test/simple_test_tick_clock.h"
12#include "google_apis/gcm/base/mcs_util.h"
13#include "google_apis/gcm/engine/fake_connection_handler.h"
14#include "google_apis/gcm/monitoring/fake_gcm_stats_recorder.h"
15#include "net/base/backoff_entry.h"
16#include "net/http/http_network_session.h"
17#include "testing/gtest/include/gtest/gtest.h"
18
19class Policy;
20
21namespace gcm {
22namespace {
23
24const char kMCSEndpoint[] = "http://my.server";
25const char kMCSEndpoint2[] = "http://my.alt.server";
26
27const int kBackoffDelayMs = 1;
28const int kBackoffMultiplier = 2;
29
30// A backoff policy with small enough delays that tests aren't burdened.
31const net::BackoffEntry::Policy kTestBackoffPolicy = {
32  // Number of initial errors (in sequence) to ignore before applying
33  // exponential back-off rules.
34  0,
35
36  // Initial delay for exponential back-off in ms.
37  kBackoffDelayMs,
38
39  // Factor by which the waiting time will be multiplied.
40  kBackoffMultiplier,
41
42  // Fuzzing percentage. ex: 10% will spread requests randomly
43  // between 90%-100% of the calculated time.
44  0,
45
46  // Maximum amount of time we are willing to delay our request in ms.
47  10,
48
49  // Time to keep an entry from being discarded even when it
50  // has no significant state, -1 to never discard.
51  -1,
52
53  // Don't use initial delay unless the last request was an error.
54  false,
55};
56
57std::vector<GURL> BuildEndpoints() {
58  std::vector<GURL> endpoints;
59  endpoints.push_back(GURL(kMCSEndpoint));
60  endpoints.push_back(GURL(kMCSEndpoint2));
61  return endpoints;
62}
63
64// Helper for calculating total expected exponential backoff delay given an
65// arbitrary number of failed attempts. See BackoffEntry::CalculateReleaseTime.
66double CalculateBackoff(int num_attempts) {
67  double delay = kBackoffDelayMs;
68  for (int i = 1; i < num_attempts; ++i) {
69    delay += kBackoffDelayMs * pow(static_cast<double>(kBackoffMultiplier),
70                                   i - 1);
71  }
72  DVLOG(1) << "Expected backoff " << delay << " milliseconds.";
73  return delay;
74}
75
76void ReadContinuation(
77    scoped_ptr<google::protobuf::MessageLite> message) {
78}
79
80void WriteContinuation() {
81}
82
83class TestBackoffEntry : public net::BackoffEntry {
84 public:
85  explicit TestBackoffEntry(base::SimpleTestTickClock* tick_clock);
86  virtual ~TestBackoffEntry();
87
88  virtual base::TimeTicks ImplGetTimeNow() const OVERRIDE;
89
90 private:
91  base::SimpleTestTickClock* tick_clock_;
92};
93
94TestBackoffEntry::TestBackoffEntry(base::SimpleTestTickClock* tick_clock)
95    : BackoffEntry(&kTestBackoffPolicy),
96      tick_clock_(tick_clock) {
97}
98
99TestBackoffEntry::~TestBackoffEntry() {}
100
101base::TimeTicks TestBackoffEntry::ImplGetTimeNow() const {
102  return tick_clock_->NowTicks();
103}
104
105// A connection factory that stubs out network requests and overrides the
106// backoff policy.
107class TestConnectionFactoryImpl : public ConnectionFactoryImpl {
108 public:
109  TestConnectionFactoryImpl(const base::Closure& finished_callback);
110  virtual ~TestConnectionFactoryImpl();
111
112  void InitializeFactory();
113
114  // Overridden stubs.
115  virtual void ConnectImpl() OVERRIDE;
116  virtual void InitHandler() OVERRIDE;
117  virtual scoped_ptr<net::BackoffEntry> CreateBackoffEntry(
118      const net::BackoffEntry::Policy* const policy) OVERRIDE;
119  virtual scoped_ptr<ConnectionHandler> CreateConnectionHandler(
120      base::TimeDelta read_timeout,
121      const ConnectionHandler::ProtoReceivedCallback& read_callback,
122      const ConnectionHandler::ProtoSentCallback& write_callback,
123      const ConnectionHandler::ConnectionChangedCallback& connection_callback)
124          OVERRIDE;
125  virtual base::TimeTicks NowTicks() OVERRIDE;
126
127  // Helpers for verifying connection attempts are made. Connection results
128  // must be consumed.
129  void SetConnectResult(int connect_result);
130  void SetMultipleConnectResults(int connect_result, int num_expected_attempts);
131
132  // Force a login handshake to be delayed.
133  void SetDelayLogin(bool delay_login);
134
135  base::SimpleTestTickClock* tick_clock() { return &tick_clock_; }
136
137 private:
138  // Clock for controlling delay.
139  base::SimpleTestTickClock tick_clock_;
140  // The result to return on the next connect attempt.
141  int connect_result_;
142  // The number of expected connection attempts;
143  int num_expected_attempts_;
144  // Whether all expected connection attempts have been fulfilled since an
145  // expectation was last set.
146  bool connections_fulfilled_;
147  // Whether to delay a login handshake completion or not.
148  bool delay_login_;
149  // Callback to invoke when all connection attempts have been made.
150  base::Closure finished_callback_;
151  // The current fake connection handler..
152  FakeConnectionHandler* fake_handler_;
153  FakeGCMStatsRecorder dummy_recorder_;
154};
155
156TestConnectionFactoryImpl::TestConnectionFactoryImpl(
157    const base::Closure& finished_callback)
158    : ConnectionFactoryImpl(BuildEndpoints(),
159                            net::BackoffEntry::Policy(),
160                            NULL,
161                            NULL,
162                            NULL,
163                            &dummy_recorder_),
164      connect_result_(net::ERR_UNEXPECTED),
165      num_expected_attempts_(0),
166      connections_fulfilled_(true),
167      delay_login_(false),
168      finished_callback_(finished_callback),
169      fake_handler_(NULL) {
170  // Set a non-null time.
171  tick_clock_.Advance(base::TimeDelta::FromMilliseconds(1));
172}
173
174TestConnectionFactoryImpl::~TestConnectionFactoryImpl() {
175  EXPECT_EQ(0, num_expected_attempts_);
176}
177
178void TestConnectionFactoryImpl::ConnectImpl() {
179  ASSERT_GT(num_expected_attempts_, 0);
180  scoped_ptr<mcs_proto::LoginRequest> request(BuildLoginRequest(0, 0, ""));
181  GetConnectionHandler()->Init(*request, NULL);
182  OnConnectDone(connect_result_);
183  if (!NextRetryAttempt().is_null()) {
184    // Advance the time to the next retry time.
185    base::TimeDelta time_till_retry =
186        NextRetryAttempt() - tick_clock_.NowTicks();
187    tick_clock_.Advance(time_till_retry);
188  }
189  --num_expected_attempts_;
190  if (num_expected_attempts_ == 0) {
191    connect_result_ = net::ERR_UNEXPECTED;
192    connections_fulfilled_ = true;
193    finished_callback_.Run();
194  }
195}
196
197void TestConnectionFactoryImpl::InitHandler() {
198  EXPECT_NE(connect_result_, net::ERR_UNEXPECTED);
199  if (!delay_login_)
200    ConnectionHandlerCallback(net::OK);
201}
202
203scoped_ptr<net::BackoffEntry> TestConnectionFactoryImpl::CreateBackoffEntry(
204    const net::BackoffEntry::Policy* const policy) {
205  return scoped_ptr<net::BackoffEntry>(new TestBackoffEntry(&tick_clock_));
206}
207
208scoped_ptr<ConnectionHandler>
209TestConnectionFactoryImpl::CreateConnectionHandler(
210    base::TimeDelta read_timeout,
211    const ConnectionHandler::ProtoReceivedCallback& read_callback,
212    const ConnectionHandler::ProtoSentCallback& write_callback,
213    const ConnectionHandler::ConnectionChangedCallback& connection_callback) {
214  fake_handler_ = new FakeConnectionHandler(
215      base::Bind(&ReadContinuation),
216      base::Bind(&WriteContinuation));
217  return make_scoped_ptr<ConnectionHandler>(fake_handler_);
218}
219
220base::TimeTicks TestConnectionFactoryImpl::NowTicks() {
221  return tick_clock_.NowTicks();
222}
223
224void TestConnectionFactoryImpl::SetConnectResult(int connect_result) {
225  DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
226  ASSERT_EQ(0, num_expected_attempts_);
227  connections_fulfilled_ = false;
228  connect_result_ = connect_result;
229  num_expected_attempts_ = 1;
230  fake_handler_->ExpectOutgoingMessage(
231      MCSMessage(kLoginRequestTag,
232                 BuildLoginRequest(0, 0, "").PassAs<
233                     const google::protobuf::MessageLite>()));
234}
235
236void TestConnectionFactoryImpl::SetMultipleConnectResults(
237    int connect_result,
238    int num_expected_attempts) {
239  DCHECK_NE(connect_result, net::ERR_UNEXPECTED);
240  DCHECK_GT(num_expected_attempts, 0);
241  ASSERT_EQ(0, num_expected_attempts_);
242  connections_fulfilled_ = false;
243  connect_result_ = connect_result;
244  num_expected_attempts_ = num_expected_attempts;
245  for (int i = 0 ; i < num_expected_attempts; ++i) {
246    fake_handler_->ExpectOutgoingMessage(
247        MCSMessage(kLoginRequestTag,
248                   BuildLoginRequest(0, 0, "").PassAs<
249                       const google::protobuf::MessageLite>()));
250  }
251}
252
253void TestConnectionFactoryImpl::SetDelayLogin(bool delay_login) {
254  delay_login_ = delay_login;
255  fake_handler_->set_fail_login(delay_login_);
256}
257
258}  // namespace
259
260class ConnectionFactoryImplTest
261    : public testing::Test,
262      public ConnectionFactory::ConnectionListener {
263 public:
264  ConnectionFactoryImplTest();
265  virtual ~ConnectionFactoryImplTest();
266
267  TestConnectionFactoryImpl* factory() { return &factory_; }
268  GURL& connected_server() { return connected_server_; }
269
270  void WaitForConnections();
271
272  // ConnectionFactory::ConnectionListener
273  virtual void OnConnected(const GURL& current_server,
274                           const net::IPEndPoint& ip_endpoint) OVERRIDE;
275  virtual void OnDisconnected() OVERRIDE;
276
277 private:
278  void ConnectionsComplete();
279
280  TestConnectionFactoryImpl factory_;
281  base::MessageLoop message_loop_;
282  scoped_ptr<base::RunLoop> run_loop_;
283
284  GURL connected_server_;
285};
286
287ConnectionFactoryImplTest::ConnectionFactoryImplTest()
288   : factory_(base::Bind(&ConnectionFactoryImplTest::ConnectionsComplete,
289                         base::Unretained(this))),
290     run_loop_(new base::RunLoop()) {
291  factory()->SetConnectionListener(this);
292  factory()->Initialize(
293      ConnectionFactory::BuildLoginRequestCallback(),
294      ConnectionHandler::ProtoReceivedCallback(),
295      ConnectionHandler::ProtoSentCallback());
296}
297ConnectionFactoryImplTest::~ConnectionFactoryImplTest() {}
298
299void ConnectionFactoryImplTest::WaitForConnections() {
300  run_loop_->Run();
301  run_loop_.reset(new base::RunLoop());
302}
303
304void ConnectionFactoryImplTest::ConnectionsComplete() {
305  if (!run_loop_)
306    return;
307  run_loop_->Quit();
308}
309
310void ConnectionFactoryImplTest::OnConnected(
311    const GURL& current_server,
312    const net::IPEndPoint& ip_endpoint) {
313  connected_server_ = current_server;
314}
315
316void ConnectionFactoryImplTest::OnDisconnected() {
317  connected_server_ = GURL();
318}
319
320// Verify building a connection handler works.
321TEST_F(ConnectionFactoryImplTest, Initialize) {
322  ConnectionHandler* handler = factory()->GetConnectionHandler();
323  ASSERT_TRUE(handler);
324  EXPECT_FALSE(factory()->IsEndpointReachable());
325  EXPECT_FALSE(connected_server().is_valid());
326}
327
328// An initial successful connection should not result in backoff.
329TEST_F(ConnectionFactoryImplTest, ConnectSuccess) {
330  factory()->SetConnectResult(net::OK);
331  factory()->Connect();
332  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
333  EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[0]);
334  EXPECT_TRUE(factory()->IsEndpointReachable());
335  EXPECT_TRUE(connected_server().is_valid());
336}
337
338// A connection failure should result in backoff, and attempting the fallback
339// endpoint next.
340TEST_F(ConnectionFactoryImplTest, ConnectFail) {
341  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
342  factory()->Connect();
343  EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
344  EXPECT_EQ(factory()->GetCurrentEndpoint(), BuildEndpoints()[1]);
345  EXPECT_FALSE(factory()->IsEndpointReachable());
346  EXPECT_FALSE(connected_server().is_valid());
347}
348
349// A connection success after a failure should reset backoff.
350TEST_F(ConnectionFactoryImplTest, FailThenSucceed) {
351  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
352  base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
353  factory()->Connect();
354  WaitForConnections();
355  EXPECT_FALSE(factory()->IsEndpointReachable());
356  EXPECT_FALSE(connected_server().is_valid());
357  base::TimeTicks retry_time = factory()->NextRetryAttempt();
358  EXPECT_FALSE(retry_time.is_null());
359  EXPECT_GE((retry_time - connect_time).InMilliseconds(), CalculateBackoff(1));
360  factory()->SetConnectResult(net::OK);
361  WaitForConnections();
362  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
363  EXPECT_TRUE(factory()->IsEndpointReachable());
364  EXPECT_TRUE(connected_server().is_valid());
365}
366
367// Multiple connection failures should retry with an exponentially increasing
368// backoff, then reset on success.
369TEST_F(ConnectionFactoryImplTest, MultipleFailuresThenSucceed) {
370  const int kNumAttempts = 5;
371  factory()->SetMultipleConnectResults(net::ERR_CONNECTION_FAILED,
372                                       kNumAttempts);
373
374  base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
375  factory()->Connect();
376  WaitForConnections();
377  EXPECT_FALSE(factory()->IsEndpointReachable());
378  EXPECT_FALSE(connected_server().is_valid());
379  base::TimeTicks retry_time = factory()->NextRetryAttempt();
380  EXPECT_FALSE(retry_time.is_null());
381  EXPECT_GE((retry_time - connect_time).InMilliseconds(),
382            CalculateBackoff(kNumAttempts));
383
384  factory()->SetConnectResult(net::OK);
385  WaitForConnections();
386  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
387  EXPECT_TRUE(factory()->IsEndpointReachable());
388  EXPECT_TRUE(connected_server().is_valid());
389}
390
391// Network change events should trigger canary connections.
392TEST_F(ConnectionFactoryImplTest, FailThenNetworkChangeEvent) {
393  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
394  factory()->Connect();
395  WaitForConnections();
396  base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
397  EXPECT_FALSE(initial_backoff.is_null());
398
399  factory()->SetConnectResult(net::ERR_FAILED);
400  factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
401  WaitForConnections();
402
403  // Backoff should increase.
404  base::TimeTicks next_backoff = factory()->NextRetryAttempt();
405  EXPECT_GT(next_backoff, initial_backoff);
406  EXPECT_FALSE(factory()->IsEndpointReachable());
407}
408
409// Verify that we reconnect even if a canary succeeded then disconnected while
410// a backoff was pending.
411TEST_F(ConnectionFactoryImplTest, CanarySucceedsThenDisconnects) {
412  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
413  factory()->Connect();
414  WaitForConnections();
415  base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
416  EXPECT_FALSE(initial_backoff.is_null());
417
418  factory()->SetConnectResult(net::OK);
419  factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_ETHERNET);
420  WaitForConnections();
421  EXPECT_TRUE(factory()->IsEndpointReachable());
422  EXPECT_TRUE(connected_server().is_valid());
423
424  factory()->SetConnectResult(net::OK);
425  factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
426  EXPECT_FALSE(factory()->IsEndpointReachable());
427  EXPECT_FALSE(connected_server().is_valid());
428  WaitForConnections();
429  EXPECT_TRUE(factory()->IsEndpointReachable());
430  EXPECT_TRUE(connected_server().is_valid());
431}
432
433// Verify that if a canary connects, but hasn't finished the handshake, a
434// pending backoff attempt doesn't interrupt the connection.
435TEST_F(ConnectionFactoryImplTest, CanarySucceedsRetryDuringLogin) {
436  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
437  factory()->Connect();
438  WaitForConnections();
439  base::TimeTicks initial_backoff = factory()->NextRetryAttempt();
440  EXPECT_FALSE(initial_backoff.is_null());
441
442  factory()->SetDelayLogin(true);
443  factory()->SetConnectResult(net::OK);
444  factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_WIFI);
445  WaitForConnections();
446  EXPECT_FALSE(factory()->IsEndpointReachable());
447
448  // Pump the loop, to ensure the pending backoff retry has no effect.
449  base::MessageLoop::current()->PostDelayedTask(
450      FROM_HERE,
451      base::MessageLoop::QuitClosure(),
452      base::TimeDelta::FromMilliseconds(1));
453  WaitForConnections();
454}
455
456// Fail after successful connection via signal reset.
457TEST_F(ConnectionFactoryImplTest, FailViaSignalReset) {
458  factory()->SetConnectResult(net::OK);
459  factory()->Connect();
460  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
461
462  factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
463  EXPECT_FALSE(factory()->NextRetryAttempt().is_null());
464  EXPECT_FALSE(factory()->IsEndpointReachable());
465}
466
467TEST_F(ConnectionFactoryImplTest, IgnoreResetWhileConnecting) {
468  factory()->SetConnectResult(net::OK);
469  factory()->Connect();
470  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
471
472  factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
473  base::TimeTicks retry_time = factory()->NextRetryAttempt();
474  EXPECT_FALSE(retry_time.is_null());
475  EXPECT_FALSE(factory()->IsEndpointReachable());
476
477  const int kNumAttempts = 5;
478  for (int i = 0; i < kNumAttempts; ++i)
479    factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
480  EXPECT_EQ(retry_time, factory()->NextRetryAttempt());
481  EXPECT_FALSE(factory()->IsEndpointReachable());
482}
483
484// Go into backoff due to connection failure. On successful connection, receive
485// a signal reset. The original backoff should be restored and extended, rather
486// than a new backoff starting from scratch.
487TEST_F(ConnectionFactoryImplTest, SignalResetRestoresBackoff) {
488  factory()->SetConnectResult(net::ERR_CONNECTION_FAILED);
489  base::TimeTicks connect_time = factory()->tick_clock()->NowTicks();
490  factory()->Connect();
491  WaitForConnections();
492  base::TimeTicks retry_time = factory()->NextRetryAttempt();
493  EXPECT_FALSE(retry_time.is_null());
494
495  factory()->SetConnectResult(net::OK);
496  connect_time = factory()->tick_clock()->NowTicks();
497  WaitForConnections();
498  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
499
500  factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
501  EXPECT_FALSE(factory()->IsEndpointReachable());
502  EXPECT_FALSE(connected_server().is_valid());
503  EXPECT_NE(retry_time, factory()->NextRetryAttempt());
504  retry_time = factory()->NextRetryAttempt();
505  EXPECT_FALSE(retry_time.is_null());
506  EXPECT_GE((retry_time - connect_time).InMilliseconds(),
507            CalculateBackoff(2));
508
509  factory()->SetConnectResult(net::OK);
510  connect_time = factory()->tick_clock()->NowTicks();
511  factory()->tick_clock()->Advance(
512      factory()->NextRetryAttempt() - connect_time);
513  WaitForConnections();
514  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
515  EXPECT_TRUE(factory()->IsEndpointReachable());
516  EXPECT_TRUE(connected_server().is_valid());
517
518  factory()->SignalConnectionReset(ConnectionFactory::SOCKET_FAILURE);
519  EXPECT_NE(retry_time, factory()->NextRetryAttempt());
520  retry_time = factory()->NextRetryAttempt();
521  EXPECT_FALSE(retry_time.is_null());
522  EXPECT_GE((retry_time - connect_time).InMilliseconds(),
523            CalculateBackoff(3));
524  EXPECT_FALSE(factory()->IsEndpointReachable());
525  EXPECT_FALSE(connected_server().is_valid());
526}
527
528// When the network is disconnected, close the socket and suppress further
529// connection attempts until the network returns.
530// Disabled while crbug.com/396687 is being investigated.
531TEST_F(ConnectionFactoryImplTest, DISABLED_SuppressConnectWhenNoNetwork) {
532  factory()->SetConnectResult(net::OK);
533  factory()->Connect();
534  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
535  EXPECT_TRUE(factory()->IsEndpointReachable());
536
537  // Advance clock so the login window reset isn't encountered.
538  factory()->tick_clock()->Advance(base::TimeDelta::FromSeconds(11));
539
540  // Will trigger reset, but will not attempt a new connection.
541  factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_NONE);
542  EXPECT_FALSE(factory()->IsEndpointReachable());
543  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
544
545  // When the network returns, attempt to connect.
546  factory()->SetConnectResult(net::OK);
547  factory()->OnNetworkChanged(net::NetworkChangeNotifier::CONNECTION_4G);
548  WaitForConnections();
549
550  EXPECT_TRUE(factory()->IsEndpointReachable());
551  EXPECT_TRUE(factory()->NextRetryAttempt().is_null());
552}
553
554}  // namespace gcm
555