1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/bind.h"
6#include "base/callback.h"
7#include "base/compiler_specific.h"
8#include "base/memory/weak_ptr.h"
9#include "base/message_loop/message_loop.h"
10#include "base/test/test_timeouts.h"
11#include "sync/engine/backoff_delay_provider.h"
12#include "sync/engine/sync_scheduler_impl.h"
13#include "sync/engine/syncer.h"
14#include "sync/internal_api/public/base/cancelation_signal.h"
15#include "sync/internal_api/public/base/model_type_test_util.h"
16#include "sync/sessions/test_util.h"
17#include "sync/test/callback_counter.h"
18#include "sync/test/engine/fake_model_worker.h"
19#include "sync/test/engine/mock_connection_manager.h"
20#include "sync/test/engine/mock_nudge_handler.h"
21#include "sync/test/engine/test_directory_setter_upper.h"
22#include "sync/test/mock_invalidation.h"
23#include "sync/util/extensions_activity.h"
24#include "testing/gmock/include/gmock/gmock.h"
25#include "testing/gtest/include/gtest/gtest.h"
26
27using base::TimeDelta;
28using base::TimeTicks;
29using testing::_;
30using testing::AtLeast;
31using testing::DoAll;
32using testing::Invoke;
33using testing::Mock;
34using testing::Return;
35using testing::WithArg;
36using testing::WithArgs;
37using testing::WithoutArgs;
38
39namespace syncer {
40using sessions::SyncSession;
41using sessions::SyncSessionContext;
42using sync_pb::GetUpdatesCallerInfo;
43
44class MockSyncer : public Syncer {
45 public:
46  MockSyncer();
47  MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
48                                     const sessions::NudgeTracker&,
49                                     sessions::SyncSession*));
50  MOCK_METHOD3(ConfigureSyncShare,
51               bool(ModelTypeSet,
52                    sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
53                    SyncSession*));
54  MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
55};
56
57MockSyncer::MockSyncer()
58  : Syncer(NULL) {}
59
60typedef std::vector<TimeTicks> SyncShareTimes;
61
62void QuitLoopNow() {
63  // We use QuitNow() instead of Quit() as the latter may get stalled
64  // indefinitely in the presence of repeated timers with low delays
65  // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
66  // delay of 5ms] run under TSAN on the trybots).
67  base::MessageLoop::current()->QuitNow();
68}
69
70void RunLoop() {
71  base::MessageLoop::current()->Run();
72}
73
74void PumpLoop() {
75  // Do it this way instead of RunAllPending to pump loop exactly once
76  // (necessary in the presence of timers; see comment in
77  // QuitLoopNow).
78  base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
79  RunLoop();
80}
81
82void PumpLoopFor(base::TimeDelta time) {
83  // Allow the loop to run for the specified amount of time.
84  base::MessageLoop::current()->PostDelayedTask(
85      FROM_HERE, base::Bind(&QuitLoopNow), time);
86  RunLoop();
87}
88
89ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
90  ModelSafeRoutingInfo routes;
91  for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
92    routes[iter.Get()] = GROUP_PASSIVE;
93  }
94  return routes;
95}
96
97
98static const size_t kMinNumSamples = 5;
99
100// Test harness for the SyncScheduler.  Test the delays and backoff timers used
101// in response to various events.
102//
103// These tests execute in real time with real timers.  We try to keep the
104// delays short, but there is a limit to how short we can make them.  The
105// timers on some platforms (ie. Windows) have a timer resolution greater than
106// 1ms.  Using 1ms delays may result in test flakiness.
107//
108// See crbug.com/402212 for more info.
109class SyncSchedulerTest : public testing::Test {
110 public:
111  SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
112
113  class MockDelayProvider : public BackoffDelayProvider {
114   public:
115    MockDelayProvider() : BackoffDelayProvider(
116        TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
117        TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
118    }
119
120    MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
121  };
122
123  virtual void SetUp() {
124    dir_maker_.SetUp();
125    syncer_ = new testing::StrictMock<MockSyncer>();
126    delay_ = NULL;
127    extensions_activity_ = new ExtensionsActivity();
128
129    routing_info_[THEMES] = GROUP_UI;
130    routing_info_[TYPED_URLS] = GROUP_DB;
131    routing_info_[THEMES] = GROUP_UI;
132    routing_info_[NIGORI] = GROUP_PASSIVE;
133
134    workers_.clear();
135    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
136    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
137    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
138
139    connection_.reset(new MockConnectionManager(directory(),
140                                                &cancelation_signal_));
141    connection_->SetServerReachable();
142
143    model_type_registry_.reset(
144        new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
145
146    context_.reset(new SyncSessionContext(
147            connection_.get(), directory(),
148            extensions_activity_.get(),
149            std::vector<SyncEngineEventListener*>(), NULL,
150            model_type_registry_.get(),
151            true,  // enable keystore encryption
152            false,  // force enable pre-commit GU avoidance
153            "fake_invalidator_client_id"));
154    context_->SetRoutingInfo(routing_info_);
155    context_->set_notifications_enabled(true);
156    context_->set_account_name("Test");
157    scheduler_.reset(
158        new SyncSchedulerImpl("TestSyncScheduler",
159            BackoffDelayProvider::FromDefaults(),
160            context(),
161            syncer_));
162    scheduler_->SetDefaultNudgeDelay(default_delay());
163  }
164
165  SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
166  const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
167  MockSyncer* syncer() { return syncer_; }
168  MockDelayProvider* delay() { return delay_; }
169  MockConnectionManager* connection() { return connection_.get(); }
170  TimeDelta default_delay() { return TimeDelta::FromSeconds(0); }
171  TimeDelta timeout() {
172    return TestTimeouts::action_timeout();
173  }
174
175  virtual void TearDown() {
176    PumpLoop();
177    scheduler_.reset();
178    PumpLoop();
179    dir_maker_.TearDown();
180  }
181
182  void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
183      const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
184    EXPECT_GE(times.size(), min_num_samples);
185    for (size_t i = 0; i < times.size(); i++) {
186      SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
187      TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
188      EXPECT_GE(times[i], optimal_next_sync);
189    }
190  }
191
192  void DoQuitLoopNow() {
193    QuitLoopNow();
194  }
195
196  void StartSyncScheduler(SyncScheduler::Mode mode) {
197    scheduler()->Start(mode);
198  }
199
200  // This stops the scheduler synchronously.
201  void StopSyncScheduler() {
202    base::MessageLoop::current()->PostTask(
203        FROM_HERE,
204        base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
205                   weak_ptr_factory_.GetWeakPtr()));
206    RunLoop();
207  }
208
209  bool RunAndGetBackoff() {
210    ModelTypeSet nudge_types(THEMES);
211    StartSyncScheduler(SyncScheduler::NORMAL_MODE);
212
213    scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
214    RunLoop();
215
216    return scheduler()->IsBackingOff();
217  }
218
219  void UseMockDelayProvider() {
220    delay_ = new MockDelayProvider();
221    scheduler_->delay_provider_.reset(delay_);
222  }
223
224  SyncSessionContext* context() { return context_.get(); }
225
226  ModelTypeSet GetThrottledTypes() {
227    return scheduler_->nudge_tracker_.GetThrottledTypes();
228  }
229
230  base::TimeDelta GetRetryTimerDelay() {
231    EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
232    return scheduler_->retry_timer_.GetCurrentDelay();
233  }
234
235  static scoped_ptr<InvalidationInterface> BuildInvalidation(
236      int64 version,
237      const std::string& payload) {
238    return MockInvalidation::Build(version, payload)
239        .PassAs<InvalidationInterface>();
240  }
241
242 private:
243  syncable::Directory* directory() {
244    return dir_maker_.directory();
245  }
246
247  base::MessageLoop loop_;
248  TestDirectorySetterUpper dir_maker_;
249  CancelationSignal cancelation_signal_;
250  scoped_ptr<MockConnectionManager> connection_;
251  scoped_ptr<ModelTypeRegistry> model_type_registry_;
252  scoped_ptr<SyncSessionContext> context_;
253  scoped_ptr<SyncSchedulerImpl> scheduler_;
254  MockNudgeHandler mock_nudge_handler_;
255  MockSyncer* syncer_;
256  MockDelayProvider* delay_;
257  std::vector<scoped_refptr<ModelSafeWorker> > workers_;
258  scoped_refptr<ExtensionsActivity> extensions_activity_;
259  ModelSafeRoutingInfo routing_info_;
260  base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
261};
262
263void RecordSyncShareImpl(SyncShareTimes* times) {
264  times->push_back(TimeTicks::Now());
265}
266
267ACTION_P(RecordSyncShare, times) {
268  RecordSyncShareImpl(times);
269  if (base::MessageLoop::current()->is_running())
270    QuitLoopNow();
271  return true;
272}
273
274ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
275  RecordSyncShareImpl(times);
276  EXPECT_LE(times->size(), quit_after);
277  if (times->size() >= quit_after &&
278      base::MessageLoop::current()->is_running()) {
279    QuitLoopNow();
280  }
281  return true;
282}
283
284ACTION_P(StopScheduler, scheduler) {
285  scheduler->Stop();
286}
287
288ACTION(AddFailureAndQuitLoopNow) {
289  ADD_FAILURE();
290  QuitLoopNow();
291  return true;
292}
293
294ACTION(QuitLoopNowAction) {
295  QuitLoopNow();
296  return true;
297}
298
299// Test nudge scheduling.
300TEST_F(SyncSchedulerTest, Nudge) {
301  SyncShareTimes times;
302  ModelTypeSet model_types(THEMES);
303
304  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
305      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
306                      RecordSyncShare(&times)))
307      .RetiresOnSaturation();
308
309  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
310
311  scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
312  RunLoop();
313
314  Mock::VerifyAndClearExpectations(syncer());
315
316  // Make sure a second, later, nudge is unaffected by first (no coalescing).
317  SyncShareTimes times2;
318  model_types.Remove(THEMES);
319  model_types.Put(TYPED_URLS);
320  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
321      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
322                      RecordSyncShare(&times2)));
323  scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
324  RunLoop();
325}
326
327// Make sure a regular config command is scheduled fine in the absence of any
328// errors.
329TEST_F(SyncSchedulerTest, Config) {
330  SyncShareTimes times;
331  const ModelTypeSet model_types(THEMES);
332
333  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
334      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
335                      RecordSyncShare(&times)));
336
337  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
338
339  CallbackCounter ready_counter;
340  CallbackCounter retry_counter;
341  ConfigurationParams params(
342      GetUpdatesCallerInfo::RECONFIGURATION,
343      model_types,
344      TypesToRoutingInfo(model_types),
345      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
346      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
347  scheduler()->ScheduleConfiguration(params);
348  PumpLoop();
349  ASSERT_EQ(1, ready_counter.times_called());
350  ASSERT_EQ(0, retry_counter.times_called());
351}
352
353// Simulate a failure and make sure the config request is retried.
354TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
355  UseMockDelayProvider();
356  EXPECT_CALL(*delay(), GetDelay(_))
357      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
358  SyncShareTimes times;
359  const ModelTypeSet model_types(THEMES);
360
361  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
362
363  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
364      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
365                      RecordSyncShare(&times)))
366      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
367                      RecordSyncShare(&times)));
368
369  CallbackCounter ready_counter;
370  CallbackCounter retry_counter;
371  ConfigurationParams params(
372      GetUpdatesCallerInfo::RECONFIGURATION,
373      model_types,
374      TypesToRoutingInfo(model_types),
375      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
376      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
377  scheduler()->ScheduleConfiguration(params);
378  RunLoop();
379  ASSERT_EQ(0, ready_counter.times_called());
380  ASSERT_EQ(1, retry_counter.times_called());
381
382  // RunLoop() will trigger TryCanaryJob which will retry configuration.
383  // Since retry_task was already called it shouldn't be called again.
384  RunLoop();
385  ASSERT_EQ(0, ready_counter.times_called());
386  ASSERT_EQ(1, retry_counter.times_called());
387
388  Mock::VerifyAndClearExpectations(syncer());
389
390  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
391      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
392                      RecordSyncShare(&times)));
393  RunLoop();
394
395  ASSERT_EQ(1, ready_counter.times_called());
396}
397
398// Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
399// This can happen if server returns NOT_MY_BIRTHDAY.
400TEST_F(SyncSchedulerTest, ConfigWithStop) {
401  UseMockDelayProvider();
402  EXPECT_CALL(*delay(), GetDelay(_))
403      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
404  SyncShareTimes times;
405  const ModelTypeSet model_types(THEMES);
406
407  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
408
409  // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
410  // retry_task or dereference configuration params.
411  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
412      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
413                      StopScheduler(scheduler()),
414                      RecordSyncShare(&times)));
415
416  CallbackCounter ready_counter;
417  CallbackCounter retry_counter;
418  ConfigurationParams params(
419      GetUpdatesCallerInfo::RECONFIGURATION,
420      model_types,
421      TypesToRoutingInfo(model_types),
422      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
423      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
424  scheduler()->ScheduleConfiguration(params);
425  PumpLoop();
426  ASSERT_EQ(0, ready_counter.times_called());
427  ASSERT_EQ(0, retry_counter.times_called());
428}
429
430// Issue a nudge when the config has failed. Make sure both the config and
431// nudge are executed.
432TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
433  const ModelTypeSet model_types(THEMES);
434  UseMockDelayProvider();
435  EXPECT_CALL(*delay(), GetDelay(_))
436      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
437  SyncShareTimes times;
438
439  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
440
441  // Request a configure and make sure it fails.
442  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
443      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
444                      RecordSyncShare(&times)));
445  CallbackCounter ready_counter;
446  CallbackCounter retry_counter;
447  ConfigurationParams params(
448      GetUpdatesCallerInfo::RECONFIGURATION,
449      model_types,
450      TypesToRoutingInfo(model_types),
451      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
452      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
453  scheduler()->ScheduleConfiguration(params);
454  RunLoop();
455  ASSERT_EQ(0, ready_counter.times_called());
456  ASSERT_EQ(1, retry_counter.times_called());
457  Mock::VerifyAndClearExpectations(syncer());
458
459  // Ask for a nudge while dealing with repeated configure failure.
460  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
461      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
462                      RecordSyncShare(&times)));
463  scheduler()->ScheduleLocalNudge(model_types, FROM_HERE);
464  RunLoop();
465  // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
466  // for the first retry attempt from the config job (after
467  // waiting ~+/- 50ms).
468  Mock::VerifyAndClearExpectations(syncer());
469  ASSERT_EQ(0, ready_counter.times_called());
470
471  // Let the next configure retry succeed.
472  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
473      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
474                      RecordSyncShare(&times)));
475  RunLoop();
476
477  // Now change the mode so nudge can execute.
478  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
479      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
480                      RecordSyncShare(&times)));
481  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
482  PumpLoop();
483}
484
485// Test that nudges are coalesced.
486TEST_F(SyncSchedulerTest, NudgeCoalescing) {
487  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
488
489  SyncShareTimes times;
490  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
491      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
492                      RecordSyncShare(&times)));
493  const ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3(THEMES);
494  TimeTicks optimal_time = TimeTicks::Now() + default_delay();
495  scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
496  scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
497  RunLoop();
498
499  ASSERT_EQ(1U, times.size());
500  EXPECT_GE(times[0], optimal_time);
501
502  Mock::VerifyAndClearExpectations(syncer());
503
504  SyncShareTimes times2;
505  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
506      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
507                      RecordSyncShare(&times2)));
508  scheduler()->ScheduleLocalNudge(types3, FROM_HERE);
509  RunLoop();
510}
511
512// Test that nudges are coalesced.
513TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
514  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
515
516  SyncShareTimes times;
517  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
518      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
519                      RecordSyncShare(&times)));
520  ModelTypeSet types1(THEMES), types2(TYPED_URLS), types3;
521
522  // Create a huge time delay.
523  TimeDelta delay = TimeDelta::FromDays(1);
524
525  std::map<ModelType, TimeDelta> delay_map;
526  delay_map[types1.First().Get()] = delay;
527  scheduler()->OnReceivedCustomNudgeDelays(delay_map);
528  scheduler()->ScheduleLocalNudge(types1, FROM_HERE);
529  scheduler()->ScheduleLocalNudge(types2, FROM_HERE);
530
531  TimeTicks min_time = TimeTicks::Now();
532  TimeTicks max_time = TimeTicks::Now() + delay;
533
534  RunLoop();
535  Mock::VerifyAndClearExpectations(syncer());
536
537  // Make sure the sync happened at the right time.
538  ASSERT_EQ(1U, times.size());
539  EXPECT_GE(times[0], min_time);
540  EXPECT_LE(times[0], max_time);
541}
542
543// Test nudge scheduling.
544TEST_F(SyncSchedulerTest, NudgeWithStates) {
545  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
546
547  SyncShareTimes times1;
548  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
549      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
550                      RecordSyncShare(&times1)))
551      .RetiresOnSaturation();
552  scheduler()->ScheduleInvalidationNudge(
553      THEMES, BuildInvalidation(10, "test"), FROM_HERE);
554  RunLoop();
555
556  Mock::VerifyAndClearExpectations(syncer());
557
558  // Make sure a second, later, nudge is unaffected by first (no coalescing).
559  SyncShareTimes times2;
560  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
561      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
562                      RecordSyncShare(&times2)));
563  scheduler()->ScheduleInvalidationNudge(
564      TYPED_URLS, BuildInvalidation(10, "test2"), FROM_HERE);
565  RunLoop();
566}
567
568// Test that polling works as expected.
569TEST_F(SyncSchedulerTest, Polling) {
570  SyncShareTimes times;
571  TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
572  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
573      .WillRepeatedly(
574          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
575                RecordSyncShareMultiple(&times, kMinNumSamples)));
576
577  scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
578
579  TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
580  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
581
582  // Run again to wait for polling.
583  RunLoop();
584
585  StopSyncScheduler();
586  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
587}
588
589// Test that the short poll interval is used.
590TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
591  SyncShareTimes times;
592  TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
593  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
594      .WillRepeatedly(
595          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
596                RecordSyncShareMultiple(&times, kMinNumSamples)));
597
598  scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
599  scheduler()->SetNotificationsEnabled(false);
600
601  TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
602  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
603
604  // Run again to wait for polling.
605  RunLoop();
606
607  StopSyncScheduler();
608  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
609}
610
611// Test that polling intervals are updated when needed.
612TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
613  SyncShareTimes times;
614  TimeDelta poll1(TimeDelta::FromMilliseconds(120));
615  TimeDelta poll2(TimeDelta::FromMilliseconds(30));
616  scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
617  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
618      .WillOnce(DoAll(
619          WithArgs<0,1>(
620              sessions::test_util::SimulatePollIntervalUpdate(poll2)),
621          Return(true)))
622      .WillRepeatedly(
623          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
624                WithArg<1>(
625                    RecordSyncShareMultiple(&times, kMinNumSamples))));
626
627  TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
628  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
629
630  // Run again to wait for polling.
631  RunLoop();
632
633  StopSyncScheduler();
634  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
635}
636
637// Test that no syncing occurs when throttled.
638TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
639  const ModelTypeSet types(THEMES);
640  TimeDelta poll(TimeDelta::FromMilliseconds(20));
641  TimeDelta throttle(TimeDelta::FromMinutes(10));
642  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
643
644  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
645      .WillOnce(DoAll(
646          WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
647          Return(true)))
648      .WillRepeatedly(AddFailureAndQuitLoopNow());
649
650  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
651
652  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
653  PumpLoop();
654
655  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
656
657  CallbackCounter ready_counter;
658  CallbackCounter retry_counter;
659  ConfigurationParams params(
660      GetUpdatesCallerInfo::RECONFIGURATION,
661      types,
662      TypesToRoutingInfo(types),
663      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
664      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
665  scheduler()->ScheduleConfiguration(params);
666  PumpLoop();
667  ASSERT_EQ(0, ready_counter.times_called());
668  ASSERT_EQ(1, retry_counter.times_called());
669
670}
671
672TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
673  SyncShareTimes times;
674  TimeDelta poll(TimeDelta::FromMilliseconds(15));
675  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
676  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
677
678  ::testing::InSequence seq;
679  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
680      .WillOnce(DoAll(
681          WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
682          Return(true)))
683      .RetiresOnSaturation();
684  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
685      .WillRepeatedly(
686          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
687                RecordSyncShareMultiple(&times, kMinNumSamples)));
688
689  TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
690  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
691
692  // Run again to wait for polling.
693  RunLoop();
694
695  StopSyncScheduler();
696  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
697}
698
699TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
700  SyncShareTimes times;
701  TimeDelta poll(TimeDelta::FromDays(1));
702  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
703  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
704
705  ::testing::InSequence seq;
706  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
707      .WillOnce(DoAll(
708          WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
709          Return(true)))
710      .RetiresOnSaturation();
711  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
712      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
713                      QuitLoopNowAction()));
714
715  const ModelTypeSet types(THEMES);
716  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
717  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
718
719  PumpLoop(); // To get PerformDelayedNudge called.
720  PumpLoop(); // To get TrySyncSessionJob called
721  EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
722  RunLoop();
723  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
724
725  StopSyncScheduler();
726}
727
728TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
729  SyncShareTimes times;
730  TimeDelta poll(TimeDelta::FromDays(1));
731  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
732  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
733
734  ::testing::InSequence seq;
735  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
736      .WillOnce(DoAll(
737          WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
738          Return(true)))
739      .RetiresOnSaturation();
740  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
741      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
742                      QuitLoopNowAction()));
743
744  const ModelTypeSet types(THEMES);
745  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
746
747  CallbackCounter ready_counter;
748  CallbackCounter retry_counter;
749  ConfigurationParams params(
750      GetUpdatesCallerInfo::RECONFIGURATION,
751      types,
752      TypesToRoutingInfo(types),
753      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
754      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
755  scheduler()->ScheduleConfiguration(params);
756  PumpLoop();
757  EXPECT_EQ(0, ready_counter.times_called());
758  EXPECT_EQ(1, retry_counter.times_called());
759  EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
760
761  RunLoop();
762  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
763
764  StopSyncScheduler();
765}
766
767TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
768  UseMockDelayProvider();
769  EXPECT_CALL(*delay(), GetDelay(_))
770      .WillRepeatedly(Return(default_delay()));
771
772  TimeDelta poll(TimeDelta::FromDays(1));
773  TimeDelta throttle1(TimeDelta::FromSeconds(60));
774  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
775
776  const ModelTypeSet types(THEMES);
777
778  ::testing::InSequence seq;
779  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
780      .WillOnce(DoAll(
781          WithArg<2>(
782              sessions::test_util::SimulateTypesThrottled(types, throttle1)),
783          Return(true)))
784      .RetiresOnSaturation();
785
786  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
787  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
788  PumpLoop(); // To get PerformDelayedNudge called.
789  PumpLoop(); // To get TrySyncSessionJob called
790  EXPECT_TRUE(GetThrottledTypes().HasAll(types));
791
792  // This won't cause a sync cycle because the types are throttled.
793  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
794  PumpLoop();
795
796  StopSyncScheduler();
797}
798
799TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
800  UseMockDelayProvider();
801  EXPECT_CALL(*delay(), GetDelay(_))
802      .WillRepeatedly(Return(default_delay()));
803
804  SyncShareTimes times;
805  TimeDelta poll(TimeDelta::FromDays(1));
806  TimeDelta throttle1(TimeDelta::FromSeconds(60));
807  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
808
809  const ModelTypeSet throttled_types(THEMES);
810  const ModelTypeSet unthrottled_types(PREFERENCES);
811
812  ::testing::InSequence seq;
813  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
814      .WillOnce(DoAll(
815          WithArg<2>(
816              sessions::test_util::SimulateTypesThrottled(
817                  throttled_types, throttle1)),
818          Return(true)))
819      .RetiresOnSaturation();
820
821  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
822  scheduler()->ScheduleLocalNudge(throttled_types, FROM_HERE);
823  PumpLoop(); // To get PerformDelayedNudge called.
824  PumpLoop(); // To get TrySyncSessionJob called
825  EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
826
827  // Ignore invalidations for throttled types.
828  scheduler()->ScheduleInvalidationNudge(
829      THEMES, BuildInvalidation(10, "test"), FROM_HERE);
830  PumpLoop();
831
832  // Ignore refresh requests for throttled types.
833  scheduler()->ScheduleLocalRefreshRequest(throttled_types, FROM_HERE);
834  PumpLoop();
835
836  Mock::VerifyAndClearExpectations(syncer());
837
838  // Local nudges for non-throttled types will trigger a sync.
839  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
840      .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
841                            RecordSyncShare(&times)));
842  scheduler()->ScheduleLocalNudge(unthrottled_types, FROM_HERE);
843  RunLoop();
844  Mock::VerifyAndClearExpectations(syncer());
845
846  StopSyncScheduler();
847}
848
849// Test nudges / polls don't run in config mode and config tasks do.
850TEST_F(SyncSchedulerTest, ConfigurationMode) {
851  TimeDelta poll(TimeDelta::FromMilliseconds(15));
852  SyncShareTimes times;
853  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
854
855  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
856
857  const ModelTypeSet nudge_types(TYPED_URLS);
858  scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
859  scheduler()->ScheduleLocalNudge(nudge_types, FROM_HERE);
860
861  const ModelTypeSet config_types(THEMES);
862
863  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
864      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
865                      RecordSyncShare(&times)))
866      .RetiresOnSaturation();
867  CallbackCounter ready_counter;
868  CallbackCounter retry_counter;
869  ConfigurationParams params(
870      GetUpdatesCallerInfo::RECONFIGURATION,
871      config_types,
872      TypesToRoutingInfo(config_types),
873      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
874      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
875  scheduler()->ScheduleConfiguration(params);
876  RunLoop();
877  ASSERT_EQ(1, ready_counter.times_called());
878  ASSERT_EQ(0, retry_counter.times_called());
879
880  Mock::VerifyAndClearExpectations(syncer());
881
882  // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
883  scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
884  SyncShareTimes times2;
885  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
886      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
887                      RecordSyncShare(&times2)));
888
889  // TODO(tim): Figure out how to remove this dangerous need to reset
890  // routing info between mode switches.
891  context()->SetRoutingInfo(routing_info());
892  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
893
894  RunLoop();
895  Mock::VerifyAndClearExpectations(syncer());
896}
897
898class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
899  virtual void SetUp() {
900    SyncSchedulerTest::SetUp();
901    UseMockDelayProvider();
902    EXPECT_CALL(*delay(), GetDelay(_))
903        .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
904  }
905
906  virtual void TearDown() {
907    StopSyncScheduler();
908    SyncSchedulerTest::TearDown();
909  }
910};
911
912// Have the sycner fail during commit.  Expect that the scheduler enters
913// backoff.
914TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
915  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
916      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
917                      QuitLoopNowAction()));
918  EXPECT_TRUE(RunAndGetBackoff());
919}
920
921// Have the syncer fail during download updates and succeed on the first
922// retry.  Expect that this clears the backoff state.
923TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
924  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
925      .WillOnce(DoAll(
926          Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
927          Return(true)))
928      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
929                      QuitLoopNowAction()));
930  EXPECT_FALSE(RunAndGetBackoff());
931}
932
933// Have the syncer fail during commit and succeed on the first retry.  Expect
934// that this clears the backoff state.
935TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
936  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
937      .WillOnce(DoAll(
938          Invoke(sessions::test_util::SimulateCommitFailed),
939          Return(true)))
940      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
941                      QuitLoopNowAction()));
942  EXPECT_FALSE(RunAndGetBackoff());
943}
944
945// Have the syncer fail to download updates and fail again on the retry.
946// Expect this will leave the scheduler in backoff.
947TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
948  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
949      .WillOnce(DoAll(
950          Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
951          Return(true)))
952      .WillRepeatedly(DoAll(
953              Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
954              QuitLoopNowAction()));
955  EXPECT_TRUE(RunAndGetBackoff());
956}
957
958// Have the syncer fail to get the encryption key yet succeed in downloading
959// updates. Expect this will leave the scheduler in backoff.
960TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
961  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
962      .WillOnce(DoAll(
963          Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
964          Return(true)))
965      .WillRepeatedly(DoAll(
966              Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
967              QuitLoopNowAction()));
968  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
969
970  ModelTypeSet types(THEMES);
971  CallbackCounter ready_counter;
972  CallbackCounter retry_counter;
973  ConfigurationParams params(
974      GetUpdatesCallerInfo::RECONFIGURATION,
975      types,
976      TypesToRoutingInfo(types),
977      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
978      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
979  scheduler()->ScheduleConfiguration(params);
980  RunLoop();
981
982  EXPECT_TRUE(scheduler()->IsBackingOff());
983}
984
985// Test that no polls or extraneous nudges occur when in backoff.
986TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
987  SyncShareTimes times;
988  TimeDelta poll(TimeDelta::FromMilliseconds(10));
989  const ModelTypeSet types(THEMES);
990  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
991  UseMockDelayProvider();
992
993  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
994      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
995                      RecordSyncShareMultiple(&times, 1U)));
996  EXPECT_CALL(*delay(), GetDelay(_)).
997      WillRepeatedly(Return(TimeDelta::FromDays(1)));
998
999  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1000
1001  // This nudge should fail and put us into backoff.  Thanks to our mock
1002  // GetDelay() setup above, this will be a long backoff.
1003  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1004  RunLoop();
1005
1006  // From this point forward, no SyncShare functions should be invoked.
1007  Mock::VerifyAndClearExpectations(syncer());
1008
1009  // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1010  PumpLoopFor(poll * 10);
1011
1012  // Try (and fail) to schedule a nudge.
1013  scheduler()->ScheduleLocalNudge(types, FROM_HERE);
1014
1015  Mock::VerifyAndClearExpectations(syncer());
1016  Mock::VerifyAndClearExpectations(delay());
1017
1018  EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1019
1020  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1021
1022  CallbackCounter ready_counter;
1023  CallbackCounter retry_counter;
1024  ConfigurationParams params(
1025      GetUpdatesCallerInfo::RECONFIGURATION,
1026      types,
1027      TypesToRoutingInfo(types),
1028      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1029      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1030  scheduler()->ScheduleConfiguration(params);
1031  PumpLoop();
1032  ASSERT_EQ(0, ready_counter.times_called());
1033  ASSERT_EQ(1, retry_counter.times_called());
1034
1035}
1036
1037// Test that backoff is shaping traffic properly with consecutive errors.
1038TEST_F(SyncSchedulerTest, BackoffElevation) {
1039  SyncShareTimes times;
1040  UseMockDelayProvider();
1041
1042  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1043      .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1044          RecordSyncShareMultiple(&times, kMinNumSamples)));
1045
1046  const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1047  const TimeDelta second = TimeDelta::FromMilliseconds(20);
1048  const TimeDelta third = TimeDelta::FromMilliseconds(30);
1049  const TimeDelta fourth = TimeDelta::FromMilliseconds(40);
1050  const TimeDelta fifth = TimeDelta::FromMilliseconds(50);
1051  const TimeDelta sixth = TimeDelta::FromDays(1);
1052
1053  EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1054          .RetiresOnSaturation();
1055  EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1056          .RetiresOnSaturation();
1057  EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1058          .RetiresOnSaturation();
1059  EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1060          .RetiresOnSaturation();
1061  EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1062
1063  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1064
1065  // Run again with a nudge.
1066  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1067  RunLoop();
1068
1069  ASSERT_EQ(kMinNumSamples, times.size());
1070  EXPECT_GE(times[1] - times[0], second);
1071  EXPECT_GE(times[2] - times[1], third);
1072  EXPECT_GE(times[3] - times[2], fourth);
1073  EXPECT_GE(times[4] - times[3], fifth);
1074}
1075
1076// Test that things go back to normal once a retry makes forward progress.
1077TEST_F(SyncSchedulerTest, BackoffRelief) {
1078  SyncShareTimes times;
1079  const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1080  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1081  UseMockDelayProvider();
1082
1083  const TimeDelta backoff = TimeDelta::FromMilliseconds(10);
1084  EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1085
1086  // Optimal start for the post-backoff poll party.
1087  TimeTicks optimal_start = TimeTicks::Now();
1088  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1089
1090  // Kick off the test with a failed nudge.
1091  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1092      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1093                      RecordSyncShare(&times)));
1094  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1095  RunLoop();
1096  Mock::VerifyAndClearExpectations(syncer());
1097  TimeTicks optimal_job_time = optimal_start;
1098  ASSERT_EQ(1U, times.size());
1099  EXPECT_GE(times[0], optimal_job_time);
1100
1101  // The retry succeeds.
1102  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1103      .WillOnce(DoAll(
1104              Invoke(sessions::test_util::SimulateNormalSuccess),
1105              RecordSyncShare(&times)));
1106  RunLoop();
1107  Mock::VerifyAndClearExpectations(syncer());
1108  optimal_job_time = optimal_job_time + backoff;
1109  ASSERT_EQ(2U, times.size());
1110  EXPECT_GE(times[1], optimal_job_time);
1111
1112  // Now let the Poll timer do its thing.
1113  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1114      .WillRepeatedly(DoAll(
1115              Invoke(sessions::test_util::SimulatePollSuccess),
1116              RecordSyncShareMultiple(&times, kMinNumSamples)));
1117  RunLoop();
1118  Mock::VerifyAndClearExpectations(syncer());
1119  ASSERT_EQ(kMinNumSamples, times.size());
1120  for (size_t i = 2; i < times.size(); i++) {
1121    optimal_job_time = optimal_job_time + poll;
1122    SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1123    EXPECT_GE(times[i], optimal_job_time);
1124  }
1125
1126  StopSyncScheduler();
1127}
1128
1129// Test that poll failures are ignored.  They should have no effect on
1130// subsequent poll attempts, nor should they trigger a backoff/retry.
1131TEST_F(SyncSchedulerTest, TransientPollFailure) {
1132  SyncShareTimes times;
1133  const TimeDelta poll_interval(TimeDelta::FromMilliseconds(10));
1134  scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1135  UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1136
1137  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1138      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1139                      RecordSyncShare(&times)))
1140      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1141                      RecordSyncShare(&times)));
1142
1143  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1144
1145  // Run the unsucessful poll. The failed poll should not trigger backoff.
1146  RunLoop();
1147  EXPECT_FALSE(scheduler()->IsBackingOff());
1148
1149  // Run the successful poll.
1150  RunLoop();
1151  EXPECT_FALSE(scheduler()->IsBackingOff());
1152}
1153
1154// Test that starting the syncer thread without a valid connection doesn't
1155// break things when a connection is detected.
1156TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1157  connection()->SetServerNotReachable();
1158  connection()->UpdateConnectionStatus();
1159  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1160    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1161                    Return(true)))
1162    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1163                    Return(true)));
1164  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1165
1166  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1167  // Should save the nudge for until after the server is reachable.
1168  base::MessageLoop::current()->RunUntilIdle();
1169
1170  scheduler()->OnConnectionStatusChange();
1171  connection()->SetServerReachable();
1172  connection()->UpdateConnectionStatus();
1173  base::MessageLoop::current()->RunUntilIdle();
1174}
1175
1176TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1177  UseMockDelayProvider();
1178  EXPECT_CALL(*delay(), GetDelay(_))
1179      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1180
1181  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1182  connection()->SetServerNotReachable();
1183  connection()->UpdateConnectionStatus();
1184
1185  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1186    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1187                    Return(true)))
1188    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1189                    Return(true)));
1190
1191  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1192  PumpLoop(); // To get PerformDelayedNudge called.
1193  PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1194  ASSERT_TRUE(scheduler()->IsBackingOff());
1195
1196  // Before we run the scheduled canary, trigger a server connection change.
1197  scheduler()->OnConnectionStatusChange();
1198  connection()->SetServerReachable();
1199  connection()->UpdateConnectionStatus();
1200  base::MessageLoop::current()->RunUntilIdle();
1201}
1202
1203// This was supposed to test the scenario where we receive a nudge while a
1204// connection change canary is scheduled, but has not run yet.  Since we've made
1205// the connection change canary synchronous, this is no longer possible.
1206TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1207  UseMockDelayProvider();
1208  EXPECT_CALL(*delay(), GetDelay(_))
1209      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1210
1211  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1212  connection()->SetServerNotReachable();
1213  connection()->UpdateConnectionStatus();
1214
1215  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1216    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1217                    Return(true)))
1218    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1219                    Return(true)))
1220    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1221                    QuitLoopNowAction()));
1222
1223  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1224
1225  PumpLoop(); // To get PerformDelayedNudge called.
1226  PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1227  ASSERT_TRUE(scheduler()->IsBackingOff());
1228
1229  // Before we run the scheduled canary, trigger a server connection change.
1230  scheduler()->OnConnectionStatusChange();
1231  PumpLoop();
1232  connection()->SetServerReachable();
1233  connection()->UpdateConnectionStatus();
1234  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1235  base::MessageLoop::current()->RunUntilIdle();
1236}
1237
1238// Tests that we don't crash trying to run two canaries at once if we receive
1239// extra connection status change notifications.  See crbug.com/190085.
1240TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1241  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1242      .WillRepeatedly(DoAll(
1243              Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1244              Return(true)));
1245  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1246  connection()->SetServerNotReachable();
1247  connection()->UpdateConnectionStatus();
1248
1249  ModelTypeSet model_types(THEMES);
1250  CallbackCounter ready_counter;
1251  CallbackCounter retry_counter;
1252  ConfigurationParams params(
1253      GetUpdatesCallerInfo::RECONFIGURATION,
1254      model_types,
1255      TypesToRoutingInfo(model_types),
1256      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1257      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1258  scheduler()->ScheduleConfiguration(params);
1259
1260  scheduler()->OnConnectionStatusChange();
1261  scheduler()->OnConnectionStatusChange();
1262
1263  PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1264}
1265
1266TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1267  SyncShareTimes times;
1268  TimeDelta poll(TimeDelta::FromMilliseconds(15));
1269  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1270
1271  ::testing::InSequence seq;
1272  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1273      .WillRepeatedly(
1274          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1275                RecordSyncShareMultiple(&times, kMinNumSamples)));
1276
1277  connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1278  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1279
1280  // Run to wait for polling.
1281  RunLoop();
1282
1283  // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1284  // but after poll finished with auth error from poll timer it should retry
1285  // poll once more
1286  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1287      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1288                      RecordSyncShare(&times)));
1289  scheduler()->OnCredentialsUpdated();
1290  connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1291  RunLoop();
1292  StopSyncScheduler();
1293}
1294
1295TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1296  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1297
1298  SyncShareTimes times;
1299  base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1300  scheduler()->OnReceivedGuRetryDelay(delay);
1301  EXPECT_EQ(delay, GetRetryTimerDelay());
1302
1303  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1304      .WillOnce(
1305          DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1306                RecordSyncShare(&times)));
1307
1308  // Run to wait for retrying.
1309  RunLoop();
1310
1311  StopSyncScheduler();
1312}
1313
1314TEST_F(SyncSchedulerTest, FailedRetry) {
1315  UseMockDelayProvider();
1316  EXPECT_CALL(*delay(), GetDelay(_))
1317      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
1318
1319  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1320
1321  base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1322  scheduler()->OnReceivedGuRetryDelay(delay);
1323
1324  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1325      .WillOnce(
1326          DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1327                QuitLoopNowAction()));
1328
1329  // Run to wait for retrying.
1330  RunLoop();
1331
1332  EXPECT_TRUE(scheduler()->IsBackingOff());
1333  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1334      .WillOnce(
1335          DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1336                QuitLoopNowAction()));
1337
1338  // Run to wait for second retrying.
1339  RunLoop();
1340
1341  StopSyncScheduler();
1342}
1343
1344ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1345  EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1346}
1347
1348TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1349  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1350
1351  SyncShareTimes times;
1352  base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1353  base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1354
1355  scheduler()->ScheduleLocalNudge(ModelTypeSet(THEMES), FROM_HERE);
1356  scheduler()->OnReceivedGuRetryDelay(delay1);
1357  EXPECT_EQ(delay1, GetRetryTimerDelay());
1358
1359  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1360      .WillOnce(DoAll(
1361          WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1362          WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1363          RecordSyncShare(&times)));
1364
1365  // Run nudge GU.
1366  RunLoop();
1367  EXPECT_EQ(delay2, GetRetryTimerDelay());
1368
1369  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1370      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1371                      RecordSyncShare(&times)));
1372
1373  // Run to wait for retrying.
1374  RunLoop();
1375
1376  StopSyncScheduler();
1377}
1378
1379}  // namespace syncer
1380