1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/bind.h"
6#include "base/callback.h"
7#include "base/compiler_specific.h"
8#include "base/memory/weak_ptr.h"
9#include "base/message_loop/message_loop.h"
10#include "base/test/test_timeouts.h"
11#include "sync/engine/backoff_delay_provider.h"
12#include "sync/engine/sync_scheduler_impl.h"
13#include "sync/engine/syncer.h"
14#include "sync/internal_api/public/base/cancelation_signal.h"
15#include "sync/internal_api/public/base/model_type_test_util.h"
16#include "sync/notifier/invalidation_util.h"
17#include "sync/notifier/object_id_invalidation_map.h"
18#include "sync/sessions/test_util.h"
19#include "sync/test/callback_counter.h"
20#include "sync/test/engine/fake_model_worker.h"
21#include "sync/test/engine/mock_connection_manager.h"
22#include "sync/test/engine/test_directory_setter_upper.h"
23#include "sync/util/extensions_activity.h"
24#include "testing/gmock/include/gmock/gmock.h"
25#include "testing/gtest/include/gtest/gtest.h"
26
27using base::TimeDelta;
28using base::TimeTicks;
29using testing::_;
30using testing::AtLeast;
31using testing::DoAll;
32using testing::Invoke;
33using testing::Mock;
34using testing::Return;
35using testing::WithArg;
36using testing::WithArgs;
37using testing::WithoutArgs;
38
39namespace syncer {
40using sessions::SyncSession;
41using sessions::SyncSessionContext;
42using sync_pb::GetUpdatesCallerInfo;
43
44class MockSyncer : public Syncer {
45 public:
46  MockSyncer();
47  MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
48                                     const sessions::NudgeTracker&,
49                                     sessions::SyncSession*));
50  MOCK_METHOD3(ConfigureSyncShare,
51               bool(ModelTypeSet,
52                    sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
53                    SyncSession*));
54  MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
55};
56
57MockSyncer::MockSyncer()
58  : Syncer(NULL) {}
59
60typedef std::vector<TimeTicks> SyncShareTimes;
61
62void QuitLoopNow() {
63  // We use QuitNow() instead of Quit() as the latter may get stalled
64  // indefinitely in the presence of repeated timers with low delays
65  // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
66  // delay of 5ms] run under TSAN on the trybots).
67  base::MessageLoop::current()->QuitNow();
68}
69
70void RunLoop() {
71  base::MessageLoop::current()->Run();
72}
73
74void PumpLoop() {
75  // Do it this way instead of RunAllPending to pump loop exactly once
76  // (necessary in the presence of timers; see comment in
77  // QuitLoopNow).
78  base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
79  RunLoop();
80}
81
82void PumpLoopFor(base::TimeDelta time) {
83  // Allow the loop to run for the specified amount of time.
84  base::MessageLoop::current()->PostDelayedTask(
85      FROM_HERE, base::Bind(&QuitLoopNow), time);
86  RunLoop();
87}
88
89ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
90  ModelSafeRoutingInfo routes;
91  for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
92    routes[iter.Get()] = GROUP_PASSIVE;
93  }
94  return routes;
95}
96
97// Convenient to use in tests wishing to analyze SyncShare calls over time.
98static const size_t kMinNumSamples = 5;
99class SyncSchedulerTest : public testing::Test {
100 public:
101  SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
102
103  class MockDelayProvider : public BackoffDelayProvider {
104   public:
105    MockDelayProvider() : BackoffDelayProvider(
106        TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
107        TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
108    }
109
110    MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
111  };
112
113  virtual void SetUp() {
114    dir_maker_.SetUp();
115    syncer_ = new testing::StrictMock<MockSyncer>();
116    delay_ = NULL;
117    extensions_activity_ = new ExtensionsActivity();
118
119    routing_info_[BOOKMARKS] = GROUP_UI;
120    routing_info_[AUTOFILL] = GROUP_DB;
121    routing_info_[THEMES] = GROUP_UI;
122    routing_info_[NIGORI] = GROUP_PASSIVE;
123
124    workers_.clear();
125    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
126    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
127    workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
128
129    connection_.reset(new MockConnectionManager(directory(),
130                                                &cancelation_signal_));
131    connection_->SetServerReachable();
132
133    model_type_registry_.reset(new ModelTypeRegistry(workers_, directory()));
134
135    context_.reset(new SyncSessionContext(
136            connection_.get(), directory(),
137            extensions_activity_.get(),
138            std::vector<SyncEngineEventListener*>(), NULL,
139            model_type_registry_.get(),
140            true,  // enable keystore encryption
141            false,  // force enable pre-commit GU avoidance
142            "fake_invalidator_client_id"));
143    context_->SetRoutingInfo(routing_info_);
144    context_->set_notifications_enabled(true);
145    context_->set_account_name("Test");
146    scheduler_.reset(
147        new SyncSchedulerImpl("TestSyncScheduler",
148            BackoffDelayProvider::FromDefaults(),
149            context(),
150            syncer_));
151  }
152
153  SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
154  const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
155  MockSyncer* syncer() { return syncer_; }
156  MockDelayProvider* delay() { return delay_; }
157  MockConnectionManager* connection() { return connection_.get(); }
158  TimeDelta zero() { return TimeDelta::FromSeconds(0); }
159  TimeDelta timeout() {
160    return TestTimeouts::action_timeout();
161  }
162
163  virtual void TearDown() {
164    PumpLoop();
165    scheduler_.reset();
166    PumpLoop();
167    dir_maker_.TearDown();
168  }
169
170  void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
171      const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
172    EXPECT_GE(times.size(), min_num_samples);
173    for (size_t i = 0; i < times.size(); i++) {
174      SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
175      TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
176      EXPECT_GE(times[i], optimal_next_sync);
177    }
178  }
179
180  void DoQuitLoopNow() {
181    QuitLoopNow();
182  }
183
184  void StartSyncScheduler(SyncScheduler::Mode mode) {
185    scheduler()->Start(mode);
186  }
187
188  // This stops the scheduler synchronously.
189  void StopSyncScheduler() {
190    base::MessageLoop::current()->PostTask(
191        FROM_HERE,
192        base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
193                   weak_ptr_factory_.GetWeakPtr()));
194    RunLoop();
195  }
196
197  bool RunAndGetBackoff() {
198    ModelTypeSet nudge_types(BOOKMARKS);
199    StartSyncScheduler(SyncScheduler::NORMAL_MODE);
200
201    scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
202    RunLoop();
203
204    return scheduler()->IsBackingOff();
205  }
206
207  void UseMockDelayProvider() {
208    delay_ = new MockDelayProvider();
209    scheduler_->delay_provider_.reset(delay_);
210  }
211
212  SyncSessionContext* context() { return context_.get(); }
213
214  ModelTypeSet GetThrottledTypes() {
215    return scheduler_->nudge_tracker_.GetThrottledTypes();
216  }
217
218  base::TimeDelta GetRetryTimerDelay() {
219    EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
220    return scheduler_->retry_timer_.GetCurrentDelay();
221  }
222
223 private:
224  syncable::Directory* directory() {
225    return dir_maker_.directory();
226  }
227
228  base::MessageLoop loop_;
229  TestDirectorySetterUpper dir_maker_;
230  CancelationSignal cancelation_signal_;
231  scoped_ptr<MockConnectionManager> connection_;
232  scoped_ptr<ModelTypeRegistry> model_type_registry_;
233  scoped_ptr<SyncSessionContext> context_;
234  scoped_ptr<SyncSchedulerImpl> scheduler_;
235  MockSyncer* syncer_;
236  MockDelayProvider* delay_;
237  std::vector<scoped_refptr<ModelSafeWorker> > workers_;
238  scoped_refptr<ExtensionsActivity> extensions_activity_;
239  ModelSafeRoutingInfo routing_info_;
240  base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
241};
242
243void RecordSyncShareImpl(SyncShareTimes* times) {
244  times->push_back(TimeTicks::Now());
245}
246
247ACTION_P(RecordSyncShare, times) {
248  RecordSyncShareImpl(times);
249  if (base::MessageLoop::current()->is_running())
250    QuitLoopNow();
251  return true;
252}
253
254ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
255  RecordSyncShareImpl(times);
256  EXPECT_LE(times->size(), quit_after);
257  if (times->size() >= quit_after &&
258      base::MessageLoop::current()->is_running()) {
259    QuitLoopNow();
260  }
261  return true;
262}
263
264ACTION_P(StopScheduler, scheduler) {
265  scheduler->Stop();
266}
267
268ACTION(AddFailureAndQuitLoopNow) {
269  ADD_FAILURE();
270  QuitLoopNow();
271  return true;
272}
273
274ACTION(QuitLoopNowAction) {
275  QuitLoopNow();
276  return true;
277}
278
279// Test nudge scheduling.
280TEST_F(SyncSchedulerTest, Nudge) {
281  SyncShareTimes times;
282  ModelTypeSet model_types(BOOKMARKS);
283
284  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
285      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
286                      RecordSyncShare(&times)))
287      .RetiresOnSaturation();
288
289  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
290
291  scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
292  RunLoop();
293
294  Mock::VerifyAndClearExpectations(syncer());
295
296  // Make sure a second, later, nudge is unaffected by first (no coalescing).
297  SyncShareTimes times2;
298  model_types.Remove(BOOKMARKS);
299  model_types.Put(AUTOFILL);
300  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
301      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
302                      RecordSyncShare(&times2)));
303  scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
304  RunLoop();
305}
306
307// Make sure a regular config command is scheduled fine in the absence of any
308// errors.
309TEST_F(SyncSchedulerTest, Config) {
310  SyncShareTimes times;
311  const ModelTypeSet model_types(BOOKMARKS);
312
313  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
314      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
315                      RecordSyncShare(&times)));
316
317  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
318
319  CallbackCounter ready_counter;
320  CallbackCounter retry_counter;
321  ConfigurationParams params(
322      GetUpdatesCallerInfo::RECONFIGURATION,
323      model_types,
324      TypesToRoutingInfo(model_types),
325      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
326      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
327  scheduler()->ScheduleConfiguration(params);
328  PumpLoop();
329  ASSERT_EQ(1, ready_counter.times_called());
330  ASSERT_EQ(0, retry_counter.times_called());
331}
332
333// Simulate a failure and make sure the config request is retried.
334TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
335  UseMockDelayProvider();
336  EXPECT_CALL(*delay(), GetDelay(_))
337      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
338  SyncShareTimes times;
339  const ModelTypeSet model_types(BOOKMARKS);
340
341  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
342
343  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
344      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
345                      RecordSyncShare(&times)))
346      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
347                      RecordSyncShare(&times)));
348
349  CallbackCounter ready_counter;
350  CallbackCounter retry_counter;
351  ConfigurationParams params(
352      GetUpdatesCallerInfo::RECONFIGURATION,
353      model_types,
354      TypesToRoutingInfo(model_types),
355      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
356      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
357  scheduler()->ScheduleConfiguration(params);
358  RunLoop();
359  ASSERT_EQ(0, ready_counter.times_called());
360  ASSERT_EQ(1, retry_counter.times_called());
361
362  // RunLoop() will trigger TryCanaryJob which will retry configuration.
363  // Since retry_task was already called it shouldn't be called again.
364  RunLoop();
365  ASSERT_EQ(0, ready_counter.times_called());
366  ASSERT_EQ(1, retry_counter.times_called());
367
368  Mock::VerifyAndClearExpectations(syncer());
369
370  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
371      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
372                      RecordSyncShare(&times)));
373  RunLoop();
374
375  ASSERT_EQ(1, ready_counter.times_called());
376}
377
378// Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
379// This can happen if server returns NOT_MY_BIRTHDAY.
380TEST_F(SyncSchedulerTest, ConfigWithStop) {
381  UseMockDelayProvider();
382  EXPECT_CALL(*delay(), GetDelay(_))
383      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
384  SyncShareTimes times;
385  const ModelTypeSet model_types(BOOKMARKS);
386
387  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
388
389  // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
390  // retry_task or dereference configuration params.
391  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
392      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
393                      StopScheduler(scheduler()),
394                      RecordSyncShare(&times)));
395
396  CallbackCounter ready_counter;
397  CallbackCounter retry_counter;
398  ConfigurationParams params(
399      GetUpdatesCallerInfo::RECONFIGURATION,
400      model_types,
401      TypesToRoutingInfo(model_types),
402      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
403      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
404  scheduler()->ScheduleConfiguration(params);
405  PumpLoop();
406  ASSERT_EQ(0, ready_counter.times_called());
407  ASSERT_EQ(0, retry_counter.times_called());
408}
409
410// Issue a nudge when the config has failed. Make sure both the config and
411// nudge are executed.
412TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
413  const ModelTypeSet model_types(BOOKMARKS);
414  UseMockDelayProvider();
415  EXPECT_CALL(*delay(), GetDelay(_))
416      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
417  SyncShareTimes times;
418
419  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
420
421  // Request a configure and make sure it fails.
422  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
423      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
424                      RecordSyncShare(&times)));
425  CallbackCounter ready_counter;
426  CallbackCounter retry_counter;
427  ConfigurationParams params(
428      GetUpdatesCallerInfo::RECONFIGURATION,
429      model_types,
430      TypesToRoutingInfo(model_types),
431      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
432      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
433  scheduler()->ScheduleConfiguration(params);
434  RunLoop();
435  ASSERT_EQ(0, ready_counter.times_called());
436  ASSERT_EQ(1, retry_counter.times_called());
437  Mock::VerifyAndClearExpectations(syncer());
438
439  // Ask for a nudge while dealing with repeated configure failure.
440  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
441      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
442                      RecordSyncShare(&times)));
443  scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
444  RunLoop();
445  // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
446  // for the first retry attempt from the config job (after
447  // waiting ~+/- 50ms).
448  Mock::VerifyAndClearExpectations(syncer());
449  ASSERT_EQ(0, ready_counter.times_called());
450
451  // Let the next configure retry succeed.
452  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
453      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
454                      RecordSyncShare(&times)));
455  RunLoop();
456
457  // Now change the mode so nudge can execute.
458  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
459      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
460                      RecordSyncShare(&times)));
461  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
462  PumpLoop();
463}
464
465// Test that nudges are coalesced.
466TEST_F(SyncSchedulerTest, NudgeCoalescing) {
467  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
468
469  SyncShareTimes times;
470  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
471      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
472                      RecordSyncShare(&times)));
473  const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
474  TimeDelta delay = zero();
475  TimeTicks optimal_time = TimeTicks::Now() + delay;
476  scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
477  scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
478  RunLoop();
479
480  ASSERT_EQ(1U, times.size());
481  EXPECT_GE(times[0], optimal_time);
482
483  Mock::VerifyAndClearExpectations(syncer());
484
485  SyncShareTimes times2;
486  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
487      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
488                      RecordSyncShare(&times2)));
489  scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
490  RunLoop();
491}
492
493// Test that nudges are coalesced.
494TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
495  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
496
497  SyncShareTimes times;
498  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
499      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
500                      RecordSyncShare(&times)));
501  ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
502
503  // Create a huge time delay.
504  TimeDelta delay = TimeDelta::FromDays(1);
505
506  scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
507  scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
508
509  TimeTicks min_time = TimeTicks::Now();
510  TimeTicks max_time = TimeTicks::Now() + delay;
511
512  RunLoop();
513  Mock::VerifyAndClearExpectations(syncer());
514
515  // Make sure the sync happened at the right time.
516  ASSERT_EQ(1U, times.size());
517  EXPECT_GE(times[0], min_time);
518  EXPECT_LE(times[0], max_time);
519}
520
521// Test nudge scheduling.
522TEST_F(SyncSchedulerTest, NudgeWithStates) {
523  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
524
525  SyncShareTimes times1;
526  ObjectIdInvalidationMap invalidations1 =
527      BuildInvalidationMap(BOOKMARKS, 10, "test");
528  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
529      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
530                      RecordSyncShare(&times1)))
531      .RetiresOnSaturation();
532  scheduler()->ScheduleInvalidationNudge(zero(), invalidations1, FROM_HERE);
533  RunLoop();
534
535  Mock::VerifyAndClearExpectations(syncer());
536
537  // Make sure a second, later, nudge is unaffected by first (no coalescing).
538  SyncShareTimes times2;
539  ObjectIdInvalidationMap invalidations2 =
540      BuildInvalidationMap(AUTOFILL, 10, "test2");
541  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
542      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
543                      RecordSyncShare(&times2)));
544  scheduler()->ScheduleInvalidationNudge(zero(), invalidations2, FROM_HERE);
545  RunLoop();
546}
547
548// Test that polling works as expected.
549TEST_F(SyncSchedulerTest, Polling) {
550  SyncShareTimes times;
551  TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
552  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
553      .WillRepeatedly(
554          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
555                RecordSyncShareMultiple(&times, kMinNumSamples)));
556
557  scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
558
559  TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
560  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
561
562  // Run again to wait for polling.
563  RunLoop();
564
565  StopSyncScheduler();
566  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
567}
568
569// Test that the short poll interval is used.
570TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
571  SyncShareTimes times;
572  TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
573  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
574      .WillRepeatedly(
575          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
576                RecordSyncShareMultiple(&times, kMinNumSamples)));
577
578  scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
579  scheduler()->SetNotificationsEnabled(false);
580
581  TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
582  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
583
584  // Run again to wait for polling.
585  RunLoop();
586
587  StopSyncScheduler();
588  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
589}
590
591// Test that polling intervals are updated when needed.
592TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
593  SyncShareTimes times;
594  TimeDelta poll1(TimeDelta::FromMilliseconds(120));
595  TimeDelta poll2(TimeDelta::FromMilliseconds(30));
596  scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
597  EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
598      .WillOnce(DoAll(
599          WithArgs<0,1>(
600              sessions::test_util::SimulatePollIntervalUpdate(poll2)),
601          Return(true)))
602      .WillRepeatedly(
603          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
604                WithArg<1>(
605                    RecordSyncShareMultiple(&times, kMinNumSamples))));
606
607  TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
608  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
609
610  // Run again to wait for polling.
611  RunLoop();
612
613  StopSyncScheduler();
614  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
615}
616
617// Test that the sessions commit delay is updated when needed.
618TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
619  SyncShareTimes times;
620  TimeDelta delay1(TimeDelta::FromMilliseconds(120));
621  TimeDelta delay2(TimeDelta::FromMilliseconds(30));
622  scheduler()->OnReceivedSessionsCommitDelay(delay1);
623
624  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
625      .WillOnce(
626          DoAll(
627              WithArgs<0,1,2>(
628                  sessions::test_util::SimulateSessionsCommitDelayUpdate(
629                      delay2)),
630              Invoke(sessions::test_util::SimulateNormalSuccess),
631              QuitLoopNowAction()));
632
633  EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
634  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
635
636  EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
637  const ModelTypeSet model_types(BOOKMARKS);
638  scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
639  RunLoop();
640
641  EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
642  StopSyncScheduler();
643}
644
645// Test that no syncing occurs when throttled.
646TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
647  const ModelTypeSet types(BOOKMARKS);
648  TimeDelta poll(TimeDelta::FromMilliseconds(5));
649  TimeDelta throttle(TimeDelta::FromMinutes(10));
650  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
651
652  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
653      .WillOnce(DoAll(
654          WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
655          Return(true)))
656      .WillRepeatedly(AddFailureAndQuitLoopNow());
657
658  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
659
660  scheduler()->ScheduleLocalNudge(
661      TimeDelta::FromMicroseconds(1), types, FROM_HERE);
662  PumpLoop();
663
664  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
665
666  CallbackCounter ready_counter;
667  CallbackCounter retry_counter;
668  ConfigurationParams params(
669      GetUpdatesCallerInfo::RECONFIGURATION,
670      types,
671      TypesToRoutingInfo(types),
672      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
673      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
674  scheduler()->ScheduleConfiguration(params);
675  PumpLoop();
676  ASSERT_EQ(0, ready_counter.times_called());
677  ASSERT_EQ(1, retry_counter.times_called());
678
679}
680
681TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
682  SyncShareTimes times;
683  TimeDelta poll(TimeDelta::FromMilliseconds(15));
684  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
685  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
686
687  ::testing::InSequence seq;
688  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
689      .WillOnce(DoAll(
690          WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
691          Return(true)))
692      .RetiresOnSaturation();
693  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
694      .WillRepeatedly(
695          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
696                RecordSyncShareMultiple(&times, kMinNumSamples)));
697
698  TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
699  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
700
701  // Run again to wait for polling.
702  RunLoop();
703
704  StopSyncScheduler();
705  AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
706}
707
708TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
709  SyncShareTimes times;
710  TimeDelta poll(TimeDelta::FromDays(1));
711  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
712  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
713
714  ::testing::InSequence seq;
715  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
716      .WillOnce(DoAll(
717          WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
718          Return(true)))
719      .RetiresOnSaturation();
720  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
721      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
722                      QuitLoopNowAction()));
723
724  const ModelTypeSet types(BOOKMARKS);
725  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
726  scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
727
728  PumpLoop(); // To get PerformDelayedNudge called.
729  PumpLoop(); // To get TrySyncSessionJob called
730  EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
731  RunLoop();
732  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
733
734  StopSyncScheduler();
735}
736
737TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
738  SyncShareTimes times;
739  TimeDelta poll(TimeDelta::FromDays(1));
740  TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
741  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
742
743  ::testing::InSequence seq;
744  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
745      .WillOnce(DoAll(
746          WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
747          Return(true)))
748      .RetiresOnSaturation();
749  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
750      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
751                      QuitLoopNowAction()));
752
753  const ModelTypeSet types(BOOKMARKS);
754  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
755
756  CallbackCounter ready_counter;
757  CallbackCounter retry_counter;
758  ConfigurationParams params(
759      GetUpdatesCallerInfo::RECONFIGURATION,
760      types,
761      TypesToRoutingInfo(types),
762      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
763      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
764  scheduler()->ScheduleConfiguration(params);
765  PumpLoop();
766  EXPECT_EQ(0, ready_counter.times_called());
767  EXPECT_EQ(1, retry_counter.times_called());
768  EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
769
770  RunLoop();
771  EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
772
773  StopSyncScheduler();
774}
775
776TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
777  UseMockDelayProvider();
778  EXPECT_CALL(*delay(), GetDelay(_))
779      .WillRepeatedly(Return(zero()));
780
781  TimeDelta poll(TimeDelta::FromDays(1));
782  TimeDelta throttle1(TimeDelta::FromSeconds(60));
783  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
784
785  const ModelTypeSet types(BOOKMARKS);
786
787  ::testing::InSequence seq;
788  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
789      .WillOnce(DoAll(
790          WithArg<2>(
791              sessions::test_util::SimulateTypesThrottled(types, throttle1)),
792          Return(true)))
793      .RetiresOnSaturation();
794
795  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
796  scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
797  PumpLoop(); // To get PerformDelayedNudge called.
798  PumpLoop(); // To get TrySyncSessionJob called
799  EXPECT_TRUE(GetThrottledTypes().HasAll(types));
800
801  // This won't cause a sync cycle because the types are throttled.
802  scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
803  PumpLoop();
804
805  StopSyncScheduler();
806}
807
808TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
809  UseMockDelayProvider();
810  EXPECT_CALL(*delay(), GetDelay(_))
811      .WillRepeatedly(Return(zero()));
812
813  SyncShareTimes times;
814  TimeDelta poll(TimeDelta::FromDays(1));
815  TimeDelta throttle1(TimeDelta::FromSeconds(60));
816  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
817
818  const ModelTypeSet throttled_types(BOOKMARKS);
819  const ModelTypeSet unthrottled_types(PREFERENCES);
820
821  ::testing::InSequence seq;
822  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
823      .WillOnce(DoAll(
824          WithArg<2>(
825              sessions::test_util::SimulateTypesThrottled(
826                  throttled_types, throttle1)),
827          Return(true)))
828      .RetiresOnSaturation();
829
830  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
831  scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
832  PumpLoop(); // To get PerformDelayedNudge called.
833  PumpLoop(); // To get TrySyncSessionJob called
834  EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
835
836  // Ignore invalidations for throttled types.
837  ObjectIdInvalidationMap invalidations =
838      BuildInvalidationMap(BOOKMARKS, 10, "test");
839  scheduler()->ScheduleInvalidationNudge(zero(), invalidations, FROM_HERE);
840  PumpLoop();
841
842  // Ignore refresh requests for throttled types.
843  scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
844  PumpLoop();
845
846  Mock::VerifyAndClearExpectations(syncer());
847
848  // Local nudges for non-throttled types will trigger a sync.
849  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
850      .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
851                            RecordSyncShare(&times)));
852  scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
853  RunLoop();
854  Mock::VerifyAndClearExpectations(syncer());
855
856  StopSyncScheduler();
857}
858
859// Test nudges / polls don't run in config mode and config tasks do.
860TEST_F(SyncSchedulerTest, ConfigurationMode) {
861  TimeDelta poll(TimeDelta::FromMilliseconds(15));
862  SyncShareTimes times;
863  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
864
865  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
866
867  const ModelTypeSet nudge_types(AUTOFILL);
868  scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
869  scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
870
871  const ModelTypeSet config_types(BOOKMARKS);
872
873  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
874      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
875                      RecordSyncShare(&times)))
876      .RetiresOnSaturation();
877  CallbackCounter ready_counter;
878  CallbackCounter retry_counter;
879  ConfigurationParams params(
880      GetUpdatesCallerInfo::RECONFIGURATION,
881      config_types,
882      TypesToRoutingInfo(config_types),
883      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
884      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
885  scheduler()->ScheduleConfiguration(params);
886  RunLoop();
887  ASSERT_EQ(1, ready_counter.times_called());
888  ASSERT_EQ(0, retry_counter.times_called());
889
890  Mock::VerifyAndClearExpectations(syncer());
891
892  // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
893  scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
894  SyncShareTimes times2;
895  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
896      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
897                      RecordSyncShare(&times2)));
898
899  // TODO(tim): Figure out how to remove this dangerous need to reset
900  // routing info between mode switches.
901  context()->SetRoutingInfo(routing_info());
902  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
903
904  RunLoop();
905  Mock::VerifyAndClearExpectations(syncer());
906}
907
908class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
909  virtual void SetUp() {
910    SyncSchedulerTest::SetUp();
911    UseMockDelayProvider();
912    EXPECT_CALL(*delay(), GetDelay(_))
913        .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
914  }
915
916  virtual void TearDown() {
917    StopSyncScheduler();
918    SyncSchedulerTest::TearDown();
919  }
920};
921
922// Have the sycner fail during commit.  Expect that the scheduler enters
923// backoff.
924TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
925  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
926      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
927                      QuitLoopNowAction()));
928  EXPECT_TRUE(RunAndGetBackoff());
929}
930
931// Have the syncer fail during download updates and succeed on the first
932// retry.  Expect that this clears the backoff state.
933TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
934  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
935      .WillOnce(DoAll(
936          Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
937          Return(true)))
938      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
939                      QuitLoopNowAction()));
940  EXPECT_FALSE(RunAndGetBackoff());
941}
942
943// Have the syncer fail during commit and succeed on the first retry.  Expect
944// that this clears the backoff state.
945TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
946  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
947      .WillOnce(DoAll(
948          Invoke(sessions::test_util::SimulateCommitFailed),
949          Return(true)))
950      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
951                      QuitLoopNowAction()));
952  EXPECT_FALSE(RunAndGetBackoff());
953}
954
955// Have the syncer fail to download updates and fail again on the retry.
956// Expect this will leave the scheduler in backoff.
957TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
958  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
959      .WillOnce(DoAll(
960          Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
961          Return(true)))
962      .WillRepeatedly(DoAll(
963              Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
964              QuitLoopNowAction()));
965  EXPECT_TRUE(RunAndGetBackoff());
966}
967
968// Have the syncer fail to get the encryption key yet succeed in downloading
969// updates. Expect this will leave the scheduler in backoff.
970TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
971  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
972      .WillOnce(DoAll(
973          Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
974          Return(true)))
975      .WillRepeatedly(DoAll(
976              Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
977              QuitLoopNowAction()));
978  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
979
980  ModelTypeSet types(BOOKMARKS);
981  CallbackCounter ready_counter;
982  CallbackCounter retry_counter;
983  ConfigurationParams params(
984      GetUpdatesCallerInfo::RECONFIGURATION,
985      types,
986      TypesToRoutingInfo(types),
987      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
988      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
989  scheduler()->ScheduleConfiguration(params);
990  RunLoop();
991
992  EXPECT_TRUE(scheduler()->IsBackingOff());
993}
994
995// Test that no polls or extraneous nudges occur when in backoff.
996TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
997  SyncShareTimes times;
998  TimeDelta poll(TimeDelta::FromMilliseconds(5));
999  const ModelTypeSet types(BOOKMARKS);
1000  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1001  UseMockDelayProvider();
1002
1003  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1004      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1005                      RecordSyncShareMultiple(&times, 1U)));
1006  EXPECT_CALL(*delay(), GetDelay(_)).
1007      WillRepeatedly(Return(TimeDelta::FromDays(1)));
1008
1009  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1010
1011  // This nudge should fail and put us into backoff.  Thanks to our mock
1012  // GetDelay() setup above, this will be a long backoff.
1013  scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
1014  RunLoop();
1015
1016  // From this point forward, no SyncShare functions should be invoked.
1017  Mock::VerifyAndClearExpectations(syncer());
1018
1019  // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1020  PumpLoopFor(poll * 10);
1021
1022  // Try (and fail) to schedule a nudge.
1023  scheduler()->ScheduleLocalNudge(
1024      base::TimeDelta::FromMilliseconds(1),
1025      types,
1026      FROM_HERE);
1027
1028  Mock::VerifyAndClearExpectations(syncer());
1029  Mock::VerifyAndClearExpectations(delay());
1030
1031  EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1032
1033  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1034
1035  CallbackCounter ready_counter;
1036  CallbackCounter retry_counter;
1037  ConfigurationParams params(
1038      GetUpdatesCallerInfo::RECONFIGURATION,
1039      types,
1040      TypesToRoutingInfo(types),
1041      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1042      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1043  scheduler()->ScheduleConfiguration(params);
1044  PumpLoop();
1045  ASSERT_EQ(0, ready_counter.times_called());
1046  ASSERT_EQ(1, retry_counter.times_called());
1047
1048}
1049
1050// Test that backoff is shaping traffic properly with consecutive errors.
1051TEST_F(SyncSchedulerTest, BackoffElevation) {
1052  SyncShareTimes times;
1053  UseMockDelayProvider();
1054
1055  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1056      .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1057          RecordSyncShareMultiple(&times, kMinNumSamples)));
1058
1059  const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1060  const TimeDelta second = TimeDelta::FromMilliseconds(2);
1061  const TimeDelta third = TimeDelta::FromMilliseconds(3);
1062  const TimeDelta fourth = TimeDelta::FromMilliseconds(4);
1063  const TimeDelta fifth = TimeDelta::FromMilliseconds(5);
1064  const TimeDelta sixth = TimeDelta::FromDays(1);
1065
1066  EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1067          .RetiresOnSaturation();
1068  EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1069          .RetiresOnSaturation();
1070  EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1071          .RetiresOnSaturation();
1072  EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1073          .RetiresOnSaturation();
1074  EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1075
1076  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1077
1078  // Run again with a nudge.
1079  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1080  RunLoop();
1081
1082  ASSERT_EQ(kMinNumSamples, times.size());
1083  EXPECT_GE(times[1] - times[0], second);
1084  EXPECT_GE(times[2] - times[1], third);
1085  EXPECT_GE(times[3] - times[2], fourth);
1086  EXPECT_GE(times[4] - times[3], fifth);
1087}
1088
1089// Test that things go back to normal once a retry makes forward progress.
1090TEST_F(SyncSchedulerTest, BackoffRelief) {
1091  SyncShareTimes times;
1092  const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1093  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1094  UseMockDelayProvider();
1095
1096  const TimeDelta backoff = TimeDelta::FromMilliseconds(5);
1097  EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1098
1099  // Optimal start for the post-backoff poll party.
1100  TimeTicks optimal_start = TimeTicks::Now();
1101  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1102
1103  // Kick off the test with a failed nudge.
1104  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1105      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1106                      RecordSyncShare(&times)));
1107  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1108  RunLoop();
1109  Mock::VerifyAndClearExpectations(syncer());
1110  TimeTicks optimal_job_time = optimal_start;
1111  ASSERT_EQ(1U, times.size());
1112  EXPECT_GE(times[0], optimal_job_time);
1113
1114  // The retry succeeds.
1115  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1116      .WillOnce(DoAll(
1117              Invoke(sessions::test_util::SimulateNormalSuccess),
1118              RecordSyncShare(&times)));
1119  RunLoop();
1120  Mock::VerifyAndClearExpectations(syncer());
1121  optimal_job_time = optimal_job_time + backoff;
1122  ASSERT_EQ(2U, times.size());
1123  EXPECT_GE(times[1], optimal_job_time);
1124
1125  // Now let the Poll timer do its thing.
1126  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1127      .WillRepeatedly(DoAll(
1128              Invoke(sessions::test_util::SimulatePollSuccess),
1129              RecordSyncShareMultiple(&times, kMinNumSamples)));
1130  RunLoop();
1131  Mock::VerifyAndClearExpectations(syncer());
1132  ASSERT_EQ(kMinNumSamples, times.size());
1133  for (size_t i = 2; i < times.size(); i++) {
1134    optimal_job_time = optimal_job_time + poll;
1135    SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1136    EXPECT_GE(times[i], optimal_job_time);
1137  }
1138
1139  StopSyncScheduler();
1140}
1141
1142// Test that poll failures are ignored.  They should have no effect on
1143// subsequent poll attempts, nor should they trigger a backoff/retry.
1144TEST_F(SyncSchedulerTest, TransientPollFailure) {
1145  SyncShareTimes times;
1146  const TimeDelta poll_interval(TimeDelta::FromMilliseconds(1));
1147  scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1148  UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1149
1150  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1151      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1152                      RecordSyncShare(&times)))
1153      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1154                      RecordSyncShare(&times)));
1155
1156  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1157
1158  // Run the unsucessful poll. The failed poll should not trigger backoff.
1159  RunLoop();
1160  EXPECT_FALSE(scheduler()->IsBackingOff());
1161
1162  // Run the successful poll.
1163  RunLoop();
1164  EXPECT_FALSE(scheduler()->IsBackingOff());
1165}
1166
1167// Test that starting the syncer thread without a valid connection doesn't
1168// break things when a connection is detected.
1169TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1170  connection()->SetServerNotReachable();
1171  connection()->UpdateConnectionStatus();
1172  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1173    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1174                    Return(true)))
1175    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1176                    Return(true)));
1177  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1178
1179  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1180  // Should save the nudge for until after the server is reachable.
1181  base::MessageLoop::current()->RunUntilIdle();
1182
1183  scheduler()->OnConnectionStatusChange();
1184  connection()->SetServerReachable();
1185  connection()->UpdateConnectionStatus();
1186  base::MessageLoop::current()->RunUntilIdle();
1187}
1188
1189TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1190  UseMockDelayProvider();
1191  EXPECT_CALL(*delay(), GetDelay(_))
1192      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1193
1194  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1195  connection()->SetServerNotReachable();
1196  connection()->UpdateConnectionStatus();
1197
1198  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1199    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1200                    Return(true)))
1201    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1202                    Return(true)));
1203
1204  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1205  PumpLoop(); // To get PerformDelayedNudge called.
1206  PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1207  ASSERT_TRUE(scheduler()->IsBackingOff());
1208
1209  // Before we run the scheduled canary, trigger a server connection change.
1210  scheduler()->OnConnectionStatusChange();
1211  connection()->SetServerReachable();
1212  connection()->UpdateConnectionStatus();
1213  base::MessageLoop::current()->RunUntilIdle();
1214}
1215
1216// This was supposed to test the scenario where we receive a nudge while a
1217// connection change canary is scheduled, but has not run yet.  Since we've made
1218// the connection change canary synchronous, this is no longer possible.
1219TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1220  UseMockDelayProvider();
1221  EXPECT_CALL(*delay(), GetDelay(_))
1222      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1223
1224  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1225  connection()->SetServerNotReachable();
1226  connection()->UpdateConnectionStatus();
1227
1228  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1229    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1230                    Return(true)))
1231    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1232                    Return(true)))
1233    .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1234                    QuitLoopNowAction()));
1235
1236  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1237
1238  PumpLoop(); // To get PerformDelayedNudge called.
1239  PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1240  ASSERT_TRUE(scheduler()->IsBackingOff());
1241
1242  // Before we run the scheduled canary, trigger a server connection change.
1243  scheduler()->OnConnectionStatusChange();
1244  PumpLoop();
1245  connection()->SetServerReachable();
1246  connection()->UpdateConnectionStatus();
1247  scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1248  base::MessageLoop::current()->RunUntilIdle();
1249}
1250
1251// Tests that we don't crash trying to run two canaries at once if we receive
1252// extra connection status change notifications.  See crbug.com/190085.
1253TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1254  EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1255      .WillRepeatedly(DoAll(
1256              Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1257              Return(true)));
1258  StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1259  connection()->SetServerNotReachable();
1260  connection()->UpdateConnectionStatus();
1261
1262  ModelTypeSet model_types(BOOKMARKS);
1263  CallbackCounter ready_counter;
1264  CallbackCounter retry_counter;
1265  ConfigurationParams params(
1266      GetUpdatesCallerInfo::RECONFIGURATION,
1267      model_types,
1268      TypesToRoutingInfo(model_types),
1269      base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1270      base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1271  scheduler()->ScheduleConfiguration(params);
1272
1273  scheduler()->OnConnectionStatusChange();
1274  scheduler()->OnConnectionStatusChange();
1275
1276  PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1277}
1278
1279TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1280  SyncShareTimes times;
1281  TimeDelta poll(TimeDelta::FromMilliseconds(15));
1282  scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1283
1284  ::testing::InSequence seq;
1285  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1286      .WillRepeatedly(
1287          DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1288                RecordSyncShareMultiple(&times, kMinNumSamples)));
1289
1290  connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1291  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1292
1293  // Run to wait for polling.
1294  RunLoop();
1295
1296  // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1297  // but after poll finished with auth error from poll timer it should retry
1298  // poll once more
1299  EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1300      .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1301                      RecordSyncShare(&times)));
1302  scheduler()->OnCredentialsUpdated();
1303  connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1304  RunLoop();
1305  StopSyncScheduler();
1306}
1307
1308TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1309  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1310
1311  SyncShareTimes times;
1312  base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1313  scheduler()->OnReceivedGuRetryDelay(delay);
1314  EXPECT_EQ(delay, GetRetryTimerDelay());
1315
1316  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1317      .WillOnce(
1318          DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1319                RecordSyncShare(&times)));
1320
1321  // Run to wait for retrying.
1322  RunLoop();
1323
1324  StopSyncScheduler();
1325}
1326
1327TEST_F(SyncSchedulerTest, FailedRetry) {
1328  UseMockDelayProvider();
1329  EXPECT_CALL(*delay(), GetDelay(_))
1330      .WillRepeatedly(Return(TimeDelta::FromMilliseconds(1)));
1331
1332  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1333
1334  base::TimeDelta delay = base::TimeDelta::FromMilliseconds(1);
1335  scheduler()->OnReceivedGuRetryDelay(delay);
1336
1337  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1338      .WillOnce(
1339          DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1340                QuitLoopNowAction()));
1341
1342  // Run to wait for retrying.
1343  RunLoop();
1344
1345  EXPECT_TRUE(scheduler()->IsBackingOff());
1346  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1347      .WillOnce(
1348          DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1349                QuitLoopNowAction()));
1350
1351  // Run to wait for second retrying.
1352  RunLoop();
1353
1354  StopSyncScheduler();
1355}
1356
1357ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1358  EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1359}
1360
1361TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1362  StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1363
1364  SyncShareTimes times;
1365  base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1366  base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1367
1368  scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
1369                                           FROM_HERE);
1370  scheduler()->OnReceivedGuRetryDelay(delay1);
1371  EXPECT_EQ(delay1, GetRetryTimerDelay());
1372
1373  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1374      .WillOnce(DoAll(
1375          WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1376          WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1377          RecordSyncShare(&times)));
1378
1379  // Run nudge GU.
1380  RunLoop();
1381  EXPECT_EQ(delay2, GetRetryTimerDelay());
1382
1383  EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1384      .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1385                      RecordSyncShare(&times)));
1386
1387  // Run to wait for retrying.
1388  RunLoop();
1389
1390  StopSyncScheduler();
1391}
1392
1393}  // namespace syncer
1394