1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_request_throttler_manager.h"
6
7#include "base/memory/scoped_ptr.h"
8#include "base/metrics/histogram_samples.h"
9#include "base/pickle.h"
10#include "base/stl_util.h"
11#include "base/strings/string_number_conversions.h"
12#include "base/strings/stringprintf.h"
13#include "base/test/histogram_tester.h"
14#include "base/time/time.h"
15#include "net/base/load_flags.h"
16#include "net/base/request_priority.h"
17#include "net/base/test_completion_callback.h"
18#include "net/url_request/url_request.h"
19#include "net/url_request/url_request_context.h"
20#include "net/url_request/url_request_test_util.h"
21#include "net/url_request/url_request_throttler_header_interface.h"
22#include "net/url_request/url_request_throttler_test_support.h"
23#include "testing/gtest/include/gtest/gtest.h"
24
25using base::TimeDelta;
26using base::TimeTicks;
27
28namespace net {
29
30namespace {
31
32const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled";
33
34class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
35 public:
36  explicit MockURLRequestThrottlerEntry(
37      URLRequestThrottlerManager* manager)
38      : URLRequestThrottlerEntry(manager, std::string()),
39        mock_backoff_entry_(&backoff_policy_) {
40    InitPolicy();
41  }
42  MockURLRequestThrottlerEntry(
43      URLRequestThrottlerManager* manager,
44      const TimeTicks& exponential_backoff_release_time,
45      const TimeTicks& sliding_window_release_time,
46      const TimeTicks& fake_now)
47      : URLRequestThrottlerEntry(manager, std::string()),
48        fake_time_now_(fake_now),
49        mock_backoff_entry_(&backoff_policy_) {
50    InitPolicy();
51
52    mock_backoff_entry_.set_fake_now(fake_now);
53    set_exponential_backoff_release_time(exponential_backoff_release_time);
54    set_sliding_window_release_time(sliding_window_release_time);
55  }
56
57  void InitPolicy() {
58    // Some tests become flaky if we have jitter.
59    backoff_policy_.jitter_factor = 0.0;
60
61    // This lets us avoid having to make multiple failures initially (this
62    // logic is already tested in the BackoffEntry unit tests).
63    backoff_policy_.num_errors_to_ignore = 0;
64  }
65
66  virtual const BackoffEntry* GetBackoffEntry() const OVERRIDE {
67    return &mock_backoff_entry_;
68  }
69
70  virtual BackoffEntry* GetBackoffEntry() OVERRIDE {
71    return &mock_backoff_entry_;
72  }
73
74  static bool ExplicitUserRequest(int load_flags) {
75    return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
76  }
77
78  void ResetToBlank(const TimeTicks& time_now) {
79    fake_time_now_ = time_now;
80    mock_backoff_entry_.set_fake_now(time_now);
81
82    GetBackoffEntry()->Reset();
83    GetBackoffEntry()->SetCustomReleaseTime(time_now);
84    set_sliding_window_release_time(time_now);
85  }
86
87  // Overridden for tests.
88  virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
89
90  void set_exponential_backoff_release_time(
91      const base::TimeTicks& release_time) {
92    GetBackoffEntry()->SetCustomReleaseTime(release_time);
93  }
94
95  base::TimeTicks sliding_window_release_time() const {
96    return URLRequestThrottlerEntry::sliding_window_release_time();
97  }
98
99  void set_sliding_window_release_time(
100      const base::TimeTicks& release_time) {
101    URLRequestThrottlerEntry::set_sliding_window_release_time(
102        release_time);
103  }
104
105  TimeTicks fake_time_now_;
106  MockBackoffEntry mock_backoff_entry_;
107
108 protected:
109  virtual ~MockURLRequestThrottlerEntry() {}
110};
111
112class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
113 public:
114  MockURLRequestThrottlerManager() : create_entry_index_(0) {}
115
116  // Method to process the URL using URLRequestThrottlerManager protected
117  // method.
118  std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
119
120  // Method to use the garbage collecting method of URLRequestThrottlerManager.
121  void DoGarbageCollectEntries() { GarbageCollectEntries(); }
122
123  // Returns the number of entries in the map.
124  int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
125
126  void CreateEntry(bool is_outdated) {
127    TimeTicks time = TimeTicks::Now();
128    if (is_outdated) {
129      time -= TimeDelta::FromMilliseconds(
130          MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
131    }
132    std::string fake_url_string("http://www.fakeurl.com/");
133    fake_url_string.append(base::IntToString(create_entry_index_++));
134    GURL fake_url(fake_url_string);
135    OverrideEntryForTests(
136        fake_url,
137        new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
138                                         TimeTicks::Now()));
139  }
140
141 private:
142  int create_entry_index_;
143};
144
145struct TimeAndBool {
146  TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
147    time = time_value;
148    result = expected;
149    line = line_num;
150  }
151  TimeTicks time;
152  bool result;
153  int line;
154};
155
156struct GurlAndString {
157  GurlAndString(const GURL& url_value,
158                const std::string& expected,
159                int line_num) {
160    url = url_value;
161    result = expected;
162    line = line_num;
163  }
164  GURL url;
165  std::string result;
166  int line;
167};
168
169}  // namespace
170
171class URLRequestThrottlerEntryTest : public testing::Test {
172 protected:
173  URLRequestThrottlerEntryTest()
174      : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL, NULL)) {
175  }
176
177  virtual void SetUp();
178
179  TimeTicks now_;
180  MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
181  scoped_refptr<MockURLRequestThrottlerEntry> entry_;
182
183  TestURLRequestContext context_;
184  scoped_ptr<URLRequest> request_;
185};
186
187void URLRequestThrottlerEntryTest::SetUp() {
188  request_->SetLoadFlags(0);
189
190  now_ = TimeTicks::Now();
191  entry_ = new MockURLRequestThrottlerEntry(&manager_);
192  entry_->ResetToBlank(now_);
193}
194
195std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
196  return out << time.ToInternalValue();
197}
198
199TEST_F(URLRequestThrottlerEntryTest, CanThrottleRequest) {
200  TestNetworkDelegate d;
201  context_.set_network_delegate(&d);
202  entry_->set_exponential_backoff_release_time(
203      entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
204
205  d.set_can_throttle_requests(false);
206  EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
207                                           context_.network_delegate()));
208  d.set_can_throttle_requests(true);
209  EXPECT_TRUE(entry_->ShouldRejectRequest(*request_,
210                                          context_.network_delegate()));
211}
212
213TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
214  base::HistogramTester histogram_tester;
215  entry_->set_exponential_backoff_release_time(
216      entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
217  EXPECT_TRUE(entry_->ShouldRejectRequest(*request_,
218                                          context_.network_delegate()));
219
220  // Also end-to-end test the load flags exceptions.
221  request_->SetLoadFlags(LOAD_MAYBE_USER_GESTURE);
222  EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
223                                           context_.network_delegate()));
224
225  histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 1);
226  histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 1);
227}
228
229TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
230  base::HistogramTester histogram_tester;
231  entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
232  EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
233                                           context_.network_delegate()));
234  entry_->set_exponential_backoff_release_time(
235      entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
236  EXPECT_FALSE(entry_->ShouldRejectRequest(*request_,
237                                           context_.network_delegate()));
238
239  histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 2);
240  histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 0);
241}
242
243TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
244  MockURLRequestThrottlerHeaderAdapter failure_response(503);
245  entry_->UpdateWithResponse(std::string(), &failure_response);
246  EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
247      << "A failure should increase the release_time";
248}
249
250TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
251  MockURLRequestThrottlerHeaderAdapter success_response(200);
252  entry_->UpdateWithResponse(std::string(), &success_response);
253  EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
254      << "A success should not add any delay";
255}
256
257TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
258  MockURLRequestThrottlerHeaderAdapter failure_response(503);
259  MockURLRequestThrottlerHeaderAdapter success_response(200);
260  entry_->UpdateWithResponse(std::string(), &success_response);
261  entry_->UpdateWithResponse(std::string(), &failure_response);
262  EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
263      << "This scenario should add delay";
264  entry_->UpdateWithResponse(std::string(), &success_response);
265}
266
267TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
268  TimeDelta lifetime = TimeDelta::FromMilliseconds(
269      MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
270  const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
271
272  TimeAndBool test_values[] = {
273      TimeAndBool(now_, false, __LINE__),
274      TimeAndBool(now_ - kFiveMs, false, __LINE__),
275      TimeAndBool(now_ + kFiveMs, false, __LINE__),
276      TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
277      TimeAndBool(now_ - lifetime, true, __LINE__),
278      TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
279
280  for (unsigned int i = 0; i < arraysize(test_values); ++i) {
281    entry_->set_exponential_backoff_release_time(test_values[i].time);
282    EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
283        "Test case #" << i << " line " << test_values[i].line << " failed";
284  }
285}
286
287TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
288  for (int i = 0; i < 30; ++i) {
289    MockURLRequestThrottlerHeaderAdapter response_adapter(503);
290    entry_->UpdateWithResponse(std::string(), &response_adapter);
291  }
292
293  TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
294  EXPECT_EQ(delay.InMilliseconds(),
295            MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
296}
297
298TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
299  MockURLRequestThrottlerHeaderAdapter response_adapter(503);
300  for (int i = 0; i < 5; ++i)
301    entry_->UpdateWithResponse(std::string(), &response_adapter);
302
303  TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
304
305  // Inform the entry that a response body was malformed, which is supposed to
306  // increase the back-off time.  Note that we also submit a successful
307  // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
308  // is what happens in practice (if a body is received, then a non-500
309  // response must also have been received).
310  entry_->ReceivedContentWasMalformed(200);
311  MockURLRequestThrottlerHeaderAdapter success_adapter(200);
312  entry_->UpdateWithResponse(std::string(), &success_adapter);
313  EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
314}
315
316TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
317  int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
318  int sliding_window =
319      URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
320
321  TimeTicks time_1 = entry_->fake_time_now_ +
322      TimeDelta::FromMilliseconds(sliding_window / 3);
323  TimeTicks time_2 = entry_->fake_time_now_ +
324      TimeDelta::FromMilliseconds(2 * sliding_window / 3);
325  TimeTicks time_3 = entry_->fake_time_now_ +
326      TimeDelta::FromMilliseconds(sliding_window);
327  TimeTicks time_4 = entry_->fake_time_now_ +
328      TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
329
330  entry_->set_exponential_backoff_release_time(time_1);
331
332  for (int i = 0; i < max_send / 2; ++i) {
333    EXPECT_EQ(2 * sliding_window / 3,
334              entry_->ReserveSendingTimeForNextRequest(time_2));
335  }
336  EXPECT_EQ(time_2, entry_->sliding_window_release_time());
337
338  entry_->fake_time_now_ = time_3;
339
340  for (int i = 0; i < (max_send + 1) / 2; ++i)
341    EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
342
343  EXPECT_EQ(time_4, entry_->sliding_window_release_time());
344}
345
346TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
347  ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
348  ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
349      LOAD_MAYBE_USER_GESTURE));
350  ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
351      ~LOAD_MAYBE_USER_GESTURE));
352}
353
354class URLRequestThrottlerManagerTest : public testing::Test {
355 protected:
356  URLRequestThrottlerManagerTest()
357      : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL, NULL)) {
358  }
359
360  virtual void SetUp() {
361    request_->SetLoadFlags(0);
362  }
363
364  void ExpectEntryAllowsAllOnErrorIfOptedOut(
365      URLRequestThrottlerEntryInterface* entry,
366      bool opted_out,
367      const URLRequest& request) {
368    EXPECT_FALSE(entry->ShouldRejectRequest(request,
369                                            context_.network_delegate()));
370    MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
371    for (int i = 0; i < 10; ++i) {
372      // Host doesn't really matter in this scenario so we skip it.
373      entry->UpdateWithResponse(std::string(), &failure_adapter);
374    }
375    EXPECT_NE(opted_out, entry->ShouldRejectRequest(
376                  request, context_.network_delegate()));
377
378    if (opted_out) {
379      // We're not mocking out GetTimeNow() in this scenario
380      // so add a 100 ms buffer to avoid flakiness (that should always
381      // give enough time to get from the TimeTicks::Now() call here
382      // to the TimeTicks::Now() call in the entry class).
383      EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
384                entry->GetExponentialBackoffReleaseTime());
385    } else {
386      // As above, add 100 ms.
387      EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
388                entry->GetExponentialBackoffReleaseTime());
389    }
390  }
391
392  // context_ must be declared before request_.
393  TestURLRequestContext context_;
394  scoped_ptr<URLRequest> request_;
395};
396
397TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
398  MockURLRequestThrottlerManager manager;
399  GurlAndString test_values[] = {
400      GurlAndString(GURL("http://www.example.com"),
401                    std::string("http://www.example.com/"),
402                    __LINE__),
403      GurlAndString(GURL("http://www.Example.com"),
404                    std::string("http://www.example.com/"),
405                    __LINE__),
406      GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
407                    std::string("http://www.ex4mple.com/pr4c71c41"),
408                    __LINE__),
409      GurlAndString(GURL("http://www.example.com/0/token/false"),
410                    std::string("http://www.example.com/0/token/false"),
411                    __LINE__),
412      GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
413                    std::string("http://www.example.com/index.php"),
414                    __LINE__),
415      GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
416                    std::string("http://www.example.com/index.php"),
417                    __LINE__),
418      GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
419                    std::string("http://www.example.com/index.php"),
420                    __LINE__),
421      GurlAndString(GURL("http://www.example.com:1234/"),
422                    std::string("http://www.example.com:1234/"),
423                    __LINE__)};
424
425  for (unsigned int i = 0; i < arraysize(test_values); ++i) {
426    std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
427    EXPECT_EQ(temp, test_values[i].result) <<
428        "Test case #" << i << " line " << test_values[i].line << " failed";
429  }
430}
431
432TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
433  MockURLRequestThrottlerManager manager;
434
435  manager.CreateEntry(true);  // true = Entry is outdated.
436  manager.CreateEntry(true);
437  manager.CreateEntry(true);
438  manager.DoGarbageCollectEntries();
439  EXPECT_EQ(0, manager.GetNumberOfEntries());
440
441  manager.CreateEntry(false);
442  manager.CreateEntry(false);
443  manager.CreateEntry(false);
444  manager.CreateEntry(true);
445  manager.DoGarbageCollectEntries();
446  EXPECT_EQ(3, manager.GetNumberOfEntries());
447}
448
449TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
450  MockURLRequestThrottlerManager manager;
451
452  manager.RegisterRequestUrl(GURL("http://www.example.com/"));
453  manager.RegisterRequestUrl(GURL("http://www.google.com/"));
454  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
455  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
456  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
457
458  EXPECT_EQ(3, manager.GetNumberOfEntries());
459}
460
461TEST_F(URLRequestThrottlerManagerTest, OptOutHeader) {
462  MockURLRequestThrottlerManager manager;
463  scoped_refptr<URLRequestThrottlerEntryInterface> entry =
464      manager.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
465
466  // Fake a response with the opt-out header.
467  MockURLRequestThrottlerHeaderAdapter response_adapter(
468      std::string(),
469      MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue,
470      200);
471  entry->UpdateWithResponse("www.google.com", &response_adapter);
472
473  // Ensure that the same entry on error always allows everything.
474  ExpectEntryAllowsAllOnErrorIfOptedOut(entry.get(), true, *request_);
475
476  // Ensure that a freshly created entry (for a different URL on an
477  // already opted-out host) also gets "always allow" behavior.
478  scoped_refptr<URLRequestThrottlerEntryInterface> other_entry =
479      manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
480  ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry.get(), true, *request_);
481
482  // Fake a response with the opt-out header incorrectly specified.
483  scoped_refptr<URLRequestThrottlerEntryInterface> no_opt_out_entry =
484      manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
485  MockURLRequestThrottlerHeaderAdapter wrong_adapter(
486      std::string(), "yesplease", 200);
487  no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
488  ExpectEntryAllowsAllOnErrorIfOptedOut(
489      no_opt_out_entry.get(), false, *request_);
490
491  // A localhost entry should always be opted out.
492  scoped_refptr<URLRequestThrottlerEntryInterface> localhost_entry =
493      manager.RegisterRequestUrl(GURL("http://localhost/hello"));
494  ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry.get(), true, *request_);
495}
496
497TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
498  for (int i = 0; i < 3; ++i) {
499    MockURLRequestThrottlerManager manager;
500    scoped_refptr<URLRequestThrottlerEntryInterface> entry_before =
501        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
502    MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
503    for (int j = 0; j < 10; ++j) {
504      // Host doesn't really matter in this scenario so we skip it.
505      entry_before->UpdateWithResponse(std::string(), &failure_adapter);
506    }
507    EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_,
508                                                  context_.network_delegate()));
509
510    switch (i) {
511      case 0:
512        manager.OnIPAddressChanged();
513        break;
514      case 1:
515        manager.OnConnectionTypeChanged(
516            NetworkChangeNotifier::CONNECTION_UNKNOWN);
517        break;
518      case 2:
519        manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE);
520        break;
521      default:
522        FAIL();
523    }
524
525    scoped_refptr<URLRequestThrottlerEntryInterface> entry_after =
526        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
527    EXPECT_FALSE(entry_after->ShouldRejectRequest(
528                     *request_, context_.network_delegate()));
529  }
530}
531
532}  // namespace net
533