1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "net/url_request/url_request_throttler_manager.h" 6 7#include "base/memory/scoped_ptr.h" 8#include "base/metrics/histogram_samples.h" 9#include "base/pickle.h" 10#include "base/stl_util.h" 11#include "base/strings/string_number_conversions.h" 12#include "base/strings/stringprintf.h" 13#include "base/test/histogram_tester.h" 14#include "base/time/time.h" 15#include "net/base/load_flags.h" 16#include "net/base/request_priority.h" 17#include "net/base/test_completion_callback.h" 18#include "net/url_request/url_request.h" 19#include "net/url_request/url_request_context.h" 20#include "net/url_request/url_request_test_util.h" 21#include "net/url_request/url_request_throttler_header_interface.h" 22#include "net/url_request/url_request_throttler_test_support.h" 23#include "testing/gtest/include/gtest/gtest.h" 24 25using base::TimeDelta; 26using base::TimeTicks; 27 28namespace net { 29 30namespace { 31 32const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled"; 33 34class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry { 35 public: 36 explicit MockURLRequestThrottlerEntry( 37 URLRequestThrottlerManager* manager) 38 : URLRequestThrottlerEntry(manager, std::string()), 39 mock_backoff_entry_(&backoff_policy_) { 40 InitPolicy(); 41 } 42 MockURLRequestThrottlerEntry( 43 URLRequestThrottlerManager* manager, 44 const TimeTicks& exponential_backoff_release_time, 45 const TimeTicks& sliding_window_release_time, 46 const TimeTicks& fake_now) 47 : URLRequestThrottlerEntry(manager, std::string()), 48 fake_time_now_(fake_now), 49 mock_backoff_entry_(&backoff_policy_) { 50 InitPolicy(); 51 52 mock_backoff_entry_.set_fake_now(fake_now); 53 set_exponential_backoff_release_time(exponential_backoff_release_time); 54 set_sliding_window_release_time(sliding_window_release_time); 55 } 56 57 void InitPolicy() { 58 // Some tests become flaky if we have jitter. 59 backoff_policy_.jitter_factor = 0.0; 60 61 // This lets us avoid having to make multiple failures initially (this 62 // logic is already tested in the BackoffEntry unit tests). 63 backoff_policy_.num_errors_to_ignore = 0; 64 } 65 66 virtual const BackoffEntry* GetBackoffEntry() const OVERRIDE { 67 return &mock_backoff_entry_; 68 } 69 70 virtual BackoffEntry* GetBackoffEntry() OVERRIDE { 71 return &mock_backoff_entry_; 72 } 73 74 static bool ExplicitUserRequest(int load_flags) { 75 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags); 76 } 77 78 void ResetToBlank(const TimeTicks& time_now) { 79 fake_time_now_ = time_now; 80 mock_backoff_entry_.set_fake_now(time_now); 81 82 GetBackoffEntry()->Reset(); 83 GetBackoffEntry()->SetCustomReleaseTime(time_now); 84 set_sliding_window_release_time(time_now); 85 } 86 87 // Overridden for tests. 88 virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; } 89 90 void set_exponential_backoff_release_time( 91 const base::TimeTicks& release_time) { 92 GetBackoffEntry()->SetCustomReleaseTime(release_time); 93 } 94 95 base::TimeTicks sliding_window_release_time() const { 96 return URLRequestThrottlerEntry::sliding_window_release_time(); 97 } 98 99 void set_sliding_window_release_time( 100 const base::TimeTicks& release_time) { 101 URLRequestThrottlerEntry::set_sliding_window_release_time( 102 release_time); 103 } 104 105 TimeTicks fake_time_now_; 106 MockBackoffEntry mock_backoff_entry_; 107 108 protected: 109 virtual ~MockURLRequestThrottlerEntry() {} 110}; 111 112class MockURLRequestThrottlerManager : public URLRequestThrottlerManager { 113 public: 114 MockURLRequestThrottlerManager() : create_entry_index_(0) {} 115 116 // Method to process the URL using URLRequestThrottlerManager protected 117 // method. 118 std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); } 119 120 // Method to use the garbage collecting method of URLRequestThrottlerManager. 121 void DoGarbageCollectEntries() { GarbageCollectEntries(); } 122 123 // Returns the number of entries in the map. 124 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); } 125 126 void CreateEntry(bool is_outdated) { 127 TimeTicks time = TimeTicks::Now(); 128 if (is_outdated) { 129 time -= TimeDelta::FromMilliseconds( 130 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000); 131 } 132 std::string fake_url_string("http://www.fakeurl.com/"); 133 fake_url_string.append(base::IntToString(create_entry_index_++)); 134 GURL fake_url(fake_url_string); 135 OverrideEntryForTests( 136 fake_url, 137 new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(), 138 TimeTicks::Now())); 139 } 140 141 private: 142 int create_entry_index_; 143}; 144 145struct TimeAndBool { 146 TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) { 147 time = time_value; 148 result = expected; 149 line = line_num; 150 } 151 TimeTicks time; 152 bool result; 153 int line; 154}; 155 156struct GurlAndString { 157 GurlAndString(const GURL& url_value, 158 const std::string& expected, 159 int line_num) { 160 url = url_value; 161 result = expected; 162 line = line_num; 163 } 164 GURL url; 165 std::string result; 166 int line; 167}; 168 169} // namespace 170 171class URLRequestThrottlerEntryTest : public testing::Test { 172 protected: 173 URLRequestThrottlerEntryTest() 174 : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL, NULL)) { 175 } 176 177 virtual void SetUp(); 178 179 TimeTicks now_; 180 MockURLRequestThrottlerManager manager_; // Dummy object, not used. 181 scoped_refptr<MockURLRequestThrottlerEntry> entry_; 182 183 TestURLRequestContext context_; 184 scoped_ptr<URLRequest> request_; 185}; 186 187void URLRequestThrottlerEntryTest::SetUp() { 188 request_->SetLoadFlags(0); 189 190 now_ = TimeTicks::Now(); 191 entry_ = new MockURLRequestThrottlerEntry(&manager_); 192 entry_->ResetToBlank(now_); 193} 194 195std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) { 196 return out << time.ToInternalValue(); 197} 198 199TEST_F(URLRequestThrottlerEntryTest, CanThrottleRequest) { 200 TestNetworkDelegate d; 201 context_.set_network_delegate(&d); 202 entry_->set_exponential_backoff_release_time( 203 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1)); 204 205 d.set_can_throttle_requests(false); 206 EXPECT_FALSE(entry_->ShouldRejectRequest(*request_, 207 context_.network_delegate())); 208 d.set_can_throttle_requests(true); 209 EXPECT_TRUE(entry_->ShouldRejectRequest(*request_, 210 context_.network_delegate())); 211} 212 213TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) { 214 base::HistogramTester histogram_tester; 215 entry_->set_exponential_backoff_release_time( 216 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1)); 217 EXPECT_TRUE(entry_->ShouldRejectRequest(*request_, 218 context_.network_delegate())); 219 220 // Also end-to-end test the load flags exceptions. 221 request_->SetLoadFlags(LOAD_MAYBE_USER_GESTURE); 222 EXPECT_FALSE(entry_->ShouldRejectRequest(*request_, 223 context_.network_delegate())); 224 225 histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 1); 226 histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 1); 227} 228 229TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) { 230 base::HistogramTester histogram_tester; 231 entry_->set_exponential_backoff_release_time(entry_->fake_time_now_); 232 EXPECT_FALSE(entry_->ShouldRejectRequest(*request_, 233 context_.network_delegate())); 234 entry_->set_exponential_backoff_release_time( 235 entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1)); 236 EXPECT_FALSE(entry_->ShouldRejectRequest(*request_, 237 context_.network_delegate())); 238 239 histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 2); 240 histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 0); 241} 242 243TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) { 244 MockURLRequestThrottlerHeaderAdapter failure_response(503); 245 entry_->UpdateWithResponse(std::string(), &failure_response); 246 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 247 << "A failure should increase the release_time"; 248} 249 250TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) { 251 MockURLRequestThrottlerHeaderAdapter success_response(200); 252 entry_->UpdateWithResponse(std::string(), &success_response); 253 EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 254 << "A success should not add any delay"; 255} 256 257TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) { 258 MockURLRequestThrottlerHeaderAdapter failure_response(503); 259 MockURLRequestThrottlerHeaderAdapter success_response(200); 260 entry_->UpdateWithResponse(std::string(), &success_response); 261 entry_->UpdateWithResponse(std::string(), &failure_response); 262 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 263 << "This scenario should add delay"; 264 entry_->UpdateWithResponse(std::string(), &success_response); 265} 266 267TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) { 268 TimeDelta lifetime = TimeDelta::FromMilliseconds( 269 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs); 270 const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5); 271 272 TimeAndBool test_values[] = { 273 TimeAndBool(now_, false, __LINE__), 274 TimeAndBool(now_ - kFiveMs, false, __LINE__), 275 TimeAndBool(now_ + kFiveMs, false, __LINE__), 276 TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__), 277 TimeAndBool(now_ - lifetime, true, __LINE__), 278 TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)}; 279 280 for (unsigned int i = 0; i < arraysize(test_values); ++i) { 281 entry_->set_exponential_backoff_release_time(test_values[i].time); 282 EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) << 283 "Test case #" << i << " line " << test_values[i].line << " failed"; 284 } 285} 286 287TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) { 288 for (int i = 0; i < 30; ++i) { 289 MockURLRequestThrottlerHeaderAdapter response_adapter(503); 290 entry_->UpdateWithResponse(std::string(), &response_adapter); 291 } 292 293 TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_; 294 EXPECT_EQ(delay.InMilliseconds(), 295 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs); 296} 297 298TEST_F(URLRequestThrottlerEntryTest, MalformedContent) { 299 MockURLRequestThrottlerHeaderAdapter response_adapter(503); 300 for (int i = 0; i < 5; ++i) 301 entry_->UpdateWithResponse(std::string(), &response_adapter); 302 303 TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime(); 304 305 // Inform the entry that a response body was malformed, which is supposed to 306 // increase the back-off time. Note that we also submit a successful 307 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that 308 // is what happens in practice (if a body is received, then a non-500 309 // response must also have been received). 310 entry_->ReceivedContentWasMalformed(200); 311 MockURLRequestThrottlerHeaderAdapter success_adapter(200); 312 entry_->UpdateWithResponse(std::string(), &success_adapter); 313 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures); 314} 315 316TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) { 317 int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold; 318 int sliding_window = 319 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs; 320 321 TimeTicks time_1 = entry_->fake_time_now_ + 322 TimeDelta::FromMilliseconds(sliding_window / 3); 323 TimeTicks time_2 = entry_->fake_time_now_ + 324 TimeDelta::FromMilliseconds(2 * sliding_window / 3); 325 TimeTicks time_3 = entry_->fake_time_now_ + 326 TimeDelta::FromMilliseconds(sliding_window); 327 TimeTicks time_4 = entry_->fake_time_now_ + 328 TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3); 329 330 entry_->set_exponential_backoff_release_time(time_1); 331 332 for (int i = 0; i < max_send / 2; ++i) { 333 EXPECT_EQ(2 * sliding_window / 3, 334 entry_->ReserveSendingTimeForNextRequest(time_2)); 335 } 336 EXPECT_EQ(time_2, entry_->sliding_window_release_time()); 337 338 entry_->fake_time_now_ = time_3; 339 340 for (int i = 0; i < (max_send + 1) / 2; ++i) 341 EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks())); 342 343 EXPECT_EQ(time_4, entry_->sliding_window_release_time()); 344} 345 346TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) { 347 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0)); 348 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest( 349 LOAD_MAYBE_USER_GESTURE)); 350 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest( 351 ~LOAD_MAYBE_USER_GESTURE)); 352} 353 354class URLRequestThrottlerManagerTest : public testing::Test { 355 protected: 356 URLRequestThrottlerManagerTest() 357 : request_(context_.CreateRequest(GURL(), DEFAULT_PRIORITY, NULL, NULL)) { 358 } 359 360 virtual void SetUp() { 361 request_->SetLoadFlags(0); 362 } 363 364 void ExpectEntryAllowsAllOnErrorIfOptedOut( 365 URLRequestThrottlerEntryInterface* entry, 366 bool opted_out, 367 const URLRequest& request) { 368 EXPECT_FALSE(entry->ShouldRejectRequest(request, 369 context_.network_delegate())); 370 MockURLRequestThrottlerHeaderAdapter failure_adapter(503); 371 for (int i = 0; i < 10; ++i) { 372 // Host doesn't really matter in this scenario so we skip it. 373 entry->UpdateWithResponse(std::string(), &failure_adapter); 374 } 375 EXPECT_NE(opted_out, entry->ShouldRejectRequest( 376 request, context_.network_delegate())); 377 378 if (opted_out) { 379 // We're not mocking out GetTimeNow() in this scenario 380 // so add a 100 ms buffer to avoid flakiness (that should always 381 // give enough time to get from the TimeTicks::Now() call here 382 // to the TimeTicks::Now() call in the entry class). 383 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), 384 entry->GetExponentialBackoffReleaseTime()); 385 } else { 386 // As above, add 100 ms. 387 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), 388 entry->GetExponentialBackoffReleaseTime()); 389 } 390 } 391 392 // context_ must be declared before request_. 393 TestURLRequestContext context_; 394 scoped_ptr<URLRequest> request_; 395}; 396 397TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) { 398 MockURLRequestThrottlerManager manager; 399 GurlAndString test_values[] = { 400 GurlAndString(GURL("http://www.example.com"), 401 std::string("http://www.example.com/"), 402 __LINE__), 403 GurlAndString(GURL("http://www.Example.com"), 404 std::string("http://www.example.com/"), 405 __LINE__), 406 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"), 407 std::string("http://www.ex4mple.com/pr4c71c41"), 408 __LINE__), 409 GurlAndString(GURL("http://www.example.com/0/token/false"), 410 std::string("http://www.example.com/0/token/false"), 411 __LINE__), 412 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"), 413 std::string("http://www.example.com/index.php"), 414 __LINE__), 415 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"), 416 std::string("http://www.example.com/index.php"), 417 __LINE__), 418 GurlAndString(GURL("http://www.example.com/index.php#superEntry"), 419 std::string("http://www.example.com/index.php"), 420 __LINE__), 421 GurlAndString(GURL("http://www.example.com:1234/"), 422 std::string("http://www.example.com:1234/"), 423 __LINE__)}; 424 425 for (unsigned int i = 0; i < arraysize(test_values); ++i) { 426 std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url); 427 EXPECT_EQ(temp, test_values[i].result) << 428 "Test case #" << i << " line " << test_values[i].line << " failed"; 429 } 430} 431 432TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) { 433 MockURLRequestThrottlerManager manager; 434 435 manager.CreateEntry(true); // true = Entry is outdated. 436 manager.CreateEntry(true); 437 manager.CreateEntry(true); 438 manager.DoGarbageCollectEntries(); 439 EXPECT_EQ(0, manager.GetNumberOfEntries()); 440 441 manager.CreateEntry(false); 442 manager.CreateEntry(false); 443 manager.CreateEntry(false); 444 manager.CreateEntry(true); 445 manager.DoGarbageCollectEntries(); 446 EXPECT_EQ(3, manager.GetNumberOfEntries()); 447} 448 449TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) { 450 MockURLRequestThrottlerManager manager; 451 452 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 453 manager.RegisterRequestUrl(GURL("http://www.google.com/")); 454 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0")); 455 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1")); 456 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure")); 457 458 EXPECT_EQ(3, manager.GetNumberOfEntries()); 459} 460 461TEST_F(URLRequestThrottlerManagerTest, OptOutHeader) { 462 MockURLRequestThrottlerManager manager; 463 scoped_refptr<URLRequestThrottlerEntryInterface> entry = 464 manager.RegisterRequestUrl(GURL("http://www.google.com/yodude")); 465 466 // Fake a response with the opt-out header. 467 MockURLRequestThrottlerHeaderAdapter response_adapter( 468 std::string(), 469 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue, 470 200); 471 entry->UpdateWithResponse("www.google.com", &response_adapter); 472 473 // Ensure that the same entry on error always allows everything. 474 ExpectEntryAllowsAllOnErrorIfOptedOut(entry.get(), true, *request_); 475 476 // Ensure that a freshly created entry (for a different URL on an 477 // already opted-out host) also gets "always allow" behavior. 478 scoped_refptr<URLRequestThrottlerEntryInterface> other_entry = 479 manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob")); 480 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry.get(), true, *request_); 481 482 // Fake a response with the opt-out header incorrectly specified. 483 scoped_refptr<URLRequestThrottlerEntryInterface> no_opt_out_entry = 484 manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit")); 485 MockURLRequestThrottlerHeaderAdapter wrong_adapter( 486 std::string(), "yesplease", 200); 487 no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter); 488 ExpectEntryAllowsAllOnErrorIfOptedOut( 489 no_opt_out_entry.get(), false, *request_); 490 491 // A localhost entry should always be opted out. 492 scoped_refptr<URLRequestThrottlerEntryInterface> localhost_entry = 493 manager.RegisterRequestUrl(GURL("http://localhost/hello")); 494 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry.get(), true, *request_); 495} 496 497TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) { 498 for (int i = 0; i < 3; ++i) { 499 MockURLRequestThrottlerManager manager; 500 scoped_refptr<URLRequestThrottlerEntryInterface> entry_before = 501 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 502 MockURLRequestThrottlerHeaderAdapter failure_adapter(503); 503 for (int j = 0; j < 10; ++j) { 504 // Host doesn't really matter in this scenario so we skip it. 505 entry_before->UpdateWithResponse(std::string(), &failure_adapter); 506 } 507 EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_, 508 context_.network_delegate())); 509 510 switch (i) { 511 case 0: 512 manager.OnIPAddressChanged(); 513 break; 514 case 1: 515 manager.OnConnectionTypeChanged( 516 NetworkChangeNotifier::CONNECTION_UNKNOWN); 517 break; 518 case 2: 519 manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE); 520 break; 521 default: 522 FAIL(); 523 } 524 525 scoped_refptr<URLRequestThrottlerEntryInterface> entry_after = 526 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 527 EXPECT_FALSE(entry_after->ShouldRejectRequest( 528 *request_, context_.network_delegate())); 529 } 530} 531 532} // namespace net 533