1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_request_throttler_manager.h"
6
7#include "base/logging.h"
8#include "base/metrics/field_trial.h"
9#include "base/metrics/histogram.h"
10#include "base/strings/string_util.h"
11#include "net/base/net_log.h"
12#include "net/base/net_util.h"
13
14namespace net {
15
16const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
17const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
18
19URLRequestThrottlerManager::URLRequestThrottlerManager()
20    : requests_since_last_gc_(0),
21      enable_thread_checks_(false),
22      logged_for_localhost_disabled_(false),
23      registered_from_thread_(base::kInvalidThreadId) {
24  url_id_replacements_.ClearPassword();
25  url_id_replacements_.ClearUsername();
26  url_id_replacements_.ClearQuery();
27  url_id_replacements_.ClearRef();
28
29  NetworkChangeNotifier::AddIPAddressObserver(this);
30  NetworkChangeNotifier::AddConnectionTypeObserver(this);
31}
32
33URLRequestThrottlerManager::~URLRequestThrottlerManager() {
34  NetworkChangeNotifier::RemoveIPAddressObserver(this);
35  NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
36
37  // Since the manager object might conceivably go away before the
38  // entries, detach the entries' back-pointer to the manager.
39  UrlEntryMap::iterator i = url_entries_.begin();
40  while (i != url_entries_.end()) {
41    if (i->second.get() != NULL) {
42      i->second->DetachManager();
43    }
44    ++i;
45  }
46
47  // Delete all entries.
48  url_entries_.clear();
49}
50
51scoped_refptr<URLRequestThrottlerEntryInterface>
52    URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
53  DCHECK(!enable_thread_checks_ || CalledOnValidThread());
54
55  // Normalize the url.
56  std::string url_id = GetIdFromUrl(url);
57
58  // Periodically garbage collect old entries.
59  GarbageCollectEntriesIfNecessary();
60
61  // Find the entry in the map or create a new NULL entry.
62  scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
63
64  // If the entry exists but could be garbage collected at this point, we
65  // start with a fresh entry so that we possibly back off a bit less
66  // aggressively (i.e. this resets the error count when the entry's URL
67  // hasn't been requested in long enough).
68  if (entry.get() && entry->IsEntryOutdated()) {
69    entry = NULL;
70  }
71
72  // Create the entry if needed.
73  if (entry.get() == NULL) {
74    entry = new URLRequestThrottlerEntry(this, url_id);
75
76    // We only disable back-off throttling on an entry that we have
77    // just constructed.  This is to allow unit tests to explicitly override
78    // the entry for localhost URLs.  Given that we do not attempt to
79    // disable throttling for entries already handed out (see comment
80    // in AddToOptOutList), this is not a problem.
81    std::string host = url.host();
82    if (opt_out_hosts_.find(host) != opt_out_hosts_.end() ||
83        IsLocalhost(host)) {
84      if (!logged_for_localhost_disabled_ && IsLocalhost(host)) {
85        logged_for_localhost_disabled_ = true;
86        net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
87                          NetLog::StringCallback("host", &host));
88      }
89
90      // TODO(joi): Once sliding window is separate from back-off throttling,
91      // we can simply return a dummy implementation of
92      // URLRequestThrottlerEntryInterface here that never blocks anything (and
93      // not keep entries in url_entries_ for opted-out sites).
94      entry->DisableBackoffThrottling();
95    }
96  }
97
98  return entry;
99}
100
101void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
102  // There is an edge case here that we are not handling, to keep things
103  // simple.  If a host starts adding the opt-out header to its responses
104  // after there are already one or more entries in url_entries_ for that
105  // host, the pre-existing entries may still perform back-off throttling.
106  // In practice, this would almost never occur.
107  if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) {
108    UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1);
109
110    net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
111                      NetLog::StringCallback("host", &host));
112    opt_out_hosts_.insert(host);
113  }
114}
115
116void URLRequestThrottlerManager::OverrideEntryForTests(
117    const GURL& url,
118    URLRequestThrottlerEntry* entry) {
119  // Normalize the url.
120  std::string url_id = GetIdFromUrl(url);
121
122  // Periodically garbage collect old entries.
123  GarbageCollectEntriesIfNecessary();
124
125  url_entries_[url_id] = entry;
126}
127
128void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
129  // Normalize the url.
130  std::string url_id = GetIdFromUrl(url);
131  url_entries_.erase(url_id);
132}
133
134void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) {
135  enable_thread_checks_ = enable;
136}
137
138bool URLRequestThrottlerManager::enable_thread_checks() const {
139  return enable_thread_checks_;
140}
141
142void URLRequestThrottlerManager::set_net_log(NetLog* net_log) {
143  DCHECK(net_log);
144  net_log_ = BoundNetLog::Make(net_log,
145                               NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING);
146}
147
148NetLog* URLRequestThrottlerManager::net_log() const {
149  return net_log_.net_log();
150}
151
152void URLRequestThrottlerManager::OnIPAddressChanged() {
153  OnNetworkChange();
154}
155
156void URLRequestThrottlerManager::OnConnectionTypeChanged(
157    NetworkChangeNotifier::ConnectionType type) {
158  OnNetworkChange();
159}
160
161std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
162  if (!url.is_valid())
163    return url.possibly_invalid_spec();
164
165  GURL id = url.ReplaceComponents(url_id_replacements_);
166  return base::StringToLowerASCII(id.spec()).c_str();
167}
168
169void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
170  requests_since_last_gc_++;
171  if (requests_since_last_gc_ < kRequestsBetweenCollecting)
172    return;
173  requests_since_last_gc_ = 0;
174
175  GarbageCollectEntries();
176}
177
178void URLRequestThrottlerManager::GarbageCollectEntries() {
179  UrlEntryMap::iterator i = url_entries_.begin();
180  while (i != url_entries_.end()) {
181    if ((i->second)->IsEntryOutdated()) {
182      url_entries_.erase(i++);
183    } else {
184      ++i;
185    }
186  }
187
188  // In case something broke we want to make sure not to grow indefinitely.
189  while (url_entries_.size() > kMaximumNumberOfEntries) {
190    url_entries_.erase(url_entries_.begin());
191  }
192}
193
194void URLRequestThrottlerManager::OnNetworkChange() {
195  // Remove all entries.  Any entries that in-flight requests have a reference
196  // to will live until those requests end, and these entries may be
197  // inconsistent with new entries for the same URLs, but since what we
198  // want is a clean slate for the new connection type, this is OK.
199  url_entries_.clear();
200  requests_since_last_gc_ = 0;
201}
202
203}  // namespace net
204