15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Copyright (c) 2012 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_ 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_ 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <map> 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <set> 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string> 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/basictypes.h" 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/memory/ref_counted.h" 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/threading/non_thread_safe.h" 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "base/threading/platform_thread.h" 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "net/base/net_export.h" 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "net/base/network_change_notifier.h" 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "net/url_request/url_request_throttler_entry.h" 197dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch#include "url/gurl.h" 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)namespace net { 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class BoundNetLog; 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class NetLog; 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// Class that registers URL request throttler entries for URLs being accessed 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// in order to supervise traffic. URL requests for HTTP contents should 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// register their URLs in this manager on each request. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// URLRequestThrottlerManager maintains a map of URL IDs to URL request 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// throttler entries. It creates URL request throttler entries when new URLs 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// are registered, and does garbage collection from time to time in order to 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// clean out outdated entries. URL ID consists of lowercased scheme, host, port 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)// and path. All URLs converted to the same ID will share the same entry. 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class NET_EXPORT URLRequestThrottlerManager 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) : NON_EXPORTED_BASE(public base::NonThreadSafe), 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public NetworkChangeNotifier::IPAddressObserver, 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public NetworkChangeNotifier::ConnectionTypeObserver { 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) public: 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) URLRequestThrottlerManager(); 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual ~URLRequestThrottlerManager(); 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Must be called for every request, returns the URL request throttler entry 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // associated with the URL. The caller must inform this entry of some events. 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Please refer to url_request_throttler_entry_interface.h for further 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // informations. 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) scoped_refptr<URLRequestThrottlerEntryInterface> RegisterRequestUrl( 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const GURL& url); 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Adds the given host to a list of sites for which exponential back-off 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // throttling will be disabled. Subdomains are not included, so they 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // must be added separately. 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void AddToOptOutList(const std::string& host); 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Registers a new entry in this service and overrides the existing entry (if 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // any) for the URL. The service will hold a reference to the entry. 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // It is only used by unit tests. 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void OverrideEntryForTests(const GURL& url, URLRequestThrottlerEntry* entry); 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Explicitly erases an entry. 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This is useful to remove those entries which have got infinite lifetime and 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // thus won't be garbage collected. 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // It is only used by unit tests. 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void EraseEntryForTests(const GURL& url); 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Turns threading model verification on or off. Any code that correctly 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // uses the network stack should preferably call this function to enable 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // verification of correct adherence to the network stack threading model. 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void set_enable_thread_checks(bool enable); 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool enable_thread_checks() const; 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Whether throttling is enabled or not. 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void set_enforce_throttling(bool enforce); 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool enforce_throttling(); 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Sets the NetLog instance to use. 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void set_net_log(NetLog* net_log); 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NetLog* net_log() const; 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // IPAddressObserver interface. 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void OnIPAddressChanged() OVERRIDE; 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // ConnectionTypeObserver interface. 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) virtual void OnConnectionTypeChanged( 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NetworkChangeNotifier::ConnectionType type) OVERRIDE; 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Method that allows us to transform a URL into an ID that can be used in our 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // map. Resulting IDs will be lowercase and consist of the scheme, host, port 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // and path (without query string, fragment, etc.). 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // If the URL is invalid, the invalid spec will be returned, without any 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // transformation. 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) std::string GetIdFromUrl(const GURL& url) const; 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Method that ensures the map gets cleaned from time to time. The period at 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // which garbage collecting happens is adjustable with the 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // kRequestBetweenCollecting constant. 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void GarbageCollectEntriesIfNecessary(); 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Method that does the actual work of garbage collecting. 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void GarbageCollectEntries(); 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // When we switch from online to offline or change IP addresses, we 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // clear all back-off history. This is a precaution in case the change in 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // online state now lets us communicate without error with servers that 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // we were previously getting 500 or 503 responses from (perhaps the 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // responses are from a badly-written proxy that should have returned a 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // 502 or 504 because it's upstream connection was down or it had no route 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // to the server). 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void OnNetworkChange(); 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Used by tests. 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int GetNumberOfEntriesForTests() const { return url_entries_.size(); } 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) private: 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // From each URL we generate an ID composed of the scheme, host, port and path 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // that allows us to uniquely map an entry to it. 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) typedef std::map<std::string, scoped_refptr<URLRequestThrottlerEntry> > 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UrlEntryMap; 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // We maintain a set of hosts that have opted out of exponential 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // back-off throttling. 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) typedef std::set<std::string> OptOutHosts; 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Maximum number of entries that we are willing to collect in our map. 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const unsigned int kMaximumNumberOfEntries; 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Number of requests that will be made between garbage collection. 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const unsigned int kRequestsBetweenCollecting; 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Map that contains a list of URL ID and their matching 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // URLRequestThrottlerEntry. 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UrlEntryMap url_entries_; 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Set of hosts that have opted out. 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) OptOutHosts opt_out_hosts_; 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // This keeps track of how many requests have been made. Used with 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // GarbageCollectEntries. 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unsigned int requests_since_last_gc_; 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Valid after construction. 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GURL::Replacements url_id_replacements_; 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Certain tests do not obey the net component's threading policy, so we 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // keep track of whether we're being used by tests, and turn off certain 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // checks. 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // TODO(joi): See if we can fix the offending unit tests and remove this 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // workaround. 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool enable_thread_checks_; 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Initially false, switches to true once we have logged because of back-off 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // being disabled for localhost. 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) bool logged_for_localhost_disabled_; 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // NetLog to use, if configured. 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BoundNetLog net_log_; 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) // Valid once we've registered for network notifications. 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base::PlatformThreadId registered_from_thread_; 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DISALLOW_COPY_AND_ASSIGN(URLRequestThrottlerManager); 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} // namespace net 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_ 167