1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#ifndef NET_URL_REQUEST_URL_FETCHER_CORE_H_ 6#define NET_URL_REQUEST_URL_FETCHER_CORE_H_ 7 8#include <set> 9#include <string> 10 11#include "base/basictypes.h" 12#include "base/compiler_specific.h" 13#include "base/debug/stack_trace.h" 14#include "base/files/file_path.h" 15#include "base/lazy_instance.h" 16#include "base/memory/ref_counted.h" 17#include "base/memory/scoped_ptr.h" 18#include "base/timer/timer.h" 19#include "net/base/host_port_pair.h" 20#include "net/http/http_request_headers.h" 21#include "net/url_request/url_fetcher.h" 22#include "net/url_request/url_request.h" 23#include "net/url_request/url_request_status.h" 24#include "url/gurl.h" 25 26namespace base { 27class SingleThreadTaskRunner; 28} // namespace base 29 30namespace net { 31class DrainableIOBuffer; 32class HttpResponseHeaders; 33class IOBuffer; 34class URLFetcherDelegate; 35class URLFetcherFileWriter; 36class URLFetcherResponseWriter; 37class URLRequestContextGetter; 38class URLRequestThrottlerEntryInterface; 39 40class URLFetcherCore 41 : public base::RefCountedThreadSafe<URLFetcherCore>, 42 public URLRequest::Delegate { 43 public: 44 URLFetcherCore(URLFetcher* fetcher, 45 const GURL& original_url, 46 URLFetcher::RequestType request_type, 47 URLFetcherDelegate* d); 48 49 // Starts the load. It's important that this not happen in the constructor 50 // because it causes the IO thread to begin AddRef()ing and Release()ing 51 // us. If our caller hasn't had time to fully construct us and take a 52 // reference, the IO thread could interrupt things, run a task, Release() 53 // us, and destroy us, leaving the caller with an already-destroyed object 54 // when construction finishes. 55 void Start(); 56 57 // Stops any in-progress load and ensures no callback will happen. It is 58 // safe to call this multiple times. 59 void Stop(); 60 61 // URLFetcher-like functions. 62 63 // For POST requests, set |content_type| to the MIME type of the 64 // content and set |content| to the data to upload. 65 void SetUploadData(const std::string& upload_content_type, 66 const std::string& upload_content); 67 void SetUploadFilePath(const std::string& upload_content_type, 68 const base::FilePath& file_path, 69 uint64 range_offset, 70 uint64 range_length, 71 scoped_refptr<base::TaskRunner> file_task_runner); 72 void SetChunkedUpload(const std::string& upload_content_type); 73 // Adds a block of data to be uploaded in a POST body. This can only be 74 // called after Start(). 75 void AppendChunkToUpload(const std::string& data, bool is_last_chunk); 76 // |flags| are flags to apply to the load operation--these should be 77 // one or more of the LOAD_* flags defined in net/base/load_flags.h. 78 void SetLoadFlags(int load_flags); 79 int GetLoadFlags() const; 80 void SetReferrer(const std::string& referrer); 81 void SetExtraRequestHeaders(const std::string& extra_request_headers); 82 void AddExtraRequestHeader(const std::string& header_line); 83 void GetExtraRequestHeaders(HttpRequestHeaders* headers) const; 84 void SetRequestContext(URLRequestContextGetter* request_context_getter); 85 // Set the URL that should be consulted for the third-party cookie 86 // blocking policy. 87 void SetFirstPartyForCookies(const GURL& first_party_for_cookies); 88 // Set the key and data callback that is used when setting the user 89 // data on any URLRequest objects this object creates. 90 void SetURLRequestUserData( 91 const void* key, 92 const URLFetcher::CreateDataCallback& create_data_callback); 93 void SetStopOnRedirect(bool stop_on_redirect); 94 void SetAutomaticallyRetryOn5xx(bool retry); 95 void SetMaxRetriesOn5xx(int max_retries); 96 int GetMaxRetriesOn5xx() const; 97 base::TimeDelta GetBackoffDelay() const; 98 void SetAutomaticallyRetryOnNetworkChanges(int max_retries); 99 void SaveResponseToFileAtPath( 100 const base::FilePath& file_path, 101 scoped_refptr<base::TaskRunner> file_task_runner); 102 void SaveResponseToTemporaryFile( 103 scoped_refptr<base::TaskRunner> file_task_runner); 104 HttpResponseHeaders* GetResponseHeaders() const; 105 HostPortPair GetSocketAddress() const; 106 bool WasFetchedViaProxy() const; 107 const GURL& GetOriginalURL() const; 108 const GURL& GetURL() const; 109 const URLRequestStatus& GetStatus() const; 110 int GetResponseCode() const; 111 const ResponseCookies& GetCookies() const; 112 bool FileErrorOccurred(int* out_error_code) const; 113 // Reports that the received content was malformed (i.e. failed parsing 114 // or validation). This makes the throttling logic that does exponential 115 // back-off when servers are having problems treat the current request as 116 // a failure. Your call to this method will be ignored if your request is 117 // already considered a failure based on the HTTP response code or response 118 // headers. 119 void ReceivedContentWasMalformed(); 120 bool GetResponseAsString(std::string* out_response_string) const; 121 bool GetResponseAsFilePath(bool take_ownership, 122 base::FilePath* out_response_path); 123 124 // Overridden from URLRequest::Delegate: 125 virtual void OnReceivedRedirect(URLRequest* request, 126 const GURL& new_url, 127 bool* defer_redirect) OVERRIDE; 128 virtual void OnResponseStarted(URLRequest* request) OVERRIDE; 129 virtual void OnReadCompleted(URLRequest* request, 130 int bytes_read) OVERRIDE; 131 virtual void OnCertificateRequested( 132 URLRequest* request, 133 SSLCertRequestInfo* cert_request_info) OVERRIDE; 134 135 URLFetcherDelegate* delegate() const { return delegate_; } 136 static void CancelAll(); 137 static int GetNumFetcherCores(); 138 static void SetEnableInterceptionForTests(bool enabled); 139 static void SetIgnoreCertificateRequests(bool ignored); 140 141 private: 142 friend class base::RefCountedThreadSafe<URLFetcherCore>; 143 144 // How should the response be stored? 145 enum ResponseDestinationType { 146 STRING, // Default: In a std::string 147 PERMANENT_FILE, // Write to a permanent file. 148 TEMP_FILE, // Write to a temporary file. 149 }; 150 151 class Registry { 152 public: 153 Registry(); 154 ~Registry(); 155 156 void AddURLFetcherCore(URLFetcherCore* core); 157 void RemoveURLFetcherCore(URLFetcherCore* core); 158 159 void CancelAll(); 160 161 int size() const { 162 return fetchers_.size(); 163 } 164 165 private: 166 std::set<URLFetcherCore*> fetchers_; 167 168 DISALLOW_COPY_AND_ASSIGN(Registry); 169 }; 170 171 virtual ~URLFetcherCore(); 172 173 // Wrapper functions that allow us to ensure actions happen on the right 174 // thread. 175 void StartOnIOThread(); 176 void StartURLRequest(); 177 void DidInitializeWriter(int result); 178 void StartURLRequestWhenAppropriate(); 179 void CancelURLRequest(); 180 void OnCompletedURLRequest(base::TimeDelta backoff_delay); 181 void InformDelegateFetchIsComplete(); 182 void NotifyMalformedContent(); 183 void DidFinishWriting(int result); 184 void RetryOrCompleteUrlFetch(); 185 186 // Deletes the request, removes it from the registry, and removes the 187 // destruction observer. 188 void ReleaseRequest(); 189 190 // Returns the max value of exponential back-off release time for 191 // |original_url_| and |url_|. 192 base::TimeTicks GetBackoffReleaseTime(); 193 194 void CompleteAddingUploadDataChunk(const std::string& data, 195 bool is_last_chunk); 196 197 // Writes all bytes stored in |data| with |response_writer_|. 198 // Returns OK if all bytes in |data| get written synchronously. Otherwise, 199 // returns ERR_IO_PENDING or a network error code. 200 int WriteBuffer(scoped_refptr<DrainableIOBuffer> data); 201 202 // Used to implement WriteBuffer(). 203 void DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data, int result); 204 205 // Read response bytes from the request. 206 void ReadResponse(); 207 208 // Notify Delegate about the progress of upload/download. 209 void InformDelegateUploadProgress(); 210 void InformDelegateUploadProgressInDelegateThread(int64 current, int64 total); 211 void InformDelegateDownloadProgress(); 212 void InformDelegateDownloadProgressInDelegateThread(int64 current, 213 int64 total); 214 void InformDelegateDownloadDataIfNecessary(int bytes_read); 215 void InformDelegateDownloadDataInDelegateThread( 216 scoped_ptr<std::string> download_data); 217 218 URLFetcher* fetcher_; // Corresponding fetcher object 219 GURL original_url_; // The URL we were asked to fetch 220 GURL url_; // The URL we eventually wound up at 221 URLFetcher::RequestType request_type_; // What type of request is this? 222 URLRequestStatus status_; // Status of the request 223 URLFetcherDelegate* delegate_; // Object to notify on completion 224 scoped_refptr<base::SingleThreadTaskRunner> delegate_task_runner_; 225 // Task runner for the creating thread. 226 scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_; 227 // Task runner for download file access. 228 scoped_refptr<base::TaskRunner> file_task_runner_; 229 // Task runner for upload file access. 230 scoped_refptr<base::TaskRunner> upload_file_task_runner_; 231 scoped_ptr<URLRequest> request_; // The actual request this wraps 232 int load_flags_; // Flags for the load operation 233 int response_code_; // HTTP status code for the request 234 std::string data_; // Results of the request, when we are 235 // storing the response as a string. 236 scoped_refptr<IOBuffer> buffer_; 237 // Read buffer 238 scoped_refptr<URLRequestContextGetter> request_context_getter_; 239 // Cookie/cache info for the request 240 GURL first_party_for_cookies_; // The first party URL for the request 241 // The user data to add to each newly-created URLRequest. 242 const void* url_request_data_key_; 243 URLFetcher::CreateDataCallback url_request_create_data_callback_; 244 ResponseCookies cookies_; // Response cookies 245 HttpRequestHeaders extra_request_headers_; 246 scoped_refptr<HttpResponseHeaders> response_headers_; 247 bool was_fetched_via_proxy_; 248 HostPortPair socket_address_; 249 250 bool upload_content_set_; // SetUploadData has been called 251 std::string upload_content_; // HTTP POST payload 252 base::FilePath upload_file_path_; // Path to file containing POST payload 253 uint64 upload_range_offset_; // Offset from the beginning of the file 254 // to be uploaded. 255 uint64 upload_range_length_; // The length of the part of file to be 256 // uploaded. 257 std::string upload_content_type_; // MIME type of POST payload 258 std::string referrer_; // HTTP Referer header value 259 bool is_chunked_upload_; // True if using chunked transfer encoding 260 261 // Used to determine how long to wait before making a request or doing a 262 // retry. 263 // 264 // Both of them can only be accessed on the IO thread. 265 // 266 // We need not only the throttler entry for |original_URL|, but also 267 // the one for |url|. For example, consider the case that URL A 268 // redirects to URL B, for which the server returns a 500 269 // response. In this case, the exponential back-off release time of 270 // URL A won't increase. If we retry without considering the 271 // back-off constraint of URL B, we may send out too many requests 272 // for URL A in a short period of time. 273 // 274 // Both of these will be NULL if 275 // URLRequestContext::throttler_manager() is NULL. 276 scoped_refptr<URLRequestThrottlerEntryInterface> 277 original_url_throttler_entry_; 278 scoped_refptr<URLRequestThrottlerEntryInterface> url_throttler_entry_; 279 280 // True if the URLFetcher has been cancelled. 281 bool was_cancelled_; 282 283 // Writer object to write response to the destination like file and string. 284 scoped_ptr<URLFetcherResponseWriter> response_writer_; 285 286 // If writing results to a file, |file_writer_| will manage creation, 287 // writing, and destruction of that file. 288 // |file_writer_| points to the same object as |response_writer_| when writing 289 // response to a file, otherwise, |file_writer_| is NULL. 290 URLFetcherFileWriter* file_writer_; 291 292 // Where should responses be saved? 293 ResponseDestinationType response_destination_; 294 295 // Path to the file where the response is written. 296 base::FilePath response_destination_file_path_; 297 298 // By default any server-initiated redirects are automatically followed. If 299 // this flag is set to true, however, a redirect will halt the fetch and call 300 // back to to the delegate immediately. 301 bool stop_on_redirect_; 302 // True when we're actually stopped due to a redirect halted by the above. We 303 // use this to ensure that |url_| is set to the redirect destination rather 304 // than the originally-fetched URL. 305 bool stopped_on_redirect_; 306 307 // If |automatically_retry_on_5xx_| is false, 5xx responses will be 308 // propagated to the observer, if it is true URLFetcher will automatically 309 // re-execute the request, after the back-off delay has expired. 310 // true by default. 311 bool automatically_retry_on_5xx_; 312 // |num_retries_on_5xx_| indicates how many times we've failed to successfully 313 // fetch this URL due to 5xx responses. Once this value exceeds the maximum 314 // number of retries specified by the owner URLFetcher instance, 315 // we'll give up. 316 int num_retries_on_5xx_; 317 // Maximum retries allowed when 5xx responses are received. 318 int max_retries_on_5xx_; 319 // Back-off time delay. 0 by default. 320 base::TimeDelta backoff_delay_; 321 322 // The number of retries that have been attempted due to ERR_NETWORK_CHANGED. 323 int num_retries_on_network_changes_; 324 // Maximum retries allowed when the request fails with ERR_NETWORK_CHANGED. 325 // 0 by default. 326 int max_retries_on_network_changes_; 327 328 // Timer to poll the progress of uploading for POST and PUT requests. 329 // When crbug.com/119629 is fixed, scoped_ptr is not necessary here. 330 scoped_ptr<base::RepeatingTimer<URLFetcherCore> > 331 upload_progress_checker_timer_; 332 // Number of bytes sent so far. 333 int64 current_upload_bytes_; 334 // Number of bytes received so far. 335 int64 current_response_bytes_; 336 // Total expected bytes to receive (-1 if it cannot be determined). 337 int64 total_response_bytes_; 338 339 // TODO(willchan): Get rid of this after debugging crbug.com/90971. 340 base::debug::StackTrace stack_trace_; 341 342 static base::LazyInstance<Registry> g_registry; 343 344 DISALLOW_COPY_AND_ASSIGN(URLFetcherCore); 345}; 346 347} // namespace net 348 349#endif // NET_URL_REQUEST_URL_FETCHER_CORE_H_ 350