1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef NET_URL_REQUEST_URL_FETCHER_CORE_H_
6#define NET_URL_REQUEST_URL_FETCHER_CORE_H_
7
8#include <set>
9#include <string>
10
11#include "base/basictypes.h"
12#include "base/compiler_specific.h"
13#include "base/debug/stack_trace.h"
14#include "base/files/file_path.h"
15#include "base/lazy_instance.h"
16#include "base/memory/ref_counted.h"
17#include "base/memory/scoped_ptr.h"
18#include "base/timer/timer.h"
19#include "net/base/host_port_pair.h"
20#include "net/http/http_request_headers.h"
21#include "net/url_request/url_fetcher.h"
22#include "net/url_request/url_request.h"
23#include "net/url_request/url_request_status.h"
24#include "url/gurl.h"
25
26namespace base {
27class SingleThreadTaskRunner;
28}  // namespace base
29
30namespace net {
31class DrainableIOBuffer;
32class HttpResponseHeaders;
33class IOBuffer;
34class URLFetcherDelegate;
35class URLFetcherResponseWriter;
36class URLRequestContextGetter;
37class URLRequestThrottlerEntryInterface;
38
39class URLFetcherCore
40    : public base::RefCountedThreadSafe<URLFetcherCore>,
41      public URLRequest::Delegate {
42 public:
43  URLFetcherCore(URLFetcher* fetcher,
44                 const GURL& original_url,
45                 URLFetcher::RequestType request_type,
46                 URLFetcherDelegate* d);
47
48  // Starts the load. It's important that this not happen in the constructor
49  // because it causes the IO thread to begin AddRef()ing and Release()ing
50  // us. If our caller hasn't had time to fully construct us and take a
51  // reference, the IO thread could interrupt things, run a task, Release()
52  // us, and destroy us, leaving the caller with an already-destroyed object
53  // when construction finishes.
54  void Start();
55
56  // Stops any in-progress load and ensures no callback will happen. It is
57  // safe to call this multiple times.
58  void Stop();
59
60  // URLFetcher-like functions.
61
62  // For POST requests, set |content_type| to the MIME type of the
63  // content and set |content| to the data to upload.
64  void SetUploadData(const std::string& upload_content_type,
65                     const std::string& upload_content);
66  void SetUploadFilePath(const std::string& upload_content_type,
67                         const base::FilePath& file_path,
68                         uint64 range_offset,
69                         uint64 range_length,
70                         scoped_refptr<base::TaskRunner> file_task_runner);
71  void SetChunkedUpload(const std::string& upload_content_type);
72  // Adds a block of data to be uploaded in a POST body. This can only be
73  // called after Start().
74  void AppendChunkToUpload(const std::string& data, bool is_last_chunk);
75  // |flags| are flags to apply to the load operation--these should be
76  // one or more of the LOAD_* flags defined in net/base/load_flags.h.
77  void SetLoadFlags(int load_flags);
78  int GetLoadFlags() const;
79  void SetReferrer(const std::string& referrer);
80  void SetReferrerPolicy(URLRequest::ReferrerPolicy referrer_policy);
81  void SetExtraRequestHeaders(const std::string& extra_request_headers);
82  void AddExtraRequestHeader(const std::string& header_line);
83  void SetRequestContext(URLRequestContextGetter* request_context_getter);
84  // Set the URL that should be consulted for the third-party cookie
85  // blocking policy.
86  void SetFirstPartyForCookies(const GURL& first_party_for_cookies);
87  // Set the key and data callback that is used when setting the user
88  // data on any URLRequest objects this object creates.
89  void SetURLRequestUserData(
90      const void* key,
91      const URLFetcher::CreateDataCallback& create_data_callback);
92  void SetStopOnRedirect(bool stop_on_redirect);
93  void SetAutomaticallyRetryOn5xx(bool retry);
94  void SetMaxRetriesOn5xx(int max_retries);
95  int GetMaxRetriesOn5xx() const;
96  base::TimeDelta GetBackoffDelay() const;
97  void SetAutomaticallyRetryOnNetworkChanges(int max_retries);
98  void SaveResponseToFileAtPath(
99      const base::FilePath& file_path,
100      scoped_refptr<base::SequencedTaskRunner> file_task_runner);
101  void SaveResponseToTemporaryFile(
102      scoped_refptr<base::SequencedTaskRunner> file_task_runner);
103  void SaveResponseWithWriter(
104      scoped_ptr<URLFetcherResponseWriter> response_writer);
105  HttpResponseHeaders* GetResponseHeaders() const;
106  HostPortPair GetSocketAddress() const;
107  bool WasFetchedViaProxy() const;
108  const GURL& GetOriginalURL() const;
109  const GURL& GetURL() const;
110  const URLRequestStatus& GetStatus() const;
111  int GetResponseCode() const;
112  const ResponseCookies& GetCookies() const;
113  // Reports that the received content was malformed (i.e. failed parsing
114  // or validation). This makes the throttling logic that does exponential
115  // back-off when servers are having problems treat the current request as
116  // a failure. Your call to this method will be ignored if your request is
117  // already considered a failure based on the HTTP response code or response
118  // headers.
119  void ReceivedContentWasMalformed();
120  bool GetResponseAsString(std::string* out_response_string) const;
121  bool GetResponseAsFilePath(bool take_ownership,
122                             base::FilePath* out_response_path);
123
124  // Overridden from URLRequest::Delegate:
125  virtual void OnReceivedRedirect(URLRequest* request,
126                                  const RedirectInfo& redirect_info,
127                                  bool* defer_redirect) OVERRIDE;
128  virtual void OnResponseStarted(URLRequest* request) OVERRIDE;
129  virtual void OnReadCompleted(URLRequest* request,
130                               int bytes_read) OVERRIDE;
131  virtual void OnCertificateRequested(
132      URLRequest* request,
133      SSLCertRequestInfo* cert_request_info) OVERRIDE;
134
135  URLFetcherDelegate* delegate() const { return delegate_; }
136  static void CancelAll();
137  static int GetNumFetcherCores();
138  static void SetEnableInterceptionForTests(bool enabled);
139  static void SetIgnoreCertificateRequests(bool ignored);
140
141 private:
142  friend class base::RefCountedThreadSafe<URLFetcherCore>;
143
144  class Registry {
145   public:
146    Registry();
147    ~Registry();
148
149    void AddURLFetcherCore(URLFetcherCore* core);
150    void RemoveURLFetcherCore(URLFetcherCore* core);
151
152    void CancelAll();
153
154    int size() const {
155      return fetchers_.size();
156    }
157
158   private:
159    std::set<URLFetcherCore*> fetchers_;
160
161    DISALLOW_COPY_AND_ASSIGN(Registry);
162  };
163
164  virtual ~URLFetcherCore();
165
166  // Wrapper functions that allow us to ensure actions happen on the right
167  // thread.
168  void StartOnIOThread();
169  void StartURLRequest();
170  void DidInitializeWriter(int result);
171  void StartURLRequestWhenAppropriate();
172  void CancelURLRequest(int error);
173  void OnCompletedURLRequest(base::TimeDelta backoff_delay);
174  void InformDelegateFetchIsComplete();
175  void NotifyMalformedContent();
176  void DidFinishWriting(int result);
177  void RetryOrCompleteUrlFetch();
178
179  // Deletes the request, removes it from the registry, and removes the
180  // destruction observer.
181  void ReleaseRequest();
182
183  // Returns the max value of exponential back-off release time for
184  // |original_url_| and |url_|.
185  base::TimeTicks GetBackoffReleaseTime();
186
187  void CompleteAddingUploadDataChunk(const std::string& data,
188                                     bool is_last_chunk);
189
190  // Writes all bytes stored in |data| with |response_writer_|.
191  // Returns OK if all bytes in |data| get written synchronously. Otherwise,
192  // returns ERR_IO_PENDING or a network error code.
193  int WriteBuffer(scoped_refptr<DrainableIOBuffer> data);
194
195  // Used to implement WriteBuffer().
196  void DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data, int result);
197
198  // Read response bytes from the request.
199  void ReadResponse();
200
201  // Notify Delegate about the progress of upload/download.
202  void InformDelegateUploadProgress();
203  void InformDelegateUploadProgressInDelegateThread(int64 current, int64 total);
204  void InformDelegateDownloadProgress();
205  void InformDelegateDownloadProgressInDelegateThread(int64 current,
206                                                      int64 total);
207
208  URLFetcher* fetcher_;              // Corresponding fetcher object
209  GURL original_url_;                // The URL we were asked to fetch
210  GURL url_;                         // The URL we eventually wound up at
211  URLFetcher::RequestType request_type_;  // What type of request is this?
212  URLRequestStatus status_;          // Status of the request
213  URLFetcherDelegate* delegate_;     // Object to notify on completion
214  // Task runner for the creating thread. Used to interact with the delegate.
215  scoped_refptr<base::SingleThreadTaskRunner> delegate_task_runner_;
216  // Task runner for network operations.
217  scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_;
218  // Task runner for upload file access.
219  scoped_refptr<base::TaskRunner> upload_file_task_runner_;
220  scoped_ptr<URLRequest> request_;   // The actual request this wraps
221  int load_flags_;                   // Flags for the load operation
222  int response_code_;                // HTTP status code for the request
223  scoped_refptr<IOBuffer> buffer_;
224                                     // Read buffer
225  scoped_refptr<URLRequestContextGetter> request_context_getter_;
226                                     // Cookie/cache info for the request
227  GURL first_party_for_cookies_;     // The first party URL for the request
228  // The user data to add to each newly-created URLRequest.
229  const void* url_request_data_key_;
230  URLFetcher::CreateDataCallback url_request_create_data_callback_;
231  ResponseCookies cookies_;          // Response cookies
232  HttpRequestHeaders extra_request_headers_;
233  scoped_refptr<HttpResponseHeaders> response_headers_;
234  bool was_fetched_via_proxy_;
235  HostPortPair socket_address_;
236
237  bool upload_content_set_;          // SetUploadData has been called
238  std::string upload_content_;       // HTTP POST payload
239  base::FilePath upload_file_path_;  // Path to file containing POST payload
240  uint64 upload_range_offset_;       // Offset from the beginning of the file
241                                     // to be uploaded.
242  uint64 upload_range_length_;       // The length of the part of file to be
243                                     // uploaded.
244  std::string upload_content_type_;  // MIME type of POST payload
245  std::string referrer_;             // HTTP Referer header value and policy
246  URLRequest::ReferrerPolicy referrer_policy_;
247  bool is_chunked_upload_;           // True if using chunked transfer encoding
248
249  // Used to determine how long to wait before making a request or doing a
250  // retry.
251  //
252  // Both of them can only be accessed on the IO thread.
253  //
254  // To determine the proper backoff timing, throttler entries for
255  // both |original_URL| and |url| are needed. For example, consider
256  // the case that URL A redirects to URL B, for which the server
257  // returns a 500 response. In this case, the exponential back-off
258  // release time of URL A won't increase. If only the backoff
259  // constraints for URL A are considered, too many requests for URL A
260  // may be sent in a short period of time.
261  //
262  // Both of these will be NULL if
263  // URLRequestContext::throttler_manager() is NULL.
264  scoped_refptr<URLRequestThrottlerEntryInterface>
265      original_url_throttler_entry_;
266  scoped_refptr<URLRequestThrottlerEntryInterface> url_throttler_entry_;
267
268  // True if the URLFetcher has been cancelled.
269  bool was_cancelled_;
270
271  // Writer object to write response to the destination like file and string.
272  scoped_ptr<URLFetcherResponseWriter> response_writer_;
273
274  // By default any server-initiated redirects are automatically followed. If
275  // this flag is set to true, however, a redirect will halt the fetch and call
276  // back to to the delegate immediately.
277  bool stop_on_redirect_;
278  // True when we're actually stopped due to a redirect halted by the above. We
279  // use this to ensure that |url_| is set to the redirect destination rather
280  // than the originally-fetched URL.
281  bool stopped_on_redirect_;
282
283  // If |automatically_retry_on_5xx_| is false, 5xx responses will be
284  // propagated to the observer, if it is true URLFetcher will automatically
285  // re-execute the request, after the back-off delay has expired.
286  // true by default.
287  bool automatically_retry_on_5xx_;
288  // |num_retries_on_5xx_| indicates how many times we've failed to successfully
289  // fetch this URL due to 5xx responses. Once this value exceeds the maximum
290  // number of retries specified by the owner URLFetcher instance,
291  // we'll give up.
292  int num_retries_on_5xx_;
293  // Maximum retries allowed when 5xx responses are received.
294  int max_retries_on_5xx_;
295  // Back-off time delay. 0 by default.
296  base::TimeDelta backoff_delay_;
297
298  // The number of retries that have been attempted due to ERR_NETWORK_CHANGED.
299  int num_retries_on_network_changes_;
300  // Maximum retries allowed when the request fails with ERR_NETWORK_CHANGED.
301  // 0 by default.
302  int max_retries_on_network_changes_;
303
304  // Timer to poll the progress of uploading for POST and PUT requests.
305  // When crbug.com/119629 is fixed, scoped_ptr is not necessary here.
306  scoped_ptr<base::RepeatingTimer<URLFetcherCore> >
307      upload_progress_checker_timer_;
308  // Number of bytes sent so far.
309  int64 current_upload_bytes_;
310  // Number of bytes received so far.
311  int64 current_response_bytes_;
312  // Total expected bytes to receive (-1 if it cannot be determined).
313  int64 total_response_bytes_;
314
315  // TODO(willchan): Get rid of this after debugging crbug.com/90971.
316  base::debug::StackTrace stack_trace_;
317
318  static base::LazyInstance<Registry> g_registry;
319
320  DISALLOW_COPY_AND_ASSIGN(URLFetcherCore);
321};
322
323}  // namespace net
324
325#endif  // NET_URL_REQUEST_URL_FETCHER_CORE_H_
326