1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/common/net/url_fetcher.h"
6
7#include <set>
8
9#include "base/compiler_specific.h"
10#include "base/lazy_instance.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/message_loop_proxy.h"
13#include "base/stl_util-inl.h"
14#include "base/string_util.h"
15#include "base/threading/thread.h"
16#include "googleurl/src/gurl.h"
17#include "net/base/load_flags.h"
18#include "net/base/io_buffer.h"
19#include "net/base/net_errors.h"
20#include "net/http/http_request_headers.h"
21#include "net/http/http_response_headers.h"
22#include "net/url_request/url_request.h"
23#include "net/url_request/url_request_context.h"
24#include "net/url_request/url_request_context_getter.h"
25#include "net/url_request/url_request_throttler_manager.h"
26
27#ifdef ANDROID
28#include "android/autofill/url_fetcher_proxy.h"
29#endif
30
31static const int kBufferSize = 4096;
32
33class URLFetcher::Core
34    : public base::RefCountedThreadSafe<URLFetcher::Core>,
35      public net::URLRequest::Delegate {
36 public:
37  // For POST requests, set |content_type| to the MIME type of the content
38  // and set |content| to the data to upload.  |flags| are flags to apply to
39  // the load operation--these should be one or more of the LOAD_* flags
40  // defined in net/base/load_flags.h.
41  Core(URLFetcher* fetcher,
42       const GURL& original_url,
43       RequestType request_type,
44       URLFetcher::Delegate* d);
45
46  // Starts the load.  It's important that this not happen in the constructor
47  // because it causes the IO thread to begin AddRef()ing and Release()ing
48  // us.  If our caller hasn't had time to fully construct us and take a
49  // reference, the IO thread could interrupt things, run a task, Release()
50  // us, and destroy us, leaving the caller with an already-destroyed object
51  // when construction finishes.
52  void Start();
53
54  // Stops any in-progress load and ensures no callback will happen.  It is
55  // safe to call this multiple times.
56  void Stop();
57
58  // Reports that the received content was malformed.
59  void ReceivedContentWasMalformed();
60
61  // Overridden from net::URLRequest::Delegate:
62  virtual void OnResponseStarted(net::URLRequest* request);
63  virtual void OnReadCompleted(net::URLRequest* request, int bytes_read);
64
65  URLFetcher::Delegate* delegate() const { return delegate_; }
66
67  static void CancelAll();
68
69 private:
70  friend class base::RefCountedThreadSafe<URLFetcher::Core>;
71
72  class Registry {
73   public:
74    Registry();
75    ~Registry();
76
77    void AddURLFetcherCore(Core* core);
78    void RemoveURLFetcherCore(Core* core);
79
80    void CancelAll();
81
82    int size() const {
83      return fetchers_.size();
84    }
85
86   private:
87    std::set<Core*> fetchers_;
88
89    DISALLOW_COPY_AND_ASSIGN(Registry);
90  };
91
92  virtual ~Core();
93
94  // Wrapper functions that allow us to ensure actions happen on the right
95  // thread.
96  void StartURLRequest();
97  void StartURLRequestWhenAppropriate();
98  void CancelURLRequest();
99  void OnCompletedURLRequest(const net::URLRequestStatus& status);
100  void NotifyMalformedContent();
101
102  // Deletes the request, removes it from the registry, and removes the
103  // destruction observer.
104  void ReleaseRequest();
105
106  // Returns the max value of exponential back-off release time for
107  // |original_url_| and |url_|.
108  base::TimeTicks GetBackoffReleaseTime();
109
110  void CompleteAddingUploadDataChunk(const std::string& data,
111                                     bool is_last_chunk);
112
113  // Adds a block of data to be uploaded in a POST body. This can only be called
114  // after Start().
115  void AppendChunkToUpload(const std::string& data, bool is_last_chunk);
116
117  URLFetcher* fetcher_;              // Corresponding fetcher object
118  GURL original_url_;                // The URL we were asked to fetch
119  GURL url_;                         // The URL we eventually wound up at
120  RequestType request_type_;         // What type of request is this?
121  URLFetcher::Delegate* delegate_;   // Object to notify on completion
122  scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_;
123                                     // Message loop proxy of the creating
124                                     // thread.
125  scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
126                                     // The message loop proxy for the thread
127                                     // on which the request IO happens.
128  scoped_ptr<net::URLRequest> request_;   // The actual request this wraps
129  int load_flags_;                   // Flags for the load operation
130  int response_code_;                // HTTP status code for the request
131  std::string data_;                 // Results of the request
132  scoped_refptr<net::IOBuffer> buffer_;
133                                     // Read buffer
134  scoped_refptr<net::URLRequestContextGetter> request_context_getter_;
135                                     // Cookie/cache info for the request
136  ResponseCookies cookies_;          // Response cookies
137  net::HttpRequestHeaders extra_request_headers_;
138  scoped_refptr<net::HttpResponseHeaders> response_headers_;
139
140  std::string upload_content_;       // HTTP POST payload
141  std::string upload_content_type_;  // MIME type of POST payload
142  std::string referrer_;             // HTTP Referer header value
143  bool is_chunked_upload_;           // True if using chunked transfer encoding
144
145  // Used to determine how long to wait before making a request or doing a
146  // retry.
147  // Both of them can only be accessed on the IO thread.
148  // We need not only the throttler entry for |original_URL|, but also the one
149  // for |url|. For example, consider the case that URL A redirects to URL B,
150  // for which the server returns a 500 response. In this case, the exponential
151  // back-off release time of URL A won't increase. If we retry without
152  // considering the back-off constraint of URL B, we may send out too many
153  // requests for URL A in a short period of time.
154  scoped_refptr<net::URLRequestThrottlerEntryInterface>
155      original_url_throttler_entry_;
156  scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_;
157
158  // |num_retries_| indicates how many times we've failed to successfully
159  // fetch this URL.  Once this value exceeds the maximum number of retries
160  // specified by the owner URLFetcher instance, we'll give up.
161  int num_retries_;
162
163  // True if the URLFetcher has been cancelled.
164  bool was_cancelled_;
165
166  // Since GetBackoffReleaseTime() can only be called on the IO thread, we cache
167  // its value to be used by OnCompletedURLRequest on the creating thread.
168  base::TimeTicks backoff_release_time_;
169
170  static base::LazyInstance<Registry> g_registry;
171
172  friend class URLFetcher;
173  DISALLOW_COPY_AND_ASSIGN(Core);
174};
175
176URLFetcher::Core::Registry::Registry() {}
177URLFetcher::Core::Registry::~Registry() {}
178
179void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) {
180  DCHECK(!ContainsKey(fetchers_, core));
181  fetchers_.insert(core);
182}
183
184void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) {
185  DCHECK(ContainsKey(fetchers_, core));
186  fetchers_.erase(core);
187}
188
189void URLFetcher::Core::Registry::CancelAll() {
190  while (!fetchers_.empty())
191    (*fetchers_.begin())->CancelURLRequest();
192}
193
194// static
195base::LazyInstance<URLFetcher::Core::Registry>
196    URLFetcher::Core::g_registry(base::LINKER_INITIALIZED);
197
198// static
199URLFetcher::Factory* URLFetcher::factory_ = NULL;
200
201// static
202bool URLFetcher::g_interception_enabled = false;
203
204URLFetcher::URLFetcher(const GURL& url,
205                       RequestType request_type,
206                       Delegate* d)
207    : ALLOW_THIS_IN_INITIALIZER_LIST(
208      core_(new Core(this, url, request_type, d))),
209      automatically_retry_on_5xx_(true),
210      max_retries_(0) {
211}
212
213URLFetcher::~URLFetcher() {
214  core_->Stop();
215}
216
217// static
218URLFetcher* URLFetcher::Create(int id, const GURL& url,
219                               RequestType request_type, Delegate* d) {
220#ifdef ANDROID
221  // TODO: Upstream.
222  return new URLFetcherProxy(url, request_type, d);
223#else
224  return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) :
225                    new URLFetcher(url, request_type, d);
226#endif
227}
228
229URLFetcher::Core::Core(URLFetcher* fetcher,
230                       const GURL& original_url,
231                       RequestType request_type,
232                       URLFetcher::Delegate* d)
233    : fetcher_(fetcher),
234      original_url_(original_url),
235      request_type_(request_type),
236      delegate_(d),
237      delegate_loop_proxy_(base::MessageLoopProxy::CreateForCurrentThread()),
238      request_(NULL),
239      load_flags_(net::LOAD_NORMAL),
240      response_code_(-1),
241      buffer_(new net::IOBuffer(kBufferSize)),
242      is_chunked_upload_(false),
243      num_retries_(0),
244      was_cancelled_(false) {
245}
246
247URLFetcher::Core::~Core() {
248  // |request_| should be NULL.  If not, it's unsafe to delete it here since we
249  // may not be on the IO thread.
250  DCHECK(!request_.get());
251}
252
253void URLFetcher::Core::Start() {
254  DCHECK(delegate_loop_proxy_);
255  CHECK(request_context_getter_) << "We need an URLRequestContext!";
256  io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy();
257  CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy";
258
259  io_message_loop_proxy_->PostTask(
260      FROM_HERE,
261      NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
262}
263
264void URLFetcher::Core::Stop() {
265  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
266  delegate_ = NULL;
267  fetcher_ = NULL;
268  if (io_message_loop_proxy_.get()) {
269    io_message_loop_proxy_->PostTask(
270        FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest));
271  }
272}
273
274void URLFetcher::Core::ReceivedContentWasMalformed() {
275  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
276  if (io_message_loop_proxy_.get()) {
277    io_message_loop_proxy_->PostTask(
278        FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent));
279  }
280}
281
282void URLFetcher::Core::CancelAll() {
283  g_registry.Get().CancelAll();
284}
285
286void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) {
287  DCHECK_EQ(request, request_.get());
288  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
289  if (request_->status().is_success()) {
290    response_code_ = request_->GetResponseCode();
291    response_headers_ = request_->response_headers();
292  }
293
294  int bytes_read = 0;
295  // Some servers may treat HEAD requests as GET requests.  To free up the
296  // network connection as soon as possible, signal that the request has
297  // completed immediately, without trying to read any data back (all we care
298  // about is the response code and headers, which we already have).
299  if (request_->status().is_success() && (request_type_ != HEAD))
300    request_->Read(buffer_, kBufferSize, &bytes_read);
301  OnReadCompleted(request_.get(), bytes_read);
302}
303
304void URLFetcher::Core::CompleteAddingUploadDataChunk(
305    const std::string& content, bool is_last_chunk) {
306  DCHECK(is_chunked_upload_);
307  DCHECK(request_.get());
308  DCHECK(!content.empty());
309  request_->AppendChunkToUpload(content.data(),
310                                static_cast<int>(content.length()),
311                                is_last_chunk);
312}
313
314void URLFetcher::Core::AppendChunkToUpload(const std::string& content,
315                                           bool is_last_chunk) {
316  DCHECK(delegate_loop_proxy_);
317  CHECK(io_message_loop_proxy_.get());
318  io_message_loop_proxy_->PostTask(
319      FROM_HERE,
320      NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content,
321                        is_last_chunk));
322}
323
324void URLFetcher::Core::OnReadCompleted(net::URLRequest* request,
325                                       int bytes_read) {
326  DCHECK(request == request_);
327  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
328
329  url_ = request->url();
330  url_throttler_entry_ =
331      net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_);
332
333  do {
334    if (!request_->status().is_success() || bytes_read <= 0)
335      break;
336    data_.append(buffer_->data(), bytes_read);
337  } while (request_->Read(buffer_, kBufferSize, &bytes_read));
338
339  if (request_->status().is_success())
340    request_->GetResponseCookies(&cookies_);
341
342  // See comments re: HEAD requests in OnResponseStarted().
343  if (!request_->status().is_io_pending() || (request_type_ == HEAD)) {
344    backoff_release_time_ = GetBackoffReleaseTime();
345
346    bool posted = delegate_loop_proxy_->PostTask(
347        FROM_HERE,
348        NewRunnableMethod(this,
349                          &Core::OnCompletedURLRequest,
350                          request_->status()));
351    // If the delegate message loop does not exist any more, then the delegate
352    // should be gone too.
353    DCHECK(posted || !delegate_);
354    ReleaseRequest();
355  }
356}
357
358void URLFetcher::Core::StartURLRequest() {
359  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
360
361  if (was_cancelled_) {
362    // Since StartURLRequest() is posted as a *delayed* task, it may
363    // run after the URLFetcher was already stopped.
364    return;
365  }
366
367  CHECK(request_context_getter_);
368  DCHECK(!request_.get());
369
370  g_registry.Get().AddURLFetcherCore(this);
371  request_.reset(new net::URLRequest(original_url_, this));
372  int flags = request_->load_flags() | load_flags_;
373  if (!g_interception_enabled) {
374    flags = flags | net::LOAD_DISABLE_INTERCEPT;
375  }
376  if (is_chunked_upload_)
377    request_->EnableChunkedUpload();
378  request_->set_load_flags(flags);
379  request_->set_context(request_context_getter_->GetURLRequestContext());
380  request_->set_referrer(referrer_);
381
382  switch (request_type_) {
383    case GET:
384      break;
385
386    case POST:
387      DCHECK(!upload_content_.empty() || is_chunked_upload_);
388      DCHECK(!upload_content_type_.empty());
389
390      request_->set_method("POST");
391      extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType,
392                                       upload_content_type_);
393      if (!upload_content_.empty()) {
394        request_->AppendBytesToUpload(
395            upload_content_.data(), static_cast<int>(upload_content_.length()));
396      }
397      break;
398
399    case HEAD:
400      request_->set_method("HEAD");
401      break;
402
403    default:
404      NOTREACHED();
405  }
406
407  if (!extra_request_headers_.IsEmpty())
408    request_->SetExtraRequestHeaders(extra_request_headers_);
409
410  // There might be data left over from a previous request attempt.
411  data_.clear();
412
413  request_->Start();
414}
415
416void URLFetcher::Core::StartURLRequestWhenAppropriate() {
417  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
418
419  if (was_cancelled_)
420    return;
421
422  if (original_url_throttler_entry_ == NULL) {
423    original_url_throttler_entry_ =
424        net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(
425            original_url_);
426  }
427
428  int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
429      GetBackoffReleaseTime());
430  if (delay == 0) {
431    StartURLRequest();
432  } else {
433    MessageLoop::current()->PostDelayedTask(
434        FROM_HERE,
435        NewRunnableMethod(this, &Core::StartURLRequest),
436        delay);
437  }
438}
439
440void URLFetcher::Core::CancelURLRequest() {
441  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
442
443  if (request_.get()) {
444    request_->Cancel();
445    ReleaseRequest();
446  }
447  // Release the reference to the request context. There could be multiple
448  // references to URLFetcher::Core at this point so it may take a while to
449  // delete the object, but we cannot delay the destruction of the request
450  // context.
451  request_context_getter_ = NULL;
452  was_cancelled_ = true;
453}
454
455void URLFetcher::Core::OnCompletedURLRequest(
456    const net::URLRequestStatus& status) {
457  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
458
459  // Checks the response from server.
460  if (response_code_ >= 500 ||
461      status.os_error() == net::ERR_TEMPORARILY_THROTTLED) {
462    // When encountering a server error, we will send the request again
463    // after backoff time.
464    ++num_retries_;
465    // Restarts the request if we still need to notify the delegate.
466    if (delegate_) {
467      fetcher_->backoff_delay_ = backoff_release_time_ - base::TimeTicks::Now();
468      if (fetcher_->backoff_delay_ < base::TimeDelta())
469        fetcher_->backoff_delay_ = base::TimeDelta();
470
471      if (fetcher_->automatically_retry_on_5xx_ &&
472          num_retries_ <= fetcher_->max_retries()) {
473        io_message_loop_proxy_->PostTask(
474            FROM_HERE,
475            NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
476      } else {
477        delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
478                                      cookies_, data_);
479      }
480    }
481  } else {
482    if (delegate_) {
483      fetcher_->backoff_delay_ = base::TimeDelta();
484      delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
485                                    cookies_, data_);
486    }
487  }
488}
489
490void URLFetcher::Core::NotifyMalformedContent() {
491  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
492  if (url_throttler_entry_ != NULL)
493    url_throttler_entry_->ReceivedContentWasMalformed();
494}
495
496void URLFetcher::Core::ReleaseRequest() {
497  request_.reset();
498  g_registry.Get().RemoveURLFetcherCore(this);
499}
500
501base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() {
502  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
503  DCHECK(original_url_throttler_entry_ != NULL);
504
505  base::TimeTicks original_url_backoff =
506      original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
507  base::TimeTicks destination_url_backoff;
508  if (url_throttler_entry_ != NULL &&
509      original_url_throttler_entry_ != url_throttler_entry_) {
510    destination_url_backoff =
511        url_throttler_entry_->GetExponentialBackoffReleaseTime();
512  }
513
514  return original_url_backoff > destination_url_backoff ?
515      original_url_backoff : destination_url_backoff;
516}
517
518void URLFetcher::set_upload_data(const std::string& upload_content_type,
519                                 const std::string& upload_content) {
520  DCHECK(!core_->is_chunked_upload_);
521  core_->upload_content_type_ = upload_content_type;
522  core_->upload_content_ = upload_content;
523}
524
525void URLFetcher::set_chunked_upload(const std::string& content_type) {
526  DCHECK(core_->is_chunked_upload_ ||
527         (core_->upload_content_type_.empty() &&
528          core_->upload_content_.empty()));
529  core_->upload_content_type_ = content_type;
530  core_->upload_content_.clear();
531  core_->is_chunked_upload_ = true;
532}
533
534void URLFetcher::AppendChunkToUpload(const std::string& data,
535                                     bool is_last_chunk) {
536  DCHECK(data.length());
537  core_->AppendChunkToUpload(data, is_last_chunk);
538}
539
540const std::string& URLFetcher::upload_data() const {
541  return core_->upload_content_;
542}
543
544void URLFetcher::set_referrer(const std::string& referrer) {
545  core_->referrer_ = referrer;
546}
547
548void URLFetcher::set_load_flags(int load_flags) {
549  core_->load_flags_ = load_flags;
550}
551
552int URLFetcher::load_flags() const {
553  return core_->load_flags_;
554}
555
556void URLFetcher::set_extra_request_headers(
557    const std::string& extra_request_headers) {
558  core_->extra_request_headers_.Clear();
559  core_->extra_request_headers_.AddHeadersFromString(extra_request_headers);
560}
561
562void URLFetcher::set_request_context(
563    net::URLRequestContextGetter* request_context_getter) {
564  core_->request_context_getter_ = request_context_getter;
565}
566
567#ifdef ANDROID
568net::URLRequestContextGetter* URLFetcher::request_context() {
569    return core_->request_context_getter_;
570}
571#endif
572
573void URLFetcher::set_automatically_retry_on_5xx(bool retry) {
574  automatically_retry_on_5xx_ = retry;
575}
576
577net::HttpResponseHeaders* URLFetcher::response_headers() const {
578  return core_->response_headers_;
579}
580
581void URLFetcher::Start() {
582  core_->Start();
583}
584
585const GURL& URLFetcher::url() const {
586  return core_->url_;
587}
588
589void URLFetcher::ReceivedContentWasMalformed() {
590  core_->ReceivedContentWasMalformed();
591}
592
593// static
594void URLFetcher::CancelAll() {
595  Core::CancelAll();
596}
597
598// static
599int URLFetcher::GetNumFetcherCores() {
600  return Core::g_registry.Get().size();
601}
602
603URLFetcher::Delegate* URLFetcher::delegate() const {
604  return core_->delegate();
605}
606