url_fetcher.cc revision 72a454cd3513ac24fbdd0e0cb9ad70b86a99b801 
1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/common/net/url_fetcher.h"
6
7#include <set>
8
9#include "base/compiler_specific.h"
10#include "base/lazy_instance.h"
11#include "base/message_loop_proxy.h"
12#include "base/scoped_ptr.h"
13#include "base/stl_util-inl.h"
14#include "base/string_util.h"
15#include "base/threading/thread.h"
16#include "chrome/common/net/url_request_context_getter.h"
17#include "googleurl/src/gurl.h"
18#include "net/base/load_flags.h"
19#include "net/base/io_buffer.h"
20#include "net/base/net_errors.h"
21#include "net/http/http_request_headers.h"
22#include "net/http/http_response_headers.h"
23#include "net/url_request/url_request.h"
24#include "net/url_request/url_request_context.h"
25#include "net/url_request/url_request_throttler_manager.h"
26
27#ifdef ANDROID
28#include "android/autofill/url_fetcher_proxy.h"
29#endif
30
31static const int kBufferSize = 4096;
32
33bool URLFetcher::g_interception_enabled = false;
34
35class URLFetcher::Core
36    : public base::RefCountedThreadSafe<URLFetcher::Core>,
37      public net::URLRequest::Delegate {
38 public:
39  // For POST requests, set |content_type| to the MIME type of the content
40  // and set |content| to the data to upload.  |flags| are flags to apply to
41  // the load operation--these should be one or more of the LOAD_* flags
42  // defined in url_request.h.
43  Core(URLFetcher* fetcher,
44       const GURL& original_url,
45       RequestType request_type,
46       URLFetcher::Delegate* d);
47
48  // Starts the load.  It's important that this not happen in the constructor
49  // because it causes the IO thread to begin AddRef()ing and Release()ing
50  // us.  If our caller hasn't had time to fully construct us and take a
51  // reference, the IO thread could interrupt things, run a task, Release()
52  // us, and destroy us, leaving the caller with an already-destroyed object
53  // when construction finishes.
54  void Start();
55
56  // Stops any in-progress load and ensures no callback will happen.  It is
57  // safe to call this multiple times.
58  void Stop();
59
60  // Reports that the received content was malformed.
61  void ReceivedContentWasMalformed();
62
63  // Overridden from net::URLRequest::Delegate:
64  virtual void OnResponseStarted(net::URLRequest* request);
65  virtual void OnReadCompleted(net::URLRequest* request, int bytes_read);
66
67  URLFetcher::Delegate* delegate() const { return delegate_; }
68
69  static void CancelAll();
70
71 private:
72  friend class base::RefCountedThreadSafe<URLFetcher::Core>;
73
74  class Registry {
75   public:
76    Registry();
77    ~Registry();
78
79    void AddURLFetcherCore(Core* core);
80    void RemoveURLFetcherCore(Core* core);
81
82    void CancelAll();
83
84   private:
85    std::set<Core*> fetchers_;
86
87    DISALLOW_COPY_AND_ASSIGN(Registry);
88  };
89
90  ~Core();
91
92  // Wrapper functions that allow us to ensure actions happen on the right
93  // thread.
94  void StartURLRequest();
95  void StartURLRequestWhenAppropriate();
96  void CancelURLRequest();
97  void OnCompletedURLRequest(const net::URLRequestStatus& status);
98  void NotifyMalformedContent();
99
100  // Deletes the request, removes it from the registry, and removes the
101  // destruction observer.
102  void ReleaseRequest();
103
104  // Returns the max value of exponential back-off release time for
105  // |original_url_| and |url_|.
106  base::TimeTicks GetBackoffReleaseTime();
107
108  void CompleteAddingUploadDataChunk(const std::string& data);
109
110  // Adds a block of data to be uploaded in a POST body. This can only be called
111  // after Start().
112  void AppendChunkToUpload(const std::string& data);
113
114  URLFetcher* fetcher_;              // Corresponding fetcher object
115  GURL original_url_;                // The URL we were asked to fetch
116  GURL url_;                         // The URL we eventually wound up at
117  RequestType request_type_;         // What type of request is this?
118  URLFetcher::Delegate* delegate_;   // Object to notify on completion
119  scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_;
120                                     // Message loop proxy of the creating
121                                     // thread.
122  scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
123                                     // The message loop proxy for the thread
124                                     // on which the request IO happens.
125  scoped_ptr<net::URLRequest> request_;   // The actual request this wraps
126  int load_flags_;                   // Flags for the load operation
127  int response_code_;                // HTTP status code for the request
128  std::string data_;                 // Results of the request
129  scoped_refptr<net::IOBuffer> buffer_;
130                                     // Read buffer
131  scoped_refptr<URLRequestContextGetter> request_context_getter_;
132                                     // Cookie/cache info for the request
133  ResponseCookies cookies_;          // Response cookies
134  net::HttpRequestHeaders extra_request_headers_;
135  scoped_refptr<net::HttpResponseHeaders> response_headers_;
136
137  std::string upload_content_;       // HTTP POST payload
138  std::string upload_content_type_;  // MIME type of POST payload
139  std::string referrer_;             // HTTP Referer header value
140  bool is_chunked_upload_;           // True if using chunked transfer encoding
141
142  // Used to determine how long to wait before making a request or doing a
143  // retry.
144  // Both of them can only be accessed on the IO thread.
145  // We need not only the throttler entry for |original_URL|, but also the one
146  // for |url|. For example, consider the case that URL A redirects to URL B,
147  // for which the server returns a 500 response. In this case, the exponential
148  // back-off release time of URL A won't increase. If we retry without
149  // considering the back-off constraint of URL B, we may send out too many
150  // requests for URL A in a short period of time.
151  scoped_refptr<net::URLRequestThrottlerEntryInterface>
152      original_url_throttler_entry_;
153  scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_;
154
155  // |num_retries_| indicates how many times we've failed to successfully
156  // fetch this URL.  Once this value exceeds the maximum number of retries
157  // specified by the owner URLFetcher instance, we'll give up.
158  int num_retries_;
159
160  // True if the URLFetcher has been cancelled.
161  bool was_cancelled_;
162
163  // Since GetBackoffReleaseTime() can only be called on the IO thread, we cache
164  // its value to be used by OnCompletedURLRequest on the creating thread.
165  base::TimeTicks backoff_release_time_;
166
167  static base::LazyInstance<Registry> g_registry;
168
169  friend class URLFetcher;
170  DISALLOW_COPY_AND_ASSIGN(Core);
171};
172
173URLFetcher::Core::Registry::Registry() {}
174URLFetcher::Core::Registry::~Registry() {}
175
176void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) {
177  DCHECK(!ContainsKey(fetchers_, core));
178  fetchers_.insert(core);
179}
180
181void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) {
182  DCHECK(ContainsKey(fetchers_, core));
183  fetchers_.erase(core);
184}
185
186void URLFetcher::Core::Registry::CancelAll() {
187  std::set<Core*> fetchers;
188  fetchers.swap(fetchers_);
189
190  for (std::set<Core*>::iterator it = fetchers.begin();
191       it != fetchers.end(); ++it)
192    (*it)->CancelURLRequest();
193}
194
195// static
196base::LazyInstance<URLFetcher::Core::Registry>
197    URLFetcher::Core::g_registry(base::LINKER_INITIALIZED);
198
199// static
200URLFetcher::Factory* URLFetcher::factory_ = NULL;
201
202URLFetcher::URLFetcher(const GURL& url,
203                       RequestType request_type,
204                       Delegate* d)
205    : ALLOW_THIS_IN_INITIALIZER_LIST(
206      core_(new Core(this, url, request_type, d))),
207      automatically_retry_on_5xx_(true),
208      max_retries_(0) {
209}
210
211URLFetcher::~URLFetcher() {
212  core_->Stop();
213}
214
215// static
216URLFetcher* URLFetcher::Create(int id, const GURL& url,
217                               RequestType request_type, Delegate* d) {
218#ifdef ANDROID
219  // TODO: Upstream.
220  return new URLFetcherProxy(url, request_type, d);
221#else
222  return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) :
223                    new URLFetcher(url, request_type, d);
224#endif
225}
226
227URLFetcher::Core::Core(URLFetcher* fetcher,
228                       const GURL& original_url,
229                       RequestType request_type,
230                       URLFetcher::Delegate* d)
231    : fetcher_(fetcher),
232      original_url_(original_url),
233      request_type_(request_type),
234      delegate_(d),
235      delegate_loop_proxy_(base::MessageLoopProxy::CreateForCurrentThread()),
236      request_(NULL),
237      load_flags_(net::LOAD_NORMAL),
238      response_code_(-1),
239      buffer_(new net::IOBuffer(kBufferSize)),
240      is_chunked_upload_(false),
241      num_retries_(0),
242      was_cancelled_(false) {
243}
244
245URLFetcher::Core::~Core() {
246  // |request_| should be NULL.  If not, it's unsafe to delete it here since we
247  // may not be on the IO thread.
248  DCHECK(!request_.get());
249}
250
251void URLFetcher::Core::Start() {
252  DCHECK(delegate_loop_proxy_);
253  CHECK(request_context_getter_) << "We need an URLRequestContext!";
254  io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy();
255  CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy";
256
257  io_message_loop_proxy_->PostTask(
258      FROM_HERE,
259      NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
260}
261
262void URLFetcher::Core::Stop() {
263  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
264  delegate_ = NULL;
265  fetcher_ = NULL;
266  if (io_message_loop_proxy_.get()) {
267    io_message_loop_proxy_->PostTask(
268        FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest));
269  }
270}
271
272void URLFetcher::Core::ReceivedContentWasMalformed() {
273  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
274  if (io_message_loop_proxy_.get()) {
275    io_message_loop_proxy_->PostTask(
276        FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent));
277  }
278}
279
280void URLFetcher::Core::CancelAll() {
281  g_registry.Get().CancelAll();
282}
283
284void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) {
285  DCHECK_EQ(request, request_.get());
286  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
287  if (request_->status().is_success()) {
288    response_code_ = request_->GetResponseCode();
289    response_headers_ = request_->response_headers();
290  }
291
292  int bytes_read = 0;
293  // Some servers may treat HEAD requests as GET requests.  To free up the
294  // network connection as soon as possible, signal that the request has
295  // completed immediately, without trying to read any data back (all we care
296  // about is the response code and headers, which we already have).
297  if (request_->status().is_success() && (request_type_ != HEAD))
298    request_->Read(buffer_, kBufferSize, &bytes_read);
299  OnReadCompleted(request_.get(), bytes_read);
300}
301
302void URLFetcher::Core::CompleteAddingUploadDataChunk(
303    const std::string& content) {
304  DCHECK(is_chunked_upload_);
305  DCHECK(request_.get());
306  if (content.length()) {
307    request_->AppendChunkToUpload(content.data(),
308                                  static_cast<int>(content.length()));
309  } else {
310    request_->MarkEndOfChunks();
311  }
312}
313
314void URLFetcher::Core::AppendChunkToUpload(const std::string& content) {
315  DCHECK(delegate_loop_proxy_);
316  CHECK(io_message_loop_proxy_.get());
317  io_message_loop_proxy_->PostTask(
318      FROM_HERE,
319      NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content));
320}
321
322void URLFetcher::Core::OnReadCompleted(net::URLRequest* request,
323                                       int bytes_read) {
324  DCHECK(request == request_);
325  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
326
327  url_ = request->url();
328  url_throttler_entry_ =
329      net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_);
330
331  do {
332    if (!request_->status().is_success() || bytes_read <= 0)
333      break;
334    data_.append(buffer_->data(), bytes_read);
335  } while (request_->Read(buffer_, kBufferSize, &bytes_read));
336
337  if (request_->status().is_success())
338    request_->GetResponseCookies(&cookies_);
339
340  // See comments re: HEAD requests in OnResponseStarted().
341  if (!request_->status().is_io_pending() || (request_type_ == HEAD)) {
342    backoff_release_time_ = GetBackoffReleaseTime();
343
344    bool posted = delegate_loop_proxy_->PostTask(
345        FROM_HERE,
346        NewRunnableMethod(this,
347                          &Core::OnCompletedURLRequest,
348                          request_->status()));
349    // If the delegate message loop does not exist any more, then the delegate
350    // should be gone too.
351    DCHECK(posted || !delegate_);
352    ReleaseRequest();
353  }
354}
355
356void URLFetcher::Core::StartURLRequest() {
357  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
358
359  if (was_cancelled_) {
360    // Since StartURLRequest() is posted as a *delayed* task, it may
361    // run after the URLFetcher was already stopped.
362    return;
363  }
364
365  CHECK(request_context_getter_);
366  DCHECK(!request_.get());
367
368  g_registry.Get().AddURLFetcherCore(this);
369  request_.reset(new net::URLRequest(original_url_, this));
370  int flags = request_->load_flags() | load_flags_;
371  if (!g_interception_enabled) {
372    flags = flags | net::LOAD_DISABLE_INTERCEPT;
373  }
374  if (is_chunked_upload_)
375    request_->EnableChunkedUpload();
376  request_->set_load_flags(flags);
377  request_->set_context(request_context_getter_->GetURLRequestContext());
378  request_->set_referrer(referrer_);
379
380  switch (request_type_) {
381    case GET:
382      break;
383
384    case POST:
385      DCHECK(!upload_content_.empty() || is_chunked_upload_);
386      DCHECK(!upload_content_type_.empty());
387
388      request_->set_method("POST");
389      extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType,
390                                       upload_content_type_);
391      if (!upload_content_.empty()) {
392        request_->AppendBytesToUpload(
393            upload_content_.data(), static_cast<int>(upload_content_.length()));
394      }
395      break;
396
397    case HEAD:
398      request_->set_method("HEAD");
399      break;
400
401    default:
402      NOTREACHED();
403  }
404
405  if (!extra_request_headers_.IsEmpty())
406    request_->SetExtraRequestHeaders(extra_request_headers_);
407
408  // There might be data left over from a previous request attempt.
409  data_.clear();
410
411  request_->Start();
412}
413
414void URLFetcher::Core::StartURLRequestWhenAppropriate() {
415  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
416
417  if (was_cancelled_)
418    return;
419
420  if (original_url_throttler_entry_ == NULL) {
421    original_url_throttler_entry_ =
422        net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(
423            original_url_);
424  }
425
426  int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
427      GetBackoffReleaseTime());
428  if (delay == 0) {
429    StartURLRequest();
430  } else {
431    MessageLoop::current()->PostDelayedTask(
432        FROM_HERE,
433        NewRunnableMethod(this, &Core::StartURLRequest),
434        delay);
435  }
436}
437
438void URLFetcher::Core::CancelURLRequest() {
439  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
440
441  if (request_.get()) {
442    request_->Cancel();
443    ReleaseRequest();
444  }
445  // Release the reference to the request context. There could be multiple
446  // references to URLFetcher::Core at this point so it may take a while to
447  // delete the object, but we cannot delay the destruction of the request
448  // context.
449  request_context_getter_ = NULL;
450  was_cancelled_ = true;
451}
452
453void URLFetcher::Core::OnCompletedURLRequest(
454    const net::URLRequestStatus& status) {
455  DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
456
457  // Checks the response from server.
458  if (response_code_ >= 500 ||
459      status.os_error() == net::ERR_TEMPORARILY_THROTTLED) {
460    // When encountering a server error, we will send the request again
461    // after backoff time.
462    ++num_retries_;
463    // Restarts the request if we still need to notify the delegate.
464    if (delegate_) {
465      fetcher_->backoff_delay_ = backoff_release_time_ - base::TimeTicks::Now();
466      if (fetcher_->backoff_delay_ < base::TimeDelta())
467        fetcher_->backoff_delay_ = base::TimeDelta();
468
469      if (fetcher_->automatically_retry_on_5xx_ &&
470          num_retries_ <= fetcher_->max_retries()) {
471        io_message_loop_proxy_->PostTask(
472            FROM_HERE,
473            NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
474      } else {
475        delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
476                                      cookies_, data_);
477      }
478    }
479  } else {
480    if (delegate_) {
481      fetcher_->backoff_delay_ = base::TimeDelta();
482      delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
483                                    cookies_, data_);
484    }
485  }
486}
487
488void URLFetcher::Core::NotifyMalformedContent() {
489  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
490  if (url_throttler_entry_ != NULL)
491    url_throttler_entry_->ReceivedContentWasMalformed();
492}
493
494void URLFetcher::Core::ReleaseRequest() {
495  request_.reset();
496  g_registry.Get().RemoveURLFetcherCore(this);
497}
498
499base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() {
500  DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
501  DCHECK(original_url_throttler_entry_ != NULL);
502
503  base::TimeTicks original_url_backoff =
504      original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
505  base::TimeTicks destination_url_backoff;
506  if (url_throttler_entry_ != NULL &&
507      original_url_throttler_entry_ != url_throttler_entry_) {
508    destination_url_backoff =
509        url_throttler_entry_->GetExponentialBackoffReleaseTime();
510  }
511
512  return original_url_backoff > destination_url_backoff ?
513      original_url_backoff : destination_url_backoff;
514}
515
516void URLFetcher::set_upload_data(const std::string& upload_content_type,
517                                 const std::string& upload_content) {
518  DCHECK(!core_->is_chunked_upload_);
519  core_->upload_content_type_ = upload_content_type;
520  core_->upload_content_ = upload_content;
521}
522
523void URLFetcher::set_chunked_upload(const std::string& content_type) {
524  DCHECK(core_->is_chunked_upload_ ||
525         (core_->upload_content_type_.empty() &&
526          core_->upload_content_.empty()));
527  core_->upload_content_type_ = content_type;
528  core_->upload_content_.clear();
529  core_->is_chunked_upload_ = true;
530}
531
532void URLFetcher::AppendChunkToUpload(const std::string& data) {
533  DCHECK(data.length());
534  core_->AppendChunkToUpload(data);
535}
536
537void URLFetcher::MarkEndOfChunks() {
538  core_->AppendChunkToUpload(std::string());
539}
540
541const std::string& URLFetcher::upload_data() const {
542  return core_->upload_content_;
543}
544
545void URLFetcher::set_referrer(const std::string& referrer) {
546  core_->referrer_ = referrer;
547}
548
549void URLFetcher::set_load_flags(int load_flags) {
550  core_->load_flags_ = load_flags;
551}
552
553int URLFetcher::load_flags() const {
554  return core_->load_flags_;
555}
556
557void URLFetcher::set_extra_request_headers(
558    const std::string& extra_request_headers) {
559  core_->extra_request_headers_.Clear();
560  core_->extra_request_headers_.AddHeadersFromString(extra_request_headers);
561}
562
563void URLFetcher::set_request_context(
564    URLRequestContextGetter* request_context_getter) {
565  core_->request_context_getter_ = request_context_getter;
566}
567
568#ifdef ANDROID
569URLRequestContextGetter* URLFetcher::request_context() {
570    return core_->request_context_getter_;
571}
572#endif
573
574void URLFetcher::set_automatically_retry_on_5xx(bool retry) {
575  automatically_retry_on_5xx_ = retry;
576}
577
578net::HttpResponseHeaders* URLFetcher::response_headers() const {
579  return core_->response_headers_;
580}
581
582void URLFetcher::Start() {
583  core_->Start();
584}
585
586const GURL& URLFetcher::url() const {
587  return core_->url_;
588}
589
590void URLFetcher::ReceivedContentWasMalformed() {
591  core_->ReceivedContentWasMalformed();
592}
593
594// static
595void URLFetcher::CancelAll() {
596  Core::CancelAll();
597}
598
599URLFetcher::Delegate* URLFetcher::delegate() const {
600  return core_->delegate();
601}
602