url_fetcher_core.cc revision 6e8cce623b6e4fe0c9e4af605d675dd9d0338c38 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_fetcher_core.h"
6
7#include "base/bind.h"
8#include "base/logging.h"
9#include "base/metrics/histogram.h"
10#include "base/sequenced_task_runner.h"
11#include "base/single_thread_task_runner.h"
12#include "base/stl_util.h"
13#include "base/thread_task_runner_handle.h"
14#include "base/tracked_objects.h"
15#include "net/base/io_buffer.h"
16#include "net/base/load_flags.h"
17#include "net/base/net_errors.h"
18#include "net/base/request_priority.h"
19#include "net/base/upload_bytes_element_reader.h"
20#include "net/base/upload_data_stream.h"
21#include "net/base/upload_file_element_reader.h"
22#include "net/http/http_response_headers.h"
23#include "net/url_request/redirect_info.h"
24#include "net/url_request/url_fetcher_delegate.h"
25#include "net/url_request/url_fetcher_response_writer.h"
26#include "net/url_request/url_request_context.h"
27#include "net/url_request/url_request_context_getter.h"
28#include "net/url_request/url_request_throttler_manager.h"
29
30namespace {
31
32const int kBufferSize = 4096;
33const int kUploadProgressTimerInterval = 100;
34bool g_interception_enabled = false;
35bool g_ignore_certificate_requests = false;
36
37void EmptyCompletionCallback(int result) {}
38
39}  // namespace
40
41namespace net {
42
43// URLFetcherCore::Registry ---------------------------------------------------
44
45URLFetcherCore::Registry::Registry() {}
46URLFetcherCore::Registry::~Registry() {}
47
48void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
49  DCHECK(!ContainsKey(fetchers_, core));
50  fetchers_.insert(core);
51}
52
53void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
54  DCHECK(ContainsKey(fetchers_, core));
55  fetchers_.erase(core);
56}
57
58void URLFetcherCore::Registry::CancelAll() {
59  while (!fetchers_.empty())
60    (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
61}
62
63// URLFetcherCore -------------------------------------------------------------
64
65// static
66base::LazyInstance<URLFetcherCore::Registry>
67    URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
68
69URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
70                               const GURL& original_url,
71                               URLFetcher::RequestType request_type,
72                               URLFetcherDelegate* d)
73    : fetcher_(fetcher),
74      original_url_(original_url),
75      request_type_(request_type),
76      delegate_(d),
77      delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
78      load_flags_(LOAD_NORMAL),
79      response_code_(URLFetcher::RESPONSE_CODE_INVALID),
80      buffer_(new IOBuffer(kBufferSize)),
81      url_request_data_key_(NULL),
82      was_fetched_via_proxy_(false),
83      upload_content_set_(false),
84      upload_range_offset_(0),
85      upload_range_length_(0),
86      referrer_policy_(
87          URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
88      is_chunked_upload_(false),
89      was_cancelled_(false),
90      stop_on_redirect_(false),
91      stopped_on_redirect_(false),
92      automatically_retry_on_5xx_(true),
93      num_retries_on_5xx_(0),
94      max_retries_on_5xx_(0),
95      num_retries_on_network_changes_(0),
96      max_retries_on_network_changes_(0),
97      current_upload_bytes_(-1),
98      current_response_bytes_(0),
99      total_response_bytes_(-1) {
100  CHECK(original_url_.is_valid());
101}
102
103void URLFetcherCore::Start() {
104  DCHECK(delegate_task_runner_.get());
105  DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
106  if (network_task_runner_.get()) {
107    DCHECK_EQ(network_task_runner_,
108              request_context_getter_->GetNetworkTaskRunner());
109  } else {
110    network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
111  }
112  DCHECK(network_task_runner_.get()) << "We need an IO task runner";
113
114  network_task_runner_->PostTask(
115      FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
116}
117
118void URLFetcherCore::Stop() {
119  if (delegate_task_runner_.get())  // May be NULL in tests.
120    DCHECK(delegate_task_runner_->BelongsToCurrentThread());
121
122  delegate_ = NULL;
123  fetcher_ = NULL;
124  if (!network_task_runner_.get())
125    return;
126  if (network_task_runner_->RunsTasksOnCurrentThread()) {
127    CancelURLRequest(ERR_ABORTED);
128  } else {
129    network_task_runner_->PostTask(
130        FROM_HERE,
131        base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
132  }
133}
134
135void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
136                                   const std::string& upload_content) {
137  DCHECK(!is_chunked_upload_);
138  DCHECK(!upload_content_set_);
139  DCHECK(upload_content_.empty());
140  DCHECK(upload_file_path_.empty());
141  DCHECK(upload_content_type_.empty());
142
143  // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
144  DCHECK(upload_content.empty() || !upload_content_type.empty());
145
146  upload_content_type_ = upload_content_type;
147  upload_content_ = upload_content;
148  upload_content_set_ = true;
149}
150
151void URLFetcherCore::SetUploadFilePath(
152    const std::string& upload_content_type,
153    const base::FilePath& file_path,
154    uint64 range_offset,
155    uint64 range_length,
156    scoped_refptr<base::TaskRunner> file_task_runner) {
157  DCHECK(!is_chunked_upload_);
158  DCHECK(!upload_content_set_);
159  DCHECK(upload_content_.empty());
160  DCHECK(upload_file_path_.empty());
161  DCHECK_EQ(upload_range_offset_, 0ULL);
162  DCHECK_EQ(upload_range_length_, 0ULL);
163  DCHECK(upload_content_type_.empty());
164  DCHECK(!upload_content_type.empty());
165
166  upload_content_type_ = upload_content_type;
167  upload_file_path_ = file_path;
168  upload_range_offset_ = range_offset;
169  upload_range_length_ = range_length;
170  upload_file_task_runner_ = file_task_runner;
171  upload_content_set_ = true;
172}
173
174void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
175  DCHECK(is_chunked_upload_ ||
176         (upload_content_type_.empty() &&
177          upload_content_.empty()));
178
179  // Empty |content_type| is not allowed here, because it is impossible
180  // to ensure non-empty upload content as it is not yet supplied.
181  DCHECK(!content_type.empty());
182
183  upload_content_type_ = content_type;
184  upload_content_.clear();
185  is_chunked_upload_ = true;
186}
187
188void URLFetcherCore::AppendChunkToUpload(const std::string& content,
189                                         bool is_last_chunk) {
190  DCHECK(delegate_task_runner_.get());
191  DCHECK(network_task_runner_.get());
192  network_task_runner_->PostTask(
193      FROM_HERE,
194      base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
195                 is_last_chunk));
196}
197
198void URLFetcherCore::SetLoadFlags(int load_flags) {
199  load_flags_ = load_flags;
200}
201
202int URLFetcherCore::GetLoadFlags() const {
203  return load_flags_;
204}
205
206void URLFetcherCore::SetReferrer(const std::string& referrer) {
207  referrer_ = referrer;
208}
209
210void URLFetcherCore::SetReferrerPolicy(
211    URLRequest::ReferrerPolicy referrer_policy) {
212  referrer_policy_ = referrer_policy;
213}
214
215void URLFetcherCore::SetExtraRequestHeaders(
216    const std::string& extra_request_headers) {
217  extra_request_headers_.Clear();
218  extra_request_headers_.AddHeadersFromString(extra_request_headers);
219}
220
221void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
222  extra_request_headers_.AddHeaderFromString(header_line);
223}
224
225void URLFetcherCore::SetRequestContext(
226    URLRequestContextGetter* request_context_getter) {
227  DCHECK(!request_context_getter_.get());
228  DCHECK(request_context_getter);
229  request_context_getter_ = request_context_getter;
230}
231
232void URLFetcherCore::SetFirstPartyForCookies(
233    const GURL& first_party_for_cookies) {
234  DCHECK(first_party_for_cookies_.is_empty());
235  first_party_for_cookies_ = first_party_for_cookies;
236}
237
238void URLFetcherCore::SetURLRequestUserData(
239    const void* key,
240    const URLFetcher::CreateDataCallback& create_data_callback) {
241  DCHECK(key);
242  DCHECK(!create_data_callback.is_null());
243  url_request_data_key_ = key;
244  url_request_create_data_callback_ = create_data_callback;
245}
246
247void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
248  stop_on_redirect_ = stop_on_redirect;
249}
250
251void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
252  automatically_retry_on_5xx_ = retry;
253}
254
255void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
256  max_retries_on_5xx_ = max_retries;
257}
258
259int URLFetcherCore::GetMaxRetriesOn5xx() const {
260  return max_retries_on_5xx_;
261}
262
263base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
264  return backoff_delay_;
265}
266
267void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
268  max_retries_on_network_changes_ = max_retries;
269}
270
271void URLFetcherCore::SaveResponseToFileAtPath(
272    const base::FilePath& file_path,
273    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
274  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
275  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
276      new URLFetcherFileWriter(file_task_runner, file_path)));
277}
278
279void URLFetcherCore::SaveResponseToTemporaryFile(
280    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
281  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
282  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
283      new URLFetcherFileWriter(file_task_runner, base::FilePath())));
284}
285
286void URLFetcherCore::SaveResponseWithWriter(
287    scoped_ptr<URLFetcherResponseWriter> response_writer) {
288  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
289  response_writer_ = response_writer.Pass();
290}
291
292HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
293  return response_headers_.get();
294}
295
296// TODO(panayiotis): socket_address_ is written in the IO thread,
297// if this is accessed in the UI thread, this could result in a race.
298// Same for response_headers_ above and was_fetched_via_proxy_ below.
299HostPortPair URLFetcherCore::GetSocketAddress() const {
300  return socket_address_;
301}
302
303bool URLFetcherCore::WasFetchedViaProxy() const {
304  return was_fetched_via_proxy_;
305}
306
307const GURL& URLFetcherCore::GetOriginalURL() const {
308  return original_url_;
309}
310
311const GURL& URLFetcherCore::GetURL() const {
312  return url_;
313}
314
315const URLRequestStatus& URLFetcherCore::GetStatus() const {
316  return status_;
317}
318
319int URLFetcherCore::GetResponseCode() const {
320  return response_code_;
321}
322
323const ResponseCookies& URLFetcherCore::GetCookies() const {
324  return cookies_;
325}
326
327void URLFetcherCore::ReceivedContentWasMalformed() {
328  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
329  if (network_task_runner_.get()) {
330    network_task_runner_->PostTask(
331        FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
332  }
333}
334
335bool URLFetcherCore::GetResponseAsString(
336    std::string* out_response_string) const {
337  URLFetcherStringWriter* string_writer =
338      response_writer_ ? response_writer_->AsStringWriter() : NULL;
339  if (!string_writer)
340    return false;
341
342  *out_response_string = string_writer->data();
343  UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
344                          (string_writer->data().length() / 1024));
345  return true;
346}
347
348bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
349                                           base::FilePath* out_response_path) {
350  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
351
352  URLFetcherFileWriter* file_writer =
353      response_writer_ ? response_writer_->AsFileWriter() : NULL;
354  if (!file_writer)
355    return false;
356
357  *out_response_path = file_writer->file_path();
358
359  if (take_ownership) {
360    // Intentionally calling a file_writer_ method directly without posting
361    // the task to network_task_runner_.
362    //
363    // This is for correctly handling the case when file_writer_->DisownFile()
364    // is soon followed by URLFetcherCore::Stop(). We have to make sure that
365    // DisownFile takes effect before Stop deletes file_writer_.
366    //
367    // This direct call should be thread-safe, since DisownFile itself does no
368    // file operation. It just flips the state to be referred in destruction.
369    file_writer->DisownFile();
370  }
371  return true;
372}
373
374void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
375                                        const RedirectInfo& redirect_info,
376                                        bool* defer_redirect) {
377  DCHECK_EQ(request, request_.get());
378  DCHECK(network_task_runner_->BelongsToCurrentThread());
379  if (stop_on_redirect_) {
380    stopped_on_redirect_ = true;
381    url_ = redirect_info.new_url;
382    response_code_ = request_->GetResponseCode();
383    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
384    request->Cancel();
385    OnReadCompleted(request, 0);
386  }
387}
388
389void URLFetcherCore::OnResponseStarted(URLRequest* request) {
390  DCHECK_EQ(request, request_.get());
391  DCHECK(network_task_runner_->BelongsToCurrentThread());
392  if (request_->status().is_success()) {
393    response_code_ = request_->GetResponseCode();
394    response_headers_ = request_->response_headers();
395    socket_address_ = request_->GetSocketAddress();
396    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
397    total_response_bytes_ = request_->GetExpectedContentSize();
398  }
399
400  ReadResponse();
401}
402
403void URLFetcherCore::OnCertificateRequested(
404    URLRequest* request,
405    SSLCertRequestInfo* cert_request_info) {
406  DCHECK_EQ(request, request_.get());
407  DCHECK(network_task_runner_->BelongsToCurrentThread());
408
409  if (g_ignore_certificate_requests) {
410    request->ContinueWithCertificate(NULL);
411  } else {
412    request->Cancel();
413  }
414}
415
416void URLFetcherCore::OnReadCompleted(URLRequest* request,
417                                     int bytes_read) {
418  DCHECK(request == request_);
419  DCHECK(network_task_runner_->BelongsToCurrentThread());
420
421  if (!stopped_on_redirect_)
422    url_ = request->url();
423  URLRequestThrottlerManager* throttler_manager =
424      request->context()->throttler_manager();
425  if (throttler_manager) {
426    url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
427  }
428
429  do {
430    if (!request_->status().is_success() || bytes_read <= 0)
431      break;
432
433    current_response_bytes_ += bytes_read;
434    InformDelegateDownloadProgress();
435
436    const int result =
437        WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
438    if (result < 0) {
439      // Write failed or waiting for write completion.
440      return;
441    }
442  } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
443
444  const URLRequestStatus status = request_->status();
445
446  if (status.is_success())
447    request_->GetResponseCookies(&cookies_);
448
449  // See comments re: HEAD requests in ReadResponse().
450  if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
451    status_ = status;
452    ReleaseRequest();
453
454    // No more data to write.
455    const int result = response_writer_->Finish(
456        base::Bind(&URLFetcherCore::DidFinishWriting, this));
457    if (result != ERR_IO_PENDING)
458      DidFinishWriting(result);
459  }
460}
461
462void URLFetcherCore::CancelAll() {
463  g_registry.Get().CancelAll();
464}
465
466int URLFetcherCore::GetNumFetcherCores() {
467  return g_registry.Get().size();
468}
469
470void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
471  g_interception_enabled = enabled;
472}
473
474void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
475  g_ignore_certificate_requests = ignored;
476}
477
478URLFetcherCore::~URLFetcherCore() {
479  // |request_| should be NULL.  If not, it's unsafe to delete it here since we
480  // may not be on the IO thread.
481  DCHECK(!request_.get());
482}
483
484void URLFetcherCore::StartOnIOThread() {
485  DCHECK(network_task_runner_->BelongsToCurrentThread());
486
487  if (!response_writer_)
488    response_writer_.reset(new URLFetcherStringWriter);
489
490  const int result = response_writer_->Initialize(
491      base::Bind(&URLFetcherCore::DidInitializeWriter, this));
492  if (result != ERR_IO_PENDING)
493    DidInitializeWriter(result);
494}
495
496void URLFetcherCore::StartURLRequest() {
497  DCHECK(network_task_runner_->BelongsToCurrentThread());
498
499  if (was_cancelled_) {
500    // Since StartURLRequest() is posted as a *delayed* task, it may
501    // run after the URLFetcher was already stopped.
502    return;
503  }
504
505  DCHECK(request_context_getter_.get());
506  DCHECK(!request_.get());
507
508  g_registry.Get().AddURLFetcherCore(this);
509  current_response_bytes_ = 0;
510  request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
511      original_url_, DEFAULT_PRIORITY, this, NULL);
512  request_->set_stack_trace(stack_trace_);
513  int flags = request_->load_flags() | load_flags_;
514  if (!g_interception_enabled)
515    flags = flags | LOAD_DISABLE_INTERCEPT;
516
517  if (is_chunked_upload_)
518    request_->EnableChunkedUpload();
519  request_->SetLoadFlags(flags);
520  request_->SetReferrer(referrer_);
521  request_->set_referrer_policy(referrer_policy_);
522  request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
523      original_url_ : first_party_for_cookies_);
524  if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
525    request_->SetUserData(url_request_data_key_,
526                          url_request_create_data_callback_.Run());
527  }
528
529  switch (request_type_) {
530    case URLFetcher::GET:
531      break;
532
533    case URLFetcher::POST:
534    case URLFetcher::PUT:
535    case URLFetcher::PATCH:
536      // Upload content must be set.
537      DCHECK(is_chunked_upload_ || upload_content_set_);
538
539      request_->set_method(
540          request_type_ == URLFetcher::POST ? "POST" :
541          request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
542      if (!upload_content_type_.empty()) {
543        extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
544                                         upload_content_type_);
545      }
546      if (!upload_content_.empty()) {
547        scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
548            upload_content_.data(), upload_content_.size()));
549        request_->set_upload(make_scoped_ptr(
550            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
551      } else if (!upload_file_path_.empty()) {
552        scoped_ptr<UploadElementReader> reader(
553            new UploadFileElementReader(upload_file_task_runner_.get(),
554                                        upload_file_path_,
555                                        upload_range_offset_,
556                                        upload_range_length_,
557                                        base::Time()));
558        request_->set_upload(make_scoped_ptr(
559            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
560      }
561
562      current_upload_bytes_ = -1;
563      // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
564      //  layer and avoid using timer here.
565      upload_progress_checker_timer_.reset(
566          new base::RepeatingTimer<URLFetcherCore>());
567      upload_progress_checker_timer_->Start(
568          FROM_HERE,
569          base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
570          this,
571          &URLFetcherCore::InformDelegateUploadProgress);
572      break;
573
574    case URLFetcher::HEAD:
575      request_->set_method("HEAD");
576      break;
577
578    case URLFetcher::DELETE_REQUEST:
579      request_->set_method("DELETE");
580      break;
581
582    default:
583      NOTREACHED();
584  }
585
586  if (!extra_request_headers_.IsEmpty())
587    request_->SetExtraRequestHeaders(extra_request_headers_);
588
589  request_->Start();
590}
591
592void URLFetcherCore::DidInitializeWriter(int result) {
593  if (result != OK) {
594    CancelURLRequest(result);
595    delegate_task_runner_->PostTask(
596        FROM_HERE,
597        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
598    return;
599  }
600  StartURLRequestWhenAppropriate();
601}
602
603void URLFetcherCore::StartURLRequestWhenAppropriate() {
604  DCHECK(network_task_runner_->BelongsToCurrentThread());
605
606  if (was_cancelled_)
607    return;
608
609  DCHECK(request_context_getter_.get());
610
611  int64 delay = 0LL;
612  if (original_url_throttler_entry_.get() == NULL) {
613    URLRequestThrottlerManager* manager =
614        request_context_getter_->GetURLRequestContext()->throttler_manager();
615    if (manager) {
616      original_url_throttler_entry_ =
617          manager->RegisterRequestUrl(original_url_);
618    }
619  }
620  if (original_url_throttler_entry_.get() != NULL) {
621    delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
622        GetBackoffReleaseTime());
623  }
624
625  if (delay == 0) {
626    StartURLRequest();
627  } else {
628    base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
629        FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
630        base::TimeDelta::FromMilliseconds(delay));
631  }
632}
633
634void URLFetcherCore::CancelURLRequest(int error) {
635  DCHECK(network_task_runner_->BelongsToCurrentThread());
636
637  if (request_.get()) {
638    request_->CancelWithError(error);
639    ReleaseRequest();
640  }
641
642  // Set the error manually.
643  // Normally, calling URLRequest::CancelWithError() results in calling
644  // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
645  // URLRequestJob::NotifyDone(). But, because the request was released
646  // immediately after being canceled, the request could not call
647  // OnReadCompleted() which overwrites |status_| with the error status.
648  status_.set_status(URLRequestStatus::CANCELED);
649  status_.set_error(error);
650
651  // Release the reference to the request context. There could be multiple
652  // references to URLFetcher::Core at this point so it may take a while to
653  // delete the object, but we cannot delay the destruction of the request
654  // context.
655  request_context_getter_ = NULL;
656  first_party_for_cookies_ = GURL();
657  url_request_data_key_ = NULL;
658  url_request_create_data_callback_.Reset();
659  was_cancelled_ = true;
660}
661
662void URLFetcherCore::OnCompletedURLRequest(
663    base::TimeDelta backoff_delay) {
664  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
665
666  // Save the status and backoff_delay so that delegates can read it.
667  if (delegate_) {
668    backoff_delay_ = backoff_delay;
669    InformDelegateFetchIsComplete();
670  }
671}
672
673void URLFetcherCore::InformDelegateFetchIsComplete() {
674  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
675  if (delegate_)
676    delegate_->OnURLFetchComplete(fetcher_);
677}
678
679void URLFetcherCore::NotifyMalformedContent() {
680  DCHECK(network_task_runner_->BelongsToCurrentThread());
681  if (url_throttler_entry_.get() != NULL) {
682    int status_code = response_code_;
683    if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
684      // The status code will generally be known by the time clients
685      // call the |ReceivedContentWasMalformed()| function (which ends up
686      // calling the current function) but if it's not, we need to assume
687      // the response was successful so that the total failure count
688      // used to calculate exponential back-off goes up.
689      status_code = 200;
690    }
691    url_throttler_entry_->ReceivedContentWasMalformed(status_code);
692  }
693}
694
695void URLFetcherCore::DidFinishWriting(int result) {
696  if (result != OK) {
697    CancelURLRequest(result);
698    delegate_task_runner_->PostTask(
699        FROM_HERE,
700        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
701    return;
702  }
703  // If the file was successfully closed, then the URL request is complete.
704  RetryOrCompleteUrlFetch();
705}
706
707void URLFetcherCore::RetryOrCompleteUrlFetch() {
708  DCHECK(network_task_runner_->BelongsToCurrentThread());
709  base::TimeDelta backoff_delay;
710
711  // Checks the response from server.
712  if (response_code_ >= 500 ||
713      status_.error() == ERR_TEMPORARILY_THROTTLED) {
714    // When encountering a server error, we will send the request again
715    // after backoff time.
716    ++num_retries_on_5xx_;
717
718    // Note that backoff_delay may be 0 because (a) the
719    // URLRequestThrottlerManager and related code does not
720    // necessarily back off on the first error, (b) it only backs off
721    // on some of the 5xx status codes, (c) not all URLRequestContexts
722    // have a throttler manager.
723    base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
724    backoff_delay = backoff_release_time - base::TimeTicks::Now();
725    if (backoff_delay < base::TimeDelta())
726      backoff_delay = base::TimeDelta();
727
728    if (automatically_retry_on_5xx_ &&
729        num_retries_on_5xx_ <= max_retries_on_5xx_) {
730      StartOnIOThread();
731      return;
732    }
733  } else {
734    backoff_delay = base::TimeDelta();
735  }
736
737  // Retry if the request failed due to network changes.
738  if (status_.error() == ERR_NETWORK_CHANGED &&
739      num_retries_on_network_changes_ < max_retries_on_network_changes_) {
740    ++num_retries_on_network_changes_;
741
742    // Retry soon, after flushing all the current tasks which may include
743    // further network change observers.
744    network_task_runner_->PostTask(
745        FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
746    return;
747  }
748
749  request_context_getter_ = NULL;
750  first_party_for_cookies_ = GURL();
751  url_request_data_key_ = NULL;
752  url_request_create_data_callback_.Reset();
753  bool posted = delegate_task_runner_->PostTask(
754      FROM_HERE,
755      base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
756
757  // If the delegate message loop does not exist any more, then the delegate
758  // should be gone too.
759  DCHECK(posted || !delegate_);
760}
761
762void URLFetcherCore::ReleaseRequest() {
763  upload_progress_checker_timer_.reset();
764  request_.reset();
765  g_registry.Get().RemoveURLFetcherCore(this);
766}
767
768base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
769  DCHECK(network_task_runner_->BelongsToCurrentThread());
770
771  if (original_url_throttler_entry_.get()) {
772    base::TimeTicks original_url_backoff =
773        original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
774    base::TimeTicks destination_url_backoff;
775    if (url_throttler_entry_.get() != NULL &&
776        original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
777      destination_url_backoff =
778          url_throttler_entry_->GetExponentialBackoffReleaseTime();
779    }
780
781    return original_url_backoff > destination_url_backoff ?
782        original_url_backoff : destination_url_backoff;
783  } else {
784    return base::TimeTicks();
785  }
786}
787
788void URLFetcherCore::CompleteAddingUploadDataChunk(
789    const std::string& content, bool is_last_chunk) {
790  if (was_cancelled_) {
791    // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
792    // may run after the URLFetcher was already stopped.
793    return;
794  }
795  DCHECK(is_chunked_upload_);
796  DCHECK(request_.get());
797  DCHECK(!content.empty());
798  request_->AppendChunkToUpload(content.data(),
799                                static_cast<int>(content.length()),
800                                is_last_chunk);
801}
802
803int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
804  while (data->BytesRemaining() > 0) {
805    const int result = response_writer_->Write(
806        data.get(),
807        data->BytesRemaining(),
808        base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
809    if (result < 0) {
810      if (result != ERR_IO_PENDING)
811        DidWriteBuffer(data, result);
812      return result;
813    }
814    data->DidConsume(result);
815  }
816  return OK;
817}
818
819void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
820                                    int result) {
821  if (result < 0) {  // Handle errors.
822    CancelURLRequest(result);
823    response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
824    delegate_task_runner_->PostTask(
825        FROM_HERE,
826        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
827    return;
828  }
829
830  // Continue writing.
831  data->DidConsume(result);
832  if (WriteBuffer(data) < 0)
833    return;
834
835  // Finished writing buffer_. Read some more, unless the request has been
836  // cancelled and deleted.
837  DCHECK_EQ(0, data->BytesRemaining());
838  if (request_.get())
839    ReadResponse();
840}
841
842void URLFetcherCore::ReadResponse() {
843  // Some servers may treat HEAD requests as GET requests.  To free up the
844  // network connection as soon as possible, signal that the request has
845  // completed immediately, without trying to read any data back (all we care
846  // about is the response code and headers, which we already have).
847  int bytes_read = 0;
848  if (request_->status().is_success() &&
849      (request_type_ != URLFetcher::HEAD))
850    request_->Read(buffer_.get(), kBufferSize, &bytes_read);
851  OnReadCompleted(request_.get(), bytes_read);
852}
853
854void URLFetcherCore::InformDelegateUploadProgress() {
855  DCHECK(network_task_runner_->BelongsToCurrentThread());
856  if (request_.get()) {
857    int64 current = request_->GetUploadProgress().position();
858    if (current_upload_bytes_ != current) {
859      current_upload_bytes_ = current;
860      int64 total = -1;
861      if (!is_chunked_upload_) {
862        total = static_cast<int64>(request_->GetUploadProgress().size());
863        // Total may be zero if the UploadDataStream::Init has not been called
864        // yet.  Don't send the upload progress until the size is initialized.
865        if (!total)
866          return;
867      }
868      delegate_task_runner_->PostTask(
869          FROM_HERE,
870          base::Bind(
871              &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
872              this, current, total));
873    }
874  }
875}
876
877void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
878    int64 current, int64 total) {
879  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
880  if (delegate_)
881    delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
882}
883
884void URLFetcherCore::InformDelegateDownloadProgress() {
885  DCHECK(network_task_runner_->BelongsToCurrentThread());
886  delegate_task_runner_->PostTask(
887      FROM_HERE,
888      base::Bind(
889          &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
890          this, current_response_bytes_, total_response_bytes_));
891}
892
893void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
894    int64 current, int64 total) {
895  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
896  if (delegate_)
897    delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
898}
899
900}  // namespace net
901