url_fetcher_core.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_fetcher_core.h"
6
7#include "base/bind.h"
8#include "base/logging.h"
9#include "base/metrics/histogram.h"
10#include "base/sequenced_task_runner.h"
11#include "base/single_thread_task_runner.h"
12#include "base/stl_util.h"
13#include "base/thread_task_runner_handle.h"
14#include "base/tracked_objects.h"
15#include "net/base/io_buffer.h"
16#include "net/base/load_flags.h"
17#include "net/base/net_errors.h"
18#include "net/base/request_priority.h"
19#include "net/base/upload_bytes_element_reader.h"
20#include "net/base/upload_data_stream.h"
21#include "net/base/upload_file_element_reader.h"
22#include "net/http/http_response_headers.h"
23#include "net/url_request/url_fetcher_delegate.h"
24#include "net/url_request/url_fetcher_response_writer.h"
25#include "net/url_request/url_request_context.h"
26#include "net/url_request/url_request_context_getter.h"
27#include "net/url_request/url_request_throttler_manager.h"
28
29namespace {
30
31const int kBufferSize = 4096;
32const int kUploadProgressTimerInterval = 100;
33bool g_interception_enabled = false;
34bool g_ignore_certificate_requests = false;
35
36void EmptyCompletionCallback(int result) {}
37
38}  // namespace
39
40namespace net {
41
42// URLFetcherCore::Registry ---------------------------------------------------
43
44URLFetcherCore::Registry::Registry() {}
45URLFetcherCore::Registry::~Registry() {}
46
47void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
48  DCHECK(!ContainsKey(fetchers_, core));
49  fetchers_.insert(core);
50}
51
52void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
53  DCHECK(ContainsKey(fetchers_, core));
54  fetchers_.erase(core);
55}
56
57void URLFetcherCore::Registry::CancelAll() {
58  while (!fetchers_.empty())
59    (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
60}
61
62// URLFetcherCore -------------------------------------------------------------
63
64// static
65base::LazyInstance<URLFetcherCore::Registry>
66    URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
67
68URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
69                               const GURL& original_url,
70                               URLFetcher::RequestType request_type,
71                               URLFetcherDelegate* d)
72    : fetcher_(fetcher),
73      original_url_(original_url),
74      request_type_(request_type),
75      delegate_(d),
76      delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
77      load_flags_(LOAD_NORMAL),
78      response_code_(URLFetcher::RESPONSE_CODE_INVALID),
79      buffer_(new IOBuffer(kBufferSize)),
80      url_request_data_key_(NULL),
81      was_fetched_via_proxy_(false),
82      upload_content_set_(false),
83      upload_range_offset_(0),
84      upload_range_length_(0),
85      referrer_policy_(
86          URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
87      is_chunked_upload_(false),
88      was_cancelled_(false),
89      stop_on_redirect_(false),
90      stopped_on_redirect_(false),
91      automatically_retry_on_5xx_(true),
92      num_retries_on_5xx_(0),
93      max_retries_on_5xx_(0),
94      num_retries_on_network_changes_(0),
95      max_retries_on_network_changes_(0),
96      current_upload_bytes_(-1),
97      current_response_bytes_(0),
98      total_response_bytes_(-1) {
99  CHECK(original_url_.is_valid());
100}
101
102void URLFetcherCore::Start() {
103  DCHECK(delegate_task_runner_.get());
104  DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
105  if (network_task_runner_.get()) {
106    DCHECK_EQ(network_task_runner_,
107              request_context_getter_->GetNetworkTaskRunner());
108  } else {
109    network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
110  }
111  DCHECK(network_task_runner_.get()) << "We need an IO task runner";
112
113  network_task_runner_->PostTask(
114      FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
115}
116
117void URLFetcherCore::Stop() {
118  if (delegate_task_runner_.get())  // May be NULL in tests.
119    DCHECK(delegate_task_runner_->BelongsToCurrentThread());
120
121  delegate_ = NULL;
122  fetcher_ = NULL;
123  if (!network_task_runner_.get())
124    return;
125  if (network_task_runner_->RunsTasksOnCurrentThread()) {
126    CancelURLRequest(ERR_ABORTED);
127  } else {
128    network_task_runner_->PostTask(
129        FROM_HERE,
130        base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
131  }
132}
133
134void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
135                                   const std::string& upload_content) {
136  DCHECK(!is_chunked_upload_);
137  DCHECK(!upload_content_set_);
138  DCHECK(upload_content_.empty());
139  DCHECK(upload_file_path_.empty());
140  DCHECK(upload_content_type_.empty());
141
142  // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
143  DCHECK(upload_content.empty() || !upload_content_type.empty());
144
145  upload_content_type_ = upload_content_type;
146  upload_content_ = upload_content;
147  upload_content_set_ = true;
148}
149
150void URLFetcherCore::SetUploadFilePath(
151    const std::string& upload_content_type,
152    const base::FilePath& file_path,
153    uint64 range_offset,
154    uint64 range_length,
155    scoped_refptr<base::TaskRunner> file_task_runner) {
156  DCHECK(!is_chunked_upload_);
157  DCHECK(!upload_content_set_);
158  DCHECK(upload_content_.empty());
159  DCHECK(upload_file_path_.empty());
160  DCHECK_EQ(upload_range_offset_, 0ULL);
161  DCHECK_EQ(upload_range_length_, 0ULL);
162  DCHECK(upload_content_type_.empty());
163  DCHECK(!upload_content_type.empty());
164
165  upload_content_type_ = upload_content_type;
166  upload_file_path_ = file_path;
167  upload_range_offset_ = range_offset;
168  upload_range_length_ = range_length;
169  upload_file_task_runner_ = file_task_runner;
170  upload_content_set_ = true;
171}
172
173void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
174  DCHECK(is_chunked_upload_ ||
175         (upload_content_type_.empty() &&
176          upload_content_.empty()));
177
178  // Empty |content_type| is not allowed here, because it is impossible
179  // to ensure non-empty upload content as it is not yet supplied.
180  DCHECK(!content_type.empty());
181
182  upload_content_type_ = content_type;
183  upload_content_.clear();
184  is_chunked_upload_ = true;
185}
186
187void URLFetcherCore::AppendChunkToUpload(const std::string& content,
188                                         bool is_last_chunk) {
189  DCHECK(delegate_task_runner_.get());
190  DCHECK(network_task_runner_.get());
191  network_task_runner_->PostTask(
192      FROM_HERE,
193      base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
194                 is_last_chunk));
195}
196
197void URLFetcherCore::SetLoadFlags(int load_flags) {
198  load_flags_ = load_flags;
199}
200
201int URLFetcherCore::GetLoadFlags() const {
202  return load_flags_;
203}
204
205void URLFetcherCore::SetReferrer(const std::string& referrer) {
206  referrer_ = referrer;
207}
208
209void URLFetcherCore::SetReferrerPolicy(
210    URLRequest::ReferrerPolicy referrer_policy) {
211  referrer_policy_ = referrer_policy;
212}
213
214void URLFetcherCore::SetExtraRequestHeaders(
215    const std::string& extra_request_headers) {
216  extra_request_headers_.Clear();
217  extra_request_headers_.AddHeadersFromString(extra_request_headers);
218}
219
220void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
221  extra_request_headers_.AddHeaderFromString(header_line);
222}
223
224void URLFetcherCore::SetRequestContext(
225    URLRequestContextGetter* request_context_getter) {
226  DCHECK(!request_context_getter_.get());
227  DCHECK(request_context_getter);
228  request_context_getter_ = request_context_getter;
229}
230
231void URLFetcherCore::SetFirstPartyForCookies(
232    const GURL& first_party_for_cookies) {
233  DCHECK(first_party_for_cookies_.is_empty());
234  first_party_for_cookies_ = first_party_for_cookies;
235}
236
237void URLFetcherCore::SetURLRequestUserData(
238    const void* key,
239    const URLFetcher::CreateDataCallback& create_data_callback) {
240  DCHECK(key);
241  DCHECK(!create_data_callback.is_null());
242  url_request_data_key_ = key;
243  url_request_create_data_callback_ = create_data_callback;
244}
245
246void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
247  stop_on_redirect_ = stop_on_redirect;
248}
249
250void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
251  automatically_retry_on_5xx_ = retry;
252}
253
254void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
255  max_retries_on_5xx_ = max_retries;
256}
257
258int URLFetcherCore::GetMaxRetriesOn5xx() const {
259  return max_retries_on_5xx_;
260}
261
262base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
263  return backoff_delay_;
264}
265
266void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
267  max_retries_on_network_changes_ = max_retries;
268}
269
270void URLFetcherCore::SaveResponseToFileAtPath(
271    const base::FilePath& file_path,
272    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
273  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
274  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
275      new URLFetcherFileWriter(file_task_runner, file_path)));
276}
277
278void URLFetcherCore::SaveResponseToTemporaryFile(
279    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
280  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
281  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
282      new URLFetcherFileWriter(file_task_runner, base::FilePath())));
283}
284
285void URLFetcherCore::SaveResponseWithWriter(
286    scoped_ptr<URLFetcherResponseWriter> response_writer) {
287  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
288  response_writer_ = response_writer.Pass();
289}
290
291HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
292  return response_headers_.get();
293}
294
295// TODO(panayiotis): socket_address_ is written in the IO thread,
296// if this is accessed in the UI thread, this could result in a race.
297// Same for response_headers_ above and was_fetched_via_proxy_ below.
298HostPortPair URLFetcherCore::GetSocketAddress() const {
299  return socket_address_;
300}
301
302bool URLFetcherCore::WasFetchedViaProxy() const {
303  return was_fetched_via_proxy_;
304}
305
306const GURL& URLFetcherCore::GetOriginalURL() const {
307  return original_url_;
308}
309
310const GURL& URLFetcherCore::GetURL() const {
311  return url_;
312}
313
314const URLRequestStatus& URLFetcherCore::GetStatus() const {
315  return status_;
316}
317
318int URLFetcherCore::GetResponseCode() const {
319  return response_code_;
320}
321
322const ResponseCookies& URLFetcherCore::GetCookies() const {
323  return cookies_;
324}
325
326void URLFetcherCore::ReceivedContentWasMalformed() {
327  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
328  if (network_task_runner_.get()) {
329    network_task_runner_->PostTask(
330        FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
331  }
332}
333
334bool URLFetcherCore::GetResponseAsString(
335    std::string* out_response_string) const {
336  URLFetcherStringWriter* string_writer =
337      response_writer_ ? response_writer_->AsStringWriter() : NULL;
338  if (!string_writer)
339    return false;
340
341  *out_response_string = string_writer->data();
342  UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
343                          (string_writer->data().length() / 1024));
344  return true;
345}
346
347bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
348                                           base::FilePath* out_response_path) {
349  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
350
351  URLFetcherFileWriter* file_writer =
352      response_writer_ ? response_writer_->AsFileWriter() : NULL;
353  if (!file_writer)
354    return false;
355
356  *out_response_path = file_writer->file_path();
357
358  if (take_ownership) {
359    // Intentionally calling a file_writer_ method directly without posting
360    // the task to network_task_runner_.
361    //
362    // This is for correctly handling the case when file_writer_->DisownFile()
363    // is soon followed by URLFetcherCore::Stop(). We have to make sure that
364    // DisownFile takes effect before Stop deletes file_writer_.
365    //
366    // This direct call should be thread-safe, since DisownFile itself does no
367    // file operation. It just flips the state to be referred in destruction.
368    file_writer->DisownFile();
369  }
370  return true;
371}
372
373void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
374                                        const GURL& new_url,
375                                        bool* defer_redirect) {
376  DCHECK_EQ(request, request_.get());
377  DCHECK(network_task_runner_->BelongsToCurrentThread());
378  if (stop_on_redirect_) {
379    stopped_on_redirect_ = true;
380    url_ = new_url;
381    response_code_ = request_->GetResponseCode();
382    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
383    request->Cancel();
384    OnReadCompleted(request, 0);
385  }
386}
387
388void URLFetcherCore::OnResponseStarted(URLRequest* request) {
389  DCHECK_EQ(request, request_.get());
390  DCHECK(network_task_runner_->BelongsToCurrentThread());
391  if (request_->status().is_success()) {
392    response_code_ = request_->GetResponseCode();
393    response_headers_ = request_->response_headers();
394    socket_address_ = request_->GetSocketAddress();
395    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
396    total_response_bytes_ = request_->GetExpectedContentSize();
397  }
398
399  ReadResponse();
400}
401
402void URLFetcherCore::OnCertificateRequested(
403    URLRequest* request,
404    SSLCertRequestInfo* cert_request_info) {
405  DCHECK_EQ(request, request_.get());
406  DCHECK(network_task_runner_->BelongsToCurrentThread());
407
408  if (g_ignore_certificate_requests) {
409    request->ContinueWithCertificate(NULL);
410  } else {
411    request->Cancel();
412  }
413}
414
415void URLFetcherCore::OnReadCompleted(URLRequest* request,
416                                     int bytes_read) {
417  DCHECK(request == request_);
418  DCHECK(network_task_runner_->BelongsToCurrentThread());
419
420  if (!stopped_on_redirect_)
421    url_ = request->url();
422  URLRequestThrottlerManager* throttler_manager =
423      request->context()->throttler_manager();
424  if (throttler_manager) {
425    url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
426  }
427
428  do {
429    if (!request_->status().is_success() || bytes_read <= 0)
430      break;
431
432    current_response_bytes_ += bytes_read;
433    InformDelegateDownloadProgress();
434
435    const int result =
436        WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
437    if (result < 0) {
438      // Write failed or waiting for write completion.
439      return;
440    }
441  } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
442
443  const URLRequestStatus status = request_->status();
444
445  if (status.is_success())
446    request_->GetResponseCookies(&cookies_);
447
448  // See comments re: HEAD requests in ReadResponse().
449  if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
450    status_ = status;
451    ReleaseRequest();
452
453    // No more data to write.
454    const int result = response_writer_->Finish(
455        base::Bind(&URLFetcherCore::DidFinishWriting, this));
456    if (result != ERR_IO_PENDING)
457      DidFinishWriting(result);
458  }
459}
460
461void URLFetcherCore::CancelAll() {
462  g_registry.Get().CancelAll();
463}
464
465int URLFetcherCore::GetNumFetcherCores() {
466  return g_registry.Get().size();
467}
468
469void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
470  g_interception_enabled = enabled;
471}
472
473void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
474  g_ignore_certificate_requests = ignored;
475}
476
477URLFetcherCore::~URLFetcherCore() {
478  // |request_| should be NULL.  If not, it's unsafe to delete it here since we
479  // may not be on the IO thread.
480  DCHECK(!request_.get());
481}
482
483void URLFetcherCore::StartOnIOThread() {
484  DCHECK(network_task_runner_->BelongsToCurrentThread());
485
486  if (!response_writer_)
487    response_writer_.reset(new URLFetcherStringWriter);
488
489  const int result = response_writer_->Initialize(
490      base::Bind(&URLFetcherCore::DidInitializeWriter, this));
491  if (result != ERR_IO_PENDING)
492    DidInitializeWriter(result);
493}
494
495void URLFetcherCore::StartURLRequest() {
496  DCHECK(network_task_runner_->BelongsToCurrentThread());
497
498  if (was_cancelled_) {
499    // Since StartURLRequest() is posted as a *delayed* task, it may
500    // run after the URLFetcher was already stopped.
501    return;
502  }
503
504  DCHECK(request_context_getter_.get());
505  DCHECK(!request_.get());
506
507  g_registry.Get().AddURLFetcherCore(this);
508  current_response_bytes_ = 0;
509  request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
510      original_url_, DEFAULT_PRIORITY, this, NULL);
511  request_->set_stack_trace(stack_trace_);
512  int flags = request_->load_flags() | load_flags_;
513  if (!g_interception_enabled)
514    flags = flags | LOAD_DISABLE_INTERCEPT;
515
516  if (is_chunked_upload_)
517    request_->EnableChunkedUpload();
518  request_->SetLoadFlags(flags);
519  request_->SetReferrer(referrer_);
520  request_->set_referrer_policy(referrer_policy_);
521  request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
522      original_url_ : first_party_for_cookies_);
523  if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
524    request_->SetUserData(url_request_data_key_,
525                          url_request_create_data_callback_.Run());
526  }
527
528  switch (request_type_) {
529    case URLFetcher::GET:
530      break;
531
532    case URLFetcher::POST:
533    case URLFetcher::PUT:
534    case URLFetcher::PATCH:
535      // Upload content must be set.
536      DCHECK(is_chunked_upload_ || upload_content_set_);
537
538      request_->set_method(
539          request_type_ == URLFetcher::POST ? "POST" :
540          request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
541      if (!upload_content_type_.empty()) {
542        extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
543                                         upload_content_type_);
544      }
545      if (!upload_content_.empty()) {
546        scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
547            upload_content_.data(), upload_content_.size()));
548        request_->set_upload(make_scoped_ptr(
549            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
550      } else if (!upload_file_path_.empty()) {
551        scoped_ptr<UploadElementReader> reader(
552            new UploadFileElementReader(upload_file_task_runner_.get(),
553                                        upload_file_path_,
554                                        upload_range_offset_,
555                                        upload_range_length_,
556                                        base::Time()));
557        request_->set_upload(make_scoped_ptr(
558            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
559      }
560
561      current_upload_bytes_ = -1;
562      // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
563      //  layer and avoid using timer here.
564      upload_progress_checker_timer_.reset(
565          new base::RepeatingTimer<URLFetcherCore>());
566      upload_progress_checker_timer_->Start(
567          FROM_HERE,
568          base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
569          this,
570          &URLFetcherCore::InformDelegateUploadProgress);
571      break;
572
573    case URLFetcher::HEAD:
574      request_->set_method("HEAD");
575      break;
576
577    case URLFetcher::DELETE_REQUEST:
578      request_->set_method("DELETE");
579      break;
580
581    default:
582      NOTREACHED();
583  }
584
585  if (!extra_request_headers_.IsEmpty())
586    request_->SetExtraRequestHeaders(extra_request_headers_);
587
588  request_->Start();
589}
590
591void URLFetcherCore::DidInitializeWriter(int result) {
592  if (result != OK) {
593    CancelURLRequest(result);
594    delegate_task_runner_->PostTask(
595        FROM_HERE,
596        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
597    return;
598  }
599  StartURLRequestWhenAppropriate();
600}
601
602void URLFetcherCore::StartURLRequestWhenAppropriate() {
603  DCHECK(network_task_runner_->BelongsToCurrentThread());
604
605  if (was_cancelled_)
606    return;
607
608  DCHECK(request_context_getter_.get());
609
610  int64 delay = 0LL;
611  if (original_url_throttler_entry_.get() == NULL) {
612    URLRequestThrottlerManager* manager =
613        request_context_getter_->GetURLRequestContext()->throttler_manager();
614    if (manager) {
615      original_url_throttler_entry_ =
616          manager->RegisterRequestUrl(original_url_);
617    }
618  }
619  if (original_url_throttler_entry_.get() != NULL) {
620    delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
621        GetBackoffReleaseTime());
622  }
623
624  if (delay == 0) {
625    StartURLRequest();
626  } else {
627    base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
628        FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
629        base::TimeDelta::FromMilliseconds(delay));
630  }
631}
632
633void URLFetcherCore::CancelURLRequest(int error) {
634  DCHECK(network_task_runner_->BelongsToCurrentThread());
635
636  if (request_.get()) {
637    request_->CancelWithError(error);
638    ReleaseRequest();
639  }
640
641  // Set the error manually.
642  // Normally, calling URLRequest::CancelWithError() results in calling
643  // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
644  // URLRequestJob::NotifyDone(). But, because the request was released
645  // immediately after being canceled, the request could not call
646  // OnReadCompleted() which overwrites |status_| with the error status.
647  status_.set_status(URLRequestStatus::CANCELED);
648  status_.set_error(error);
649
650  // Release the reference to the request context. There could be multiple
651  // references to URLFetcher::Core at this point so it may take a while to
652  // delete the object, but we cannot delay the destruction of the request
653  // context.
654  request_context_getter_ = NULL;
655  first_party_for_cookies_ = GURL();
656  url_request_data_key_ = NULL;
657  url_request_create_data_callback_.Reset();
658  was_cancelled_ = true;
659}
660
661void URLFetcherCore::OnCompletedURLRequest(
662    base::TimeDelta backoff_delay) {
663  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
664
665  // Save the status and backoff_delay so that delegates can read it.
666  if (delegate_) {
667    backoff_delay_ = backoff_delay;
668    InformDelegateFetchIsComplete();
669  }
670}
671
672void URLFetcherCore::InformDelegateFetchIsComplete() {
673  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
674  if (delegate_)
675    delegate_->OnURLFetchComplete(fetcher_);
676}
677
678void URLFetcherCore::NotifyMalformedContent() {
679  DCHECK(network_task_runner_->BelongsToCurrentThread());
680  if (url_throttler_entry_.get() != NULL) {
681    int status_code = response_code_;
682    if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
683      // The status code will generally be known by the time clients
684      // call the |ReceivedContentWasMalformed()| function (which ends up
685      // calling the current function) but if it's not, we need to assume
686      // the response was successful so that the total failure count
687      // used to calculate exponential back-off goes up.
688      status_code = 200;
689    }
690    url_throttler_entry_->ReceivedContentWasMalformed(status_code);
691  }
692}
693
694void URLFetcherCore::DidFinishWriting(int result) {
695  if (result != OK) {
696    CancelURLRequest(result);
697    delegate_task_runner_->PostTask(
698        FROM_HERE,
699        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
700    return;
701  }
702  // If the file was successfully closed, then the URL request is complete.
703  RetryOrCompleteUrlFetch();
704}
705
706void URLFetcherCore::RetryOrCompleteUrlFetch() {
707  DCHECK(network_task_runner_->BelongsToCurrentThread());
708  base::TimeDelta backoff_delay;
709
710  // Checks the response from server.
711  if (response_code_ >= 500 ||
712      status_.error() == ERR_TEMPORARILY_THROTTLED) {
713    // When encountering a server error, we will send the request again
714    // after backoff time.
715    ++num_retries_on_5xx_;
716
717    // Note that backoff_delay may be 0 because (a) the
718    // URLRequestThrottlerManager and related code does not
719    // necessarily back off on the first error, (b) it only backs off
720    // on some of the 5xx status codes, (c) not all URLRequestContexts
721    // have a throttler manager.
722    base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
723    backoff_delay = backoff_release_time - base::TimeTicks::Now();
724    if (backoff_delay < base::TimeDelta())
725      backoff_delay = base::TimeDelta();
726
727    if (automatically_retry_on_5xx_ &&
728        num_retries_on_5xx_ <= max_retries_on_5xx_) {
729      StartOnIOThread();
730      return;
731    }
732  } else {
733    backoff_delay = base::TimeDelta();
734  }
735
736  // Retry if the request failed due to network changes.
737  if (status_.error() == ERR_NETWORK_CHANGED &&
738      num_retries_on_network_changes_ < max_retries_on_network_changes_) {
739    ++num_retries_on_network_changes_;
740
741    // Retry soon, after flushing all the current tasks which may include
742    // further network change observers.
743    network_task_runner_->PostTask(
744        FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
745    return;
746  }
747
748  request_context_getter_ = NULL;
749  first_party_for_cookies_ = GURL();
750  url_request_data_key_ = NULL;
751  url_request_create_data_callback_.Reset();
752  bool posted = delegate_task_runner_->PostTask(
753      FROM_HERE,
754      base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
755
756  // If the delegate message loop does not exist any more, then the delegate
757  // should be gone too.
758  DCHECK(posted || !delegate_);
759}
760
761void URLFetcherCore::ReleaseRequest() {
762  upload_progress_checker_timer_.reset();
763  request_.reset();
764  g_registry.Get().RemoveURLFetcherCore(this);
765}
766
767base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
768  DCHECK(network_task_runner_->BelongsToCurrentThread());
769
770  if (original_url_throttler_entry_.get()) {
771    base::TimeTicks original_url_backoff =
772        original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
773    base::TimeTicks destination_url_backoff;
774    if (url_throttler_entry_.get() != NULL &&
775        original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
776      destination_url_backoff =
777          url_throttler_entry_->GetExponentialBackoffReleaseTime();
778    }
779
780    return original_url_backoff > destination_url_backoff ?
781        original_url_backoff : destination_url_backoff;
782  } else {
783    return base::TimeTicks();
784  }
785}
786
787void URLFetcherCore::CompleteAddingUploadDataChunk(
788    const std::string& content, bool is_last_chunk) {
789  if (was_cancelled_) {
790    // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
791    // may run after the URLFetcher was already stopped.
792    return;
793  }
794  DCHECK(is_chunked_upload_);
795  DCHECK(request_.get());
796  DCHECK(!content.empty());
797  request_->AppendChunkToUpload(content.data(),
798                                static_cast<int>(content.length()),
799                                is_last_chunk);
800}
801
802int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
803  while (data->BytesRemaining() > 0) {
804    const int result = response_writer_->Write(
805        data.get(),
806        data->BytesRemaining(),
807        base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
808    if (result < 0) {
809      if (result != ERR_IO_PENDING)
810        DidWriteBuffer(data, result);
811      return result;
812    }
813    data->DidConsume(result);
814  }
815  return OK;
816}
817
818void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
819                                    int result) {
820  if (result < 0) {  // Handle errors.
821    CancelURLRequest(result);
822    response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
823    delegate_task_runner_->PostTask(
824        FROM_HERE,
825        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
826    return;
827  }
828
829  // Continue writing.
830  data->DidConsume(result);
831  if (WriteBuffer(data) < 0)
832    return;
833
834  // Finished writing buffer_. Read some more, unless the request has been
835  // cancelled and deleted.
836  DCHECK_EQ(0, data->BytesRemaining());
837  if (request_.get())
838    ReadResponse();
839}
840
841void URLFetcherCore::ReadResponse() {
842  // Some servers may treat HEAD requests as GET requests.  To free up the
843  // network connection as soon as possible, signal that the request has
844  // completed immediately, without trying to read any data back (all we care
845  // about is the response code and headers, which we already have).
846  int bytes_read = 0;
847  if (request_->status().is_success() &&
848      (request_type_ != URLFetcher::HEAD))
849    request_->Read(buffer_.get(), kBufferSize, &bytes_read);
850  OnReadCompleted(request_.get(), bytes_read);
851}
852
853void URLFetcherCore::InformDelegateUploadProgress() {
854  DCHECK(network_task_runner_->BelongsToCurrentThread());
855  if (request_.get()) {
856    int64 current = request_->GetUploadProgress().position();
857    if (current_upload_bytes_ != current) {
858      current_upload_bytes_ = current;
859      int64 total = -1;
860      if (!is_chunked_upload_) {
861        total = static_cast<int64>(request_->GetUploadProgress().size());
862        // Total may be zero if the UploadDataStream::Init has not been called
863        // yet.  Don't send the upload progress until the size is initialized.
864        if (!total)
865          return;
866      }
867      delegate_task_runner_->PostTask(
868          FROM_HERE,
869          base::Bind(
870              &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
871              this, current, total));
872    }
873  }
874}
875
876void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
877    int64 current, int64 total) {
878  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
879  if (delegate_)
880    delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
881}
882
883void URLFetcherCore::InformDelegateDownloadProgress() {
884  DCHECK(network_task_runner_->BelongsToCurrentThread());
885  delegate_task_runner_->PostTask(
886      FROM_HERE,
887      base::Bind(
888          &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
889          this, current_response_bytes_, total_response_bytes_));
890}
891
892void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
893    int64 current, int64 total) {
894  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
895  if (delegate_)
896    delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
897}
898
899}  // namespace net
900