url_fetcher_core.cc revision 1320f92c476a1ad9d19dba2a48c72b75566198e9 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_fetcher_core.h"
6
7#include <stdint.h>
8
9#include "base/bind.h"
10#include "base/logging.h"
11#include "base/metrics/histogram.h"
12#include "base/sequenced_task_runner.h"
13#include "base/single_thread_task_runner.h"
14#include "base/stl_util.h"
15#include "base/thread_task_runner_handle.h"
16#include "base/tracked_objects.h"
17#include "net/base/io_buffer.h"
18#include "net/base/load_flags.h"
19#include "net/base/net_errors.h"
20#include "net/base/request_priority.h"
21#include "net/base/upload_bytes_element_reader.h"
22#include "net/base/upload_data_stream.h"
23#include "net/base/upload_file_element_reader.h"
24#include "net/http/http_response_headers.h"
25#include "net/url_request/redirect_info.h"
26#include "net/url_request/url_fetcher_delegate.h"
27#include "net/url_request/url_fetcher_response_writer.h"
28#include "net/url_request/url_request_context.h"
29#include "net/url_request/url_request_context_getter.h"
30#include "net/url_request/url_request_throttler_manager.h"
31
32namespace {
33
34const int kBufferSize = 4096;
35const int kUploadProgressTimerInterval = 100;
36bool g_interception_enabled = false;
37bool g_ignore_certificate_requests = false;
38
39void EmptyCompletionCallback(int result) {}
40
41}  // namespace
42
43namespace net {
44
45// URLFetcherCore::Registry ---------------------------------------------------
46
47URLFetcherCore::Registry::Registry() {}
48URLFetcherCore::Registry::~Registry() {}
49
50void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
51  DCHECK(!ContainsKey(fetchers_, core));
52  fetchers_.insert(core);
53}
54
55void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
56  DCHECK(ContainsKey(fetchers_, core));
57  fetchers_.erase(core);
58}
59
60void URLFetcherCore::Registry::CancelAll() {
61  while (!fetchers_.empty())
62    (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
63}
64
65// URLFetcherCore -------------------------------------------------------------
66
67// static
68base::LazyInstance<URLFetcherCore::Registry>
69    URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
70
71URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
72                               const GURL& original_url,
73                               URLFetcher::RequestType request_type,
74                               URLFetcherDelegate* d)
75    : fetcher_(fetcher),
76      original_url_(original_url),
77      request_type_(request_type),
78      delegate_(d),
79      delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
80      load_flags_(LOAD_NORMAL),
81      response_code_(URLFetcher::RESPONSE_CODE_INVALID),
82      buffer_(new IOBuffer(kBufferSize)),
83      url_request_data_key_(NULL),
84      was_fetched_via_proxy_(false),
85      upload_content_set_(false),
86      upload_range_offset_(0),
87      upload_range_length_(0),
88      referrer_policy_(
89          URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
90      is_chunked_upload_(false),
91      was_cancelled_(false),
92      stop_on_redirect_(false),
93      stopped_on_redirect_(false),
94      automatically_retry_on_5xx_(true),
95      num_retries_on_5xx_(0),
96      max_retries_on_5xx_(0),
97      num_retries_on_network_changes_(0),
98      max_retries_on_network_changes_(0),
99      current_upload_bytes_(-1),
100      current_response_bytes_(0),
101      total_response_bytes_(-1) {
102  CHECK(original_url_.is_valid());
103}
104
105void URLFetcherCore::Start() {
106  DCHECK(delegate_task_runner_.get());
107  DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
108  if (network_task_runner_.get()) {
109    DCHECK_EQ(network_task_runner_,
110              request_context_getter_->GetNetworkTaskRunner());
111  } else {
112    network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
113  }
114  DCHECK(network_task_runner_.get()) << "We need an IO task runner";
115
116  network_task_runner_->PostTask(
117      FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
118}
119
120void URLFetcherCore::Stop() {
121  if (delegate_task_runner_.get())  // May be NULL in tests.
122    DCHECK(delegate_task_runner_->BelongsToCurrentThread());
123
124  delegate_ = NULL;
125  fetcher_ = NULL;
126  if (!network_task_runner_.get())
127    return;
128  if (network_task_runner_->RunsTasksOnCurrentThread()) {
129    CancelURLRequest(ERR_ABORTED);
130  } else {
131    network_task_runner_->PostTask(
132        FROM_HERE,
133        base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
134  }
135}
136
137void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
138                                   const std::string& upload_content) {
139  DCHECK(!is_chunked_upload_);
140  DCHECK(!upload_content_set_);
141  DCHECK(upload_content_.empty());
142  DCHECK(upload_file_path_.empty());
143  DCHECK(upload_content_type_.empty());
144
145  // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
146  DCHECK(upload_content.empty() || !upload_content_type.empty());
147
148  upload_content_type_ = upload_content_type;
149  upload_content_ = upload_content;
150  upload_content_set_ = true;
151}
152
153void URLFetcherCore::SetUploadFilePath(
154    const std::string& upload_content_type,
155    const base::FilePath& file_path,
156    uint64 range_offset,
157    uint64 range_length,
158    scoped_refptr<base::TaskRunner> file_task_runner) {
159  DCHECK(!is_chunked_upload_);
160  DCHECK(!upload_content_set_);
161  DCHECK(upload_content_.empty());
162  DCHECK(upload_file_path_.empty());
163  DCHECK_EQ(upload_range_offset_, 0ULL);
164  DCHECK_EQ(upload_range_length_, 0ULL);
165  DCHECK(upload_content_type_.empty());
166  DCHECK(!upload_content_type.empty());
167
168  upload_content_type_ = upload_content_type;
169  upload_file_path_ = file_path;
170  upload_range_offset_ = range_offset;
171  upload_range_length_ = range_length;
172  upload_file_task_runner_ = file_task_runner;
173  upload_content_set_ = true;
174}
175
176void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
177  DCHECK(is_chunked_upload_ ||
178         (upload_content_type_.empty() &&
179          upload_content_.empty()));
180
181  // Empty |content_type| is not allowed here, because it is impossible
182  // to ensure non-empty upload content as it is not yet supplied.
183  DCHECK(!content_type.empty());
184
185  upload_content_type_ = content_type;
186  upload_content_.clear();
187  is_chunked_upload_ = true;
188}
189
190void URLFetcherCore::AppendChunkToUpload(const std::string& content,
191                                         bool is_last_chunk) {
192  DCHECK(delegate_task_runner_.get());
193  DCHECK(network_task_runner_.get());
194  network_task_runner_->PostTask(
195      FROM_HERE,
196      base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
197                 is_last_chunk));
198}
199
200void URLFetcherCore::SetLoadFlags(int load_flags) {
201  load_flags_ = load_flags;
202}
203
204int URLFetcherCore::GetLoadFlags() const {
205  return load_flags_;
206}
207
208void URLFetcherCore::SetReferrer(const std::string& referrer) {
209  referrer_ = referrer;
210}
211
212void URLFetcherCore::SetReferrerPolicy(
213    URLRequest::ReferrerPolicy referrer_policy) {
214  referrer_policy_ = referrer_policy;
215}
216
217void URLFetcherCore::SetExtraRequestHeaders(
218    const std::string& extra_request_headers) {
219  extra_request_headers_.Clear();
220  extra_request_headers_.AddHeadersFromString(extra_request_headers);
221}
222
223void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
224  extra_request_headers_.AddHeaderFromString(header_line);
225}
226
227void URLFetcherCore::SetRequestContext(
228    URLRequestContextGetter* request_context_getter) {
229  DCHECK(!request_context_getter_.get());
230  DCHECK(request_context_getter);
231  request_context_getter_ = request_context_getter;
232}
233
234void URLFetcherCore::SetFirstPartyForCookies(
235    const GURL& first_party_for_cookies) {
236  DCHECK(first_party_for_cookies_.is_empty());
237  first_party_for_cookies_ = first_party_for_cookies;
238}
239
240void URLFetcherCore::SetURLRequestUserData(
241    const void* key,
242    const URLFetcher::CreateDataCallback& create_data_callback) {
243  DCHECK(key);
244  DCHECK(!create_data_callback.is_null());
245  url_request_data_key_ = key;
246  url_request_create_data_callback_ = create_data_callback;
247}
248
249void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
250  stop_on_redirect_ = stop_on_redirect;
251}
252
253void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
254  automatically_retry_on_5xx_ = retry;
255}
256
257void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
258  max_retries_on_5xx_ = max_retries;
259}
260
261int URLFetcherCore::GetMaxRetriesOn5xx() const {
262  return max_retries_on_5xx_;
263}
264
265base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
266  return backoff_delay_;
267}
268
269void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
270  max_retries_on_network_changes_ = max_retries;
271}
272
273void URLFetcherCore::SaveResponseToFileAtPath(
274    const base::FilePath& file_path,
275    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
276  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
277  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
278      new URLFetcherFileWriter(file_task_runner, file_path)));
279}
280
281void URLFetcherCore::SaveResponseToTemporaryFile(
282    scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
283  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
284  SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
285      new URLFetcherFileWriter(file_task_runner, base::FilePath())));
286}
287
288void URLFetcherCore::SaveResponseWithWriter(
289    scoped_ptr<URLFetcherResponseWriter> response_writer) {
290  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
291  response_writer_ = response_writer.Pass();
292}
293
294HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
295  return response_headers_.get();
296}
297
298// TODO(panayiotis): socket_address_ is written in the IO thread,
299// if this is accessed in the UI thread, this could result in a race.
300// Same for response_headers_ above and was_fetched_via_proxy_ below.
301HostPortPair URLFetcherCore::GetSocketAddress() const {
302  return socket_address_;
303}
304
305bool URLFetcherCore::WasFetchedViaProxy() const {
306  return was_fetched_via_proxy_;
307}
308
309const GURL& URLFetcherCore::GetOriginalURL() const {
310  return original_url_;
311}
312
313const GURL& URLFetcherCore::GetURL() const {
314  return url_;
315}
316
317const URLRequestStatus& URLFetcherCore::GetStatus() const {
318  return status_;
319}
320
321int URLFetcherCore::GetResponseCode() const {
322  return response_code_;
323}
324
325const ResponseCookies& URLFetcherCore::GetCookies() const {
326  return cookies_;
327}
328
329void URLFetcherCore::ReceivedContentWasMalformed() {
330  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
331  if (network_task_runner_.get()) {
332    network_task_runner_->PostTask(
333        FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
334  }
335}
336
337bool URLFetcherCore::GetResponseAsString(
338    std::string* out_response_string) const {
339  URLFetcherStringWriter* string_writer =
340      response_writer_ ? response_writer_->AsStringWriter() : NULL;
341  if (!string_writer)
342    return false;
343
344  *out_response_string = string_writer->data();
345  UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
346                          (string_writer->data().length() / 1024));
347  return true;
348}
349
350bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
351                                           base::FilePath* out_response_path) {
352  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
353
354  URLFetcherFileWriter* file_writer =
355      response_writer_ ? response_writer_->AsFileWriter() : NULL;
356  if (!file_writer)
357    return false;
358
359  *out_response_path = file_writer->file_path();
360
361  if (take_ownership) {
362    // Intentionally calling a file_writer_ method directly without posting
363    // the task to network_task_runner_.
364    //
365    // This is for correctly handling the case when file_writer_->DisownFile()
366    // is soon followed by URLFetcherCore::Stop(). We have to make sure that
367    // DisownFile takes effect before Stop deletes file_writer_.
368    //
369    // This direct call should be thread-safe, since DisownFile itself does no
370    // file operation. It just flips the state to be referred in destruction.
371    file_writer->DisownFile();
372  }
373  return true;
374}
375
376void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
377                                        const RedirectInfo& redirect_info,
378                                        bool* defer_redirect) {
379  DCHECK_EQ(request, request_.get());
380  DCHECK(network_task_runner_->BelongsToCurrentThread());
381  if (stop_on_redirect_) {
382    stopped_on_redirect_ = true;
383    url_ = redirect_info.new_url;
384    response_code_ = request_->GetResponseCode();
385    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
386    request->Cancel();
387    OnReadCompleted(request, 0);
388  }
389}
390
391void URLFetcherCore::OnResponseStarted(URLRequest* request) {
392  DCHECK_EQ(request, request_.get());
393  DCHECK(network_task_runner_->BelongsToCurrentThread());
394  if (request_->status().is_success()) {
395    response_code_ = request_->GetResponseCode();
396    response_headers_ = request_->response_headers();
397    socket_address_ = request_->GetSocketAddress();
398    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
399    total_response_bytes_ = request_->GetExpectedContentSize();
400  }
401
402  ReadResponse();
403}
404
405void URLFetcherCore::OnCertificateRequested(
406    URLRequest* request,
407    SSLCertRequestInfo* cert_request_info) {
408  DCHECK_EQ(request, request_.get());
409  DCHECK(network_task_runner_->BelongsToCurrentThread());
410
411  if (g_ignore_certificate_requests) {
412    request->ContinueWithCertificate(NULL);
413  } else {
414    request->Cancel();
415  }
416}
417
418void URLFetcherCore::OnReadCompleted(URLRequest* request,
419                                     int bytes_read) {
420  DCHECK(request == request_);
421  DCHECK(network_task_runner_->BelongsToCurrentThread());
422
423  if (!stopped_on_redirect_)
424    url_ = request->url();
425  URLRequestThrottlerManager* throttler_manager =
426      request->context()->throttler_manager();
427  if (throttler_manager) {
428    url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
429  }
430
431  do {
432    if (!request_->status().is_success() || bytes_read <= 0)
433      break;
434
435    current_response_bytes_ += bytes_read;
436    InformDelegateDownloadProgress();
437
438    const int result =
439        WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
440    if (result < 0) {
441      // Write failed or waiting for write completion.
442      return;
443    }
444  } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
445
446  const URLRequestStatus status = request_->status();
447
448  if (status.is_success())
449    request_->GetResponseCookies(&cookies_);
450
451  // See comments re: HEAD requests in ReadResponse().
452  if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
453    status_ = status;
454    ReleaseRequest();
455
456    // No more data to write.
457    const int result = response_writer_->Finish(
458        base::Bind(&URLFetcherCore::DidFinishWriting, this));
459    if (result != ERR_IO_PENDING)
460      DidFinishWriting(result);
461  }
462}
463
464void URLFetcherCore::CancelAll() {
465  g_registry.Get().CancelAll();
466}
467
468int URLFetcherCore::GetNumFetcherCores() {
469  return g_registry.Get().size();
470}
471
472void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
473  g_interception_enabled = enabled;
474}
475
476void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
477  g_ignore_certificate_requests = ignored;
478}
479
480URLFetcherCore::~URLFetcherCore() {
481  // |request_| should be NULL. If not, it's unsafe to delete it here since we
482  // may not be on the IO thread.
483  DCHECK(!request_.get());
484}
485
486void URLFetcherCore::StartOnIOThread() {
487  DCHECK(network_task_runner_->BelongsToCurrentThread());
488
489  if (!response_writer_)
490    response_writer_.reset(new URLFetcherStringWriter);
491
492  const int result = response_writer_->Initialize(
493      base::Bind(&URLFetcherCore::DidInitializeWriter, this));
494  if (result != ERR_IO_PENDING)
495    DidInitializeWriter(result);
496}
497
498void URLFetcherCore::StartURLRequest() {
499  DCHECK(network_task_runner_->BelongsToCurrentThread());
500
501  if (was_cancelled_) {
502    // Since StartURLRequest() is posted as a *delayed* task, it may
503    // run after the URLFetcher was already stopped.
504    return;
505  }
506
507  DCHECK(request_context_getter_.get());
508  DCHECK(!request_.get());
509
510  g_registry.Get().AddURLFetcherCore(this);
511  current_response_bytes_ = 0;
512  request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
513      original_url_, DEFAULT_PRIORITY, this, NULL);
514  request_->set_stack_trace(stack_trace_);
515  int flags = request_->load_flags() | load_flags_;
516  if (!g_interception_enabled)
517    flags = flags | LOAD_DISABLE_INTERCEPT;
518
519  if (is_chunked_upload_)
520    request_->EnableChunkedUpload();
521  request_->SetLoadFlags(flags);
522  request_->SetReferrer(referrer_);
523  request_->set_referrer_policy(referrer_policy_);
524  request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
525      original_url_ : first_party_for_cookies_);
526  if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
527    request_->SetUserData(url_request_data_key_,
528                          url_request_create_data_callback_.Run());
529  }
530
531  switch (request_type_) {
532    case URLFetcher::GET:
533      break;
534
535    case URLFetcher::POST:
536    case URLFetcher::PUT:
537    case URLFetcher::PATCH:
538      // Upload content must be set.
539      DCHECK(is_chunked_upload_ || upload_content_set_);
540
541      request_->set_method(
542          request_type_ == URLFetcher::POST ? "POST" :
543          request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
544      if (!upload_content_type_.empty()) {
545        extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
546                                         upload_content_type_);
547      }
548      if (!upload_content_.empty()) {
549        scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
550            upload_content_.data(), upload_content_.size()));
551        request_->set_upload(make_scoped_ptr(
552            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
553      } else if (!upload_file_path_.empty()) {
554        scoped_ptr<UploadElementReader> reader(
555            new UploadFileElementReader(upload_file_task_runner_.get(),
556                                        upload_file_path_,
557                                        upload_range_offset_,
558                                        upload_range_length_,
559                                        base::Time()));
560        request_->set_upload(make_scoped_ptr(
561            UploadDataStream::CreateWithReader(reader.Pass(), 0)));
562      }
563
564      current_upload_bytes_ = -1;
565      // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
566      //  layer and avoid using timer here.
567      upload_progress_checker_timer_.reset(
568          new base::RepeatingTimer<URLFetcherCore>());
569      upload_progress_checker_timer_->Start(
570          FROM_HERE,
571          base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
572          this,
573          &URLFetcherCore::InformDelegateUploadProgress);
574      break;
575
576    case URLFetcher::HEAD:
577      request_->set_method("HEAD");
578      break;
579
580    case URLFetcher::DELETE_REQUEST:
581      request_->set_method("DELETE");
582      break;
583
584    default:
585      NOTREACHED();
586  }
587
588  if (!extra_request_headers_.IsEmpty())
589    request_->SetExtraRequestHeaders(extra_request_headers_);
590
591  request_->Start();
592}
593
594void URLFetcherCore::DidInitializeWriter(int result) {
595  if (result != OK) {
596    CancelURLRequest(result);
597    delegate_task_runner_->PostTask(
598        FROM_HERE,
599        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
600    return;
601  }
602  StartURLRequestWhenAppropriate();
603}
604
605void URLFetcherCore::StartURLRequestWhenAppropriate() {
606  DCHECK(network_task_runner_->BelongsToCurrentThread());
607
608  if (was_cancelled_)
609    return;
610
611  DCHECK(request_context_getter_.get());
612
613  int64 delay = INT64_C(0);
614  if (!original_url_throttler_entry_.get()) {
615    URLRequestThrottlerManager* manager =
616        request_context_getter_->GetURLRequestContext()->throttler_manager();
617    if (manager) {
618      original_url_throttler_entry_ =
619          manager->RegisterRequestUrl(original_url_);
620    }
621  }
622  if (original_url_throttler_entry_.get()) {
623    delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
624        GetBackoffReleaseTime());
625  }
626
627  if (delay == INT64_C(0)) {
628    StartURLRequest();
629  } else {
630    base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
631        FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
632        base::TimeDelta::FromMilliseconds(delay));
633  }
634}
635
636void URLFetcherCore::CancelURLRequest(int error) {
637  DCHECK(network_task_runner_->BelongsToCurrentThread());
638
639  if (request_.get()) {
640    request_->CancelWithError(error);
641    ReleaseRequest();
642  }
643
644  // Set the error manually.
645  // Normally, calling URLRequest::CancelWithError() results in calling
646  // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
647  // URLRequestJob::NotifyDone(). But, because the request was released
648  // immediately after being canceled, the request could not call
649  // OnReadCompleted() which overwrites |status_| with the error status.
650  status_.set_status(URLRequestStatus::CANCELED);
651  status_.set_error(error);
652
653  // Release the reference to the request context. There could be multiple
654  // references to URLFetcher::Core at this point so it may take a while to
655  // delete the object, but we cannot delay the destruction of the request
656  // context.
657  request_context_getter_ = NULL;
658  first_party_for_cookies_ = GURL();
659  url_request_data_key_ = NULL;
660  url_request_create_data_callback_.Reset();
661  was_cancelled_ = true;
662}
663
664void URLFetcherCore::OnCompletedURLRequest(
665    base::TimeDelta backoff_delay) {
666  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
667
668  // Save the status and backoff_delay so that delegates can read it.
669  if (delegate_) {
670    backoff_delay_ = backoff_delay;
671    InformDelegateFetchIsComplete();
672  }
673}
674
675void URLFetcherCore::InformDelegateFetchIsComplete() {
676  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
677  if (delegate_)
678    delegate_->OnURLFetchComplete(fetcher_);
679}
680
681void URLFetcherCore::NotifyMalformedContent() {
682  DCHECK(network_task_runner_->BelongsToCurrentThread());
683  if (url_throttler_entry_.get()) {
684    int status_code = response_code_;
685    if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
686      // The status code will generally be known by the time clients
687      // call the |ReceivedContentWasMalformed()| function (which ends up
688      // calling the current function) but if it's not, we need to assume
689      // the response was successful so that the total failure count
690      // used to calculate exponential back-off goes up.
691      status_code = 200;
692    }
693    url_throttler_entry_->ReceivedContentWasMalformed(status_code);
694  }
695}
696
697void URLFetcherCore::DidFinishWriting(int result) {
698  if (result != OK) {
699    CancelURLRequest(result);
700    delegate_task_runner_->PostTask(
701        FROM_HERE,
702        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
703    return;
704  }
705  // If the file was successfully closed, then the URL request is complete.
706  RetryOrCompleteUrlFetch();
707}
708
709void URLFetcherCore::RetryOrCompleteUrlFetch() {
710  DCHECK(network_task_runner_->BelongsToCurrentThread());
711  base::TimeDelta backoff_delay;
712
713  // Checks the response from server.
714  if (response_code_ >= 500 ||
715      status_.error() == ERR_TEMPORARILY_THROTTLED) {
716    // When encountering a server error, we will send the request again
717    // after backoff time.
718    ++num_retries_on_5xx_;
719
720    // Note that backoff_delay may be 0 because (a) the
721    // URLRequestThrottlerManager and related code does not
722    // necessarily back off on the first error, (b) it only backs off
723    // on some of the 5xx status codes, (c) not all URLRequestContexts
724    // have a throttler manager.
725    base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
726    backoff_delay = backoff_release_time - base::TimeTicks::Now();
727    if (backoff_delay < base::TimeDelta())
728      backoff_delay = base::TimeDelta();
729
730    if (automatically_retry_on_5xx_ &&
731        num_retries_on_5xx_ <= max_retries_on_5xx_) {
732      StartOnIOThread();
733      return;
734    }
735  } else {
736    backoff_delay = base::TimeDelta();
737  }
738
739  // Retry if the request failed due to network changes.
740  if (status_.error() == ERR_NETWORK_CHANGED &&
741      num_retries_on_network_changes_ < max_retries_on_network_changes_) {
742    ++num_retries_on_network_changes_;
743
744    // Retry soon, after flushing all the current tasks which may include
745    // further network change observers.
746    network_task_runner_->PostTask(
747        FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
748    return;
749  }
750
751  request_context_getter_ = NULL;
752  first_party_for_cookies_ = GURL();
753  url_request_data_key_ = NULL;
754  url_request_create_data_callback_.Reset();
755  bool posted = delegate_task_runner_->PostTask(
756      FROM_HERE,
757      base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
758
759  // If the delegate message loop does not exist any more, then the delegate
760  // should be gone too.
761  DCHECK(posted || !delegate_);
762}
763
764void URLFetcherCore::ReleaseRequest() {
765  upload_progress_checker_timer_.reset();
766  request_.reset();
767  g_registry.Get().RemoveURLFetcherCore(this);
768}
769
770base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
771  DCHECK(network_task_runner_->BelongsToCurrentThread());
772
773  if (!original_url_throttler_entry_.get())
774    return base::TimeTicks();
775
776  base::TimeTicks original_url_backoff =
777      original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
778  base::TimeTicks destination_url_backoff;
779  if (url_throttler_entry_.get() &&
780      original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
781    destination_url_backoff =
782        url_throttler_entry_->GetExponentialBackoffReleaseTime();
783  }
784
785  return original_url_backoff > destination_url_backoff ?
786      original_url_backoff : destination_url_backoff;
787}
788
789void URLFetcherCore::CompleteAddingUploadDataChunk(
790    const std::string& content, bool is_last_chunk) {
791  if (was_cancelled_) {
792    // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
793    // may run after the URLFetcher was already stopped.
794    return;
795  }
796  DCHECK(is_chunked_upload_);
797  DCHECK(request_.get());
798  DCHECK(!content.empty());
799  request_->AppendChunkToUpload(content.data(),
800                                static_cast<int>(content.length()),
801                                is_last_chunk);
802}
803
804int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
805  while (data->BytesRemaining() > 0) {
806    const int result = response_writer_->Write(
807        data.get(),
808        data->BytesRemaining(),
809        base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
810    if (result < 0) {
811      if (result != ERR_IO_PENDING)
812        DidWriteBuffer(data, result);
813      return result;
814    }
815    data->DidConsume(result);
816  }
817  return OK;
818}
819
820void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
821                                    int result) {
822  if (result < 0) {  // Handle errors.
823    CancelURLRequest(result);
824    response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
825    delegate_task_runner_->PostTask(
826        FROM_HERE,
827        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
828    return;
829  }
830
831  // Continue writing.
832  data->DidConsume(result);
833  if (WriteBuffer(data) < 0)
834    return;
835
836  // Finished writing buffer_. Read some more, unless the request has been
837  // cancelled and deleted.
838  DCHECK_EQ(0, data->BytesRemaining());
839  if (request_.get())
840    ReadResponse();
841}
842
843void URLFetcherCore::ReadResponse() {
844  // Some servers may treat HEAD requests as GET requests. To free up the
845  // network connection as soon as possible, signal that the request has
846  // completed immediately, without trying to read any data back (all we care
847  // about is the response code and headers, which we already have).
848  int bytes_read = 0;
849  if (request_->status().is_success() &&
850      (request_type_ != URLFetcher::HEAD)) {
851    if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
852      bytes_read = -1;  // Match OnReadCompleted() interface contract.
853  }
854  OnReadCompleted(request_.get(), bytes_read);
855}
856
857void URLFetcherCore::InformDelegateUploadProgress() {
858  DCHECK(network_task_runner_->BelongsToCurrentThread());
859  if (request_.get()) {
860    int64 current = request_->GetUploadProgress().position();
861    if (current_upload_bytes_ != current) {
862      current_upload_bytes_ = current;
863      int64 total = -1;
864      if (!is_chunked_upload_) {
865        total = static_cast<int64>(request_->GetUploadProgress().size());
866        // Total may be zero if the UploadDataStream::Init has not been called
867        // yet. Don't send the upload progress until the size is initialized.
868        if (!total)
869          return;
870      }
871      delegate_task_runner_->PostTask(
872          FROM_HERE,
873          base::Bind(
874              &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
875              this, current, total));
876    }
877  }
878}
879
880void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
881    int64 current, int64 total) {
882  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
883  if (delegate_)
884    delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
885}
886
887void URLFetcherCore::InformDelegateDownloadProgress() {
888  DCHECK(network_task_runner_->BelongsToCurrentThread());
889  delegate_task_runner_->PostTask(
890      FROM_HERE,
891      base::Bind(
892          &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
893          this, current_response_bytes_, total_response_bytes_));
894}
895
896void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
897    int64 current, int64 total) {
898  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
899  if (delegate_)
900    delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
901}
902
903}  // namespace net
904