url_fetcher_core.cc revision 5821806d5e7f356e8fa4b058a389a808ea183019 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_fetcher_core.h"
6
7#include "base/bind.h"
8#include "base/file_util_proxy.h"
9#include "base/logging.h"
10#include "base/single_thread_task_runner.h"
11#include "base/metrics/histogram.h"
12#include "base/stl_util.h"
13#include "base/thread_task_runner_handle.h"
14#include "base/tracked_objects.h"
15#include "net/base/io_buffer.h"
16#include "net/base/load_flags.h"
17#include "net/base/net_errors.h"
18#include "net/http/http_response_headers.h"
19#include "net/url_request/url_fetcher_delegate.h"
20#include "net/url_request/url_request_context.h"
21#include "net/url_request/url_request_context_getter.h"
22#include "net/url_request/url_request_throttler_manager.h"
23
24namespace {
25
26const int kBufferSize = 4096;
27const int kUploadProgressTimerInterval = 100;
28bool g_interception_enabled = false;
29
30}  // namespace
31
32namespace net {
33
34// URLFetcherCore::Registry ---------------------------------------------------
35
36URLFetcherCore::Registry::Registry() {}
37URLFetcherCore::Registry::~Registry() {}
38
39void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
40  DCHECK(!ContainsKey(fetchers_, core));
41  fetchers_.insert(core);
42}
43
44void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
45  DCHECK(ContainsKey(fetchers_, core));
46  fetchers_.erase(core);
47}
48
49void URLFetcherCore::Registry::CancelAll() {
50  while (!fetchers_.empty())
51    (*fetchers_.begin())->CancelURLRequest();
52}
53
54
55// URLFetcherCore::FileWriter -------------------------------------------------
56
57URLFetcherCore::FileWriter::FileWriter(
58    URLFetcherCore* core,
59    scoped_refptr<base::TaskRunner> file_task_runner)
60    : core_(core),
61      error_code_(base::PLATFORM_FILE_OK),
62      ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)),
63      file_task_runner_(file_task_runner),
64      file_handle_(base::kInvalidPlatformFileValue) {
65}
66
67URLFetcherCore::FileWriter::~FileWriter() {
68  CloseAndDeleteFile();
69}
70
71void URLFetcherCore::FileWriter::CreateFileAtPath(
72    const FilePath& file_path) {
73  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
74  DCHECK(file_task_runner_.get());
75  base::FileUtilProxy::CreateOrOpen(
76      file_task_runner_,
77      file_path,
78      base::PLATFORM_FILE_CREATE_ALWAYS | base::PLATFORM_FILE_WRITE,
79      base::Bind(&URLFetcherCore::FileWriter::DidCreateFile,
80                 weak_factory_.GetWeakPtr(),
81                 file_path));
82}
83
84void URLFetcherCore::FileWriter::CreateTempFile() {
85  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
86  DCHECK(file_task_runner_.get());
87  base::FileUtilProxy::CreateTemporary(
88      file_task_runner_,
89      0,  // No additional file flags.
90      base::Bind(&URLFetcherCore::FileWriter::DidCreateTempFile,
91                 weak_factory_.GetWeakPtr()));
92}
93
94void URLFetcherCore::FileWriter::WriteBuffer(int num_bytes) {
95  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
96
97  // Start writing to the file by setting the initial state
98  // of |pending_bytes_| and |buffer_offset_| to indicate that the
99  // entire buffer has not yet been written.
100  pending_bytes_ = num_bytes;
101  buffer_offset_ = 0;
102  ContinueWrite(base::PLATFORM_FILE_OK, 0);
103}
104
105void URLFetcherCore::FileWriter::ContinueWrite(
106    base::PlatformFileError error_code,
107    int bytes_written) {
108  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
109
110  if (file_handle_ == base::kInvalidPlatformFileValue) {
111    // While a write was being done on the file thread, a request
112    // to close or disown the file occured on the IO thread.  At
113    // this point a request to close the file is pending on the
114    // file thread.
115    return;
116  }
117
118  // Every code path that resets |core_->request_| should reset
119  // |core->file_writer_| or cause the file writer to disown the file.  In the
120  // former case, this callback can not be called, because the weak pointer to
121  // |this| will be NULL. In the latter case, the check of |file_handle_| at the
122  // start of this method ensures that we can not reach this point.
123  CHECK(core_->request_.get());
124
125  if (base::PLATFORM_FILE_OK != error_code) {
126    error_code_ = error_code;
127    CloseAndDeleteFile();
128    core_->delegate_task_runner_->PostTask(
129        FROM_HERE,
130        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, core_));
131    return;
132  }
133
134  total_bytes_written_ += bytes_written;
135  buffer_offset_ += bytes_written;
136  pending_bytes_ -= bytes_written;
137
138  if (pending_bytes_ > 0) {
139    base::FileUtilProxy::Write(
140        file_task_runner_, file_handle_,
141        total_bytes_written_,  // Append to the end
142        (core_->buffer_->data() + buffer_offset_), pending_bytes_,
143        base::Bind(&URLFetcherCore::FileWriter::ContinueWrite,
144                   weak_factory_.GetWeakPtr()));
145  } else {
146    // Finished writing core_->buffer_ to the file. Read some more.
147    core_->ReadResponse();
148  }
149}
150
151void URLFetcherCore::FileWriter::DisownFile() {
152  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
153
154  // Disowning is done by the delegate's OnURLFetchComplete method.
155  // The file should be closed by the time that method is called.
156  DCHECK(file_handle_ == base::kInvalidPlatformFileValue);
157
158  // Forget about any file by reseting the path.
159  file_path_.clear();
160}
161
162void URLFetcherCore::FileWriter::CloseFileAndCompleteRequest() {
163  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
164
165  if (file_handle_ != base::kInvalidPlatformFileValue) {
166    base::FileUtilProxy::Close(
167        file_task_runner_, file_handle_,
168        base::Bind(&URLFetcherCore::FileWriter::DidCloseFile,
169                   weak_factory_.GetWeakPtr()));
170    file_handle_ = base::kInvalidPlatformFileValue;
171  }
172}
173
174void URLFetcherCore::FileWriter::CloseAndDeleteFile() {
175  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
176
177  if (file_handle_ == base::kInvalidPlatformFileValue) {
178    DeleteFile(base::PLATFORM_FILE_OK);
179    return;
180  }
181  // Close the file if it is open.
182  base::FileUtilProxy::Close(
183      file_task_runner_, file_handle_,
184      base::Bind(&URLFetcherCore::FileWriter::DeleteFile,
185                 weak_factory_.GetWeakPtr()));
186  file_handle_ = base::kInvalidPlatformFileValue;
187}
188
189void URLFetcherCore::FileWriter::DeleteFile(
190    base::PlatformFileError error_code) {
191  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
192  if (file_path_.empty())
193    return;
194
195  base::FileUtilProxy::Delete(
196      file_task_runner_, file_path_,
197      false,  // No need to recurse, as the path is to a file.
198      base::FileUtilProxy::StatusCallback());
199  DisownFile();
200}
201
202void URLFetcherCore::FileWriter::DidCreateFile(
203    const FilePath& file_path,
204    base::PlatformFileError error_code,
205    base::PassPlatformFile file_handle,
206    bool created) {
207  DidCreateFileInternal(file_path, error_code, file_handle);
208}
209
210void URLFetcherCore::FileWriter::DidCreateTempFile(
211    base::PlatformFileError error_code,
212    base::PassPlatformFile file_handle,
213    const FilePath& file_path) {
214  DidCreateFileInternal(file_path, error_code, file_handle);
215}
216
217void URLFetcherCore::FileWriter::DidCreateFileInternal(
218    const FilePath& file_path,
219    base::PlatformFileError error_code,
220    base::PassPlatformFile file_handle) {
221  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
222
223  if (base::PLATFORM_FILE_OK != error_code) {
224    error_code_ = error_code;
225    CloseAndDeleteFile();
226    core_->delegate_task_runner_->PostTask(
227        FROM_HERE,
228        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, core_));
229    return;
230  }
231
232  file_path_ = file_path;
233  file_handle_ = file_handle.ReleaseValue();
234  total_bytes_written_ = 0;
235
236  core_->network_task_runner_->PostTask(
237      FROM_HERE,
238      base::Bind(&URLFetcherCore::StartURLRequestWhenAppropriate, core_));
239}
240
241void URLFetcherCore::FileWriter::DidCloseFile(
242    base::PlatformFileError error_code) {
243  DCHECK(core_->network_task_runner_->BelongsToCurrentThread());
244
245  if (base::PLATFORM_FILE_OK != error_code) {
246    error_code_ = error_code;
247    CloseAndDeleteFile();
248    core_->delegate_task_runner_->PostTask(
249        FROM_HERE,
250        base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, core_));
251    return;
252  }
253
254  // If the file was successfully closed, then the URL request is complete.
255  core_->RetryOrCompleteUrlFetch();
256}
257
258
259// URLFetcherCore -------------------------------------------------------------
260
261// static
262base::LazyInstance<URLFetcherCore::Registry>
263    URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
264
265URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
266                               const GURL& original_url,
267                               URLFetcher::RequestType request_type,
268                               URLFetcherDelegate* d)
269    : fetcher_(fetcher),
270      original_url_(original_url),
271      request_type_(request_type),
272      delegate_(d),
273      delegate_task_runner_(
274          base::ThreadTaskRunnerHandle::Get()),
275      request_(NULL),
276      load_flags_(LOAD_NORMAL),
277      response_code_(URLFetcher::RESPONSE_CODE_INVALID),
278      buffer_(new IOBuffer(kBufferSize)),
279      url_request_data_key_(NULL),
280      was_fetched_via_proxy_(false),
281      is_chunked_upload_(false),
282      num_retries_(0),
283      was_cancelled_(false),
284      response_destination_(STRING),
285      stop_on_redirect_(false),
286      stopped_on_redirect_(false),
287      automatically_retry_on_5xx_(true),
288      max_retries_(0),
289      current_upload_bytes_(-1),
290      current_response_bytes_(0),
291      total_response_bytes_(-1) {
292  CHECK(original_url_.is_valid());
293}
294
295void URLFetcherCore::Start() {
296  DCHECK(delegate_task_runner_);
297  DCHECK(request_context_getter_) << "We need an URLRequestContext!";
298  if (network_task_runner_) {
299    DCHECK_EQ(network_task_runner_,
300              request_context_getter_->GetNetworkTaskRunner());
301  } else {
302    network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
303  }
304  DCHECK(network_task_runner_.get()) << "We need an IO task runner";
305
306  network_task_runner_->PostTask(
307      FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
308}
309
310void URLFetcherCore::Stop() {
311  if (delegate_task_runner_)  // May be NULL in tests.
312    DCHECK(delegate_task_runner_->BelongsToCurrentThread());
313
314  delegate_ = NULL;
315  fetcher_ = NULL;
316  if (!network_task_runner_.get())
317    return;
318  if (network_task_runner_->RunsTasksOnCurrentThread()) {
319    CancelURLRequest();
320  } else {
321    network_task_runner_->PostTask(
322        FROM_HERE, base::Bind(&URLFetcherCore::CancelURLRequest, this));
323  }
324}
325
326void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
327                                   const std::string& upload_content) {
328  DCHECK(!is_chunked_upload_);
329  upload_content_type_ = upload_content_type;
330  upload_content_ = upload_content;
331}
332
333void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
334  DCHECK(is_chunked_upload_ ||
335         (upload_content_type_.empty() &&
336          upload_content_.empty()));
337  upload_content_type_ = content_type;
338  upload_content_.clear();
339  is_chunked_upload_ = true;
340}
341
342void URLFetcherCore::AppendChunkToUpload(const std::string& content,
343                                         bool is_last_chunk) {
344  DCHECK(delegate_task_runner_);
345  DCHECK(network_task_runner_.get());
346  network_task_runner_->PostTask(
347      FROM_HERE,
348      base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
349                 is_last_chunk));
350}
351
352void URLFetcherCore::SetLoadFlags(int load_flags) {
353  load_flags_ = load_flags;
354}
355
356int URLFetcherCore::GetLoadFlags() const {
357  return load_flags_;
358}
359
360void URLFetcherCore::SetReferrer(const std::string& referrer) {
361  referrer_ = referrer;
362}
363
364void URLFetcherCore::SetExtraRequestHeaders(
365    const std::string& extra_request_headers) {
366  extra_request_headers_.Clear();
367  extra_request_headers_.AddHeadersFromString(extra_request_headers);
368}
369
370void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
371  extra_request_headers_.AddHeaderFromString(header_line);
372}
373
374void URLFetcherCore::GetExtraRequestHeaders(
375    HttpRequestHeaders* headers) const {
376  headers->CopyFrom(extra_request_headers_);
377}
378
379void URLFetcherCore::SetRequestContext(
380    URLRequestContextGetter* request_context_getter) {
381  DCHECK(!request_context_getter_);
382  DCHECK(request_context_getter);
383  request_context_getter_ = request_context_getter;
384}
385
386void URLFetcherCore::SetFirstPartyForCookies(
387    const GURL& first_party_for_cookies) {
388  DCHECK(first_party_for_cookies_.is_empty());
389  first_party_for_cookies_ = first_party_for_cookies;
390}
391
392void URLFetcherCore::SetURLRequestUserData(
393    const void* key,
394    const URLFetcher::CreateDataCallback& create_data_callback) {
395  DCHECK(key);
396  DCHECK(!create_data_callback.is_null());
397  url_request_data_key_ = key;
398  url_request_create_data_callback_ = create_data_callback;
399}
400
401void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
402  stop_on_redirect_ = stop_on_redirect;
403}
404
405void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
406  automatically_retry_on_5xx_ = retry;
407}
408
409void URLFetcherCore::SetMaxRetries(int max_retries) {
410  max_retries_ = max_retries;
411}
412
413int URLFetcherCore::GetMaxRetries() const {
414  return max_retries_;
415}
416
417base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
418  return backoff_delay_;
419}
420
421void URLFetcherCore::SaveResponseToFileAtPath(
422    const FilePath& file_path,
423    scoped_refptr<base::TaskRunner> file_task_runner) {
424  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
425  file_task_runner_ = file_task_runner;
426  response_destination_ = URLFetcherCore::PERMANENT_FILE;
427  response_destination_file_path_ = file_path;
428}
429
430void URLFetcherCore::SaveResponseToTemporaryFile(
431    scoped_refptr<base::TaskRunner> file_task_runner) {
432  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
433  file_task_runner_ = file_task_runner;
434  response_destination_ = URLFetcherCore::TEMP_FILE;
435}
436
437HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
438  return response_headers_;
439}
440
441// TODO(panayiotis): socket_address_ is written in the IO thread,
442// if this is accessed in the UI thread, this could result in a race.
443// Same for response_headers_ above and was_fetched_via_proxy_ below.
444HostPortPair URLFetcherCore::GetSocketAddress() const {
445  return socket_address_;
446}
447
448bool URLFetcherCore::WasFetchedViaProxy() const {
449  return was_fetched_via_proxy_;
450}
451
452const GURL& URLFetcherCore::GetOriginalURL() const {
453  return original_url_;
454}
455
456const GURL& URLFetcherCore::GetURL() const {
457  return url_;
458}
459
460const URLRequestStatus& URLFetcherCore::GetStatus() const {
461  return status_;
462}
463
464int URLFetcherCore::GetResponseCode() const {
465  return response_code_;
466}
467
468const ResponseCookies& URLFetcherCore::GetCookies() const {
469  return cookies_;
470}
471
472bool URLFetcherCore::FileErrorOccurred(
473    base::PlatformFileError* out_error_code) const {
474
475  // Can't have a file error if no file is being created or written to.
476  if (!file_writer_.get())
477    return false;
478
479  base::PlatformFileError error_code = file_writer_->error_code();
480  if (error_code == base::PLATFORM_FILE_OK)
481    return false;
482
483  *out_error_code = error_code;
484  return true;
485}
486
487void URLFetcherCore::ReceivedContentWasMalformed() {
488  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
489  if (network_task_runner_.get()) {
490    network_task_runner_->PostTask(
491        FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
492  }
493}
494
495bool URLFetcherCore::GetResponseAsString(
496    std::string* out_response_string) const {
497  if (response_destination_ != URLFetcherCore::STRING)
498    return false;
499
500  *out_response_string = data_;
501  UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
502                          (data_.length() / 1024));
503
504  return true;
505}
506
507bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
508                                           FilePath* out_response_path) {
509  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
510  const bool destination_is_file =
511      response_destination_ == URLFetcherCore::TEMP_FILE ||
512      response_destination_ == URLFetcherCore::PERMANENT_FILE;
513  if (!destination_is_file || !file_writer_.get())
514    return false;
515
516  *out_response_path = file_writer_->file_path();
517
518  if (take_ownership) {
519    network_task_runner_->PostTask(
520        FROM_HERE,
521        base::Bind(&URLFetcherCore::DisownFile, this));
522  }
523  return true;
524}
525
526void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
527                                        const GURL& new_url,
528                                        bool* defer_redirect) {
529  DCHECK_EQ(request, request_.get());
530  DCHECK(network_task_runner_->BelongsToCurrentThread());
531  if (stop_on_redirect_) {
532    stopped_on_redirect_ = true;
533    url_ = new_url;
534    response_code_ = request_->GetResponseCode();
535    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
536    request->Cancel();
537    OnReadCompleted(request, 0);
538  }
539}
540
541void URLFetcherCore::OnResponseStarted(URLRequest* request) {
542  DCHECK_EQ(request, request_.get());
543  DCHECK(network_task_runner_->BelongsToCurrentThread());
544  if (request_->status().is_success()) {
545    response_code_ = request_->GetResponseCode();
546    response_headers_ = request_->response_headers();
547    socket_address_ = request_->GetSocketAddress();
548    was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
549    total_response_bytes_ = request_->GetExpectedContentSize();
550  }
551
552  ReadResponse();
553}
554
555void URLFetcherCore::OnReadCompleted(URLRequest* request,
556                                     int bytes_read) {
557  DCHECK(request == request_);
558  DCHECK(network_task_runner_->BelongsToCurrentThread());
559
560  if (!stopped_on_redirect_)
561    url_ = request->url();
562  URLRequestThrottlerManager* throttler_manager =
563      request->context()->throttler_manager();
564  if (throttler_manager) {
565    url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
566  }
567
568  bool waiting_on_write = false;
569  do {
570    if (!request_->status().is_success() || bytes_read <= 0)
571      break;
572
573    current_response_bytes_ += bytes_read;
574    InformDelegateDownloadProgress();
575    InformDelegateDownloadDataIfNecessary(bytes_read);
576
577    if (!WriteBuffer(bytes_read)) {
578      // If WriteBuffer() returns false, we have a pending write to
579      // wait on before reading further.
580      waiting_on_write = true;
581      break;
582    }
583  } while (request_->Read(buffer_, kBufferSize, &bytes_read));
584
585  const URLRequestStatus status = request_->status();
586
587  if (status.is_success())
588    request_->GetResponseCookies(&cookies_);
589
590  // See comments re: HEAD requests in ReadResponse().
591  if ((!status.is_io_pending() && !waiting_on_write) ||
592      (request_type_ == URLFetcher::HEAD)) {
593    status_ = status;
594    ReleaseRequest();
595
596    // If a file is open, close it.
597    if (file_writer_.get()) {
598      // If the file is open, close it.  After closing the file,
599      // RetryOrCompleteUrlFetch() will be called.
600      file_writer_->CloseFileAndCompleteRequest();
601    } else {
602      // Otherwise, complete or retry the URL request directly.
603      RetryOrCompleteUrlFetch();
604    }
605  }
606}
607
608void URLFetcherCore::CancelAll() {
609  g_registry.Get().CancelAll();
610}
611
612int URLFetcherCore::GetNumFetcherCores() {
613  return g_registry.Get().size();
614}
615
616void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
617  g_interception_enabled = enabled;
618}
619
620URLFetcherCore::~URLFetcherCore() {
621  // |request_| should be NULL.  If not, it's unsafe to delete it here since we
622  // may not be on the IO thread.
623  DCHECK(!request_.get());
624}
625
626void URLFetcherCore::StartOnIOThread() {
627  DCHECK(network_task_runner_->BelongsToCurrentThread());
628
629  switch (response_destination_) {
630    case STRING:
631      StartURLRequestWhenAppropriate();
632      break;
633
634    case PERMANENT_FILE:
635    case TEMP_FILE:
636      DCHECK(file_task_runner_.get())
637          << "Need to set the file task runner.";
638
639      file_writer_.reset(new FileWriter(this, file_task_runner_));
640
641      // If the file is successfully created,
642      // URLFetcherCore::StartURLRequestWhenAppropriate() will be called.
643      switch (response_destination_) {
644        case PERMANENT_FILE:
645          file_writer_->CreateFileAtPath(response_destination_file_path_);
646          break;
647        case TEMP_FILE:
648          file_writer_->CreateTempFile();
649          break;
650        default:
651          NOTREACHED();
652      }
653      break;
654
655    default:
656      NOTREACHED();
657  }
658}
659
660void URLFetcherCore::StartURLRequest() {
661  DCHECK(network_task_runner_->BelongsToCurrentThread());
662
663  if (was_cancelled_) {
664    // Since StartURLRequest() is posted as a *delayed* task, it may
665    // run after the URLFetcher was already stopped.
666    return;
667  }
668
669  DCHECK(request_context_getter_);
670  DCHECK(!request_.get());
671
672  g_registry.Get().AddURLFetcherCore(this);
673  current_response_bytes_ = 0;
674  request_.reset(request_context_getter_->GetURLRequestContext()->CreateRequest(
675      original_url_, this));
676  request_->set_stack_trace(stack_trace_);
677  int flags = request_->load_flags() | load_flags_;
678  if (!g_interception_enabled)
679    flags = flags | LOAD_DISABLE_INTERCEPT;
680
681  if (is_chunked_upload_)
682    request_->EnableChunkedUpload();
683  request_->set_load_flags(flags);
684  request_->set_referrer(referrer_);
685  request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
686      original_url_ : first_party_for_cookies_);
687  if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
688    request_->SetUserData(url_request_data_key_,
689                          url_request_create_data_callback_.Run());
690  }
691
692  switch (request_type_) {
693    case URLFetcher::GET:
694      break;
695
696    case URLFetcher::POST:
697    case URLFetcher::PUT:
698      DCHECK(!upload_content_type_.empty());
699
700      request_->set_method(
701          request_type_ == URLFetcher::POST ? "POST" : "PUT");
702      extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
703                                       upload_content_type_);
704      if (!upload_content_.empty()) {
705        request_->AppendBytesToUpload(
706            upload_content_.data(), static_cast<int>(upload_content_.length()));
707      }
708
709      current_upload_bytes_ = -1;
710      // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
711      //  layer and avoid using timer here.
712      upload_progress_checker_timer_.reset(
713          new base::RepeatingTimer<URLFetcherCore>());
714      upload_progress_checker_timer_->Start(
715          FROM_HERE,
716          base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
717          this,
718          &URLFetcherCore::InformDelegateUploadProgress);
719      break;
720
721    case URLFetcher::HEAD:
722      request_->set_method("HEAD");
723      break;
724
725    case URLFetcher::DELETE_REQUEST:
726      request_->set_method("DELETE");
727      break;
728
729    default:
730      NOTREACHED();
731  }
732
733  if (!extra_request_headers_.IsEmpty())
734    request_->SetExtraRequestHeaders(extra_request_headers_);
735
736  // There might be data left over from a previous request attempt.
737  data_.clear();
738
739  // If we are writing the response to a file, the only caller
740  // of this function should have created it and not written yet.
741  DCHECK(!file_writer_.get() || file_writer_->total_bytes_written() == 0);
742
743  request_->Start();
744}
745
746void URLFetcherCore::StartURLRequestWhenAppropriate() {
747  DCHECK(network_task_runner_->BelongsToCurrentThread());
748
749  if (was_cancelled_)
750    return;
751
752  DCHECK(request_context_getter_);
753
754  int64 delay = 0LL;
755  if (original_url_throttler_entry_ == NULL) {
756    URLRequestThrottlerManager* manager =
757        request_context_getter_->GetURLRequestContext()->throttler_manager();
758    if (manager) {
759      original_url_throttler_entry_ =
760          manager->RegisterRequestUrl(original_url_);
761    }
762  }
763  if (original_url_throttler_entry_ != NULL) {
764    delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
765        GetBackoffReleaseTime());
766  }
767
768  if (delay == 0) {
769    StartURLRequest();
770  } else {
771    base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
772        FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
773        base::TimeDelta::FromMilliseconds(delay));
774  }
775}
776
777void URLFetcherCore::CancelURLRequest() {
778  DCHECK(network_task_runner_->BelongsToCurrentThread());
779
780  if (request_.get()) {
781    request_->Cancel();
782    ReleaseRequest();
783  }
784  // Release the reference to the request context. There could be multiple
785  // references to URLFetcher::Core at this point so it may take a while to
786  // delete the object, but we cannot delay the destruction of the request
787  // context.
788  request_context_getter_ = NULL;
789  first_party_for_cookies_ = GURL();
790  url_request_data_key_ = NULL;
791  url_request_create_data_callback_.Reset();
792  was_cancelled_ = true;
793  file_writer_.reset();
794}
795
796void URLFetcherCore::OnCompletedURLRequest(
797    base::TimeDelta backoff_delay) {
798  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
799
800  // Save the status and backoff_delay so that delegates can read it.
801  if (delegate_) {
802    backoff_delay_ = backoff_delay;
803    InformDelegateFetchIsComplete();
804  }
805}
806
807void URLFetcherCore::InformDelegateFetchIsComplete() {
808  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
809  if (delegate_)
810    delegate_->OnURLFetchComplete(fetcher_);
811}
812
813void URLFetcherCore::NotifyMalformedContent() {
814  DCHECK(network_task_runner_->BelongsToCurrentThread());
815  if (url_throttler_entry_ != NULL) {
816    int status_code = response_code_;
817    if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
818      // The status code will generally be known by the time clients
819      // call the |ReceivedContentWasMalformed()| function (which ends up
820      // calling the current function) but if it's not, we need to assume
821      // the response was successful so that the total failure count
822      // used to calculate exponential back-off goes up.
823      status_code = 200;
824    }
825    url_throttler_entry_->ReceivedContentWasMalformed(status_code);
826  }
827}
828
829void URLFetcherCore::RetryOrCompleteUrlFetch() {
830  DCHECK(network_task_runner_->BelongsToCurrentThread());
831  base::TimeDelta backoff_delay;
832
833  // Checks the response from server.
834  if (response_code_ >= 500 ||
835      status_.error() == ERR_TEMPORARILY_THROTTLED) {
836    // When encountering a server error, we will send the request again
837    // after backoff time.
838    ++num_retries_;
839
840    // Note that backoff_delay may be 0 because (a) the
841    // URLRequestThrottlerManager and related code does not
842    // necessarily back off on the first error, (b) it only backs off
843    // on some of the 5xx status codes, (c) not all URLRequestContexts
844    // have a throttler manager.
845    base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
846    backoff_delay = backoff_release_time - base::TimeTicks::Now();
847    if (backoff_delay < base::TimeDelta())
848      backoff_delay = base::TimeDelta();
849
850    if (automatically_retry_on_5xx_ && num_retries_ <= max_retries_) {
851      StartOnIOThread();
852      return;
853    }
854  } else {
855    backoff_delay = base::TimeDelta();
856  }
857  request_context_getter_ = NULL;
858  first_party_for_cookies_ = GURL();
859  url_request_data_key_ = NULL;
860  url_request_create_data_callback_.Reset();
861  bool posted = delegate_task_runner_->PostTask(
862      FROM_HERE,
863      base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
864
865  // If the delegate message loop does not exist any more, then the delegate
866  // should be gone too.
867  DCHECK(posted || !delegate_);
868}
869
870void URLFetcherCore::ReleaseRequest() {
871  upload_progress_checker_timer_.reset();
872  request_.reset();
873  g_registry.Get().RemoveURLFetcherCore(this);
874}
875
876base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
877  DCHECK(network_task_runner_->BelongsToCurrentThread());
878
879  if (original_url_throttler_entry_) {
880    base::TimeTicks original_url_backoff =
881        original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
882    base::TimeTicks destination_url_backoff;
883    if (url_throttler_entry_ != NULL &&
884        original_url_throttler_entry_ != url_throttler_entry_) {
885      destination_url_backoff =
886          url_throttler_entry_->GetExponentialBackoffReleaseTime();
887    }
888
889    return original_url_backoff > destination_url_backoff ?
890        original_url_backoff : destination_url_backoff;
891  } else {
892    return base::TimeTicks();
893  }
894}
895
896void URLFetcherCore::CompleteAddingUploadDataChunk(
897    const std::string& content, bool is_last_chunk) {
898  if (was_cancelled_) {
899    // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
900    // may run after the URLFetcher was already stopped.
901    return;
902  }
903  DCHECK(is_chunked_upload_);
904  DCHECK(request_.get());
905  DCHECK(!content.empty());
906  request_->AppendChunkToUpload(content.data(),
907                                static_cast<int>(content.length()),
908                                is_last_chunk);
909}
910
911// Return true if the write was done and reading may continue.
912// Return false if the write is pending, and the next read will
913// be done later.
914bool URLFetcherCore::WriteBuffer(int num_bytes) {
915  bool write_complete = false;
916  switch (response_destination_) {
917    case STRING:
918      data_.append(buffer_->data(), num_bytes);
919      write_complete = true;
920      break;
921
922    case PERMANENT_FILE:
923    case TEMP_FILE:
924      file_writer_->WriteBuffer(num_bytes);
925      // WriteBuffer() sends a request the file thread.
926      // The write is not done yet.
927      write_complete = false;
928      break;
929
930    default:
931      NOTREACHED();
932  }
933  return write_complete;
934}
935
936void URLFetcherCore::ReadResponse() {
937  // Some servers may treat HEAD requests as GET requests.  To free up the
938  // network connection as soon as possible, signal that the request has
939  // completed immediately, without trying to read any data back (all we care
940  // about is the response code and headers, which we already have).
941  int bytes_read = 0;
942  if (request_->status().is_success() &&
943      (request_type_ != URLFetcher::HEAD))
944    request_->Read(buffer_, kBufferSize, &bytes_read);
945  OnReadCompleted(request_.get(), bytes_read);
946}
947
948void URLFetcherCore::DisownFile() {
949  file_writer_->DisownFile();
950}
951
952void URLFetcherCore::InformDelegateUploadProgress() {
953  DCHECK(network_task_runner_->BelongsToCurrentThread());
954  if (request_.get()) {
955    int64 current = request_->GetUploadProgress().position();
956    if (current_upload_bytes_ != current) {
957      current_upload_bytes_ = current;
958      int64 total = -1;
959      if (!is_chunked_upload_)
960        total = static_cast<int64>(upload_content_.size());
961      delegate_task_runner_->PostTask(
962          FROM_HERE,
963          base::Bind(
964              &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
965              this, current, total));
966    }
967  }
968}
969
970void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
971    int64 current, int64 total) {
972  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
973  if (delegate_)
974    delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
975}
976
977void URLFetcherCore::InformDelegateDownloadProgress() {
978  DCHECK(network_task_runner_->BelongsToCurrentThread());
979  delegate_task_runner_->PostTask(
980      FROM_HERE,
981      base::Bind(
982          &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
983          this, current_response_bytes_, total_response_bytes_));
984}
985
986void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
987    int64 current, int64 total) {
988  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
989  if (delegate_)
990    delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
991}
992
993void URLFetcherCore::InformDelegateDownloadDataIfNecessary(int bytes_read) {
994  DCHECK(network_task_runner_->BelongsToCurrentThread());
995  if (delegate_ && delegate_->ShouldSendDownloadData()) {
996    scoped_ptr<std::string> download_data(
997        new std::string(buffer_->data(), bytes_read));
998    delegate_task_runner_->PostTask(
999        FROM_HERE,
1000        base::Bind(
1001            &URLFetcherCore::InformDelegateDownloadDataInDelegateThread,
1002            this, base::Passed(&download_data)));
1003  }
1004}
1005
1006void URLFetcherCore::InformDelegateDownloadDataInDelegateThread(
1007    scoped_ptr<std::string> download_data) {
1008  DCHECK(delegate_task_runner_->BelongsToCurrentThread());
1009  if (delegate_)
1010    delegate_->OnURLFetchDownloadData(fetcher_, download_data.Pass());
1011}
1012
1013}  // namespace net
1014