1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_request.h"
6
7#include "base/compiler_specific.h"
8#include "base/memory/singleton.h"
9#include "base/message_loop.h"
10#include "base/metrics/stats_counters.h"
11#include "base/synchronization/lock.h"
12#include "net/base/host_port_pair.h"
13#include "net/base/load_flags.h"
14#include "net/base/net_errors.h"
15#include "net/base/net_log.h"
16#include "net/base/network_delegate.h"
17#include "net/base/ssl_cert_request_info.h"
18#include "net/base/upload_data.h"
19#include "net/http/http_response_headers.h"
20#include "net/http/http_util.h"
21#include "net/url_request/url_request_context.h"
22#include "net/url_request/url_request_error_job.h"
23#include "net/url_request/url_request_job.h"
24#include "net/url_request/url_request_job_manager.h"
25#include "net/url_request/url_request_netlog_params.h"
26#include "net/url_request/url_request_redirect_job.h"
27
28using base::Time;
29using std::string;
30
31namespace net {
32
33namespace {
34
35// Max number of http redirects to follow.  Same number as gecko.
36const int kMaxRedirects = 20;
37
38// Discard headers which have meaning in POST (Content-Length, Content-Type,
39// Origin).
40void StripPostSpecificHeaders(HttpRequestHeaders* headers) {
41  // These are headers that may be attached to a POST.
42  headers->RemoveHeader(HttpRequestHeaders::kContentLength);
43  headers->RemoveHeader(HttpRequestHeaders::kContentType);
44  headers->RemoveHeader(HttpRequestHeaders::kOrigin);
45}
46
47// This counter keeps track of the identifiers used for URL requests so far.
48// 0 is reserved to represent an invalid ID.
49uint64 g_next_url_request_identifier = 1;
50
51// This lock protects g_next_url_request_identifier.
52base::Lock g_next_url_request_identifier_lock;
53
54// Returns an prior unused identifier for URL requests.
55uint64 GenerateURLRequestIdentifier() {
56  base::AutoLock lock(g_next_url_request_identifier_lock);
57  return g_next_url_request_identifier++;
58}
59
60}  // namespace
61
62///////////////////////////////////////////////////////////////////////////////
63// URLRequest::Interceptor
64
65URLRequestJob* URLRequest::Interceptor::MaybeInterceptRedirect(
66    URLRequest* request,
67    const GURL& location) {
68  return NULL;
69}
70
71URLRequestJob* URLRequest::Interceptor::MaybeInterceptResponse(
72    URLRequest* request) {
73  return NULL;
74}
75
76///////////////////////////////////////////////////////////////////////////////
77// URLRequest::Delegate
78
79void URLRequest::Delegate::OnReceivedRedirect(URLRequest* request,
80                                              const GURL& new_url,
81                                              bool* defer_redirect) {
82}
83
84void URLRequest::Delegate::OnAuthRequired(URLRequest* request,
85                                          AuthChallengeInfo* auth_info) {
86  request->CancelAuth();
87}
88
89void URLRequest::Delegate::OnCertificateRequested(
90    URLRequest* request,
91    SSLCertRequestInfo* cert_request_info) {
92  request->ContinueWithCertificate(NULL);
93}
94
95void URLRequest::Delegate::OnSSLCertificateError(URLRequest* request,
96                                                 int cert_error,
97                                                 X509Certificate* cert) {
98  request->Cancel();
99}
100
101void URLRequest::Delegate::OnGetCookies(URLRequest* request,
102                                        bool blocked_by_policy) {
103}
104
105void URLRequest::Delegate::OnSetCookie(URLRequest* request,
106                                       const std::string& cookie_line,
107                                       const CookieOptions& options,
108                                       bool blocked_by_policy) {
109}
110
111///////////////////////////////////////////////////////////////////////////////
112// URLRequest
113
114URLRequest::URLRequest(const GURL& url, Delegate* delegate)
115    : url_chain_(1, url),
116      method_("GET"),
117      load_flags_(LOAD_NORMAL),
118      delegate_(delegate),
119      is_pending_(false),
120      redirect_limit_(kMaxRedirects),
121      final_upload_progress_(0),
122      priority_(LOWEST),
123      identifier_(GenerateURLRequestIdentifier()),
124      ALLOW_THIS_IN_INITIALIZER_LIST(
125          before_request_callback_(this, &URLRequest::BeforeRequestComplete)) {
126  SIMPLE_STATS_COUNTER("URLRequestCount");
127
128  // Sanity check out environment.
129  DCHECK(MessageLoop::current()) <<
130      "The current MessageLoop must exist";
131  DCHECK_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type()) <<
132      "The current MessageLoop must be TYPE_IO";
133}
134
135URLRequest::~URLRequest() {
136  if (context_ && context_->network_delegate())
137    context_->network_delegate()->NotifyURLRequestDestroyed(this);
138
139  Cancel();
140
141  if (job_)
142    OrphanJob();
143
144  set_context(NULL);
145}
146
147// static
148URLRequest::ProtocolFactory* URLRequest::RegisterProtocolFactory(
149    const string& scheme, ProtocolFactory* factory) {
150  return URLRequestJobManager::GetInstance()->RegisterProtocolFactory(scheme,
151                                                                      factory);
152}
153
154// static
155void URLRequest::RegisterRequestInterceptor(Interceptor* interceptor) {
156  URLRequestJobManager::GetInstance()->RegisterRequestInterceptor(interceptor);
157}
158
159// static
160void URLRequest::UnregisterRequestInterceptor(Interceptor* interceptor) {
161  URLRequestJobManager::GetInstance()->UnregisterRequestInterceptor(
162      interceptor);
163}
164
165void URLRequest::AppendBytesToUpload(const char* bytes, int bytes_len) {
166  DCHECK(bytes_len > 0 && bytes);
167  if (!upload_)
168    upload_ = new UploadData();
169  upload_->AppendBytes(bytes, bytes_len);
170}
171
172void URLRequest::AppendFileRangeToUpload(
173    const FilePath& file_path,
174    uint64 offset,
175    uint64 length,
176    const base::Time& expected_modification_time) {
177  DCHECK(file_path.value().length() > 0 && length > 0);
178  if (!upload_)
179    upload_ = new UploadData();
180  upload_->AppendFileRange(file_path, offset, length,
181                           expected_modification_time);
182}
183
184void URLRequest::EnableChunkedUpload() {
185  DCHECK(!upload_ || upload_->is_chunked());
186  if (!upload_) {
187    upload_ = new UploadData();
188    upload_->set_is_chunked(true);
189  }
190}
191
192void URLRequest::AppendChunkToUpload(const char* bytes,
193                                     int bytes_len,
194                                     bool is_last_chunk) {
195  DCHECK(upload_);
196  DCHECK(upload_->is_chunked());
197  DCHECK_GT(bytes_len, 0);
198  upload_->AppendChunk(bytes, bytes_len, is_last_chunk);
199}
200
201void URLRequest::set_upload(UploadData* upload) {
202  upload_ = upload;
203}
204
205// Get the upload data directly.
206UploadData* URLRequest::get_upload() {
207  return upload_.get();
208}
209
210bool URLRequest::has_upload() const {
211  return upload_ != NULL;
212}
213
214void URLRequest::SetExtraRequestHeaderById(int id, const string& value,
215                                           bool overwrite) {
216  DCHECK(!is_pending_);
217  NOTREACHED() << "implement me!";
218}
219
220void URLRequest::SetExtraRequestHeaderByName(const string& name,
221                                             const string& value,
222                                             bool overwrite) {
223  DCHECK(!is_pending_);
224  if (overwrite) {
225    extra_request_headers_.SetHeader(name, value);
226  } else {
227    extra_request_headers_.SetHeaderIfMissing(name, value);
228  }
229}
230
231void URLRequest::SetExtraRequestHeaders(
232    const HttpRequestHeaders& headers) {
233  DCHECK(!is_pending_);
234  extra_request_headers_ = headers;
235
236  // NOTE: This method will likely become non-trivial once the other setters
237  // for request headers are implemented.
238}
239
240LoadState URLRequest::GetLoadState() const {
241  return job_ ? job_->GetLoadState() : LOAD_STATE_IDLE;
242}
243
244uint64 URLRequest::GetUploadProgress() const {
245  if (!job_) {
246    // We haven't started or the request was cancelled
247    return 0;
248  }
249  if (final_upload_progress_) {
250    // The first job completed and none of the subsequent series of
251    // GETs when following redirects will upload anything, so we return the
252    // cached results from the initial job, the POST.
253    return final_upload_progress_;
254  }
255  return job_->GetUploadProgress();
256}
257
258void URLRequest::GetResponseHeaderById(int id, string* value) {
259  DCHECK(job_);
260  NOTREACHED() << "implement me!";
261}
262
263void URLRequest::GetResponseHeaderByName(const string& name, string* value) {
264  DCHECK(value);
265  if (response_info_.headers) {
266    response_info_.headers->GetNormalizedHeader(name, value);
267  } else {
268    value->clear();
269  }
270}
271
272void URLRequest::GetAllResponseHeaders(string* headers) {
273  DCHECK(headers);
274  if (response_info_.headers) {
275    response_info_.headers->GetNormalizedHeaders(headers);
276  } else {
277    headers->clear();
278  }
279}
280
281HostPortPair URLRequest::GetSocketAddress() const {
282  DCHECK(job_);
283  return job_->GetSocketAddress();
284}
285
286HttpResponseHeaders* URLRequest::response_headers() const {
287  return response_info_.headers.get();
288}
289
290bool URLRequest::GetResponseCookies(ResponseCookies* cookies) {
291  DCHECK(job_);
292  return job_->GetResponseCookies(cookies);
293}
294
295void URLRequest::GetMimeType(string* mime_type) {
296  DCHECK(job_);
297  job_->GetMimeType(mime_type);
298}
299
300void URLRequest::GetCharset(string* charset) {
301  DCHECK(job_);
302  job_->GetCharset(charset);
303}
304
305int URLRequest::GetResponseCode() {
306  DCHECK(job_);
307  return job_->GetResponseCode();
308}
309
310// static
311bool URLRequest::IsHandledProtocol(const std::string& scheme) {
312  return URLRequestJobManager::GetInstance()->SupportsScheme(scheme);
313}
314
315// static
316bool URLRequest::IsHandledURL(const GURL& url) {
317  if (!url.is_valid()) {
318    // We handle error cases.
319    return true;
320  }
321
322  return IsHandledProtocol(url.scheme());
323}
324
325// static
326void URLRequest::AllowFileAccess() {
327  URLRequestJobManager::GetInstance()->set_enable_file_access(true);
328}
329
330// static
331bool URLRequest::IsFileAccessAllowed() {
332  return URLRequestJobManager::GetInstance()->enable_file_access();
333}
334
335
336void URLRequest::set_first_party_for_cookies(
337    const GURL& first_party_for_cookies) {
338  first_party_for_cookies_ = first_party_for_cookies;
339}
340
341void URLRequest::set_method(const std::string& method) {
342  DCHECK(!is_pending_);
343  method_ = method;
344}
345
346void URLRequest::set_referrer(const std::string& referrer) {
347  DCHECK(!is_pending_);
348  referrer_ = referrer;
349}
350
351GURL URLRequest::GetSanitizedReferrer() const {
352  GURL ret(referrer());
353
354  // Ensure that we do not send username and password fields in the referrer.
355  if (ret.has_username() || ret.has_password()) {
356    GURL::Replacements referrer_mods;
357    referrer_mods.ClearUsername();
358    referrer_mods.ClearPassword();
359    ret = ret.ReplaceComponents(referrer_mods);
360  }
361
362  return ret;
363}
364
365void URLRequest::Start() {
366  response_info_.request_time = Time::Now();
367
368  // Only notify the delegate for the initial request.
369  if (context_ && context_->network_delegate()) {
370    if (context_->network_delegate()->NotifyBeforeURLRequest(
371            this, &before_request_callback_, &delegate_redirect_url_) ==
372            net::ERR_IO_PENDING) {
373      net_log_.BeginEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL);
374      return;  // paused
375    }
376  }
377
378  StartInternal();
379}
380
381///////////////////////////////////////////////////////////////////////////////
382
383void URLRequest::BeforeRequestComplete(int error) {
384  DCHECK(!job_);
385  DCHECK_NE(ERR_IO_PENDING, error);
386
387  net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL);
388  if (error != OK) {
389    StartJob(new URLRequestErrorJob(this, error));
390  } else if (!delegate_redirect_url_.is_empty()) {
391    GURL new_url;
392    new_url.Swap(&delegate_redirect_url_);
393    StartJob(new URLRequestRedirectJob(this, new_url));
394  } else {
395    StartInternal();
396  }
397}
398
399void URLRequest::StartInternal() {
400  StartJob(URLRequestJobManager::GetInstance()->CreateJob(this));
401}
402
403void URLRequest::StartJob(URLRequestJob* job) {
404  DCHECK(!is_pending_);
405  DCHECK(!job_);
406
407  net_log_.BeginEvent(
408      NetLog::TYPE_URL_REQUEST_START_JOB,
409      make_scoped_refptr(new URLRequestStartEventParameters(
410          url(), method_, load_flags_, priority_)));
411
412  job_ = job;
413  job_->SetExtraRequestHeaders(extra_request_headers_);
414
415  if (upload_.get())
416    job_->SetUpload(upload_.get());
417
418  is_pending_ = true;
419
420  response_info_.was_cached = false;
421
422  // Don't allow errors to be sent from within Start().
423  // TODO(brettw) this may cause NotifyDone to be sent synchronously,
424  // we probably don't want this: they should be sent asynchronously so
425  // the caller does not get reentered.
426  job_->Start();
427}
428
429void URLRequest::Restart() {
430  // Should only be called if the original job didn't make any progress.
431  DCHECK(job_ && !job_->has_response_started());
432  RestartWithJob(URLRequestJobManager::GetInstance()->CreateJob(this));
433}
434
435void URLRequest::RestartWithJob(URLRequestJob *job) {
436  DCHECK(job->request() == this);
437  PrepareToRestart();
438  StartJob(job);
439}
440
441void URLRequest::Cancel() {
442  DoCancel(ERR_ABORTED, SSLInfo());
443}
444
445void URLRequest::SimulateError(int os_error) {
446  DoCancel(os_error, SSLInfo());
447}
448
449void URLRequest::SimulateSSLError(int os_error, const SSLInfo& ssl_info) {
450  // This should only be called on a started request.
451  if (!is_pending_ || !job_ || job_->has_response_started()) {
452    NOTREACHED();
453    return;
454  }
455  DoCancel(os_error, ssl_info);
456}
457
458void URLRequest::DoCancel(int os_error, const SSLInfo& ssl_info) {
459  DCHECK(os_error < 0);
460
461  // If the URL request already has an error status, then canceling is a no-op.
462  // Plus, we don't want to change the error status once it has been set.
463  if (status_.is_success()) {
464    status_.set_status(URLRequestStatus::CANCELED);
465    status_.set_os_error(os_error);
466    response_info_.ssl_info = ssl_info;
467  }
468
469  // There's nothing to do if we are not waiting on a Job.
470  if (!is_pending_ || !job_)
471    return;
472
473  job_->Kill();
474
475  // The Job will call our NotifyDone method asynchronously.  This is done so
476  // that the Delegate implementation can call Cancel without having to worry
477  // about being called recursively.
478}
479
480bool URLRequest::Read(IOBuffer* dest, int dest_size, int* bytes_read) {
481  DCHECK(job_);
482  DCHECK(bytes_read);
483  DCHECK(!job_->is_done());
484  *bytes_read = 0;
485
486  if (dest_size == 0) {
487    // Caller is not too bright.  I guess we've done what they asked.
488    return true;
489  }
490
491  // Once the request fails or is cancelled, read will just return 0 bytes
492  // to indicate end of stream.
493  if (!status_.is_success()) {
494    return true;
495  }
496
497  return job_->Read(dest, dest_size, bytes_read);
498}
499
500void URLRequest::StopCaching() {
501  DCHECK(job_);
502  job_->StopCaching();
503}
504
505void URLRequest::ReceivedRedirect(const GURL& location, bool* defer_redirect) {
506  URLRequestJob* job =
507      URLRequestJobManager::GetInstance()->MaybeInterceptRedirect(this,
508                                                                  location);
509  if (job) {
510    RestartWithJob(job);
511  } else if (delegate_) {
512    delegate_->OnReceivedRedirect(this, location, defer_redirect);
513  }
514}
515
516void URLRequest::ResponseStarted() {
517  scoped_refptr<NetLog::EventParameters> params;
518  if (!status_.is_success())
519    params = new NetLogIntegerParameter("net_error", status_.os_error());
520  net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, params);
521
522  URLRequestJob* job =
523      URLRequestJobManager::GetInstance()->MaybeInterceptResponse(this);
524  if (job) {
525    RestartWithJob(job);
526  } else {
527    if (context_ && context_->network_delegate())
528      context_->network_delegate()->NotifyResponseStarted(this);
529    if (delegate_)
530      delegate_->OnResponseStarted(this);
531  }
532}
533
534void URLRequest::FollowDeferredRedirect() {
535  CHECK(job_);
536  CHECK(status_.is_success());
537
538  job_->FollowDeferredRedirect();
539}
540
541void URLRequest::SetAuth(const string16& username, const string16& password) {
542  DCHECK(job_);
543  DCHECK(job_->NeedsAuth());
544
545  job_->SetAuth(username, password);
546}
547
548void URLRequest::CancelAuth() {
549  DCHECK(job_);
550  DCHECK(job_->NeedsAuth());
551
552  job_->CancelAuth();
553}
554
555void URLRequest::ContinueWithCertificate(X509Certificate* client_cert) {
556  DCHECK(job_);
557
558  job_->ContinueWithCertificate(client_cert);
559}
560
561void URLRequest::ContinueDespiteLastError() {
562  DCHECK(job_);
563
564  job_->ContinueDespiteLastError();
565}
566
567void URLRequest::PrepareToRestart() {
568  DCHECK(job_);
569
570  // Close the current URL_REQUEST_START_JOB, since we will be starting a new
571  // one.
572  net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, NULL);
573
574  OrphanJob();
575
576  response_info_ = HttpResponseInfo();
577  response_info_.request_time = Time::Now();
578  status_ = URLRequestStatus();
579  is_pending_ = false;
580}
581
582void URLRequest::OrphanJob() {
583  job_->Kill();
584  job_->DetachRequest();  // ensures that the job will not call us again
585  job_ = NULL;
586}
587
588int URLRequest::Redirect(const GURL& location, int http_status_code) {
589  if (net_log_.IsLoggingAllEvents()) {
590    net_log_.AddEvent(
591        NetLog::TYPE_URL_REQUEST_REDIRECTED,
592        make_scoped_refptr(new NetLogStringParameter(
593            "location", location.possibly_invalid_spec())));
594  }
595  if (redirect_limit_ <= 0) {
596    DVLOG(1) << "disallowing redirect: exceeds limit";
597    return ERR_TOO_MANY_REDIRECTS;
598  }
599
600  if (!location.is_valid())
601    return ERR_INVALID_URL;
602
603  if (!job_->IsSafeRedirect(location)) {
604    DVLOG(1) << "disallowing redirect: unsafe protocol";
605    return ERR_UNSAFE_REDIRECT;
606  }
607
608  bool strip_post_specific_headers = false;
609  if (http_status_code != 307) {
610    // NOTE: Even though RFC 2616 says to preserve the request method when
611    // following a 302 redirect, normal browsers don't do that.  Instead, they
612    // all convert a POST into a GET in response to a 302 and so shall we.  For
613    // 307 redirects, browsers preserve the method.  The RFC says to prompt the
614    // user to confirm the generation of a new POST request, but IE omits this
615    // prompt and so shall we.
616    strip_post_specific_headers = method_ == "POST";
617    method_ = "GET";
618    upload_ = NULL;
619  }
620
621  // Suppress the referrer if we're redirecting out of https.
622  if (GURL(referrer_).SchemeIsSecure() && !location.SchemeIsSecure())
623    referrer_.clear();
624
625  url_chain_.push_back(location);
626  --redirect_limit_;
627
628  if (strip_post_specific_headers) {
629    // If being switched from POST to GET, must remove headers that were
630    // specific to the POST and don't have meaning in GET. For example
631    // the inclusion of a multipart Content-Type header in GET can cause
632    // problems with some servers:
633    // http://code.google.com/p/chromium/issues/detail?id=843
634    StripPostSpecificHeaders(&extra_request_headers_);
635  }
636
637  if (!final_upload_progress_)
638    final_upload_progress_ = job_->GetUploadProgress();
639
640  PrepareToRestart();
641  StartInternal();
642  return OK;
643}
644
645URLRequestContext* URLRequest::context() const {
646  return context_.get();
647}
648
649void URLRequest::set_context(URLRequestContext* context) {
650  scoped_refptr<URLRequestContext> prev_context = context_;
651
652  context_ = context;
653
654  // If the context this request belongs to has changed, update the tracker.
655  if (prev_context != context) {
656    net_log_.EndEvent(NetLog::TYPE_REQUEST_ALIVE, NULL);
657    net_log_ = BoundNetLog();
658
659    if (context) {
660      net_log_ = BoundNetLog::Make(context->net_log(),
661                                   NetLog::SOURCE_URL_REQUEST);
662      net_log_.BeginEvent(NetLog::TYPE_REQUEST_ALIVE, NULL);
663    }
664  }
665}
666
667int64 URLRequest::GetExpectedContentSize() const {
668  int64 expected_content_size = -1;
669  if (job_)
670    expected_content_size = job_->expected_content_size();
671
672  return expected_content_size;
673}
674
675URLRequest::UserData* URLRequest::GetUserData(const void* key) const {
676  UserDataMap::const_iterator found = user_data_.find(key);
677  if (found != user_data_.end())
678    return found->second.get();
679  return NULL;
680}
681
682void URLRequest::SetUserData(const void* key, UserData* data) {
683  user_data_[key] = linked_ptr<UserData>(data);
684}
685
686}  // namespace net
687