1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "net/url_request/url_request_job.h"
6
7#include "base/compiler_specific.h"
8#include "base/message_loop.h"
9#include "base/string_number_conversions.h"
10#include "base/string_util.h"
11#include "net/base/auth.h"
12#include "net/base/host_port_pair.h"
13#include "net/base/io_buffer.h"
14#include "net/base/load_states.h"
15#include "net/base/net_errors.h"
16#include "net/base/network_delegate.h"
17#include "net/http/http_response_headers.h"
18#include "net/url_request/url_request.h"
19#include "net/url_request/url_request_context.h"
20#include "net/url_request/url_request_job_tracker.h"
21
22namespace net {
23
24URLRequestJob::URLRequestJob(URLRequest* request)
25    : request_(request),
26      done_(false),
27      prefilter_bytes_read_(0),
28      postfilter_bytes_read_(0),
29      filter_input_byte_count_(0),
30      filter_needs_more_output_space_(false),
31      filtered_read_buffer_len_(0),
32      has_handled_response_(false),
33      expected_content_size_(-1),
34      deferred_redirect_status_code_(-1),
35      ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {
36  g_url_request_job_tracker.AddNewJob(this);
37}
38
39void URLRequestJob::SetUpload(UploadData* upload) {
40}
41
42void URLRequestJob::SetExtraRequestHeaders(
43    const HttpRequestHeaders& headers) {
44}
45
46void URLRequestJob::Kill() {
47  // Make sure the request is notified that we are done.  We assume that the
48  // request took care of setting its error status before calling Kill.
49  if (request_)
50    NotifyCanceled();
51}
52
53void URLRequestJob::DetachRequest() {
54  request_ = NULL;
55}
56
57// This function calls ReadData to get stream data. If a filter exists, passes
58// the data to the attached filter. Then returns the output from filter back to
59// the caller.
60bool URLRequestJob::Read(IOBuffer* buf, int buf_size, int *bytes_read) {
61  bool rv = false;
62
63  DCHECK_LT(buf_size, 1000000);  // sanity check
64  DCHECK(buf);
65  DCHECK(bytes_read);
66  DCHECK(filtered_read_buffer_ == NULL);
67  DCHECK_EQ(0, filtered_read_buffer_len_);
68
69  *bytes_read = 0;
70
71  // Skip Filter if not present
72  if (!filter_.get()) {
73    rv = ReadRawDataHelper(buf, buf_size, bytes_read);
74  } else {
75    // Save the caller's buffers while we do IO
76    // in the filter's buffers.
77    filtered_read_buffer_ = buf;
78    filtered_read_buffer_len_ = buf_size;
79
80    if (ReadFilteredData(bytes_read)) {
81      rv = true;   // we have data to return
82    } else {
83      rv = false;  // error, or a new IO is pending
84    }
85  }
86  if (rv && *bytes_read == 0)
87    NotifyDone(URLRequestStatus());
88  return rv;
89}
90
91void URLRequestJob::StopCaching() {
92  // Nothing to do here.
93}
94
95LoadState URLRequestJob::GetLoadState() const {
96  return LOAD_STATE_IDLE;
97}
98
99uint64 URLRequestJob::GetUploadProgress() const {
100  return 0;
101}
102
103bool URLRequestJob::GetCharset(std::string* charset) {
104  return false;
105}
106
107void URLRequestJob::GetResponseInfo(HttpResponseInfo* info) {
108}
109
110bool URLRequestJob::GetResponseCookies(std::vector<std::string>* cookies) {
111  return false;
112}
113
114Filter* URLRequestJob::SetupFilter() const {
115  return NULL;
116}
117
118bool URLRequestJob::IsRedirectResponse(GURL* location,
119                                       int* http_status_code) {
120  // For non-HTTP jobs, headers will be null.
121  HttpResponseHeaders* headers = request_->response_headers();
122  if (!headers)
123    return false;
124
125  std::string value;
126  if (!headers->IsRedirect(&value))
127    return false;
128
129  *location = request_->url().Resolve(value);
130  *http_status_code = headers->response_code();
131  return true;
132}
133
134bool URLRequestJob::IsSafeRedirect(const GURL& location) {
135  return true;
136}
137
138bool URLRequestJob::NeedsAuth() {
139  return false;
140}
141
142void URLRequestJob::GetAuthChallengeInfo(
143    scoped_refptr<AuthChallengeInfo>* auth_info) {
144  // This will only be called if NeedsAuth() returns true, in which
145  // case the derived class should implement this!
146  NOTREACHED();
147}
148
149void URLRequestJob::SetAuth(const string16& username,
150                            const string16& password) {
151  // This will only be called if NeedsAuth() returns true, in which
152  // case the derived class should implement this!
153  NOTREACHED();
154}
155
156void URLRequestJob::CancelAuth() {
157  // This will only be called if NeedsAuth() returns true, in which
158  // case the derived class should implement this!
159  NOTREACHED();
160}
161
162void URLRequestJob::ContinueWithCertificate(
163    X509Certificate* client_cert) {
164  // The derived class should implement this!
165  NOTREACHED();
166}
167
168void URLRequestJob::ContinueDespiteLastError() {
169  // Implementations should know how to recover from errors they generate.
170  // If this code was reached, we are trying to recover from an error that
171  // we don't know how to recover from.
172  NOTREACHED();
173}
174
175void URLRequestJob::FollowDeferredRedirect() {
176  DCHECK(deferred_redirect_status_code_ != -1);
177
178  // NOTE: deferred_redirect_url_ may be invalid, and attempting to redirect to
179  // such an URL will fail inside FollowRedirect.  The DCHECK above asserts
180  // that we called OnReceivedRedirect.
181
182  // It is also possible that FollowRedirect will drop the last reference to
183  // this job, so we need to reset our members before calling it.
184
185  GURL redirect_url = deferred_redirect_url_;
186  int redirect_status_code = deferred_redirect_status_code_;
187
188  deferred_redirect_url_ = GURL();
189  deferred_redirect_status_code_ = -1;
190
191  FollowRedirect(redirect_url, redirect_status_code);
192}
193
194bool URLRequestJob::GetMimeType(std::string* mime_type) const {
195  return false;
196}
197
198int URLRequestJob::GetResponseCode() const {
199  return -1;
200}
201
202HostPortPair URLRequestJob::GetSocketAddress() const {
203  return HostPortPair();
204}
205
206URLRequestJob::~URLRequestJob() {
207  g_url_request_job_tracker.RemoveJob(this);
208}
209
210void URLRequestJob::NotifyHeadersComplete() {
211  if (!request_ || !request_->delegate())
212    return;  // The request was destroyed, so there is no more work to do.
213
214  if (has_handled_response_)
215    return;
216
217  DCHECK(!request_->status().is_io_pending());
218
219  // Initialize to the current time, and let the subclass optionally override
220  // the time stamps if it has that information.  The default request_time is
221  // set by URLRequest before it calls our Start method.
222  request_->response_info_.response_time = base::Time::Now();
223  GetResponseInfo(&request_->response_info_);
224
225  // When notifying the delegate, the delegate can release the request
226  // (and thus release 'this').  After calling to the delgate, we must
227  // check the request pointer to see if it still exists, and return
228  // immediately if it has been destroyed.  self_preservation ensures our
229  // survival until we can get out of this method.
230  scoped_refptr<URLRequestJob> self_preservation(this);
231
232  GURL new_location;
233  int http_status_code;
234  if (IsRedirectResponse(&new_location, &http_status_code)) {
235    const GURL& url = request_->url();
236
237    // Move the reference fragment of the old location to the new one if the
238    // new one has none. This duplicates mozilla's behavior.
239    if (url.is_valid() && url.has_ref() && !new_location.has_ref()) {
240      GURL::Replacements replacements;
241      // Reference the |ref| directly out of the original URL to avoid a
242      // malloc.
243      replacements.SetRef(url.spec().data(),
244                          url.parsed_for_possibly_invalid_spec().ref);
245      new_location = new_location.ReplaceComponents(replacements);
246    }
247
248    bool defer_redirect = false;
249    request_->ReceivedRedirect(new_location, &defer_redirect);
250
251    // Ensure that the request wasn't detached or destroyed in ReceivedRedirect
252    if (!request_ || !request_->delegate())
253      return;
254
255    // If we were not cancelled, then maybe follow the redirect.
256    if (request_->status().is_success()) {
257      if (defer_redirect) {
258        deferred_redirect_url_ = new_location;
259        deferred_redirect_status_code_ = http_status_code;
260      } else {
261        FollowRedirect(new_location, http_status_code);
262      }
263      return;
264    }
265  } else if (NeedsAuth()) {
266    scoped_refptr<AuthChallengeInfo> auth_info;
267    GetAuthChallengeInfo(&auth_info);
268    // Need to check for a NULL auth_info because the server may have failed
269    // to send a challenge with the 401 response.
270    if (auth_info) {
271      request_->delegate()->OnAuthRequired(request_, auth_info);
272      // Wait for SetAuth or CancelAuth to be called.
273      return;
274    }
275  }
276
277  has_handled_response_ = true;
278  if (request_->status().is_success())
279    filter_.reset(SetupFilter());
280
281  if (!filter_.get()) {
282    std::string content_length;
283    request_->GetResponseHeaderByName("content-length", &content_length);
284    if (!content_length.empty())
285      base::StringToInt64(content_length, &expected_content_size_);
286  }
287
288  request_->ResponseStarted();
289}
290
291void URLRequestJob::NotifyReadComplete(int bytes_read) {
292  if (!request_ || !request_->delegate())
293    return;  // The request was destroyed, so there is no more work to do.
294
295  // TODO(darin): Bug 1004233. Re-enable this test once all of the chrome
296  // unit_tests have been fixed to not trip this.
297  //DCHECK(!request_->status().is_io_pending());
298
299  // The headers should be complete before reads complete
300  DCHECK(has_handled_response_);
301
302  OnRawReadComplete(bytes_read);
303
304  // Don't notify if we had an error.
305  if (!request_->status().is_success())
306    return;
307
308  // When notifying the delegate, the delegate can release the request
309  // (and thus release 'this').  After calling to the delgate, we must
310  // check the request pointer to see if it still exists, and return
311  // immediately if it has been destroyed.  self_preservation ensures our
312  // survival until we can get out of this method.
313  scoped_refptr<URLRequestJob> self_preservation(this);
314
315  prefilter_bytes_read_ += bytes_read;
316  if (filter_.get()) {
317    // Tell the filter that it has more data
318    FilteredDataRead(bytes_read);
319
320    // Filter the data.
321    int filter_bytes_read = 0;
322    if (ReadFilteredData(&filter_bytes_read)) {
323      postfilter_bytes_read_ += filter_bytes_read;
324      if (request_->context() && request_->context()->network_delegate()) {
325        request_->context()->network_delegate()->NotifyReadCompleted(
326            request_, filter_bytes_read);
327      }
328      request_->delegate()->OnReadCompleted(request_, filter_bytes_read);
329    }
330  } else {
331    postfilter_bytes_read_ += bytes_read;
332    if (request_->context() && request_->context()->network_delegate()) {
333      request_->context()->network_delegate()->NotifyReadCompleted(
334          request_, bytes_read);
335    }
336    request_->delegate()->OnReadCompleted(request_, bytes_read);
337  }
338}
339
340void URLRequestJob::NotifyStartError(const URLRequestStatus &status) {
341  DCHECK(!has_handled_response_);
342  has_handled_response_ = true;
343  if (request_) {
344    request_->set_status(status);
345    request_->ResponseStarted();
346  }
347}
348
349void URLRequestJob::NotifyDone(const URLRequestStatus &status) {
350  DCHECK(!done_) << "Job sending done notification twice";
351  if (done_)
352    return;
353  done_ = true;
354
355  // Unless there was an error, we should have at least tried to handle
356  // the response before getting here.
357  DCHECK(has_handled_response_ || !status.is_success());
358
359  // As with NotifyReadComplete, we need to take care to notice if we were
360  // destroyed during a delegate callback.
361  if (request_) {
362    request_->set_is_pending(false);
363    // With async IO, it's quite possible to have a few outstanding
364    // requests.  We could receive a request to Cancel, followed shortly
365    // by a successful IO.  For tracking the status(), once there is
366    // an error, we do not change the status back to success.  To
367    // enforce this, only set the status if the job is so far
368    // successful.
369    if (request_->status().is_success())
370      request_->set_status(status);
371  }
372
373  g_url_request_job_tracker.OnJobDone(this, status);
374
375  // Complete this notification later.  This prevents us from re-entering the
376  // delegate if we're done because of a synchronous call.
377  MessageLoop::current()->PostTask(
378      FROM_HERE,
379      method_factory_.NewRunnableMethod(&URLRequestJob::CompleteNotifyDone));
380}
381
382void URLRequestJob::CompleteNotifyDone() {
383  // Check if we should notify the delegate that we're done because of an error.
384  if (request_ &&
385      !request_->status().is_success() &&
386      request_->delegate()) {
387    // We report the error differently depending on whether we've called
388    // OnResponseStarted yet.
389    if (has_handled_response_) {
390      // We signal the error by calling OnReadComplete with a bytes_read of -1.
391      if (request_->context() && request_->context()->network_delegate())
392        request_->context()->network_delegate()->NotifyReadCompleted(
393            request_, -1);
394      request_->delegate()->OnReadCompleted(request_, -1);
395    } else {
396      has_handled_response_ = true;
397      request_->ResponseStarted();
398    }
399  }
400}
401
402void URLRequestJob::NotifyCanceled() {
403  if (!done_) {
404    NotifyDone(URLRequestStatus(URLRequestStatus::CANCELED,
405                                ERR_ABORTED));
406  }
407}
408
409void URLRequestJob::NotifyRestartRequired() {
410  DCHECK(!has_handled_response_);
411  if (GetStatus().status() != URLRequestStatus::CANCELED)
412    request_->Restart();
413}
414
415bool URLRequestJob::ReadRawData(IOBuffer* buf, int buf_size,
416                                int *bytes_read) {
417  DCHECK(bytes_read);
418  *bytes_read = 0;
419  NotifyDone(URLRequestStatus());
420  return false;
421}
422
423void URLRequestJob::FilteredDataRead(int bytes_read) {
424  DCHECK(filter_.get());  // don't add data if there is no filter
425  filter_->FlushStreamBuffer(bytes_read);
426}
427
428bool URLRequestJob::ReadFilteredData(int* bytes_read) {
429  DCHECK(filter_.get());  // don't add data if there is no filter
430  DCHECK(filtered_read_buffer_ != NULL);  // we need to have a buffer to fill
431  DCHECK_GT(filtered_read_buffer_len_, 0);  // sanity check
432  DCHECK_LT(filtered_read_buffer_len_, 1000000);  // sanity check
433  DCHECK(raw_read_buffer_ == NULL);  // there should be no raw read buffer yet
434
435  bool rv = false;
436  *bytes_read = 0;
437
438  if (is_done())
439    return true;
440
441  if (!filter_needs_more_output_space_ && !filter_->stream_data_len()) {
442    // We don't have any raw data to work with, so
443    // read from the socket.
444    int filtered_data_read;
445    if (ReadRawDataForFilter(&filtered_data_read)) {
446      if (filtered_data_read > 0) {
447        filter_->FlushStreamBuffer(filtered_data_read);  // Give data to filter.
448      } else {
449        return true;  // EOF
450      }
451    } else {
452      return false;  // IO Pending (or error)
453    }
454  }
455
456  if ((filter_->stream_data_len() || filter_needs_more_output_space_)
457      && !is_done()) {
458    // Get filtered data.
459    int filtered_data_len = filtered_read_buffer_len_;
460    Filter::FilterStatus status;
461    int output_buffer_size = filtered_data_len;
462    status = filter_->ReadData(filtered_read_buffer_->data(),
463                               &filtered_data_len);
464
465    if (filter_needs_more_output_space_ && 0 == filtered_data_len) {
466      // filter_needs_more_output_space_ was mistaken... there are no more bytes
467      // and we should have at least tried to fill up the filter's input buffer.
468      // Correct the state, and try again.
469      filter_needs_more_output_space_ = false;
470      return ReadFilteredData(bytes_read);
471    }
472
473    switch (status) {
474      case Filter::FILTER_DONE: {
475        filter_needs_more_output_space_ = false;
476        *bytes_read = filtered_data_len;
477        rv = true;
478        break;
479      }
480      case Filter::FILTER_NEED_MORE_DATA: {
481        filter_needs_more_output_space_ =
482            (filtered_data_len == output_buffer_size);
483        // We have finished filtering all data currently in the buffer.
484        // There might be some space left in the output buffer. One can
485        // consider reading more data from the stream to feed the filter
486        // and filling up the output buffer. This leads to more complicated
487        // buffer management and data notification mechanisms.
488        // We can revisit this issue if there is a real perf need.
489        if (filtered_data_len > 0) {
490          *bytes_read = filtered_data_len;
491          rv = true;
492        } else {
493          // Read again since we haven't received enough data yet (e.g., we may
494          // not have a complete gzip header yet)
495          rv = ReadFilteredData(bytes_read);
496        }
497        break;
498      }
499      case Filter::FILTER_OK: {
500        filter_needs_more_output_space_ =
501            (filtered_data_len == output_buffer_size);
502        *bytes_read = filtered_data_len;
503        rv = true;
504        break;
505      }
506      case Filter::FILTER_ERROR: {
507        filter_needs_more_output_space_ = false;
508        NotifyDone(URLRequestStatus(URLRequestStatus::FAILED,
509                   ERR_CONTENT_DECODING_FAILED));
510        rv = false;
511        break;
512      }
513      default: {
514        NOTREACHED();
515        filter_needs_more_output_space_ = false;
516        rv = false;
517        break;
518      }
519    }
520  } else {
521    // we are done, or there is no data left.
522    rv = true;
523  }
524
525  if (rv) {
526    // When we successfully finished a read, we no longer need to
527    // save the caller's buffers. Release our reference.
528    filtered_read_buffer_ = NULL;
529    filtered_read_buffer_len_ = 0;
530  }
531  return rv;
532}
533
534const URLRequestStatus URLRequestJob::GetStatus() {
535  if (request_)
536    return request_->status();
537  // If the request is gone, we must be cancelled.
538  return URLRequestStatus(URLRequestStatus::CANCELED,
539                          ERR_ABORTED);
540}
541
542void URLRequestJob::SetStatus(const URLRequestStatus &status) {
543  if (request_)
544    request_->set_status(status);
545}
546
547bool URLRequestJob::ReadRawDataForFilter(int* bytes_read) {
548  bool rv = false;
549
550  DCHECK(bytes_read);
551  DCHECK(filter_.get());
552
553  *bytes_read = 0;
554
555  // Get more pre-filtered data if needed.
556  // TODO(mbelshe): is it possible that the filter needs *MORE* data
557  //    when there is some data already in the buffer?
558  if (!filter_->stream_data_len() && !is_done()) {
559    IOBuffer* stream_buffer = filter_->stream_buffer();
560    int stream_buffer_size = filter_->stream_buffer_size();
561    rv = ReadRawDataHelper(stream_buffer, stream_buffer_size, bytes_read);
562  }
563  return rv;
564}
565
566bool URLRequestJob::ReadRawDataHelper(IOBuffer* buf, int buf_size,
567                                      int* bytes_read) {
568  DCHECK(!request_->status().is_io_pending());
569  DCHECK(raw_read_buffer_ == NULL);
570
571  // Keep a pointer to the read buffer, so we have access to it in the
572  // OnRawReadComplete() callback in the event that the read completes
573  // asynchronously.
574  raw_read_buffer_ = buf;
575  bool rv = ReadRawData(buf, buf_size, bytes_read);
576
577  if (!request_->status().is_io_pending()) {
578    // If the read completes synchronously, either success or failure,
579    // invoke the OnRawReadComplete callback so we can account for the
580    // completed read.
581    OnRawReadComplete(*bytes_read);
582  }
583  return rv;
584}
585
586void URLRequestJob::FollowRedirect(const GURL& location, int http_status_code) {
587  g_url_request_job_tracker.OnJobRedirect(this, location, http_status_code);
588
589  int rv = request_->Redirect(location, http_status_code);
590  if (rv != OK)
591    NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
592}
593
594void URLRequestJob::OnRawReadComplete(int bytes_read) {
595  DCHECK(raw_read_buffer_);
596  if (bytes_read > 0) {
597    RecordBytesRead(bytes_read);
598  }
599  raw_read_buffer_ = NULL;
600}
601
602void URLRequestJob::RecordBytesRead(int bytes_read) {
603  filter_input_byte_count_ += bytes_read;
604  UpdatePacketReadTimes();  // Facilitate stats recording if it is active.
605  g_url_request_job_tracker.OnBytesRead(this, raw_read_buffer_->data(),
606                                        bytes_read);
607}
608
609bool URLRequestJob::FilterHasData() {
610    return filter_.get() && filter_->stream_data_len();
611}
612
613void URLRequestJob::UpdatePacketReadTimes() {
614}
615
616}  // namespace net
617