1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/automation/url_request_automation_job.h"
6
7#include "base/bind.h"
8#include "base/compiler_specific.h"
9#include "base/message_loop/message_loop.h"
10#include "base/time/time.h"
11#include "chrome/browser/automation/automation_resource_message_filter.h"
12#include "chrome/common/automation_messages.h"
13#include "content/public/browser/browser_thread.h"
14#include "content/public/browser/render_view_host.h"
15#include "content/public/browser/resource_request_info.h"
16#include "net/base/host_port_pair.h"
17#include "net/base/io_buffer.h"
18#include "net/base/net_errors.h"
19#include "net/base/upload_bytes_element_reader.h"
20#include "net/base/upload_data_stream.h"
21#include "net/base/upload_file_element_reader.h"
22#include "net/cookies/cookie_monster.h"
23#include "net/http/http_request_headers.h"
24#include "net/http/http_response_headers.h"
25#include "net/http/http_util.h"
26#include "net/url_request/http_user_agent_settings.h"
27#include "net/url_request/url_request.h"
28#include "net/url_request/url_request_context.h"
29
30using base::Time;
31using base::TimeDelta;
32using content::BrowserThread;
33using content::ResourceRequestInfo;
34
35namespace {
36
37// The list of filtered headers that are removed from requests sent via
38// StartAsync(). These must be lower case.
39const char* const kFilteredHeaderStrings[] = {
40  "connection",
41  "cookie",
42  "expect",
43  "max-forwards",
44  "proxy-authorization",
45  "referer",
46  "te",
47  "upgrade",
48  "via"
49};
50
51// Creates UploadData from UploadDataStream.
52net::UploadData* CreateUploadData(
53    const net::UploadDataStream* upload_data_stream) {
54  net::UploadData* upload_data = new net::UploadData();
55  const ScopedVector<net::UploadElementReader>& element_readers =
56      upload_data_stream->element_readers();
57  for (size_t i = 0; i < element_readers.size(); ++i) {
58    const net::UploadElementReader* reader = element_readers[i];
59    if (reader->AsBytesReader()) {
60      const net::UploadBytesElementReader* bytes_reader =
61          reader->AsBytesReader();
62      upload_data->AppendBytes(bytes_reader->bytes(), bytes_reader->length());
63    } else if (reader->AsFileReader()) {
64      const net::UploadFileElementReader* file_reader =
65          reader->AsFileReader();
66      upload_data->AppendFileRange(file_reader->path(),
67                                   file_reader->range_offset(),
68                                   file_reader->range_length(),
69                                   file_reader->expected_modification_time());
70    } else {
71      NOTIMPLEMENTED();
72    }
73  }
74  upload_data->set_identifier(upload_data_stream->identifier());
75  upload_data->set_is_chunked(upload_data_stream->is_chunked());
76  upload_data->set_last_chunk_appended(
77      upload_data_stream->last_chunk_appended());
78  return upload_data;
79}
80
81}  // namespace
82
83int URLRequestAutomationJob::instance_count_ = 0;
84bool URLRequestAutomationJob::is_protocol_factory_registered_ = false;
85
86net::URLRequest::ProtocolFactory* URLRequestAutomationJob::old_http_factory_
87    = NULL;
88net::URLRequest::ProtocolFactory* URLRequestAutomationJob::old_https_factory_
89    = NULL;
90
91URLRequestAutomationJob::URLRequestAutomationJob(
92    net::URLRequest* request,
93    net::NetworkDelegate* network_delegate,
94    const net::HttpUserAgentSettings* http_user_agent_settings,
95    int tab,
96    int request_id,
97    AutomationResourceMessageFilter* filter,
98    bool is_pending)
99    : net::URLRequestJob(request, network_delegate),
100      http_user_agent_settings_(http_user_agent_settings),
101      id_(0),
102      tab_(tab),
103      message_filter_(filter),
104      pending_buf_size_(0),
105      redirect_status_(0),
106      request_id_(request_id),
107      is_pending_(is_pending),
108      upload_size_(0),
109      weak_factory_(this) {
110  DVLOG(1) << "URLRequestAutomationJob create. Count: " << ++instance_count_;
111  DCHECK(message_filter_.get() != NULL);
112
113  if (message_filter_.get()) {
114    id_ = message_filter_->NewAutomationRequestId();
115    DCHECK_NE(id_, 0);
116  }
117}
118
119URLRequestAutomationJob::~URLRequestAutomationJob() {
120  DVLOG(1) << "URLRequestAutomationJob delete. Count: " << --instance_count_;
121  Cleanup();
122}
123
124void URLRequestAutomationJob::EnsureProtocolFactoryRegistered() {
125  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
126
127  if (!is_protocol_factory_registered_) {
128    old_http_factory_ =
129        net::URLRequest::Deprecated::RegisterProtocolFactory(
130            "http", &URLRequestAutomationJob::Factory);
131    old_https_factory_ =
132        net::URLRequest::Deprecated::RegisterProtocolFactory(
133            "https", &URLRequestAutomationJob::Factory);
134    is_protocol_factory_registered_ = true;
135  }
136}
137
138net::URLRequestJob* URLRequestAutomationJob::Factory(
139    net::URLRequest* request,
140    net::NetworkDelegate* network_delegate,
141    const std::string& scheme) {
142  bool scheme_is_http = request->url().SchemeIs("http");
143  bool scheme_is_https = request->url().SchemeIs("https");
144
145  // Returning null here just means that the built-in handler will be used.
146  if (scheme_is_http || scheme_is_https) {
147    const ResourceRequestInfo* info = ResourceRequestInfo::ForRequest(request);
148    if (info) {
149      int child_id = info->GetChildID();
150      int route_id = info->GetRouteID();
151      AutomationResourceMessageFilter::AutomationDetails details;
152      if (AutomationResourceMessageFilter::LookupRegisteredRenderView(
153              child_id, route_id, &details)) {
154        URLRequestAutomationJob* job = new URLRequestAutomationJob(
155            request,
156            network_delegate,
157            request->context()->http_user_agent_settings(),
158            details.tab_handle,
159            info->GetRequestID(),
160            details.filter.get(),
161            details.is_pending_render_view);
162        return job;
163      }
164    }
165
166    if (scheme_is_http && old_http_factory_)
167      return old_http_factory_(request, network_delegate, scheme);
168    else if (scheme_is_https && old_https_factory_)
169      return old_https_factory_(request, network_delegate, scheme);
170  }
171  return NULL;
172}
173
174// net::URLRequestJob Implementation.
175void URLRequestAutomationJob::Start() {
176  if (!is_pending()) {
177    // Start reading asynchronously so that all error reporting and data
178    // callbacks happen as they would for network requests.
179    base::MessageLoop::current()->PostTask(
180        FROM_HERE,
181        base::Bind(&URLRequestAutomationJob::StartAsync,
182                   weak_factory_.GetWeakPtr()));
183  } else {
184    // If this is a pending job, then register it immediately with the message
185    // filter so it can be serviced later when we receive a request from the
186    // external host to connect to the corresponding external tab.
187    message_filter_->RegisterRequest(this);
188  }
189}
190
191void URLRequestAutomationJob::Kill() {
192  if (message_filter_.get()) {
193    if (!is_pending()) {
194      message_filter_->Send(new AutomationMsg_RequestEnd(tab_, id_,
195          net::URLRequestStatus(net::URLRequestStatus::CANCELED,
196                                net::ERR_ABORTED)));
197    }
198  }
199  DisconnectFromMessageFilter();
200  receive_headers_end_ = base::TimeTicks();
201  net::URLRequestJob::Kill();
202}
203
204bool URLRequestAutomationJob::ReadRawData(
205    net::IOBuffer* buf, int buf_size, int* bytes_read) {
206  DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
207           << " - read pending: " << buf_size;
208
209  // We should not receive a read request for a pending job.
210  DCHECK(!is_pending());
211
212  pending_buf_ = buf;
213  pending_buf_size_ = buf_size;
214
215  if (message_filter_.get()) {
216    message_filter_->Send(new AutomationMsg_RequestRead(tab_, id_, buf_size));
217    SetStatus(net::URLRequestStatus(net::URLRequestStatus::IO_PENDING, 0));
218  } else {
219    base::MessageLoop::current()->PostTask(
220        FROM_HERE,
221        base::Bind(&URLRequestAutomationJob::NotifyJobCompletionTask,
222                   weak_factory_.GetWeakPtr()));
223  }
224  return false;
225}
226
227bool URLRequestAutomationJob::GetMimeType(std::string* mime_type) const {
228  if (!mime_type_.empty()) {
229    *mime_type = mime_type_;
230  } else if (headers_.get()) {
231    headers_->GetMimeType(mime_type);
232  }
233
234  return (!mime_type->empty());
235}
236
237bool URLRequestAutomationJob::GetCharset(std::string* charset) {
238  if (headers_.get())
239    return headers_->GetCharset(charset);
240  return false;
241}
242
243void URLRequestAutomationJob::GetResponseInfo(net::HttpResponseInfo* info) {
244  if (headers_.get())
245    info->headers = headers_;
246  if (request_->url().SchemeIsSecure()) {
247    // Make up a fake certificate for this response since we don't have
248    // access to the real SSL info.
249    const char* kCertIssuer = "Chrome Internal";
250    const int kLifetimeDays = 100;
251
252    info->ssl_info.cert =
253        new net::X509Certificate(request_->url().GetWithEmptyPath().spec(),
254                                 kCertIssuer,
255                                 Time::Now(),
256                                 Time::Now() +
257                                     TimeDelta::FromDays(kLifetimeDays));
258    info->ssl_info.cert_status = 0;
259    info->ssl_info.security_bits = -1;
260  }
261}
262
263void URLRequestAutomationJob::GetLoadTimingInfo(
264    net::LoadTimingInfo* load_timing_info) const {
265  if (!receive_headers_end_.is_null()) {
266    load_timing_info->send_start = request_start_;
267    // The send ended some time ago, but that information is not available on
268    // this side of the automation channel. Consider the send to have ended at
269    // the same time we received the response headers.
270    load_timing_info->send_end = receive_headers_end_;
271    load_timing_info->receive_headers_end = receive_headers_end_;
272  }
273}
274
275int URLRequestAutomationJob::GetResponseCode() const {
276  if (headers_.get())
277    return headers_->response_code();
278
279  static const int kDefaultResponseCode = 200;
280  return kDefaultResponseCode;
281}
282
283bool URLRequestAutomationJob::IsRedirectResponse(
284    GURL* location, int* http_status_code) {
285  if (!net::HttpResponseHeaders::IsRedirectResponseCode(redirect_status_))
286    return false;
287
288  *http_status_code = redirect_status_;
289  *location = GURL(redirect_url_);
290  return true;
291}
292
293net::UploadProgress URLRequestAutomationJob::GetUploadProgress() const {
294  uint64 progress = 0;
295  if (request_ && request_->status().is_success()) {
296    // We don't support incremental progress notifications in ChromeFrame. When
297    // we receive a response for the POST request from Chromeframe, it means
298    // that the upload is fully complete.
299    progress = upload_size_;
300  }
301  return net::UploadProgress(progress, upload_size_);
302}
303
304net::HostPortPair URLRequestAutomationJob::GetSocketAddress() const {
305  return socket_address_;
306}
307
308bool URLRequestAutomationJob::MayFilterMessage(const IPC::Message& message,
309                                               int* request_id) {
310  switch (message.type()) {
311    case AutomationMsg_RequestStarted::ID:
312    case AutomationMsg_RequestData::ID:
313    case AutomationMsg_RequestEnd::ID: {
314      PickleIterator iter(message);
315      if (message.ReadInt(&iter, request_id))
316        return true;
317      break;
318    }
319  }
320
321  return false;
322}
323
324void URLRequestAutomationJob::OnMessage(const IPC::Message& message) {
325  if (!request_) {
326    NOTREACHED() << __FUNCTION__
327                 << ": Unexpected request received for job:"
328                 << id();
329    return;
330  }
331
332  bool deserialize_success = false;
333  IPC_BEGIN_MESSAGE_MAP_EX(URLRequestAutomationJob,
334                           message,
335                           deserialize_success)
336    IPC_MESSAGE_HANDLER(AutomationMsg_RequestStarted, OnRequestStarted)
337    IPC_MESSAGE_HANDLER(AutomationMsg_RequestData, OnDataAvailable)
338    IPC_MESSAGE_HANDLER(AutomationMsg_RequestEnd, OnRequestEnd)
339  IPC_END_MESSAGE_MAP_EX()
340
341  if (!deserialize_success) {
342    LOG(ERROR) << "Failed to deserialize IPC message.";
343  }
344}
345
346void URLRequestAutomationJob::OnRequestStarted(
347    int id, const AutomationURLResponse& response) {
348  DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
349           << " - response started.";
350  set_expected_content_size(response.content_length);
351  mime_type_ = response.mime_type;
352
353  receive_headers_end_ = base::TimeTicks::Now();
354
355  redirect_url_ = response.redirect_url;
356  redirect_status_ = response.redirect_status;
357  DCHECK(redirect_status_ == 0 || redirect_status_ == 200 ||
358         (redirect_status_ >= 300 && redirect_status_ < 400));
359
360  if (!response.headers.empty()) {
361    headers_ = new net::HttpResponseHeaders(
362        net::HttpUtil::AssembleRawHeaders(response.headers.data(),
363                                          response.headers.size()));
364  }
365  socket_address_ = response.socket_address;
366  upload_size_ = response.upload_size;
367  NotifyHeadersComplete();
368}
369
370void URLRequestAutomationJob::OnDataAvailable(
371    int id, const std::string& bytes) {
372  DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
373           << " - data available, Size: " << bytes.size();
374  DCHECK(!bytes.empty());
375
376  // The request completed, and we have all the data.
377  // Clear any IO pending status.
378  SetStatus(net::URLRequestStatus());
379
380  if (pending_buf_.get() && pending_buf_->data()) {
381    DCHECK_GE(pending_buf_size_, bytes.size());
382    const int bytes_to_copy = std::min(bytes.size(), pending_buf_size_);
383    memcpy(pending_buf_->data(), &bytes[0], bytes_to_copy);
384
385    pending_buf_ = NULL;
386    pending_buf_size_ = 0;
387
388    NotifyReadComplete(bytes_to_copy);
389  } else {
390    NOTREACHED() << "Received unexpected data of length:" << bytes.size();
391  }
392}
393
394void URLRequestAutomationJob::OnRequestEnd(
395    int id, const net::URLRequestStatus& status) {
396#ifndef NDEBUG
397  std::string url;
398  if (request_)
399    url = request_->url().spec();
400  DVLOG(1) << "URLRequestAutomationJob: " << url << " - request end. Status: "
401           << status.status();
402#endif
403
404  // TODO(tommi): When we hit certificate errors, notify the delegate via
405  // OnSSLCertificateError().  Right now we don't have the certificate
406  // so we don't.  We could possibly call OnSSLCertificateError with a NULL
407  // certificate, but I'm not sure if all implementations expect it.
408  // if (status.status() == net::URLRequestStatus::FAILED &&
409  //    net::IsCertificateError(status.error()) && request_->delegate()) {
410  //  request_->delegate()->OnSSLCertificateError(request_, status.error());
411  // }
412
413  DisconnectFromMessageFilter();
414  // NotifyDone may have been called on the job if the original request was
415  // redirected.
416  if (!is_done()) {
417    // We can complete the job if we have a valid response or a pending read.
418    // An end request can be received in the following cases
419    // 1. We failed to connect to the server, in which case we did not receive
420    //    a valid response.
421    // 2. In response to a read request.
422    if (!has_response_started()) {
423      NotifyStartError(status);
424    } else if (pending_buf_.get()) {
425      pending_buf_ = NULL;
426      pending_buf_size_ = 0;
427      NotifyDone(status);
428      NotifyReadComplete(0);
429    } else {
430      // Wait for the http stack to issue a Read request where we will notify
431      // that the job has completed.
432      request_status_ = status;
433    }
434  }
435  // Note
436  // The job could have been destroyed above. Please don't attempt to access
437  // member variables here.
438}
439
440void URLRequestAutomationJob::Cleanup() {
441  headers_ = NULL;
442  mime_type_.erase();
443
444  id_ = 0;
445  tab_ = 0;
446
447  DCHECK(!message_filter_.get());
448  DisconnectFromMessageFilter();
449
450  pending_buf_ = NULL;
451  pending_buf_size_ = 0;
452}
453
454void URLRequestAutomationJob::StartAsync() {
455  DVLOG(1) << "URLRequestAutomationJob: start request: "
456           << (request_ ? request_->url().spec() : "NULL request");
457
458  // If the job is cancelled before we got a chance to start it
459  // we have nothing much to do here.
460  if (is_done())
461    return;
462
463  // We should not receive a Start request for a pending job.
464  DCHECK(!is_pending());
465
466  if (!request_) {
467    NotifyStartError(net::URLRequestStatus(net::URLRequestStatus::FAILED,
468                                           net::ERR_FAILED));
469    return;
470  }
471
472  // Register this request with automation message filter.
473  message_filter_->RegisterRequest(this);
474
475  // Strip unwanted headers.
476  net::HttpRequestHeaders new_request_headers;
477  new_request_headers.MergeFrom(request_->extra_request_headers());
478  for (size_t i = 0; i < arraysize(kFilteredHeaderStrings); ++i)
479    new_request_headers.RemoveHeader(kFilteredHeaderStrings[i]);
480
481  // Only add default Accept-Language if the request didn't have it specified.
482  if (!new_request_headers.HasHeader(
483      net::HttpRequestHeaders::kAcceptLanguage) &&
484      http_user_agent_settings_) {
485    std::string accept_language =
486        http_user_agent_settings_->GetAcceptLanguage();
487    if (!accept_language.empty()) {
488      new_request_headers.SetHeader(net::HttpRequestHeaders::kAcceptLanguage,
489                                    accept_language);
490    }
491  }
492
493  // URLRequest::SetReferrer() ensures that we do not send username and
494  // password fields in the referrer.
495  GURL referrer(request_->referrer());
496
497  // The referrer header must be suppressed if the preceding URL was
498  // a secure one and the new one is not.
499  if (referrer.SchemeIsSecure() && !request_->url().SchemeIsSecure()) {
500    DVLOG(1) << "Suppressing referrer header since going from secure to "
501                "non-secure";
502    referrer = GURL();
503  }
504
505  // Get the resource type (main_frame/script/image/stylesheet etc.
506  const ResourceRequestInfo* info = ResourceRequestInfo::ForRequest(request_);
507  ResourceType::Type resource_type = ResourceType::MAIN_FRAME;
508  if (info) {
509    resource_type = info->GetResourceType();
510  }
511
512  // Construct UploadData from UploadDataStream.
513  scoped_refptr<net::UploadData> upload_data;
514  if (request_->get_upload())
515    upload_data = CreateUploadData(request_->get_upload());
516
517  request_start_ = base::TimeTicks::Now();
518
519  // Ask automation to start this request.
520  AutomationURLRequest automation_request;
521  automation_request.url = request_->url().spec();
522  automation_request.method = request_->method();
523  automation_request.referrer = referrer.spec();
524  automation_request.extra_request_headers = new_request_headers.ToString();
525  automation_request.upload_data = upload_data;
526  automation_request.resource_type = resource_type;
527  automation_request.load_flags = request_->load_flags();
528
529  DCHECK(message_filter_.get());
530  message_filter_->Send(
531      new AutomationMsg_RequestStart(tab_, id_, automation_request));
532}
533
534void URLRequestAutomationJob::DisconnectFromMessageFilter() {
535  if (message_filter_.get()) {
536    message_filter_->UnRegisterRequest(this);
537    message_filter_ = NULL;
538  }
539}
540
541void URLRequestAutomationJob::StartPendingJob(
542    int new_tab_handle,
543    AutomationResourceMessageFilter* new_filter) {
544  DCHECK(new_filter != NULL);
545  tab_ = new_tab_handle;
546  message_filter_ = new_filter;
547  is_pending_ = false;
548  Start();
549}
550
551void URLRequestAutomationJob::NotifyJobCompletionTask() {
552  if (!is_done()) {
553    NotifyDone(request_status_);
554  }
555  // Reset any pending reads.
556  if (pending_buf_.get()) {
557    pending_buf_ = NULL;
558    pending_buf_size_ = 0;
559    NotifyReadComplete(0);
560  }
561}
562