libcurl_http_fetcher.cc revision 34135a9f24c72137aa85378b57e7698162c687f2 
1// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "update_engine/libcurl_http_fetcher.h"
6
7#include <algorithm>
8#include <string>
9
10#include <base/logging.h>
11#include <base/string_util.h>
12#include <base/stringprintf.h>
13
14#include "update_engine/certificate_checker.h"
15#include "update_engine/dbus_interface.h"
16#include "update_engine/utils.h"
17
18using google::protobuf::NewCallback;
19using std::max;
20using std::make_pair;
21using std::string;
22
23// This is a concrete implementation of HttpFetcher that uses libcurl to do the
24// http work.
25
26namespace chromeos_update_engine {
27
28namespace {
29const int kNoNetworkRetrySeconds = 10;
30const char kCACertificatesPath[] = "/usr/share/chromeos-ca-certificates";
31}  // namespace {}
32
33LibcurlHttpFetcher::~LibcurlHttpFetcher() {
34  LOG_IF(ERROR, transfer_in_progress_)
35      << "Destroying the fetcher while a transfer is in progress.";
36  CleanUp();
37}
38
39// On error, returns false.
40bool LibcurlHttpFetcher::IsUpdateAllowedOverCurrentConnection() const {
41  NetworkConnectionType type;
42  ConcreteDbusGlib dbus_iface;
43  ConnectionManager* connection_manager = system_state_->connection_manager();
44  TEST_AND_RETURN_FALSE(connection_manager->GetConnectionType(&dbus_iface,
45                                                              &type));
46  bool is_allowed = connection_manager->IsUpdateAllowedOver(type);
47  LOG(INFO) << "We are connected via "
48            << connection_manager->StringForConnectionType(type)
49            << ", Updates allowed: " << (is_allowed ? "Yes" : "No");
50  return is_allowed;
51}
52
53bool LibcurlHttpFetcher::IsOfficialBuild() const {
54  return force_build_type_ ? forced_official_build_ : utils::IsOfficialBuild();
55}
56
57bool LibcurlHttpFetcher::GetProxyType(const std::string& proxy,
58                                      curl_proxytype* out_type) {
59  if (utils::StringHasPrefix(proxy, "socks5://") ||
60      utils::StringHasPrefix(proxy, "socks://")) {
61    *out_type = CURLPROXY_SOCKS5_HOSTNAME;
62    return true;
63  }
64  if (utils::StringHasPrefix(proxy, "socks4://")) {
65    *out_type = CURLPROXY_SOCKS4A;
66    return true;
67  }
68  if (utils::StringHasPrefix(proxy, "http://") ||
69      utils::StringHasPrefix(proxy, "https://")) {
70    *out_type = CURLPROXY_HTTP;
71    return true;
72  }
73  if (utils::StringHasPrefix(proxy, kNoProxy)) {
74    // known failure case. don't log.
75    return false;
76  }
77  LOG(INFO) << "Unknown proxy type: " << proxy;
78  return false;
79}
80
81void LibcurlHttpFetcher::ResumeTransfer(const std::string& url) {
82  LOG(INFO) << "Starting/Resuming transfer";
83  CHECK(!transfer_in_progress_);
84  url_ = url;
85  curl_multi_handle_ = curl_multi_init();
86  CHECK(curl_multi_handle_);
87
88  curl_handle_ = curl_easy_init();
89  CHECK(curl_handle_);
90
91  CHECK(HasProxy());
92  bool is_direct = (GetCurrentProxy() == kNoProxy);
93  LOG(INFO) << "Using proxy: " << (is_direct ? "no" : "yes");
94  if (is_direct) {
95    CHECK_EQ(curl_easy_setopt(curl_handle_,
96                              CURLOPT_PROXY,
97                              ""), CURLE_OK);
98  } else {
99    CHECK_EQ(curl_easy_setopt(curl_handle_,
100                              CURLOPT_PROXY,
101                              GetCurrentProxy().c_str()), CURLE_OK);
102    // Curl seems to require us to set the protocol
103    curl_proxytype type;
104    if (GetProxyType(GetCurrentProxy(), &type)) {
105      CHECK_EQ(curl_easy_setopt(curl_handle_,
106                                CURLOPT_PROXYTYPE,
107                                type), CURLE_OK);
108    }
109  }
110
111  if (post_data_set_) {
112    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK);
113    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS,
114                              &post_data_[0]),
115             CURLE_OK);
116    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDSIZE,
117                              post_data_.size()),
118             CURLE_OK);
119
120    // Set the Content-Type HTTP header, if one was specifically set.
121    CHECK(!curl_http_headers_);
122    if (post_content_type_ != kHttpContentTypeUnspecified) {
123      const string content_type_attr =
124        base::StringPrintf("Content-Type: %s",
125                           GetHttpContentTypeString(post_content_type_));
126      curl_http_headers_ = curl_slist_append(NULL, content_type_attr.c_str());
127      CHECK(curl_http_headers_);
128      CHECK_EQ(
129          curl_easy_setopt(curl_handle_, CURLOPT_HTTPHEADER,
130                           curl_http_headers_),
131          CURLE_OK);
132    } else {
133      LOG(WARNING) << "no content type set, using libcurl default";
134    }
135  }
136
137  if (bytes_downloaded_ > 0 || download_length_) {
138    // Resume from where we left off.
139    resume_offset_ = bytes_downloaded_;
140    CHECK_GE(resume_offset_, 0);
141
142    // Compute end offset, if one is specified. As per HTTP specification, this
143    // is an inclusive boundary. Make sure it doesn't overflow.
144    size_t end_offset = 0;
145    if (download_length_) {
146      end_offset = static_cast<size_t>(resume_offset_) + download_length_ - 1;
147      CHECK_LE((size_t) resume_offset_, end_offset);
148    }
149
150    // Create a string representation of the desired range.
151    std::string range_str = (end_offset ?
152                             StringPrintf("%jd-%zu", resume_offset_,
153                                          end_offset) :
154                             StringPrintf("%jd-", resume_offset_));
155    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_RANGE, range_str.c_str()),
156             CURLE_OK);
157  }
158
159  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK);
160  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION,
161                            StaticLibcurlWrite), CURLE_OK);
162
163  string url_to_use(url_);
164  if (!IsUpdateAllowedOverCurrentConnection()) {
165    LOG(INFO) << "Not initiating HTTP connection b/c updates are disabled "
166              << "over this connection";
167    url_to_use = "";  // Sabotage the URL
168  }
169
170  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_to_use.c_str()),
171           CURLE_OK);
172
173  // If the connection drops under |low_speed_limit_bps_| (10
174  // bytes/sec by default) for |low_speed_time_seconds_| (90 seconds,
175  // 180 on non-official builds), reconnect.
176  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_LIMIT,
177                            low_speed_limit_bps_),
178           CURLE_OK);
179  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_TIME,
180                            low_speed_time_seconds_),
181           CURLE_OK);
182  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CONNECTTIMEOUT,
183                            connect_timeout_seconds_),
184           CURLE_OK);
185
186  // By default, libcurl doesn't follow redirections. Allow up to
187  // |kDownloadMaxRedirects| redirections.
188  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK);
189  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS,
190                            kDownloadMaxRedirects),
191           CURLE_OK);
192
193  // If we are running in test mode or using a dev/test build, then lock down
194  // the appropriate curl options for HTTP or HTTPS depending on the url.
195  if (!is_test_mode_ && IsOfficialBuild()) {
196    if (StartsWithASCII(url_to_use, "http://", false))
197      SetCurlOptionsForHttp();
198    else
199      SetCurlOptionsForHttps();
200  } else {
201    LOG(INFO) << "Not setting http(s) curl options because we are in "
202              << "test mode or running a dev/test image";
203  }
204
205  CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK);
206  transfer_in_progress_ = true;
207}
208
209// Lock down only the protocol in case of HTTP.
210void LibcurlHttpFetcher::SetCurlOptionsForHttp() {
211  LOG(INFO) << "Setting up curl options for HTTP";
212  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTP),
213           CURLE_OK);
214  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
215                            CURLPROTO_HTTP),
216           CURLE_OK);
217}
218
219// Security lock-down in official builds: makes sure that peer certificate
220// verification is enabled, restricts the set of trusted certificates,
221// restricts protocols to HTTPS, restricts ciphers to HIGH.
222void LibcurlHttpFetcher::SetCurlOptionsForHttps() {
223  LOG(INFO) << "Setting up curl options for HTTPS";
224  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1),
225           CURLE_OK);
226  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CAPATH, kCACertificatesPath),
227           CURLE_OK);
228  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS),
229           CURLE_OK);
230  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
231                            CURLPROTO_HTTPS),
232           CURLE_OK);
233  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST, "HIGH:!ADH"),
234           CURLE_OK);
235  if (check_certificate_ != CertificateChecker::kNone) {
236    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA,
237                              &check_certificate_),
238             CURLE_OK);
239    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_FUNCTION,
240                              CertificateChecker::ProcessSSLContext),
241             CURLE_OK);
242  }
243}
244
245
246// Begins the transfer, which must not have already been started.
247void LibcurlHttpFetcher::BeginTransfer(const std::string& url) {
248  CHECK(!transfer_in_progress_);
249  url_ = url;
250  if (!ResolveProxiesForUrl(
251          url_,
252          NewCallback(this, &LibcurlHttpFetcher::ProxiesResolved))) {
253    LOG(ERROR) << "Couldn't resolve proxies";
254    if (delegate_)
255      delegate_->TransferComplete(this, false);
256  }
257}
258
259void LibcurlHttpFetcher::ProxiesResolved() {
260  transfer_size_ = -1;
261  resume_offset_ = 0;
262  retry_count_ = 0;
263  no_network_retry_count_ = 0;
264  http_response_code_ = 0;
265  terminate_requested_ = false;
266  sent_byte_ = false;
267  ResumeTransfer(url_);
268  CurlPerformOnce();
269}
270
271void LibcurlHttpFetcher::ForceTransferTermination() {
272  CleanUp();
273  if (delegate_) {
274    // Note that after the callback returns this object may be destroyed.
275    delegate_->TransferTerminated(this);
276  }
277}
278
279void LibcurlHttpFetcher::TerminateTransfer() {
280  if (in_write_callback_) {
281    terminate_requested_ = true;
282  } else {
283    ForceTransferTermination();
284  }
285}
286
287void LibcurlHttpFetcher::CurlPerformOnce() {
288  CHECK(transfer_in_progress_);
289  int running_handles = 0;
290  CURLMcode retcode = CURLM_CALL_MULTI_PERFORM;
291
292  // libcurl may request that we immediately call curl_multi_perform after it
293  // returns, so we do. libcurl promises that curl_multi_perform will not block.
294  while (CURLM_CALL_MULTI_PERFORM == retcode) {
295    retcode = curl_multi_perform(curl_multi_handle_, &running_handles);
296    if (terminate_requested_) {
297      ForceTransferTermination();
298      return;
299    }
300  }
301  if (0 == running_handles) {
302    GetHttpResponseCode();
303    if (http_response_code_) {
304      LOG(INFO) << "HTTP response code: " << http_response_code_;
305      no_network_retry_count_ = 0;
306    } else {
307      LOG(ERROR) << "Unable to get http response code.";
308    }
309
310    // we're done!
311    CleanUp();
312
313    // TODO(petkov): This temporary code tries to deal with the case where the
314    // update engine performs an update check while the network is not ready
315    // (e.g., right after resume). Longer term, we should check if the network
316    // is online/offline and return an appropriate error code.
317    if (!sent_byte_ &&
318        http_response_code_ == 0 &&
319        no_network_retry_count_ < no_network_max_retries_) {
320      no_network_retry_count_++;
321      g_timeout_add_seconds(kNoNetworkRetrySeconds,
322                            &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
323                            this);
324      LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_;
325      return;
326    }
327
328    if ((!sent_byte_ && !IsHttpResponseSuccess()) || IsHttpResponseError()) {
329      // The transfer completed w/ error and we didn't get any bytes.
330      // If we have another proxy to try, try that.
331      //
332      // TODO(garnold) in fact there are two separate cases here: one case is an
333      // other-than-success return code (including no return code) and no
334      // received bytes, which is necessary due to the way callbacks are
335      // currently processing error conditions;  the second is an explicit HTTP
336      // error code, where some data may have been received (as in the case of a
337      // semi-successful multi-chunk fetch).  This is a confusing behavior and
338      // should be unified into a complete, coherent interface.
339      LOG(INFO) << "Transfer resulted in an error (" << http_response_code_
340                << "), " << bytes_downloaded_ << " bytes downloaded";
341
342      PopProxy();  // Delete the proxy we just gave up on.
343
344      if (HasProxy()) {
345        // We have another proxy. Retry immediately.
346        LOG(INFO) << "Retrying with next proxy setting";
347        g_idle_add(&LibcurlHttpFetcher::StaticRetryTimeoutCallback, this);
348      } else {
349        // Out of proxies. Give up.
350        LOG(INFO) << "No further proxies, indicating transfer complete";
351        if (delegate_)
352          delegate_->TransferComplete(this, false);  // signal fail
353      }
354    } else if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) {
355      retry_count_++;
356      LOG(INFO) << "Transfer interrupted after downloading "
357                << bytes_downloaded_ << " of " << transfer_size_ << " bytes. "
358                << transfer_size_ - bytes_downloaded_ << " bytes remaining "
359                << "after " << retry_count_ << " attempt(s)";
360
361      if (retry_count_ > max_retry_count_) {
362        LOG(INFO) << "Reached max attempts (" << retry_count_ << ")";
363        if (delegate_)
364          delegate_->TransferComplete(this, false);  // signal fail
365      } else {
366        // Need to restart transfer
367        LOG(INFO) << "Restarting transfer to download the remaining bytes";
368        g_timeout_add_seconds(retry_seconds_,
369                              &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
370                              this);
371      }
372    } else {
373      LOG(INFO) << "Transfer completed (" << http_response_code_
374                << "), " << bytes_downloaded_ << " bytes downloaded";
375      if (delegate_) {
376        bool success = IsHttpResponseSuccess();
377        delegate_->TransferComplete(this, success);
378      }
379    }
380  } else {
381    // set up callback
382    SetupMainloopSources();
383  }
384}
385
386size_t LibcurlHttpFetcher::LibcurlWrite(void *ptr, size_t size, size_t nmemb) {
387  // Update HTTP response first.
388  GetHttpResponseCode();
389  const size_t payload_size = size * nmemb;
390
391  // Do nothing if no payload or HTTP response is an error.
392  if (payload_size == 0 || !IsHttpResponseSuccess()) {
393    LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_
394              << ") or no payload (" << payload_size << "), nothing to do";
395    return 0;
396  }
397
398  sent_byte_ = true;
399  {
400    double transfer_size_double;
401    CHECK_EQ(curl_easy_getinfo(curl_handle_,
402                               CURLINFO_CONTENT_LENGTH_DOWNLOAD,
403                               &transfer_size_double), CURLE_OK);
404    off_t new_transfer_size = static_cast<off_t>(transfer_size_double);
405    if (new_transfer_size > 0) {
406      transfer_size_ = resume_offset_ + new_transfer_size;
407    }
408  }
409  bytes_downloaded_ += payload_size;
410  in_write_callback_ = true;
411  if (delegate_)
412    delegate_->ReceivedBytes(this, reinterpret_cast<char*>(ptr), payload_size);
413  in_write_callback_ = false;
414  return payload_size;
415}
416
417void LibcurlHttpFetcher::Pause() {
418  CHECK(curl_handle_);
419  CHECK(transfer_in_progress_);
420  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK);
421}
422
423void LibcurlHttpFetcher::Unpause() {
424  CHECK(curl_handle_);
425  CHECK(transfer_in_progress_);
426  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK);
427}
428
429// This method sets up callbacks with the glib main loop.
430void LibcurlHttpFetcher::SetupMainloopSources() {
431  fd_set fd_read;
432  fd_set fd_write;
433  fd_set fd_exc;
434
435  FD_ZERO(&fd_read);
436  FD_ZERO(&fd_write);
437  FD_ZERO(&fd_exc);
438
439  int fd_max = 0;
440
441  // Ask libcurl for the set of file descriptors we should track on its
442  // behalf.
443  CHECK_EQ(curl_multi_fdset(curl_multi_handle_, &fd_read, &fd_write,
444                            &fd_exc, &fd_max), CURLM_OK);
445
446  // We should iterate through all file descriptors up to libcurl's fd_max or
447  // the highest one we're tracking, whichever is larger.
448  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
449    if (!io_channels_[t].empty())
450      fd_max = max(fd_max, io_channels_[t].rbegin()->first);
451  }
452
453  // For each fd, if we're not tracking it, track it. If we are tracking it, but
454  // libcurl doesn't care about it anymore, stop tracking it. After this loop,
455  // there should be exactly as many GIOChannel objects in io_channels_[0|1] as
456  // there are read/write fds that we're tracking.
457  for (int fd = 0; fd <= fd_max; ++fd) {
458    // Note that fd_exc is unused in the current version of libcurl so is_exc
459    // should always be false.
460    bool is_exc = FD_ISSET(fd, &fd_exc) != 0;
461    bool must_track[2] = {
462      is_exc || (FD_ISSET(fd, &fd_read) != 0),  // track 0 -- read
463      is_exc || (FD_ISSET(fd, &fd_write) != 0)  // track 1 -- write
464    };
465
466    for (size_t t = 0; t < arraysize(io_channels_); ++t) {
467      bool tracked = io_channels_[t].find(fd) != io_channels_[t].end();
468
469      if (!must_track[t]) {
470        // If we have an outstanding io_channel, remove it.
471        if (tracked) {
472          g_source_remove(io_channels_[t][fd].second);
473          g_io_channel_unref(io_channels_[t][fd].first);
474          io_channels_[t].erase(io_channels_[t].find(fd));
475        }
476        continue;
477      }
478
479      // If we are already tracking this fd, continue -- nothing to do.
480      if (tracked)
481        continue;
482
483      // Set conditions appropriately -- read for track 0, write for track 1.
484      GIOCondition condition = static_cast<GIOCondition>(
485          ((t == 0) ? (G_IO_IN | G_IO_PRI) : G_IO_OUT) | G_IO_ERR | G_IO_HUP);
486
487      // Track a new fd.
488      GIOChannel* io_channel = g_io_channel_unix_new(fd);
489      guint tag =
490          g_io_add_watch(io_channel, condition, &StaticFDCallback, this);
491
492      io_channels_[t][fd] = make_pair(io_channel, tag);
493      static int io_counter = 0;
494      io_counter++;
495      if (io_counter % 50 == 0) {
496        LOG(INFO) << "io_counter = " << io_counter;
497      }
498    }
499  }
500
501  // Set up a timeout callback for libcurl.
502  if (!timeout_source_) {
503    LOG(INFO) << "Setting up timeout source: " << idle_seconds_ << " seconds.";
504    timeout_source_ = g_timeout_source_new_seconds(idle_seconds_);
505    g_source_set_callback(timeout_source_, StaticTimeoutCallback, this, NULL);
506    g_source_attach(timeout_source_, NULL);
507  }
508}
509
510bool LibcurlHttpFetcher::FDCallback(GIOChannel *source,
511                                    GIOCondition condition) {
512  CurlPerformOnce();
513  // We handle removing of this source elsewhere, so we always return true.
514  // The docs say, "the function should return FALSE if the event source
515  // should be removed."
516  // http://www.gtk.org/api/2.6/glib/glib-IO-Channels.html#GIOFunc
517  return true;
518}
519
520gboolean LibcurlHttpFetcher::RetryTimeoutCallback() {
521  ResumeTransfer(url_);
522  CurlPerformOnce();
523  return FALSE;  // Don't have glib auto call this callback again
524}
525
526gboolean LibcurlHttpFetcher::TimeoutCallback() {
527  // We always return true, even if we don't want glib to call us back.
528  // We will remove the event source separately if we don't want to
529  // be called back.
530  if (!transfer_in_progress_)
531    return TRUE;
532  CurlPerformOnce();
533  return TRUE;
534}
535
536void LibcurlHttpFetcher::CleanUp() {
537  if (timeout_source_) {
538    g_source_destroy(timeout_source_);
539    timeout_source_ = NULL;
540  }
541
542  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
543    for (IOChannels::iterator it = io_channels_[t].begin();
544         it != io_channels_[t].end(); ++it) {
545      g_source_remove(it->second.second);
546      g_io_channel_unref(it->second.first);
547    }
548    io_channels_[t].clear();
549  }
550
551  if (curl_http_headers_) {
552    curl_slist_free_all(curl_http_headers_);
553    curl_http_headers_ = NULL;
554  }
555  if (curl_handle_) {
556    if (curl_multi_handle_) {
557      CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_),
558               CURLM_OK);
559    }
560    curl_easy_cleanup(curl_handle_);
561    curl_handle_ = NULL;
562  }
563  if (curl_multi_handle_) {
564    CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK);
565    curl_multi_handle_ = NULL;
566  }
567  transfer_in_progress_ = false;
568}
569
570void LibcurlHttpFetcher::GetHttpResponseCode() {
571  long http_response_code = 0;
572  if (curl_easy_getinfo(curl_handle_,
573                        CURLINFO_RESPONSE_CODE,
574                        &http_response_code) == CURLE_OK) {
575    http_response_code_ = static_cast<int>(http_response_code);
576  }
577}
578
579}  // namespace chromeos_update_engine
580