libcurl_http_fetcher.cc revision 7d623ebf9eee346fdeee71f1ccb32ebaf4bd3318 
1// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "update_engine/libcurl_http_fetcher.h"
6
7#include <algorithm>
8#include <string>
9
10#include <base/logging.h>
11#include <base/strings/string_util.h>
12#include <base/strings/stringprintf.h>
13
14#include "update_engine/certificate_checker.h"
15#include "update_engine/hardware_interface.h"
16#include "update_engine/real_dbus_wrapper.h"
17#include "update_engine/utils.h"
18
19using google::protobuf::NewCallback;
20using std::max;
21using std::make_pair;
22using std::string;
23
24// This is a concrete implementation of HttpFetcher that uses libcurl to do the
25// http work.
26
27namespace chromeos_update_engine {
28
29namespace {
30const int kNoNetworkRetrySeconds = 10;
31const char kCACertificatesPath[] = "/usr/share/chromeos-ca-certificates";
32}  // namespace {}
33
34LibcurlHttpFetcher::~LibcurlHttpFetcher() {
35  LOG_IF(ERROR, transfer_in_progress_)
36      << "Destroying the fetcher while a transfer is in progress.";
37  CleanUp();
38}
39
40// On error, returns false.
41bool LibcurlHttpFetcher::IsUpdateAllowedOverCurrentConnection() const {
42  NetworkConnectionType type;
43  NetworkTethering tethering;
44  RealDBusWrapper dbus_iface;
45  ConnectionManager* connection_manager = system_state_->connection_manager();
46  if (!connection_manager->GetConnectionProperties(&dbus_iface,
47                                                   &type, &tethering)) {
48    LOG(INFO) << "We could not determine our connection type. "
49              << "Defaulting to allow updates.";
50    return true;
51  }
52  bool is_allowed = connection_manager->IsUpdateAllowedOver(type, tethering);
53  LOG(INFO) << "We are connected via "
54            << connection_manager->StringForConnectionType(type)
55            << ", Updates allowed: " << (is_allowed ? "Yes" : "No");
56  return is_allowed;
57}
58
59bool LibcurlHttpFetcher::GetProxyType(const std::string& proxy,
60                                      curl_proxytype* out_type) {
61  if (utils::StringHasPrefix(proxy, "socks5://") ||
62      utils::StringHasPrefix(proxy, "socks://")) {
63    *out_type = CURLPROXY_SOCKS5_HOSTNAME;
64    return true;
65  }
66  if (utils::StringHasPrefix(proxy, "socks4://")) {
67    *out_type = CURLPROXY_SOCKS4A;
68    return true;
69  }
70  if (utils::StringHasPrefix(proxy, "http://") ||
71      utils::StringHasPrefix(proxy, "https://")) {
72    *out_type = CURLPROXY_HTTP;
73    return true;
74  }
75  if (utils::StringHasPrefix(proxy, kNoProxy)) {
76    // known failure case. don't log.
77    return false;
78  }
79  LOG(INFO) << "Unknown proxy type: " << proxy;
80  return false;
81}
82
83void LibcurlHttpFetcher::ResumeTransfer(const std::string& url) {
84  LOG(INFO) << "Starting/Resuming transfer";
85  CHECK(!transfer_in_progress_);
86  url_ = url;
87  curl_multi_handle_ = curl_multi_init();
88  CHECK(curl_multi_handle_);
89
90  curl_handle_ = curl_easy_init();
91  CHECK(curl_handle_);
92
93  CHECK(HasProxy());
94  bool is_direct = (GetCurrentProxy() == kNoProxy);
95  LOG(INFO) << "Using proxy: " << (is_direct ? "no" : "yes");
96  if (is_direct) {
97    CHECK_EQ(curl_easy_setopt(curl_handle_,
98                              CURLOPT_PROXY,
99                              ""), CURLE_OK);
100  } else {
101    CHECK_EQ(curl_easy_setopt(curl_handle_,
102                              CURLOPT_PROXY,
103                              GetCurrentProxy().c_str()), CURLE_OK);
104    // Curl seems to require us to set the protocol
105    curl_proxytype type;
106    if (GetProxyType(GetCurrentProxy(), &type)) {
107      CHECK_EQ(curl_easy_setopt(curl_handle_,
108                                CURLOPT_PROXYTYPE,
109                                type), CURLE_OK);
110    }
111  }
112
113  if (post_data_set_) {
114    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK);
115    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS,
116                              &post_data_[0]),
117             CURLE_OK);
118    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDSIZE,
119                              post_data_.size()),
120             CURLE_OK);
121
122    // Set the Content-Type HTTP header, if one was specifically set.
123    CHECK(!curl_http_headers_);
124    if (post_content_type_ != kHttpContentTypeUnspecified) {
125      const string content_type_attr =
126        base::StringPrintf("Content-Type: %s",
127                           GetHttpContentTypeString(post_content_type_));
128      curl_http_headers_ = curl_slist_append(NULL, content_type_attr.c_str());
129      CHECK(curl_http_headers_);
130      CHECK_EQ(
131          curl_easy_setopt(curl_handle_, CURLOPT_HTTPHEADER,
132                           curl_http_headers_),
133          CURLE_OK);
134    } else {
135      LOG(WARNING) << "no content type set, using libcurl default";
136    }
137  }
138
139  if (bytes_downloaded_ > 0 || download_length_) {
140    // Resume from where we left off.
141    resume_offset_ = bytes_downloaded_;
142    CHECK_GE(resume_offset_, 0);
143
144    // Compute end offset, if one is specified. As per HTTP specification, this
145    // is an inclusive boundary. Make sure it doesn't overflow.
146    size_t end_offset = 0;
147    if (download_length_) {
148      end_offset = static_cast<size_t>(resume_offset_) + download_length_ - 1;
149      CHECK_LE((size_t) resume_offset_, end_offset);
150    }
151
152    // Create a string representation of the desired range.
153    std::string range_str = (end_offset ?
154                             base::StringPrintf("%jd-%zu", resume_offset_,
155                                                end_offset) :
156                             base::StringPrintf("%jd-", resume_offset_));
157    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_RANGE, range_str.c_str()),
158             CURLE_OK);
159  }
160
161  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK);
162  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION,
163                            StaticLibcurlWrite), CURLE_OK);
164
165  string url_to_use(url_);
166  if (!IsUpdateAllowedOverCurrentConnection()) {
167    LOG(INFO) << "Not initiating HTTP connection b/c updates are disabled "
168              << "over this connection";
169    url_to_use = "";  // Sabotage the URL
170  }
171
172  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_to_use.c_str()),
173           CURLE_OK);
174
175  // If the connection drops under |low_speed_limit_bps_| (10
176  // bytes/sec by default) for |low_speed_time_seconds_| (90 seconds,
177  // 180 on non-official builds), reconnect.
178  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_LIMIT,
179                            low_speed_limit_bps_),
180           CURLE_OK);
181  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_TIME,
182                            low_speed_time_seconds_),
183           CURLE_OK);
184  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CONNECTTIMEOUT,
185                            connect_timeout_seconds_),
186           CURLE_OK);
187
188  // By default, libcurl doesn't follow redirections. Allow up to
189  // |kDownloadMaxRedirects| redirections.
190  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK);
191  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS,
192                            kDownloadMaxRedirects),
193           CURLE_OK);
194
195  // Lock down the appropriate curl options for HTTP or HTTPS depending on
196  // the url.
197  if (GetSystemState()->hardware()->IsOfficialBuild()) {
198    if (StartsWithASCII(url_to_use, "http://", false))
199      SetCurlOptionsForHttp();
200    else
201      SetCurlOptionsForHttps();
202  } else {
203    LOG(INFO) << "Not setting http(s) curl options because we are "
204              << "running a dev/test image";
205  }
206
207  CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK);
208  transfer_in_progress_ = true;
209}
210
211// Lock down only the protocol in case of HTTP.
212void LibcurlHttpFetcher::SetCurlOptionsForHttp() {
213  LOG(INFO) << "Setting up curl options for HTTP";
214  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTP),
215           CURLE_OK);
216  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
217                            CURLPROTO_HTTP),
218           CURLE_OK);
219}
220
221// Security lock-down in official builds: makes sure that peer certificate
222// verification is enabled, restricts the set of trusted certificates,
223// restricts protocols to HTTPS, restricts ciphers to HIGH.
224void LibcurlHttpFetcher::SetCurlOptionsForHttps() {
225  LOG(INFO) << "Setting up curl options for HTTPS";
226  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1),
227           CURLE_OK);
228  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CAPATH, kCACertificatesPath),
229           CURLE_OK);
230  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS),
231           CURLE_OK);
232  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
233                            CURLPROTO_HTTPS),
234           CURLE_OK);
235  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST, "HIGH:!ADH"),
236           CURLE_OK);
237  if (check_certificate_ != CertificateChecker::kNone) {
238    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA,
239                              &check_certificate_),
240             CURLE_OK);
241    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_FUNCTION,
242                              CertificateChecker::ProcessSSLContext),
243             CURLE_OK);
244  }
245}
246
247
248// Begins the transfer, which must not have already been started.
249void LibcurlHttpFetcher::BeginTransfer(const std::string& url) {
250  CHECK(!transfer_in_progress_);
251  url_ = url;
252  if (!ResolveProxiesForUrl(
253          url_,
254          NewCallback(this, &LibcurlHttpFetcher::ProxiesResolved))) {
255    LOG(ERROR) << "Couldn't resolve proxies";
256    if (delegate_)
257      delegate_->TransferComplete(this, false);
258  }
259}
260
261void LibcurlHttpFetcher::ProxiesResolved() {
262  transfer_size_ = -1;
263  resume_offset_ = 0;
264  retry_count_ = 0;
265  no_network_retry_count_ = 0;
266  http_response_code_ = 0;
267  terminate_requested_ = false;
268  sent_byte_ = false;
269  ResumeTransfer(url_);
270  CurlPerformOnce();
271}
272
273void LibcurlHttpFetcher::ForceTransferTermination() {
274  CleanUp();
275  if (delegate_) {
276    // Note that after the callback returns this object may be destroyed.
277    delegate_->TransferTerminated(this);
278  }
279}
280
281void LibcurlHttpFetcher::TerminateTransfer() {
282  if (in_write_callback_) {
283    terminate_requested_ = true;
284  } else {
285    ForceTransferTermination();
286  }
287}
288
289void LibcurlHttpFetcher::CurlPerformOnce() {
290  CHECK(transfer_in_progress_);
291  int running_handles = 0;
292  CURLMcode retcode = CURLM_CALL_MULTI_PERFORM;
293
294  // libcurl may request that we immediately call curl_multi_perform after it
295  // returns, so we do. libcurl promises that curl_multi_perform will not block.
296  while (CURLM_CALL_MULTI_PERFORM == retcode) {
297    retcode = curl_multi_perform(curl_multi_handle_, &running_handles);
298    if (terminate_requested_) {
299      ForceTransferTermination();
300      return;
301    }
302  }
303  if (0 == running_handles) {
304    GetHttpResponseCode();
305    if (http_response_code_) {
306      LOG(INFO) << "HTTP response code: " << http_response_code_;
307      no_network_retry_count_ = 0;
308    } else {
309      LOG(ERROR) << "Unable to get http response code.";
310    }
311
312    // we're done!
313    CleanUp();
314
315    // TODO(petkov): This temporary code tries to deal with the case where the
316    // update engine performs an update check while the network is not ready
317    // (e.g., right after resume). Longer term, we should check if the network
318    // is online/offline and return an appropriate error code.
319    if (!sent_byte_ &&
320        http_response_code_ == 0 &&
321        no_network_retry_count_ < no_network_max_retries_) {
322      no_network_retry_count_++;
323      g_timeout_add_seconds(kNoNetworkRetrySeconds,
324                            &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
325                            this);
326      LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_;
327      return;
328    }
329
330    if ((!sent_byte_ && !IsHttpResponseSuccess()) || IsHttpResponseError()) {
331      // The transfer completed w/ error and we didn't get any bytes.
332      // If we have another proxy to try, try that.
333      //
334      // TODO(garnold) in fact there are two separate cases here: one case is an
335      // other-than-success return code (including no return code) and no
336      // received bytes, which is necessary due to the way callbacks are
337      // currently processing error conditions;  the second is an explicit HTTP
338      // error code, where some data may have been received (as in the case of a
339      // semi-successful multi-chunk fetch).  This is a confusing behavior and
340      // should be unified into a complete, coherent interface.
341      LOG(INFO) << "Transfer resulted in an error (" << http_response_code_
342                << "), " << bytes_downloaded_ << " bytes downloaded";
343
344      PopProxy();  // Delete the proxy we just gave up on.
345
346      if (HasProxy()) {
347        // We have another proxy. Retry immediately.
348        LOG(INFO) << "Retrying with next proxy setting";
349        g_idle_add(&LibcurlHttpFetcher::StaticRetryTimeoutCallback, this);
350      } else {
351        // Out of proxies. Give up.
352        LOG(INFO) << "No further proxies, indicating transfer complete";
353        if (delegate_)
354          delegate_->TransferComplete(this, false);  // signal fail
355      }
356    } else if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) {
357      retry_count_++;
358      LOG(INFO) << "Transfer interrupted after downloading "
359                << bytes_downloaded_ << " of " << transfer_size_ << " bytes. "
360                << transfer_size_ - bytes_downloaded_ << " bytes remaining "
361                << "after " << retry_count_ << " attempt(s)";
362
363      if (retry_count_ > max_retry_count_) {
364        LOG(INFO) << "Reached max attempts (" << retry_count_ << ")";
365        if (delegate_)
366          delegate_->TransferComplete(this, false);  // signal fail
367      } else {
368        // Need to restart transfer
369        LOG(INFO) << "Restarting transfer to download the remaining bytes";
370        g_timeout_add_seconds(retry_seconds_,
371                              &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
372                              this);
373      }
374    } else {
375      LOG(INFO) << "Transfer completed (" << http_response_code_
376                << "), " << bytes_downloaded_ << " bytes downloaded";
377      if (delegate_) {
378        bool success = IsHttpResponseSuccess();
379        delegate_->TransferComplete(this, success);
380      }
381    }
382  } else {
383    // set up callback
384    SetupMainloopSources();
385  }
386}
387
388size_t LibcurlHttpFetcher::LibcurlWrite(void *ptr, size_t size, size_t nmemb) {
389  // Update HTTP response first.
390  GetHttpResponseCode();
391  const size_t payload_size = size * nmemb;
392
393  // Do nothing if no payload or HTTP response is an error.
394  if (payload_size == 0 || !IsHttpResponseSuccess()) {
395    LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_
396              << ") or no payload (" << payload_size << "), nothing to do";
397    return 0;
398  }
399
400  sent_byte_ = true;
401  {
402    double transfer_size_double;
403    CHECK_EQ(curl_easy_getinfo(curl_handle_,
404                               CURLINFO_CONTENT_LENGTH_DOWNLOAD,
405                               &transfer_size_double), CURLE_OK);
406    off_t new_transfer_size = static_cast<off_t>(transfer_size_double);
407    if (new_transfer_size > 0) {
408      transfer_size_ = resume_offset_ + new_transfer_size;
409    }
410  }
411  bytes_downloaded_ += payload_size;
412  in_write_callback_ = true;
413  if (delegate_)
414    delegate_->ReceivedBytes(this, reinterpret_cast<char*>(ptr), payload_size);
415  in_write_callback_ = false;
416  return payload_size;
417}
418
419void LibcurlHttpFetcher::Pause() {
420  CHECK(curl_handle_);
421  CHECK(transfer_in_progress_);
422  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK);
423}
424
425void LibcurlHttpFetcher::Unpause() {
426  CHECK(curl_handle_);
427  CHECK(transfer_in_progress_);
428  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK);
429}
430
431// This method sets up callbacks with the glib main loop.
432void LibcurlHttpFetcher::SetupMainloopSources() {
433  fd_set fd_read;
434  fd_set fd_write;
435  fd_set fd_exc;
436
437  FD_ZERO(&fd_read);
438  FD_ZERO(&fd_write);
439  FD_ZERO(&fd_exc);
440
441  int fd_max = 0;
442
443  // Ask libcurl for the set of file descriptors we should track on its
444  // behalf.
445  CHECK_EQ(curl_multi_fdset(curl_multi_handle_, &fd_read, &fd_write,
446                            &fd_exc, &fd_max), CURLM_OK);
447
448  // We should iterate through all file descriptors up to libcurl's fd_max or
449  // the highest one we're tracking, whichever is larger.
450  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
451    if (!io_channels_[t].empty())
452      fd_max = max(fd_max, io_channels_[t].rbegin()->first);
453  }
454
455  // For each fd, if we're not tracking it, track it. If we are tracking it, but
456  // libcurl doesn't care about it anymore, stop tracking it. After this loop,
457  // there should be exactly as many GIOChannel objects in io_channels_[0|1] as
458  // there are read/write fds that we're tracking.
459  for (int fd = 0; fd <= fd_max; ++fd) {
460    // Note that fd_exc is unused in the current version of libcurl so is_exc
461    // should always be false.
462    bool is_exc = FD_ISSET(fd, &fd_exc) != 0;
463    bool must_track[2] = {
464      is_exc || (FD_ISSET(fd, &fd_read) != 0),  // track 0 -- read
465      is_exc || (FD_ISSET(fd, &fd_write) != 0)  // track 1 -- write
466    };
467
468    for (size_t t = 0; t < arraysize(io_channels_); ++t) {
469      bool tracked = io_channels_[t].find(fd) != io_channels_[t].end();
470
471      if (!must_track[t]) {
472        // If we have an outstanding io_channel, remove it.
473        if (tracked) {
474          g_source_remove(io_channels_[t][fd].second);
475          g_io_channel_unref(io_channels_[t][fd].first);
476          io_channels_[t].erase(io_channels_[t].find(fd));
477        }
478        continue;
479      }
480
481      // If we are already tracking this fd, continue -- nothing to do.
482      if (tracked)
483        continue;
484
485      // Set conditions appropriately -- read for track 0, write for track 1.
486      GIOCondition condition = static_cast<GIOCondition>(
487          ((t == 0) ? (G_IO_IN | G_IO_PRI) : G_IO_OUT) | G_IO_ERR | G_IO_HUP);
488
489      // Track a new fd.
490      GIOChannel* io_channel = g_io_channel_unix_new(fd);
491      guint tag =
492          g_io_add_watch(io_channel, condition, &StaticFDCallback, this);
493
494      io_channels_[t][fd] = make_pair(io_channel, tag);
495      static int io_counter = 0;
496      io_counter++;
497      if (io_counter % 50 == 0) {
498        LOG(INFO) << "io_counter = " << io_counter;
499      }
500    }
501  }
502
503  // Set up a timeout callback for libcurl.
504  if (!timeout_source_) {
505    LOG(INFO) << "Setting up timeout source: " << idle_seconds_ << " seconds.";
506    timeout_source_ = g_timeout_source_new_seconds(idle_seconds_);
507    g_source_set_callback(timeout_source_, StaticTimeoutCallback, this, NULL);
508    g_source_attach(timeout_source_, NULL);
509  }
510}
511
512bool LibcurlHttpFetcher::FDCallback(GIOChannel *source,
513                                    GIOCondition condition) {
514  CurlPerformOnce();
515  // We handle removing of this source elsewhere, so we always return true.
516  // The docs say, "the function should return FALSE if the event source
517  // should be removed."
518  // http://www.gtk.org/api/2.6/glib/glib-IO-Channels.html#GIOFunc
519  return true;
520}
521
522gboolean LibcurlHttpFetcher::RetryTimeoutCallback() {
523  ResumeTransfer(url_);
524  CurlPerformOnce();
525  return FALSE;  // Don't have glib auto call this callback again
526}
527
528gboolean LibcurlHttpFetcher::TimeoutCallback() {
529  // We always return true, even if we don't want glib to call us back.
530  // We will remove the event source separately if we don't want to
531  // be called back.
532  if (!transfer_in_progress_)
533    return TRUE;
534  CurlPerformOnce();
535  return TRUE;
536}
537
538void LibcurlHttpFetcher::CleanUp() {
539  if (timeout_source_) {
540    g_source_destroy(timeout_source_);
541    timeout_source_ = NULL;
542  }
543
544  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
545    for (IOChannels::iterator it = io_channels_[t].begin();
546         it != io_channels_[t].end(); ++it) {
547      g_source_remove(it->second.second);
548      g_io_channel_unref(it->second.first);
549    }
550    io_channels_[t].clear();
551  }
552
553  if (curl_http_headers_) {
554    curl_slist_free_all(curl_http_headers_);
555    curl_http_headers_ = NULL;
556  }
557  if (curl_handle_) {
558    if (curl_multi_handle_) {
559      CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_),
560               CURLM_OK);
561    }
562    curl_easy_cleanup(curl_handle_);
563    curl_handle_ = NULL;
564  }
565  if (curl_multi_handle_) {
566    CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK);
567    curl_multi_handle_ = NULL;
568  }
569  transfer_in_progress_ = false;
570}
571
572void LibcurlHttpFetcher::GetHttpResponseCode() {
573  long http_response_code = 0;
574  if (curl_easy_getinfo(curl_handle_,
575                        CURLINFO_RESPONSE_CODE,
576                        &http_response_code) == CURLE_OK) {
577    http_response_code_ = static_cast<int>(http_response_code);
578  }
579}
580
581}  // namespace chromeos_update_engine
582