libcurl_http_fetcher.cc revision 1b9d6ae00b3c3badf9785584c2c867710f214a62 
1// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "update_engine/libcurl_http_fetcher.h"
6
7#include <algorithm>
8#include <string>
9
10#include <base/logging.h>
11#include <base/string_util.h>
12#include <base/stringprintf.h>
13
14#include "update_engine/certificate_checker.h"
15#include "update_engine/hardware_interface.h"
16#include "update_engine/real_dbus_wrapper.h"
17#include "update_engine/utils.h"
18
19using google::protobuf::NewCallback;
20using std::max;
21using std::make_pair;
22using std::string;
23
24// This is a concrete implementation of HttpFetcher that uses libcurl to do the
25// http work.
26
27namespace chromeos_update_engine {
28
29namespace {
30const int kNoNetworkRetrySeconds = 10;
31const char kCACertificatesPath[] = "/usr/share/chromeos-ca-certificates";
32}  // namespace {}
33
34LibcurlHttpFetcher::~LibcurlHttpFetcher() {
35  LOG_IF(ERROR, transfer_in_progress_)
36      << "Destroying the fetcher while a transfer is in progress.";
37  CleanUp();
38}
39
40// On error, returns false.
41bool LibcurlHttpFetcher::IsUpdateAllowedOverCurrentConnection() const {
42  NetworkConnectionType type;
43  RealDBusWrapper dbus_iface;
44  ConnectionManager* connection_manager = system_state_->connection_manager();
45  if (!connection_manager->GetConnectionType(&dbus_iface, &type)) {
46    LOG(INFO) << "We could not determine our connection type. "
47              << "Defaulting to allow updates.";
48    return true;
49  }
50  bool is_allowed = connection_manager->IsUpdateAllowedOver(type);
51  LOG(INFO) << "We are connected via "
52            << connection_manager->StringForConnectionType(type)
53            << ", Updates allowed: " << (is_allowed ? "Yes" : "No");
54  return is_allowed;
55}
56
57bool LibcurlHttpFetcher::IsOfficialBuild() const {
58  return force_build_type_ ? forced_official_build_
59                           : system_state_->hardware()->IsOfficialBuild();
60}
61
62bool LibcurlHttpFetcher::GetProxyType(const std::string& proxy,
63                                      curl_proxytype* out_type) {
64  if (utils::StringHasPrefix(proxy, "socks5://") ||
65      utils::StringHasPrefix(proxy, "socks://")) {
66    *out_type = CURLPROXY_SOCKS5_HOSTNAME;
67    return true;
68  }
69  if (utils::StringHasPrefix(proxy, "socks4://")) {
70    *out_type = CURLPROXY_SOCKS4A;
71    return true;
72  }
73  if (utils::StringHasPrefix(proxy, "http://") ||
74      utils::StringHasPrefix(proxy, "https://")) {
75    *out_type = CURLPROXY_HTTP;
76    return true;
77  }
78  if (utils::StringHasPrefix(proxy, kNoProxy)) {
79    // known failure case. don't log.
80    return false;
81  }
82  LOG(INFO) << "Unknown proxy type: " << proxy;
83  return false;
84}
85
86void LibcurlHttpFetcher::ResumeTransfer(const std::string& url) {
87  LOG(INFO) << "Starting/Resuming transfer";
88  CHECK(!transfer_in_progress_);
89  url_ = url;
90  curl_multi_handle_ = curl_multi_init();
91  CHECK(curl_multi_handle_);
92
93  curl_handle_ = curl_easy_init();
94  CHECK(curl_handle_);
95
96  CHECK(HasProxy());
97  bool is_direct = (GetCurrentProxy() == kNoProxy);
98  LOG(INFO) << "Using proxy: " << (is_direct ? "no" : "yes");
99  if (is_direct) {
100    CHECK_EQ(curl_easy_setopt(curl_handle_,
101                              CURLOPT_PROXY,
102                              ""), CURLE_OK);
103  } else {
104    CHECK_EQ(curl_easy_setopt(curl_handle_,
105                              CURLOPT_PROXY,
106                              GetCurrentProxy().c_str()), CURLE_OK);
107    // Curl seems to require us to set the protocol
108    curl_proxytype type;
109    if (GetProxyType(GetCurrentProxy(), &type)) {
110      CHECK_EQ(curl_easy_setopt(curl_handle_,
111                                CURLOPT_PROXYTYPE,
112                                type), CURLE_OK);
113    }
114  }
115
116  if (post_data_set_) {
117    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK);
118    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS,
119                              &post_data_[0]),
120             CURLE_OK);
121    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDSIZE,
122                              post_data_.size()),
123             CURLE_OK);
124
125    // Set the Content-Type HTTP header, if one was specifically set.
126    CHECK(!curl_http_headers_);
127    if (post_content_type_ != kHttpContentTypeUnspecified) {
128      const string content_type_attr =
129        base::StringPrintf("Content-Type: %s",
130                           GetHttpContentTypeString(post_content_type_));
131      curl_http_headers_ = curl_slist_append(NULL, content_type_attr.c_str());
132      CHECK(curl_http_headers_);
133      CHECK_EQ(
134          curl_easy_setopt(curl_handle_, CURLOPT_HTTPHEADER,
135                           curl_http_headers_),
136          CURLE_OK);
137    } else {
138      LOG(WARNING) << "no content type set, using libcurl default";
139    }
140  }
141
142  if (bytes_downloaded_ > 0 || download_length_) {
143    // Resume from where we left off.
144    resume_offset_ = bytes_downloaded_;
145    CHECK_GE(resume_offset_, 0);
146
147    // Compute end offset, if one is specified. As per HTTP specification, this
148    // is an inclusive boundary. Make sure it doesn't overflow.
149    size_t end_offset = 0;
150    if (download_length_) {
151      end_offset = static_cast<size_t>(resume_offset_) + download_length_ - 1;
152      CHECK_LE((size_t) resume_offset_, end_offset);
153    }
154
155    // Create a string representation of the desired range.
156    std::string range_str = (end_offset ?
157                             StringPrintf("%jd-%zu", resume_offset_,
158                                          end_offset) :
159                             StringPrintf("%jd-", resume_offset_));
160    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_RANGE, range_str.c_str()),
161             CURLE_OK);
162  }
163
164  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK);
165  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION,
166                            StaticLibcurlWrite), CURLE_OK);
167
168  string url_to_use(url_);
169  if (!IsUpdateAllowedOverCurrentConnection()) {
170    LOG(INFO) << "Not initiating HTTP connection b/c updates are disabled "
171              << "over this connection";
172    url_to_use = "";  // Sabotage the URL
173  }
174
175  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_to_use.c_str()),
176           CURLE_OK);
177
178  // If the connection drops under |low_speed_limit_bps_| (10
179  // bytes/sec by default) for |low_speed_time_seconds_| (90 seconds,
180  // 180 on non-official builds), reconnect.
181  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_LIMIT,
182                            low_speed_limit_bps_),
183           CURLE_OK);
184  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_TIME,
185                            low_speed_time_seconds_),
186           CURLE_OK);
187  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CONNECTTIMEOUT,
188                            connect_timeout_seconds_),
189           CURLE_OK);
190
191  // By default, libcurl doesn't follow redirections. Allow up to
192  // |kDownloadMaxRedirects| redirections.
193  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK);
194  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS,
195                            kDownloadMaxRedirects),
196           CURLE_OK);
197
198  // If we are running in test mode or using a dev/test build, then lock down
199  // the appropriate curl options for HTTP or HTTPS depending on the url.
200  if (!is_test_mode_ && IsOfficialBuild()) {
201    if (StartsWithASCII(url_to_use, "http://", false))
202      SetCurlOptionsForHttp();
203    else
204      SetCurlOptionsForHttps();
205  } else {
206    LOG(INFO) << "Not setting http(s) curl options because we are in "
207              << "test mode or running a dev/test image";
208  }
209
210  CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK);
211  transfer_in_progress_ = true;
212}
213
214// Lock down only the protocol in case of HTTP.
215void LibcurlHttpFetcher::SetCurlOptionsForHttp() {
216  LOG(INFO) << "Setting up curl options for HTTP";
217  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTP),
218           CURLE_OK);
219  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
220                            CURLPROTO_HTTP),
221           CURLE_OK);
222}
223
224// Security lock-down in official builds: makes sure that peer certificate
225// verification is enabled, restricts the set of trusted certificates,
226// restricts protocols to HTTPS, restricts ciphers to HIGH.
227void LibcurlHttpFetcher::SetCurlOptionsForHttps() {
228  LOG(INFO) << "Setting up curl options for HTTPS";
229  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1),
230           CURLE_OK);
231  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CAPATH, kCACertificatesPath),
232           CURLE_OK);
233  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS),
234           CURLE_OK);
235  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_REDIR_PROTOCOLS,
236                            CURLPROTO_HTTPS),
237           CURLE_OK);
238  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST, "HIGH:!ADH"),
239           CURLE_OK);
240  if (check_certificate_ != CertificateChecker::kNone) {
241    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA,
242                              &check_certificate_),
243             CURLE_OK);
244    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_FUNCTION,
245                              CertificateChecker::ProcessSSLContext),
246             CURLE_OK);
247  }
248}
249
250
251// Begins the transfer, which must not have already been started.
252void LibcurlHttpFetcher::BeginTransfer(const std::string& url) {
253  CHECK(!transfer_in_progress_);
254  url_ = url;
255  if (!ResolveProxiesForUrl(
256          url_,
257          NewCallback(this, &LibcurlHttpFetcher::ProxiesResolved))) {
258    LOG(ERROR) << "Couldn't resolve proxies";
259    if (delegate_)
260      delegate_->TransferComplete(this, false);
261  }
262}
263
264void LibcurlHttpFetcher::ProxiesResolved() {
265  transfer_size_ = -1;
266  resume_offset_ = 0;
267  retry_count_ = 0;
268  no_network_retry_count_ = 0;
269  http_response_code_ = 0;
270  terminate_requested_ = false;
271  sent_byte_ = false;
272  ResumeTransfer(url_);
273  CurlPerformOnce();
274}
275
276void LibcurlHttpFetcher::ForceTransferTermination() {
277  CleanUp();
278  if (delegate_) {
279    // Note that after the callback returns this object may be destroyed.
280    delegate_->TransferTerminated(this);
281  }
282}
283
284void LibcurlHttpFetcher::TerminateTransfer() {
285  if (in_write_callback_) {
286    terminate_requested_ = true;
287  } else {
288    ForceTransferTermination();
289  }
290}
291
292void LibcurlHttpFetcher::CurlPerformOnce() {
293  CHECK(transfer_in_progress_);
294  int running_handles = 0;
295  CURLMcode retcode = CURLM_CALL_MULTI_PERFORM;
296
297  // libcurl may request that we immediately call curl_multi_perform after it
298  // returns, so we do. libcurl promises that curl_multi_perform will not block.
299  while (CURLM_CALL_MULTI_PERFORM == retcode) {
300    retcode = curl_multi_perform(curl_multi_handle_, &running_handles);
301    if (terminate_requested_) {
302      ForceTransferTermination();
303      return;
304    }
305  }
306  if (0 == running_handles) {
307    GetHttpResponseCode();
308    if (http_response_code_) {
309      LOG(INFO) << "HTTP response code: " << http_response_code_;
310      no_network_retry_count_ = 0;
311    } else {
312      LOG(ERROR) << "Unable to get http response code.";
313    }
314
315    // we're done!
316    CleanUp();
317
318    // TODO(petkov): This temporary code tries to deal with the case where the
319    // update engine performs an update check while the network is not ready
320    // (e.g., right after resume). Longer term, we should check if the network
321    // is online/offline and return an appropriate error code.
322    if (!sent_byte_ &&
323        http_response_code_ == 0 &&
324        no_network_retry_count_ < no_network_max_retries_) {
325      no_network_retry_count_++;
326      g_timeout_add_seconds(kNoNetworkRetrySeconds,
327                            &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
328                            this);
329      LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_;
330      return;
331    }
332
333    if ((!sent_byte_ && !IsHttpResponseSuccess()) || IsHttpResponseError()) {
334      // The transfer completed w/ error and we didn't get any bytes.
335      // If we have another proxy to try, try that.
336      //
337      // TODO(garnold) in fact there are two separate cases here: one case is an
338      // other-than-success return code (including no return code) and no
339      // received bytes, which is necessary due to the way callbacks are
340      // currently processing error conditions;  the second is an explicit HTTP
341      // error code, where some data may have been received (as in the case of a
342      // semi-successful multi-chunk fetch).  This is a confusing behavior and
343      // should be unified into a complete, coherent interface.
344      LOG(INFO) << "Transfer resulted in an error (" << http_response_code_
345                << "), " << bytes_downloaded_ << " bytes downloaded";
346
347      PopProxy();  // Delete the proxy we just gave up on.
348
349      if (HasProxy()) {
350        // We have another proxy. Retry immediately.
351        LOG(INFO) << "Retrying with next proxy setting";
352        g_idle_add(&LibcurlHttpFetcher::StaticRetryTimeoutCallback, this);
353      } else {
354        // Out of proxies. Give up.
355        LOG(INFO) << "No further proxies, indicating transfer complete";
356        if (delegate_)
357          delegate_->TransferComplete(this, false);  // signal fail
358      }
359    } else if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) {
360      retry_count_++;
361      LOG(INFO) << "Transfer interrupted after downloading "
362                << bytes_downloaded_ << " of " << transfer_size_ << " bytes. "
363                << transfer_size_ - bytes_downloaded_ << " bytes remaining "
364                << "after " << retry_count_ << " attempt(s)";
365
366      if (retry_count_ > max_retry_count_) {
367        LOG(INFO) << "Reached max attempts (" << retry_count_ << ")";
368        if (delegate_)
369          delegate_->TransferComplete(this, false);  // signal fail
370      } else {
371        // Need to restart transfer
372        LOG(INFO) << "Restarting transfer to download the remaining bytes";
373        g_timeout_add_seconds(retry_seconds_,
374                              &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
375                              this);
376      }
377    } else {
378      LOG(INFO) << "Transfer completed (" << http_response_code_
379                << "), " << bytes_downloaded_ << " bytes downloaded";
380      if (delegate_) {
381        bool success = IsHttpResponseSuccess();
382        delegate_->TransferComplete(this, success);
383      }
384    }
385  } else {
386    // set up callback
387    SetupMainloopSources();
388  }
389}
390
391size_t LibcurlHttpFetcher::LibcurlWrite(void *ptr, size_t size, size_t nmemb) {
392  // Update HTTP response first.
393  GetHttpResponseCode();
394  const size_t payload_size = size * nmemb;
395
396  // Do nothing if no payload or HTTP response is an error.
397  if (payload_size == 0 || !IsHttpResponseSuccess()) {
398    LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_
399              << ") or no payload (" << payload_size << "), nothing to do";
400    return 0;
401  }
402
403  sent_byte_ = true;
404  {
405    double transfer_size_double;
406    CHECK_EQ(curl_easy_getinfo(curl_handle_,
407                               CURLINFO_CONTENT_LENGTH_DOWNLOAD,
408                               &transfer_size_double), CURLE_OK);
409    off_t new_transfer_size = static_cast<off_t>(transfer_size_double);
410    if (new_transfer_size > 0) {
411      transfer_size_ = resume_offset_ + new_transfer_size;
412    }
413  }
414  bytes_downloaded_ += payload_size;
415  in_write_callback_ = true;
416  if (delegate_)
417    delegate_->ReceivedBytes(this, reinterpret_cast<char*>(ptr), payload_size);
418  in_write_callback_ = false;
419  return payload_size;
420}
421
422void LibcurlHttpFetcher::Pause() {
423  CHECK(curl_handle_);
424  CHECK(transfer_in_progress_);
425  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK);
426}
427
428void LibcurlHttpFetcher::Unpause() {
429  CHECK(curl_handle_);
430  CHECK(transfer_in_progress_);
431  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK);
432}
433
434// This method sets up callbacks with the glib main loop.
435void LibcurlHttpFetcher::SetupMainloopSources() {
436  fd_set fd_read;
437  fd_set fd_write;
438  fd_set fd_exc;
439
440  FD_ZERO(&fd_read);
441  FD_ZERO(&fd_write);
442  FD_ZERO(&fd_exc);
443
444  int fd_max = 0;
445
446  // Ask libcurl for the set of file descriptors we should track on its
447  // behalf.
448  CHECK_EQ(curl_multi_fdset(curl_multi_handle_, &fd_read, &fd_write,
449                            &fd_exc, &fd_max), CURLM_OK);
450
451  // We should iterate through all file descriptors up to libcurl's fd_max or
452  // the highest one we're tracking, whichever is larger.
453  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
454    if (!io_channels_[t].empty())
455      fd_max = max(fd_max, io_channels_[t].rbegin()->first);
456  }
457
458  // For each fd, if we're not tracking it, track it. If we are tracking it, but
459  // libcurl doesn't care about it anymore, stop tracking it. After this loop,
460  // there should be exactly as many GIOChannel objects in io_channels_[0|1] as
461  // there are read/write fds that we're tracking.
462  for (int fd = 0; fd <= fd_max; ++fd) {
463    // Note that fd_exc is unused in the current version of libcurl so is_exc
464    // should always be false.
465    bool is_exc = FD_ISSET(fd, &fd_exc) != 0;
466    bool must_track[2] = {
467      is_exc || (FD_ISSET(fd, &fd_read) != 0),  // track 0 -- read
468      is_exc || (FD_ISSET(fd, &fd_write) != 0)  // track 1 -- write
469    };
470
471    for (size_t t = 0; t < arraysize(io_channels_); ++t) {
472      bool tracked = io_channels_[t].find(fd) != io_channels_[t].end();
473
474      if (!must_track[t]) {
475        // If we have an outstanding io_channel, remove it.
476        if (tracked) {
477          g_source_remove(io_channels_[t][fd].second);
478          g_io_channel_unref(io_channels_[t][fd].first);
479          io_channels_[t].erase(io_channels_[t].find(fd));
480        }
481        continue;
482      }
483
484      // If we are already tracking this fd, continue -- nothing to do.
485      if (tracked)
486        continue;
487
488      // Set conditions appropriately -- read for track 0, write for track 1.
489      GIOCondition condition = static_cast<GIOCondition>(
490          ((t == 0) ? (G_IO_IN | G_IO_PRI) : G_IO_OUT) | G_IO_ERR | G_IO_HUP);
491
492      // Track a new fd.
493      GIOChannel* io_channel = g_io_channel_unix_new(fd);
494      guint tag =
495          g_io_add_watch(io_channel, condition, &StaticFDCallback, this);
496
497      io_channels_[t][fd] = make_pair(io_channel, tag);
498      static int io_counter = 0;
499      io_counter++;
500      if (io_counter % 50 == 0) {
501        LOG(INFO) << "io_counter = " << io_counter;
502      }
503    }
504  }
505
506  // Set up a timeout callback for libcurl.
507  if (!timeout_source_) {
508    LOG(INFO) << "Setting up timeout source: " << idle_seconds_ << " seconds.";
509    timeout_source_ = g_timeout_source_new_seconds(idle_seconds_);
510    g_source_set_callback(timeout_source_, StaticTimeoutCallback, this, NULL);
511    g_source_attach(timeout_source_, NULL);
512  }
513}
514
515bool LibcurlHttpFetcher::FDCallback(GIOChannel *source,
516                                    GIOCondition condition) {
517  CurlPerformOnce();
518  // We handle removing of this source elsewhere, so we always return true.
519  // The docs say, "the function should return FALSE if the event source
520  // should be removed."
521  // http://www.gtk.org/api/2.6/glib/glib-IO-Channels.html#GIOFunc
522  return true;
523}
524
525gboolean LibcurlHttpFetcher::RetryTimeoutCallback() {
526  ResumeTransfer(url_);
527  CurlPerformOnce();
528  return FALSE;  // Don't have glib auto call this callback again
529}
530
531gboolean LibcurlHttpFetcher::TimeoutCallback() {
532  // We always return true, even if we don't want glib to call us back.
533  // We will remove the event source separately if we don't want to
534  // be called back.
535  if (!transfer_in_progress_)
536    return TRUE;
537  CurlPerformOnce();
538  return TRUE;
539}
540
541void LibcurlHttpFetcher::CleanUp() {
542  if (timeout_source_) {
543    g_source_destroy(timeout_source_);
544    timeout_source_ = NULL;
545  }
546
547  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
548    for (IOChannels::iterator it = io_channels_[t].begin();
549         it != io_channels_[t].end(); ++it) {
550      g_source_remove(it->second.second);
551      g_io_channel_unref(it->second.first);
552    }
553    io_channels_[t].clear();
554  }
555
556  if (curl_http_headers_) {
557    curl_slist_free_all(curl_http_headers_);
558    curl_http_headers_ = NULL;
559  }
560  if (curl_handle_) {
561    if (curl_multi_handle_) {
562      CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_),
563               CURLM_OK);
564    }
565    curl_easy_cleanup(curl_handle_);
566    curl_handle_ = NULL;
567  }
568  if (curl_multi_handle_) {
569    CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK);
570    curl_multi_handle_ = NULL;
571  }
572  transfer_in_progress_ = false;
573}
574
575void LibcurlHttpFetcher::GetHttpResponseCode() {
576  long http_response_code = 0;
577  if (curl_easy_getinfo(curl_handle_,
578                        CURLINFO_RESPONSE_CODE,
579                        &http_response_code) == CURLE_OK) {
580    http_response_code_ = static_cast<int>(http_response_code);
581  }
582}
583
584}  // namespace chromeos_update_engine
585