libcurl_http_fetcher.cc revision 48085ba58516e94f045d3ab7e26c8f36e6a6936f 
1// Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "update_engine/libcurl_http_fetcher.h"
6
7#include <algorithm>
8#include <string>
9
10#include <base/logging.h>
11
12#include "update_engine/certificate_checker.h"
13#include "update_engine/chrome_proxy_resolver.h"
14#include "update_engine/dbus_interface.h"
15#include "update_engine/flimflam_proxy.h"
16#include "update_engine/utils.h"
17
18using google::protobuf::NewCallback;
19using std::max;
20using std::make_pair;
21using std::string;
22
23// This is a concrete implementation of HttpFetcher that uses libcurl to do the
24// http work.
25
26namespace chromeos_update_engine {
27
28namespace {
29const int kMaxRetriesCount = 20;
30const int kNoNetworkRetrySeconds = 10;
31const char kCACertificatesPath[] = "/usr/share/chromeos-ca-certificates";
32}  // namespace {}
33
34LibcurlHttpFetcher::~LibcurlHttpFetcher() {
35  LOG_IF(ERROR, transfer_in_progress_)
36      << "Destroying the fetcher while a transfer is in progress.";
37  CleanUp();
38}
39
40// On error, returns false.
41bool LibcurlHttpFetcher::ConnectionIsExpensive() const {
42  if (force_connection_type_)
43    return forced_expensive_connection_;
44  NetworkConnectionType type;
45  ConcreteDbusGlib dbus_iface;
46  TEST_AND_RETURN_FALSE(FlimFlamProxy::GetConnectionType(&dbus_iface, &type));
47  LOG(INFO) << "We are connected via "
48            << FlimFlamProxy::StringForConnectionType(type);
49  return FlimFlamProxy::IsExpensiveConnectionType(type);
50}
51
52bool LibcurlHttpFetcher::IsOfficialBuild() const {
53  return force_build_type_ ? forced_official_build_ : utils::IsOfficialBuild();
54}
55
56void LibcurlHttpFetcher::ResumeTransfer(const std::string& url) {
57  LOG(INFO) << "Starting/Resuming transfer";
58  CHECK(!transfer_in_progress_);
59  url_ = url;
60  curl_multi_handle_ = curl_multi_init();
61  CHECK(curl_multi_handle_);
62
63  curl_handle_ = curl_easy_init();
64  CHECK(curl_handle_);
65
66  CHECK(HasProxy());
67  LOG(INFO) << "Using proxy: " << GetCurrentProxy();
68  if (GetCurrentProxy() == kNoProxy) {
69    CHECK_EQ(curl_easy_setopt(curl_handle_,
70                              CURLOPT_PROXY,
71                              ""), CURLE_OK);
72  } else {
73    CHECK_EQ(curl_easy_setopt(curl_handle_,
74                              CURLOPT_PROXY,
75                              GetCurrentProxy().c_str()), CURLE_OK);
76    // Curl seems to require us to set the protocol
77    curl_proxytype type;
78    if (ChromeProxyResolver::GetProxyType(GetCurrentProxy(), &type)) {
79      CHECK_EQ(curl_easy_setopt(curl_handle_,
80                                CURLOPT_PROXYTYPE,
81                                type), CURLE_OK);
82    }
83  }
84
85  if (post_data_set_) {
86    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POST, 1), CURLE_OK);
87    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDS,
88                              &post_data_[0]),
89             CURLE_OK);
90    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_POSTFIELDSIZE,
91                              post_data_.size()),
92             CURLE_OK);
93  }
94
95  if (bytes_downloaded_ > 0) {
96    // Resume from where we left off
97    resume_offset_ = bytes_downloaded_;
98    CHECK_EQ(curl_easy_setopt(curl_handle_,
99                              CURLOPT_RESUME_FROM_LARGE,
100                              bytes_downloaded_), CURLE_OK);
101  }
102
103  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEDATA, this), CURLE_OK);
104  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_WRITEFUNCTION,
105                            StaticLibcurlWrite), CURLE_OK);
106
107  string url_to_use(url_);
108  if (ConnectionIsExpensive()) {
109    LOG(INFO) << "Not initiating HTTP connection b/c we are on an expensive"
110              << " connection";
111    url_to_use = "";  // Sabotage the URL
112  }
113
114  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_URL, url_to_use.c_str()),
115           CURLE_OK);
116
117  // If the connection drops under 10 bytes/sec for 3 minutes, reconnect.
118  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_LIMIT, 10),
119           CURLE_OK);
120  // Use a smaller timeout on official builds, larger for dev. Dev users
121  // want a longer timeout b/c they may be waiting on the dev server to
122  // build an image.
123  const int kTimeout = IsOfficialBuild() ? 90 : 3 * 60;
124  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_LOW_SPEED_TIME, kTimeout),
125           CURLE_OK);
126  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_CONNECTTIMEOUT, 30),
127           CURLE_OK);
128
129  // By default, libcurl doesn't follow redirections. Allow up to
130  // |kMaxRedirects| redirections.
131  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_FOLLOWLOCATION, 1), CURLE_OK);
132  CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_MAXREDIRS, kMaxRedirects),
133           CURLE_OK);
134
135  // Security lock-down in official builds: makes sure that peer certificate
136  // verification is enabled, restricts the set of trusted certificates,
137  // restricts protocols to HTTPS, restricts ciphers to HIGH.
138  if (IsOfficialBuild()) {
139    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_VERIFYPEER, 1),
140             CURLE_OK);
141    CHECK_EQ(curl_easy_setopt(curl_handle_,
142                              CURLOPT_CAPATH,
143                              kCACertificatesPath),
144             CURLE_OK);
145    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS),
146             CURLE_OK);
147    CHECK_EQ(curl_easy_setopt(curl_handle_,
148                              CURLOPT_REDIR_PROTOCOLS,
149                              CURLPROTO_HTTPS),
150             CURLE_OK);
151    CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CIPHER_LIST,
152                              "HIGH:!ADH"),
153             CURLE_OK);
154    if (check_certificate_ != CertificateChecker::kNone) {
155      CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_DATA,
156                                &check_certificate_),
157               CURLE_OK);
158      CHECK_EQ(curl_easy_setopt(curl_handle_, CURLOPT_SSL_CTX_FUNCTION,
159                                CertificateChecker::ProcessSSLContext),
160               CURLE_OK);
161    }
162  }
163
164  CHECK_EQ(curl_multi_add_handle(curl_multi_handle_, curl_handle_), CURLM_OK);
165  transfer_in_progress_ = true;
166}
167
168// Begins the transfer, which must not have already been started.
169void LibcurlHttpFetcher::BeginTransfer(const std::string& url) {
170  CHECK(!transfer_in_progress_);
171  url_ = url;
172  if (!ResolveProxiesForUrl(
173          url_,
174          NewCallback(this, &LibcurlHttpFetcher::ProxiesResolved))) {
175    LOG(ERROR) << "Couldn't resolve proxies";
176    if (delegate_)
177      delegate_->TransferComplete(this, false);
178  }
179}
180
181void LibcurlHttpFetcher::ProxiesResolved() {
182  transfer_size_ = -1;
183  resume_offset_ = 0;
184  retry_count_ = 0;
185  no_network_retry_count_ = 0;
186  http_response_code_ = 0;
187  terminate_requested_ = false;
188  ResumeTransfer(url_);
189  CurlPerformOnce();
190}
191
192void LibcurlHttpFetcher::ForceTransferTermination() {
193  CleanUp();
194  if (delegate_) {
195    // Note that after the callback returns this object may be destroyed.
196    delegate_->TransferTerminated(this);
197  }
198}
199
200void LibcurlHttpFetcher::TerminateTransfer() {
201  if (in_write_callback_) {
202    terminate_requested_ = true;
203  } else {
204    ForceTransferTermination();
205  }
206}
207
208void LibcurlHttpFetcher::CurlPerformOnce() {
209  CHECK(transfer_in_progress_);
210  int running_handles = 0;
211  CURLMcode retcode = CURLM_CALL_MULTI_PERFORM;
212
213  // libcurl may request that we immediately call curl_multi_perform after it
214  // returns, so we do. libcurl promises that curl_multi_perform will not block.
215  while (CURLM_CALL_MULTI_PERFORM == retcode) {
216    retcode = curl_multi_perform(curl_multi_handle_, &running_handles);
217    if (terminate_requested_) {
218      ForceTransferTermination();
219      return;
220    }
221  }
222  if (0 == running_handles) {
223    GetHttpResponseCode();
224    if (http_response_code_) {
225      LOG(INFO) << "HTTP response code: " << http_response_code_;
226      no_network_retry_count_ = 0;
227    } else {
228      LOG(ERROR) << "Unable to get http response code.";
229    }
230
231    // we're done!
232    CleanUp();
233
234    // TODO(petkov): This temporary code tries to deal with the case where the
235    // update engine performs an update check while the network is not ready
236    // (e.g., right after resume). Longer term, we should check if the network
237    // is online/offline and return an appropriate error code.
238    if (!sent_byte_ &&
239        http_response_code_ == 0 &&
240        no_network_retry_count_ < no_network_max_retries_) {
241      no_network_retry_count_++;
242      g_timeout_add_seconds(kNoNetworkRetrySeconds,
243                            &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
244                            this);
245      LOG(INFO) << "No HTTP response, retry " << no_network_retry_count_;
246      return;
247    }
248
249    if (!sent_byte_ && ! IsHttpResponseSuccess()) {
250      // The transfer completed w/ error and we didn't get any bytes.
251      // If we have another proxy to try, try that.
252
253      PopProxy();  // Delete the proxy we just gave up on.
254
255      if (HasProxy()) {
256        // We have another proxy. Retry immediately.
257        g_idle_add(&LibcurlHttpFetcher::StaticRetryTimeoutCallback, this);
258      } else {
259        // Out of proxies. Give up.
260        if (delegate_)
261          delegate_->TransferComplete(this, false);  // success
262      }
263      return;
264    }
265
266    if ((transfer_size_ >= 0) && (bytes_downloaded_ < transfer_size_)) {
267      // Need to restart transfer
268      retry_count_++;
269      LOG(INFO) << "Restarting transfer b/c we finished, had downloaded "
270                << bytes_downloaded_ << " bytes, but transfer_size_ is "
271                << transfer_size_ << ". retry_count: " << retry_count_;
272      if (retry_count_ > kMaxRetriesCount) {
273        if (delegate_)
274          delegate_->TransferComplete(this, false);  // success
275      } else {
276        g_timeout_add_seconds(retry_seconds_,
277                              &LibcurlHttpFetcher::StaticRetryTimeoutCallback,
278                              this);
279      }
280      return;
281    } else {
282      if (delegate_) {
283        // success is when http_response_code is 2xx
284        bool success = IsHttpResponseSuccess();
285        delegate_->TransferComplete(this, success);
286      }
287    }
288  } else {
289    // set up callback
290    SetupMainloopSources();
291  }
292}
293
294size_t LibcurlHttpFetcher::LibcurlWrite(void *ptr, size_t size, size_t nmemb) {
295  // Update HTTP response first.
296  GetHttpResponseCode();
297  const size_t payload_size = size * nmemb;
298
299  // Do nothing if no payload or HTTP response is an error.
300  if (payload_size == 0 || ! IsHttpResponseSuccess()) {
301    LOG(INFO) << "HTTP response unsuccessful (" << http_response_code_
302              << ") or no payload (" << payload_size << "), nothing to do";
303    return 0;
304  }
305
306  sent_byte_ = true;
307  {
308    double transfer_size_double;
309    CHECK_EQ(curl_easy_getinfo(curl_handle_,
310                               CURLINFO_CONTENT_LENGTH_DOWNLOAD,
311                               &transfer_size_double), CURLE_OK);
312    off_t new_transfer_size = static_cast<off_t>(transfer_size_double);
313    if (new_transfer_size > 0) {
314      transfer_size_ = resume_offset_ + new_transfer_size;
315    }
316  }
317  bytes_downloaded_ += payload_size;
318  in_write_callback_ = true;
319  if (delegate_)
320    delegate_->ReceivedBytes(this, reinterpret_cast<char*>(ptr), payload_size);
321  in_write_callback_ = false;
322  return payload_size;
323}
324
325void LibcurlHttpFetcher::Pause() {
326  CHECK(curl_handle_);
327  CHECK(transfer_in_progress_);
328  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_ALL), CURLE_OK);
329}
330
331void LibcurlHttpFetcher::Unpause() {
332  CHECK(curl_handle_);
333  CHECK(transfer_in_progress_);
334  CHECK_EQ(curl_easy_pause(curl_handle_, CURLPAUSE_CONT), CURLE_OK);
335}
336
337// This method sets up callbacks with the glib main loop.
338void LibcurlHttpFetcher::SetupMainloopSources() {
339  fd_set fd_read;
340  fd_set fd_write;
341  fd_set fd_exc;
342
343  FD_ZERO(&fd_read);
344  FD_ZERO(&fd_write);
345  FD_ZERO(&fd_exc);
346
347  int fd_max = 0;
348
349  // Ask libcurl for the set of file descriptors we should track on its
350  // behalf.
351  CHECK_EQ(curl_multi_fdset(curl_multi_handle_, &fd_read, &fd_write,
352                            &fd_exc, &fd_max), CURLM_OK);
353
354  // We should iterate through all file descriptors up to libcurl's fd_max or
355  // the highest one we're tracking, whichever is larger.
356  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
357    if (!io_channels_[t].empty())
358      fd_max = max(fd_max, io_channels_[t].rbegin()->first);
359  }
360
361  // For each fd, if we're not tracking it, track it. If we are tracking it, but
362  // libcurl doesn't care about it anymore, stop tracking it. After this loop,
363  // there should be exactly as many GIOChannel objects in io_channels_[0|1] as
364  // there are read/write fds that we're tracking.
365  for (int fd = 0; fd <= fd_max; ++fd) {
366    // Note that fd_exc is unused in the current version of libcurl so is_exc
367    // should always be false.
368    bool is_exc = FD_ISSET(fd, &fd_exc) != 0;
369    bool must_track[2] = {
370      is_exc || (FD_ISSET(fd, &fd_read) != 0),  // track 0 -- read
371      is_exc || (FD_ISSET(fd, &fd_write) != 0)  // track 1 -- write
372    };
373
374    for (size_t t = 0; t < arraysize(io_channels_); ++t) {
375      bool tracked = io_channels_[t].find(fd) != io_channels_[t].end();
376
377      if (!must_track[t]) {
378        // If we have an outstanding io_channel, remove it.
379        if (tracked) {
380          g_source_remove(io_channels_[t][fd].second);
381          g_io_channel_unref(io_channels_[t][fd].first);
382          io_channels_[t].erase(io_channels_[t].find(fd));
383        }
384        continue;
385      }
386
387      // If we are already tracking this fd, continue -- nothing to do.
388      if (tracked)
389        continue;
390
391      // Set conditions appropriately -- read for track 0, write for track 1.
392      GIOCondition condition = static_cast<GIOCondition>(
393          ((t == 0) ? (G_IO_IN | G_IO_PRI) : G_IO_OUT) | G_IO_ERR | G_IO_HUP);
394
395      // Track a new fd.
396      GIOChannel* io_channel = g_io_channel_unix_new(fd);
397      guint tag =
398          g_io_add_watch(io_channel, condition, &StaticFDCallback, this);
399
400      io_channels_[t][fd] = make_pair(io_channel, tag);
401      static int io_counter = 0;
402      io_counter++;
403      if (io_counter % 50 == 0) {
404        LOG(INFO) << "io_counter = " << io_counter;
405      }
406    }
407  }
408
409  // Set up a timeout callback for libcurl.
410  if (!timeout_source_) {
411    LOG(INFO) << "Setting up timeout source: " << idle_seconds_ << " seconds.";
412    timeout_source_ = g_timeout_source_new_seconds(idle_seconds_);
413    g_source_set_callback(timeout_source_, StaticTimeoutCallback, this, NULL);
414    g_source_attach(timeout_source_, NULL);
415  }
416}
417
418bool LibcurlHttpFetcher::FDCallback(GIOChannel *source,
419                                    GIOCondition condition) {
420  CurlPerformOnce();
421  // We handle removing of this source elsewhere, so we always return true.
422  // The docs say, "the function should return FALSE if the event source
423  // should be removed."
424  // http://www.gtk.org/api/2.6/glib/glib-IO-Channels.html#GIOFunc
425  return true;
426}
427
428gboolean LibcurlHttpFetcher::RetryTimeoutCallback() {
429  ResumeTransfer(url_);
430  CurlPerformOnce();
431  return FALSE;  // Don't have glib auto call this callback again
432}
433
434gboolean LibcurlHttpFetcher::TimeoutCallback() {
435  // We always return true, even if we don't want glib to call us back.
436  // We will remove the event source separately if we don't want to
437  // be called back.
438  if (!transfer_in_progress_)
439    return TRUE;
440  CurlPerformOnce();
441  return TRUE;
442}
443
444void LibcurlHttpFetcher::CleanUp() {
445  if (timeout_source_) {
446    g_source_destroy(timeout_source_);
447    timeout_source_ = NULL;
448  }
449
450  for (size_t t = 0; t < arraysize(io_channels_); ++t) {
451    for (IOChannels::iterator it = io_channels_[t].begin();
452         it != io_channels_[t].end(); ++it) {
453      g_source_remove(it->second.second);
454      g_io_channel_unref(it->second.first);
455    }
456    io_channels_[t].clear();
457  }
458
459  if (curl_handle_) {
460    if (curl_multi_handle_) {
461      CHECK_EQ(curl_multi_remove_handle(curl_multi_handle_, curl_handle_),
462               CURLM_OK);
463    }
464    curl_easy_cleanup(curl_handle_);
465    curl_handle_ = NULL;
466  }
467  if (curl_multi_handle_) {
468    CHECK_EQ(curl_multi_cleanup(curl_multi_handle_), CURLM_OK);
469    curl_multi_handle_ = NULL;
470  }
471  transfer_in_progress_ = false;
472}
473
474void LibcurlHttpFetcher::GetHttpResponseCode() {
475  long http_response_code = 0;
476  if (curl_easy_getinfo(curl_handle_,
477                        CURLINFO_RESPONSE_CODE,
478                        &http_response_code) == CURLE_OK) {
479    http_response_code_ = static_cast<int>(http_response_code);
480  }
481}
482
483}  // namespace chromeos_update_engine
484