1//
2// Copyright (C) 2012 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include <netinet/in.h>
18#include <netinet/ip.h>
19#include <sys/socket.h>
20#include <unistd.h>
21
22#include <memory>
23#include <string>
24#include <utility>
25#include <vector>
26
27#include <base/bind.h>
28#include <base/location.h>
29#include <base/logging.h>
30#include <base/message_loop/message_loop.h>
31#include <base/strings/string_number_conversions.h>
32#include <base/strings/string_util.h>
33#include <base/strings/stringprintf.h>
34#include <base/time/time.h>
35#include <brillo/bind_lambda.h>
36#include <brillo/message_loops/base_message_loop.h>
37#include <brillo/message_loops/message_loop.h>
38#include <brillo/message_loops/message_loop_utils.h>
39#include <brillo/process.h>
40#include <brillo/streams/file_stream.h>
41#include <brillo/streams/stream.h>
42#include <gtest/gtest.h>
43
44#include "update_engine/common/fake_hardware.h"
45#include "update_engine/common/file_fetcher.h"
46#include "update_engine/common/http_common.h"
47#include "update_engine/common/mock_http_fetcher.h"
48#include "update_engine/common/multi_range_http_fetcher.h"
49#include "update_engine/common/test_utils.h"
50#include "update_engine/common/utils.h"
51#include "update_engine/libcurl_http_fetcher.h"
52#include "update_engine/mock_proxy_resolver.h"
53#include "update_engine/proxy_resolver.h"
54
55using brillo::MessageLoop;
56using std::make_pair;
57using std::pair;
58using std::string;
59using std::unique_ptr;
60using std::vector;
61using testing::DoAll;
62using testing::Return;
63using testing::SaveArg;
64using testing::_;
65
66namespace {
67
68const int kBigLength           = 100000;
69const int kMediumLength        = 1000;
70const int kFlakyTruncateLength = 29000;
71const int kFlakySleepEvery     = 3;
72const int kFlakySleepSecs      = 10;
73
74}  // namespace
75
76namespace chromeos_update_engine {
77
78static const char *kUnusedUrl = "unused://unused";
79
80static inline string LocalServerUrlForPath(in_port_t port,
81                                           const string& path) {
82  string port_str = (port ? base::StringPrintf(":%hu", port) : "");
83  return base::StringPrintf("http://127.0.0.1%s%s", port_str.c_str(),
84                            path.c_str());
85}
86
87//
88// Class hierarchy for HTTP server implementations.
89//
90
91class HttpServer {
92 public:
93  // This makes it an abstract class (dirty but works).
94  virtual ~HttpServer() = 0;
95
96  virtual in_port_t GetPort() const {
97    return 0;
98  }
99
100  bool started_;
101};
102
103HttpServer::~HttpServer() {}
104
105
106class NullHttpServer : public HttpServer {
107 public:
108  NullHttpServer() {
109    started_ = true;
110  }
111};
112
113
114class PythonHttpServer : public HttpServer {
115 public:
116  PythonHttpServer() : port_(0) {
117    started_ = false;
118
119    // Spawn the server process.
120    unique_ptr<brillo::Process> http_server(new brillo::ProcessImpl());
121    base::FilePath test_server_path =
122        test_utils::GetBuildArtifactsPath().Append("test_http_server");
123    http_server->AddArg(test_server_path.value());
124    http_server->RedirectUsingPipe(STDOUT_FILENO, false);
125
126    if (!http_server->Start()) {
127      ADD_FAILURE() << "failed to spawn http server process";
128      return;
129    }
130    LOG(INFO) << "started http server with pid " << http_server->pid();
131
132    // Wait for server to begin accepting connections, obtain its port.
133    brillo::StreamPtr stdout = brillo::FileStream::FromFileDescriptor(
134        http_server->GetPipe(STDOUT_FILENO), false /* own */, nullptr);
135    if (!stdout)
136      return;
137
138    vector<char> buf(128);
139    string line;
140    while (line.find('\n') == string::npos) {
141      size_t read;
142      if (!stdout->ReadBlocking(buf.data(), buf.size(), &read, nullptr)) {
143        ADD_FAILURE() << "error reading http server stdout";
144        return;
145      }
146      line.append(buf.data(), read);
147      if (read == 0)
148        break;
149    }
150    // Parse the port from the output line.
151    const size_t listening_msg_prefix_len = strlen(kServerListeningMsgPrefix);
152    if (line.size() < listening_msg_prefix_len) {
153      ADD_FAILURE() << "server output too short";
154      return;
155    }
156
157    EXPECT_EQ(kServerListeningMsgPrefix,
158              line.substr(0, listening_msg_prefix_len));
159    string port_str = line.substr(listening_msg_prefix_len);
160    port_str.resize(port_str.find('\n'));
161    EXPECT_TRUE(base::StringToUint(port_str, &port_));
162
163    started_ = true;
164    LOG(INFO) << "server running, listening on port " << port_;
165
166    // Any failure before this point will SIGKILL the test server if started
167    // when the |http_server| goes out of scope.
168    http_server_ = std::move(http_server);
169  }
170
171  ~PythonHttpServer() {
172    // If there's no process, do nothing.
173    if (!http_server_)
174      return;
175    // Wait up to 10 seconds for the process to finish. Destroying the process
176    // will kill it with a SIGKILL otherwise.
177    http_server_->Kill(SIGTERM, 10);
178  }
179
180  in_port_t GetPort() const override {
181    return port_;
182  }
183
184 private:
185  static const char* kServerListeningMsgPrefix;
186
187  unique_ptr<brillo::Process> http_server_;
188  unsigned int port_;
189};
190
191const char* PythonHttpServer::kServerListeningMsgPrefix = "listening on port ";
192
193//
194// Class hierarchy for HTTP fetcher test wrappers.
195//
196
197class AnyHttpFetcherTest {
198 public:
199  AnyHttpFetcherTest() {}
200  virtual ~AnyHttpFetcherTest() {}
201
202  virtual HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) = 0;
203  HttpFetcher* NewLargeFetcher(size_t num_proxies) {
204    proxy_resolver_.set_num_proxies(num_proxies);
205    return NewLargeFetcher(&proxy_resolver_);
206  }
207  HttpFetcher* NewLargeFetcher() {
208    return NewLargeFetcher(1);
209  }
210
211  virtual HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) = 0;
212  HttpFetcher* NewSmallFetcher() {
213    proxy_resolver_.set_num_proxies(1);
214    return NewSmallFetcher(&proxy_resolver_);
215  }
216
217  virtual string BigUrl(in_port_t port) const { return kUnusedUrl; }
218  virtual string SmallUrl(in_port_t port) const { return kUnusedUrl; }
219  virtual string ErrorUrl(in_port_t port) const { return kUnusedUrl; }
220
221  virtual bool IsMock() const = 0;
222  virtual bool IsMulti() const = 0;
223  virtual bool IsHttpSupported() const = 0;
224
225  virtual void IgnoreServerAborting(HttpServer* server) const {}
226
227  virtual HttpServer* CreateServer() = 0;
228
229  FakeHardware* fake_hardware() {
230    return &fake_hardware_;
231  }
232
233 protected:
234  DirectProxyResolver proxy_resolver_;
235  FakeHardware fake_hardware_;
236};
237
238class MockHttpFetcherTest : public AnyHttpFetcherTest {
239 public:
240  // Necessary to unhide the definition in the base class.
241  using AnyHttpFetcherTest::NewLargeFetcher;
242  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
243    brillo::Blob big_data(1000000);
244    return new MockHttpFetcher(
245        big_data.data(), big_data.size(), proxy_resolver);
246  }
247
248  // Necessary to unhide the definition in the base class.
249  using AnyHttpFetcherTest::NewSmallFetcher;
250  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
251    return new MockHttpFetcher("x", 1, proxy_resolver);
252  }
253
254  bool IsMock() const override { return true; }
255  bool IsMulti() const override { return false; }
256  bool IsHttpSupported() const override { return true; }
257
258  HttpServer* CreateServer() override {
259    return new NullHttpServer;
260  }
261};
262
263class LibcurlHttpFetcherTest : public AnyHttpFetcherTest {
264 public:
265  // Necessary to unhide the definition in the base class.
266  using AnyHttpFetcherTest::NewLargeFetcher;
267  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
268    LibcurlHttpFetcher* ret =
269        new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_);
270    // Speed up test execution.
271    ret->set_idle_seconds(1);
272    ret->set_retry_seconds(1);
273    fake_hardware_.SetIsOfficialBuild(false);
274    return ret;
275  }
276
277  // Necessary to unhide the definition in the base class.
278  using AnyHttpFetcherTest::NewSmallFetcher;
279  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
280    return NewLargeFetcher(proxy_resolver);
281  }
282
283  string BigUrl(in_port_t port) const override {
284    return LocalServerUrlForPath(port,
285                                 base::StringPrintf("/download/%d",
286                                                    kBigLength));
287  }
288  string SmallUrl(in_port_t port) const override {
289    return LocalServerUrlForPath(port, "/foo");
290  }
291  string ErrorUrl(in_port_t port) const override {
292    return LocalServerUrlForPath(port, "/error");
293  }
294
295  bool IsMock() const override { return false; }
296  bool IsMulti() const override { return false; }
297  bool IsHttpSupported() const override { return true; }
298
299  void IgnoreServerAborting(HttpServer* server) const override {
300    // Nothing to do.
301  }
302
303  HttpServer* CreateServer() override {
304    return new PythonHttpServer;
305  }
306};
307
308class MultiRangeHttpFetcherTest : public LibcurlHttpFetcherTest {
309 public:
310  // Necessary to unhide the definition in the base class.
311  using AnyHttpFetcherTest::NewLargeFetcher;
312  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
313    MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(
314        new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_));
315    ret->ClearRanges();
316    ret->AddRange(0);
317    // Speed up test execution.
318    ret->set_idle_seconds(1);
319    ret->set_retry_seconds(1);
320    fake_hardware_.SetIsOfficialBuild(false);
321    return ret;
322  }
323
324  // Necessary to unhide the definition in the base class.
325  using AnyHttpFetcherTest::NewSmallFetcher;
326  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
327    return NewLargeFetcher(proxy_resolver);
328  }
329
330  bool IsMulti() const override { return true; }
331};
332
333class FileFetcherTest : public AnyHttpFetcherTest {
334 public:
335  // Necessary to unhide the definition in the base class.
336  using AnyHttpFetcherTest::NewLargeFetcher;
337  HttpFetcher* NewLargeFetcher(ProxyResolver* /* proxy_resolver */) override {
338    return new FileFetcher();
339  }
340
341  // Necessary to unhide the definition in the base class.
342  using AnyHttpFetcherTest::NewSmallFetcher;
343  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
344    return NewLargeFetcher(proxy_resolver);
345  }
346
347  string BigUrl(in_port_t port) const override {
348    return "file://" + temp_file_.path();
349  }
350  string SmallUrl(in_port_t port) const override {
351    test_utils::WriteFileString(temp_file_.path(), "small contents");
352    return "file://" + temp_file_.path();
353  }
354  string ErrorUrl(in_port_t port) const override {
355    return "file:///path/to/non-existing-file";
356  }
357
358  bool IsMock() const override { return false; }
359  bool IsMulti() const override { return false; }
360  bool IsHttpSupported() const override { return false; }
361
362  void IgnoreServerAborting(HttpServer* server) const override {}
363
364  HttpServer* CreateServer() override { return new NullHttpServer; }
365
366 private:
367  test_utils::ScopedTempFile temp_file_{"ue_file_fetcher.XXXXXX"};
368};
369
370//
371// Infrastructure for type tests of HTTP fetcher.
372// See: http://code.google.com/p/googletest/wiki/AdvancedGuide#Typed_Tests
373//
374
375// Fixture class template. We use an explicit constraint to guarantee that it
376// can only be instantiated with an AnyHttpFetcherTest type, see:
377// http://www2.research.att.com/~bs/bs_faq2.html#constraints
378template <typename T>
379class HttpFetcherTest : public ::testing::Test {
380 public:
381  base::MessageLoopForIO base_loop_;
382  brillo::BaseMessageLoop loop_{&base_loop_};
383
384  T test_;
385
386 protected:
387  HttpFetcherTest() {
388    loop_.SetAsCurrent();
389  }
390
391  void TearDown() override {
392    EXPECT_EQ(0, brillo::MessageLoopRunMaxIterations(&loop_, 1));
393  }
394
395 private:
396  static void TypeConstraint(T* a) {
397    AnyHttpFetcherTest *b = a;
398    if (b == 0)  // Silence compiler warning of unused variable.
399      *b = a;
400  }
401};
402
403// Test case types list.
404typedef ::testing::Types<LibcurlHttpFetcherTest,
405                         MockHttpFetcherTest,
406                         MultiRangeHttpFetcherTest,
407                         FileFetcherTest>
408    HttpFetcherTestTypes;
409TYPED_TEST_CASE(HttpFetcherTest, HttpFetcherTestTypes);
410
411
412namespace {
413class HttpFetcherTestDelegate : public HttpFetcherDelegate {
414 public:
415  HttpFetcherTestDelegate() = default;
416
417  void ReceivedBytes(HttpFetcher* /* fetcher */,
418                     const void* bytes,
419                     size_t length) override {
420    data.append(reinterpret_cast<const char*>(bytes), length);
421    // Update counters
422    times_received_bytes_called_++;
423  }
424
425  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
426    if (is_expect_error_)
427      EXPECT_EQ(kHttpResponseNotFound, fetcher->http_response_code());
428    else
429      EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
430    MessageLoop::current()->BreakLoop();
431
432    // Update counter
433    times_transfer_complete_called_++;
434  }
435
436  void TransferTerminated(HttpFetcher* fetcher) override {
437    ADD_FAILURE();
438    times_transfer_terminated_called_++;
439    MessageLoop::current()->BreakLoop();
440  }
441
442  // Are we expecting an error response? (default: no)
443  bool is_expect_error_{false};
444
445  // Counters for callback invocations.
446  int times_transfer_complete_called_{0};
447  int times_transfer_terminated_called_{0};
448  int times_received_bytes_called_{0};
449
450  // The received data bytes.
451  string data;
452};
453
454
455void StartTransfer(HttpFetcher* http_fetcher, const string& url) {
456  http_fetcher->BeginTransfer(url);
457}
458}  // namespace
459
460TYPED_TEST(HttpFetcherTest, SimpleTest) {
461  HttpFetcherTestDelegate delegate;
462  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
463  fetcher->set_delegate(&delegate);
464
465  unique_ptr<HttpServer> server(this->test_.CreateServer());
466  ASSERT_TRUE(server->started_);
467
468  this->loop_.PostTask(FROM_HERE, base::Bind(
469      StartTransfer,
470      fetcher.get(),
471      this->test_.SmallUrl(server->GetPort())));
472  this->loop_.Run();
473}
474
475TYPED_TEST(HttpFetcherTest, SimpleBigTest) {
476  HttpFetcherTestDelegate delegate;
477  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
478  fetcher->set_delegate(&delegate);
479
480  unique_ptr<HttpServer> server(this->test_.CreateServer());
481  ASSERT_TRUE(server->started_);
482
483  this->loop_.PostTask(FROM_HERE, base::Bind(
484      StartTransfer,
485      fetcher.get(),
486      this->test_.BigUrl(server->GetPort())));
487  this->loop_.Run();
488}
489
490// Issue #9648: when server returns an error HTTP response, the fetcher needs to
491// terminate transfer prematurely, rather than try to process the error payload.
492TYPED_TEST(HttpFetcherTest, ErrorTest) {
493  if (this->test_.IsMock() || this->test_.IsMulti())
494    return;
495  HttpFetcherTestDelegate delegate;
496
497  // Delegate should expect an error response.
498  delegate.is_expect_error_ = true;
499
500  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
501  fetcher->set_delegate(&delegate);
502
503  unique_ptr<HttpServer> server(this->test_.CreateServer());
504  ASSERT_TRUE(server->started_);
505
506  this->loop_.PostTask(FROM_HERE, base::Bind(
507      StartTransfer,
508      fetcher.get(),
509      this->test_.ErrorUrl(server->GetPort())));
510  this->loop_.Run();
511
512  // Make sure that no bytes were received.
513  CHECK_EQ(delegate.times_received_bytes_called_, 0);
514  CHECK_EQ(fetcher->GetBytesDownloaded(), static_cast<size_t>(0));
515
516  // Make sure that transfer completion was signaled once, and no termination
517  // was signaled.
518  CHECK_EQ(delegate.times_transfer_complete_called_, 1);
519  CHECK_EQ(delegate.times_transfer_terminated_called_, 0);
520}
521
522TYPED_TEST(HttpFetcherTest, ExtraHeadersInRequestTest) {
523  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
524    return;
525
526  HttpFetcherTestDelegate delegate;
527  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
528  fetcher->set_delegate(&delegate);
529  fetcher->SetHeader("User-Agent", "MyTest");
530  fetcher->SetHeader("user-agent", "Override that header");
531  fetcher->SetHeader("Authorization", "Basic user:passwd");
532
533  // Invalid headers.
534  fetcher->SetHeader("X-Foo", "Invalid\nHeader\nIgnored");
535  fetcher->SetHeader("X-Bar: ", "I do not know how to parse");
536
537  // Hide Accept header normally added by default.
538  fetcher->SetHeader("Accept", "");
539
540  PythonHttpServer server;
541  int port = server.GetPort();
542  ASSERT_TRUE(server.started_);
543
544  this->loop_.PostTask(
545      FROM_HERE,
546      base::Bind(StartTransfer,
547                 fetcher.get(),
548                 LocalServerUrlForPath(port, "/echo-headers")));
549  this->loop_.Run();
550
551  EXPECT_NE(string::npos,
552            delegate.data.find("user-agent: Override that header\r\n"));
553  EXPECT_NE(string::npos,
554            delegate.data.find("Authorization: Basic user:passwd\r\n"));
555
556  EXPECT_EQ(string::npos, delegate.data.find("\nAccept:"));
557  EXPECT_EQ(string::npos, delegate.data.find("X-Foo: Invalid"));
558  EXPECT_EQ(string::npos, delegate.data.find("X-Bar: I do not"));
559}
560
561namespace {
562class PausingHttpFetcherTestDelegate : public HttpFetcherDelegate {
563 public:
564  void ReceivedBytes(HttpFetcher* fetcher,
565                     const void* /* bytes */, size_t /* length */) override {
566    CHECK(!paused_);
567    paused_ = true;
568    fetcher->Pause();
569  }
570  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
571    MessageLoop::current()->BreakLoop();
572  }
573  void TransferTerminated(HttpFetcher* fetcher) override {
574    ADD_FAILURE();
575  }
576  void Unpause() {
577    CHECK(paused_);
578    paused_ = false;
579    fetcher_->Unpause();
580  }
581  bool paused_;
582  HttpFetcher* fetcher_;
583};
584
585void UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate* delegate,
586                              MessageLoop::TaskId* my_id) {
587  if (delegate->paused_)
588    delegate->Unpause();
589  // Update the task id with the new scheduled callback.
590  *my_id = MessageLoop::current()->PostDelayedTask(
591      FROM_HERE,
592      base::Bind(&UnpausingTimeoutCallback, delegate, my_id),
593      base::TimeDelta::FromMilliseconds(200));
594}
595}  // namespace
596
597TYPED_TEST(HttpFetcherTest, PauseTest) {
598  PausingHttpFetcherTestDelegate delegate;
599  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
600  delegate.paused_ = false;
601  delegate.fetcher_ = fetcher.get();
602  fetcher->set_delegate(&delegate);
603
604  unique_ptr<HttpServer> server(this->test_.CreateServer());
605  ASSERT_TRUE(server->started_);
606
607  MessageLoop::TaskId callback_id;
608  callback_id = this->loop_.PostDelayedTask(
609      FROM_HERE,
610      base::Bind(&UnpausingTimeoutCallback, &delegate, &callback_id),
611      base::TimeDelta::FromMilliseconds(200));
612  fetcher->BeginTransfer(this->test_.BigUrl(server->GetPort()));
613
614  this->loop_.Run();
615  EXPECT_TRUE(this->loop_.CancelTask(callback_id));
616}
617
618// This test will pause the fetcher while the download is not yet started
619// because it is waiting for the proxy to be resolved.
620TYPED_TEST(HttpFetcherTest, PauseWhileResolvingProxyTest) {
621  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
622    return;
623  MockProxyResolver mock_resolver;
624  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
625
626  // Saved arguments from the proxy call.
627  ProxiesResolvedFn proxy_callback = nullptr;
628  void* proxy_data = nullptr;
629
630  EXPECT_CALL(mock_resolver, GetProxiesForUrl("http://fake_url", _, _))
631      .WillOnce(DoAll(
632          SaveArg<1>(&proxy_callback), SaveArg<2>(&proxy_data), Return(true)));
633  fetcher->BeginTransfer("http://fake_url");
634  testing::Mock::VerifyAndClearExpectations(&mock_resolver);
635
636  // Pausing and unpausing while resolving the proxy should not affect anything.
637  fetcher->Pause();
638  fetcher->Unpause();
639  fetcher->Pause();
640  // Proxy resolver comes back after we paused the fetcher.
641  ASSERT_TRUE(proxy_callback);
642  (*proxy_callback)({1, kNoProxy}, proxy_data);
643}
644
645namespace {
646class AbortingHttpFetcherTestDelegate : public HttpFetcherDelegate {
647 public:
648  void ReceivedBytes(HttpFetcher* fetcher,
649                     const void* bytes, size_t length) override {}
650  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
651    ADD_FAILURE();  // We should never get here
652    MessageLoop::current()->BreakLoop();
653  }
654  void TransferTerminated(HttpFetcher* fetcher) override {
655    EXPECT_EQ(fetcher, fetcher_.get());
656    EXPECT_FALSE(once_);
657    EXPECT_TRUE(callback_once_);
658    callback_once_ = false;
659    // The fetcher could have a callback scheduled on the ProxyResolver that
660    // can fire after this callback. We wait until the end of the test to
661    // delete the fetcher.
662  }
663  void TerminateTransfer() {
664    CHECK(once_);
665    once_ = false;
666    fetcher_->TerminateTransfer();
667  }
668  void EndLoop() {
669    MessageLoop::current()->BreakLoop();
670  }
671  bool once_;
672  bool callback_once_;
673  unique_ptr<HttpFetcher> fetcher_;
674};
675
676void AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate* delegate,
677                             MessageLoop::TaskId* my_id) {
678  if (delegate->once_) {
679    delegate->TerminateTransfer();
680    *my_id = MessageLoop::current()->PostTask(
681        FROM_HERE,
682        base::Bind(AbortingTimeoutCallback, delegate, my_id));
683  } else {
684    delegate->EndLoop();
685    *my_id = MessageLoop::kTaskIdNull;
686  }
687}
688}  // namespace
689
690TYPED_TEST(HttpFetcherTest, AbortTest) {
691  AbortingHttpFetcherTestDelegate delegate;
692  delegate.fetcher_.reset(this->test_.NewLargeFetcher());
693  delegate.once_ = true;
694  delegate.callback_once_ = true;
695  delegate.fetcher_->set_delegate(&delegate);
696
697  unique_ptr<HttpServer> server(this->test_.CreateServer());
698  this->test_.IgnoreServerAborting(server.get());
699  ASSERT_TRUE(server->started_);
700
701  MessageLoop::TaskId task_id = MessageLoop::kTaskIdNull;
702
703  task_id = this->loop_.PostTask(
704      FROM_HERE,
705      base::Bind(AbortingTimeoutCallback, &delegate, &task_id));
706  delegate.fetcher_->BeginTransfer(this->test_.BigUrl(server->GetPort()));
707
708  this->loop_.Run();
709  CHECK(!delegate.once_);
710  CHECK(!delegate.callback_once_);
711  this->loop_.CancelTask(task_id);
712}
713
714namespace {
715class FlakyHttpFetcherTestDelegate : public HttpFetcherDelegate {
716 public:
717  void ReceivedBytes(HttpFetcher* fetcher,
718                     const void* bytes, size_t length) override {
719    data.append(reinterpret_cast<const char*>(bytes), length);
720  }
721  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
722    EXPECT_TRUE(successful);
723    EXPECT_EQ(kHttpResponsePartialContent, fetcher->http_response_code());
724    MessageLoop::current()->BreakLoop();
725  }
726  void TransferTerminated(HttpFetcher* fetcher) override {
727    ADD_FAILURE();
728  }
729  string data;
730};
731}  // namespace
732
733TYPED_TEST(HttpFetcherTest, FlakyTest) {
734  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
735    return;
736  {
737    FlakyHttpFetcherTestDelegate delegate;
738    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
739    fetcher->set_delegate(&delegate);
740
741    unique_ptr<HttpServer> server(this->test_.CreateServer());
742    ASSERT_TRUE(server->started_);
743
744    this->loop_.PostTask(FROM_HERE, base::Bind(
745        &StartTransfer,
746        fetcher.get(),
747        LocalServerUrlForPath(server->GetPort(),
748                              base::StringPrintf("/flaky/%d/%d/%d/%d",
749                                                 kBigLength,
750                                                 kFlakyTruncateLength,
751                                                 kFlakySleepEvery,
752                                                 kFlakySleepSecs))));
753    this->loop_.Run();
754
755    // verify the data we get back
756    ASSERT_EQ(kBigLength, static_cast<int>(delegate.data.size()));
757    for (int i = 0; i < kBigLength; i += 10) {
758      // Assert so that we don't flood the screen w/ EXPECT errors on failure.
759      ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
760    }
761  }
762}
763
764namespace {
765// This delegate kills the server attached to it after receiving any bytes.
766// This can be used for testing what happens when you try to fetch data and
767// the server dies.
768class FailureHttpFetcherTestDelegate : public HttpFetcherDelegate {
769 public:
770  explicit FailureHttpFetcherTestDelegate(PythonHttpServer* server)
771      : server_(server) {}
772
773  ~FailureHttpFetcherTestDelegate() override {
774    if (server_) {
775      LOG(INFO) << "Stopping server in destructor";
776      delete server_;
777      LOG(INFO) << "server stopped";
778    }
779  }
780
781  void ReceivedBytes(HttpFetcher* fetcher,
782                     const void* bytes, size_t length) override {
783    if (server_) {
784      LOG(INFO) << "Stopping server in ReceivedBytes";
785      delete server_;
786      LOG(INFO) << "server stopped";
787      server_ = nullptr;
788    }
789  }
790  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
791    EXPECT_FALSE(successful);
792    EXPECT_EQ(0, fetcher->http_response_code());
793    MessageLoop::current()->BreakLoop();
794  }
795  void TransferTerminated(HttpFetcher* fetcher) override {
796    ADD_FAILURE();
797  }
798  PythonHttpServer* server_;
799};
800}  // namespace
801
802
803TYPED_TEST(HttpFetcherTest, FailureTest) {
804  // This test ensures that a fetcher responds correctly when a server isn't
805  // available at all.
806  if (this->test_.IsMock())
807    return;
808  {
809    FailureHttpFetcherTestDelegate delegate(nullptr);
810    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
811    fetcher->set_delegate(&delegate);
812
813    this->loop_.PostTask(FROM_HERE,
814                         base::Bind(StartTransfer,
815                                    fetcher.get(),
816                                    "http://host_doesnt_exist99999999"));
817    this->loop_.Run();
818
819    // Exiting and testing happens in the delegate
820  }
821}
822
823TYPED_TEST(HttpFetcherTest, NoResponseTest) {
824  // This test starts a new http server but the server doesn't respond and just
825  // closes the connection.
826  if (this->test_.IsMock())
827    return;
828
829  PythonHttpServer* server = new PythonHttpServer();
830  int port = server->GetPort();
831  ASSERT_TRUE(server->started_);
832
833  // Handles destruction and claims ownership.
834  FailureHttpFetcherTestDelegate delegate(server);
835  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
836  fetcher->set_delegate(&delegate);
837  // The server will not reply at all, so we can limit the execution time of the
838  // test by reducing the low-speed timeout to something small. The test will
839  // finish once the TimeoutCallback() triggers (every second) and the timeout
840  // expired.
841  fetcher->set_low_speed_limit(kDownloadLowSpeedLimitBps, 1);
842
843  this->loop_.PostTask(FROM_HERE, base::Bind(
844      StartTransfer,
845      fetcher.get(),
846      LocalServerUrlForPath(port, "/hang")));
847  this->loop_.Run();
848
849  // Check that no other callback runs in the next two seconds. That would
850  // indicate a leaked callback.
851  bool timeout = false;
852  auto callback = base::Bind([&timeout]{ timeout = true;});
853  this->loop_.PostDelayedTask(FROM_HERE, callback,
854                              base::TimeDelta::FromSeconds(2));
855  EXPECT_TRUE(this->loop_.RunOnce(true));
856  EXPECT_TRUE(timeout);
857}
858
859TYPED_TEST(HttpFetcherTest, ServerDiesTest) {
860  // This test starts a new http server and kills it after receiving its first
861  // set of bytes. It test whether or not our fetcher eventually gives up on
862  // retries and aborts correctly.
863  if (this->test_.IsMock())
864    return;
865  {
866    PythonHttpServer* server = new PythonHttpServer();
867    int port = server->GetPort();
868    ASSERT_TRUE(server->started_);
869
870    // Handles destruction and claims ownership.
871    FailureHttpFetcherTestDelegate delegate(server);
872    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
873    fetcher->set_delegate(&delegate);
874
875    this->loop_.PostTask(FROM_HERE, base::Bind(
876        StartTransfer,
877        fetcher.get(),
878        LocalServerUrlForPath(port,
879                              base::StringPrintf("/flaky/%d/%d/%d/%d",
880                                                 kBigLength,
881                                                 kFlakyTruncateLength,
882                                                 kFlakySleepEvery,
883                                                 kFlakySleepSecs))));
884    this->loop_.Run();
885
886    // Exiting and testing happens in the delegate
887  }
888}
889
890namespace {
891const HttpResponseCode kRedirectCodes[] = {
892  kHttpResponseMovedPermanently, kHttpResponseFound, kHttpResponseSeeOther,
893  kHttpResponseTempRedirect
894};
895
896class RedirectHttpFetcherTestDelegate : public HttpFetcherDelegate {
897 public:
898  explicit RedirectHttpFetcherTestDelegate(bool expected_successful)
899      : expected_successful_(expected_successful) {}
900  void ReceivedBytes(HttpFetcher* fetcher,
901                     const void* bytes, size_t length) override {
902    data.append(reinterpret_cast<const char*>(bytes), length);
903  }
904  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
905    EXPECT_EQ(expected_successful_, successful);
906    if (expected_successful_) {
907      EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
908    } else {
909      EXPECT_GE(fetcher->http_response_code(), kHttpResponseMovedPermanently);
910      EXPECT_LE(fetcher->http_response_code(), kHttpResponseTempRedirect);
911    }
912    MessageLoop::current()->BreakLoop();
913  }
914  void TransferTerminated(HttpFetcher* fetcher) override {
915    ADD_FAILURE();
916  }
917  bool expected_successful_;
918  string data;
919};
920
921// RedirectTest takes ownership of |http_fetcher|.
922void RedirectTest(const HttpServer* server,
923                  bool expected_successful,
924                  const string& url,
925                  HttpFetcher* http_fetcher) {
926  RedirectHttpFetcherTestDelegate delegate(expected_successful);
927  unique_ptr<HttpFetcher> fetcher(http_fetcher);
928  fetcher->set_delegate(&delegate);
929
930  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
931      StartTransfer,
932      fetcher.get(),
933      LocalServerUrlForPath(server->GetPort(), url)));
934  MessageLoop::current()->Run();
935  if (expected_successful) {
936    // verify the data we get back
937    ASSERT_EQ(static_cast<size_t>(kMediumLength), delegate.data.size());
938    for (int i = 0; i < kMediumLength; i += 10) {
939      // Assert so that we don't flood the screen w/ EXPECT errors on failure.
940      ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
941    }
942  }
943}
944}  // namespace
945
946TYPED_TEST(HttpFetcherTest, SimpleRedirectTest) {
947  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
948    return;
949
950  unique_ptr<HttpServer> server(this->test_.CreateServer());
951  ASSERT_TRUE(server->started_);
952
953  for (size_t c = 0; c < arraysize(kRedirectCodes); ++c) {
954    const string url = base::StringPrintf("/redirect/%d/download/%d",
955                                          kRedirectCodes[c],
956                                          kMediumLength);
957    RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
958  }
959}
960
961TYPED_TEST(HttpFetcherTest, MaxRedirectTest) {
962  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
963    return;
964
965  unique_ptr<HttpServer> server(this->test_.CreateServer());
966  ASSERT_TRUE(server->started_);
967
968  string url;
969  for (int r = 0; r < kDownloadMaxRedirects; r++) {
970    url += base::StringPrintf("/redirect/%d",
971                              kRedirectCodes[r % arraysize(kRedirectCodes)]);
972  }
973  url += base::StringPrintf("/download/%d", kMediumLength);
974  RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
975}
976
977TYPED_TEST(HttpFetcherTest, BeyondMaxRedirectTest) {
978  if (this->test_.IsMock() || !this->test_.IsHttpSupported())
979    return;
980
981  unique_ptr<HttpServer> server(this->test_.CreateServer());
982  ASSERT_TRUE(server->started_);
983
984  string url;
985  for (int r = 0; r < kDownloadMaxRedirects + 1; r++) {
986    url += base::StringPrintf("/redirect/%d",
987                              kRedirectCodes[r % arraysize(kRedirectCodes)]);
988  }
989  url += base::StringPrintf("/download/%d", kMediumLength);
990  RedirectTest(server.get(), false, url, this->test_.NewLargeFetcher());
991}
992
993namespace {
994class MultiHttpFetcherTestDelegate : public HttpFetcherDelegate {
995 public:
996  explicit MultiHttpFetcherTestDelegate(int expected_response_code)
997      : expected_response_code_(expected_response_code) {}
998
999  void ReceivedBytes(HttpFetcher* fetcher,
1000                     const void* bytes, size_t length) override {
1001    EXPECT_EQ(fetcher, fetcher_.get());
1002    data.append(reinterpret_cast<const char*>(bytes), length);
1003  }
1004
1005  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1006    EXPECT_EQ(fetcher, fetcher_.get());
1007    EXPECT_EQ(expected_response_code_ != kHttpResponseUndefined, successful);
1008    if (expected_response_code_ != 0)
1009      EXPECT_EQ(expected_response_code_, fetcher->http_response_code());
1010    // Destroy the fetcher (because we're allowed to).
1011    fetcher_.reset(nullptr);
1012    MessageLoop::current()->BreakLoop();
1013  }
1014
1015  void TransferTerminated(HttpFetcher* fetcher) override {
1016    ADD_FAILURE();
1017  }
1018
1019  unique_ptr<HttpFetcher> fetcher_;
1020  int expected_response_code_;
1021  string data;
1022};
1023
1024void MultiTest(HttpFetcher* fetcher_in,
1025               FakeHardware* fake_hardware,
1026               const string& url,
1027               const vector<pair<off_t, off_t>>& ranges,
1028               const string& expected_prefix,
1029               size_t expected_size,
1030               HttpResponseCode expected_response_code) {
1031  MultiHttpFetcherTestDelegate delegate(expected_response_code);
1032  delegate.fetcher_.reset(fetcher_in);
1033
1034  MultiRangeHttpFetcher* multi_fetcher =
1035      static_cast<MultiRangeHttpFetcher*>(fetcher_in);
1036  ASSERT_TRUE(multi_fetcher);
1037  multi_fetcher->ClearRanges();
1038  for (vector<pair<off_t, off_t>>::const_iterator it = ranges.begin(),
1039           e = ranges.end(); it != e; ++it) {
1040    string tmp_str = base::StringPrintf("%jd+", it->first);
1041    if (it->second > 0) {
1042      base::StringAppendF(&tmp_str, "%jd", it->second);
1043      multi_fetcher->AddRange(it->first, it->second);
1044    } else {
1045      base::StringAppendF(&tmp_str, "?");
1046      multi_fetcher->AddRange(it->first);
1047    }
1048    LOG(INFO) << "added range: " << tmp_str;
1049  }
1050  fake_hardware->SetIsOfficialBuild(false);
1051  multi_fetcher->set_delegate(&delegate);
1052
1053  MessageLoop::current()->PostTask(
1054      FROM_HERE,
1055      base::Bind(StartTransfer, multi_fetcher, url));
1056  MessageLoop::current()->Run();
1057
1058  EXPECT_EQ(expected_size, delegate.data.size());
1059  EXPECT_EQ(expected_prefix,
1060            string(delegate.data.data(), expected_prefix.size()));
1061}
1062}  // namespace
1063
1064TYPED_TEST(HttpFetcherTest, MultiHttpFetcherSimpleTest) {
1065  if (!this->test_.IsMulti())
1066    return;
1067
1068  unique_ptr<HttpServer> server(this->test_.CreateServer());
1069  ASSERT_TRUE(server->started_);
1070
1071  vector<pair<off_t, off_t>> ranges;
1072  ranges.push_back(make_pair(0, 25));
1073  ranges.push_back(make_pair(99, 0));
1074  MultiTest(this->test_.NewLargeFetcher(),
1075            this->test_.fake_hardware(),
1076            this->test_.BigUrl(server->GetPort()),
1077            ranges,
1078            "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1079            kBigLength - (99 - 25),
1080            kHttpResponsePartialContent);
1081}
1082
1083TYPED_TEST(HttpFetcherTest, MultiHttpFetcherLengthLimitTest) {
1084  if (!this->test_.IsMulti())
1085    return;
1086
1087  unique_ptr<HttpServer> server(this->test_.CreateServer());
1088  ASSERT_TRUE(server->started_);
1089
1090  vector<pair<off_t, off_t>> ranges;
1091  ranges.push_back(make_pair(0, 24));
1092  MultiTest(this->test_.NewLargeFetcher(),
1093            this->test_.fake_hardware(),
1094            this->test_.BigUrl(server->GetPort()),
1095            ranges,
1096            "abcdefghijabcdefghijabcd",
1097            24,
1098            kHttpResponsePartialContent);
1099}
1100
1101TYPED_TEST(HttpFetcherTest, MultiHttpFetcherMultiEndTest) {
1102  if (!this->test_.IsMulti())
1103    return;
1104
1105  unique_ptr<HttpServer> server(this->test_.CreateServer());
1106  ASSERT_TRUE(server->started_);
1107
1108  vector<pair<off_t, off_t>> ranges;
1109  ranges.push_back(make_pair(kBigLength - 2, 0));
1110  ranges.push_back(make_pair(kBigLength - 3, 0));
1111  MultiTest(this->test_.NewLargeFetcher(),
1112            this->test_.fake_hardware(),
1113            this->test_.BigUrl(server->GetPort()),
1114            ranges,
1115            "ijhij",
1116            5,
1117            kHttpResponsePartialContent);
1118}
1119
1120TYPED_TEST(HttpFetcherTest, MultiHttpFetcherInsufficientTest) {
1121  if (!this->test_.IsMulti())
1122    return;
1123
1124  unique_ptr<HttpServer> server(this->test_.CreateServer());
1125  ASSERT_TRUE(server->started_);
1126
1127  vector<pair<off_t, off_t>> ranges;
1128  ranges.push_back(make_pair(kBigLength - 2, 4));
1129  for (int i = 0; i < 2; ++i) {
1130    LOG(INFO) << "i = " << i;
1131    MultiTest(this->test_.NewLargeFetcher(),
1132              this->test_.fake_hardware(),
1133              this->test_.BigUrl(server->GetPort()),
1134              ranges,
1135              "ij",
1136              2,
1137              kHttpResponseUndefined);
1138    ranges.push_back(make_pair(0, 5));
1139  }
1140}
1141
1142// Issue #18143: when a fetch of a secondary chunk out of a chain, then it
1143// should retry with other proxies listed before giving up.
1144//
1145// (1) successful recovery: The offset fetch will fail twice but succeed with
1146// the third proxy.
1147TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetRecoverableTest) {
1148  if (!this->test_.IsMulti())
1149    return;
1150
1151  unique_ptr<HttpServer> server(this->test_.CreateServer());
1152  ASSERT_TRUE(server->started_);
1153
1154  vector<pair<off_t, off_t>> ranges;
1155  ranges.push_back(make_pair(0, 25));
1156  ranges.push_back(make_pair(99, 0));
1157  MultiTest(this->test_.NewLargeFetcher(3),
1158            this->test_.fake_hardware(),
1159            LocalServerUrlForPath(server->GetPort(),
1160                                  base::StringPrintf("/error-if-offset/%d/2",
1161                                                     kBigLength)),
1162            ranges,
1163            "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1164            kBigLength - (99 - 25),
1165            kHttpResponsePartialContent);
1166}
1167
1168// (2) unsuccessful recovery: The offset fetch will fail repeatedly.  The
1169// fetcher will signal a (failed) completed transfer to the delegate.
1170TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetUnrecoverableTest) {
1171  if (!this->test_.IsMulti())
1172    return;
1173
1174  unique_ptr<HttpServer> server(this->test_.CreateServer());
1175  ASSERT_TRUE(server->started_);
1176
1177  vector<pair<off_t, off_t>> ranges;
1178  ranges.push_back(make_pair(0, 25));
1179  ranges.push_back(make_pair(99, 0));
1180  MultiTest(this->test_.NewLargeFetcher(2),
1181            this->test_.fake_hardware(),
1182            LocalServerUrlForPath(server->GetPort(),
1183                                  base::StringPrintf("/error-if-offset/%d/3",
1184                                                     kBigLength)),
1185            ranges,
1186            "abcdefghijabcdefghijabcde",  // only received the first chunk
1187            25,
1188            kHttpResponseUndefined);
1189}
1190
1191
1192
1193namespace {
1194class BlockedTransferTestDelegate : public HttpFetcherDelegate {
1195 public:
1196  void ReceivedBytes(HttpFetcher* fetcher,
1197                     const void* bytes, size_t length) override {
1198    ADD_FAILURE();
1199  }
1200  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1201    EXPECT_FALSE(successful);
1202    MessageLoop::current()->BreakLoop();
1203  }
1204  void TransferTerminated(HttpFetcher* fetcher) override {
1205    ADD_FAILURE();
1206  }
1207};
1208
1209void BlockedTransferTestHelper(AnyHttpFetcherTest* fetcher_test,
1210                               bool is_official_build) {
1211  if (fetcher_test->IsMock() || fetcher_test->IsMulti())
1212    return;
1213
1214  unique_ptr<HttpServer> server(fetcher_test->CreateServer());
1215  ASSERT_TRUE(server->started_);
1216
1217  BlockedTransferTestDelegate delegate;
1218  unique_ptr<HttpFetcher> fetcher(fetcher_test->NewLargeFetcher());
1219  LOG(INFO) << "is_official_build: " << is_official_build;
1220  // NewLargeFetcher creates the HttpFetcher* with a FakeSystemState.
1221  fetcher_test->fake_hardware()->SetIsOfficialBuild(is_official_build);
1222  fetcher->set_delegate(&delegate);
1223
1224  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
1225      StartTransfer,
1226      fetcher.get(),
1227      LocalServerUrlForPath(server->GetPort(),
1228                            fetcher_test->SmallUrl(server->GetPort()))));
1229  MessageLoop::current()->Run();
1230}
1231}  // namespace
1232
1233TYPED_TEST(HttpFetcherTest, BlockedTransferTest) {
1234  BlockedTransferTestHelper(&this->test_, false);
1235}
1236
1237TYPED_TEST(HttpFetcherTest, BlockedTransferOfficialBuildTest) {
1238  BlockedTransferTestHelper(&this->test_, true);
1239}
1240
1241}  // namespace chromeos_update_engine
1242