1//
2// Copyright (C) 2012 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include <netinet/in.h>
18#include <netinet/ip.h>
19#include <sys/socket.h>
20#include <unistd.h>
21
22#include <memory>
23#include <string>
24#include <utility>
25#include <vector>
26
27#include <base/bind.h>
28#include <base/location.h>
29#include <base/logging.h>
30#include <base/message_loop/message_loop.h>
31#include <base/strings/string_number_conversions.h>
32#include <base/strings/string_util.h>
33#include <base/strings/stringprintf.h>
34#include <base/time/time.h>
35#include <brillo/bind_lambda.h>
36#include <brillo/message_loops/base_message_loop.h>
37#include <brillo/message_loops/message_loop.h>
38#include <brillo/message_loops/message_loop_utils.h>
39#include <brillo/process.h>
40#include <brillo/streams/file_stream.h>
41#include <brillo/streams/stream.h>
42#include <gtest/gtest.h>
43
44#include "update_engine/common/fake_hardware.h"
45#include "update_engine/common/http_common.h"
46#include "update_engine/common/libcurl_http_fetcher.h"
47#include "update_engine/common/mock_http_fetcher.h"
48#include "update_engine/common/multi_range_http_fetcher.h"
49#include "update_engine/common/test_utils.h"
50#include "update_engine/common/utils.h"
51#include "update_engine/mock_proxy_resolver.h"
52#include "update_engine/proxy_resolver.h"
53
54using brillo::MessageLoop;
55using std::make_pair;
56using std::pair;
57using std::string;
58using std::unique_ptr;
59using std::vector;
60using testing::DoAll;
61using testing::Return;
62using testing::SaveArg;
63using testing::_;
64
65namespace {
66
67const int kBigLength           = 100000;
68const int kMediumLength        = 1000;
69const int kFlakyTruncateLength = 29000;
70const int kFlakySleepEvery     = 3;
71const int kFlakySleepSecs      = 10;
72
73}  // namespace
74
75namespace chromeos_update_engine {
76
77static const char *kUnusedUrl = "unused://unused";
78
79static inline string LocalServerUrlForPath(in_port_t port,
80                                           const string& path) {
81  string port_str = (port ? base::StringPrintf(":%hu", port) : "");
82  return base::StringPrintf("http://127.0.0.1%s%s", port_str.c_str(),
83                            path.c_str());
84}
85
86//
87// Class hierarchy for HTTP server implementations.
88//
89
90class HttpServer {
91 public:
92  // This makes it an abstract class (dirty but works).
93  virtual ~HttpServer() = 0;
94
95  virtual in_port_t GetPort() const {
96    return 0;
97  }
98
99  bool started_;
100};
101
102HttpServer::~HttpServer() {}
103
104
105class NullHttpServer : public HttpServer {
106 public:
107  NullHttpServer() {
108    started_ = true;
109  }
110};
111
112
113class PythonHttpServer : public HttpServer {
114 public:
115  PythonHttpServer() : port_(0) {
116    started_ = false;
117
118    // Spawn the server process.
119    unique_ptr<brillo::Process> http_server(new brillo::ProcessImpl());
120    base::FilePath test_server_path =
121        test_utils::GetBuildArtifactsPath().Append("test_http_server");
122    http_server->AddArg(test_server_path.value());
123    http_server->RedirectUsingPipe(STDOUT_FILENO, false);
124
125    if (!http_server->Start()) {
126      ADD_FAILURE() << "failed to spawn http server process";
127      return;
128    }
129    LOG(INFO) << "started http server with pid " << http_server->pid();
130
131    // Wait for server to begin accepting connections, obtain its port.
132    brillo::StreamPtr stdout = brillo::FileStream::FromFileDescriptor(
133        http_server->GetPipe(STDOUT_FILENO), false /* own */, nullptr);
134    if (!stdout)
135      return;
136
137    vector<char> buf(128);
138    string line;
139    while (line.find('\n') == string::npos) {
140      size_t read;
141      if (!stdout->ReadBlocking(buf.data(), buf.size(), &read, nullptr)) {
142        ADD_FAILURE() << "error reading http server stdout";
143        return;
144      }
145      line.append(buf.data(), read);
146      if (read == 0)
147        break;
148    }
149    // Parse the port from the output line.
150    const size_t listening_msg_prefix_len = strlen(kServerListeningMsgPrefix);
151    if (line.size() < listening_msg_prefix_len) {
152      ADD_FAILURE() << "server output too short";
153      return;
154    }
155
156    EXPECT_EQ(kServerListeningMsgPrefix,
157              line.substr(0, listening_msg_prefix_len));
158    string port_str = line.substr(listening_msg_prefix_len);
159    port_str.resize(port_str.find('\n'));
160    EXPECT_TRUE(base::StringToUint(port_str, &port_));
161
162    started_ = true;
163    LOG(INFO) << "server running, listening on port " << port_;
164
165    // Any failure before this point will SIGKILL the test server if started
166    // when the |http_server| goes out of scope.
167    http_server_ = std::move(http_server);
168  }
169
170  ~PythonHttpServer() {
171    // If there's no process, do nothing.
172    if (!http_server_)
173      return;
174    // Wait up to 10 seconds for the process to finish. Destroying the process
175    // will kill it with a SIGKILL otherwise.
176    http_server_->Kill(SIGTERM, 10);
177  }
178
179  in_port_t GetPort() const override {
180    return port_;
181  }
182
183 private:
184  static const char* kServerListeningMsgPrefix;
185
186  unique_ptr<brillo::Process> http_server_;
187  unsigned int port_;
188};
189
190const char* PythonHttpServer::kServerListeningMsgPrefix = "listening on port ";
191
192//
193// Class hierarchy for HTTP fetcher test wrappers.
194//
195
196class AnyHttpFetcherTest {
197 public:
198  AnyHttpFetcherTest() {}
199  virtual ~AnyHttpFetcherTest() {}
200
201  virtual HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) = 0;
202  HttpFetcher* NewLargeFetcher(size_t num_proxies) {
203    proxy_resolver_.set_num_proxies(num_proxies);
204    return NewLargeFetcher(&proxy_resolver_);
205  }
206  HttpFetcher* NewLargeFetcher() {
207    return NewLargeFetcher(1);
208  }
209
210  virtual HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) = 0;
211  HttpFetcher* NewSmallFetcher() {
212    proxy_resolver_.set_num_proxies(1);
213    return NewSmallFetcher(&proxy_resolver_);
214  }
215
216  virtual string BigUrl(in_port_t port) const { return kUnusedUrl; }
217  virtual string SmallUrl(in_port_t port) const { return kUnusedUrl; }
218  virtual string ErrorUrl(in_port_t port) const { return kUnusedUrl; }
219
220  virtual bool IsMock() const = 0;
221  virtual bool IsMulti() const = 0;
222
223  virtual void IgnoreServerAborting(HttpServer* server) const {}
224
225  virtual HttpServer* CreateServer() = 0;
226
227  FakeHardware* fake_hardware() {
228    return &fake_hardware_;
229  }
230
231 protected:
232  DirectProxyResolver proxy_resolver_;
233  FakeHardware fake_hardware_;
234};
235
236class MockHttpFetcherTest : public AnyHttpFetcherTest {
237 public:
238  // Necessary to unhide the definition in the base class.
239  using AnyHttpFetcherTest::NewLargeFetcher;
240  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
241    brillo::Blob big_data(1000000);
242    return new MockHttpFetcher(
243        big_data.data(), big_data.size(), proxy_resolver);
244  }
245
246  // Necessary to unhide the definition in the base class.
247  using AnyHttpFetcherTest::NewSmallFetcher;
248  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
249    return new MockHttpFetcher("x", 1, proxy_resolver);
250  }
251
252  bool IsMock() const override { return true; }
253  bool IsMulti() const override { return false; }
254
255  HttpServer* CreateServer() override {
256    return new NullHttpServer;
257  }
258};
259
260class LibcurlHttpFetcherTest : public AnyHttpFetcherTest {
261 public:
262  // Necessary to unhide the definition in the base class.
263  using AnyHttpFetcherTest::NewLargeFetcher;
264  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
265    LibcurlHttpFetcher* ret =
266        new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_);
267    // Speed up test execution.
268    ret->set_idle_seconds(1);
269    ret->set_retry_seconds(1);
270    fake_hardware_.SetIsOfficialBuild(false);
271    return ret;
272  }
273
274  // Necessary to unhide the definition in the base class.
275  using AnyHttpFetcherTest::NewSmallFetcher;
276  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
277    return NewLargeFetcher(proxy_resolver);
278  }
279
280  string BigUrl(in_port_t port) const override {
281    return LocalServerUrlForPath(port,
282                                 base::StringPrintf("/download/%d",
283                                                    kBigLength));
284  }
285  string SmallUrl(in_port_t port) const override {
286    return LocalServerUrlForPath(port, "/foo");
287  }
288  string ErrorUrl(in_port_t port) const override {
289    return LocalServerUrlForPath(port, "/error");
290  }
291
292  bool IsMock() const override { return false; }
293  bool IsMulti() const override { return false; }
294
295  void IgnoreServerAborting(HttpServer* server) const override {
296    // Nothing to do.
297  }
298
299  HttpServer* CreateServer() override {
300    return new PythonHttpServer;
301  }
302};
303
304class MultiRangeHttpFetcherTest : public LibcurlHttpFetcherTest {
305 public:
306  // Necessary to unhide the definition in the base class.
307  using AnyHttpFetcherTest::NewLargeFetcher;
308  HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
309    MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(
310        new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_));
311    ret->ClearRanges();
312    ret->AddRange(0);
313    // Speed up test execution.
314    ret->set_idle_seconds(1);
315    ret->set_retry_seconds(1);
316    fake_hardware_.SetIsOfficialBuild(false);
317    return ret;
318  }
319
320  // Necessary to unhide the definition in the base class.
321  using AnyHttpFetcherTest::NewSmallFetcher;
322  HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
323    return NewLargeFetcher(proxy_resolver);
324  }
325
326  bool IsMulti() const override { return true; }
327};
328
329
330//
331// Infrastructure for type tests of HTTP fetcher.
332// See: http://code.google.com/p/googletest/wiki/AdvancedGuide#Typed_Tests
333//
334
335// Fixture class template. We use an explicit constraint to guarantee that it
336// can only be instantiated with an AnyHttpFetcherTest type, see:
337// http://www2.research.att.com/~bs/bs_faq2.html#constraints
338template <typename T>
339class HttpFetcherTest : public ::testing::Test {
340 public:
341  base::MessageLoopForIO base_loop_;
342  brillo::BaseMessageLoop loop_{&base_loop_};
343
344  T test_;
345
346 protected:
347  HttpFetcherTest() {
348    loop_.SetAsCurrent();
349  }
350
351  void TearDown() override {
352    EXPECT_EQ(0, brillo::MessageLoopRunMaxIterations(&loop_, 1));
353  }
354
355 private:
356  static void TypeConstraint(T* a) {
357    AnyHttpFetcherTest *b = a;
358    if (b == 0)  // Silence compiler warning of unused variable.
359      *b = a;
360  }
361};
362
363// Test case types list.
364typedef ::testing::Types<LibcurlHttpFetcherTest,
365                         MockHttpFetcherTest,
366                         MultiRangeHttpFetcherTest> HttpFetcherTestTypes;
367TYPED_TEST_CASE(HttpFetcherTest, HttpFetcherTestTypes);
368
369
370namespace {
371class HttpFetcherTestDelegate : public HttpFetcherDelegate {
372 public:
373  HttpFetcherTestDelegate() = default;
374
375  void ReceivedBytes(HttpFetcher* /* fetcher */,
376                     const void* bytes,
377                     size_t length) override {
378    data.append(reinterpret_cast<const char*>(bytes), length);
379    // Update counters
380    times_received_bytes_called_++;
381  }
382
383  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
384    if (is_expect_error_)
385      EXPECT_EQ(kHttpResponseNotFound, fetcher->http_response_code());
386    else
387      EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
388    MessageLoop::current()->BreakLoop();
389
390    // Update counter
391    times_transfer_complete_called_++;
392  }
393
394  void TransferTerminated(HttpFetcher* fetcher) override {
395    ADD_FAILURE();
396    times_transfer_terminated_called_++;
397  }
398
399  // Are we expecting an error response? (default: no)
400  bool is_expect_error_{false};
401
402  // Counters for callback invocations.
403  int times_transfer_complete_called_{0};
404  int times_transfer_terminated_called_{0};
405  int times_received_bytes_called_{0};
406
407  // The received data bytes.
408  string data;
409};
410
411
412void StartTransfer(HttpFetcher* http_fetcher, const string& url) {
413  http_fetcher->BeginTransfer(url);
414}
415}  // namespace
416
417TYPED_TEST(HttpFetcherTest, SimpleTest) {
418  HttpFetcherTestDelegate delegate;
419  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
420  fetcher->set_delegate(&delegate);
421
422  unique_ptr<HttpServer> server(this->test_.CreateServer());
423  ASSERT_TRUE(server->started_);
424
425  this->loop_.PostTask(FROM_HERE, base::Bind(
426      StartTransfer,
427      fetcher.get(),
428      this->test_.SmallUrl(server->GetPort())));
429  this->loop_.Run();
430}
431
432TYPED_TEST(HttpFetcherTest, SimpleBigTest) {
433  HttpFetcherTestDelegate delegate;
434  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
435  fetcher->set_delegate(&delegate);
436
437  unique_ptr<HttpServer> server(this->test_.CreateServer());
438  ASSERT_TRUE(server->started_);
439
440  this->loop_.PostTask(FROM_HERE, base::Bind(
441      StartTransfer,
442      fetcher.get(),
443      this->test_.BigUrl(server->GetPort())));
444  this->loop_.Run();
445}
446
447// Issue #9648: when server returns an error HTTP response, the fetcher needs to
448// terminate transfer prematurely, rather than try to process the error payload.
449TYPED_TEST(HttpFetcherTest, ErrorTest) {
450  if (this->test_.IsMock() || this->test_.IsMulti())
451    return;
452  HttpFetcherTestDelegate delegate;
453
454  // Delegate should expect an error response.
455  delegate.is_expect_error_ = true;
456
457  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
458  fetcher->set_delegate(&delegate);
459
460  unique_ptr<HttpServer> server(this->test_.CreateServer());
461  ASSERT_TRUE(server->started_);
462
463  this->loop_.PostTask(FROM_HERE, base::Bind(
464      StartTransfer,
465      fetcher.get(),
466      this->test_.ErrorUrl(server->GetPort())));
467  this->loop_.Run();
468
469  // Make sure that no bytes were received.
470  CHECK_EQ(delegate.times_received_bytes_called_, 0);
471  CHECK_EQ(fetcher->GetBytesDownloaded(), static_cast<size_t>(0));
472
473  // Make sure that transfer completion was signaled once, and no termination
474  // was signaled.
475  CHECK_EQ(delegate.times_transfer_complete_called_, 1);
476  CHECK_EQ(delegate.times_transfer_terminated_called_, 0);
477}
478
479TYPED_TEST(HttpFetcherTest, ExtraHeadersInRequestTest) {
480  if (this->test_.IsMock())
481    return;
482
483  HttpFetcherTestDelegate delegate;
484  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
485  fetcher->set_delegate(&delegate);
486  fetcher->SetHeader("User-Agent", "MyTest");
487  fetcher->SetHeader("user-agent", "Override that header");
488  fetcher->SetHeader("Authorization", "Basic user:passwd");
489
490  // Invalid headers.
491  fetcher->SetHeader("X-Foo", "Invalid\nHeader\nIgnored");
492  fetcher->SetHeader("X-Bar: ", "I do not know how to parse");
493
494  // Hide Accept header normally added by default.
495  fetcher->SetHeader("Accept", "");
496
497  PythonHttpServer server;
498  int port = server.GetPort();
499  ASSERT_TRUE(server.started_);
500
501  StartTransfer(fetcher.get(), LocalServerUrlForPath(port, "/echo-headers"));
502  this->loop_.Run();
503
504  EXPECT_NE(string::npos,
505            delegate.data.find("user-agent: Override that header\r\n"));
506  EXPECT_NE(string::npos,
507            delegate.data.find("Authorization: Basic user:passwd\r\n"));
508
509  EXPECT_EQ(string::npos, delegate.data.find("\nAccept:"));
510  EXPECT_EQ(string::npos, delegate.data.find("X-Foo: Invalid"));
511  EXPECT_EQ(string::npos, delegate.data.find("X-Bar: I do not"));
512}
513
514namespace {
515class PausingHttpFetcherTestDelegate : public HttpFetcherDelegate {
516 public:
517  void ReceivedBytes(HttpFetcher* fetcher,
518                     const void* /* bytes */, size_t /* length */) override {
519    CHECK(!paused_);
520    paused_ = true;
521    fetcher->Pause();
522  }
523  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
524    MessageLoop::current()->BreakLoop();
525  }
526  void TransferTerminated(HttpFetcher* fetcher) override {
527    ADD_FAILURE();
528  }
529  void Unpause() {
530    CHECK(paused_);
531    paused_ = false;
532    fetcher_->Unpause();
533  }
534  bool paused_;
535  HttpFetcher* fetcher_;
536};
537
538void UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate* delegate,
539                              MessageLoop::TaskId* my_id) {
540  if (delegate->paused_)
541    delegate->Unpause();
542  // Update the task id with the new scheduled callback.
543  *my_id = MessageLoop::current()->PostDelayedTask(
544      FROM_HERE,
545      base::Bind(&UnpausingTimeoutCallback, delegate, my_id),
546      base::TimeDelta::FromMilliseconds(200));
547}
548}  // namespace
549
550TYPED_TEST(HttpFetcherTest, PauseTest) {
551  PausingHttpFetcherTestDelegate delegate;
552  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
553  delegate.paused_ = false;
554  delegate.fetcher_ = fetcher.get();
555  fetcher->set_delegate(&delegate);
556
557  unique_ptr<HttpServer> server(this->test_.CreateServer());
558  ASSERT_TRUE(server->started_);
559
560  MessageLoop::TaskId callback_id;
561  callback_id = this->loop_.PostDelayedTask(
562      FROM_HERE,
563      base::Bind(&UnpausingTimeoutCallback, &delegate, &callback_id),
564      base::TimeDelta::FromMilliseconds(200));
565  fetcher->BeginTransfer(this->test_.BigUrl(server->GetPort()));
566
567  this->loop_.Run();
568  EXPECT_TRUE(this->loop_.CancelTask(callback_id));
569}
570
571// This test will pause the fetcher while the download is not yet started
572// because it is waiting for the proxy to be resolved.
573TYPED_TEST(HttpFetcherTest, PauseWhileResolvingProxyTest) {
574  if (this->test_.IsMock())
575    return;
576  MockProxyResolver mock_resolver;
577  unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
578
579  // Saved arguments from the proxy call.
580  ProxiesResolvedFn proxy_callback = nullptr;
581  void* proxy_data = nullptr;
582
583  EXPECT_CALL(mock_resolver, GetProxiesForUrl("http://fake_url", _, _))
584      .WillOnce(DoAll(
585          SaveArg<1>(&proxy_callback), SaveArg<2>(&proxy_data), Return(true)));
586  fetcher->BeginTransfer("http://fake_url");
587  testing::Mock::VerifyAndClearExpectations(&mock_resolver);
588
589  // Pausing and unpausing while resolving the proxy should not affect anything.
590  fetcher->Pause();
591  fetcher->Unpause();
592  fetcher->Pause();
593  // Proxy resolver comes back after we paused the fetcher.
594  ASSERT_TRUE(proxy_callback);
595  (*proxy_callback)({1, kNoProxy}, proxy_data);
596}
597
598namespace {
599class AbortingHttpFetcherTestDelegate : public HttpFetcherDelegate {
600 public:
601  void ReceivedBytes(HttpFetcher* fetcher,
602                     const void* bytes, size_t length) override {}
603  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
604    ADD_FAILURE();  // We should never get here
605    MessageLoop::current()->BreakLoop();
606  }
607  void TransferTerminated(HttpFetcher* fetcher) override {
608    EXPECT_EQ(fetcher, fetcher_.get());
609    EXPECT_FALSE(once_);
610    EXPECT_TRUE(callback_once_);
611    callback_once_ = false;
612    // The fetcher could have a callback scheduled on the ProxyResolver that
613    // can fire after this callback. We wait until the end of the test to
614    // delete the fetcher.
615  }
616  void TerminateTransfer() {
617    CHECK(once_);
618    once_ = false;
619    fetcher_->TerminateTransfer();
620  }
621  void EndLoop() {
622    MessageLoop::current()->BreakLoop();
623  }
624  bool once_;
625  bool callback_once_;
626  unique_ptr<HttpFetcher> fetcher_;
627};
628
629void AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate* delegate,
630                             MessageLoop::TaskId* my_id) {
631  if (delegate->once_) {
632    delegate->TerminateTransfer();
633    *my_id = MessageLoop::current()->PostTask(
634        FROM_HERE,
635        base::Bind(AbortingTimeoutCallback, delegate, my_id));
636  } else {
637    delegate->EndLoop();
638    *my_id = MessageLoop::kTaskIdNull;
639  }
640}
641}  // namespace
642
643TYPED_TEST(HttpFetcherTest, AbortTest) {
644  AbortingHttpFetcherTestDelegate delegate;
645  delegate.fetcher_.reset(this->test_.NewLargeFetcher());
646  delegate.once_ = true;
647  delegate.callback_once_ = true;
648  delegate.fetcher_->set_delegate(&delegate);
649
650  unique_ptr<HttpServer> server(this->test_.CreateServer());
651  this->test_.IgnoreServerAborting(server.get());
652  ASSERT_TRUE(server->started_);
653
654  MessageLoop::TaskId task_id = MessageLoop::kTaskIdNull;
655
656  task_id = this->loop_.PostTask(
657      FROM_HERE,
658      base::Bind(AbortingTimeoutCallback, &delegate, &task_id));
659  delegate.fetcher_->BeginTransfer(this->test_.BigUrl(server->GetPort()));
660
661  this->loop_.Run();
662  CHECK(!delegate.once_);
663  CHECK(!delegate.callback_once_);
664  this->loop_.CancelTask(task_id);
665}
666
667namespace {
668class FlakyHttpFetcherTestDelegate : public HttpFetcherDelegate {
669 public:
670  void ReceivedBytes(HttpFetcher* fetcher,
671                     const void* bytes, size_t length) override {
672    data.append(reinterpret_cast<const char*>(bytes), length);
673  }
674  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
675    EXPECT_TRUE(successful);
676    EXPECT_EQ(kHttpResponsePartialContent, fetcher->http_response_code());
677    MessageLoop::current()->BreakLoop();
678  }
679  void TransferTerminated(HttpFetcher* fetcher) override {
680    ADD_FAILURE();
681  }
682  string data;
683};
684}  // namespace
685
686TYPED_TEST(HttpFetcherTest, FlakyTest) {
687  if (this->test_.IsMock())
688    return;
689  {
690    FlakyHttpFetcherTestDelegate delegate;
691    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
692    fetcher->set_delegate(&delegate);
693
694    unique_ptr<HttpServer> server(this->test_.CreateServer());
695    ASSERT_TRUE(server->started_);
696
697    this->loop_.PostTask(FROM_HERE, base::Bind(
698        &StartTransfer,
699        fetcher.get(),
700        LocalServerUrlForPath(server->GetPort(),
701                              base::StringPrintf("/flaky/%d/%d/%d/%d",
702                                                 kBigLength,
703                                                 kFlakyTruncateLength,
704                                                 kFlakySleepEvery,
705                                                 kFlakySleepSecs))));
706    this->loop_.Run();
707
708    // verify the data we get back
709    ASSERT_EQ(kBigLength, static_cast<int>(delegate.data.size()));
710    for (int i = 0; i < kBigLength; i += 10) {
711      // Assert so that we don't flood the screen w/ EXPECT errors on failure.
712      ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
713    }
714  }
715}
716
717namespace {
718// This delegate kills the server attached to it after receiving any bytes.
719// This can be used for testing what happens when you try to fetch data and
720// the server dies.
721class FailureHttpFetcherTestDelegate : public HttpFetcherDelegate {
722 public:
723  explicit FailureHttpFetcherTestDelegate(PythonHttpServer* server)
724      : server_(server) {}
725
726  ~FailureHttpFetcherTestDelegate() override {
727    if (server_) {
728      LOG(INFO) << "Stopping server in destructor";
729      delete server_;
730      LOG(INFO) << "server stopped";
731    }
732  }
733
734  void ReceivedBytes(HttpFetcher* fetcher,
735                     const void* bytes, size_t length) override {
736    if (server_) {
737      LOG(INFO) << "Stopping server in ReceivedBytes";
738      delete server_;
739      LOG(INFO) << "server stopped";
740      server_ = nullptr;
741    }
742  }
743  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
744    EXPECT_FALSE(successful);
745    EXPECT_EQ(0, fetcher->http_response_code());
746    MessageLoop::current()->BreakLoop();
747  }
748  void TransferTerminated(HttpFetcher* fetcher) override {
749    ADD_FAILURE();
750  }
751  PythonHttpServer* server_;
752};
753}  // namespace
754
755
756TYPED_TEST(HttpFetcherTest, FailureTest) {
757  // This test ensures that a fetcher responds correctly when a server isn't
758  // available at all.
759  if (this->test_.IsMock())
760    return;
761  {
762    FailureHttpFetcherTestDelegate delegate(nullptr);
763    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
764    fetcher->set_delegate(&delegate);
765
766    this->loop_.PostTask(FROM_HERE,
767                         base::Bind(StartTransfer,
768                                    fetcher.get(),
769                                    "http://host_doesnt_exist99999999"));
770    this->loop_.Run();
771
772    // Exiting and testing happens in the delegate
773  }
774}
775
776TYPED_TEST(HttpFetcherTest, NoResponseTest) {
777  // This test starts a new http server but the server doesn't respond and just
778  // closes the connection.
779  if (this->test_.IsMock())
780    return;
781
782  PythonHttpServer* server = new PythonHttpServer();
783  int port = server->GetPort();
784  ASSERT_TRUE(server->started_);
785
786  // Handles destruction and claims ownership.
787  FailureHttpFetcherTestDelegate delegate(server);
788  unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
789  fetcher->set_delegate(&delegate);
790  // The server will not reply at all, so we can limit the execution time of the
791  // test by reducing the low-speed timeout to something small. The test will
792  // finish once the TimeoutCallback() triggers (every second) and the timeout
793  // expired.
794  fetcher->set_low_speed_limit(kDownloadLowSpeedLimitBps, 1);
795
796  this->loop_.PostTask(FROM_HERE, base::Bind(
797      StartTransfer,
798      fetcher.get(),
799      LocalServerUrlForPath(port, "/hang")));
800  this->loop_.Run();
801
802  // Check that no other callback runs in the next two seconds. That would
803  // indicate a leaked callback.
804  bool timeout = false;
805  auto callback = base::Bind([&timeout]{ timeout = true;});
806  this->loop_.PostDelayedTask(FROM_HERE, callback,
807                              base::TimeDelta::FromSeconds(2));
808  EXPECT_TRUE(this->loop_.RunOnce(true));
809  EXPECT_TRUE(timeout);
810}
811
812TYPED_TEST(HttpFetcherTest, ServerDiesTest) {
813  // This test starts a new http server and kills it after receiving its first
814  // set of bytes. It test whether or not our fetcher eventually gives up on
815  // retries and aborts correctly.
816  if (this->test_.IsMock())
817    return;
818  {
819    PythonHttpServer* server = new PythonHttpServer();
820    int port = server->GetPort();
821    ASSERT_TRUE(server->started_);
822
823    // Handles destruction and claims ownership.
824    FailureHttpFetcherTestDelegate delegate(server);
825    unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
826    fetcher->set_delegate(&delegate);
827
828    this->loop_.PostTask(FROM_HERE, base::Bind(
829        StartTransfer,
830        fetcher.get(),
831        LocalServerUrlForPath(port,
832                              base::StringPrintf("/flaky/%d/%d/%d/%d",
833                                                 kBigLength,
834                                                 kFlakyTruncateLength,
835                                                 kFlakySleepEvery,
836                                                 kFlakySleepSecs))));
837    this->loop_.Run();
838
839    // Exiting and testing happens in the delegate
840  }
841}
842
843namespace {
844const HttpResponseCode kRedirectCodes[] = {
845  kHttpResponseMovedPermanently, kHttpResponseFound, kHttpResponseSeeOther,
846  kHttpResponseTempRedirect
847};
848
849class RedirectHttpFetcherTestDelegate : public HttpFetcherDelegate {
850 public:
851  explicit RedirectHttpFetcherTestDelegate(bool expected_successful)
852      : expected_successful_(expected_successful) {}
853  void ReceivedBytes(HttpFetcher* fetcher,
854                     const void* bytes, size_t length) override {
855    data.append(reinterpret_cast<const char*>(bytes), length);
856  }
857  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
858    EXPECT_EQ(expected_successful_, successful);
859    if (expected_successful_) {
860      EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
861    } else {
862      EXPECT_GE(fetcher->http_response_code(), kHttpResponseMovedPermanently);
863      EXPECT_LE(fetcher->http_response_code(), kHttpResponseTempRedirect);
864    }
865    MessageLoop::current()->BreakLoop();
866  }
867  void TransferTerminated(HttpFetcher* fetcher) override {
868    ADD_FAILURE();
869  }
870  bool expected_successful_;
871  string data;
872};
873
874// RedirectTest takes ownership of |http_fetcher|.
875void RedirectTest(const HttpServer* server,
876                  bool expected_successful,
877                  const string& url,
878                  HttpFetcher* http_fetcher) {
879  RedirectHttpFetcherTestDelegate delegate(expected_successful);
880  unique_ptr<HttpFetcher> fetcher(http_fetcher);
881  fetcher->set_delegate(&delegate);
882
883  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
884      StartTransfer,
885      fetcher.get(),
886      LocalServerUrlForPath(server->GetPort(), url)));
887  MessageLoop::current()->Run();
888  if (expected_successful) {
889    // verify the data we get back
890    ASSERT_EQ(static_cast<size_t>(kMediumLength), delegate.data.size());
891    for (int i = 0; i < kMediumLength; i += 10) {
892      // Assert so that we don't flood the screen w/ EXPECT errors on failure.
893      ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
894    }
895  }
896}
897}  // namespace
898
899TYPED_TEST(HttpFetcherTest, SimpleRedirectTest) {
900  if (this->test_.IsMock())
901    return;
902
903  unique_ptr<HttpServer> server(this->test_.CreateServer());
904  ASSERT_TRUE(server->started_);
905
906  for (size_t c = 0; c < arraysize(kRedirectCodes); ++c) {
907    const string url = base::StringPrintf("/redirect/%d/download/%d",
908                                          kRedirectCodes[c],
909                                          kMediumLength);
910    RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
911  }
912}
913
914TYPED_TEST(HttpFetcherTest, MaxRedirectTest) {
915  if (this->test_.IsMock())
916    return;
917
918  unique_ptr<HttpServer> server(this->test_.CreateServer());
919  ASSERT_TRUE(server->started_);
920
921  string url;
922  for (int r = 0; r < kDownloadMaxRedirects; r++) {
923    url += base::StringPrintf("/redirect/%d",
924                              kRedirectCodes[r % arraysize(kRedirectCodes)]);
925  }
926  url += base::StringPrintf("/download/%d", kMediumLength);
927  RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
928}
929
930TYPED_TEST(HttpFetcherTest, BeyondMaxRedirectTest) {
931  if (this->test_.IsMock())
932    return;
933
934  unique_ptr<HttpServer> server(this->test_.CreateServer());
935  ASSERT_TRUE(server->started_);
936
937  string url;
938  for (int r = 0; r < kDownloadMaxRedirects + 1; r++) {
939    url += base::StringPrintf("/redirect/%d",
940                              kRedirectCodes[r % arraysize(kRedirectCodes)]);
941  }
942  url += base::StringPrintf("/download/%d", kMediumLength);
943  RedirectTest(server.get(), false, url, this->test_.NewLargeFetcher());
944}
945
946namespace {
947class MultiHttpFetcherTestDelegate : public HttpFetcherDelegate {
948 public:
949  explicit MultiHttpFetcherTestDelegate(int expected_response_code)
950      : expected_response_code_(expected_response_code) {}
951
952  void ReceivedBytes(HttpFetcher* fetcher,
953                     const void* bytes, size_t length) override {
954    EXPECT_EQ(fetcher, fetcher_.get());
955    data.append(reinterpret_cast<const char*>(bytes), length);
956  }
957
958  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
959    EXPECT_EQ(fetcher, fetcher_.get());
960    EXPECT_EQ(expected_response_code_ != kHttpResponseUndefined, successful);
961    if (expected_response_code_ != 0)
962      EXPECT_EQ(expected_response_code_, fetcher->http_response_code());
963    // Destroy the fetcher (because we're allowed to).
964    fetcher_.reset(nullptr);
965    MessageLoop::current()->BreakLoop();
966  }
967
968  void TransferTerminated(HttpFetcher* fetcher) override {
969    ADD_FAILURE();
970  }
971
972  unique_ptr<HttpFetcher> fetcher_;
973  int expected_response_code_;
974  string data;
975};
976
977void MultiTest(HttpFetcher* fetcher_in,
978               FakeHardware* fake_hardware,
979               const string& url,
980               const vector<pair<off_t, off_t>>& ranges,
981               const string& expected_prefix,
982               size_t expected_size,
983               HttpResponseCode expected_response_code) {
984  MultiHttpFetcherTestDelegate delegate(expected_response_code);
985  delegate.fetcher_.reset(fetcher_in);
986
987  MultiRangeHttpFetcher* multi_fetcher =
988      static_cast<MultiRangeHttpFetcher*>(fetcher_in);
989  ASSERT_TRUE(multi_fetcher);
990  multi_fetcher->ClearRanges();
991  for (vector<pair<off_t, off_t>>::const_iterator it = ranges.begin(),
992           e = ranges.end(); it != e; ++it) {
993    string tmp_str = base::StringPrintf("%jd+", it->first);
994    if (it->second > 0) {
995      base::StringAppendF(&tmp_str, "%jd", it->second);
996      multi_fetcher->AddRange(it->first, it->second);
997    } else {
998      base::StringAppendF(&tmp_str, "?");
999      multi_fetcher->AddRange(it->first);
1000    }
1001    LOG(INFO) << "added range: " << tmp_str;
1002  }
1003  fake_hardware->SetIsOfficialBuild(false);
1004  multi_fetcher->set_delegate(&delegate);
1005
1006  MessageLoop::current()->PostTask(
1007      FROM_HERE,
1008      base::Bind(StartTransfer, multi_fetcher, url));
1009  MessageLoop::current()->Run();
1010
1011  EXPECT_EQ(expected_size, delegate.data.size());
1012  EXPECT_EQ(expected_prefix,
1013            string(delegate.data.data(), expected_prefix.size()));
1014}
1015}  // namespace
1016
1017TYPED_TEST(HttpFetcherTest, MultiHttpFetcherSimpleTest) {
1018  if (!this->test_.IsMulti())
1019    return;
1020
1021  unique_ptr<HttpServer> server(this->test_.CreateServer());
1022  ASSERT_TRUE(server->started_);
1023
1024  vector<pair<off_t, off_t>> ranges;
1025  ranges.push_back(make_pair(0, 25));
1026  ranges.push_back(make_pair(99, 0));
1027  MultiTest(this->test_.NewLargeFetcher(),
1028            this->test_.fake_hardware(),
1029            this->test_.BigUrl(server->GetPort()),
1030            ranges,
1031            "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1032            kBigLength - (99 - 25),
1033            kHttpResponsePartialContent);
1034}
1035
1036TYPED_TEST(HttpFetcherTest, MultiHttpFetcherLengthLimitTest) {
1037  if (!this->test_.IsMulti())
1038    return;
1039
1040  unique_ptr<HttpServer> server(this->test_.CreateServer());
1041  ASSERT_TRUE(server->started_);
1042
1043  vector<pair<off_t, off_t>> ranges;
1044  ranges.push_back(make_pair(0, 24));
1045  MultiTest(this->test_.NewLargeFetcher(),
1046            this->test_.fake_hardware(),
1047            this->test_.BigUrl(server->GetPort()),
1048            ranges,
1049            "abcdefghijabcdefghijabcd",
1050            24,
1051            kHttpResponsePartialContent);
1052}
1053
1054TYPED_TEST(HttpFetcherTest, MultiHttpFetcherMultiEndTest) {
1055  if (!this->test_.IsMulti())
1056    return;
1057
1058  unique_ptr<HttpServer> server(this->test_.CreateServer());
1059  ASSERT_TRUE(server->started_);
1060
1061  vector<pair<off_t, off_t>> ranges;
1062  ranges.push_back(make_pair(kBigLength - 2, 0));
1063  ranges.push_back(make_pair(kBigLength - 3, 0));
1064  MultiTest(this->test_.NewLargeFetcher(),
1065            this->test_.fake_hardware(),
1066            this->test_.BigUrl(server->GetPort()),
1067            ranges,
1068            "ijhij",
1069            5,
1070            kHttpResponsePartialContent);
1071}
1072
1073TYPED_TEST(HttpFetcherTest, MultiHttpFetcherInsufficientTest) {
1074  if (!this->test_.IsMulti())
1075    return;
1076
1077  unique_ptr<HttpServer> server(this->test_.CreateServer());
1078  ASSERT_TRUE(server->started_);
1079
1080  vector<pair<off_t, off_t>> ranges;
1081  ranges.push_back(make_pair(kBigLength - 2, 4));
1082  for (int i = 0; i < 2; ++i) {
1083    LOG(INFO) << "i = " << i;
1084    MultiTest(this->test_.NewLargeFetcher(),
1085              this->test_.fake_hardware(),
1086              this->test_.BigUrl(server->GetPort()),
1087              ranges,
1088              "ij",
1089              2,
1090              kHttpResponseUndefined);
1091    ranges.push_back(make_pair(0, 5));
1092  }
1093}
1094
1095// Issue #18143: when a fetch of a secondary chunk out of a chain, then it
1096// should retry with other proxies listed before giving up.
1097//
1098// (1) successful recovery: The offset fetch will fail twice but succeed with
1099// the third proxy.
1100TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetRecoverableTest) {
1101  if (!this->test_.IsMulti())
1102    return;
1103
1104  unique_ptr<HttpServer> server(this->test_.CreateServer());
1105  ASSERT_TRUE(server->started_);
1106
1107  vector<pair<off_t, off_t>> ranges;
1108  ranges.push_back(make_pair(0, 25));
1109  ranges.push_back(make_pair(99, 0));
1110  MultiTest(this->test_.NewLargeFetcher(3),
1111            this->test_.fake_hardware(),
1112            LocalServerUrlForPath(server->GetPort(),
1113                                  base::StringPrintf("/error-if-offset/%d/2",
1114                                                     kBigLength)),
1115            ranges,
1116            "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1117            kBigLength - (99 - 25),
1118            kHttpResponsePartialContent);
1119}
1120
1121// (2) unsuccessful recovery: The offset fetch will fail repeatedly.  The
1122// fetcher will signal a (failed) completed transfer to the delegate.
1123TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetUnrecoverableTest) {
1124  if (!this->test_.IsMulti())
1125    return;
1126
1127  unique_ptr<HttpServer> server(this->test_.CreateServer());
1128  ASSERT_TRUE(server->started_);
1129
1130  vector<pair<off_t, off_t>> ranges;
1131  ranges.push_back(make_pair(0, 25));
1132  ranges.push_back(make_pair(99, 0));
1133  MultiTest(this->test_.NewLargeFetcher(2),
1134            this->test_.fake_hardware(),
1135            LocalServerUrlForPath(server->GetPort(),
1136                                  base::StringPrintf("/error-if-offset/%d/3",
1137                                                     kBigLength)),
1138            ranges,
1139            "abcdefghijabcdefghijabcde",  // only received the first chunk
1140            25,
1141            kHttpResponseUndefined);
1142}
1143
1144
1145
1146namespace {
1147class BlockedTransferTestDelegate : public HttpFetcherDelegate {
1148 public:
1149  void ReceivedBytes(HttpFetcher* fetcher,
1150                     const void* bytes, size_t length) override {
1151    ADD_FAILURE();
1152  }
1153  void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1154    EXPECT_FALSE(successful);
1155    MessageLoop::current()->BreakLoop();
1156  }
1157  void TransferTerminated(HttpFetcher* fetcher) override {
1158    ADD_FAILURE();
1159  }
1160};
1161
1162void BlockedTransferTestHelper(AnyHttpFetcherTest* fetcher_test,
1163                               bool is_official_build) {
1164  if (fetcher_test->IsMock() || fetcher_test->IsMulti())
1165    return;
1166
1167  unique_ptr<HttpServer> server(fetcher_test->CreateServer());
1168  ASSERT_TRUE(server->started_);
1169
1170  BlockedTransferTestDelegate delegate;
1171  unique_ptr<HttpFetcher> fetcher(fetcher_test->NewLargeFetcher());
1172  LOG(INFO) << "is_official_build: " << is_official_build;
1173  // NewLargeFetcher creates the HttpFetcher* with a FakeSystemState.
1174  fetcher_test->fake_hardware()->SetIsOfficialBuild(is_official_build);
1175  fetcher->set_delegate(&delegate);
1176
1177  MessageLoop::current()->PostTask(FROM_HERE, base::Bind(
1178      StartTransfer,
1179      fetcher.get(),
1180      LocalServerUrlForPath(server->GetPort(),
1181                            fetcher_test->SmallUrl(server->GetPort()))));
1182  MessageLoop::current()->Run();
1183}
1184}  // namespace
1185
1186TYPED_TEST(HttpFetcherTest, BlockedTransferTest) {
1187  BlockedTransferTestHelper(&this->test_, false);
1188}
1189
1190TYPED_TEST(HttpFetcherTest, BlockedTransferOfficialBuildTest) {
1191  BlockedTransferTestHelper(&this->test_, true);
1192}
1193
1194}  // namespace chromeos_update_engine
1195