1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <time.h>
6
7#include <algorithm>
8#include <sstream>
9#include <string>
10
11#include "base/memory/scoped_ptr.h"
12#include "base/message_loop/message_loop.h"
13#include "base/strings/string_number_conversions.h"
14#include "base/timer/timer.h"
15#include "base/values.h"
16#include "chrome/browser/net/predictor.h"
17#include "chrome/browser/net/spdyproxy/proxy_advisor.h"
18#include "chrome/browser/net/url_info.h"
19#include "chrome/common/net/predictor_common.h"
20#include "content/public/test/test_browser_thread.h"
21#include "net/base/address_list.h"
22#include "net/base/winsock_init.h"
23#include "net/dns/mock_host_resolver.h"
24#include "net/http/transport_security_state.h"
25#include "testing/gmock/include/gmock/gmock.h"
26#include "testing/gtest/include/gtest/gtest.h"
27
28using base::Time;
29using base::TimeDelta;
30using content::BrowserThread;
31
32namespace chrome_browser_net {
33
34class WaitForResolutionHelper;
35
36typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
37
38class WaitForResolutionHelper {
39 public:
40  WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
41                          HelperTimer* timer, int checks_until_quit)
42      : predictor_(predictor),
43        hosts_(hosts),
44        timer_(timer),
45        checks_until_quit_(checks_until_quit) {
46  }
47
48  void CheckIfResolutionsDone() {
49    if (--checks_until_quit_ > 0) {
50      for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
51        if (predictor_->GetResolutionDuration(*i) ==
52            UrlInfo::NullDuration())
53          return;  // We don't have resolution for that host.
54    }
55
56    // When all hostnames have been resolved, or we've hit the limit,
57    // exit the loop.
58    timer_->Stop();
59    base::MessageLoop::current()->Quit();
60    delete timer_;
61    delete this;
62  }
63
64 private:
65  Predictor* predictor_;
66  const UrlList hosts_;
67  HelperTimer* timer_;
68  int checks_until_quit_;
69};
70
71class PredictorTest : public testing::Test {
72 public:
73  PredictorTest()
74      : ui_thread_(BrowserThread::UI, &loop_),
75        io_thread_(BrowserThread::IO, &loop_),
76        host_resolver_(new net::MockCachingHostResolver()) {
77  }
78
79 protected:
80  virtual void SetUp() {
81#if defined(OS_WIN)
82    net::EnsureWinsockInit();
83#endif
84    Predictor::set_max_parallel_resolves(
85        Predictor::kMaxSpeculativeParallelResolves);
86    Predictor::set_max_queueing_delay(
87        Predictor::kMaxSpeculativeResolveQueueDelayMs);
88    // Since we are using a caching HostResolver, the following latencies will
89    // only be incurred by the first request, after which the result will be
90    // cached internally by |host_resolver_|.
91    net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
92    rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
93    rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
94    rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
95    rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
96  }
97
98  void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
99    HelperTimer* timer = new HelperTimer();
100    // By default allow the loop to run for a minute -- 600 iterations.
101    timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
102                 new WaitForResolutionHelper(predictor, hosts, timer, 600),
103                 &WaitForResolutionHelper::CheckIfResolutionsDone);
104    base::MessageLoop::current()->Run();
105  }
106
107  void WaitForResolutionWithLimit(
108      Predictor* predictor, const UrlList& hosts, int limit) {
109    HelperTimer* timer = new HelperTimer();
110    timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
111                 new WaitForResolutionHelper(predictor, hosts, timer, limit),
112                 &WaitForResolutionHelper::CheckIfResolutionsDone);
113    base::MessageLoop::current()->Run();
114  }
115
116 private:
117  // IMPORTANT: do not move this below |host_resolver_|; the host resolver
118  // must not outlive the message loop, otherwise bad things can happen
119  // (like posting to a deleted message loop).
120  base::MessageLoopForUI loop_;
121  content::TestBrowserThread ui_thread_;
122  content::TestBrowserThread io_thread_;
123
124 protected:
125  scoped_ptr<net::MockCachingHostResolver> host_resolver_;
126};
127
128//------------------------------------------------------------------------------
129
130TEST_F(PredictorTest, StartupShutdownTest) {
131  Predictor testing_master(true, true);
132  testing_master.Shutdown();
133}
134
135
136TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
137  scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
138
139  Predictor testing_master(true, true);
140  testing_master.SetHostResolver(host_resolver.get());
141
142  GURL localhost("http://localhost:80");
143  UrlList names;
144  names.push_back(localhost);
145
146  testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
147
148  base::MessageLoop::current()->PostDelayedTask(
149      FROM_HERE,
150      base::MessageLoop::QuitClosure(),
151      base::TimeDelta::FromMilliseconds(500));
152  base::MessageLoop::current()->Run();
153
154  EXPECT_FALSE(testing_master.WasFound(localhost));
155
156  testing_master.Shutdown();
157
158  // Clean up after ourselves.
159  base::MessageLoop::current()->RunUntilIdle();
160}
161
162TEST_F(PredictorTest, SingleLookupTest) {
163  Predictor testing_master(true, true);
164  testing_master.SetHostResolver(host_resolver_.get());
165
166  GURL goog("http://www.google.com:80");
167
168  UrlList names;
169  names.push_back(goog);
170
171  // Try to flood the predictor with many concurrent requests.
172  for (int i = 0; i < 10; i++)
173    testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
174
175  WaitForResolution(&testing_master, names);
176
177  EXPECT_TRUE(testing_master.WasFound(goog));
178
179  base::MessageLoop::current()->RunUntilIdle();
180
181  EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
182  EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
183  EXPECT_LE(testing_master.peak_pending_lookups(),
184            testing_master.max_concurrent_dns_lookups());
185
186  testing_master.Shutdown();
187}
188
189TEST_F(PredictorTest, ConcurrentLookupTest) {
190  host_resolver_->rules()->AddSimulatedFailure("*.notfound");
191
192  Predictor testing_master(true, true);
193  testing_master.SetHostResolver(host_resolver_.get());
194
195  GURL goog("http://www.google.com:80"),
196      goog2("http://gmail.google.com.com:80"),
197      goog3("http://mail.google.com:80"),
198      goog4("http://gmail.com:80");
199  GURL bad1("http://bad1.notfound:80"),
200      bad2("http://bad2.notfound:80");
201
202  UrlList names;
203  names.push_back(goog);
204  names.push_back(goog3);
205  names.push_back(bad1);
206  names.push_back(goog2);
207  names.push_back(bad2);
208  names.push_back(goog4);
209  names.push_back(goog);
210
211  // Try to flood the predictor with many concurrent requests.
212  for (int i = 0; i < 10; i++)
213    testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
214
215  WaitForResolution(&testing_master, names);
216
217  EXPECT_TRUE(testing_master.WasFound(goog));
218  EXPECT_TRUE(testing_master.WasFound(goog3));
219  EXPECT_TRUE(testing_master.WasFound(goog2));
220  EXPECT_TRUE(testing_master.WasFound(goog4));
221  EXPECT_FALSE(testing_master.WasFound(bad1));
222  EXPECT_FALSE(testing_master.WasFound(bad2));
223
224  base::MessageLoop::current()->RunUntilIdle();
225
226  EXPECT_FALSE(testing_master.WasFound(bad1));
227  EXPECT_FALSE(testing_master.WasFound(bad2));
228
229  EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
230  EXPECT_LE(testing_master.peak_pending_lookups(),
231            testing_master.max_concurrent_dns_lookups());
232
233  testing_master.Shutdown();
234}
235
236TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
237  host_resolver_->rules()->AddSimulatedFailure("*.notfound");
238
239  Predictor testing_master(true, true);
240  testing_master.SetHostResolver(host_resolver_.get());
241
242  UrlList names;
243  for (int i = 0; i < 100; i++)
244    names.push_back(GURL(
245        "http://host" + base::IntToString(i) + ".notfound:80"));
246
247  // Try to flood the predictor with many concurrent requests.
248  for (int i = 0; i < 10; i++)
249    testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
250
251  WaitForResolution(&testing_master, names);
252
253  base::MessageLoop::current()->RunUntilIdle();
254
255  EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
256  EXPECT_LE(testing_master.peak_pending_lookups(),
257            testing_master.max_concurrent_dns_lookups());
258
259  testing_master.Shutdown();
260}
261
262//------------------------------------------------------------------------------
263// Functions to help synthesize and test serializations of subresource referrer
264// lists.
265
266// Return a motivation_list if we can find one for the given motivating_host (or
267// NULL if a match is not found).
268static const base::ListValue* FindSerializationMotivation(
269    const GURL& motivation,
270    const base::ListValue* referral_list) {
271  CHECK_LT(0u, referral_list->GetSize());  // Room for version.
272  int format_version = -1;
273  CHECK(referral_list->GetInteger(0, &format_version));
274  CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
275  const base::ListValue* motivation_list(NULL);
276  for (size_t i = 1; i < referral_list->GetSize(); ++i) {
277    referral_list->GetList(i, &motivation_list);
278    std::string existing_spec;
279    EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
280    if (motivation == GURL(existing_spec))
281      return motivation_list;
282  }
283  return NULL;
284}
285
286static base::ListValue* FindSerializationMotivation(
287    const GURL& motivation,
288    base::ListValue* referral_list) {
289  return const_cast<base::ListValue*>(FindSerializationMotivation(
290      motivation, static_cast<const base::ListValue*>(referral_list)));
291}
292
293// Create a new empty serialization list.
294static base::ListValue* NewEmptySerializationList() {
295  base::ListValue* list = new base::ListValue;
296  list->Append(
297      new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
298  return list;
299}
300
301// Add a motivating_url and a subresource_url to a serialized list, using
302// this given latency. This is a helper function for quickly building these
303// lists.
304static void AddToSerializedList(const GURL& motivation,
305                                const GURL& subresource,
306                                double use_rate,
307                                base::ListValue* referral_list) {
308  // Find the motivation if it is already used.
309  base::ListValue* motivation_list = FindSerializationMotivation(motivation,
310                                                           referral_list);
311  if (!motivation_list) {
312    // This is the first mention of this motivation, so build a list.
313    motivation_list = new base::ListValue;
314    motivation_list->Append(new base::StringValue(motivation.spec()));
315    // Provide empty subresource list.
316    motivation_list->Append(new base::ListValue());
317
318    // ...and make it part of the serialized referral_list.
319    referral_list->Append(motivation_list);
320  }
321
322  base::ListValue* subresource_list(NULL);
323  // 0 == url; 1 == subresource_list.
324  EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
325
326  // We won't bother to check for the subresource being there already.  Worst
327  // case, during deserialization, the latency value we supply plus the
328  // existing value(s) will be added to the referrer.
329
330  subresource_list->Append(new base::StringValue(subresource.spec()));
331  subresource_list->Append(new base::FundamentalValue(use_rate));
332}
333
334// For a given motivation, and subresource, find what latency is currently
335// listed.  This assume a well formed serialization, which has at most one such
336// entry for any pair of names.  If no such pair is found, then return false.
337// Data is written into use_rate arguments.
338static bool GetDataFromSerialization(const GURL& motivation,
339                                     const GURL& subresource,
340                                     const base::ListValue& referral_list,
341                                     double* use_rate) {
342  const base::ListValue* motivation_list =
343      FindSerializationMotivation(motivation, &referral_list);
344  if (!motivation_list)
345    return false;
346  const base::ListValue* subresource_list;
347  EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
348  for (size_t i = 0; i < subresource_list->GetSize();) {
349    std::string url_spec;
350    EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
351    EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
352    if (subresource == GURL(url_spec)) {
353      return true;
354    }
355  }
356  return false;
357}
358
359//------------------------------------------------------------------------------
360
361// Make sure nil referral lists really have no entries, and no latency listed.
362TEST_F(PredictorTest, ReferrerSerializationNilTest) {
363  Predictor predictor(true, true);
364  predictor.SetHostResolver(host_resolver_.get());
365
366  scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
367  predictor.SerializeReferrers(referral_list.get());
368  EXPECT_EQ(1U, referral_list->GetSize());
369  EXPECT_FALSE(GetDataFromSerialization(
370    GURL("http://a.com:79"), GURL("http://b.com:78"),
371      *referral_list.get(), NULL));
372
373  predictor.Shutdown();
374}
375
376// Make sure that when a serialization list includes a value, that it can be
377// deserialized into the database, and can be extracted back out via
378// serialization without being changed.
379TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
380  Predictor predictor(true, true);
381  predictor.SetHostResolver(host_resolver_.get());
382  const GURL motivation_url("http://www.google.com:91");
383  const GURL subresource_url("http://icons.google.com:90");
384  const double kUseRate = 23.4;
385  scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
386
387  AddToSerializedList(motivation_url, subresource_url,
388      kUseRate, referral_list.get());
389
390  predictor.DeserializeReferrers(*referral_list.get());
391
392  base::ListValue recovered_referral_list;
393  predictor.SerializeReferrers(&recovered_referral_list);
394  EXPECT_EQ(2U, recovered_referral_list.GetSize());
395  double rate;
396  EXPECT_TRUE(GetDataFromSerialization(
397      motivation_url, subresource_url, recovered_referral_list, &rate));
398  EXPECT_EQ(rate, kUseRate);
399
400  predictor.Shutdown();
401}
402
403// Check that GetHtmlReferrerLists() doesn't crash when given duplicated
404// domains for referring URL, and that it sorts the results in the
405// correct order.
406TEST_F(PredictorTest, GetHtmlReferrerLists) {
407  Predictor predictor(true, true);
408  predictor.SetHostResolver(host_resolver_.get());
409  const double kUseRate = 23.4;
410  scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
411
412  AddToSerializedList(
413      GURL("http://d.google.com/x1"),
414      GURL("http://foo.com/"),
415      kUseRate, referral_list.get());
416
417  // Duplicated hostname (d.google.com). This should not cause any crashes
418  // (i.e. crbug.com/116345)
419  AddToSerializedList(
420      GURL("http://d.google.com/x2"),
421      GURL("http://foo.com/"),
422      kUseRate, referral_list.get());
423
424  AddToSerializedList(
425      GURL("http://a.yahoo.com/y"),
426      GURL("http://foo1.com/"),
427      kUseRate, referral_list.get());
428
429  AddToSerializedList(
430      GURL("http://b.google.com/x3"),
431      GURL("http://foo2.com/"),
432      kUseRate, referral_list.get());
433
434  AddToSerializedList(
435      GURL("http://d.yahoo.com/x5"),
436      GURL("http://i.like.turtles/"),
437      kUseRate, referral_list.get());
438
439  AddToSerializedList(
440      GURL("http://c.yahoo.com/x4"),
441      GURL("http://foo3.com/"),
442      kUseRate, referral_list.get());
443
444  predictor.DeserializeReferrers(*referral_list.get());
445
446  std::string html;
447  predictor.GetHtmlReferrerLists(&html);
448
449  // The lexicographic sorting of hostnames would be:
450  //   a.yahoo.com
451  //   b.google.com
452  //   c.yahoo.com
453  //   d.google.com
454  //   d.yahoo.com
455  //
456  // However we expect to sort them by domain in the output:
457  //   b.google.com
458  //   d.google.com
459  //   a.yahoo.com
460  //   c.yahoo.com
461  //   d.yahoo.com
462
463  size_t pos[] = {
464      html.find("<td rowspan=1>http://b.google.com/x3"),
465      html.find("<td rowspan=1>http://d.google.com/x1"),
466      html.find("<td rowspan=1>http://d.google.com/x2"),
467      html.find("<td rowspan=1>http://a.yahoo.com/y"),
468      html.find("<td rowspan=1>http://c.yahoo.com/x4"),
469      html.find("<td rowspan=1>http://d.yahoo.com/x5"),
470  };
471
472  // Make sure things appeared in the expected order.
473  for (size_t i = 1; i < arraysize(pos); ++i) {
474    EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
475  }
476
477  predictor.Shutdown();
478}
479
480// Verify that two floats are within 1% of each other in value.
481#define EXPECT_SIMILAR(a, b) do { \
482    double espilon_ratio = 1.01;  \
483    if ((a) < 0.)  \
484      espilon_ratio = 1 / espilon_ratio;  \
485    EXPECT_LT(a, espilon_ratio * (b));   \
486    EXPECT_GT((a) * espilon_ratio, b);   \
487    } while (0)
488
489
490// Make sure the Trim() functionality works as expected.
491TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
492  Predictor predictor(true, true);
493  predictor.SetHostResolver(host_resolver_.get());
494  GURL motivation_url("http://www.google.com:110");
495
496  GURL icon_subresource_url("http://icons.google.com:111");
497  const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
498  GURL img_subresource_url("http://img.google.com:118");
499  const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
500
501  scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
502  AddToSerializedList(
503      motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
504  AddToSerializedList(
505      motivation_url, img_subresource_url, kRateImg, referral_list.get());
506
507  predictor.DeserializeReferrers(*referral_list.get());
508
509  base::ListValue recovered_referral_list;
510  predictor.SerializeReferrers(&recovered_referral_list);
511  EXPECT_EQ(2U, recovered_referral_list.GetSize());
512  double rate;
513  EXPECT_TRUE(GetDataFromSerialization(
514      motivation_url, icon_subresource_url, recovered_referral_list,
515      &rate));
516  EXPECT_SIMILAR(rate, kRateIcon);
517
518  EXPECT_TRUE(GetDataFromSerialization(
519      motivation_url, img_subresource_url, recovered_referral_list, &rate));
520  EXPECT_SIMILAR(rate, kRateImg);
521
522  // Each time we Trim 24 times, the user_rate figures should reduce by a factor
523  // of two,  until they are small, and then a trim will delete the whole entry.
524  for (int i = 0; i < 24; ++i)
525    predictor.TrimReferrersNow();
526  predictor.SerializeReferrers(&recovered_referral_list);
527  EXPECT_EQ(2U, recovered_referral_list.GetSize());
528  EXPECT_TRUE(GetDataFromSerialization(
529      motivation_url, icon_subresource_url, recovered_referral_list, &rate));
530  EXPECT_SIMILAR(rate, kRateIcon / 2);
531
532  EXPECT_TRUE(GetDataFromSerialization(
533      motivation_url, img_subresource_url, recovered_referral_list, &rate));
534  EXPECT_SIMILAR(rate, kRateImg / 2);
535
536  for (int i = 0; i < 24; ++i)
537    predictor.TrimReferrersNow();
538  predictor.SerializeReferrers(&recovered_referral_list);
539  EXPECT_EQ(2U, recovered_referral_list.GetSize());
540  EXPECT_TRUE(GetDataFromSerialization(
541      motivation_url, icon_subresource_url, recovered_referral_list, &rate));
542  EXPECT_SIMILAR(rate, kRateIcon / 4);
543  EXPECT_TRUE(GetDataFromSerialization(
544      motivation_url, img_subresource_url, recovered_referral_list, &rate));
545  EXPECT_SIMILAR(rate, kRateImg / 4);
546
547  for (int i = 0; i < 24; ++i)
548    predictor.TrimReferrersNow();
549  predictor.SerializeReferrers(&recovered_referral_list);
550  EXPECT_EQ(2U, recovered_referral_list.GetSize());
551  EXPECT_TRUE(GetDataFromSerialization(
552      motivation_url, icon_subresource_url, recovered_referral_list, &rate));
553  EXPECT_SIMILAR(rate, kRateIcon / 8);
554
555  // Img is below threshold, and so it gets deleted.
556  EXPECT_FALSE(GetDataFromSerialization(
557      motivation_url, img_subresource_url, recovered_referral_list, &rate));
558
559  for (int i = 0; i < 24; ++i)
560    predictor.TrimReferrersNow();
561  predictor.SerializeReferrers(&recovered_referral_list);
562  // Icon is also trimmed away, so entire set gets discarded.
563  EXPECT_EQ(1U, recovered_referral_list.GetSize());
564  EXPECT_FALSE(GetDataFromSerialization(
565      motivation_url, icon_subresource_url, recovered_referral_list, &rate));
566  EXPECT_FALSE(GetDataFromSerialization(
567      motivation_url, img_subresource_url, recovered_referral_list, &rate));
568
569  predictor.Shutdown();
570}
571
572
573TEST_F(PredictorTest, PriorityQueuePushPopTest) {
574  Predictor::HostNameQueue queue;
575
576  GURL first("http://first:80"), second("http://second:90");
577
578  // First check high priority queue FIFO functionality.
579  EXPECT_TRUE(queue.IsEmpty());
580  queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
581  EXPECT_FALSE(queue.IsEmpty());
582  queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
583  EXPECT_FALSE(queue.IsEmpty());
584  EXPECT_EQ(queue.Pop(), first);
585  EXPECT_FALSE(queue.IsEmpty());
586  EXPECT_EQ(queue.Pop(), second);
587  EXPECT_TRUE(queue.IsEmpty());
588
589  // Then check low priority queue FIFO functionality.
590  queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
591  EXPECT_FALSE(queue.IsEmpty());
592  queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
593  EXPECT_FALSE(queue.IsEmpty());
594  EXPECT_EQ(queue.Pop(), first);
595  EXPECT_FALSE(queue.IsEmpty());
596  EXPECT_EQ(queue.Pop(), second);
597  EXPECT_TRUE(queue.IsEmpty());
598}
599
600TEST_F(PredictorTest, PriorityQueueReorderTest) {
601  Predictor::HostNameQueue queue;
602
603  // Push all the low priority items.
604  GURL low1("http://low1:80"),
605      low2("http://low2:80"),
606      low3("http://low3:443"),
607      low4("http://low4:80"),
608      low5("http://low5:80"),
609      hi1("http://hi1:80"),
610      hi2("http://hi2:80"),
611      hi3("http://hi3:80");
612
613  EXPECT_TRUE(queue.IsEmpty());
614  queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
615  queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
616  queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
617  queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
618  queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
619  queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
620
621  // Push all the high prority items
622  queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
623  queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
624  queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
625
626  // Check that high priority stuff comes out first, and in FIFO order.
627  EXPECT_EQ(queue.Pop(), hi1);
628  EXPECT_EQ(queue.Pop(), hi2);
629  EXPECT_EQ(queue.Pop(), hi3);
630
631  // ...and then low priority strings.
632  EXPECT_EQ(queue.Pop(), low1);
633  EXPECT_EQ(queue.Pop(), low2);
634  EXPECT_EQ(queue.Pop(), low3);
635  EXPECT_EQ(queue.Pop(), low4);
636  EXPECT_EQ(queue.Pop(), low5);
637  EXPECT_EQ(queue.Pop(), low4);
638
639  EXPECT_TRUE(queue.IsEmpty());
640}
641
642TEST_F(PredictorTest, CanonicalizeUrl) {
643  // Base case, only handles HTTP and HTTPS.
644  EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
645
646  // Remove path testing.
647  GURL long_url("http://host:999/path?query=value");
648  EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
649
650  // Default port cannoncalization.
651  GURL implied_port("http://test");
652  GURL explicit_port("http://test:80");
653  EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
654            Predictor::CanonicalizeUrl(explicit_port));
655
656  // Port is still maintained.
657  GURL port_80("http://test:80");
658  GURL port_90("http://test:90");
659  EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
660            Predictor::CanonicalizeUrl(port_90));
661
662  // Host is still maintained.
663  GURL host_1("http://test_1");
664  GURL host_2("http://test_2");
665  EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
666            Predictor::CanonicalizeUrl(host_2));
667
668  // Scheme is maintained (mismatch identified).
669  GURL http("http://test");
670  GURL https("https://test");
671  EXPECT_NE(Predictor::CanonicalizeUrl(http),
672            Predictor::CanonicalizeUrl(https));
673
674  // Https works fine.
675  GURL long_https("https://host:999/path?query=value");
676  EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
677            long_https.GetWithEmptyPath());
678}
679
680TEST_F(PredictorTest, DiscardPredictorResults) {
681  SimplePredictor predictor(true, true);
682  predictor.SetHostResolver(host_resolver_.get());
683  base::ListValue referral_list;
684  predictor.SerializeReferrers(&referral_list);
685  EXPECT_EQ(1U, referral_list.GetSize());
686
687  GURL host_1("http://test_1");
688  GURL host_2("http://test_2");
689  predictor.LearnFromNavigation(host_1, host_2);
690
691  predictor.SerializeReferrers(&referral_list);
692  EXPECT_EQ(2U, referral_list.GetSize());
693
694  predictor.DiscardAllResults();
695  predictor.SerializeReferrers(&referral_list);
696  EXPECT_EQ(1U, referral_list.GetSize());
697
698  predictor.Shutdown();
699}
700
701class TestPredictorObserver : public PredictorObserver {
702 public:
703  // PredictorObserver implementation:
704  virtual void OnPreconnectUrl(const GURL& url,
705                               const GURL& first_party_for_cookies,
706                               UrlInfo::ResolutionMotivation motivation,
707                               int count) OVERRIDE {
708    preconnected_urls_.push_back(url);
709  }
710
711  std::vector<GURL> preconnected_urls_;
712};
713
714// Tests that preconnects apply the HSTS list.
715TEST_F(PredictorTest, HSTSRedirect) {
716  const GURL kHttpUrl("http://example.com");
717  const GURL kHttpsUrl("https://example.com");
718
719  const base::Time expiry =
720      base::Time::Now() + base::TimeDelta::FromSeconds(1000);
721  net::TransportSecurityState state;
722  state.AddHSTS(kHttpUrl.host(), expiry, false);
723
724  Predictor predictor(true, true);
725  TestPredictorObserver observer;
726  predictor.SetObserver(&observer);
727  predictor.SetTransportSecurityState(&state);
728
729  predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
730  ASSERT_EQ(1u, observer.preconnected_urls_.size());
731  EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
732
733  predictor.Shutdown();
734}
735
736// Tests that preconnecting a URL on the HSTS list preconnects the subresources
737// for the SSL version.
738TEST_F(PredictorTest, HSTSRedirectSubresources) {
739  const GURL kHttpUrl("http://example.com");
740  const GURL kHttpsUrl("https://example.com");
741  const GURL kSubresourceUrl("https://images.example.com");
742  const double kUseRate = 23.4;
743
744  const base::Time expiry =
745      base::Time::Now() + base::TimeDelta::FromSeconds(1000);
746  net::TransportSecurityState state;
747  state.AddHSTS(kHttpUrl.host(), expiry, false);
748
749  SimplePredictor predictor(true, true);
750  TestPredictorObserver observer;
751  predictor.SetObserver(&observer);
752  predictor.SetTransportSecurityState(&state);
753
754  scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
755  AddToSerializedList(
756      kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
757  predictor.DeserializeReferrers(*referral_list.get());
758
759  predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
760  ASSERT_EQ(2u, observer.preconnected_urls_.size());
761  EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
762  EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
763
764  predictor.Shutdown();
765}
766
767#if defined(OS_ANDROID) || defined(OS_IOS)
768// Tests for the predictor with a proxy advisor
769
770class TestProxyAdvisor : public ProxyAdvisor {
771 public:
772  TestProxyAdvisor()
773      : ProxyAdvisor(NULL, NULL),
774        would_proxy_(false),
775        advise_count_(0),
776        would_proxy_count_(0) {
777  }
778
779  virtual ~TestProxyAdvisor() {}
780
781  virtual void Advise(const GURL& url,
782                      UrlInfo::ResolutionMotivation motivation,
783                      bool is_preconnect) OVERRIDE {
784    ++advise_count_;
785  }
786
787  virtual bool WouldProxyURL(const GURL& url) OVERRIDE {
788    ++would_proxy_count_;
789    return would_proxy_;
790  }
791
792  bool would_proxy_;
793  int advise_count_;
794  int would_proxy_count_;
795};
796
797TEST_F(PredictorTest, SingleLookupTestWithDisabledAdvisor) {
798  Predictor testing_master(true, true);
799  TestProxyAdvisor* advisor = new TestProxyAdvisor();
800  testing_master.SetHostResolver(host_resolver_.get());
801  testing_master.proxy_advisor_.reset(advisor);
802
803  GURL goog("http://www.google.com:80");
804
805  advisor->would_proxy_ = false;
806
807  UrlList names;
808  names.push_back(goog);
809  testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
810
811  WaitForResolution(&testing_master, names);
812  EXPECT_TRUE(testing_master.WasFound(goog));
813  EXPECT_EQ(advisor->would_proxy_count_, 1);
814  EXPECT_EQ(advisor->advise_count_, 1);
815
816  base::MessageLoop::current()->RunUntilIdle();
817
818  testing_master.Shutdown();
819}
820
821TEST_F(PredictorTest, SingleLookupTestWithEnabledAdvisor) {
822  Predictor testing_master(true, true);
823  testing_master.SetHostResolver(host_resolver_.get());
824  TestProxyAdvisor* advisor = new TestProxyAdvisor();
825  testing_master.proxy_advisor_.reset(advisor);
826
827  GURL goog("http://www.google.com:80");
828
829  advisor->would_proxy_ = true;
830
831  UrlList names;
832  names.push_back(goog);
833
834  testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
835
836  // Attempt to resolve a few times.
837  WaitForResolutionWithLimit(&testing_master, names, 10);
838
839  // Because the advisor indicated that the url would be proxied,
840  // no resolution should have occurred.
841  EXPECT_FALSE(testing_master.WasFound(goog));
842  EXPECT_EQ(advisor->would_proxy_count_, 1);
843  EXPECT_EQ(advisor->advise_count_, 1);
844
845  base::MessageLoop::current()->RunUntilIdle();
846
847  testing_master.Shutdown();
848}
849
850TEST_F(PredictorTest, TestSimplePreconnectAdvisor) {
851  Predictor testing_master(true, true);
852  testing_master.SetHostResolver(host_resolver_.get());
853  TestProxyAdvisor* advisor = new TestProxyAdvisor();
854  testing_master.proxy_advisor_.reset(advisor);
855
856  GURL goog("http://www.google.com:80");
857
858  testing_master.PreconnectUrl(goog, goog, UrlInfo::OMNIBOX_MOTIVATED, 2);
859
860  EXPECT_EQ(advisor->would_proxy_count_, 0);
861  EXPECT_EQ(advisor->advise_count_, 1);
862
863  testing_master.Shutdown();
864}
865
866#endif  // defined(OS_ANDROID) || defined(OS_IOS)
867
868}  // namespace chrome_browser_net
869