1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4//
5// Unit tests for the SafeBrowsing storage system.
6
7#include "chrome/browser/safe_browsing/safe_browsing_database.h"
8
9#include "base/files/file_util.h"
10#include "base/files/scoped_temp_dir.h"
11#include "base/logging.h"
12#include "base/memory/scoped_vector.h"
13#include "base/message_loop/message_loop.h"
14#include "base/sha1.h"
15#include "base/strings/string_number_conversions.h"
16#include "base/strings/string_split.h"
17#include "base/time/time.h"
18#include "chrome/browser/safe_browsing/chunk.pb.h"
19#include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
20#include "content/public/test/test_browser_thread_bundle.h"
21#include "crypto/sha2.h"
22#include "net/base/net_util.h"
23#include "sql/connection.h"
24#include "sql/statement.h"
25#include "testing/gtest/include/gtest/gtest.h"
26#include "testing/platform_test.h"
27#include "url/gurl.h"
28
29using base::Time;
30using base::TimeDelta;
31
32namespace {
33
34const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
35
36SBPrefix SBPrefixForString(const std::string& str) {
37  return SBFullHashForString(str).prefix;
38}
39
40// Construct a full hash which has the given prefix, with the given
41// suffix data coming after the prefix.
42SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
43                                        const base::StringPiece& suffix) {
44  SBFullHash full_hash;
45  memset(&full_hash, 0, sizeof(SBFullHash));
46  full_hash.prefix = prefix;
47  CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
48  memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
49  return full_hash;
50}
51
52std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
53  net::IPAddressNumber ip_number;
54  EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
55  EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
56  const std::string hashed_ip_prefix = base::SHA1HashString(
57      net::IPAddressToPackedString(ip_number));
58  std::string hash(crypto::kSHA256Length, '\0');
59  hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
60  hash[base::kSHA1Length] = static_cast<char>(prefix_size);
61  return hash;
62}
63
64// Helper to build a chunk.  Caller takes ownership.
65SBChunkData* BuildChunk(int chunk_number,
66                        safe_browsing::ChunkData::ChunkType chunk_type,
67                        safe_browsing::ChunkData::PrefixType prefix_type,
68                        const void* data, size_t data_size,
69                        const std::vector<int>& add_chunk_numbers) {
70  scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
71  raw_data->set_chunk_number(chunk_number);
72  raw_data->set_chunk_type(chunk_type);
73  raw_data->set_prefix_type(prefix_type);
74  raw_data->set_hashes(data, data_size);
75  raw_data->clear_add_numbers();
76  for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
77    raw_data->add_add_numbers(add_chunk_numbers[i]);
78  }
79
80  return new SBChunkData(raw_data.release());
81}
82
83// Create add chunk with a single prefix.
84SBChunkData* AddChunkPrefix(int chunk_number,  SBPrefix prefix) {
85  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
86                    safe_browsing::ChunkData::PREFIX_4B,
87                    &prefix, sizeof(prefix),
88                    std::vector<int>());
89}
90
91// Create add chunk with a single prefix generated from |value|.
92SBChunkData* AddChunkPrefixValue(int chunk_number,
93                                 const std::string& value) {
94  return AddChunkPrefix(chunk_number, SBPrefixForString(value));
95}
96
97// Generate an add chunk with two prefixes.
98SBChunkData* AddChunkPrefix2Value(int chunk_number,
99                                  const std::string& value1,
100                                  const std::string& value2) {
101  const SBPrefix prefixes[2] = {
102    SBPrefixForString(value1),
103    SBPrefixForString(value2),
104  };
105  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
106                    safe_browsing::ChunkData::PREFIX_4B,
107                    &prefixes[0], sizeof(prefixes),
108                    std::vector<int>());
109}
110
111// Generate an add chunk with four prefixes.
112SBChunkData* AddChunkPrefix4Value(int chunk_number,
113                                  const std::string& value1,
114                                  const std::string& value2,
115                                  const std::string& value3,
116                                  const std::string& value4) {
117  const SBPrefix prefixes[4] = {
118    SBPrefixForString(value1),
119    SBPrefixForString(value2),
120    SBPrefixForString(value3),
121    SBPrefixForString(value4),
122  };
123  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
124                    safe_browsing::ChunkData::PREFIX_4B,
125                    &prefixes[0], sizeof(prefixes),
126                    std::vector<int>());
127}
128
129// Generate an add chunk with a full hash.
130SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
131  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
132                    safe_browsing::ChunkData::FULL_32B,
133                    &full_hash, sizeof(full_hash),
134                    std::vector<int>());
135}
136
137// Generate an add chunk with a full hash generated from |value|.
138SBChunkData* AddChunkFullHashValue(int chunk_number,
139                                   const std::string& value) {
140  return AddChunkFullHash(chunk_number, SBFullHashForString(value));
141}
142
143// Generate an add chunk with two full hashes.
144SBChunkData* AddChunkFullHash2Value(int chunk_number,
145                                   const std::string& value1,
146                                   const std::string& value2) {
147  const SBFullHash full_hashes[2] = {
148    SBFullHashForString(value1),
149    SBFullHashForString(value2),
150  };
151  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
152                    safe_browsing::ChunkData::FULL_32B,
153                    &full_hashes[0], sizeof(full_hashes),
154                    std::vector<int>());
155}
156
157// Generate a sub chunk with a prefix generated from |value|.
158SBChunkData* SubChunkPrefixValue(int chunk_number,
159                                 const std::string& value,
160                                 int add_chunk_number) {
161  const SBPrefix prefix = SBPrefixForString(value);
162  return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
163                    safe_browsing::ChunkData::PREFIX_4B,
164                    &prefix, sizeof(prefix),
165                    std::vector<int>(1, add_chunk_number));
166}
167
168// Generate a sub chunk with two prefixes.
169SBChunkData* SubChunkPrefix2Value(int chunk_number,
170                                  const std::string& value1,
171                                  int add_chunk_number1,
172                                  const std::string& value2,
173                                  int add_chunk_number2) {
174  const SBPrefix prefixes[2] = {
175    SBPrefixForString(value1),
176    SBPrefixForString(value2),
177  };
178  std::vector<int> add_chunk_numbers;
179  add_chunk_numbers.push_back(add_chunk_number1);
180  add_chunk_numbers.push_back(add_chunk_number2);
181  return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
182                    safe_browsing::ChunkData::PREFIX_4B,
183                    &prefixes[0], sizeof(prefixes),
184                    add_chunk_numbers);
185}
186
187// Generate a sub chunk with a full hash.
188SBChunkData* SubChunkFullHash(int chunk_number,
189                              SBFullHash full_hash,
190                              int add_chunk_number) {
191  return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
192                    safe_browsing::ChunkData::FULL_32B,
193                    &full_hash, sizeof(full_hash),
194                    std::vector<int>(1, add_chunk_number));
195}
196
197// Generate a sub chunk with a full hash generated from |value|.
198SBChunkData* SubChunkFullHashValue(int chunk_number,
199                                   const std::string& value,
200                                   int add_chunk_number) {
201  return SubChunkFullHash(chunk_number,
202                          SBFullHashForString(value),
203                          add_chunk_number);
204}
205
206// Generate an add chunk with a single full hash for the ip blacklist.
207SBChunkData* AddChunkHashedIpValue(int chunk_number,
208                                   const std::string& ip_str,
209                                   size_t prefix_size) {
210  const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
211  EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
212  SBFullHash full_hash;
213  std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
214  return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
215                    safe_browsing::ChunkData::FULL_32B,
216                    &full_hash, sizeof(full_hash),
217                    std::vector<int>());
218}
219
220// Prevent DCHECK from killing tests.
221// TODO(shess): Pawel disputes the use of this, so the test which uses
222// it is DISABLED.  http://crbug.com/56448
223class ScopedLogMessageIgnorer {
224 public:
225  ScopedLogMessageIgnorer() {
226    logging::SetLogMessageHandler(&LogMessageIgnorer);
227  }
228  ~ScopedLogMessageIgnorer() {
229    // TODO(shess): Would be better to verify whether anyone else
230    // changed it, and then restore it to the previous value.
231    logging::SetLogMessageHandler(NULL);
232  }
233
234 private:
235  static bool LogMessageIgnorer(int severity, const char* file, int line,
236      size_t message_start, const std::string& str) {
237    // Intercept FATAL, strip the stack backtrace, and log it without
238    // the crash part.
239    if (severity == logging::LOG_FATAL) {
240      size_t newline = str.find('\n');
241      if (newline != std::string::npos) {
242        const std::string msg = str.substr(0, newline + 1);
243        fprintf(stderr, "%s", msg.c_str());
244        fflush(stderr);
245      }
246      return true;
247    }
248
249    return false;
250  }
251};
252
253}  // namespace
254
255class SafeBrowsingDatabaseTest : public PlatformTest {
256 public:
257  virtual void SetUp() {
258    PlatformTest::SetUp();
259
260    // Setup a database in a temporary directory.
261    ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
262    database_.reset(new SafeBrowsingDatabaseNew);
263    database_filename_ =
264        temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
265    database_->Init(database_filename_);
266  }
267
268  virtual void TearDown() {
269    database_.reset();
270
271    PlatformTest::TearDown();
272  }
273
274  void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
275    lists->clear();
276    ASSERT_TRUE(database_->UpdateStarted(lists));
277    database_->UpdateFinished(true);
278  }
279
280  // Helper function to do an AddDel or SubDel command.
281  void DelChunk(const std::string& list,
282                int chunk_id,
283                bool is_sub_del) {
284    std::vector<SBChunkDelete> deletes;
285    SBChunkDelete chunk_delete;
286    chunk_delete.list_name = list;
287    chunk_delete.is_sub_del = is_sub_del;
288    chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
289    deletes.push_back(chunk_delete);
290    database_->DeleteChunks(deletes);
291  }
292
293  void AddDelChunk(const std::string& list, int chunk_id) {
294    DelChunk(list, chunk_id, false);
295  }
296
297  void SubDelChunk(const std::string& list, int chunk_id) {
298    DelChunk(list, chunk_id, true);
299  }
300
301  // Utility function for setting up the database for the caching test.
302  void PopulateDatabaseForCacheTest();
303
304  scoped_ptr<SafeBrowsingDatabaseNew> database_;
305  base::FilePath database_filename_;
306  base::ScopedTempDir temp_dir_;
307};
308
309// Tests retrieving list name information.
310TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
311  std::vector<SBListChunkRanges> lists;
312  ScopedVector<SBChunkData> chunks;
313
314  chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
315  chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
316  chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
317
318  ASSERT_TRUE(database_->UpdateStarted(&lists));
319  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
320  database_->UpdateFinished(true);
321
322  GetListsInfo(&lists);
323  ASSERT_LE(1U, lists.size());
324  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
325  EXPECT_EQ("1-3", lists[0].adds);
326  EXPECT_TRUE(lists[0].subs.empty());
327
328  // Insert a malware sub chunk.
329  chunks.clear();
330  chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
331
332  ASSERT_TRUE(database_->UpdateStarted(&lists));
333  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
334  database_->UpdateFinished(true);
335
336  GetListsInfo(&lists);
337  ASSERT_LE(1U, lists.size());
338  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
339  EXPECT_EQ("1-3", lists[0].adds);
340  EXPECT_EQ("7", lists[0].subs);
341  if (lists.size() == 2) {
342    // Old style database won't have the second entry since it creates the lists
343    // when it receives an update containing that list. The filter-based
344    // database has these values hard coded.
345    EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
346    EXPECT_TRUE(lists[1].adds.empty());
347    EXPECT_TRUE(lists[1].subs.empty());
348  }
349
350  // Add phishing chunks.
351  chunks.clear();
352  chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
353  chunks.push_back(
354      SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
355  chunks.push_back(
356      SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
357
358  ASSERT_TRUE(database_->UpdateStarted(&lists));
359  database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
360  database_->UpdateFinished(true);
361
362  GetListsInfo(&lists);
363  ASSERT_EQ(2U, lists.size());
364  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
365  EXPECT_EQ("1-3", lists[0].adds);
366  EXPECT_EQ("7", lists[0].subs);
367  EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
368  EXPECT_EQ("47", lists[1].adds);
369  EXPECT_EQ("200-201", lists[1].subs);
370}
371
372TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
373  database_.reset();
374  base::MessageLoop loop;
375  SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
376  SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
377  SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
378  SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
379  SafeBrowsingStoreFile* extension_blacklist_store =
380      new SafeBrowsingStoreFile();
381  SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
382  database_.reset(new SafeBrowsingDatabaseNew(browse_store,
383                                              download_store,
384                                              csd_whitelist_store,
385                                              download_whitelist_store,
386                                              extension_blacklist_store,
387                                              NULL,
388                                              ip_blacklist_store));
389  database_->Init(database_filename_);
390
391  ScopedVector<SBChunkData> chunks;
392
393  std::vector<SBListChunkRanges> lists;
394  ASSERT_TRUE(database_->UpdateStarted(&lists));
395
396  // Insert malware, phish, binurl and bindownload add chunks.
397  chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
398  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
399
400  chunks.clear();
401  chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
402  database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
403
404  chunks.clear();
405  chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
406  database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
407
408  chunks.clear();
409  chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
410  database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
411
412  chunks.clear();
413  chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
414  database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
415
416  chunks.clear();
417  chunks.push_back(AddChunkFullHashValue(8,
418                                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
419                                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
420  database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
421                          chunks.get());
422
423  chunks.clear();
424  chunks.push_back(AddChunkHashedIpValue(9, "::ffff:192.168.1.0", 120));
425  database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
426
427  database_->UpdateFinished(true);
428
429  GetListsInfo(&lists);
430  ASSERT_EQ(7U, lists.size());
431  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
432  EXPECT_EQ("1", lists[0].adds);
433  EXPECT_TRUE(lists[0].subs.empty());
434  EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
435  EXPECT_EQ("2", lists[1].adds);
436  EXPECT_TRUE(lists[1].subs.empty());
437  EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
438  EXPECT_EQ("3", lists[2].adds);
439  EXPECT_TRUE(lists[2].subs.empty());
440  EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
441  EXPECT_EQ("5", lists[3].adds);
442  EXPECT_TRUE(lists[3].subs.empty());
443  EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
444  EXPECT_EQ("6", lists[4].adds);
445  EXPECT_TRUE(lists[4].subs.empty());
446  EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[5].name);
447  EXPECT_EQ("8", lists[5].adds);
448  EXPECT_TRUE(lists[5].subs.empty());
449  EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[6].name);
450  EXPECT_EQ("9", lists[6].adds);
451  EXPECT_TRUE(lists[6].subs.empty());
452
453  database_.reset();
454}
455
456// Checks database reading and writing for browse.
457TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
458  std::vector<SBListChunkRanges> lists;
459  ScopedVector<SBChunkData> chunks;
460
461  chunks.push_back(AddChunkPrefix2Value(1,
462                                        "www.evil.com/phishing.html",
463                                        "www.evil.com/malware.html"));
464  chunks.push_back(AddChunkPrefix4Value(2,
465                                        "www.evil.com/notevil1.html",
466                                        "www.evil.com/notevil2.html",
467                                        "www.good.com/good1.html",
468                                        "www.good.com/good2.html"));
469  chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
470  chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
471
472  ASSERT_TRUE(database_->UpdateStarted(&lists));
473  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
474  database_->UpdateFinished(true);
475
476  // Make sure they were added correctly.
477  GetListsInfo(&lists);
478  ASSERT_LE(1U, lists.size());
479  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
480  EXPECT_EQ("1-3,7", lists[0].adds);
481  EXPECT_TRUE(lists[0].subs.empty());
482
483  std::vector<SBPrefix> prefix_hits;
484  std::vector<SBFullHashResult> cache_hits;
485  EXPECT_TRUE(database_->ContainsBrowseUrl(
486      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
487  ASSERT_EQ(1U, prefix_hits.size());
488  EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
489  EXPECT_TRUE(cache_hits.empty());
490
491  EXPECT_TRUE(database_->ContainsBrowseUrl(
492      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
493
494  EXPECT_TRUE(database_->ContainsBrowseUrl(
495      GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
496
497  EXPECT_TRUE(database_->ContainsBrowseUrl(
498      GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
499
500  EXPECT_TRUE(database_->ContainsBrowseUrl(
501      GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
502
503  EXPECT_TRUE(database_->ContainsBrowseUrl(
504      GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
505
506  EXPECT_TRUE(database_->ContainsBrowseUrl(
507      GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
508
509  EXPECT_FALSE(database_->ContainsBrowseUrl(
510      GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
511  EXPECT_TRUE(prefix_hits.empty());
512  EXPECT_TRUE(cache_hits.empty());
513
514  EXPECT_FALSE(database_->ContainsBrowseUrl(
515      GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
516
517  EXPECT_TRUE(database_->ContainsBrowseUrl(
518      GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
519  ASSERT_EQ(1U, prefix_hits.size());
520  EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
521
522  // Attempt to re-add the first chunk (should be a no-op).
523  // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
524  chunks.clear();
525  chunks.push_back(AddChunkPrefix2Value(1,
526                                        "www.evil.com/phishing.html",
527                                        "www.evil.com/malware.html"));
528  ASSERT_TRUE(database_->UpdateStarted(&lists));
529  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
530  database_->UpdateFinished(true);
531
532  GetListsInfo(&lists);
533  ASSERT_LE(1U, lists.size());
534  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
535  EXPECT_EQ("1-3,7", lists[0].adds);
536  EXPECT_TRUE(lists[0].subs.empty());
537
538  // Test removing a single prefix from the add chunk.
539  chunks.clear();
540  chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
541  ASSERT_TRUE(database_->UpdateStarted(&lists));
542  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
543  database_->UpdateFinished(true);
544
545  EXPECT_TRUE(database_->ContainsBrowseUrl(
546      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
547  ASSERT_EQ(1U, prefix_hits.size());
548  EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
549  EXPECT_TRUE(cache_hits.empty());
550
551  EXPECT_FALSE(database_->ContainsBrowseUrl(
552      GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
553  EXPECT_TRUE(prefix_hits.empty());
554  EXPECT_TRUE(cache_hits.empty());
555
556  EXPECT_TRUE(database_->ContainsBrowseUrl(
557      GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
558
559  EXPECT_TRUE(database_->ContainsBrowseUrl(
560      GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
561
562  EXPECT_TRUE(database_->ContainsBrowseUrl(
563      GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
564
565  GetListsInfo(&lists);
566  ASSERT_LE(1U, lists.size());
567  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
568  EXPECT_EQ("1-3,7", lists[0].adds);
569  EXPECT_EQ("4", lists[0].subs);
570
571  // Test the same sub chunk again.  This should be a no-op.
572  // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
573  chunks.clear();
574  chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
575
576  ASSERT_TRUE(database_->UpdateStarted(&lists));
577  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
578  database_->UpdateFinished(true);
579
580  GetListsInfo(&lists);
581  ASSERT_LE(1U, lists.size());
582  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
583  EXPECT_EQ("1-3,7", lists[0].adds);
584  EXPECT_EQ("4", lists[0].subs);
585
586  // Test removing all the prefixes from an add chunk.
587  ASSERT_TRUE(database_->UpdateStarted(&lists));
588  AddDelChunk(safe_browsing_util::kMalwareList, 2);
589  database_->UpdateFinished(true);
590
591  EXPECT_FALSE(database_->ContainsBrowseUrl(
592      GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
593
594  EXPECT_FALSE(database_->ContainsBrowseUrl(
595      GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
596
597  EXPECT_FALSE(database_->ContainsBrowseUrl(
598      GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
599
600  GetListsInfo(&lists);
601  ASSERT_LE(1U, lists.size());
602  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
603  EXPECT_EQ("1,3,7", lists[0].adds);
604  EXPECT_EQ("4", lists[0].subs);
605
606  // The adddel command exposed a bug in the transaction code where any
607  // transaction after it would fail.  Add a dummy entry and remove it to
608  // make sure the transcation works fine.
609  chunks.clear();
610  chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
611  ASSERT_TRUE(database_->UpdateStarted(&lists));
612  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
613
614  // Now remove the dummy entry.  If there are any problems with the
615  // transactions, asserts will fire.
616  AddDelChunk(safe_browsing_util::kMalwareList, 44);
617
618  // Test the subdel command.
619  SubDelChunk(safe_browsing_util::kMalwareList, 4);
620  database_->UpdateFinished(true);
621
622  GetListsInfo(&lists);
623  ASSERT_LE(1U, lists.size());
624  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
625  EXPECT_EQ("1,3,7", lists[0].adds);
626  EXPECT_TRUE(lists[0].subs.empty());
627
628  // Test a sub command coming in before the add.
629  chunks.clear();
630  chunks.push_back(SubChunkPrefix2Value(5,
631                                        "www.notevilanymore.com/index.html",
632                                        10,
633                                        "www.notevilanymore.com/good.html",
634                                        10));
635  ASSERT_TRUE(database_->UpdateStarted(&lists));
636  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
637  database_->UpdateFinished(true);
638
639  EXPECT_FALSE(database_->ContainsBrowseUrl(
640      GURL("http://www.notevilanymore.com/index.html"),
641      &prefix_hits,
642      &cache_hits));
643
644  // Now insert the tardy add chunk and we don't expect them to appear
645  // in database because of the previous sub chunk.
646  chunks.clear();
647  chunks.push_back(AddChunkPrefix2Value(10,
648                                        "www.notevilanymore.com/index.html",
649                                        "www.notevilanymore.com/good.html"));
650  ASSERT_TRUE(database_->UpdateStarted(&lists));
651  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
652  database_->UpdateFinished(true);
653
654  EXPECT_FALSE(database_->ContainsBrowseUrl(
655      GURL("http://www.notevilanymore.com/index.html"),
656      &prefix_hits,
657      &cache_hits));
658
659  EXPECT_FALSE(database_->ContainsBrowseUrl(
660      GURL("http://www.notevilanymore.com/good.html"),
661      &prefix_hits,
662      &cache_hits));
663
664  // Reset and reload the database.  The database will rely on the prefix set.
665  database_.reset(new SafeBrowsingDatabaseNew);
666  database_->Init(database_filename_);
667
668  // Check that a prefix still hits.
669  EXPECT_TRUE(database_->ContainsBrowseUrl(
670      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
671  ASSERT_EQ(1U, prefix_hits.size());
672  EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
673
674  // Also check that it's not just always returning true in this case.
675  EXPECT_FALSE(database_->ContainsBrowseUrl(
676      GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
677
678  // Check that the full hash is still present.
679  EXPECT_TRUE(database_->ContainsBrowseUrl(
680      GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
681  ASSERT_EQ(1U, prefix_hits.size());
682  EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
683}
684
685// Test adding zero length chunks to the database.
686TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
687  std::vector<SBListChunkRanges> lists;
688  ScopedVector<SBChunkData> chunks;
689
690  // Populate with a couple of normal chunks.
691  chunks.push_back(AddChunkPrefix2Value(1,
692                                        "www.test.com/test1.html",
693                                        "www.test.com/test2.html"));
694  chunks.push_back(AddChunkPrefix2Value(10,
695                                        "www.random.com/random1.html",
696                                        "www.random.com/random2.html"));
697
698  ASSERT_TRUE(database_->UpdateStarted(&lists));
699  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
700  database_->UpdateFinished(true);
701
702  // Add an empty ADD and SUB chunk.
703  GetListsInfo(&lists);
704  ASSERT_LE(1U, lists.size());
705  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
706  EXPECT_EQ("1,10", lists[0].adds);
707  EXPECT_TRUE(lists[0].subs.empty());
708
709  chunks.clear();
710  chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
711                              safe_browsing::ChunkData::PREFIX_4B,
712                              NULL, 0, std::vector<int>()));
713  chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
714                              safe_browsing::ChunkData::PREFIX_4B,
715                              NULL, 0, std::vector<int>()));
716
717  ASSERT_TRUE(database_->UpdateStarted(&lists));
718  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
719  database_->UpdateFinished(true);
720
721  GetListsInfo(&lists);
722  ASSERT_LE(1U, lists.size());
723  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
724  EXPECT_EQ("1,10,19", lists[0].adds);
725  EXPECT_EQ("7", lists[0].subs);
726
727  // Add an empty chunk along with a couple that contain data. This should
728  // result in the chunk range being reduced in size.
729  chunks.clear();
730  chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
731  chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
732                              safe_browsing::ChunkData::PREFIX_4B,
733                              NULL, 0, std::vector<int>()));
734  chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
735
736  ASSERT_TRUE(database_->UpdateStarted(&lists));
737  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
738  database_->UpdateFinished(true);
739
740  std::vector<SBPrefix> prefix_hits;
741  std::vector<SBFullHashResult> cache_hits;
742  EXPECT_TRUE(database_->ContainsBrowseUrl(
743      GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
744  EXPECT_TRUE(database_->ContainsBrowseUrl(
745      GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
746
747  GetListsInfo(&lists);
748  ASSERT_LE(1U, lists.size());
749  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
750  EXPECT_EQ("1,10,19-22", lists[0].adds);
751  EXPECT_EQ("7", lists[0].subs);
752
753  // Handle AddDel and SubDel commands for empty chunks.
754  ASSERT_TRUE(database_->UpdateStarted(&lists));
755  AddDelChunk(safe_browsing_util::kMalwareList, 21);
756  database_->UpdateFinished(true);
757
758  GetListsInfo(&lists);
759  ASSERT_LE(1U, lists.size());
760  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
761  EXPECT_EQ("1,10,19-20,22", lists[0].adds);
762  EXPECT_EQ("7", lists[0].subs);
763
764  ASSERT_TRUE(database_->UpdateStarted(&lists));
765  SubDelChunk(safe_browsing_util::kMalwareList, 7);
766  database_->UpdateFinished(true);
767
768  GetListsInfo(&lists);
769  ASSERT_LE(1U, lists.size());
770  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
771  EXPECT_EQ("1,10,19-20,22", lists[0].adds);
772  EXPECT_TRUE(lists[0].subs.empty());
773}
774
775// Utility function for setting up the database for the caching test.
776void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
777  // Add a couple prefixes.
778  ScopedVector<SBChunkData> chunks;
779  chunks.push_back(AddChunkPrefix2Value(1,
780                                        "www.evil.com/phishing.html",
781                                        "www.evil.com/malware.html"));
782
783  std::vector<SBListChunkRanges> lists;
784  ASSERT_TRUE(database_->UpdateStarted(&lists));
785  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
786  database_->UpdateFinished(true);
787
788  // Cache should be cleared after updating.
789  EXPECT_TRUE(database_->browse_gethash_cache_.empty());
790
791  SBFullHashResult full_hash;
792  full_hash.list_id = safe_browsing_util::MALWARE;
793
794  std::vector<SBFullHashResult> results;
795  std::vector<SBPrefix> prefixes;
796
797  // Add a fullhash result for each prefix.
798  full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
799  results.push_back(full_hash);
800  prefixes.push_back(full_hash.hash.prefix);
801
802  full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
803  results.push_back(full_hash);
804  prefixes.push_back(full_hash.hash.prefix);
805
806  database_->CacheHashResults(prefixes, results, kCacheLifetime);
807}
808
809TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
810  PopulateDatabaseForCacheTest();
811
812  // We should have both full hashes in the cache.
813  EXPECT_EQ(2U, database_->browse_gethash_cache_.size());
814
815  // Test the cache lookup for the first prefix.
816  std::vector<SBPrefix> prefix_hits;
817  std::vector<SBFullHashResult> cache_hits;
818  EXPECT_TRUE(database_->ContainsBrowseUrl(
819      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
820  EXPECT_TRUE(prefix_hits.empty());
821  ASSERT_EQ(1U, cache_hits.size());
822  EXPECT_TRUE(SBFullHashEqual(
823      cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
824
825  prefix_hits.clear();
826  cache_hits.clear();
827
828  // Test the cache lookup for the second prefix.
829  EXPECT_TRUE(database_->ContainsBrowseUrl(
830      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
831  EXPECT_TRUE(prefix_hits.empty());
832  ASSERT_EQ(1U, cache_hits.size());
833  EXPECT_TRUE(SBFullHashEqual(
834      cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
835
836  prefix_hits.clear();
837  cache_hits.clear();
838
839  // Test removing a prefix via a sub chunk.
840  ScopedVector<SBChunkData> chunks;
841  chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
842
843  std::vector<SBListChunkRanges> lists;
844  ASSERT_TRUE(database_->UpdateStarted(&lists));
845  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
846  database_->UpdateFinished(true);
847
848  // This prefix should still be there, but cached fullhash should be gone.
849  EXPECT_TRUE(database_->ContainsBrowseUrl(
850      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
851  ASSERT_EQ(1U, prefix_hits.size());
852  EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
853  EXPECT_TRUE(cache_hits.empty());
854  prefix_hits.clear();
855  cache_hits.clear();
856
857  // This prefix should be gone.
858  EXPECT_FALSE(database_->ContainsBrowseUrl(
859      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
860  prefix_hits.clear();
861  cache_hits.clear();
862
863  // Test that an AddDel for the original chunk removes the last cached entry.
864  ASSERT_TRUE(database_->UpdateStarted(&lists));
865  AddDelChunk(safe_browsing_util::kMalwareList, 1);
866  database_->UpdateFinished(true);
867  EXPECT_FALSE(database_->ContainsBrowseUrl(
868      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
869  EXPECT_TRUE(database_->browse_gethash_cache_.empty());
870  prefix_hits.clear();
871  cache_hits.clear();
872
873  // Test that the cache won't return expired values. First we have to adjust
874  // the cached entries' received time to make them older, since the database
875  // cache insert uses Time::Now(). First, store some entries.
876  PopulateDatabaseForCacheTest();
877
878  std::map<SBPrefix, SBCachedFullHashResult>* hash_cache =
879      &database_->browse_gethash_cache_;
880  EXPECT_EQ(2U, hash_cache->size());
881
882  // Now adjust one of the entries times to be in the past.
883  const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
884  std::map<SBPrefix, SBCachedFullHashResult>::iterator iter =
885      hash_cache->find(key);
886  ASSERT_TRUE(iter != hash_cache->end());
887  iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
888
889  EXPECT_TRUE(database_->ContainsBrowseUrl(
890      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
891  EXPECT_EQ(1U, prefix_hits.size());
892  EXPECT_TRUE(cache_hits.empty());
893  // Expired entry should have been removed from cache.
894  EXPECT_EQ(1U, hash_cache->size());
895
896  // This entry should still exist.
897  EXPECT_TRUE(database_->ContainsBrowseUrl(
898      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
899  EXPECT_TRUE(prefix_hits.empty());
900  EXPECT_EQ(1U, cache_hits.size());
901
902  // Testing prefix miss caching. First, we clear out the existing database,
903  // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
904  // chunks.
905  ASSERT_TRUE(database_->UpdateStarted(&lists));
906  AddDelChunk(safe_browsing_util::kMalwareList, 1);
907  database_->UpdateFinished(true);
908
909  // Cache should be cleared after updating.
910  EXPECT_TRUE(hash_cache->empty());
911
912  std::vector<SBPrefix> prefix_misses;
913  std::vector<SBFullHashResult> empty_full_hash;
914  prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
915  prefix_misses.push_back(
916      SBPrefixForString("http://www.bad.com/phishing.html"));
917  database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
918
919  // Prefixes with no full results are misses.
920  EXPECT_EQ(hash_cache->size(), prefix_misses.size());
921  ASSERT_TRUE(
922      hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
923  EXPECT_TRUE(
924      hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
925          ->second.full_hashes.empty());
926  ASSERT_TRUE(
927      hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
928  EXPECT_TRUE(
929      hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
930          ->second.full_hashes.empty());
931
932  // Update the database.
933  PopulateDatabaseForCacheTest();
934
935  // Cache a GetHash miss for a particular prefix, and even though the prefix is
936  // in the database, it is flagged as a miss so looking up the associated URL
937  // will not succeed.
938  prefix_hits.clear();
939  cache_hits.clear();
940  prefix_misses.clear();
941  empty_full_hash.clear();
942  prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
943  database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
944  EXPECT_FALSE(database_->ContainsBrowseUrl(
945      GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
946  prefix_hits.clear();
947  cache_hits.clear();
948
949  // Test receiving a full add chunk.
950  chunks.clear();
951  chunks.push_back(AddChunkFullHash2Value(20,
952                                          "www.fullevil.com/bad1.html",
953                                          "www.fullevil.com/bad2.html"));
954  ASSERT_TRUE(database_->UpdateStarted(&lists));
955  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
956  database_->UpdateFinished(true);
957
958  EXPECT_TRUE(database_->ContainsBrowseUrl(
959      GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
960  ASSERT_EQ(1U, prefix_hits.size());
961  EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
962  EXPECT_TRUE(cache_hits.empty());
963  prefix_hits.clear();
964  cache_hits.clear();
965
966  EXPECT_TRUE(database_->ContainsBrowseUrl(
967      GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
968  ASSERT_EQ(1U, prefix_hits.size());
969  EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
970  EXPECT_TRUE(cache_hits.empty());
971  prefix_hits.clear();
972  cache_hits.clear();
973
974  // Test receiving a full sub chunk, which will remove one of the full adds.
975  chunks.clear();
976  chunks.push_back(SubChunkFullHashValue(200,
977                                         "www.fullevil.com/bad1.html",
978                                         20));
979  ASSERT_TRUE(database_->UpdateStarted(&lists));
980  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
981  database_->UpdateFinished(true);
982
983  EXPECT_FALSE(database_->ContainsBrowseUrl(
984      GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
985
986  // There should be one remaining full add.
987  EXPECT_TRUE(database_->ContainsBrowseUrl(
988      GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
989  ASSERT_EQ(1U, prefix_hits.size());
990  EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
991  EXPECT_TRUE(cache_hits.empty());
992  prefix_hits.clear();
993  cache_hits.clear();
994
995  // Now test an AddDel for the remaining full add.
996  ASSERT_TRUE(database_->UpdateStarted(&lists));
997  AddDelChunk(safe_browsing_util::kMalwareList, 20);
998  database_->UpdateFinished(true);
999
1000  EXPECT_FALSE(database_->ContainsBrowseUrl(
1001      GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1002  EXPECT_FALSE(database_->ContainsBrowseUrl(
1003      GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1004
1005  // Add a fullhash which has a prefix collision for a known url.
1006  static const char kExampleFine[] = "www.example.com/fine.html";
1007  static const char kExampleCollision[] =
1008      "www.example.com/3123364814/malware.htm";
1009  ASSERT_EQ(SBPrefixForString(kExampleFine),
1010            SBPrefixForString(kExampleCollision));
1011  ASSERT_TRUE(database_->UpdateStarted(&lists));
1012  {
1013    ScopedVector<SBChunkData> chunks;
1014    chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1015    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1016  }
1017  database_->UpdateFinished(true);
1018
1019  // Expect a prefix hit due to the collision between |kExampleFine| and
1020  // |kExampleCollision|.
1021  EXPECT_TRUE(database_->ContainsBrowseUrl(
1022      GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1023  ASSERT_EQ(1U, prefix_hits.size());
1024  EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1025  EXPECT_TRUE(cache_hits.empty());
1026
1027  // Cache gethash response for |kExampleCollision|.
1028  {
1029    SBFullHashResult result;
1030    result.hash = SBFullHashForString(kExampleCollision);
1031    result.list_id = safe_browsing_util::MALWARE;
1032    database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1033                                std::vector<SBFullHashResult>(1, result),
1034                                kCacheLifetime);
1035  }
1036
1037  // The cached response means the collision no longer causes a hit.
1038  EXPECT_FALSE(database_->ContainsBrowseUrl(
1039      GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1040}
1041
1042// Test that corrupt databases are appropriately handled, even if the
1043// corruption is detected in the midst of the update.
1044// TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1045// http://crbug.com/56448
1046TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1047  // Re-create the database in a captive message loop so that we can
1048  // influence task-posting.  Database specifically needs to the
1049  // file-backed.
1050  database_.reset();
1051  base::MessageLoop loop;
1052  SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1053  database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1054                                              NULL, NULL));
1055  database_->Init(database_filename_);
1056
1057  // This will cause an empty database to be created.
1058  std::vector<SBListChunkRanges> lists;
1059  ASSERT_TRUE(database_->UpdateStarted(&lists));
1060  database_->UpdateFinished(true);
1061
1062  // Create a sub chunk to insert.
1063  ScopedVector<SBChunkData> chunks;
1064  chunks.push_back(SubChunkPrefixValue(7,
1065                                       "www.subbed.com/notevil1.html",
1066                                       19));
1067
1068  // Corrupt the file by corrupting the checksum, which is not checked
1069  // until the entire table is read in |UpdateFinished()|.
1070  FILE* fp = base::OpenFile(database_filename_, "r+");
1071  ASSERT_TRUE(fp);
1072  ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1073  for (size_t i = 0; i < 8; ++i) {
1074    fputc('!', fp);
1075  }
1076  fclose(fp);
1077
1078  {
1079    // The following code will cause DCHECKs, so suppress the crashes.
1080    ScopedLogMessageIgnorer ignorer;
1081
1082    // Start an update.  The insert will fail due to corruption.
1083    ASSERT_TRUE(database_->UpdateStarted(&lists));
1084    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1085    database_->UpdateFinished(true);
1086
1087    // Database file still exists until the corruption handler has run.
1088    EXPECT_TRUE(base::PathExists(database_filename_));
1089
1090    // Flush through the corruption-handler task.
1091    VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1092    base::MessageLoop::current()->RunUntilIdle();
1093  }
1094
1095  // Database file should not exist.
1096  EXPECT_FALSE(base::PathExists(database_filename_));
1097
1098  // Run the update again successfully.
1099  ASSERT_TRUE(database_->UpdateStarted(&lists));
1100  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1101  database_->UpdateFinished(true);
1102  EXPECT_TRUE(base::PathExists(database_filename_));
1103
1104  database_.reset();
1105}
1106
1107// Checks database reading and writing.
1108TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1109  database_.reset();
1110  base::MessageLoop loop;
1111  SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1112  SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
1113  SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1114  database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1115                                              download_store,
1116                                              csd_whitelist_store,
1117                                              NULL,
1118                                              NULL,
1119                                              NULL,
1120                                              NULL));
1121  database_->Init(database_filename_);
1122
1123  const char kEvil1Url1[] = "www.evil1.com/download1/";
1124  const char kEvil1Url2[] = "www.evil1.com/download2.html";
1125
1126  // Add a simple chunk with one hostkey for download url list.
1127  ScopedVector<SBChunkData> chunks;
1128  chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1129
1130  std::vector<SBListChunkRanges> lists;
1131  ASSERT_TRUE(database_->UpdateStarted(&lists));
1132  database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1133  database_->UpdateFinished(true);
1134
1135  std::vector<SBPrefix> prefix_hits;
1136  std::vector<GURL> urls(1);
1137
1138  urls[0] = GURL(std::string("http://") + kEvil1Url1);
1139  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1140  ASSERT_EQ(1U, prefix_hits.size());
1141  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1142
1143  urls[0] = GURL(std::string("http://") + kEvil1Url2);
1144  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1145  ASSERT_EQ(1U, prefix_hits.size());
1146  EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1147
1148  urls[0] = GURL(std::string("https://") + kEvil1Url2);
1149  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1150  ASSERT_EQ(1U, prefix_hits.size());
1151  EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1152
1153  urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1154  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1155  ASSERT_EQ(1U, prefix_hits.size());
1156  EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1157
1158  urls[0] = GURL("http://www.randomevil.com");
1159  EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1160
1161  // Should match with query args stripped.
1162  urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1163  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1164  ASSERT_EQ(1U, prefix_hits.size());
1165  EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1166
1167  // Should match with extra path stuff and query args stripped.
1168  urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1169  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1170  ASSERT_EQ(1U, prefix_hits.size());
1171  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1172
1173  // First hit in redirect chain is malware.
1174  urls.clear();
1175  urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1176  urls.push_back(GURL("http://www.randomevil.com"));
1177  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1178  ASSERT_EQ(1U, prefix_hits.size());
1179  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1180
1181  // Middle hit in redirect chain is malware.
1182  urls.clear();
1183  urls.push_back(GURL("http://www.randomevil.com"));
1184  urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1185  urls.push_back(GURL("http://www.randomevil2.com"));
1186  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1187  ASSERT_EQ(1U, prefix_hits.size());
1188  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1189
1190  // Final hit in redirect chain is malware.
1191  urls.clear();
1192  urls.push_back(GURL("http://www.randomevil.com"));
1193  urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1194  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1195  ASSERT_EQ(1U, prefix_hits.size());
1196  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1197
1198  // Multiple hits in redirect chain are in malware list.
1199  urls.clear();
1200  urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1201  urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1202  EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1203  ASSERT_EQ(2U, prefix_hits.size());
1204  EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1205  EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1206  database_.reset();
1207}
1208
1209// Checks that the whitelists are handled properly.
1210TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1211  database_.reset();
1212
1213  // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1214  // from the IO thread.  In general the whitelist lookups are thread-safe.
1215  content::TestBrowserThreadBundle thread_bundle_;
1216
1217  // If the whitelist is disabled everything should match the whitelist.
1218  database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1219                                              NULL, NULL, NULL, NULL, NULL,
1220                                              NULL));
1221  database_->Init(database_filename_);
1222  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1223      GURL(std::string("http://www.phishing.com/"))));
1224  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1225      GURL(std::string("http://www.phishing.com/"))));
1226  EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1227
1228  SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1229  SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1230  SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
1231  SafeBrowsingStoreFile* extension_blacklist_store =
1232      new SafeBrowsingStoreFile();
1233  database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
1234                                              csd_whitelist_store,
1235                                              download_whitelist_store,
1236                                              extension_blacklist_store,
1237                                              NULL, NULL));
1238  database_->Init(database_filename_);
1239
1240  const char kGood1Host[] = "www.good1.com/";
1241  const char kGood1Url1[] = "www.good1.com/a/b.html";
1242  const char kGood1Url2[] = "www.good1.com/b/";
1243
1244  const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1245
1246  // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1247  const char kGood3Url1[] = "good3.com/";
1248
1249  const char kGoodString[] = "good_string";
1250
1251  ScopedVector<SBChunkData> csd_chunks;
1252  ScopedVector<SBChunkData> download_chunks;
1253
1254  // Add two simple chunks to the csd whitelist.
1255  csd_chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1256  csd_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1257  download_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1258  download_chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1259  download_chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1260
1261  std::vector<SBListChunkRanges> lists;
1262  ASSERT_TRUE(database_->UpdateStarted(&lists));
1263  database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1264  database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1265                          download_chunks.get());
1266  database_->UpdateFinished(true);
1267
1268  EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1269      GURL(std::string("http://") + kGood1Host)));
1270
1271  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1272      GURL(std::string("http://") + kGood1Url1)));
1273  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1274      GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1275
1276  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1277      GURL(std::string("http://") + kGood1Url2)));
1278  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1279      GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1280
1281  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1282      GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1283
1284  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1285      GURL(std::string("http://") + kGood2Url1 + "/c")));
1286  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1287      GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1288  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1289      GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1290
1291  EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1292      GURL(std::string("http://www.google.com/"))));
1293
1294  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1295      GURL(std::string("http://") + kGood2Url1 + "/c")));
1296  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1297      GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1298  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1299      GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1300
1301  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1302      GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1303  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1304      GURL(std::string("http://a.b.good3.com/"))));
1305
1306  EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1307  EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1308
1309  EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1310      GURL(std::string("http://www.google.com/"))));
1311
1312  // The CSD whitelist killswitch is not present.
1313  EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1314
1315  // Test only add the malware IP killswitch
1316  csd_chunks.clear();
1317  csd_chunks.push_back(AddChunkFullHashValue(
1318      15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1319
1320  ASSERT_TRUE(database_->UpdateStarted(&lists));
1321  database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1322  database_->UpdateFinished(true);
1323
1324  EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1325  // The CSD whitelist killswitch is not present.
1326  EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1327
1328  // Test that the kill-switch works as intended.
1329  csd_chunks.clear();
1330  download_chunks.clear();
1331  lists.clear();
1332  csd_chunks.push_back(AddChunkFullHashValue(
1333      5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1334  download_chunks.push_back(AddChunkFullHashValue(
1335      5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1336
1337  ASSERT_TRUE(database_->UpdateStarted(&lists));
1338  database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1339  database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1340                          download_chunks.get());
1341  database_->UpdateFinished(true);
1342
1343  // The CSD whitelist killswitch is present.
1344  EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1345  EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1346  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1347      GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1348  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1349      GURL(std::string("http://www.google.com/"))));
1350  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1351      GURL(std::string("http://www.phishing_url.com/"))));
1352
1353  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1354      GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1355  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1356      GURL(std::string("http://www.google.com/"))));
1357  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1358      GURL(std::string("http://www.phishing_url.com/"))));
1359
1360  EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1361  EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1362
1363  // Remove the kill-switch and verify that we can recover.
1364  csd_chunks.clear();
1365  download_chunks.clear();
1366  lists.clear();
1367  csd_chunks.push_back(SubChunkFullHashValue(
1368      1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1369  csd_chunks.push_back(SubChunkFullHashValue(
1370      10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1371  download_chunks.push_back(SubChunkFullHashValue(
1372      1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1373
1374  ASSERT_TRUE(database_->UpdateStarted(&lists));
1375  database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1376  database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1377                          download_chunks.get());
1378  database_->UpdateFinished(true);
1379
1380  EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1381  EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1382  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1383      GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1384  EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1385      GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1386  EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1387      GURL(std::string("http://www.google.com/"))));
1388  EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1389      GURL(std::string("http://www.phishing_url.com/"))));
1390
1391  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1392      GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1393  EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1394      GURL(std::string("https://good3.com/"))));
1395  EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1396  EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1397      GURL(std::string("http://www.google.com/"))));
1398  EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1399      GURL(std::string("http://www.phishing_url.com/"))));
1400  EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1401
1402  database_.reset();
1403}
1404
1405// Test to make sure we could insert chunk list that
1406// contains entries for the same host.
1407TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1408  ScopedVector<SBChunkData> chunks;
1409
1410  // Add a malware add chunk with two entries of the same host.
1411  chunks.push_back(AddChunkPrefix2Value(1,
1412                                        "www.evil.com/malware1.html",
1413                                        "www.evil.com/malware2.html"));
1414
1415  // Insert the testing chunks into database.
1416  std::vector<SBListChunkRanges> lists;
1417  ASSERT_TRUE(database_->UpdateStarted(&lists));
1418  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1419  database_->UpdateFinished(true);
1420
1421  GetListsInfo(&lists);
1422  ASSERT_LE(1U, lists.size());
1423  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1424  EXPECT_EQ("1", lists[0].adds);
1425  EXPECT_TRUE(lists[0].subs.empty());
1426
1427  // Add a phishing add chunk with two entries of the same host.
1428  chunks.clear();
1429  chunks.push_back(AddChunkPrefix2Value(47,
1430                                        "www.evil.com/phishing1.html",
1431                                        "www.evil.com/phishing2.html"));
1432
1433  ASSERT_TRUE(database_->UpdateStarted(&lists));
1434  database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1435  database_->UpdateFinished(true);
1436
1437  GetListsInfo(&lists);
1438  ASSERT_EQ(2U, lists.size());
1439  EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1440  EXPECT_EQ("1", lists[0].adds);
1441  EXPECT_TRUE(lists[0].subs.empty());
1442  EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1443  EXPECT_EQ("47", lists[1].adds);
1444  EXPECT_TRUE(lists[1].subs.empty());
1445
1446  std::vector<SBPrefix> prefix_hits;
1447  std::vector<SBFullHashResult> cache_hits;
1448
1449  EXPECT_TRUE(database_->ContainsBrowseUrl(
1450      GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1451  EXPECT_TRUE(database_->ContainsBrowseUrl(
1452      GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1453  EXPECT_TRUE(database_->ContainsBrowseUrl(
1454      GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1455  EXPECT_TRUE(database_->ContainsBrowseUrl(
1456      GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1457
1458  // Test removing a single prefix from the add chunk.
1459  // Remove the prefix that added first.
1460  chunks.clear();
1461  chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1462  ASSERT_TRUE(database_->UpdateStarted(&lists));
1463  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1464  database_->UpdateFinished(true);
1465
1466  // Remove the prefix that added last.
1467  chunks.clear();
1468  chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1469  ASSERT_TRUE(database_->UpdateStarted(&lists));
1470  database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1471  database_->UpdateFinished(true);
1472
1473  // Verify that the database contains urls expected.
1474  EXPECT_FALSE(database_->ContainsBrowseUrl(
1475      GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1476  EXPECT_TRUE(database_->ContainsBrowseUrl(
1477      GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1478  EXPECT_TRUE(database_->ContainsBrowseUrl(
1479      GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1480  EXPECT_FALSE(database_->ContainsBrowseUrl(
1481      GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1482}
1483
1484// Test that an empty update doesn't actually update the database.
1485// This isn't a functionality requirement, but it is a useful
1486// optimization.
1487TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1488  ScopedVector<SBChunkData> chunks;
1489
1490  base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1491
1492  // Prime the database.
1493  std::vector<SBListChunkRanges> lists;
1494  ASSERT_TRUE(database_->UpdateStarted(&lists));
1495  chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1496  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1497  database_->UpdateFinished(true);
1498
1499  // Get an older time to reset the lastmod time for detecting whether
1500  // the file has been updated.
1501  base::File::Info before_info, after_info;
1502  ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1503  const Time old_last_modified =
1504      before_info.last_modified - TimeDelta::FromSeconds(10);
1505
1506  // Inserting another chunk updates the database file.  The sleep is
1507  // needed because otherwise the entire test can finish w/in the
1508  // resolution of the lastmod time.
1509  ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1510  ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1511  ASSERT_TRUE(database_->UpdateStarted(&lists));
1512  chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1513  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1514  database_->UpdateFinished(true);
1515  ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1516  EXPECT_LT(before_info.last_modified, after_info.last_modified);
1517
1518  // Deleting a chunk updates the database file.
1519  ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1520  ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1521  ASSERT_TRUE(database_->UpdateStarted(&lists));
1522  AddDelChunk(safe_browsing_util::kMalwareList, 2);
1523  database_->UpdateFinished(true);
1524  ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1525  EXPECT_LT(before_info.last_modified, after_info.last_modified);
1526
1527  // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1528  // update the database file.
1529  ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1530  ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1531  ASSERT_TRUE(database_->UpdateStarted(&lists));
1532  database_->UpdateFinished(true);
1533  ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1534  EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1535}
1536
1537// Test that a filter file is written out during update and read back
1538// in during setup.
1539TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1540  // Create a database with trivial example data and write it out.
1541  {
1542    // Prime the database.
1543    std::vector<SBListChunkRanges> lists;
1544    ASSERT_TRUE(database_->UpdateStarted(&lists));
1545
1546    ScopedVector<SBChunkData> chunks;
1547    chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1548    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1549    database_->UpdateFinished(true);
1550  }
1551
1552  // Find the malware url in the database, don't find a good url.
1553  std::vector<SBPrefix> prefix_hits;
1554  std::vector<SBFullHashResult> cache_hits;
1555  EXPECT_TRUE(database_->ContainsBrowseUrl(
1556      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1557  EXPECT_FALSE(database_->ContainsBrowseUrl(
1558      GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1559
1560  base::FilePath filter_file = database_->PrefixSetForFilename(
1561      database_->BrowseDBFilename(database_filename_));
1562
1563  // After re-creating the database, it should have a filter read from
1564  // a file, so it should find the same results.
1565  ASSERT_TRUE(base::PathExists(filter_file));
1566  database_.reset(new SafeBrowsingDatabaseNew);
1567  database_->Init(database_filename_);
1568  EXPECT_TRUE(database_->ContainsBrowseUrl(
1569      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1570  EXPECT_FALSE(database_->ContainsBrowseUrl(
1571      GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1572
1573  // If there is no filter file, the database cannot find malware urls.
1574  base::DeleteFile(filter_file, false);
1575  ASSERT_FALSE(base::PathExists(filter_file));
1576  database_.reset(new SafeBrowsingDatabaseNew);
1577  database_->Init(database_filename_);
1578  EXPECT_FALSE(database_->ContainsBrowseUrl(
1579      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1580  EXPECT_FALSE(database_->ContainsBrowseUrl(
1581      GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1582}
1583
1584TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1585  const SBPrefix kPrefix1 = 1001U;
1586  const SBFullHash kFullHash1_1 =
1587      SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1588
1589  const SBPrefix kPrefix2 = 1002U;
1590  const SBFullHash kFullHash2_1 =
1591      SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1592
1593  // Insert prefix kPrefix1 and kPrefix2 into database.
1594  ScopedVector<SBChunkData> chunks;
1595  chunks.push_back(AddChunkPrefix(1, kPrefix1));
1596  chunks.push_back(AddChunkPrefix(2, kPrefix2));
1597
1598  std::vector<SBListChunkRanges> lists;
1599  ASSERT_TRUE(database_->UpdateStarted(&lists));
1600  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1601  database_->UpdateFinished(true);
1602
1603  {
1604    // Cache a full miss result for kPrefix1.
1605    std::vector<SBPrefix> prefixes(1, kPrefix1);
1606    std::vector<SBFullHashResult> cache_results;
1607    database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1608  }
1609
1610  {
1611    // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1612    // not have a cache hit.
1613    std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1614    std::vector<SBPrefix> prefix_hits;
1615    std::vector<SBFullHashResult> cache_hits;
1616    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1617        full_hashes, &prefix_hits, &cache_hits));
1618
1619    // kFullHash2_1 gets a hit from the prefix in the database.
1620    full_hashes.push_back(kFullHash2_1);
1621    prefix_hits.clear();
1622    cache_hits.clear();
1623    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1624        full_hashes, &prefix_hits, &cache_hits));
1625    ASSERT_EQ(1U, prefix_hits.size());
1626    EXPECT_EQ(kPrefix2, prefix_hits[0]);
1627    EXPECT_TRUE(cache_hits.empty());
1628  }
1629}
1630
1631TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1632  const SBPrefix kPrefix1 = 1001U;
1633  const SBFullHash kFullHash1_1 =
1634      SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1635  const SBFullHash kFullHash1_2 =
1636      SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1637  const SBFullHash kFullHash1_3 =
1638      SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1639
1640  const SBPrefix kPrefix2 = 1002U;
1641  const SBFullHash kFullHash2_1 =
1642      SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1643
1644  const SBPrefix kPrefix3 = 1003U;
1645  const SBFullHash kFullHash3_1 =
1646      SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1647
1648  // Insert prefix kPrefix1 and kPrefix2 into database.
1649  ScopedVector<SBChunkData> chunks;
1650  chunks.push_back(AddChunkPrefix(1, kPrefix1));
1651  chunks.push_back(AddChunkPrefix(2, kPrefix2));
1652
1653  std::vector<SBListChunkRanges> lists;
1654  ASSERT_TRUE(database_->UpdateStarted(&lists));
1655  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1656  database_->UpdateFinished(true);
1657
1658  {
1659    // kFullHash1_1 has a prefix hit of kPrefix1.
1660    std::vector<SBFullHash> full_hashes;
1661    full_hashes.push_back(kFullHash1_1);
1662    std::vector<SBPrefix> prefix_hits;
1663    std::vector<SBFullHashResult> cache_hits;
1664    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1665        full_hashes, &prefix_hits, &cache_hits));
1666    ASSERT_EQ(1U, prefix_hits.size());
1667    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1668    EXPECT_TRUE(cache_hits.empty());
1669
1670    // kFullHash2_1 has a prefix hit of kPrefix2.
1671    full_hashes.push_back(kFullHash2_1);
1672    prefix_hits.clear();
1673    cache_hits.clear();
1674    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1675        full_hashes, &prefix_hits, &cache_hits));
1676    ASSERT_EQ(2U, prefix_hits.size());
1677    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1678    EXPECT_EQ(kPrefix2, prefix_hits[1]);
1679    EXPECT_TRUE(cache_hits.empty());
1680
1681    // kFullHash3_1 has no hits.
1682    full_hashes.push_back(kFullHash3_1);
1683    prefix_hits.clear();
1684    cache_hits.clear();
1685    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1686        full_hashes, &prefix_hits, &cache_hits));
1687    ASSERT_EQ(2U, prefix_hits.size());
1688    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1689    EXPECT_EQ(kPrefix2, prefix_hits[1]);
1690    EXPECT_TRUE(cache_hits.empty());
1691  }
1692
1693  {
1694    // Cache a fullhash result for two kPrefix1 full hashes.
1695    std::vector<SBPrefix> prefixes(1, kPrefix1);
1696    std::vector<SBFullHashResult> cache_results;
1697
1698    SBFullHashResult full_hash_result;
1699    full_hash_result.list_id = safe_browsing_util::MALWARE;
1700
1701    full_hash_result.hash = kFullHash1_1;
1702    cache_results.push_back(full_hash_result);
1703
1704    full_hash_result.hash = kFullHash1_3;
1705    cache_results.push_back(full_hash_result);
1706
1707    database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1708  }
1709
1710  {
1711    // kFullHash1_1 should now see a cache hit.
1712    std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1713    std::vector<SBPrefix> prefix_hits;
1714    std::vector<SBFullHashResult> cache_hits;
1715    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1716        full_hashes, &prefix_hits, &cache_hits));
1717    EXPECT_TRUE(prefix_hits.empty());
1718    ASSERT_EQ(1U, cache_hits.size());
1719    EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1720
1721    // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1722    // for kPrefix2.
1723    full_hashes.push_back(kFullHash2_1);
1724    prefix_hits.clear();
1725    cache_hits.clear();
1726    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1727        full_hashes, &prefix_hits, &cache_hits));
1728    ASSERT_EQ(1U, prefix_hits.size());
1729    EXPECT_EQ(kPrefix2, prefix_hits[0]);
1730    ASSERT_EQ(1U, cache_hits.size());
1731    EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1732
1733    // kFullHash1_3 also gets a cache hit.
1734    full_hashes.push_back(kFullHash1_3);
1735    prefix_hits.clear();
1736    cache_hits.clear();
1737    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1738        full_hashes, &prefix_hits, &cache_hits));
1739    ASSERT_EQ(1U, prefix_hits.size());
1740    EXPECT_EQ(kPrefix2, prefix_hits[0]);
1741    ASSERT_EQ(2U, cache_hits.size());
1742    EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1743    EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1744  }
1745
1746  {
1747    // Check if DB contains only kFullHash1_3. Should return a cache hit.
1748    std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1749    std::vector<SBPrefix> prefix_hits;
1750    std::vector<SBFullHashResult> cache_hits;
1751    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1752        full_hashes, &prefix_hits, &cache_hits));
1753    EXPECT_TRUE(prefix_hits.empty());
1754    ASSERT_EQ(1U, cache_hits.size());
1755    EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1756  }
1757
1758  {
1759    // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1760    // kPrefix1.
1761    std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1762    std::vector<SBPrefix> prefix_hits;
1763    std::vector<SBFullHashResult> cache_hits;
1764    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1765        full_hashes, &prefix_hits, &cache_hits));
1766
1767    // Other prefix hits possible when kFullHash1_2 hits nothing.
1768    full_hashes.push_back(kFullHash2_1);
1769    prefix_hits.clear();
1770    cache_hits.clear();
1771    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1772        full_hashes, &prefix_hits, &cache_hits));
1773    ASSERT_EQ(1U, prefix_hits.size());
1774    EXPECT_EQ(kPrefix2, prefix_hits[0]);
1775    EXPECT_TRUE(cache_hits.empty());
1776  }
1777}
1778
1779TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1780  const SBPrefix kPrefix1 = 1001U;
1781  const SBFullHash kFullHash1_1 =
1782      SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1783  const SBFullHash kFullHash1_2 =
1784      SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1785  const SBFullHash kFullHash1_3 =
1786      SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1787
1788  // Insert two full hashes with a shared prefix.
1789  ScopedVector<SBChunkData> chunks;
1790  chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1791  chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1792
1793  std::vector<SBListChunkRanges> lists;
1794  ASSERT_TRUE(database_->UpdateStarted(&lists));
1795  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1796  database_->UpdateFinished(true);
1797
1798  {
1799    // Check a full hash which isn't present.
1800    std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1801    std::vector<SBPrefix> prefix_hits;
1802    std::vector<SBFullHashResult> cache_hits;
1803    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1804        full_hashes, &prefix_hits, &cache_hits));
1805
1806    // Also one which is present, should have a prefix hit.
1807    full_hashes.push_back(kFullHash1_1);
1808    prefix_hits.clear();
1809    cache_hits.clear();
1810    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1811        full_hashes, &prefix_hits, &cache_hits));
1812    ASSERT_EQ(1U, prefix_hits.size());
1813    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1814    EXPECT_TRUE(cache_hits.empty());
1815
1816    // Two full hash matches with the same prefix should return one prefix hit.
1817    full_hashes.push_back(kFullHash1_2);
1818    prefix_hits.clear();
1819    cache_hits.clear();
1820    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1821        full_hashes, &prefix_hits, &cache_hits));
1822    ASSERT_EQ(1U, prefix_hits.size());
1823    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1824    EXPECT_TRUE(cache_hits.empty());
1825  }
1826
1827  {
1828    // Cache a gethash result for kFullHash1_2.
1829    SBFullHashResult full_hash_result;
1830    full_hash_result.list_id = safe_browsing_util::MALWARE;
1831    full_hash_result.hash = kFullHash1_2;
1832
1833    std::vector<SBPrefix> prefixes(1, kPrefix1);
1834    std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1835
1836    database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1837  }
1838
1839  {
1840    // kFullHash1_3 should still return false, because the cached
1841    // result for kPrefix1 doesn't contain kFullHash1_3.
1842    std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1843    std::vector<SBPrefix> prefix_hits;
1844    std::vector<SBFullHashResult> cache_hits;
1845    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1846        full_hashes, &prefix_hits, &cache_hits));
1847
1848    // kFullHash1_1 is also not in the cached result, which takes
1849    // priority over the database.
1850    prefix_hits.clear();
1851    full_hashes.push_back(kFullHash1_1);
1852    cache_hits.clear();
1853    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1854        full_hashes, &prefix_hits, &cache_hits));
1855
1856    // kFullHash1_2 is in the cached result.
1857    full_hashes.push_back(kFullHash1_2);
1858    prefix_hits.clear();
1859    cache_hits.clear();
1860    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1861        full_hashes, &prefix_hits, &cache_hits));
1862    EXPECT_TRUE(prefix_hits.empty());
1863    ASSERT_EQ(1U, cache_hits.size());
1864    EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1865  }
1866
1867  // Remove kFullHash1_1 from the database.
1868  chunks.clear();
1869  chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1870
1871  ASSERT_TRUE(database_->UpdateStarted(&lists));
1872  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1873  database_->UpdateFinished(true);
1874
1875  // Cache should be cleared after updating.
1876  EXPECT_TRUE(database_->browse_gethash_cache_.empty());
1877
1878  {
1879    // Now the database doesn't contain kFullHash1_1.
1880    std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1881    std::vector<SBPrefix> prefix_hits;
1882    std::vector<SBFullHashResult> cache_hits;
1883    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1884        full_hashes, &prefix_hits, &cache_hits));
1885
1886    // Nor kFullHash1_3.
1887    full_hashes.push_back(kFullHash1_3);
1888    prefix_hits.clear();
1889    cache_hits.clear();
1890    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1891        full_hashes, &prefix_hits, &cache_hits));
1892
1893    // Still has kFullHash1_2.
1894    full_hashes.push_back(kFullHash1_2);
1895    prefix_hits.clear();
1896    cache_hits.clear();
1897    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1898        full_hashes, &prefix_hits, &cache_hits));
1899    ASSERT_EQ(1U, prefix_hits.size());
1900    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1901    EXPECT_TRUE(cache_hits.empty());
1902  }
1903
1904  // Remove kFullHash1_2 from the database.
1905  chunks.clear();
1906  chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
1907
1908  ASSERT_TRUE(database_->UpdateStarted(&lists));
1909  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1910  database_->UpdateFinished(true);
1911
1912  // Cache should be cleared after updating.
1913  EXPECT_TRUE(database_->browse_gethash_cache_.empty());
1914
1915  {
1916    // None are present.
1917    std::vector<SBFullHash> full_hashes;
1918    std::vector<SBPrefix> prefix_hits;
1919    std::vector<SBFullHashResult> cache_hits;
1920    full_hashes.push_back(kFullHash1_1);
1921    full_hashes.push_back(kFullHash1_2);
1922    full_hashes.push_back(kFullHash1_3);
1923    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1924        full_hashes, &prefix_hits, &cache_hits));
1925  }
1926}
1927
1928TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
1929  const SBPrefix kPrefix1 = 1001U;
1930  const SBFullHash kFullHash1_1 =
1931      SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1932  const SBFullHash kFullHash1_2 =
1933      SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1934
1935  ScopedVector<SBChunkData> chunks;
1936  chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1937
1938  std::vector<SBListChunkRanges> lists;
1939  ASSERT_TRUE(database_->UpdateStarted(&lists));
1940  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1941  database_->UpdateFinished(true);
1942
1943  {
1944    // kFullHash1_2 does not match kFullHash1_1.
1945    std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1946    std::vector<SBPrefix> prefix_hits;
1947    std::vector<SBFullHashResult> cache_hits;
1948    EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1949        full_hashes, &prefix_hits, &cache_hits));
1950  }
1951
1952  // Add a prefix match.
1953  chunks.clear();
1954  chunks.push_back(AddChunkPrefix(2, kPrefix1));
1955
1956  ASSERT_TRUE(database_->UpdateStarted(&lists));
1957  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1958  database_->UpdateFinished(true);
1959
1960  {
1961    // kFullHash1_2 does match kPrefix1.
1962    std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1963    std::vector<SBPrefix> prefix_hits;
1964    std::vector<SBFullHashResult> cache_hits;
1965    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1966        full_hashes, &prefix_hits, &cache_hits));
1967    ASSERT_EQ(1U, prefix_hits.size());
1968    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1969    EXPECT_TRUE(cache_hits.empty());
1970  }
1971
1972  // Remove the full hash.
1973  chunks.clear();
1974  chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1975
1976  ASSERT_TRUE(database_->UpdateStarted(&lists));
1977  database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1978  database_->UpdateFinished(true);
1979
1980  {
1981    // kFullHash1_2 still returns true due to the prefix hit.
1982    std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1983    std::vector<SBPrefix> prefix_hits;
1984    std::vector<SBFullHashResult> cache_hits;
1985    EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1986        full_hashes, &prefix_hits, &cache_hits));
1987    ASSERT_EQ(1U, prefix_hits.size());
1988    EXPECT_EQ(kPrefix1, prefix_hits[0]);
1989    EXPECT_TRUE(cache_hits.empty());
1990  }
1991}
1992
1993TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
1994  database_.reset();
1995  SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1996  SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
1997  database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1998                                              NULL,
1999                                              NULL,
2000                                              NULL,
2001                                              NULL,
2002                                              NULL,
2003                                              ip_blacklist_store));
2004  database_->Init(database_filename_);
2005  std::vector<SBListChunkRanges> lists;
2006  ASSERT_TRUE(database_->UpdateStarted(&lists));
2007
2008  ScopedVector<SBChunkData> chunks;
2009
2010  // IPv4 prefix match for ::ffff:192.168.1.0/120.
2011  chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2012
2013  // IPv4 exact match for ::ffff:192.1.1.1.
2014  chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2015
2016  // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2017  chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2018
2019  // IPv6 prefix match for: 2620:0:1000:3103::/64.
2020  chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2021
2022  // IPv4 prefix match for ::ffff:192.1.122.0/119.
2023  chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2024
2025  // IPv4 prefix match for ::ffff:192.1.128.0/113.
2026  chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2027
2028  database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2029  database_->UpdateFinished(true);
2030
2031  EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2032  EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2033  EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2034  EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2035  EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2036  EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2037
2038  EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2039  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2040  EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2041
2042  EXPECT_FALSE(database_->ContainsMalwareIP(
2043      "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2044  EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2045  EXPECT_TRUE(database_->ContainsMalwareIP(
2046      "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2047  EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2048
2049  EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2050  EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2051  EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2052
2053  EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2054  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2055  EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2056  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2057  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2058  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2059  EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2060
2061  EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2062  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2063  EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2064  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2065  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2066  EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2067  EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2068}
2069
2070TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2071  std::vector<SBListChunkRanges> lists;
2072  ASSERT_TRUE(database_->UpdateStarted(&lists));
2073
2074  // Add a host-level hit.
2075  {
2076    ScopedVector<SBChunkData> chunks;
2077    chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2078    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2079  }
2080
2081  // Add a specific fullhash.
2082  static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2083  {
2084    ScopedVector<SBChunkData> chunks;
2085    chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2086    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2087  }
2088
2089  // Add a fullhash which has a prefix collision for a known url.
2090  static const char kExampleFine[] = "www.example.com/fine.html";
2091  static const char kExampleCollision[] =
2092      "www.example.com/3123364814/malware.htm";
2093  ASSERT_EQ(SBPrefixForString(kExampleFine),
2094            SBPrefixForString(kExampleCollision));
2095  {
2096    ScopedVector<SBChunkData> chunks;
2097    chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2098    database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2099  }
2100
2101  database_->UpdateFinished(true);
2102
2103  std::vector<SBPrefix> prefix_hits;
2104  std::vector<SBFullHashResult> cache_hits;
2105
2106  // Anything will hit the host prefix.
2107  EXPECT_TRUE(database_->ContainsBrowseUrl(
2108      GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2109  ASSERT_EQ(1U, prefix_hits.size());
2110  EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2111  EXPECT_TRUE(cache_hits.empty());
2112
2113  // Hit the specific URL prefix.
2114  EXPECT_TRUE(database_->ContainsBrowseUrl(
2115      GURL(std::string("http://") + kWhateverMalware),
2116      &prefix_hits, &cache_hits));
2117  ASSERT_EQ(1U, prefix_hits.size());
2118  EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2119  EXPECT_TRUE(cache_hits.empty());
2120
2121  // Other URLs at that host are fine.
2122  EXPECT_FALSE(database_->ContainsBrowseUrl(
2123      GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2124  EXPECT_TRUE(prefix_hits.empty());
2125  EXPECT_TRUE(cache_hits.empty());
2126
2127  // Hit the specific URL full hash.
2128  EXPECT_TRUE(database_->ContainsBrowseUrl(
2129      GURL(std::string("http://") + kExampleCollision),
2130      &prefix_hits, &cache_hits));
2131  ASSERT_EQ(1U, prefix_hits.size());
2132  EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2133  EXPECT_TRUE(cache_hits.empty());
2134
2135  // This prefix collides, but no full hash match.
2136  EXPECT_FALSE(database_->ContainsBrowseUrl(
2137      GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2138}
2139