1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/safe_browsing/database_manager.h"
6
7#include <algorithm>
8
9#include "base/bind.h"
10#include "base/bind_helpers.h"
11#include "base/callback.h"
12#include "base/command_line.h"
13#include "base/debug/leak_tracker.h"
14#include "base/path_service.h"
15#include "base/stl_util.h"
16#include "base/strings/string_util.h"
17#include "base/threading/thread.h"
18#include "base/threading/thread_restrictions.h"
19#include "chrome/browser/browser_process.h"
20#include "chrome/browser/chrome_notification_types.h"
21#include "chrome/browser/prerender/prerender_field_trial.h"
22#include "chrome/browser/safe_browsing/client_side_detection_service.h"
23#include "chrome/browser/safe_browsing/download_protection_service.h"
24#include "chrome/browser/safe_browsing/malware_details.h"
25#include "chrome/browser/safe_browsing/protocol_manager.h"
26#include "chrome/browser/safe_browsing/safe_browsing_database.h"
27#include "chrome/browser/safe_browsing/safe_browsing_service.h"
28#include "chrome/browser/safe_browsing/ui_manager.h"
29#include "chrome/common/chrome_constants.h"
30#include "chrome/common/chrome_paths.h"
31#include "chrome/common/chrome_switches.h"
32#include "components/metrics/metrics_service.h"
33#include "components/startup_metric_utils/startup_metric_utils.h"
34#include "content/public/browser/browser_thread.h"
35#include "content/public/browser/notification_service.h"
36#include "url/url_constants.h"
37
38using content::BrowserThread;
39
40namespace {
41
42// Timeout for match checks, e.g. download URLs, hashes.
43const int kCheckTimeoutMs = 10000;
44
45// Records disposition information about the check.  |hit| should be
46// |true| if there were any prefix hits in |full_hashes|.
47void RecordGetHashCheckStatus(
48    bool hit,
49    safe_browsing_util::ListType check_type,
50    const std::vector<SBFullHashResult>& full_hashes) {
51  SafeBrowsingProtocolManager::ResultType result;
52  if (full_hashes.empty()) {
53    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
54  } else if (hit) {
55    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
56  } else {
57    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
58  }
59  bool is_download = check_type == safe_browsing_util::BINURL;
60  SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
61}
62
63bool IsExpectedThreat(
64    const SBThreatType threat_type,
65    const std::vector<SBThreatType>& expected_threats) {
66  return expected_threats.end() != std::find(expected_threats.begin(),
67                                             expected_threats.end(),
68                                             threat_type);
69}
70
71// Return the list id from the first result in |full_hashes| which matches
72// |hash|, or INVALID if none match.
73safe_browsing_util::ListType GetHashThreatListType(
74    const SBFullHash& hash,
75    const std::vector<SBFullHashResult>& full_hashes,
76    size_t* index) {
77  for (size_t i = 0; i < full_hashes.size(); ++i) {
78    if (SBFullHashEqual(hash, full_hashes[i].hash)) {
79      if (index)
80        *index = i;
81      return static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
82    }
83  }
84  return safe_browsing_util::INVALID;
85}
86
87// Given a URL, compare all the possible host + path full hashes to the set of
88// provided full hashes.  Returns the list id of the a matching result from
89// |full_hashes|, or INVALID if none match.
90safe_browsing_util::ListType GetUrlThreatListType(
91    const GURL& url,
92    const std::vector<SBFullHashResult>& full_hashes,
93    size_t* index) {
94  if (full_hashes.empty())
95    return safe_browsing_util::INVALID;
96
97  std::vector<std::string> patterns;
98  safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
99
100  for (size_t i = 0; i < patterns.size(); ++i) {
101    safe_browsing_util::ListType threat = GetHashThreatListType(
102        SBFullHashForString(patterns[i]), full_hashes, index);
103    if (threat != safe_browsing_util::INVALID)
104      return threat;
105  }
106  return safe_browsing_util::INVALID;
107}
108
109SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
110  switch (list_type) {
111    case safe_browsing_util::PHISH:
112      return SB_THREAT_TYPE_URL_PHISHING;
113    case safe_browsing_util::MALWARE:
114      return SB_THREAT_TYPE_URL_MALWARE;
115    case safe_browsing_util::BINURL:
116      return SB_THREAT_TYPE_BINARY_MALWARE_URL;
117    case safe_browsing_util::EXTENSIONBLACKLIST:
118      return SB_THREAT_TYPE_EXTENSION;
119    default:
120      DVLOG(1) << "Unknown safe browsing list id " << list_type;
121      return SB_THREAT_TYPE_SAFE;
122  }
123}
124
125}  // namespace
126
127// static
128SBThreatType SafeBrowsingDatabaseManager::GetHashThreatType(
129    const SBFullHash& hash,
130    const std::vector<SBFullHashResult>& full_hashes) {
131  return GetThreatTypeFromListType(
132      GetHashThreatListType(hash, full_hashes, NULL));
133}
134
135// static
136SBThreatType SafeBrowsingDatabaseManager::GetUrlThreatType(
137    const GURL& url,
138    const std::vector<SBFullHashResult>& full_hashes,
139    size_t* index) {
140  return GetThreatTypeFromListType(
141      GetUrlThreatListType(url, full_hashes, index));
142}
143
144SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
145    const std::vector<GURL>& urls,
146    const std::vector<SBFullHash>& full_hashes,
147    Client* client,
148    safe_browsing_util::ListType check_type,
149    const std::vector<SBThreatType>& expected_threats)
150    : urls(urls),
151      url_results(urls.size(), SB_THREAT_TYPE_SAFE),
152      url_metadata(urls.size()),
153      full_hashes(full_hashes),
154      full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
155      client(client),
156      need_get_hash(false),
157      check_type(check_type),
158      expected_threats(expected_threats) {
159  DCHECK_EQ(urls.empty(), !full_hashes.empty())
160      << "Exactly one of urls and full_hashes must be set";
161}
162
163SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
164
165void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
166    const SafeBrowsingCheck& check) {
167  DCHECK_EQ(check.urls.size(), check.url_results.size());
168  DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
169  if (!check.urls.empty()) {
170    DCHECK(check.full_hashes.empty());
171    switch (check.check_type) {
172      case safe_browsing_util::MALWARE:
173      case safe_browsing_util::PHISH:
174        DCHECK_EQ(1u, check.urls.size());
175        OnCheckBrowseUrlResult(
176            check.urls[0], check.url_results[0], check.url_metadata[0]);
177        break;
178      case safe_browsing_util::BINURL:
179        DCHECK_EQ(check.urls.size(), check.url_results.size());
180        OnCheckDownloadUrlResult(
181            check.urls,
182            *std::max_element(check.url_results.begin(),
183                              check.url_results.end()));
184        break;
185      default:
186        NOTREACHED();
187    }
188  } else if (!check.full_hashes.empty()) {
189    switch (check.check_type) {
190      case safe_browsing_util::EXTENSIONBLACKLIST: {
191        std::set<std::string> unsafe_extension_ids;
192        for (size_t i = 0; i < check.full_hashes.size(); ++i) {
193          std::string extension_id =
194              safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
195          if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
196            unsafe_extension_ids.insert(extension_id);
197        }
198        OnCheckExtensionsResult(unsafe_extension_ids);
199        break;
200      }
201      default:
202        NOTREACHED();
203    }
204  } else {
205    NOTREACHED();
206  }
207}
208
209SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
210    const scoped_refptr<SafeBrowsingService>& service)
211    : sb_service_(service),
212      database_(NULL),
213      enabled_(false),
214      enable_download_protection_(false),
215      enable_csd_whitelist_(false),
216      enable_download_whitelist_(false),
217      enable_extension_blacklist_(false),
218      enable_side_effect_free_whitelist_(false),
219      enable_ip_blacklist_(false),
220      update_in_progress_(false),
221      database_update_in_progress_(false),
222      closing_database_(false),
223      check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
224  DCHECK(sb_service_.get() != NULL);
225
226  // Android only supports a subset of FULL_SAFE_BROWSING.
227  // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
228#if !defined(OS_ANDROID)
229  CommandLine* cmdline = CommandLine::ForCurrentProcess();
230  enable_download_protection_ =
231      !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
232
233  // We only download the csd-whitelist if client-side phishing detection is
234  // enabled.
235  enable_csd_whitelist_ =
236      !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
237
238  // TODO(noelutz): remove this boolean variable since it should always be true
239  // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
240  // list right now.  This means that we need to be able to disable this list
241  // for the SafeBrowsing test to pass.
242  enable_download_whitelist_ = enable_csd_whitelist_;
243
244  // TODO(kalman): there really shouldn't be a flag for this.
245  enable_extension_blacklist_ =
246      !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
247
248  enable_side_effect_free_whitelist_ =
249      prerender::IsSideEffectFreeWhitelistEnabled() &&
250      !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
251
252  // The client-side IP blacklist feature is tightly integrated with client-side
253  // phishing protection for now.
254  enable_ip_blacklist_ = enable_csd_whitelist_;
255
256  enum SideEffectFreeWhitelistStatus {
257    SIDE_EFFECT_FREE_WHITELIST_ENABLED,
258    SIDE_EFFECT_FREE_WHITELIST_DISABLED,
259    SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
260  };
261
262  SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
263      enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
264      SIDE_EFFECT_FREE_WHITELIST_DISABLED;
265
266  UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
267                            side_effect_free_whitelist_status,
268                            SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
269#endif
270}
271
272SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
273  // We should have already been shut down. If we're still enabled, then the
274  // database isn't going to be closed properly, which could lead to corruption.
275  DCHECK(!enabled_);
276}
277
278bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
279  return url.SchemeIs(url::kFtpScheme) ||
280         url.SchemeIs(url::kHttpScheme) ||
281         url.SchemeIs(url::kHttpsScheme);
282}
283
284bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
285    const std::vector<GURL>& url_chain,
286    Client* client) {
287  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
288  if (!enabled_ || !enable_download_protection_)
289    return true;
290
291  // We need to check the database for url prefix, and later may fetch the url
292  // from the safebrowsing backends. These need to be asynchronous.
293  SafeBrowsingCheck* check =
294      new SafeBrowsingCheck(url_chain,
295                            std::vector<SBFullHash>(),
296                            client,
297                            safe_browsing_util::BINURL,
298                            std::vector<SBThreatType>(1,
299                                SB_THREAT_TYPE_BINARY_MALWARE_URL));
300  StartSafeBrowsingCheck(
301      check,
302      base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
303                 check));
304  return false;
305}
306
307bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
308    const std::set<std::string>& extension_ids,
309    Client* client) {
310  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
311
312  if (!enabled_ || !enable_extension_blacklist_)
313    return true;
314
315  std::vector<SBFullHash> extension_id_hashes;
316  std::transform(extension_ids.begin(), extension_ids.end(),
317                 std::back_inserter(extension_id_hashes),
318                 safe_browsing_util::StringToSBFullHash);
319
320  SafeBrowsingCheck* check = new SafeBrowsingCheck(
321      std::vector<GURL>(),
322      extension_id_hashes,
323      client,
324      safe_browsing_util::EXTENSIONBLACKLIST,
325      std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
326
327  StartSafeBrowsingCheck(
328      check,
329      base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
330                 this,
331                 check));
332  return false;
333}
334
335bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
336    const GURL& url) {
337  if (!enabled_)
338    return false;
339
340  if (!CanCheckUrl(url))
341    return false;
342
343  return database_->ContainsSideEffectFreeWhitelistUrl(url);
344}
345
346bool SafeBrowsingDatabaseManager::MatchMalwareIP(
347    const std::string& ip_address) {
348  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
349  if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
350    return false;  // Fail open.
351  }
352  return database_->ContainsMalwareIP(ip_address);
353}
354
355bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
356  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
357  if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
358    // There is something funky going on here -- for example, perhaps the user
359    // has not restarted since enabling metrics reporting, so we haven't
360    // enabled the csd whitelist yet.  Just to be safe we return true in this
361    // case.
362    return true;
363  }
364  return database_->ContainsCsdWhitelistedUrl(url);
365}
366
367bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
368  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
369  if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
370    return true;
371  }
372  return database_->ContainsDownloadWhitelistedUrl(url);
373}
374
375bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
376    const std::string& str) {
377  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
378  if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
379    return true;
380  }
381  return database_->ContainsDownloadWhitelistedString(str);
382}
383
384bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
385  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
386  if (!enabled_ || !MakeDatabaseAvailable()) {
387    return true;
388  }
389  return database_->IsMalwareIPMatchKillSwitchOn();
390}
391
392bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
393  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
394  if (!enabled_ || !MakeDatabaseAvailable()) {
395    return true;
396  }
397  return database_->IsCsdWhitelistKillSwitchOn();
398}
399
400bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
401                                                 Client* client) {
402  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
403  if (!enabled_)
404    return true;
405
406  if (!CanCheckUrl(url))
407    return true;
408
409  std::vector<SBThreatType> expected_threats;
410  expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
411  expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
412
413  const base::TimeTicks start = base::TimeTicks::Now();
414  if (!MakeDatabaseAvailable()) {
415    QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
416                             client,
417                             url,
418                             expected_threats,
419                             start);
420    queued_checks_.push_back(queued_check);
421    return false;
422  }
423
424  std::vector<SBPrefix> prefix_hits;
425  std::vector<SBFullHashResult> cache_hits;
426
427  bool prefix_match =
428      database_->ContainsBrowseUrl(url, &prefix_hits, &cache_hits);
429
430  UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
431
432  if (!prefix_match)
433    return true;  // URL is okay.
434
435  // Needs to be asynchronous, since we could be in the constructor of a
436  // ResourceDispatcherHost event handler which can't pause there.
437  SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
438                                                   std::vector<SBFullHash>(),
439                                                   client,
440                                                   safe_browsing_util::MALWARE,
441                                                   expected_threats);
442  check->need_get_hash = cache_hits.empty();
443  check->prefix_hits.swap(prefix_hits);
444  check->cache_hits.swap(cache_hits);
445  checks_.insert(check);
446
447  BrowserThread::PostTask(
448      BrowserThread::IO, FROM_HERE,
449      base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
450
451  return false;
452}
453
454void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
455  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
456  for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
457    // We can't delete matching checks here because the db thread has a copy of
458    // the pointer.  Instead, we simply NULL out the client, and when the db
459    // thread calls us back, we'll clean up the check.
460    if ((*i)->client == client)
461      (*i)->client = NULL;
462  }
463
464  // Scan the queued clients store. Clients may be here if they requested a URL
465  // check before the database has finished loading.
466  for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
467       it != queued_checks_.end(); ) {
468    // In this case it's safe to delete matches entirely since nothing has a
469    // pointer to them.
470    if (it->client == client)
471      it = queued_checks_.erase(it);
472    else
473      ++it;
474  }
475}
476
477void SafeBrowsingDatabaseManager::HandleGetHashResults(
478    SafeBrowsingCheck* check,
479    const std::vector<SBFullHashResult>& full_hashes,
480    const base::TimeDelta& cache_lifetime) {
481  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
482
483  if (!enabled_)
484    return;
485
486  // If the service has been shut down, |check| should have been deleted.
487  DCHECK(checks_.find(check) != checks_.end());
488
489  // |start| is set before calling |GetFullHash()|, which should be
490  // the only path which gets to here.
491  DCHECK(!check->start.is_null());
492  UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
493                           base::TimeTicks::Now() - check->start);
494
495  std::vector<SBPrefix> prefixes = check->prefix_hits;
496  OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
497
498  // Cache the GetHash results.
499  if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
500    database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
501}
502
503void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
504  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
505  DCHECK(enabled_);
506  DCHECK(!callback.is_null());
507  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
508      &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
509}
510
511void SafeBrowsingDatabaseManager::AddChunks(
512    const std::string& list,
513    scoped_ptr<ScopedVector<SBChunkData> > chunks,
514    AddChunksCallback callback) {
515  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
516  DCHECK(enabled_);
517  DCHECK(!callback.is_null());
518  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
519      &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
520      base::Passed(&chunks), callback));
521}
522
523void SafeBrowsingDatabaseManager::DeleteChunks(
524    scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
525  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
526  DCHECK(enabled_);
527  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
528      &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
529      base::Passed(&chunk_deletes)));
530}
531
532void SafeBrowsingDatabaseManager::UpdateStarted() {
533  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
534  DCHECK(enabled_);
535  DCHECK(!update_in_progress_);
536  update_in_progress_ = true;
537}
538
539void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
540  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
541  DCHECK(enabled_);
542  if (update_in_progress_) {
543    update_in_progress_ = false;
544    safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
545      base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
546                 this, update_succeeded));
547  }
548}
549
550void SafeBrowsingDatabaseManager::ResetDatabase() {
551  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
552  DCHECK(enabled_);
553  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
554      &SafeBrowsingDatabaseManager::OnResetDatabase, this));
555}
556
557void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
558  UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
559}
560
561void SafeBrowsingDatabaseManager::StartOnIOThread() {
562  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
563  if (enabled_)
564    return;
565
566  DCHECK(!safe_browsing_thread_.get());
567  safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
568  if (!safe_browsing_thread_->Start())
569    return;
570  enabled_ = true;
571
572  MakeDatabaseAvailable();
573}
574
575void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
576  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
577
578  DoStopOnIOThread();
579  if (shutdown) {
580    sb_service_ = NULL;
581  }
582}
583
584void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
585    bool update_succeeded) {
586  content::NotificationService::current()->Notify(
587      chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
588      content::Source<SafeBrowsingDatabaseManager>(this),
589      content::Details<bool>(&update_succeeded));
590}
591
592SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
593    const safe_browsing_util::ListType check_type,
594    Client* client,
595    const GURL& url,
596    const std::vector<SBThreatType>& expected_threats,
597    const base::TimeTicks& start)
598    : check_type(check_type),
599      client(client),
600      url(url),
601      expected_threats(expected_threats),
602      start(start) {
603}
604
605SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
606}
607
608void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
609  if (!enabled_)
610    return;
611
612  enabled_ = false;
613
614  // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
615  while (!queued_checks_.empty()) {
616    QueuedCheck queued = queued_checks_.front();
617    if (queued.client) {
618      SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
619                                 std::vector<SBFullHash>(),
620                                 queued.client,
621                                 queued.check_type,
622                                 queued.expected_threats);
623      queued.client->OnSafeBrowsingResult(sb_check);
624    }
625    queued_checks_.pop_front();
626  }
627
628  // Close the database.  Cases to avoid:
629  //  * If |closing_database_| is true, continuing will queue up a second
630  //    request, |closing_database_| will be reset after handling the first
631  //    request, and if any functions on the db thread recreate the database, we
632  //    could start using it on the IO thread and then have the second request
633  //    handler delete it out from under us.
634  //  * If |database_| is NULL, then either no creation request is in flight, in
635  //    which case we don't need to do anything, or one is in flight, in which
636  //    case the database will be recreated before our deletion request is
637  //    handled, and could be used on the IO thread in that time period, leading
638  //    to the same problem as above.
639  // Checking DatabaseAvailable() avoids both of these.
640  if (DatabaseAvailable()) {
641    closing_database_ = true;
642    safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
643        base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
644  }
645
646  // Flush the database thread. Any in-progress database check results will be
647  // ignored and cleaned up below.
648  //
649  // Note that to avoid leaking the database, we rely on the fact that no new
650  // tasks will be added to the db thread between the call above and this one.
651  // See comments on the declaration of |safe_browsing_thread_|.
652  {
653    // A ScopedAllowIO object is required to join the thread when calling Stop.
654    // See http://crbug.com/72696. Note that we call Stop() first to clear out
655    // any remaining tasks before clearing safe_browsing_thread_.
656    base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
657    safe_browsing_thread_->Stop();
658    safe_browsing_thread_.reset();
659  }
660
661  // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
662  // We have to do this after the db thread returns because methods on it can
663  // have copies of these pointers, so deleting them might lead to accessing
664  // garbage.
665  for (CurrentChecks::iterator it = checks_.begin();
666       it != checks_.end(); ++it) {
667    SafeBrowsingCheck* check = *it;
668    if (check->client)
669      check->client->OnSafeBrowsingResult(*check);
670  }
671  STLDeleteElements(&checks_);
672
673  gethash_requests_.clear();
674}
675
676bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
677  base::AutoLock lock(database_lock_);
678  return !closing_database_ && (database_ != NULL);
679}
680
681bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
682  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
683  DCHECK(enabled_);
684  if (DatabaseAvailable())
685    return true;
686  safe_browsing_thread_->message_loop()->PostTask(
687      FROM_HERE,
688      base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
689                 this));
690  return false;
691}
692
693SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
694  DCHECK_EQ(base::MessageLoop::current(),
695            safe_browsing_thread_->message_loop());
696  if (database_)
697    return database_;
698  startup_metric_utils::ScopedSlowStartupUMA
699      scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
700  const base::TimeTicks before = base::TimeTicks::Now();
701
702  SafeBrowsingDatabase* database =
703      SafeBrowsingDatabase::Create(enable_download_protection_,
704                                   enable_csd_whitelist_,
705                                   enable_download_whitelist_,
706                                   enable_extension_blacklist_,
707                                   enable_side_effect_free_whitelist_,
708                                   enable_ip_blacklist_);
709
710  database->Init(SafeBrowsingService::GetBaseFilename());
711  {
712    // Acquiring the lock here guarantees correct ordering between the writes to
713    // the new database object above, and the setting of |databse_| below.
714    base::AutoLock lock(database_lock_);
715    database_ = database;
716  }
717
718  BrowserThread::PostTask(
719      BrowserThread::IO, FROM_HERE,
720      base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
721
722  UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
723  return database_;
724}
725
726void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
727  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
728
729  if (!enabled_)
730    return;
731
732  // If the service has been shut down, |check| should have been deleted.
733  DCHECK(checks_.find(check) != checks_.end());
734
735  if (check->client && check->need_get_hash) {
736    // We have a partial match so we need to query Google for the full hash.
737    // Clean up will happen in HandleGetHashResults.
738
739    // See if we have a GetHash request already in progress for this particular
740    // prefix. If so, we just append ourselves to the list of interested parties
741    // when the results arrive. We only do this for checks involving one prefix,
742    // since that is the common case (multiple prefixes will issue the request
743    // as normal).
744    if (check->prefix_hits.size() == 1) {
745      SBPrefix prefix = check->prefix_hits[0];
746      GetHashRequests::iterator it = gethash_requests_.find(prefix);
747      if (it != gethash_requests_.end()) {
748        // There's already a request in progress.
749        it->second.push_back(check);
750        return;
751      }
752
753      // No request in progress, so we're the first for this prefix.
754      GetHashRequestors requestors;
755      requestors.push_back(check);
756      gethash_requests_[prefix] = requestors;
757    }
758
759    // Reset the start time so that we can measure the network time without the
760    // database time.
761    check->start = base::TimeTicks::Now();
762    // Note: If |this| is deleted or stopped, the protocol_manager will
763    // be destroyed as well - hence it's OK to do unretained in this case.
764    bool is_download = check->check_type == safe_browsing_util::BINURL;
765    sb_service_->protocol_manager()->GetFullHash(
766        check->prefix_hits,
767        base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
768                   base::Unretained(this),
769                   check),
770        is_download);
771  } else {
772    // We may have cached results for previous GetHash queries.  Since
773    // this data comes from cache, don't histogram hits.
774    bool is_threat = HandleOneCheck(check, check->cache_hits);
775    // cache_hits should only contain hits for a fullhash we searched for, so if
776    // we got to this point it should always result in a threat match.
777    DCHECK(is_threat);
778  }
779}
780
781void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
782    GetChunksCallback callback) {
783  DCHECK_EQ(base::MessageLoop::current(),
784            safe_browsing_thread_->message_loop());
785
786  bool database_error = true;
787  std::vector<SBListChunkRanges> lists;
788  DCHECK(!database_update_in_progress_);
789  database_update_in_progress_ = true;
790  GetDatabase();  // This guarantees that |database_| is non-NULL.
791  if (database_->UpdateStarted(&lists)) {
792    database_error = false;
793  } else {
794    database_->UpdateFinished(false);
795  }
796
797  BrowserThread::PostTask(
798      BrowserThread::IO, FROM_HERE,
799      base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
800                 this, lists, database_error, callback));
801}
802
803void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
804    const std::vector<SBListChunkRanges>& lists, bool database_error,
805    GetChunksCallback callback) {
806  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
807  if (enabled_)
808    callback.Run(lists, database_error);
809}
810
811void SafeBrowsingDatabaseManager::OnAddChunksComplete(
812    AddChunksCallback callback) {
813  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
814  if (enabled_)
815    callback.Run();
816}
817
818void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
819  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
820  if (!enabled_)
821    return;
822
823  LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
824  if (queued_checks_.empty())
825    return;
826
827  // If the database isn't already available, calling CheckUrl() in the loop
828  // below will add the check back to the queue, and we'll infinite-loop.
829  DCHECK(DatabaseAvailable());
830  while (!queued_checks_.empty()) {
831    QueuedCheck check = queued_checks_.front();
832    DCHECK(!check.start.is_null());
833    LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
834                          base::TimeTicks::Now() - check.start);
835    // If CheckUrl() determines the URL is safe immediately, it doesn't call the
836    // client's handler function (because normally it's being directly called by
837    // the client).  Since we're not the client, we have to convey this result.
838    if (check.client && CheckBrowseUrl(check.url, check.client)) {
839      SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
840                                 std::vector<SBFullHash>(),
841                                 check.client,
842                                 check.check_type,
843                                 check.expected_threats);
844      check.client->OnSafeBrowsingResult(sb_check);
845    }
846    queued_checks_.pop_front();
847  }
848}
849
850void SafeBrowsingDatabaseManager::AddDatabaseChunks(
851    const std::string& list_name,
852    scoped_ptr<ScopedVector<SBChunkData> > chunks,
853    AddChunksCallback callback) {
854  DCHECK_EQ(base::MessageLoop::current(),
855            safe_browsing_thread_->message_loop());
856  if (chunks)
857    GetDatabase()->InsertChunks(list_name, chunks->get());
858  BrowserThread::PostTask(
859      BrowserThread::IO, FROM_HERE,
860      base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
861                 callback));
862}
863
864void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
865    scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
866  DCHECK_EQ(base::MessageLoop::current(),
867            safe_browsing_thread_->message_loop());
868  if (chunk_deletes)
869    GetDatabase()->DeleteChunks(*chunk_deletes);
870}
871
872void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
873    bool update_succeeded) {
874  DCHECK_EQ(base::MessageLoop::current(),
875            safe_browsing_thread_->message_loop());
876  GetDatabase()->UpdateFinished(update_succeeded);
877  DCHECK(database_update_in_progress_);
878  database_update_in_progress_ = false;
879  BrowserThread::PostTask(
880      BrowserThread::UI, FROM_HERE,
881      base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
882                 this, update_succeeded));
883}
884
885void SafeBrowsingDatabaseManager::OnCloseDatabase() {
886  DCHECK_EQ(base::MessageLoop::current(),
887            safe_browsing_thread_->message_loop());
888  DCHECK(closing_database_);
889
890  // Because |closing_database_| is true, nothing on the IO thread will be
891  // accessing the database, so it's safe to delete and then NULL the pointer.
892  delete database_;
893  database_ = NULL;
894
895  // Acquiring the lock here guarantees correct ordering between the resetting
896  // of |database_| above and of |closing_database_| below, which ensures there
897  // won't be a window during which the IO thread falsely believes the database
898  // is available.
899  base::AutoLock lock(database_lock_);
900  closing_database_ = false;
901}
902
903void SafeBrowsingDatabaseManager::OnResetDatabase() {
904  DCHECK_EQ(base::MessageLoop::current(),
905            safe_browsing_thread_->message_loop());
906  GetDatabase()->ResetDatabase();
907}
908
909void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
910    SafeBrowsingCheck* check,
911    const std::vector<SBFullHashResult>& full_hashes) {
912  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
913  safe_browsing_util::ListType check_type = check->check_type;
914  SBPrefix prefix = check->prefix_hits[0];
915  GetHashRequests::iterator it = gethash_requests_.find(prefix);
916  if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
917    const bool hit = HandleOneCheck(check, full_hashes);
918    RecordGetHashCheckStatus(hit, check_type, full_hashes);
919    return;
920  }
921
922  // Call back all interested parties, noting if any has a hit.
923  GetHashRequestors& requestors = it->second;
924  bool hit = false;
925  for (GetHashRequestors::iterator r = requestors.begin();
926       r != requestors.end(); ++r) {
927    if (HandleOneCheck(*r, full_hashes))
928      hit = true;
929  }
930  RecordGetHashCheckStatus(hit, check_type, full_hashes);
931
932  gethash_requests_.erase(it);
933}
934
935bool SafeBrowsingDatabaseManager::HandleOneCheck(
936    SafeBrowsingCheck* check,
937    const std::vector<SBFullHashResult>& full_hashes) {
938  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
939  DCHECK(check);
940
941  bool is_threat = false;
942
943  // TODO(shess): GetHashThreadListType() contains a loop,
944  // GetUrlThreatListType() a loop around that loop.  Having another loop out
945  // here concerns me.  It is likely that SAFE is an expected outcome, which
946  // means all of those loops run to completion.  Refactoring this to generate a
947  // set of sorted items to compare in sequence would probably improve things.
948  //
949  // Additionally, the set of patterns generated from the urls is very similar
950  // to the patterns generated in ContainsBrowseUrl() and other database checks,
951  // which are called from this code.  Refactoring that across the checks could
952  // interact well with batching the checks here.
953
954  for (size_t i = 0; i < check->urls.size(); ++i) {
955    size_t threat_index;
956    SBThreatType threat =
957        GetUrlThreatType(check->urls[i], full_hashes, &threat_index);
958    if (threat != SB_THREAT_TYPE_SAFE &&
959        IsExpectedThreat(threat, check->expected_threats)) {
960      check->url_results[i] = threat;
961      check->url_metadata[i] = full_hashes[threat_index].metadata;
962      is_threat = true;
963    }
964  }
965
966  for (size_t i = 0; i < check->full_hashes.size(); ++i) {
967    SBThreatType threat = GetHashThreatType(check->full_hashes[i], full_hashes);
968    if (threat != SB_THREAT_TYPE_SAFE &&
969        IsExpectedThreat(threat, check->expected_threats)) {
970      check->full_hash_results[i] = threat;
971      is_threat = true;
972    }
973  }
974
975  SafeBrowsingCheckDone(check);
976  return is_threat;
977}
978
979void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
980    SafeBrowsingCheck* check) {
981  DCHECK_EQ(base::MessageLoop::current(),
982            safe_browsing_thread_->message_loop());
983  DCHECK(enable_download_protection_);
984
985  std::vector<SBPrefix> prefix_hits;
986
987  if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
988    // Good, we don't have hash for this url prefix.
989    BrowserThread::PostTask(
990        BrowserThread::IO, FROM_HERE,
991        base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
992                   check));
993    return;
994  }
995
996  check->need_get_hash = true;
997  check->prefix_hits.clear();
998  check->prefix_hits = prefix_hits;
999  BrowserThread::PostTask(
1000      BrowserThread::IO, FROM_HERE,
1001      base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1002}
1003
1004void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1005    SafeBrowsingCheck* check) {
1006  DCHECK_EQ(base::MessageLoop::current(),
1007            safe_browsing_thread_->message_loop());
1008
1009  std::vector<SBPrefix> prefixes;
1010  for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
1011       it != check->full_hashes.end(); ++it) {
1012    prefixes.push_back((*it).prefix);
1013  }
1014  database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
1015
1016  if (check->prefix_hits.empty()) {
1017    // No matches for any extensions.
1018    BrowserThread::PostTask(
1019        BrowserThread::IO,
1020        FROM_HERE,
1021        base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
1022                   check));
1023  } else {
1024    // Some prefixes matched, we need to ask Google whether they're legit.
1025    check->need_get_hash = true;
1026    BrowserThread::PostTask(
1027        BrowserThread::IO,
1028        FROM_HERE,
1029        base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1030  }
1031}
1032
1033void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1034  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1035  DCHECK(check);
1036
1037  if (!enabled_)
1038    return;
1039
1040  DCHECK(checks_.find(check) != checks_.end());
1041  if (check->client) {
1042    check->client->OnSafeBrowsingResult(*check);
1043    check->client = NULL;
1044  }
1045}
1046
1047void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1048    SafeBrowsingCheck* check) {
1049  DCHECK(enable_download_protection_);
1050  SafeBrowsingCheckDone(check);
1051}
1052
1053void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1054    SafeBrowsingCheck* check) {
1055  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1056  DCHECK(check);
1057
1058  if (!enabled_)
1059    return;
1060
1061  VLOG(1) << "SafeBrowsingCheckDone";
1062  DCHECK(checks_.find(check) != checks_.end());
1063  if (check->client)
1064    check->client->OnSafeBrowsingResult(*check);
1065  checks_.erase(check);
1066  delete check;
1067}
1068
1069void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1070    SafeBrowsingCheck* check,
1071    const base::Closure& task) {
1072  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1073  check->timeout_factory_.reset(
1074      new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1075  checks_.insert(check);
1076
1077  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
1078
1079  base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1080      base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1081                 check->timeout_factory_->GetWeakPtr(), check),
1082      check_timeout_);
1083}
1084