1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/safe_browsing/database_manager.h"
6
7#include <algorithm>
8
9#include "base/bind.h"
10#include "base/bind_helpers.h"
11#include "base/callback.h"
12#include "base/command_line.h"
13#include "base/debug/leak_tracker.h"
14#include "base/path_service.h"
15#include "base/stl_util.h"
16#include "base/strings/string_util.h"
17#include "base/threading/thread.h"
18#include "base/threading/thread_restrictions.h"
19#include "chrome/browser/browser_process.h"
20#include "chrome/browser/chrome_notification_types.h"
21#include "chrome/browser/metrics/metrics_service.h"
22#include "chrome/browser/prerender/prerender_field_trial.h"
23#include "chrome/browser/safe_browsing/client_side_detection_service.h"
24#include "chrome/browser/safe_browsing/download_protection_service.h"
25#include "chrome/browser/safe_browsing/malware_details.h"
26#include "chrome/browser/safe_browsing/protocol_manager.h"
27#include "chrome/browser/safe_browsing/safe_browsing_database.h"
28#include "chrome/browser/safe_browsing/safe_browsing_service.h"
29#include "chrome/browser/safe_browsing/ui_manager.h"
30#include "chrome/common/chrome_constants.h"
31#include "chrome/common/chrome_paths.h"
32#include "chrome/common/chrome_switches.h"
33#include "chrome/common/url_constants.h"
34#include "components/startup_metric_utils/startup_metric_utils.h"
35#include "content/public/browser/browser_thread.h"
36#include "content/public/browser/notification_service.h"
37
38using content::BrowserThread;
39
40namespace {
41
42// Timeout for match checks, e.g. download URLs, hashes.
43const int kCheckTimeoutMs = 10000;
44
45// Records disposition information about the check.  |hit| should be
46// |true| if there were any prefix hits in |full_hashes|.
47void RecordGetHashCheckStatus(
48    bool hit,
49    safe_browsing_util::ListType check_type,
50    const std::vector<SBFullHashResult>& full_hashes) {
51  SafeBrowsingProtocolManager::ResultType result;
52  if (full_hashes.empty()) {
53    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
54  } else if (hit) {
55    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
56  } else {
57    result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
58  }
59  bool is_download = check_type == safe_browsing_util::BINURL ||
60                     check_type == safe_browsing_util::BINHASH;
61  SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
62}
63
64bool IsExpectedThreat(
65    const SBThreatType threat_type,
66    const std::vector<SBThreatType>& expected_threats) {
67  return expected_threats.end() != std::find(expected_threats.begin(),
68                                             expected_threats.end(),
69                                             threat_type);
70}
71
72}  // namespace
73
74SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
75    const std::vector<GURL>& urls,
76    const std::vector<SBFullHash>& full_hashes,
77    Client* client,
78    safe_browsing_util::ListType check_type,
79    const std::vector<SBThreatType>& expected_threats)
80    : urls(urls),
81      url_results(urls.size(), SB_THREAT_TYPE_SAFE),
82      full_hashes(full_hashes),
83      full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
84      client(client),
85      need_get_hash(false),
86      check_type(check_type),
87      expected_threats(expected_threats) {
88  DCHECK_EQ(urls.empty(), !full_hashes.empty())
89      << "Exactly one of urls and full_hashes must be set";
90}
91
92SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
93
94void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
95    const SafeBrowsingCheck& check) {
96  DCHECK_EQ(check.urls.size(), check.url_results.size());
97  DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
98  if (!check.urls.empty()) {
99    DCHECK(check.full_hashes.empty());
100    switch (check.check_type) {
101      case safe_browsing_util::MALWARE:
102      case safe_browsing_util::PHISH:
103        DCHECK_EQ(1u, check.urls.size());
104        OnCheckBrowseUrlResult(check.urls[0], check.url_results[0]);
105        break;
106      case safe_browsing_util::BINURL:
107        DCHECK_EQ(check.urls.size(), check.url_results.size());
108        OnCheckDownloadUrlResult(
109            check.urls,
110            *std::max_element(check.url_results.begin(),
111                              check.url_results.end()));
112        break;
113      default:
114        NOTREACHED();
115    }
116  } else if (!check.full_hashes.empty()) {
117    switch (check.check_type) {
118      case safe_browsing_util::BINHASH:
119        DCHECK_EQ(1u, check.full_hashes.size());
120        OnCheckDownloadHashResult(
121            safe_browsing_util::SBFullHashToString(check.full_hashes[0]),
122            check.full_hash_results[0]);
123        break;
124      case safe_browsing_util::EXTENSIONBLACKLIST: {
125        std::set<std::string> unsafe_extension_ids;
126        for (size_t i = 0; i < check.full_hashes.size(); ++i) {
127          std::string extension_id =
128              safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
129          if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
130            unsafe_extension_ids.insert(extension_id);
131        }
132        OnCheckExtensionsResult(unsafe_extension_ids);
133        break;
134      }
135      default:
136        NOTREACHED();
137    }
138  } else {
139    NOTREACHED();
140  }
141}
142
143SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
144    const scoped_refptr<SafeBrowsingService>& service)
145    : sb_service_(service),
146      database_(NULL),
147      enabled_(false),
148      enable_download_protection_(false),
149      enable_csd_whitelist_(false),
150      enable_download_whitelist_(false),
151      enable_extension_blacklist_(false),
152      enable_side_effect_free_whitelist_(false),
153      enable_ip_blacklist_(false),
154      update_in_progress_(false),
155      database_update_in_progress_(false),
156      closing_database_(false),
157      check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
158  DCHECK(sb_service_.get() != NULL);
159
160  CommandLine* cmdline = CommandLine::ForCurrentProcess();
161  enable_download_protection_ =
162      !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
163
164  // We only download the csd-whitelist if client-side phishing detection is
165  // enabled.
166  enable_csd_whitelist_ =
167      !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
168
169  // TODO(noelutz): remove this boolean variable since it should always be true
170  // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
171  // list right now.  This means that we need to be able to disable this list
172  // for the SafeBrowsing test to pass.
173  enable_download_whitelist_ = enable_csd_whitelist_;
174
175  // TODO(kalman): there really shouldn't be a flag for this.
176  enable_extension_blacklist_ =
177      !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
178
179  enable_side_effect_free_whitelist_ =
180      prerender::IsSideEffectFreeWhitelistEnabled() &&
181      !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
182
183  // The client-side IP blacklist feature is tightly integrated with client-side
184  // phishing protection for now.
185  enable_ip_blacklist_ = enable_csd_whitelist_;
186
187  enum SideEffectFreeWhitelistStatus {
188    SIDE_EFFECT_FREE_WHITELIST_ENABLED,
189    SIDE_EFFECT_FREE_WHITELIST_DISABLED,
190    SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
191  };
192
193  SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
194      enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
195      SIDE_EFFECT_FREE_WHITELIST_DISABLED;
196
197  UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
198                            side_effect_free_whitelist_status,
199                            SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
200}
201
202SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
203  // We should have already been shut down. If we're still enabled, then the
204  // database isn't going to be closed properly, which could lead to corruption.
205  DCHECK(!enabled_);
206}
207
208bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
209  return url.SchemeIs(content::kFtpScheme) ||
210         url.SchemeIs(content::kHttpScheme) ||
211         url.SchemeIs(content::kHttpsScheme);
212}
213
214bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
215    const std::vector<GURL>& url_chain,
216    Client* client) {
217  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
218  if (!enabled_ || !enable_download_protection_)
219    return true;
220
221  // We need to check the database for url prefix, and later may fetch the url
222  // from the safebrowsing backends. These need to be asynchronous.
223  SafeBrowsingCheck* check =
224      new SafeBrowsingCheck(url_chain,
225                            std::vector<SBFullHash>(),
226                            client,
227                            safe_browsing_util::BINURL,
228                            std::vector<SBThreatType>(1,
229                                SB_THREAT_TYPE_BINARY_MALWARE_URL));
230  StartSafeBrowsingCheck(
231      check,
232      base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
233                 check));
234  return false;
235}
236
237bool SafeBrowsingDatabaseManager::CheckDownloadHash(
238    const std::string& full_hash,
239    Client* client) {
240  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
241  DCHECK(!full_hash.empty());
242  if (!enabled_ || !enable_download_protection_ || full_hash.empty())
243    return true;
244
245  // We need to check the database for url prefix, and later may fetch the url
246  // from the safebrowsing backends. These need to be asynchronous.
247  std::vector<SBFullHash> full_hashes(
248      1, safe_browsing_util::StringToSBFullHash(full_hash));
249  SafeBrowsingCheck* check =
250      new SafeBrowsingCheck(std::vector<GURL>(),
251                            full_hashes,
252                            client,
253                            safe_browsing_util::BINHASH,
254                            std::vector<SBThreatType>(1,
255                                SB_THREAT_TYPE_BINARY_MALWARE_HASH));
256  StartSafeBrowsingCheck(
257      check,
258      base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadHashOnSBThread,this,
259                 check));
260  return false;
261}
262
263bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
264    const std::set<std::string>& extension_ids,
265    Client* client) {
266  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
267
268  if (!enabled_ || !enable_extension_blacklist_)
269    return true;
270
271  std::vector<SBFullHash> extension_id_hashes;
272  std::transform(extension_ids.begin(), extension_ids.end(),
273                 std::back_inserter(extension_id_hashes),
274                 safe_browsing_util::StringToSBFullHash);
275
276  SafeBrowsingCheck* check = new SafeBrowsingCheck(
277      std::vector<GURL>(),
278      extension_id_hashes,
279      client,
280      safe_browsing_util::EXTENSIONBLACKLIST,
281      std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
282
283  StartSafeBrowsingCheck(
284      check,
285      base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
286                 this,
287                 check));
288  return false;
289}
290
291bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
292    const GURL& url) {
293  if (!enabled_)
294    return false;
295
296  if (!CanCheckUrl(url))
297    return false;
298
299  return database_->ContainsSideEffectFreeWhitelistUrl(url);
300}
301
302bool SafeBrowsingDatabaseManager::MatchMalwareIP(
303    const std::string& ip_address) {
304  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
305  if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
306    return false;  // Fail open.
307  }
308  return database_->ContainsMalwareIP(ip_address);
309}
310
311bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
312  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
313  if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
314    // There is something funky going on here -- for example, perhaps the user
315    // has not restarted since enabling metrics reporting, so we haven't
316    // enabled the csd whitelist yet.  Just to be safe we return true in this
317    // case.
318    return true;
319  }
320  return database_->ContainsCsdWhitelistedUrl(url);
321}
322
323bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
324  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
325  if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
326    return true;
327  }
328  return database_->ContainsDownloadWhitelistedUrl(url);
329}
330
331bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
332    const std::string& str) {
333  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
334  if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
335    return true;
336  }
337  return database_->ContainsDownloadWhitelistedString(str);
338}
339
340bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
341  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
342  if (!enabled_ || !MakeDatabaseAvailable()) {
343    return true;
344  }
345  return database_->IsMalwareIPMatchKillSwitchOn();
346}
347
348bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
349                                                 Client* client) {
350  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
351  if (!enabled_)
352    return true;
353
354  if (!CanCheckUrl(url))
355    return true;
356
357  std::vector<SBThreatType> expected_threats;
358  expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
359  expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
360
361  const base::TimeTicks start = base::TimeTicks::Now();
362  if (!MakeDatabaseAvailable()) {
363    QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
364                             client,
365                             url,
366                             expected_threats,
367                             start);
368    queued_checks_.push_back(queued_check);
369    return false;
370  }
371
372  std::string list;
373  std::vector<SBPrefix> prefix_hits;
374  std::vector<SBFullHashResult> full_hits;
375
376  bool prefix_match =
377      database_->ContainsBrowseUrl(url, &list, &prefix_hits, &full_hits,
378          sb_service_->protocol_manager()->last_update());
379
380  UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
381
382  if (!prefix_match)
383    return true;  // URL is okay.
384
385  // Needs to be asynchronous, since we could be in the constructor of a
386  // ResourceDispatcherHost event handler which can't pause there.
387  SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
388                                                   std::vector<SBFullHash>(),
389                                                   client,
390                                                   safe_browsing_util::MALWARE,
391                                                   expected_threats);
392  check->need_get_hash = full_hits.empty();
393  check->prefix_hits.swap(prefix_hits);
394  check->full_hits.swap(full_hits);
395  checks_.insert(check);
396
397  BrowserThread::PostTask(
398      BrowserThread::IO, FROM_HERE,
399      base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
400
401  return false;
402}
403
404void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
405  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
406  for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
407    // We can't delete matching checks here because the db thread has a copy of
408    // the pointer.  Instead, we simply NULL out the client, and when the db
409    // thread calls us back, we'll clean up the check.
410    if ((*i)->client == client)
411      (*i)->client = NULL;
412  }
413
414  // Scan the queued clients store. Clients may be here if they requested a URL
415  // check before the database has finished loading.
416  for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
417       it != queued_checks_.end(); ) {
418    // In this case it's safe to delete matches entirely since nothing has a
419    // pointer to them.
420    if (it->client == client)
421      it = queued_checks_.erase(it);
422    else
423      ++it;
424  }
425}
426
427void SafeBrowsingDatabaseManager::HandleGetHashResults(
428    SafeBrowsingCheck* check,
429    const std::vector<SBFullHashResult>& full_hashes,
430    bool can_cache) {
431  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
432
433  if (!enabled_)
434    return;
435
436  // If the service has been shut down, |check| should have been deleted.
437  DCHECK(checks_.find(check) != checks_.end());
438
439  // |start| is set before calling |GetFullHash()|, which should be
440  // the only path which gets to here.
441  DCHECK(!check->start.is_null());
442  UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
443                           base::TimeTicks::Now() - check->start);
444
445  std::vector<SBPrefix> prefixes = check->prefix_hits;
446  OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
447
448  if (can_cache && MakeDatabaseAvailable()) {
449    // Cache the GetHash results in memory:
450    database_->CacheHashResults(prefixes, full_hashes);
451  }
452}
453
454void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
455  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
456  DCHECK(enabled_);
457  DCHECK(!callback.is_null());
458  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
459      &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
460}
461
462void SafeBrowsingDatabaseManager::AddChunks(const std::string& list,
463                                            SBChunkList* chunks,
464                                            AddChunksCallback callback) {
465  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
466  DCHECK(enabled_);
467  DCHECK(!callback.is_null());
468  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
469      &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
470      chunks, callback));
471}
472
473void SafeBrowsingDatabaseManager::DeleteChunks(
474    std::vector<SBChunkDelete>* chunk_deletes) {
475  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
476  DCHECK(enabled_);
477  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
478      &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this, chunk_deletes));
479}
480
481void SafeBrowsingDatabaseManager::UpdateStarted() {
482  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
483  DCHECK(enabled_);
484  DCHECK(!update_in_progress_);
485  update_in_progress_ = true;
486}
487
488void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
489  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
490  DCHECK(enabled_);
491  if (update_in_progress_) {
492    update_in_progress_ = false;
493    safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
494      base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
495                 this, update_succeeded));
496  }
497}
498
499void SafeBrowsingDatabaseManager::ResetDatabase() {
500  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
501  DCHECK(enabled_);
502  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
503      &SafeBrowsingDatabaseManager::OnResetDatabase, this));
504}
505
506void SafeBrowsingDatabaseManager::PurgeMemory() {
507  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
508  CloseDatabase();
509}
510
511void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
512  UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
513}
514
515void SafeBrowsingDatabaseManager::StartOnIOThread() {
516  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
517  if (enabled_)
518    return;
519
520  DCHECK(!safe_browsing_thread_.get());
521  safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
522  if (!safe_browsing_thread_->Start())
523    return;
524  enabled_ = true;
525
526  MakeDatabaseAvailable();
527}
528
529void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
530  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
531
532  DoStopOnIOThread();
533  if (shutdown) {
534    sb_service_ = NULL;
535  }
536}
537
538void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
539    bool update_succeeded) {
540  content::NotificationService::current()->Notify(
541      chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
542      content::Source<SafeBrowsingDatabaseManager>(this),
543      content::Details<bool>(&update_succeeded));
544}
545
546SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
547    const safe_browsing_util::ListType check_type,
548    Client* client,
549    const GURL& url,
550    const std::vector<SBThreatType>& expected_threats,
551    const base::TimeTicks& start)
552    : check_type(check_type),
553      client(client),
554      url(url),
555      expected_threats(expected_threats),
556      start(start) {
557}
558
559SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
560}
561
562void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
563  if (!enabled_)
564    return;
565
566  enabled_ = false;
567
568  // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
569  // If we don't do this here we may fail to close the database below.
570  while (!queued_checks_.empty()) {
571    QueuedCheck queued = queued_checks_.front();
572    if (queued.client) {
573      SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
574                                 std::vector<SBFullHash>(),
575                                 queued.client,
576                                 queued.check_type,
577                                 queued.expected_threats);
578      queued.client->OnSafeBrowsingResult(sb_check);
579    }
580    queued_checks_.pop_front();
581  }
582
583  // Close the database.  We don't simply DeleteSoon() because if a close is
584  // already pending, we'll double-free, and we don't set |database_| to NULL
585  // because if there is still anything running on the db thread, it could
586  // create a new database object (via GetDatabase()) that would then leak.
587  CloseDatabase();
588
589  // Flush the database thread. Any in-progress database check results will be
590  // ignored and cleaned up below.
591  //
592  // Note that to avoid leaking the database, we rely on the fact that no new
593  // tasks will be added to the db thread between the call above and this one.
594  // See comments on the declaration of |safe_browsing_thread_|.
595  {
596    // A ScopedAllowIO object is required to join the thread when calling Stop.
597    // See http://crbug.com/72696. Note that we call Stop() first to clear out
598    // any remaining tasks before clearing safe_browsing_thread_.
599    base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
600    safe_browsing_thread_->Stop();
601    safe_browsing_thread_.reset();
602  }
603
604  // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
605  // We have to do this after the db thread returns because methods on it can
606  // have copies of these pointers, so deleting them might lead to accessing
607  // garbage.
608  for (CurrentChecks::iterator it = checks_.begin();
609       it != checks_.end(); ++it) {
610    SafeBrowsingCheck* check = *it;
611    if (check->client)
612      check->client->OnSafeBrowsingResult(*check);
613  }
614  STLDeleteElements(&checks_);
615
616  gethash_requests_.clear();
617}
618
619bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
620  base::AutoLock lock(database_lock_);
621  return !closing_database_ && (database_ != NULL);
622}
623
624bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
625  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
626  DCHECK(enabled_);
627  if (DatabaseAvailable())
628    return true;
629  safe_browsing_thread_->message_loop()->PostTask(
630      FROM_HERE,
631      base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
632                 this));
633  return false;
634}
635
636void SafeBrowsingDatabaseManager::CloseDatabase() {
637  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
638
639  // Cases to avoid:
640  //  * If |closing_database_| is true, continuing will queue up a second
641  //    request, |closing_database_| will be reset after handling the first
642  //    request, and if any functions on the db thread recreate the database, we
643  //    could start using it on the IO thread and then have the second request
644  //    handler delete it out from under us.
645  //  * If |database_| is NULL, then either no creation request is in flight, in
646  //    which case we don't need to do anything, or one is in flight, in which
647  //    case the database will be recreated before our deletion request is
648  //    handled, and could be used on the IO thread in that time period, leading
649  //    to the same problem as above.
650  //  * If |queued_checks_| is non-empty and |database_| is non-NULL, we're
651  //    about to be called back (in DatabaseLoadComplete()).  This will call
652  //    CheckUrl(), which will want the database.  Closing the database here
653  //    would lead to an infinite loop in DatabaseLoadComplete(), and even if it
654  //    didn't, it would be pointless since we'd just want to recreate.
655  //
656  // The first two cases above are handled by checking DatabaseAvailable().
657  if (!DatabaseAvailable() || !queued_checks_.empty())
658    return;
659
660  closing_database_ = true;
661  if (safe_browsing_thread_.get()) {
662    safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
663        base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
664  }
665}
666
667SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
668  DCHECK_EQ(base::MessageLoop::current(),
669            safe_browsing_thread_->message_loop());
670  if (database_)
671    return database_;
672  startup_metric_utils::ScopedSlowStartupUMA
673      scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
674  const base::TimeTicks before = base::TimeTicks::Now();
675
676  SafeBrowsingDatabase* database =
677      SafeBrowsingDatabase::Create(enable_download_protection_,
678                                   enable_csd_whitelist_,
679                                   enable_download_whitelist_,
680                                   enable_extension_blacklist_,
681                                   enable_side_effect_free_whitelist_,
682                                   enable_ip_blacklist_);
683
684  database->Init(SafeBrowsingService::GetBaseFilename());
685  {
686    // Acquiring the lock here guarantees correct ordering between the writes to
687    // the new database object above, and the setting of |databse_| below.
688    base::AutoLock lock(database_lock_);
689    database_ = database;
690  }
691
692  BrowserThread::PostTask(
693      BrowserThread::IO, FROM_HERE,
694      base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
695
696  UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
697  return database_;
698}
699
700void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
701  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
702
703  if (!enabled_)
704    return;
705
706  // If the service has been shut down, |check| should have been deleted.
707  DCHECK(checks_.find(check) != checks_.end());
708
709  if (check->client && check->need_get_hash) {
710    // We have a partial match so we need to query Google for the full hash.
711    // Clean up will happen in HandleGetHashResults.
712
713    // See if we have a GetHash request already in progress for this particular
714    // prefix. If so, we just append ourselves to the list of interested parties
715    // when the results arrive. We only do this for checks involving one prefix,
716    // since that is the common case (multiple prefixes will issue the request
717    // as normal).
718    if (check->prefix_hits.size() == 1) {
719      SBPrefix prefix = check->prefix_hits[0];
720      GetHashRequests::iterator it = gethash_requests_.find(prefix);
721      if (it != gethash_requests_.end()) {
722        // There's already a request in progress.
723        it->second.push_back(check);
724        return;
725      }
726
727      // No request in progress, so we're the first for this prefix.
728      GetHashRequestors requestors;
729      requestors.push_back(check);
730      gethash_requests_[prefix] = requestors;
731    }
732
733    // Reset the start time so that we can measure the network time without the
734    // database time.
735    check->start = base::TimeTicks::Now();
736    // Note: If |this| is deleted or stopped, the protocol_manager will
737    // be destroyed as well - hence it's OK to do unretained in this case.
738    bool is_download = check->check_type == safe_browsing_util::BINURL ||
739                       check->check_type == safe_browsing_util::BINHASH;
740    sb_service_->protocol_manager()->GetFullHash(
741        check->prefix_hits,
742        base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
743                   base::Unretained(this),
744                   check),
745        is_download);
746  } else {
747    // We may have cached results for previous GetHash queries.  Since
748    // this data comes from cache, don't histogram hits.
749    HandleOneCheck(check, check->full_hits);
750  }
751}
752
753void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
754    GetChunksCallback callback) {
755  DCHECK_EQ(base::MessageLoop::current(),
756            safe_browsing_thread_->message_loop());
757
758  bool database_error = true;
759  std::vector<SBListChunkRanges> lists;
760  DCHECK(!database_update_in_progress_);
761  database_update_in_progress_ = true;
762  GetDatabase();  // This guarantees that |database_| is non-NULL.
763  if (database_->UpdateStarted(&lists)) {
764    database_error = false;
765  } else {
766    database_->UpdateFinished(false);
767  }
768
769  BrowserThread::PostTask(
770      BrowserThread::IO, FROM_HERE,
771      base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
772                 this, lists, database_error, callback));
773}
774
775void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
776    const std::vector<SBListChunkRanges>& lists, bool database_error,
777    GetChunksCallback callback) {
778  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
779  if (enabled_)
780    callback.Run(lists, database_error);
781}
782
783void SafeBrowsingDatabaseManager::OnAddChunksComplete(
784    AddChunksCallback callback) {
785  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
786  if (enabled_)
787    callback.Run();
788}
789
790void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
791  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
792  if (!enabled_)
793    return;
794
795  HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
796  if (queued_checks_.empty())
797    return;
798
799  // If the database isn't already available, calling CheckUrl() in the loop
800  // below will add the check back to the queue, and we'll infinite-loop.
801  DCHECK(DatabaseAvailable());
802  while (!queued_checks_.empty()) {
803    QueuedCheck check = queued_checks_.front();
804    DCHECK(!check.start.is_null());
805    HISTOGRAM_TIMES("SB.QueueDelay", base::TimeTicks::Now() - check.start);
806    // If CheckUrl() determines the URL is safe immediately, it doesn't call the
807    // client's handler function (because normally it's being directly called by
808    // the client).  Since we're not the client, we have to convey this result.
809    if (check.client && CheckBrowseUrl(check.url, check.client)) {
810      SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
811                                 std::vector<SBFullHash>(),
812                                 check.client,
813                                 check.check_type,
814                                 check.expected_threats);
815      check.client->OnSafeBrowsingResult(sb_check);
816    }
817    queued_checks_.pop_front();
818  }
819}
820
821void SafeBrowsingDatabaseManager::AddDatabaseChunks(
822    const std::string& list_name, SBChunkList* chunks,
823    AddChunksCallback callback) {
824  DCHECK_EQ(base::MessageLoop::current(),
825            safe_browsing_thread_->message_loop());
826  if (chunks) {
827    GetDatabase()->InsertChunks(list_name, *chunks);
828    delete chunks;
829  }
830  BrowserThread::PostTask(
831      BrowserThread::IO, FROM_HERE,
832      base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
833                 callback));
834}
835
836void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
837    std::vector<SBChunkDelete>* chunk_deletes) {
838  DCHECK_EQ(base::MessageLoop::current(),
839            safe_browsing_thread_->message_loop());
840  if (chunk_deletes) {
841    GetDatabase()->DeleteChunks(*chunk_deletes);
842    delete chunk_deletes;
843  }
844}
845
846SBThreatType SafeBrowsingDatabaseManager::GetThreatTypeFromListname(
847    const std::string& list_name) {
848  if (safe_browsing_util::IsPhishingList(list_name)) {
849    return SB_THREAT_TYPE_URL_PHISHING;
850  }
851
852  if (safe_browsing_util::IsMalwareList(list_name)) {
853    return SB_THREAT_TYPE_URL_MALWARE;
854  }
855
856  if (safe_browsing_util::IsBadbinurlList(list_name)) {
857    return SB_THREAT_TYPE_BINARY_MALWARE_URL;
858  }
859
860  if (safe_browsing_util::IsBadbinhashList(list_name)) {
861    return SB_THREAT_TYPE_BINARY_MALWARE_HASH;
862  }
863
864  if (safe_browsing_util::IsExtensionList(list_name)) {
865    return SB_THREAT_TYPE_EXTENSION;
866  }
867
868  DVLOG(1) << "Unknown safe browsing list " << list_name;
869  return SB_THREAT_TYPE_SAFE;
870}
871
872void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
873    bool update_succeeded) {
874  DCHECK_EQ(base::MessageLoop::current(),
875            safe_browsing_thread_->message_loop());
876  GetDatabase()->UpdateFinished(update_succeeded);
877  DCHECK(database_update_in_progress_);
878  database_update_in_progress_ = false;
879  BrowserThread::PostTask(
880      BrowserThread::UI, FROM_HERE,
881      base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
882                 this, update_succeeded));
883}
884
885void SafeBrowsingDatabaseManager::OnCloseDatabase() {
886  DCHECK_EQ(base::MessageLoop::current(),
887            safe_browsing_thread_->message_loop());
888  DCHECK(closing_database_);
889
890  // Because |closing_database_| is true, nothing on the IO thread will be
891  // accessing the database, so it's safe to delete and then NULL the pointer.
892  delete database_;
893  database_ = NULL;
894
895  // Acquiring the lock here guarantees correct ordering between the resetting
896  // of |database_| above and of |closing_database_| below, which ensures there
897  // won't be a window during which the IO thread falsely believes the database
898  // is available.
899  base::AutoLock lock(database_lock_);
900  closing_database_ = false;
901}
902
903void SafeBrowsingDatabaseManager::OnResetDatabase() {
904  DCHECK_EQ(base::MessageLoop::current(),
905            safe_browsing_thread_->message_loop());
906  GetDatabase()->ResetDatabase();
907}
908
909void SafeBrowsingDatabaseManager::CacheHashResults(
910  const std::vector<SBPrefix>& prefixes,
911  const std::vector<SBFullHashResult>& full_hashes) {
912  DCHECK_EQ(base::MessageLoop::current(),
913            safe_browsing_thread_->message_loop());
914  GetDatabase()->CacheHashResults(prefixes, full_hashes);
915}
916
917void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
918    SafeBrowsingCheck* check,
919    const std::vector<SBFullHashResult>& full_hashes) {
920  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
921  safe_browsing_util::ListType check_type = check->check_type;
922  SBPrefix prefix = check->prefix_hits[0];
923  GetHashRequests::iterator it = gethash_requests_.find(prefix);
924  if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
925    const bool hit = HandleOneCheck(check, full_hashes);
926    RecordGetHashCheckStatus(hit, check_type, full_hashes);
927    return;
928  }
929
930  // Call back all interested parties, noting if any has a hit.
931  GetHashRequestors& requestors = it->second;
932  bool hit = false;
933  for (GetHashRequestors::iterator r = requestors.begin();
934       r != requestors.end(); ++r) {
935    if (HandleOneCheck(*r, full_hashes))
936      hit = true;
937  }
938  RecordGetHashCheckStatus(hit, check_type, full_hashes);
939
940  gethash_requests_.erase(it);
941}
942
943bool SafeBrowsingDatabaseManager::HandleOneCheck(
944    SafeBrowsingCheck* check,
945    const std::vector<SBFullHashResult>& full_hashes) {
946  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
947  DCHECK(check);
948
949  bool is_threat = false;
950
951  for (size_t i = 0; i < check->urls.size(); ++i) {
952    int index =
953        safe_browsing_util::GetUrlHashIndex(check->urls[i], full_hashes);
954    if (index == -1)
955      continue;
956    SBThreatType threat =
957        GetThreatTypeFromListname(full_hashes[index].list_name);
958    if (threat != SB_THREAT_TYPE_SAFE &&
959        IsExpectedThreat(threat, check->expected_threats)) {
960      check->url_results[i] = threat;
961      is_threat = true;
962    }
963  }
964
965  for (size_t i = 0; i < check->full_hashes.size(); ++i) {
966    int index =
967        safe_browsing_util::GetHashIndex(check->full_hashes[i], full_hashes);
968    if (index == -1)
969      continue;
970    SBThreatType threat =
971        GetThreatTypeFromListname(full_hashes[index].list_name);
972    if (threat != SB_THREAT_TYPE_SAFE &&
973        IsExpectedThreat(threat, check->expected_threats)) {
974      check->full_hash_results[i] = threat;
975      is_threat = true;
976    }
977  }
978
979  SafeBrowsingCheckDone(check);
980  return is_threat;
981}
982
983void SafeBrowsingDatabaseManager::CheckDownloadHashOnSBThread(
984    SafeBrowsingCheck* check) {
985  DCHECK_EQ(base::MessageLoop::current(),
986            safe_browsing_thread_->message_loop());
987  DCHECK(enable_download_protection_);
988
989  DCHECK_EQ(1u, check->full_hashes.size());
990  SBFullHash full_hash = check->full_hashes[0];
991
992  if (!database_->ContainsDownloadHashPrefix(full_hash.prefix)) {
993    // Good, we don't have hash for this url prefix.
994    BrowserThread::PostTask(
995        BrowserThread::IO, FROM_HERE,
996        base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadHashDone, this,
997                   check));
998    return;
999  }
1000
1001  check->need_get_hash = true;
1002  check->prefix_hits.push_back(full_hash.prefix);
1003  BrowserThread::PostTask(
1004      BrowserThread::IO, FROM_HERE,
1005      base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1006}
1007
1008void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1009    SafeBrowsingCheck* check) {
1010  DCHECK_EQ(base::MessageLoop::current(),
1011            safe_browsing_thread_->message_loop());
1012  DCHECK(enable_download_protection_);
1013
1014  std::vector<SBPrefix> prefix_hits;
1015
1016  if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
1017    // Good, we don't have hash for this url prefix.
1018    BrowserThread::PostTask(
1019        BrowserThread::IO, FROM_HERE,
1020        base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
1021                   check));
1022    return;
1023  }
1024
1025  check->need_get_hash = true;
1026  check->prefix_hits.clear();
1027  check->prefix_hits = prefix_hits;
1028  BrowserThread::PostTask(
1029      BrowserThread::IO, FROM_HERE,
1030      base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1031}
1032
1033void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1034    SafeBrowsingCheck* check) {
1035  DCHECK_EQ(base::MessageLoop::current(),
1036            safe_browsing_thread_->message_loop());
1037
1038  std::vector<SBPrefix> prefixes;
1039  for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
1040       it != check->full_hashes.end(); ++it) {
1041    prefixes.push_back((*it).prefix);
1042  }
1043  database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
1044
1045  if (check->prefix_hits.empty()) {
1046    // No matches for any extensions.
1047    BrowserThread::PostTask(
1048        BrowserThread::IO,
1049        FROM_HERE,
1050        base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
1051                   check));
1052  } else {
1053    // Some prefixes matched, we need to ask Google whether they're legit.
1054    check->need_get_hash = true;
1055    BrowserThread::PostTask(
1056        BrowserThread::IO,
1057        FROM_HERE,
1058        base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1059  }
1060}
1061
1062void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1063  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1064  DCHECK(check);
1065
1066  if (!enabled_)
1067    return;
1068
1069  DCHECK(checks_.find(check) != checks_.end());
1070  if (check->client) {
1071    check->client->OnSafeBrowsingResult(*check);
1072    check->client = NULL;
1073  }
1074}
1075
1076void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1077    SafeBrowsingCheck* check) {
1078  DCHECK(enable_download_protection_);
1079  SafeBrowsingCheckDone(check);
1080}
1081
1082void SafeBrowsingDatabaseManager::CheckDownloadHashDone(
1083    SafeBrowsingCheck* check) {
1084  DCHECK(enable_download_protection_);
1085  SafeBrowsingCheckDone(check);
1086}
1087
1088void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1089    SafeBrowsingCheck* check) {
1090  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1091  DCHECK(check);
1092
1093  if (!enabled_)
1094    return;
1095
1096  VLOG(1) << "SafeBrowsingCheckDone";
1097  DCHECK(checks_.find(check) != checks_.end());
1098  if (check->client)
1099    check->client->OnSafeBrowsingResult(*check);
1100  checks_.erase(check);
1101  delete check;
1102}
1103
1104void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1105    SafeBrowsingCheck* check,
1106    const base::Closure& task) {
1107  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1108  check->timeout_factory_.reset(
1109      new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1110  checks_.insert(check);
1111
1112  safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
1113
1114  base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1115      base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1116                 check->timeout_factory_->GetWeakPtr(), check),
1117      check_timeout_);
1118}
1119