download_protection_service.cc revision eb525c5499e34cc9c4b825d6d9e75bb07cc06ace 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/safe_browsing/download_protection_service.h"
6
7#include "base/bind.h"
8#include "base/compiler_specific.h"
9#include "base/format_macros.h"
10#include "base/memory/scoped_ptr.h"
11#include "base/memory/weak_ptr.h"
12#include "base/metrics/histogram.h"
13#include "base/sequenced_task_runner_helpers.h"
14#include "base/stl_util.h"
15#include "base/strings/string_number_conversions.h"
16#include "base/strings/string_util.h"
17#include "base/strings/stringprintf.h"
18#include "base/threading/sequenced_worker_pool.h"
19#include "base/time/time.h"
20#include "chrome/browser/safe_browsing/download_feedback_service.h"
21#include "chrome/browser/safe_browsing/safe_browsing_service.h"
22#include "chrome/browser/safe_browsing/sandboxed_zip_analyzer.h"
23#include "chrome/browser/safe_browsing/signature_util.h"
24#include "chrome/browser/ui/browser.h"
25#include "chrome/browser/ui/browser_list.h"
26#include "chrome/common/safe_browsing/csd.pb.h"
27#include "chrome/common/safe_browsing/download_protection_util.h"
28#include "chrome/common/safe_browsing/zip_analyzer.h"
29#include "chrome/common/url_constants.h"
30#include "content/public/browser/browser_thread.h"
31#include "content/public/browser/download_item.h"
32#include "content/public/browser/page_navigator.h"
33#include "google_apis/google_api_keys.h"
34#include "net/base/escape.h"
35#include "net/base/load_flags.h"
36#include "net/cert/x509_cert_types.h"
37#include "net/cert/x509_certificate.h"
38#include "net/http/http_status_code.h"
39#include "net/url_request/url_fetcher.h"
40#include "net/url_request/url_fetcher_delegate.h"
41#include "net/url_request/url_request_context_getter.h"
42#include "net/url_request/url_request_status.h"
43
44using content::BrowserThread;
45
46namespace {
47static const int64 kDownloadRequestTimeoutMs = 3000;
48}  // namespace
49
50namespace safe_browsing {
51
52const char DownloadProtectionService::kDownloadRequestUrl[] =
53    "https://sb-ssl.google.com/safebrowsing/clientreport/download";
54
55namespace {
56ClientDownloadRequest::DownloadType GetDownloadType(
57    const base::FilePath& file) {
58  DCHECK(download_protection_util::IsBinaryFile(file));
59  if (file.MatchesExtension(FILE_PATH_LITERAL(".apk")))
60    return ClientDownloadRequest::ANDROID_APK;
61  else if (file.MatchesExtension(FILE_PATH_LITERAL(".crx")))
62    return ClientDownloadRequest::CHROME_EXTENSION;
63  // For zip files, we use the ZIPPED_EXECUTABLE type since we will only send
64  // the pingback if we find an executable inside the zip archive.
65  else if (file.MatchesExtension(FILE_PATH_LITERAL(".zip")))
66    return ClientDownloadRequest::ZIPPED_EXECUTABLE;
67  return ClientDownloadRequest::WIN_EXECUTABLE;
68}
69
70// List of extensions for which we track some UMA stats.
71enum MaliciousExtensionType {
72  EXTENSION_EXE,
73  EXTENSION_MSI,
74  EXTENSION_CAB,
75  EXTENSION_SYS,
76  EXTENSION_SCR,
77  EXTENSION_DRV,
78  EXTENSION_BAT,
79  EXTENSION_ZIP,
80  EXTENSION_RAR,
81  EXTENSION_DLL,
82  EXTENSION_PIF,
83  EXTENSION_COM,
84  EXTENSION_JAR,
85  EXTENSION_CLASS,
86  EXTENSION_PDF,
87  EXTENSION_VB,
88  EXTENSION_REG,
89  EXTENSION_GRP,
90  EXTENSION_OTHER,  // Groups all other extensions into one bucket.
91  EXTENSION_CRX,
92  EXTENSION_APK,
93  EXTENSION_MAX,
94};
95
96MaliciousExtensionType GetExtensionType(const base::FilePath& f) {
97  if (f.MatchesExtension(FILE_PATH_LITERAL(".exe"))) return EXTENSION_EXE;
98  if (f.MatchesExtension(FILE_PATH_LITERAL(".msi"))) return EXTENSION_MSI;
99  if (f.MatchesExtension(FILE_PATH_LITERAL(".cab"))) return EXTENSION_CAB;
100  if (f.MatchesExtension(FILE_PATH_LITERAL(".sys"))) return EXTENSION_SYS;
101  if (f.MatchesExtension(FILE_PATH_LITERAL(".scr"))) return EXTENSION_SCR;
102  if (f.MatchesExtension(FILE_PATH_LITERAL(".drv"))) return EXTENSION_DRV;
103  if (f.MatchesExtension(FILE_PATH_LITERAL(".bat"))) return EXTENSION_BAT;
104  if (f.MatchesExtension(FILE_PATH_LITERAL(".zip"))) return EXTENSION_ZIP;
105  if (f.MatchesExtension(FILE_PATH_LITERAL(".rar"))) return EXTENSION_RAR;
106  if (f.MatchesExtension(FILE_PATH_LITERAL(".dll"))) return EXTENSION_DLL;
107  if (f.MatchesExtension(FILE_PATH_LITERAL(".pif"))) return EXTENSION_PIF;
108  if (f.MatchesExtension(FILE_PATH_LITERAL(".com"))) return EXTENSION_COM;
109  if (f.MatchesExtension(FILE_PATH_LITERAL(".jar"))) return EXTENSION_JAR;
110  if (f.MatchesExtension(FILE_PATH_LITERAL(".class"))) return EXTENSION_CLASS;
111  if (f.MatchesExtension(FILE_PATH_LITERAL(".pdf"))) return EXTENSION_PDF;
112  if (f.MatchesExtension(FILE_PATH_LITERAL(".vb"))) return EXTENSION_VB;
113  if (f.MatchesExtension(FILE_PATH_LITERAL(".reg"))) return EXTENSION_REG;
114  if (f.MatchesExtension(FILE_PATH_LITERAL(".grp"))) return EXTENSION_GRP;
115  if (f.MatchesExtension(FILE_PATH_LITERAL(".crx"))) return EXTENSION_CRX;
116  if (f.MatchesExtension(FILE_PATH_LITERAL(".apk"))) return EXTENSION_APK;
117  return EXTENSION_OTHER;
118}
119
120void RecordFileExtensionType(const base::FilePath& file) {
121  UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadExtensions",
122                            GetExtensionType(file),
123                            EXTENSION_MAX);
124}
125
126// Enumerate for histogramming purposes.
127// DO NOT CHANGE THE ORDERING OF THESE VALUES (different histogram data will
128// be mixed together based on their values).
129enum SBStatsType {
130  DOWNLOAD_URL_CHECKS_TOTAL,
131  DOWNLOAD_URL_CHECKS_CANCELED,
132  DOWNLOAD_URL_CHECKS_MALWARE,
133
134  DOWNLOAD_HASH_CHECKS_TOTAL,
135  DOWNLOAD_HASH_CHECKS_MALWARE,
136
137  // Memory space for histograms is determined by the max.
138  // ALWAYS ADD NEW VALUES BEFORE THIS ONE.
139  DOWNLOAD_CHECKS_MAX
140};
141}  // namespace
142
143// Parent SafeBrowsing::Client class used to lookup the bad binary
144// URL and digest list.  There are two sub-classes (one for each list).
145class DownloadSBClient
146    : public SafeBrowsingDatabaseManager::Client,
147      public base::RefCountedThreadSafe<DownloadSBClient> {
148 public:
149  DownloadSBClient(
150      const content::DownloadItem& item,
151      const DownloadProtectionService::CheckDownloadCallback& callback,
152      const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
153      SBStatsType total_type,
154      SBStatsType dangerous_type)
155      : sha256_hash_(item.GetHash()),
156        url_chain_(item.GetUrlChain()),
157        referrer_url_(item.GetReferrerUrl()),
158        callback_(callback),
159        ui_manager_(ui_manager),
160        start_time_(base::TimeTicks::Now()),
161        total_type_(total_type),
162        dangerous_type_(dangerous_type) {}
163
164  virtual void StartCheck() = 0;
165  virtual bool IsDangerous(SBThreatType threat_type) const = 0;
166
167 protected:
168  friend class base::RefCountedThreadSafe<DownloadSBClient>;
169  virtual ~DownloadSBClient() {}
170
171  void CheckDone(SBThreatType threat_type) {
172    DownloadProtectionService::DownloadCheckResult result =
173        IsDangerous(threat_type) ?
174        DownloadProtectionService::DANGEROUS :
175        DownloadProtectionService::SAFE;
176    BrowserThread::PostTask(BrowserThread::UI,
177                            FROM_HERE,
178                            base::Bind(callback_, result));
179    UpdateDownloadCheckStats(total_type_);
180    if (threat_type != SB_THREAT_TYPE_SAFE) {
181      UpdateDownloadCheckStats(dangerous_type_);
182      BrowserThread::PostTask(
183          BrowserThread::UI,
184          FROM_HERE,
185          base::Bind(&DownloadSBClient::ReportMalware,
186                     this, threat_type));
187    }
188  }
189
190  void ReportMalware(SBThreatType threat_type) {
191    std::string post_data;
192    if (!sha256_hash_.empty())
193      post_data += base::HexEncode(sha256_hash_.data(),
194                                   sha256_hash_.size()) + "\n";
195    for (size_t i = 0; i < url_chain_.size(); ++i) {
196      post_data += url_chain_[i].spec() + "\n";
197    }
198    ui_manager_->ReportSafeBrowsingHit(
199        url_chain_.back(),  // malicious_url
200        url_chain_.front(), // page_url
201        referrer_url_,
202        true,  // is_subresource
203        threat_type,
204        post_data);
205  }
206
207  void UpdateDownloadCheckStats(SBStatsType stat_type) {
208    UMA_HISTOGRAM_ENUMERATION("SB2.DownloadChecks",
209                              stat_type,
210                              DOWNLOAD_CHECKS_MAX);
211  }
212
213  std::string sha256_hash_;
214  std::vector<GURL> url_chain_;
215  GURL referrer_url_;
216  DownloadProtectionService::CheckDownloadCallback callback_;
217  scoped_refptr<SafeBrowsingUIManager> ui_manager_;
218  base::TimeTicks start_time_;
219
220 private:
221  const SBStatsType total_type_;
222  const SBStatsType dangerous_type_;
223
224  DISALLOW_COPY_AND_ASSIGN(DownloadSBClient);
225};
226
227class DownloadUrlSBClient : public DownloadSBClient {
228 public:
229  DownloadUrlSBClient(
230      const content::DownloadItem& item,
231      const DownloadProtectionService::CheckDownloadCallback& callback,
232      const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
233      const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager)
234      : DownloadSBClient(item, callback, ui_manager,
235                         DOWNLOAD_URL_CHECKS_TOTAL,
236                         DOWNLOAD_URL_CHECKS_MALWARE),
237        database_manager_(database_manager) { }
238
239  virtual void StartCheck() OVERRIDE {
240    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
241    if (!database_manager_.get() ||
242        database_manager_->CheckDownloadUrl(url_chain_, this)) {
243      CheckDone(SB_THREAT_TYPE_SAFE);
244    } else {
245      AddRef();  // SafeBrowsingService takes a pointer not a scoped_refptr.
246    }
247  }
248
249  virtual bool IsDangerous(SBThreatType threat_type) const OVERRIDE {
250    return threat_type == SB_THREAT_TYPE_BINARY_MALWARE_URL;
251  }
252
253  virtual void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
254                                        SBThreatType threat_type) OVERRIDE {
255    CheckDone(threat_type);
256    UMA_HISTOGRAM_TIMES("SB2.DownloadUrlCheckDuration",
257                        base::TimeTicks::Now() - start_time_);
258    Release();
259  }
260
261 protected:
262  virtual ~DownloadUrlSBClient() {}
263
264 private:
265  scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
266
267  DISALLOW_COPY_AND_ASSIGN(DownloadUrlSBClient);
268};
269
270class DownloadProtectionService::CheckClientDownloadRequest
271    : public base::RefCountedThreadSafe<
272          DownloadProtectionService::CheckClientDownloadRequest,
273          BrowserThread::DeleteOnUIThread>,
274      public net::URLFetcherDelegate,
275      public content::DownloadItem::Observer {
276 public:
277  CheckClientDownloadRequest(
278      content::DownloadItem* item,
279      const CheckDownloadCallback& callback,
280      DownloadProtectionService* service,
281      const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager,
282      SignatureUtil* signature_util)
283      : item_(item),
284        url_chain_(item->GetUrlChain()),
285        referrer_url_(item->GetReferrerUrl()),
286        zipped_executable_(false),
287        callback_(callback),
288        service_(service),
289        signature_util_(signature_util),
290        database_manager_(database_manager),
291        pingback_enabled_(service_->enabled()),
292        finished_(false),
293        type_(ClientDownloadRequest::WIN_EXECUTABLE),
294        weakptr_factory_(this),
295        start_time_(base::TimeTicks::Now()) {
296    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
297    item_->AddObserver(this);
298  }
299
300  void Start() {
301    VLOG(2) << "Starting SafeBrowsing download check for: "
302            << item_->DebugString(true);
303    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
304    // TODO(noelutz): implement some cache to make sure we don't issue the same
305    // request over and over again if a user downloads the same binary multiple
306    // times.
307    DownloadCheckResultReason reason = REASON_MAX;
308    if (!IsSupportedDownload(
309        *item_, item_->GetTargetFilePath(), &reason, &type_)) {
310      switch (reason) {
311        case REASON_EMPTY_URL_CHAIN:
312        case REASON_INVALID_URL:
313          PostFinishTask(SAFE, reason);
314          return;
315
316        case REASON_NOT_BINARY_FILE:
317          RecordFileExtensionType(item_->GetTargetFilePath());
318          PostFinishTask(SAFE, reason);
319          return;
320
321        default:
322          // We only expect the reasons explicitly handled above.
323          NOTREACHED();
324      }
325    }
326    RecordFileExtensionType(item_->GetTargetFilePath());
327
328    // Compute features from the file contents. Note that we record histograms
329    // based on the result, so this runs regardless of whether the pingbacks
330    // are enabled.
331    if (item_->GetTargetFilePath().MatchesExtension(
332        FILE_PATH_LITERAL(".zip"))) {
333      StartExtractZipFeatures();
334    } else {
335      DCHECK(!download_protection_util::IsArchiveFile(
336          item_->GetTargetFilePath()));
337      StartExtractSignatureFeatures();
338    }
339  }
340
341  // Start a timeout to cancel the request if it takes too long.
342  // This should only be called after we have finished accessing the file.
343  void StartTimeout() {
344    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
345    if (!service_) {
346      // Request has already been cancelled.
347      return;
348    }
349    BrowserThread::PostDelayedTask(
350        BrowserThread::UI,
351        FROM_HERE,
352        base::Bind(&CheckClientDownloadRequest::Cancel,
353                   weakptr_factory_.GetWeakPtr()),
354        base::TimeDelta::FromMilliseconds(
355            service_->download_request_timeout_ms()));
356  }
357
358  // Canceling a request will cause us to always report the result as SAFE
359  // unless a pending request is about to call FinishRequest.
360  void Cancel() {
361    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
362    if (fetcher_.get()) {
363      // The DownloadProtectionService is going to release its reference, so we
364      // might be destroyed before the URLFetcher completes.  Cancel the
365      // fetcher so it does not try to invoke OnURLFetchComplete.
366      fetcher_.reset();
367    }
368    // Note: If there is no fetcher, then some callback is still holding a
369    // reference to this object.  We'll eventually wind up in some method on
370    // the UI thread that will call FinishRequest() again.  If FinishRequest()
371    // is called a second time, it will be a no-op.
372    FinishRequest(SAFE, REASON_REQUEST_CANCELED);
373    // Calling FinishRequest might delete this object, we may be deleted by
374    // this point.
375  }
376
377  // content::DownloadItem::Observer implementation.
378  virtual void OnDownloadDestroyed(content::DownloadItem* download) OVERRIDE {
379    Cancel();
380    DCHECK(item_ == NULL);
381  }
382
383  // From the net::URLFetcherDelegate interface.
384  virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE {
385    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
386    DCHECK_EQ(source, fetcher_.get());
387    VLOG(2) << "Received a response for URL: "
388            << item_->GetUrlChain().back() << ": success="
389            << source->GetStatus().is_success() << " response_code="
390            << source->GetResponseCode();
391    DownloadCheckResultReason reason = REASON_SERVER_PING_FAILED;
392    DownloadCheckResult result = SAFE;
393    if (source->GetStatus().is_success() &&
394        net::HTTP_OK == source->GetResponseCode()) {
395      ClientDownloadResponse response;
396      std::string data;
397      bool got_data = source->GetResponseAsString(&data);
398      DCHECK(got_data);
399      if (!response.ParseFromString(data)) {
400        reason = REASON_INVALID_RESPONSE_PROTO;
401      } else if (response.verdict() == ClientDownloadResponse::SAFE) {
402        reason = REASON_DOWNLOAD_SAFE;
403      } else if (service_ && !service_->IsSupportedDownload(
404          *item_, item_->GetTargetFilePath())) {
405        // The client of the download protection service assumes that we don't
406        // support this download so we cannot return any other verdict than
407        // SAFE even if the server says it's dangerous to download this file.
408        // Note: if service_ is NULL we already cancelled the request and
409        // returned SAFE.
410        reason = REASON_DOWNLOAD_NOT_SUPPORTED;
411      } else if (response.verdict() == ClientDownloadResponse::DANGEROUS) {
412        reason = REASON_DOWNLOAD_DANGEROUS;
413        result = DANGEROUS;
414      } else if (response.verdict() == ClientDownloadResponse::UNCOMMON) {
415        reason = REASON_DOWNLOAD_UNCOMMON;
416        result = UNCOMMON;
417      } else if (response.verdict() == ClientDownloadResponse::DANGEROUS_HOST) {
418        reason = REASON_DOWNLOAD_DANGEROUS_HOST;
419        result = DANGEROUS_HOST;
420      } else {
421        LOG(DFATAL) << "Unknown download response verdict: "
422                    << response.verdict();
423        reason = REASON_INVALID_RESPONSE_VERDICT;
424      }
425      DownloadFeedbackService::MaybeStorePingsForDownload(
426          result, item_, client_download_request_data_, data);
427    }
428    // We don't need the fetcher anymore.
429    fetcher_.reset();
430    UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestDuration",
431                        base::TimeTicks::Now() - start_time_);
432    FinishRequest(result, reason);
433  }
434
435  static bool IsSupportedDownload(const content::DownloadItem& item,
436                                  const base::FilePath& target_path,
437                                  DownloadCheckResultReason* reason,
438                                  ClientDownloadRequest::DownloadType* type) {
439    if (item.GetUrlChain().empty()) {
440      *reason = REASON_EMPTY_URL_CHAIN;
441      return false;
442    }
443    const GURL& final_url = item.GetUrlChain().back();
444    if (!final_url.is_valid() || final_url.is_empty() ||
445        !final_url.IsStandard() || final_url.SchemeIsFile()) {
446      *reason = REASON_INVALID_URL;
447      return false;
448    }
449    if (!download_protection_util::IsBinaryFile(target_path)) {
450      *reason = REASON_NOT_BINARY_FILE;
451      return false;
452    }
453    *type = GetDownloadType(target_path);
454    return true;
455  }
456
457 private:
458  friend struct BrowserThread::DeleteOnThread<BrowserThread::UI>;
459  friend class base::DeleteHelper<CheckClientDownloadRequest>;
460
461  virtual ~CheckClientDownloadRequest() {
462    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
463    DCHECK(item_ == NULL);
464  }
465
466  void OnFileFeatureExtractionDone() {
467    // This can run in any thread, since it just posts more messages.
468
469    // TODO(noelutz): DownloadInfo should also contain the IP address of
470    // every URL in the redirect chain.  We also should check whether the
471    // download URL is hosted on the internal network.
472    BrowserThread::PostTask(
473        BrowserThread::IO,
474        FROM_HERE,
475        base::Bind(&CheckClientDownloadRequest::CheckWhitelists, this));
476
477    // We wait until after the file checks finish to start the timeout, as
478    // windows can cause permissions errors if the timeout fired while we were
479    // checking the file signature and we tried to complete the download.
480    BrowserThread::PostTask(
481        BrowserThread::UI,
482        FROM_HERE,
483        base::Bind(&CheckClientDownloadRequest::StartTimeout, this));
484  }
485
486  void StartExtractSignatureFeatures() {
487    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
488    DCHECK(item_);  // Called directly from Start(), item should still exist.
489    // Since we do blocking I/O, offload this to a worker thread.
490    // The task does not need to block shutdown.
491    BrowserThread::GetBlockingPool()->PostWorkerTaskWithShutdownBehavior(
492        FROM_HERE,
493        base::Bind(&CheckClientDownloadRequest::ExtractSignatureFeatures,
494                   this, item_->GetFullPath()),
495        base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN);
496  }
497
498  void ExtractSignatureFeatures(const base::FilePath& file_path) {
499    base::TimeTicks start_time = base::TimeTicks::Now();
500    signature_util_->CheckSignature(file_path, &signature_info_);
501    bool is_signed = (signature_info_.certificate_chain_size() > 0);
502    if (is_signed) {
503      VLOG(2) << "Downloaded a signed binary: " << file_path.value();
504    } else {
505      VLOG(2) << "Downloaded an unsigned binary: "
506              << file_path.value();
507    }
508    UMA_HISTOGRAM_BOOLEAN("SBClientDownload.SignedBinaryDownload", is_signed);
509    UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractSignatureFeaturesTime",
510                        base::TimeTicks::Now() - start_time);
511
512    OnFileFeatureExtractionDone();
513  }
514
515  void StartExtractZipFeatures() {
516    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
517    DCHECK(item_);  // Called directly from Start(), item should still exist.
518    zip_analysis_start_time_ = base::TimeTicks::Now();
519    // We give the zip analyzer a weak pointer to this object.  Since the
520    // analyzer is refcounted, it might outlive the request.
521    analyzer_ = new SandboxedZipAnalyzer(
522        item_->GetFullPath(),
523        base::Bind(&CheckClientDownloadRequest::OnZipAnalysisFinished,
524                   weakptr_factory_.GetWeakPtr()));
525    analyzer_->Start();
526  }
527
528  void OnZipAnalysisFinished(const zip_analyzer::Results& results) {
529    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
530    if (!service_)
531      return;
532    if (results.success) {
533      zipped_executable_ = results.has_executable;
534      VLOG(1) << "Zip analysis finished for " << item_->GetFullPath().value()
535              << ", has_executable=" << results.has_executable
536              << " has_archive=" << results.has_archive;
537    } else {
538      VLOG(1) << "Zip analysis failed for " << item_->GetFullPath().value();
539    }
540    UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasExecutable",
541                          zipped_executable_);
542    UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasArchiveButNoExecutable",
543                          results.has_archive && !zipped_executable_);
544    UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractZipFeaturesTime",
545                        base::TimeTicks::Now() - zip_analysis_start_time_);
546
547    if (!zipped_executable_) {
548      PostFinishTask(SAFE, REASON_ARCHIVE_WITHOUT_BINARIES);
549      return;
550    }
551    OnFileFeatureExtractionDone();
552  }
553
554  void CheckWhitelists() {
555    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
556    DownloadCheckResultReason reason = REASON_MAX;
557    if (!database_manager_.get()) {
558      reason = REASON_SB_DISABLED;
559    } else {
560      for (size_t i = 0; i < url_chain_.size(); ++i) {
561        const GURL& url = url_chain_[i];
562        if (url.is_valid() &&
563            database_manager_->MatchDownloadWhitelistUrl(url)) {
564          VLOG(2) << url << " is on the download whitelist.";
565          reason = REASON_WHITELISTED_URL;
566          break;
567        }
568      }
569      if (referrer_url_.is_valid() && reason == REASON_MAX &&
570          database_manager_->MatchDownloadWhitelistUrl(
571              referrer_url_)) {
572        VLOG(2) << "Referrer url " << referrer_url_
573                << " is on the download whitelist.";
574        reason = REASON_WHITELISTED_REFERRER;
575      }
576      if (reason != REASON_MAX || signature_info_.trusted()) {
577        UMA_HISTOGRAM_COUNTS("SBClientDownload.SignedOrWhitelistedDownload", 1);
578      }
579    }
580    if (reason == REASON_MAX && signature_info_.trusted()) {
581      for (int i = 0; i < signature_info_.certificate_chain_size(); ++i) {
582        if (CertificateChainIsWhitelisted(
583                signature_info_.certificate_chain(i))) {
584          reason = REASON_TRUSTED_EXECUTABLE;
585          break;
586        }
587      }
588    }
589    if (reason != REASON_MAX) {
590      PostFinishTask(SAFE, reason);
591    } else if (!pingback_enabled_) {
592      PostFinishTask(SAFE, REASON_PING_DISABLED);
593    } else {
594      // Currently, the UI only works on Windows so we don't even bother
595      // with pinging the server if we're not on Windows.  TODO(noelutz):
596      // change this code once the UI is done for Linux and Mac.
597#if defined(OS_WIN)
598      // The URLFetcher is owned by the UI thread, so post a message to
599      // start the pingback.
600      BrowserThread::PostTask(
601          BrowserThread::UI,
602          FROM_HERE,
603          base::Bind(&CheckClientDownloadRequest::SendRequest, this));
604#else
605      PostFinishTask(SAFE, REASON_OS_NOT_SUPPORTED);
606#endif
607    }
608  }
609
610  void SendRequest() {
611    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
612
613    // This is our last chance to check whether the request has been canceled
614    // before sending it.
615    if (!service_)
616      return;
617
618    ClientDownloadRequest request;
619    request.set_url(item_->GetUrlChain().back().spec());
620    request.mutable_digests()->set_sha256(item_->GetHash());
621    request.set_length(item_->GetReceivedBytes());
622    for (size_t i = 0; i < item_->GetUrlChain().size(); ++i) {
623      ClientDownloadRequest::Resource* resource = request.add_resources();
624      resource->set_url(item_->GetUrlChain()[i].spec());
625      if (i == item_->GetUrlChain().size() - 1) {
626        // The last URL in the chain is the download URL.
627        resource->set_type(ClientDownloadRequest::DOWNLOAD_URL);
628        resource->set_referrer(item_->GetReferrerUrl().spec());
629        if (!item_->GetRemoteAddress().empty()) {
630          resource->set_remote_ip(item_->GetRemoteAddress());
631        }
632      } else {
633        resource->set_type(ClientDownloadRequest::DOWNLOAD_REDIRECT);
634      }
635      // TODO(noelutz): fill out the remote IP addresses.
636    }
637    request.set_user_initiated(item_->HasUserGesture());
638    request.set_file_basename(
639        item_->GetTargetFilePath().BaseName().AsUTF8Unsafe());
640    request.set_download_type(type_);
641    request.mutable_signature()->CopyFrom(signature_info_);
642    if (!request.SerializeToString(&client_download_request_data_)) {
643      FinishRequest(SAFE, REASON_INVALID_REQUEST_PROTO);
644      return;
645    }
646
647    VLOG(2) << "Sending a request for URL: "
648            << item_->GetUrlChain().back();
649    fetcher_.reset(net::URLFetcher::Create(0 /* ID used for testing */,
650                                           GURL(GetDownloadRequestUrl()),
651                                           net::URLFetcher::POST,
652                                           this));
653    fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE);
654    fetcher_->SetAutomaticallyRetryOn5xx(false);  // Don't retry on error.
655    fetcher_->SetRequestContext(service_->request_context_getter_.get());
656    fetcher_->SetUploadData("application/octet-stream",
657                            client_download_request_data_);
658    fetcher_->Start();
659  }
660
661  void PostFinishTask(DownloadCheckResult result,
662                      DownloadCheckResultReason reason) {
663    BrowserThread::PostTask(
664        BrowserThread::UI,
665        FROM_HERE,
666        base::Bind(&CheckClientDownloadRequest::FinishRequest, this, result,
667                   reason));
668  }
669
670  void FinishRequest(DownloadCheckResult result,
671                     DownloadCheckResultReason reason) {
672    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
673    if (finished_) {
674      return;
675    }
676    finished_ = true;
677    // Ensure the timeout task is cancelled while we still have a non-zero
678    // refcount. (crbug.com/240449)
679    weakptr_factory_.InvalidateWeakPtrs();
680    if (service_) {
681      VLOG(2) << "SafeBrowsing download verdict for: "
682              << item_->DebugString(true) << " verdict:" << reason;
683      UMA_HISTOGRAM_ENUMERATION("SBClientDownload.CheckDownloadStats",
684                                reason,
685                                REASON_MAX);
686      callback_.Run(result);
687      item_->RemoveObserver(this);
688      item_ = NULL;
689      DownloadProtectionService* service = service_;
690      service_ = NULL;
691      service->RequestFinished(this);
692      // DownloadProtectionService::RequestFinished will decrement our refcount,
693      // so we may be deleted now.
694    } else {
695      callback_.Run(SAFE);
696    }
697  }
698
699  bool CertificateChainIsWhitelisted(
700      const ClientDownloadRequest_CertificateChain& chain) {
701    DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
702    if (chain.element_size() < 2) {
703      // We need to have both a signing certificate and its issuer certificate
704      // present to construct a whitelist entry.
705      return false;
706    }
707    scoped_refptr<net::X509Certificate> cert =
708        net::X509Certificate::CreateFromBytes(
709            chain.element(0).certificate().data(),
710            chain.element(0).certificate().size());
711    if (!cert.get()) {
712      return false;
713    }
714
715    for (int i = 1; i < chain.element_size(); ++i) {
716      scoped_refptr<net::X509Certificate> issuer =
717          net::X509Certificate::CreateFromBytes(
718              chain.element(i).certificate().data(),
719              chain.element(i).certificate().size());
720      if (!issuer.get()) {
721        return false;
722      }
723      std::vector<std::string> whitelist_strings;
724      DownloadProtectionService::GetCertificateWhitelistStrings(
725          *cert.get(), *issuer.get(), &whitelist_strings);
726      for (size_t j = 0; j < whitelist_strings.size(); ++j) {
727        if (database_manager_->MatchDownloadWhitelistString(
728                whitelist_strings[j])) {
729          VLOG(2) << "Certificate matched whitelist, cert="
730                  << cert->subject().GetDisplayName()
731                  << " issuer=" << issuer->subject().GetDisplayName();
732          return true;
733        }
734      }
735      cert = issuer;
736    }
737    return false;
738  }
739
740  // The DownloadItem we are checking. Will be NULL if the request has been
741  // canceled. Must be accessed only on UI thread.
742  content::DownloadItem* item_;
743  // Copies of data from |item_| for access on other threads.
744  std::vector<GURL> url_chain_;
745  GURL referrer_url_;
746
747  bool zipped_executable_;
748  ClientDownloadRequest_SignatureInfo signature_info_;
749  CheckDownloadCallback callback_;
750  // Will be NULL if the request has been canceled.
751  DownloadProtectionService* service_;
752  scoped_refptr<SignatureUtil> signature_util_;
753  scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
754  const bool pingback_enabled_;
755  scoped_ptr<net::URLFetcher> fetcher_;
756  scoped_refptr<SandboxedZipAnalyzer> analyzer_;
757  base::TimeTicks zip_analysis_start_time_;
758  bool finished_;
759  ClientDownloadRequest::DownloadType type_;
760  std::string client_download_request_data_;
761  base::WeakPtrFactory<CheckClientDownloadRequest> weakptr_factory_;
762  base::TimeTicks start_time_;  // Used for stats.
763
764  DISALLOW_COPY_AND_ASSIGN(CheckClientDownloadRequest);
765};
766
767DownloadProtectionService::DownloadProtectionService(
768    SafeBrowsingService* sb_service,
769    net::URLRequestContextGetter* request_context_getter)
770    : request_context_getter_(request_context_getter),
771      enabled_(false),
772      signature_util_(new SignatureUtil()),
773      download_request_timeout_ms_(kDownloadRequestTimeoutMs),
774      feedback_service_(new DownloadFeedbackService(
775          request_context_getter, BrowserThread::GetBlockingPool())) {
776
777  if (sb_service) {
778    ui_manager_ = sb_service->ui_manager();
779    database_manager_ = sb_service->database_manager();
780  }
781}
782
783DownloadProtectionService::~DownloadProtectionService() {
784  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
785  CancelPendingRequests();
786}
787
788void DownloadProtectionService::SetEnabled(bool enabled) {
789  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
790  if (enabled == enabled_) {
791    return;
792  }
793  enabled_ = enabled;
794  if (!enabled_) {
795    CancelPendingRequests();
796  }
797}
798
799void DownloadProtectionService::CheckClientDownload(
800    content::DownloadItem* item,
801    const CheckDownloadCallback& callback) {
802  scoped_refptr<CheckClientDownloadRequest> request(
803      new CheckClientDownloadRequest(item, callback, this,
804                                     database_manager_, signature_util_.get()));
805  download_requests_.insert(request);
806  request->Start();
807}
808
809void DownloadProtectionService::CheckDownloadUrl(
810    const content::DownloadItem& item,
811    const CheckDownloadCallback& callback) {
812  DCHECK(!item.GetUrlChain().empty());
813  scoped_refptr<DownloadUrlSBClient> client(
814      new DownloadUrlSBClient(item, callback, ui_manager_, database_manager_));
815  // The client will release itself once it is done.
816  BrowserThread::PostTask(
817        BrowserThread::IO,
818        FROM_HERE,
819        base::Bind(&DownloadUrlSBClient::StartCheck, client));
820}
821
822bool DownloadProtectionService::IsSupportedDownload(
823    const content::DownloadItem& item,
824    const base::FilePath& target_path) const {
825  // Currently, the UI only works on Windows.  On Linux and Mac we still
826  // want to show the dangerous file type warning if the file is possibly
827  // dangerous which means we have to always return false here.
828#if defined(OS_WIN)
829  DownloadCheckResultReason reason = REASON_MAX;
830  ClientDownloadRequest::DownloadType type =
831      ClientDownloadRequest::WIN_EXECUTABLE;
832  return (CheckClientDownloadRequest::IsSupportedDownload(item, target_path,
833                                                          &reason, &type) &&
834          (ClientDownloadRequest::ANDROID_APK == type ||
835           ClientDownloadRequest::WIN_EXECUTABLE == type ||
836           ClientDownloadRequest::ZIPPED_EXECUTABLE == type));
837#else
838  return false;
839#endif
840}
841
842void DownloadProtectionService::CancelPendingRequests() {
843  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
844  for (std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
845           download_requests_.begin();
846       it != download_requests_.end();) {
847    // We need to advance the iterator before we cancel because canceling
848    // the request will invalidate it when RequestFinished is called below.
849    scoped_refptr<CheckClientDownloadRequest> tmp = *it++;
850    tmp->Cancel();
851  }
852  DCHECK(download_requests_.empty());
853}
854
855void DownloadProtectionService::RequestFinished(
856    CheckClientDownloadRequest* request) {
857  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
858  std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
859      download_requests_.find(request);
860  DCHECK(it != download_requests_.end());
861  download_requests_.erase(*it);
862}
863
864void DownloadProtectionService::ShowDetailsForDownload(
865    const content::DownloadItem& item,
866    content::PageNavigator* navigator) {
867  navigator->OpenURL(
868      content::OpenURLParams(GURL(chrome::kDownloadScanningLearnMoreURL),
869                             content::Referrer(),
870                             NEW_FOREGROUND_TAB,
871                             content::PAGE_TRANSITION_LINK,
872                             false));
873}
874
875namespace {
876// Escapes a certificate attribute so that it can be used in a whitelist
877// entry.  Currently, we only escape slashes, since they are used as a
878// separator between attributes.
879std::string EscapeCertAttribute(const std::string& attribute) {
880  std::string escaped;
881  for (size_t i = 0; i < attribute.size(); ++i) {
882    if (attribute[i] == '%') {
883      escaped.append("%25");
884    } else if (attribute[i] == '/') {
885      escaped.append("%2F");
886    } else {
887      escaped.push_back(attribute[i]);
888    }
889  }
890  return escaped;
891}
892}  // namespace
893
894// static
895void DownloadProtectionService::GetCertificateWhitelistStrings(
896    const net::X509Certificate& certificate,
897    const net::X509Certificate& issuer,
898    std::vector<std::string>* whitelist_strings) {
899  // The whitelist paths are in the format:
900  // cert/<ascii issuer fingerprint>[/CN=common_name][/O=org][/OU=unit]
901  //
902  // Any of CN, O, or OU may be omitted from the whitelist entry, in which
903  // case they match anything.  However, the attributes that do appear will
904  // always be in the order shown above.  At least one attribute will always
905  // be present.
906
907  const net::CertPrincipal& subject = certificate.subject();
908  std::vector<std::string> ou_tokens;
909  for (size_t i = 0; i < subject.organization_unit_names.size(); ++i) {
910    ou_tokens.push_back(
911        "/OU=" + EscapeCertAttribute(subject.organization_unit_names[i]));
912  }
913
914  std::vector<std::string> o_tokens;
915  for (size_t i = 0; i < subject.organization_names.size(); ++i) {
916    o_tokens.push_back(
917        "/O=" + EscapeCertAttribute(subject.organization_names[i]));
918  }
919
920  std::string cn_token;
921  if (!subject.common_name.empty()) {
922    cn_token = "/CN=" + EscapeCertAttribute(subject.common_name);
923  }
924
925  std::set<std::string> paths_to_check;
926  if (!cn_token.empty()) {
927    paths_to_check.insert(cn_token);
928  }
929  for (size_t i = 0; i < o_tokens.size(); ++i) {
930    paths_to_check.insert(cn_token + o_tokens[i]);
931    paths_to_check.insert(o_tokens[i]);
932    for (size_t j = 0; j < ou_tokens.size(); ++j) {
933      paths_to_check.insert(cn_token + o_tokens[i] + ou_tokens[j]);
934      paths_to_check.insert(o_tokens[i] + ou_tokens[j]);
935    }
936  }
937  for (size_t i = 0; i < ou_tokens.size(); ++i) {
938    paths_to_check.insert(cn_token + ou_tokens[i]);
939    paths_to_check.insert(ou_tokens[i]);
940  }
941
942  std::string issuer_fp = base::HexEncode(issuer.fingerprint().data,
943                                          sizeof(issuer.fingerprint().data));
944  for (std::set<std::string>::iterator it = paths_to_check.begin();
945       it != paths_to_check.end(); ++it) {
946    whitelist_strings->push_back("cert/" + issuer_fp + *it);
947  }
948}
949
950// static
951std::string DownloadProtectionService::GetDownloadRequestUrl() {
952  std::string url = kDownloadRequestUrl;
953  std::string api_key = google_apis::GetAPIKey();
954  if (!api_key.empty()) {
955    base::StringAppendF(&url, "?key=%s",
956                        net::EscapeQueryParamValue(api_key, true).c_str());
957  }
958  return url;
959}
960
961}  // namespace safe_browsing
962