1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/media_galleries/media_scan_manager.h"
6
7#include "base/files/file_enumerator.h"
8#include "base/files/file_util.h"
9#include "base/logging.h"
10#include "base/metrics/histogram.h"
11#include "base/time/time.h"
12#include "chrome/browser/extensions/extension_service.h"
13#include "chrome/browser/media_galleries/media_galleries_preferences.h"
14#include "chrome/browser/media_galleries/media_galleries_preferences_factory.h"
15#include "chrome/browser/media_galleries/media_scan_manager_observer.h"
16#include "chrome/browser/profiles/profile.h"
17#include "chrome/common/extensions/api/media_galleries.h"
18#include "content/public/browser/browser_thread.h"
19#include "extensions/browser/extension_registry.h"
20#include "extensions/browser/extension_system.h"
21#include "extensions/common/extension.h"
22
23using extensions::ExtensionRegistry;
24
25namespace media_galleries = extensions::api::media_galleries;
26
27namespace {
28
29typedef std::set<std::string /*extension id*/> ScanningExtensionIdSet;
30
31// When multiple scan results have the same parent, sometimes it makes sense
32// to combine them into a single scan result at the parent. This constant
33// governs when that happens; kContainerDirectoryMinimumPercent percent of the
34// directories in the parent directory must be scan results.
35const int kContainerDirectoryMinimumPercent = 80;
36
37// How long after a completed media scan can we provide the cached results.
38const int kScanResultsExpiryTimeInHours = 24;
39
40struct LocationInfo {
41  LocationInfo()
42      : pref_id(kInvalidMediaGalleryPrefId),
43        type(MediaGalleryPrefInfo::kInvalidType) {}
44  LocationInfo(MediaGalleryPrefId pref_id, MediaGalleryPrefInfo::Type type,
45               base::FilePath path)
46      : pref_id(pref_id), type(type), path(path) {}
47  // Highest priority comparison by path, next by type (scan result last),
48  // then by pref id (invalid last).
49  bool operator<(const LocationInfo& rhs) const {
50    if (path.value() == rhs.path.value()) {
51      if (type == rhs.type) {
52        return pref_id > rhs.pref_id;
53      }
54      return rhs.type == MediaGalleryPrefInfo::kScanResult;
55    }
56    return path.value() < rhs.path.value();
57  }
58
59  MediaGalleryPrefId pref_id;
60  MediaGalleryPrefInfo::Type type;
61  base::FilePath path;
62  MediaGalleryScanResult file_counts;
63};
64
65// Finds new scan results that are shadowed (the same location, or a child) by
66// existing locations and moves them from |found_folders| to |child_folders|.
67// Also moves new scan results that are shadowed by other new scan results
68// to |child_folders|.
69void PartitionChildScanResults(
70    MediaGalleriesPreferences* preferences,
71    MediaFolderFinder::MediaFolderFinderResults* found_folders,
72    MediaFolderFinder::MediaFolderFinderResults* child_folders) {
73  // Construct a list with everything in it.
74  std::vector<LocationInfo> all_locations;
75  for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it =
76           found_folders->begin(); it != found_folders->end(); ++it) {
77    all_locations.push_back(LocationInfo(kInvalidMediaGalleryPrefId,
78                                         MediaGalleryPrefInfo::kScanResult,
79                                         it->first));
80    all_locations.back().file_counts = it->second;
81  }
82  const MediaGalleriesPrefInfoMap& known_galleries =
83      preferences->known_galleries();
84  for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin();
85       it != known_galleries.end();
86       ++it) {
87    all_locations.push_back(LocationInfo(it->second.pref_id, it->second.type,
88                                         it->second.AbsolutePath()));
89  }
90  // Sorting on path should put all paths that are prefixes of other paths
91  // next to each other, with the shortest one first.
92  std::sort(all_locations.begin(), all_locations.end());
93
94  size_t previous_parent_index = 0;
95  for (size_t i = 1; i < all_locations.size(); i++) {
96    const LocationInfo& current = all_locations[i];
97    const LocationInfo& previous_parent = all_locations[previous_parent_index];
98    bool is_child = previous_parent.path.IsParent(current.path);
99    if (current.type == MediaGalleryPrefInfo::kScanResult &&
100        current.pref_id == kInvalidMediaGalleryPrefId &&
101        (is_child || previous_parent.path == current.path)) {
102      // Move new scan results that are shadowed.
103      (*child_folders)[current.path] = current.file_counts;
104      found_folders->erase(current.path);
105    } else if (!is_child) {
106      previous_parent_index = i;
107    }
108  }
109}
110
111MediaGalleryScanResult SumFilesUnderPath(
112    const base::FilePath& path,
113    const MediaFolderFinder::MediaFolderFinderResults& candidates) {
114  MediaGalleryScanResult results;
115  for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it =
116           candidates.begin(); it != candidates.end(); ++it) {
117    if (it->first == path || path.IsParent(it->first)) {
118      results.audio_count += it->second.audio_count;
119      results.image_count += it->second.image_count;
120      results.video_count += it->second.video_count;
121    }
122  }
123  return results;
124}
125
126void AddScanResultsForProfile(
127    MediaGalleriesPreferences* preferences,
128    const MediaFolderFinder::MediaFolderFinderResults& found_folders) {
129  // First, remove any existing scan results where no app has been granted
130  // permission - either it is gone, or is already in the new scan results.
131  // This burns some pref ids, but not at an appreciable rate.
132  MediaGalleryPrefIdSet to_remove;
133  const MediaGalleriesPrefInfoMap& known_galleries =
134      preferences->known_galleries();
135  for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin();
136       it != known_galleries.end();
137       ++it) {
138    if (it->second.type == MediaGalleryPrefInfo::kScanResult &&
139        !preferences->NonAutoGalleryHasPermission(it->first)) {
140      to_remove.insert(it->first);
141    }
142  }
143  for (MediaGalleryPrefIdSet::const_iterator it = to_remove.begin();
144       it != to_remove.end();
145       ++it) {
146    preferences->EraseGalleryById(*it);
147  }
148
149  MediaFolderFinder::MediaFolderFinderResults child_folders;
150  MediaFolderFinder::MediaFolderFinderResults
151      unique_found_folders(found_folders);
152  PartitionChildScanResults(preferences, &unique_found_folders, &child_folders);
153
154  // Updating prefs while iterating them will invalidate the pointer, so
155  // calculate the changes first and then apply them.
156  std::map<MediaGalleryPrefId, MediaGalleryScanResult> to_update;
157  for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin();
158       it != known_galleries.end();
159       ++it) {
160    const MediaGalleryPrefInfo& gallery = it->second;
161    if (!gallery.IsBlackListedType()) {
162      MediaGalleryScanResult file_counts =
163          SumFilesUnderPath(gallery.AbsolutePath(), child_folders);
164      if (gallery.audio_count != file_counts.audio_count ||
165          gallery.image_count != file_counts.image_count ||
166          gallery.video_count != file_counts.video_count) {
167        to_update[it->first] = file_counts;
168      }
169    }
170  }
171
172  for (std::map<MediaGalleryPrefId,
173                MediaGalleryScanResult>::const_iterator it = to_update.begin();
174       it != to_update.end();
175       ++it) {
176    const MediaGalleryPrefInfo& gallery =
177        preferences->known_galleries().find(it->first)->second;
178      preferences->AddGallery(gallery.device_id, gallery.path, gallery.type,
179                              gallery.volume_label, gallery.vendor_name,
180                              gallery.model_name, gallery.total_size_in_bytes,
181                              gallery.last_attach_time,
182                              it->second.audio_count,
183                              it->second.image_count,
184                              it->second.video_count);
185  }
186
187  // Add new scan results.
188  for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it =
189           unique_found_folders.begin();
190       it != unique_found_folders.end();
191       ++it) {
192    MediaGalleryScanResult file_counts =
193        SumFilesUnderPath(it->first, child_folders);
194    // The top level scan result is not in |child_folders|. Add it in as well.
195    file_counts.audio_count += it->second.audio_count;
196    file_counts.image_count += it->second.image_count;
197    file_counts.video_count += it->second.video_count;
198
199    MediaGalleryPrefInfo gallery;
200    bool existing = preferences->LookUpGalleryByPath(it->first, &gallery);
201    DCHECK(!existing);
202    preferences->AddGallery(gallery.device_id, gallery.path,
203                            MediaGalleryPrefInfo::kScanResult,
204                            gallery.volume_label, gallery.vendor_name,
205                            gallery.model_name, gallery.total_size_in_bytes,
206                            gallery.last_attach_time, file_counts.audio_count,
207                            file_counts.image_count, file_counts.video_count);
208  }
209  UMA_HISTOGRAM_COUNTS_10000("MediaGalleries.ScanGalleriesPopulated",
210                             unique_found_folders.size() + to_update.size());
211}
212
213int CountScanResultsForExtension(MediaGalleriesPreferences* preferences,
214                                 const extensions::Extension* extension,
215                                 MediaGalleryScanResult* file_counts) {
216  int gallery_count = 0;
217
218  MediaGalleryPrefIdSet permitted_galleries =
219      preferences->GalleriesForExtension(*extension);
220  const MediaGalleriesPrefInfoMap& known_galleries =
221      preferences->known_galleries();
222  for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin();
223       it != known_galleries.end();
224       ++it) {
225    if (it->second.type == MediaGalleryPrefInfo::kScanResult &&
226        !ContainsKey(permitted_galleries, it->first)) {
227      gallery_count++;
228      file_counts->audio_count += it->second.audio_count;
229      file_counts->image_count += it->second.image_count;
230      file_counts->video_count += it->second.video_count;
231    }
232  }
233  return gallery_count;
234}
235
236int CountDirectoryEntries(const base::FilePath& path) {
237  base::FileEnumerator dir_counter(
238      path, false /*recursive*/, base::FileEnumerator::DIRECTORIES);
239  int count = 0;
240  base::FileEnumerator::FileInfo info;
241  for (base::FilePath name = dir_counter.Next(); !name.empty();
242       name = dir_counter.Next()) {
243    if (!base::IsLink(name))
244      ++count;
245  }
246  return count;
247}
248
249struct ContainerCount {
250  int seen_count, entries_count;
251  bool is_qualified;
252
253  ContainerCount() : seen_count(0), entries_count(-1), is_qualified(false) {}
254};
255
256typedef std::map<base::FilePath, ContainerCount> ContainerCandidates;
257
258}  // namespace
259
260MediaScanManager::MediaScanManager()
261    : scoped_extension_registry_observer_(this),
262      weak_factory_(this) {
263  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
264}
265
266MediaScanManager::~MediaScanManager() {
267  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
268}
269
270void MediaScanManager::AddObserver(Profile* profile,
271                                   MediaScanManagerObserver* observer) {
272  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
273  DCHECK(!ContainsKey(observers_, profile));
274  observers_[profile].observer = observer;
275}
276
277void MediaScanManager::RemoveObserver(Profile* profile) {
278  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
279  bool scan_in_progress = ScanInProgress();
280  observers_.erase(profile);
281  DCHECK_EQ(scan_in_progress, ScanInProgress());
282}
283
284void MediaScanManager::CancelScansForProfile(Profile* profile) {
285  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
286  observers_[profile].scanning_extensions.clear();
287
288  if (!ScanInProgress())
289    folder_finder_.reset();
290}
291
292void MediaScanManager::StartScan(Profile* profile,
293                                 const extensions::Extension* extension,
294                                 bool user_gesture) {
295  DCHECK(extension);
296  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
297
298  ScanObserverMap::iterator scans_for_profile = observers_.find(profile);
299  // We expect that an MediaScanManagerObserver has already been registered.
300  DCHECK(scans_for_profile != observers_.end());
301  bool scan_in_progress = ScanInProgress();
302  // Ignore requests for extensions that are already scanning.
303  ScanningExtensionIdSet* scanning_extensions;
304  scanning_extensions = &scans_for_profile->second.scanning_extensions;
305  if (scan_in_progress && ContainsKey(*scanning_extensions, extension->id()))
306    return;
307
308  // Provide cached result if there is not already a scan in progress,
309  // there is no user gesture, and the previous results are unexpired.
310  MediaGalleriesPreferences* preferences =
311      MediaGalleriesPreferencesFactory::GetForProfile(profile);
312  base::TimeDelta time_since_last_scan =
313      base::Time::Now() - preferences->GetLastScanCompletionTime();
314  if (!scan_in_progress && !user_gesture && time_since_last_scan <
315          base::TimeDelta::FromHours(kScanResultsExpiryTimeInHours)) {
316    MediaGalleryScanResult file_counts;
317    int gallery_count =
318        CountScanResultsForExtension(preferences, extension, &file_counts);
319    scans_for_profile->second.observer->OnScanStarted(extension->id());
320    scans_for_profile->second.observer->OnScanFinished(extension->id(),
321                                                       gallery_count,
322                                                       file_counts);
323    return;
324  }
325
326  // On first scan for the |profile|, register to listen for extension unload.
327  if (scanning_extensions->empty())
328    scoped_extension_registry_observer_.Add(ExtensionRegistry::Get(profile));
329
330  scanning_extensions->insert(extension->id());
331  scans_for_profile->second.observer->OnScanStarted(extension->id());
332
333  if (folder_finder_)
334    return;
335
336  MediaFolderFinder::MediaFolderFinderResultsCallback callback =
337      base::Bind(&MediaScanManager::OnScanCompleted,
338                 weak_factory_.GetWeakPtr());
339  if (testing_folder_finder_factory_.is_null()) {
340    folder_finder_.reset(new MediaFolderFinder(callback));
341  } else {
342    folder_finder_.reset(testing_folder_finder_factory_.Run(callback));
343  }
344  scan_start_time_ = base::Time::Now();
345  folder_finder_->StartScan();
346}
347
348void MediaScanManager::CancelScan(Profile* profile,
349                                  const extensions::Extension* extension) {
350  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
351
352  // Erases the logical scan if found, early exit otherwise.
353  ScanObserverMap::iterator scans_for_profile = observers_.find(profile);
354  if (scans_for_profile == observers_.end() ||
355      !scans_for_profile->second.scanning_extensions.erase(extension->id())) {
356    return;
357  }
358
359  scans_for_profile->second.observer->OnScanCancelled(extension->id());
360
361  // No more scanning extensions for |profile|, so stop listening for unloads.
362  if (scans_for_profile->second.scanning_extensions.empty())
363    scoped_extension_registry_observer_.Remove(ExtensionRegistry::Get(profile));
364
365  if (!ScanInProgress()) {
366    folder_finder_.reset();
367    DCHECK(!scan_start_time_.is_null());
368    UMA_HISTOGRAM_LONG_TIMES("MediaGalleries.ScanCancelTime",
369                             base::Time::Now() - scan_start_time_);
370    scan_start_time_ = base::Time();
371  }
372}
373
374void MediaScanManager::SetMediaFolderFinderFactory(
375    const MediaFolderFinderFactory& factory) {
376  testing_folder_finder_factory_ = factory;
377}
378
379// A single directory may contain many folders with media in them, without
380// containing any media itself. In fact, the primary purpose of that directory
381// may be to contain media directories. This function tries to find those
382// container directories.
383MediaFolderFinder::MediaFolderFinderResults
384MediaScanManager::FindContainerScanResults(
385    const MediaFolderFinder::MediaFolderFinderResults& found_folders,
386    const std::vector<base::FilePath>& sensitive_locations) {
387  DCHECK_CURRENTLY_ON(content::BrowserThread::FILE);
388  std::vector<base::FilePath> abs_sensitive_locations;
389  for (size_t i = 0; i < sensitive_locations.size(); ++i) {
390    base::FilePath path = base::MakeAbsoluteFilePath(sensitive_locations[i]);
391    if (!path.empty())
392      abs_sensitive_locations.push_back(path);
393  }
394  // Recursively find parent directories with majority of media directories,
395  // or container directories.
396  // |candidates| keeps track of directories which might have enough
397  // such directories to have us return them.
398  typedef std::map<base::FilePath, ContainerCount> ContainerCandidates;
399  ContainerCandidates candidates;
400  for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it =
401           found_folders.begin();
402       it != found_folders.end();
403       ++it) {
404    base::FilePath child_directory = it->first;
405    base::FilePath parent_directory = child_directory.DirName();
406
407    // Parent of root is root.
408    while (!parent_directory.empty() && child_directory != parent_directory) {
409      // Skip sensitive folders and their ancestors.
410      base::FilePath abs_parent_directory =
411          base::MakeAbsoluteFilePath(parent_directory);
412      if (abs_parent_directory.empty())
413        break;
414      bool is_sensitive = false;
415      for (size_t i = 0; i < abs_sensitive_locations.size(); ++i) {
416        if (abs_parent_directory == abs_sensitive_locations[i] ||
417            abs_parent_directory.IsParent(abs_sensitive_locations[i])) {
418          is_sensitive = true;
419          break;
420        }
421      }
422      if (is_sensitive)
423        break;
424
425      // Don't bother with ones we already have.
426      if (found_folders.find(parent_directory) != found_folders.end())
427        continue;
428
429      ContainerCandidates::iterator parent_it =
430          candidates.find(parent_directory);
431      if (parent_it == candidates.end()) {
432        ContainerCount count;
433        count.seen_count = 1;
434        count.entries_count = CountDirectoryEntries(parent_directory);
435        parent_it =
436            candidates.insert(std::make_pair(parent_directory, count)).first;
437      } else {
438        ++candidates[parent_directory].seen_count;
439      }
440      // If previously sufficient, or not sufficient, bail.
441      if (parent_it->second.is_qualified ||
442          parent_it->second.seen_count * 100 / parent_it->second.entries_count <
443              kContainerDirectoryMinimumPercent) {
444        break;
445      }
446      // Otherwise, mark qualified and check parent.
447      parent_it->second.is_qualified = true;
448      child_directory = parent_directory;
449      parent_directory = child_directory.DirName();
450    }
451  }
452  MediaFolderFinder::MediaFolderFinderResults result;
453  // Copy and return worthy results.
454  for (ContainerCandidates::const_iterator it = candidates.begin();
455       it != candidates.end();
456       ++it) {
457    if (it->second.is_qualified && it->second.seen_count >= 2)
458      result[it->first] = MediaGalleryScanResult();
459  }
460  return result;
461}
462
463MediaScanManager::ScanObservers::ScanObservers() : observer(NULL) {}
464MediaScanManager::ScanObservers::~ScanObservers() {}
465
466void MediaScanManager::OnExtensionUnloaded(
467    content::BrowserContext* browser_context,
468    const extensions::Extension* extension,
469    extensions::UnloadedExtensionInfo::Reason reason) {
470  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
471  CancelScan(Profile::FromBrowserContext(browser_context), extension);
472}
473
474bool MediaScanManager::ScanInProgress() const {
475  for (ScanObserverMap::const_iterator it = observers_.begin();
476       it != observers_.end();
477       ++it) {
478    if (!it->second.scanning_extensions.empty())
479      return true;
480  }
481  return false;
482}
483
484void MediaScanManager::OnScanCompleted(
485    bool success,
486    const MediaFolderFinder::MediaFolderFinderResults& found_folders) {
487  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
488  if (!folder_finder_ || !success) {
489    folder_finder_.reset();
490    return;
491  }
492
493  UMA_HISTOGRAM_COUNTS_10000("MediaGalleries.ScanDirectoriesFound",
494                             found_folders.size());
495  DCHECK(!scan_start_time_.is_null());
496  UMA_HISTOGRAM_LONG_TIMES("MediaGalleries.ScanFinishedTime",
497                           base::Time::Now() - scan_start_time_);
498  scan_start_time_ = base::Time();
499
500  content::BrowserThread::PostTaskAndReplyWithResult(
501      content::BrowserThread::FILE, FROM_HERE,
502      base::Bind(FindContainerScanResults,
503                 found_folders,
504                 folder_finder_->graylisted_folders()),
505      base::Bind(&MediaScanManager::OnFoundContainerDirectories,
506                 weak_factory_.GetWeakPtr(),
507                 found_folders));
508}
509
510void MediaScanManager::OnFoundContainerDirectories(
511    const MediaFolderFinder::MediaFolderFinderResults& found_folders,
512    const MediaFolderFinder::MediaFolderFinderResults& container_folders) {
513  MediaFolderFinder::MediaFolderFinderResults folders;
514  folders.insert(found_folders.begin(), found_folders.end());
515  folders.insert(container_folders.begin(), container_folders.end());
516
517  for (ScanObserverMap::iterator scans_for_profile = observers_.begin();
518       scans_for_profile != observers_.end();
519       ++scans_for_profile) {
520    if (scans_for_profile->second.scanning_extensions.empty())
521      continue;
522    Profile* profile = scans_for_profile->first;
523    MediaGalleriesPreferences* preferences =
524        MediaGalleriesPreferencesFactory::GetForProfile(profile);
525    ExtensionService* extension_service =
526        extensions::ExtensionSystem::Get(profile)->extension_service();
527    if (!extension_service)
528      continue;
529
530    AddScanResultsForProfile(preferences, folders);
531
532    ScanningExtensionIdSet* scanning_extensions =
533        &scans_for_profile->second.scanning_extensions;
534    for (ScanningExtensionIdSet::const_iterator extension_id_it =
535             scanning_extensions->begin();
536         extension_id_it != scanning_extensions->end();
537         ++extension_id_it) {
538      const extensions::Extension* extension =
539          extension_service->GetExtensionById(*extension_id_it, false);
540      if (extension) {
541        MediaGalleryScanResult file_counts;
542        int gallery_count = CountScanResultsForExtension(preferences, extension,
543                                                         &file_counts);
544        scans_for_profile->second.observer->OnScanFinished(*extension_id_it,
545                                                           gallery_count,
546                                                           file_counts);
547      }
548    }
549    scanning_extensions->clear();
550    preferences->SetLastScanCompletionTime(base::Time::Now());
551  }
552  scoped_extension_registry_observer_.RemoveAll();
553  folder_finder_.reset();
554}
555