local_file_change_tracker.cc revision 116680a4aac90f2aa7413d9095a592090648e557 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
6
7#include <queue>
8
9#include "base/location.h"
10#include "base/logging.h"
11#include "base/sequenced_task_runner.h"
12#include "base/stl_util.h"
13#include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14#include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15#include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
16#include "third_party/leveldatabase/src/include/leveldb/db.h"
17#include "third_party/leveldatabase/src/include/leveldb/env.h"
18#include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
19#include "webkit/browser/fileapi/file_system_context.h"
20#include "webkit/browser/fileapi/file_system_file_util.h"
21#include "webkit/browser/fileapi/file_system_operation_context.h"
22#include "webkit/common/fileapi/file_system_util.h"
23
24using fileapi::FileSystemContext;
25using fileapi::FileSystemFileUtil;
26using fileapi::FileSystemOperationContext;
27using fileapi::FileSystemURL;
28using fileapi::FileSystemURLSet;
29
30namespace sync_file_system {
31
32namespace {
33const base::FilePath::CharType kDatabaseName[] =
34    FILE_PATH_LITERAL("LocalFileChangeTracker");
35const char kMark[] = "d";
36}  // namespace
37
38// A database class that stores local file changes in a local database. This
39// object must be destructed on file_task_runner.
40class LocalFileChangeTracker::TrackerDB {
41 public:
42  TrackerDB(const base::FilePath& base_path,
43            leveldb::Env* env_override);
44
45  SyncStatusCode MarkDirty(const std::string& url);
46  SyncStatusCode ClearDirty(const std::string& url);
47  SyncStatusCode GetDirtyEntries(
48      std::queue<FileSystemURL>* dirty_files);
49  SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch);
50
51 private:
52  enum RecoveryOption {
53    REPAIR_ON_CORRUPTION,
54    FAIL_ON_CORRUPTION,
55  };
56
57  SyncStatusCode Init(RecoveryOption recovery_option);
58  SyncStatusCode Repair(const std::string& db_path);
59  void HandleError(const tracked_objects::Location& from_here,
60                   const leveldb::Status& status);
61
62  const base::FilePath base_path_;
63  leveldb::Env* env_override_;
64  scoped_ptr<leveldb::DB> db_;
65  SyncStatusCode db_status_;
66
67  DISALLOW_COPY_AND_ASSIGN(TrackerDB);
68};
69
70LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
71LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
72
73// LocalFileChangeTracker ------------------------------------------------------
74
75LocalFileChangeTracker::LocalFileChangeTracker(
76    const base::FilePath& base_path,
77    leveldb::Env* env_override,
78    base::SequencedTaskRunner* file_task_runner)
79    : initialized_(false),
80      file_task_runner_(file_task_runner),
81      tracker_db_(new TrackerDB(base_path, env_override)),
82      current_change_seq_(0),
83      num_changes_(0) {
84}
85
86LocalFileChangeTracker::~LocalFileChangeTracker() {
87  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
88  tracker_db_.reset();
89}
90
91void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) {
92  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
93  if (ContainsKey(changes_, url))
94    return;
95  // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
96  MarkDirtyOnDatabase(url);
97}
98
99void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {}
100
101void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) {
102  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
103                               SYNC_FILE_TYPE_FILE));
104}
105
106void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url,
107                                              const FileSystemURL& src) {
108  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
109                               SYNC_FILE_TYPE_FILE));
110}
111
112void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) {
113  RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
114                               SYNC_FILE_TYPE_FILE));
115}
116
117void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) {
118  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
119                               SYNC_FILE_TYPE_FILE));
120}
121
122void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) {
123  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
124                               SYNC_FILE_TYPE_DIRECTORY));
125}
126
127void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) {
128  RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
129                               SYNC_FILE_TYPE_DIRECTORY));
130}
131
132void LocalFileChangeTracker::GetNextChangedURLs(
133    std::deque<FileSystemURL>* urls, int max_urls) {
134  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
135  DCHECK(urls);
136  urls->clear();
137  // Mildly prioritizes the URLs that older changes and have not been updated
138  // for a while.
139  for (ChangeSeqMap::iterator iter = change_seqs_.begin();
140       iter != change_seqs_.end() &&
141       (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls));
142       ++iter) {
143    urls->push_back(iter->second);
144  }
145}
146
147void LocalFileChangeTracker::GetChangesForURL(
148    const FileSystemURL& url, FileChangeList* changes) {
149  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
150  DCHECK(changes);
151  changes->clear();
152  FileChangeMap::iterator found = changes_.find(url);
153  if (found == changes_.end()) {
154    found = demoted_changes_.find(url);
155    if (found == demoted_changes_.end())
156      return;
157  }
158  *changes = found->second.change_list;
159}
160
161void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) {
162  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
163  ClearDirtyOnDatabase(url);
164  mirror_changes_.erase(url);
165  demoted_changes_.erase(url);
166  FileChangeMap::iterator found = changes_.find(url);
167  if (found == changes_.end())
168    return;
169  change_seqs_.erase(found->second.change_seq);
170  changes_.erase(found);
171  UpdateNumChanges();
172}
173
174void LocalFileChangeTracker::CreateFreshMirrorForURL(
175    const fileapi::FileSystemURL& url) {
176  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
177  DCHECK(!ContainsKey(mirror_changes_, url));
178  mirror_changes_[url] = ChangeInfo();
179}
180
181void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
182    const fileapi::FileSystemURL& url) {
183  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
184  FileChangeMap::iterator found = mirror_changes_.find(url);
185  if (found == mirror_changes_.end())
186    return;
187  mirror_changes_.erase(found);
188
189  if (ContainsKey(changes_, url))
190    MarkDirtyOnDatabase(url);
191  else
192    ClearDirtyOnDatabase(url);
193  UpdateNumChanges();
194}
195
196void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
197    const fileapi::FileSystemURL& url) {
198  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
199  FileChangeMap::iterator found = mirror_changes_.find(url);
200  if (found == mirror_changes_.end() || found->second.change_list.empty()) {
201    ClearChangesForURL(url);
202    return;
203  }
204  const ChangeInfo& info = found->second;
205  change_seqs_[info.change_seq] = url;
206  changes_[url] = info;
207  RemoveMirrorAndCommitChangesForURL(url);
208}
209
210void LocalFileChangeTracker::DemoteChangesForURL(
211    const fileapi::FileSystemURL& url) {
212  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
213  FileChangeMap::iterator found = changes_.find(url);
214  if (found == changes_.end())
215    return;
216  FileChangeList changes = found->second.change_list;
217
218  mirror_changes_.erase(url);
219  change_seqs_.erase(found->second.change_seq);
220  changes_.erase(found);
221
222  FileChangeList::List change_list = changes.list();
223  while (!change_list.empty()) {
224    RecordChangeToChangeMaps(url, change_list.front(), 0,
225                             &demoted_changes_, NULL);
226    change_list.pop_front();
227  }
228}
229
230void LocalFileChangeTracker::PromoteDemotedChangesForURL(
231    const fileapi::FileSystemURL& url) {
232  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
233
234  FileChangeMap::iterator iter = demoted_changes_.find(url);
235  if (iter == demoted_changes_.end())
236    return;
237
238  FileChangeList::List change_list = iter->second.change_list.list();
239  // Make sure that this URL is in no queues.
240  DCHECK(!ContainsKey(changes_, url));
241  DCHECK(!ContainsKey(mirror_changes_, url));
242
243  demoted_changes_.erase(iter);
244
245  while (!change_list.empty()) {
246    RecordChange(url, change_list.front());
247    change_list.pop_front();
248  }
249}
250
251bool LocalFileChangeTracker::PromoteDemotedChanges() {
252  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
253  if (demoted_changes_.empty())
254    return false;
255  while (!demoted_changes_.empty()) {
256    fileapi::FileSystemURL url = demoted_changes_.begin()->first;
257    PromoteDemotedChangesForURL(url);
258  }
259  return true;
260}
261
262SyncStatusCode LocalFileChangeTracker::Initialize(
263    FileSystemContext* file_system_context) {
264  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
265  DCHECK(!initialized_);
266  DCHECK(file_system_context);
267
268  SyncStatusCode status = CollectLastDirtyChanges(file_system_context);
269  if (status == SYNC_STATUS_OK)
270    initialized_ = true;
271  return status;
272}
273
274void LocalFileChangeTracker::ResetForFileSystem(
275    const GURL& origin,
276    fileapi::FileSystemType type) {
277  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
278  scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch);
279  for (FileChangeMap::iterator iter = changes_.begin();
280       iter != changes_.end();) {
281    fileapi::FileSystemURL url = iter->first;
282    if (url.origin() != origin || url.type() != type) {
283      ++iter;
284      continue;
285    }
286    mirror_changes_.erase(url);
287    demoted_changes_.erase(url);
288    change_seqs_.erase(iter->second.change_seq);
289    changes_.erase(iter++);
290
291    std::string serialized_url;
292    const bool should_success =
293        SerializeSyncableFileSystemURL(url, &serialized_url);
294    if (!should_success) {
295      NOTREACHED() << "Failed to serialize: " << url.DebugString();
296      continue;
297    }
298    batch->Delete(serialized_url);
299  }
300  // Fail to apply batch to database wouldn't have critical effect, they'll be
301  // just marked deleted on next relaunch.
302  tracker_db_->WriteBatch(batch.Pass());
303  UpdateNumChanges();
304}
305
306void LocalFileChangeTracker::UpdateNumChanges() {
307  base::AutoLock lock(num_changes_lock_);
308  num_changes_ = static_cast<int64>(change_seqs_.size());
309}
310
311void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) {
312  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
313  std::deque<FileSystemURL> url_deque;
314  GetNextChangedURLs(&url_deque, 0);
315  urls->clear();
316  urls->insert(url_deque.begin(), url_deque.end());
317}
318
319void LocalFileChangeTracker::DropAllChanges() {
320  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
321  changes_.clear();
322  change_seqs_.clear();
323  mirror_changes_.clear();
324}
325
326SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase(
327    const FileSystemURL& url) {
328  std::string serialized_url;
329  if (!SerializeSyncableFileSystemURL(url, &serialized_url))
330    return SYNC_FILE_ERROR_INVALID_URL;
331
332  return tracker_db_->MarkDirty(serialized_url);
333}
334
335SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase(
336    const FileSystemURL& url) {
337  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
338  std::string serialized_url;
339  if (!SerializeSyncableFileSystemURL(url, &serialized_url))
340    return SYNC_FILE_ERROR_INVALID_URL;
341
342  return tracker_db_->ClearDirty(serialized_url);
343}
344
345SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges(
346    FileSystemContext* file_system_context) {
347  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
348
349  std::queue<FileSystemURL> dirty_files;
350  const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files);
351  if (status != SYNC_STATUS_OK)
352    return status;
353
354  FileSystemFileUtil* file_util =
355      file_system_context->sandbox_delegate()->sync_file_util();
356  DCHECK(file_util);
357  scoped_ptr<FileSystemOperationContext> context(
358      new FileSystemOperationContext(file_system_context));
359
360  base::File::Info file_info;
361  base::FilePath platform_path;
362
363  while (!dirty_files.empty()) {
364    const FileSystemURL url = dirty_files.front();
365    dirty_files.pop();
366    DCHECK_EQ(url.type(), fileapi::kFileSystemTypeSyncable);
367
368    switch (file_util->GetFileInfo(context.get(), url,
369                                   &file_info, &platform_path)) {
370      case base::File::FILE_OK: {
371        if (!file_info.is_directory) {
372          RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
373                                       SYNC_FILE_TYPE_FILE));
374          break;
375        }
376
377        RecordChange(url, FileChange(
378            FileChange::FILE_CHANGE_ADD_OR_UPDATE,
379            SYNC_FILE_TYPE_DIRECTORY));
380
381        // Push files and directories in this directory into |dirty_files|.
382        scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator(
383            file_util->CreateFileEnumerator(context.get(), url));
384        base::FilePath path_each;
385        while (!(path_each = enumerator->Next()).empty()) {
386          dirty_files.push(CreateSyncableFileSystemURL(
387                  url.origin(), path_each));
388        }
389        break;
390      }
391      case base::File::FILE_ERROR_NOT_FOUND: {
392        // File represented by |url| has already been deleted. Since we cannot
393        // figure out if this file was directory or not from the URL, file
394        // type is treated as SYNC_FILE_TYPE_UNKNOWN.
395        //
396        // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
397        // also treated as FILE_CHANGE_DELETE.
398        RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
399                                     SYNC_FILE_TYPE_UNKNOWN));
400        break;
401      }
402      case base::File::FILE_ERROR_FAILED:
403      default:
404        // TODO(nhiroki): handle file access error (http://crbug.com/155251).
405        LOG(WARNING) << "Failed to access local file.";
406        break;
407    }
408  }
409  return SYNC_STATUS_OK;
410}
411
412void LocalFileChangeTracker::RecordChange(
413    const FileSystemURL& url, const FileChange& change) {
414  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
415  if (ContainsKey(demoted_changes_, url)) {
416    RecordChangeToChangeMaps(url, change, 0, &demoted_changes_, NULL);
417    return;
418  }
419  int change_seq = current_change_seq_++;
420  RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_);
421  if (ContainsKey(mirror_changes_, url))
422    RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_, NULL);
423  UpdateNumChanges();
424}
425
426// static
427void LocalFileChangeTracker::RecordChangeToChangeMaps(
428    const FileSystemURL& url,
429    const FileChange& change,
430    int new_change_seq,
431    FileChangeMap* changes,
432    ChangeSeqMap* change_seqs) {
433  ChangeInfo& info = (*changes)[url];
434  if (info.change_seq >= 0 && change_seqs)
435    change_seqs->erase(info.change_seq);
436  info.change_list.Update(change);
437  if (info.change_list.empty()) {
438    changes->erase(url);
439    return;
440  }
441  info.change_seq = new_change_seq;
442  if (change_seqs)
443    (*change_seqs)[info.change_seq] = url;
444}
445
446// TrackerDB -------------------------------------------------------------------
447
448LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path,
449                                             leveldb::Env* env_override)
450  : base_path_(base_path),
451    env_override_(env_override),
452    db_status_(SYNC_STATUS_OK) {}
453
454SyncStatusCode LocalFileChangeTracker::TrackerDB::Init(
455    RecoveryOption recovery_option) {
456  if (db_.get() && db_status_ == SYNC_STATUS_OK)
457    return SYNC_STATUS_OK;
458
459  std::string path = fileapi::FilePathToString(
460      base_path_.Append(kDatabaseName));
461  leveldb::Options options;
462  options.max_open_files = 0;  // Use minimum.
463  options.create_if_missing = true;
464  if (env_override_)
465    options.env = env_override_;
466  leveldb::DB* db;
467  leveldb::Status status = leveldb::DB::Open(options, path, &db);
468  if (status.ok()) {
469    db_.reset(db);
470    return SYNC_STATUS_OK;
471  }
472
473  HandleError(FROM_HERE, status);
474  if (!status.IsCorruption())
475    return LevelDBStatusToSyncStatusCode(status);
476
477  // Try to repair the corrupted DB.
478  switch (recovery_option) {
479    case FAIL_ON_CORRUPTION:
480      return SYNC_DATABASE_ERROR_CORRUPTION;
481    case REPAIR_ON_CORRUPTION:
482      return Repair(path);
483  }
484  NOTREACHED();
485  return SYNC_DATABASE_ERROR_FAILED;
486}
487
488SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair(
489    const std::string& db_path) {
490  DCHECK(!db_.get());
491  LOG(WARNING) << "Attempting to repair TrackerDB.";
492
493  leveldb::Options options;
494  options.max_open_files = 0;  // Use minimum.
495  if (leveldb::RepairDB(db_path, options).ok() &&
496      Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) {
497    // TODO(nhiroki): perform some consistency checks between TrackerDB and
498    // syncable file system.
499    LOG(WARNING) << "Repairing TrackerDB completed.";
500    return SYNC_STATUS_OK;
501  }
502
503  LOG(WARNING) << "Failed to repair TrackerDB.";
504  return SYNC_DATABASE_ERROR_CORRUPTION;
505}
506
507// TODO(nhiroki): factor out the common methods into somewhere else.
508void LocalFileChangeTracker::TrackerDB::HandleError(
509    const tracked_objects::Location& from_here,
510    const leveldb::Status& status) {
511  LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: "
512             << from_here.ToString() << " with error: " << status.ToString();
513}
514
515SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty(
516    const std::string& url) {
517  if (db_status_ != SYNC_STATUS_OK)
518    return db_status_;
519
520  db_status_ = Init(REPAIR_ON_CORRUPTION);
521  if (db_status_ != SYNC_STATUS_OK) {
522    db_.reset();
523    return db_status_;
524  }
525
526  leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark);
527  if (!status.ok()) {
528    HandleError(FROM_HERE, status);
529    db_status_ = LevelDBStatusToSyncStatusCode(status);
530    db_.reset();
531    return db_status_;
532  }
533  return SYNC_STATUS_OK;
534}
535
536SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty(
537    const std::string& url) {
538  if (db_status_ != SYNC_STATUS_OK)
539    return db_status_;
540
541  // Should not reach here before initializing the database. The database should
542  // be cleared after read, and should be initialized during read if
543  // uninitialized.
544  DCHECK(db_.get());
545
546  leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url);
547  if (!status.ok() && !status.IsNotFound()) {
548    HandleError(FROM_HERE, status);
549    db_status_ = LevelDBStatusToSyncStatusCode(status);
550    db_.reset();
551    return db_status_;
552  }
553  return SYNC_STATUS_OK;
554}
555
556SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
557    std::queue<FileSystemURL>* dirty_files) {
558  if (db_status_ != SYNC_STATUS_OK)
559    return db_status_;
560
561  db_status_ = Init(REPAIR_ON_CORRUPTION);
562  if (db_status_ != SYNC_STATUS_OK) {
563    db_.reset();
564    return db_status_;
565  }
566
567  scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
568  iter->SeekToFirst();
569  FileSystemURL url;
570  while (iter->Valid()) {
571    if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) {
572      LOG(WARNING) << "Failed to deserialize an URL. "
573                   << "TrackerDB might be corrupted.";
574      db_status_ = SYNC_DATABASE_ERROR_CORRUPTION;
575      db_.reset();
576      return db_status_;
577    }
578    dirty_files->push(url);
579    iter->Next();
580  }
581  return SYNC_STATUS_OK;
582}
583
584SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch(
585    scoped_ptr<leveldb::WriteBatch> batch) {
586  if (db_status_ != SYNC_STATUS_OK)
587    return db_status_;
588
589  leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get());
590  if (!status.ok() && !status.IsNotFound()) {
591    HandleError(FROM_HERE, status);
592    db_status_ = LevelDBStatusToSyncStatusCode(status);
593    db_.reset();
594    return db_status_;
595  }
596  return SYNC_STATUS_OK;
597}
598
599}  // namespace sync_file_system
600