local_file_change_tracker.cc revision a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7 
1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/sync_file_system/local/local_file_change_tracker.h"
6
7#include <queue>
8
9#include "base/location.h"
10#include "base/logging.h"
11#include "base/sequenced_task_runner.h"
12#include "base/stl_util.h"
13#include "chrome/browser/sync_file_system/local/local_file_sync_status.h"
14#include "chrome/browser/sync_file_system/syncable_file_system_util.h"
15#include "third_party/leveldatabase/src/include/leveldb/db.h"
16#include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
17#include "webkit/browser/fileapi/file_system_context.h"
18#include "webkit/browser/fileapi/file_system_file_util.h"
19#include "webkit/browser/fileapi/file_system_operation_context.h"
20#include "webkit/common/fileapi/file_system_util.h"
21
22using fileapi::FileSystemContext;
23using fileapi::FileSystemFileUtil;
24using fileapi::FileSystemOperationContext;
25using fileapi::FileSystemURL;
26using fileapi::FileSystemURLSet;
27
28namespace sync_file_system {
29
30namespace {
31const base::FilePath::CharType kDatabaseName[] =
32    FILE_PATH_LITERAL("LocalFileChangeTracker");
33const char kMark[] = "d";
34}  // namespace
35
36// A database class that stores local file changes in a local database. This
37// object must be destructed on file_task_runner.
38class LocalFileChangeTracker::TrackerDB {
39 public:
40  explicit TrackerDB(const base::FilePath& base_path);
41
42  SyncStatusCode MarkDirty(const std::string& url);
43  SyncStatusCode ClearDirty(const std::string& url);
44  SyncStatusCode GetDirtyEntries(
45      std::queue<FileSystemURL>* dirty_files);
46  SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch);
47
48 private:
49  enum RecoveryOption {
50    REPAIR_ON_CORRUPTION,
51    FAIL_ON_CORRUPTION,
52  };
53
54  SyncStatusCode Init(RecoveryOption recovery_option);
55  SyncStatusCode Repair(const std::string& db_path);
56  void HandleError(const tracked_objects::Location& from_here,
57                   const leveldb::Status& status);
58
59  const base::FilePath base_path_;
60  scoped_ptr<leveldb::DB> db_;
61  SyncStatusCode db_status_;
62
63  DISALLOW_COPY_AND_ASSIGN(TrackerDB);
64};
65
66LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {}
67LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {}
68
69// LocalFileChangeTracker ------------------------------------------------------
70
71LocalFileChangeTracker::LocalFileChangeTracker(
72    const base::FilePath& base_path,
73    base::SequencedTaskRunner* file_task_runner)
74    : initialized_(false),
75      file_task_runner_(file_task_runner),
76      tracker_db_(new TrackerDB(base_path)),
77      current_change_seq_(0),
78      num_changes_(0) {
79}
80
81LocalFileChangeTracker::~LocalFileChangeTracker() {
82  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
83  tracker_db_.reset();
84}
85
86void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) {
87  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
88  if (ContainsKey(changes_, url))
89    return;
90  // TODO(nhiroki): propagate the error code (see http://crbug.com/152127).
91  MarkDirtyOnDatabase(url);
92}
93
94void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {}
95
96void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) {
97  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
98                               SYNC_FILE_TYPE_FILE));
99}
100
101void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url,
102                                              const FileSystemURL& src) {
103  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
104                               SYNC_FILE_TYPE_FILE));
105}
106
107void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) {
108  RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
109                               SYNC_FILE_TYPE_FILE));
110}
111
112void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) {
113  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
114                               SYNC_FILE_TYPE_FILE));
115}
116
117void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) {
118  RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
119                               SYNC_FILE_TYPE_DIRECTORY));
120}
121
122void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) {
123  RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
124                               SYNC_FILE_TYPE_DIRECTORY));
125}
126
127void LocalFileChangeTracker::GetNextChangedURLs(
128    std::deque<FileSystemURL>* urls, int max_urls) {
129  DCHECK(urls);
130  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
131  urls->clear();
132  // Mildly prioritizes the URLs that older changes and have not been updated
133  // for a while.
134  for (ChangeSeqMap::iterator iter = change_seqs_.begin();
135       iter != change_seqs_.end() &&
136       (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls));
137       ++iter) {
138    urls->push_back(iter->second);
139  }
140}
141
142void LocalFileChangeTracker::GetChangesForURL(
143    const FileSystemURL& url, FileChangeList* changes) {
144  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
145  DCHECK(changes);
146  changes->clear();
147  FileChangeMap::iterator found = changes_.find(url);
148  if (found == changes_.end())
149    return;
150  *changes = found->second.change_list;
151}
152
153void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) {
154  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
155  ClearDirtyOnDatabase(url);
156  mirror_changes_.erase(url);
157  FileChangeMap::iterator found = changes_.find(url);
158  if (found == changes_.end())
159    return;
160  change_seqs_.erase(found->second.change_seq);
161  changes_.erase(found);
162  UpdateNumChanges();
163}
164
165void LocalFileChangeTracker::CreateFreshMirrorForURL(
166    const fileapi::FileSystemURL& url) {
167  DCHECK(!ContainsKey(mirror_changes_, url));
168  mirror_changes_[url] = ChangeInfo();
169}
170
171void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL(
172    const fileapi::FileSystemURL& url) {
173  FileChangeMap::iterator found = mirror_changes_.find(url);
174  if (found == mirror_changes_.end())
175    return;
176  mirror_changes_.erase(found);
177
178  if (ContainsKey(changes_, url))
179    MarkDirtyOnDatabase(url);
180  else
181    ClearDirtyOnDatabase(url);
182  UpdateNumChanges();
183}
184
185void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL(
186    const fileapi::FileSystemURL& url) {
187  FileChangeMap::iterator found = mirror_changes_.find(url);
188  if (found == mirror_changes_.end() || found->second.change_list.empty()) {
189    ClearChangesForURL(url);
190    return;
191  }
192  const ChangeInfo& info = found->second;
193  change_seqs_[info.change_seq] = url;
194  changes_[url] = info;
195  RemoveMirrorAndCommitChangesForURL(url);
196}
197
198SyncStatusCode LocalFileChangeTracker::Initialize(
199    FileSystemContext* file_system_context) {
200  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
201  DCHECK(!initialized_);
202  DCHECK(file_system_context);
203
204  SyncStatusCode status = CollectLastDirtyChanges(file_system_context);
205  if (status == SYNC_STATUS_OK)
206    initialized_ = true;
207  return status;
208}
209
210void LocalFileChangeTracker::ResetForFileSystem(
211    const GURL& origin,
212    fileapi::FileSystemType type) {
213  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
214  scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch);
215  for (FileChangeMap::iterator iter = changes_.begin();
216       iter != changes_.end();) {
217    fileapi::FileSystemURL url = iter->first;
218    if (url.origin() != origin || url.type() != type) {
219      ++iter;
220      continue;
221    }
222    mirror_changes_.erase(url);
223    change_seqs_.erase(iter->second.change_seq);
224    changes_.erase(iter++);
225
226    std::string serialized_url;
227    const bool should_success =
228        SerializeSyncableFileSystemURL(url, &serialized_url);
229    if (!should_success) {
230      NOTREACHED() << "Failed to serialize: " << url.DebugString();
231      continue;
232    }
233    batch->Delete(serialized_url);
234  }
235  // Fail to apply batch to database wouldn't have critical effect, they'll be
236  // just marked deleted on next relaunch.
237  tracker_db_->WriteBatch(batch.Pass());
238  UpdateNumChanges();
239}
240
241void LocalFileChangeTracker::UpdateNumChanges() {
242  base::AutoLock lock(num_changes_lock_);
243  num_changes_ = static_cast<int64>(change_seqs_.size());
244}
245
246void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) {
247  std::deque<FileSystemURL> url_deque;
248  GetNextChangedURLs(&url_deque, 0);
249  urls->clear();
250  urls->insert(url_deque.begin(), url_deque.end());
251}
252
253void LocalFileChangeTracker::DropAllChanges() {
254  changes_.clear();
255  change_seqs_.clear();
256  mirror_changes_.clear();
257}
258
259SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase(
260    const FileSystemURL& url) {
261  std::string serialized_url;
262  if (!SerializeSyncableFileSystemURL(url, &serialized_url))
263    return SYNC_FILE_ERROR_INVALID_URL;
264
265  return tracker_db_->MarkDirty(serialized_url);
266}
267
268SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase(
269    const FileSystemURL& url) {
270  std::string serialized_url;
271  if (!SerializeSyncableFileSystemURL(url, &serialized_url))
272    return SYNC_FILE_ERROR_INVALID_URL;
273
274  return tracker_db_->ClearDirty(serialized_url);
275}
276
277SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges(
278    FileSystemContext* file_system_context) {
279  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
280
281  std::queue<FileSystemURL> dirty_files;
282  const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files);
283  if (status != SYNC_STATUS_OK)
284    return status;
285
286  FileSystemFileUtil* file_util =
287      file_system_context->sandbox_delegate()->sync_file_util();
288  DCHECK(file_util);
289  scoped_ptr<FileSystemOperationContext> context(
290      new FileSystemOperationContext(file_system_context));
291
292  base::PlatformFileInfo file_info;
293  base::FilePath platform_path;
294
295  while (!dirty_files.empty()) {
296    const FileSystemURL url = dirty_files.front();
297    dirty_files.pop();
298    DCHECK_EQ(url.type(), fileapi::kFileSystemTypeSyncable);
299
300    switch (file_util->GetFileInfo(context.get(), url,
301                                   &file_info, &platform_path)) {
302      case base::PLATFORM_FILE_OK: {
303        if (!file_info.is_directory) {
304          RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE,
305                                       SYNC_FILE_TYPE_FILE));
306          break;
307        }
308
309        RecordChange(url, FileChange(
310            FileChange::FILE_CHANGE_ADD_OR_UPDATE,
311            SYNC_FILE_TYPE_DIRECTORY));
312
313        // Push files and directories in this directory into |dirty_files|.
314        scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator(
315            file_util->CreateFileEnumerator(context.get(), url));
316        base::FilePath path_each;
317        while (!(path_each = enumerator->Next()).empty()) {
318          dirty_files.push(CreateSyncableFileSystemURL(
319                  url.origin(), path_each));
320        }
321        break;
322      }
323      case base::PLATFORM_FILE_ERROR_NOT_FOUND: {
324        // File represented by |url| has already been deleted. Since we cannot
325        // figure out if this file was directory or not from the URL, file
326        // type is treated as SYNC_FILE_TYPE_UNKNOWN.
327        //
328        // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is
329        // also treated as FILE_CHANGE_DELETE.
330        RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE,
331                                     SYNC_FILE_TYPE_UNKNOWN));
332        break;
333      }
334      case base::PLATFORM_FILE_ERROR_FAILED:
335      default:
336        // TODO(nhiroki): handle file access error (http://crbug.com/155251).
337        LOG(WARNING) << "Failed to access local file.";
338        break;
339    }
340  }
341  return SYNC_STATUS_OK;
342}
343
344void LocalFileChangeTracker::RecordChange(
345    const FileSystemURL& url, const FileChange& change) {
346  DCHECK(file_task_runner_->RunsTasksOnCurrentThread());
347  int change_seq = current_change_seq_++;
348  RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_);
349  if (ContainsKey(mirror_changes_, url))
350    RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_, NULL);
351  UpdateNumChanges();
352}
353
354void LocalFileChangeTracker::RecordChangeToChangeMaps(
355    const FileSystemURL& url,
356    const FileChange& change,
357    int new_change_seq,
358    FileChangeMap* changes,
359    ChangeSeqMap* change_seqs) {
360  ChangeInfo& info = (*changes)[url];
361  if (info.change_seq >= 0 && change_seqs)
362    change_seqs->erase(info.change_seq);
363  info.change_list.Update(change);
364  if (info.change_list.empty()) {
365    changes->erase(url);
366    return;
367  }
368  info.change_seq = new_change_seq;
369  if (change_seqs)
370    (*change_seqs)[info.change_seq] = url;
371}
372
373// TrackerDB -------------------------------------------------------------------
374
375LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path)
376  : base_path_(base_path),
377    db_status_(SYNC_STATUS_OK) {}
378
379SyncStatusCode LocalFileChangeTracker::TrackerDB::Init(
380    RecoveryOption recovery_option) {
381  if (db_.get() && db_status_ == SYNC_STATUS_OK)
382    return SYNC_STATUS_OK;
383
384  std::string path = fileapi::FilePathToString(
385      base_path_.Append(kDatabaseName));
386  leveldb::Options options;
387  options.max_open_files = 0;  // Use minimum.
388  options.create_if_missing = true;
389  leveldb::DB* db;
390  leveldb::Status status = leveldb::DB::Open(options, path, &db);
391  if (status.ok()) {
392    db_.reset(db);
393    return SYNC_STATUS_OK;
394  }
395
396  HandleError(FROM_HERE, status);
397  if (!status.IsCorruption())
398    return LevelDBStatusToSyncStatusCode(status);
399
400  // Try to repair the corrupted DB.
401  switch (recovery_option) {
402    case FAIL_ON_CORRUPTION:
403      return SYNC_DATABASE_ERROR_CORRUPTION;
404    case REPAIR_ON_CORRUPTION:
405      return Repair(path);
406  }
407  NOTREACHED();
408  return SYNC_DATABASE_ERROR_FAILED;
409}
410
411SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair(
412    const std::string& db_path) {
413  DCHECK(!db_.get());
414  LOG(WARNING) << "Attempting to repair TrackerDB.";
415
416  leveldb::Options options;
417  options.max_open_files = 0;  // Use minimum.
418  if (leveldb::RepairDB(db_path, options).ok() &&
419      Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) {
420    // TODO(nhiroki): perform some consistency checks between TrackerDB and
421    // syncable file system.
422    LOG(WARNING) << "Repairing TrackerDB completed.";
423    return SYNC_STATUS_OK;
424  }
425
426  LOG(WARNING) << "Failed to repair TrackerDB.";
427  return SYNC_DATABASE_ERROR_CORRUPTION;
428}
429
430// TODO(nhiroki): factor out the common methods into somewhere else.
431void LocalFileChangeTracker::TrackerDB::HandleError(
432    const tracked_objects::Location& from_here,
433    const leveldb::Status& status) {
434  LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: "
435             << from_here.ToString() << " with error: " << status.ToString();
436}
437
438SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty(
439    const std::string& url) {
440  if (db_status_ != SYNC_STATUS_OK)
441    return db_status_;
442
443  db_status_ = Init(REPAIR_ON_CORRUPTION);
444  if (db_status_ != SYNC_STATUS_OK) {
445    db_.reset();
446    return db_status_;
447  }
448
449  leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark);
450  if (!status.ok()) {
451    HandleError(FROM_HERE, status);
452    db_status_ = LevelDBStatusToSyncStatusCode(status);
453    db_.reset();
454    return db_status_;
455  }
456  return SYNC_STATUS_OK;
457}
458
459SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty(
460    const std::string& url) {
461  if (db_status_ != SYNC_STATUS_OK)
462    return db_status_;
463
464  // Should not reach here before initializing the database. The database should
465  // be cleared after read, and should be initialized during read if
466  // uninitialized.
467  DCHECK(db_.get());
468
469  leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url);
470  if (!status.ok() && !status.IsNotFound()) {
471    HandleError(FROM_HERE, status);
472    db_status_ = LevelDBStatusToSyncStatusCode(status);
473    db_.reset();
474    return db_status_;
475  }
476  return SYNC_STATUS_OK;
477}
478
479SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries(
480    std::queue<FileSystemURL>* dirty_files) {
481  if (db_status_ != SYNC_STATUS_OK)
482    return db_status_;
483
484  db_status_ = Init(REPAIR_ON_CORRUPTION);
485  if (db_status_ != SYNC_STATUS_OK) {
486    db_.reset();
487    return db_status_;
488  }
489
490  scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions()));
491  iter->SeekToFirst();
492  FileSystemURL url;
493  while (iter->Valid()) {
494    if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) {
495      LOG(WARNING) << "Failed to deserialize an URL. "
496                   << "TrackerDB might be corrupted.";
497      db_status_ = SYNC_DATABASE_ERROR_CORRUPTION;
498      db_.reset();
499      return db_status_;
500    }
501    dirty_files->push(url);
502    iter->Next();
503  }
504  return SYNC_STATUS_OK;
505}
506
507SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch(
508    scoped_ptr<leveldb::WriteBatch> batch) {
509  if (db_status_ != SYNC_STATUS_OK)
510    return db_status_;
511
512  leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get());
513  if (!status.ok() && !status.IsNotFound()) {
514    HandleError(FROM_HERE, status);
515    db_status_ = LevelDBStatusToSyncStatusCode(status);
516    db_.reset();
517    return db_status_;
518  }
519  return SYNC_STATUS_OK;
520}
521
522}  // namespace sync_file_system
523