change_list_processor.cc revision a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "chrome/browser/chromeos/drive/change_list_processor.h"
6
7#include "base/metrics/histogram.h"
8#include "base/strings/string_number_conversions.h"
9#include "chrome/browser/chromeos/drive/drive.pb.h"
10#include "chrome/browser/chromeos/drive/file_system_util.h"
11#include "chrome/browser/chromeos/drive/resource_entry_conversion.h"
12#include "chrome/browser/chromeos/drive/resource_metadata.h"
13#include "google_apis/drive/drive_api_parser.h"
14#include "google_apis/drive/gdata_wapi_parser.h"
15
16namespace drive {
17namespace internal {
18
19namespace {
20
21class ChangeListToEntryMapUMAStats {
22 public:
23  ChangeListToEntryMapUMAStats()
24    : num_regular_files_(0),
25      num_hosted_documents_(0),
26      num_shared_with_me_entries_(0) {
27  }
28
29  // Increments number of files.
30  void IncrementNumFiles(bool is_hosted_document) {
31    is_hosted_document ? num_hosted_documents_++ : num_regular_files_++;
32  }
33
34  // Increments number of shared-with-me entries.
35  void IncrementNumSharedWithMeEntries() {
36    num_shared_with_me_entries_++;
37  }
38
39  // Updates UMA histograms with file counts.
40  void UpdateFileCountUmaHistograms() {
41    const int num_total_files = num_hosted_documents_ + num_regular_files_;
42    UMA_HISTOGRAM_COUNTS("Drive.NumberOfRegularFiles", num_regular_files_);
43    UMA_HISTOGRAM_COUNTS("Drive.NumberOfHostedDocuments",
44                         num_hosted_documents_);
45    UMA_HISTOGRAM_COUNTS("Drive.NumberOfTotalFiles", num_total_files);
46    UMA_HISTOGRAM_COUNTS("Drive.NumberOfSharedWithMeEntries",
47                         num_shared_with_me_entries_);
48  }
49
50 private:
51  int num_regular_files_;
52  int num_hosted_documents_;
53  int num_shared_with_me_entries_;
54};
55
56}  // namespace
57
58std::string DirectoryFetchInfo::ToString() const {
59  return ("resource_id: " + resource_id_ +
60          ", changestamp: " + base::Int64ToString(changestamp_));
61}
62
63ChangeList::ChangeList() {}
64
65ChangeList::ChangeList(const google_apis::ResourceList& resource_list)
66    : largest_changestamp_(resource_list.largest_changestamp()) {
67  resource_list.GetNextFeedURL(&next_url_);
68
69  entries_.resize(resource_list.entries().size());
70  parent_resource_ids_.resize(resource_list.entries().size());
71  size_t entries_index = 0;
72  for (size_t i = 0; i < resource_list.entries().size(); ++i) {
73    if (ConvertToResourceEntry(*resource_list.entries()[i],
74                               &entries_[entries_index],
75                               &parent_resource_ids_[entries_index]))
76      ++entries_index;
77  }
78  entries_.resize(entries_index);
79  parent_resource_ids_.resize(entries_index);
80}
81
82ChangeList::~ChangeList() {}
83
84ChangeListProcessor::ChangeListProcessor(ResourceMetadata* resource_metadata)
85  : resource_metadata_(resource_metadata) {
86}
87
88ChangeListProcessor::~ChangeListProcessor() {
89}
90
91FileError ChangeListProcessor::Apply(
92    scoped_ptr<google_apis::AboutResource> about_resource,
93    ScopedVector<ChangeList> change_lists,
94    bool is_delta_update) {
95  DCHECK(about_resource);
96
97  int64 largest_changestamp = 0;
98  if (is_delta_update) {
99    if (!change_lists.empty()) {
100      // The changestamp appears in the first page of the change list.
101      // The changestamp does not appear in the full resource list.
102      largest_changestamp = change_lists[0]->largest_changestamp();
103      DCHECK_GE(change_lists[0]->largest_changestamp(), 0);
104    }
105  } else {
106    largest_changestamp = about_resource->largest_change_id();
107
108    DVLOG(1) << "Root folder ID is " << about_resource->root_folder_id();
109    DCHECK(!about_resource->root_folder_id().empty());
110  }
111
112  // Convert ChangeList to map.
113  ChangeListToEntryMapUMAStats uma_stats;
114  for (size_t i = 0; i < change_lists.size(); ++i) {
115    ChangeList* change_list = change_lists[i];
116
117    std::vector<ResourceEntry>* entries = change_list->mutable_entries();
118    for (size_t i = 0; i < entries->size(); ++i) {
119      ResourceEntry* entry = &(*entries)[i];
120
121      // Count the number of files.
122      if (!entry->file_info().is_directory()) {
123        uma_stats.IncrementNumFiles(
124            entry->file_specific_info().is_hosted_document());
125        if (entry->shared_with_me())
126          uma_stats.IncrementNumSharedWithMeEntries();
127      }
128      parent_resource_id_map_[entry->resource_id()] =
129          change_list->parent_resource_ids()[i];
130      entry_map_[entry->resource_id()].Swap(entry);
131      LOG_IF(WARNING, !entry->resource_id().empty())
132          << "Found duplicated file: " << entry->base_name();
133    }
134  }
135
136  // Add the largest changestamp for directories.
137  for (ResourceEntryMap::iterator it = entry_map_.begin();
138       it != entry_map_.end(); ++it) {
139    if (it->second.file_info().is_directory()) {
140      it->second.mutable_directory_specific_info()->set_changestamp(
141          largest_changestamp);
142    }
143  }
144
145  FileError error = ApplyEntryMap(largest_changestamp, about_resource.Pass());
146  if (error != FILE_ERROR_OK) {
147    DLOG(ERROR) << "ApplyEntryMap failed: " << FileErrorToString(error);
148    return error;
149  }
150
151  // Update changestamp.
152  error = resource_metadata_->SetLargestChangestamp(largest_changestamp);
153  if (error != FILE_ERROR_OK) {
154    DLOG(ERROR) << "SetLargestChangeStamp failed: " << FileErrorToString(error);
155    return error;
156  }
157
158  // Shouldn't record histograms when processing delta update.
159  if (!is_delta_update)
160    uma_stats.UpdateFileCountUmaHistograms();
161
162  return FILE_ERROR_OK;
163}
164
165FileError ChangeListProcessor::ApplyEntryMap(
166    int64 changestamp,
167    scoped_ptr<google_apis::AboutResource> about_resource) {
168  DCHECK(about_resource);
169
170  // Create the entry for "My Drive" directory with the latest changestamp.
171  ResourceEntry root;
172  FileError error = resource_metadata_->GetResourceEntryByPath(
173      util::GetDriveMyDriveRootPath(), &root);
174  switch (error) {
175    case FILE_ERROR_OK:  // Refresh the existing My Drive root entry.
176      root.mutable_directory_specific_info()->set_changestamp(changestamp);
177      error = resource_metadata_->RefreshEntry(root);
178      break;
179    case FILE_ERROR_NOT_FOUND: {  // Add the My Drive root directory.
180      changed_dirs_.insert(util::GetDriveGrandRootPath());
181      changed_dirs_.insert(util::GetDriveMyDriveRootPath());
182
183      root = util::CreateMyDriveRootEntry(about_resource->root_folder_id());
184      root.mutable_directory_specific_info()->set_changestamp(changestamp);
185      std::string local_id;
186      error = resource_metadata_->AddEntry(root, &local_id);
187      break;
188    }
189    default:
190      break;
191  }
192  if (error != FILE_ERROR_OK) {
193    LOG(ERROR) << "Failed to update root entry: " << FileErrorToString(error);
194    return error;
195  }
196
197  // Gather the set of changes in the old path.
198  // Note that we want to notify the change in both old and new paths (suppose
199  // /a/b/c is moved to /x/y/c. We want to notify both "/a/b" and "/x/y".)
200  // The old paths must be calculated before we apply any actual changes.
201  // The new paths are calculated after each change is applied. It correctly
202  // sets the new path because we apply changes in such an order (see below).
203  for (ResourceEntryMap::iterator it = entry_map_.begin();
204       it != entry_map_.end(); ++it) {
205    UpdateChangedDirs(it->second);
206  }
207
208  // Apply all entries except deleted ones to the metadata.
209  std::vector<ResourceEntry> deleted_entries;
210  deleted_entries.reserve(entry_map_.size());
211  while (!entry_map_.empty()) {
212    ResourceEntryMap::iterator it = entry_map_.begin();
213
214    // Process deleted entries later to avoid deleting moved entries under it.
215    if (it->second.deleted()) {
216      deleted_entries.push_back(ResourceEntry());
217      deleted_entries.back().Swap(&it->second);
218      entry_map_.erase(it);
219      continue;
220    }
221
222    // Start from entry_map_.begin() and traverse ancestors using the
223    // parent-child relationships in the result (after this apply) tree.
224    // Then apply the topmost change first.
225    //
226    // By doing this, assuming the result tree does not contain any cycles, we
227    // can guarantee that no cycle is made during this apply (i.e. no entry gets
228    // moved under any of its descendants) because the following conditions are
229    // always satisfied in any move:
230    // - The new parent entry is not a descendant of the moved entry.
231    // - The new parent and its ancestors will no longer move during this apply.
232    std::vector<ResourceEntryMap::iterator> entries;
233    for (ResourceEntryMap::iterator it = entry_map_.begin();
234         it != entry_map_.end();) {
235      entries.push_back(it);
236
237      const std::string& parent_resource_id =
238          parent_resource_id_map_[it->first];
239      DCHECK(!parent_resource_id.empty()) << it->first;
240
241      ResourceEntryMap::iterator it_parent =
242          entry_map_.find(parent_resource_id);
243      if (it_parent == entry_map_.end()) {
244        // Current entry's parent is already updated or not going to be updated,
245        // get the parent from the local tree.
246        std::string parent_local_id;
247        FileError error = resource_metadata_->GetIdByResourceId(
248            parent_resource_id, &parent_local_id);
249        if (error != FILE_ERROR_OK) {
250          LOG(ERROR) << "Failed to get local ID: " << parent_resource_id
251                     << ", error = " << FileErrorToString(error);
252          break;
253        }
254        ResourceEntry parent_entry;
255        while (it_parent == entry_map_.end() && !parent_local_id.empty()) {
256          error = resource_metadata_->GetResourceEntryById(
257              parent_local_id, &parent_entry);
258          if (error != FILE_ERROR_OK) {
259            LOG(ERROR) << "Failed to get local entry: "
260                       << FileErrorToString(error);
261            break;
262          }
263          it_parent = entry_map_.find(parent_entry.resource_id());
264          parent_local_id = parent_entry.parent_local_id();
265        }
266      }
267      it = it_parent;
268    }
269
270    // Apply the parent first.
271    std::reverse(entries.begin(), entries.end());
272    for (size_t i = 0; i < entries.size(); ++i) {
273      // Skip root entry in the change list. We don't expect servers to send
274      // root entry, but we should better be defensive (see crbug.com/297259).
275      ResourceEntryMap::iterator it = entries[i];
276      if (it->first != root.resource_id()) {
277        // TODO(hashimoto): Handle ApplyEntry errors correctly.
278        FileError error = ApplyEntry(it->second);
279        DLOG_IF(WARNING, error != FILE_ERROR_OK)
280            << "ApplyEntry failed: " << FileErrorToString(error)
281            << ", title = " << it->second.title();
282      }
283      entry_map_.erase(it);
284    }
285  }
286
287  // Apply deleted entries.
288  for (size_t i = 0; i < deleted_entries.size(); ++i) {
289    // TODO(hashimoto): Handle ApplyEntry errors correctly.
290    FileError error = ApplyEntry(deleted_entries[i]);
291    DLOG_IF(WARNING, error != FILE_ERROR_OK)
292        << "ApplyEntry failed: " << FileErrorToString(error)
293        << ", title = " << deleted_entries[i].title();
294  }
295
296  return FILE_ERROR_OK;
297}
298
299FileError ChangeListProcessor::ApplyEntry(const ResourceEntry& entry) {
300  // Lookup the entry.
301  std::string local_id;
302  FileError error = resource_metadata_->GetIdByResourceId(entry.resource_id(),
303                                                          &local_id);
304  ResourceEntry existing_entry;
305  if (error == FILE_ERROR_OK)
306    error = resource_metadata_->GetResourceEntryById(local_id, &existing_entry);
307
308  const FileError get_existing_entry_result = error;
309  if (entry.deleted()) {
310    // Deleted file/directory.
311    switch (get_existing_entry_result) {
312      case FILE_ERROR_OK:
313        error = resource_metadata_->RemoveEntry(local_id);
314        break;
315      case FILE_ERROR_NOT_FOUND:  // Already deleted.
316        error = FILE_ERROR_OK;
317        break;
318      default:
319        error = get_existing_entry_result;
320    }
321  } else {
322    const std::string& parent_resource_id =
323        parent_resource_id_map_[entry.resource_id()];
324    DCHECK(!parent_resource_id.empty()) << entry.resource_id();
325
326    std::string parent_local_id;
327    error = resource_metadata_->GetIdByResourceId(
328        parent_resource_id, &parent_local_id);
329    if (error == FILE_ERROR_OK) {
330      ResourceEntry new_entry(entry);
331      new_entry.set_parent_local_id(parent_local_id);
332
333      switch (get_existing_entry_result) {
334        case FILE_ERROR_OK:  // Entry exists and needs to be refreshed.
335          new_entry.set_local_id(local_id);
336          error = resource_metadata_->RefreshEntry(new_entry);
337          break;
338        case FILE_ERROR_NOT_FOUND: {  // Adding a new entry.
339          std::string local_id;
340          error = resource_metadata_->AddEntry(new_entry, &local_id);
341          break;
342        }
343        default:
344          error = get_existing_entry_result;
345      }
346
347      if (error == FILE_ERROR_OK)
348        UpdateChangedDirs(entry);
349    }
350  }
351
352  return error;
353}
354
355// static
356FileError ChangeListProcessor::RefreshDirectory(
357    ResourceMetadata* resource_metadata,
358    const DirectoryFetchInfo& directory_fetch_info,
359    ScopedVector<ChangeList> change_lists,
360    base::FilePath* out_file_path) {
361  DCHECK(!directory_fetch_info.empty());
362
363  std::string directory_local_id;
364  FileError error = resource_metadata->GetIdByResourceId(
365      directory_fetch_info.resource_id(), &directory_local_id);
366  if (error != FILE_ERROR_OK)
367    return error;
368
369  ResourceEntry directory;
370  error = resource_metadata->GetResourceEntryById(directory_local_id,
371                                                  &directory);
372  if (error != FILE_ERROR_OK)
373    return error;
374
375  if (!directory.file_info().is_directory())
376    return FILE_ERROR_NOT_A_DIRECTORY;
377
378  for (size_t i = 0; i < change_lists.size(); ++i) {
379    ChangeList* change_list = change_lists[i];
380    std::vector<ResourceEntry>* entries = change_list->mutable_entries();
381    for (size_t i = 0; i < entries->size(); ++i) {
382      ResourceEntry* entry = &(*entries)[i];
383      const std::string& parent_resource_id =
384          change_list->parent_resource_ids()[i];
385
386      // Skip if the parent resource ID does not match. This is needed to
387      // handle entries with multiple parents. For such entries, the first
388      // parent is picked and other parents are ignored, hence some entries may
389      // have a parent resource ID which does not match the target directory's.
390      if (parent_resource_id != directory_fetch_info.resource_id()) {
391        DVLOG(1) << "Wrong-parent entry rejected: " << entry->resource_id();
392        continue;
393      }
394
395      entry->set_parent_local_id(directory_local_id);
396
397      std::string local_id;
398      error = resource_metadata->GetIdByResourceId(entry->resource_id(),
399                                                   &local_id);
400      if (error == FILE_ERROR_OK) {
401        entry->set_local_id(local_id);
402        error = resource_metadata->RefreshEntry(*entry);
403      }
404
405      if (error == FILE_ERROR_NOT_FOUND) {  // If refreshing fails, try adding.
406        std::string local_id;
407        entry->clear_local_id();
408        error = resource_metadata->AddEntry(*entry, &local_id);
409      }
410
411      if (error != FILE_ERROR_OK)
412        return error;
413    }
414  }
415
416  directory.mutable_directory_specific_info()->set_changestamp(
417      directory_fetch_info.changestamp());
418  error = resource_metadata->RefreshEntry(directory);
419  if (error != FILE_ERROR_OK)
420    return error;
421
422  *out_file_path = resource_metadata->GetFilePath(directory_local_id);
423  return FILE_ERROR_OK;
424}
425
426void ChangeListProcessor::UpdateChangedDirs(const ResourceEntry& entry) {
427  DCHECK(!entry.resource_id().empty());
428
429  std::string local_id;
430  base::FilePath file_path;
431  if (resource_metadata_->GetIdByResourceId(
432          entry.resource_id(), &local_id) == FILE_ERROR_OK)
433    file_path = resource_metadata_->GetFilePath(local_id);
434
435  if (!file_path.empty()) {
436    // Notify parent.
437    changed_dirs_.insert(file_path.DirName());
438
439    if (entry.file_info().is_directory()) {
440      // Notify self if entry is a directory.
441      changed_dirs_.insert(file_path);
442
443      // Notify all descendants if it is a directory deletion.
444      if (entry.deleted()) {
445        std::set<base::FilePath> sub_directories;
446        resource_metadata_->GetSubDirectoriesRecursively(local_id,
447                                                         &sub_directories);
448        changed_dirs_.insert(sub_directories.begin(), sub_directories.end());
449      }
450    }
451  }
452}
453
454}  // namespace internal
455}  // namespace drive
456