indexed_db_context_impl.cc revision a1401311d1ab56c4ed0a474bd38c108f75cb0cd9 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/indexed_db/indexed_db_context_impl.h"
6
7#include <algorithm>
8
9#include "base/bind.h"
10#include "base/command_line.h"
11#include "base/file_util.h"
12#include "base/files/file_enumerator.h"
13#include "base/logging.h"
14#include "base/metrics/histogram.h"
15#include "base/sequenced_task_runner.h"
16#include "base/strings/string_util.h"
17#include "base/strings/utf_string_conversions.h"
18#include "base/threading/thread_restrictions.h"
19#include "base/time/time.h"
20#include "base/values.h"
21#include "content/browser/browser_main_loop.h"
22#include "content/browser/indexed_db/indexed_db_connection.h"
23#include "content/browser/indexed_db/indexed_db_database.h"
24#include "content/browser/indexed_db/indexed_db_dispatcher_host.h"
25#include "content/browser/indexed_db/indexed_db_factory.h"
26#include "content/browser/indexed_db/indexed_db_quota_client.h"
27#include "content/browser/indexed_db/indexed_db_tracing.h"
28#include "content/browser/indexed_db/indexed_db_transaction.h"
29#include "content/public/browser/browser_thread.h"
30#include "content/public/browser/indexed_db_info.h"
31#include "content/public/common/content_switches.h"
32#include "ui/base/text/bytes_formatting.h"
33#include "webkit/browser/database/database_util.h"
34#include "webkit/browser/quota/quota_manager_proxy.h"
35#include "webkit/browser/quota/special_storage_policy.h"
36#include "webkit/common/database/database_identifier.h"
37
38using base::DictionaryValue;
39using base::ListValue;
40using webkit_database::DatabaseUtil;
41
42namespace content {
43const base::FilePath::CharType IndexedDBContextImpl::kIndexedDBDirectory[] =
44    FILE_PATH_LITERAL("IndexedDB");
45
46static const base::FilePath::CharType kIndexedDBExtension[] =
47    FILE_PATH_LITERAL(".indexeddb");
48
49static const base::FilePath::CharType kLevelDBExtension[] =
50    FILE_PATH_LITERAL(".leveldb");
51
52namespace {
53
54// This may be called after the IndexedDBContext is destroyed.
55void GetAllOriginsAndPaths(const base::FilePath& indexeddb_path,
56                           std::vector<GURL>* origins,
57                           std::vector<base::FilePath>* file_paths) {
58  // TODO(jsbell): DCHECK that this is running on an IndexedDB thread,
59  // if a global handle to it is ever available.
60  if (indexeddb_path.empty())
61    return;
62  base::FileEnumerator file_enumerator(
63      indexeddb_path, false, base::FileEnumerator::DIRECTORIES);
64  for (base::FilePath file_path = file_enumerator.Next(); !file_path.empty();
65       file_path = file_enumerator.Next()) {
66    if (file_path.Extension() == kLevelDBExtension &&
67        file_path.RemoveExtension().Extension() == kIndexedDBExtension) {
68      std::string origin_id = file_path.BaseName().RemoveExtension()
69          .RemoveExtension().MaybeAsASCII();
70      origins->push_back(webkit_database::GetOriginFromIdentifier(origin_id));
71      if (file_paths)
72        file_paths->push_back(file_path);
73    }
74  }
75}
76
77// This will be called after the IndexedDBContext is destroyed.
78void ClearSessionOnlyOrigins(
79    const base::FilePath& indexeddb_path,
80    scoped_refptr<quota::SpecialStoragePolicy> special_storage_policy) {
81  // TODO(jsbell): DCHECK that this is running on an IndexedDB thread,
82  // if a global handle to it is ever available.
83  std::vector<GURL> origins;
84  std::vector<base::FilePath> file_paths;
85  GetAllOriginsAndPaths(indexeddb_path, &origins, &file_paths);
86  DCHECK_EQ(origins.size(), file_paths.size());
87  std::vector<base::FilePath>::const_iterator file_path_iter =
88      file_paths.begin();
89  for (std::vector<GURL>::const_iterator iter = origins.begin();
90       iter != origins.end();
91       ++iter, ++file_path_iter) {
92    if (!special_storage_policy->IsStorageSessionOnly(*iter))
93      continue;
94    if (special_storage_policy->IsStorageProtected(*iter))
95      continue;
96    base::DeleteFile(*file_path_iter, true);
97  }
98}
99
100}  // namespace
101
102IndexedDBContextImpl::IndexedDBContextImpl(
103    const base::FilePath& data_path,
104    quota::SpecialStoragePolicy* special_storage_policy,
105    quota::QuotaManagerProxy* quota_manager_proxy,
106    base::SequencedTaskRunner* task_runner)
107    : force_keep_session_state_(false),
108      special_storage_policy_(special_storage_policy),
109      quota_manager_proxy_(quota_manager_proxy),
110      task_runner_(task_runner) {
111  IDB_TRACE("init");
112  if (!data_path.empty())
113    data_path_ = data_path.Append(kIndexedDBDirectory);
114  if (quota_manager_proxy) {
115    quota_manager_proxy->RegisterClient(new IndexedDBQuotaClient(this));
116  }
117}
118
119IndexedDBFactory* IndexedDBContextImpl::GetIDBFactory() {
120  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
121  if (!factory_) {
122    // Prime our cache of origins with existing databases so we can
123    // detect when dbs are newly created.
124    GetOriginSet();
125    factory_ = new IndexedDBFactory(this);
126  }
127  return factory_;
128}
129
130std::vector<GURL> IndexedDBContextImpl::GetAllOrigins() {
131  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
132  std::vector<GURL> origins;
133  std::set<GURL>* origins_set = GetOriginSet();
134  for (std::set<GURL>::const_iterator iter = origins_set->begin();
135       iter != origins_set->end();
136       ++iter) {
137    origins.push_back(*iter);
138  }
139  return origins;
140}
141
142std::vector<IndexedDBInfo> IndexedDBContextImpl::GetAllOriginsInfo() {
143  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
144  std::vector<GURL> origins = GetAllOrigins();
145  std::vector<IndexedDBInfo> result;
146  for (std::vector<GURL>::const_iterator iter = origins.begin();
147       iter != origins.end();
148       ++iter) {
149    const GURL& origin_url = *iter;
150
151    base::FilePath idb_directory = GetFilePath(origin_url);
152    size_t connection_count = GetConnectionCount(origin_url);
153    result.push_back(IndexedDBInfo(origin_url,
154                                   GetOriginDiskUsage(origin_url),
155                                   GetOriginLastModified(origin_url),
156                                   idb_directory,
157                                   connection_count));
158  }
159  return result;
160}
161
162static bool HostNameComparator(const GURL& i, const GURL& j) {
163  return i.host() < j.host();
164}
165
166base::ListValue* IndexedDBContextImpl::GetAllOriginsDetails() {
167  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
168  std::vector<GURL> origins = GetAllOrigins();
169
170  std::sort(origins.begin(), origins.end(), HostNameComparator);
171
172  scoped_ptr<base::ListValue> list(new base::ListValue());
173  for (std::vector<GURL>::const_iterator iter = origins.begin();
174       iter != origins.end();
175       ++iter) {
176    const GURL& origin_url = *iter;
177
178    scoped_ptr<base::DictionaryValue> info(new base::DictionaryValue());
179    info->SetString("url", origin_url.spec());
180    info->SetString("size", ui::FormatBytes(GetOriginDiskUsage(origin_url)));
181    info->SetDouble("last_modified",
182                    GetOriginLastModified(origin_url).ToJsTime());
183    info->SetString("path", GetFilePath(origin_url).value());
184    info->SetDouble("connection_count", GetConnectionCount(origin_url));
185
186    // This ends up being O(n^2) since we iterate over all open databases
187    // to extract just those in the origin, and we're iterating over all
188    // origins in the outer loop.
189
190    if (factory_) {
191      std::pair<IndexedDBFactory::OriginDBMapIterator,
192                IndexedDBFactory::OriginDBMapIterator> range =
193          factory_->GetOpenDatabasesForOrigin(origin_url);
194      // TODO(jsbell): Sort by name?
195      scoped_ptr<base::ListValue> database_list(new base::ListValue());
196
197      for (IndexedDBFactory::OriginDBMapIterator it = range.first;
198           it != range.second;
199           ++it) {
200
201        const IndexedDBDatabase* db = it->second;
202        scoped_ptr<base::DictionaryValue> db_info(new base::DictionaryValue());
203
204        db_info->SetString("name", db->name());
205        db_info->SetDouble("pending_opens", db->PendingOpenCount());
206        db_info->SetDouble("pending_upgrades", db->PendingUpgradeCount());
207        db_info->SetDouble("running_upgrades", db->RunningUpgradeCount());
208        db_info->SetDouble("pending_deletes", db->PendingDeleteCount());
209        db_info->SetDouble("connection_count",
210                           db->ConnectionCount() - db->PendingUpgradeCount() -
211                               db->RunningUpgradeCount());
212
213        scoped_ptr<base::ListValue> transaction_list(new base::ListValue());
214        std::vector<const IndexedDBTransaction*> transactions =
215            db->transaction_coordinator().GetTransactions();
216        for (std::vector<const IndexedDBTransaction*>::iterator trans_it =
217                 transactions.begin();
218             trans_it != transactions.end();
219             ++trans_it) {
220
221          const IndexedDBTransaction* transaction = *trans_it;
222          scoped_ptr<base::DictionaryValue> transaction_info(
223              new base::DictionaryValue());
224
225          const char* kModes[] = { "readonly", "readwrite", "versionchange" };
226          transaction_info->SetString("mode", kModes[transaction->mode()]);
227          switch (transaction->state()) {
228            case IndexedDBTransaction::CREATED:
229              transaction_info->SetString("status", "blocked");
230              break;
231            case IndexedDBTransaction::STARTED:
232              if (transaction->diagnostics().tasks_scheduled > 0)
233                transaction_info->SetString("status", "running");
234              else
235                transaction_info->SetString("status", "started");
236              break;
237            case IndexedDBTransaction::FINISHED:
238              transaction_info->SetString("status", "finished");
239              break;
240          }
241
242          transaction_info->SetDouble(
243              "pid",
244              IndexedDBDispatcherHost::TransactionIdToProcessId(
245                  transaction->id()));
246          transaction_info->SetDouble(
247              "tid",
248              IndexedDBDispatcherHost::TransactionIdToRendererTransactionId(
249                  transaction->id()));
250          transaction_info->SetDouble(
251              "age",
252              (base::Time::Now() - transaction->diagnostics().creation_time)
253                  .InMillisecondsF());
254          transaction_info->SetDouble(
255              "runtime",
256              (base::Time::Now() - transaction->diagnostics().start_time)
257                  .InMillisecondsF());
258          transaction_info->SetDouble(
259              "tasks_scheduled", transaction->diagnostics().tasks_scheduled);
260          transaction_info->SetDouble(
261              "tasks_completed", transaction->diagnostics().tasks_completed);
262
263          scoped_ptr<base::ListValue> scope(new base::ListValue());
264          for (std::set<int64>::const_iterator scope_it =
265                   transaction->scope().begin();
266               scope_it != transaction->scope().end();
267               ++scope_it) {
268            IndexedDBDatabaseMetadata::ObjectStoreMap::const_iterator it =
269                db->metadata().object_stores.find(*scope_it);
270            if (it != db->metadata().object_stores.end())
271              scope->AppendString(it->second.name);
272          }
273
274          transaction_info->Set("scope", scope.release());
275          transaction_list->Append(transaction_info.release());
276        }
277        db_info->Set("transactions", transaction_list.release());
278
279        database_list->Append(db_info.release());
280      }
281      info->Set("databases", database_list.release());
282    }
283
284    list->Append(info.release());
285  }
286  return list.release();
287}
288
289int64 IndexedDBContextImpl::GetOriginDiskUsage(const GURL& origin_url) {
290  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
291  if (data_path_.empty() || !IsInOriginSet(origin_url))
292    return 0;
293  EnsureDiskUsageCacheInitialized(origin_url);
294  return origin_size_map_[origin_url];
295}
296
297base::Time IndexedDBContextImpl::GetOriginLastModified(const GURL& origin_url) {
298  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
299  if (data_path_.empty() || !IsInOriginSet(origin_url))
300    return base::Time();
301  base::FilePath idb_directory = GetFilePath(origin_url);
302  base::File::Info file_info;
303  if (!base::GetFileInfo(idb_directory, &file_info))
304    return base::Time();
305  return file_info.last_modified;
306}
307
308void IndexedDBContextImpl::DeleteForOrigin(const GURL& origin_url) {
309  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
310  ForceClose(origin_url, FORCE_CLOSE_DELETE_ORIGIN);
311  if (data_path_.empty() || !IsInOriginSet(origin_url))
312    return;
313
314  base::FilePath idb_directory = GetFilePath(origin_url);
315  EnsureDiskUsageCacheInitialized(origin_url);
316  leveldb::Status s = LevelDBDatabase::Destroy(idb_directory);
317  if (!s.ok()) {
318    LOG(WARNING) << "Failed to delete LevelDB database: "
319                 << idb_directory.AsUTF8Unsafe();
320  } else {
321    // LevelDB does not delete empty directories; work around this.
322    // TODO(jsbell): Remove when upstream bug is fixed.
323    // https://code.google.com/p/leveldb/issues/detail?id=209
324    const bool kNonRecursive = false;
325    base::DeleteFile(idb_directory, kNonRecursive);
326  }
327
328  QueryDiskAndUpdateQuotaUsage(origin_url);
329  if (s.ok()) {
330    RemoveFromOriginSet(origin_url);
331    origin_size_map_.erase(origin_url);
332    space_available_map_.erase(origin_url);
333  }
334}
335
336void IndexedDBContextImpl::ForceClose(const GURL origin_url,
337                                      ForceCloseReason reason) {
338  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
339  UMA_HISTOGRAM_ENUMERATION("WebCore.IndexedDB.Context.ForceCloseReason",
340                            reason,
341                            FORCE_CLOSE_REASON_MAX);
342
343  if (data_path_.empty() || !IsInOriginSet(origin_url))
344    return;
345
346  if (factory_)
347    factory_->ForceClose(origin_url);
348  DCHECK_EQ(0UL, GetConnectionCount(origin_url));
349}
350
351size_t IndexedDBContextImpl::GetConnectionCount(const GURL& origin_url) {
352  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
353  if (data_path_.empty() || !IsInOriginSet(origin_url))
354    return 0;
355
356  if (!factory_)
357    return 0;
358
359  return factory_->GetConnectionCount(origin_url);
360}
361
362base::FilePath IndexedDBContextImpl::GetFilePath(const GURL& origin_url) const {
363  std::string origin_id = webkit_database::GetIdentifierFromOrigin(origin_url);
364  return GetIndexedDBFilePath(origin_id);
365}
366
367base::FilePath IndexedDBContextImpl::GetFilePathForTesting(
368    const std::string& origin_id) const {
369  return GetIndexedDBFilePath(origin_id);
370}
371
372void IndexedDBContextImpl::SetTaskRunnerForTesting(
373    base::SequencedTaskRunner* task_runner) {
374  DCHECK(!task_runner_);
375  task_runner_ = task_runner;
376}
377
378void IndexedDBContextImpl::ConnectionOpened(const GURL& origin_url,
379                                            IndexedDBConnection* connection) {
380  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
381  if (quota_manager_proxy()) {
382    quota_manager_proxy()->NotifyStorageAccessed(
383        quota::QuotaClient::kIndexedDatabase,
384        origin_url,
385        quota::kStorageTypeTemporary);
386  }
387  if (AddToOriginSet(origin_url)) {
388    // A newly created db, notify the quota system.
389    QueryDiskAndUpdateQuotaUsage(origin_url);
390  } else {
391    EnsureDiskUsageCacheInitialized(origin_url);
392  }
393  QueryAvailableQuota(origin_url);
394}
395
396void IndexedDBContextImpl::ConnectionClosed(const GURL& origin_url,
397                                            IndexedDBConnection* connection) {
398  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
399  if (quota_manager_proxy()) {
400    quota_manager_proxy()->NotifyStorageAccessed(
401        quota::QuotaClient::kIndexedDatabase,
402        origin_url,
403        quota::kStorageTypeTemporary);
404  }
405  if (factory_ && factory_->GetConnectionCount(origin_url) == 0)
406    QueryDiskAndUpdateQuotaUsage(origin_url);
407}
408
409void IndexedDBContextImpl::TransactionComplete(const GURL& origin_url) {
410  DCHECK(!factory_ || factory_->GetConnectionCount(origin_url) > 0);
411  QueryDiskAndUpdateQuotaUsage(origin_url);
412  QueryAvailableQuota(origin_url);
413}
414
415void IndexedDBContextImpl::DatabaseDeleted(const GURL& origin_url) {
416  AddToOriginSet(origin_url);
417  QueryDiskAndUpdateQuotaUsage(origin_url);
418  QueryAvailableQuota(origin_url);
419}
420
421bool IndexedDBContextImpl::WouldBeOverQuota(const GURL& origin_url,
422                                            int64 additional_bytes) {
423  if (space_available_map_.find(origin_url) == space_available_map_.end()) {
424    // We haven't heard back from the QuotaManager yet, just let it through.
425    return false;
426  }
427  bool over_quota = additional_bytes > space_available_map_[origin_url];
428  return over_quota;
429}
430
431bool IndexedDBContextImpl::IsOverQuota(const GURL& origin_url) {
432  const int kOneAdditionalByte = 1;
433  return WouldBeOverQuota(origin_url, kOneAdditionalByte);
434}
435
436quota::QuotaManagerProxy* IndexedDBContextImpl::quota_manager_proxy() {
437  return quota_manager_proxy_;
438}
439
440IndexedDBContextImpl::~IndexedDBContextImpl() {
441  if (factory_) {
442    TaskRunner()->PostTask(
443        FROM_HERE, base::Bind(&IndexedDBFactory::ContextDestroyed, factory_));
444    factory_ = NULL;
445  }
446
447  if (data_path_.empty())
448    return;
449
450  if (force_keep_session_state_)
451    return;
452
453  bool has_session_only_databases =
454      special_storage_policy_ &&
455      special_storage_policy_->HasSessionOnlyOrigins();
456
457  // Clearning only session-only databases, and there are none.
458  if (!has_session_only_databases)
459    return;
460
461  TaskRunner()->PostTask(
462      FROM_HERE,
463      base::Bind(
464          &ClearSessionOnlyOrigins, data_path_, special_storage_policy_));
465}
466
467base::FilePath IndexedDBContextImpl::GetIndexedDBFilePath(
468    const std::string& origin_id) const {
469  DCHECK(!data_path_.empty());
470  return data_path_.AppendASCII(origin_id).AddExtension(kIndexedDBExtension)
471      .AddExtension(kLevelDBExtension);
472}
473
474int64 IndexedDBContextImpl::ReadUsageFromDisk(const GURL& origin_url) const {
475  if (data_path_.empty())
476    return 0;
477  base::FilePath file_path = GetFilePath(origin_url);
478  return base::ComputeDirectorySize(file_path);
479}
480
481void IndexedDBContextImpl::EnsureDiskUsageCacheInitialized(
482    const GURL& origin_url) {
483  if (origin_size_map_.find(origin_url) == origin_size_map_.end())
484    origin_size_map_[origin_url] = ReadUsageFromDisk(origin_url);
485}
486
487void IndexedDBContextImpl::QueryDiskAndUpdateQuotaUsage(
488    const GURL& origin_url) {
489  int64 former_disk_usage = origin_size_map_[origin_url];
490  int64 current_disk_usage = ReadUsageFromDisk(origin_url);
491  int64 difference = current_disk_usage - former_disk_usage;
492  if (difference) {
493    origin_size_map_[origin_url] = current_disk_usage;
494    // quota_manager_proxy() is NULL in unit tests.
495    if (quota_manager_proxy()) {
496      quota_manager_proxy()->NotifyStorageModified(
497          quota::QuotaClient::kIndexedDatabase,
498          origin_url,
499          quota::kStorageTypeTemporary,
500          difference);
501    }
502  }
503}
504
505void IndexedDBContextImpl::GotUsageAndQuota(const GURL& origin_url,
506                                            quota::QuotaStatusCode status,
507                                            int64 usage,
508                                            int64 quota) {
509  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
510  DCHECK(status == quota::kQuotaStatusOk || status == quota::kQuotaErrorAbort)
511      << "status was " << status;
512  if (status == quota::kQuotaErrorAbort) {
513    // We seem to no longer care to wait around for the answer.
514    return;
515  }
516  TaskRunner()->PostTask(FROM_HERE,
517                         base::Bind(&IndexedDBContextImpl::GotUpdatedQuota,
518                                    this,
519                                    origin_url,
520                                    usage,
521                                    quota));
522}
523
524void IndexedDBContextImpl::GotUpdatedQuota(const GURL& origin_url,
525                                           int64 usage,
526                                           int64 quota) {
527  DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
528  space_available_map_[origin_url] = quota - usage;
529}
530
531void IndexedDBContextImpl::QueryAvailableQuota(const GURL& origin_url) {
532  if (!BrowserThread::CurrentlyOn(BrowserThread::IO)) {
533    DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
534    if (quota_manager_proxy()) {
535      BrowserThread::PostTask(
536          BrowserThread::IO,
537          FROM_HERE,
538          base::Bind(
539              &IndexedDBContextImpl::QueryAvailableQuota, this, origin_url));
540    }
541    return;
542  }
543  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
544  if (!quota_manager_proxy() || !quota_manager_proxy()->quota_manager())
545    return;
546  quota_manager_proxy()->quota_manager()->GetUsageAndQuota(
547      origin_url,
548      quota::kStorageTypeTemporary,
549      base::Bind(&IndexedDBContextImpl::GotUsageAndQuota, this, origin_url));
550}
551
552std::set<GURL>* IndexedDBContextImpl::GetOriginSet() {
553  if (!origin_set_) {
554    origin_set_.reset(new std::set<GURL>);
555    std::vector<GURL> origins;
556    GetAllOriginsAndPaths(data_path_, &origins, NULL);
557    for (std::vector<GURL>::const_iterator iter = origins.begin();
558         iter != origins.end();
559         ++iter) {
560      origin_set_->insert(*iter);
561    }
562  }
563  return origin_set_.get();
564}
565
566void IndexedDBContextImpl::ResetCaches() {
567  origin_set_.reset();
568  origin_size_map_.clear();
569  space_available_map_.clear();
570}
571
572base::TaskRunner* IndexedDBContextImpl::TaskRunner() const {
573  return task_runner_;
574}
575
576}  // namespace content
577