appcache_storage_impl.cc revision 5e3f23d412006dc4db4e659864679f29341e113f 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "webkit/browser/appcache/appcache_storage_impl.h"
6
7#include <algorithm>
8#include <functional>
9#include <set>
10#include <vector>
11
12#include "base/bind.h"
13#include "base/bind_helpers.h"
14#include "base/file_util.h"
15#include "base/logging.h"
16#include "base/message_loop.h"
17#include "base/stl_util.h"
18#include "base/strings/string_util.h"
19#include "net/base/cache_type.h"
20#include "net/base/net_errors.h"
21#include "sql/connection.h"
22#include "sql/transaction.h"
23#include "webkit/browser/appcache/appcache.h"
24#include "webkit/browser/appcache/appcache_database.h"
25#include "webkit/browser/appcache/appcache_entry.h"
26#include "webkit/browser/appcache/appcache_group.h"
27#include "webkit/browser/appcache/appcache_histograms.h"
28#include "webkit/browser/appcache/appcache_quota_client.h"
29#include "webkit/browser/appcache/appcache_response.h"
30#include "webkit/browser/appcache/appcache_service.h"
31#include "webkit/browser/quota/quota_client.h"
32#include "webkit/browser/quota/quota_manager.h"
33#include "webkit/browser/quota/special_storage_policy.h"
34
35namespace appcache {
36
37// Hard coded default when not using quota management.
38static const int kDefaultQuota = 5 * 1024 * 1024;
39
40static const int kMaxDiskCacheSize = 250 * 1024 * 1024;
41static const int kMaxMemDiskCacheSize = 10 * 1024 * 1024;
42static const base::FilePath::CharType kDiskCacheDirectoryName[] =
43    FILE_PATH_LITERAL("Cache");
44
45namespace {
46
47// Helpers for clearing data from the AppCacheDatabase.
48bool DeleteGroupAndRelatedRecords(AppCacheDatabase* database,
49                                  int64 group_id,
50                                  std::vector<int64>* deletable_response_ids) {
51  AppCacheDatabase::CacheRecord cache_record;
52  bool success = false;
53  if (database->FindCacheForGroup(group_id, &cache_record)) {
54    database->FindResponseIdsForCacheAsVector(cache_record.cache_id,
55                                              deletable_response_ids);
56    success =
57        database->DeleteGroup(group_id) &&
58        database->DeleteCache(cache_record.cache_id) &&
59        database->DeleteEntriesForCache(cache_record.cache_id) &&
60        database->DeleteNamespacesForCache(cache_record.cache_id) &&
61        database->DeleteOnlineWhiteListForCache(cache_record.cache_id) &&
62        database->InsertDeletableResponseIds(*deletable_response_ids);
63  } else {
64    NOTREACHED() << "A existing group without a cache is unexpected";
65    success = database->DeleteGroup(group_id);
66  }
67  return success;
68}
69
70// Destroys |database|. If there is appcache data to be deleted
71// (|force_keep_session_state| is false), deletes session-only appcache data.
72void ClearSessionOnlyOrigins(
73    AppCacheDatabase* database,
74    scoped_refptr<quota::SpecialStoragePolicy> special_storage_policy,
75    bool force_keep_session_state) {
76  scoped_ptr<AppCacheDatabase> database_to_delete(database);
77
78  // If saving session state, only delete the database.
79  if (force_keep_session_state)
80    return;
81
82  bool has_session_only_appcaches =
83      special_storage_policy.get() &&
84      special_storage_policy->HasSessionOnlyOrigins();
85
86  // Clearning only session-only databases, and there are none.
87  if (!has_session_only_appcaches)
88    return;
89
90  std::set<GURL> origins;
91  database->FindOriginsWithGroups(&origins);
92  if (origins.empty())
93    return;  // nothing to delete
94
95  sql::Connection* connection = database->db_connection();
96  if (!connection) {
97    NOTREACHED() << "Missing database connection.";
98    return;
99  }
100
101  std::set<GURL>::const_iterator origin;
102  for (origin = origins.begin(); origin != origins.end(); ++origin) {
103    if (!special_storage_policy->IsStorageSessionOnly(*origin))
104      continue;
105    if (special_storage_policy.get() &&
106        special_storage_policy->IsStorageProtected(*origin))
107      continue;
108
109    std::vector<AppCacheDatabase::GroupRecord> groups;
110    database->FindGroupsForOrigin(*origin, &groups);
111    std::vector<AppCacheDatabase::GroupRecord>::const_iterator group;
112    for (group = groups.begin(); group != groups.end(); ++group) {
113      sql::Transaction transaction(connection);
114      if (!transaction.Begin()) {
115        NOTREACHED() << "Failed to start transaction";
116        return;
117      }
118      std::vector<int64> deletable_response_ids;
119      bool success = DeleteGroupAndRelatedRecords(database,
120                                                  group->group_id,
121                                                  &deletable_response_ids);
122      success = success && transaction.Commit();
123      DCHECK(success);
124    }  // for each group
125  }  // for each origin
126}
127
128}  // namespace
129
130// DatabaseTask -----------------------------------------
131
132class AppCacheStorageImpl::DatabaseTask
133    : public base::RefCountedThreadSafe<DatabaseTask> {
134 public:
135  explicit DatabaseTask(AppCacheStorageImpl* storage)
136      : storage_(storage), database_(storage->database_),
137        io_thread_(base::MessageLoopProxy::current()) {
138    DCHECK(io_thread_.get());
139  }
140
141  void AddDelegate(DelegateReference* delegate_reference) {
142    delegates_.push_back(make_scoped_refptr(delegate_reference));
143  }
144
145  // Schedules a task to be Run() on the DB thread. Tasks
146  // are run in the order in which they are scheduled.
147  void Schedule();
148
149  // Called on the DB thread.
150  virtual void Run() = 0;
151
152  // Called on the IO thread after Run() has completed.
153  virtual void RunCompleted() {}
154
155  // Once scheduled a task cannot be cancelled, but the
156  // call to RunCompleted may be. This method should only be
157  // called on the IO thread. This is used by AppCacheStorageImpl
158  // to cancel the completion calls when AppCacheStorageImpl is
159  // destructed. This method may be overriden to release or delete
160  // additional data associated with the task that is not DB thread
161  // safe. If overriden, this base class method must be called from
162  // within the override.
163  virtual void CancelCompletion();
164
165 protected:
166  friend class base::RefCountedThreadSafe<DatabaseTask>;
167  virtual ~DatabaseTask() {}
168
169  AppCacheStorageImpl* storage_;
170  AppCacheDatabase* database_;
171  DelegateReferenceVector delegates_;
172
173 private:
174  void CallRun(base::TimeTicks schedule_time);
175  void CallRunCompleted(base::TimeTicks schedule_time);
176  void CallDisableStorage();
177
178  scoped_refptr<base::MessageLoopProxy> io_thread_;
179};
180
181void AppCacheStorageImpl::DatabaseTask::Schedule() {
182  DCHECK(storage_);
183  DCHECK(io_thread_->BelongsToCurrentThread());
184  if (storage_->db_thread_->PostTask(
185      FROM_HERE,
186      base::Bind(&DatabaseTask::CallRun, this, base::TimeTicks::Now()))) {
187    storage_->scheduled_database_tasks_.push_back(this);
188  } else {
189    NOTREACHED() << "Thread for database tasks is not running. "
190                 << "This is not always the DB thread.";
191  }
192}
193
194void AppCacheStorageImpl::DatabaseTask::CancelCompletion() {
195  DCHECK(io_thread_->BelongsToCurrentThread());
196  delegates_.clear();
197  storage_ = NULL;
198}
199
200void AppCacheStorageImpl::DatabaseTask::CallRun(
201    base::TimeTicks schedule_time) {
202  AppCacheHistograms::AddTaskQueueTimeSample(
203      base::TimeTicks::Now() - schedule_time);
204  if (!database_->is_disabled()) {
205    base::TimeTicks run_time = base::TimeTicks::Now();
206    Run();
207    AppCacheHistograms::AddTaskRunTimeSample(
208        base::TimeTicks::Now() - run_time);
209    if (database_->is_disabled()) {
210      io_thread_->PostTask(
211          FROM_HERE,
212          base::Bind(&DatabaseTask::CallDisableStorage, this));
213    }
214  }
215  io_thread_->PostTask(
216      FROM_HERE,
217      base::Bind(&DatabaseTask::CallRunCompleted, this,
218                 base::TimeTicks::Now()));
219}
220
221void AppCacheStorageImpl::DatabaseTask::CallRunCompleted(
222    base::TimeTicks schedule_time) {
223  AppCacheHistograms::AddCompletionQueueTimeSample(
224      base::TimeTicks::Now() - schedule_time);
225  if (storage_) {
226    DCHECK(io_thread_->BelongsToCurrentThread());
227    DCHECK(storage_->scheduled_database_tasks_.front() == this);
228    storage_->scheduled_database_tasks_.pop_front();
229    base::TimeTicks run_time = base::TimeTicks::Now();
230    RunCompleted();
231    AppCacheHistograms::AddCompletionRunTimeSample(
232        base::TimeTicks::Now() - run_time);
233    delegates_.clear();
234  }
235}
236
237void AppCacheStorageImpl::DatabaseTask::CallDisableStorage() {
238  if (storage_) {
239    DCHECK(io_thread_->BelongsToCurrentThread());
240    storage_->Disable();
241  }
242}
243
244// InitTask -------
245
246class AppCacheStorageImpl::InitTask : public DatabaseTask {
247 public:
248  explicit InitTask(AppCacheStorageImpl* storage)
249      : DatabaseTask(storage), last_group_id_(0),
250        last_cache_id_(0), last_response_id_(0),
251        last_deletable_response_rowid_(0) {}
252
253  // DatabaseTask:
254  virtual void Run() OVERRIDE;
255  virtual void RunCompleted() OVERRIDE;
256
257 protected:
258  virtual ~InitTask() {}
259
260 private:
261  int64 last_group_id_;
262  int64 last_cache_id_;
263  int64 last_response_id_;
264  int64 last_deletable_response_rowid_;
265  std::map<GURL, int64> usage_map_;
266};
267
268void AppCacheStorageImpl::InitTask::Run() {
269  database_->FindLastStorageIds(
270      &last_group_id_, &last_cache_id_, &last_response_id_,
271      &last_deletable_response_rowid_);
272  database_->GetAllOriginUsage(&usage_map_);
273}
274
275void AppCacheStorageImpl::InitTask::RunCompleted() {
276  storage_->last_group_id_ = last_group_id_;
277  storage_->last_cache_id_ = last_cache_id_;
278  storage_->last_response_id_ = last_response_id_;
279  storage_->last_deletable_response_rowid_ = last_deletable_response_rowid_;
280
281  if (!storage_->is_disabled()) {
282    storage_->usage_map_.swap(usage_map_);
283    const base::TimeDelta kDelay = base::TimeDelta::FromMinutes(5);
284    base::MessageLoop::current()->PostDelayedTask(
285        FROM_HERE,
286        base::Bind(&AppCacheStorageImpl::DelayedStartDeletingUnusedResponses,
287                   storage_->weak_factory_.GetWeakPtr()),
288        kDelay);
289  }
290
291  if (storage_->service()->quota_client())
292    storage_->service()->quota_client()->NotifyAppCacheReady();
293}
294
295// CloseConnectionTask -------
296
297class AppCacheStorageImpl::CloseConnectionTask : public DatabaseTask {
298 public:
299  explicit CloseConnectionTask(AppCacheStorageImpl* storage)
300      : DatabaseTask(storage) {}
301
302  // DatabaseTask:
303  virtual void Run() OVERRIDE { database_->CloseConnection(); }
304
305 protected:
306  virtual ~CloseConnectionTask() {}
307};
308
309// DisableDatabaseTask -------
310
311class AppCacheStorageImpl::DisableDatabaseTask : public DatabaseTask {
312 public:
313  explicit DisableDatabaseTask(AppCacheStorageImpl* storage)
314      : DatabaseTask(storage) {}
315
316  // DatabaseTask:
317  virtual void Run() OVERRIDE { database_->Disable(); }
318
319 protected:
320  virtual ~DisableDatabaseTask() {}
321};
322
323// GetAllInfoTask -------
324
325class AppCacheStorageImpl::GetAllInfoTask : public DatabaseTask {
326 public:
327  explicit GetAllInfoTask(AppCacheStorageImpl* storage)
328      : DatabaseTask(storage),
329        info_collection_(new AppCacheInfoCollection()) {
330  }
331
332  // DatabaseTask:
333  virtual void Run() OVERRIDE;
334  virtual void RunCompleted() OVERRIDE;
335
336 protected:
337  virtual ~GetAllInfoTask() {}
338
339 private:
340  scoped_refptr<AppCacheInfoCollection> info_collection_;
341};
342
343void AppCacheStorageImpl::GetAllInfoTask::Run() {
344  std::set<GURL> origins;
345  database_->FindOriginsWithGroups(&origins);
346  for (std::set<GURL>::const_iterator origin = origins.begin();
347       origin != origins.end(); ++origin) {
348    AppCacheInfoVector& infos =
349        info_collection_->infos_by_origin[*origin];
350    std::vector<AppCacheDatabase::GroupRecord> groups;
351    database_->FindGroupsForOrigin(*origin, &groups);
352    for (std::vector<AppCacheDatabase::GroupRecord>::const_iterator
353         group = groups.begin();
354         group != groups.end(); ++group) {
355      AppCacheDatabase::CacheRecord cache_record;
356      database_->FindCacheForGroup(group->group_id, &cache_record);
357      AppCacheInfo info;
358      info.manifest_url = group->manifest_url;
359      info.creation_time = group->creation_time;
360      info.size = cache_record.cache_size;
361      info.last_access_time = group->last_access_time;
362      info.last_update_time = cache_record.update_time;
363      info.cache_id = cache_record.cache_id;
364      info.group_id = group->group_id;
365      info.is_complete = true;
366      infos.push_back(info);
367    }
368  }
369}
370
371void AppCacheStorageImpl::GetAllInfoTask::RunCompleted() {
372  DCHECK(delegates_.size() == 1);
373  FOR_EACH_DELEGATE(delegates_, OnAllInfo(info_collection_.get()));
374}
375
376// StoreOrLoadTask -------
377
378class AppCacheStorageImpl::StoreOrLoadTask : public DatabaseTask {
379 protected:
380  explicit StoreOrLoadTask(AppCacheStorageImpl* storage)
381      : DatabaseTask(storage) {}
382  virtual ~StoreOrLoadTask() {}
383
384  bool FindRelatedCacheRecords(int64 cache_id);
385  void CreateCacheAndGroupFromRecords(
386      scoped_refptr<AppCache>* cache, scoped_refptr<AppCacheGroup>* group);
387
388  AppCacheDatabase::GroupRecord group_record_;
389  AppCacheDatabase::CacheRecord cache_record_;
390  std::vector<AppCacheDatabase::EntryRecord> entry_records_;
391  std::vector<AppCacheDatabase::NamespaceRecord>
392      intercept_namespace_records_;
393  std::vector<AppCacheDatabase::NamespaceRecord>
394      fallback_namespace_records_;
395  std::vector<AppCacheDatabase::OnlineWhiteListRecord>
396      online_whitelist_records_;
397};
398
399bool AppCacheStorageImpl::StoreOrLoadTask::FindRelatedCacheRecords(
400    int64 cache_id) {
401  return database_->FindEntriesForCache(cache_id, &entry_records_) &&
402         database_->FindNamespacesForCache(
403             cache_id, &intercept_namespace_records_,
404             &fallback_namespace_records_) &&
405         database_->FindOnlineWhiteListForCache(
406             cache_id, &online_whitelist_records_);
407}
408
409void AppCacheStorageImpl::StoreOrLoadTask::CreateCacheAndGroupFromRecords(
410    scoped_refptr<AppCache>* cache, scoped_refptr<AppCacheGroup>* group) {
411  DCHECK(storage_ && cache && group);
412
413  (*cache) = storage_->working_set_.GetCache(cache_record_.cache_id);
414  if (cache->get()) {
415    (*group) = cache->get()->owning_group();
416    DCHECK(group->get());
417    DCHECK_EQ(group_record_.group_id, group->get()->group_id());
418
419    // TODO(michaeln): histogram is fishing for clues to crbug/95101
420    if (!cache->get()->GetEntry(group_record_.manifest_url)) {
421      AppCacheHistograms::AddMissingManifestDetectedAtCallsite(
422          AppCacheHistograms::CALLSITE_0);
423    }
424
425    storage_->NotifyStorageAccessed(group_record_.origin);
426    return;
427  }
428
429  (*cache) = new AppCache(storage_, cache_record_.cache_id);
430  cache->get()->InitializeWithDatabaseRecords(
431      cache_record_, entry_records_,
432      intercept_namespace_records_,
433      fallback_namespace_records_,
434      online_whitelist_records_);
435  cache->get()->set_complete(true);
436
437  (*group) = storage_->working_set_.GetGroup(group_record_.manifest_url);
438  if (group->get()) {
439    DCHECK(group_record_.group_id == group->get()->group_id());
440    group->get()->AddCache(cache->get());
441
442    // TODO(michaeln): histogram is fishing for clues to crbug/95101
443    if (!cache->get()->GetEntry(group_record_.manifest_url)) {
444      AppCacheHistograms::AddMissingManifestDetectedAtCallsite(
445          AppCacheHistograms::CALLSITE_1);
446    }
447  } else {
448    (*group) = new AppCacheGroup(
449        storage_, group_record_.manifest_url,
450        group_record_.group_id);
451    group->get()->set_creation_time(group_record_.creation_time);
452    group->get()->AddCache(cache->get());
453
454    // TODO(michaeln): histogram is fishing for clues to crbug/95101
455    if (!cache->get()->GetEntry(group_record_.manifest_url)) {
456      AppCacheHistograms::AddMissingManifestDetectedAtCallsite(
457          AppCacheHistograms::CALLSITE_2);
458    }
459  }
460  DCHECK(group->get()->newest_complete_cache() == cache->get());
461
462  // We have to update foriegn entries if MarkEntryAsForeignTasks
463  // are in flight.
464  std::vector<GURL> urls;
465  storage_->GetPendingForeignMarkingsForCache(cache->get()->cache_id(), &urls);
466  for (std::vector<GURL>::iterator iter = urls.begin();
467       iter != urls.end(); ++iter) {
468    DCHECK(cache->get()->GetEntry(*iter));
469    cache->get()->GetEntry(*iter)->add_types(AppCacheEntry::FOREIGN);
470  }
471
472  storage_->NotifyStorageAccessed(group_record_.origin);
473
474  // TODO(michaeln): Maybe verify that the responses we expect to exist
475  // do actually exist in the disk_cache (and if not then what?)
476}
477
478// CacheLoadTask -------
479
480class AppCacheStorageImpl::CacheLoadTask : public StoreOrLoadTask {
481 public:
482  CacheLoadTask(int64 cache_id, AppCacheStorageImpl* storage)
483      : StoreOrLoadTask(storage), cache_id_(cache_id),
484        success_(false) {}
485
486  // DatabaseTask:
487  virtual void Run() OVERRIDE;
488  virtual void RunCompleted() OVERRIDE;
489
490 protected:
491  virtual ~CacheLoadTask() {}
492
493 private:
494  int64 cache_id_;
495  bool success_;
496};
497
498void AppCacheStorageImpl::CacheLoadTask::Run() {
499  success_ =
500      database_->FindCache(cache_id_, &cache_record_) &&
501      database_->FindGroup(cache_record_.group_id, &group_record_) &&
502      FindRelatedCacheRecords(cache_id_);
503
504  if (success_)
505    database_->UpdateGroupLastAccessTime(group_record_.group_id,
506                                         base::Time::Now());
507}
508
509void AppCacheStorageImpl::CacheLoadTask::RunCompleted() {
510  storage_->pending_cache_loads_.erase(cache_id_);
511  scoped_refptr<AppCache> cache;
512  scoped_refptr<AppCacheGroup> group;
513  if (success_ && !storage_->is_disabled()) {
514    DCHECK(cache_record_.cache_id == cache_id_);
515    CreateCacheAndGroupFromRecords(&cache, &group);
516  }
517  FOR_EACH_DELEGATE(delegates_, OnCacheLoaded(cache.get(), cache_id_));
518}
519
520// GroupLoadTask -------
521
522class AppCacheStorageImpl::GroupLoadTask : public StoreOrLoadTask {
523 public:
524  GroupLoadTask(GURL manifest_url, AppCacheStorageImpl* storage)
525      : StoreOrLoadTask(storage), manifest_url_(manifest_url),
526        success_(false) {}
527
528  // DatabaseTask:
529  virtual void Run() OVERRIDE;
530  virtual void RunCompleted() OVERRIDE;
531
532 protected:
533  virtual ~GroupLoadTask() {}
534
535 private:
536  GURL manifest_url_;
537  bool success_;
538};
539
540void AppCacheStorageImpl::GroupLoadTask::Run() {
541  success_ =
542      database_->FindGroupForManifestUrl(manifest_url_, &group_record_) &&
543      database_->FindCacheForGroup(group_record_.group_id, &cache_record_) &&
544      FindRelatedCacheRecords(cache_record_.cache_id);
545
546  if (success_)
547    database_->UpdateGroupLastAccessTime(group_record_.group_id,
548                                         base::Time::Now());
549}
550
551void AppCacheStorageImpl::GroupLoadTask::RunCompleted() {
552  storage_->pending_group_loads_.erase(manifest_url_);
553  scoped_refptr<AppCacheGroup> group;
554  scoped_refptr<AppCache> cache;
555  if (!storage_->is_disabled()) {
556    if (success_) {
557      DCHECK(group_record_.manifest_url == manifest_url_);
558      CreateCacheAndGroupFromRecords(&cache, &group);
559    } else {
560      group = storage_->working_set_.GetGroup(manifest_url_);
561      if (!group.get()) {
562        group =
563            new AppCacheGroup(storage_, manifest_url_, storage_->NewGroupId());
564      }
565    }
566  }
567  FOR_EACH_DELEGATE(delegates_, OnGroupLoaded(group.get(), manifest_url_));
568}
569
570// StoreGroupAndCacheTask -------
571
572class AppCacheStorageImpl::StoreGroupAndCacheTask : public StoreOrLoadTask {
573 public:
574  StoreGroupAndCacheTask(AppCacheStorageImpl* storage, AppCacheGroup* group,
575                         AppCache* newest_cache);
576
577  void GetQuotaThenSchedule();
578  void OnQuotaCallback(
579      quota::QuotaStatusCode status, int64 usage, int64 quota);
580
581  // DatabaseTask:
582  virtual void Run() OVERRIDE;
583  virtual void RunCompleted() OVERRIDE;
584  virtual void CancelCompletion() OVERRIDE;
585
586 protected:
587  virtual ~StoreGroupAndCacheTask() {}
588
589 private:
590  scoped_refptr<AppCacheGroup> group_;
591  scoped_refptr<AppCache> cache_;
592  bool success_;
593  bool would_exceed_quota_;
594  int64 space_available_;
595  int64 new_origin_usage_;
596  std::vector<int64> newly_deletable_response_ids_;
597};
598
599AppCacheStorageImpl::StoreGroupAndCacheTask::StoreGroupAndCacheTask(
600    AppCacheStorageImpl* storage, AppCacheGroup* group, AppCache* newest_cache)
601    : StoreOrLoadTask(storage), group_(group), cache_(newest_cache),
602      success_(false), would_exceed_quota_(false),
603      space_available_(-1), new_origin_usage_(-1) {
604  group_record_.group_id = group->group_id();
605  group_record_.manifest_url = group->manifest_url();
606  group_record_.origin = group_record_.manifest_url.GetOrigin();
607  newest_cache->ToDatabaseRecords(
608      group,
609      &cache_record_, &entry_records_,
610      &intercept_namespace_records_,
611      &fallback_namespace_records_,
612      &online_whitelist_records_);
613}
614
615void AppCacheStorageImpl::StoreGroupAndCacheTask::GetQuotaThenSchedule() {
616  quota::QuotaManager* quota_manager = NULL;
617  if (storage_->service()->quota_manager_proxy()) {
618    quota_manager =
619        storage_->service()->quota_manager_proxy()->quota_manager();
620  }
621
622  if (!quota_manager) {
623    if (storage_->service()->special_storage_policy() &&
624        storage_->service()->special_storage_policy()->IsStorageUnlimited(
625            group_record_.origin))
626      space_available_ = kint64max;
627    Schedule();
628    return;
629  }
630
631  // We have to ask the quota manager for the value.
632  storage_->pending_quota_queries_.insert(this);
633  quota_manager->GetUsageAndQuota(
634      group_record_.origin, quota::kStorageTypeTemporary,
635      base::Bind(&StoreGroupAndCacheTask::OnQuotaCallback, this));
636}
637
638void AppCacheStorageImpl::StoreGroupAndCacheTask::OnQuotaCallback(
639    quota::QuotaStatusCode status, int64 usage, int64 quota) {
640  if (storage_) {
641    if (status == quota::kQuotaStatusOk)
642      space_available_ = std::max(static_cast<int64>(0), quota - usage);
643    else
644      space_available_ = 0;
645    storage_->pending_quota_queries_.erase(this);
646    Schedule();
647  }
648}
649
650void AppCacheStorageImpl::StoreGroupAndCacheTask::Run() {
651  DCHECK(!success_);
652  sql::Connection* connection = database_->db_connection();
653  if (!connection)
654    return;
655
656  sql::Transaction transaction(connection);
657  if (!transaction.Begin())
658    return;
659
660  int64 old_origin_usage = database_->GetOriginUsage(group_record_.origin);
661
662  AppCacheDatabase::GroupRecord existing_group;
663  success_ = database_->FindGroup(group_record_.group_id, &existing_group);
664  if (!success_) {
665    group_record_.creation_time = base::Time::Now();
666    group_record_.last_access_time = base::Time::Now();
667    success_ = database_->InsertGroup(&group_record_);
668  } else {
669    DCHECK(group_record_.group_id == existing_group.group_id);
670    DCHECK(group_record_.manifest_url == existing_group.manifest_url);
671    DCHECK(group_record_.origin == existing_group.origin);
672
673    database_->UpdateGroupLastAccessTime(group_record_.group_id,
674                                         base::Time::Now());
675
676    AppCacheDatabase::CacheRecord cache;
677    if (database_->FindCacheForGroup(group_record_.group_id, &cache)) {
678      // Get the set of response ids in the old cache.
679      std::set<int64> existing_response_ids;
680      database_->FindResponseIdsForCacheAsSet(cache.cache_id,
681                                              &existing_response_ids);
682
683      // Remove those that remain in the new cache.
684      std::vector<AppCacheDatabase::EntryRecord>::const_iterator entry_iter =
685          entry_records_.begin();
686      while (entry_iter != entry_records_.end()) {
687        existing_response_ids.erase(entry_iter->response_id);
688        ++entry_iter;
689      }
690
691      // The rest are deletable.
692      std::set<int64>::const_iterator id_iter = existing_response_ids.begin();
693      while (id_iter != existing_response_ids.end()) {
694        newly_deletable_response_ids_.push_back(*id_iter);
695        ++id_iter;
696      }
697
698      success_ =
699          database_->DeleteCache(cache.cache_id) &&
700          database_->DeleteEntriesForCache(cache.cache_id) &&
701          database_->DeleteNamespacesForCache(cache.cache_id) &&
702          database_->DeleteOnlineWhiteListForCache(cache.cache_id) &&
703          database_->InsertDeletableResponseIds(newly_deletable_response_ids_);
704          // TODO(michaeln): store group_id too with deletable ids
705    } else {
706      NOTREACHED() << "A existing group without a cache is unexpected";
707    }
708  }
709
710  success_ =
711      success_ &&
712      database_->InsertCache(&cache_record_) &&
713      database_->InsertEntryRecords(entry_records_) &&
714      database_->InsertNamespaceRecords(intercept_namespace_records_) &&
715      database_->InsertNamespaceRecords(fallback_namespace_records_) &&
716      database_->InsertOnlineWhiteListRecords(online_whitelist_records_);
717
718  if (!success_)
719    return;
720
721  new_origin_usage_ = database_->GetOriginUsage(group_record_.origin);
722
723  // Only check quota when the new usage exceeds the old usage.
724  if (new_origin_usage_ <= old_origin_usage) {
725    success_ = transaction.Commit();
726    return;
727  }
728
729  // Use a simple hard-coded value when not using quota management.
730  if (space_available_ == -1) {
731    if (new_origin_usage_ > kDefaultQuota) {
732      would_exceed_quota_ = true;
733      success_ = false;
734      return;
735    }
736    success_ = transaction.Commit();
737    return;
738  }
739
740  // Check limits based on the space availbable given to us via the
741  // quota system.
742  int64 delta = new_origin_usage_ - old_origin_usage;
743  if (delta > space_available_) {
744    would_exceed_quota_ = true;
745    success_ = false;
746    return;
747  }
748
749  success_ = transaction.Commit();
750}
751
752void AppCacheStorageImpl::StoreGroupAndCacheTask::RunCompleted() {
753  if (success_) {
754    storage_->UpdateUsageMapAndNotify(
755        group_->manifest_url().GetOrigin(), new_origin_usage_);
756    if (cache_.get() != group_->newest_complete_cache()) {
757      cache_->set_complete(true);
758      group_->AddCache(cache_.get());
759    }
760    if (group_->creation_time().is_null())
761      group_->set_creation_time(group_record_.creation_time);
762    group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
763  }
764  FOR_EACH_DELEGATE(
765      delegates_,
766      OnGroupAndNewestCacheStored(
767          group_.get(), cache_.get(), success_, would_exceed_quota_));
768  group_ = NULL;
769  cache_ = NULL;
770
771  // TODO(michaeln): if (would_exceed_quota_) what if the current usage
772  // also exceeds the quota? http://crbug.com/83968
773}
774
775void AppCacheStorageImpl::StoreGroupAndCacheTask::CancelCompletion() {
776  // Overriden to safely drop our reference to the group and cache
777  // which are not thread safe refcounted.
778  DatabaseTask::CancelCompletion();
779  group_ = NULL;
780  cache_ = NULL;
781}
782
783// FindMainResponseTask -------
784
785// Helpers for FindMainResponseTask::Run()
786namespace {
787class SortByCachePreference
788    : public std::binary_function<
789        AppCacheDatabase::EntryRecord,
790        AppCacheDatabase::EntryRecord,
791        bool> {
792 public:
793  SortByCachePreference(int64 preferred_id, const std::set<int64>& in_use_ids)
794      : preferred_id_(preferred_id), in_use_ids_(in_use_ids) {
795  }
796  bool operator()(
797      const AppCacheDatabase::EntryRecord& lhs,
798      const AppCacheDatabase::EntryRecord& rhs) {
799    return compute_value(lhs) > compute_value(rhs);
800  }
801 private:
802  int compute_value(const AppCacheDatabase::EntryRecord& entry) {
803    if (entry.cache_id == preferred_id_)
804      return 100;
805    else if (in_use_ids_.find(entry.cache_id) != in_use_ids_.end())
806      return 50;
807    return 0;
808  }
809  int64 preferred_id_;
810  const std::set<int64>& in_use_ids_;
811};
812
813bool SortByLength(
814    const AppCacheDatabase::NamespaceRecord& lhs,
815    const AppCacheDatabase::NamespaceRecord& rhs) {
816  return lhs.namespace_.namespace_url.spec().length() >
817         rhs.namespace_.namespace_url.spec().length();
818}
819
820class NetworkNamespaceHelper {
821 public:
822  explicit NetworkNamespaceHelper(AppCacheDatabase* database)
823      : database_(database) {
824  }
825
826  bool IsInNetworkNamespace(const GURL& url, int64 cache_id) {
827    typedef std::pair<WhiteListMap::iterator, bool> InsertResult;
828    InsertResult result = namespaces_map_.insert(
829        WhiteListMap::value_type(cache_id, NamespaceVector()));
830    if (result.second)
831      GetOnlineWhiteListForCache(cache_id, &result.first->second);
832    return AppCache::FindNamespace(result.first->second, url) != NULL;
833  }
834
835 private:
836  void GetOnlineWhiteListForCache(
837      int64 cache_id, NamespaceVector* namespaces) {
838    DCHECK(namespaces && namespaces->empty());
839    typedef std::vector<AppCacheDatabase::OnlineWhiteListRecord>
840        WhiteListVector;
841    WhiteListVector records;
842    if (!database_->FindOnlineWhiteListForCache(cache_id, &records))
843      return;
844    WhiteListVector::const_iterator iter = records.begin();
845    while (iter != records.end()) {
846      namespaces->push_back(
847            Namespace(NETWORK_NAMESPACE, iter->namespace_url, GURL(),
848                      iter->is_pattern));
849      ++iter;
850    }
851  }
852
853  // Key is cache id
854  typedef std::map<int64, NamespaceVector> WhiteListMap;
855  WhiteListMap namespaces_map_;
856  AppCacheDatabase* database_;
857};
858
859}  // namespace
860
861class AppCacheStorageImpl::FindMainResponseTask : public DatabaseTask {
862 public:
863  FindMainResponseTask(AppCacheStorageImpl* storage,
864                       const GURL& url,
865                       const GURL& preferred_manifest_url,
866                       const AppCacheWorkingSet::GroupMap* groups_in_use)
867      : DatabaseTask(storage), url_(url),
868        preferred_manifest_url_(preferred_manifest_url),
869        cache_id_(kNoCacheId), group_id_(0) {
870    if (groups_in_use) {
871      for (AppCacheWorkingSet::GroupMap::const_iterator it =
872               groups_in_use->begin();
873           it != groups_in_use->end(); ++it) {
874        AppCacheGroup* group = it->second;
875        AppCache* cache = group->newest_complete_cache();
876        if (group->is_obsolete() || !cache)
877          continue;
878        cache_ids_in_use_.insert(cache->cache_id());
879      }
880    }
881  }
882
883  // DatabaseTask:
884  virtual void Run() OVERRIDE;
885  virtual void RunCompleted() OVERRIDE;
886
887 protected:
888  virtual ~FindMainResponseTask() {}
889
890 private:
891  typedef std::vector<AppCacheDatabase::NamespaceRecord*>
892      NamespaceRecordPtrVector;
893
894  bool FindExactMatch(int64 preferred_id);
895  bool FindNamespaceMatch(int64 preferred_id);
896  bool FindNamespaceHelper(
897      int64 preferred_cache_id,
898      AppCacheDatabase::NamespaceRecordVector* namespaces,
899      NetworkNamespaceHelper* network_namespace_helper);
900  bool FindFirstValidNamespace(const NamespaceRecordPtrVector& namespaces);
901
902  GURL url_;
903  GURL preferred_manifest_url_;
904  std::set<int64> cache_ids_in_use_;
905  AppCacheEntry entry_;
906  AppCacheEntry fallback_entry_;
907  GURL namespace_entry_url_;
908  int64 cache_id_;
909  int64 group_id_;
910  GURL manifest_url_;
911};
912
913
914
915void AppCacheStorageImpl::FindMainResponseTask::Run() {
916  // NOTE: The heuristics around choosing amoungst multiple candidates
917  // is underspecified, and just plain not fully understood. This needs
918  // to be refined.
919
920  // The 'preferred_manifest_url' is the url of the manifest associated
921  // with the page that opened or embedded the page being loaded now.
922  // We have a strong preference to use resources from that cache.
923  // We also have a lesser bias to use resources from caches that are currently
924  // being used by other unrelated pages.
925  // TODO(michaeln): come up with a 'preferred_manifest_url' in more cases
926  // - when navigating a frame whose current contents are from an appcache
927  // - when clicking an href in a frame that is appcached
928  int64 preferred_cache_id = kNoCacheId;
929  if (!preferred_manifest_url_.is_empty()) {
930    AppCacheDatabase::GroupRecord preferred_group;
931    AppCacheDatabase::CacheRecord preferred_cache;
932    if (database_->FindGroupForManifestUrl(
933            preferred_manifest_url_, &preferred_group) &&
934        database_->FindCacheForGroup(
935            preferred_group.group_id, &preferred_cache)) {
936      preferred_cache_id = preferred_cache.cache_id;
937    }
938  }
939
940  if (FindExactMatch(preferred_cache_id) ||
941      FindNamespaceMatch(preferred_cache_id)) {
942    // We found something.
943    DCHECK(cache_id_ != kNoCacheId && !manifest_url_.is_empty() &&
944           group_id_ != 0);
945    return;
946  }
947
948  // We didn't find anything.
949  DCHECK(cache_id_ == kNoCacheId && manifest_url_.is_empty() &&
950         group_id_ == 0);
951}
952
953bool AppCacheStorageImpl::
954FindMainResponseTask::FindExactMatch(int64 preferred_cache_id) {
955  std::vector<AppCacheDatabase::EntryRecord> entries;
956  if (database_->FindEntriesForUrl(url_, &entries) && !entries.empty()) {
957    // Sort them in order of preference, from the preferred_cache first,
958    // followed by hits from caches that are 'in use', then the rest.
959    std::sort(entries.begin(), entries.end(),
960              SortByCachePreference(preferred_cache_id, cache_ids_in_use_));
961
962    // Take the first with a valid, non-foreign entry.
963    std::vector<AppCacheDatabase::EntryRecord>::iterator iter;
964    for (iter = entries.begin(); iter < entries.end(); ++iter) {
965      AppCacheDatabase::GroupRecord group_record;
966      if ((iter->flags & AppCacheEntry::FOREIGN) ||
967          !database_->FindGroupForCache(iter->cache_id, &group_record)) {
968        continue;
969      }
970      manifest_url_ = group_record.manifest_url;
971      group_id_ = group_record.group_id;
972      entry_ = AppCacheEntry(iter->flags, iter->response_id);
973      cache_id_ = iter->cache_id;
974      return true;  // We found an exact match.
975    }
976  }
977  return false;
978}
979
980bool AppCacheStorageImpl::
981FindMainResponseTask::FindNamespaceMatch(int64 preferred_cache_id) {
982  AppCacheDatabase::NamespaceRecordVector all_intercepts;
983  AppCacheDatabase::NamespaceRecordVector all_fallbacks;
984  if (!database_->FindNamespacesForOrigin(
985          url_.GetOrigin(), &all_intercepts, &all_fallbacks)
986      || (all_intercepts.empty() && all_fallbacks.empty())) {
987    return false;
988  }
989
990  NetworkNamespaceHelper network_namespace_helper(database_);
991  if (FindNamespaceHelper(preferred_cache_id,
992                          &all_intercepts,
993                          &network_namespace_helper) ||
994      FindNamespaceHelper(preferred_cache_id,
995                          &all_fallbacks,
996                          &network_namespace_helper)) {
997    return true;
998  }
999  return false;
1000}
1001
1002bool AppCacheStorageImpl::
1003FindMainResponseTask::FindNamespaceHelper(
1004    int64 preferred_cache_id,
1005    AppCacheDatabase::NamespaceRecordVector* namespaces,
1006    NetworkNamespaceHelper* network_namespace_helper) {
1007  // Sort them by length, longer matches within the same cache/bucket take
1008  // precedence.
1009  std::sort(namespaces->begin(), namespaces->end(), SortByLength);
1010
1011  NamespaceRecordPtrVector preferred_namespaces;
1012  NamespaceRecordPtrVector inuse_namespaces;
1013  NamespaceRecordPtrVector other_namespaces;
1014  std::vector<AppCacheDatabase::NamespaceRecord>::iterator iter;
1015  for (iter = namespaces->begin(); iter < namespaces->end(); ++iter) {
1016    // Skip those that aren't a match.
1017    if (!iter->namespace_.IsMatch(url_))
1018      continue;
1019
1020    // Skip namespaces where the requested url falls into a network
1021    // namespace of its containing appcache.
1022    if (network_namespace_helper->IsInNetworkNamespace(url_, iter->cache_id))
1023      continue;
1024
1025    // Bin them into one of our three buckets.
1026    if (iter->cache_id == preferred_cache_id)
1027      preferred_namespaces.push_back(&(*iter));
1028    else if (cache_ids_in_use_.find(iter->cache_id) != cache_ids_in_use_.end())
1029      inuse_namespaces.push_back(&(*iter));
1030    else
1031      other_namespaces.push_back(&(*iter));
1032  }
1033
1034  if (FindFirstValidNamespace(preferred_namespaces) ||
1035      FindFirstValidNamespace(inuse_namespaces) ||
1036      FindFirstValidNamespace(other_namespaces))
1037    return true;  // We found one.
1038
1039  // We didn't find anything.
1040  return false;
1041}
1042
1043bool AppCacheStorageImpl::
1044FindMainResponseTask::FindFirstValidNamespace(
1045    const NamespaceRecordPtrVector& namespaces) {
1046  // Take the first with a valid, non-foreign entry.
1047  NamespaceRecordPtrVector::const_iterator iter;
1048  for (iter = namespaces.begin(); iter < namespaces.end();  ++iter) {
1049    AppCacheDatabase::EntryRecord entry_record;
1050    if (database_->FindEntry((*iter)->cache_id, (*iter)->namespace_.target_url,
1051                             &entry_record)) {
1052      AppCacheDatabase::GroupRecord group_record;
1053      if ((entry_record.flags & AppCacheEntry::FOREIGN) ||
1054          !database_->FindGroupForCache(entry_record.cache_id, &group_record)) {
1055        continue;
1056      }
1057      manifest_url_ = group_record.manifest_url;
1058      group_id_ = group_record.group_id;
1059      cache_id_ = (*iter)->cache_id;
1060      namespace_entry_url_ = (*iter)->namespace_.target_url;
1061      if ((*iter)->namespace_.type == FALLBACK_NAMESPACE)
1062        fallback_entry_ = AppCacheEntry(entry_record.flags,
1063                                        entry_record.response_id);
1064      else
1065        entry_ = AppCacheEntry(entry_record.flags, entry_record.response_id);
1066      return true;  // We found one.
1067    }
1068  }
1069  return false;  // We didn't find a match.
1070}
1071
1072void AppCacheStorageImpl::FindMainResponseTask::RunCompleted() {
1073  storage_->CallOnMainResponseFound(
1074      &delegates_, url_, entry_, namespace_entry_url_, fallback_entry_,
1075      cache_id_, group_id_, manifest_url_);
1076}
1077
1078// MarkEntryAsForeignTask -------
1079
1080class AppCacheStorageImpl::MarkEntryAsForeignTask : public DatabaseTask {
1081 public:
1082  MarkEntryAsForeignTask(
1083      AppCacheStorageImpl* storage, const GURL& url, int64 cache_id)
1084      : DatabaseTask(storage), cache_id_(cache_id), entry_url_(url) {}
1085
1086  // DatabaseTask:
1087  virtual void Run() OVERRIDE;
1088  virtual void RunCompleted() OVERRIDE;
1089
1090 protected:
1091  virtual ~MarkEntryAsForeignTask() {}
1092
1093 private:
1094  int64 cache_id_;
1095  GURL entry_url_;
1096};
1097
1098void AppCacheStorageImpl::MarkEntryAsForeignTask::Run() {
1099  database_->AddEntryFlags(entry_url_, cache_id_, AppCacheEntry::FOREIGN);
1100}
1101
1102void AppCacheStorageImpl::MarkEntryAsForeignTask::RunCompleted() {
1103  DCHECK(storage_->pending_foreign_markings_.front().first == entry_url_ &&
1104         storage_->pending_foreign_markings_.front().second == cache_id_);
1105  storage_->pending_foreign_markings_.pop_front();
1106}
1107
1108// MakeGroupObsoleteTask -------
1109
1110class AppCacheStorageImpl::MakeGroupObsoleteTask : public DatabaseTask {
1111 public:
1112  MakeGroupObsoleteTask(AppCacheStorageImpl* storage, AppCacheGroup* group);
1113
1114  // DatabaseTask:
1115  virtual void Run() OVERRIDE;
1116  virtual void RunCompleted() OVERRIDE;
1117  virtual void CancelCompletion() OVERRIDE;
1118
1119 protected:
1120  virtual ~MakeGroupObsoleteTask() {}
1121
1122 private:
1123  scoped_refptr<AppCacheGroup> group_;
1124  int64 group_id_;
1125  GURL origin_;
1126  bool success_;
1127  int64 new_origin_usage_;
1128  std::vector<int64> newly_deletable_response_ids_;
1129};
1130
1131AppCacheStorageImpl::MakeGroupObsoleteTask::MakeGroupObsoleteTask(
1132    AppCacheStorageImpl* storage, AppCacheGroup* group)
1133    : DatabaseTask(storage), group_(group), group_id_(group->group_id()),
1134      origin_(group->manifest_url().GetOrigin()),
1135      success_(false), new_origin_usage_(-1) {
1136}
1137
1138void AppCacheStorageImpl::MakeGroupObsoleteTask::Run() {
1139  DCHECK(!success_);
1140  sql::Connection* connection = database_->db_connection();
1141  if (!connection)
1142    return;
1143
1144  sql::Transaction transaction(connection);
1145  if (!transaction.Begin())
1146    return;
1147
1148  AppCacheDatabase::GroupRecord group_record;
1149  if (!database_->FindGroup(group_id_, &group_record)) {
1150    // This group doesn't exists in the database, nothing todo here.
1151    new_origin_usage_ = database_->GetOriginUsage(origin_);
1152    success_ = true;
1153    return;
1154  }
1155
1156  DCHECK_EQ(group_record.origin, origin_);
1157  success_ = DeleteGroupAndRelatedRecords(database_,
1158                                          group_id_,
1159                                          &newly_deletable_response_ids_);
1160
1161  new_origin_usage_ = database_->GetOriginUsage(origin_);
1162  success_ = success_ && transaction.Commit();
1163}
1164
1165void AppCacheStorageImpl::MakeGroupObsoleteTask::RunCompleted() {
1166  if (success_) {
1167    group_->set_obsolete(true);
1168    if (!storage_->is_disabled()) {
1169      storage_->UpdateUsageMapAndNotify(origin_, new_origin_usage_);
1170      group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
1171
1172      // Also remove from the working set, caches for an 'obsolete' group
1173      // may linger in use, but the group itself cannot be looked up by
1174      // 'manifest_url' in the working set any longer.
1175      storage_->working_set()->RemoveGroup(group_.get());
1176    }
1177  }
1178  FOR_EACH_DELEGATE(delegates_, OnGroupMadeObsolete(group_.get(), success_));
1179  group_ = NULL;
1180}
1181
1182void AppCacheStorageImpl::MakeGroupObsoleteTask::CancelCompletion() {
1183  // Overriden to safely drop our reference to the group
1184  // which is not thread safe refcounted.
1185  DatabaseTask::CancelCompletion();
1186  group_ = NULL;
1187}
1188
1189// GetDeletableResponseIdsTask -------
1190
1191class AppCacheStorageImpl::GetDeletableResponseIdsTask : public DatabaseTask {
1192 public:
1193  GetDeletableResponseIdsTask(AppCacheStorageImpl* storage, int64 max_rowid)
1194      : DatabaseTask(storage), max_rowid_(max_rowid) {}
1195
1196  // DatabaseTask:
1197  virtual void Run() OVERRIDE;
1198  virtual void RunCompleted() OVERRIDE;
1199
1200 protected:
1201  virtual ~GetDeletableResponseIdsTask() {}
1202
1203 private:
1204  int64 max_rowid_;
1205  std::vector<int64> response_ids_;
1206};
1207
1208void AppCacheStorageImpl::GetDeletableResponseIdsTask::Run() {
1209  const int kSqlLimit = 1000;
1210  database_->GetDeletableResponseIds(&response_ids_, max_rowid_, kSqlLimit);
1211  // TODO(michaeln): retrieve group_ids too
1212}
1213
1214void AppCacheStorageImpl::GetDeletableResponseIdsTask::RunCompleted() {
1215  if (!response_ids_.empty())
1216    storage_->StartDeletingResponses(response_ids_);
1217}
1218
1219// InsertDeletableResponseIdsTask -------
1220
1221class AppCacheStorageImpl::InsertDeletableResponseIdsTask
1222    : public DatabaseTask {
1223 public:
1224  explicit InsertDeletableResponseIdsTask(AppCacheStorageImpl* storage)
1225      : DatabaseTask(storage) {}
1226
1227  // DatabaseTask:
1228  virtual void Run() OVERRIDE;
1229
1230  std::vector<int64> response_ids_;
1231
1232 protected:
1233  virtual ~InsertDeletableResponseIdsTask() {}
1234};
1235
1236void AppCacheStorageImpl::InsertDeletableResponseIdsTask::Run() {
1237  database_->InsertDeletableResponseIds(response_ids_);
1238  // TODO(michaeln): store group_ids too
1239}
1240
1241// DeleteDeletableResponseIdsTask -------
1242
1243class AppCacheStorageImpl::DeleteDeletableResponseIdsTask
1244    : public DatabaseTask {
1245 public:
1246  explicit DeleteDeletableResponseIdsTask(AppCacheStorageImpl* storage)
1247      : DatabaseTask(storage) {}
1248
1249  // DatabaseTask:
1250  virtual void Run() OVERRIDE;
1251
1252  std::vector<int64> response_ids_;
1253
1254 protected:
1255  virtual ~DeleteDeletableResponseIdsTask() {}
1256};
1257
1258void AppCacheStorageImpl::DeleteDeletableResponseIdsTask::Run() {
1259  database_->DeleteDeletableResponseIds(response_ids_);
1260}
1261
1262// UpdateGroupLastAccessTimeTask -------
1263
1264class AppCacheStorageImpl::UpdateGroupLastAccessTimeTask
1265    : public DatabaseTask {
1266 public:
1267  UpdateGroupLastAccessTimeTask(
1268      AppCacheStorageImpl* storage, AppCacheGroup* group, base::Time time)
1269      : DatabaseTask(storage), group_id_(group->group_id()),
1270        last_access_time_(time) {
1271    storage->NotifyStorageAccessed(group->manifest_url().GetOrigin());
1272  }
1273
1274  // DatabaseTask:
1275  virtual void Run() OVERRIDE;
1276
1277 protected:
1278  virtual ~UpdateGroupLastAccessTimeTask() {}
1279
1280 private:
1281  int64 group_id_;
1282  base::Time last_access_time_;
1283};
1284
1285void AppCacheStorageImpl::UpdateGroupLastAccessTimeTask::Run() {
1286  database_->UpdateGroupLastAccessTime(group_id_, last_access_time_);
1287}
1288
1289
1290// AppCacheStorageImpl ---------------------------------------------------
1291
1292AppCacheStorageImpl::AppCacheStorageImpl(AppCacheService* service)
1293    : AppCacheStorage(service),
1294      is_incognito_(false),
1295      is_response_deletion_scheduled_(false),
1296      did_start_deleting_responses_(false),
1297      last_deletable_response_rowid_(0),
1298      database_(NULL),
1299      is_disabled_(false),
1300      weak_factory_(this) {
1301}
1302
1303AppCacheStorageImpl::~AppCacheStorageImpl() {
1304  std::for_each(pending_quota_queries_.begin(),
1305                pending_quota_queries_.end(),
1306                std::mem_fun(&DatabaseTask::CancelCompletion));
1307  std::for_each(scheduled_database_tasks_.begin(),
1308                scheduled_database_tasks_.end(),
1309                std::mem_fun(&DatabaseTask::CancelCompletion));
1310
1311  if (database_ &&
1312      !db_thread_->PostTask(
1313          FROM_HERE,
1314          base::Bind(&ClearSessionOnlyOrigins, database_,
1315                     make_scoped_refptr(service_->special_storage_policy()),
1316                     service()->force_keep_session_state()))) {
1317    delete database_;
1318  }
1319}
1320
1321void AppCacheStorageImpl::Initialize(const base::FilePath& cache_directory,
1322                                     base::MessageLoopProxy* db_thread,
1323                                     base::MessageLoopProxy* cache_thread) {
1324  DCHECK(db_thread);
1325
1326  cache_directory_ = cache_directory;
1327  is_incognito_ = cache_directory_.empty();
1328
1329  base::FilePath db_file_path;
1330  if (!is_incognito_)
1331    db_file_path = cache_directory_.Append(kAppCacheDatabaseName);
1332  database_ = new AppCacheDatabase(db_file_path);
1333
1334  db_thread_ = db_thread;
1335  cache_thread_ = cache_thread;
1336
1337  scoped_refptr<InitTask> task(new InitTask(this));
1338  task->Schedule();
1339}
1340
1341void AppCacheStorageImpl::Disable() {
1342  if (is_disabled_)
1343    return;
1344  VLOG(1) << "Disabling appcache storage.";
1345  is_disabled_ = true;
1346  ClearUsageMapAndNotify();
1347  working_set()->Disable();
1348  if (disk_cache_)
1349    disk_cache_->Disable();
1350  scoped_refptr<DisableDatabaseTask> task(new DisableDatabaseTask(this));
1351  task->Schedule();
1352}
1353
1354void AppCacheStorageImpl::GetAllInfo(Delegate* delegate) {
1355  DCHECK(delegate);
1356  scoped_refptr<GetAllInfoTask> task(new GetAllInfoTask(this));
1357  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1358  task->Schedule();
1359}
1360
1361void AppCacheStorageImpl::LoadCache(int64 id, Delegate* delegate) {
1362  DCHECK(delegate);
1363  if (is_disabled_) {
1364    delegate->OnCacheLoaded(NULL, id);
1365    return;
1366  }
1367
1368  AppCache* cache = working_set_.GetCache(id);
1369  if (cache) {
1370    delegate->OnCacheLoaded(cache, id);
1371    if (cache->owning_group()) {
1372      scoped_refptr<DatabaseTask> update_task(
1373          new UpdateGroupLastAccessTimeTask(
1374              this, cache->owning_group(), base::Time::Now()));
1375      update_task->Schedule();
1376    }
1377    return;
1378  }
1379  scoped_refptr<CacheLoadTask> task(GetPendingCacheLoadTask(id));
1380  if (task.get()) {
1381    task->AddDelegate(GetOrCreateDelegateReference(delegate));
1382    return;
1383  }
1384  task = new CacheLoadTask(id, this);
1385  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1386  task->Schedule();
1387  pending_cache_loads_[id] = task.get();
1388}
1389
1390void AppCacheStorageImpl::LoadOrCreateGroup(
1391    const GURL& manifest_url, Delegate* delegate) {
1392  DCHECK(delegate);
1393  if (is_disabled_) {
1394    delegate->OnGroupLoaded(NULL, manifest_url);
1395    return;
1396  }
1397
1398  AppCacheGroup* group = working_set_.GetGroup(manifest_url);
1399  if (group) {
1400    delegate->OnGroupLoaded(group, manifest_url);
1401    scoped_refptr<DatabaseTask> update_task(
1402        new UpdateGroupLastAccessTimeTask(
1403            this, group, base::Time::Now()));
1404    update_task->Schedule();
1405    return;
1406  }
1407
1408  scoped_refptr<GroupLoadTask> task(GetPendingGroupLoadTask(manifest_url));
1409  if (task.get()) {
1410    task->AddDelegate(GetOrCreateDelegateReference(delegate));
1411    return;
1412  }
1413
1414  if (usage_map_.find(manifest_url.GetOrigin()) == usage_map_.end()) {
1415    // No need to query the database, return a new group immediately.
1416    scoped_refptr<AppCacheGroup> group(new AppCacheGroup(
1417        service_->storage(), manifest_url, NewGroupId()));
1418    delegate->OnGroupLoaded(group.get(), manifest_url);
1419    return;
1420  }
1421
1422  task = new GroupLoadTask(manifest_url, this);
1423  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1424  task->Schedule();
1425  pending_group_loads_[manifest_url] = task.get();
1426}
1427
1428void AppCacheStorageImpl::StoreGroupAndNewestCache(
1429    AppCacheGroup* group, AppCache* newest_cache, Delegate* delegate) {
1430  // TODO(michaeln): distinguish between a simple update of an existing
1431  // cache that just adds new master entry(s), and the insertion of a
1432  // whole new cache. The StoreGroupAndCacheTask as written will handle
1433  // the simple update case in a very heavy weight way (delete all and
1434  // the reinsert all over again).
1435  DCHECK(group && delegate && newest_cache);
1436  scoped_refptr<StoreGroupAndCacheTask> task(
1437      new StoreGroupAndCacheTask(this, group, newest_cache));
1438  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1439  task->GetQuotaThenSchedule();
1440
1441  // TODO(michaeln): histogram is fishing for clues to crbug/95101
1442  if (!newest_cache->GetEntry(group->manifest_url())) {
1443    AppCacheHistograms::AddMissingManifestDetectedAtCallsite(
1444        AppCacheHistograms::CALLSITE_3);
1445  }
1446}
1447
1448void AppCacheStorageImpl::FindResponseForMainRequest(
1449    const GURL& url, const GURL& preferred_manifest_url,
1450    Delegate* delegate) {
1451  DCHECK(delegate);
1452
1453  const GURL* url_ptr = &url;
1454  GURL url_no_ref;
1455  if (url.has_ref()) {
1456    GURL::Replacements replacements;
1457    replacements.ClearRef();
1458    url_no_ref = url.ReplaceComponents(replacements);
1459    url_ptr = &url_no_ref;
1460  }
1461
1462  const GURL origin = url.GetOrigin();
1463
1464  // First look in our working set for a direct hit without having to query
1465  // the database.
1466  const AppCacheWorkingSet::GroupMap* groups_in_use =
1467      working_set()->GetGroupsInOrigin(origin);
1468  if (groups_in_use) {
1469    if (!preferred_manifest_url.is_empty()) {
1470      AppCacheWorkingSet::GroupMap::const_iterator found =
1471          groups_in_use->find(preferred_manifest_url);
1472      if (found != groups_in_use->end() &&
1473          FindResponseForMainRequestInGroup(
1474              found->second, *url_ptr, delegate)) {
1475          return;
1476      }
1477    } else {
1478      for (AppCacheWorkingSet::GroupMap::const_iterator it =
1479              groups_in_use->begin();
1480           it != groups_in_use->end(); ++it) {
1481        if (FindResponseForMainRequestInGroup(
1482                it->second, *url_ptr, delegate)) {
1483          return;
1484        }
1485      }
1486    }
1487  }
1488
1489  if (IsInitTaskComplete() &&  usage_map_.find(origin) == usage_map_.end()) {
1490    // No need to query the database, return async'ly but without going thru
1491    // the DB thread.
1492    scoped_refptr<AppCacheGroup> no_group;
1493    scoped_refptr<AppCache> no_cache;
1494    ScheduleSimpleTask(
1495        base::Bind(&AppCacheStorageImpl::DeliverShortCircuitedFindMainResponse,
1496                   weak_factory_.GetWeakPtr(), url, AppCacheEntry(), no_group,
1497                   no_cache,
1498                   make_scoped_refptr(GetOrCreateDelegateReference(delegate))));
1499    return;
1500  }
1501
1502  // We have to query the database, schedule a database task to do so.
1503  scoped_refptr<FindMainResponseTask> task(
1504      new FindMainResponseTask(this, *url_ptr, preferred_manifest_url,
1505                               groups_in_use));
1506  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1507  task->Schedule();
1508}
1509
1510bool AppCacheStorageImpl::FindResponseForMainRequestInGroup(
1511    AppCacheGroup* group,  const GURL& url, Delegate* delegate) {
1512  AppCache* cache = group->newest_complete_cache();
1513  if (group->is_obsolete() || !cache)
1514    return false;
1515
1516  AppCacheEntry* entry = cache->GetEntry(url);
1517  if (!entry || entry->IsForeign())
1518    return false;
1519
1520  ScheduleSimpleTask(
1521      base::Bind(&AppCacheStorageImpl::DeliverShortCircuitedFindMainResponse,
1522                 weak_factory_.GetWeakPtr(), url, *entry,
1523                 make_scoped_refptr(group), make_scoped_refptr(cache),
1524                 make_scoped_refptr(GetOrCreateDelegateReference(delegate))));
1525  return true;
1526}
1527
1528void AppCacheStorageImpl::DeliverShortCircuitedFindMainResponse(
1529    const GURL& url,
1530    const AppCacheEntry& found_entry,
1531    scoped_refptr<AppCacheGroup> group,
1532    scoped_refptr<AppCache> cache,
1533    scoped_refptr<DelegateReference> delegate_ref) {
1534  if (delegate_ref->delegate) {
1535    DelegateReferenceVector delegates(1, delegate_ref);
1536    CallOnMainResponseFound(
1537        &delegates, url, found_entry,
1538        GURL(), AppCacheEntry(),
1539        cache.get() ? cache->cache_id() : kNoCacheId,
1540        group.get() ? group->group_id() : kNoCacheId,
1541        group.get() ? group->manifest_url() : GURL());
1542  }
1543}
1544
1545void AppCacheStorageImpl::CallOnMainResponseFound(
1546    DelegateReferenceVector* delegates,
1547    const GURL& url, const AppCacheEntry& entry,
1548    const GURL& namespace_entry_url, const AppCacheEntry& fallback_entry,
1549    int64 cache_id, int64 group_id, const GURL& manifest_url) {
1550  FOR_EACH_DELEGATE(
1551      (*delegates),
1552      OnMainResponseFound(url, entry,
1553                          namespace_entry_url, fallback_entry,
1554                          cache_id, group_id, manifest_url));
1555}
1556
1557void AppCacheStorageImpl::FindResponseForSubRequest(
1558    AppCache* cache, const GURL& url,
1559    AppCacheEntry* found_entry, AppCacheEntry* found_fallback_entry,
1560    bool* found_network_namespace) {
1561  DCHECK(cache && cache->is_complete());
1562
1563  // When a group is forcibly deleted, all subresource loads for pages
1564  // using caches in the group will result in a synthesized network errors.
1565  // Forcible deletion is not a function that is covered by the HTML5 spec.
1566  if (cache->owning_group()->is_being_deleted()) {
1567    *found_entry = AppCacheEntry();
1568    *found_fallback_entry = AppCacheEntry();
1569    *found_network_namespace = false;
1570    return;
1571  }
1572
1573  GURL fallback_namespace_not_used;
1574  GURL intercept_namespace_not_used;
1575  cache->FindResponseForRequest(
1576      url, found_entry, &intercept_namespace_not_used,
1577      found_fallback_entry, &fallback_namespace_not_used,
1578      found_network_namespace);
1579}
1580
1581void AppCacheStorageImpl::MarkEntryAsForeign(
1582    const GURL& entry_url, int64 cache_id) {
1583  AppCache* cache = working_set_.GetCache(cache_id);
1584  if (cache) {
1585    AppCacheEntry* entry = cache->GetEntry(entry_url);
1586    DCHECK(entry);
1587    if (entry)
1588      entry->add_types(AppCacheEntry::FOREIGN);
1589  }
1590  scoped_refptr<MarkEntryAsForeignTask> task(
1591      new MarkEntryAsForeignTask(this, entry_url, cache_id));
1592  task->Schedule();
1593  pending_foreign_markings_.push_back(std::make_pair(entry_url, cache_id));
1594}
1595
1596void AppCacheStorageImpl::MakeGroupObsolete(
1597    AppCacheGroup* group, Delegate* delegate) {
1598  DCHECK(group && delegate);
1599  scoped_refptr<MakeGroupObsoleteTask> task(
1600      new MakeGroupObsoleteTask(this, group));
1601  task->AddDelegate(GetOrCreateDelegateReference(delegate));
1602  task->Schedule();
1603}
1604
1605AppCacheResponseReader* AppCacheStorageImpl::CreateResponseReader(
1606    const GURL& manifest_url, int64 group_id, int64 response_id) {
1607  return new AppCacheResponseReader(response_id, group_id, disk_cache());
1608}
1609
1610AppCacheResponseWriter* AppCacheStorageImpl::CreateResponseWriter(
1611    const GURL& manifest_url, int64 group_id) {
1612  return new AppCacheResponseWriter(NewResponseId(), group_id, disk_cache());
1613}
1614
1615void AppCacheStorageImpl::DoomResponses(
1616    const GURL& manifest_url, const std::vector<int64>& response_ids) {
1617  if (response_ids.empty())
1618    return;
1619
1620  // Start deleting them from the disk cache lazily.
1621  StartDeletingResponses(response_ids);
1622
1623  // Also schedule a database task to record these ids in the
1624  // deletable responses table.
1625  // TODO(michaeln): There is a race here. If the browser crashes
1626  // prior to committing these rows to the database and prior to us
1627  // having deleted them from the disk cache, we'll never delete them.
1628  scoped_refptr<InsertDeletableResponseIdsTask> task(
1629      new InsertDeletableResponseIdsTask(this));
1630  task->response_ids_ = response_ids;
1631  task->Schedule();
1632}
1633
1634void AppCacheStorageImpl::DeleteResponses(
1635    const GURL& manifest_url, const std::vector<int64>& response_ids) {
1636  if (response_ids.empty())
1637    return;
1638  StartDeletingResponses(response_ids);
1639}
1640
1641void AppCacheStorageImpl::PurgeMemory() {
1642  scoped_refptr<CloseConnectionTask> task(new CloseConnectionTask(this));
1643  task->Schedule();
1644}
1645
1646void AppCacheStorageImpl::DelayedStartDeletingUnusedResponses() {
1647  // Only if we haven't already begun.
1648  if (!did_start_deleting_responses_) {
1649    scoped_refptr<GetDeletableResponseIdsTask> task(
1650        new GetDeletableResponseIdsTask(this, last_deletable_response_rowid_));
1651    task->Schedule();
1652  }
1653}
1654
1655void AppCacheStorageImpl::StartDeletingResponses(
1656    const std::vector<int64>& response_ids) {
1657  DCHECK(!response_ids.empty());
1658  did_start_deleting_responses_ = true;
1659  deletable_response_ids_.insert(
1660      deletable_response_ids_.end(),
1661      response_ids.begin(), response_ids.end());
1662  if (!is_response_deletion_scheduled_)
1663    ScheduleDeleteOneResponse();
1664}
1665
1666void AppCacheStorageImpl::ScheduleDeleteOneResponse() {
1667  DCHECK(!is_response_deletion_scheduled_);
1668  const base::TimeDelta kDelay = base::TimeDelta::FromMilliseconds(10);
1669  base::MessageLoop::current()->PostDelayedTask(
1670      FROM_HERE,
1671      base::Bind(&AppCacheStorageImpl::DeleteOneResponse,
1672                 weak_factory_.GetWeakPtr()),
1673      kDelay);
1674  is_response_deletion_scheduled_ = true;
1675}
1676
1677void AppCacheStorageImpl::DeleteOneResponse() {
1678  DCHECK(is_response_deletion_scheduled_);
1679  DCHECK(!deletable_response_ids_.empty());
1680
1681  if (!disk_cache()) {
1682    DCHECK(is_disabled_);
1683    deletable_response_ids_.clear();
1684    deleted_response_ids_.clear();
1685    is_response_deletion_scheduled_ = false;
1686    return;
1687  }
1688
1689  // TODO(michaeln): add group_id to DoomEntry args
1690  int64 id = deletable_response_ids_.front();
1691  int rv = disk_cache_->DoomEntry(
1692      id, base::Bind(&AppCacheStorageImpl::OnDeletedOneResponse,
1693                     base::Unretained(this)));
1694  if (rv != net::ERR_IO_PENDING)
1695    OnDeletedOneResponse(rv);
1696}
1697
1698void AppCacheStorageImpl::OnDeletedOneResponse(int rv) {
1699  is_response_deletion_scheduled_ = false;
1700  if (is_disabled_)
1701    return;
1702
1703  int64 id = deletable_response_ids_.front();
1704  deletable_response_ids_.pop_front();
1705  if (rv != net::ERR_ABORTED)
1706    deleted_response_ids_.push_back(id);
1707
1708  const size_t kBatchSize = 50U;
1709  if (deleted_response_ids_.size() >= kBatchSize ||
1710      deletable_response_ids_.empty()) {
1711    scoped_refptr<DeleteDeletableResponseIdsTask> task(
1712        new DeleteDeletableResponseIdsTask(this));
1713    task->response_ids_.swap(deleted_response_ids_);
1714    task->Schedule();
1715  }
1716
1717  if (deletable_response_ids_.empty()) {
1718    scoped_refptr<GetDeletableResponseIdsTask> task(
1719        new GetDeletableResponseIdsTask(this, last_deletable_response_rowid_));
1720    task->Schedule();
1721    return;
1722  }
1723
1724  ScheduleDeleteOneResponse();
1725}
1726
1727AppCacheStorageImpl::CacheLoadTask*
1728AppCacheStorageImpl::GetPendingCacheLoadTask(int64 cache_id) {
1729  PendingCacheLoads::iterator found = pending_cache_loads_.find(cache_id);
1730  if (found != pending_cache_loads_.end())
1731    return found->second;
1732  return NULL;
1733}
1734
1735AppCacheStorageImpl::GroupLoadTask*
1736AppCacheStorageImpl::GetPendingGroupLoadTask(const GURL& manifest_url) {
1737  PendingGroupLoads::iterator found = pending_group_loads_.find(manifest_url);
1738  if (found != pending_group_loads_.end())
1739    return found->second;
1740  return NULL;
1741}
1742
1743void AppCacheStorageImpl::GetPendingForeignMarkingsForCache(
1744    int64 cache_id, std::vector<GURL>* urls) {
1745  PendingForeignMarkings::iterator iter = pending_foreign_markings_.begin();
1746  while (iter != pending_foreign_markings_.end()) {
1747    if (iter->second == cache_id)
1748      urls->push_back(iter->first);
1749    ++iter;
1750  }
1751}
1752
1753void AppCacheStorageImpl::ScheduleSimpleTask(const base::Closure& task) {
1754  pending_simple_tasks_.push_back(task);
1755  base::MessageLoop::current()->PostTask(
1756      FROM_HERE,
1757      base::Bind(&AppCacheStorageImpl::RunOnePendingSimpleTask,
1758                 weak_factory_.GetWeakPtr()));
1759}
1760
1761void AppCacheStorageImpl::RunOnePendingSimpleTask() {
1762  DCHECK(!pending_simple_tasks_.empty());
1763  base::Closure task = pending_simple_tasks_.front();
1764  pending_simple_tasks_.pop_front();
1765  task.Run();
1766}
1767
1768AppCacheDiskCache* AppCacheStorageImpl::disk_cache() {
1769  DCHECK(IsInitTaskComplete());
1770
1771  if (is_disabled_)
1772    return NULL;
1773
1774  if (!disk_cache_) {
1775    int rv = net::OK;
1776    disk_cache_.reset(new AppCacheDiskCache);
1777    if (is_incognito_) {
1778      rv = disk_cache_->InitWithMemBackend(
1779          kMaxMemDiskCacheSize,
1780          base::Bind(&AppCacheStorageImpl::OnDiskCacheInitialized,
1781                     base::Unretained(this)));
1782    } else {
1783      rv = disk_cache_->InitWithDiskBackend(
1784          cache_directory_.Append(kDiskCacheDirectoryName),
1785          kMaxDiskCacheSize,
1786          false,
1787          cache_thread_.get(),
1788          base::Bind(&AppCacheStorageImpl::OnDiskCacheInitialized,
1789                     base::Unretained(this)));
1790    }
1791
1792    // We should not keep this reference around.
1793    cache_thread_ = NULL;
1794
1795    if (rv != net::ERR_IO_PENDING)
1796      OnDiskCacheInitialized(rv);
1797  }
1798  return disk_cache_.get();
1799}
1800
1801void AppCacheStorageImpl::OnDiskCacheInitialized(int rv) {
1802  if (rv != net::OK) {
1803    LOG(ERROR) << "Failed to open the appcache diskcache.";
1804    AppCacheHistograms::CountInitResult(AppCacheHistograms::DISK_CACHE_ERROR);
1805
1806    // We're unable to open the disk cache, this is a fatal error that we can't
1807    // really recover from. We handle it by disabling the appcache for this
1808    // browser session and deleting the directory on disk. The next browser
1809    // session should start with a clean slate.
1810    Disable();
1811    if (!is_incognito_) {
1812      VLOG(1) << "Deleting existing appcache data and starting over.";
1813      db_thread_->PostTask(
1814          FROM_HERE, base::Bind(base::IgnoreResult(&file_util::Delete),
1815                                cache_directory_, true));
1816    }
1817  }
1818}
1819
1820}  // namespace appcache
1821