1c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 3c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// found in the LICENSE file. 4c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 5c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ 6c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#define NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ 7c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 8c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include <string> 9eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch#include <vector> 10c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 11c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/basictypes.h" 127d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)#include "base/containers/hash_tables.h" 13c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/files/file_path.h" 14c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/gtest_prod_util.h" 15c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/logging.h" 16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/memory/scoped_ptr.h" 17c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/pickle.h" 18c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "base/port.h" 1958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "net/base/cache_type.h" 20c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "net/base/net_export.h" 21c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#include "net/disk_cache/simple/simple_index.h" 22c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 237d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)namespace base { 247d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)class SingleThreadTaskRunner; 257d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)class TaskRunner; 267d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles)} 277d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) 28c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)namespace disk_cache { 29c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 30c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)const uint64 kSimpleIndexMagicNumber = GG_UINT64_C(0x656e74657220796f); 31c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 32558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdochstruct NET_EXPORT_PRIVATE SimpleIndexLoadResult { 33558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult(); 34558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch ~SimpleIndexLoadResult(); 35558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch void Reset(); 36558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch 37558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch bool did_load; 38558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndex::EntrySet entries; 39558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch bool flush_required; 40558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch}; 41558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch 42c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// Simple Index File format is a pickle serialized data of IndexMetadata and 434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// EntryMetadata objects. The file format is as follows: one instance of 44c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// serialized |IndexMetadata| followed serialized |EntryMetadata| entries 454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// repeated |number_of_entries| amount of times. To know more about the format, 46c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// see SimpleIndexFile::Serialize() and SeeSimpleIndexFile::LoadFromDisk() 47c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)// methods. 48868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)// 494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// The non-static methods must run on the IO thread. All the real 50868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)// work is done in the static methods, which are run on the cache thread 514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)// or in worker threads. Synchronization between methods is the 52868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)// responsibility of the caller. 53c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)class NET_EXPORT_PRIVATE SimpleIndexFile { 54c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) public: 55c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) class NET_EXPORT_PRIVATE IndexMetadata { 56c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) public: 57c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) IndexMetadata(); 58c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) IndexMetadata(uint64 number_of_entries, uint64 cache_size); 59c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 60c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) void Serialize(Pickle* pickle) const; 61c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) bool Deserialize(PickleIterator* it); 62c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 63c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) bool CheckIndexMetadata(); 64c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 65c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint64 GetNumberOfEntries() { return number_of_entries_; } 66c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 67c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) private: 68c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) FRIEND_TEST_ALL_PREFIXES(IndexMetadataTest, Basics); 69c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) FRIEND_TEST_ALL_PREFIXES(IndexMetadataTest, Serialize); 70c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 71c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint64 magic_number_; 72c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint32 version_; 73c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint64 number_of_entries_; 74c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint64 cache_size_; // Total cache storage size in bytes. 75c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) }; 76c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 7703b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) SimpleIndexFile( 7803b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) const scoped_refptr<base::SingleThreadTaskRunner>& cache_thread, 7903b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) const scoped_refptr<base::TaskRunner>& worker_pool, 8003b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) net::CacheType cache_type, 8103b57e008b61dfcb1fbad3aea950ae0e001748b0Torne (Richard Coles) const base::FilePath& cache_directory); 82868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) virtual ~SimpleIndexFile(); 83868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 84868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) // Get index entries based on current disk context. 85558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch virtual void LoadIndexEntries(base::Time cache_last_modified, 86558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch const base::Closure& callback, 87558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult* out_result); 88868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 89868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) // Write the specified set of entries to disk. 90868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) virtual void WriteToDisk(const SimpleIndex::EntrySet& entry_set, 91868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) uint64 cache_size, 92868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) const base::TimeTicks& start, 93868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) bool app_on_background); 94868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 95868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) private: 96eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch friend class WrappedSimpleIndexFile; 97868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 983551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Used for cache directory traversal. 993551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) typedef base::Callback<void (const base::FilePath&)> EntryFileCallback; 1003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 101ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch // When loading the entries from disk, add this many extra hash buckets to 102ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch // prevent reallocation on the IO thread when merging in new live entries. 103ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch static const int kExtraSizeForMerge = 512; 104ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 1057dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch // Synchronous (IO performing) implementation of LoadIndexEntries. 10658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) static void SyncLoadIndexEntries(net::CacheType cache_type, 10758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) base::Time cache_last_modified, 108558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch const base::FilePath& cache_directory, 109558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch const base::FilePath& index_file_path, 110558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult* out_result); 111868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 112d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // Load the index file from disk returning an EntrySet. 113558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch static void SyncLoadFromDisk(const base::FilePath& index_filename, 114d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) base::Time* out_last_cache_seen_by_index, 115558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult* out_result); 116c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 117c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) // Returns a scoped_ptr for a newly allocated Pickle containing the serialized 118d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // data to be written to a file. Note: the pickle is not in a consistent state 119d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // immediately after calling this menthod, one needs to call 120d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // SerializeFinalData to make it ready to write to a file. 121c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) static scoped_ptr<Pickle> Serialize( 122c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) const SimpleIndexFile::IndexMetadata& index_metadata, 123c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) const SimpleIndex::EntrySet& entries); 124c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 125d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // Appends cache modification time data to the serialized format. This is 126d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // performed on a thread accessing the disk. It is not combined with the main 127d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // serialization path to avoid extra thread hops or copying the pickle to the 128d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // worker thread. 129d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static bool SerializeFinalData(base::Time cache_modified, Pickle* pickle); 130d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 1317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch // Given the contents of an index file |data| of length |data_len|, returns 1327dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch // the corresponding EntrySet. Returns NULL on error. 133558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch static void Deserialize(const char* data, int data_len, 134d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) base::Time* out_cache_last_modified, 135558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult* out_result); 136c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Implemented either in simple_index_file_posix.cc or 1383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // simple_index_file_win.cc. base::FileEnumerator turned out to be very 1393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // expensive in terms of memory usage therefore it's used only on non-POSIX 1403551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // environments for convenience (for now). Returns whether the traversal 1413551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // succeeded. 1423551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) static bool TraverseCacheDirectory( 1433551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const base::FilePath& cache_path, 1443551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const EntryFileCallback& entry_file_callback); 1453551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 146d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // Writes the index file to disk atomically. 147d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static void SyncWriteToDisk(net::CacheType cache_type, 148d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const base::FilePath& cache_directory, 149d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const base::FilePath& index_filename, 150d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const base::FilePath& temp_index_filename, 151d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) scoped_ptr<Pickle> pickle, 152d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const base::TimeTicks& start_time, 153d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) bool app_on_background); 154d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 1557dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch // Scan the index directory for entries, returning an EntrySet of all entries 1567dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch // found. 157558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch static void SyncRestoreFromDisk(const base::FilePath& cache_directory, 158558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch const base::FilePath& index_file_path, 159558790d6acca3451cf3a6b497803a5f07d0bec58Ben Murdoch SimpleIndexLoadResult* out_result); 160868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 161bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch // Determines if an index file is stale relative to the time of last 162d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // modification of the cache directory. Obsolete, used only for a histogram to 163d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // compare with the new method. 164d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) // TODO(pasko): remove this method after getting enough data. 165d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static bool LegacyIsIndexFileStale(base::Time cache_last_modified, 166d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const base::FilePath& index_file_path); 1677dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch 168c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) struct PickleHeader : public Pickle::Header { 169c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) uint32 crc; 170c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) }; 171c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1727d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) const scoped_refptr<base::SingleThreadTaskRunner> cache_thread_; 1737d4cd473f85ac64c3747c96c277f9e506a0d2246Torne (Richard Coles) const scoped_refptr<base::TaskRunner> worker_pool_; 17458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const net::CacheType cache_type_; 175bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch const base::FilePath cache_directory_; 176bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch const base::FilePath index_file_; 177bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch const base::FilePath temp_index_file_; 178868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 179d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static const char kIndexDirectory[]; 180d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static const char kIndexFileName[]; 181d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) static const char kTempIndexFileName[]; 182d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 183c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) DISALLOW_COPY_AND_ASSIGN(SimpleIndexFile); 184c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)}; 185c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 186c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 187c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)} // namespace disk_cache 188c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 189c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#endif // NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ 190