caching_file_system_test.py revision 5d1f7b1de12d16ceb2c938c56701a3e8bfa558f7 
1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import os
7import sys
8import unittest
9
10from caching_file_system import CachingFileSystem
11from extensions_paths import SERVER2
12from file_system import  StatInfo
13from local_file_system import LocalFileSystem
14from mock_file_system import MockFileSystem
15from object_store_creator import ObjectStoreCreator
16from test_file_system import TestFileSystem
17from test_object_store import TestObjectStore
18
19
20def _CreateLocalFs():
21  return LocalFileSystem.Create(SERVER2, 'test_data', 'file_system/')
22
23
24class CachingFileSystemTest(unittest.TestCase):
25  def setUp(self):
26    # Use this to make sure that every time _CreateCachingFileSystem is called
27    # the underlying object store data is the same, within each test.
28    self._object_store_dbs = {}
29
30  def _CreateCachingFileSystem(self, fs, start_empty=False):
31    def store_type_constructor(namespace, start_empty=False):
32      '''Returns an ObjectStore backed onto test-lifetime-persistent objects
33      in |_object_store_dbs|.
34      '''
35      if namespace not in self._object_store_dbs:
36        self._object_store_dbs[namespace] = {}
37      db = self._object_store_dbs[namespace]
38      if start_empty:
39        db.clear()
40      return TestObjectStore(namespace, init=db)
41    object_store_creator = ObjectStoreCreator(start_empty=start_empty,
42                                              store_type=store_type_constructor)
43    return CachingFileSystem(fs, object_store_creator)
44
45  def testReadFiles(self):
46    file_system = self._CreateCachingFileSystem(
47        _CreateLocalFs(), start_empty=False)
48    expected = {
49      './test1.txt': 'test1\n',
50      './test2.txt': 'test2\n',
51      './test3.txt': 'test3\n',
52    }
53    self.assertEqual(
54        expected,
55        file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
56
57  def testListDir(self):
58    file_system = self._CreateCachingFileSystem(
59        _CreateLocalFs(), start_empty=False)
60    expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
61    file_system._read_object_store.Set(
62        'list/',
63        (expected, file_system.Stat('list/').version))
64    self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
65
66    expected.remove('file0.html')
67    file_system._read_object_store.Set(
68        'list/',
69        (expected, file_system.Stat('list/').version))
70    self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
71
72  def testCaching(self):
73    test_fs = TestFileSystem({
74      'bob': {
75        'bob0': 'bob/bob0 contents',
76        'bob1': 'bob/bob1 contents',
77        'bob2': 'bob/bob2 contents',
78        'bob3': 'bob/bob3 contents',
79      }
80    })
81    mock_fs = MockFileSystem(test_fs)
82    def create_empty_caching_fs():
83      return self._CreateCachingFileSystem(mock_fs, start_empty=True)
84
85    file_system = create_empty_caching_fs()
86
87    # The stat/read should happen before resolving the Future, and resolving
88    # the future shouldn't do any additional work.
89    get_future = file_system.ReadSingle('bob/bob0')
90    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
91    self.assertEqual('bob/bob0 contents', get_future.Get())
92    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
93
94    # Resource has been cached, so test resource is not re-fetched.
95    self.assertEqual('bob/bob0 contents',
96                     file_system.ReadSingle('bob/bob0').Get())
97    self.assertTrue(*mock_fs.CheckAndReset())
98
99    # Test if the Stat version is the same the resource is not re-fetched.
100    file_system = create_empty_caching_fs()
101    self.assertEqual('bob/bob0 contents',
102                     file_system.ReadSingle('bob/bob0').Get())
103    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
104
105    # Test if there is a newer version, the resource is re-fetched.
106    file_system = create_empty_caching_fs()
107    test_fs.IncrementStat();
108    future = file_system.ReadSingle('bob/bob0')
109    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
110    self.assertEqual('bob/bob0 contents', future.Get())
111    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
112
113    # Test directory and subdirectory stats are cached.
114    file_system = create_empty_caching_fs()
115    file_system._stat_object_store.Del('bob/bob0')
116    file_system._read_object_store.Del('bob/bob0')
117    file_system._stat_object_store.Del('bob/bob1')
118    test_fs.IncrementStat();
119    futures = (file_system.ReadSingle('bob/bob1'),
120               file_system.ReadSingle('bob/bob0'))
121    self.assertTrue(*mock_fs.CheckAndReset(read_count=2, stat_count=1))
122    self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'),
123                     tuple(future.Get() for future in futures))
124    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2))
125    self.assertEqual('bob/bob1 contents',
126                     file_system.ReadSingle('bob/bob1').Get())
127    self.assertTrue(*mock_fs.CheckAndReset())
128
129    # Test a more recent parent directory doesn't force a refetch of children.
130    file_system = create_empty_caching_fs()
131    file_system._read_object_store.Del('bob/bob0')
132    file_system._read_object_store.Del('bob/bob1')
133    futures = (file_system.ReadSingle('bob/bob1'),
134               file_system.ReadSingle('bob/bob2'),
135               file_system.ReadSingle('bob/bob3'))
136    self.assertTrue(*mock_fs.CheckAndReset(read_count=3, stat_count=1))
137    self.assertEqual(
138        ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'),
139        tuple(future.Get() for future in futures))
140    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3))
141
142    test_fs.IncrementStat(path='bob/bob0')
143    file_system = create_empty_caching_fs()
144    self.assertEqual('bob/bob1 contents',
145                     file_system.ReadSingle('bob/bob1').Get())
146    self.assertEqual('bob/bob2 contents',
147                     file_system.ReadSingle('bob/bob2').Get())
148    self.assertEqual('bob/bob3 contents',
149                     file_system.ReadSingle('bob/bob3').Get())
150    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
151
152    file_system = create_empty_caching_fs()
153    file_system._stat_object_store.Del('bob/bob0')
154    future = file_system.ReadSingle('bob/bob0')
155    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
156    self.assertEqual('bob/bob0 contents', future.Get())
157    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
158    self.assertEqual('bob/bob0 contents',
159                     file_system.ReadSingle('bob/bob0').Get())
160    self.assertTrue(*mock_fs.CheckAndReset())
161
162  def testCachedStat(self):
163    test_fs = TestFileSystem({
164      'bob': {
165        'bob0': 'bob/bob0 contents',
166        'bob1': 'bob/bob1 contents'
167      }
168    })
169    mock_fs = MockFileSystem(test_fs)
170
171    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False)
172
173    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
174    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
175    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
176    self.assertTrue(*mock_fs.CheckAndReset())
177
178    # Caching happens on a directory basis, so reading other files from that
179    # directory won't result in a stat.
180    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
181    self.assertEqual(
182        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
183        file_system.Stat('bob/'))
184    self.assertTrue(*mock_fs.CheckAndReset())
185
186    # Even though the stat is bumped, the object store still has it cached so
187    # this won't update.
188    test_fs.IncrementStat()
189    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
190    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
191    self.assertEqual(
192        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
193        file_system.Stat('bob/'))
194    self.assertTrue(*mock_fs.CheckAndReset())
195
196  def testFreshStat(self):
197    test_fs = TestFileSystem({
198      'bob': {
199        'bob0': 'bob/bob0 contents',
200        'bob1': 'bob/bob1 contents'
201      }
202    })
203    mock_fs = MockFileSystem(test_fs)
204
205    def run_expecting_stat(stat):
206      def run():
207        file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
208        self.assertEqual(
209            StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}),
210            file_system.Stat('bob/'))
211        self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
212        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
213        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
214        self.assertTrue(*mock_fs.CheckAndReset())
215      run()
216      run()
217
218    run_expecting_stat('0')
219    test_fs.IncrementStat()
220    run_expecting_stat('1')
221
222if __name__ == '__main__':
223  unittest.main()
224