1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import os
7import sys
8import unittest
9
10from caching_file_system import CachingFileSystem
11from extensions_paths import SERVER2
12from file_system import FileNotFoundError, StatInfo
13from local_file_system import LocalFileSystem
14from mock_file_system import MockFileSystem
15from object_store_creator import ObjectStoreCreator
16from test_file_system import TestFileSystem
17from test_object_store import TestObjectStore
18
19
20def _CreateLocalFs():
21  return LocalFileSystem.Create(SERVER2, 'test_data', 'file_system/')
22
23
24class CachingFileSystemTest(unittest.TestCase):
25  def setUp(self):
26    # Use this to make sure that every time _CreateCachingFileSystem is called
27    # the underlying object store data is the same, within each test.
28    self._object_store_dbs = {}
29
30  def _CreateCachingFileSystem(self, fs, start_empty=False):
31    def store_type_constructor(namespace, start_empty=False):
32      '''Returns an ObjectStore backed onto test-lifetime-persistent objects
33      in |_object_store_dbs|.
34      '''
35      if namespace not in self._object_store_dbs:
36        self._object_store_dbs[namespace] = {}
37      db = self._object_store_dbs[namespace]
38      if start_empty:
39        db.clear()
40      return TestObjectStore(namespace, init=db)
41    object_store_creator = ObjectStoreCreator(start_empty=start_empty,
42                                              store_type=store_type_constructor)
43    return CachingFileSystem(fs, object_store_creator)
44
45  def testReadFiles(self):
46    file_system = self._CreateCachingFileSystem(
47        _CreateLocalFs(), start_empty=False)
48    expected = {
49      './test1.txt': 'test1\n',
50      './test2.txt': 'test2\n',
51      './test3.txt': 'test3\n',
52    }
53    self.assertEqual(
54        expected,
55        file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
56
57  def testListDir(self):
58    file_system = self._CreateCachingFileSystem(
59        _CreateLocalFs(), start_empty=False)
60    expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
61    file_system._read_cache.Set(
62        'list/',
63        (expected, file_system.Stat('list/').version))
64    self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
65
66    expected.remove('file0.html')
67    file_system._read_cache.Set(
68        'list/',
69        (expected, file_system.Stat('list/').version))
70    self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
71
72  def testCaching(self):
73    test_fs = TestFileSystem({
74      'bob': {
75        'bob0': 'bob/bob0 contents',
76        'bob1': 'bob/bob1 contents',
77        'bob2': 'bob/bob2 contents',
78        'bob3': 'bob/bob3 contents',
79      }
80    })
81    mock_fs = MockFileSystem(test_fs)
82    def create_empty_caching_fs():
83      return self._CreateCachingFileSystem(mock_fs, start_empty=True)
84
85    file_system = create_empty_caching_fs()
86
87    # The stat/read should happen before resolving the Future, and resolving
88    # the future shouldn't do any additional work.
89    get_future = file_system.ReadSingle('bob/bob0')
90    self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
91    self.assertEqual('bob/bob0 contents', get_future.Get())
92    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
93
94    # Resource has been cached, so test resource is not re-fetched.
95    self.assertEqual('bob/bob0 contents',
96                     file_system.ReadSingle('bob/bob0').Get())
97    self.assertTrue(*mock_fs.CheckAndReset())
98
99    # Test if the Stat version is the same the resource is not re-fetched.
100    file_system = create_empty_caching_fs()
101    self.assertEqual('bob/bob0 contents',
102                     file_system.ReadSingle('bob/bob0').Get())
103    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
104
105    # Test if there is a newer version, the resource is re-fetched.
106    file_system = create_empty_caching_fs()
107    test_fs.IncrementStat();
108    future = file_system.ReadSingle('bob/bob0')
109    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
110    self.assertEqual('bob/bob0 contents', future.Get())
111    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
112
113    # Test directory and subdirectory stats are cached.
114    file_system = create_empty_caching_fs()
115    file_system._stat_cache.Del('bob/bob0')
116    file_system._read_cache.Del('bob/bob0')
117    file_system._stat_cache.Del('bob/bob1')
118    test_fs.IncrementStat();
119    futures = (file_system.ReadSingle('bob/bob1'),
120               file_system.ReadSingle('bob/bob0'))
121    self.assertTrue(*mock_fs.CheckAndReset(read_count=2))
122    self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'),
123                     tuple(future.Get() for future in futures))
124    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2, stat_count=1))
125    self.assertEqual('bob/bob1 contents',
126                     file_system.ReadSingle('bob/bob1').Get())
127    self.assertTrue(*mock_fs.CheckAndReset())
128
129    # Test a more recent parent directory doesn't force a refetch of children.
130    file_system = create_empty_caching_fs()
131    file_system._read_cache.Del('bob/bob0')
132    file_system._read_cache.Del('bob/bob1')
133    futures = (file_system.ReadSingle('bob/bob1'),
134               file_system.ReadSingle('bob/bob2'),
135               file_system.ReadSingle('bob/bob3'))
136    self.assertTrue(*mock_fs.CheckAndReset(read_count=3))
137    self.assertEqual(
138        ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'),
139        tuple(future.Get() for future in futures))
140    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3, stat_count=1))
141
142    test_fs.IncrementStat(path='bob/bob0')
143    file_system = create_empty_caching_fs()
144    self.assertEqual('bob/bob1 contents',
145                     file_system.ReadSingle('bob/bob1').Get())
146    self.assertEqual('bob/bob2 contents',
147                     file_system.ReadSingle('bob/bob2').Get())
148    self.assertEqual('bob/bob3 contents',
149                     file_system.ReadSingle('bob/bob3').Get())
150    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
151
152    file_system = create_empty_caching_fs()
153    file_system._stat_cache.Del('bob/bob0')
154    future = file_system.ReadSingle('bob/bob0')
155    self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
156    self.assertEqual('bob/bob0 contents', future.Get())
157    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
158    self.assertEqual('bob/bob0 contents',
159                     file_system.ReadSingle('bob/bob0').Get())
160    self.assertTrue(*mock_fs.CheckAndReset())
161
162    # Test skip_not_found caching behavior.
163    file_system = create_empty_caching_fs()
164    future = file_system.ReadSingle('bob/no_file', skip_not_found=True)
165    self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
166    self.assertEqual(None, future.Get())
167    self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
168    future = file_system.ReadSingle('bob/no_file', skip_not_found=True)
169    # There shouldn't be another read/stat from the file system;
170    # we know the file is not there.
171    self.assertTrue(*mock_fs.CheckAndReset())
172    future = file_system.ReadSingle('bob/no_file')
173    self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
174    # Even though we cached information about non-existent files,
175    # trying to read one without specifiying skip_not_found should
176    # still raise an error.
177    self.assertRaises(FileNotFoundError, future.Get)
178
179  def testCachedStat(self):
180    test_fs = TestFileSystem({
181      'bob': {
182        'bob0': 'bob/bob0 contents',
183        'bob1': 'bob/bob1 contents'
184      }
185    })
186    mock_fs = MockFileSystem(test_fs)
187
188    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False)
189
190    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
191    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
192    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
193    self.assertTrue(*mock_fs.CheckAndReset())
194
195    # Caching happens on a directory basis, so reading other files from that
196    # directory won't result in a stat.
197    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
198    self.assertEqual(
199        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
200        file_system.Stat('bob/'))
201    self.assertTrue(*mock_fs.CheckAndReset())
202
203    # Even though the stat is bumped, the object store still has it cached so
204    # this won't update.
205    test_fs.IncrementStat()
206    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
207    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
208    self.assertEqual(
209        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
210        file_system.Stat('bob/'))
211    self.assertTrue(*mock_fs.CheckAndReset())
212
213  def testFreshStat(self):
214    test_fs = TestFileSystem({
215      'bob': {
216        'bob0': 'bob/bob0 contents',
217        'bob1': 'bob/bob1 contents'
218      }
219    })
220    mock_fs = MockFileSystem(test_fs)
221
222    def run_expecting_stat(stat):
223      def run():
224        file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
225        self.assertEqual(
226            StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}),
227            file_system.Stat('bob/'))
228        self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
229        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
230        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
231        self.assertTrue(*mock_fs.CheckAndReset())
232      run()
233      run()
234
235    run_expecting_stat('0')
236    test_fs.IncrementStat()
237    run_expecting_stat('1')
238
239  def testSkipNotFound(self):
240    caching_fs = self._CreateCachingFileSystem(TestFileSystem({
241      'bob': {
242        'bob0': 'bob/bob0 contents',
243        'bob1': 'bob/bob1 contents'
244      }
245    }))
246    def read_skip_not_found(paths):
247      return caching_fs.Read(paths, skip_not_found=True).Get()
248    self.assertEqual({}, read_skip_not_found(('grub',)))
249    self.assertEqual({}, read_skip_not_found(('bob/bob2',)))
250    self.assertEqual({
251      'bob/bob0': 'bob/bob0 contents',
252    }, read_skip_not_found(('bob/bob0', 'bob/bob2')))
253
254  def testWalkCaching(self):
255    test_fs = TestFileSystem({
256      'root': {
257        'file1': 'file1',
258        'file2': 'file2',
259        'dir1': {
260          'dir1_file1': 'dir1_file1',
261          'dir2': {},
262          'dir3': {
263            'dir3_file1': 'dir3_file1',
264            'dir3_file2': 'dir3_file2'
265          }
266        }
267      }
268    })
269    mock_fs = MockFileSystem(test_fs)
270    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
271    for walkinfo in file_system.Walk(''):
272      pass
273    self.assertTrue(*mock_fs.CheckAndReset(
274        read_resolve_count=5, read_count=5, stat_count=5))
275
276    all_dirs, all_files = [], []
277    for root, dirs, files in file_system.Walk(''):
278      all_dirs.extend(dirs)
279      all_files.extend(files)
280    self.assertEqual(sorted(['root/', 'dir1/', 'dir2/', 'dir3/']),
281                     sorted(all_dirs))
282    self.assertEqual(
283        sorted(['file1', 'file2', 'dir1_file1', 'dir3_file1', 'dir3_file2']),
284        sorted(all_files))
285    # All data should be cached.
286    self.assertTrue(*mock_fs.CheckAndReset())
287
288    # Starting from a different root should still pull cached data.
289    for walkinfo in file_system.Walk('root/dir1/'):
290      pass
291    self.assertTrue(*mock_fs.CheckAndReset())
292    # TODO(ahernandez): Test with a new instance CachingFileSystem so a
293    # different object store is utilized.
294
295if __name__ == '__main__':
296  unittest.main()
297