1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import os
7import sys
8import unittest
9
10from caching_file_system import CachingFileSystem
11from file_system import FileSystem, StatInfo
12from future import Future
13from local_file_system import LocalFileSystem
14from mock_file_system import MockFileSystem
15from object_store_creator import ObjectStoreCreator
16from test_file_system import TestFileSystem
17from test_object_store import TestObjectStore
18
19def _CreateLocalFs():
20  return LocalFileSystem(
21      os.path.join(sys.path[0], 'test_data', 'file_system'))
22
23class CachingFileSystemTest(unittest.TestCase):
24  def setUp(self):
25    # Use this to make sure that every time _CreateCachingFileSystem is called
26    # the underlying object store data is the same, within each test.
27    self._object_store_dbs = {}
28
29  def _CreateCachingFileSystem(self, fs, start_empty=False):
30    def store_type_constructor(namespace, start_empty=False):
31      '''Returns an ObjectStore backed onto test-lifetime-persistent objects
32      in |_object_store_dbs|.
33      '''
34      if namespace not in self._object_store_dbs:
35        self._object_store_dbs[namespace] = {}
36      db = self._object_store_dbs[namespace]
37      if start_empty:
38        db.clear()
39      return TestObjectStore(namespace, init=db)
40    object_store_creator = ObjectStoreCreator(start_empty=start_empty,
41                                              store_type=store_type_constructor)
42    return CachingFileSystem(fs, object_store_creator)
43
44  def testReadFiles(self):
45    file_system = self._CreateCachingFileSystem(
46        _CreateLocalFs(), start_empty=False)
47    expected = {
48      './test1.txt': 'test1\n',
49      './test2.txt': 'test2\n',
50      './test3.txt': 'test3\n',
51    }
52    self.assertEqual(
53        expected,
54        file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
55
56  def testListDir(self):
57    file_system = self._CreateCachingFileSystem(
58        _CreateLocalFs(), start_empty=False)
59    expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
60    file_system._read_object_store.Set(
61        'list/',
62        (expected, file_system.Stat('list/').version))
63    self.assertEqual(expected, sorted(file_system.ReadSingle('list/')))
64
65    expected.remove('file0.html')
66    file_system._read_object_store.Set(
67        'list/',
68        (expected, file_system.Stat('list/').version))
69    self.assertEqual(expected, sorted(file_system.ReadSingle('list/')))
70
71  def testCaching(self):
72    test_fs = TestFileSystem({
73      'bob': {
74        'bob0': 'bob/bob0 contents',
75        'bob1': 'bob/bob1 contents',
76        'bob2': 'bob/bob2 contents',
77        'bob3': 'bob/bob3 contents',
78      }
79    })
80    mock_fs = MockFileSystem(test_fs)
81    def create_empty_caching_fs():
82      return self._CreateCachingFileSystem(mock_fs, start_empty=True)
83
84    file_system = create_empty_caching_fs()
85    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
86    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
87
88    # Resource has been cached, so test resource is not re-fetched.
89    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
90    self.assertTrue(*mock_fs.CheckAndReset())
91
92    # Test if the Stat version is the same the resource is not re-fetched.
93    file_system = create_empty_caching_fs()
94    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
95    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
96
97    # Test if there is a newer version, the resource is re-fetched.
98    file_system = create_empty_caching_fs()
99    test_fs.IncrementStat();
100    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
101    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
102
103    # Test directory and subdirectory stats are cached.
104    file_system = create_empty_caching_fs()
105    file_system._stat_object_store.Del('bob/bob0')
106    file_system._read_object_store.Del('bob/bob0')
107    file_system._stat_object_store.Del('bob/bob1')
108    test_fs.IncrementStat();
109    self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
110    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
111    self.assertTrue(*mock_fs.CheckAndReset(read_count=2, stat_count=1))
112    self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
113    self.assertTrue(*mock_fs.CheckAndReset())
114
115    # Test a more recent parent directory doesn't force a refetch of children.
116    file_system = create_empty_caching_fs()
117    file_system._read_object_store.Del('bob/bob0')
118    file_system._read_object_store.Del('bob/bob1')
119    self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
120    self.assertEqual('bob/bob2 contents', file_system.ReadSingle('bob/bob2'))
121    self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3'))
122    self.assertTrue(*mock_fs.CheckAndReset(read_count=3, stat_count=1))
123
124    test_fs.IncrementStat(path='bob/')
125    file_system = create_empty_caching_fs()
126    self.assertEqual('bob/bob1 contents', file_system.ReadSingle('bob/bob1'))
127    self.assertEqual('bob/bob2 contents', file_system.ReadSingle('bob/bob2'))
128    self.assertEqual('bob/bob3 contents', file_system.ReadSingle('bob/bob3'))
129    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
130
131    file_system = create_empty_caching_fs()
132    file_system._stat_object_store.Del('bob/bob0')
133    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
134    self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
135    self.assertEqual('bob/bob0 contents', file_system.ReadSingle('bob/bob0'))
136    self.assertTrue(*mock_fs.CheckAndReset())
137
138  def testCachedStat(self):
139    test_fs = TestFileSystem({
140      'bob': {
141        'bob0': 'bob/bob0 contents',
142        'bob1': 'bob/bob1 contents'
143      }
144    })
145    mock_fs = MockFileSystem(test_fs)
146
147    file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False)
148
149    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
150    self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
151    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
152    self.assertTrue(*mock_fs.CheckAndReset())
153
154    # Caching happens on a directory basis, so reading other files from that
155    # directory won't result in a stat.
156    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
157    self.assertEqual(
158        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
159        file_system.Stat('bob/'))
160    self.assertTrue(*mock_fs.CheckAndReset())
161
162    # Even though the stat is bumped, the object store still has it cached so
163    # this won't update.
164    test_fs.IncrementStat()
165    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
166    self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
167    self.assertEqual(
168        StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
169        file_system.Stat('bob/'))
170    self.assertTrue(*mock_fs.CheckAndReset())
171
172  def testFreshStat(self):
173    test_fs = TestFileSystem({
174      'bob': {
175        'bob0': 'bob/bob0 contents',
176        'bob1': 'bob/bob1 contents'
177      }
178    })
179    mock_fs = MockFileSystem(test_fs)
180
181    def run_expecting_stat(stat):
182      def run():
183        file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
184        self.assertEqual(
185            StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}),
186            file_system.Stat('bob/'))
187        self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
188        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
189        self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
190        self.assertTrue(*mock_fs.CheckAndReset())
191      run()
192      run()
193
194    run_expecting_stat('0')
195    test_fs.IncrementStat()
196    run_expecting_stat('1')
197
198if __name__ == '__main__':
199  unittest.main()
200