1c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)#!/usr/bin/env python 2c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 4c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)# found in the LICENSE file. 5c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 6c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import os 7c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import sys 8c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import unittest 9c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 10c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)from caching_file_system import CachingFileSystem 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from extensions_paths import SERVER2 121320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccifrom file_system import FileNotFoundError, StatInfo 13c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)from local_file_system import LocalFileSystem 14b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles)from mock_file_system import MockFileSystem 15c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)from object_store_creator import ObjectStoreCreator 16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)from test_file_system import TestFileSystem 17c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)from test_object_store import TestObjectStore 18c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 20c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)def _CreateLocalFs(): 215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return LocalFileSystem.Create(SERVER2, 'test_data', 'file_system/') 225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 23c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 24c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)class CachingFileSystemTest(unittest.TestCase): 25c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def setUp(self): 26c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Use this to make sure that every time _CreateCachingFileSystem is called 27c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # the underlying object store data is the same, within each test. 28c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self._object_store_dbs = {} 29c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 30b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def _CreateCachingFileSystem(self, fs, start_empty=False): 31b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def store_type_constructor(namespace, start_empty=False): 32b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) '''Returns an ObjectStore backed onto test-lifetime-persistent objects 33b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) in |_object_store_dbs|. 34b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) ''' 35c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) if namespace not in self._object_store_dbs: 36c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self._object_store_dbs[namespace] = {} 37c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) db = self._object_store_dbs[namespace] 38c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) if start_empty: 39c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) db.clear() 40c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) return TestObjectStore(namespace, init=db) 41ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch object_store_creator = ObjectStoreCreator(start_empty=start_empty, 42b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) store_type=store_type_constructor) 43b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return CachingFileSystem(fs, object_store_creator) 44c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 45c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testReadFiles(self): 46b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = self._CreateCachingFileSystem( 47b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) _CreateLocalFs(), start_empty=False) 48c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) expected = { 49c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) './test1.txt': 'test1\n', 50c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) './test2.txt': 'test2\n', 51c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) './test3.txt': 'test3\n', 52c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) } 53c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual( 54c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) expected, 55c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get()) 56c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 57c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testListDir(self): 58b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = self._CreateCachingFileSystem( 59b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) _CreateLocalFs(), start_empty=False) 60c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) expected = ['dir/'] + ['file%d.html' % i for i in range(7)] 611320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._read_cache.Set( 62c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'list/', 63c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) (expected, file_system.Stat('list/').version)) 644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get())) 65c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 66c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) expected.remove('file0.html') 671320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._read_cache.Set( 68c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'list/', 69c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) (expected, file_system.Stat('list/').version)) 704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get())) 71c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 72c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testCaching(self): 73b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs = TestFileSystem({ 74c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob': { 75c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob0': 'bob/bob0 contents', 76c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob1': 'bob/bob1 contents', 77c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob2': 'bob/bob2 contents', 78c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob3': 'bob/bob3 contents', 79c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) } 80c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) }) 81b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) mock_fs = MockFileSystem(test_fs) 82b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def create_empty_caching_fs(): 83b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) return self._CreateCachingFileSystem(mock_fs, start_empty=True) 84c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 85b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 864e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 874e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # The stat/read should happen before resolving the Future, and resolving 884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # the future shouldn't do any additional work. 894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) get_future = file_system.ReadSingle('bob/bob0') 90e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 914e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', get_future.Get()) 92e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1)) 93c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 94c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Resource has been cached, so test resource is not re-fetched. 954e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', 964e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob0').Get()) 97b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 98c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 99c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Test if the Stat version is the same the resource is not re-fetched. 100b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 1014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', 1024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob0').Get()) 103b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 104c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 105c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Test if there is a newer version, the resource is re-fetched. 106b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 107b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs.IncrementStat(); 1084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) future = file_system.ReadSingle('bob/bob0') 109b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1)) 1104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', future.Get()) 1114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1)) 112c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 113c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Test directory and subdirectory stats are cached. 114b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 1151320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._stat_cache.Del('bob/bob0') 1161320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._read_cache.Del('bob/bob0') 1171320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._stat_cache.Del('bob/bob1') 118b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs.IncrementStat(); 1194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) futures = (file_system.ReadSingle('bob/bob1'), 1204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob0')) 121e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_count=2)) 1224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'), 1234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tuple(future.Get() for future in futures)) 124e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2, stat_count=1)) 1254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob1 contents', 1264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob1').Get()) 127b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 128c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 129c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Test a more recent parent directory doesn't force a refetch of children. 130b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 1311320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._read_cache.Del('bob/bob0') 1321320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._read_cache.Del('bob/bob1') 1334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) futures = (file_system.ReadSingle('bob/bob1'), 1344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob2'), 1354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob3')) 136e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_count=3)) 1374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual( 1384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'), 1394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tuple(future.Get() for future in futures)) 140e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3, stat_count=1)) 141c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_fs.IncrementStat(path='bob/bob0') 143b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 1444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob1 contents', 1454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob1').Get()) 1464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob2 contents', 1474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob2').Get()) 1484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob3 contents', 1494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob3').Get()) 150b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 151c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 152b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = create_empty_caching_fs() 1531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system._stat_cache.Del('bob/bob0') 1544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) future = file_system.ReadSingle('bob/bob0') 155e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 1564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', future.Get()) 157e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1)) 1584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self.assertEqual('bob/bob0 contents', 1594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) file_system.ReadSingle('bob/bob0').Get()) 160b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 161c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1621320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # Test skip_not_found caching behavior. 1631320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system = create_empty_caching_fs() 1641320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci future = file_system.ReadSingle('bob/no_file', skip_not_found=True) 1651320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 1661320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual(None, future.Get()) 1671320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1)) 1681320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci future = file_system.ReadSingle('bob/no_file', skip_not_found=True) 1691320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # There shouldn't be another read/stat from the file system; 1701320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # we know the file is not there. 1711320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset()) 1721320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci future = file_system.ReadSingle('bob/no_file') 1731320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 1741320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # Even though we cached information about non-existent files, 1751320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # trying to read one without specifiying skip_not_found should 1761320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # still raise an error. 1771320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertRaises(FileNotFoundError, future.Get) 1781320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 179c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testCachedStat(self): 180b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs = TestFileSystem({ 181c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob': { 182c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob0': 'bob/bob0 contents', 183c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob1': 'bob/bob1 contents' 184c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) } 185c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) }) 186b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) mock_fs = MockFileSystem(test_fs) 187b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) 188b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False) 189c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 190c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 191b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 192c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 193b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 194c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 195c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Caching happens on a directory basis, so reading other files from that 196c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # directory won't result in a stat. 197c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1')) 198c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual( 199c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}), 200c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) file_system.Stat('bob/')) 201b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 202c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 203c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Even though the stat is bumped, the object store still has it cached so 204c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # this won't update. 205b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs.IncrementStat() 206c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 207c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1')) 208c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual( 209c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}), 210c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) file_system.Stat('bob/')) 211b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 212c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 213c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def testFreshStat(self): 214b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs = TestFileSystem({ 215c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob': { 216c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob0': 'bob/bob0 contents', 217c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'bob1': 'bob/bob1 contents' 218c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) } 219c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) }) 220b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) mock_fs = MockFileSystem(test_fs) 221c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 222c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def run_expecting_stat(stat): 223c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def run(): 224b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True) 225c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual( 226c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}), 227c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) file_system.Stat('bob/')) 228b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 229c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0')) 230c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0')) 231b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) self.assertTrue(*mock_fs.CheckAndReset()) 232c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) run() 233c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) run() 234c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 235c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) run_expecting_stat('0') 236b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) test_fs.IncrementStat() 237c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) run_expecting_stat('1') 238c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 2391320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci def testSkipNotFound(self): 2401320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci caching_fs = self._CreateCachingFileSystem(TestFileSystem({ 2411320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'bob': { 2421320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'bob0': 'bob/bob0 contents', 2431320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'bob1': 'bob/bob1 contents' 2441320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 2451320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci })) 2461320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci def read_skip_not_found(paths): 2471320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci return caching_fs.Read(paths, skip_not_found=True).Get() 2481320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual({}, read_skip_not_found(('grub',))) 2491320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual({}, read_skip_not_found(('bob/bob2',))) 2501320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual({ 2511320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'bob/bob0': 'bob/bob0 contents', 2521320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci }, read_skip_not_found(('bob/bob0', 'bob/bob2'))) 2531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 2541320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci def testWalkCaching(self): 2551320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci test_fs = TestFileSystem({ 2561320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'root': { 2571320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'file1': 'file1', 2581320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'file2': 'file2', 2591320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir1': { 2601320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir1_file1': 'dir1_file1', 2611320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir2': {}, 2621320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir3': { 2631320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir3_file1': 'dir3_file1', 2641320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'dir3_file2': 'dir3_file2' 2651320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 266e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 2671320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 2681320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci }) 2691320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci mock_fs = MockFileSystem(test_fs) 2701320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True) 2711320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci for walkinfo in file_system.Walk(''): 2721320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci pass 2731320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset( 2741320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci read_resolve_count=5, read_count=5, stat_count=5)) 275e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 2761320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci all_dirs, all_files = [], [] 2771320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci for root, dirs, files in file_system.Walk(''): 2781320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci all_dirs.extend(dirs) 2791320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci all_files.extend(files) 2801320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual(sorted(['root/', 'dir1/', 'dir2/', 'dir3/']), 2811320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci sorted(all_dirs)) 2821320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertEqual( 2831320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci sorted(['file1', 'file2', 'dir1_file1', 'dir3_file1', 'dir3_file2']), 2841320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci sorted(all_files)) 2851320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # All data should be cached. 2861320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset()) 2871320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 2881320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # Starting from a different root should still pull cached data. 2891320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci for walkinfo in file_system.Walk('root/dir1/'): 2901320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci pass 2911320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci self.assertTrue(*mock_fs.CheckAndReset()) 2921320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # TODO(ahernandez): Test with a new instance CachingFileSystem so a 2931320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci # different object store is utilized. 294e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 295c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)if __name__ == '__main__': 296c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) unittest.main() 297