12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved. 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from file_system import FileSystem, FileNotFoundError, StatInfo 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from future import Future 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)from path_util import AssertIsValid, AssertIsDirectory, IsDirectory 82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 10f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def MoveTo(base, obj): 114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '''Returns an object as |obj| moved to |base|. That is, 12f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) MoveTo('foo/bar', {'a': 'b'}) -> {'foo': {'bar': {'a': 'b'}}} 134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ''' 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) AssertIsDirectory(base) 154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result = {} 164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) leaf = result 175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for k in base.rstrip('/').split('/'): 184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) leaf[k] = {} 194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) leaf = leaf[k] 204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) leaf.update(obj) 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return result 224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 24f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def MoveAllTo(base, obj): 25f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '''Moves every value in |obj| to |base|. See MoveTo. 26f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) ''' 27f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) result = {} 28f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) for key, value in obj.iteritems(): 29f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) result[key] = MoveTo(base, value) 30f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return result 31f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def _List(file_system): 345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Returns a list of '/' separated paths derived from |file_system|. 355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) For example, {'index.html': '', 'www': {'file.txt': ''}} would return 365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ['index.html', 'www/file.txt']. 375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) assert isinstance(file_system, dict) 395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) result = {} 405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def update_result(item, path): 415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) AssertIsValid(path) 425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if isinstance(item, dict): 435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path != '': 445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) path += '/' 455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) result[path] = [p if isinstance(content, basestring) else (p + '/') 465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for p, content in item.iteritems()] 475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for subpath, subitem in item.iteritems(): 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) update_result(subitem, path + subpath) 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif isinstance(item, basestring): 505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) result[path] = item 515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) raise ValueError('Unsupported item type: %s' % type(item)) 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) update_result(file_system, '') 545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return result 555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)class _StatTracker(object): 585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '''Maintains the versions of paths in a file system. The versions of files 595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) are changed either by |Increment| or |SetVersion|. The versions of 605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) directories are derived from the versions of files within it. 615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ''' 625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def __init__(self): 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._path_stats = {} 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._global_stat = 0 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def Increment(self, path=None, by=1): 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path is None: 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._global_stat += by 705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self.SetVersion(path, self._path_stats.get(path, 0) + by) 725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def SetVersion(self, path, new_version): 745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if IsDirectory(path): 755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) raise ValueError('Only files have an incrementable stat, ' 765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'but "%s" is a directory' % path) 775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Update version of that file. 795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._path_stats[path] = new_version 805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Update all parent directory versions as well. 825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) slash_index = 0 # (deliberately including '' in the dir paths) 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) while slash_index != -1: 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dir_path = path[:slash_index] + '/' 855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._path_stats[dir_path] = max(self._path_stats.get(dir_path, 0), 865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) new_version) 875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if dir_path == '/': 885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Legacy support for '/' being the root of the file system rather 895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # than ''. Eventually when the path normalisation logic is complete 905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # this will be impossible and this logic will change slightly. 915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._path_stats[''] = self._path_stats['/'] 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) slash_index = path.find('/', slash_index + 1) 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def GetVersion(self, path): 955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return self._global_stat + self._path_stats.get(path, 0) 965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class TestFileSystem(FileSystem): 992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '''A FileSystem backed by an object. Create with an object representing file 1002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) paths such that {'a': {'b': 'hello'}} will resolve Read('a/b') as 'hello', 1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Read('a/') as ['b'], and Stat determined by a value incremented via 1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) IncrementStat. 1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ''' 104c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) def __init__(self, obj, relative_to=None, identity=None): 106a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) assert obj is not None 1075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if relative_to is not None: 1085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) obj = MoveTo(relative_to, obj) 1094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) self._identity = identity or type(self).__name__ 1105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._path_values = _List(obj) 1115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._stat_tracker = _StatTracker() 112c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 113c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 114c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # FileSystem implementation. 115c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 117e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch def Read(self, paths, skip_not_found=False): 1185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for path in paths: 1195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path not in self._path_values: 120e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch if skip_not_found: continue 1215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return FileNotFoundError.RaiseInFuture( 1225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '%s not in %s' % (path, '\n'.join(self._path_values))) 1235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return Future(value=dict((k, v) for k, v in self._path_values.iteritems() 1245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if k in paths)) 1252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 126f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) def Refresh(self): 127f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) return Future(value=()) 128f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 1292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def Stat(self, path): 1305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) read_result = self.ReadSingle(path).Get() 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) stat_result = StatInfo(str(self._stat_tracker.GetVersion(path))) 1322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if isinstance(read_result, list): 133c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) stat_result.child_versions = dict( 1345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) (file_result, 1355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) str(self._stat_tracker.GetVersion('%s%s' % (path, file_result)))) 136c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) for file_result in read_result) 1372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return stat_result 1382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 139c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 140c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # Testing methods. 141c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 142c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 1431e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) def IncrementStat(self, path=None, by=1): 1445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._stat_tracker.Increment(path, by=by) 145c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 146b2df76ea8fec9e32f6f3718986dba0d95315b29cTorne (Richard Coles) def GetIdentity(self): 1474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return self._identity 148