1a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 2a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 3a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# found in the LICENSE file. 4a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 5a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)"""This module handles file-backed storage of the core classes. 6a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 7a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)The storage is logically organized as follows: 8a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)Storage -> N Archives -> 1 Symbol index 9a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) N Snapshots -> 1 Mmaps dump. 10a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) -> 0/1 Native heap dump. 11a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 12a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)Where an "archive" is essentially a collection of snapshots taken for a given 13a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)app at a given point in time. 14a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)""" 15a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 16a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)import datetime 17a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)import json 18a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)import os 19a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 20a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from memory_inspector.core import memory_map 21a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from memory_inspector.core import native_heap 22a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from memory_inspector.core import symbol 23a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from memory_inspector.data import serialization 24a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 25a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 26a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)class Storage(object): 27a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 28a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _SETTINGS_FILE = 'settings-%s.json' 29a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 30a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def __init__(self, root_path): 31a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Creates a file-backed storage. Files will be placed in |root_path|.""" 32a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._root = root_path 33a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if not os.path.exists(self._root): 34a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) os.makedirs(self._root) 35a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 36a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def LoadSettings(self, name): 37a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Loads a key-value dict from the /settings-name.json file. 38a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 39a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) This is for backend and device settings (e.g., symbols path, adb path).""" 40a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._root, Storage._SETTINGS_FILE % name) 41a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if not os.path.exists(file_path): 42a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return {} 43a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path) as f: 44a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return json.load(f) 45a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 46a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def StoreSettings(self, name, settings): 47a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Stores a key-value dict into /settings-name.json file.""" 48a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(isinstance(settings, dict)) 49a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._root, Storage._SETTINGS_FILE % name) 50a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if not settings: 51a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if os.path.exists(file_path): 52a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) os.unlink(file_path) 53a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return 54a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path, 'w') as f: 55a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return json.dump(settings, f) 56a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 57a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def ListArchives(self): 58a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Lists archives. Each of them is a sub-folder inside the |root_path|.""" 59a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return sorted( 60a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) [name for name in os.listdir(self._root) 61a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if os.path.isdir(os.path.join(self._root, name))]) 62a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 63a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def OpenArchive(self, archive_name, create=False): 64a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Returns an instance of |Archive|.""" 65a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) archive_path = os.path.join(self._root, archive_name) 66a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if not os.path.exists(archive_path) and create: 67a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) os.makedirs(archive_path) 68a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return Archive(archive_name, archive_path) 69a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 70a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def DeleteArchive(self, archive_name): 71a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Deletes the archive (removing its folder).""" 72a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) archive_path = os.path.join(self._root, archive_name) 73a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) for f in os.listdir(archive_path): 74a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) os.unlink(os.path.join(archive_path, f)) 75a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) os.rmdir(archive_path) 76a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 77a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 78a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)class Archive(object): 79a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """A collection of snapshots, each one holding one memory dump (per kind).""" 80a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 81a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _MMAP_EXT = '-mmap.json' 82a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _NHEAP_EXT = '-nheap.json' 83a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _SNAP_EXT = '.snapshot' 84a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _SYM_FILE = 'syms.json' 85a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) _TIME_FMT = '%Y-%m-%d_%H-%M-%S-%f' 86a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 87a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def __init__(self, name, path): 88a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(os.path.isdir(path)) 89a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._name = name 90a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._path = path 91a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._cur_snapshot = None 92a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 93a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def StoreSymbols(self, symbols): 94a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Stores the symbol db (one per the overall archive).""" 95a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(isinstance(symbols, symbol.Symbols)) 96a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, Archive._SYM_FILE) 97a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path, 'w') as f: 98a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) json.dump(symbols, f, cls=serialization.Encoder) 99a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 100a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def HasSymbols(self): 101a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return os.path.exists(os.path.join(self._path, Archive._SYM_FILE)) 102a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 103a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def LoadSymbols(self): 104a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(self.HasSymbols()) 105a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, Archive._SYM_FILE) 106a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path) as f: 107a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return json.load(f, cls=serialization.SymbolsDecoder) 108a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 109a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def StartNewSnapshot(self): 110a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Creates a 2014-01-01_02:03:04.snapshot marker (an empty file).""" 111a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._cur_snapshot = Archive._TimestampToStr(datetime.datetime.now()) 112a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, 113a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._cur_snapshot + Archive._SNAP_EXT) 114a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(not os.path.exists(file_path)) 115a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) open(file_path, 'w').close() 116a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return datetime.datetime.strptime(self._cur_snapshot, Archive._TIME_FMT) 117a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 118a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def ListSnapshots(self): 119a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) """Returns a list of timestamps (datetime.datetime instances).""" 120a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_names = sorted( 121a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) [name[:-(len(Archive._SNAP_EXT))] for name in os.listdir(self._path) 122a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if name.endswith(Archive._SNAP_EXT)]) 123a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) timestamps = [datetime.datetime.strptime(x, Archive._TIME_FMT) 124a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) for x in file_names] 125a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return timestamps 126a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 127a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def StoreMemMaps(self, mmaps): 128a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(isinstance(mmaps, memory_map.Map)) 129a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(self._cur_snapshot), 'Must call StartNewSnapshot first' 130a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, self._cur_snapshot + Archive._MMAP_EXT) 131a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path, 'w') as f: 132a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) json.dump(mmaps, f, cls=serialization.Encoder) 133a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 134a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def HasMemMaps(self, timestamp): 135a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return self._HasSnapshotFile(timestamp, Archive._MMAP_EXT) 136a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 137a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def LoadMemMaps(self, timestamp): 138a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(self.HasMemMaps(timestamp)) 139a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) snapshot_name = Archive._TimestampToStr(timestamp) 140a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, snapshot_name + Archive._MMAP_EXT) 141a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path) as f: 142a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return json.load(f, cls=serialization.MmapDecoder) 143a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 144a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def StoreNativeHeap(self, nheap): 145a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(isinstance(nheap, native_heap.NativeHeap)) 146a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(self._cur_snapshot), 'Must call StartNewSnapshot first' 147a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, 148a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) self._cur_snapshot + Archive._NHEAP_EXT) 149a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path, 'w') as f: 150a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) json.dump(nheap, f, cls=serialization.Encoder) 151a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 152a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def HasNativeHeap(self, timestamp): 153a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return self._HasSnapshotFile(timestamp, Archive._NHEAP_EXT) 154a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 155a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def LoadNativeHeap(self, timestamp): 156a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) assert(self.HasNativeHeap(timestamp)) 157a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) snapshot_name = Archive._TimestampToStr(timestamp) 158a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) file_path = os.path.join(self._path, snapshot_name + Archive._NHEAP_EXT) 159a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) with open(file_path) as f: 160a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return json.load(f, cls=serialization.NativeHeapDecoder) 161a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 162a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def _HasSnapshotFile(self, timestamp, ext): 163a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) name = Archive._TimestampToStr(timestamp) 164a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return os.path.exists(os.path.join(self._path, name + ext)) 165a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 166a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) @staticmethod 167a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def _TimestampToStr(timestamp): 168a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) return timestamp.strftime(Archive._TIME_FMT)