bucket.py revision 7dbb3d5cf0c15f500944d211057644d6a2f37371
1f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org# Copyright 2013 The Chromium Authors. All rights reserved. 2f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org# Use of this source code is governed by a BSD-style license that can be 3f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org# found in the LICENSE file. 4f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 5f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgimport logging 6f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgimport os 7f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 8f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgfrom lib.symbol import FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS, TYPEINFO_SYMBOLS 9f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 10f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 11f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgLOGGER = logging.getLogger('dmprof') 12f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 13f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org# Indexes in dumped heap profile dumps. 14f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgVIRTUAL, COMMITTED, ALLOC_COUNT, FREE_COUNT, _, BUCKET_ID = range(6) 15f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 16f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 17f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgclass Bucket(object): 18f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """Represents a bucket, which is a unit of memory block classification.""" 19f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 20f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def __init__(self, stacktrace, allocator_type, typeinfo, typeinfo_name): 21f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._stacktrace = stacktrace 22f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._allocator_type = allocator_type 23f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._typeinfo = typeinfo 24f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._typeinfo_name = typeinfo_name 25f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 26f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stackfunction = stacktrace 27f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_joined_stackfunction = '' 28f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stacksourcefile = stacktrace 29f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_joined_stacksourcefile = '' 30f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_typeinfo = typeinfo_name 31f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 32f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self.component_cache = '' 33f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 34f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def __str__(self): 35f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result = [] 36f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result.append(self._allocator_type) 37f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if self._symbolized_typeinfo == 'no typeinfo': 38f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result.append('tno_typeinfo') 39f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org else: 40f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result.append('t' + self._symbolized_typeinfo) 41f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result.append('n' + self._typeinfo_name) 42f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org result.extend(['%s(@%s)' % (function, sourcefile) 43f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for function, sourcefile 44f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org in zip(self._symbolized_stackfunction, 45f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stacksourcefile)]) 46f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return ' '.join(result) 47f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 48f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolize(self, symbol_mapping_cache): 49f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """Makes a symbolized stacktrace and typeinfo with |symbol_mapping_cache|. 50f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 51f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org Args: 52f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org symbol_mapping_cache: A SymbolMappingCache object. 53f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """ 54f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org # TODO(dmikurube): Fill explicitly with numbers if symbol not found. 55f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stackfunction = [ 56f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org symbol_mapping_cache.lookup(FUNCTION_SYMBOLS, address) 57f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for address in self._stacktrace] 58f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_joined_stackfunction = ' '.join( 59f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stackfunction) 60f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stacksourcefile = [ 61f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org symbol_mapping_cache.lookup(SOURCEFILE_SYMBOLS, address) 62f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for address in self._stacktrace] 63f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_joined_stacksourcefile = ' '.join( 64f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_stacksourcefile) 65f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if not self._typeinfo: 66f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_typeinfo = 'no typeinfo' 67f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org else: 68f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_typeinfo = symbol_mapping_cache.lookup( 69f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org TYPEINFO_SYMBOLS, self._typeinfo) 70f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if not self._symbolized_typeinfo: 71f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._symbolized_typeinfo = 'no typeinfo' 72f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 73f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def clear_component_cache(self): 74f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self.component_cache = '' 75f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 76f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 77f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def stacktrace(self): 78f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._stacktrace 79f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 80f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 81f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def allocator_type(self): 82f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._allocator_type 83f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 84f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 85f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def typeinfo(self): 86f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._typeinfo 87f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 88f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 89f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def typeinfo_name(self): 90f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._typeinfo_name 91f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 92f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 93f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolized_stackfunction(self): 94f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._symbolized_stackfunction 95f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 96f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 97f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolized_joined_stackfunction(self): 98f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._symbolized_joined_stackfunction 99f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 100f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 101f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolized_stacksourcefile(self): 102f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._symbolized_stacksourcefile 103f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 104f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 105f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolized_joined_stacksourcefile(self): 106f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._symbolized_joined_stacksourcefile 107f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 108f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org @property 109f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolized_typeinfo(self): 110f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._symbolized_typeinfo 111f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 112f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 113f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.orgclass BucketSet(object): 114f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """Represents a set of bucket.""" 115f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def __init__(self): 116f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._buckets = {} 117f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._code_addresses = set() 118f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._typeinfo_addresses = set() 119f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 120f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def load(self, prefix): 121f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """Loads all related bucket files. 122f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 123f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org Args: 124f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org prefix: A prefix string for bucket file names. 125f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org """ 126f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org LOGGER.info('Loading bucket files.') 127f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 128f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org n = 0 129f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org skipped = 0 130f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org while True: 131f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org path = '%s.%04d.buckets' % (prefix, n) 132f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if not os.path.exists(path) or not os.stat(path).st_size: 133f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if skipped > 10: 134f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org break 135f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org n += 1 136f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org skipped += 1 137f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org continue 138f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org LOGGER.info(' %s' % path) 139f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org with open(path, 'r') as f: 140f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._load_file(f) 141f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org n += 1 142f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org skipped = 0 143f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 144f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def _load_file(self, bucket_f): 145f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for line in bucket_f: 146f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org words = line.split() 147f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org typeinfo = None 148f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org typeinfo_name = '' 149f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org stacktrace_begin = 2 150f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for index, word in enumerate(words): 151f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if index < 2: 152f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org continue 153f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if word[0] == 't': 154f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org typeinfo = int(word[1:], 16) 155f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._typeinfo_addresses.add(typeinfo) 156f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org elif word[0] == 'n': 157f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org typeinfo_name = word[1:] 158f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org else: 159f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org stacktrace_begin = index 160f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org break 161f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] 162f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for frame in stacktrace: 163f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._code_addresses.add(frame) 164f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org self._buckets[int(words[0])] = Bucket( 165f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org stacktrace, words[1], typeinfo, typeinfo_name) 166f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 167f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def __iter__(self): 168f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for bucket_id, bucket_content in self._buckets.iteritems(): 169f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org yield bucket_id, bucket_content 170f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 171f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def __getitem__(self, bucket_id): 172f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._buckets[bucket_id] 173f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 174f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def get(self, bucket_id): 175f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org return self._buckets.get(bucket_id) 176f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 177f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def symbolize(self, symbol_mapping_cache): 178f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for bucket_content in self._buckets.itervalues(): 179f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org bucket_content.symbolize(symbol_mapping_cache) 180f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 181f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def clear_component_cache(self): 182f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for bucket_content in self._buckets.itervalues(): 183f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org bucket_content.clear_component_cache() 184f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org 185f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org def iter_addresses(self, symbol_type): 186f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org if symbol_type in [FUNCTION_SYMBOLS, SOURCEFILE_SYMBOLS]: 187f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for function in self._code_addresses: 188f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org yield function 189f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org else: 190f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org for function in self._typeinfo_addresses: 191f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org yield function 192f2304cf60bf51f05cc720689e8b89958f25ca5dakbr@chromium.org