1cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)#!/usr/bin/env python 25d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 35d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 45d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)# found in the LICENSE file. 55d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 65d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)"""Generate a spatial analysis against an arbitrary library. 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 85d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)To use, build the 'binary_size_tool' target. Then run this tool, passing 95d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)in the location of the library to be analyzed along with any other options 105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)you desire. 115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)""" 125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import collections 145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import json 15cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import logging 16cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import multiprocessing 175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import optparse 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import os 195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import re 205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import shutil 215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import struct 225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import subprocess 235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import sys 245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import tempfile 25cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import time 26cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 27cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import binary_size_utils 28cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 29cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# This path changee is not beautiful. Temporary (I hope) measure until 30cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# the chromium project has figured out a proper way to organize the 31cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# library of python tools. http://crbug.com/375725 32cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)elf_symbolizer_path = os.path.abspath(os.path.join( 33cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) os.path.dirname(__file__), 34cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) '..', 35cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) '..', 36cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'build', 37cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'android', 38cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'pylib')) 39cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)sys.path.append(elf_symbolizer_path) 40cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)import symbols.elf_symbolizer as elf_symbolizer # pylint: disable=F0401 415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 43f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# Node dictionary keys. These are output in json read by the webapp so 44f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# keep them short to save file size. 45f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# Note: If these change, the webapp must also change. 46f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_TYPE_KEY = 'k' 47f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_NAME_KEY = 'n' 48f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_CHILDREN_KEY = 'children' 49f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_SYMBOL_TYPE_KEY = 't' 50f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_SYMBOL_SIZE_KEY = 'value' 51f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_MAX_DEPTH_KEY = 'maxDepth' 52f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NODE_LAST_PATH_ELEMENT_KEY = 'lastPathElement' 53f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 54f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# The display name of the bucket where we put symbols without path. 55f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)NAME_NO_PATH_BUCKET = '(No Path)' 56f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 57f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# Try to keep data buckets smaller than this to avoid killing the 58f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)# graphing lib. 59f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)BIG_BUCKET_LIMIT = 3000 60f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 61f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 620529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 63cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def FormatBytes(byte_count): 645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """Pretty-print a number of bytes.""" 65cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if byte_count > 1e6: 66cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) byte_count = byte_count / 1.0e6 67cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return '%.1fm' % byte_count 68cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if byte_count > 1e3: 69cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) byte_count = byte_count / 1.0e3 70cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return '%.1fk' % byte_count 71cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return str(byte_count) 725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 740529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 75cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def SymbolTypeToHuman(symbol_type): 765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """Convert a symbol type as printed by nm into a human-readable name.""" 775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return {'b': 'bss', 785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'd': 'data', 795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'r': 'read-only data', 805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 't': 'code', 815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'w': 'weak symbol', 82cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'v': 'weak symbol'}[symbol_type] 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 85cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def _MkChild(node, name): 86f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) child = node[NODE_CHILDREN_KEY].get(name) 87cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if child is None: 88f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) child = {NODE_NAME_KEY: name, 89f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) NODE_CHILDREN_KEY: {}} 90f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_CHILDREN_KEY][name] = child 91cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return child 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 94f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 95f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)def SplitNoPathBucket(node): 96f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) """NAME_NO_PATH_BUCKET can be too large for the graphing lib to 97f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) handle. Split it into sub-buckets in that case.""" 98f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) root_children = node[NODE_CHILDREN_KEY] 99f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if NAME_NO_PATH_BUCKET in root_children: 100f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) no_path_bucket = root_children[NAME_NO_PATH_BUCKET] 101f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) old_children = no_path_bucket[NODE_CHILDREN_KEY] 102f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) count = 0 103f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for symbol_type, symbol_bucket in old_children.iteritems(): 104f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) count += len(symbol_bucket[NODE_CHILDREN_KEY]) 105f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if count > BIG_BUCKET_LIMIT: 106f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) new_children = {} 107f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) no_path_bucket[NODE_CHILDREN_KEY] = new_children 108f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) current_bucket = None 109f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) index = 0 110f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for symbol_type, symbol_bucket in old_children.iteritems(): 111f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for symbol_name, value in symbol_bucket[NODE_CHILDREN_KEY].iteritems(): 112f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if index % BIG_BUCKET_LIMIT == 0: 113f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) group_no = (index / BIG_BUCKET_LIMIT) + 1 114f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) current_bucket = _MkChild(no_path_bucket, 115f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) '%s subgroup %d' % (NAME_NO_PATH_BUCKET, 116f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) group_no)) 117f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'p' 118f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_TYPE_KEY] = 'p' # p for path 119f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) index += 1 120f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) symbol_size = value[NODE_SYMBOL_SIZE_KEY] 121f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) AddSymbolIntoFileNode(current_bucket, symbol_type, 122f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) symbol_name, symbol_size) 123f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 124f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 125cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def MakeChildrenDictsIntoLists(node): 126cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) largest_list_len = 0 127f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if NODE_CHILDREN_KEY in node: 128f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) largest_list_len = len(node[NODE_CHILDREN_KEY]) 129cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) child_list = [] 130f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) for child in node[NODE_CHILDREN_KEY].itervalues(): 131cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) child_largest_list_len = MakeChildrenDictsIntoLists(child) 132cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if child_largest_list_len > largest_list_len: 133cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) largest_list_len = child_largest_list_len 134cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) child_list.append(child) 135f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_CHILDREN_KEY] = child_list 1365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 137cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return largest_list_len 1380529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 1390529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 140f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)def AddSymbolIntoFileNode(node, symbol_type, symbol_name, symbol_size): 141f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) """Puts symbol into the file path node |node|. 142f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) Returns the number of added levels in tree. I.e. returns 2.""" 143f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 144f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # 'node' is the file node and first step is to find its symbol-type bucket. 145f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_LAST_PATH_ELEMENT_KEY] = True 146f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node = _MkChild(node, symbol_type) 147f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'b' 148f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_SYMBOL_TYPE_KEY] = symbol_type 149f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_TYPE_KEY] = 'b' # b for bucket 150f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 151f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # 'node' is now the symbol-type bucket. Make the child entry. 152f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node = _MkChild(node, symbol_name) 153f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if NODE_CHILDREN_KEY in node: 154f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if node[NODE_CHILDREN_KEY]: 155f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) logging.warning('A container node used as symbol for %s.' % symbol_name) 156f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # This is going to be used as a leaf so no use for child list. 157f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) del node[NODE_CHILDREN_KEY] 158f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_SYMBOL_SIZE_KEY] = symbol_size 159f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_SYMBOL_TYPE_KEY] = symbol_type 160f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_TYPE_KEY] = 's' # s for symbol 161f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 162f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return 2 # Depth of the added subtree. 163f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 164f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 165116680a4aac90f2aa7413d9095a592090648e557Ben Murdochdef MakeCompactTree(symbols, symbol_path_origin_dir): 166f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) result = {NODE_NAME_KEY: '/', 167f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) NODE_CHILDREN_KEY: {}, 168f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) NODE_TYPE_KEY: 'p', 169f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) NODE_MAX_DEPTH_KEY: 0} 170cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) seen_symbol_with_path = False 171116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch cwd = os.path.abspath(os.getcwd()) 1720529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch for symbol_name, symbol_type, symbol_size, file_path in symbols: 1730529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 1740529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if 'vtable for ' in symbol_name: 175f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) symbol_type = '@' # hack to categorize these separately 1760529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch # Take path like '/foo/bar/baz', convert to ['foo', 'bar', 'baz'] 177116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if file_path and file_path != "??": 178116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch file_path = os.path.abspath(os.path.join(symbol_path_origin_dir, 179116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch file_path)) 180116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # Let the output structure be relative to $CWD if inside $CWD, 181116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # otherwise relative to the disk root. This is to avoid 182116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # unnecessary click-through levels in the output. 183116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if file_path.startswith(cwd + os.sep): 184116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch file_path = file_path[len(cwd):] 185116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if file_path.startswith('/'): 186116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch file_path = file_path[1:] 187cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) seen_symbol_with_path = True 1880529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch else: 189f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) file_path = NAME_NO_PATH_BUCKET 1900529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 1910529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch path_parts = file_path.split('/') 1920529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 1930529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch # Find pre-existing node in tree, or update if it already exists 1940529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch node = result 1950529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch depth = 0 1960529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch while len(path_parts) > 0: 1970529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch path_part = path_parts.pop(0) 1980529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if len(path_part) == 0: 1990529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch continue 2000529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch depth += 1 201cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) node = _MkChild(node, path_part) 202f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'p' 203f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) node[NODE_TYPE_KEY] = 'p' # p for path 204f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 205f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) depth += AddSymbolIntoFileNode(node, symbol_type, symbol_name, symbol_size) 206f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) result[NODE_MAX_DEPTH_KEY] = max(result[NODE_MAX_DEPTH_KEY], depth) 207cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 208cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if not seen_symbol_with_path: 209cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) logging.warning('Symbols lack paths. Data will not be structured.') 2100529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 211f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # The (no path) bucket can be extremely large if we failed to get 212f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) # path information. Split it into subgroups if needed. 213f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) SplitNoPathBucket(result) 214f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 215cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) largest_list_len = MakeChildrenDictsIntoLists(result) 216cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 217f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if largest_list_len > BIG_BUCKET_LIMIT: 218cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) logging.warning('There are sections with %d nodes. ' 219cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'Results might be unusable.' % largest_list_len) 2200529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch return result 2210529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 2220529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 2230529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 2245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def TreeifySymbols(symbols): 2255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """Convert symbols into a path-based tree, calculating size information 2265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) along the way. 2275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) The result is a dictionary that contains two kinds of nodes: 2295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1. Leaf nodes, representing source code locations (e.g., c++ files) 2305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) These nodes have the following dictionary entries: 2315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sizes: a dictionary whose keys are categories (such as code, data, 2325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) vtable, etceteras) and whose values are the size, in bytes, of 2335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) those categories; 2345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) size: the total size, in bytes, of all the entries in the sizes dict 2355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2. Non-leaf nodes, representing directories 2365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) These nodes have the following dictionary entries: 2375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) children: a dictionary whose keys are names (path entries; either 2385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) directory or file names) and whose values are other nodes; 2395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) size: the total size, in bytes, of all the leaf nodes that are 2405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) contained within the children dict (recursively expanded) 2415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) The result object is itself a dictionary that represents the common ancestor 2435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) of all child nodes, e.g. a path to which all other nodes beneath it are 2445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) relative. The 'size' attribute of this dict yields the sum of the size of all 2455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) leaf nodes within the data structure. 2465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """ 2475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dirs = {'children': {}, 'size': 0} 248cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for sym, symbol_type, size, path in symbols: 2495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dirs['size'] += size 2505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path: 2515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) path = os.path.normpath(path) 2525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path.startswith('/'): 2535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) path = path[1:] 2545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parts = None 2565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path: 2575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parts = path.split('/') 2585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if parts: 2605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) assert path 2615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) file_key = parts.pop() 2625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree = dirs 2635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 2645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Traverse the tree to the parent of the file node, creating as needed 2655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for part in parts: 2665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) assert part != '' 2675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if part not in tree['children']: 2685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['children'][part] = {'children': {}, 'size': 0} 2695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree = tree['children'][part] 2705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['size'] += size 2715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Get (creating if necessary) the node for the file 2735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # This node doesn't have a 'children' attribute 2745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if file_key not in tree['children']: 2755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['children'][file_key] = {'sizes': collections.defaultdict(int), 2765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'size': 0} 2775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree = tree['children'][file_key] 2785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['size'] += size 2795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Accumulate size into a bucket within the file 281cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) symbol_type = symbol_type.lower() 2825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'vtable for ' in sym: 2835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[vtable]'] += size 284cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) elif 'r' == symbol_type: 2855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[rodata]'] += size 286cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) elif 'd' == symbol_type: 2875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[data]'] += size 288cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) elif 'b' == symbol_type: 2895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[bss]'] += size 290cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) elif 't' == symbol_type: 2915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # 'text' in binary parlance means 'code'. 2925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[code]'] += size 293cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) elif 'w' == symbol_type: 2945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[weak]'] += size 2955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 2965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes']['[other]'] += size 2975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) except: 298cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print >> sys.stderr, sym, parts, file_key 2995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) raise 3005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 3015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) key = 'symbols without paths' 3025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if key not in dirs['children']: 3035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dirs['children'][key] = {'sizes': collections.defaultdict(int), 3045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'size': 0} 3055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree = dirs['children'][key] 3065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) subkey = 'misc' 3075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (sym.endswith('::__FUNCTION__') or 3085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sym.endswith('::__PRETTY_FUNCTION__')): 3095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) subkey = '__FUNCTION__' 3105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif sym.startswith('CSWTCH.'): 3115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) subkey = 'CSWTCH' 3125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif '::' in sym: 3135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) subkey = sym[0:sym.find('::') + 2] 3145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['sizes'][subkey] = tree['sizes'].get(subkey, 0) + size 3155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tree['size'] += size 3165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return dirs 3175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3190529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 3205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def JsonifyTree(tree, name): 3215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """Convert TreeifySymbols output to a JSON treemap. 3225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) The format is very similar, with the notable exceptions being 3245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) lists of children instead of maps and some different attribute names.""" 3255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) children = [] 3265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) css_class_map = { 3275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[vtable]': 'vtable', 3285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[rodata]': 'read-only_data', 3295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[data]': 'data', 3305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[bss]': 'bss', 3315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[code]': 'code', 3325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '[weak]': 'weak_symbol' 3335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) } 3345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'children' in tree: 3355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Non-leaf node. Recurse. 3365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for child_name, child in tree['children'].iteritems(): 3375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) children.append(JsonifyTree(child, child_name)) 3385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 3395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Leaf node; dump per-file stats as entries in the treemap 3405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for kind, size in tree['sizes'].iteritems(): 3415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) child_json = {'name': kind + ' (' + FormatBytes(size) + ')', 3425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'data': { '$area': size }} 3435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) css_class = css_class_map.get(kind) 344cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if css_class is not None: 345cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) child_json['data']['$symbol'] = css_class 3465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) children.append(child_json) 3475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Sort children by size, largest to smallest. 3485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) children.sort(key=lambda child: -child['data']['$area']) 3495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # For leaf nodes, the 'size' attribute is the size of the leaf; 3515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Non-leaf nodes don't really have a size, but their 'size' attribute is 3525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # the sum of the sizes of all their children. 3535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return {'name': name + ' (' + FormatBytes(tree['size']) + ')', 3545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'data': { '$area': tree['size'] }, 3555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'children': children } 3565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 357116680a4aac90f2aa7413d9095a592090648e557Ben Murdochdef DumpCompactTree(symbols, symbol_path_origin_dir, outfile): 358116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch tree_root = MakeCompactTree(symbols, symbol_path_origin_dir) 359cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) with open(outfile, 'w') as out: 360116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch out.write('var tree_data=') 361116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # Use separators without whitespace to get a smaller file. 362116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch json.dump(tree_root, out, separators=(',', ':')) 363cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print('Writing %d bytes json' % os.path.getsize(outfile)) 3645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3650529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 3660529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 3675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def DumpTreemap(symbols, outfile): 3685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dirs = TreeifySymbols(symbols) 3695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out = open(outfile, 'w') 3705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 3715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('var kTree = ' + json.dumps(JsonifyTree(dirs, '/'))) 3725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 3735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.flush() 3745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.close() 3755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 3770529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 3785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def DumpLargestSymbols(symbols, outfile, n): 379cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # a list of (sym, symbol_type, size, path); sort by size. 3805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) symbols = sorted(symbols, key=lambda x: -x[2]) 3815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped = 0 3825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out = open(outfile, 'w') 3835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 3845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('var largestSymbols = [\n') 385cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for sym, symbol_type, size, path in symbols: 386cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if symbol_type in ('b', 'w'): 3875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) continue # skip bss and weak symbols 3885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path is None: 3895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) path = '' 3905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) entry = {'size': FormatBytes(size), 3915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'symbol': sym, 392cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'type': SymbolTypeToHuman(symbol_type), 3935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'location': path } 3945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(json.dumps(entry)) 3955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(',\n') 3965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped += 1 3975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if dumped >= n: 3985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return 3995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 4005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('];\n') 4015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.flush() 4025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.close() 4035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def MakeSourceMap(symbols): 4065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sources = {} 407cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for _sym, _symbol_type, size, path in symbols: 4085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) key = None 4095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if path: 4105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) key = os.path.normpath(path) 4115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 4125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) key = '[no path]' 4135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if key not in sources: 4145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sources[key] = {'path': path, 'symbol_count': 0, 'size': 0} 4155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) record = sources[key] 4165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) record['size'] += size 4175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) record['symbol_count'] += 1 4185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return sources 4195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4210529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 4225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def DumpLargestSources(symbols, outfile, n): 423cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) source_map = MakeSourceMap(symbols) 424cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) sources = sorted(source_map.values(), key=lambda x: -x['size']) 4255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped = 0 4265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out = open(outfile, 'w') 4275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 4285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('var largestSources = [\n') 4295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for record in sources: 4305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) entry = {'size': FormatBytes(record['size']), 4315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'symbol_count': str(record['symbol_count']), 4325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'location': record['path']} 4335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(json.dumps(entry)) 4345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(',\n') 4355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped += 1 4365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if dumped >= n: 4375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return 4385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 4395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('];\n') 4405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.flush() 4415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.close() 4425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4440529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch# TODO(andrewhayden): Only used for legacy reports. Delete. 4455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def DumpLargestVTables(symbols, outfile, n): 4465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) vtables = [] 447cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for symbol, _type, size, path in symbols: 4485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if 'vtable for ' in symbol: 4495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) vtables.append({'symbol': symbol, 'path': path, 'size': size}) 4505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) vtables = sorted(vtables, key=lambda x: -x['size']) 4515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped = 0 4525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out = open(outfile, 'w') 4535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) try: 4545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('var largestVTables = [\n') 4555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for record in vtables: 4565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) entry = {'size': FormatBytes(record['size']), 4575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'symbol': record['symbol'], 4585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'location': record['path']} 4595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(json.dumps(entry)) 4605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write(',\n') 4615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) dumped += 1 4625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if dumped >= n: 4635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return 4645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) finally: 4655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.write('];\n') 4665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.flush() 4675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) out.close() 4685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 4695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 470cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# Regex for parsing "nm" output. A sample line looks like this: 471cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# 0167b39c 00000018 t ACCESS_DESCRIPTION_free /path/file.c:95 472cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# 473cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# The fields are: address, size, type, name, source location 474cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# Regular expression explained ( see also: https://xkcd.com/208 ): 475cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# ([0-9a-f]{8,}+) The address 476cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# [\s]+ Whitespace separator 477cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# ([0-9a-f]{8,}+) The size. From here on out it's all optional. 478cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# [\s]+ Whitespace separator 479cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# (\S?) The symbol type, which is any non-whitespace char 480cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# [\s*] Whitespace separator 481cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# ([^\t]*) Symbol name, any non-tab character (spaces ok!) 482cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# [\t]? Tab separator 483cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)# (.*) The location (filename[:linennum|?][ (discriminator n)] 484cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)sNmPattern = re.compile( 485cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) r'([0-9a-f]{8,})[\s]+([0-9a-f]{8,})[\s]*(\S?)[\s*]([^\t]*)[\t]?(.*)') 486cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 487cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)class Progress(): 488cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def __init__(self): 489cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.count = 0 490cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.skip_count = 0 491cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.collisions = 0 492cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.time_last_output = time.time() 493cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) self.count_last_output = 0 494116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.disambiguations = 0 495116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch self.was_ambiguous = 0 496cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 497cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 498116680a4aac90f2aa7413d9095a592090648e557Ben Murdochdef RunElfSymbolizer(outfile, library, addr2line_binary, nm_binary, jobs, 499116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch disambiguate, src_path): 500cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_output = RunNm(library, nm_binary) 501cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_output_lines = nm_output.splitlines() 502cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_output_lines_len = len(nm_output_lines) 503cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) address_symbol = {} 504cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress = Progress() 505cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def map_address_symbol(symbol, addr): 506cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress.count += 1 507cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if addr in address_symbol: 508cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # 'Collision between %s and %s.' % (str(symbol.name), 509cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # str(address_symbol[addr].name)) 510cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress.collisions += 1 511cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) else: 512116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if symbol.disambiguated: 513116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch progress.disambiguations += 1 514116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if symbol.was_ambiguous: 515116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch progress.was_ambiguous += 1 516116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 517cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) address_symbol[addr] = symbol 518cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 519116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch progress_output() 520116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 521116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch def progress_output(): 522cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress_chunk = 100 523cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if progress.count % progress_chunk == 0: 524cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) time_now = time.time() 525cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) time_spent = time_now - progress.time_last_output 526cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if time_spent > 1.0: 527cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Only output at most once per second. 528cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress.time_last_output = time_now 529cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) chunk_size = progress.count - progress.count_last_output 530cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress.count_last_output = progress.count 531cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if time_spent > 0: 532cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) speed = chunk_size / time_spent 533cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) else: 534cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) speed = 0 535cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) progress_percent = (100.0 * (progress.count + progress.skip_count) / 536cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_output_lines_len) 537116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch disambiguation_percent = 0 538116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if progress.disambiguations != 0: 539116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch disambiguation_percent = (100.0 * progress.disambiguations / 540116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch progress.was_ambiguous) 541116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 542116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch sys.stdout.write('\r%.1f%%: Looked up %d symbols (%d collisions, ' 543116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch '%d disambiguations where %.1f%% succeeded)' 5441320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ' - %.1f lookups/s.' % 545116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch (progress_percent, progress.count, progress.collisions, 546116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch progress.disambiguations, disambiguation_percent, speed)) 547116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 548116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # In case disambiguation was disabled, we remove the source path (which upon 549116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # being set signals the symbolizer to enable disambiguation) 550116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if not disambiguate: 551116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch src_path = None 552cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) symbolizer = elf_symbolizer.ELFSymbolizer(library, addr2line_binary, 553cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) map_address_symbol, 554116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch max_concurrent_jobs=jobs, 555116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch source_root_path=src_path) 55646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) user_interrupted = False 55746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) try: 55846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) for line in nm_output_lines: 55946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) match = sNmPattern.match(line) 56046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if match: 56146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) location = match.group(5) 56246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if not location: 56346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) addr = int(match.group(1), 16) 56446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) size = int(match.group(2), 16) 56546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if addr in address_symbol: # Already looked up, shortcut 56646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # ELFSymbolizer. 56746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) map_address_symbol(address_symbol[addr], addr) 56846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) continue 56946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) elif size == 0: 57046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # Save time by not looking up empty symbols (do they even exist?) 57146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) print('Empty symbol: ' + line) 57246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) else: 57346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) symbolizer.SymbolizeAsync(addr, addr) 57446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) continue 57546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 57646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) progress.skip_count += 1 57746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) except KeyboardInterrupt: 57846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) user_interrupted = True 57946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) print('Interrupting - killing subprocesses. Please wait.') 580cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 58146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) try: 58246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) symbolizer.Join() 58346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) except KeyboardInterrupt: 58446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # Don't want to abort here since we will be finished in a few seconds. 58546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) user_interrupted = True 58646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) print('Patience you must have my young padawan.') 587cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 588116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch print '' 589116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 59046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if user_interrupted: 59146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) print('Skipping the rest of the file mapping. ' 59246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'Output will not be fully classified.') 593cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 594116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch symbol_path_origin_dir = os.path.dirname(os.path.abspath(library)) 595116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 596cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) with open(outfile, 'w') as out: 597cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for line in nm_output_lines: 598cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) match = sNmPattern.match(line) 599cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if match: 600cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) location = match.group(5) 601cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if not location: 602cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) addr = int(match.group(1), 16) 60346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) symbol = address_symbol.get(addr) 60446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if symbol is not None: 60546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) path = '??' 60646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if symbol.source_path is not None: 607116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch path = os.path.abspath(os.path.join(symbol_path_origin_dir, 608116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch symbol.source_path)) 60946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) line_number = 0 61046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if symbol.source_line is not None: 61146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) line_number = symbol.source_line 61246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) out.write('%s\t%s:%d\n' % (line, path, line_number)) 61346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) continue 614cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 615cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) out.write('%s\n' % line) 616cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 617cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print('%d symbols in the results.' % len(address_symbol)) 618cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 619cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 620cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def RunNm(binary, nm_binary): 62146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) cmd = [nm_binary, '-C', '--print-size', '--size-sort', '--reverse-sort', 62246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) binary] 623cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_process = subprocess.Popen(cmd, 624cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) stdout=subprocess.PIPE, 625cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) stderr=subprocess.PIPE) 626cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) (process_output, err_output) = nm_process.communicate() 627cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 628cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if nm_process.returncode != 0: 629cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if err_output: 630cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise Exception, err_output 6315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 632cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise Exception, process_output 633cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 634cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return process_output 635cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 636cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 637cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def GetNmSymbols(nm_infile, outfile, library, jobs, verbose, 638116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch addr2line_binary, nm_binary, disambiguate, src_path): 639cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if nm_infile is None: 640cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if outfile is None: 641cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) outfile = tempfile.NamedTemporaryFile(delete=False).name 6425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 6435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if verbose: 644cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print 'Running parallel addr2line, dumping symbols to ' + outfile 645116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch RunElfSymbolizer(outfile, library, addr2line_binary, nm_binary, jobs, 646116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch disambiguate, src_path) 647cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 648cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_infile = outfile 649cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 6505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif verbose: 651cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print 'Using nm input from ' + nm_infile 652cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) with file(nm_infile, 'r') as infile: 653cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return list(binary_size_utils.ParseNm(infile)) 654cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 655cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 6565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)PAK_RESOURCE_ID_TO_STRING = { "inited": False } 6575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def LoadPakIdsFromResourceFile(filename): 6595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Given a file name, it loads everything that looks like a resource id 6605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) into PAK_RESOURCE_ID_TO_STRING.""" 6615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) with open(filename) as resource_header: 6625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for line in resource_header: 6635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if line.startswith("#define "): 6645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) line_data = line.split() 6655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if len(line_data) == 3: 6665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) try: 6675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) resource_number = int(line_data[2]) 6685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) resource_name = line_data[1] 6695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) PAK_RESOURCE_ID_TO_STRING[resource_number] = resource_name 6705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) except ValueError: 6715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pass 6725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetReadablePakResourceName(pak_file, resource_id): 6745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Pak resources have a numeric identifier. It is not helpful when 6755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) trying to locate where footprint is generated. This does its best to 6765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) map the number to a usable string.""" 6775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not PAK_RESOURCE_ID_TO_STRING['inited']: 6785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Try to find resource header files generated by grit when 6795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # building the pak file. We'll look for files named *resources.h" 6805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # and lines of the type: 6815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # #define MY_RESOURCE_JS 1234 6825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) PAK_RESOURCE_ID_TO_STRING['inited'] = True 6835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) gen_dir = os.path.join(os.path.dirname(pak_file), 'gen') 6845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if os.path.isdir(gen_dir): 6855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for dirname, _dirs, files in os.walk(gen_dir): 6865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for filename in files: 6875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if filename.endswith('resources.h'): 6885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) LoadPakIdsFromResourceFile(os.path.join(dirname, filename)) 6895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return PAK_RESOURCE_ID_TO_STRING.get(resource_id, 6905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'Pak Resource %d' % resource_id) 6915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def AddPakData(symbols, pak_file): 6935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Adds pseudo-symbols from a pak file.""" 6945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) pak_file = os.path.abspath(pak_file) 6955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) with open(pak_file, 'rb') as pak: 6965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) data = pak.read() 6975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) PAK_FILE_VERSION = 4 6995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) HEADER_LENGTH = 2 * 4 + 1 # Two uint32s. (file version, number of entries) 7005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # and one uint8 (encoding of text resources) 7015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) INDEX_ENTRY_SIZE = 2 + 4 # Each entry is a uint16 and a uint32. 7025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) version, num_entries, _encoding = struct.unpack('<IIB', data[:HEADER_LENGTH]) 7035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert version == PAK_FILE_VERSION, ('Unsupported pak file ' 7045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'version (%d) in %s. Only ' 7055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'support version %d' % 7065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) (version, pak_file, PAK_FILE_VERSION)) 7075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if num_entries > 0: 7085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Read the index and data. 7095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) data = data[HEADER_LENGTH:] 7105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for _ in range(num_entries): 7115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) resource_id, offset = struct.unpack('<HI', data[:INDEX_ENTRY_SIZE]) 7125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) data = data[INDEX_ENTRY_SIZE:] 7135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _next_id, next_offset = struct.unpack('<HI', data[:INDEX_ENTRY_SIZE]) 7145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) resource_size = next_offset - offset 7155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 7165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_name = GetReadablePakResourceName(pak_file, resource_id) 7175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_path = pak_file 7185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_type = 'd' # Data. Approximation. 7195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_size = resource_size 7205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbols.append((symbol_name, symbol_type, symbol_size, symbol_path)) 7215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 722cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def _find_in_system_path(binary): 723cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) """Locate the full path to binary in the system path or return None 724cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if not found.""" 725cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) system_path = os.environ["PATH"].split(os.pathsep) 726cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) for path in system_path: 727cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) binary_path = os.path.join(path, binary) 728cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if os.path.isfile(binary_path): 729cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return binary_path 730cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) return None 7315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 732f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)def CheckDebugFormatSupport(library, addr2line_binary): 733f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) """Kills the program if debug data is in an unsupported format. 734f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 735f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) There are two common versions of the DWARF debug formats and 736f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) since we are right now transitioning from DWARF2 to newer formats, 737f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) it's possible to have a mix of tools that are not compatible. Detect 738f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) that and abort rather than produce meaningless output.""" 739f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) tool_output = subprocess.check_output([addr2line_binary, '--version']) 740f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) version_re = re.compile(r'^GNU [^ ]+ .* (\d+).(\d+).*?$', re.M) 741f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) parsed_output = version_re.match(tool_output) 742f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) major = int(parsed_output.group(1)) 743f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) minor = int(parsed_output.group(2)) 744f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) supports_dwarf4 = major > 2 or major == 2 and minor > 22 745f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 746f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if supports_dwarf4: 747f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) return 748f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 749f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) print('Checking version of debug information in %s.' % library) 750f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) debug_info = subprocess.check_output(['readelf', '--debug-dump=info', 751f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) '--dwarf-depth=1', library]) 752f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) dwarf_version_re = re.compile(r'^\s+Version:\s+(\d+)$', re.M) 753f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) parsed_dwarf_format_output = dwarf_version_re.search(debug_info) 754f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) version = int(parsed_dwarf_format_output.group(1)) 755f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if version > 2: 756f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) print('The supplied tools only support DWARF2 debug data but the binary\n' + 757f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 'uses DWARF%d. Update the tools or compile the binary\n' % version + 758f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 'with -gdwarf-2.') 759f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) sys.exit(1) 760f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 7615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def main(): 763cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) usage = """%prog [options] 7645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Runs a spatial analysis on a given library, looking up the source locations 7665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) of its symbols and calculating how much space each directory, source file, 7675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) and so on is taking. The result is a report that can be used to pinpoint 7685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) sources of large portions of the binary, etceteras. 7695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Under normal circumstances, you only need to pass two arguments, thusly: 7715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) %prog --library /path/to/library --destdir /path/to/output 7735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) In this mode, the program will dump the symbols from the specified library 7755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) and map those symbols back to source locations, producing a web-based 7765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) report in the specified output directory. 7775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 7785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Other options are available via '--help'. 7795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """ 7805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser = optparse.OptionParser(usage=usage) 7815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--nm-in', metavar='PATH', 7825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='if specified, use nm input from <path> instead of ' 7835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'generating it. Note that source locations should be ' 7845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'present in the file; i.e., no addr2line symbol lookups ' 7855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'will be performed when this option is specified. ' 7865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'Mutually exclusive with --library.') 7875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--destdir', metavar='PATH', 7885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='write output to the specified directory. An HTML ' 7895d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'report is generated here along with supporting files; ' 7905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'any existing report will be overwritten.') 7915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--library', metavar='PATH', 7925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='if specified, process symbols in the library at ' 7935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'the specified path. Mutually exclusive with --nm-in.') 7945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) parser.add_option('--pak', metavar='PATH', 7955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) help='if specified, includes the contents of the ' 7965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'specified *.pak file in the output.') 797cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) parser.add_option('--nm-binary', 798cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) help='use the specified nm binary to analyze library. ' 799cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'This is to be used when the nm in the path is not for ' 800cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'the right architecture or of the right version.') 801cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) parser.add_option('--addr2line-binary', 802cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) help='use the specified addr2line binary to analyze ' 803cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'library. This is to be used when the addr2line in ' 804cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'the path is not for the right architecture or ' 805cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'of the right version.') 806f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) parser.add_option('--jobs', type='int', 8075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='number of jobs to use for the parallel ' 8085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'addr2line processing pool; defaults to 1. More ' 8095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'jobs greatly improve throughput but eat RAM like ' 8105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'popcorn, and take several gigabytes each. Start low ' 8115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'and ramp this number up until your machine begins to ' 8125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'struggle with RAM. ' 8135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'This argument is only valid when using --library.') 8145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('-v', dest='verbose', action='store_true', 8155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='be verbose, printing lots of status information.') 8165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--nm-out', metavar='PATH', 8175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='keep the nm output file, and store it at the ' 8185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'specified path. This is useful if you want to see the ' 8195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'fully processed nm output after the symbols have been ' 8205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'mapped to source locations. By default, a tempfile is ' 8215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'used and is deleted when the program terminates.' 8225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'This argument is only valid when using --library.') 8230529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch parser.add_option('--legacy', action='store_true', 8240529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch help='emit legacy binary size report instead of modern') 825116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch parser.add_option('--disable-disambiguation', action='store_true', 826116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch help='disables the disambiguation process altogether,' 827116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ' NOTE: this may, depending on your toolchain, produce' 828116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ' output with some symbols at the top layer if addr2line' 829116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ' could not get the entire source path.') 830116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch parser.add_option('--source-path', default='./', 831116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch help='the path to the source code of the output binary, ' 832116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 'default set to current directory. Used in the' 833116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch ' disambiguation process.') 834cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) opts, _args = parser.parse_args() 8355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 8365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if ((not opts.library) and (not opts.nm_in)) or (opts.library and opts.nm_in): 8375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.error('exactly one of --library or --nm-in is required') 8385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (opts.nm_in): 8395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if opts.jobs: 8405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) print >> sys.stderr, ('WARNING: --jobs has no effect ' 8415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'when used with --nm-in') 8425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not opts.destdir: 8435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.error('--destdir is required argument') 8445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not opts.jobs: 845cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Use the number of processors but cap between 2 and 4 since raw 846cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # CPU power isn't the limiting factor. It's I/O limited, memory 847cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # bus limited and available-memory-limited. Too many processes and 848cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # the computer will run out of memory and it will be slow. 849cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) opts.jobs = max(2, min(4, str(multiprocessing.cpu_count()))) 850cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 851cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if opts.addr2line_binary: 852cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) assert os.path.isfile(opts.addr2line_binary) 853cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) addr2line_binary = opts.addr2line_binary 854cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) else: 855cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) addr2line_binary = _find_in_system_path('addr2line') 856cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) assert addr2line_binary, 'Unable to find addr2line in the path. '\ 857cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'Use --addr2line-binary to specify location.' 858cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 859cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if opts.nm_binary: 860cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) assert os.path.isfile(opts.nm_binary) 861cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_binary = opts.nm_binary 862cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) else: 863cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) nm_binary = _find_in_system_path('nm') 864cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) assert nm_binary, 'Unable to find nm in the path. Use --nm-binary '\ 865cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'to specify location.' 866cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 8675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if opts.pak: 8685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert os.path.isfile(opts.pak), 'Could not find ' % opts.pak 8695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 870cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) print('addr2line: %s' % addr2line_binary) 871f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) print('nm: %s' % nm_binary) 872f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) 8731320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if opts.library: 8741320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci CheckDebugFormatSupport(opts.library, addr2line_binary) 8755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 876cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) symbols = GetNmSymbols(opts.nm_in, opts.nm_out, opts.library, 877cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) opts.jobs, opts.verbose is True, 878116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch addr2line_binary, nm_binary, 879116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch opts.disable_disambiguation is None, 880116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch opts.source_path) 8815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 8825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if opts.pak: 8835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) AddPakData(symbols, opts.pak) 8845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 8855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not os.path.exists(opts.destdir): 8865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) os.makedirs(opts.destdir, 0755) 8875d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 8880529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 8890529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if opts.legacy: # legacy report 8900529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch DumpTreemap(symbols, os.path.join(opts.destdir, 'treemap-dump.js')) 8910529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch DumpLargestSymbols(symbols, 8920529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch os.path.join(opts.destdir, 'largest-symbols.js'), 100) 8930529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch DumpLargestSources(symbols, 8940529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch os.path.join(opts.destdir, 'largest-sources.js'), 100) 8950529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch DumpLargestVTables(symbols, 8960529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch os.path.join(opts.destdir, 'largest-vtables.js'), 100) 8970529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch treemap_out = os.path.join(opts.destdir, 'webtreemap') 8980529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if not os.path.exists(treemap_out): 8990529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch os.makedirs(treemap_out, 0755) 9000529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch treemap_src = os.path.join('third_party', 'webtreemap', 'src') 9010529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(treemap_src, 'COPYING'), treemap_out) 9020529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(treemap_src, 'webtreemap.js'), treemap_out) 9030529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(treemap_src, 'webtreemap.css'), treemap_out) 9040529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join('tools', 'binary_size', 'legacy_template', 9050529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 'index.html'), opts.destdir) 9060529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch else: # modern report 907116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch if opts.library: 908116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch symbol_path_origin_dir = os.path.dirname(os.path.abspath(opts.library)) 909116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch else: 910116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch # Just a guess. Hopefully all paths in the input file are absolute. 911116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch symbol_path_origin_dir = os.path.abspath(os.getcwd()) 912116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch data_js_file_name = os.path.join(opts.destdir, 'data.js') 913116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch DumpCompactTree(symbols, symbol_path_origin_dir, data_js_file_name) 9140529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch d3_out = os.path.join(opts.destdir, 'd3') 9150529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if not os.path.exists(d3_out): 9160529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch os.makedirs(d3_out, 0755) 917cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) d3_src = os.path.join(os.path.dirname(__file__), 918cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) '..', 919cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) '..', 920cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'third_party', 'd3', 'src') 921cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) template_src = os.path.join(os.path.dirname(__file__), 9220529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 'template') 9230529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(d3_src, 'LICENSE'), d3_out) 9240529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(d3_src, 'd3.js'), d3_out) 9250529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(template_src, 'index.html'), opts.destdir) 9260529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch shutil.copy(os.path.join(template_src, 'D3SymbolTreeMap.js'), opts.destdir) 9270529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 92846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) print 'Report saved to ' + opts.destdir + '/index.html' 9295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 9305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 9315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)if __name__ == '__main__': 9320529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch sys.exit(main()) 933