1#!/usr/bin/env python
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""binary_cache_builder.py: read perf.data, collect binaries needed by
19    it, and put them in binary_cache.
20"""
21
22from __future__ import print_function
23import argparse
24import os
25import os.path
26import re
27import shutil
28import subprocess
29import sys
30import time
31
32from simpleperf_report_lib import *
33from utils import *
34
35
36class BinaryCacheBuilder(object):
37    """Collect all binaries needed by perf.data in binary_cache."""
38    def __init__(self, config):
39        config_names = ['perf_data_path', 'symfs_dirs']
40        for name in config_names:
41            if name not in config:
42                log_exit('config for "%s" is missing' % name)
43
44        self.perf_data_path = config.get('perf_data_path')
45        if not os.path.isfile(self.perf_data_path):
46            log_exit("can't find file %s" % self.perf_data_path)
47        self.symfs_dirs = config.get('symfs_dirs')
48        for symfs_dir in self.symfs_dirs:
49            if not os.path.isdir(symfs_dir):
50                log_exit("symfs_dir '%s' is not a directory" % symfs_dir)
51        self.adb = AdbHelper()
52        self.readelf_path = find_tool_path('readelf')
53        if not self.readelf_path and self.symfs_dirs:
54            log_warning("Debug shared libraries on host are not used because can't find readelf.")
55        self.binary_cache_dir = 'binary_cache'
56        if not os.path.isdir(self.binary_cache_dir):
57            os.makedirs(self.binary_cache_dir)
58
59
60    def build_binary_cache(self):
61        self._collect_used_binaries()
62        self._copy_binaries_from_symfs_dirs()
63        self._pull_binaries_from_device()
64        self._pull_kernel_symbols()
65
66
67    def _collect_used_binaries(self):
68        """read perf.data, collect all used binaries and their build id (if available)."""
69        # A dict mapping from binary name to build_id
70        binaries = dict()
71        lib = ReportLib()
72        lib.SetRecordFile(self.perf_data_path)
73        lib.SetLogSeverity('error')
74        while True:
75            sample = lib.GetNextSample()
76            if sample is None:
77                lib.Close()
78                break
79            symbols = [lib.GetSymbolOfCurrentSample()]
80            callchain = lib.GetCallChainOfCurrentSample()
81            for i in range(callchain.nr):
82                symbols.append(callchain.entries[i].symbol)
83
84            for symbol in symbols:
85                dso_name = symbol.dso_name
86                if dso_name not in binaries:
87                    binaries[dso_name] = lib.GetBuildIdForPath(dso_name)
88        self.binaries = binaries
89
90
91    def _copy_binaries_from_symfs_dirs(self):
92        """collect all files in symfs_dirs."""
93        if not self.symfs_dirs:
94            return
95
96        # It is possible that the path of the binary in symfs_dirs doesn't match
97        # the one recorded in perf.data. For example, a file in symfs_dirs might
98        # be "debug/arm/obj/armeabi-v7a/libsudo-game-jni.so", but the path in
99        # perf.data is "/data/app/xxxx/lib/arm/libsudo-game-jni.so". So we match
100        # binaries if they have the same filename (like libsudo-game-jni.so)
101        # and same build_id.
102
103        # Map from filename to binary paths.
104        filename_dict = dict()
105        for binary in self.binaries:
106            index = binary.rfind('/')
107            filename = binary[index+1:]
108            paths = filename_dict.get(filename)
109            if paths is None:
110                filename_dict[filename] = paths = []
111            paths.append(binary)
112
113        # Walk through all files in symfs_dirs, and copy matching files to build_cache.
114        for symfs_dir in self.symfs_dirs:
115            for root, _, files in os.walk(symfs_dir):
116                for file in files:
117                    paths = filename_dict.get(file)
118                    if paths is not None:
119                        build_id = self._read_build_id(os.path.join(root, file))
120                        if not build_id:
121                            continue
122                        for binary in paths:
123                            expected_build_id = self.binaries.get(binary)
124                            if expected_build_id == build_id:
125                                self._copy_to_binary_cache(os.path.join(root, file),
126                                                           expected_build_id, binary)
127
128
129    def _copy_to_binary_cache(self, from_path, expected_build_id, target_file):
130        if target_file[0] == '/':
131            target_file = target_file[1:]
132        target_file = target_file.replace('/', os.sep)
133        target_file = os.path.join(self.binary_cache_dir, target_file)
134        if (os.path.isfile(target_file) and self._read_build_id(target_file) == expected_build_id
135            and self._file_has_symbol_table(target_file)):
136            # The existing file in binary_cache can provide more information, so no
137            # need to copy.
138            return
139        target_dir = os.path.dirname(target_file)
140        if not os.path.isdir(target_dir):
141            os.makedirs(target_dir)
142        log_info('copy to binary_cache: %s to %s' % (from_path, target_file))
143        shutil.copy(from_path, target_file)
144
145
146    def _pull_binaries_from_device(self):
147        """pull binaries needed in perf.data to binary_cache."""
148        for binary in self.binaries:
149            build_id = self.binaries[binary]
150            if binary[0] != '/' or binary == "//anon":
151                # [kernel.kallsyms] or unknown, or something we can't find binary.
152                continue
153            binary_cache_file = binary[1:].replace('/', os.sep)
154            binary_cache_file = os.path.join(self.binary_cache_dir, binary_cache_file)
155            self._check_and_pull_binary(binary, build_id, binary_cache_file)
156
157
158    def _check_and_pull_binary(self, binary, expected_build_id, binary_cache_file):
159        """If the binary_cache_file exists and has the expected_build_id, there
160           is no need to pull the binary from device. Otherwise, pull it.
161        """
162        need_pull = True
163        if os.path.isfile(binary_cache_file):
164            need_pull = False
165            if expected_build_id:
166                build_id = self._read_build_id(binary_cache_file)
167                if expected_build_id != build_id:
168                    need_pull = True
169        if need_pull:
170            target_dir = os.path.dirname(binary_cache_file)
171            if not os.path.isdir(target_dir):
172                os.makedirs(target_dir)
173            if os.path.isfile(binary_cache_file):
174                os.remove(binary_cache_file)
175            log_info('pull file to binary_cache: %s to %s' % (binary, binary_cache_file))
176            self._pull_file_from_device(binary, binary_cache_file)
177        else:
178            log_info('use current file in binary_cache: %s' % binary_cache_file)
179
180
181    def _read_build_id(self, file):
182        """read build id of a binary on host."""
183        if not self.readelf_path:
184            return ""
185        output = subprocess.check_output([self.readelf_path, '-n', file])
186        output = bytes_to_str(output)
187        result = re.search(r'Build ID:\s*(\S+)', output)
188        if result:
189            build_id = result.group(1)
190            if len(build_id) < 40:
191                build_id += '0' * (40 - len(build_id))
192            build_id = '0x' + build_id
193            return build_id
194        return ""
195
196
197    def _file_has_symbol_table(self, file):
198        """Test if an elf file has symbol table section."""
199        if not self.readelf_path:
200            return False
201        output = subprocess.check_output([self.readelf_path, '-S', file])
202        output = bytes_to_str(output)
203        if output.find('.symtab') != -1:
204            return True
205        return False
206
207
208    def _pull_file_from_device(self, device_path, host_path):
209        if self.adb.run(['pull', device_path, host_path]):
210            return True
211        # In non-root device, we can't pull /data/app/XXX/base.odex directly.
212        # Instead, we can first copy the file to /data/local/tmp, then pull it.
213        filename = device_path[device_path.rfind('/')+1:]
214        if (self.adb.run(['shell', 'cp', device_path, '/data/local/tmp']) and
215            self.adb.run(['pull', '/data/local/tmp/' + filename, host_path])):
216            self.adb.run(['shell', 'rm', '/data/local/tmp/' + filename])
217            return True
218        log_warning('failed to pull %s from device' % device_path)
219        return False
220
221
222    def _pull_kernel_symbols(self):
223        file = os.path.join(self.binary_cache_dir, 'kallsyms')
224        if os.path.isfile(file):
225            os.remove(file)
226        if self.adb.switch_to_root():
227            self.adb.run(['shell', '"echo 0 >/proc/sys/kernel/kptr_restrict"'])
228            self.adb.run(['pull', '/proc/kallsyms', file])
229
230
231def main():
232    parser = argparse.ArgumentParser(description=
233"""Pull binaries needed by perf.data from device to binary_cache directory.""")
234    parser.add_argument('-i', '--perf_data_path', default='perf.data', help=
235"""The path of profiling data.""")
236    parser.add_argument('-lib', '--native_lib_dir', nargs='+', help=
237"""Path to find debug version of native shared libraries used in the app.""",
238                        action='append')
239    args = parser.parse_args()
240    config = {}
241    config['perf_data_path'] = args.perf_data_path
242    config['symfs_dirs'] = flatten_arg_list(args.native_lib_dir)
243
244    builder = BinaryCacheBuilder(config)
245    builder.build_binary_cache()
246
247
248if __name__ == '__main__':
249    main()