1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Simpleperf runtest runner: run simpleperf runtests on host or on device.
18
19For a simpleperf runtest like one_function test, it contains following steps:
201. Run simpleperf record command to record simpleperf_runtest_one_function's
21   running samples, which is generated in perf.data.
222. Run simpleperf report command to parse perf.data, generate perf.report.
234. Parse perf.report and see if it matches expectation.
24
25The information of all runtests is stored in runtest.conf.
26"""
27
28import os
29import os.path
30import re
31import subprocess
32import sys
33import xml.etree.ElementTree as ET
34
35
36class CallTreeNode(object):
37
38  def __init__(self, name):
39    self.name = name
40    self.children = []
41
42  def add_child(self, child):
43    self.children.append(child)
44
45  def __str__(self):
46    return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
47
48  def _dump(self, indent):
49    indent_str = '  ' * indent
50    strs = [indent_str + self.name]
51    for child in self.children:
52      strs.extend(child._dump(indent + 1))
53    return strs
54
55
56class Symbol(object):
57
58  def __init__(self, name, comm, overhead, children_overhead):
59    self.name = name
60    self.comm = comm
61    self.overhead = overhead
62    # children_overhead is the overhead sum of this symbol and functions
63    # called by this symbol.
64    self.children_overhead = children_overhead
65    self.call_tree = None
66
67  def set_call_tree(self, call_tree):
68    self.call_tree = call_tree
69
70  def __str__(self):
71    strs = []
72    strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
73        self.name, self.comm, self.overhead, self.children_overhead))
74    if self.call_tree:
75      strs.append('\t%s' % self.call_tree)
76    return '\n'.join(strs)
77
78
79class SymbolOverheadRequirement(object):
80
81  def __init__(self, symbol_name=None, comm=None, min_overhead=None,
82               max_overhead=None):
83    self.symbol_name = symbol_name
84    self.comm = comm
85    self.min_overhead = min_overhead
86    self.max_overhead = max_overhead
87
88  def __str__(self):
89    strs = []
90    strs.append('SymbolOverheadRequirement')
91    if self.symbol_name is not None:
92      strs.append('symbol_name=%s' % self.symbol_name)
93    if self.comm is not None:
94      strs.append('comm=%s' % self.comm)
95    if self.min_overhead is not None:
96      strs.append('min_overhead=%f' % self.min_overhead)
97    if self.max_overhead is not None:
98      strs.append('max_overhead=%f' % self.max_overhead)
99    return ' '.join(strs)
100
101  def is_match(self, symbol):
102    if self.symbol_name is not None:
103      if self.symbol_name != symbol.name:
104        return False
105    if self.comm is not None:
106      if self.comm != symbol.comm:
107        return False
108    return True
109
110  def check_overhead(self, overhead):
111    if self.min_overhead is not None:
112      if self.min_overhead > overhead:
113        return False
114    if self.max_overhead is not None:
115      if self.max_overhead < overhead:
116        return False
117    return True
118
119
120class SymbolRelationRequirement(object):
121
122  def __init__(self, symbol_name, comm=None):
123    self.symbol_name = symbol_name
124    self.comm = comm
125    self.children = []
126
127  def add_child(self, child):
128    self.children.append(child)
129
130  def __str__(self):
131    return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
132
133  def _dump(self, indent):
134    indent_str = '  ' * indent
135    strs = [indent_str + self.symbol_name +
136            (' ' + self.comm if self.comm else '')]
137    for child in self.children:
138      strs.extend(child._dump(indent + 1))
139    return strs
140
141  def is_match(self, symbol):
142    if symbol.name != self.symbol_name:
143      return False
144    if self.comm is not None:
145      if symbol.comm != self.comm:
146        return False
147    return True
148
149  def check_relation(self, call_tree):
150    if not call_tree:
151      return False
152    if self.symbol_name != call_tree.name:
153      return False
154    for child in self.children:
155      child_matched = False
156      for node in call_tree.children:
157        if child.check_relation(node):
158          child_matched = True
159          break
160      if not child_matched:
161        return False
162    return True
163
164
165class Test(object):
166
167  def __init__(
168          self,
169          test_name,
170          executable_name,
171          report_options,
172          symbol_overhead_requirements,
173          symbol_children_overhead_requirements,
174          symbol_relation_requirements):
175    self.test_name = test_name
176    self.executable_name = executable_name
177    self.report_options = report_options
178    self.symbol_overhead_requirements = symbol_overhead_requirements
179    self.symbol_children_overhead_requirements = (
180        symbol_children_overhead_requirements)
181    self.symbol_relation_requirements = symbol_relation_requirements
182
183  def __str__(self):
184    strs = []
185    strs.append('Test test_name=%s' % self.test_name)
186    strs.append('\texecutable_name=%s' % self.executable_name)
187    strs.append('\treport_options=%s' % (' '.join(self.report_options)))
188    strs.append('\tsymbol_overhead_requirements:')
189    for req in self.symbol_overhead_requirements:
190      strs.append('\t\t%s' % req)
191    strs.append('\tsymbol_children_overhead_requirements:')
192    for req in self.symbol_children_overhead_requirements:
193      strs.append('\t\t%s' % req)
194    strs.append('\tsymbol_relation_requirements:')
195    for req in self.symbol_relation_requirements:
196      strs.append('\t\t%s' % req)
197    return '\n'.join(strs)
198
199
200def load_config_file(config_file):
201  tests = []
202  tree = ET.parse(config_file)
203  root = tree.getroot()
204  assert root.tag == 'runtests'
205  for test in root:
206    assert test.tag == 'test'
207    test_name = test.attrib['name']
208    executable_name = None
209    report_options = []
210    symbol_overhead_requirements = []
211    symbol_children_overhead_requirements = []
212    symbol_relation_requirements = []
213    for test_item in test:
214      if test_item.tag == 'executable':
215        executable_name = test_item.attrib['name']
216      elif test_item.tag == 'report':
217        report_options = test_item.attrib['option'].split()
218      elif (test_item.tag == 'symbol_overhead' or
219              test_item.tag == 'symbol_children_overhead'):
220        for symbol_item in test_item:
221          assert symbol_item.tag == 'symbol'
222          symbol_name = None
223          if 'name' in symbol_item.attrib:
224            symbol_name = symbol_item.attrib['name']
225          comm = None
226          if 'comm' in symbol_item.attrib:
227            comm = symbol_item.attrib['comm']
228          overhead_min = None
229          if 'min' in symbol_item.attrib:
230            overhead_min = float(symbol_item.attrib['min'])
231          overhead_max = None
232          if 'max' in symbol_item.attrib:
233            overhead_max = float(symbol_item.attrib['max'])
234
235          if test_item.tag == 'symbol_overhead':
236            symbol_overhead_requirements.append(
237                SymbolOverheadRequirement(
238                    symbol_name,
239                    comm,
240                    overhead_min,
241                    overhead_max)
242            )
243          else:
244            symbol_children_overhead_requirements.append(
245                SymbolOverheadRequirement(
246                    symbol_name,
247                    comm,
248                    overhead_min,
249                    overhead_max))
250      elif test_item.tag == 'symbol_callgraph_relation':
251        for symbol_item in test_item:
252          req = load_symbol_relation_requirement(symbol_item)
253          symbol_relation_requirements.append(req)
254
255    tests.append(
256        Test(
257            test_name,
258            executable_name,
259            report_options,
260            symbol_overhead_requirements,
261            symbol_children_overhead_requirements,
262            symbol_relation_requirements))
263  return tests
264
265
266def load_symbol_relation_requirement(symbol_item):
267  symbol_name = symbol_item.attrib['name']
268  comm = None
269  if 'comm' in symbol_item.attrib:
270    comm = symbol_item.attrib['comm']
271  req = SymbolRelationRequirement(symbol_name, comm)
272  for item in symbol_item:
273    child_req = load_symbol_relation_requirement(item)
274    req.add_child(child_req)
275  return req
276
277
278class Runner(object):
279
280  def __init__(self, target, perf_path):
281    self.target = target
282    self.is32 = target.endswith('32')
283    self.perf_path = perf_path
284    self.use_callgraph = False
285    self.sampler = 'cpu-cycles'
286
287  def record(self, test_executable_name, record_file, additional_options=[]):
288    call_args = [self.perf_path, 'record']
289    call_args += ['--duration', '2']
290    call_args += ['-e', '%s:u' % self.sampler]
291    if self.use_callgraph:
292      call_args += ['-f', '1000', '-g']
293    call_args += ['-o', record_file]
294    call_args += additional_options
295    test_executable_name += '32' if self.is32 else '64'
296    call_args += [test_executable_name]
297    self._call(call_args)
298
299  def report(self, record_file, report_file, additional_options=[]):
300    call_args = [self.perf_path, 'report']
301    call_args += ['-i', record_file]
302    if self.use_callgraph:
303      call_args += ['-g', 'callee']
304    call_args += additional_options
305    self._call(call_args, report_file)
306
307  def _call(self, args, output_file=None):
308    pass
309
310
311class HostRunner(Runner):
312
313  """Run perf test on host."""
314
315  def __init__(self, target):
316    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
317    super(HostRunner, self).__init__(target, perf_path)
318
319  def _call(self, args, output_file=None):
320    output_fh = None
321    if output_file is not None:
322      output_fh = open(output_file, 'w')
323    subprocess.check_call(args, stdout=output_fh)
324    if output_fh is not None:
325      output_fh.close()
326
327
328class DeviceRunner(Runner):
329
330  """Run perf test on device."""
331
332  def __init__(self, target):
333    self.tmpdir = '/data/local/tmp/'
334    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
335    super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path)
336    self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir)
337    lib = 'lib' if self.is32 else 'lib64'
338    self._download(os.environ['OUT'] + '/system/' + lib + '/libsimpleperf_inplace_sampler.so',
339                   self.tmpdir)
340
341  def _call(self, args, output_file=None):
342    output_fh = None
343    if output_file is not None:
344      output_fh = open(output_file, 'w')
345    args_with_adb = ['adb', 'shell']
346    args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args))
347    subprocess.check_call(args_with_adb, stdout=output_fh)
348    if output_fh is not None:
349      output_fh.close()
350
351  def _download(self, file, to_dir):
352    args = ['adb', 'push', file, to_dir]
353    subprocess.check_call(args)
354
355  def record(self, test_executable_name, record_file, additional_options=[]):
356    self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name +
357                   ('32' if self.is32 else '64'), self.tmpdir)
358    super(DeviceRunner, self).record(self.tmpdir + test_executable_name,
359                                     self.tmpdir + record_file,
360                                     additional_options)
361
362  def report(self, record_file, report_file, additional_options=[]):
363    super(DeviceRunner, self).report(self.tmpdir + record_file,
364                                     report_file,
365                                     additional_options)
366
367class ReportAnalyzer(object):
368
369  """Check if perf.report matches expectation in Configuration."""
370
371  def _read_report_file(self, report_file, has_callgraph):
372    fh = open(report_file, 'r')
373    lines = fh.readlines()
374    fh.close()
375
376    lines = [x.rstrip() for x in lines]
377    blank_line_index = -1
378    for i in range(len(lines)):
379      if not lines[i]:
380        blank_line_index = i
381    assert blank_line_index != -1
382    assert blank_line_index + 1 < len(lines)
383    title_line = lines[blank_line_index + 1]
384    report_item_lines = lines[blank_line_index + 2:]
385
386    if has_callgraph:
387      assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
388    else:
389      assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
390
391    return self._parse_report_items(report_item_lines, has_callgraph)
392
393  def _parse_report_items(self, lines, has_callgraph):
394    symbols = []
395    cur_symbol = None
396    call_tree_stack = {}
397    vertical_columns = []
398    last_node = None
399    last_depth = -1
400
401    for line in lines:
402      if not line:
403        continue
404      if not line[0].isspace():
405        if has_callgraph:
406          m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
407          children_overhead = float(m.group(1))
408          overhead = float(m.group(2))
409          comm = m.group(3)
410          symbol_name = m.group(4)
411          cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
412          symbols.append(cur_symbol)
413        else:
414          m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
415          overhead = float(m.group(1))
416          comm = m.group(2)
417          symbol_name = m.group(3)
418          cur_symbol = Symbol(symbol_name, comm, overhead, 0)
419          symbols.append(cur_symbol)
420        # Each report item can have different column depths.
421        vertical_columns = []
422      else:
423        for i in range(len(line)):
424          if line[i] == '|':
425            if not vertical_columns or vertical_columns[-1] < i:
426              vertical_columns.append(i)
427
428        if not line.strip('| \t'):
429          continue
430        if line.find('-') == -1:
431          function_name = line.strip('| \t')
432          node = CallTreeNode(function_name)
433          last_node.add_child(node)
434          last_node = node
435          call_tree_stack[last_depth] = node
436        else:
437          pos = line.find('-')
438          depth = -1
439          for i in range(len(vertical_columns)):
440            if pos >= vertical_columns[i]:
441              depth = i
442          assert depth != -1
443
444          line = line.strip('|- \t')
445          m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
446          if m:
447            function_name = m.group(1)
448          else:
449            function_name = line
450
451          node = CallTreeNode(function_name)
452          if depth == 0:
453            cur_symbol.set_call_tree(node)
454
455          else:
456            call_tree_stack[depth - 1].add_child(node)
457          call_tree_stack[depth] = node
458          last_node = node
459          last_depth = depth
460
461    return symbols
462
463  def check_report_file(self, test, report_file, has_callgraph):
464    symbols = self._read_report_file(report_file, has_callgraph)
465    if not self._check_symbol_overhead_requirements(test, symbols):
466      return False
467    if has_callgraph:
468      if not self._check_symbol_children_overhead_requirements(test, symbols):
469        return False
470      if not self._check_symbol_relation_requirements(test, symbols):
471        return False
472    return True
473
474  def _check_symbol_overhead_requirements(self, test, symbols):
475    result = True
476    matched = [False] * len(test.symbol_overhead_requirements)
477    matched_overhead = [0] * len(test.symbol_overhead_requirements)
478    for symbol in symbols:
479      for i in range(len(test.symbol_overhead_requirements)):
480        req = test.symbol_overhead_requirements[i]
481        if req.is_match(symbol):
482          matched[i] = True
483          matched_overhead[i] += symbol.overhead
484    for i in range(len(matched)):
485      if not matched[i]:
486        print 'requirement (%s) has no matched symbol in test %s' % (
487            test.symbol_overhead_requirements[i], test)
488        result = False
489      else:
490        fulfilled = req.check_overhead(matched_overhead[i])
491        if not fulfilled:
492          print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
493              symbol, req, test)
494          result = False
495    return result
496
497  def _check_symbol_children_overhead_requirements(self, test, symbols):
498    result = True
499    matched = [False] * len(test.symbol_children_overhead_requirements)
500    for symbol in symbols:
501      for i in range(len(test.symbol_children_overhead_requirements)):
502        req = test.symbol_children_overhead_requirements[i]
503        if req.is_match(symbol):
504          matched[i] = True
505          fulfilled = req.check_overhead(symbol.children_overhead)
506          if not fulfilled:
507            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
508                symbol, req, test)
509            result = False
510    for i in range(len(matched)):
511      if not matched[i]:
512        print 'requirement (%s) has no matched symbol in test %s' % (
513            test.symbol_children_overhead_requirements[i], test)
514        result = False
515    return result
516
517  def _check_symbol_relation_requirements(self, test, symbols):
518    result = True
519    matched = [False] * len(test.symbol_relation_requirements)
520    for symbol in symbols:
521      for i in range(len(test.symbol_relation_requirements)):
522        req = test.symbol_relation_requirements[i]
523        if req.is_match(symbol):
524          matched[i] = True
525          fulfilled = req.check_relation(symbol.call_tree)
526          if not fulfilled:
527            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
528                symbol, req, test)
529            result = False
530    for i in range(len(matched)):
531      if not matched[i]:
532        print 'requirement (%s) has no matched symbol in test %s' % (
533            test.symbol_relation_requirements[i], test)
534        result = False
535    return result
536
537
538def build_runner(target, use_callgraph, sampler):
539  if target == 'host32' and use_callgraph:
540    print "Current 64bit linux host doesn't support `simpleperf32 record -g`"
541    return None
542  if target.startswith('host'):
543    runner = HostRunner(target)
544  else:
545    runner = DeviceRunner(target)
546  runner.use_callgraph = use_callgraph
547  runner.sampler = sampler
548  return runner
549
550
551def test_with_runner(runner, tests):
552  report_analyzer = ReportAnalyzer()
553  for test in tests:
554    runner.record(test.executable_name, 'perf.data')
555    if runner.sampler == 'inplace-sampler':
556      # TODO: fix this when inplace-sampler actually works.
557      runner.report('perf.data', 'perf.report')
558      symbols = report_analyzer._read_report_file('perf.report', runner.use_callgraph)
559      result = False
560      if len(symbols) == 1 and symbols[0].name.find('FakeFunction()') != -1:
561        result = True
562    else:
563      runner.report('perf.data', 'perf.report', additional_options = test.report_options)
564      result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph)
565    str = 'test %s on %s ' % (test.test_name, runner.target)
566    if runner.use_callgraph:
567      str += 'with call graph '
568    str += 'using %s ' % runner.sampler
569    str += ' Succeeded' if result else 'Failed'
570    print str
571    if not result:
572      exit(1)
573
574
575def runtest(target_options, use_callgraph_options, sampler_options, selected_tests):
576  tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \
577                           '/runtest.conf')
578  if selected_tests is not None:
579    new_tests = []
580    for test in tests:
581      if test.test_name in selected_tests:
582        new_tests.append(test)
583    tests = new_tests
584  for target in target_options:
585    for use_callgraph in use_callgraph_options:
586      for sampler in sampler_options:
587        runner = build_runner(target, use_callgraph, sampler)
588        if runner is not None:
589          test_with_runner(runner, tests)
590
591
592def main():
593  target_options = ['host64', 'host32', 'device64', 'device32']
594  use_callgraph_options = [False, True]
595  sampler_options = ['cpu-cycles', 'inplace-sampler']
596  selected_tests = None
597  i = 1
598  while i < len(sys.argv):
599    if sys.argv[i] == '--host':
600      target_options = ['host64', 'host32']
601    elif sys.argv[i] == '--device':
602      target_options = ['device64', 'device32']
603    elif sys.argv[i] == '--normal':
604      use_callgraph_options = [False]
605    elif sys.argv[i] == '--callgraph':
606      use_callgraph_options = [True]
607    elif sys.argv[i] == '--no-inplace-sampler':
608      sampler_options = ['cpu-cycles']
609    elif sys.argv[i] == '--inplace-sampler':
610      sampler_options = ['inplace-sampler']
611    elif sys.argv[i] == '--test':
612      if i < len(sys.argv):
613        i += 1
614        for test in sys.argv[i].split(','):
615          if selected_tests is None:
616            selected_tests = {}
617          selected_tests[test] = True
618    i += 1
619  runtest(target_options, use_callgraph_options, sampler_options, selected_tests)
620
621if __name__ == '__main__':
622  main()
623