1ed69676d435b7b6983271ed8fab200627a0b966eGeorge Burgess IV#!/usr/bin/env python2 22e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# 32e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# Copyright 2016 The Chromium OS Authors. All rights reserved. 42e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# Use of this source code is governed by a BSD-style license that can be 52e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# found in the LICENSE file. 62e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 72e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV"""Unittest for the results reporter.""" 82e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 92e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom __future__ import division 102e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom __future__ import print_function 112e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 122e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom StringIO import StringIO 132e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 1463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVimport collections 1563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVimport mock 162e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVimport os 172e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVimport test_flag 182e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVimport unittest 192e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 202e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom benchmark_run import MockBenchmarkRun 212e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom cros_utils import logger 222e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom experiment_factory import ExperimentFactory 232e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom experiment_file import ExperimentFile 2463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVfrom machine_manager import MockCrosMachine 252e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom machine_manager import MockMachineManager 262e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom results_cache import MockResult 272368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IVfrom results_report import BenchmarkResults 2863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVfrom results_report import HTMLResultsReport 292e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom results_report import JSONResultsReport 302e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVfrom results_report import ParseChromeosImage 312368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IVfrom results_report import ParseStandardPerfReport 3263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVfrom results_report import TextResultsReport 332e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 342e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 352e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVclass FreeFunctionsTest(unittest.TestCase): 362e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """Tests for any free functions in results_report.""" 372e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 382e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV def testParseChromeosImage(self): 392e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # N.B. the cases with blank versions aren't explicitly supported by 402e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # ParseChromeosImage. I'm not sure if they need to be supported, but the 412e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # goal of this was to capture existing functionality as much as possible. 422e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV base_case = '/my/chroot/src/build/images/x86-generic/R01-1.0.date-time' \ 432e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV '/chromiumos_test_image.bin' 442e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(base_case), ('R01-1.0', base_case)) 452e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 462e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV dir_base_case = os.path.dirname(base_case) 472e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(dir_base_case), ('', dir_base_case)) 482e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 492e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV buildbot_case = '/my/chroot/chroot/tmp/buildbot-build/R02-1.0.date-time' \ 502e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV '/chromiumos_test_image.bin' 512e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV buildbot_img = buildbot_case.split('/chroot/tmp')[1] 522e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 532e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(buildbot_case), 542e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV ('R02-1.0', buildbot_img)) 552e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(os.path.dirname(buildbot_case)), 562e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV ('', os.path.dirname(buildbot_img))) 572e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 582e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # Ensure we don't act completely insanely given a few mildly insane paths. 592e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV fun_case = '/chromiumos_test_image.bin' 602e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(fun_case), ('', fun_case)) 612e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 622e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV fun_case2 = 'chromiumos_test_image.bin' 632e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(ParseChromeosImage(fun_case2), ('', fun_case2)) 642e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 652e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 662e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# There are many ways for this to be done better, but the linter complains 672e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV# about all of them (that I can think of, at least). 682e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV_fake_path_number = [0] 692e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVdef FakePath(ext): 702e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """Makes a unique path that shouldn't exist on the host system. 712e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 722e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV Each call returns a different path, so if said path finds its way into an 732e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV error message, it may be easier to track it to its source. 742e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """ 752e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV _fake_path_number[0] += 1 762e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV prefix = '/tmp/should/not/exist/%d/' % (_fake_path_number[0], ) 772e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV return os.path.join(prefix, ext) 782e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 792e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 802e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVdef MakeMockExperiment(compiler='gcc'): 812e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """Mocks an experiment using the given compiler.""" 822e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV mock_experiment_file = StringIO(""" 832e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV board: x86-alex 842e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV remote: 127.0.0.1 852e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV perf_args: record -a -e cycles 862e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV benchmark: PageCycler { 872e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV iterations: 3 882e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV } 892e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 902e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV image1 { 912e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV chromeos_image: %s 922e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV } 932e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 942e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV image2 { 952e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV remote: 127.0.0.2 962e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV chromeos_image: %s 972e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV } 982e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """ % (FakePath('cros_image1.bin'), FakePath('cros_image2.bin'))) 992e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV efile = ExperimentFile(mock_experiment_file) 1002e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV experiment = ExperimentFactory().GetExperiment(efile, 1012e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV FakePath('working_directory'), 1022e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV FakePath('log_dir')) 1032e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV for label in experiment.labels: 1042e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV label.compiler = compiler 1052e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV return experiment 1062e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 1072e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 10863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVdef _InjectSuccesses(experiment, how_many, keyvals, for_benchmark=0, 10963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV label=None): 11063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV """Injects successful experiment runs (for each label) into the experiment.""" 11163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # Defensive copy of keyvals, so if it's modified, we'll know. 11263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV keyvals = dict(keyvals) 11363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_configs = len(experiment.benchmarks) * len(experiment.labels) 11463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_runs = len(experiment.benchmark_runs) // num_configs 11563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 11663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # TODO(gbiv): Centralize the mocking of these, maybe? (It's also done in 11763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # benchmark_run_unittest) 11863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV bench = experiment.benchmarks[for_benchmark] 11963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV cache_conditions = [] 12063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV log_level = 'average' 12163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV share_cache = '' 12263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV locks_dir = '' 12363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV log = logger.GetLogger() 12463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV machine_manager = MockMachineManager(FakePath('chromeos_root'), 0, 12563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV log_level, locks_dir) 12663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV machine_manager.AddMachine('testing_machine') 12763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV machine = next(m for m in machine_manager.GetMachines() 12863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV if m.name == 'testing_machine') 12963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV for label in experiment.labels: 13063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV def MakeSuccessfulRun(n): 13163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV run = MockBenchmarkRun('mock_success%d' % (n, ), bench, label, 13263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 1 + n + num_runs, cache_conditions, 13363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV machine_manager, log, log_level, share_cache) 13463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV mock_result = MockResult(log, label, log_level, machine) 13563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV mock_result.keyvals = keyvals 13663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV run.result = mock_result 13763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV return run 13863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 13963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV experiment.benchmark_runs.extend(MakeSuccessfulRun(n) 14063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV for n in xrange(how_many)) 14163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV return experiment 14263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 14363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 14463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVclass TextResultsReportTest(unittest.TestCase): 14563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV """Tests that the output of a text report contains the things we pass in. 14663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 14763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV At the moment, this doesn't care deeply about the format in which said 14863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV things are displayed. It just cares that they're present. 14963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV """ 15063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 15163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV def _checkReport(self, email): 15263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_success = 2 15363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV success_keyvals = {'retval': 0, 'machine': 'some bot', 'a_float': 3.96} 15463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV experiment = _InjectSuccesses(MakeMockExperiment(), num_success, 15563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV success_keyvals) 1562368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV text_report = TextResultsReport.FromExperiment(experiment, email=email) \ 1572368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV .GetReport() 15863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn(str(success_keyvals['a_float']), text_report) 15963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn(success_keyvals['machine'], text_report) 16063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn(MockCrosMachine.CPUINFO_STRING, text_report) 16163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV return text_report 16263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 16363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 16463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV def testOutput(self): 16563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV email_report = self._checkReport(email=True) 16663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV text_report = self._checkReport(email=False) 16763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 16863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # Ensure that the reports somehow different. Otherwise, having the 16963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # distinction is useless. 17063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertNotEqual(email_report, text_report) 17163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 17263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 17363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IVclass HTMLResultsReportTest(unittest.TestCase): 17463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV """Tests that the output of a HTML report contains the things we pass in. 17563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 17663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV At the moment, this doesn't care deeply about the format in which said 17763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV things are displayed. It just cares that they're present. 17863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV """ 17963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 18063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV _TestOutput = collections.namedtuple('TestOutput', ['summary_table', 18163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 'perf_html', 18243fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV 'chart_js', 18363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 'charts', 1842368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'full_table', 18563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 'experiment_file']) 18663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 18763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV @staticmethod 18843fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV def _GetTestOutput(perf_table, chart_js, summary_table, print_table, 1892368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV chart_divs, full_table, experiment_file): 19043fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV # N.B. Currently we don't check chart_js; it's just passed through because 19143fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV # cros lint complains otherwise. 1922368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV summary_table = print_table(summary_table, 'HTML') 1932368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV perf_html = print_table(perf_table, 'HTML') 1942368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV full_table = print_table(full_table, 'HTML') 1952368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV return HTMLResultsReportTest._TestOutput(summary_table=summary_table, 1962368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV perf_html=perf_html, 19743fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV chart_js=chart_js, 1982368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV charts=chart_divs, 1992368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV full_table=full_table, 2002368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV experiment_file=experiment_file) 2012368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2022368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def _GetOutput(self, experiment=None, benchmark_results=None): 2032368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV with mock.patch('results_report_templates.GenerateHTMLPage') as standin: 2042368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV if experiment is not None: 2052368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV HTMLResultsReport.FromExperiment(experiment).GetReport() 2062368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV else: 2072368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV HTMLResultsReport(benchmark_results).GetReport() 2082368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV mod_mock = standin 20963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertEquals(mod_mock.call_count, 1) 2102368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # call_args[0] is positional args, call_args[1] is kwargs. 2112368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEquals(mod_mock.call_args[0], tuple()) 2122368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV fmt_args = mod_mock.call_args[1] 2132368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV return self._GetTestOutput(**fmt_args) 21463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 21563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV def testNoSuccessOutput(self): 21663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV output = self._GetOutput(MakeMockExperiment()) 21763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn('no result', output.summary_table) 2182368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn('no result', output.full_table) 21963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertEqual(output.charts, '') 2202368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotEqual(output.experiment_file, '') 22163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 22263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV def testSuccessfulOutput(self): 22363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_success = 2 22463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV success_keyvals = {'retval': 0, 'a_float': 3.96} 22563f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV output = self._GetOutput(_InjectSuccesses(MakeMockExperiment(), num_success, 22663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV success_keyvals)) 22763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 22863f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertNotIn('no result', output.summary_table) 22963f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV #self.assertIn(success_keyvals['machine'], output.summary_table) 23063f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn('a_float', output.summary_table) 23163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn(str(success_keyvals['a_float']), output.summary_table) 2322368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn('a_float', output.full_table) 23363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV # The _ in a_float is filtered out when we're generating HTML. 23463f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV self.assertIn('afloat', output.charts) 2352368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # And make sure we have our experiment file... 2362368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotEqual(output.experiment_file, '') 2372368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2382368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testBenchmarkResultFailure(self): 2392368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV labels = ['label1'] 2402368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_names_and_iterations = [('bench1', 1)] 2412368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals = {'bench1': [[]]} 2422368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = BenchmarkResults(labels, benchmark_names_and_iterations, 2432368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals) 2442368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV output = self._GetOutput(benchmark_results=results) 2452368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn('no result', output.summary_table) 2462368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(output.charts, '') 2472368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(output.experiment_file, '') 2482368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2492368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testBenchmarkResultSuccess(self): 2502368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV labels = ['label1'] 2512368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_names_and_iterations = [('bench1', 1)] 2522368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals = {'bench1': [[{'retval': 1, 'foo': 2.0}]]} 2532368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = BenchmarkResults(labels, benchmark_names_and_iterations, 2542368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals) 2552368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV output = self._GetOutput(benchmark_results=results) 2562368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotIn('no result', output.summary_table) 2572368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn('bench1', output.summary_table) 2582368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn('bench1', output.full_table) 2592368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotEqual(output.charts, '') 2602368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(output.experiment_file, '') 26163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 26263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 2632e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVclass JSONResultsReportTest(unittest.TestCase): 2642e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV """Tests JSONResultsReport.""" 2652368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2662368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV REQUIRED_REPORT_KEYS = ('date', 'time', 'label', 'test_name', 'pass') 2672368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV EXPERIMENT_REPORT_KEYS = ('board', 'chromeos_image', 'chromeos_version', 2682368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'chrome_version', 'compiler') 2692368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2702368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV @staticmethod 2712368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def _GetRequiredKeys(is_experiment): 2722368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV required_keys = JSONResultsReportTest.REQUIRED_REPORT_KEYS 2732368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV if is_experiment: 2742368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV required_keys += JSONResultsReportTest.EXPERIMENT_REPORT_KEYS 2752368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV return required_keys 2762368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 2772368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def _CheckRequiredKeys(self, test_output, is_experiment): 2782368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV required_keys = self._GetRequiredKeys(is_experiment) 2792e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV for output in test_output: 2802368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV for key in required_keys: 2812e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertIn(key, output) 2822e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 2832e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV def testAllFailedJSONReportOutput(self): 2842368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV experiment = MakeMockExperiment() 2852368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = JSONResultsReport.FromExperiment(experiment).GetReportObject() 2862368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self._CheckRequiredKeys(results, is_experiment=True) 2872e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # Nothing succeeded; we don't send anything more than what's required. 2882368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV required_keys = self._GetRequiredKeys(is_experiment=True) 2892e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV for result in results: 2902368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertItemsEqual(result.iterkeys(), required_keys) 2912e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 2922e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV def testJSONReportOutputWithSuccesses(self): 2932e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV success_keyvals = { 2942e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 'retval': 0, 2952e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 'a_float': '2.3', 2962e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 'many_floats': [['1.0', '2.0'], ['3.0']], 2972e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 'machine': "i'm a pirate" 2982e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV } 2992e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 3002e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # 2 is arbitrary. 30163f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_success = 2 30263f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV experiment = _InjectSuccesses(MakeMockExperiment(), num_success, 30363f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV success_keyvals) 3042368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = JSONResultsReport.FromExperiment(experiment).GetReportObject() 3052368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self._CheckRequiredKeys(results, is_experiment=True) 30663f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV 30763f13489b1a02bbbc75b46fec6ae7a817442df94George Burgess IV num_passes = num_success * len(experiment.labels) 3082e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV non_failures = [r for r in results if r['pass']] 3092e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(num_passes, len(non_failures)) 3102e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 3112e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV # TODO(gbiv): ...Is the 3.0 *actually* meant to be dropped? 3122e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV expected_detailed = {'a_float': 2.3, 'many_floats': [1.0, 2.0]} 3132e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV for pass_ in non_failures: 3142e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertIn('detailed_results', pass_) 3152e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertDictEqual(expected_detailed, pass_['detailed_results']) 3162e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertIn('machine', pass_) 3172e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV self.assertEqual(success_keyvals['machine'], pass_['machine']) 3182e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 3192368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testFailedJSONReportOutputWithoutExperiment(self): 3202368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV labels = ['label1'] 3212368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_names_and_iterations = [('bench1', 1), ('bench2', 2), 3222368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV ('bench3', 1), ('bench4', 0)] 3232368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals = { 3242368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench1': [[{'retval': 1, 'foo': 2.0}]], 3252368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench2': [[{'retval': 1, 'foo': 4.0}, {'retval': -1, 'bar': 999}]], 3262368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # lack of retval is considered a failure. 3272368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench3': [[{}]], 3282368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench4': [[]] 3292368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV } 3302368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, 3312368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals) 3322368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = JSONResultsReport(bench_results).GetReportObject() 3332368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self._CheckRequiredKeys(results, is_experiment=False) 3342368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertFalse(any(r['pass'] for r in results)) 3352368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3362368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testJSONGetReportObeysJSONSettings(self): 3372368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV labels = ['label1'] 3382368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_names_and_iterations = [('bench1', 1)] 3392368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # These can be anything, really. So long as they're distinctive. 3402368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV separators = (',\t\n\t', ':\t\n\t') 3412368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals = {'bench1': [[{'retval': 0, 'foo': 2.0}]]} 3422368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, 3432368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals) 3442368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV reporter = JSONResultsReport(bench_results, 3452368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV json_args={'separators': separators}) 3462368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV result_str = reporter.GetReport() 3472368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn(separators[0], result_str) 3482368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn(separators[1], result_str) 3492368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3502368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testSuccessfulJSONReportOutputWithoutExperiment(self): 3512368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV labels = ['label1'] 3522368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_names_and_iterations = [('bench1', 1), ('bench2', 2)] 3532368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals = { 3542368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench1': [[{'retval': 0, 'foo': 2.0}]], 3552368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 'bench2': [[{'retval': 0, 'foo': 4.0}, {'retval': 0, 'bar': 999}]] 3562368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV } 3572368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV bench_results = BenchmarkResults(labels, benchmark_names_and_iterations, 3582368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV benchmark_keyvals) 3592368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV results = JSONResultsReport(bench_results).GetReportObject() 3602368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self._CheckRequiredKeys(results, is_experiment=False) 3612368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertTrue(all(r['pass'] for r in results)) 3622368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # Enforce that the results have *some* deterministic order. 3632368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV keyfn = lambda r: (r['test_name'], r['detailed_results'].get('foo', 5.0)) 3642368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV sorted_results = sorted(results, key=keyfn) 3652368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV detailed_results = [r['detailed_results'] for r in sorted_results] 3662368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV bench1, bench2_foo, bench2_bar = detailed_results 3672368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(bench1['foo'], 2.0) 3682368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(bench2_foo['foo'], 4.0) 3692368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(bench2_bar['bar'], 999) 3702368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotIn('bar', bench1) 3712368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotIn('bar', bench2_foo) 3722368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertNotIn('foo', bench2_bar) 3732368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3742368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3752368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IVclass PerfReportParserTest(unittest.TestCase): 37643fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV """Tests for the perf report parser in results_report.""" 3772368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV @staticmethod 3782368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def _ReadRealPerfReport(): 3792368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV my_dir = os.path.dirname(os.path.realpath(__file__)) 3802368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV with open(os.path.join(my_dir, 'perf_files/perf.data.report.0')) as f: 3812368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV return f.read() 3822368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3832368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV def testParserParsesRealWorldPerfReport(self): 3842368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV report = ParseStandardPerfReport(self._ReadRealPerfReport()) 3852368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertItemsEqual(['cycles', 'instructions'], report.keys()) 3862368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 3872368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV # Arbitrarily selected known percentages from the perf report. 3882368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV known_cycles_percentages = { 3892368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV '0xffffffffa4a1f1c9': 0.66, 3902368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV '0x0000115bb7ba9b54': 0.47, 3912368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV '0x0000000000082e08': 0.00, 3922368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV '0xffffffffa4a13e63': 0.00, 3932368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV } 3942368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV report_cycles = report['cycles'] 3952368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(len(report_cycles), 214) 3962368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV for k, v in known_cycles_percentages.iteritems(): 3972368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn(k, report_cycles) 3982368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(v, report_cycles[k]) 3992368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 4002368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV known_instrunctions_percentages = { 40143fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV '0x0000115bb6c35d7a': 1.65, 40243fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV '0x0000115bb7ba9b54': 0.67, 40343fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV '0x0000000000024f56': 0.00, 40443fe499320c0d77c7cd2b3189896a0d9791821c5George Burgess IV '0xffffffffa4a0ee03': 0.00, 4052368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV } 4062368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV report_instructions = report['instructions'] 4072368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(len(report_instructions), 492) 4082368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV for k, v in known_instrunctions_percentages.iteritems(): 4092368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertIn(k, report_instructions) 4102368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV self.assertEqual(v, report_instructions[k]) 4112368b41ea869a6904ae8320ad69f1b8b9a9a3714George Burgess IV 4122e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV 4132e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IVif __name__ == '__main__': 4142e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV test_flag.SetTestMode(True) 4152e9f8a097c095ca93052b368ffab4c850d4d3d0fGeorge Burgess IV unittest.main() 416