generate_report_unittest.py revision 4bf85d17e38c3f36f64c761a0743591d296f41a5
1#!/usr/bin/env python2 2# 3# Copyright 2016 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Test for generate_report.py.""" 7 8from __future__ import division 9from __future__ import print_function 10 11from StringIO import StringIO 12 13import copy 14import json 15import mock 16import test_flag 17import unittest 18 19import generate_report 20import results_report 21 22class _ContextualStringIO(StringIO): 23 """StringIO that can be used in `with` statements.""" 24 def __init__(self, *args): 25 StringIO.__init__(self, *args) 26 27 def __enter__(self): 28 return self 29 30 def __exit__(self, _type, _value, _traceback): 31 pass 32 33 34class GenerateReportTests(unittest.TestCase): 35 """Tests for generate_report.py.""" 36 def testCountBenchmarks(self): 37 runs = { 38 'foo': [[{}, {}, {}], [{}, {}, {}, {}]], 39 'bar': [], 40 'baz': [[], [{}], [{}, {}, {}]] 41 } 42 results = generate_report.CountBenchmarks(runs) 43 expected_results = [('foo', 4), ('bar', 0), ('baz', 3)] 44 self.assertItemsEqual(expected_results, results) 45 46 def testCutResultsInPlace(self): 47 bench_data = { 48 'foo': [[{'a': 1, 'b': 2, 'c': 3}, {'a': 3, 'b': 2.5, 'c': 1}]], 49 'bar': [[{'d': 11, 'e': 12, 'f': 13}]], 50 'baz': [[{'g': 12, 'h': 13}]], 51 'qux': [[{'i': 11}]], 52 } 53 original_bench_data = copy.deepcopy(bench_data) 54 55 max_keys = 2 56 results = generate_report.CutResultsInPlace(bench_data, max_keys=max_keys, 57 complain_on_update=False) 58 # Cuts should be in-place. 59 self.assertIs(results, bench_data) 60 self.assertItemsEqual(original_bench_data.keys(), bench_data.keys()) 61 for bench_name, original_runs in original_bench_data.iteritems(): 62 bench_runs = bench_data[bench_name] 63 self.assertEquals(len(original_runs), len(bench_runs)) 64 # Order of these sub-lists shouldn't have changed. 65 for original_list, new_list in zip(original_runs, bench_runs): 66 self.assertEqual(len(original_list), len(new_list)) 67 for original_keyvals, sub_keyvals in zip(original_list, new_list): 68 # sub_keyvals must be a subset of original_keyvals 69 self.assertDictContainsSubset(sub_keyvals, original_keyvals) 70 71 72 def testCutResultsInPlaceLeavesRetval(self): 73 bench_data = { 74 'foo': [[{'retval': 0, 'a': 1}]], 75 'bar': [[{'retval': 1}]], 76 'baz': [[{'RETVAL': 1}]], 77 } 78 results = generate_report.CutResultsInPlace(bench_data, max_keys=0, 79 complain_on_update=False) 80 # Just reach into results assuming we know it otherwise outputs things 81 # sanely. If it doesn't, testCutResultsInPlace should give an indication as 82 # to what, exactly, is broken. 83 self.assertEqual(results['foo'][0][0].items(), [('retval', 0)]) 84 self.assertEqual(results['bar'][0][0].items(), [('retval', 1)]) 85 self.assertEqual(results['baz'][0][0].items(), []) 86 87 def _RunMainWithInput(self, args, input_obj): 88 assert '-i' not in args 89 args += ['-i', '-'] 90 input_buf = _ContextualStringIO(json.dumps(input_obj)) 91 with mock.patch('generate_report.PickInputFile', return_value=input_buf) \ 92 as patched_pick: 93 result = generate_report.Main(args) 94 patched_pick.assert_called_once_with('-') 95 return result 96 97 @mock.patch('generate_report.RunActions') 98 def testMain(self, mock_run_actions): 99 # Email is left out because it's a bit more difficult to test, and it'll be 100 # mildly obvious if it's failing. 101 args = ['--json', '--html', '--text'] 102 return_code = self._RunMainWithInput(args, {'platforms': [], 'data': {}}) 103 self.assertEqual(0, return_code) 104 self.assertEqual(mock_run_actions.call_count, 1) 105 ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] 106 self.assertItemsEqual(ctors, [ 107 results_report.JSONResultsReport, 108 results_report.TextResultsReport, 109 results_report.HTMLResultsReport, 110 ]) 111 112 @mock.patch('generate_report.RunActions') 113 def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions): 114 args = [] 115 return_code = self._RunMainWithInput(args, {'platforms': [], 'data': {}}) 116 self.assertEqual(0, return_code) 117 self.assertEqual(mock_run_actions.call_count, 1) 118 ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] 119 self.assertItemsEqual(ctors, [results_report.HTMLResultsReport]) 120 121 # We only mock print_exc so we don't have exception info printed to stdout. 122 @mock.patch('generate_report.WriteFile', side_effect=ValueError('Oh noo')) 123 @mock.patch('traceback.print_exc') 124 def testRunActionsRunsAllActionsRegardlessOfExceptions(self, mock_print_exc, 125 mock_write_file): 126 actions = [(None, 'json'), (None, 'html'), (None, 'text'), (None, 'email')] 127 output_prefix = '-' 128 ok = generate_report.RunActions(actions, {}, output_prefix, overwrite=False, 129 verbose=False) 130 self.assertFalse(ok) 131 self.assertEqual(mock_write_file.call_count, len(actions)) 132 self.assertEqual(mock_print_exc.call_count, len(actions)) 133 134 @mock.patch('generate_report.WriteFile') 135 def testRunActionsReturnsTrueIfAllActionsSucceed(self, mock_write_file): 136 actions = [(None, 'json'), (None, 'html'), (None, 'text')] 137 output_prefix = '-' 138 ok = generate_report.RunActions(actions, {}, output_prefix, overwrite=False, 139 verbose=False) 140 self.assertEqual(mock_write_file.call_count, len(actions)) 141 self.assertTrue(ok) 142 143 144if __name__ == '__main__': 145 test_flag.SetTestMode(True) 146 unittest.main() 147