1a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com#!/usr/bin/env python 2a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com# Use of this source code is governed by a BSD-style license that can be 4a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com# found in the LICENSE file. 5a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 6a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com"""Utility to display a summary of JSON-format GM results, and exit with 7a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.coma nonzero errorcode if there were non-ignored failures in the GM results. 8a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 9a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comUsage: 10a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com python display_json_results.py <filename> 11a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 12a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comTODO(epoger): We may want to add flags to set the following: 13a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com- which error types cause a nonzero return code 14a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com- maximum number of tests to list for any one ResultAccumulator 15a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com (to keep the output reasonably short) 16a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com""" 17a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 18a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com__author__ = 'Elliot Poger' 19a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 20a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 219d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# system-level imports 22a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comimport sys 23a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 249d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# local imports 259d33154b1362d89454297c21663b2403f1a75de5epoger@google.comimport gm_json 26a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 27a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 28a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comclass ResultAccumulator(object): 29a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """Object that accumulates results of a given type, and can generate a 30a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary upon request.""" 31a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 32a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com def __init__(self, name, do_list, do_fail): 33a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """name: name of the category this result type falls into 34a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_list: whether to list all of the tests with this results type 35a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_fail: whether to return with nonzero exit code if there are any 36a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com results of this type 37a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """ 38a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com self._name = name 39a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com self._do_list = do_list 40a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com self._do_fail = do_fail 41a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com self._testnames = [] 42a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 43a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com def AddResult(self, testname): 44a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """Adds a result of this particular type. 45a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com testname: (string) name of the test""" 46a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com self._testnames.append(testname) 47a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 48a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com def ShouldSignalFailure(self): 49a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """Returns true if this result type is serious (self._do_fail is True) 50a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com and there were any results of this type.""" 51a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if self._do_fail and self._testnames: 52a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com return True 53a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com else: 54a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com return False 55a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 56a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com def GetSummaryLine(self): 57a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """Returns a single-line string summary of all results added to this 58a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com accumulator so far.""" 59a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary = '' 60a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if self._do_fail: 61a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += '[*] ' 62a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com else: 63a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += '[ ] ' 64a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += str(len(self._testnames)) 65a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += ' ' 66a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += self._name 67a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if self._do_list: 68a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += ': ' 69a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com for testname in self._testnames: 70a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += testname 71a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com summary += ' ' 72a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com return summary 73a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 74a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 75a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comdef Display(filepath): 76a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com """Displays a summary of the results in a JSON file. 77a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com Returns True if the results are free of any significant failures. 78a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com filepath: (string) path to JSON file""" 79a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 80a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com # Map labels within the JSON file to the ResultAccumulator for each label. 81a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com results_map = { 829d33154b1362d89454297c21663b2403f1a75de5epoger@google.com gm_json.JSONKEY_ACTUALRESULTS_FAILED: 83a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com ResultAccumulator(name='ExpectationsMismatch', 84a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_list=True, do_fail=True), 859d33154b1362d89454297c21663b2403f1a75de5epoger@google.com gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED: 86a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com ResultAccumulator(name='IgnoredExpectationsMismatch', 87a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_list=True, do_fail=False), 889d33154b1362d89454297c21663b2403f1a75de5epoger@google.com gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON: 89a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com ResultAccumulator(name='MissingExpectations', 90a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_list=False, do_fail=False), 919d33154b1362d89454297c21663b2403f1a75de5epoger@google.com gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: 92a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com ResultAccumulator(name='Passed', 93a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com do_list=False, do_fail=False), 94a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com } 95a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 96a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com success = True 974075fd485101eea80cc67c396e6839e555ab948aepoger@google.com json_dict = gm_json.LoadFromFile(filepath) 989d33154b1362d89454297c21663b2403f1a75de5epoger@google.com actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] 99a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com for label, accumulator in results_map.iteritems(): 100a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com results = actual_results[label] 101a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if results: 102a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com for result in results: 103a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com accumulator.AddResult(result) 104a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com print accumulator.GetSummaryLine() 105a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if accumulator.ShouldSignalFailure(): 106a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com success = False 107a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com print '(results marked with [*] will cause nonzero return value)' 108a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com return success 109a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 110a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com 111a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.comif '__main__' == __name__: 112a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com if len(sys.argv) != 2: 113a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) 114a55e48d6ef67daa90d17731b49d45b0ac94dcc34epoger@google.com sys.exit(0 if Display(sys.argv[1]) else 1) 115