17839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger#!/usr/bin/env python 27839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger# Copyright (c) 2013 The Chromium Authors. All rights reserved. 37839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger# Use of this source code is governed by a BSD-style license that can be 47839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger# found in the LICENSE file. 57839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 67839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger"""Utility to display a summary of JSON-format GM results, and exit with 77839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenbergera nonzero errorcode if there were non-ignored failures in the GM results. 87839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 97839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek SollenbergerUsage: 107839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger python display_json_results.py <filename> 117839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 127839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek SollenbergerTODO(epoger): We may want to add flags to set the following: 137839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger- which error types cause a nonzero return code 147839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger- maximum number of tests to list for any one ResultAccumulator 157839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger (to keep the output reasonably short) 167839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger""" 177839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 187839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger__author__ = 'Elliot Poger' 197839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 207839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 2158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# system-level imports 227839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenbergerimport sys 237839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 2458190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger# local imports 2558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenbergerimport gm_json 267839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 277839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 287839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenbergerclass ResultAccumulator(object): 297839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """Object that accumulates results of a given type, and can generate a 307839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary upon request.""" 317839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 327839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger def __init__(self, name, do_list, do_fail): 337839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """name: name of the category this result type falls into 347839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_list: whether to list all of the tests with this results type 357839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_fail: whether to return with nonzero exit code if there are any 367839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger results of this type 377839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """ 387839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger self._name = name 397839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger self._do_list = do_list 407839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger self._do_fail = do_fail 417839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger self._testnames = [] 427839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 437839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger def AddResult(self, testname): 447839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """Adds a result of this particular type. 457839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger testname: (string) name of the test""" 467839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger self._testnames.append(testname) 477839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 487839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger def ShouldSignalFailure(self): 497839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """Returns true if this result type is serious (self._do_fail is True) 507839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger and there were any results of this type.""" 517839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if self._do_fail and self._testnames: 527839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger return True 537839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger else: 547839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger return False 557839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 567839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger def GetSummaryLine(self): 577839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """Returns a single-line string summary of all results added to this 587839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger accumulator so far.""" 597839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary = '' 607839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if self._do_fail: 617839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += '[*] ' 627839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger else: 637839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += '[ ] ' 647839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += str(len(self._testnames)) 657839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += ' ' 667839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += self._name 677839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if self._do_list: 687839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += ': ' 697839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger for testname in self._testnames: 707839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += testname 717839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger summary += ' ' 727839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger return summary 737839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 747839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 757839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenbergerdef Display(filepath): 767839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger """Displays a summary of the results in a JSON file. 777839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger Returns True if the results are free of any significant failures. 787839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger filepath: (string) path to JSON file""" 797839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 807839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger # Map labels within the JSON file to the ResultAccumulator for each label. 817839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger results_map = { 8258190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger gm_json.JSONKEY_ACTUALRESULTS_FAILED: 837839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger ResultAccumulator(name='ExpectationsMismatch', 847839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_list=True, do_fail=True), 8558190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED: 867839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger ResultAccumulator(name='IgnoredExpectationsMismatch', 877839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_list=True, do_fail=False), 8858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON: 897839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger ResultAccumulator(name='MissingExpectations', 907839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_list=False, do_fail=False), 9158190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: 927839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger ResultAccumulator(name='Passed', 937839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger do_list=False, do_fail=False), 947839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger } 957839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 967839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger success = True 9758190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger json_dict = gm_json.LoadFromFile(filepath) 9858190644c30e1c4aa8e527f3503c58f841e0fcf3Derek Sollenberger actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] 997839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger for label, accumulator in results_map.iteritems(): 1007839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger results = actual_results[label] 1017839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if results: 1027839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger for result in results: 1037839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger accumulator.AddResult(result) 1047839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger print accumulator.GetSummaryLine() 1057839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if accumulator.ShouldSignalFailure(): 1067839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger success = False 1077839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger print '(results marked with [*] will cause nonzero return value)' 1087839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger return success 1097839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 1107839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger 1117839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenbergerif '__main__' == __name__: 1127839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger if len(sys.argv) != 2: 1137839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) 1147839ce1af63bf12fe7b3caa866970bbbb3afb13dDerek Sollenberger sys.exit(0 if Display(sys.argv[1]) else 1) 115