1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Utility to display a summary of JSON-format GM results, and exit with
7a nonzero errorcode if there were non-ignored failures in the GM results.
8
9Usage:
10  python display_json_results.py <filename>
11
12TODO(epoger): We may want to add flags to set the following:
13- which error types cause a nonzero return code
14- maximum number of tests to list for any one ResultAccumulator
15  (to keep the output reasonably short)
16"""
17
18__author__ = 'Elliot Poger'
19
20
21# system-level imports
22import sys
23
24# local imports
25import gm_json
26
27
28class ResultAccumulator(object):
29  """Object that accumulates results of a given type, and can generate a
30     summary upon request."""
31
32  def __init__(self, name, do_list, do_fail):
33    """name: name of the category this result type falls into
34       do_list: whether to list all of the tests with this results type
35       do_fail: whether to return with nonzero exit code if there are any
36                results of this type
37    """
38    self._name = name
39    self._do_list = do_list
40    self._do_fail = do_fail
41    self._testnames = []
42
43  def AddResult(self, testname):
44    """Adds a result of this particular type.
45       testname: (string) name of the test"""
46    self._testnames.append(testname)
47
48  def ShouldSignalFailure(self):
49    """Returns true if this result type is serious (self._do_fail is True)
50       and there were any results of this type."""
51    if self._do_fail and self._testnames:
52      return True
53    else:
54      return False
55
56  def GetSummaryLine(self):
57    """Returns a single-line string summary of all results added to this
58       accumulator so far."""
59    summary = ''
60    if self._do_fail:
61      summary += '[*] '
62    else:
63      summary += '[ ] '
64    summary += str(len(self._testnames))
65    summary += ' '
66    summary += self._name
67    if self._do_list:
68      summary += ': '
69      for testname in self._testnames:
70        summary += testname
71        summary += ' '
72    return summary
73
74
75def Display(filepath):
76  """Displays a summary of the results in a JSON file.
77     Returns True if the results are free of any significant failures.
78     filepath: (string) path to JSON file"""
79
80  # Map labels within the JSON file to the ResultAccumulator for each label.
81  results_map = {
82    gm_json.JSONKEY_ACTUALRESULTS_FAILED:
83        ResultAccumulator(name='ExpectationsMismatch',
84                          do_list=True, do_fail=True),
85    gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED:
86        ResultAccumulator(name='IgnoredExpectationsMismatch',
87                          do_list=True, do_fail=False),
88    gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON:
89        ResultAccumulator(name='MissingExpectations',
90                          do_list=False, do_fail=False),
91    gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
92        ResultAccumulator(name='Passed',
93                          do_list=False, do_fail=False),
94  }
95
96  success = True
97  json_dict = gm_json.LoadFromFile(filepath)
98  actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
99  for label, accumulator in results_map.iteritems():
100    results = actual_results[label]
101    if results:
102      for result in results:
103        accumulator.AddResult(result)
104    print accumulator.GetSummaryLine()
105    if accumulator.ShouldSignalFailure():
106      success = False
107  print '(results marked with [*] will cause nonzero return value)'
108  return success
109
110
111if '__main__' == __name__:
112  if len(sys.argv) != 2:
113    raise Exception('usage: %s <input-json-filepath>' % sys.argv[0])
114  sys.exit(0 if Display(sys.argv[1]) else 1)
115