1#!/usr/bin/python
2# Copyright 2016 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# pylint: disable-msg=W0311
7
8import argparse
9import json
10import os
11
12
13gpu_list = [
14    #'pinetrail',
15    'sandybridge',
16    'ivybridge',
17    'baytrail',
18    'haswell',
19    'broadwell',
20    'braswell',
21    'skylake',
22    'mali-t604',
23    'mali-t628',
24    'mali-t760',
25    'rogue',
26    'tegra',
27]
28
29_PROBLEM_STATUS = ['Fail', 'Flaky']
30_UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None]
31
32status_dict = {
33    'Fail': 'FAIL ',
34    'Flaky': 'flaky',
35    'Pass': '  +  ',
36    'NotSupported': ' --- ',
37    'Skipped': ' === ',
38    'QualityWarning': 'qw   ',
39    'CompatibilityWarning': 'cw   ',
40    'Unknown': ' ??? ',
41    None: ' n/a ',
42}
43
44def load_expectation_dict(json_file):
45  data = {}
46  if os.path.isfile(json_file):
47    with open(json_file, 'r') as f:
48      text = f.read()
49      data = json.loads(text)
50  return data
51
52
53def load_expectations(json_file):
54  data = load_expectation_dict(json_file)
55  expectations = {}
56  # Convert from dictionary of lists to dictionary of sets.
57  for key in data:
58    expectations[key] = set(data[key])
59  return expectations
60
61
62def get_problem_count(dict, gpu):
63  if gpu in dict:
64    if not dict[gpu]:
65      return None
66    count = 0
67    for status in dict[gpu]:
68      if status not in _UNKNOWN_STATUS:
69        count = count + len((dict[gpu])[status])
70    # If every result has an unknown status then don't return a count.
71    if count < 1:
72      return None
73    count = 0
74    # Return counts of truly problematic statuses.
75    for status in _PROBLEM_STATUS:
76      if status in dict[gpu]:
77        count = count + len((dict[gpu])[status])
78  else:
79    print 'Warning: %s not found in dict!' % gpu
80  return count
81
82
83def get_problem_tests(dict):
84  tests = set([])
85  for gpu in dict:
86    for status in _PROBLEM_STATUS:
87      if status in dict[gpu]:
88        tests = tests.union((dict[gpu])[status])
89  return sorted(list(tests))
90
91
92def get_test_result(dict, test):
93  for key in dict:
94    if test in dict[key]:
95      return key
96  return None
97
98
99argparser = argparse.ArgumentParser(
100    description='Create a matrix of failing tests per GPU.')
101argparser.add_argument('interface',
102                       default='gles2',
103                       help='Interface for matrix (gles2, gles3, gles31).')
104args = argparser.parse_args()
105status = '%s-master.json' % args.interface
106
107dict = {}
108for gpu in gpu_list:
109  filename = 'expectations/%s/%s' % (gpu, status)
110  dict[gpu] = load_expectations(filename)
111
112tests = get_problem_tests(dict)
113
114print 'Legend:'
115for key in status_dict:
116  print '%s  -->  %s' % (status_dict[key], key)
117print
118
119offset = ''
120for gpu in gpu_list:
121  print '%s%s' % (offset, gpu)
122  offset = '%s    |    ' % offset
123print offset
124
125text_count = ''
126text_del = ''
127for gpu in gpu_list:
128  problem_count = get_problem_count(dict, gpu)
129  if problem_count is None:
130    text_count = '%s  %s  ' % (text_count, status_dict[problem_count])
131  else:
132    text_count = '%s%5d    ' % (text_count, problem_count)
133  text_del = '%s=========' % text_del
134text_count = '%s  Total failure count (Fail + Flaky)' % text_count
135print text_del
136print text_count
137print text_del
138
139for test in tests:
140  text = ''
141  for gpu in gpu_list:
142    result = get_test_result(dict[gpu], test)
143    status = status_dict[result]
144    text = '%s  %s  ' % (text, status)
145  text = '%s  %s' % (text, test)
146  print text
147
148print text_del
149print '%s repeated' % text_count
150print text_del
151
152