1#!/usr/bin/python 2# Copyright 2016 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# pylint: disable-msg=W0311 7 8import argparse 9import json 10import os 11 12 13gpu_list = [ 14 #'pinetrail', 15 'sandybridge', 16 'ivybridge', 17 'baytrail', 18 'haswell', 19 'broadwell', 20 'braswell', 21 'skylake', 22 'mali-t604', 23 'mali-t628', 24 'mali-t760', 25 'rogue', 26 'tegra', 27] 28 29_PROBLEM_STATUS = ['Fail', 'Flaky'] 30_UNKNOWN_STATUS = ['NotSupported', 'Skipped', 'Unknown', None] 31 32status_dict = { 33 'Fail': 'FAIL ', 34 'Flaky': 'flaky', 35 'Pass': ' + ', 36 'NotSupported': ' --- ', 37 'Skipped': ' === ', 38 'QualityWarning': 'qw ', 39 'CompatibilityWarning': 'cw ', 40 'Unknown': ' ??? ', 41 None: ' n/a ', 42} 43 44def load_expectation_dict(json_file): 45 data = {} 46 if os.path.isfile(json_file): 47 with open(json_file, 'r') as f: 48 text = f.read() 49 data = json.loads(text) 50 return data 51 52 53def load_expectations(json_file): 54 data = load_expectation_dict(json_file) 55 expectations = {} 56 # Convert from dictionary of lists to dictionary of sets. 57 for key in data: 58 expectations[key] = set(data[key]) 59 return expectations 60 61 62def get_problem_count(dict, gpu): 63 if gpu in dict: 64 if not dict[gpu]: 65 return None 66 count = 0 67 for status in dict[gpu]: 68 if status not in _UNKNOWN_STATUS: 69 count = count + len((dict[gpu])[status]) 70 # If every result has an unknown status then don't return a count. 71 if count < 1: 72 return None 73 count = 0 74 # Return counts of truly problematic statuses. 75 for status in _PROBLEM_STATUS: 76 if status in dict[gpu]: 77 count = count + len((dict[gpu])[status]) 78 else: 79 print 'Warning: %s not found in dict!' % gpu 80 return count 81 82 83def get_problem_tests(dict): 84 tests = set([]) 85 for gpu in dict: 86 for status in _PROBLEM_STATUS: 87 if status in dict[gpu]: 88 tests = tests.union((dict[gpu])[status]) 89 return sorted(list(tests)) 90 91 92def get_test_result(dict, test): 93 for key in dict: 94 if test in dict[key]: 95 return key 96 return None 97 98 99argparser = argparse.ArgumentParser( 100 description='Create a matrix of failing tests per GPU.') 101argparser.add_argument('interface', 102 default='gles2', 103 help='Interface for matrix (gles2, gles3, gles31).') 104args = argparser.parse_args() 105status = '%s-master.json' % args.interface 106 107dict = {} 108for gpu in gpu_list: 109 filename = 'expectations/%s/%s' % (gpu, status) 110 dict[gpu] = load_expectations(filename) 111 112tests = get_problem_tests(dict) 113 114print 'Legend:' 115for key in status_dict: 116 print '%s --> %s' % (status_dict[key], key) 117print 118 119offset = '' 120for gpu in gpu_list: 121 print '%s%s' % (offset, gpu) 122 offset = '%s | ' % offset 123print offset 124 125text_count = '' 126text_del = '' 127for gpu in gpu_list: 128 problem_count = get_problem_count(dict, gpu) 129 if problem_count is None: 130 text_count = '%s %s ' % (text_count, status_dict[problem_count]) 131 else: 132 text_count = '%s%5d ' % (text_count, problem_count) 133 text_del = '%s=========' % text_del 134text_count = '%s Total failure count (Fail + Flaky)' % text_count 135print text_del 136print text_count 137print text_del 138 139for test in tests: 140 text = '' 141 for gpu in gpu_list: 142 result = get_test_result(dict[gpu], test) 143 status = status_dict[result] 144 text = '%s %s ' % (text, status) 145 text = '%s %s' % (text, test) 146 print text 147 148print text_del 149print '%s repeated' % text_count 150print text_del 151 152