1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4import json 5import optparse 6import os 7import sys 8 9import webgl_conformance_expectations 10 11from telemetry import benchmark as benchmark_module 12from telemetry.core import util 13from telemetry.page import page_set 14from telemetry.page import page as page_module 15from telemetry.page import page_test 16 17 18conformance_path = os.path.join( 19 util.GetChromiumSrcDir(), 20 'third_party', 'webgl', 'src', 'sdk', 'tests') 21 22conformance_harness_script = r""" 23 var testHarness = {}; 24 testHarness._allTestSucceeded = true; 25 testHarness._messages = ''; 26 testHarness._failures = 0; 27 testHarness._finished = false; 28 testHarness._originalLog = window.console.log; 29 30 testHarness.log = function(msg) { 31 testHarness._messages += msg + "\n"; 32 testHarness._originalLog.apply(window.console, [msg]); 33 } 34 35 testHarness.reportResults = function(url, success, msg) { 36 testHarness._allTestSucceeded = testHarness._allTestSucceeded && !!success; 37 if(!success) { 38 testHarness._failures++; 39 if(msg) { 40 testHarness.log(msg); 41 } 42 } 43 }; 44 testHarness.notifyFinished = function(url) { 45 testHarness._finished = true; 46 }; 47 testHarness.navigateToPage = function(src) { 48 var testFrame = document.getElementById("test-frame"); 49 testFrame.src = src; 50 }; 51 52 window.webglTestHarness = testHarness; 53 window.parent.webglTestHarness = testHarness; 54 window.console.log = testHarness.log; 55 window.onerror = function(message, url, line) { 56 testHarness.reportResults(null, false, message); 57 testHarness.notifyFinished(null); 58 }; 59""" 60 61def _DidWebGLTestSucceed(tab): 62 return tab.EvaluateJavaScript('webglTestHarness._allTestSucceeded') 63 64def _WebGLTestMessages(tab): 65 return tab.EvaluateJavaScript('webglTestHarness._messages') 66 67class WebglConformanceValidator(page_test.PageTest): 68 def __init__(self): 69 super(WebglConformanceValidator, self).__init__(attempts=1, max_failures=10) 70 71 def ValidateAndMeasurePage(self, page, tab, results): 72 if not _DidWebGLTestSucceed(tab): 73 raise page_test.Failure(_WebGLTestMessages(tab)) 74 75 def CustomizeBrowserOptions(self, options): 76 options.AppendExtraBrowserArgs([ 77 '--disable-gesture-requirement-for-media-playback', 78 '--disable-domain-blocking-for-3d-apis', 79 '--disable-gpu-process-crash-limit' 80 ]) 81 82 83class WebglConformancePage(page_module.Page): 84 def __init__(self, page_set, test): 85 super(WebglConformancePage, self).__init__( 86 url='file://' + test, page_set=page_set, base_dir=page_set.base_dir, 87 name=('WebglConformance.%s' % 88 test.replace('/', '_').replace('-', '_'). 89 replace('\\', '_').rpartition('.')[0].replace('.', '_'))) 90 self.script_to_evaluate_on_commit = conformance_harness_script 91 92 def RunNavigateSteps(self, action_runner): 93 action_runner.NavigateToPage(self) 94 action_runner.WaitForJavaScriptCondition( 95 'webglTestHarness._finished', timeout_in_seconds=180) 96 97 98class WebglConformance(benchmark_module.Benchmark): 99 """Conformance with Khronos WebGL Conformance Tests""" 100 test = WebglConformanceValidator 101 102 @classmethod 103 def AddTestCommandLineArgs(cls, group): 104 group.add_option('--webgl-conformance-version', 105 help='Version of the WebGL conformance tests to run.', 106 default='1.0.3') 107 108 def CreatePageSet(self, options): 109 tests = self._ParseTests('00_test_list.txt', 110 options.webgl_conformance_version) 111 112 ps = page_set.PageSet( 113 user_agent_type='desktop', 114 serving_dirs=[''], 115 file_path=conformance_path) 116 117 for test in tests: 118 ps.AddPage(WebglConformancePage(ps, test)) 119 120 return ps 121 122 def CreateExpectations(self, page_set): 123 return webgl_conformance_expectations.WebGLConformanceExpectations() 124 125 @staticmethod 126 def _ParseTests(path, version=None): 127 test_paths = [] 128 current_dir = os.path.dirname(path) 129 full_path = os.path.normpath(os.path.join(conformance_path, path)) 130 131 if not os.path.exists(full_path): 132 raise Exception('The WebGL conformance test path specified ' + 133 'does not exist: ' + full_path) 134 135 with open(full_path, 'r') as f: 136 for line in f: 137 line = line.strip() 138 139 if not line: 140 continue 141 142 if line.startswith('//') or line.startswith('#'): 143 continue 144 145 line_tokens = line.split(' ') 146 147 i = 0 148 min_version = None 149 while i < len(line_tokens): 150 token = line_tokens[i] 151 if token == '--min-version': 152 i += 1 153 min_version = line_tokens[i] 154 i += 1 155 156 if version and min_version and version < min_version: 157 continue 158 159 test_name = line_tokens[-1] 160 161 if '.txt' in test_name: 162 include_path = os.path.join(current_dir, test_name) 163 test_paths += WebglConformance._ParseTests( 164 include_path, version) 165 else: 166 test = os.path.join(current_dir, test_name) 167 test_paths.append(test) 168 169 return test_paths 170