1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import json
5import optparse
6import os
7import sys
8
9import webgl_conformance_expectations
10
11from telemetry import benchmark as benchmark_module
12from telemetry.core import util
13from telemetry.page import page_set
14from telemetry.page import page as page_module
15from telemetry.page import page_test
16
17
18conformance_path = os.path.join(
19    util.GetChromiumSrcDir(),
20    'third_party', 'webgl', 'src', 'sdk', 'tests')
21
22conformance_harness_script = r"""
23  var testHarness = {};
24  testHarness._allTestSucceeded = true;
25  testHarness._messages = '';
26  testHarness._failures = 0;
27  testHarness._finished = false;
28  testHarness._originalLog = window.console.log;
29
30  testHarness.log = function(msg) {
31    testHarness._messages += msg + "\n";
32    testHarness._originalLog.apply(window.console, [msg]);
33  }
34
35  testHarness.reportResults = function(url, success, msg) {
36    testHarness._allTestSucceeded = testHarness._allTestSucceeded && !!success;
37    if(!success) {
38      testHarness._failures++;
39      if(msg) {
40        testHarness.log(msg);
41      }
42    }
43  };
44  testHarness.notifyFinished = function(url) {
45    testHarness._finished = true;
46  };
47  testHarness.navigateToPage = function(src) {
48    var testFrame = document.getElementById("test-frame");
49    testFrame.src = src;
50  };
51
52  window.webglTestHarness = testHarness;
53  window.parent.webglTestHarness = testHarness;
54  window.console.log = testHarness.log;
55  window.onerror = function(message, url, line) {
56    testHarness.reportResults(null, false, message);
57    testHarness.notifyFinished(null);
58  };
59"""
60
61def _DidWebGLTestSucceed(tab):
62  return tab.EvaluateJavaScript('webglTestHarness._allTestSucceeded')
63
64def _WebGLTestMessages(tab):
65  return tab.EvaluateJavaScript('webglTestHarness._messages')
66
67class WebglConformanceValidator(page_test.PageTest):
68  def __init__(self):
69    super(WebglConformanceValidator, self).__init__(attempts=1, max_failures=10)
70
71  def ValidateAndMeasurePage(self, page, tab, results):
72    if not _DidWebGLTestSucceed(tab):
73      raise page_test.Failure(_WebGLTestMessages(tab))
74
75  def CustomizeBrowserOptions(self, options):
76    options.AppendExtraBrowserArgs([
77        '--disable-gesture-requirement-for-media-playback',
78        '--disable-domain-blocking-for-3d-apis',
79        '--disable-gpu-process-crash-limit'
80    ])
81
82
83class WebglConformancePage(page_module.Page):
84  def __init__(self, page_set, test):
85    super(WebglConformancePage, self).__init__(
86      url='file://' + test, page_set=page_set, base_dir=page_set.base_dir,
87      name=('WebglConformance.%s' %
88              test.replace('/', '_').replace('-', '_').
89                 replace('\\', '_').rpartition('.')[0].replace('.', '_')))
90    self.script_to_evaluate_on_commit = conformance_harness_script
91
92  def RunNavigateSteps(self, action_runner):
93    action_runner.NavigateToPage(self)
94    action_runner.WaitForJavaScriptCondition(
95        'webglTestHarness._finished', timeout_in_seconds=180)
96
97
98class WebglConformance(benchmark_module.Benchmark):
99  """Conformance with Khronos WebGL Conformance Tests"""
100  test = WebglConformanceValidator
101
102  @classmethod
103  def AddTestCommandLineArgs(cls, group):
104    group.add_option('--webgl-conformance-version',
105        help='Version of the WebGL conformance tests to run.',
106        default='1.0.3')
107
108  def CreatePageSet(self, options):
109    tests = self._ParseTests('00_test_list.txt',
110        options.webgl_conformance_version)
111
112    ps = page_set.PageSet(
113      user_agent_type='desktop',
114      serving_dirs=[''],
115      file_path=conformance_path)
116
117    for test in tests:
118      ps.AddPage(WebglConformancePage(ps, test))
119
120    return ps
121
122  def CreateExpectations(self, page_set):
123    return webgl_conformance_expectations.WebGLConformanceExpectations()
124
125  @staticmethod
126  def _ParseTests(path, version=None):
127    test_paths = []
128    current_dir = os.path.dirname(path)
129    full_path = os.path.normpath(os.path.join(conformance_path, path))
130
131    if not os.path.exists(full_path):
132      raise Exception('The WebGL conformance test path specified ' +
133        'does not exist: ' + full_path)
134
135    with open(full_path, 'r') as f:
136      for line in f:
137        line = line.strip()
138
139        if not line:
140          continue
141
142        if line.startswith('//') or line.startswith('#'):
143          continue
144
145        line_tokens = line.split(' ')
146
147        i = 0
148        min_version = None
149        while i < len(line_tokens):
150          token = line_tokens[i]
151          if token == '--min-version':
152            i += 1
153            min_version = line_tokens[i]
154          i += 1
155
156        if version and min_version and version < min_version:
157          continue
158
159        test_name = line_tokens[-1]
160
161        if '.txt' in test_name:
162          include_path = os.path.join(current_dir, test_name)
163          test_paths += WebglConformance._ParseTests(
164            include_path, version)
165        else:
166          test = os.path.join(current_dir, test_name)
167          test_paths.append(test)
168
169    return test_paths
170