graphics_dEQP.py revision 8af26fa68550b4a04bfe80ece52682caec08a9af 
1# Copyright 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import bz2
6import glob
7import logging
8import os
9import shutil
10import tempfile
11import xml.etree.cElementTree as et
12from autotest_lib.client.bin import test, utils
13from autotest_lib.client.common_lib import error
14from autotest_lib.client.cros import service_stopper
15
16
17class graphics_dEQP(test.test):
18    """Run the drawElements Quality Program test suite.
19    """
20    version = 1
21    _services = None
22    _board = None
23    _cpu_type = None
24    _gpu_type = None
25    _filter = None
26    _width = 256  # Use smallest width for which all tests run/pass.
27    _height = 256  # Use smallest height for which all tests run/pass.
28    _timeout = 40  # Slightly larger than the dEQP watchdog timeout at 30s.
29
30    DEQP_BASEDIR = '/usr/local/deqp'
31    DEQP_MODULES = {
32        'dEQP-EGL':    'egl',
33        'dEQP-GLES2':  'gles2',
34        'dEQP-GLES3':  'gles3',
35        'dEQP-GLES31': 'gles31'
36    }
37
38    def initialize(self):
39        self._gpu_type = utils.get_gpu_family()
40        self._cpu_type = utils.get_cpu_soc_family()
41        self._board = utils.get_board()
42        self._services = service_stopper.ServiceStopper(['ui', 'powerd'])
43
44    def cleanup(self):
45        if self._services:
46            self._services.restore_services()
47
48    def _parse_test_results(self, result_filename):
49        """Handles result files with one or more test results.
50
51        @param result_filename: log file to parse.
52
53        @return: dictionary of parsed test results.
54        """
55        xml = ''
56        xml_start = False
57        xml_complete = False
58        xml_bad = False
59        result = 'ParseTestResultFail'
60        test_results = {}
61
62        if not os.path.isfile(result_filename):
63            return {}
64
65        with open(result_filename) as result_file:
66            for line in result_file.readlines():
67                # If the test terminates early, the XML will be incomplete
68                # and should not be parsed.
69                if line.startswith('#terminateTestCaseResult'):
70                    result = line.strip().split()[1]
71                    xml_bad = True
72                # Will only see #endTestCaseResult if the test does not
73                # terminate early.
74                elif line.startswith('#endTestCaseResult'):
75                    xml_complete = True
76                elif xml_start:
77                    xml += line
78                elif line.startswith('#beginTestCaseResult'):
79                    # If we see another begin before an end then something is
80                    # wrong.
81                    if xml_start:
82                        xml_bad = True
83                    else:
84                        xml_start = True
85
86                if xml_complete or xml_bad:
87                    if xml_complete:
88                        root = et.fromstring(xml)
89                        result = root.find('Result').get('StatusCode').strip()
90                        xml_complete = False
91                    test_results[result] = test_results.get(result, 0) + 1
92                    xml_bad = False
93                    xml_start = False
94                    result = 'ParseTestResultFail'
95                    xml = ''
96
97        return test_results
98
99    def _bootstrap_new_test_cases(self, executable, test_filter):
100        """Ask dEQP for all test cases and removes non-Pass'ing ones.
101
102        This function assumes that the '*.Pass' file does not exist and that
103        everything else found in the directory should be removed from the tests
104        to run. This can be used incrementally updating failing/hangin tests
105        over several runs.
106
107        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
108        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
109
110        @return: List of dEQP tests to run.
111        """
112        test_cases = []
113        not_passing_cases = []
114        # We did not find passing cases in expectations. Assume everything else
115        # that is there should not be run this time.
116        expectations_dir = os.path.join(self.bindir, 'expectations',
117                                        self._gpu_type)
118        subset_spec = '%s.*' % test_filter
119        subset_paths = glob.glob(os.path.join(expectations_dir, subset_spec))
120        for subset_file in subset_paths:
121            not_passing_cases.extend(
122                bz2.BZ2File(subset_file).read().splitlines())
123
124        # Now ask dEQP executable nicely for whole list of tests. Needs to be
125        # run in executable directory. Output file is plain text file named
126        # e.g. 'dEQP-GLES2-cases.txt'.
127        command = ('%s '
128                   '--deqp-runmode=txt-caselist '
129                   '--deqp-surface-type=fbo ' % executable)
130        logging.info('Running command %s', command)
131        utils.run(command,
132                  timeout=60,
133                  stderr_is_expected=False,
134                  ignore_status=False,
135                  stdin=None,
136                  stdout_tee=utils.TEE_TO_LOGS,
137                  stderr_tee=utils.TEE_TO_LOGS)
138
139        # Now read this caselist file.
140        caselist_name = '%s-cases.txt' % test_filter.split('.')[0]
141        caselist_file = os.path.join(os.path.dirname(executable), caselist_name)
142        if not os.path.isfile(caselist_file):
143            raise error.TestError('No caselist file at %s!' % caselist_file)
144
145        # And remove non-Pass'ing expectations from caselist.
146        caselist = open(caselist_file).read().splitlines()
147        # Contains lines like "TEST: dEQP-GLES2.capability"
148        test_cases = []
149        match = 'TEST: %s' % test_filter
150        logging.info('Bootstrapping test cases matching "%s".', match)
151        for case in caselist:
152            if case.startswith(match):
153                test = case.split('TEST: ')[1]
154                logging.info('Found test "%s".', test)
155                test_cases.append(test)
156
157        test_cases = list(set(test_cases) - set(not_passing_cases))
158        if not test_cases:
159            raise error.TestError('Unable to bootstrap %s!' % test_filter)
160        test_cases.sort()
161        return test_cases
162
163    def _get_test_cases(self, executable, test_filter, subset):
164        """Gets the test cases for 'Pass', 'Fail' etc. expectations.
165
166        This function supports bootstrapping of new GPU families and dEQP
167        binaries. In particular if there are not 'Pass' expectations found for
168        this GPU family it will query the dEQP executable for a list of all
169        available tests. It will then remove known non-'Pass'ing tests from this
170        list to avoid getting into hangs/crashes etc.
171
172        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
173        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
174        @param subset: string from 'Pass', 'Fail', 'Timeout' etc.
175
176        @return: List of dEQP tests to run.
177        """
178        expectations_dir = os.path.join(self.bindir, 'expectations',
179                                        self._gpu_type)
180        subset_name = '%s.%s.bz2' % (test_filter, subset)
181        subset_path = os.path.join(expectations_dir, subset_name)
182        if not os.path.isfile(subset_path):
183            if subset != 'Pass':
184                raise error.TestError('No subset file found for %s!' %
185                                      subset_path)
186            return self._bootstrap_new_test_cases(executable, test_filter)
187
188        test_cases = bz2.BZ2File(subset_path).read().splitlines()
189        if not test_cases:
190            raise error.TestError('No test cases found in subset file %s!' %
191                                  subset_path)
192        return test_cases
193
194    def run_tests_individually(self, executable, test_cases):
195        """Runs tests as isolated from each other, but slowly.
196
197        This function runs each test case separately as a command.
198        This means a new context for each test etc. Failures will be more
199        isolated, but runtime quite high due to overhead.
200
201        @param executable: dEQP executable path.
202        @param test_cases: List of dEQP test case strings.
203
204        @return: dictionary of test results.
205        """
206        test_results = {}
207        width = self._width
208        height = self._height
209
210        log_path = os.path.join(tempfile.gettempdir(), '%s-logs' % self._filter)
211        shutil.rmtree(log_path, ignore_errors=True)
212        os.mkdir(log_path)
213
214        i = 0
215        for test_case in test_cases:
216            i += 1
217            logging.info('[%d/%d] TestCase: %s', i, len(test_cases), test_case)
218            log_file = '%s.log' % os.path.join(log_path, test_case)
219            command = ('%s '
220                       '--deqp-case=%s '
221                       '--deqp-surface-type=fbo '
222                       '--deqp-log-images=disable '
223                       '--deqp-watchdog=enable '
224                       '--deqp-surface-width=%d '
225                       '--deqp-surface-height=%d '
226                       '--deqp-log-filename=%s' %
227                       (executable, test_case, width, height, log_file))
228
229            try:
230                logging.info('Running single: %s', command)
231                utils.run(command,
232                          timeout=self._timeout,
233                          stderr_is_expected=False,
234                          ignore_status=True,
235                          stdout_tee=utils.TEE_TO_LOGS,
236                          stderr_tee=utils.TEE_TO_LOGS)
237                result_counts = self._parse_test_results(log_file)
238                if result_counts:
239                    result = result_counts.keys()[0]
240                else:
241                    result = 'Unknown'
242            except error.CmdTimeoutError:
243                result = 'TestTimeout'
244            except error.CmdError:
245                result = 'CommandFailed'
246            except:
247                result = 'UnexpectedError'
248
249            logging.info('Result: %s', result)
250            test_results[result] = test_results.get(result, 0) + 1
251
252        return test_results
253
254    def run_tests_hasty(self, executable, test_cases):
255        """Runs tests as quickly as possible.
256
257        This function runs all the test cases, but does not isolate tests and
258        may take shortcuts/not run all tests to provide maximum coverage at
259        minumum runtime.
260
261        @param executable: dEQP executable path.
262        @param test_cases: List of dEQP test case strings.
263
264        @return: dictionary of test results.
265        """
266        # TODO(ihf): It saves half the test time to use 32*32 but a few tests
267        # fail as they need surfaces larger than 200*200.
268        width = self._width
269        height = self._height
270        command = ('%s '
271                   '--deqp-stdin-caselist '
272                   '--deqp-surface-type=fbo '
273                   '--deqp-log-images=disable '
274                   '--deqp-visibility=hidden '
275                   '--deqp-watchdog=enable '
276                   '--deqp-surface-width=%d '
277                   '--deqp-surface-height=%d ' % (executable, width, height))
278
279        log_file = os.path.join(tempfile.gettempdir(),
280                                '%s_hasty.log' % self._filter)
281        if os.path.exists(log_file):
282            os.remove(log_file)
283        command += '--deqp-log-filename=' + log_file
284
285        logging.info('Running hasty %d test cases: %s', len(test_cases),
286                     command)
287        # We are trying to handle all errors by parsing the log file.
288        try:
289            utils.run(command,
290                      timeout=3600,  # All tests combined less than 1h in hasty.
291                      stderr_is_expected=False,
292                      ignore_status=False,
293                      stdin='\n'.join(test_cases),
294                      stdout_tee=utils.TEE_TO_LOGS,
295                      stderr_tee=utils.TEE_TO_LOGS)
296        except:
297            pass
298
299        return self._parse_test_results(log_file)
300
301    def run_once(self, opts=[]):
302        options = dict(filter='',
303                       timeout=self._timeout,
304                       subset_to_run='Pass',  # Pass, Fail, Timeout etc.
305                       hasty='False',
306                       shard=1,  # TODO(ihf): Support sharding for bvt-cq.
307                       shards=1)
308        options.update(utils.args_to_dict(opts))
309        logging.info('Test Options: %s', options)
310
311        hasty = (options['hasty'] == 'True')
312        self._timeout = int(options['timeout'])
313        self._filter = options['filter']
314        if not self._filter:
315            raise error.TestError('No dEQP test filter specified')
316
317        # Some information to help postprocess logs into blacklists later.
318        logging.info('ChromeOS BOARD = %s', self._board)
319        logging.info('ChromeOS CPU family = %s', self._cpu_type)
320        logging.info('ChromeOS GPU family = %s', self._gpu_type)
321        logging.info('dEQP test filter = %s', self._filter)
322
323        # Determine module from filter.
324        test_prefix, test_group = self._filter.split('.')
325        if test_prefix in self.DEQP_MODULES:
326            module = self.DEQP_MODULES[test_prefix]
327        else:
328            raise error.TestError('Invalid test filter: %s' % self._filter)
329
330        executable_path = os.path.join(self.DEQP_BASEDIR, 'modules', module)
331        executable = os.path.join(executable_path, 'deqp-%s' % module)
332
333        self._services.stop_services()
334
335        # Must be in the executable directory when running for it to find it's
336        # test data files!
337        os.chdir(executable_path)
338        test_cases = self._get_test_cases(executable, self._filter,
339                                          options['subset_to_run'])
340        logging.info('Test cases:')
341        logging.info(test_cases)
342
343        test_results = {}
344        if hasty:
345            logging.info('Running in hasty mode.')
346            test_results = self.run_tests_hasty(executable, test_cases)
347        else:
348            logging.info('Running each test individually.')
349            test_results = self.run_tests_individually(executable, test_cases)
350
351        logging.info('Test results:')
352        logging.info(test_results)
353        self.write_perf_keyval(test_results)
354
355        test_count = 0
356        test_failures = 0
357        for result in test_results:
358            test_count += test_results[result]
359            if result.lower() not in ['pass', 'notsupported']:
360                test_failures += test_results[result]
361        # The text "Completed all tests." is used by the process_log.py script
362        # and should always appear at the end of a completed test run.
363        logging.info('Completed all tests. Saw %d tests and %d failures.',
364                     test_count, test_failures)
365
366        if test_failures:
367            raise error.TestFail('%d/%d tests failed.' %
368                                 (test_failures, test_count))
369
370        if test_count == 0:
371            raise error.TestError('No test cases found for filter: %s!' %
372                                  self._filter)
373