graphics_dEQP.py revision 3152d7b9e9fc8f6b724bb426b8af68ba81a0077f 
1# Copyright 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import bz2
6import glob
7import logging
8import os
9import shutil
10import tempfile
11import xml.etree.cElementTree as et
12from autotest_lib.client.bin import test, utils
13from autotest_lib.client.common_lib import error
14from autotest_lib.client.cros import service_stopper
15
16
17class graphics_dEQP(test.test):
18    """Run the drawElements Quality Program test suite.
19    """
20    version = 1
21    _services = None
22    _hasty = False
23    _hasty_batch_size = 100  # Batch size in hasty mode.
24    _board = None
25    _cpu_type = None
26    _gpu_type = None
27    _filter = None
28    _width = 256  # Use smallest width for which all tests run/pass.
29    _height = 256  # Use smallest height for which all tests run/pass.
30    _timeout = 70  # Larger than twice the dEQP watchdog timeout at 30s.
31    _test_names = None
32
33    DEQP_BASEDIR = '/usr/local/deqp'
34    DEQP_MODULES = {
35        'dEQP-EGL': 'egl',
36        'dEQP-GLES2': 'gles2',
37        'dEQP-GLES3': 'gles3',
38        'dEQP-GLES31': 'gles31'
39    }
40
41    def initialize(self):
42        self._gpu_type = utils.get_gpu_family()
43        self._cpu_type = utils.get_cpu_soc_family()
44        self._board = utils.get_board()
45        self._services = service_stopper.ServiceStopper(['ui', 'powerd'])
46
47    def cleanup(self):
48        if self._services:
49            self._services.restore_services()
50
51    def _parse_test_results(self, result_filename, test_results=None):
52        """Handles result files with one or more test results.
53
54        @param result_filename: log file to parse.
55
56        @return: dictionary of parsed test results.
57        """
58        xml = ''
59        xml_start = False
60        xml_complete = False
61        xml_bad = False
62        result = 'ParseTestResultFail'
63
64        if test_results is None:
65            test_results = {}
66
67        if not os.path.isfile(result_filename):
68            return test_results
69        # TODO(ihf): Add names of failing tests to a list in the results.
70        with open(result_filename) as result_file:
71            for line in result_file.readlines():
72                # If the test terminates early, the XML will be incomplete
73                # and should not be parsed.
74                if line.startswith('#terminateTestCaseResult'):
75                    result = line.strip().split()[1]
76                    xml_bad = True
77                # Will only see #endTestCaseResult if the test does not
78                # terminate early.
79                elif line.startswith('#endTestCaseResult'):
80                    xml_complete = True
81                elif xml_start:
82                    xml += line
83                elif line.startswith('#beginTestCaseResult'):
84                    # If we see another begin before an end then something is
85                    # wrong.
86                    if xml_start:
87                        xml_bad = True
88                    else:
89                        xml_start = True
90
91                if xml_complete or xml_bad:
92                    if xml_complete:
93                        root = et.fromstring(xml)
94                        result = root.find('Result').get('StatusCode').strip()
95                        xml_complete = False
96                    test_results[result] = test_results.get(result, 0) + 1
97                    xml_bad = False
98                    xml_start = False
99                    result = 'ParseTestResultFail'
100                    xml = ''
101
102        return test_results
103
104    def _bootstrap_new_test_cases(self, executable, test_filter):
105        """Ask dEQP for all test cases and removes non-Pass'ing ones.
106
107        This function assumes that the '*.Pass' file does not exist and that
108        everything else found in the directory should be removed from the tests
109        to run. This can be used incrementally updating failing/hangin tests
110        over several runs.
111
112        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
113        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
114
115        @return: List of dEQP tests to run.
116        """
117        test_cases = []
118        not_passing_cases = []
119        # We did not find passing cases in expectations. Assume everything else
120        # that is there should not be run this time.
121        expectations_dir = os.path.join(self.bindir, 'expectations',
122                                        self._gpu_type)
123        subset_spec = '%s.*' % test_filter
124        subset_paths = glob.glob(os.path.join(expectations_dir, subset_spec))
125        for subset_file in subset_paths:
126            # Filter against hasty failures only in hasty mode.
127            if self._hasty or '.hasty.bz2' not in subset_file:
128                not_passing_cases.extend(
129                    bz2.BZ2File(subset_file).read().splitlines())
130
131        # Now ask dEQP executable nicely for whole list of tests. Needs to be
132        # run in executable directory. Output file is plain text file named
133        # e.g. 'dEQP-GLES2-cases.txt'.
134        command = ('%s '
135                   '--deqp-runmode=txt-caselist '
136                   '--deqp-surface-type=fbo ' % executable)
137        logging.info('Running command %s', command)
138        utils.run(command,
139                  timeout=60,
140                  stderr_is_expected=False,
141                  ignore_status=False,
142                  stdin=None,
143                  stdout_tee=utils.TEE_TO_LOGS,
144                  stderr_tee=utils.TEE_TO_LOGS)
145
146        # Now read this caselist file.
147        caselist_name = '%s-cases.txt' % test_filter.split('.')[0]
148        caselist_file = os.path.join(os.path.dirname(executable), caselist_name)
149        if not os.path.isfile(caselist_file):
150            raise error.TestError('No caselist file at %s!' % caselist_file)
151
152        # And remove non-Pass'ing expectations from caselist.
153        caselist = open(caselist_file).read().splitlines()
154        # Contains lines like "TEST: dEQP-GLES2.capability"
155        test_cases = []
156        match = 'TEST: %s' % test_filter
157        logging.info('Bootstrapping test cases matching "%s".', match)
158        for case in caselist:
159            if case.startswith(match):
160                case = case.split('TEST: ')[1]
161                test_cases.append(case)
162
163        test_cases = list(set(test_cases) - set(not_passing_cases))
164        if not test_cases:
165            raise error.TestError('Unable to bootstrap %s!' % test_filter)
166        # TODO(ihf): Sorting is nice but can introduce errors due to changed
167        # ordering and state leakage. Consider deleting this if it causes
168        # too much problems.
169        test_cases.sort()
170        return test_cases
171
172    def _get_test_cases(self, executable, test_filter, subset):
173        """Gets the test cases for 'Pass', 'Fail' etc. expectations.
174
175        This function supports bootstrapping of new GPU families and dEQP
176        binaries. In particular if there are not 'Pass' expectations found for
177        this GPU family it will query the dEQP executable for a list of all
178        available tests. It will then remove known non-'Pass'ing tests from this
179        list to avoid getting into hangs/crashes etc.
180
181        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
182        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
183        @param subset: string from 'Pass', 'Fail', 'Timeout' etc.
184
185        @return: List of dEQP tests to run.
186        """
187        expectations_dir = os.path.join(self.bindir, 'expectations',
188                                        self._gpu_type)
189        subset_name = '%s.%s.bz2' % (test_filter, subset)
190        subset_path = os.path.join(expectations_dir, subset_name)
191        if not os.path.isfile(subset_path):
192            if subset != 'Pass':
193                raise error.TestError('No subset file found for %s!' %
194                                      subset_path)
195            return self._bootstrap_new_test_cases(executable, test_filter)
196
197        test_cases = bz2.BZ2File(subset_path).read().splitlines()
198        if not test_cases:
199            raise error.TestError('No test cases found in subset file %s!' %
200                                  subset_path)
201        return test_cases
202
203    def run_tests_individually(self, executable, test_cases):
204        """Runs tests as isolated from each other, but slowly.
205
206        This function runs each test case separately as a command.
207        This means a new context for each test etc. Failures will be more
208        isolated, but runtime quite high due to overhead.
209
210        @param executable: dEQP executable path.
211        @param test_cases: List of dEQP test case strings.
212
213        @return: dictionary of test results.
214        """
215        test_results = {}
216        width = self._width
217        height = self._height
218
219        log_path = os.path.join(tempfile.gettempdir(), '%s-logs' % self._filter)
220        shutil.rmtree(log_path, ignore_errors=True)
221        os.mkdir(log_path)
222
223        i = 0
224        for test_case in test_cases:
225            i += 1
226            logging.info('[%d/%d] TestCase: %s', i, len(test_cases), test_case)
227            log_file = '%s.log' % os.path.join(log_path, test_case)
228            command = ('%s '
229                       '--deqp-case=%s '
230                       '--deqp-surface-type=fbo '
231                       '--deqp-log-images=disable '
232                       '--deqp-watchdog=enable '
233                       '--deqp-surface-width=%d '
234                       '--deqp-surface-height=%d '
235                       '--deqp-log-filename=%s' %
236                       (executable, test_case, width, height, log_file))
237
238            try:
239                logging.info('Running single: %s', command)
240                utils.run(command,
241                          timeout=self._timeout,
242                          stderr_is_expected=False,
243                          ignore_status=True,
244                          stdout_tee=utils.TEE_TO_LOGS,
245                          stderr_tee=utils.TEE_TO_LOGS)
246                result_counts = self._parse_test_results(log_file)
247                if result_counts:
248                    result = result_counts.keys()[0]
249                else:
250                    result = 'Unknown'
251            except error.CmdTimeoutError:
252                result = 'TestTimeout'
253            except error.CmdError:
254                result = 'CommandFailed'
255            except Exception:
256                result = 'UnexpectedError'
257
258            logging.info('Result: %s', result)
259            test_results[result] = test_results.get(result, 0) + 1
260
261        return test_results
262
263    def run_tests_hasty(self, executable, test_cases):
264        """Runs tests as quickly as possible.
265
266        This function runs all the test cases, but does not isolate tests and
267        may take shortcuts/not run all tests to provide maximum coverage at
268        minumum runtime.
269
270        @param executable: dEQP executable path.
271        @param test_cases: List of dEQP test case strings.
272
273        @return: dictionary of test results.
274        """
275        # TODO(ihf): It saves half the test time to use 32*32 but a few tests
276        # fail as they need surfaces larger than 200*200.
277        width = self._width
278        height = self._height
279        results = {}
280        # All tests combined less than 1h in hasty.
281        batch_timeout = min(3600, self._timeout * self._hasty_batch_size)
282        num_test_cases = len(test_cases)
283        # We are trying to handle all errors by parsing the log file.
284        for batch in xrange(0, num_test_cases, self._hasty_batch_size):
285            batch_to = min(batch + self._hasty_batch_size, num_test_cases)
286            batch_cases = '\n'.join(test_cases[batch:batch_to])
287            command = ('%s '
288                       '--deqp-stdin-caselist '
289                       '--deqp-surface-type=fbo '
290                       '--deqp-log-images=disable '
291                       '--deqp-visibility=hidden '
292                       '--deqp-watchdog=enable '
293                       '--deqp-surface-width=%d '
294                       '--deqp-surface-height=%d ' % (executable, width,
295                                                      height))
296
297            log_file = os.path.join(tempfile.gettempdir(),
298                                    '%s_hasty_%d.log' % (self._filter, batch))
299            if os.path.exists(log_file):
300                os.remove(log_file)
301            command += '--deqp-log-filename=' + log_file
302            logging.info('Running tests %d...%d out of %d:\n%s\n%s', batch + 1,
303                         batch_to, num_test_cases, command, batch_cases)
304
305            try:
306                utils.run(command,
307                          timeout=batch_timeout,
308                          stderr_is_expected=False,
309                          ignore_status=False,
310                          stdin=batch_cases,
311                          stdout_tee=utils.TEE_TO_LOGS,
312                          stderr_tee=utils.TEE_TO_LOGS)
313            except Exception:
314                pass
315            results = self._parse_test_results(log_file, results)
316            logging.info(results)
317        return results
318
319    def run_once(self, opts=None):
320        options = dict(filter='',
321                       test_names='',  # e.g., dEQP-GLES3.info.version,
322                                       # dEQP-GLES2.functional,
323                                       # dEQP-GLES3.accuracy.texture, etc.
324                       timeout=self._timeout,
325                       subset_to_run='Pass',  # Pass, Fail, Timeout etc.
326                       hasty='False',
327                       shard=1,  # TODO(ihf): Support sharding for bvt-cq.
328                       shards=1)
329        if opts is None:
330            opts = []
331        options.update(utils.args_to_dict(opts))
332        logging.info('Test Options: %s', options)
333
334        self._hasty = (options['hasty'] == 'True')
335        self._timeout = int(options['timeout'])
336        self._test_names = options['test_names']
337        if not self._test_names:
338            self._filter = options['filter']
339            if not self._filter:
340                raise error.TestError('No dEQP test filter specified')
341
342        # Some information to help postprocess logs into blacklists later.
343        logging.info('ChromeOS BOARD = %s', self._board)
344        logging.info('ChromeOS CPU family = %s', self._cpu_type)
345        logging.info('ChromeOS GPU family = %s', self._gpu_type)
346        logging.info('dEQP test filter = %s', self._filter)
347
348        # Determine module from test_names or filter.
349        if self._test_names:
350            test_prefix = self._test_names.split('.')[0]
351            self._filter = '%s.filter_args' % test_prefix
352        elif self._filter:
353            test_prefix = self._filter.split('.')[0]
354        if test_prefix in self.DEQP_MODULES:
355            module = self.DEQP_MODULES[test_prefix]
356        elif self._test_names:
357            raise error.TestError('Invalid test names: %s' % self._test_names)
358        else:
359            raise error.TestError('Invalid test filter: %s' % self._filter)
360
361        executable_path = os.path.join(self.DEQP_BASEDIR, 'modules', module)
362        executable = os.path.join(executable_path, 'deqp-%s' % module)
363
364        self._services.stop_services()
365
366        # Must be in the executable directory when running for it to find it's
367        # test data files!
368        os.chdir(executable_path)
369        if self._test_names:
370            test_cases = []
371            for name in self._test_names.split(','):
372                test_cases.extend(self._get_test_cases(executable, name,
373                                                       'Pass'))
374        else:
375            test_cases = self._get_test_cases(executable, self._filter,
376                                              options['subset_to_run'])
377
378        test_results = {}
379        if self._hasty:
380            logging.info('Running in hasty mode.')
381            test_results = self.run_tests_hasty(executable, test_cases)
382        else:
383            logging.info('Running each test individually.')
384            test_results = self.run_tests_individually(executable, test_cases)
385
386        logging.info('Test results:')
387        logging.info(test_results)
388        self.write_perf_keyval(test_results)
389
390        test_count = 0
391        test_failures = 0
392        for result in test_results:
393            test_count += test_results[result]
394            if result.lower() not in ['pass', 'notsupported', 'internalerror']:
395                test_failures += test_results[result]
396        # The text "Completed all tests." is used by the process_log.py script
397        # and should always appear at the end of a completed test run.
398        logging.info('Completed all tests. Saw %d tests and %d failures.',
399                     test_count, test_failures)
400
401        if test_failures:
402            raise error.TestFail('%d/%d tests failed.' %
403                                 (test_failures, test_count))
404
405        if test_count == 0:
406            raise error.TestError('No test cases found for filter: %s!' %
407                                  self._filter)
408