graphics_dEQP.py revision 6a0b82bdd5f69d093448934ba0db14bcbfbc0e3d 
1# Copyright 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import bz2
6import glob
7import logging
8import os
9import shutil
10import tempfile
11import xml.etree.cElementTree as et
12from autotest_lib.client.bin import test, utils
13from autotest_lib.client.common_lib import error
14from autotest_lib.client.cros import service_stopper
15
16
17class graphics_dEQP(test.test):
18    """Run the drawElements Quality Program test suite.
19    """
20    version = 1
21    _services = None
22    _hasty = False
23    _hasty_batch_size = 100  # Batch size in hasty mode.
24    _board = None
25    _cpu_type = None
26    _gpu_type = None
27    _filter = None
28    _width = 256  # Use smallest width for which all tests run/pass.
29    _height = 256  # Use smallest height for which all tests run/pass.
30    _timeout = 70  # Larger than twice the dEQP watchdog timeout at 30s.
31
32    DEQP_BASEDIR = '/usr/local/deqp'
33    DEQP_MODULES = {
34        'dEQP-EGL':    'egl',
35        'dEQP-GLES2':  'gles2',
36        'dEQP-GLES3':  'gles3',
37        'dEQP-GLES31': 'gles31'
38    }
39
40    def initialize(self):
41        self._gpu_type = utils.get_gpu_family()
42        self._cpu_type = utils.get_cpu_soc_family()
43        self._board = utils.get_board()
44        self._services = service_stopper.ServiceStopper(['ui', 'powerd'])
45
46    def cleanup(self):
47        if self._services:
48            self._services.restore_services()
49
50    def _parse_test_results(self, result_filename, test_results={}):
51        """Handles result files with one or more test results.
52
53        @param result_filename: log file to parse.
54
55        @return: dictionary of parsed test results.
56        """
57        xml = ''
58        xml_start = False
59        xml_complete = False
60        xml_bad = False
61        result = 'ParseTestResultFail'
62
63        if not os.path.isfile(result_filename):
64            return test_results
65        # TODO(ihf): Add names of failing tests to a list in the results.
66        with open(result_filename) as result_file:
67            for line in result_file.readlines():
68                # If the test terminates early, the XML will be incomplete
69                # and should not be parsed.
70                if line.startswith('#terminateTestCaseResult'):
71                    result = line.strip().split()[1]
72                    xml_bad = True
73                # Will only see #endTestCaseResult if the test does not
74                # terminate early.
75                elif line.startswith('#endTestCaseResult'):
76                    xml_complete = True
77                elif xml_start:
78                    xml += line
79                elif line.startswith('#beginTestCaseResult'):
80                    # If we see another begin before an end then something is
81                    # wrong.
82                    if xml_start:
83                        xml_bad = True
84                    else:
85                        xml_start = True
86
87                if xml_complete or xml_bad:
88                    if xml_complete:
89                        root = et.fromstring(xml)
90                        result = root.find('Result').get('StatusCode').strip()
91                        xml_complete = False
92                    test_results[result] = test_results.get(result, 0) + 1
93                    xml_bad = False
94                    xml_start = False
95                    result = 'ParseTestResultFail'
96                    xml = ''
97
98        return test_results
99
100    def _bootstrap_new_test_cases(self, executable, test_filter):
101        """Ask dEQP for all test cases and removes non-Pass'ing ones.
102
103        This function assumes that the '*.Pass' file does not exist and that
104        everything else found in the directory should be removed from the tests
105        to run. This can be used incrementally updating failing/hangin tests
106        over several runs.
107
108        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
109        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
110
111        @return: List of dEQP tests to run.
112        """
113        test_cases = []
114        not_passing_cases = []
115        # We did not find passing cases in expectations. Assume everything else
116        # that is there should not be run this time.
117        expectations_dir = os.path.join(self.bindir, 'expectations',
118                                        self._gpu_type)
119        subset_spec = '%s.*' % test_filter
120        subset_paths = glob.glob(os.path.join(expectations_dir, subset_spec))
121        for subset_file in subset_paths:
122            # Filter against hasty failures only in hasty mode.
123            if self._hasty or not '.hasty.bz2' in subset_file:
124                not_passing_cases.extend(
125                    bz2.BZ2File(subset_file).read().splitlines())
126
127        # Now ask dEQP executable nicely for whole list of tests. Needs to be
128        # run in executable directory. Output file is plain text file named
129        # e.g. 'dEQP-GLES2-cases.txt'.
130        command = ('%s '
131                   '--deqp-runmode=txt-caselist '
132                   '--deqp-surface-type=fbo ' % executable)
133        logging.info('Running command %s', command)
134        utils.run(command,
135                  timeout=60,
136                  stderr_is_expected=False,
137                  ignore_status=False,
138                  stdin=None,
139                  stdout_tee=utils.TEE_TO_LOGS,
140                  stderr_tee=utils.TEE_TO_LOGS)
141
142        # Now read this caselist file.
143        caselist_name = '%s-cases.txt' % test_filter.split('.')[0]
144        caselist_file = os.path.join(os.path.dirname(executable), caselist_name)
145        if not os.path.isfile(caselist_file):
146            raise error.TestError('No caselist file at %s!' % caselist_file)
147
148        # And remove non-Pass'ing expectations from caselist.
149        caselist = open(caselist_file).read().splitlines()
150        # Contains lines like "TEST: dEQP-GLES2.capability"
151        test_cases = []
152        match = 'TEST: %s' % test_filter
153        logging.info('Bootstrapping test cases matching "%s".', match)
154        for case in caselist:
155            if case.startswith(match):
156                test = case.split('TEST: ')[1]
157                test_cases.append(test)
158
159        test_cases = list(set(test_cases) - set(not_passing_cases))
160        if not test_cases:
161            raise error.TestError('Unable to bootstrap %s!' % test_filter)
162        # TODO(ihf): Sorting is nice but can introduce errors due to changed
163        # ordering and state leakage. Consider deleting this if it causes
164        # too much problems.
165        test_cases.sort()
166        return test_cases
167
168    def _get_test_cases(self, executable, test_filter, subset):
169        """Gets the test cases for 'Pass', 'Fail' etc. expectations.
170
171        This function supports bootstrapping of new GPU families and dEQP
172        binaries. In particular if there are not 'Pass' expectations found for
173        this GPU family it will query the dEQP executable for a list of all
174        available tests. It will then remove known non-'Pass'ing tests from this
175        list to avoid getting into hangs/crashes etc.
176
177        @param executable: location '/usr/local/deqp/modules/gles2/deqp-gles2'.
178        @param test_filter: string like 'dEQP-GLES2.info', 'dEQP-GLES3.stress'.
179        @param subset: string from 'Pass', 'Fail', 'Timeout' etc.
180
181        @return: List of dEQP tests to run.
182        """
183        expectations_dir = os.path.join(self.bindir, 'expectations',
184                                        self._gpu_type)
185        subset_name = '%s.%s.bz2' % (test_filter, subset)
186        subset_path = os.path.join(expectations_dir, subset_name)
187        if not os.path.isfile(subset_path):
188            if subset != 'Pass':
189                raise error.TestError('No subset file found for %s!' %
190                                      subset_path)
191            return self._bootstrap_new_test_cases(executable, test_filter)
192
193        test_cases = bz2.BZ2File(subset_path).read().splitlines()
194        if not test_cases:
195            raise error.TestError('No test cases found in subset file %s!' %
196                                  subset_path)
197        return test_cases
198
199    def run_tests_individually(self, executable, test_cases):
200        """Runs tests as isolated from each other, but slowly.
201
202        This function runs each test case separately as a command.
203        This means a new context for each test etc. Failures will be more
204        isolated, but runtime quite high due to overhead.
205
206        @param executable: dEQP executable path.
207        @param test_cases: List of dEQP test case strings.
208
209        @return: dictionary of test results.
210        """
211        test_results = {}
212        width = self._width
213        height = self._height
214
215        log_path = os.path.join(tempfile.gettempdir(), '%s-logs' % self._filter)
216        shutil.rmtree(log_path, ignore_errors=True)
217        os.mkdir(log_path)
218
219        i = 0
220        for test_case in test_cases:
221            i += 1
222            logging.info('[%d/%d] TestCase: %s', i, len(test_cases), test_case)
223            log_file = '%s.log' % os.path.join(log_path, test_case)
224            command = ('%s '
225                       '--deqp-case=%s '
226                       '--deqp-surface-type=fbo '
227                       '--deqp-log-images=disable '
228                       '--deqp-watchdog=enable '
229                       '--deqp-surface-width=%d '
230                       '--deqp-surface-height=%d '
231                       '--deqp-log-filename=%s' %
232                       (executable, test_case, width, height, log_file))
233
234            try:
235                logging.info('Running single: %s', command)
236                utils.run(command,
237                          timeout=self._timeout,
238                          stderr_is_expected=False,
239                          ignore_status=True,
240                          stdout_tee=utils.TEE_TO_LOGS,
241                          stderr_tee=utils.TEE_TO_LOGS)
242                result_counts = self._parse_test_results(log_file)
243                if result_counts:
244                    result = result_counts.keys()[0]
245                else:
246                    result = 'Unknown'
247            except error.CmdTimeoutError:
248                result = 'TestTimeout'
249            except error.CmdError:
250                result = 'CommandFailed'
251            except:
252                result = 'UnexpectedError'
253
254            logging.info('Result: %s', result)
255            test_results[result] = test_results.get(result, 0) + 1
256
257        return test_results
258
259    def run_tests_hasty(self, executable, test_cases):
260        """Runs tests as quickly as possible.
261
262        This function runs all the test cases, but does not isolate tests and
263        may take shortcuts/not run all tests to provide maximum coverage at
264        minumum runtime.
265
266        @param executable: dEQP executable path.
267        @param test_cases: List of dEQP test case strings.
268
269        @return: dictionary of test results.
270        """
271        # TODO(ihf): It saves half the test time to use 32*32 but a few tests
272        # fail as they need surfaces larger than 200*200.
273        width = self._width
274        height = self._height
275        results = {}
276        # All tests combined less than 1h in hasty.
277        batch_timeout = min(3600, self._timeout * self._hasty_batch_size)
278        num_test_cases = len(test_cases)
279        # We are trying to handle all errors by parsing the log file.
280        for batch in xrange(0, num_test_cases, self._hasty_batch_size):
281            batch_to = min(batch + self._hasty_batch_size, num_test_cases)
282            batch_cases = '\n'.join(test_cases[batch:batch_to])
283            command = ('%s '
284                       '--deqp-stdin-caselist '
285                       '--deqp-surface-type=fbo '
286                       '--deqp-log-images=disable '
287                       '--deqp-visibility=hidden '
288                       '--deqp-watchdog=enable '
289                       '--deqp-surface-width=%d '
290                       '--deqp-surface-height=%d ' % (executable, width, height))
291
292            log_file = os.path.join(tempfile.gettempdir(),
293                                    '%s_hasty_%d.log' % (self._filter, batch))
294            if os.path.exists(log_file):
295                os.remove(log_file)
296            command += '--deqp-log-filename=' + log_file
297            logging.info('Running tests %d...%d out of %d:\n%s\n%s', batch + 1,
298                         batch_to, num_test_cases, command, batch_cases)
299
300            try:
301                utils.run(command,
302                          timeout=batch_timeout,
303                          stderr_is_expected=False,
304                          ignore_status=False,
305                          stdin=batch_cases,
306                          stdout_tee=utils.TEE_TO_LOGS,
307                          stderr_tee=utils.TEE_TO_LOGS)
308            except:
309                pass
310            results = self._parse_test_results(log_file, results)
311            logging.info(results)
312        return results
313
314    def run_once(self, opts=[]):
315        options = dict(filter='',
316                       timeout=self._timeout,
317                       subset_to_run='Pass',  # Pass, Fail, Timeout etc.
318                       hasty='False',
319                       shard=1,  # TODO(ihf): Support sharding for bvt-cq.
320                       shards=1)
321        options.update(utils.args_to_dict(opts))
322        logging.info('Test Options: %s', options)
323
324        self._hasty = (options['hasty'] == 'True')
325        self._timeout = int(options['timeout'])
326        self._filter = options['filter']
327        if not self._filter:
328            raise error.TestError('No dEQP test filter specified')
329
330        # Some information to help postprocess logs into blacklists later.
331        logging.info('ChromeOS BOARD = %s', self._board)
332        logging.info('ChromeOS CPU family = %s', self._cpu_type)
333        logging.info('ChromeOS GPU family = %s', self._gpu_type)
334        logging.info('dEQP test filter = %s', self._filter)
335
336        # Determine module from filter.
337        test_prefix, test_group = self._filter.split('.')
338        if test_prefix in self.DEQP_MODULES:
339            module = self.DEQP_MODULES[test_prefix]
340        else:
341            raise error.TestError('Invalid test filter: %s' % self._filter)
342
343        executable_path = os.path.join(self.DEQP_BASEDIR, 'modules', module)
344        executable = os.path.join(executable_path, 'deqp-%s' % module)
345
346        self._services.stop_services()
347
348        # Must be in the executable directory when running for it to find it's
349        # test data files!
350        os.chdir(executable_path)
351        test_cases = self._get_test_cases(executable, self._filter,
352                                          options['subset_to_run'])
353
354        test_results = {}
355        if self._hasty:
356            logging.info('Running in hasty mode.')
357            test_results = self.run_tests_hasty(executable, test_cases)
358        else:
359            logging.info('Running each test individually.')
360            test_results = self.run_tests_individually(executable, test_cases)
361
362        logging.info('Test results:')
363        logging.info(test_results)
364        self.write_perf_keyval(test_results)
365
366        test_count = 0
367        test_failures = 0
368        for result in test_results:
369            test_count += test_results[result]
370            if result.lower() not in ['pass', 'notsupported', 'internalerror']:
371                test_failures += test_results[result]
372        # The text "Completed all tests." is used by the process_log.py script
373        # and should always appear at the end of a completed test run.
374        logging.info('Completed all tests. Saw %d tests and %d failures.',
375                     test_count, test_failures)
376
377        if test_failures:
378            raise error.TestFail('%d/%d tests failed.' %
379                                 (test_failures, test_count))
380
381        if test_count == 0:
382            raise error.TestError('No test cases found for filter: %s!' %
383                                  self._filter)
384