1#!/usr/bin/env python
2
3"""
4    This script can generate XLS reports from OpenCV tests' XML output files.
5
6    To use it, first, create a directory for each machine you ran tests on.
7    Each such directory will become a sheet in the report. Put each XML file
8    into the corresponding directory.
9
10    Then, create your configuration file(s). You can have a global configuration
11    file (specified with the -c option), and per-sheet configuration files, which
12    must be called sheet.conf and placed in the directory corresponding to the sheet.
13    The settings in the per-sheet configuration file will override those in the
14    global configuration file, if both are present.
15
16    A configuration file must consist of a Python dictionary. The following keys
17    will be recognized:
18
19    * 'comparisons': [{'from': string, 'to': string}]
20        List of configurations to compare performance between. For each item,
21        the sheet will have a column showing speedup from configuration named
22        'from' to configuration named "to".
23
24    * 'configuration_matchers': [{'properties': {string: object}, 'name': string}]
25        Instructions for matching test run property sets to configuration names.
26
27        For each found XML file:
28
29        1) All attributes of the root element starting with the prefix 'cv_' are
30           placed in a dictionary, with the cv_ prefix stripped and the cv_module_name
31           element deleted.
32
33        2) The first matcher for which the XML's file property set contains the same
34           keys with equal values as its 'properties' dictionary is searched for.
35           A missing property can be matched by using None as the value.
36
37           Corollary 1: you should place more specific matchers before less specific
38           ones.
39
40           Corollary 2: an empty 'properties' dictionary matches every property set.
41
42        3) If a matching matcher is found, its 'name' string is presumed to be the name
43           of the configuration the XML file corresponds to. A warning is printed if
44           two different property sets match to the same configuration name.
45
46        4) If a such a matcher isn't found, if --include-unmatched was specified, the
47           configuration name is assumed to be the relative path from the sheet's
48           directory to the XML file's containing directory. If the XML file isinstance
49           directly inside the sheet's directory, the configuration name is instead
50           a dump of all its properties. If --include-unmatched wasn't specified,
51           the XML file is ignored and a warning is printed.
52
53    * 'configurations': [string]
54        List of names for compile-time and runtime configurations of OpenCV.
55        Each item will correspond to a column of the sheet.
56
57    * 'module_colors': {string: string}
58        Mapping from module name to color name. In the sheet, cells containing module
59        names from this mapping will be colored with the corresponding color. You can
60        find the list of available colors here:
61        <http://www.simplistix.co.uk/presentations/python-excel.pdf>.
62
63    * 'sheet_name': string
64        Name for the sheet. If this parameter is missing, the name of sheet's directory
65        will be used.
66
67    * 'sheet_properties': [(string, string)]
68        List of arbitrary (key, value) pairs that somehow describe the sheet. Will be
69        dumped into the first row of the sheet in string form.
70
71    Note that all keys are optional, although to get useful results, you'll want to
72    specify at least 'configurations' and 'configuration_matchers'.
73
74    Finally, run the script. Use the --help option for usage information.
75"""
76
77from __future__ import division
78
79import ast
80import errno
81import fnmatch
82import logging
83import numbers
84import os, os.path
85import re
86
87from argparse import ArgumentParser
88from glob import glob
89from itertools import ifilter
90
91import xlwt
92
93from testlog_parser import parseLogFile
94
95re_image_size = re.compile(r'^ \d+ x \d+$', re.VERBOSE)
96re_data_type = re.compile(r'^ (?: 8 | 16 | 32 | 64 ) [USF] C [1234] $', re.VERBOSE)
97
98time_style = xlwt.easyxf(num_format_str='#0.00')
99no_time_style = xlwt.easyxf('pattern: pattern solid, fore_color gray25')
100failed_style = xlwt.easyxf('pattern: pattern solid, fore_color red')
101noimpl_style = xlwt.easyxf('pattern: pattern solid, fore_color orange')
102style_dict = {"failed": failed_style, "noimpl":noimpl_style}
103
104speedup_style = time_style
105good_speedup_style = xlwt.easyxf('font: color green', num_format_str='#0.00')
106bad_speedup_style = xlwt.easyxf('font: color red', num_format_str='#0.00')
107no_speedup_style = no_time_style
108error_speedup_style = xlwt.easyxf('pattern: pattern solid, fore_color orange')
109header_style = xlwt.easyxf('font: bold true; alignment: horizontal centre, vertical top, wrap True')
110subheader_style = xlwt.easyxf('alignment: horizontal centre, vertical top')
111
112class Collector(object):
113    def __init__(self, config_match_func, include_unmatched):
114        self.__config_cache = {}
115        self.config_match_func = config_match_func
116        self.include_unmatched = include_unmatched
117        self.tests = {}
118        self.extra_configurations = set()
119
120    # Format a sorted sequence of pairs as if it was a dictionary.
121    # We can't just use a dictionary instead, since we want to preserve the sorted order of the keys.
122    @staticmethod
123    def __format_config_cache_key(pairs, multiline=False):
124        return (
125          ('{\n' if multiline else '{') +
126          (',\n' if multiline else ', ').join(
127             ('  ' if multiline else '') + repr(k) + ': ' + repr(v) for (k, v) in pairs) +
128          ('\n}\n' if multiline else '}')
129        )
130
131    def collect_from(self, xml_path, default_configuration):
132        run = parseLogFile(xml_path)
133
134        module = run.properties['module_name']
135
136        properties = run.properties.copy()
137        del properties['module_name']
138
139        props_key = tuple(sorted(properties.iteritems())) # dicts can't be keys
140
141        if props_key in self.__config_cache:
142            configuration = self.__config_cache[props_key]
143        else:
144            configuration = self.config_match_func(properties)
145
146            if configuration is None:
147                if self.include_unmatched:
148                    if default_configuration is not None:
149                        configuration = default_configuration
150                    else:
151                        configuration = Collector.__format_config_cache_key(props_key, multiline=True)
152
153                    self.extra_configurations.add(configuration)
154                else:
155                    logging.warning('failed to match properties to a configuration: %s',
156                        Collector.__format_config_cache_key(props_key))
157
158            else:
159                same_config_props = [it[0] for it in self.__config_cache.iteritems() if it[1] == configuration]
160                if len(same_config_props) > 0:
161                    logging.warning('property set %s matches the same configuration %r as property set %s',
162                        Collector.__format_config_cache_key(props_key),
163                        configuration,
164                        Collector.__format_config_cache_key(same_config_props[0]))
165
166            self.__config_cache[props_key] = configuration
167
168        if configuration is None: return
169
170        module_tests = self.tests.setdefault(module, {})
171
172        for test in run.tests:
173            test_results = module_tests.setdefault((test.shortName(), test.param()), {})
174            new_result = test.get("gmean") if test.status == 'run' else test.status
175            test_results[configuration] = min(
176              test_results.get(configuration), new_result,
177              key=lambda r: (1, r) if isinstance(r, numbers.Number) else
178                            (2,) if r is not None else
179                            (3,)
180            ) # prefer lower result; prefer numbers to errors and errors to nothing
181
182def make_match_func(matchers):
183    def match_func(properties):
184        for matcher in matchers:
185            if all(properties.get(name) == value
186                   for (name, value) in matcher['properties'].iteritems()):
187                return matcher['name']
188
189        return None
190
191    return match_func
192
193def main():
194    arg_parser = ArgumentParser(description='Build an XLS performance report.')
195    arg_parser.add_argument('sheet_dirs', nargs='+', metavar='DIR', help='directory containing perf test logs')
196    arg_parser.add_argument('-o', '--output', metavar='XLS', default='report.xls', help='name of output file')
197    arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file')
198    arg_parser.add_argument('--include-unmatched', action='store_true',
199        help='include results from XML files that were not recognized by configuration matchers')
200    arg_parser.add_argument('--show-times-per-pixel', action='store_true',
201        help='for tests that have an image size parameter, show per-pixel time, as well as total time')
202
203    args = arg_parser.parse_args()
204
205    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
206
207    if args.config is not None:
208        with open(args.config) as global_conf_file:
209            global_conf = ast.literal_eval(global_conf_file.read())
210    else:
211        global_conf = {}
212
213    wb = xlwt.Workbook()
214
215    for sheet_path in args.sheet_dirs:
216        try:
217            with open(os.path.join(sheet_path, 'sheet.conf')) as sheet_conf_file:
218                sheet_conf = ast.literal_eval(sheet_conf_file.read())
219        except IOError as ioe:
220            if ioe.errno != errno.ENOENT: raise
221            sheet_conf = {}
222            logging.debug('no sheet.conf for %s', sheet_path)
223
224        sheet_conf = dict(global_conf.items() + sheet_conf.items())
225
226        config_names = sheet_conf.get('configurations', [])
227        config_matchers = sheet_conf.get('configuration_matchers', [])
228
229        collector = Collector(make_match_func(config_matchers), args.include_unmatched)
230
231        for root, _, filenames in os.walk(sheet_path):
232            logging.info('looking in %s', root)
233            for filename in fnmatch.filter(filenames, '*.xml'):
234                if os.path.normpath(sheet_path) == os.path.normpath(root):
235                  default_conf = None
236                else:
237                  default_conf = os.path.relpath(root, sheet_path)
238                collector.collect_from(os.path.join(root, filename), default_conf)
239
240        config_names.extend(sorted(collector.extra_configurations - set(config_names)))
241
242        sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path))))
243
244        sheet_properties = sheet_conf.get('sheet_properties', [])
245
246        sheet.write(0, 0, 'Properties:')
247
248        sheet.write(0, 1,
249          'N/A' if len(sheet_properties) == 0 else
250          ' '.join(str(k) + '=' + repr(v) for (k, v) in sheet_properties))
251
252        sheet.row(2).height = 800
253        sheet.panes_frozen = True
254        sheet.remove_splits = True
255
256        sheet_comparisons = sheet_conf.get('comparisons', [])
257
258        row = 2
259
260        col = 0
261
262        for (w, caption) in [
263                (2500, 'Module'),
264                (10000, 'Test'),
265                (2000, 'Image\nwidth'),
266                (2000, 'Image\nheight'),
267                (2000, 'Data\ntype'),
268                (7500, 'Other parameters')]:
269            sheet.col(col).width = w
270            if args.show_times_per_pixel:
271                sheet.write_merge(row, row + 1, col, col, caption, header_style)
272            else:
273                sheet.write(row, col, caption, header_style)
274            col += 1
275
276        for config_name in config_names:
277            if args.show_times_per_pixel:
278                sheet.col(col).width = 3000
279                sheet.col(col + 1).width = 3000
280                sheet.write_merge(row, row, col, col + 1, config_name, header_style)
281                sheet.write(row + 1, col, 'total, ms', subheader_style)
282                sheet.write(row + 1, col + 1, 'per pixel, ns', subheader_style)
283                col += 2
284            else:
285                sheet.col(col).width = 4000
286                sheet.write(row, col, config_name, header_style)
287                col += 1
288
289        col += 1 # blank column between configurations and comparisons
290
291        for comp in sheet_comparisons:
292            sheet.col(col).width = 4000
293            caption = comp['to'] + '\nvs\n' + comp['from']
294            if args.show_times_per_pixel:
295                sheet.write_merge(row, row + 1, col, col, caption, header_style)
296            else:
297                sheet.write(row, col, caption, header_style)
298            col += 1
299
300        row += 2 if args.show_times_per_pixel else 1
301
302        sheet.horz_split_pos = row
303        sheet.horz_split_first_visible = row
304
305        module_colors = sheet_conf.get('module_colors', {})
306        module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color))
307                         for module, color in module_colors.iteritems()}
308
309        for module, tests in sorted(collector.tests.iteritems()):
310            for ((test, param), configs) in sorted(tests.iteritems()):
311                sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style))
312                sheet.write(row, 1, test)
313
314                param_list = param[1:-1].split(', ') if param.startswith('(') and param.endswith(')') else [param]
315
316                image_size = next(ifilter(re_image_size.match, param_list), None)
317                if image_size is not None:
318                    (image_width, image_height) = map(int, image_size.split('x', 1))
319                    sheet.write(row, 2, image_width)
320                    sheet.write(row, 3, image_height)
321                    del param_list[param_list.index(image_size)]
322
323                data_type = next(ifilter(re_data_type.match, param_list), None)
324                if data_type is not None:
325                    sheet.write(row, 4, data_type)
326                    del param_list[param_list.index(data_type)]
327
328                sheet.row(row).write(5, ' | '.join(param_list))
329
330                col = 6
331
332                for c in config_names:
333                    if c in configs:
334                        sheet.write(row, col, configs[c], style_dict.get(configs[c], time_style))
335                    else:
336                        sheet.write(row, col, None, no_time_style)
337                    col += 1
338                    if args.show_times_per_pixel:
339                        sheet.write(row, col,
340                          xlwt.Formula('{0} * 1000000 / ({1} * {2})'.format(
341                              xlwt.Utils.rowcol_to_cell(row, col - 1),
342                              xlwt.Utils.rowcol_to_cell(row, 2),
343                              xlwt.Utils.rowcol_to_cell(row, 3)
344                          )),
345                          time_style
346                        )
347                        col += 1
348
349                col += 1 # blank column
350
351                for comp in sheet_comparisons:
352                    cmp_from = configs.get(comp["from"])
353                    cmp_to = configs.get(comp["to"])
354
355                    if isinstance(cmp_from, numbers.Number) and isinstance(cmp_to, numbers.Number):
356                        try:
357                            speedup = cmp_from / cmp_to
358                            sheet.write(row, col, speedup, good_speedup_style if speedup > 1.1 else
359                                                           bad_speedup_style  if speedup < 0.9 else
360                                                           speedup_style)
361                        except ArithmeticError as e:
362                            sheet.write(row, col, None, error_speedup_style)
363                    else:
364                        sheet.write(row, col, None, no_speedup_style)
365
366                    col += 1
367
368                row += 1
369                if row % 1000 == 0: sheet.flush_row_data()
370
371    wb.save(args.output)
372
373if __name__ == '__main__':
374    main()
375