1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Main function to run the layout test analyzer.
7
8The purpose of this script is to run the layout test analyzer for various
9teams based on the run configuration file in CSV format. The CSV file is based
10on https://sites.google.com/a/chromium.org/dev/developers/testing/
11webkit-layout-tests/layout-test-stats-1.
12"""
13
14import optparse
15import os
16import shutil
17from subprocess import Popen
18
19# TODO(shadi): Re-examine the need of external files. Inline data instead?
20DEFAULT_RUN_CONFIG = {
21    # test_group_name: ('test_files.csv', 'report_email_address')
22    'media': ('testname/media.csv', 'layout-test-analyzer-result@google.com')
23}
24
25# Predefined result/graph directory.
26DEFAULT_RESULT_DIR = 'result'
27DEFAULT_GRAPH_DIR = 'graph'
28
29
30def ParseOption():
31  """Parse command-line options using OptionParser.
32
33  Returns:
34      an object containing all command-line option information.
35  """
36  option_parser = optparse.OptionParser()
37  option_parser.add_option('-d', '--result-directory-location',
38                           dest='result_directory_location',
39                           help=('Name of result directory location '
40                                 '(default to %default)'),
41                           default=DEFAULT_RESULT_DIR)
42  option_parser.add_option('-p', '--graph-directory-location',
43                           dest='graph_directory_location',
44                           help=('Name of graph directory location '
45                                 '(default to %default)'),
46                           default=DEFAULT_GRAPH_DIR)
47  option_parser.add_option('-e', '--email-only-change-mode',
48                           dest='email_only_change_mode',
49                           help=('With this mode, email is sent out '
50                                 'only when there is a change in the '
51                                 'analyzer result compared to the previous '
52                                 'result (off by default)'),
53                           action='store_true', default=False)
54  option_parser.add_option('-z', '--issue-detail-mode',
55                           dest='issue_detail_mode',
56                           help=('With this mode, email includes issue details'
57                                 ' including links to the flakiness dashboard'
58                                 ' (off by default)'),
59                           action='store_true', default=False)
60  return option_parser.parse_args()[0]
61
62
63def GenerateDashboardHTMLFile(file_name, test_group_list):
64  """Generate dashboard HTML file.
65
66  Currently, it is simple table that shows all the analyzer results.
67
68  Args:
69    file_name: the file name of the dashboard.
70    test_group_list: a list of test group names such as 'media' or 'composite'.
71  """
72  file_object = open(file_name, 'wb')
73  legend_txt = """
74<style type="text/css">
75th {
76  width: 30px; overflow: hidden;
77}
78tr.d0 td {
79  background-color: #CC9999; color: black;
80  text-align: right;
81  width: 30px; overflow: hidden;
82}
83tr.d1 td {
84  background-color: #9999CC; color: black;
85  text-align: right;
86  width: 30px; overflow: hidden;
87}
88</style>
89<h2>Chromium Layout Test Analyzer Result</h2>
90Legend:
91<ul>
92<li>#Tests: the number of tests for the given test group
93<li>#Skipped Tests: the number of tests that are skipped in the
94<a href='http://svn.webkit.org/repository/webkit/trunk/LayoutTests/platform/\
95chromium/test_expectations.txt'>test expectaion file</a> (e.g., BUGWK60877
96SKIP : loader/navigation-while-deferring-loads.html = FAIL)
97<li>#Non-Skipped Failing Tests: the number of tests that appeared in the
98test expectation file and were not skipped.
99<li>Failing rate: #NonSkippedFailing / (#Tests - #Skipped)
100<li>Passing rate: 100 - (Failing rate)
101</ul>
102  """
103  file_object.write(legend_txt)
104  file_object.write('<table border="1">')
105  file_object.write('<tr><th>Base Directory</th>')
106  file_object.write('<th>Trend Graph</th>')
107  file_object.write('<th>#Tests</th>')
108  file_object.write('<th>#Skipped Tests</th>')
109  file_object.write('<th>#Non-Skipped Failing Tests</th>')
110  file_object.write('<th>Failing Rate</th>')
111  file_object.write('<th>Passing Rate</th>')
112  file_object.write('<th>Last Revision Number</th>')
113  file_object.write('<th>Last Revision Date</th>')
114  file_object.write('<th>Owner Email</th>')
115  file_object.write('<th>Bug Information</th></tr>\n')
116  test_group_list.sort()
117  for i, test_group in enumerate(test_group_list):
118    file_object.write('<tr class="d' + str(i % 2) + '">\n')
119    file_object.write('<td>' + test_group + '</td>\n')
120    file_object.write('</tr>\n')
121  file_object.write('</table>')
122  file_object.close()
123
124
125# TODO(shadi): Use only one file with main()! Remove this file in favor of
126# layouttest_analyzer.py main().
127def main():
128  """A main function for the analyzer runner."""
129  options = ParseOption()
130  run_config_map = DEFAULT_RUN_CONFIG
131  test_group_list = run_config_map.keys()
132  dashboard_file_location = os.path.join(options.graph_directory_location,
133                                         'index.html')
134  if not os.path.exists(dashboard_file_location):
135    GenerateDashboardHTMLFile(dashboard_file_location, test_group_list)
136  for test_group in test_group_list:
137    # Prepare the result if it does not exist.
138    # The directory name should be changed to avoid collision
139    # with the file separator.
140    test_group_name_for_data = test_group.replace('/', '_')
141    result_dir = os.path.join(options.result_directory_location,
142                              test_group_name_for_data)
143    if not os.path.exists(result_dir):
144      os.mkdir(result_dir)
145    graph_file = os.path.join(options.graph_directory_location,
146                              test_group_name_for_data + '.html')
147    if not os.path.exists(graph_file):
148      # Copy the template file.
149      shutil.copy(os.path.join('graph', 'graph.html'),
150                  graph_file)
151      os.chmod(graph_file, 0744)
152    cmd = ('python layouttest_analyzer.py -x %s -d %s -t %s'
153           ' -q %s ') % (
154               test_group, result_dir, graph_file, dashboard_file_location)
155    if run_config_map[test_group][0]:
156      cmd += '-n ' + run_config_map[test_group][0] + ' '
157    if run_config_map[test_group][1]:
158      cmd += '-r ' + run_config_map[test_group][1] + ' '
159    if options.email_only_change_mode:
160      cmd += ' -c '
161    if options.issue_detail_mode:
162      cmd += ' -z '
163    print 'Running ' + cmd
164    proc = Popen(cmd, shell=True)
165    proc.communicate()
166
167
168if '__main__' == __name__:
169  main()
170