1#!/bin/env python
2# Copyright (c) 2011 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6
7"""Module to setup and generate code coverage data
8
9This module first sets up the environment for code coverage, instruments the
10binaries, runs the tests and collects the code coverage data.
11
12
13Usage:
14  coverage.py --upload=<upload_location>
15              --revision=<revision_number>
16              --src_root=<root_of_source_tree>
17              [--tools_path=<tools_path>]
18"""
19
20import logging
21import optparse
22import os
23import shutil
24import subprocess
25import sys
26import tempfile
27
28import google.logging_utils
29import google.process_utils as proc
30
31
32# The list of binaries that will be instrumented for code coverage
33# TODO(niranjan): Re-enable instrumentation of chrome.exe and chrome.dll once we
34# resolve the issue where vsinstr.exe is confused while reading symbols.
35windows_binaries = [#'chrome.exe',
36                    #'chrome.dll',
37                    'unit_tests.exe',
38                    'automated_ui_tests.exe',
39                    'installer_util_unittests.exe',
40                    'ipc_tests.exe',
41                    'memory_test.exe',
42                    'page_cycler_tests.exe',
43                    'perf_tests.exe',
44                    'reliability_tests.exe',
45                    'security_tests.dll',
46                    'startup_tests.exe',
47                    'tab_switching_test.exe',
48                    'test_shell.exe']
49
50# The list of [tests, args] that will be run.
51# Failing tests have been commented out.
52# TODO(niranjan): Need to add layout tests that excercise the test shell.
53windows_tests = [
54                 ['unit_tests.exe', ''],
55#                 ['automated_ui_tests.exe', ''],
56                 ['installer_util_unittests.exe', ''],
57                 ['ipc_tests.exe', ''],
58                 ['page_cycler_tests.exe', '--gtest_filter=*File --no-sandbox'],
59                 ['reliability_tests.exe', '--no-sandbox'],
60                 ['startup_tests.exe', '--no-sandbox'],
61                 ['tab_switching_test.exe', '--no-sandbox'],
62                ]
63
64
65def IsWindows():
66  """Checks if the current platform is Windows.
67  """
68  return sys.platform[:3] == 'win'
69
70
71class Coverage(object):
72  """Class to set up and generate code coverage.
73
74  This class contains methods that are useful to set up the environment for
75  code coverage.
76
77  Attributes:
78    instrumented: A boolean indicating if all the binaries have been
79                  instrumented.
80  """
81
82  def __init__(self,
83               revision,
84               src_path = None,
85               tools_path = None,
86               archive=None):
87    """Init method for the Coverage class.
88
89    Args:
90      revision: Revision number of the Chromium source tree.
91      src_path: Location of the Chromium source base.
92      tools_path: Location of the Visual Studio Team Tools. (Win32 only)
93      archive: Archive location for the intermediate .coverage results.
94    """
95    google.logging_utils.config_root()
96    self.revision = revision
97    self.instrumented = False
98    self.tools_path = tools_path
99    self.src_path = src_path
100    self._dir = tempfile.mkdtemp()
101    self._archive = archive
102
103  def SetUp(self, binaries):
104    """Set up the platform specific environment and instrument the binaries for
105    coverage.
106
107    This method sets up the environment, instruments all the compiled binaries
108    and sets up the code coverage counters.
109
110    Args:
111      binaries: List of binaries that need to be instrumented.
112
113    Returns:
114      True on success.
115      False on error.
116    """
117    if self.instrumented:
118      logging.error('Binaries already instrumented')
119      return False
120    if IsWindows():
121      # Stop all previous instance of VSPerfMon counters
122      counters_command = ('%s -shutdown' %
123                          (os.path.join(self.tools_path, 'vsperfcmd.exe')))
124      (retcode, output) = proc.RunCommandFull(counters_command,
125                                              collect_output=True)
126      # TODO(niranjan): Add a check that to verify that the binaries were built
127      # using the /PROFILE linker flag.
128      if self.tools_path == None:
129        logging.error('Could not locate Visual Studio Team Server tools')
130        return False
131      # Remove trailing slashes
132      self.tools_path = self.tools_path.rstrip('\\')
133      # Add this to the env PATH.
134      os.environ['PATH'] = os.environ['PATH'] + ';' + self.tools_path
135      instrument_command = '%s /COVERAGE ' % (os.path.join(self.tools_path,
136                                                           'vsinstr.exe'))
137      for binary in binaries:
138        logging.info('binary = %s' % (binary))
139        logging.info('instrument_command = %s' % (instrument_command))
140        # Instrument each binary in the list
141        binary = os.path.join(self.src_path, 'chrome', 'Release', binary)
142        (retcode, output) = proc.RunCommandFull(instrument_command + binary,
143                                                collect_output=True)
144        # Check if the file has been instrumented correctly.
145        if output.pop().rfind('Successfully instrumented') == -1:
146          logging.error('Error instrumenting %s' % (binary))
147          return False
148      # We are now ready to run tests and measure code coverage.
149      self.instrumented = True
150      return True
151
152  def TearDown(self):
153    """Tear down method.
154
155    This method shuts down the counters, and cleans up all the intermediate
156    artifacts.
157    """
158    if self.instrumented == False:
159      return
160
161    if IsWindows():
162      # Stop counters
163      counters_command = ('%s -shutdown' %
164                         (os.path.join(self.tools_path, 'vsperfcmd.exe')))
165      (retcode, output) = proc.RunCommandFull(counters_command,
166                                              collect_output=True)
167      logging.info('Counters shut down: %s' % (output))
168      # TODO(niranjan): Revert the instrumented binaries to their original
169      # versions.
170    else:
171      return
172    if self._archive:
173      shutil.copytree(self._dir, os.path.join(self._archive, self.revision))
174      logging.info('Archived the .coverage files')
175    # Delete all the temp files and folders
176    if self._dir != None:
177      shutil.rmtree(self._dir, ignore_errors=True)
178      logging.info('Cleaned up temporary files and folders')
179    # Reset the instrumented flag.
180    self.instrumented = False
181
182  def RunTest(self, src_root, test):
183    """Run tests and collect the .coverage file
184
185    Args:
186      src_root: Path to the root of the source.
187      test: Path to the test to be run.
188
189    Returns:
190      Path of the intermediate .coverage file on success.
191      None on error.
192    """
193    # Generate the intermediate file name for the coverage results
194    test_name = os.path.split(test[0])[1].strip('.exe')
195    # test_command = binary + args
196    test_command = '%s %s' % (os.path.join(src_root,
197                                           'chrome',
198                                           'Release',
199                                           test[0]),
200                              test[1])
201
202    coverage_file = os.path.join(self._dir, '%s_win32_%s.coverage' %
203                                            (test_name, self.revision))
204    logging.info('.coverage file for test %s: %s' % (test_name, coverage_file))
205
206    # After all the binaries have been instrumented, we start the counters.
207    counters_command = ('%s -start:coverage -output:%s' %
208                        (os.path.join(self.tools_path, 'vsperfcmd.exe'),
209                         coverage_file))
210    # Here we use subprocess.call() instead of the RunCommandFull because the
211    # VSPerfCmd spawns another process before terminating and this confuses
212    # the subprocess.Popen() used by RunCommandFull.
213    retcode = subprocess.call(counters_command)
214
215    # Run the test binary
216    logging.info('Executing test %s: ' % test_command)
217    (retcode, output) = proc.RunCommandFull(test_command, collect_output=True)
218    if retcode != 0: # Return error if the tests fail
219      logging.error('One or more tests failed in %s.' % test_command)
220      return None
221
222    # Stop the counters
223    counters_command = ('%s -shutdown' %
224                        (os.path.join(self.tools_path, 'vsperfcmd.exe')))
225    (retcode, output) = proc.RunCommandFull(counters_command,
226                                            collect_output=True)
227    logging.info('Counters shut down: %s' % (output))
228    # Return the intermediate .coverage file
229    return coverage_file
230
231  def Upload(self, list_coverage, upload_path, sym_path=None, src_root=None):
232    """Upload the results to the dashboard.
233
234    This method uploads the coverage data to a dashboard where it will be
235    processed. On Windows, this method will first convert the .coverage file to
236    the lcov format. This method needs to be called before the TearDown method.
237
238    Args:
239      list_coverage: The list of coverage data files to consoliate and upload.
240      upload_path: Destination where the coverage data will be processed.
241      sym_path: Symbol path for the build (Win32 only)
242      src_root: Root folder of the source tree (Win32 only)
243
244    Returns:
245      True on success.
246      False on failure.
247    """
248    if upload_path == None:
249      logging.info('Upload path not specified. Will not convert to LCOV')
250      return True
251
252    if IsWindows():
253      # Stop counters
254      counters_command = ('%s -shutdown' %
255                          (os.path.join(self.tools_path, 'vsperfcmd.exe')))
256      (retcode, output) = proc.RunCommandFull(counters_command,
257                                              collect_output=True)
258      logging.info('Counters shut down: %s' % (output))
259      lcov_file = os.path.join(upload_path, 'chrome_win32_%s.lcov' %
260                                            (self.revision))
261      lcov = open(lcov_file, 'w')
262      for coverage_file in list_coverage:
263        # Convert the intermediate .coverage file to lcov format
264        if self.tools_path == None:
265          logging.error('Lcov converter tool not found')
266          return False
267        self.tools_path = self.tools_path.rstrip('\\')
268        convert_command = ('%s -sym_path=%s -src_root=%s %s' %
269                           (os.path.join(self.tools_path,
270                                         'coverage_analyzer.exe'),
271                           sym_path,
272                           src_root,
273                           coverage_file))
274        (retcode, output) = proc.RunCommandFull(convert_command,
275                                                collect_output=True)
276        # TODO(niranjan): Fix this to check for the correct return code.
277#        if output != 0:
278#          logging.error('Conversion to LCOV failed. Exiting.')
279        tmp_lcov_file = coverage_file + '.lcov'
280        logging.info('Conversion to lcov complete for %s' % (coverage_file))
281        # Now append this .lcov file to the cumulative lcov file
282        logging.info('Consolidating LCOV file: %s' % (tmp_lcov_file))
283        tmp_lcov = open(tmp_lcov_file, 'r')
284        lcov.write(tmp_lcov.read())
285        tmp_lcov.close()
286      lcov.close()
287      logging.info('LCOV file uploaded to %s' % (upload_path))
288
289
290def main():
291  # Command line parsing
292  parser = optparse.OptionParser()
293  # Path where the .coverage to .lcov converter tools are stored.
294  parser.add_option('-t',
295                    '--tools_path',
296                    dest='tools_path',
297                    default=None,
298                    help='Location of the coverage tools (windows only)')
299  parser.add_option('-u',
300                    '--upload',
301                    dest='upload_path',
302                    default=None,
303                    help='Location where the results should be uploaded')
304  # We need the revision number so that we can generate the output file of the
305  # format chrome_<platform>_<revision>.lcov
306  parser.add_option('-r',
307                    '--revision',
308                    dest='revision',
309                    default=None,
310                    help='Revision number of the Chromium source repo')
311  # Root of the source tree. Needed for converting the generated .coverage file
312  # on Windows to the open source lcov format.
313  parser.add_option('-s',
314                    '--src_root',
315                    dest='src_root',
316                    default=None,
317                    help='Root of the source repository')
318  parser.add_option('-a',
319                    '--archive',
320                    dest='archive',
321                    default=None,
322                    help='Archive location of the intermediate .coverage data')
323
324  (options, args) = parser.parse_args()
325
326  if options.revision == None:
327    parser.error('Revision number not specified')
328  if options.src_root == None:
329    parser.error('Source root not specified')
330
331  if IsWindows():
332    # Initialize coverage
333    cov = Coverage(options.revision,
334                   options.src_root,
335                   options.tools_path,
336                   options.archive)
337    list_coverage = []
338    # Instrument the binaries
339    if cov.SetUp(windows_binaries):
340      # Run all the tests
341      for test in windows_tests:
342        coverage = cov.RunTest(options.src_root, test)
343        if coverage == None: # Indicate failure to the buildbots.
344          return 1
345        # Collect the intermediate file
346        list_coverage.append(coverage)
347    else:
348      logging.error('Error during instrumentation.')
349      sys.exit(1)
350
351    cov.Upload(list_coverage,
352               options.upload_path,
353               os.path.join(options.src_root, 'chrome', 'Release'),
354               options.src_root)
355    cov.TearDown()
356
357
358if __name__ == '__main__':
359  sys.exit(main())
360