results_test.py revision d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76 
1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Test results.py
10
11TODO(epoger): Create a command to update the expected results (in
12OUTPUT_DIR_EXPECTED) when appropriate.  For now, you should:
131. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok
142. rm -rf OUTPUT_DIR_EXPECTED
153. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED
16Although, if you're using an SVN checkout, this will blow away .svn directories
17within OUTPUT_DIR_EXPECTED, which wouldn't be good...
18
19"""
20
21import filecmp
22import os
23import shutil
24import sys
25import tempfile
26import unittest
27
28# Imports from within Skia
29import results
30import gm_json  # must import results first, so that gm_json will be in sys.path
31
32PARENT_DIR = os.path.dirname(os.path.realpath(__file__))
33INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs')
34OUTPUT_DIR_ACTUAL   = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual')
35OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected')
36
37
38class ResultsTest(unittest.TestCase):
39
40  def setUp(self):
41    self._temp_dir = tempfile.mkdtemp()
42    self._output_dir_actual   = os.path.join(OUTPUT_DIR_ACTUAL, self.id())
43    self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id())
44    create_empty_dir(self._output_dir_actual)
45
46  def tearDown(self):
47    shutil.rmtree(self._temp_dir)
48    different_files = find_different_files(self._output_dir_actual,
49                                           self._output_dir_expected)
50    # Maybe we should move this assert elsewhere?  It's unusual to see an
51    # assert within tearDown(), but my thinking was:
52    # 1. Every test case will have some collection of output files that need to
53    #    be validated.
54    # 2. So put that validation within tearDown(), which will be called after
55    #    every test case!
56    #
57    # I have confirmed that the test really does fail if this assert is
58    # triggered.
59    #
60    # Ravi notes: if somebody later comes along and adds cleanup code below the
61    # assert, then if tests fail, the artifacts will not be cleaned up.
62    assert (not different_files), \
63      ('found differing files between actual dir %s and expected dir %s: %s' %
64       (self._output_dir_actual, self._output_dir_expected, different_files))
65
66  def shortDescription(self):
67    """Tell unittest framework to not print docstrings for test cases."""
68    return None
69
70  def test_gm(self):
71    """Process results of a GM run with the Results object."""
72    results_obj = results.Results(
73        actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'),
74        expected_root=os.path.join(INPUT_DIR, 'gm-expectations'),
75        generated_images_root=self._temp_dir)
76    gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL),
77                        os.path.join(self._output_dir_actual, 'gm.json'))
78
79
80# TODO(epoger): create_empty_dir(), find_different_files(), etc. should be
81# extracted from this file to some common location, where they can be shared
82# with download_test.py and other users.
83
84def create_empty_dir(path):
85  """Create an empty directory at the given path."""
86  if os.path.isdir(path):
87    shutil.rmtree(path)
88  elif os.path.lexists(path):
89    os.remove(path)
90  os.makedirs(path)
91
92
93def find_different_files(dir1, dir2, ignore_subtree_names=None):
94  """Returns a list of any files that differ between the directory trees rooted
95  at dir1 and dir2.
96
97  Args:
98    dir1: root of a directory tree; if nonexistent, will raise OSError
99    dir2: root of another directory tree; if nonexistent, will raise OSError
100    ignore_subtree_names: list of subtree directory names to ignore;
101          defaults to ['.svn'], so all SVN files are ignores
102
103  TODO(epoger): include the dirname within each filename (not just the
104  basename), to make it easier to locate any differences
105  """
106  differing_files = []
107  if ignore_subtree_names is None:
108    ignore_subtree_names = ['.svn']
109  dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names)
110  differing_files.extend(dircmp.left_only)
111  differing_files.extend(dircmp.right_only)
112  differing_files.extend(dircmp.common_funny)
113  differing_files.extend(dircmp.diff_files)
114  differing_files.extend(dircmp.funny_files)
115  for common_dir in dircmp.common_dirs:
116    differing_files.extend(find_different_files(
117        os.path.join(dir1, common_dir), os.path.join(dir2, common_dir)))
118  return differing_files
119
120
121def main():
122  suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest)
123  unittest.TextTestRunner(verbosity=2).run(suite)
124
125
126if __name__ == '__main__':
127  main()
128