results_test.py revision 0bf919c89a5cd8d1779cdbd5d17536c2fbf534c8 
1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Test results.py
10
11TODO(epoger): Launch this (and other unittests within this dir) automatically
12on the housekeeper bot, but first make sure it works properly after having been
13checked out (from both git and svn)
14
15TODO(epoger): Create a command to update the expected results (in
16OUTPUT_DIR_EXPECTED) when appropriate.  For now, you should:
171. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok
182. rm -rf OUTPUT_DIR_EXPECTED
193. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED
20Although, if you're using an SVN checkout, this will blow away .svn directories
21within OUTPUT_DIR_EXPECTED, which wouldn't be good...
22
23"""
24
25import filecmp
26import os
27import shutil
28import sys
29import tempfile
30import unittest
31
32# Imports from within Skia
33import results
34import gm_json  # must import results first, so that gm_json will be in sys.path
35
36PARENT_DIR = os.path.dirname(os.path.realpath(__file__))
37INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs')
38OUTPUT_DIR_ACTUAL   = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual')
39OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected')
40
41
42class ResultsTest(unittest.TestCase):
43
44  def setUp(self):
45    self._temp_dir = tempfile.mkdtemp()
46    self._output_dir_actual   = os.path.join(OUTPUT_DIR_ACTUAL, self.id())
47    self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id())
48    create_empty_dir(self._output_dir_actual)
49
50  def tearDown(self):
51    shutil.rmtree(self._temp_dir)
52    different_files = find_different_files(self._output_dir_actual,
53                                           self._output_dir_expected)
54    # Maybe we should move this assert elsewhere?  It's unusual to see an
55    # assert within tearDown(), but my thinking was:
56    # 1. Every test case will have some collection of output files that need to
57    #    be validated.
58    # 2. So put that validation within tearDown(), which will be called after
59    #    every test case!
60    #
61    # I have confirmed that the test really does fail if this assert is
62    # triggered.
63    #
64    # Ravi notes: if somebody later comes along and adds cleanup code below the
65    # assert, then if tests fail, the artifacts will not be cleaned up.
66    assert (not different_files), \
67      ('found differing files between actual dir %s and expected dir %s: %s' %
68       (self._output_dir_actual, self._output_dir_expected, different_files))
69
70  def test_gm(self):
71    """Process results of a GM run with the Results object."""
72    results_obj = results.Results(
73        actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'),
74        expected_root=os.path.join(INPUT_DIR, 'gm-expectations'),
75        generated_images_root=self._temp_dir)
76    gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL),
77                        os.path.join(self._output_dir_actual, 'gm.json'))
78
79
80def create_empty_dir(path):
81  """Create an empty directory at the given path."""
82  if os.path.isdir(path):
83    shutil.rmtree(path)
84  elif os.path.lexists(path):
85    os.remove(path)
86  os.makedirs(path)
87
88
89def find_different_files(dir1, dir2, ignore_subtree_names=None):
90  """Returns a list of any files that differ between the directory trees rooted
91  at dir1 and dir2.
92
93  Args:
94    dir1: root of a directory tree; if nonexistent, will raise OSError
95    dir2: root of another directory tree; if nonexistent, will raise OSError
96    ignore_subtree_names: list of subtree directory names to ignore;
97          defaults to ['.svn'], so all SVN files are ignores
98
99  TODO(epoger): include the dirname within each filename (not just the
100  basename), to make it easier to locate any differences
101  """
102  differing_files = []
103  if ignore_subtree_names is None:
104    ignore_subtree_names = ['.svn']
105  dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names)
106  differing_files.extend(dircmp.left_only)
107  differing_files.extend(dircmp.right_only)
108  differing_files.extend(dircmp.common_funny)
109  differing_files.extend(dircmp.diff_files)
110  differing_files.extend(dircmp.funny_files)
111  for common_dir in dircmp.common_dirs:
112    differing_files.extend(find_different_files(
113        os.path.join(dir1, common_dir), os.path.join(dir2, common_dir)))
114  return differing_files
115
116
117def main():
118  suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest)
119  unittest.TextTestRunner(verbosity=2).run(suite)
120
121
122if __name__ == '__main__':
123  main()
124