results_test.py revision 7b06c8ee61bb3d361eea940c2feaecfb43125bbc 
1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Test results.py
10
11TODO(epoger): Launch this (and other unittests within this dir) automatically
12on the housekeeper bot, but first make sure it works properly after having been
13checked out (from both git and svn)
14
15TODO(epoger): Create a command to update the expected results (in
16OUTPUT_DIR_EXPECTATIONS) when appropriate.  For now, you should:
171. examine the results in OUTPUT_DIR and make sure they are ok
182. rm -rf OUTPUT_DIR_EXPECTATIONS
193. mv OUTPUT_DIR OUTPUT_DIR_EXPECTATIONS
20Although, if you're using an SVN checkout, this will blow away .svn directories
21within OUTPUT_DIR_EXPECTATIONS, which wouldn't be good...
22
23"""
24
25import filecmp
26import os
27import shutil
28import sys
29import tempfile
30import unittest
31
32# Imports from within Skia
33import results
34import gm_json  # must import results first, so that gm_json will be in sys.path
35
36PARENT_DIR = os.path.dirname(os.path.realpath(__file__))
37INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs')
38OUTPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual')
39OUTPUT_DIR_EXPECTATIONS = os.path.join(
40    PARENT_DIR, 'tests', 'outputs', 'expected')
41
42class ResultsTest(unittest.TestCase):
43
44  def setUp(self):
45    self._tempdir = tempfile.mkdtemp()
46
47  def tearDown(self):
48    shutil.rmtree(self._tempdir)
49
50  def test_gm(self):
51    """Process results of a GM run with the Results object."""
52    results_obj = results.Results(
53        actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'),
54        expected_root=os.path.join(INPUT_DIR, 'gm-expectations'),
55        generated_images_root=self._tempdir)
56    gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL),
57                        os.path.join(OUTPUT_DIR, 'gm.json'))
58
59
60def find_different_files(dir1, dir2, ignore_subtree_names=None):
61  """Returns a list of any files that differ between the directory trees rooted
62  at dir1 and dir2.
63
64  Args:
65    dir1: root of a directory tree; if nonexistent, will raise OSError
66    dir2: root of another directory tree; if nonexistent, will raise OSError
67    ignore_subtree_names: list of subtree directory names to ignore;
68          defaults to ['.svn'], so all SVN files are ignores
69
70  TODO(epoger): include the dirname within each filename (not just the
71  basename), to make it easier to locate any differences
72  """
73  differing_files = []
74  if ignore_subtree_names is None:
75    ignore_subtree_names = ['.svn']
76  dircmp = filecmp.dircmp(dir1, dir2, ignore=ignore_subtree_names)
77  differing_files.extend(dircmp.left_only)
78  differing_files.extend(dircmp.right_only)
79  differing_files.extend(dircmp.common_funny)
80  differing_files.extend(dircmp.diff_files)
81  differing_files.extend(dircmp.funny_files)
82  for common_dir in dircmp.common_dirs:
83    differing_files.extend(find_different_files(
84        os.path.join(dir1, common_dir), os.path.join(dir2, common_dir)))
85  return differing_files
86
87
88def main():
89  if not os.path.isdir(OUTPUT_DIR):
90    os.makedirs(OUTPUT_DIR)
91  suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest)
92  unittest.TextTestRunner(verbosity=2).run(suite)
93  different_files = find_different_files(OUTPUT_DIR, OUTPUT_DIR_EXPECTATIONS)
94  assert (not different_files), 'found differing files: %s' % different_files
95
96
97if __name__ == '__main__':
98  main()
99