1#!/usr/bin/python
2
3"""
4Copyright 2014 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Test compare_rendered_pictures.py
10
11TODO(epoger): Create a command to update the expected results (in
12self._output_dir_expected) when appropriate.  For now, you should:
131. examine the results in self.output_dir_actual and make sure they are ok
142. rm -rf self._output_dir_expected
153. mv self.output_dir_actual self._output_dir_expected
16Although, if you're using an SVN checkout, this will blow away .svn directories
17within self._output_dir_expected, which wouldn't be good...
18
19"""
20
21# System-level imports
22import os
23import posixpath
24import subprocess
25
26# Must fix up PYTHONPATH before importing from within Skia
27import rs_fixpypath  # pylint: disable=W0611
28
29# Imports from within Skia
30import base_unittest
31import compare_rendered_pictures
32import find_run_binary
33import gm_json
34import imagediffdb
35import imagepairset
36import results
37
38
39class CompareRenderedPicturesTest(base_unittest.TestCase):
40
41  def test_endToEnd(self):
42    """Generate two sets of SKPs, run render_pictures over both, and compare
43    the results."""
44    setA_subdir = 'before_patch'
45    setB_subdir = 'after_patch'
46    self._generate_skps_and_run_render_pictures(
47        subdir=setA_subdir, skpdict={
48            'changed.skp': 200,
49            'unchanged.skp': 100,
50            'only-in-before.skp': 128,
51        })
52    self._generate_skps_and_run_render_pictures(
53        subdir=setB_subdir, skpdict={
54            'changed.skp': 201,
55            'unchanged.skp': 100,
56            'only-in-after.skp': 128,
57        })
58
59    results_obj = compare_rendered_pictures.RenderedPicturesComparisons(
60        setA_dir=os.path.join(self.temp_dir, setA_subdir),
61        setB_dir=os.path.join(self.temp_dir, setB_subdir),
62        setA_section=gm_json.JSONKEY_ACTUALRESULTS,
63        setB_section=gm_json.JSONKEY_ACTUALRESULTS,
64        image_diff_db=imagediffdb.ImageDiffDB(self.temp_dir),
65        image_base_gs_url='gs://fakebucket/fake/path',
66        diff_base_url='/static/generated-images')
67    results_obj.get_timestamp = mock_get_timestamp
68
69    # Overwrite elements within the results that change from one test run
70    # to the next.
71    # pylint: disable=W0212
72    results_obj._setA_descriptions[results.KEY__SET_DESCRIPTIONS__DIR] = [
73        'before-patch-fake-dir']
74    results_obj._setB_descriptions[results.KEY__SET_DESCRIPTIONS__DIR] = [
75        'after-patch-fake-dir']
76
77    gm_json.WriteToFile(
78        results_obj.get_packaged_results_of_type(
79            results.KEY__HEADER__RESULTS_ALL),
80        os.path.join(self.output_dir_actual, 'compare_rendered_pictures.json'))
81
82  def test_endToEnd_withImageBaseGSUrl(self):
83    """Generate two sets of SKPs, run render_pictures over both, and compare
84    the results."""
85    setA_subdir = 'before_patch'
86    setB_subdir = 'after_patch'
87    imageA_gs_base = 'superman/kent-camera/pictures'
88    imageB_gs_base = 'batman/batarang/pictures'
89    self._generate_skps_and_run_render_pictures(
90        subdir=setA_subdir, skpdict={
91            'changed.skp': 200,
92            'unchanged.skp': 100,
93            'only-in-before.skp': 128,
94        },
95        image_base_gs_url='gs://%s' % imageA_gs_base)
96    self._generate_skps_and_run_render_pictures(
97        subdir=setB_subdir, skpdict={
98            'changed.skp': 201,
99            'unchanged.skp': 100,
100            'only-in-after.skp': 128,
101        },
102        image_base_gs_url='gs://%s' % imageB_gs_base)
103
104    results_obj = compare_rendered_pictures.RenderedPicturesComparisons(
105        setA_dir=os.path.join(self.temp_dir, setA_subdir),
106        setB_dir=os.path.join(self.temp_dir, setB_subdir),
107        setA_section=gm_json.JSONKEY_ACTUALRESULTS,
108        setB_section=gm_json.JSONKEY_ACTUALRESULTS,
109        image_diff_db=imagediffdb.ImageDiffDB(self.temp_dir),
110        image_base_gs_url='gs://fakebucket/fake/path',
111        diff_base_url='/static/generated-images')
112    results_obj.get_timestamp = mock_get_timestamp
113
114    output_dict = results_obj.get_packaged_results_of_type(
115        results.KEY__HEADER__RESULTS_ALL)
116    # Assert that the baseURLs are as expected.
117    self.assertEquals(
118        output_dict[imagepairset.KEY__ROOT__IMAGESETS]
119                   [imagepairset.KEY__IMAGESETS__SET__IMAGE_A]
120                   [imagepairset.KEY__IMAGESETS__FIELD__BASE_URL],
121        'http://storage.cloud.google.com/%s' % imageA_gs_base)
122    self.assertEquals(
123        output_dict[imagepairset.KEY__ROOT__IMAGESETS]
124                   [imagepairset.KEY__IMAGESETS__SET__IMAGE_B]
125                   [imagepairset.KEY__IMAGESETS__FIELD__BASE_URL],
126        'http://storage.cloud.google.com/%s' % imageB_gs_base)
127    # Overwrite elements within the results that change from one test run
128    # to the next.
129    # pylint: disable=W0212
130    results_obj._setA_descriptions[results.KEY__SET_DESCRIPTIONS__DIR] = [
131        'before-patch-fake-dir']
132    results_obj._setB_descriptions[results.KEY__SET_DESCRIPTIONS__DIR] = [
133        'after-patch-fake-dir']
134
135    gm_json.WriteToFile(
136        output_dict,
137        os.path.join(self.output_dir_actual,
138                               'compare_rendered_pictures.json'))
139
140  def test_repo_url(self):
141    """Use repo: URL to specify summary files."""
142    base_repo_url = 'repo:gm/rebaseline_server/testdata/inputs/skp-summaries'
143    results_obj = compare_rendered_pictures.RenderedPicturesComparisons(
144        setA_dir=posixpath.join(base_repo_url, 'expectations'),
145        setB_dir=posixpath.join(base_repo_url, 'actuals'),
146        setA_section=gm_json.JSONKEY_EXPECTEDRESULTS,
147        setB_section=gm_json.JSONKEY_ACTUALRESULTS,
148        image_diff_db=imagediffdb.ImageDiffDB(self.temp_dir),
149        image_base_gs_url='gs://fakebucket/fake/path',
150        diff_base_url='/static/generated-images')
151    results_obj.get_timestamp = mock_get_timestamp
152
153    # Overwrite elements within the results that change from one test run
154    # to the next.
155    # pylint: disable=W0212
156    results_obj._setA_descriptions\
157        [results.KEY__SET_DESCRIPTIONS__REPO_REVISION] = 'fake-repo-revision'
158    results_obj._setB_descriptions\
159        [results.KEY__SET_DESCRIPTIONS__REPO_REVISION] = 'fake-repo-revision'
160
161    gm_json.WriteToFile(
162        results_obj.get_packaged_results_of_type(
163            results.KEY__HEADER__RESULTS_ALL),
164        os.path.join(self.output_dir_actual, 'compare_rendered_pictures.json'))
165
166  def _generate_skps_and_run_render_pictures(self, subdir, skpdict,
167                                             image_base_gs_url=None):
168    """Generate SKPs and run render_pictures on them.
169
170    Args:
171      subdir: subdirectory (within self.temp_dir) to write all files into
172      skpdict: {skpname: redvalue} dictionary describing the SKP files to render
173    """
174    out_path = os.path.join(self.temp_dir, subdir)
175    os.makedirs(out_path)
176    for skpname, redvalue in skpdict.iteritems():
177      self._run_skpmaker(
178          output_path=os.path.join(out_path, skpname), red=redvalue)
179
180    # TODO(epoger): Add --mode tile 256 256 --writeWholeImage to the unittest,
181    # and fix its result!  (imageURLs within whole-image entries are wrong when
182    # I tried adding that)
183    binary = find_run_binary.find_path_to_program('render_pictures')
184    render_pictures_cmd = [
185        binary,
186        '--config', '8888',
187        '-r', out_path,
188        '--writeChecksumBasedFilenames',
189        '--writeJsonSummaryPath', os.path.join(out_path, 'summary.json'),
190        '--writePath', out_path]
191    if image_base_gs_url:
192      render_pictures_cmd.extend(['--imageBaseGSUrl', image_base_gs_url])
193    return subprocess.check_output(render_pictures_cmd)
194
195  def _run_skpmaker(self, output_path, red=0, green=0, blue=0,
196                    width=640, height=400):
197    """Runs the skpmaker binary to generate SKP with known characteristics.
198
199    Args:
200      output_path: Filepath to write the SKP into.
201      red: Value of red color channel in image, 0-255.
202      green: Value of green color channel in image, 0-255.
203      blue: Value of blue color channel in image, 0-255.
204      width: Width of canvas to create.
205      height: Height of canvas to create.
206    """
207    binary = find_run_binary.find_path_to_program('skpmaker')
208    return subprocess.check_output([
209        binary,
210        '--red', str(red),
211        '--green', str(green),
212        '--blue', str(blue),
213        '--width', str(width),
214        '--height', str(height),
215        '--writePath', str(output_path)])
216
217def mock_get_timestamp():
218  """Mock version of BaseComparisons.get_timestamp() for testing."""
219  return 12345678
220
221
222def main():
223  base_unittest.main(CompareRenderedPicturesTest)
224
225
226if __name__ == '__main__':
227  main()
228