1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# Self-test for skimage.
7
8import filecmp
9import os
10import subprocess
11import sys
12import tempfile
13
14class BinaryNotFoundException(Exception):
15    def __str__ (self):
16        return ("Could not find binary!\n"
17                "Did you forget to build the tools project?\n"
18                "Self tests failed")
19
20# Find a path to the binary to use. Iterates through a list of possible
21# locations the binary may be.
22def PickBinaryPath(base_dir):
23    POSSIBLE_BINARY_PATHS = [
24        'out/Debug/skimage',
25        'out/Release/skimage',
26        'xcodebuild/Debug/skimage',
27        'xcodebuild/Release/skimage',
28    ]
29    for binary in POSSIBLE_BINARY_PATHS:
30        binary_full_path = os.path.join(base_dir, binary)
31        if (os.path.exists(binary_full_path)):
32            return binary_full_path
33    raise BinaryNotFoundException
34
35# Quit early if two files have different content.
36def DieIfFilesMismatch(expected, actual):
37    if not filecmp.cmp(expected, actual):
38        raise Exception("Error: file mismatch! expected=%s , actual=%s" % (
39            expected, actual))
40
41def test_invalid_file(file_dir, skimage_binary):
42    """ Test the return value of skimage when an invalid file is decoded.
43        If there is no expectation file, or the file expects a particular
44        result, skimage should return nonzero indicating failure.
45        If the file has no expectation, or ignore-failure is set to true,
46        skimage should return zero indicating success. """
47    invalid_file = os.path.join(file_dir, "skimage", "input", "bad-images",
48                                "invalid.png")
49    # No expectations file:
50    args = [skimage_binary, "--readPath", invalid_file]
51    result = subprocess.call(args)
52    if 0 == result:
53      raise Exception("'%s' should have reported failure!" % " ".join(args))
54
55    # Directory holding all expectations files
56    expectations_dir = os.path.join(file_dir, "skimage", "input", "bad-images")
57
58    # Expectations file expecting a valid decode:
59    incorrect_expectations = os.path.join(expectations_dir,
60                                          "incorrect-results.json")
61    args = [skimage_binary, "--readPath", invalid_file,
62            "--readExpectationsPath", incorrect_expectations]
63    result = subprocess.call(args)
64    if 0 == result:
65      raise Exception("'%s' should have reported failure!" % " ".join(args))
66
67    # Empty expectations:
68    empty_expectations = os.path.join(expectations_dir, "empty-results.json")
69    output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
70                                      "--readExpectationsPath",
71                                      empty_expectations],
72                                     stderr=subprocess.STDOUT)
73    if not "Missing" in output:
74      # Another test (in main()) tests to ensure that "Missing" does not appear
75      # in the output. That test could be passed if the output changed so
76      # "Missing" never appears. This ensures that an error is not missed if
77      # that happens.
78      raise Exception(
79          "skimage output changed! This may cause other self tests to fail!")
80
81    # Ignore failure:
82    ignore_expectations = os.path.join(expectations_dir, "ignore-results.json")
83    output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
84                                      "--readExpectationsPath",
85                                      ignore_expectations],
86                                     stderr=subprocess.STDOUT)
87    if not "failures" in output:
88      # Another test (in main()) tests to ensure that "failures" does not
89      # appear in the output. That test could be passed if the output changed
90      # so "failures" never appears. This ensures that an error is not missed
91      # if that happens.
92      raise Exception(
93          "skimage output changed! This may cause other self tests to fail!")
94
95def test_incorrect_expectations(file_dir, skimage_binary):
96    """ Test that comparing to incorrect expectations fails, unless
97        ignore-failures is set to true. """
98    valid_file = os.path.join(file_dir, "skimage", "input",
99                                    "images-with-known-hashes",
100                                    "1209453360120438698.png")
101    expectations_dir = os.path.join(file_dir, "skimage", "input",
102                                    "images-with-known-hashes")
103
104    incorrect_results = os.path.join(expectations_dir,
105                                     "incorrect-results.json")
106    args = [skimage_binary, "--readPath", valid_file, "--readExpectationsPath",
107            incorrect_results]
108    result = subprocess.call(args)
109    if 0 == result:
110      raise Exception("'%s' should have reported failure!" % " ".join(args))
111
112    ignore_results = os.path.join(expectations_dir, "ignore-failures.json")
113    subprocess.check_call([skimage_binary, "--readPath", valid_file,
114                           "--readExpectationsPath", ignore_results])
115
116def main():
117    # Use the directory of this file as the out directory
118    file_dir = os.path.abspath(os.path.dirname(__file__))
119
120    trunk_dir = os.path.normpath(os.path.join(file_dir, os.pardir, os.pardir))
121
122    # Find the binary
123    skimage_binary = PickBinaryPath(trunk_dir)
124    print "Running " + skimage_binary
125
126    # Generate an expectations file from known images.
127    images_dir = os.path.join(file_dir, "skimage", "input",
128                              "images-with-known-hashes")
129    expectations_path = os.path.join(file_dir, "skimage", "output-actual",
130                                     "create-expectations", "expectations.json")
131    subprocess.check_call([skimage_binary, "--readPath", images_dir,
132                           "--createExpectationsPath", expectations_path])
133
134    # Make sure the expectations file was generated correctly.
135    golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
136                                       "create-expectations",
137                                       "expectations.json")
138    DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
139
140    # Tell skimage to read back the expectations file it just wrote, and
141    # confirm that the images in images_dir match it.
142    output = subprocess.check_output([skimage_binary, "--readPath", images_dir,
143                                      "--readExpectationsPath",
144                                      expectations_path],
145                                     stderr=subprocess.STDOUT)
146
147    # Although skimage succeeded, it would have reported success if the file
148    # was missing from the expectations file. Consider this a failure, since
149    # the expectations file was created from this same image. (It will print
150    # "Missing" in this case before listing the missing expectations).
151    if "Missing" in output:
152      raise Exception("Expectations file was missing expectations: %s" % output)
153
154    # Again, skimage would succeed if there were known failures (and print
155    # "failures"), but there should be no failures, since the file just
156    # created did not include failures to ignore.
157    if "failures" in output:
158      raise Exception("Image failed: %s" % output)
159
160
161    test_incorrect_expectations(file_dir=file_dir,
162                                skimage_binary=skimage_binary)
163
164    # Generate an expectations file from an empty directory.
165    empty_dir = tempfile.mkdtemp()
166    expectations_path = os.path.join(file_dir, "skimage", "output-actual",
167                                     "empty-dir", "expectations.json")
168    subprocess.check_call([skimage_binary, "--readPath", empty_dir,
169                           "--createExpectationsPath", expectations_path])
170    golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
171                                       "empty-dir", "expectations.json")
172    DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
173    os.rmdir(empty_dir)
174
175    # Generate an expectations file from a nonexistent directory.
176    expectations_path = os.path.join(file_dir, "skimage", "output-actual",
177                                     "nonexistent-dir", "expectations.json")
178    subprocess.check_call([skimage_binary, "--readPath", "/nonexistent/dir",
179                           "--createExpectationsPath", expectations_path])
180    golden_expectations = os.path.join(file_dir, "skimage", "output-expected",
181                                       "nonexistent-dir", "expectations.json")
182    DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path)
183
184    test_invalid_file(file_dir=file_dir, skimage_binary=skimage_binary)
185
186    # Done with all tests.
187    print "Self tests succeeded!"
188
189if __name__ == "__main__":
190    main()
191