1#!/bin/bash
2
3# Tests for our tools.
4#
5# TODO: currently, this only passes on Linux (which is the platform that
6# the housekeeper bot runs on, e.g.
7# http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/steps/RunToolSelfTests/logs/stdio )
8# See https://code.google.com/p/skia/issues/detail?id=677
9# ('make tools/tests/run.sh work cross-platform')
10# Ideally, these tests should pass on all development platforms...
11# otherwise, how can developers be expected to test them before committing a
12# change?
13
14# cd into .../trunk so all the paths will work
15cd $(dirname $0)/../..
16
17# TODO: make it look in Release and/or Debug
18SKDIFF_BINARY=out/Debug/skdiff
19
20# Suffixes of the raw bench data files we want to process.
21BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES=\
22"data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles_bbh_rtree "\
23"data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles"
24BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES=\
25"data_skp_multi_4_scale_1.3061_config_8888_mode_tile_256_256 "\
26"data_skp_scale_1.3061_config_8888_mode_record"
27
28# Compare contents of all files within directories $1 and $2,
29# EXCEPT for any dotfiles.
30# If there are any differences, a description is written to stdout and
31# we exit with a nonzero return value.
32# Otherwise, we write nothing to stdout and return.
33function compare_directories {
34  if [ $# != 2 ]; then
35    echo "compare_directories requires exactly 2 parameters, got $#"
36    exit 1
37  fi
38  diff --recursive --exclude=.* $1 $2
39  if [ $? != 0 ]; then
40    echo "failed in: compare_directories $1 $2"
41    exit 1
42  fi
43}
44
45# Run skdiff with arguments in $1 (plus implicit final argument causing skdiff
46# to write its output, if any, to directory $2/output-actual).
47# Then compare its results against those in $2/output-expected.
48function skdiff_test {
49  if [ $# != 2 ]; then
50    echo "skdiff_test requires exactly 2 parameters, got $#"
51    exit 1
52  fi
53  SKDIFF_ARGS="$1"
54  ACTUAL_OUTPUT_DIR="$2/output-actual"
55  EXPECTED_OUTPUT_DIR="$2/output-expected"
56
57  rm -rf $ACTUAL_OUTPUT_DIR
58  mkdir -p $ACTUAL_OUTPUT_DIR
59  COMMAND="$SKDIFF_BINARY $SKDIFF_ARGS $ACTUAL_OUTPUT_DIR"
60  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
61  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
62  echo $? >$ACTUAL_OUTPUT_DIR/return_value
63
64  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
65}
66
67# Download a subset of the raw bench data for platform $1 at revision $2.
68# (For the subset, download all files matching any of the suffixes in
69# whitespace-separated list $3.)
70# If any of those files already exist locally, we assume that they are
71# correct and up to date, and we don't download them again.
72function download_bench_rawdata {
73  if [ $# != 3 ]; then
74    echo "download_bench_rawdata requires exactly 3 parameters, got $#"
75    exit 1
76  fi
77  PLATFORM="$1"
78  REV="$2"
79  FILE_SUFFIXES="$3"
80
81  PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM"
82  RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data"
83  mkdir -p $RAW_BENCH_DATA_DIR
84
85  for FILE_SUFFIX in $FILE_SUFFIXES; do
86    FILE=bench_${REV}_${FILE_SUFFIX}
87    DESTFILE=$RAW_BENCH_DATA_DIR/$FILE
88    if [ ! -f $DESTFILE ];
89    then
90      URL=http://chromium-skia-gm.commondatastorage.googleapis.com/perfdata/${PLATFORM}/${FILE}
91      echo Downloading $URL ...
92      curl $URL --output $DESTFILE
93    fi
94  done
95}
96
97# Run check_bench_regressions.py across the data from platform $1,
98# writing its output to output-actual and comparing those results against
99# output-expected.
100function benchalert_test {
101  if [ $# != 2 ]; then
102    echo "benchalert_test requires exactly 2 parameter, got $#"
103    exit 1
104  fi
105  PLATFORM="$1"
106  REVISION="$2"
107
108  PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM"
109  RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data"
110  ACTUAL_OUTPUT_DIR="$PLATFORM_DIR/output-actual"
111  EXPECTED_OUTPUT_DIR="$PLATFORM_DIR/output-expected"
112
113  # Run check_bench_regressions.py .
114  rm -rf $ACTUAL_OUTPUT_DIR
115  mkdir -p $ACTUAL_OUTPUT_DIR
116  COMMAND="python bench/check_bench_regressions.py -a 25th -b $PLATFORM -d $RAW_BENCH_DATA_DIR -e $PLATFORM_DIR/expectations.txt -r $REVISION"
117  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
118  START_TIMESTAMP=$(date +%s)
119  $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr
120  echo $? >$ACTUAL_OUTPUT_DIR/return_value
121  END_TIMESTAMP=$(date +%s)
122
123  SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP)
124  echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete"
125
126  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
127}
128
129# Test rebaseline.py's JSON-format expectations rebaselining capability.
130#
131# Copy expected-results.json files from $1 into a dir where they can be modified.
132# Run rebaseline.py with arguments in $2, recording its output.
133# Then compare the output (and modified expected-results.json files) to the
134# content of $2/output-expected.
135function rebaseline_test {
136  if [ $# != 3 ]; then
137    echo "rebaseline_test requires exactly 3 parameters, got $#"
138    exit 1
139  fi
140  COPY_EXPECTATIONS_FROM_DIR="$1"
141  ARGS="$2"
142  ACTUAL_OUTPUT_DIR="$3/output-actual"
143  EXPECTED_OUTPUT_DIR="$3/output-expected"
144
145  rm -rf $ACTUAL_OUTPUT_DIR
146  mkdir -p $ACTUAL_OUTPUT_DIR
147  EXPECTATIONS_TO_MODIFY_DIR="$ACTUAL_OUTPUT_DIR/gm-expectations"
148  BUILDERS=$(ls $COPY_EXPECTATIONS_FROM_DIR)
149  for BUILDER in $BUILDERS; do
150    mkdir -p $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER
151    cp $COPY_EXPECTATIONS_FROM_DIR/$BUILDER/expected-results.json \
152       $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER
153  done
154  COMMAND="python tools/rebaseline.py --expectations-root $EXPECTATIONS_TO_MODIFY_DIR $ARGS"
155  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
156  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
157  echo $? >$ACTUAL_OUTPUT_DIR/return_value
158
159  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
160}
161
162# Run jsondiff.py with arguments in $1, recording its output.
163# Then compare that output to the content of $2/output-expected.
164function jsondiff_test {
165  if [ $# != 2 ]; then
166    echo "jsondiff_test requires exactly 2 parameters, got $#"
167    exit 1
168  fi
169  ARGS="$1"
170  ACTUAL_OUTPUT_DIR="$2/output-actual"
171  EXPECTED_OUTPUT_DIR="$2/output-expected"
172
173  rm -rf $ACTUAL_OUTPUT_DIR
174  mkdir -p $ACTUAL_OUTPUT_DIR
175  COMMAND="python tools/jsondiff.py $ARGS"
176  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
177  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
178  echo $? >$ACTUAL_OUTPUT_DIR/return_value
179
180  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
181}
182
183
184
185#
186# Run skdiff tests...
187#
188
189SKDIFF_TESTDIR=tools/tests/skdiff
190
191# Run skdiff over a variety of file pair types: identical bits, identical pixels, missing from
192# baseDir, etc.
193skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1"
194
195# Run skdiff over the same set of files, but with arguments as used by our buildbots:
196# - return the number of mismatching file pairs (but ignore any files missing from either
197#   baseDir or comparisonDir)
198# - list filenames with each result type to stdout
199# - don't generate HTML output files
200skdiff_test "--failonresult DifferentPixels --failonresult DifferentSizes --failonresult Unknown --failonstatus CouldNotDecode,CouldNotRead any --failonstatus any CouldNotDecode,CouldNotRead --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
201
202# Run skdiff over just the files that have identical bits.
203skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
204
205# Run skdiff over just the files that have identical bits or identical pixels.
206skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
207
208#
209# Run bench alerts tests...
210#
211
212# Parse a collection of bench data
213PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release
214REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34
215download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES"
216download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES"
217benchalert_test $PLATFORM $REVISION
218
219#
220# Run self test for skimage ...
221#
222
223COMMAND="python tools/tests/skimage_self_test.py"
224echo "$COMMAND"
225$COMMAND
226ret=$?
227if [ $ret -ne 0 ]; then
228    echo "skimage self tests failed."
229    exit 1
230fi
231
232#
233# Test rebaseline.py ...
234#
235
236REBASELINE_INPUT=tools/tests/rebaseline/input
237REBASELINE_OUTPUT=tools/tests/rebaseline/output
238rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/using-json1-expectations"
239rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --bugs 1234 5678 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release --notes notes_content --unreviewed" "$REBASELINE_OUTPUT/marked-unreviewed"
240rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --add-new --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/add-new"
241
242#
243# Test jsondiff.py ...
244#
245
246JSONDIFF_INPUT=tools/tests/jsondiff/input
247JSONDIFF_OUTPUT=tools/tests/jsondiff/output
248jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUTPUT/old-vs-new"
249
250
251echo "All tests passed."
252