1#!/bin/bash
2
3# Tests for our tools.
4#
5# TODO: currently, this only passes on Linux (which is the platform that
6# the housekeeper bot runs on, e.g.
7# http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/steps/RunToolSelfTests/logs/stdio )
8# See https://code.google.com/p/skia/issues/detail?id=677
9# ('make tools/tests/run.sh work cross-platform')
10# Ideally, these tests should pass on all development platforms...
11# otherwise, how can developers be expected to test them before committing a
12# change?
13
14# cd into .../trunk so all the paths will work
15cd $(dirname $0)/../..
16
17# TODO: make it look in Release and/or Debug
18SKDIFF_BINARY=out/Debug/skdiff
19
20# Suffixes of the raw bench data files we want to process.
21BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES=\
22"data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles_bbh_rtree "\
23"data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles"
24BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES=\
25"data_skp_multi_4_scale_1.3061_config_8888_mode_tile_256_256 "\
26"data_skp_scale_1.3061_config_8888_mode_record"
27
28# Compare contents of all files within directories $1 and $2,
29# EXCEPT for any dotfiles.
30# If there are any differences, a description is written to stdout and
31# we exit with a nonzero return value.
32# Otherwise, we write nothing to stdout and return.
33function compare_directories {
34  if [ $# != 2 ]; then
35    echo "compare_directories requires exactly 2 parameters, got $#"
36    exit 1
37  fi
38  diff --recursive --exclude=.* $1 $2
39  if [ $? != 0 ]; then
40    echo "failed in: compare_directories $1 $2"
41    exit 1
42  fi
43}
44
45# Run skdiff with arguments in $1 (plus implicit final argument causing skdiff
46# to write its output, if any, to directory $2/output-actual).
47# Then compare its results against those in $2/output-expected.
48function skdiff_test {
49  if [ $# != 2 ]; then
50    echo "skdiff_test requires exactly 2 parameters, got $#"
51    exit 1
52  fi
53  SKDIFF_ARGS="$1"
54  ACTUAL_OUTPUT_DIR="$2/output-actual"
55  EXPECTED_OUTPUT_DIR="$2/output-expected"
56
57  rm -rf $ACTUAL_OUTPUT_DIR
58  mkdir -p $ACTUAL_OUTPUT_DIR
59  COMMAND="$SKDIFF_BINARY $SKDIFF_ARGS $ACTUAL_OUTPUT_DIR"
60  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
61  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
62  echo $? >$ACTUAL_OUTPUT_DIR/return_value
63
64  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
65}
66
67# Download a subset of the raw bench data for platform $1 at revision $2.
68# (For the subset, download all files matching any of the suffixes in
69# whitespace-separated list $3.)
70# If any of those files already exist locally, we assume that they are
71# correct and up to date, and we don't download them again.
72function download_bench_rawdata {
73  if [ $# != 3 ]; then
74    echo "download_bench_rawdata requires exactly 3 parameters, got $#"
75    exit 1
76  fi
77  PLATFORM="$1"
78  REV="$2"
79  FILE_SUFFIXES="$3"
80
81  PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM"
82  RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data"
83  mkdir -p $RAW_BENCH_DATA_DIR
84
85  for FILE_SUFFIX in $FILE_SUFFIXES; do
86    FILE=bench_${REV}_${FILE_SUFFIX}
87    DESTFILE=$RAW_BENCH_DATA_DIR/$FILE
88    if [ ! -f $DESTFILE ];
89    then
90      URL=http://chromium-skia-gm.commondatastorage.googleapis.com/perfdata/${PLATFORM}/${FILE}
91      echo Downloading $URL ...
92      curl $URL --output $DESTFILE
93    fi
94  done
95}
96
97# Run check_bench_regressions.py across the data from platform $1,
98# writing its output to output-actual and comparing those results against
99# output-expected.
100function benchalert_test {
101  if [ $# != 2 ]; then
102    echo "benchalert_test requires exactly 2 parameter, got $#"
103    exit 1
104  fi
105  PLATFORM="$1"
106  REVISION="$2"
107
108  PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM"
109  RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data"
110  ACTUAL_OUTPUT_DIR="$PLATFORM_DIR/output-actual"
111  EXPECTED_OUTPUT_DIR="$PLATFORM_DIR/output-expected"
112
113  # Run check_bench_regressions.py .
114  rm -rf $ACTUAL_OUTPUT_DIR
115  mkdir -p $ACTUAL_OUTPUT_DIR
116  COMMAND="python bench/check_bench_regressions.py -a 25th -b $PLATFORM -d $RAW_BENCH_DATA_DIR -e $PLATFORM_DIR/expectations.txt -r $REVISION"
117  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
118  START_TIMESTAMP=$(date +%s)
119  $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr
120  echo $? >$ACTUAL_OUTPUT_DIR/return_value
121  END_TIMESTAMP=$(date +%s)
122
123  SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP)
124  echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete"
125
126  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
127}
128
129# Run jsondiff.py with arguments in $1, recording its output.
130# Then compare that output to the content of $2/output-expected.
131function jsondiff_test {
132  if [ $# != 2 ]; then
133    echo "jsondiff_test requires exactly 2 parameters, got $#"
134    exit 1
135  fi
136  ARGS="$1"
137  ACTUAL_OUTPUT_DIR="$2/output-actual"
138  EXPECTED_OUTPUT_DIR="$2/output-expected"
139
140  rm -rf $ACTUAL_OUTPUT_DIR
141  mkdir -p $ACTUAL_OUTPUT_DIR
142  COMMAND="python tools/jsondiff.py $ARGS"
143  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
144  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
145  echo $? >$ACTUAL_OUTPUT_DIR/return_value
146
147  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
148}
149
150
151
152#
153# Run skdiff tests...
154#
155
156SKDIFF_TESTDIR=tools/tests/skdiff
157
158# Run skdiff over a variety of file pair types: identical bits, identical pixels, missing from
159# baseDir, etc.
160skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1"
161
162# Run skdiff over the same set of files, but with arguments as used by our buildbots:
163# - return the number of mismatching file pairs (but ignore any files missing from either
164#   baseDir or comparisonDir)
165# - list filenames with each result type to stdout
166# - don't generate HTML output files
167skdiff_test "--failonresult DifferentPixels --failonresult DifferentSizes --failonresult Unknown --failonstatus CouldNotDecode,CouldNotRead any --failonstatus any CouldNotDecode,CouldNotRead --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
168
169# Run skdiff over just the files that have identical bits.
170skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
171
172# Run skdiff over just the files that have identical bits or identical pixels.
173skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
174
175#
176# Run bench alerts tests...
177#
178
179# Parse a collection of bench data
180PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release
181REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34
182download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES"
183download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES"
184benchalert_test $PLATFORM $REVISION
185
186#
187# Test jsondiff.py ...
188#
189
190JSONDIFF_INPUT=tools/tests/jsondiff/input
191JSONDIFF_OUTPUT=tools/tests/jsondiff/output
192jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUTPUT/old-vs-new"
193
194
195#
196# Launch all the self-tests which have been written in Python.
197#
198# TODO: Over time, we should move all of our tests into Python, and delete
199# the bash tests above.
200# See https://code.google.com/p/skia/issues/detail?id=677
201# ('make tools/tests/run.sh work cross-platform')
202#
203
204COMMAND="python tools/test_all.py"
205echo "$COMMAND"
206$COMMAND
207ret=$?
208if [ $ret -ne 0 ]; then
209    echo "failure in Python self-tests; see stack trace above"
210    exit 1
211fi
212
213
214echo "All tests passed."
215