1#!/usr/bin/perl -w
2
3# Copyright (C) 2010, 2011 Apple Inc. All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions
7# are met:
8# 1. Redistributions of source code must retain the above copyright
9#    notice, this list of conditions and the following disclaimer.
10# 2. Redistributions in binary form must reproduce the above copyright
11#    notice, this list of conditions and the following disclaimer in the
12#    documentation and/or other materials provided with the distribution.
13#
14# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24# THE POSSIBILITY OF SUCH DAMAGE.
25
26# Features to add:
27#   - Command line option to run a single test.
28#   - Command line option to run all tests in a suite.
29
30use strict;
31use warnings;
32
33use File::Basename;
34use FindBin;
35use Getopt::Long qw(:config pass_through);
36use IPC::Open3;
37use lib $FindBin::Bin;
38use webkitdirs;
39use Term::ANSIColor qw(:constants);
40
41sub dumpAllTests();
42sub runAllTests();
43sub runAllTestsInSuite($);
44sub runTest($$);
45sub populateTests();
46sub buildTestTool();
47
48my $showHelp = 0;
49my $verbose = 0;
50my $dump = 0;
51
52my $programName = basename($0);
53my $usage = <<EOF;
54Usage: $programName [options]
55  --help                Show this help message
56  -v|--verbose          Verbose output
57  -d|--dump-tests       Dump the names of testcases without running them
58EOF
59
60GetOptions(
61    'help' => \$showHelp,
62    'verbose|v' => \$verbose,
63    'dump|d' => \$dump,
64);
65
66if ($showHelp) {
67   print STDERR $usage;
68   exit 1;
69}
70
71setConfiguration();
72buildTestTool();
73setPathForRunningWebKitApp(\%ENV);
74my %testsToRun = populateTests();
75
76if ($dump) {
77    dumpAllTests();
78    exit 0;
79}
80
81runAllTests();
82
83sub dumpAllTests()
84{
85    print "Dumping test cases\n";
86    print "------------------\n";
87    for my $suite (keys %testsToRun) {
88        print $suite . ":\n";
89        print map { "    " . $_ . "\n" } @{ $testsToRun{$suite} };
90    }
91    print "------------------\n";
92}
93
94sub runAllTests()
95{
96    my $anyFailures = 0;
97    for my $suite (keys %testsToRun) {
98        my $failed = runAllTestsInSuite($suite);
99        if ($failed) {
100            $anyFailures = 1;
101        }
102    }
103    return $anyFailures;
104}
105
106sub runAllTestsInSuite($)
107{
108    my ($suite) = @_;
109    print "Suite: $suite\n";
110
111    my $anyFailures = 0;
112    for my $test (@{$testsToRun{$suite}}) {
113        my $failed = runTest($suite, $test);
114        if ($failed) {
115            $anyFailures = 1;
116        }
117    }
118    
119    return $anyFailures;
120}
121
122sub runTest($$)
123{
124    my ($suite, $testName) = @_;
125    my $test = $suite . "/" . $testName;
126
127    print "    Test: $testName -> ";
128
129    my $result = 0;
130    if (isAppleMacWebKit()) {
131        my $productDir = productDir();
132        $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
133        $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
134        my $apiTesterPath = "$productDir/TestWebKitAPI";
135
136        local *DEVNULL;
137        my ($childIn, $childOut, $childErr);
138        unless ($verbose) {
139            open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
140            $childOut = ">&DEVNULL";
141            $childErr = ">&DEVNULL";
142        } else {
143            $childOut = ">&STDOUT";
144            $childErr = ">&STDERR";
145        }
146
147        my $pid;
148        if (architecture()) {
149            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, $test, @ARGV) or die "Failed to run test: $test.";
150        } else {
151            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, $test, @ARGV) or die "Failed to run test: $test.";
152        }
153
154        close($childIn);
155        close($childOut);
156        close($childErr);
157        close(DEVNULL) unless ($verbose);
158
159        waitpid($pid, 0);
160        $result = $?;
161    } elsif (isAppleWinWebKit()) {
162        my $apiTesterNameSuffix;
163        if (configurationForVisualStudio() ne "Debug_All") {
164            $apiTesterNameSuffix = "";
165        } else {
166            $apiTesterNameSuffix = "_debug";
167        }
168        my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
169        $result = system $apiTesterPath, $test, @ARGV;
170    } else {
171        die "run-api-tests is not supported on this platform.\n"
172    }
173    
174    if ($result == 0) {
175        print BOLD GREEN, "Passed", RESET, "\n";
176    } else {
177        print BOLD RED, "Failed", RESET, "\n";
178    }
179}
180
181sub populateTests()
182{
183    my @tests;
184
185    if (isAppleMacWebKit()) {
186        my $productDir = productDir();
187        $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
188        $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
189        my $apiTesterPath = "$productDir/TestWebKitAPI";
190
191        local *DEVNULL;
192        my ($childIn, $childOut, $childErr);
193        unless ($verbose) {
194            open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
195            $childErr = ">&DEVNULL";
196        } else {
197            $childErr = ">&STDERR";
198        }
199
200        my $pid;
201        if (architecture()) {
202            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, "--dump-tests") or die "Failed to build list of tests!";
203        } else {
204            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, "--dump-tests") or die "Failed to build list of tests!";
205        }
206
207        close($childIn);
208        @tests = <$childOut>;
209        close($childOut);
210        close($childErr);
211        close(DEVNULL) unless ($verbose);
212
213        waitpid($pid, 0);
214        my $result = $?;
215
216        if ($result) {
217            print STDERR "Failed to build list of tests!\n";
218            exit exitStatus($result);
219        }
220    } elsif (isAppleWinWebKit()) {
221        my $apiTesterNameSuffix;
222        if (configurationForVisualStudio() ne "Debug_All") {
223            $apiTesterNameSuffix = "";
224        } else {
225            $apiTesterNameSuffix = "_debug";
226        }
227        my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
228        open(TESTS, "-|", $apiTesterPath, "--dump-tests") or die $!;
229        @tests = <TESTS>;
230        close(TESTS) or die $!;
231    } else {
232        die "run-api-tests is not supported on this platform.\n"
233    }
234
235    my %keyedTests = ();
236    for my $test (@tests) {
237        $test =~ s/[\r\n]*$//;
238        my ($suite, $testName) = split(/\//, $test);
239        push @{$keyedTests{$suite}}, $testName;
240    }
241    
242    return %keyedTests;
243}
244
245sub buildTestTool()
246{
247    chdirWebKit();
248
249    my $buildTestTool = "build-api-tests";
250    print STDERR "Running $buildTestTool\n";
251
252    local *DEVNULL;
253    my ($childIn, $childOut, $childErr);
254    unless ($verbose) {
255        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
256        $childOut = ">&DEVNULL";
257        $childErr = ">&DEVNULL";
258    } else {
259        # When not quiet, let the child use our stdout/stderr.
260        $childOut = ">&STDOUT";
261        $childErr = ">&STDERR";
262    }
263
264    my @args = argumentsForConfiguration();
265    my $buildProcess = open3($childIn, $childOut, $childErr, "Tools/Scripts/$buildTestTool", @args) or die "Failed to run " . $buildTestTool;
266
267    close($childIn);
268    close($childOut);
269    close($childErr);
270    close(DEVNULL) unless ($verbose);
271
272    waitpid($buildProcess, 0);
273    my $buildResult = $?;
274
275    if ($buildResult) {
276        print STDERR "Compiling TestWebKitAPI failed!\n";
277        exit exitStatus($buildResult);
278    }
279}
280