1#!/usr/bin/perl -w
2
3# Copyright (C) 2007, 2008, 2011 Apple Inc.  All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions
7# are met:
8#
9# 1.  Redistributions of source code must retain the above copyright
10#     notice, this list of conditions and the following disclaimer. 
11# 2.  Redistributions in binary form must reproduce the above copyright
12#     notice, this list of conditions and the following disclaimer in the
13#     documentation and/or other materials provided with the distribution. 
14# 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
15#     its contributors may be used to endorse or promote products derived
16#     from this software without specific prior written permission. 
17#
18# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
19# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
22# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29# This script attempts to find the point at which a regression (or progression)
30# of behavior occurred by searching WebKit nightly builds.
31
32# To override the location where the nightly builds are downloaded or the path
33# to the Safari web browser, create a ~/.bisect-buildsrc file with one or more of
34# the following lines (use "~/" to specify a path from your home directory):
35#
36# $branch = "branch-name";
37# $nightlyDownloadDirectory = "~/path/to/nightly/downloads";
38# $safariPath = "/path/to/Safari.app";
39
40use strict;
41
42use File::Basename;
43use File::Path;
44use File::Spec;
45use File::Temp qw(tempfile);
46use Getopt::Long;
47use Time::HiRes qw(usleep);
48
49sub createTempFile($);
50sub downloadNightly($$$);
51sub findMacOSXVersion();
52sub findNearestNightlyIndex(\@$$);
53sub findSafariVersion($);
54sub loadSettings();
55sub makeNightlyList($$$$);
56sub max($$) { return $_[0] > $_[1] ? $_[0] : $_[1]; }
57sub mountAndRunNightly($$$$);
58sub parseRevisions($$;$);
59sub printStatus($$$);
60sub printTracLink($$);
61sub promptForTest($);
62
63loadSettings();
64
65my %validBranches = map { $_ => 1 } qw(feature-branch trunk);
66my $branch = $Settings::branch;
67my $nightlyDownloadDirectory = $Settings::nightlyDownloadDirectory;
68my $safariPath = $Settings::safariPath;
69
70my @nightlies;
71
72my $isProgression;
73my $localOnly;
74my @revisions;
75my $sanityCheck;
76my $showHelp;
77my $testURL;
78
79# Fix up -r switches in @ARGV
80@ARGV = map { /^(-r)(.+)$/ ? ($1, $2) : $_ } @ARGV;
81
82my $result = GetOptions(
83    "b|branch=s"             => \$branch,
84    "d|download-directory=s" => \$nightlyDownloadDirectory,
85    "h|help"                 => \$showHelp,
86    "l|local!"               => \$localOnly,
87    "p|progression!"         => \$isProgression,
88    "r|revisions=s"          => \&parseRevisions,
89    "safari-path=s"          => \$safariPath,
90    "s|sanity-check!"        => \$sanityCheck,
91);
92$testURL = shift @ARGV;
93
94$branch = "feature-branch" if $branch eq "feature";
95if (!exists $validBranches{$branch}) {
96    print STDERR "ERROR: Invalid branch '$branch'\n";
97    $showHelp = 1;
98}
99
100if (!$result || $showHelp || scalar(@ARGV) > 0) {
101    print STDERR "Search WebKit nightly builds for changes in behavior.\n";
102    print STDERR "Usage: " . basename($0) . " [options] [url]\n";
103    print STDERR <<END;
104  [-b|--branch name]             name of the nightly build branch (default: trunk)
105  [-d|--download-directory dir]  nightly build download directory (default: ~/Library/Caches/WebKit-Nightlies)
106  [-h|--help]                    show this help message
107  [-l|--local]                   only use local (already downloaded) nightlies
108  [-p|--progression]             searching for a progression, not a regression
109  [-r|--revision M[:N]]          specify starting (and optional ending) revisions to search
110  [--safari-path path]           path to Safari application bundle (default: /Applications/Safari.app)
111  [-s|--sanity-check]            verify both starting and ending revisions before bisecting
112END
113    exit 1;
114}
115
116my $nightlyWebSite = "http://nightly.webkit.org";
117my $nightlyBuildsURLBase = $nightlyWebSite . File::Spec->catdir("/builds", $branch, "mac");
118my $nightlyFilesURLBase = $nightlyWebSite . File::Spec->catdir("/files", $branch, "mac");
119
120$nightlyDownloadDirectory = glob($nightlyDownloadDirectory) if $nightlyDownloadDirectory =~ /^~/;
121$safariPath = glob($safariPath) if $safariPath =~ /^~/;
122$safariPath = File::Spec->catdir($safariPath, "Contents/MacOS/Safari") if $safariPath =~ m#\.app/*#;
123
124$nightlyDownloadDirectory = File::Spec->catdir($nightlyDownloadDirectory, $branch);
125if (! -d $nightlyDownloadDirectory) {
126    mkpath($nightlyDownloadDirectory, 0, 0755) || die "Could not create $nightlyDownloadDirectory: $!";
127}
128
129@nightlies = makeNightlyList($localOnly, $nightlyDownloadDirectory, findMacOSXVersion(), findSafariVersion($safariPath));
130
131my $startIndex = $revisions[0] ? findNearestNightlyIndex(@nightlies, $revisions[0], 'ceil') : 0;
132my $endIndex = $revisions[1] ? findNearestNightlyIndex(@nightlies, $revisions[1], 'floor') : $#nightlies;
133
134my $tempFile = createTempFile($testURL);
135
136if ($sanityCheck) {
137    my $didReproduceBug;
138
139    do {
140        printf "\nChecking starting revision r%s...\n",
141            $nightlies[$startIndex]->{rev};
142        downloadNightly($nightlies[$startIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
143        mountAndRunNightly($nightlies[$startIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
144        $didReproduceBug = promptForTest($nightlies[$startIndex]->{rev});
145        $startIndex-- if $didReproduceBug < 0;
146    } while ($didReproduceBug < 0);
147    die "ERROR: Bug reproduced in starting revision!  Do you need to test an earlier revision or for a progression?"
148        if $didReproduceBug && !$isProgression;
149    die "ERROR: Bug not reproduced in starting revision!  Do you need to test an earlier revision or for a regression?"
150        if !$didReproduceBug && $isProgression;
151
152    do {
153        printf "\nChecking ending revision r%s...\n",
154            $nightlies[$endIndex]->{rev};
155        downloadNightly($nightlies[$endIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
156        mountAndRunNightly($nightlies[$endIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
157        $didReproduceBug = promptForTest($nightlies[$endIndex]->{rev});
158        $endIndex++ if $didReproduceBug < 0;
159    } while ($didReproduceBug < 0);
160    die "ERROR: Bug NOT reproduced in ending revision!  Do you need to test a later revision or for a progression?"
161        if !$didReproduceBug && !$isProgression;
162    die "ERROR: Bug reproduced in ending revision!  Do you need to test a later revision or for a regression?"
163        if $didReproduceBug && $isProgression;
164}
165
166printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
167
168my %brokenRevisions = ();
169while (abs($endIndex - $startIndex) > 1) {
170    my $index = $startIndex + int(($endIndex - $startIndex) / 2);
171
172    my $didReproduceBug;
173    do {
174        if (exists $nightlies[$index]) {
175            my $buildsLeft = max(max(0, $endIndex - $index - 1), max(0, $index - $startIndex - 1));
176            my $plural = $buildsLeft == 1 ? "" : "s";
177            printf "\nChecking revision r%s (%d build%s left to test after this)...\n", $nightlies[$index]->{rev}, $buildsLeft, $plural;
178            downloadNightly($nightlies[$index]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
179            mountAndRunNightly($nightlies[$index]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
180            $didReproduceBug = promptForTest($nightlies[$index]->{rev});
181        }
182        if ($didReproduceBug < 0) {
183            $brokenRevisions{$nightlies[$index]->{rev}} = $nightlies[$index]->{file};
184            delete $nightlies[$index];
185            $endIndex--;
186            $index = $startIndex + int(($endIndex - $startIndex) / 2);
187        }
188    } while ($didReproduceBug < 0);
189
190    if ($didReproduceBug && !$isProgression || !$didReproduceBug && $isProgression) {
191        $endIndex = $index;
192    } else {
193        $startIndex = $index;
194    }
195
196    print "\nBroken revisions skipped: r" . join(", r", keys %brokenRevisions) . "\n"
197        if scalar keys %brokenRevisions > 0;
198    printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
199}
200
201printTracLink($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev});
202
203unlink $tempFile if $tempFile;
204
205exit 0;
206
207sub createTempFile($)
208{
209    my ($url) = @_;
210
211    return undef if !$url;
212
213    my ($fh, $tempFile) = tempfile(
214        basename($0) . "-XXXXXXXX",
215        DIR => File::Spec->tmpdir(),
216        SUFFIX => ".html",
217        UNLINK => 0,
218    );
219    print $fh "<meta http-equiv=\"refresh\" content=\"0; $url\">\n";
220    close($fh);
221
222    return $tempFile;
223}
224
225sub downloadNightly($$$)
226{
227    my ($filename, $urlBase, $directory) = @_;
228    my $path = File::Spec->catfile($directory, $filename);
229    if (! -f $path) {
230        print "Downloading $filename to $directory...\n";
231        `curl -# -o '$path' '$urlBase/$filename'`;
232    }
233}
234
235sub findMacOSXVersion()
236{
237    my $version;
238    open(SW_VERS, "-|", "/usr/bin/sw_vers") || die;
239    while (<SW_VERS>) {
240        $version = $1 if /^ProductVersion:\s+([^\s]+)/;
241    }
242    close(SW_VERS);
243    return $version;
244}
245
246sub findNearestNightlyIndex(\@$$)
247{
248    my ($nightlies, $revision, $round) = @_;
249
250    my $lowIndex = 0;
251    my $highIndex = $#{$nightlies};
252
253    return $highIndex if uc($revision) eq 'HEAD' || $revision >= $nightlies->[$highIndex]->{rev};
254    return $lowIndex if $revision <= $nightlies->[$lowIndex]->{rev};
255
256    while (abs($highIndex - $lowIndex) > 1) {
257        my $index = $lowIndex + int(($highIndex - $lowIndex) / 2);
258        if ($revision < $nightlies->[$index]->{rev}) {
259            $highIndex = $index;
260        } elsif ($revision > $nightlies->[$index]->{rev}) {
261            $lowIndex = $index;
262        } else {
263            return $index;
264        }
265    }
266
267    return ($round eq "floor") ? $lowIndex : $highIndex;
268}
269
270sub findSafariVersion($)
271{
272    my ($path) = @_;
273    my $versionPlist = File::Spec->catdir(dirname(dirname($path)), "version.plist");
274    my $version;
275    open(PLIST, "< $versionPlist") || die;
276    while (<PLIST>) {
277        if (m#^\s*<key>CFBundleShortVersionString</key>#) {
278            $version = <PLIST>;
279            $version =~ s#^\s*<string>([0-9.]+)[^<]*</string>\s*[\r\n]*#$1#;
280        }
281    }
282    close(PLIST);
283    return $version;
284}
285
286sub loadSettings()
287{
288    package Settings;
289
290    our $branch = "trunk";
291    our $nightlyDownloadDirectory = File::Spec->catdir($ENV{HOME}, "Library/Caches/WebKit-Nightlies");
292    our $safariPath = "/Applications/Safari.app";
293
294    my $rcfile = File::Spec->catdir($ENV{HOME}, ".bisect-buildsrc");
295    return if !-f $rcfile;
296
297    my $result = do $rcfile;
298    die "Could not parse $rcfile: $@" if $@;
299}
300
301sub makeNightlyList($$$$)
302{
303    my ($useLocalFiles, $localDirectory, $macOSXVersion, $safariVersion) = @_;
304    my @files;
305
306    if ($useLocalFiles) {
307        opendir(DIR, $localDirectory) || die "$!";
308        foreach my $file (readdir(DIR)) {
309            if ($file =~ /^WebKit-SVN-r([0-9]+)\.dmg$/) {
310                push(@files, +{ rev => $1, file => $file });
311            }
312        }
313        closedir(DIR);
314    } else {
315        open(NIGHTLIES, "curl -s $nightlyBuildsURLBase/all |") || die;
316
317        while (my $line = <NIGHTLIES>) {
318            chomp $line;
319            my ($revision, $timestamp, $url) = split(/,/, $line);
320            my $nightly = basename($url);
321            push(@files, +{ rev => $revision, file => $nightly });
322        }
323        close(NIGHTLIES);
324    }
325
326    if (eval "v$macOSXVersion" ge v10.5) {
327        if ($safariVersion eq "4 Public Beta") {
328            @files = grep { $_->{rev} >= 39682 } @files;
329        } elsif (eval "v$safariVersion" ge v3.2) {
330            @files = grep { $_->{rev} >= 37348 } @files;
331        } elsif (eval "v$safariVersion" ge v3.1) {
332            @files = grep { $_->{rev} >= 29711 } @files;
333        } elsif (eval "v$safariVersion" ge v3.0) {
334            @files = grep { $_->{rev} >= 25124 } @files;
335        } elsif (eval "v$safariVersion" ge v2.0) {
336            @files = grep { $_->{rev} >= 19594 } @files;
337        } else {
338            die "Requires Safari 2.0 or newer";
339        }
340    } elsif (eval "v$macOSXVersion" ge v10.4) {
341        if ($safariVersion eq "4 Public Beta") {
342            @files = grep { $_->{rev} >= 39682 } @files;
343        } elsif (eval "v$safariVersion" ge v3.2) {
344            @files = grep { $_->{rev} >= 37348 } @files;
345        } elsif (eval "v$safariVersion" ge v3.1) {
346            @files = grep { $_->{rev} >= 29711 } @files;
347        } elsif (eval "v$safariVersion" ge v3.0) {
348            @files = grep { $_->{rev} >= 19992 } @files;
349        } elsif (eval "v$safariVersion" ge v2.0) {
350            @files = grep { $_->{rev} >= 11976 } @files;
351        } else {
352            die "Requires Safari 2.0 or newer";
353        }
354    } else {
355        die "Requires Mac OS X 10.4 (Tiger) or 10.5 (Leopard)";
356    }
357
358    my $nightlycmp = sub { return $a->{rev} <=> $b->{rev}; };
359
360    return sort $nightlycmp @files;
361}
362
363sub mountAndRunNightly($$$$)
364{
365    my ($filename, $directory, $safari, $tempFile) = @_;
366    my $mountPath = "/Volumes/WebKit";
367    my $webkitApp = File::Spec->catfile($mountPath, "WebKit.app");
368    my $diskImage = File::Spec->catfile($directory, $filename);
369    my $devNull = File::Spec->devnull();
370
371    my $i = 0;
372    while (-e $mountPath) {
373        $i++;
374        usleep 100 if $i > 1;
375        `hdiutil detach '$mountPath' 2> $devNull`;
376        die "Could not unmount $diskImage at $mountPath" if $i > 100;
377    }
378    die "Can't mount $diskImage: $mountPath already exists!" if -e $mountPath;
379
380    print "Mounting disk image and running WebKit...\n";
381    `hdiutil attach '$diskImage'`;
382    $i = 0;
383    while (! -e $webkitApp) {
384        usleep 100;
385        $i++;
386        die "Could not mount $diskImage at $mountPath" if $i > 100;
387    }
388
389    my $frameworkPath;
390    if (-d "/Volumes/WebKit/WebKit.app/Contents/Frameworks") {
391        my $osXVersion = join('.', (split(/\./, findMacOSXVersion()))[0..1]);
392        $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Frameworks/$osXVersion";
393    } else {
394        $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Resources";
395    }
396
397    $tempFile ||= "";
398    `DYLD_FRAMEWORK_PATH=$frameworkPath WEBKIT_UNSET_DYLD_FRAMEWORK_PATH=YES $safari $tempFile`;
399
400    `hdiutil detach '$mountPath' 2> $devNull`;
401}
402
403sub parseRevisions($$;$)
404{
405    my ($optionName, $value, $ignored) = @_;
406
407    if ($value =~ /^r?([0-9]+|HEAD):?$/i) {
408        push(@revisions, $1);
409        die "Too many revision arguments specified" if scalar @revisions > 2;
410    } elsif ($value =~ /^r?([0-9]+):?r?([0-9]+|HEAD)$/i) {
411        $revisions[0] = $1;
412        $revisions[1] = $2;
413    } else {
414        die "Unknown revision '$value':  expected 'M' or 'M:N'";
415    }
416}
417
418sub printStatus($$$)
419{
420    my ($startRevision, $endRevision, $isProgression) = @_;
421    printf "\n%s: r%s  %s: r%s\n",
422        $isProgression ? "Fails" : "Works", $startRevision,
423        $isProgression ? "Works" : "Fails", $endRevision;
424}
425
426sub printTracLink($$)
427{
428    my ($startRevision, $endRevision) = @_;
429    printf("http://trac.webkit.org/log/trunk/?rev=%s&stop_rev=%s\n", $endRevision, $startRevision + 1);
430}
431
432sub promptForTest($)
433{
434    my ($revision) = @_;
435    print "Did the bug reproduce in r$revision (yes/no/broken)? ";
436    my $answer = <STDIN>;
437    return 1 if $answer =~ /^(1|y.*)$/i;
438    return -1 if $answer =~ /^(-1|b.*)$/i; # Broken
439    return 0;
440}
441
442