1#!/usr/bin/perl
2use POSIX qw(strftime);
3use File::Copy;
4use File::Find;
5use Socket;
6
7#
8# Program:  NewNightlyTest.pl
9#
10# Synopsis: Perform a series of tests which are designed to be run nightly.
11#           This is used to keep track of the status of the LLVM tree, tracking
12#           regressions and performance changes. Submits this information
13#           to llvm.org where it is placed into the nightlytestresults database.
14#
15# Syntax:   NightlyTest.pl [OPTIONS] [CVSROOT BUILDDIR WEBDIR]
16#   where
17# OPTIONS may include one or more of the following:
18#
19# MAIN OPTIONS:
20#  -config LLVMPATH If specified, use an existing LLVM build and only run and
21#                   report the test information. The LLVMCONFIG argument should
22#                   be the path to the llvm-config executable in the LLVM build.
23#                   This should be the first argument if given. NOT YET
24#                   IMPLEMENTED.
25#  -nickname NAME   The NAME argument specifieds the nickname this script
26#                   will submit to the nightlytest results repository.
27#  -nouname         Don't include uname data (machine will be identified by nickname only).
28#  -submit-server   Specifies a server to submit the test results too. If this
29#                   option is not specified it defaults to
30#                   llvm.org. This is basically just the address of the
31#                   webserver
32#  -submit-script   Specifies which script to call on the submit server. If
33#                   this option is not specified it defaults to
34#                   /nightlytest/NightlyTestAccept.php. This is basically
35#                   everything after the www.yourserver.org.
36#  -submit-aux      If specified, an auxiliary script to run in addition to the
37#                   normal submit script. The script will be passed the path to
38#                   the "sentdata.txt" file as its sole argument.
39#  -nosubmit        Do not report the test results back to a submit server.
40#
41#
42# BUILD OPTIONS (not used with -config):
43#  -nocheckout      Do not create, checkout, update, or configure
44#                   the source tree.
45#  -noremove        Do not remove the BUILDDIR after it has been built.
46#  -noremoveresults Do not remove the WEBDIR after it has been built.
47#  -noclean         Do not run 'make clean' before building.
48#  -nobuild         Do not build llvm. If tests are enabled perform them
49#                   on the llvm build specified in the build directory
50#  -release         Build an LLVM Release+Asserts version
51#  -release-asserts Build an LLVM Release version
52#  -disable-bindings     Disable building LLVM bindings.
53#  -with-clang      Checkout Clang source into tools/clang.
54#  -compileflags    Next argument specifies extra options passed to make when
55#                   building LLVM.
56#  -use-gmake       Use gmake instead of the default make command to build
57#                   llvm and run tests.
58#  -llvmgccdir      Next argument specifies the llvm-gcc install prefix.
59#
60# TESTING OPTIONS:
61#  -notest          Do not even attempt to run the test programs.
62#  -nodejagnu       Do not run feature or regression tests
63#  -enable-llcbeta  Enable testing of beta features in llc.
64#  -enable-lli      Enable testing of lli (interpreter) features, default is off
65#  -disable-pic	    Disable building with Position Independent Code.
66#  -disable-llc     Disable LLC tests in the nightly tester.
67#  -disable-jit     Disable JIT tests in the nightly tester.
68#  -disable-cbe     Disable C backend tests in the nightly tester.
69#  -disable-lto     Disable link time optimization.
70#  -test-cflags     Next argument specifies that C compilation options that
71#                   override the default when running the testsuite.
72#  -test-cxxflags   Next argument specifies that C++ compilation options that
73#                   override the default when running the testsuite.
74#  -extraflags      Next argument specifies extra options that are passed to
75#                   compile the tests.
76#  -noexternals     Do not run the external tests (for cases where povray
77#                   or SPEC are not installed)
78#  -with-externals  Specify a directory where the external tests are located.
79#
80# OTHER OPTIONS:
81#  -parallel        Run parallel jobs with GNU Make (see -parallel-jobs).
82#  -parallel-jobs   The number of parallel Make jobs to use (default is two).
83#  -parallel-test   Allow parallel execution of llvm-test
84#  -verbose         Turn on some debug output
85#  -nice            Checkout/Configure/Build with "nice" to reduce impact
86#                   on busy servers.
87#  -f2c             Next argument specifies path to F2C utility
88#  -gccpath         Path to gcc/g++ used to build LLVM
89#  -target          Specify the target triplet
90#  -cflags          Next argument specifies that C compilation options that
91#                   override the default.
92#  -cxxflags        Next argument specifies that C++ compilation options that
93#                   override the default.
94#  -ldflags         Next argument specifies that linker options that override
95#                   the default.
96#
97# CVSROOT is ignored, it is passed for backwards compatibility.
98# BUILDDIR is the directory where sources for this test run will be checked out
99#  AND objects for this test run will be built. This directory MUST NOT
100#  exist before the script is run; it will be created by the svn checkout
101#  process and erased (unless -noremove is specified; see above.)
102# WEBDIR is the directory into which the test results web page will be written,
103#  AND in which the "index.html" is assumed to be a symlink to the most recent
104#  copy of the results. This directory will be created if it does not exist.
105# LLVMGCCDIR is the directory in which the LLVM GCC Front End is installed
106#  to. This is the same as you would have for a normal LLVM build.
107#
108##############################################################
109#
110# Getting environment variables
111#
112##############################################################
113my $HOME       = $ENV{'HOME'};
114my $SVNURL     = $ENV{"SVNURL"};
115$SVNURL        = 'http://llvm.org/svn/llvm-project' unless $SVNURL;
116my $TestSVNURL = $ENV{"TestSVNURL"};
117$TestSVNURL    = 'http://llvm.org/svn/llvm-project' unless $TestSVNURL;
118my $BuildDir   = $ENV{'BUILDDIR'};
119my $WebDir     = $ENV{'WEBDIR'};
120
121##############################################################
122#
123# Calculate the date prefix...
124#
125##############################################################
126use POSIX;
127@TIME = localtime;
128my $DATE = strftime("%Y-%m-%d_%H-%M-%S", localtime());
129
130##############################################################
131#
132# Parse arguments...
133#
134##############################################################
135$CONFIG_PATH="";
136$CONFIGUREARGS="";
137$nickname="";
138$NOTEST=0;
139$MAKECMD="make";
140$SUBMITSERVER = "llvm.org";
141$SUBMITSCRIPT = "/nightlytest/NightlyTestAccept.php";
142$SUBMITAUX="";
143$SUBMIT = 1;
144$PARALLELJOBS = "2";
145my $TESTFLAGS="";
146
147if ($ENV{'LLVMGCCDIR'}) {
148  $CONFIGUREARGS .= " --with-llvmgccdir=" . $ENV{'LLVMGCCDIR'};
149  $LLVMGCCPATH = $ENV{'LLVMGCCDIR'} . '/bin';
150}
151else {
152  $LLVMGCCPATH = "";
153}
154
155while (scalar(@ARGV) and ($_ = $ARGV[0], /^[-+]/)) {
156  shift;
157  last if /^--$/;  # Stop processing arguments on --
158
159  # List command line options here...
160  if (/^-config$/)         { $CONFIG_PATH = "$ARGV[0]"; shift; next; }
161  if (/^-nocheckout$/)     { $NOCHECKOUT = 1; next; }
162  if (/^-noclean$/)        { $NOCLEAN = 1; next; }
163  if (/^-noremove$/)       { $NOREMOVE = 1; next; }
164  if (/^-noremoveatend$/)  { $NOREMOVEATEND = 1; next; }
165  if (/^-noremoveresults$/){ $NOREMOVERESULTS = 1; next; }
166  if (/^-notest$/)         { $NOTEST = 1; next; }
167  if (/^-norunningtests$/) { next; } # Backward compatibility, ignored.
168  if (/^-parallel-jobs$/)  { $PARALLELJOBS = "$ARGV[0]"; shift; next;}
169  if (/^-parallel$/)       { $MAKEOPTS = "$MAKEOPTS -j$PARALLELJOBS"; next; }
170  if (/^-parallel-test$/)  { $PROGTESTOPTS .= " ENABLE_PARALLEL_REPORT=1"; next; }
171  if (/^-with-clang$/)     { $WITHCLANG = 1; next; }
172  if (/^-release$/)        { $MAKEOPTS = "$MAKEOPTS ENABLE_OPTIMIZED=1 ".
173                             "OPTIMIZE_OPTION=-O2"; next;}
174  if (/^-release-asserts$/){ $MAKEOPTS = "$MAKEOPTS ENABLE_OPTIMIZED=1 ".
175                             "DISABLE_ASSERTIONS=1 ".
176                             "OPTIMIZE_OPTION=-O2"; next;}
177  if (/^-enable-llcbeta$/) { $PROGTESTOPTS .= " ENABLE_LLCBETA=1"; next; }
178  if (/^-disable-pic$/)    { $CONFIGUREARGS .= " --enable-pic=no"; next; }
179  if (/^-enable-lli$/)     { $PROGTESTOPTS .= " ENABLE_LLI=1";
180                             $CONFIGUREARGS .= " --enable-lli"; next; }
181  if (/^-disable-llc$/)    { $PROGTESTOPTS .= " DISABLE_LLC=1";
182                             $CONFIGUREARGS .= " --disable-llc_diffs"; next; }
183  if (/^-disable-jit$/)    { $PROGTESTOPTS .= " DISABLE_JIT=1";
184                             $CONFIGUREARGS .= " --disable-jit"; next; }
185  if (/^-disable-bindings$/)    { $CONFIGUREARGS .= " --disable-bindings"; next; }
186  if (/^-disable-cbe$/)    { $PROGTESTOPTS .= " DISABLE_CBE=1"; next; }
187  if (/^-disable-lto$/)    { $PROGTESTOPTS .= " DISABLE_LTO=1"; next; }
188  if (/^-test-opts$/)      { $PROGTESTOPTS .= " $ARGV[0]"; shift; next; }
189  if (/^-verbose$/)        { $VERBOSE = 1; next; }
190  if (/^-teelogs$/)        { $TEELOGS = 1; next; }
191  if (/^-nice$/)           { $NICE = "nice "; next; }
192  if (/^-f2c$/)            { $CONFIGUREARGS .= " --with-f2c=$ARGV[0]";
193                             shift; next; }
194  if (/^-with-externals$/) { $CONFIGUREARGS .= " --with-externals=$ARGV[0]";
195                             shift; next; }
196  if (/^-configure-args$/) { $CONFIGUREARGS .= " $ARGV[0]";
197                             shift; next; }
198  if (/^-submit-server/)   { $SUBMITSERVER = "$ARGV[0]"; shift; next; }
199  if (/^-submit-script/)   { $SUBMITSCRIPT = "$ARGV[0]"; shift; next; }
200  if (/^-submit-aux/)      { $SUBMITAUX = "$ARGV[0]"; shift; next; }
201  if (/^-nosubmit$/)       { $SUBMIT = 0; next; }
202  if (/^-nickname$/)       { $nickname = "$ARGV[0]"; shift; next; }
203  if (/^-gccpath/)         { $CONFIGUREARGS .=
204                             " CC=$ARGV[0]/gcc CXX=$ARGV[0]/g++";
205                             $GCCPATH=$ARGV[0]; shift;  next; }
206  else                     { $GCCPATH=""; }
207  if (/^-target/)          { $CONFIGUREARGS .= " --target=$ARGV[0]";
208                             shift; next; }
209  if (/^-cflags/)          { $MAKEOPTS = "$MAKEOPTS C.Flags=\'$ARGV[0]\'";
210                             shift; next; }
211  if (/^-cxxflags/)        { $MAKEOPTS = "$MAKEOPTS CXX.Flags=\'$ARGV[0]\'";
212                             shift; next; }
213  if (/^-ldflags/)         { $MAKEOPTS = "$MAKEOPTS LD.Flags=\'$ARGV[0]\'";
214                             shift; next; }
215  if (/^-test-cflags/)     { $TESTFLAGS = "$TESTFLAGS CFLAGS=\'$ARGV[0]\'";
216                             shift; next; }
217  if (/^-test-cxxflags/)   { $TESTFLAGS = "$TESTFLAGS CXXFLAGS=\'$ARGV[0]\'";
218                             shift; next; }
219  if (/^-compileflags/)    { $MAKEOPTS = "$MAKEOPTS $ARGV[0]"; shift; next; }
220  if (/^-llvmgccdir/)      { $CONFIGUREARGS .= " --with-llvmgccdir=\'$ARGV[0]\'";
221                             $LLVMGCCPATH = $ARGV[0] . '/bin';
222                             shift; next;}
223  if (/^-noexternals$/)    { $NOEXTERNALS = 1; next; }
224  if (/^-nouname$/)        { $NOUNAME = 1; next; }
225  if (/^-use-gmake/)       { $MAKECMD = "gmake"; shift; next; }
226  if (/^-extraflags/)      { $CONFIGUREARGS .=
227                             " --with-extra-options=\'$ARGV[0]\'"; shift; next;}
228  if (/^-noexternals$/)    { $NOEXTERNALS = 1; next; }
229  if (/^-nodejagnu$/)      { next; }
230  if (/^-nobuild$/)        { $NOBUILD = 1; next; }
231  print "Unknown option: $_ : ignoring!\n";
232}
233
234if ($CONFIGUREARGS !~ /--disable-jit/) {
235  $CONFIGUREARGS .= " --enable-jit";
236}
237
238if (@ARGV != 0 and @ARGV != 3) {
239  die "error: must specify 0 or 3 options!";
240}
241
242if (@ARGV == 3) {
243  if ($CONFIG_PATH ne "") {
244      die "error: arguments are unsupported in -config mode,";
245  }
246
247  # ARGV[0] used to be the CVS root, ignored for backward compatibility.
248  $BuildDir   = $ARGV[1];
249  $WebDir     = $ARGV[2];
250}
251
252if ($CONFIG_PATH ne "") {
253  $BuildDir = "";
254  $SVNURL = $TestSVNURL = "";
255  if ($WebDir     eq "") {
256    die("please specify a web directory");
257  }
258} else {
259  if ($BuildDir   eq "" or
260      $WebDir     eq "") {
261    die("please specify a build directory, and a web directory");
262  }
263}
264
265if ($nickname eq "") {
266  die ("Please invoke NewNightlyTest.pl with command line option " .
267       "\"-nickname <nickname>\"");
268}
269
270my $LLVMSrcDir   = $ENV{'LLVMSRCDIR'};
271$LLVMSrcDir    = "$BuildDir/llvm" unless $LLVMSrcDir;
272my $LLVMObjDir   = $ENV{'LLVMOBJDIR'};
273$LLVMObjDir    = "$BuildDir/llvm" unless $LLVMObjDir;
274my $LLVMTestDir   = $ENV{'LLVMTESTDIR'};
275$LLVMTestDir    = "$BuildDir/llvm/projects/llvm-test" unless $LLVMTestDir;
276
277##############################################################
278#
279# Define the file names we'll use
280#
281##############################################################
282
283my $Prefix = "$WebDir/$DATE";
284my $SingleSourceLog = "$Prefix-SingleSource-ProgramTest.txt.gz";
285my $MultiSourceLog = "$Prefix-MultiSource-ProgramTest.txt.gz";
286my $ExternalLog = "$Prefix-External-ProgramTest.txt.gz";
287
288# These are only valid in non-config mode.
289my $ConfigureLog = "", $BuildLog = "", $COLog = "";
290my $DejagnuLog = "", $DejagnuSum = "", $DejagnuLog = "";
291
292# Are we in config mode?
293my $ConfigMode = 0;
294
295##############################################################
296#
297# Helper functions
298#
299##############################################################
300
301sub GetDir {
302  my $Suffix = shift;
303  opendir DH, $WebDir;
304  my @Result = reverse sort grep !/$DATE/, grep /[-0-9]+$Suffix/, readdir DH;
305  closedir DH;
306  return @Result;
307}
308
309sub RunLoggedCommand {
310  my $Command = shift;
311  my $Log = shift;
312  my $Title = shift;
313  if ($TEELOGS) {
314      if ($VERBOSE) {
315          print "$Title\n";
316          print "$Command 2>&1 | tee $Log\n";
317      }
318      system "$Command 2>&1 | tee $Log";
319  } else {
320      if ($VERBOSE) {
321          print "$Title\n";
322          print "$Command > $Log 2>&1\n";
323      }
324      system "$Command > $Log 2>&1";
325  }
326}
327
328sub RunAppendingLoggedCommand {
329  my $Command = shift;
330  my $Log = shift;
331  my $Title = shift;
332  if ($TEELOGS) {
333      if ($VERBOSE) {
334          print "$Title\n";
335          print "$Command 2>&1 | tee -a $Log\n";
336      }
337      system "$Command 2>&1 | tee -a $Log";
338  } else {
339      if ($VERBOSE) {
340          print "$Title\n";
341          print "$Command >> $Log 2>&1\n";
342      }
343      system "$Command >> $Log 2>&1";
344  }
345}
346
347sub GetRegex {   # (Regex with ()'s, value)
348  if ($_[1] =~ /$_[0]/m) {
349    return $1;
350  }
351  return "0";
352}
353
354sub ChangeDir { # directory, logical name
355  my ($dir,$name) = @_;
356  chomp($dir);
357  if ( $VERBOSE ) { print "Changing To: $name ($dir)\n"; }
358  $result = chdir($dir);
359  if (!$result) {
360    print "ERROR!!! Cannot change directory to: $name ($dir) because $!\n";
361    return false;
362  }
363  return true;
364}
365
366sub ReadFile {
367  if (open (FILE, $_[0])) {
368    undef $/;
369    my $Ret = <FILE>;
370    close FILE;
371    $/ = '\n';
372    return $Ret;
373  } else {
374    print "Could not open file '$_[0]' for reading!\n";
375    return "";
376  }
377}
378
379sub WriteFile {  # (filename, contents)
380  open (FILE, ">$_[0]") or die "Could not open file '$_[0]' for writing!\n";
381  print FILE $_[1];
382  close FILE;
383}
384
385sub CopyFile { #filename, newfile
386  my ($file, $newfile) = @_;
387  chomp($file);
388  if ($VERBOSE) { print "Copying $file to $newfile\n"; }
389  copy($file, $newfile);
390}
391
392#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
393#
394# This function acts as a mini web browswer submitting data
395# to our central server via the post method
396#
397#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
398sub WriteSentData {
399    $variables = $_[0];
400
401    # Write out the "...-sentdata.txt" file.
402
403    my $sentdata="";
404    foreach $x (keys (%$variables)){
405        $value = $variables->{$x};
406        $sentdata.= "$x  => $value\n";
407    }
408    WriteFile "$Prefix-sentdata.txt", $sentdata;
409}
410
411sub SendData {
412    $host = $_[0];
413    $file = $_[1];
414    $variables = $_[2];
415
416    if (!($SUBMITAUX eq "")) {
417        system "$SUBMITAUX \"$Prefix-sentdata.txt\"";
418    }
419
420    if (!$SUBMIT) {
421        return "Skipped standard submit.\n";
422    }
423
424    # Create the content to send to the server.
425
426    my $content;
427    foreach $key (keys (%$variables)){
428        $value = $variables->{$key};
429        $value =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
430        $content .= "$key=$value&";
431    }
432
433    # Send the data to the server.
434    #
435    # FIXME: This code should be more robust?
436
437    $port=80;
438    $socketaddr= sockaddr_in $port, inet_aton $host or die "Bad hostname\n";
439    socket SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp') or
440      die "Bad socket\n";
441    connect SOCK, $socketaddr or die "Bad connection\n";
442    select((select(SOCK), $| = 1)[0]);
443
444    $length = length($content);
445
446    my $send= "POST $file HTTP/1.0\n";
447    $send.= "Host: $host\n";
448    $send.= "Content-Type: application/x-www-form-urlencoded\n";
449    $send.= "Content-length: $length\n\n";
450    $send.= "$content";
451
452    print SOCK $send;
453    my $result;
454    while(<SOCK>){
455        $result  .= $_;
456    }
457    close(SOCK);
458
459    return $result;
460}
461
462##############################################################
463#
464# Individual Build & Test Functions
465#
466##############################################################
467
468# Create the source repository directory.
469sub CheckoutSource {
470  die "Invalid call!" unless $ConfigMode == 0;
471  if (-d $BuildDir) {
472    if (!$NOREMOVE) {
473      if ( $VERBOSE ) {
474        print "Build directory exists! Removing it\n";
475      }
476      system "rm -rf $BuildDir";
477      mkdir $BuildDir or die "Could not create checkout directory $BuildDir!";
478    } else {
479      if ( $VERBOSE ) {
480        print "Build directory exists!\n";
481      }
482    }
483  } else {
484    mkdir $BuildDir or die "Could not create checkout directory $BuildDir!";
485  }
486
487  ChangeDir( $BuildDir, "checkout directory" );
488  my $SVNCMD = "$NICE svn co --non-interactive";
489  RunLoggedCommand("( time -p $SVNCMD $SVNURL/llvm/trunk llvm; cd llvm/projects ; " .
490                   "  $SVNCMD $TestSVNURL/test-suite/trunk llvm-test )", $COLog,
491                   "CHECKOUT LLVM");
492  if ($WITHCLANG) {
493      RunLoggedCommand("( cd llvm/tools ; " .
494                       "  $SVNCMD $SVNURL/cfe/trunk clang )", $COLog,
495                       "CHECKOUT CLANG");
496  }
497}
498
499# Build the entire tree, saving build messages to the build log. Returns false
500# on build failure.
501sub BuildLLVM {
502  die "Invalid call!" unless $ConfigMode == 0;
503  my $EXTRAFLAGS = "--enable-spec --with-objroot=.";
504  RunLoggedCommand("(time -p $NICE ./configure $CONFIGUREARGS $EXTRAFLAGS) ",
505                   $ConfigureLog, "CONFIGURE");
506  # Build the entire tree, capturing the output into $BuildLog
507  if (!$NOCLEAN) {
508      RunAppendingLoggedCommand("($NICE $MAKECMD $MAKEOPTS clean)", $BuildLog, "BUILD CLEAN");
509  }
510  RunAppendingLoggedCommand("(time -p $NICE $MAKECMD $MAKEOPTS)", $BuildLog, "BUILD");
511
512  if (`grep -a '^$MAKECMD\[^:]*: .*Error' $BuildLog | wc -l` + 0 ||
513      `grep -a '^$MAKECMD: \*\*\*.*Stop.' $BuildLog | wc -l` + 0) {
514    return 0;
515  }
516
517  return 1;
518}
519
520# Run the named tests (i.e. "SingleSource" "MultiSource" "External")
521sub TestDirectory {
522  my $SubDir = shift;
523  ChangeDir( "$LLVMTestDir/$SubDir",
524             "Programs Test Subdirectory" ) || return ("", "");
525
526  my $ProgramTestLog = "$Prefix-$SubDir-ProgramTest.txt";
527
528  # Make sure to clean the test results.
529  RunLoggedCommand("$MAKECMD -k $MAKEOPTS $PROGTESTOPTS clean $TESTFLAGS",
530                   $ProgramTestLog, "TEST DIRECTORY $SubDir");
531
532  # Run the programs tests... creating a report.nightly.csv file.
533  my $LLCBetaOpts = "";
534  RunLoggedCommand("$MAKECMD -k $MAKEOPTS $PROGTESTOPTS report.nightly.csv ".
535                   "$TESTFLAGS TEST=nightly",
536                   $ProgramTestLog, "TEST DIRECTORY $SubDir");
537  $LLCBetaOpts = `$MAKECMD print-llcbeta-option`;
538
539  my $ProgramsTable;
540  if (`grep -a '^$MAKECMD\[^:]: .*Error' $ProgramTestLog | wc -l` + 0) {
541    $ProgramsTable="Error running test $SubDir\n";
542    print "ERROR TESTING\n";
543  } elsif (`grep -a '^$MAKECMD\[^:]: .*No rule to make target' $ProgramTestLog | wc -l` + 0) {
544    $ProgramsTable="Makefile error running tests $SubDir!\n";
545    print "ERROR TESTING\n";
546  } else {
547    # Create a list of the tests which were run...
548    system "egrep -a 'TEST-(PASS|FAIL)' < $ProgramTestLog ".
549           "| sort > $Prefix-$SubDir-Tests.txt";
550  }
551  $ProgramsTable = ReadFile "report.nightly.csv";
552
553  ChangeDir( "../../..", "Programs Test Parent Directory" );
554  return ($ProgramsTable, $LLCBetaOpts);
555}
556
557# Run all the nightly tests and return the program tables and the list of tests,
558# passes, fails, and xfails.
559sub RunNightlyTest() {
560  ($SSProgs, $llcbeta_options) = TestDirectory("SingleSource");
561  WriteFile "$Prefix-SingleSource-Performance.txt", $SSProgs;
562  ($MSProgs, $llcbeta_options) = TestDirectory("MultiSource");
563  WriteFile "$Prefix-MultiSource-Performance.txt", $MSProgs;
564  if ( ! $NOEXTERNALS ) {
565    ($ExtProgs, $llcbeta_options) = TestDirectory("External");
566    WriteFile "$Prefix-External-Performance.txt", $ExtProgs;
567    system "cat $Prefix-SingleSource-Tests.txt " .
568               "$Prefix-MultiSource-Tests.txt ".
569               "$Prefix-External-Tests.txt | sort > $Prefix-Tests.txt";
570    system "cat $Prefix-SingleSource-Performance.txt " .
571               "$Prefix-MultiSource-Performance.txt ".
572               "$Prefix-External-Performance.txt | sort > $Prefix-Performance.txt";
573  } else {
574    $ExtProgs = "External TEST STAGE SKIPPED\n";
575    if ( $VERBOSE ) {
576      print "External TEST STAGE SKIPPED\n";
577    }
578    system "cat $Prefix-SingleSource-Tests.txt " .
579               "$Prefix-MultiSource-Tests.txt ".
580               " | sort > $Prefix-Tests.txt";
581    system "cat $Prefix-SingleSource-Performance.txt " .
582               "$Prefix-MultiSource-Performance.txt ".
583               " | sort > $Prefix-Performance.txt";
584  }
585
586  # Compile passes, fails, xfails.
587  my $All = (ReadFile "$Prefix-Tests.txt");
588  my @TestSuiteResultLines = split "\n", $All;
589  my ($Passes, $Fails, $XFails) = "";
590
591  for ($x=0; $x < @TestSuiteResultLines; $x++) {
592    if (@TestSuiteResultLines[$x] =~ m/^PASS:/) {
593      $Passes .= "$TestSuiteResultLines[$x]\n";
594    }
595    elsif (@TestSuiteResultLines[$x] =~ m/^FAIL:/) {
596      $Fails .= "$TestSuiteResultLines[$x]\n";
597    }
598    elsif (@TestSuiteResultLines[$x] =~ m/^XFAIL:/) {
599      $XFails .= "$TestSuiteResultLines[$x]\n";
600    }
601  }
602
603  return ($SSProgs, $MSProgs, $ExtProgs, $All, $Passes, $Fails, $XFails);
604}
605
606##############################################################
607#
608# Initialize filenames
609#
610##############################################################
611
612if (! -d $WebDir) {
613  mkdir $WebDir, 0777 or die "Unable to create web directory: '$WebDir'.";
614  if($VERBOSE){
615    warn "$WebDir did not exist; creating it.\n";
616  }
617}
618
619if ($CONFIG_PATH ne "") {
620  $ConfigMode = 1;
621  $LLVMSrcDir = GetRegex "^(.*)\\s+", `$CONFIG_PATH --src-root`;
622  $LLVMObjDir = GetRegex "^(.*)\\s+", `$CONFIG_PATH --obj-root`;
623  # FIXME: Add llvm-config hook for this?
624  $LLVMTestDir = $LLVMObjDir . "/projects/test-suite";
625} else {
626  $ConfigureLog = "$Prefix-Configure-Log.txt";
627  $BuildLog = "$Prefix-Build-Log.txt";
628  $COLog = "$Prefix-CVS-Log.txt";
629}
630
631if ($VERBOSE) {
632  if ($CONFIG_PATH ne "") {
633    print "INITIALIZED (config mode)\n";
634    print "WebDir    = $WebDir\n";
635    print "Prefix    = $Prefix\n";
636    print "LLVM Src  = $LLVMSrcDir\n";
637    print "LLVM Obj  = $LLVMObjDir\n";
638    print "LLVM Test = $LLVMTestDir\n";
639  } else {
640    print "INITIALIZED\n";
641    print "SVN URL  = $SVNURL\n";
642    print "COLog    = $COLog\n";
643    print "BuildDir = $BuildDir\n";
644    print "WebDir   = $WebDir\n";
645    print "Prefix   = $Prefix\n";
646    print "BuildLog = $BuildLog\n";
647  }
648}
649
650##############################################################
651#
652# The actual NewNightlyTest logic.
653#
654##############################################################
655
656$starttime = `date "+20%y-%m-%d %H:%M:%S"`;
657
658my $BuildError = 0, $BuildStatus = "OK";
659if ($ConfigMode == 0) {
660  if (!$NOCHECKOUT) {
661    CheckoutSource();
662  }
663
664  # Build LLVM.
665  ChangeDir( $LLVMSrcDir , "llvm source directory") ;
666  if ($NOCHECKOUT || $NOBUILD) {
667    $BuildStatus = "Skipped by user";
668  } else {
669    if (!BuildLLVM()) {
670      if( $VERBOSE) { print  "\n***ERROR BUILDING TREE\n\n"; }
671      $BuildError = 1;
672      $BuildStatus = "Error: compilation aborted";
673    }
674  }
675}
676
677# Run the llvm-test tests.
678my ($SingleSourceProgramsTable, $MultiSourceProgramsTable, $ExternalProgramsTable,
679    $all_tests, $passes, $fails, $xfails) = "";
680if (!$NOTEST && !$BuildError) {
681  ($SingleSourceProgramsTable, $MultiSourceProgramsTable, $ExternalProgramsTable,
682   $all_tests, $passes, $fails, $xfails) = RunNightlyTest();
683}
684
685$endtime = `date "+20%y-%m-%d %H:%M:%S"`;
686
687# The last bit of logic is to remove the build and web dirs, after sending data
688# to the server.
689
690##############################################################
691#
692# Accumulate the information to send to the server.
693#
694##############################################################
695
696if ( $VERBOSE ) { print "PREPARING LOGS TO BE SENT TO SERVER\n"; }
697
698if ( ! $NOUNAME ) {
699    $machine_data = "uname: ".`uname -a`.
700        "hardware: ".`uname -m`.
701        "os: ".`uname -sr`.
702        "name: ".`uname -n`.
703        "date: ".`date \"+20%y-%m-%d\"`.
704        "time: ".`date +\"%H:%M:%S\"`;
705} else {
706    $machine_data = "uname: (excluded)\n".
707        "hardware: ".`uname -m`.
708        "os: ".`uname -sr`.
709        "name: $nickname\n".
710        "date: ".`date \"+20%y-%m-%d\"`.
711        "time: ".`date +\"%H:%M:%S\"`;
712}
713
714# Get gcc version.
715my $gcc_version_long = "";
716if ($GCCPATH ne "") {
717  $gcc_version_long = `$GCCPATH/gcc --version`;
718} elsif ($ENV{"CC"}) {
719  $gcc_version_long = `$ENV{"CC"} --version`;
720} else {
721  $gcc_version_long = `gcc --version`;
722}
723my $gcc_version = (split '\n', $gcc_version_long)[0];
724
725# Get llvm-gcc target triple.
726#
727# FIXME: This shouldn't be hardwired to llvm-gcc.
728my $llvmgcc_version_long = "";
729if ($LLVMGCCPATH ne "") {
730  $llvmgcc_version_long = `$LLVMGCCPATH/llvm-gcc -v 2>&1`;
731} else {
732  $llvmgcc_version_long = `llvm-gcc -v 2>&1`;
733}
734(split '\n', $llvmgcc_version_long)[1] =~ /Target: (.+)/;
735my $targetTriple = $1;
736
737# Logs.
738my ($ConfigureLogData, $BuildLogData, $CheckoutLogData) = "";
739if ($ConfigMode == 0) {
740  $ConfigureLogData = ReadFile $ConfigureLog;
741  $BuildLogData = ReadFile $BuildLog;
742  $CheckoutLogData = ReadFile $COLog;
743}
744
745# Checkout info.
746my $CheckoutTime_Wall = GetRegex "^real ([0-9.]+)", $CheckoutLogData;
747my $CheckoutTime_User = GetRegex "^user ([0-9.]+)", $CheckoutLogData;
748my $CheckoutTime_Sys = GetRegex "^sys ([0-9.]+)", $CheckoutLogData;
749my $CheckoutTime_CPU = $CVSCheckoutTime_User + $CVSCheckoutTime_Sys;
750
751# Configure info.
752my $ConfigTimeU = GetRegex "^user ([0-9.]+)", $ConfigureLogData;
753my $ConfigTimeS = GetRegex "^sys ([0-9.]+)", $ConfigureLogData;
754my $ConfigTime  = $ConfigTimeU+$ConfigTimeS;  # ConfigTime = User+System
755my $ConfigWallTime = GetRegex "^real ([0-9.]+)",$ConfigureLogData;
756$ConfigTime=-1 unless $ConfigTime;
757$ConfigWallTime=-1 unless $ConfigWallTime;
758
759# Build info.
760my $BuildTimeU = GetRegex "^user ([0-9.]+)", $BuildLogData;
761my $BuildTimeS = GetRegex "^sys ([0-9.]+)", $BuildLogData;
762my $BuildTime  = $BuildTimeU+$BuildTimeS;  # BuildTime = User+System
763my $BuildWallTime = GetRegex "^real ([0-9.]+)", $BuildLogData;
764$BuildTime=-1 unless $BuildTime;
765$BuildWallTime=-1 unless $BuildWallTime;
766
767if ( $VERBOSE ) { print "SEND THE DATA VIA THE POST REQUEST\n"; }
768
769my %hash_of_data = (
770  'machine_data' => $machine_data,
771  'build_data' => $ConfigureLogData . $BuildLogData,
772  'gcc_version' => $gcc_version,
773  'nickname' => $nickname,
774  'dejagnutime_wall' => "0.0",
775  'dejagnutime_cpu' => "0.0",
776  'cvscheckouttime_wall' => $CheckoutTime_Wall,
777  'cvscheckouttime_cpu' => $CheckoutTime_CPU,
778  'configtime_wall' => $ConfigWallTime,
779  'configtime_cpu'=> $ConfigTime,
780  'buildtime_wall' => $BuildWallTime,
781  'buildtime_cpu' => $BuildTime,
782  'buildstatus' => $BuildStatus,
783  'singlesource_programstable' => $SingleSourceProgramsTable,
784  'multisource_programstable' => $MultiSourceProgramsTable,
785  'externalsource_programstable' => $ExternalProgramsTable,
786  'llcbeta_options' => $llcbeta_options,
787  'passing_tests' => $passes,
788  'expfail_tests' => $xfails,
789  'unexpfail_tests' => $fails,
790  'all_tests' => $all_tests,
791  'dejagnutests_results' => "Dejagnu skipped by user choice.",
792  'dejagnutests_log' => "",
793  'starttime' => $starttime,
794  'endtime' => $endtime,
795  'target_triple' => $targetTriple,
796
797  # Unused, but left around for backwards compatibility.
798  'warnings' => "",
799  'cvsusercommitlist' => "",
800  'cvsuserupdatelist' => "",
801  'cvsaddedfiles' => "",
802  'cvsmodifiedfiles' => "",
803  'cvsremovedfiles' => "",
804  'lines_of_code' => "",
805  'cvs_file_count' => 0,
806  'cvs_dir_count' => 0,
807  'warnings_removed' => "",
808  'warnings_added' => "",
809  'new_tests' => "",
810  'removed_tests' => "",
811  'o_file_sizes' => "",
812  'a_file_sizes' => ""
813);
814
815# Write out the "...-sentdata.txt" file.
816WriteSentData \%hash_of_data;
817
818if ($SUBMIT || !($SUBMITAUX eq "")) {
819  my $response = SendData $SUBMITSERVER,$SUBMITSCRIPT,\%hash_of_data;
820  if( $VERBOSE) { print "============================\n$response"; }
821} else {
822  print "============================\n";
823  foreach $x(keys %hash_of_data){
824      print "$x  => $hash_of_data{$x}\n";
825  }
826}
827
828##############################################################
829#
830# Remove the source tree...
831#
832##############################################################
833system ( "$NICE rm -rf $BuildDir")
834  if (!$NOCHECKOUT and !$NOREMOVE and !$NOREMOVEATEND);
835system ( "$NICE rm -rf $WebDir")
836  if (!$NOCHECKOUT and !$NOREMOVE and !$NOREMOVERESULTS);
837