1//
2//                     The LLVM Compiler Infrastructure
3//
4// This file is distributed under the University of Illinois Open Source
5// License. See LICENSE.TXT for details.
6
7//
8//  testfilerunner.m
9//  testObjects
10//
11//  Created by Blaine Garst on 9/24/08.
12//
13
14#import "testfilerunner.h"
15#import <Foundation/Foundation.h>
16#include <stdio.h>
17#include <unistd.h>
18#include <fcntl.h>
19#include <string.h>
20#include <stdlib.h>
21#include <stdbool.h>
22
23bool Everything = false; // do it also with 3 levels of optimization
24bool DoClang = false;
25
26static bool isDirectory(char *path);
27static bool isExecutable(char *path);
28static bool isYounger(char *source, char *binary);
29static bool readErrorFile(char *buffer, const char *from);
30
31__strong char *gcstrcpy2(__strong const char *arg, char *endp) {
32    unsigned size = endp - arg + 1;
33    __strong char *result = NSAllocateCollectable(size, 0);
34    strncpy(result, arg, size);
35    result[size-1] = 0;
36    return result;
37}
38__strong char *gcstrcpy1(__strong char *arg) {
39    unsigned size = strlen(arg) + 1;
40    __strong char *result = NSAllocateCollectable(size, 0);
41    strncpy(result, arg, size);
42    result[size-1] = 0;
43    return result;
44}
45
46@implementation TestFileExe
47
48@synthesize options, compileLine, shouldFail, binaryName, sourceName;
49@synthesize generator;
50@synthesize libraryPath, frameworkPath;
51
52- (NSString *)description {
53    NSMutableString *result = [NSMutableString new];
54    if (shouldFail) [result appendString:@"fail"];
55    for (id x  in compileLine) {
56        [result appendString:[NSString stringWithFormat:@" %s", (char *)x]];
57    }
58    return result;
59}
60
61- (__strong char *)radar {
62    return generator.radar;
63}
64  
65- (bool) compileUnlessExists:(bool)skip {
66    if (shouldFail) {
67        printf("don't use this to compile anymore!\n");
68        return false;
69    }
70    if (skip && isExecutable(binaryName) && !isYounger(sourceName, binaryName)) return true;
71    int argc = [compileLine count];
72    char *argv[argc+1];
73    for (int i = 0; i < argc; ++i)
74        argv[i] = (char *)[compileLine pointerAtIndex:i];
75    argv[argc] = NULL;
76    pid_t child = fork();
77    if (child == 0) {
78        execv(argv[0], argv);
79        exit(10); // shouldn't happen
80    }
81    if (child < 0) {
82        printf("fork failed\n");
83        return false;
84    }
85    int status = 0;
86    pid_t deadchild = wait(&status);
87    if (deadchild != child) {
88        printf("wait got %d instead of %d\n", deadchild, child);
89        exit(1);
90    }
91    if (WEXITSTATUS(status) == 0) {
92        return true;
93    }
94    printf("run failed\n");
95    return false;
96}
97
98bool lookforIn(char *lookfor, const char *format, pid_t child) {
99    char buffer[512];
100    char got[512];
101    sprintf(buffer, format, child);    
102    bool gotOutput = readErrorFile(got, buffer);
103    if (!gotOutput) {
104        printf("**** didn't get an output file %s to analyze!!??\n", buffer);
105        return false;
106    }
107    char *where = strstr(got, lookfor);
108    if (!where) {
109        printf("didn't find '%s' in output file %s\n", lookfor, buffer);
110        return false;
111    }
112    unlink(buffer);
113    return true;
114}
115
116- (bool) compileWithExpectedFailure {
117    if (!shouldFail) {
118        printf("Why am I being called?\n");
119        return false;
120    }
121    int argc = [compileLine count];
122    char *argv[argc+1];
123    for (int i = 0; i < argc; ++i)
124        argv[i] = (char *)[compileLine pointerAtIndex:i];
125    argv[argc] = NULL;
126    pid_t child = fork();
127    char buffer[512];
128    if (child == 0) {
129        // in child
130        sprintf(buffer, "/tmp/errorfile_%d", getpid());
131        close(1);
132        int fd = creat(buffer, 0777);
133        if (fd != 1) {
134            fprintf(stderr, "didn't open custom error file %s as 1, got %d\n", buffer, fd);
135            exit(1);
136        }
137        close(2);
138        dup(1);
139        int result = execv(argv[0], argv);
140        exit(10);
141    }
142    if (child < 0) {
143        printf("fork failed\n");
144        return false;
145    }
146    int status = 0;
147    pid_t deadchild = wait(&status);
148    if (deadchild != child) {
149        printf("wait got %d instead of %d\n", deadchild, child);
150        exit(11);
151    }
152    if (WIFEXITED(status)) {
153        if (WEXITSTATUS(status) == 0) {
154            return false;
155        }
156    }
157    else {
158        printf("***** compiler borked/ICEd/died unexpectedly (status %x)\n", status);
159        return false;
160    }
161    char *error = generator.errorString;
162    
163    if (!error) return true;
164#if 0
165    char got[512];
166    sprintf(buffer, "/tmp/errorfile_%d", child);    
167    bool gotOutput = readErrorFile(got, buffer);
168    if (!gotOutput) {
169        printf("**** didn't get an error file %s to analyze!!??\n", buffer);
170        return false;
171    }
172    char *where = strstr(got, error);
173    if (!where) {
174        printf("didn't find '%s' in error file %s\n", error, buffer);
175        return false;
176    }
177    unlink(buffer);
178#else
179    if (!lookforIn(error, "/tmp/errorfile_%d", child)) return false;
180#endif
181    return true;
182}
183
184- (bool) run {
185    if (shouldFail) return true;
186    if (sizeof(long) == 4 && options & Do64) {
187        return true;    // skip 64-bit tests
188    }
189    int argc = 1;
190    char *argv[argc+1];
191    argv[0] = binaryName;
192    argv[argc] = NULL;
193    pid_t child = fork();
194    if (child == 0) {
195        // set up environment
196        char lpath[1024];
197        char fpath[1024];
198        char *myenv[3];
199        int counter = 0;
200        if (libraryPath) {
201            sprintf(lpath, "DYLD_LIBRARY_PATH=%s", libraryPath);
202            myenv[counter++] = lpath;
203        }
204        if (frameworkPath) {
205            sprintf(fpath, "DYLD_FRAMEWORK_PATH=%s", frameworkPath);
206            myenv[counter++] = fpath;
207        }
208        myenv[counter] = NULL;
209        if (generator.warningString) {
210            // set up stdout/stderr
211            char outfile[1024];
212            sprintf(outfile, "/tmp/stdout_%d", getpid());
213            close(2);
214            close(1);
215            creat(outfile, 0700);
216            dup(1);
217        }
218        execve(argv[0], argv, myenv);
219        exit(10); // shouldn't happen
220    }
221    if (child < 0) {
222        printf("fork failed\n");
223        return false;
224    }
225    int status = 0;
226    pid_t deadchild = wait(&status);
227    if (deadchild != child) {
228        printf("wait got %d instead of %d\n", deadchild, child);
229        exit(1);
230    }
231    if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
232        if (generator.warningString) {
233            if (!lookforIn(generator.warningString, "/tmp/stdout_%d", child)) return false;
234        }
235        return true;
236    }
237    printf("**** run failed for %s\n", binaryName);
238    return false;
239}
240
241@end
242
243@implementation TestFileExeGenerator
244@synthesize filename, compilerPath, errorString;
245@synthesize hasObjC, hasRR, hasGC, hasCPlusPlus, wantsC99, supposedToNotCompile, open, wants32, wants64;
246@synthesize radar;
247@synthesize warningString;
248
249- (void)setFilename:(__strong char *)name {
250    filename = gcstrcpy1(name);
251}
252- (void)setCompilerPath:(__strong char *)name {
253    compilerPath = gcstrcpy1(name);
254}
255
256- (void)forMostThings:(NSMutableArray *)lines options:(int)options {
257    TestFileExe *item = nil;
258    item = [self lineForOptions:options];
259    if (item) [lines addObject:item];
260    item = [self lineForOptions:options|Do64];
261    if (item) [lines addObject:item];
262    item = [self lineForOptions:options|DoCPP];
263    if (item) [lines addObject:item];
264    item = [self lineForOptions:options|Do64|DoCPP];
265    if (item) [lines addObject:item];
266}
267
268/*
269    DoDashG = (1 << 8),
270    DoDashO = (1 << 9),
271    DoDashOs = (1 << 10),
272    DoDashO2 = (1 << 11),
273*/
274
275- (void)forAllThings:(NSMutableArray *)lines options:(int)options {
276    [self forMostThings:lines options:options];
277    if (!Everything) {
278        return;
279    }
280    // now do it with three explicit optimization flags
281    [self forMostThings:lines options:options | DoDashO];
282    [self forMostThings:lines options:options | DoDashOs];
283    [self forMostThings:lines options:options | DoDashO2];
284}
285
286- (NSArray *)allLines {
287    NSMutableArray *result = [NSMutableArray new];
288    TestFileExe *item = nil;
289    
290    int options = 0;
291    [self forAllThings:result options:0];
292    [self forAllThings:result options:DoOBJC | DoRR];
293    [self forAllThings:result options:DoOBJC | DoGC];
294    [self forAllThings:result options:DoOBJC | DoGCRR];
295    //[self forAllThings:result options:DoOBJC | DoRRGC];
296    
297    return result;
298}
299
300- (void)addLibrary:(const char *)dashLSomething {
301    if (!extraLibraries) {
302        extraLibraries = [NSPointerArray pointerArrayWithOptions:
303            NSPointerFunctionsStrongMemory |
304            NSPointerFunctionsCStringPersonality];
305    }
306    [extraLibraries addPointer:(void *)dashLSomething];
307}
308
309- (TestFileExe *)lineForOptions:(int)options { // nil if no can do
310    if (hasObjC && !(options & DoOBJC)) return nil;
311    if (hasCPlusPlus && !(options & DoCPP)) return nil;
312    if (hasObjC) {
313        if (!hasGC && (options & (DoGC|DoGCRR))) return nil; // not smart enough
314        if (!hasRR && (options & (DoRR|DoRRGC))) return nil;
315    }
316    NSPointerArray *pa = [NSPointerArray pointerArrayWithOptions:
317        NSPointerFunctionsStrongMemory |
318        NSPointerFunctionsCStringPersonality];
319    // construct path
320    char path[512];
321    path[0] = 0;
322    if (!compilerPath) compilerPath = "/usr/bin";
323    if (compilerPath) {
324        strcat(path, compilerPath);
325        strcat(path, "/");
326    }
327    if (options & DoCPP) {
328        strcat(path, DoClang ? "clang++" : "g++-4.2");
329    }
330    else {
331        strcat(path, DoClang ? "clang" : "gcc-4.2");
332    }
333    [pa addPointer:gcstrcpy1(path)];
334    if (options & DoOBJC) {
335        if (options & DoCPP) {
336            [pa addPointer:"-ObjC++"];
337        }
338        else {
339            [pa addPointer:"-ObjC"];
340        }
341    }
342    [pa addPointer:"-g"];
343    if (options & DoDashO) [pa addPointer:"-O"];
344    else if (options & DoDashO2) [pa addPointer:"-O2"];
345    else if (options & DoDashOs) [pa addPointer:"-Os"];
346    if (wantsC99 && (! (options & DoCPP))) {
347        [pa addPointer:"-std=c99"];
348        [pa addPointer:"-fblocks"];
349    }
350    [pa addPointer:"-arch"];
351    [pa addPointer: (options & Do64) ? "x86_64" : "i386"];
352    
353    if (options & DoOBJC) {
354        switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) {
355        case DoRR:
356            break;
357        case DoGC:
358            [pa addPointer:"-fobjc-gc-only"];
359            break;
360        case DoGCRR:
361            [pa addPointer:"-fobjc-gc"];
362            break;
363        case DoRRGC:
364            printf("DoRRGC unsupported right now\n");
365            [pa addPointer:"-c"];
366            return nil;
367        }
368        [pa addPointer:"-framework"];
369        [pa addPointer:"Foundation"];
370    }
371    [pa addPointer:gcstrcpy1(filename)];
372    [pa addPointer:"-o"];
373    
374    path[0] = 0;
375    strcat(path, filename);
376    strcat(path, ".");
377    strcat(path, (options & Do64) ? "64" : "32");
378    if (options & DoOBJC) {
379        switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) {
380        case DoRR: strcat(path, "-rr"); break;
381        case DoGC: strcat(path, "-gconly"); break;
382        case DoGCRR: strcat(path, "-gcrr"); break;
383        case DoRRGC: strcat(path, "-rrgc"); break;
384        }
385    }
386    if (options & DoCPP) strcat(path, "++");
387    if (options & DoDashO) strcat(path, "-O");
388    else if (options & DoDashO2) strcat(path, "-O2");
389    else if (options & DoDashOs) strcat(path, "-Os");
390    if (wantsC99) strcat(path, "-C99");
391    strcat(path, DoClang ? "-clang" : "-gcc");
392    strcat(path, "-bin");
393    TestFileExe *result = [TestFileExe new];
394    result.binaryName = gcstrcpy1(path); // could snarf copy in pa
395    [pa addPointer:result.binaryName];
396    for (id cString in extraLibraries) {
397        [pa addPointer:cString];
398    }
399    
400    result.sourceName = gcstrcpy1(filename); // could snarf copy in pa
401    result.compileLine = pa;
402    result.options = options;
403    result.shouldFail = supposedToNotCompile;
404    result.generator = self;
405    return result;
406}
407
408+ (NSArray *)generatorsFromPath:(NSString *)path {
409    FILE *fp = fopen([path fileSystemRepresentation], "r");
410    if (fp == NULL) return nil;
411    NSArray *result = [self generatorsFromFILE:fp];
412    fclose(fp);
413    return result;
414}
415
416#define LOOKFOR "CON" "FIG"
417
418char *__strong parseRadar(char *line) {
419    line = strstr(line, "rdar:");   // returns beginning
420    char *endp = line + strlen("rdar:");
421    while (*endp && *endp != ' ' && *endp != '\n')
422        ++endp;
423    return gcstrcpy2(line, endp);
424}
425
426- (void)parseLibraries:(const char *)line {
427  start:
428    line = strstr(line, "-l");
429    char *endp = (char *)line + 2;
430    while (*endp && *endp != ' ' && *endp != '\n')
431        ++endp;
432    [self addLibrary:gcstrcpy2(line, endp)];
433    if (strstr(endp, "-l")) {
434        line = endp;
435        goto start;
436    }
437}
438
439+ (TestFileExeGenerator *)generatorFromLine:(char *)line filename:(char *)filename {
440    TestFileExeGenerator *item = [TestFileExeGenerator new];
441    item.filename = gcstrcpy1(filename);
442    if (strstr(line, "GC")) item.hasGC = true;
443    if (strstr(line, "RR")) item.hasRR = true;
444    if (strstr(line, "C++")) item.hasCPlusPlus = true;
445    if (strstr(line, "-C99")) {
446        item.wantsC99 = true;
447    }
448    if (strstr(line, "64")) item.wants64 = true;
449    if (strstr(line, "32")) item.wants32 = true;
450    if (strstr(line, "-l")) [item parseLibraries:line];
451    if (strstr(line, "open")) item.open = true;
452    if (strstr(line, "FAIL")) item.supposedToNotCompile = true; // old
453    // compile time error
454    if (strstr(line, "error:")) {
455        item.supposedToNotCompile = true;
456        // zap newline
457        char *error = strstr(line, "error:") + strlen("error:");
458        // make sure we have something before the newline
459        char *newline = strstr(error, "\n");
460        if (newline && ((newline-error) > 1)) {
461            *newline = 0;
462            item.errorString = gcstrcpy1(strstr(line, "error:") + strlen("error: "));
463        }
464    }
465    // run time warning
466    if (strstr(line, "runtime:")) {
467        // zap newline
468        char *error = strstr(line, "runtime:") + strlen("runtime:");
469        // make sure we have something before the newline
470        char *newline = strstr(error, "\n");
471        if (newline && ((newline-error) > 1)) {
472            *newline = 0;
473            item.warningString = gcstrcpy1(strstr(line, "runtime:") + strlen("runtime:"));
474        }
475    }
476    if (strstr(line, "rdar:")) item.radar = parseRadar(line);
477    if (item.hasGC || item.hasRR) item.hasObjC = true;
478    if (!item.wants32 && !item.wants64) { // give them both if they ask for neither
479        item.wants32 = item.wants64 = true;
480    }
481    return item;
482}
483
484+ (NSArray *)generatorsFromFILE:(FILE *)fp {
485    NSMutableArray *result = [NSMutableArray new];
486    // pretend this is a grep LOOKFOR *.[cmCM][cmCM] input
487    // look for
488    // filename: ... LOOKFOR [GC] [RR] [C++] [FAIL ...]
489    char buf[512];
490    while (fgets(buf, 512, fp)) {
491        char *config = strstr(buf, LOOKFOR);
492        if (!config) continue;
493        char *filename = buf;
494        char *end = strchr(buf, ':');
495        *end = 0;
496        [result addObject:[self generatorFromLine:config filename:filename]];
497    }
498    return result;
499}
500
501+ (TestFileExeGenerator *)generatorFromFilename:(char *)filename {
502    FILE *fp = fopen(filename, "r");
503    if (!fp) {
504        printf("didn't open %s!!\n", filename);
505        return nil;
506    }
507    char buf[512];
508    while (fgets(buf, 512, fp)) {
509        char *config = strstr(buf, LOOKFOR);
510        if (!config) continue;
511        fclose(fp);
512        return [self generatorFromLine:config filename:filename];
513    }
514    fclose(fp);
515    // guess from filename
516    char *ext = strrchr(filename, '.');
517    if (!ext) return nil;
518    TestFileExeGenerator *result = [TestFileExeGenerator new];
519    result.filename = gcstrcpy1(filename);
520    if (!strncmp(ext, ".m", 2)) {
521        result.hasObjC = true;
522        result.hasRR = true;
523        result.hasGC = true;
524    }
525    else if (!strcmp(ext, ".c")) {
526        ;
527    }
528    else if (!strcmp(ext, ".M") || !strcmp(ext, ".mm")) {
529        result.hasObjC = true;
530        result.hasRR = true;
531        result.hasGC = true;
532        result.hasCPlusPlus = true;
533    }
534    else if (!strcmp(ext, ".cc")
535        || !strcmp(ext, ".cp")
536        || !strcmp(ext, ".cxx")
537        || !strcmp(ext, ".cpp")
538        || !strcmp(ext, ".CPP")
539        || !strcmp(ext, ".c++")
540        || !strcmp(ext, ".C")) {
541        result.hasCPlusPlus = true;
542    }
543    else {
544        printf("unknown extension, file %s ignored\n", filename);
545        result = nil;
546    }
547    return result;
548        
549}
550
551- (NSString *)description {
552    return [NSString stringWithFormat:@"%s: %s%s%s%s%s%s",
553        filename,
554        LOOKFOR,
555        hasGC ? " GC" : "",
556        hasRR ? " RR" : "",
557        hasCPlusPlus ? " C++" : "",
558        wantsC99 ? "C99" : "",
559        supposedToNotCompile ? " FAIL" : ""];
560}
561
562@end
563
564void printDetails(NSArray *failures, const char *whatAreThey) {
565    if ([failures count]) {
566        NSMutableString *output = [NSMutableString new];
567        printf("%s:\n", whatAreThey);
568        for (TestFileExe *line in failures) {
569            printf("%s", line.binaryName);
570            char *radar = line.generator.radar;
571            if (radar)
572                printf(" (due to %s?),", radar);
573            printf(" recompile via:\n%s\n\n", line.description.UTF8String);
574        }
575        printf("\n");
576    }
577}
578
579void help(const char *whoami) {
580    printf("Usage: %s [-fast] [-e] [-dyld librarypath] [gcc4.2dir] [-- | source1 ...]\n", whoami);
581    printf("     -fast              don't recompile if binary younger than source\n");
582    printf("     -open              only run tests that are thought to still be unresolved\n");
583    printf("     -clang             use the clang and clang++ compilers\n");
584    printf("     -e                 compile all variations also with -Os, -O2, -O3\n");
585    printf("     -dyld p            override DYLD_LIBRARY_PATH and DYLD_FRAMEWORK_PATH to p when running tests\n");
586    printf("     <compilerpath>     directory containing gcc-4.2 (or clang) that you wish to use to compile the tests\n");
587    printf("     --                 assume stdin is a grep CON" "FIG across the test sources\n");
588    printf("     otherwise treat each remaining argument as a single test file source\n");
589    printf("%s will compile and run individual test files under a variety of compilers, c, obj-c, c++, and objc++\n", whoami);
590    printf("  .c files are compiled with all four compilers\n");
591    printf("  .m files are compiled with objc and objc++ compilers\n");
592    printf("  .C files are compiled with c++ and objc++ compilers\n");
593    printf("  .M files are compiled only with the objc++ compiler\n");
594    printf("(actually all forms of extensions recognized by the compilers are honored, .cc, .c++ etc.)\n");
595    printf("\nTest files should run to completion with no output and exit (return) 0 on success.\n");
596    printf("Further they should be able to be compiled and run with GC on or off and by the C++ compilers\n");
597    printf("A line containing the string CON" "FIG within the source enables restrictions to the above assumptions\n");
598    printf("and other options.\n");
599    printf("Following CON" "FIG the string\n");
600    printf("    C++ restricts the test to only be run by c++ and objc++ compilers\n");
601    printf("    GC  restricts the test to only be compiled and run with GC on\n");
602    printf("    RR  (retain/release) restricts the test to only be compiled and run with GC off\n");
603    printf("Additionally,\n");
604    printf("    -C99 restricts the C versions of the test to -fstd=c99 -fblocks\n");
605    printf("    -O   adds the -O optimization level\n");
606    printf("    -O2  adds the -O2 optimization level\n");
607    printf("    -Os  adds the -Os optimization level\n");
608    printf("Files that are known to exhibit unresolved problems can provide the term \"open\" and this can");
609    printf("in turn allow highlighting of fixes that have regressed as well as identify that fixes are now available.\n");
610    printf("Files that exhibit known bugs may provide\n");
611    printf("    rdar://whatever such that if they fail the rdar will get cited\n");
612    printf("Files that are expected to fail to compile should provide, as their last token sequence,\n");
613    printf("    error:\n");
614    printf(" or error: substring to match.\n");
615    printf("Files that are expected to produce a runtime error message should provide, as their last token sequence,\n");
616    printf("    warning: string to match\n");
617    printf("\n%s will compile and run all configurations of the test files and report a summary at the end. Good luck.\n", whoami);
618    printf("       Blaine Garst blaine@apple.com\n");
619}
620
621int main(int argc, char *argv[]) {
622    printf("running on %s-bit architecture\n", sizeof(long) == 4 ? "32" : "64");
623    char *compilerDir = "/usr/bin";
624    NSMutableArray *generators = [NSMutableArray new];
625    bool doFast = false;
626    bool doStdin = false;
627    bool onlyOpen = false;
628    char *libraryPath = getenv("DYLD_LIBRARY_PATH");
629    char *frameworkPath = getenv("DYLD_FRAMEWORK_PATH");
630    // process options
631    while (argc > 1) {
632        if (!strcmp(argv[1], "-fast")) {
633            doFast = true;
634            --argc;
635            ++argv;
636        }
637        else if (!strcmp(argv[1], "-dyld")) {
638            doFast = true;
639            --argc;
640            ++argv;
641            frameworkPath = argv[1];
642            libraryPath = argv[1];
643            --argc;
644            ++argv;
645        }
646        else if (!strcmp(argv[1], "-open")) {
647            onlyOpen = true;
648            --argc;
649            ++argv;
650        }
651        else if (!strcmp(argv[1], "-clang")) {
652            DoClang = true;
653            --argc;
654            ++argv;
655        }
656        else if (!strcmp(argv[1], "-e")) {
657            Everything = true;
658            --argc;
659            ++argv;
660        }
661        else if (!strcmp(argv[1], "--")) {
662            doStdin = true;
663            --argc;
664            ++argv;
665        }
666        else if (!strcmp(argv[1], "-")) {
667            help(argv[0]);
668            return 1;
669        }
670        else if (argc > 1 && isDirectory(argv[1])) {
671            compilerDir = argv[1];
672            ++argv;
673            --argc;
674        }
675        else
676            break;
677    }
678    // process remaining arguments, or stdin
679    if (argc == 1) {
680        if (doStdin)
681            generators = (NSMutableArray *)[TestFileExeGenerator generatorsFromFILE:stdin];
682        else {
683            help(argv[0]);
684            return 1;
685        }
686    }
687    else while (argc > 1) {
688        TestFileExeGenerator *generator = [TestFileExeGenerator generatorFromFilename:argv[1]];
689        if (generator) [generators addObject:generator];
690        ++argv;
691        --argc;
692    }
693    // see if we can generate all possibilities
694    NSMutableArray *failureToCompile = [NSMutableArray new];
695    NSMutableArray *failureToFailToCompile = [NSMutableArray new];
696    NSMutableArray *failureToRun = [NSMutableArray new];
697    NSMutableArray *successes = [NSMutableArray new];
698    for (TestFileExeGenerator *generator in generators) {
699        //NSLog(@"got %@", generator);
700        if (onlyOpen && !generator.open) {
701            //printf("skipping resolved test %s\n", generator.filename);
702            continue;  // skip closed if onlyOpen
703        }
704        if (!onlyOpen && generator.open) {
705            //printf("skipping open test %s\n", generator.filename);
706            continue;  // skip open if not asked for onlyOpen
707        }
708        generator.compilerPath = compilerDir;
709        NSArray *tests = [generator allLines];
710        for (TestFileExe *line in tests) {
711            line.frameworkPath = frameworkPath;   // tell generators about it instead XXX
712            line.libraryPath = libraryPath;   // tell generators about it instead XXX
713            if ([line shouldFail]) {
714                if (doFast) continue; // don't recompile & don't count as success
715                if ([line compileWithExpectedFailure]) {
716                    [successes addObject:line];
717                }
718                else
719                    [failureToFailToCompile addObject:line];
720            }
721            else if ([line compileUnlessExists:doFast]) {
722                if ([line run]) {
723                    printf("%s ran successfully\n", line.binaryName);
724                    [successes addObject:line];
725                }
726                else {
727                    [failureToRun addObject:line];
728                }
729            }
730            else {
731                [failureToCompile addObject:line];
732            }
733        }
734    }
735    printf("\n--- results ---\n\n%lu successes\n%lu unexpected compile failures\n%lu failure to fail to compile errors\n%lu run failures\n",
736        [successes count], [failureToCompile count], [failureToFailToCompile count], [failureToRun count]);
737    printDetails(failureToCompile, "unexpected compile failures");
738    printDetails(failureToFailToCompile, "should have failed to compile but didn't failures");
739    printDetails(failureToRun, "run failures");
740    
741    if (onlyOpen && [successes count]) {
742        NSMutableSet *radars = [NSMutableSet new];
743        printf("The following tests ran successfully suggesting that they are now resolved:\n");
744        for (TestFileExe *line in successes) {
745            printf("%s\n", line.binaryName);
746            if (line.radar) [radars addObject:line.generator];
747        }
748        if ([radars count]) {
749            printf("The following radars may be resolved:\n");
750            for (TestFileExeGenerator *line in radars) {
751                printf("%s\n", line.radar);
752            }
753        }
754    }
755            
756    return [failureToCompile count] + [failureToRun count];
757}
758
759#include <sys/stat.h>
760
761static bool isDirectory(char *path) {
762    struct stat statb;
763    int retval = stat(path, &statb);
764    if (retval != 0) return false;
765    if (statb.st_mode & S_IFDIR) return true;
766    return false;
767}
768
769static bool isExecutable(char *path) {
770    struct stat statb;
771    int retval = stat(path, &statb);
772    if (retval != 0) return false;
773    if (!(statb.st_mode & S_IFREG)) return false;
774    if (statb.st_mode & S_IXUSR) return true;
775    return false;
776}
777
778static bool isYounger(char *source, char *binary) {
779    struct stat statb;
780    int retval = stat(binary, &statb);
781    if (retval != 0) return true;  // if doesn't exit, lie
782    
783    struct stat stata;
784    retval = stat(source, &stata);
785    if (retval != 0) return true; // we're hosed
786    // the greater the timeval the younger it is
787    if (stata.st_mtimespec.tv_sec > statb.st_mtimespec.tv_sec) return true;
788    if (stata.st_mtimespec.tv_nsec > statb.st_mtimespec.tv_nsec) return true;
789    return false;
790}
791
792static bool readErrorFile(char *buffer, const char *from) {
793    int fd = open(from, 0);
794    if (fd < 0) {
795        printf("didn't open %s, (might not have been created?)\n", buffer);
796        return false;
797    }
798    int count = read(fd, buffer, 512);
799    if (count < 1) {
800        printf("read error on %s\n", buffer);
801        return false;
802    }
803    buffer[count-1] = 0; // zap newline
804    return true;
805}
806