1#!/usr/bin/python3
2##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
3# 
4#                     The LLVM Compiler Infrastructure
5#
6# This file is distributed under the University of Illinois Open Source
7# License. See LICENSE.TXT for details.
8# 
9##===----------------------------------------------------------------------===##
10#
11# This script builds many different flavors of the LLVM ecosystem.  It
12# will build LLVM, Clang, llvm-gcc, and dragonegg as well as run tests
13# on them.  This script is convenient to use to check builds and tests
14# before committing changes to the upstream repository
15#
16# A typical source setup uses three trees and looks like this:
17#
18# official
19#   dragonegg
20#     trunk
21#   gcc
22#     trunk
23#   llvm
24#     trunk
25#       tools
26#         clang
27#     tags
28#       RELEASE_28
29#         tools
30#           clang
31#   llvm-gcc
32#     trunk
33#     tags
34#       RELEASE_28
35# staging
36#   dragonegg
37#     trunk
38#   gcc
39#     trunk
40#   llvm
41#     trunk
42#       tools
43#         clang
44#     tags
45#       RELEASE_28
46#         tools
47#           clang
48#   llvm-gcc
49#     trunk
50#     tags
51#       RELEASE_28
52# commit
53#   dragonegg
54#     trunk
55#   gcc
56#     trunk
57#   llvm
58#     trunk
59#       tools
60#         clang
61#     tags
62#       RELEASE_28
63#         tools
64#           clang
65#   llvm-gcc
66#     trunk
67#     tags
68#       RELEASE_28
69#
70# "gcc" above is the upstream FSF gcc and "gcc/trunk" refers to the
71# 4.5 branch as discussed in the dragonegg build guide.
72#
73# In a typical workflow, the "official" tree always contains unchanged
74# sources from the main LLVM project repositories.  The "staging" tree
75# is where local work is done.  A set of changes resides there waiting
76# to be moved upstream.  The "commit" tree is where changes from
77# "staging" make their way upstream.  Individual incremental changes
78# from "staging" are applied to "commit" and committed upstream after
79# a successful build and test run.  A successful build is one in which
80# testing results in no more failures than seen in the testing of the
81# "official" tree.
82# 
83# A build may be invoked as such:
84#
85# llvmbuild --src=~/llvm/commit --src=~/llvm/staging
86#   --src=~/llvm/official --branch=trunk --branch=tags/RELEASE_28
87#   --build=debug --build=release --build=paranoid
88#   --prefix=/home/greened/install --builddir=/home/greened/build
89#
90# This will build the LLVM ecosystem, including LLVM, Clang, llvm-gcc,
91# gcc 4.5 and dragonegg, putting build results in ~/build and
92# installing tools in ~/install.  llvmbuild creates separate build and
93# install directories for each source/branch/build flavor.  In the
94# above example, llvmbuild will build debug, release and paranoid
95# (debug+checks) flavors of the trunk and RELEASE_28 branches from
96# each source tree (official, staging and commit) for a total of
97# eighteen builds.  All builds will be run in parallel.
98#
99# The user may control parallelism via the --jobs and --threads
100# switches.  --jobs tells llvmbuild the maximum total number of builds
101# to activate in parallel.  The user may think of it as equivalent to
102# the GNU make -j switch.  --threads tells llvmbuild how many worker
103# threads to use to accomplish those builds.  If --threads is less
104# than --jobs, --threads workers will be launched and each one will
105# pick a source/branch/flavor combination to build.  Then llvmbuild
106# will invoke GNU make with -j (--jobs / --threads) to use up the
107# remaining job capacity.  Once a worker is finished with a build, it
108# will pick another combination off the list and start building it.
109#
110##===----------------------------------------------------------------------===##
111
112import optparse
113import os
114import sys
115import threading
116import queue
117import logging
118import traceback
119import subprocess
120import re
121
122# TODO: Use shutil.which when it is available (3.2 or later)
123def find_executable(executable, path=None):
124    """Try to find 'executable' in the directories listed in 'path' (a
125    string listing directories separated by 'os.pathsep'; defaults to
126    os.environ['PATH']).  Returns the complete filename or None if not
127    found
128    """
129    if path is None:
130        path = os.environ['PATH']
131    paths = path.split(os.pathsep)
132    extlist = ['']
133    if os.name == 'os2':
134        (base, ext) = os.path.splitext(executable)
135        # executable files on OS/2 can have an arbitrary extension, but
136        # .exe is automatically appended if no dot is present in the name
137        if not ext:
138            executable = executable + ".exe"
139    elif sys.platform == 'win32':
140        pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
141        (base, ext) = os.path.splitext(executable)
142        if ext.lower() not in pathext:
143            extlist = pathext
144    for ext in extlist:
145        execname = executable + ext
146        if os.path.isfile(execname):
147            return execname
148        else:
149            for p in paths:
150                f = os.path.join(p, execname)
151                if os.path.isfile(f):
152                    return f
153    else:
154        return None
155
156def is_executable(fpath):
157    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
158
159def add_options(parser):
160    parser.add_option("-v", "--verbose", action="store_true",
161                      default=False,
162                      help=("Output informational messages"
163                            " [default: %default]"))
164    parser.add_option("--src", action="append",
165                      help=("Top-level source directory [default: %default]"))
166    parser.add_option("--build", action="append",
167                      help=("Build types to run [default: %default]"))
168    parser.add_option("--branch", action="append",
169                      help=("Source branch to build [default: %default]"))
170    parser.add_option("--cc", default=find_executable("cc"),
171                      help=("The C compiler to use [default: %default]"))
172    parser.add_option("--cxx", default=find_executable("c++"),
173                      help=("The C++ compiler to use [default: %default]"))
174    parser.add_option("--threads", default=4, type="int",
175                      help=("The number of worker threads to use "
176                            "[default: %default]"))
177    parser.add_option("--jobs", "-j", default=8, type="int",
178                      help=("The number of simultaneous build jobs "
179                            "[default: %default]"))
180    parser.add_option("--prefix",
181                      help=("Root install directory [default: %default]"))
182    parser.add_option("--builddir",
183                      help=("Root build directory [default: %default]"))
184    parser.add_option("--extra-llvm-config-flags", default="",
185                      help=("Extra flags to pass to llvm configure [default: %default]"))
186    parser.add_option("--extra-llvm-gcc-config-flags", default="",
187                      help=("Extra flags to pass to llvm-gcc configure [default: %default]"))
188    parser.add_option("--extra-gcc-config-flags", default="",
189                      help=("Extra flags to pass to gcc configure [default: %default]"))
190    parser.add_option("--force-configure", default=False, action="store_true",
191                      help=("Force reconfigure of all components"))
192    parser.add_option("--no-gcc", default=False, action="store_true",
193                      help=("Do not build dragonegg and gcc"))
194    parser.add_option("--no-install", default=False, action="store_true",
195                      help=("Do not do installs"))
196    return
197
198def check_options(parser, options, valid_builds):
199    # See if we're building valid flavors.
200    for build in options.build:
201        if (build not in valid_builds):
202            parser.error("'" + build + "' is not a valid build flavor "
203                         + str(valid_builds))
204
205    # See if we can find source directories.
206    for src in options.src:
207        for component in components:
208            component = component.rstrip("2")
209            compsrc = src + "/" + component
210            if (not os.path.isdir(compsrc)):
211                parser.error("'" + compsrc + "' does not exist")
212                if (options.branch is not None):
213                    for branch in options.branch:
214                        if (not os.path.isdir(os.path.join(compsrc, branch))):
215                            parser.error("'" + os.path.join(compsrc, branch)
216                                         + "' does not exist")
217
218    # See if we can find the compilers
219    options.cc = find_executable(options.cc)
220    options.cxx = find_executable(options.cxx)
221
222    return
223
224# Find a unique short name for the given set of paths.  This searches
225# back through path components until it finds unique component names
226# among all given paths.
227def get_path_abbrevs(paths):
228    # Find the number of common starting characters in the last component
229    # of the paths.
230    unique_paths = list(paths)
231
232    class NotFoundException(Exception): pass
233
234    # Find a unique component of each path.
235    unique_bases = unique_paths[:]
236    found = 0
237    while len(unique_paths) > 0:
238        bases = [os.path.basename(src) for src in unique_paths]
239        components = { c for c in bases }
240        # Account for single entry in paths.
241        if len(components) > 1 or len(components) == len(bases):
242            # We found something unique.
243            for c in components:
244                if bases.count(c) == 1:
245                   index = bases.index(c)
246                   unique_bases[index] = c
247                   # Remove the corresponding path from the set under
248                   # consideration.
249                   unique_paths[index] = None
250            unique_paths = [ p for p in unique_paths if p is not None ]
251        unique_paths = [os.path.dirname(src) for src in unique_paths]
252
253    if len(unique_paths) > 0:
254        raise NotFoundException()
255
256    abbrevs = dict(zip(paths, [base for base in unique_bases]))
257
258    return abbrevs
259
260# Given a set of unique names, find a short character sequence that
261# uniquely identifies them.
262def get_short_abbrevs(unique_bases):
263    # Find a unique start character for each path base.
264    my_unique_bases = unique_bases[:]
265    unique_char_starts = unique_bases[:]
266    while len(my_unique_bases) > 0:
267        for start, char_tuple in enumerate(zip(*[base
268                                                 for base in my_unique_bases])):
269            chars = { c for c in char_tuple }
270            # Account for single path.
271            if len(chars) > 1 or len(chars) == len(char_tuple):
272                # We found something unique.
273                for c in chars:
274                    if char_tuple.count(c) == 1:
275                        index = char_tuple.index(c)
276                        unique_char_starts[index] = start
277                        # Remove the corresponding path from the set under
278                        # consideration.
279                        my_unique_bases[index] = None
280                my_unique_bases = [ b for b in my_unique_bases
281                                    if b is not None ]
282                break
283
284    if len(my_unique_bases) > 0:
285        raise NotFoundException()
286
287    abbrevs = [abbrev[start_index:start_index+3]
288               for abbrev, start_index
289               in zip([base for base in unique_bases],
290                      [index for index in unique_char_starts])]
291
292    abbrevs = dict(zip(unique_bases, abbrevs))
293
294    return abbrevs
295
296class Builder(threading.Thread):
297    class ExecutableNotFound(Exception): pass
298    class FileNotExecutable(Exception): pass
299
300    def __init__(self, work_queue, jobs,
301                 build_abbrev, source_abbrev, branch_abbrev,
302                 options):
303        super().__init__()
304        self.work_queue = work_queue
305        self.jobs = jobs
306        self.cc = options.cc
307        self.cxx = options.cxx
308        self.build_abbrev = build_abbrev
309        self.source_abbrev = source_abbrev
310        self.branch_abbrev = branch_abbrev
311        self.build_prefix = options.builddir
312        self.install_prefix = options.prefix
313        self.options = options
314        self.component_abbrev = dict(
315            llvm="llvm",
316            llvm_gcc="lgcc",
317            llvm2="llv2",
318            gcc="ugcc",
319            dagonegg="degg")
320    def run(self):
321        while True:
322            try:
323                source, branch, build = self.work_queue.get()
324                self.dobuild(source, branch, build)
325            except:
326                traceback.print_exc()
327            finally:
328                self.work_queue.task_done()
329
330    def execute(self, command, execdir, env, component):
331        prefix = self.component_abbrev[component.replace("-", "_")]
332        pwd = os.getcwd()
333        if not os.path.exists(execdir):
334            os.makedirs(execdir)
335
336        execenv = os.environ.copy()
337
338        for key, value in env.items():
339            execenv[key] = value
340 
341        self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
342                          + " ".join(command));
343
344        try:
345            proc = subprocess.Popen(command,
346                                    cwd=execdir,
347                                    env=execenv,
348                                    stdout=subprocess.PIPE,
349                                    stderr=subprocess.STDOUT)
350
351            line = proc.stdout.readline()
352            while line:
353                self.logger.info("[" + prefix + "] "
354                                 + str(line, "utf-8").rstrip())
355                line = proc.stdout.readline()
356
357        except:
358            traceback.print_exc()
359
360    # Get a list of C++ include directories to pass to clang.
361    def get_includes(self):
362        # Assume we're building with g++ for now.
363        command = [self.cxx]
364        command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
365        includes = []
366        self.logger.debug(command)
367        try:
368            proc = subprocess.Popen(command,
369                                    stdout=subprocess.PIPE,
370                                    stderr=subprocess.STDOUT)
371
372            gather = False
373            line = proc.stdout.readline()
374            while line:
375                self.logger.debug(line)
376                if re.search("End of search list", str(line)) is not None:
377                    self.logger.debug("Stop Gather")
378                    gather = False
379                if gather:
380                    includes.append(str(line, "utf-8").strip())
381                if re.search("#include <...> search starts", str(line)) is not None:
382                    self.logger.debug("Start Gather")
383                    gather = True
384                line = proc.stdout.readline()
385        except:
386            traceback.print_exc()
387        self.logger.debug(includes)
388        return includes
389
390    def dobuild(self, source, branch, build):
391        build_suffix = ""
392
393        ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
394
395        if branch is not None:
396            sbabbrev = get_short_abbrevs([ab for ab in self.branch_abbrev.values()])
397
398            prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + sbabbrev[self.branch_abbrev[branch]] + "-" + self.build_abbrev[build] + "]"
399            self.install_prefix += "/" + self.source_abbrev[source] + "/" + branch + "/" + build
400            build_suffix += self.source_abbrev[source] + "/" + branch + "/" + build
401        else:
402            prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
403            self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
404            build_suffix += "/" + self.source_abbrev[source] + "/" + build
405
406        self.logger = logging.getLogger(prefix)
407
408        self.logger.debug(self.install_prefix)
409
410        # Assume we're building with gcc for now.
411        cxxincludes = self.get_includes()
412        cxxroot = cxxincludes[0]
413        cxxarch = os.path.basename(cxxincludes[1])
414
415        configure_flags = dict(
416            llvm=dict(debug=["--prefix=" + self.install_prefix,
417                             "--with-extra-options=-Werror",
418                             "--enable-assertions",
419                             "--disable-optimized",
420                             "--with-cxx-include-root=" + cxxroot,
421                             "--with-cxx-include-arch=" + cxxarch],
422                      release=["--prefix=" + self.install_prefix,
423                               "--with-extra-options=-Werror",
424                               "--enable-optimized",
425                               "--with-cxx-include-root=" + cxxroot,
426                               "--with-cxx-include-arch=" + cxxarch],
427                      paranoid=["--prefix=" + self.install_prefix,
428                                "--with-extra-options=-Werror",
429                                "--enable-assertions",
430                                "--enable-expensive-checks",
431                                "--disable-optimized",
432                                "--with-cxx-include-root=" + cxxroot,
433                                "--with-cxx-include-arch=" + cxxarch]),
434            llvm_gcc=dict(debug=["--prefix=" + self.install_prefix,
435                                 "--enable-checking",
436                                 "--program-prefix=llvm-",
437                                 "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
438# Fortran install seems to be broken.
439#                                 "--enable-languages=c,c++,fortran"],
440                                 "--enable-languages=c,c++"],
441                          release=["--prefix=" + self.install_prefix,
442                                   "--program-prefix=llvm-",
443                                   "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
444# Fortran install seems to be broken.
445#                                   "--enable-languages=c,c++,fortran"],
446                                   "--enable-languages=c,c++"],
447                          paranoid=["--prefix=" + self.install_prefix,
448                                    "--enable-checking",
449                                    "--program-prefix=llvm-",
450                                    "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix,
451# Fortran install seems to be broken.
452#                                    "--enable-languages=c,c++,fortran"]),
453                                    "--enable-languages=c,c++"]),
454            llvm2=dict(debug=["--prefix=" + self.install_prefix,
455                              "--with-extra-options=-Werror",
456                              "--enable-assertions",
457                              "--disable-optimized",
458                              "--with-llvmgccdir=" + self.install_prefix + "/bin",
459                              "--with-cxx-include-root=" + cxxroot,
460                              "--with-cxx-include-arch=" + cxxarch],
461                       release=["--prefix=" + self.install_prefix,
462                                "--with-extra-options=-Werror",
463                                "--enable-optimized",
464                                "--with-llvmgccdir=" + self.install_prefix + "/bin",
465                                "--with-cxx-include-root=" + cxxroot,
466                                "--with-cxx-include-arch=" + cxxarch],
467                       paranoid=["--prefix=" + self.install_prefix,
468                                 "--with-extra-options=-Werror",
469                                 "--enable-assertions",
470                                 "--enable-expensive-checks",
471                                 "--disable-optimized",
472                                 "--with-llvmgccdir=" + self.install_prefix + "/bin",
473                                 "--with-cxx-include-root=" + cxxroot,
474                                 "--with-cxx-include-arch=" + cxxarch]),
475            gcc=dict(debug=["--prefix=" + self.install_prefix,
476                            "--enable-checking"],
477                     release=["--prefix=" + self.install_prefix],
478                     paranoid=["--prefix=" + self.install_prefix,
479                               "--enable-checking"]),
480            dragonegg=dict(debug=[],
481                           release=[],
482                           paranoid=[]))
483
484        configure_env = dict(
485            llvm=dict(debug=dict(CC=self.cc,
486                                 CXX=self.cxx),
487                      release=dict(CC=self.cc,
488                                   CXX=self.cxx),
489                      paranoid=dict(CC=self.cc,
490                                    CXX=self.cxx)),
491            llvm_gcc=dict(debug=dict(CC=self.cc,
492                                     CXX=self.cxx),
493                          release=dict(CC=self.cc,
494                                       CXX=self.cxx),
495                          paranoid=dict(CC=self.cc,
496                                        CXX=self.cxx)),
497            llvm2=dict(debug=dict(CC=self.cc,
498                                  CXX=self.cxx),
499                       release=dict(CC=self.cc,
500                                    CXX=self.cxx),
501                       paranoid=dict(CC=self.cc,
502                                     CXX=self.cxx)),
503            gcc=dict(debug=dict(CC=self.cc,
504                                CXX=self.cxx),
505                     release=dict(CC=self.cc,
506                                  CXX=self.cxx),
507                     paranoid=dict(CC=self.cc,
508                                   CXX=self.cxx)),
509            dragonegg=dict(debug=dict(CC=self.cc,
510                                      CXX=self.cxx),
511                           release=dict(CC=self.cc,
512                                        CXX=self.cxx),
513                           paranoid=dict(CC=self.cc,
514                                         CXX=self.cxx)))
515
516        make_flags = dict(
517            llvm=dict(debug=["-j" + str(self.jobs)],
518                      release=["-j" + str(self.jobs)],
519                      paranoid=["-j" + str(self.jobs)]),
520            llvm_gcc=dict(debug=["-j" + str(self.jobs),
521                                 "bootstrap"],
522                          release=["-j" + str(self.jobs),
523                                   "bootstrap"],
524                          paranoid=["-j" + str(self.jobs),
525                                    "bootstrap"]),
526            llvm2=dict(debug=["-j" + str(self.jobs)],
527                       release=["-j" + str(self.jobs)],
528                       paranoid=["-j" + str(self.jobs)]),
529            gcc=dict(debug=["-j" + str(self.jobs),
530                            "bootstrap"],
531                     release=["-j" + str(self.jobs),
532                              "bootstrap"],
533                     paranoid=["-j" + str(self.jobs),
534                               "bootstrap"]),
535            dragonegg=dict(debug=["-j" + str(self.jobs)],
536                           release=["-j" + str(self.jobs)],
537                           paranoid=["-j" + str(self.jobs)]))
538
539        make_env = dict(
540            llvm=dict(debug=dict(),
541                      release=dict(),
542                      paranoid=dict()),
543            llvm_gcc=dict(debug=dict(),
544                          release=dict(),
545                          paranoid=dict()),
546            llvm2=dict(debug=dict(),
547                       release=dict(),
548                       paranoid=dict()),
549            gcc=dict(debug=dict(),
550                     release=dict(),
551                     paranoid=dict()),
552            dragonegg=dict(debug=dict(GCC=self.install_prefix + "/bin/gcc",
553                                      LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
554                           release=dict(GCC=self.install_prefix + "/bin/gcc",
555                                        LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
556                           paranoid=dict(GCC=self.install_prefix + "/bin/gcc",
557                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
558
559        make_install_flags = dict(
560            llvm=dict(debug=["install"],
561                      release=["install"],
562                      paranoid=["install"]),
563            llvm_gcc=dict(debug=["install"],
564                          release=["install"],
565                          paranoid=["install"]),
566            llvm2=dict(debug=["install"],
567                       release=["install"],
568                       paranoid=["install"]),
569            gcc=dict(debug=["install"],
570                     release=["install"],
571                     paranoid=["install"]),
572            dragonegg=dict(debug=["install"],
573                           release=["install"],
574                           paranoid=["install"]))
575
576        make_install_env = dict(
577            llvm=dict(debug=dict(),
578                      release=dict(),
579                      paranoid=dict()),
580            llvm_gcc=dict(debug=dict(),
581                          release=dict(),
582                          paranoid=dict()),
583            llvm2=dict(debug=dict(),
584                       release=dict(),
585                       paranoid=dict()),
586            gcc=dict(debug=dict(),
587                     release=dict(),
588                     paranoid=dict()),
589            dragonegg=dict(debug=dict(),
590                           release=dict(),
591                           paranoid=dict()))
592
593        make_check_flags = dict(
594            llvm=dict(debug=["check"],
595                      release=["check"],
596                      paranoid=["check"]),
597            llvm_gcc=dict(debug=["check"],
598                          release=["check"],
599                          paranoid=["check"]),
600            llvm2=dict(debug=["check"],
601                       release=["check"],
602                       paranoid=["check"]),
603            gcc=dict(debug=["check"],
604                     release=["check"],
605                     paranoid=["check"]),
606            dragonegg=dict(debug=["check"],
607                           release=["check"],
608                           paranoid=["check"]))
609
610        make_check_env = dict(
611            llvm=dict(debug=dict(),
612                      release=dict(),
613                      paranoid=dict()),
614            llvm_gcc=dict(debug=dict(),
615                          release=dict(),
616                          paranoid=dict()),
617            llvm2=dict(debug=dict(),
618                       release=dict(),
619                       paranoid=dict()),
620            gcc=dict(debug=dict(),
621                     release=dict(),
622                     paranoid=dict()),
623            dragonegg=dict(debug=dict(),
624                           release=dict(),
625                           paranoid=dict()))
626
627        for component in components:
628            comp = component[:]
629            
630            if (self.options.no_gcc):
631                if (comp == 'gcc' or comp == 'dragonegg' or comp == 'llvm2'):
632                    self.logger.info("Skipping " + component + " in "
633                                     + builddir)
634                    continue
635
636            srcdir = source + "/" + comp.rstrip("2")
637            builddir = self.build_prefix + "/" + comp + "/" + build_suffix
638            installdir = self.install_prefix
639
640            if (branch is not None):
641                srcdir += "/" + branch
642
643            comp_key = comp.replace("-", "_")
644
645            config_args = configure_flags[comp_key][build][:]
646            config_args.extend(getattr(self.options,
647                                       "extra_" + comp_key.rstrip("2")
648                                       + "_config_flags").split())
649
650            self.logger.info("Configuring " + component + " in " + builddir)
651            self.configure(component, srcdir, builddir,
652                           config_args,
653                           configure_env[comp_key][build])
654
655            self.logger.info("Building " + component + " in " + builddir)
656            self.logger.info("Build: make " + str(make_flags[comp_key][build]))
657            self.make(component, srcdir, builddir,
658                      make_flags[comp_key][build],
659                      make_env[comp_key][build])
660
661            if (not self.options.no_install):
662                self.logger.info("Installing " + component + " in " + installdir)
663                self.make(component, srcdir, builddir,
664                          make_install_flags[comp_key][build],
665                          make_install_env[comp_key][build])
666
667            self.logger.info("Testing " + component + " in " + builddir)
668            self.logger.info("Test: make "
669                             + str(make_check_flags[comp_key][build]))
670            self.make(component, srcdir, builddir,
671                      make_check_flags[comp_key][build],
672                      make_check_env[comp_key][build])
673
674
675    def configure(self, component, srcdir, builddir, flags, env):
676        self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
677                          + str(builddir))
678
679        configure_files = dict(
680            llvm=[(srcdir + "/configure", builddir + "/Makefile")],
681            llvm_gcc=[(srcdir + "/configure", builddir + "/Makefile"),
682                      (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
683            llvm2=[(srcdir + "/configure", builddir + "/Makefile")],
684            gcc=[(srcdir + "/configure", builddir + "/Makefile"),
685                 (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")],
686            dragonegg=[()])
687
688
689        doconfig = False
690        for conf, mf in configure_files[component.replace("-", "_")]:
691            if not os.path.exists(conf):
692                return
693            if os.path.exists(conf) and os.path.exists(mf):
694                confstat = os.stat(conf)
695                makestat = os.stat(mf)
696                if confstat.st_mtime > makestat.st_mtime:
697                    doconfig = True
698                    break
699            else:
700                doconfig = True
701                break
702
703        if not doconfig and not self.options.force_configure:
704            return
705
706        program = srcdir + "/configure"
707        if not is_executable(program):
708            return
709
710        args = [program]
711        args += ["--verbose"]
712        args += flags
713        self.execute(args, builddir, env, component)
714
715    def make(self, component, srcdir, builddir, flags, env):
716        program = find_executable("make")
717        if program is None:
718            raise ExecutableNotFound
719
720        if not is_executable(program):
721            raise FileNotExecutable
722
723        args = [program]
724        args += flags
725        self.execute(args, builddir, env, component)
726
727# Global constants
728build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
729#components = ["llvm", "llvm-gcc", "llvm2", "gcc", "dragonegg"]
730components = ["llvm", "llvm2", "gcc", "dragonegg"]
731
732# Parse options
733parser = optparse.OptionParser(version="%prog 1.0")
734add_options(parser)
735(options, args) = parser.parse_args()
736check_options(parser, options, build_abbrev.keys());
737
738if options.verbose:
739    logging.basicConfig(level=logging.DEBUG,
740                        format='%(name)-13s: %(message)s')
741else:
742    logging.basicConfig(level=logging.INFO,
743                        format='%(name)-13s: %(message)s')
744
745source_abbrev = get_path_abbrevs(set(options.src))
746
747branch_abbrev = None
748if options.branch is not None:
749    branch_abbrev = get_path_abbrevs(set(options.branch))
750
751work_queue = queue.Queue()
752
753jobs = options.jobs // options.threads
754if jobs == 0:
755    jobs = 1
756
757numthreads = options.threads
758
759logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
760                         + str(numthreads) + " threads using " + str(jobs)
761                         + " make jobs")
762
763for t in range(numthreads):
764    builder = Builder(work_queue, jobs,
765                      build_abbrev, source_abbrev, branch_abbrev,
766                      options)
767    builder.daemon = True
768    builder.start()
769
770for build in set(options.build):
771    for source in set(options.src):
772        if options.branch is not None:
773            for branch in set(options.branch):
774                work_queue.put((source, branch, build))
775        else:
776            work_queue.put((source, None, build))
777
778work_queue.join()
779