1#!/usr/bin/python3
2##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
3#
4#                     The LLVM Compiler Infrastructure
5#
6# This file is distributed under the University of Illinois Open Source
7# License. See LICENSE.TXT for details.
8#
9##===----------------------------------------------------------------------===##
10#
11# This script builds many different flavors of the LLVM ecosystem.  It
12# will build LLVM, Clang and dragonegg as well as run tests on them.
13# This script is convenient to use to check builds and tests before
14# committing changes to the upstream repository
15#
16# A typical source setup uses three trees and looks like this:
17#
18# official
19#   dragonegg
20#   llvm
21#     tools
22#       clang
23# staging
24#   dragonegg
25#   llvm
26#     tools
27#       clang
28# commit
29#   dragonegg
30#   llvm
31#     tools
32#       clang
33#
34# In a typical workflow, the "official" tree always contains unchanged
35# sources from the main LLVM project repositories.  The "staging" tree
36# is where local work is done.  A set of changes resides there waiting
37# to be moved upstream.  The "commit" tree is where changes from
38# "staging" make their way upstream.  Individual incremental changes
39# from "staging" are applied to "commit" and committed upstream after
40# a successful build and test run.  A successful build is one in which
41# testing results in no more failures than seen in the testing of the
42# "official" tree.
43# 
44# A build may be invoked as such:
45#
46# llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
47#   --build=debug --build=release --build=paranoid
48#   --prefix=/home/greened/install --builddir=/home/greened/build
49#
50# This will build the LLVM ecosystem, including LLVM, Clangand
51# dragonegg, putting build results in ~/build and installing tools in
52# ~/install.  llvm-compilers-check creates separate build and install
53# directories for each source/build flavor.  In the above example,
54# llvmbuild will build debug, release and paranoid (debug+checks)
55# flavors from each source tree (official, staging and commit) for a
56# total of nine builds.  All builds will be run in parallel.
57#
58# The user may control parallelism via the --jobs and --threads
59# switches.  --jobs tells llvm-compilers-checl the maximum total
60# number of builds to activate in parallel.  The user may think of it
61# as equivalent to the GNU make -j switch.  --threads tells
62# llvm-compilers-check how many worker threads to use to accomplish
63# those builds.  If --threads is less than --jobs, --threads workers
64# will be launched and each one will pick a source/flavor combination
65# to build.  Then llvm-compilers-check will invoke GNU make with -j
66# (--jobs / --threads) to use up the remaining job capacity.  Once a
67# worker is finished with a build, it will pick another combination
68# off the list and start building it.
69#
70##===----------------------------------------------------------------------===##
71
72import optparse
73import os
74import sys
75import threading
76import queue
77import logging
78import traceback
79import subprocess
80import re
81
82# TODO: Use shutil.which when it is available (3.2 or later)
83def find_executable(executable, path=None):
84    """Try to find 'executable' in the directories listed in 'path' (a
85    string listing directories separated by 'os.pathsep'; defaults to
86    os.environ['PATH']).  Returns the complete filename or None if not
87    found
88    """
89    if path is None:
90        path = os.environ['PATH']
91    paths = path.split(os.pathsep)
92    extlist = ['']
93    if os.name == 'os2':
94        (base, ext) = os.path.splitext(executable)
95        # executable files on OS/2 can have an arbitrary extension, but
96        # .exe is automatically appended if no dot is present in the name
97        if not ext:
98            executable = executable + ".exe"
99    elif sys.platform == 'win32':
100        pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
101        (base, ext) = os.path.splitext(executable)
102        if ext.lower() not in pathext:
103            extlist = pathext
104    for ext in extlist:
105        execname = executable + ext
106        if os.path.isfile(execname):
107            return execname
108        else:
109            for p in paths:
110                f = os.path.join(p, execname)
111                if os.path.isfile(f):
112                    return f
113    else:
114        return None
115
116def is_executable(fpath):
117    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
118
119def add_options(parser):
120    parser.add_option("-v", "--verbose", action="store_true",
121                      default=False,
122                      help=("Output informational messages"
123                            " [default: %default]"))
124    parser.add_option("--src", action="append",
125                      help=("Top-level source directory [default: %default]"))
126    parser.add_option("--build", action="append",
127                      help=("Build types to run [default: %default]"))
128    parser.add_option("--cc", default=find_executable("cc"),
129                      help=("The C compiler to use [default: %default]"))
130    parser.add_option("--cxx", default=find_executable("c++"),
131                      help=("The C++ compiler to use [default: %default]"))
132    parser.add_option("--threads", default=4, type="int",
133                      help=("The number of worker threads to use "
134                            "[default: %default]"))
135    parser.add_option("--jobs", "-j", default=8, type="int",
136                      help=("The number of simultaneous build jobs "
137                            "[default: %default]"))
138    parser.add_option("--prefix",
139                      help=("Root install directory [default: %default]"))
140    parser.add_option("--builddir",
141                      help=("Root build directory [default: %default]"))
142    parser.add_option("--extra-llvm-config-flags", default="",
143                      help=("Extra flags to pass to llvm configure [default: %default]"))
144    parser.add_option("--force-configure", default=False, action="store_true",
145                      help=("Force reconfigure of all components"))
146    parser.add_option("--no-dragonegg", default=False, action="store_true",
147                      help=("Do not build dragonegg"))
148    parser.add_option("--no-install", default=False, action="store_true",
149                      help=("Do not do installs"))
150    parser.add_option("--keep-going", default=False, action="store_true",
151                      help=("Keep going after failures"))
152    parser.add_option("--no-flavor-prefix", default=False, action="store_true",
153                      help=("Do not append the build flavor to the install path"))
154    parser.add_option("--enable-werror", default=False, action="store_true",
155                      help=("Build with -Werror"))
156    return
157
158def check_options(parser, options, valid_builds):
159    # See if we're building valid flavors.
160    for build in options.build:
161        if (build not in valid_builds):
162            parser.error("'" + build + "' is not a valid build flavor "
163                         + str(valid_builds))
164
165    # See if we can find source directories.
166    for src in options.src:
167        for component in components:
168            component = component.rstrip("2")
169            compsrc = src + "/" + component
170            if (not os.path.isdir(compsrc)):
171                parser.error("'" + compsrc + "' does not exist")
172
173    # See if we can find the compilers
174    options.cc = find_executable(options.cc)
175    options.cxx = find_executable(options.cxx)
176
177    return
178
179# Find a unique short name for the given set of paths.  This searches
180# back through path components until it finds unique component names
181# among all given paths.
182def get_path_abbrevs(paths):
183    # Find the number of common starting characters in the last component
184    # of the paths.
185    unique_paths = list(paths)
186
187    class NotFoundException(Exception): pass
188
189    # Find a unique component of each path.
190    unique_bases = unique_paths[:]
191    found = 0
192    while len(unique_paths) > 0:
193        bases = [os.path.basename(src) for src in unique_paths]
194        components = { c for c in bases }
195        # Account for single entry in paths.
196        if len(components) > 1 or len(components) == len(bases):
197            # We found something unique.
198            for c in components:
199                if bases.count(c) == 1:
200                   index = bases.index(c)
201                   unique_bases[index] = c
202                   # Remove the corresponding path from the set under
203                   # consideration.
204                   unique_paths[index] = None
205            unique_paths = [ p for p in unique_paths if p is not None ]
206        unique_paths = [os.path.dirname(src) for src in unique_paths]
207
208    if len(unique_paths) > 0:
209        raise NotFoundException()
210
211    abbrevs = dict(zip(paths, [base for base in unique_bases]))
212
213    return abbrevs
214
215# Given a set of unique names, find a short character sequence that
216# uniquely identifies them.
217def get_short_abbrevs(unique_bases):
218    # Find a unique start character for each path base.
219    my_unique_bases = unique_bases[:]
220    unique_char_starts = unique_bases[:]
221    while len(my_unique_bases) > 0:
222        for start, char_tuple in enumerate(zip(*[base
223                                                 for base in my_unique_bases])):
224            chars = { c for c in char_tuple }
225            # Account for single path.
226            if len(chars) > 1 or len(chars) == len(char_tuple):
227                # We found something unique.
228                for c in chars:
229                    if char_tuple.count(c) == 1:
230                        index = char_tuple.index(c)
231                        unique_char_starts[index] = start
232                        # Remove the corresponding path from the set under
233                        # consideration.
234                        my_unique_bases[index] = None
235                my_unique_bases = [ b for b in my_unique_bases
236                                    if b is not None ]
237                break
238
239    if len(my_unique_bases) > 0:
240        raise NotFoundException()
241
242    abbrevs = [abbrev[start_index:start_index+3]
243               for abbrev, start_index
244               in zip([base for base in unique_bases],
245                      [index for index in unique_char_starts])]
246
247    abbrevs = dict(zip(unique_bases, abbrevs))
248
249    return abbrevs
250
251class Builder(threading.Thread):
252    class ExecutableNotFound(Exception): pass
253    class FileNotExecutable(Exception): pass
254
255    def __init__(self, work_queue, jobs,
256                 build_abbrev, source_abbrev,
257                 options):
258        super().__init__()
259        self.work_queue = work_queue
260        self.jobs = jobs
261        self.cc = options.cc
262        self.cxx = options.cxx
263        self.build_abbrev = build_abbrev
264        self.source_abbrev = source_abbrev
265        self.build_prefix = options.builddir
266        self.install_prefix = options.prefix
267        self.options = options
268        self.component_abbrev = dict(
269            llvm="llvm",
270            dragonegg="degg")
271    def run(self):
272        while True:
273            try:
274                source, build = self.work_queue.get()
275                self.dobuild(source, build)
276            except:
277                traceback.print_exc()
278            finally:
279                self.work_queue.task_done()
280
281    def execute(self, command, execdir, env, component):
282        prefix = self.component_abbrev[component.replace("-", "_")]
283        pwd = os.getcwd()
284        if not os.path.exists(execdir):
285            os.makedirs(execdir)
286
287        execenv = os.environ.copy()
288
289        for key, value in env.items():
290            execenv[key] = value
291
292        self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
293                          + " ".join(command));
294
295        try:
296            proc = subprocess.Popen(command,
297                                    cwd=execdir,
298                                    env=execenv,
299                                    stdout=subprocess.PIPE,
300                                    stderr=subprocess.STDOUT)
301
302            line = proc.stdout.readline()
303            while line:
304                self.logger.info("[" + prefix + "] "
305                                 + str(line, "utf-8").rstrip())
306                line = proc.stdout.readline()
307
308            (stdoutdata, stderrdata) = proc.communicate()
309            retcode = proc.wait()
310
311            return retcode
312
313        except:
314            traceback.print_exc()
315
316    # Get a list of C++ include directories to pass to clang.
317    def get_includes(self):
318        # Assume we're building with g++ for now.
319        command = [self.cxx]
320        command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
321        includes = []
322        self.logger.debug(command)
323        try:
324            proc = subprocess.Popen(command,
325                                    stdout=subprocess.PIPE,
326                                    stderr=subprocess.STDOUT)
327
328            gather = False
329            line = proc.stdout.readline()
330            while line:
331                self.logger.debug(line)
332                if re.search("End of search list", str(line)) is not None:
333                    self.logger.debug("Stop Gather")
334                    gather = False
335                if gather:
336                    includes.append(str(line, "utf-8").strip())
337                if re.search("#include <...> search starts", str(line)) is not None:
338                    self.logger.debug("Start Gather")
339                    gather = True
340                line = proc.stdout.readline()
341
342        except:
343            traceback.print_exc()
344        self.logger.debug(includes)
345        return includes
346
347    def dobuild(self, source, build):
348        build_suffix = ""
349
350        ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
351
352        prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
353        if (not self.options.no_flavor_prefix):
354            self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
355
356        build_suffix += "/" + self.source_abbrev[source] + "/" + build
357
358        self.logger = logging.getLogger(prefix)
359
360        self.logger.debug(self.install_prefix)
361
362        # Assume we're building with gcc for now.
363        cxxincludes = self.get_includes()
364        cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
365        cxxroot = os.path.dirname(cxxroot)        # Remove the c++
366        cxxroot = os.path.dirname(cxxroot)        # Remove the include
367
368        configure_flags = dict(
369            llvm=dict(debug=["--prefix=" + self.install_prefix,
370                             "--enable-assertions",
371                             "--disable-optimized",
372                             "--with-gcc-toolchain=" + cxxroot],
373                      release=["--prefix=" + self.install_prefix,
374                               "--enable-optimized",
375                               "--with-gcc-toolchain=" + cxxroot],
376                      paranoid=["--prefix=" + self.install_prefix,
377                                "--enable-assertions",
378                                "--enable-expensive-checks",
379                                "--disable-optimized",
380                                "--with-gcc-toolchain=" + cxxroot]),
381            dragonegg=dict(debug=[],
382                           release=[],
383                           paranoid=[]))
384
385        if (self.options.enable_werror):
386            configure_flags["llvm"]["debug"].append("--enable-werror")
387            configure_flags["llvm"]["release"].append("--enable-werror")
388            configure_flags["llvm"]["paranoid"].append("--enable-werror")
389
390        configure_env = dict(
391            llvm=dict(debug=dict(CC=self.cc,
392                                 CXX=self.cxx),
393                      release=dict(CC=self.cc,
394                                   CXX=self.cxx),
395                      paranoid=dict(CC=self.cc,
396                                    CXX=self.cxx)),
397            dragonegg=dict(debug=dict(CC=self.cc,
398                                      CXX=self.cxx),
399                           release=dict(CC=self.cc,
400                                        CXX=self.cxx),
401                           paranoid=dict(CC=self.cc,
402                                         CXX=self.cxx)))
403
404        make_flags = dict(
405            llvm=dict(debug=["-j" + str(self.jobs)],
406                      release=["-j" + str(self.jobs)],
407                      paranoid=["-j" + str(self.jobs)]),
408            dragonegg=dict(debug=["-j" + str(self.jobs)],
409                           release=["-j" + str(self.jobs)],
410                           paranoid=["-j" + str(self.jobs)]))
411
412        make_env = dict(
413            llvm=dict(debug=dict(),
414                      release=dict(),
415                      paranoid=dict()),
416            dragonegg=dict(debug=dict(GCC=self.cc,
417                                      LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
418                           release=dict(GCC=self.cc,
419                                        LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
420                           paranoid=dict(GCC=self.cc,
421                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
422
423        make_install_flags = dict(
424            llvm=dict(debug=["install"],
425                      release=["install"],
426                      paranoid=["install"]),
427            dragonegg=dict(debug=["install"],
428                           release=["install"],
429                           paranoid=["install"]))
430
431        make_install_env = dict(
432            llvm=dict(debug=dict(),
433                      release=dict(),
434                      paranoid=dict()),
435            dragonegg=dict(debug=dict(),
436                           release=dict(),
437                           paranoid=dict()))
438
439        make_check_flags = dict(
440            llvm=dict(debug=["check"],
441                      release=["check"],
442                      paranoid=["check"]),
443            dragonegg=dict(debug=["check"],
444                           release=["check"],
445                           paranoid=["check"]))
446
447        make_check_env = dict(
448            llvm=dict(debug=dict(),
449                      release=dict(),
450                      paranoid=dict()),
451            dragonegg=dict(debug=dict(),
452                           release=dict(),
453                           paranoid=dict()))
454
455        for component in components:
456            comp = component[:]
457
458            if (self.options.no_dragonegg):
459                if (comp == 'dragonegg'):
460                    self.logger.info("Skipping " + component + " in "
461                                     + builddir)
462                    continue
463
464            srcdir = source + "/" + comp.rstrip("2")
465            builddir = self.build_prefix + "/" + comp + "/" + build_suffix
466            installdir = self.install_prefix
467
468            comp_key = comp.replace("-", "_")
469
470            config_args = configure_flags[comp_key][build][:]
471            config_args.extend(getattr(self.options,
472                                       "extra_" + comp_key.rstrip("2")
473                                       + "_config_flags",
474                                       "").split())
475
476            self.logger.info("Configuring " + component + " in " + builddir)
477            configrc = self.configure(component, srcdir, builddir,
478                                      config_args,
479                                      configure_env[comp_key][build])
480
481            if (configrc == None) :
482                self.logger.info("[None] Failed to configure " + component + " in " + installdir)
483
484            if (configrc == 0 or self.options.keep_going) :
485                self.logger.info("Building " + component + " in " + builddir)
486                self.logger.info("Build: make " + str(make_flags[comp_key][build]))
487                buildrc = self.make(component, srcdir, builddir,
488                                    make_flags[comp_key][build],
489                                    make_env[comp_key][build])
490
491                if (buildrc == None) :
492                    self.logger.info("[None] Failed to build " + component + " in " + installdir)
493
494                if (buildrc == 0 or self.options.keep_going) :
495                    self.logger.info("Testing " + component + " in " + builddir)
496                    self.logger.info("Test: make "
497                                     + str(make_check_flags[comp_key][build]))
498                    testrc = self.make(component, srcdir, builddir,
499                                       make_check_flags[comp_key][build],
500                                       make_check_env[comp_key][build])
501
502                    if (testrc == None) :
503                        self.logger.info("[None] Failed to test " + component + " in " + installdir)
504
505                    if ((testrc == 0  or self.options.keep_going)
506                        and not self.options.no_install):
507                        self.logger.info("Installing " + component + " in " + installdir)
508                        self.make(component, srcdir, builddir,
509                                  make_install_flags[comp_key][build],
510                                  make_install_env[comp_key][build])
511                    else :
512                        self.logger.info("Failed testing " + component + " in " + installdir)
513
514                else :
515                    self.logger.info("Failed to build " + component + " in " + installdir)
516
517            else :
518                self.logger.info("Failed to configure " + component + " in " + installdir)
519
520    def configure(self, component, srcdir, builddir, flags, env):
521        prefix = self.component_abbrev[component.replace("-", "_")]
522
523        self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
524                          + str(builddir))
525
526        configure_files = dict(
527            llvm=[(srcdir + "/configure", builddir + "/Makefile")],
528            dragonegg=[(None,None)])
529
530
531        doconfig = False
532        for conf, mf in configure_files[component.replace("-", "_")]:
533            if conf is None:
534                # No configure necessary
535                return 0
536
537            if not os.path.exists(conf):
538                self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
539                return -1
540
541            if os.path.exists(conf) and os.path.exists(mf):
542                confstat = os.stat(conf)
543                makestat = os.stat(mf)
544                if confstat.st_mtime > makestat.st_mtime:
545                    doconfig = True
546                    break
547            else:
548                doconfig = True
549                break
550
551        if not doconfig and not self.options.force_configure:
552            return 0
553
554        program = srcdir + "/configure"
555        if not is_executable(program):
556            self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
557            return -1
558
559        args = [program]
560        args += ["--verbose"]
561        args += flags
562        return self.execute(args, builddir, env, component)
563
564    def make(self, component, srcdir, builddir, flags, env):
565        program = find_executable("make")
566        if program is None:
567            raise ExecutableNotFound
568
569        if not is_executable(program):
570            raise FileNotExecutable
571
572        args = [program]
573        args += flags
574        return self.execute(args, builddir, env, component)
575
576# Global constants
577build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
578components = ["llvm", "dragonegg"]
579
580# Parse options
581parser = optparse.OptionParser(version="%prog 1.0")
582add_options(parser)
583(options, args) = parser.parse_args()
584check_options(parser, options, build_abbrev.keys());
585
586if options.verbose:
587    logging.basicConfig(level=logging.DEBUG,
588                        format='%(name)-13s: %(message)s')
589else:
590    logging.basicConfig(level=logging.INFO,
591                        format='%(name)-13s: %(message)s')
592
593source_abbrev = get_path_abbrevs(set(options.src))
594
595work_queue = queue.Queue()
596
597jobs = options.jobs // options.threads
598if jobs == 0:
599    jobs = 1
600
601numthreads = options.threads
602
603logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
604                         + str(numthreads) + " threads using " + str(jobs)
605                         + " make jobs")
606
607logging.getLogger().info("CC  = " + str(options.cc))
608logging.getLogger().info("CXX = " + str(options.cxx))
609
610for t in range(numthreads):
611    builder = Builder(work_queue, jobs,
612                      build_abbrev, source_abbrev,
613                      options)
614    builder.daemon = True
615    builder.start()
616
617for build in set(options.build):
618    for source in set(options.src):
619        work_queue.put((source, build))
620
621work_queue.join()
622