1#!/usr/bin/python3
2##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7#
8##===----------------------------------------------------------------------===##
9#
10# This script builds many different flavors of the LLVM ecosystem.  It
11# will build LLVM, Clang and dragonegg as well as run tests on them.
12# This script is convenient to use to check builds and tests before
13# committing changes to the upstream repository
14#
15# A typical source setup uses three trees and looks like this:
16#
17# official
18#   dragonegg
19#   llvm
20#     tools
21#       clang
22# staging
23#   dragonegg
24#   llvm
25#     tools
26#       clang
27# commit
28#   dragonegg
29#   llvm
30#     tools
31#       clang
32#
33# In a typical workflow, the "official" tree always contains unchanged
34# sources from the main LLVM project repositories.  The "staging" tree
35# is where local work is done.  A set of changes resides there waiting
36# to be moved upstream.  The "commit" tree is where changes from
37# "staging" make their way upstream.  Individual incremental changes
38# from "staging" are applied to "commit" and committed upstream after
39# a successful build and test run.  A successful build is one in which
40# testing results in no more failures than seen in the testing of the
41# "official" tree.
42#
43# A build may be invoked as such:
44#
45# llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
46#   --build=debug --build=release --build=paranoid
47#   --prefix=/home/greened/install --builddir=/home/greened/build
48#
49# This will build the LLVM ecosystem, including LLVM, Clangand
50# dragonegg, putting build results in ~/build and installing tools in
51# ~/install.  llvm-compilers-check creates separate build and install
52# directories for each source/build flavor.  In the above example,
53# llvmbuild will build debug, release and paranoid (debug+checks)
54# flavors from each source tree (official, staging and commit) for a
55# total of nine builds.  All builds will be run in parallel.
56#
57# The user may control parallelism via the --jobs and --threads
58# switches.  --jobs tells llvm-compilers-check the maximum total
59# number of builds to activate in parallel.  The user may think of it
60# as equivalent to the GNU make -j switch.  --threads tells
61# llvm-compilers-check how many worker threads to use to accomplish
62# those builds.  If --threads is less than --jobs, --threads workers
63# will be launched and each one will pick a source/flavor combination
64# to build.  Then llvm-compilers-check will invoke GNU make with -j
65# (--jobs / --threads) to use up the remaining job capacity.  Once a
66# worker is finished with a build, it will pick another combination
67# off the list and start building it.
68#
69##===----------------------------------------------------------------------===##
70
71import optparse
72import os
73import sys
74import threading
75import queue
76import logging
77import traceback
78import subprocess
79import re
80
81# TODO: Use shutil.which when it is available (3.2 or later)
82def find_executable(executable, path=None):
83    """Try to find 'executable' in the directories listed in 'path' (a
84    string listing directories separated by 'os.pathsep'; defaults to
85    os.environ['PATH']).  Returns the complete filename or None if not
86    found
87    """
88    if path is None:
89        path = os.environ['PATH']
90    paths = path.split(os.pathsep)
91    extlist = ['']
92    if os.name == 'os2':
93        (base, ext) = os.path.splitext(executable)
94        # executable files on OS/2 can have an arbitrary extension, but
95        # .exe is automatically appended if no dot is present in the name
96        if not ext:
97            executable = executable + ".exe"
98    elif sys.platform == 'win32':
99        pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
100        (base, ext) = os.path.splitext(executable)
101        if ext.lower() not in pathext:
102            extlist = pathext
103    for ext in extlist:
104        execname = executable + ext
105        if os.path.isfile(execname):
106            return execname
107        else:
108            for p in paths:
109                f = os.path.join(p, execname)
110                if os.path.isfile(f):
111                    return f
112    else:
113        return None
114
115def is_executable(fpath):
116    return os.path.exists(fpath) and os.access(fpath, os.X_OK)
117
118def add_options(parser):
119    parser.add_option("-v", "--verbose", action="store_true",
120                      default=False,
121                      help=("Output informational messages"
122                            " [default: %default]"))
123    parser.add_option("--src", action="append",
124                      help=("Top-level source directory [default: %default]"))
125    parser.add_option("--build", action="append",
126                      help=("Build types to run [default: %default]"))
127    parser.add_option("--cc", default=find_executable("cc"),
128                      help=("The C compiler to use [default: %default]"))
129    parser.add_option("--cxx", default=find_executable("c++"),
130                      help=("The C++ compiler to use [default: %default]"))
131    parser.add_option("--threads", default=4, type="int",
132                      help=("The number of worker threads to use "
133                            "[default: %default]"))
134    parser.add_option("--jobs", "-j", default=8, type="int",
135                      help=("The number of simultaneous build jobs "
136                            "[default: %default]"))
137    parser.add_option("--prefix",
138                      help=("Root install directory [default: %default]"))
139    parser.add_option("--builddir",
140                      help=("Root build directory [default: %default]"))
141    parser.add_option("--extra-llvm-config-flags", default="",
142                      help=("Extra flags to pass to llvm configure [default: %default]"))
143    parser.add_option("--force-configure", default=False, action="store_true",
144                      help=("Force reconfigure of all components"))
145    parser.add_option("--no-dragonegg", default=False, action="store_true",
146                      help=("Do not build dragonegg"))
147    parser.add_option("--no-install", default=False, action="store_true",
148                      help=("Do not do installs"))
149    parser.add_option("--keep-going", default=False, action="store_true",
150                      help=("Keep going after failures"))
151    parser.add_option("--no-flavor-prefix", default=False, action="store_true",
152                      help=("Do not append the build flavor to the install path"))
153    parser.add_option("--enable-werror", default=False, action="store_true",
154                      help=("Build with -Werror"))
155    return
156
157def check_options(parser, options, valid_builds):
158    # See if we're building valid flavors.
159    for build in options.build:
160        if (build not in valid_builds):
161            parser.error("'" + build + "' is not a valid build flavor "
162                         + str(valid_builds))
163
164    # See if we can find source directories.
165    for src in options.src:
166        for component in components:
167            component = component.rstrip("2")
168            compsrc = src + "/" + component
169            if (not os.path.isdir(compsrc)):
170                parser.error("'" + compsrc + "' does not exist")
171
172    # See if we can find the compilers
173    options.cc = find_executable(options.cc)
174    options.cxx = find_executable(options.cxx)
175
176    return
177
178# Find a unique short name for the given set of paths.  This searches
179# back through path components until it finds unique component names
180# among all given paths.
181def get_path_abbrevs(paths):
182    # Find the number of common starting characters in the last component
183    # of the paths.
184    unique_paths = list(paths)
185
186    class NotFoundException(Exception): pass
187
188    # Find a unique component of each path.
189    unique_bases = unique_paths[:]
190    found = 0
191    while len(unique_paths) > 0:
192        bases = [os.path.basename(src) for src in unique_paths]
193        components = { c for c in bases }
194        # Account for single entry in paths.
195        if len(components) > 1 or len(components) == len(bases):
196            # We found something unique.
197            for c in components:
198                if bases.count(c) == 1:
199                   index = bases.index(c)
200                   unique_bases[index] = c
201                   # Remove the corresponding path from the set under
202                   # consideration.
203                   unique_paths[index] = None
204            unique_paths = [ p for p in unique_paths if p is not None ]
205        unique_paths = [os.path.dirname(src) for src in unique_paths]
206
207    if len(unique_paths) > 0:
208        raise NotFoundException()
209
210    abbrevs = dict(zip(paths, [base for base in unique_bases]))
211
212    return abbrevs
213
214# Given a set of unique names, find a short character sequence that
215# uniquely identifies them.
216def get_short_abbrevs(unique_bases):
217    # Find a unique start character for each path base.
218    my_unique_bases = unique_bases[:]
219    unique_char_starts = unique_bases[:]
220    while len(my_unique_bases) > 0:
221        for start, char_tuple in enumerate(zip(*[base
222                                                 for base in my_unique_bases])):
223            chars = { c for c in char_tuple }
224            # Account for single path.
225            if len(chars) > 1 or len(chars) == len(char_tuple):
226                # We found something unique.
227                for c in chars:
228                    if char_tuple.count(c) == 1:
229                        index = char_tuple.index(c)
230                        unique_char_starts[index] = start
231                        # Remove the corresponding path from the set under
232                        # consideration.
233                        my_unique_bases[index] = None
234                my_unique_bases = [ b for b in my_unique_bases
235                                    if b is not None ]
236                break
237
238    if len(my_unique_bases) > 0:
239        raise NotFoundException()
240
241    abbrevs = [abbrev[start_index:start_index+3]
242               for abbrev, start_index
243               in zip([base for base in unique_bases],
244                      [index for index in unique_char_starts])]
245
246    abbrevs = dict(zip(unique_bases, abbrevs))
247
248    return abbrevs
249
250class Builder(threading.Thread):
251    class ExecutableNotFound(Exception): pass
252    class FileNotExecutable(Exception): pass
253
254    def __init__(self, work_queue, jobs,
255                 build_abbrev, source_abbrev,
256                 options):
257        super().__init__()
258        self.work_queue = work_queue
259        self.jobs = jobs
260        self.cc = options.cc
261        self.cxx = options.cxx
262        self.build_abbrev = build_abbrev
263        self.source_abbrev = source_abbrev
264        self.build_prefix = options.builddir
265        self.install_prefix = options.prefix
266        self.options = options
267        self.component_abbrev = dict(
268            llvm="llvm",
269            dragonegg="degg")
270    def run(self):
271        while True:
272            try:
273                source, build = self.work_queue.get()
274                self.dobuild(source, build)
275            except:
276                traceback.print_exc()
277            finally:
278                self.work_queue.task_done()
279
280    def execute(self, command, execdir, env, component):
281        prefix = self.component_abbrev[component.replace("-", "_")]
282        pwd = os.getcwd()
283        if not os.path.exists(execdir):
284            os.makedirs(execdir)
285
286        execenv = os.environ.copy()
287
288        for key, value in env.items():
289            execenv[key] = value
290
291        self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
292                          + " ".join(command));
293
294        try:
295            proc = subprocess.Popen(command,
296                                    cwd=execdir,
297                                    env=execenv,
298                                    stdout=subprocess.PIPE,
299                                    stderr=subprocess.STDOUT)
300
301            line = proc.stdout.readline()
302            while line:
303                self.logger.info("[" + prefix + "] "
304                                 + str(line, "utf-8").rstrip())
305                line = proc.stdout.readline()
306
307            (stdoutdata, stderrdata) = proc.communicate()
308            retcode = proc.wait()
309
310            return retcode
311
312        except:
313            traceback.print_exc()
314
315    # Get a list of C++ include directories to pass to clang.
316    def get_includes(self):
317        # Assume we're building with g++ for now.
318        command = [self.cxx]
319        command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
320        includes = []
321        self.logger.debug(command)
322        try:
323            proc = subprocess.Popen(command,
324                                    stdout=subprocess.PIPE,
325                                    stderr=subprocess.STDOUT)
326
327            gather = False
328            line = proc.stdout.readline()
329            while line:
330                self.logger.debug(line)
331                if re.search("End of search list", str(line)) is not None:
332                    self.logger.debug("Stop Gather")
333                    gather = False
334                if gather:
335                    includes.append(str(line, "utf-8").strip())
336                if re.search("#include <...> search starts", str(line)) is not None:
337                    self.logger.debug("Start Gather")
338                    gather = True
339                line = proc.stdout.readline()
340
341        except:
342            traceback.print_exc()
343        self.logger.debug(includes)
344        return includes
345
346    def dobuild(self, source, build):
347        build_suffix = ""
348
349        ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
350
351        prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
352        if (not self.options.no_flavor_prefix):
353            self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
354
355        build_suffix += "/" + self.source_abbrev[source] + "/" + build
356
357        self.logger = logging.getLogger(prefix)
358
359        self.logger.debug(self.install_prefix)
360
361        # Assume we're building with gcc for now.
362        cxxincludes = self.get_includes()
363        cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
364        cxxroot = os.path.dirname(cxxroot)        # Remove the c++
365        cxxroot = os.path.dirname(cxxroot)        # Remove the include
366
367        configure_flags = dict(
368            llvm=dict(debug=["--prefix=" + self.install_prefix,
369                             "--enable-assertions",
370                             "--disable-optimized",
371                             "--with-gcc-toolchain=" + cxxroot],
372                      release=["--prefix=" + self.install_prefix,
373                               "--enable-optimized",
374                               "--with-gcc-toolchain=" + cxxroot],
375                      paranoid=["--prefix=" + self.install_prefix,
376                                "--enable-assertions",
377                                "--enable-expensive-checks",
378                                "--disable-optimized",
379                                "--with-gcc-toolchain=" + cxxroot]),
380            dragonegg=dict(debug=[],
381                           release=[],
382                           paranoid=[]))
383
384        if (self.options.enable_werror):
385            configure_flags["llvm"]["debug"].append("--enable-werror")
386            configure_flags["llvm"]["release"].append("--enable-werror")
387            configure_flags["llvm"]["paranoid"].append("--enable-werror")
388
389        configure_env = dict(
390            llvm=dict(debug=dict(CC=self.cc,
391                                 CXX=self.cxx),
392                      release=dict(CC=self.cc,
393                                   CXX=self.cxx),
394                      paranoid=dict(CC=self.cc,
395                                    CXX=self.cxx)),
396            dragonegg=dict(debug=dict(CC=self.cc,
397                                      CXX=self.cxx),
398                           release=dict(CC=self.cc,
399                                        CXX=self.cxx),
400                           paranoid=dict(CC=self.cc,
401                                         CXX=self.cxx)))
402
403        make_flags = dict(
404            llvm=dict(debug=["-j" + str(self.jobs)],
405                      release=["-j" + str(self.jobs)],
406                      paranoid=["-j" + str(self.jobs)]),
407            dragonegg=dict(debug=["-j" + str(self.jobs)],
408                           release=["-j" + str(self.jobs)],
409                           paranoid=["-j" + str(self.jobs)]))
410
411        make_env = dict(
412            llvm=dict(debug=dict(),
413                      release=dict(),
414                      paranoid=dict()),
415            dragonegg=dict(debug=dict(GCC=self.cc,
416                                      LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
417                           release=dict(GCC=self.cc,
418                                        LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
419                           paranoid=dict(GCC=self.cc,
420                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
421
422        make_install_flags = dict(
423            llvm=dict(debug=["install"],
424                      release=["install"],
425                      paranoid=["install"]),
426            dragonegg=dict(debug=["install"],
427                           release=["install"],
428                           paranoid=["install"]))
429
430        make_install_env = dict(
431            llvm=dict(debug=dict(),
432                      release=dict(),
433                      paranoid=dict()),
434            dragonegg=dict(debug=dict(),
435                           release=dict(),
436                           paranoid=dict()))
437
438        make_check_flags = dict(
439            llvm=dict(debug=["check"],
440                      release=["check"],
441                      paranoid=["check"]),
442            dragonegg=dict(debug=["check"],
443                           release=["check"],
444                           paranoid=["check"]))
445
446        make_check_env = dict(
447            llvm=dict(debug=dict(),
448                      release=dict(),
449                      paranoid=dict()),
450            dragonegg=dict(debug=dict(),
451                           release=dict(),
452                           paranoid=dict()))
453
454        for component in components:
455            comp = component[:]
456
457            if (self.options.no_dragonegg):
458                if (comp == 'dragonegg'):
459                    self.logger.info("Skipping " + component + " in "
460                                     + builddir)
461                    continue
462
463            srcdir = source + "/" + comp.rstrip("2")
464            builddir = self.build_prefix + "/" + comp + "/" + build_suffix
465            installdir = self.install_prefix
466
467            comp_key = comp.replace("-", "_")
468
469            config_args = configure_flags[comp_key][build][:]
470            config_args.extend(getattr(self.options,
471                                       "extra_" + comp_key.rstrip("2")
472                                       + "_config_flags",
473                                       "").split())
474
475            self.logger.info("Configuring " + component + " in " + builddir)
476            configrc = self.configure(component, srcdir, builddir,
477                                      config_args,
478                                      configure_env[comp_key][build])
479
480            if (configrc == None) :
481                self.logger.info("[None] Failed to configure " + component + " in " + installdir)
482
483            if (configrc == 0 or self.options.keep_going) :
484                self.logger.info("Building " + component + " in " + builddir)
485                self.logger.info("Build: make " + str(make_flags[comp_key][build]))
486                buildrc = self.make(component, srcdir, builddir,
487                                    make_flags[comp_key][build],
488                                    make_env[comp_key][build])
489
490                if (buildrc == None) :
491                    self.logger.info("[None] Failed to build " + component + " in " + installdir)
492
493                if (buildrc == 0 or self.options.keep_going) :
494                    self.logger.info("Testing " + component + " in " + builddir)
495                    self.logger.info("Test: make "
496                                     + str(make_check_flags[comp_key][build]))
497                    testrc = self.make(component, srcdir, builddir,
498                                       make_check_flags[comp_key][build],
499                                       make_check_env[comp_key][build])
500
501                    if (testrc == None) :
502                        self.logger.info("[None] Failed to test " + component + " in " + installdir)
503
504                    if ((testrc == 0  or self.options.keep_going)
505                        and not self.options.no_install):
506                        self.logger.info("Installing " + component + " in " + installdir)
507                        self.make(component, srcdir, builddir,
508                                  make_install_flags[comp_key][build],
509                                  make_install_env[comp_key][build])
510                    else :
511                        self.logger.info("Failed testing " + component + " in " + installdir)
512
513                else :
514                    self.logger.info("Failed to build " + component + " in " + installdir)
515
516            else :
517                self.logger.info("Failed to configure " + component + " in " + installdir)
518
519    def configure(self, component, srcdir, builddir, flags, env):
520        prefix = self.component_abbrev[component.replace("-", "_")]
521
522        self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
523                          + str(builddir))
524
525        configure_files = dict(
526            llvm=[(srcdir + "/configure", builddir + "/Makefile")],
527            dragonegg=[(None,None)])
528
529
530        doconfig = False
531        for conf, mf in configure_files[component.replace("-", "_")]:
532            if conf is None:
533                # No configure necessary
534                return 0
535
536            if not os.path.exists(conf):
537                self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
538                return -1
539
540            if os.path.exists(conf) and os.path.exists(mf):
541                confstat = os.stat(conf)
542                makestat = os.stat(mf)
543                if confstat.st_mtime > makestat.st_mtime:
544                    doconfig = True
545                    break
546            else:
547                doconfig = True
548                break
549
550        if not doconfig and not self.options.force_configure:
551            return 0
552
553        program = srcdir + "/configure"
554        if not is_executable(program):
555            self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
556            return -1
557
558        args = [program]
559        args += ["--verbose"]
560        args += flags
561        return self.execute(args, builddir, env, component)
562
563    def make(self, component, srcdir, builddir, flags, env):
564        program = find_executable("make")
565        if program is None:
566            raise ExecutableNotFound
567
568        if not is_executable(program):
569            raise FileNotExecutable
570
571        args = [program]
572        args += flags
573        return self.execute(args, builddir, env, component)
574
575# Global constants
576build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
577components = ["llvm", "dragonegg"]
578
579# Parse options
580parser = optparse.OptionParser(version="%prog 1.0")
581add_options(parser)
582(options, args) = parser.parse_args()
583check_options(parser, options, build_abbrev.keys());
584
585if options.verbose:
586    logging.basicConfig(level=logging.DEBUG,
587                        format='%(name)-13s: %(message)s')
588else:
589    logging.basicConfig(level=logging.INFO,
590                        format='%(name)-13s: %(message)s')
591
592source_abbrev = get_path_abbrevs(set(options.src))
593
594work_queue = queue.Queue()
595
596jobs = options.jobs // options.threads
597if jobs == 0:
598    jobs = 1
599
600numthreads = options.threads
601
602logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
603                         + str(numthreads) + " threads using " + str(jobs)
604                         + " make jobs")
605
606logging.getLogger().info("CC  = " + str(options.cc))
607logging.getLogger().info("CXX = " + str(options.cxx))
608
609for t in range(numthreads):
610    builder = Builder(work_queue, jobs,
611                      build_abbrev, source_abbrev,
612                      options)
613    builder.daemon = True
614    builder.start()
615
616for build in set(options.build):
617    for source in set(options.src):
618        work_queue.put((source, build))
619
620work_queue.join()
621