1#!/usr/bin/env python3
2
3#  Copyright 2021 Google, Inc.
4#
5#  Licensed under the Apache License, Version 2.0 (the "License");
6#  you may not use this file except in compliance with the License.
7#  You may obtain a copy of the License at:
8#
9#  http://www.apache.org/licenses/LICENSE-2.0
10#
11#  Unless required by applicable law or agreed to in writing, software
12#  distributed under the License is distributed on an "AS IS" BASIS,
13#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#  See the License for the specific language governing permissions and
15#  limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
33import shutil
34import six
35import subprocess
36import sys
37
38# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
39COMMON_MK_USES = [
40    'asan',
41    'coverage',
42    'cros_host',
43    'fuzzer',
44    'fuzzer',
45    'msan',
46    'profiling',
47    'tcmalloc',
48    'test',
49    'ubsan',
50]
51
52# Default use flags.
53USE_DEFAULTS = {
54    'android': False,
55    'bt_nonstandard_codecs': False,
56    'test': False,
57}
58
59VALID_TARGETS = [
60    'prepare',  # Prepare the output directory (gn gen + rust setup)
61    'tools',  # Build the host tools (i.e. packetgen)
62    'rust',  # Build only the rust components + copy artifacts to output dir
63    'main',  # Build the main C++ codebase
64    'test',  # Run the unit tests
65    'clean',  # Clean up output directory
66    'all',  # All targets except test and clean
67]
68
69HOST_TESTS = [
70    'bluetooth_test_common',
71    'bluetoothtbd_test',
72    'net_test_avrcp',
73    'net_test_btcore',
74    'net_test_types',
75    'net_test_btm_iso',
76    'net_test_btpackets',
77]
78
79
80class UseFlags():
81
82    def __init__(self, use_flags):
83        """ Construct the use flags.
84
85        Args:
86            use_flags: List of use flags parsed from the command.
87        """
88        self.flags = {}
89
90        # Import use flags required by common-mk
91        for use in COMMON_MK_USES:
92            self.set_flag(use, False)
93
94        # Set our defaults
95        for use, value in USE_DEFAULTS.items():
96            self.set_flag(use, value)
97
98        # Set use flags - value is set to True unless the use starts with -
99        # All given use flags always override the defaults
100        for use in use_flags:
101            value = not use.startswith('-')
102            self.set_flag(use, value)
103
104    def set_flag(self, key, value=True):
105        setattr(self, key, value)
106        self.flags[key] = value
107
108
109class HostBuild():
110
111    def __init__(self, args):
112        """ Construct the builder.
113
114        Args:
115            args: Parsed arguments from ArgumentParser
116        """
117        self.args = args
118
119        # Set jobs to number of cpus unless explicitly set
120        self.jobs = self.args.jobs
121        if not self.jobs:
122            self.jobs = multiprocessing.cpu_count()
123            print("Number of jobs = {}".format(self.jobs))
124
125        # Normalize all directories
126        self.output_dir = os.path.abspath(self.args.output)
127        self.platform_dir = os.path.abspath(self.args.platform_dir)
128        self.sysroot = self.args.sysroot
129        self.use_board = os.path.abspath(self.args.use_board) if self.args.use_board else None
130        self.libdir = self.args.libdir
131
132        # If default target isn't set, build everything
133        self.target = 'all'
134        if hasattr(self.args, 'target') and self.args.target:
135            self.target = self.args.target
136
137        target_use = self.args.use if self.args.use else []
138
139        # Unless set, always build test code
140        if not self.args.notest:
141            target_use.append('test')
142
143        self.use = UseFlags(target_use)
144
145        # Validate platform directory
146        assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
147        assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
148
149        # Make sure output directory exists (or create it)
150        os.makedirs(self.output_dir, exist_ok=True)
151
152        # Set some default attributes
153        self.libbase_ver = None
154
155        self.configure_environ()
156
157    def _generate_rustflags(self):
158        """ Rustflags to include for the build.
159      """
160        rust_flags = [
161            '-L',
162            '{}/out/Default/'.format(self.output_dir),
163            '-C',
164            'link-arg=-Wl,--allow-multiple-definition',
165        ]
166
167        return ' '.join(rust_flags)
168
169    def configure_environ(self):
170        """ Configure environment variables for GN and Cargo.
171        """
172        self.env = os.environ.copy()
173
174        # Make sure cargo home dir exists and has a bin directory
175        cargo_home = os.path.join(self.output_dir, 'cargo_home')
176        os.makedirs(cargo_home, exist_ok=True)
177        os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
178
179        # Configure Rust env variables
180        self.env['CARGO_TARGET_DIR'] = self.output_dir
181        self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
182        self.env['RUSTFLAGS'] = self._generate_rustflags()
183
184        # Configure some GN variables
185        if self.use_board:
186            self.env['PKG_CONFIG_PATH'] = os.path.join(self.use_board, self.libdir, 'pkgconfig')
187            libdir = os.path.join(self.use_board, self.libdir)
188            if self.env.get('LIBRARY_PATH'):
189                libpath = self.env['LIBRARY_PATH']
190                self.env['LIBRARY_PATH'] = '{}:{}'.format(libdir, libpath)
191            else:
192                self.env['LIBRARY_PATH'] = libdir
193
194    def run_command(self, target, args, cwd=None, env=None):
195        """ Run command and stream the output.
196        """
197        # Set some defaults
198        if not cwd:
199            cwd = self.platform_dir
200        if not env:
201            env = self.env
202
203        log_file = os.path.join(self.output_dir, '{}.log'.format(target))
204        with open(log_file, 'wb') as lf:
205            rc = 0
206            process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
207            while True:
208                line = process.stdout.readline()
209                print(line.decode('utf-8'), end="")
210                lf.write(line)
211                if not line:
212                    rc = process.poll()
213                    if rc is not None:
214                        break
215
216                    time.sleep(0.1)
217
218            if rc != 0:
219                raise Exception("Return code is {}".format(rc))
220
221    def _get_basever(self):
222        if self.libbase_ver:
223            return self.libbase_ver
224
225        self.libbase_ver = os.environ.get('BASE_VER', '')
226        if not self.libbase_ver:
227            base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
228            try:
229                with open(base_file, 'r') as f:
230                    self.libbase_ver = f.read().strip('\n')
231            except:
232                self.libbase_ver = 'NOT-INSTALLED'
233
234        return self.libbase_ver
235
236    def _gn_default_output(self):
237        return os.path.join(self.output_dir, 'out/Default')
238
239    def _gn_configure(self):
240        """ Configure all required parameters for platform2.
241
242        Mostly copied from //common-mk/platform2.py
243        """
244        clang = self.args.clang
245
246        def to_gn_string(s):
247            return '"%s"' % s.replace('"', '\\"')
248
249        def to_gn_list(strs):
250            return '[%s]' % ','.join([to_gn_string(s) for s in strs])
251
252        def to_gn_args_args(gn_args):
253            for k, v in gn_args.items():
254                if isinstance(v, bool):
255                    v = str(v).lower()
256                elif isinstance(v, list):
257                    v = to_gn_list(v)
258                elif isinstance(v, six.string_types):
259                    v = to_gn_string(v)
260                else:
261                    raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
262                yield '%s=%s' % (k.replace('-', '_'), v)
263
264        gn_args = {
265            'platform_subdir': 'bt',
266            'cc': 'clang' if clang else 'gcc',
267            'cxx': 'clang++' if clang else 'g++',
268            'ar': 'llvm-ar' if clang else 'ar',
269            'pkg-config': 'pkg-config',
270            'clang_cc': clang,
271            'clang_cxx': clang,
272            'OS': 'linux',
273            'sysroot': self.sysroot,
274            'libdir': os.path.join(self.sysroot, self.libdir),
275            'build_root': self.output_dir,
276            'platform2_root': self.platform_dir,
277            'libbase_ver': self._get_basever(),
278            'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
279            'external_cflags': [],
280            'external_cxxflags': [],
281            'enable_werror': False,
282        }
283
284        if clang:
285            # Make sure to mark the clang use flag as true
286            self.use.set_flag('clang', True)
287            gn_args['external_cxxflags'] += ['-I/usr/include/']
288
289        # EXTREME HACK ALERT
290        #
291        # In my laziness, I am supporting building against an already built
292        # sysroot path (i.e. chromeos board) so that I don't have to build
293        # libchrome or modp_b64 locally.
294        if self.use_board:
295            includedir = os.path.join(self.use_board, 'usr/include')
296            gn_args['external_cxxflags'] += [
297                '-I{}'.format(includedir),
298                '-I{}/libchrome'.format(includedir),
299                '-I{}/gtest'.format(includedir),
300                '-I{}/gmock'.format(includedir),
301                '-I{}/modp_b64'.format(includedir),
302            ]
303        gn_args_args = list(to_gn_args_args(gn_args))
304        use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
305        gn_args_args += ['use={%s}' % (' '.join(use_args))]
306
307        gn_args = [
308            'gn',
309            'gen',
310        ]
311
312        if self.args.verbose:
313            gn_args.append('-v')
314
315        gn_args += [
316            '--root=%s' % self.platform_dir,
317            '--args=%s' % ' '.join(gn_args_args),
318            self._gn_default_output(),
319        ]
320
321        if 'PKG_CONFIG_PATH' in self.env:
322            print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
323
324        self.run_command('configure', gn_args)
325
326    def _gn_build(self, target):
327        """ Generate the ninja command for the target and run it.
328        """
329        args = ['%s:%s' % ('bt', target)]
330        ninja_args = ['ninja', '-C', self._gn_default_output()]
331        if self.jobs:
332            ninja_args += ['-j', str(self.jobs)]
333        ninja_args += args
334
335        if self.args.verbose:
336            ninja_args.append('-v')
337
338        self.run_command('build', ninja_args)
339
340    def _rust_configure(self):
341        """ Generate config file at cargo_home so we use vendored crates.
342        """
343        template = """
344        [source.systembt]
345        directory = "{}/external/rust/vendor"
346
347        [source.crates-io]
348        replace-with = "systembt"
349        local-registry = "/nonexistent"
350        """
351
352        if self.args.vendored_rust:
353            contents = template.format(self.platform_dir)
354            with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
355                f.write(contents)
356
357    def _rust_build(self):
358        """ Run `cargo build` from platform2/bt directory.
359        """
360        self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
361
362    def _target_prepare(self):
363        """ Target to prepare the output directory for building.
364
365        This runs gn gen to generate all rquired files and set up the Rust
366        config properly. This will be run
367        """
368        self._gn_configure()
369        self._rust_configure()
370
371    def _target_tools(self):
372        """ Build the tools target in an already prepared environment.
373        """
374        self._gn_build('tools')
375
376        # Also copy bluetooth_packetgen to CARGO_HOME so it's available
377        shutil.copy(
378            os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
379
380    def _target_rust(self):
381        """ Build rust artifacts in an already prepared environment.
382        """
383        self._rust_build()
384        rust_dir = os.path.join(self._gn_default_output(), 'rust')
385        if os.path.exists(rust_dir):
386            shutil.rmtree(rust_dir)
387        shutil.copytree(os.path.join(self.output_dir, 'debug'), rust_dir)
388
389    def _target_main(self):
390        """ Build the main GN artifacts in an already prepared environment.
391        """
392        self._gn_build('all')
393
394    def _target_test(self):
395        """ Runs the host tests.
396        """
397        # Rust tests first
398        self.run_command('test', ['cargo', 'test'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
399
400        # Host tests second based on host test list
401        for t in HOST_TESTS:
402            self.run_command(
403                'test', [os.path.join(self.output_dir, 'out/Default', t)],
404                cwd=os.path.join(self.output_dir),
405                env=self.env)
406
407    def _target_clean(self):
408        """ Delete the output directory entirely.
409        """
410        shutil.rmtree(self.output_dir)
411
412    def _target_all(self):
413        """ Build all common targets (skipping test and clean).
414        """
415        self._target_prepare()
416        self._target_tools()
417        self._target_main()
418        self._target_rust()
419
420    def build(self):
421        """ Builds according to self.target
422        """
423        print('Building target ', self.target)
424
425        if self.target == 'prepare':
426            self._target_prepare()
427        elif self.target == 'tools':
428            self._target_tools()
429        elif self.target == 'rust':
430            self._target_rust()
431        elif self.target == 'main':
432            self._target_main()
433        elif self.target == 'test':
434            self._target_test()
435        elif self.target == 'clean':
436            self._target_clean()
437        elif self.target == 'all':
438            self._target_all()
439
440
441if __name__ == '__main__':
442    parser = argparse.ArgumentParser(description='Simple build for host.')
443    parser.add_argument('--output', help='Output directory for the build.', required=True)
444    parser.add_argument('--platform-dir', help='Directory where platform2 is staged.', required=True)
445    parser.add_argument('--clang', help='Use clang compiler.', default=False, action='store_true')
446    parser.add_argument('--use', help='Set a specific use flag.')
447    parser.add_argument('--notest', help="Don't compile test code.", default=False, action='store_true')
448    parser.add_argument('--target', help='Run specific build target')
449    parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
450    parser.add_argument('--libdir', help='Libdir - default = usr/lib64', default='usr/lib64')
451    parser.add_argument('--use-board', help='Use a built x86 board for dependencies. Provide path.')
452    parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
453    parser.add_argument('--vendored-rust', help='Use vendored rust crates', default=False, action='store_true')
454    parser.add_argument('--verbose', help='Verbose logs for build.')
455
456    args = parser.parse_args()
457    build = HostBuild(args)
458    build.build()
459