1# Copyright (C) 2021 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Core logic for generating, syncing, and cleaning up a Bazel environment."""
15import abc
16import datetime
17import logging
18import os
19import pathlib
20import re
21import shutil
22import subprocess
23from typing import Set, Dict, List
24
25# Regex for BUILD files used to identify them since they can be named
26# BUILD or BUILD.bazel.
27BUILD_FILENAME_REGEX = re.compile("(BUILD|BUILD.bazel)")
28
29
30class Error(Exception):
31    """Base Error that all other errors the system throws are descendants of."""
32    pass
33
34
35class SoongExecutionError(Error):
36    """Raised when Soong fails to build provided targets."""
37    pass
38
39
40class Soong:
41    """Interface for the Soong build system.
42
43    Attributes:
44        soong_workspace: the top of the Android workspace that Soong
45        will be operating in.
46        soong_executable: the path to the executable for the soong_ui.bash
47        launcher.
48    """
49    soong_workspace: pathlib.Path
50    soong_executable: pathlib.Path
51
52    def __init__(self, soong_workspace: pathlib.Path):
53        self.soong_workspace = soong_workspace
54
55        self.soong_executable = self.soong_workspace.joinpath(
56            "build/soong/soong_ui.bash")
57        if not self.soong_executable.exists():
58            raise SoongExecutionError(
59                "Unable to find Soong executable, expected location: %s" %
60                self.soong_executable)
61
62    def build(self, build_targets: Set[str]) -> None:
63        """Builds the provided set of targets with Soong.
64
65        Of note, there is no verification for the targets that get passed in,
66        rather that responsibility passes to Soong which will fail to build if a
67        target is invalid.
68
69        Args:
70            build_targets: a set of targets to build with Soong.
71        """
72        cmd_args = [
73            str(self.soong_executable), "--build-mode", "--all-modules",
74            f"--dir={self.soong_workspace}"
75        ]
76        cmd_args.extend(build_targets)
77
78        logging.info("Building targets with Soong: %s", build_targets)
79        logging.info("Please be patient, this may take a while...")
80        logging.debug("Soong Command is: %s", " ".join(cmd_args))
81
82        try:
83            subprocess.run(cmd_args,
84                           cwd=self.soong_workspace,
85                           check=True,
86                           stdout=subprocess.PIPE,
87                           stderr=subprocess.STDOUT)
88        except subprocess.CalledProcessError as cpe:
89            raise SoongExecutionError(
90                "There was an error during the Soong build process. Please "
91                "correct the error with Soong, and try running this script "
92                "again. Soong output follows:\n\n"
93                f"{cpe.stdout.decode('utf-8')}") from cpe
94
95        logging.info("Soong command completed successfully.")
96
97
98class Resource(abc.ABC):
99    """
100    Represents a resource file that is used for scaffolding the Bazel env.
101
102    Resource is an abstract class. Inheriting classes must provide the following
103    attributes which are used by the default method implementations below.
104
105    Attributes:
106        stage_path: the path where this resource should be written when staged.
107        workspace_path: the path where this resource should be synced to in the
108        workspace.
109    """
110    stage_path: pathlib.Path
111    workspace_path: pathlib.Path
112
113    def build_targets(self) -> set:
114        return set()
115
116    @abc.abstractmethod
117    def stage(self, _):
118        """Writes a resource to its stage location."""
119        pass
120
121    @abc.abstractmethod
122    def sync(self):
123        """Syncs a resource to its workspace location."""
124
125        # Overwrite any existing file in the workspace when synced.
126        self.workspace_path.unlink(missing_ok=True)
127        self.workspace_path.symlink_to(self.stage_path)
128
129    @abc.abstractmethod
130    def clean(self):
131        """Cleans a resource from its workspace location."""
132        self.workspace_path.unlink(missing_ok=True)
133
134
135class StaticResource(Resource):
136    """Resource representing a static file to be copied.
137
138    Attributes:
139        resource_path: path to the resource on disk.
140    """
141    def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
142                 resource_path: pathlib.Path):
143        self.stage_path = stage_path
144        self.workspace_path = workspace_path
145        self._resource_path = resource_path
146
147    def stage(self, _):
148        _verify_directory(self.stage_path.parent)
149        shutil.copy(self._resource_path, self.stage_path)
150        _make_executable_if_script(self.stage_path)
151
152    def sync(self):
153        super().sync()
154
155    def clean(self):
156        super().sync()
157
158    def __repr__(self):
159        return (f"StaticResource(stage_path={self.stage_path}, "
160                f"workspace_path={self.workspace_path}, "
161                f"resource_path={self._resource_path})")
162
163
164class TemplateError(Error):
165    """Raised when there is an issue while templating a template file."""
166    pass
167
168
169class TemplateResource(Resource):
170    """Resource that represents a file to be templated.
171
172    When staged, the resource is templated using the "mapping" provided
173    to the stage function.
174
175    Attributes:
176        resource_path: path to the resource on disk.
177    """
178    resource_path: pathlib.Path
179
180    # Key within templates that identifies a Soong target to build for a
181    # provided template.
182    SOONG_TARGET_KEY = "SOONG_TARGET"
183    _KEY_VALUE_SEP = ":"
184
185    # For a provided template, lines matching this regex are ignored when
186    # loading the template.
187    #
188    # This enables to contain metadata that, while visible to the script, is
189    # not visible in the generated templates. This is currently used with the
190    # SOONG_TARGET key/value pairs in templates.
191    _IGNORE_LINE_REGEX = re.compile(f"({SOONG_TARGET_KEY})")
192
193    def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
194                 resource_path: pathlib.Path):
195        self.stage_path = stage_path
196        self.workspace_path = workspace_path
197        self.resource_path = resource_path
198
199    def __repr__(self):
200        return (f"TemplateResource(stage_path={self.stage_path}, "
201                f"workspace_path={self.workspace_path}, "
202                f"resource_path={self.resource_path})")
203
204    def stage(self, mapping: Dict[str, str]):
205        _verify_directory(self.stage_path.parent)
206        lines = self.resource_path.open().readlines()
207        lines = [
208            line for line in lines if not self._IGNORE_LINE_REGEX.search(line)
209        ]
210
211        try:
212            output = "".join(lines).format_map(mapping)
213        except KeyError as ke:
214            raise TemplateError(
215                f"Malformed template file: {self.resource_path}") from ke
216
217        with self.stage_path.open("w") as output_file:
218            output_file.write(output)
219        _make_executable_if_script(self.stage_path)
220
221    def sync(self):
222        super().sync()
223
224    def clean(self):
225        super().clean()
226
227    @classmethod
228    def read_value_from_template_var(cls, line: str) -> str:
229        value = line.split(cls._KEY_VALUE_SEP)[-1]
230        return value.strip()
231
232
233class BuildTemplateResource(Resource):
234    """Resource that represents a BUILD file.
235
236    This operates in the same way as a TemplateResource, however also
237    sets up the prebuilts directory needed by BUILD files.
238
239    Attributes:
240        _template_resource: underlying TemplateResource for this resource.
241        global_prebuilts_dir: the directory in the filesystem where prebuilts
242        live, this directory is symlinked to by a directory adjacent
243        to the BUILD file represented by this Resource.
244        prebuilts_stage_path: the path of the staged prebuilts directory
245        which is a symlink to the global_prebuilts_dir.
246        prebuilts_workspace_path: the path of the workspace prebuilts directory,
247        which is a symlink to the global_prebuilts_dir.
248    """
249    _template_resource: TemplateResource
250    global_prebuilts_dir: pathlib.Path
251    prebuilts_stage_path: pathlib.Path
252    prebuilts_workspace_path: pathlib.Path
253
254    def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
255                 resource_path: pathlib.Path,
256                 global_prebuilts_dir: pathlib.Path, prebuilts_dir_name: str):
257        self.stage_path = stage_path
258        self.workspace_path = workspace_path
259
260        self._template_resource = TemplateResource(stage_path, workspace_path,
261                                                   resource_path)
262        self.global_prebuilts_dir = global_prebuilts_dir
263        self.prebuilts_stage_path = self.stage_path.parent.joinpath(
264            prebuilts_dir_name)
265        self.prebuilts_workspace_path = self.workspace_path.parent.joinpath(
266            prebuilts_dir_name)
267
268    def __repr__(self):
269        return ("BuildTemplateResource("
270                f"template_resource={self._template_resource}, "
271                f"global_prebuilts_dir={self.global_prebuilts_dir}, "
272                f"prebuilts_stage_path={self.prebuilts_stage_path}, "
273                f"prebuilts_workspace_path={self.prebuilts_workspace_path})")
274
275    def build_targets(self) -> set:
276        """Overrides build_targets() to read targets from the BUILD template."""
277        targets = set()
278        with self._template_resource.resource_path.open() as build_file:
279            while line := build_file.readline():
280                if self._template_resource.SOONG_TARGET_KEY in line:
281                    targets.add(
282                        self._template_resource.read_value_from_template_var(
283                            line))
284
285        return targets
286
287    def stage(self, mapping: Dict[str, str]):
288        """Overrides stage() to stage a BUILD resource.
289
290        Delegates most actions to the _template_resource, while ensuring that
291        a generated prebuilts directory is also staged.
292        """
293        self._template_resource.stage(mapping)
294        self.prebuilts_stage_path.symlink_to(self.global_prebuilts_dir,
295                                             target_is_directory=True)
296
297    def sync(self):
298        """Overrides sync() to sync a BUILD resource with prebuilts.
299
300        Delegates to the _template_resource while also ensuring that the
301        generated prebuilts directory is written to the workspace.
302        """
303        self._template_resource.sync()
304
305        # Overwrite the existing prebuilts directory, if it exists.
306        self.prebuilts_workspace_path.unlink(missing_ok=True)
307        self.prebuilts_workspace_path.symlink_to(self.global_prebuilts_dir,
308                                                 target_is_directory=True)
309
310    def clean(self):
311        """Overrides clean() to clean a BUILD resource.
312
313        Delegates most actions to the _template_resource, while also ensuring
314        that the generated prebuilts directory is removed from the workspace.
315        """
316        self._template_resource.clean()
317        self.prebuilts_workspace_path.unlink(missing_ok=True)
318
319
320def _make_executable_if_script(path: pathlib.Path) -> None:
321    """Makes the provided path executable if it is a script.
322    Args:
323        path: the path to check and conditionally make executable.
324    """
325    if path.name.endswith(".sh"):
326        # Grant full permissions (read/write/execute) for current user and
327        # read/write permissions for group.
328        path.chmod(mode=0o750)
329
330
331class ResourcesNotFoundError(Error):
332    """Raised when the required resources are not found."""
333    pass
334
335
336class Resources:
337    """Manages and loads resources from disk.
338
339    Attributes:
340        workspace_base_path: the path to the root of the workspace where
341        staged files will be synced to.
342        gendir_base_path: the path to the root of the staging directory.
343        global_prebuilts_dir_path: the path to the root of the global
344        prebuilts directory to which all generated prebuilts directories will
345        symlink to.
346        prebuilts_dir_name: the name to use for generated prebuilts
347        directories.
348        static_path: the path to the static resources directory.
349        template_path: the path to the template resources directory.
350    """
351    workspace_base_path: pathlib.Path
352    gendir_base_path: pathlib.Path
353    global_prebuilts_dir_path: pathlib.Path
354    prebuilts_dir_name: str
355    static_path: pathlib.Path
356    template_path: pathlib.Path
357
358    # Name of the directory where script runfiles are located.
359    _DATA_DIRNAME = "data"
360
361    # Name of the directory where the templates should be located.
362    _TEMPLATES_DIRNAME = "templates"
363
364    # Name of the directory where static files (to be copied to the environment)
365    # should be located.
366    _STATIC_DIRNAME = "static"
367
368    # If the script is executed from the root of the runfiles directory, this is
369    # where the data should be.
370    _RESOURCES_RUNFILES_BASEPATH = pathlib.Path(
371        "build/pesto/experiments/prepare_bazel_test_env", _DATA_DIRNAME)
372
373    # File extension for templates, determining whether or not a given file is a
374    # template.
375    _TEMPLATE_FILE_EXT = ".template"
376
377    def __init__(self,
378                 workspace_base_path: pathlib.Path,
379                 gendir_base_path: pathlib.Path,
380                 global_prebuilts_dir_path: pathlib.Path,
381                 prebuilts_dir_name: str,
382                 path: pathlib.Path = _RESOURCES_RUNFILES_BASEPATH):
383        logging.debug("Resources(path=%s)", path)
384
385        self.workspace_base_path = workspace_base_path
386        self.gendir_base_path = gendir_base_path
387        self.global_prebuilts_dir_path = global_prebuilts_dir_path
388        self.prebuilts_dir_name = prebuilts_dir_name
389
390        path = path.resolve()
391
392        self.template_path = path.joinpath(Resources._TEMPLATES_DIRNAME)
393        self.static_path = path.joinpath(Resources._STATIC_DIRNAME)
394
395        if not self.template_path.exists() or not self.static_path.exists():
396            raise ResourcesNotFoundError("Unable to find resources at path "
397                                         f"{path}, expected the following "
398                                         "directories: "
399                                         f"{Resources._TEMPLATES_DIRNAME}, "
400                                         f"{Resources._STATIC_DIRNAME}")
401
402    def __repr__(self):
403        return ("Resources("
404                f"template_path={self.template_path}"
405                f"static_path={self.static_path})")
406
407    def stage(self, mapping: Dict[str, str]) -> None:
408        for resource in self.load():
409            logging.debug("Staging resource: %s", resource)
410            resource.stage(mapping)
411
412    def sync(self) -> None:
413        for resource in self.load():
414            logging.debug("Syncing resource: %s", resource)
415            resource.sync()
416
417    def clean(self) -> None:
418        for resource in self.load():
419            logging.debug("Cleaning resource: %s", resource)
420            resource.clean()
421
422    def build_targets(self) -> Set[str]:
423        return {
424            t
425            for resource in self.load() for t in resource.build_targets()
426        }
427
428    def load(self) -> List[Resource]:
429        """Loads the Resources used to scaffold the Bazel env.
430
431        Returns:
432            a list of Resource objects representing the files
433            that should be used to template the environment.
434        """
435        resources = []
436
437        # Add all templates.
438        for p in self._template_resource_paths():
439            template_relpath = Resources._strip_template_identifier_from_path(
440                p.relative_to(self.template_path))
441            stage_path = self.gendir_base_path.joinpath(template_relpath)
442            workspace_path = self.workspace_base_path.joinpath(
443                template_relpath)
444
445            if BUILD_FILENAME_REGEX.match(template_relpath.name):
446                resources.append(
447                    BuildTemplateResource(
448                        stage_path=stage_path,
449                        workspace_path=workspace_path,
450                        resource_path=p,
451                        global_prebuilts_dir=self.global_prebuilts_dir_path,
452                        prebuilts_dir_name=self.prebuilts_dir_name))
453            else:
454                resources.append(
455                    TemplateResource(stage_path=stage_path,
456                                     workspace_path=workspace_path,
457                                     resource_path=p))
458
459        # Add all static files.
460        for p in self._static_resource_paths():
461            static_relpath = p.relative_to(self.static_path)
462            stage_path = self.gendir_base_path.joinpath(static_relpath)
463            workspace_path = self.workspace_base_path.joinpath(static_relpath)
464            resources.append(
465                StaticResource(stage_path=stage_path,
466                               workspace_path=workspace_path,
467                               resource_path=p))
468
469        return resources
470
471    def _static_resource_paths(self) -> List[pathlib.Path]:
472        return [p for p in self.static_path.glob("**/*") if p.is_file()]
473
474    def _template_resource_paths(self) -> List[pathlib.Path]:
475        return [p for p in self.template_path.glob("**/*") if p.is_file()]
476
477    @staticmethod
478    def _strip_template_identifier_from_path(p: pathlib.Path):
479        """Strips the template file extension from a provided path."""
480        if p.name.endswith(Resources._TEMPLATE_FILE_EXT):
481            p = p.with_name(p.name[:len(p.name) -
482                                   len(Resources._TEMPLATE_FILE_EXT)])
483        return p
484
485
486class AndroidBuildEnvironmentError(Error):
487    """Raised when the Android Build Environment is not properly set."""
488    pass
489
490
491class BazelTestEnvGenerator:
492    """Context for the Bazel environment generation.
493
494  This class provides access to locations within the filesystem pertinent to the
495  current execution as members that can be accessed by users of the class.
496
497  Attributes:
498      workspace_top: the top of the codebase, which also serves as the top of
499      the Bazel WORKSPACE.
500      host_out: Host artifact staging directory location.
501      host_testcases: Host testcase staging directory location.
502      product_out: Product/Target artifact staging directory location.
503      target_testcases: Product/Target testcase staging directory location.
504      staging_dir: Staging directory for generated Bazel artifacts.
505      prebuilts_dir_name: Name of the directory that should be used for the
506      directories that prebuilts are placed into when synced into the source
507      tree.
508      global_prebuilts_dir_path: The absolute path to the global prebuilts
509      directory.
510      year: the current calendar year.
511      gen_dir: Subdirectory of the staging dir where the Bazel environment
512      should be generated to.
513  """
514    workspace_top: pathlib.Path
515    host_out: pathlib.Path
516    host_testcases: pathlib.Path
517    product_out: pathlib.Path
518    target_testcases: pathlib.Path
519    staging_dir: pathlib.Path
520    prebuilts_dir_name: str = ".soong_prebuilts"
521    prebuilts_dir_path: pathlib.Path
522    gen_dir: pathlib.Path
523    year: str
524    _resources: Resources
525    _soong: Soong
526
527    # Name of the subdirectory, within the output directory, where the
528    # prebuilts directory is scaffolded.
529    #
530    # This directory then serves as the target of symlinks from across the
531    # source tree that gives any Bazel target access to the Soong staging
532    # directories.
533    GLOBAL_PREBUILTS_DIR_PATH = "prebuilts"
534
535    def __init__(self, env_dict: Dict[str, str] = os.environ):
536        try:
537            self.workspace_top = pathlib.Path(env_dict["ANDROID_BUILD_TOP"])
538            self.host_out = pathlib.Path(env_dict["ANDROID_HOST_OUT"])
539            self.host_testcases = pathlib.Path(
540                env_dict["ANDROID_HOST_OUT_TESTCASES"])
541            self.product_out = pathlib.Path(env_dict["ANDROID_PRODUCT_OUT"])
542            self.target_testcases = pathlib.Path(
543                env_dict["ANDROID_TARGET_OUT_TESTCASES"])
544        except KeyError as e:
545            raise AndroidBuildEnvironmentError(
546                "Missing expected environment variable.") from e
547
548        self.staging_dir = pathlib.Path(self.workspace_top,
549                                        "out/pesto-environment")
550        self.prebuilts_dir_name = BazelTestEnvGenerator.prebuilts_dir_name
551        self.global_prebuilts_dir_path = self.staging_dir.joinpath(
552            self.GLOBAL_PREBUILTS_DIR_PATH)
553        self.year = str(datetime.date.today().year)
554        self.gen_dir = pathlib.Path(self.staging_dir, "gen")
555
556        self._resources = Resources(self.workspace_top, self.gen_dir,
557                                    self.global_prebuilts_dir_path,
558                                    self.prebuilts_dir_name)
559        self._soong = Soong(self.workspace_top)
560
561    def __repr__(self):
562        return "GenerationContext(%s)" % vars(self)
563
564    def generate(self):
565        logging.info("Starting generation of Bazel environment.")
566
567        self._soong.build(self._resources.build_targets())
568
569        logging.debug("Creating fresh staging dir at: %s", self.staging_dir)
570        _verify_directory(self.staging_dir, clean=True)
571
572        logging.debug("Creating fresh gen dir at: %s", self.gen_dir)
573        _verify_directory(self.gen_dir, clean=True)
574
575        logging.debug("Creating global prebuilts directory at: %s",
576                      self.global_prebuilts_dir_path)
577        _verify_directory(self.global_prebuilts_dir_path, clean=True)
578
579        # Symlink the build system provided staging directories to the
580        # global prebuilts directory.
581        self.global_prebuilts_dir_path.joinpath("host").symlink_to(
582            self.host_out)
583        self.global_prebuilts_dir_path.joinpath("host_testcases").symlink_to(
584            self.host_testcases)
585        self.global_prebuilts_dir_path.joinpath("product").symlink_to(
586            self.product_out)
587        self.global_prebuilts_dir_path.joinpath("target_testcases").symlink_to(
588            self.target_testcases)
589
590        # Load and process each resource into the gen directory.
591        self._resources.stage(mapping=vars(self))
592
593        logging.info(
594            "Generation of Bazel environment to staging directory "
595            "(%s) completed successfully.", self.staging_dir)
596
597    def sync(self):
598        logging.info(
599            "Starting synchronization of generated environment to source tree."
600        )
601
602        if not self.staging_dir.exists():
603            raise FileNotFoundError("Staging directory does not exist, "
604                                    "the generate function should be called "
605                                    " to create this directory.")
606
607        self._resources.sync()
608
609        logging.info(
610            "Successfully synchronized generated Bazel environment from "
611            "%s to %s", self.gen_dir, self.workspace_top)
612
613    def clean(self):
614        logging.info("Starting clean of generated environment.")
615
616        logging.info("Cleaning up synchronized files from the source tree.")
617        # For all of our configured templates, attempt to find the corresponding
618        # location in the source tree and remove them.
619        self._resources.clean()
620
621        logging.info("Cleaning up staging directory: %s", self.staging_dir)
622        try:
623            shutil.rmtree(self.staging_dir)
624        except FileNotFoundError:
625            logging.debug("Staging directory not found during cleanup "
626                          "and may have already been removed.")
627        logging.info("Successfully cleaned up generated environment.")
628
629
630def _verify_directory(directory: pathlib.Path, clean: bool = False) -> None:
631    """Verifies that the provided directory exists, creating it if it does not.
632
633    Args:
634      directory: path to the directory to create.
635      clean: whether or not the existing directory should be removed if found or
636        reused.
637    """
638    if directory.exists() and clean:
639        logging.debug("Cleaning existing directory %s", directory)
640        shutil.rmtree(directory)
641
642    logging.debug("Verifying directory exists at %s", directory)
643    directory.mkdir(parents=True, exist_ok=True)
644