1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Builds SDK snapshots. 17 18If the environment variable TARGET_BUILD_APPS is nonempty then only the SDKs for 19the APEXes in it are built, otherwise all configured SDKs are built. 20""" 21import argparse 22import dataclasses 23import datetime 24import enum 25import functools 26import io 27import json 28import os 29from pathlib import Path 30import re 31import shutil 32import subprocess 33import sys 34import tempfile 35import typing 36from collections import defaultdict 37from typing import Callable, List 38import zipfile 39 40COPYRIGHT_BOILERPLATE = """ 41// 42// Copyright (C) 2020 The Android Open Source Project 43// 44// Licensed under the Apache License, Version 2.0 (the "License"); 45// you may not use this file except in compliance with the License. 46// You may obtain a copy of the License at 47// 48// http://www.apache.org/licenses/LICENSE-2.0 49// 50// Unless required by applicable law or agreed to in writing, software 51// distributed under the License is distributed on an "AS IS" BASIS, 52// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 53// See the License for the specific language governing permissions and 54// limitations under the License. 55// 56""".lstrip() 57 58 59@dataclasses.dataclass(frozen=True) 60class ConfigVar: 61 """Represents a Soong configuration variable""" 62 # The config variable namespace, e.g. ANDROID. 63 namespace: str 64 65 # The name of the variable within the namespace. 66 name: str 67 68 69@dataclasses.dataclass(frozen=True) 70class FileTransformation: 71 """Performs a transformation on a file within an SDK snapshot zip file.""" 72 73 # The path of the file within the SDK snapshot zip file. 74 path: str 75 76 def apply(self, producer, path, build_release): 77 """Apply the transformation to the path; changing it in place.""" 78 with open(path, "r+", encoding="utf8") as file: 79 self._apply_transformation(producer, file, build_release) 80 81 def _apply_transformation(self, producer, file, build_release): 82 """Apply the transformation to the file. 83 84 The file has been opened in read/write mode so the implementation of 85 this must read the contents and then reset the file to the beginning 86 and write the altered contents. 87 """ 88 raise NotImplementedError 89 90 91@dataclasses.dataclass(frozen=True) 92class SoongConfigVarTransformation(FileTransformation): 93 94 # The configuration variable that will control the prefer setting. 95 configVar: ConfigVar 96 97 # The line containing the prefer property. 98 PREFER_LINE = " prefer: false," 99 100 def _apply_transformation(self, producer, file, build_release): 101 raise NotImplementedError 102 103 104@dataclasses.dataclass(frozen=True) 105class SoongConfigBoilerplateInserter(SoongConfigVarTransformation): 106 """Transforms an Android.bp file to add soong config boilerplate. 107 108 The boilerplate allows the prefer setting of the modules to be controlled 109 through a Soong configuration variable. 110 """ 111 112 # The configuration variable that will control the prefer setting. 113 configVar: ConfigVar 114 115 # The prefix to use for the soong config module types. 116 configModuleTypePrefix: str 117 118 def config_module_type(self, module_type): 119 return self.configModuleTypePrefix + module_type 120 121 def _apply_transformation(self, producer, file, build_release): 122 # TODO(b/174997203): Remove this when we have a proper way to control 123 # prefer flags in Mainline modules. 124 125 header_lines = [] 126 for line in file: 127 line = line.rstrip("\n") 128 if not line.startswith("//"): 129 break 130 header_lines.append(line) 131 132 config_module_types = set() 133 134 content_lines = [] 135 for line in file: 136 line = line.rstrip("\n") 137 138 # Check to see whether the line is the start of a new module type, 139 # e.g. <module-type> { 140 module_header = re.match("([a-z0-9_]+) +{$", line) 141 if not module_header: 142 # It is not so just add the line to the output and skip to the 143 # next line. 144 content_lines.append(line) 145 continue 146 147 module_type = module_header.group(1) 148 module_content = [] 149 150 # Iterate over the Soong module contents 151 for module_line in file: 152 module_line = module_line.rstrip("\n") 153 154 # When the end of the module has been reached then exit. 155 if module_line == "}": 156 break 157 158 # Check to see if the module is an unversioned module, i.e. 159 # without @<version>. If it is then it needs to have the soong 160 # config boilerplate added to control the setting of the prefer 161 # property. Versioned modules do not need that because they are 162 # never preferred. 163 # At the moment this differentiation between versioned and 164 # unversioned relies on the fact that the unversioned modules 165 # set "prefer: false", while the versioned modules do not. That 166 # is a little bit fragile so may require some additional checks. 167 if module_line != self.PREFER_LINE: 168 # The line does not indicate that the module needs the 169 # soong config boilerplate so add the line and skip to the 170 # next one. 171 module_content.append(module_line) 172 continue 173 174 # Add the soong config boilerplate instead of the line: 175 # prefer: false, 176 namespace = self.configVar.namespace 177 name = self.configVar.name 178 module_content.append(f"""\ 179 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 180 prefer: true, 181 soong_config_variables: {{ 182 {name}: {{ 183 prefer: false, 184 }}, 185 }},""") 186 187 # Add the module type to the list of module types that need to 188 # have corresponding config module types. 189 config_module_types.add(module_type) 190 191 # Change the module type to the corresponding soong config 192 # module type by adding the prefix. 193 module_type = self.config_module_type(module_type) 194 195 # Generate the module, possibly with the new module type and 196 # containing the soong config variables entry. 197 content_lines.append(module_type + " {") 198 content_lines.extend(module_content) 199 content_lines.append("}") 200 201 # Add the soong_config_module_type module definitions to the header 202 # lines so that they appear before any uses. 203 header_lines.append("") 204 for module_type in sorted(config_module_types): 205 # Create the corresponding soong config module type name by adding 206 # the prefix. 207 config_module_type = self.configModuleTypePrefix + module_type 208 header_lines.append(f""" 209// Soong config variable module type added by {producer.script}. 210soong_config_module_type {{ 211 name: "{config_module_type}", 212 module_type: "{module_type}", 213 config_namespace: "{self.configVar.namespace}", 214 bool_variables: ["{self.configVar.name}"], 215 properties: ["prefer"], 216}} 217""".lstrip()) 218 219 # Overwrite the file with the updated contents. 220 file.seek(0) 221 file.truncate() 222 file.write("\n".join(header_lines + content_lines) + "\n") 223 224 225@dataclasses.dataclass(frozen=True) 226class UseSourceConfigVarTransformation(SoongConfigVarTransformation): 227 228 def _apply_transformation(self, producer, file, build_release): 229 lines = [] 230 for line in file: 231 line = line.rstrip("\n") 232 if line != self.PREFER_LINE: 233 lines.append(line) 234 continue 235 236 # Replace "prefer: false" with "use_source_config_var {...}". 237 namespace = self.configVar.namespace 238 name = self.configVar.name 239 lines.append(f"""\ 240 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 241 use_source_config_var: {{ 242 config_namespace: "{namespace}", 243 var_name: "{name}", 244 }},""") 245 246 # Overwrite the file with the updated contents. 247 file.seek(0) 248 file.truncate() 249 file.write("\n".join(lines) + "\n") 250 251# Removes any lines containing prefer 252@dataclasses.dataclass(frozen=True) 253class UseNoPreferPropertyTransformation(SoongConfigVarTransformation): 254 255 def _apply_transformation(self, producer, file, build_release): 256 lines = [] 257 for line in file: 258 line = line.rstrip("\n") 259 if line != self.PREFER_LINE: 260 lines.append(line) 261 continue 262 263 # Overwrite the file with the updated contents. 264 file.seek(0) 265 file.truncate() 266 file.write("\n".join(lines) + "\n") 267 268@dataclasses.dataclass() 269class SubprocessRunner: 270 """Runs subprocesses""" 271 272 # Destination for stdout from subprocesses. 273 # 274 # This (and the following stderr) are needed to allow the tests to be run 275 # in Intellij. This ensures that the tests are run with stdout/stderr 276 # objects that work when passed to subprocess.run(stdout/stderr). Without it 277 # the tests are run with a FlushingStringIO object that has no fileno 278 # attribute - https://youtrack.jetbrains.com/issue/PY-27883. 279 stdout: io.TextIOBase = sys.stdout 280 281 # Destination for stderr from subprocesses. 282 stderr: io.TextIOBase = sys.stderr 283 284 def run(self, *args, **kwargs): 285 return subprocess.run( 286 *args, check=True, stdout=self.stdout, stderr=self.stderr, **kwargs) 287 288 289def sdk_snapshot_zip_file(snapshots_dir, sdk_name): 290 """Get the path to the sdk snapshot zip file.""" 291 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.zip") 292 293 294def sdk_snapshot_info_file(snapshots_dir, sdk_name): 295 """Get the path to the sdk snapshot info file.""" 296 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.info") 297 298 299def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name): 300 """Get the path to the sdk snapshot api diff file.""" 301 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}-api-diff.txt") 302 303 304def sdk_snapshot_gantry_metadata_json_file(snapshots_dir, sdk_name): 305 """Get the path to the sdk snapshot gantry metadata json file.""" 306 return os.path.join(snapshots_dir, 307 f"{sdk_name}-{SDK_VERSION}-gantry-metadata.json") 308 309 310# The default time to use in zip entries. Ideally, this should be the same as is 311# used by soong_zip and ziptime but there is no strict need for that to be the 312# case. What matters is this is a fixed time so that the contents of zip files 313# created by this script do not depend on when it is run, only the inputs. 314default_zip_time = datetime.datetime(2008, 1, 1, 0, 0, 0, 0, 315 datetime.timezone.utc) 316 317 318# set the timestamps of the paths to the default_zip_time. 319def set_default_timestamp(base_dir, paths): 320 for path in paths: 321 timestamp = default_zip_time.timestamp() 322 p = os.path.join(base_dir, path) 323 os.utime(p, (timestamp, timestamp)) 324 325 326# Find the git project path of the module_sdk for given module. 327def module_sdk_project_for_module(module, root_dir): 328 module = module.rsplit(".", 1)[1] 329 # git_master-art and aosp-master-art branches does not contain project for 330 # art, hence adding special case for art. 331 if module == "art": 332 return "prebuilts/module_sdk/art" 333 if module == "btservices": 334 return "prebuilts/module_sdk/Bluetooth" 335 if module == "media": 336 return "prebuilts/module_sdk/Media" 337 if module == "rkpd": 338 return "prebuilts/module_sdk/RemoteKeyProvisioning" 339 if module == "tethering": 340 return "prebuilts/module_sdk/Connectivity" 341 342 target_dir = "" 343 for dir in os.listdir(os.path.join(root_dir, "prebuilts/module_sdk/")): 344 if module.lower() in dir.lower(): 345 if target_dir: 346 print( 347 'Multiple target dirs matched "%s": %s' 348 % (module, (target_dir, dir)) 349 ) 350 sys.exit(1) 351 target_dir = dir 352 if not target_dir: 353 print("Could not find a target dir for %s" % module) 354 sys.exit(1) 355 356 return "prebuilts/module_sdk/%s" % target_dir 357 358 359@dataclasses.dataclass() 360class SnapshotBuilder: 361 """Builds sdk snapshots""" 362 363 # The path to this tool. 364 tool_path: str 365 366 # Used to run subprocesses for building snapshots. 367 subprocess_runner: SubprocessRunner 368 369 # The OUT_DIR environment variable. 370 out_dir: str 371 372 # The out/soong/mainline-sdks directory. 373 mainline_sdks_dir: str = "" 374 375 # True if apex-allowed-deps-check is to be skipped. 376 skip_allowed_deps_check: bool = False 377 378 def __post_init__(self): 379 self.mainline_sdks_dir = os.path.join(self.out_dir, 380 "soong/mainline-sdks") 381 382 def get_sdk_path(self, sdk_name): 383 """Get the path to the sdk snapshot zip file produced by soong""" 384 return os.path.join(self.mainline_sdks_dir, 385 f"{sdk_name}-{SDK_VERSION}.zip") 386 387 def build_target_paths(self, build_release, target_paths): 388 # Extra environment variables to pass to the build process. 389 extraEnv = { 390 # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but 391 # we currently break without it. 392 "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", 393 # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside 394 # sdk zip files as expected by prebuilt drop. 395 "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", 396 } 397 extraEnv.update(build_release.soong_env) 398 399 # Unless explicitly specified in the calling environment set 400 # TARGET_BUILD_VARIANT=user. 401 # This MUST be identical to the TARGET_BUILD_VARIANT used to build 402 # the corresponding APEXes otherwise it could result in different 403 # hidden API flags, see http://b/202398851#comment29 for more info. 404 target_build_variant = os.environ.get("TARGET_BUILD_VARIANT", "user") 405 cmd = [ 406 "build/soong/soong_ui.bash", 407 "--make-mode", 408 "--soong-only", 409 f"TARGET_BUILD_VARIANT={target_build_variant}", 410 "TARGET_PRODUCT=mainline_sdk", 411 "MODULE_BUILD_FROM_SOURCE=true", 412 ] + target_paths 413 if not self.skip_allowed_deps_check: 414 cmd += ["apex-allowed-deps-check"] 415 print_command(extraEnv, cmd) 416 env = os.environ.copy() 417 env.update(extraEnv) 418 self.subprocess_runner.run(cmd, env=env) 419 420 def build_snapshots(self, build_release, modules): 421 # Compute the paths to all the Soong generated sdk snapshot files 422 # required by this script. 423 paths = [ 424 sdk_snapshot_zip_file(self.mainline_sdks_dir, sdk) 425 for module in modules 426 for sdk in module.sdks 427 ] 428 429 if paths: 430 self.build_target_paths(build_release, paths) 431 return self.mainline_sdks_dir 432 433 def build_snapshots_for_build_r(self, build_release, modules): 434 # Build the snapshots as standard. 435 snapshot_dir = self.build_snapshots(build_release, modules) 436 437 # Each module will extract needed files from the original snapshot zip 438 # file and then use that to create a replacement zip file. 439 r_snapshot_dir = os.path.join(snapshot_dir, "for-R-build") 440 shutil.rmtree(r_snapshot_dir, ignore_errors=True) 441 442 build_number_file = os.path.join(self.out_dir, "soong/build_number.txt") 443 444 for module in modules: 445 apex = module.apex 446 dest_dir = os.path.join(r_snapshot_dir, apex) 447 os.makedirs(dest_dir, exist_ok=True) 448 449 # Write the bp file in the sdk_library sub-directory rather than the 450 # root of the zip file as it will be unpacked in a directory that 451 # already contains an Android.bp file that defines the corresponding 452 # apex_set. 453 bp_file = os.path.join(dest_dir, "sdk_library/Android.bp") 454 os.makedirs(os.path.dirname(bp_file), exist_ok=True) 455 456 # The first sdk in the list is the name to use. 457 sdk_name = module.sdks[0] 458 459 with open(bp_file, "w", encoding="utf8") as bp: 460 bp.write("// DO NOT EDIT. Auto-generated by the following:\n") 461 bp.write(f"// {self.tool_path}\n") 462 bp.write(COPYRIGHT_BOILERPLATE) 463 aosp_apex = google_to_aosp_name(apex) 464 465 for library in module.for_r_build.sdk_libraries: 466 module_name = library.name 467 shared_library = str(library.shared_library).lower() 468 sdk_file = sdk_snapshot_zip_file(snapshot_dir, sdk_name) 469 extract_matching_files_from_zip( 470 sdk_file, dest_dir, 471 sdk_library_files_pattern( 472 scope_pattern=r"(public|system|module-lib)", 473 name_pattern=fr"({module_name}(-removed|-stubs)?)")) 474 475 available_apexes = [f'"{aosp_apex}"'] 476 if aosp_apex != "com.android.tethering": 477 available_apexes.append(f'"test_{aosp_apex}"') 478 apex_available = ",\n ".join(available_apexes) 479 480 bp.write(f""" 481java_sdk_library_import {{ 482 name: "{module_name}", 483 owner: "google", 484 prefer: true, 485 shared_library: {shared_library}, 486 apex_available: [ 487 {apex_available}, 488 ], 489 public: {{ 490 jars: ["public/{module_name}-stubs.jar"], 491 current_api: "public/{module_name}.txt", 492 removed_api: "public/{module_name}-removed.txt", 493 sdk_version: "module_current", 494 }}, 495 system: {{ 496 jars: ["system/{module_name}-stubs.jar"], 497 current_api: "system/{module_name}.txt", 498 removed_api: "system/{module_name}-removed.txt", 499 sdk_version: "module_current", 500 }}, 501 module_lib: {{ 502 jars: ["module-lib/{module_name}-stubs.jar"], 503 current_api: "module-lib/{module_name}.txt", 504 removed_api: "module-lib/{module_name}-removed.txt", 505 sdk_version: "module_current", 506 }}, 507}} 508""") 509 510 # Copy the build_number.txt file into the snapshot. 511 snapshot_build_number_file = os.path.join( 512 dest_dir, "snapshot-creation-build-number.txt") 513 shutil.copy(build_number_file, snapshot_build_number_file) 514 515 # Make sure that all the paths being added to the zip file have a 516 # fixed timestamp so that the contents of the zip file do not depend 517 # on when this script is run, only the inputs. 518 for root, dirs, files in os.walk(dest_dir): 519 set_default_timestamp(root, dirs) 520 set_default_timestamp(root, files) 521 522 # Now zip up the files into a snapshot zip file. 523 base_file = os.path.join(r_snapshot_dir, sdk_name + "-current") 524 shutil.make_archive(base_file, "zip", dest_dir) 525 526 return r_snapshot_dir 527 528 @staticmethod 529 def does_sdk_library_support_latest_api(sdk_library): 530 if sdk_library == "conscrypt.module.platform.api" or \ 531 sdk_library == "conscrypt.module.intra.core.api": 532 return False 533 return True 534 535 def latest_api_file_targets(self, sdk_info_file): 536 # Read the sdk info file and fetch the latest scope targets. 537 with open(sdk_info_file, "r", encoding="utf8") as sdk_info_file_object: 538 sdk_info_file_json = json.loads(sdk_info_file_object.read()) 539 540 target_paths = [] 541 target_dict = {} 542 for jsonItem in sdk_info_file_json: 543 if not jsonItem["@type"] == "java_sdk_library": 544 continue 545 546 sdk_library = jsonItem["@name"] 547 if not self.does_sdk_library_support_latest_api(sdk_library): 548 continue 549 550 target_dict[sdk_library] = {} 551 for scope in jsonItem["scopes"]: 552 scope_json = jsonItem["scopes"][scope] 553 target_dict[sdk_library][scope] = {} 554 target_list = [ 555 "current_api", "latest_api", "removed_api", 556 "latest_removed_api" 557 ] 558 for target in target_list: 559 target_dict[sdk_library][scope][target] = scope_json[target] 560 target_paths.append(scope_json["latest_api"]) 561 target_paths.append(scope_json["latest_removed_api"]) 562 target_paths.append(scope_json["latest_api"] 563 .replace(".latest", ".latest.extension_version")) 564 target_paths.append(scope_json["latest_removed_api"] 565 .replace(".latest", ".latest.extension_version")) 566 567 return target_paths, target_dict 568 569 def build_sdk_scope_targets(self, build_release, modules): 570 # Build the latest scope targets for each module sdk 571 # Compute the paths to all the latest scope targets for each module sdk. 572 target_paths = [] 573 target_dict = {} 574 for module in modules: 575 for sdk in module.sdks: 576 sdk_type = sdk_type_from_name(sdk) 577 if not sdk_type.providesApis: 578 continue 579 580 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir, 581 sdk) 582 paths, dict_item = self.latest_api_file_targets(sdk_info_file) 583 target_paths.extend(paths) 584 target_dict[sdk_info_file] = dict_item 585 if target_paths: 586 self.build_target_paths(build_release, target_paths) 587 return target_dict 588 589 def appendDiffToFile(self, file_object, sdk_zip_file, current_api, 590 latest_api, snapshots_dir): 591 """Extract current api and find its diff with the latest api.""" 592 with zipfile.ZipFile(sdk_zip_file, "r") as zipObj: 593 extracted_current_api = zipObj.extract( 594 member=current_api, path=snapshots_dir) 595 # The diff tool has an exit code of 0, 1 or 2 depending on whether 596 # it find no differences, some differences or an error (like missing 597 # file). As 0 or 1 are both valid results this cannot use check=True 598 # so disable the pylint check. 599 # pylint: disable=subprocess-run-check 600 diff = subprocess.run([ 601 "diff", "-u0", latest_api, extracted_current_api, "--label", 602 latest_api, "--label", extracted_current_api 603 ], 604 capture_output=True).stdout.decode("utf-8") 605 file_object.write(diff) 606 607 def create_snapshot_gantry_metadata_and_api_diff(self, sdk, target_dict, 608 snapshots_dir, 609 module_extension_version): 610 """Creates gantry metadata and api diff files for each module sdk. 611 612 For each module sdk, the scope targets are obtained for each java sdk 613 library and the api diff files are generated by performing a diff 614 operation between the current api file vs the latest api file. 615 """ 616 sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk) 617 sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk) 618 sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 619 620 gantry_metadata_dict = {} 621 with open( 622 sdk_api_diff_file, "w", 623 encoding="utf8") as sdk_api_diff_file_object: 624 last_finalized_version_set = set() 625 for sdk_library in target_dict[sdk_info_file]: 626 for scope in target_dict[sdk_info_file][sdk_library]: 627 scope_json = target_dict[sdk_info_file][sdk_library][scope] 628 current_api = scope_json["current_api"] 629 latest_api = scope_json["latest_api"] 630 self.appendDiffToFile(sdk_api_diff_file_object, 631 sdk_zip_file, current_api, latest_api, 632 snapshots_dir) 633 634 removed_api = scope_json["removed_api"] 635 latest_removed_api = scope_json["latest_removed_api"] 636 self.appendDiffToFile(sdk_api_diff_file_object, 637 sdk_zip_file, removed_api, 638 latest_removed_api, snapshots_dir) 639 640 def read_extension_version(target): 641 extension_target = target.replace( 642 ".latest", ".latest.extension_version") 643 with open( 644 extension_target, "r", encoding="utf8") as file: 645 version = int(file.read()) 646 # version equal to -1 means "not an extension version". 647 if version != -1: 648 last_finalized_version_set.add(version) 649 650 read_extension_version(scope_json["latest_api"]) 651 read_extension_version(scope_json["latest_removed_api"]) 652 653 if len(last_finalized_version_set) == 0: 654 # Either there is no java sdk library or all java sdk libraries 655 # have not been finalized in sdk extensions yet and hence have 656 # last finalized version set as -1. 657 gantry_metadata_dict["last_finalized_version"] = -1 658 elif len(last_finalized_version_set) == 1: 659 # All java sdk library extension version match. 660 gantry_metadata_dict["last_finalized_version"] =\ 661 last_finalized_version_set.pop() 662 else: 663 # Fail the build 664 raise ValueError( 665 "Not all sdk libraries finalized with the same version.\n") 666 667 gantry_metadata_dict["api_diff_file"] = sdk_api_diff_file.rsplit( 668 "/", 1)[-1] 669 gantry_metadata_dict["api_diff_file_size"] = os.path.getsize( 670 sdk_api_diff_file) 671 gantry_metadata_dict[ 672 "module_extension_version"] = module_extension_version 673 sdk_metadata_json_file = sdk_snapshot_gantry_metadata_json_file( 674 snapshots_dir, sdk) 675 676 gantry_metadata_json_object = json.dumps(gantry_metadata_dict, indent=4) 677 with open(sdk_metadata_json_file, 678 "w") as gantry_metadata_json_file_object: 679 gantry_metadata_json_file_object.write(gantry_metadata_json_object) 680 681 if os.path.getsize(sdk_metadata_json_file) > 1048576: # 1 MB 682 raise ValueError("Metadata file size should not exceed 1 MB.\n") 683 684 def get_module_extension_version(self): 685 return int( 686 subprocess.run([ 687 "build/soong/soong_ui.bash", "--dumpvar-mode", 688 "PLATFORM_SDK_EXTENSION_VERSION" 689 ], 690 capture_output=True).stdout.decode("utf-8").strip()) 691 692 def build_snapshot_gantry_metadata_and_api_diff(self, modules, target_dict, 693 snapshots_dir): 694 """For each module sdk, create the metadata and api diff file.""" 695 module_extension_version = self.get_module_extension_version() 696 for module in modules: 697 for sdk in module.sdks: 698 sdk_type = sdk_type_from_name(sdk) 699 if not sdk_type.providesApis: 700 continue 701 self.create_snapshot_gantry_metadata_and_api_diff( 702 sdk, target_dict, snapshots_dir, module_extension_version) 703 704 705# The sdk version to build 706# 707# This is legacy from the time when this could generate versioned sdk snapshots. 708SDK_VERSION = "current" 709 710# The initially empty list of build releases. Every BuildRelease that is created 711# automatically appends itself to this list. 712ALL_BUILD_RELEASES = [] 713 714 715class PreferHandling(enum.Enum): 716 """Enumeration of the various ways of handling prefer properties""" 717 718 # No special prefer property handling is required. 719 NONE = enum.auto() 720 721 # Apply the SoongConfigBoilerplateInserter transformation. 722 SOONG_CONFIG = enum.auto() 723 724 # Use the use_source_config_var property added in T. 725 USE_SOURCE_CONFIG_VAR_PROPERTY = enum.auto() 726 727 # No prefer in Android.bp file 728 # Starting with V, prebuilts will be enabled using apex_contributions flags. 729 USE_NO_PREFER_PROPERTY = enum.auto() 730 731 732@dataclasses.dataclass(frozen=True) 733@functools.total_ordering 734class BuildRelease: 735 """Represents a build release""" 736 737 # The name of the build release, e.g. Q, R, S, T, etc. 738 name: str 739 740 # The function to call to create the snapshot in the dist, that covers 741 # building and copying the snapshot into the dist. 742 creator: Callable[ 743 ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] 744 745 # The sub-directory of dist/mainline-sdks into which the build release 746 # specific snapshots will be copied. 747 # 748 # Defaults to for-<name>-build. 749 sub_dir: str = None 750 751 # Additional environment variables to pass to Soong when building the 752 # snapshots for this build release. 753 # 754 # Defaults to { 755 # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>, 756 # } 757 soong_env: typing.Dict[str, str] = None 758 759 # The position of this instance within the BUILD_RELEASES list. 760 ordinal: int = dataclasses.field(default=-1, init=False) 761 762 # Whether this build release supports the Soong config boilerplate that is 763 # used to control the prefer setting of modules via a Soong config variable. 764 preferHandling: PreferHandling = \ 765 PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY 766 767 # Whether the generated snapshots should include flagged APIs. Defaults to 768 # false because flagged APIs are not suitable for use outside Android. 769 include_flagged_apis: bool = False 770 771 # Whether the build release should generate Gantry metadata and API diff. 772 generate_gantry_metadata_and_api_diff: bool = False 773 774 def __post_init__(self): 775 # The following use object.__setattr__ as this object is frozen and 776 # attempting to set the fields directly would cause an exception to be 777 # thrown. 778 object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) 779 # Add this to the end of the list of all build releases. 780 ALL_BUILD_RELEASES.append(self) 781 # If no sub_dir was specified then set the default. 782 if self.sub_dir is None: 783 object.__setattr__(self, "sub_dir", f"for-{self.name}-build") 784 # If no soong_env was specified then set the default. 785 if self.soong_env is None: 786 object.__setattr__( 787 self, 788 "soong_env", 789 { 790 # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a 791 # snapshot suitable for a specific target build release. 792 "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, 793 }) 794 795 def __eq__(self, other): 796 return self.ordinal == other.ordinal 797 798 def __le__(self, other): 799 return self.ordinal <= other.ordinal 800 801 802def create_no_dist_snapshot(_: BuildRelease, __: "SdkDistProducer", 803 modules: List["MainlineModule"]): 804 """A place holder dist snapshot creation function that does nothing.""" 805 print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") 806 807 808def create_dist_snapshot_for_r(build_release: BuildRelease, 809 producer: "SdkDistProducer", 810 modules: List["MainlineModule"]): 811 """Generate a snapshot suitable for use in an R build.""" 812 producer.product_dist_for_build_r(build_release, modules) 813 814 815def create_sdk_snapshots_in_soong(build_release: BuildRelease, 816 producer: "SdkDistProducer", 817 modules: List["MainlineModule"]): 818 """Builds sdks and populates the dist for unbundled modules.""" 819 producer.produce_unbundled_dist_for_build_release(build_release, modules) 820 821 822def create_latest_sdk_snapshots(build_release: BuildRelease, 823 producer: "SdkDistProducer", 824 modules: List["MainlineModule"]): 825 """Builds and populates the latest release, including bundled modules.""" 826 producer.produce_unbundled_dist_for_build_release(build_release, modules) 827 producer.produce_bundled_dist_for_build_release(build_release, modules) 828 829 830Q = BuildRelease( 831 name="Q", 832 # At the moment we do not generate a snapshot for Q. 833 creator=create_no_dist_snapshot, 834 # This does not support or need any special prefer property handling. 835 preferHandling=PreferHandling.NONE, 836) 837R = BuildRelease( 838 name="R", 839 # Generate a simple snapshot for R. 840 creator=create_dist_snapshot_for_r, 841 # By default a BuildRelease creates an environment to pass to Soong that 842 # creates a release specific snapshot. However, Soong does not yet (and is 843 # unlikely to) support building an sdk snapshot for R so create an empty 844 # environment to pass to Soong instead. 845 soong_env={}, 846 # This does not support or need any special prefer property handling. 847 preferHandling=PreferHandling.NONE, 848) 849S = BuildRelease( 850 name="S", 851 # Generate a snapshot for this build release using Soong. 852 creator=create_sdk_snapshots_in_soong, 853 # This requires the SoongConfigBoilerplateInserter transformation to be 854 # applied. 855 preferHandling=PreferHandling.SOONG_CONFIG, 856) 857Tiramisu = BuildRelease( 858 name="Tiramisu", 859 # Generate a snapshot for this build release using Soong. 860 creator=create_sdk_snapshots_in_soong, 861 # This build release supports the use_source_config_var property. 862 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 863) 864UpsideDownCake = BuildRelease( 865 name="UpsideDownCake", 866 # Generate a snapshot for this build release using Soong. 867 creator=create_sdk_snapshots_in_soong, 868 # This build release supports the use_source_config_var property. 869 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 870) 871 872# Insert additional BuildRelease definitions for following releases here, 873# before LATEST. 874 875# A build release for the latest build excluding flagged apis. 876NEXT = BuildRelease( 877 name="next", 878 creator=create_latest_sdk_snapshots, 879 # There are no build release specific environment variables to pass to 880 # Soong. 881 soong_env={}, 882 generate_gantry_metadata_and_api_diff=True, 883 # Starting with V, setting `prefer|use_source_config_var` on soong modules 884 # in prebuilts/module_sdk is not necessary. 885 # prebuilts will be enabled using apex_contributions release build flags. 886 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 887) 888 889# The build release for the latest build supported by this build, i.e. the 890# current build. This must be the last BuildRelease defined in this script. 891LATEST = BuildRelease( 892 name="latest", 893 creator=create_latest_sdk_snapshots, 894 # There are no build release specific environment variables to pass to 895 # Soong. 896 soong_env={}, 897 # Latest must include flagged APIs because it may be dropped into the main 898 # Android branches. 899 include_flagged_apis=True, 900 generate_gantry_metadata_and_api_diff=True, 901 # Starting with V, setting `prefer|use_source_config_var` on soong modules 902 # in prebuilts/module_sdk is not necessary. 903 # prebuilts will be enabled using apex_contributions release build flags. 904 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 905) 906 907 908@dataclasses.dataclass(frozen=True) 909class SdkLibrary: 910 """Information about a java_sdk_library.""" 911 912 # The name of java_sdk_library module. 913 name: str 914 915 # True if the sdk_library module is a shared library. 916 shared_library: bool = False 917 918 919@dataclasses.dataclass(frozen=True) 920class ForRBuild: 921 """Data structure needed for generating a snapshot for an R build.""" 922 923 # The java_sdk_library modules to export to the r snapshot. 924 sdk_libraries: typing.List[SdkLibrary] = dataclasses.field( 925 default_factory=list) 926 927 928@dataclasses.dataclass(frozen=True) 929class MainlineModule: 930 """Represents an unbundled mainline module. 931 932 This is a module that is distributed as a prebuilt and intended to be 933 updated with Mainline trains. 934 """ 935 # The name of the apex. 936 apex: str 937 938 # The names of the sdk and module_exports. 939 sdks: list[str] 940 941 # The first build release in which the SDK snapshot for this module is 942 # needed. 943 # 944 # Note: This is not necessarily the same build release in which the SDK 945 # source was first included. So, a module that was added in build T 946 # could potentially be used in an S release and so its SDK will need 947 # to be made available for S builds. 948 first_release: BuildRelease 949 950 # The configuration variable, defaults to ANDROID:module_build_from_source 951 configVar: ConfigVar = ConfigVar( 952 namespace="ANDROID", 953 name="module_build_from_source", 954 ) 955 956 for_r_build: typing.Optional[ForRBuild] = None 957 958 # The last release on which this module was optional. 959 # 960 # Some modules are optional when they are first released, usually because 961 # some vendors of Android devices have their own customizations of the 962 # module that they would like to preserve and which cannot yet be achieved 963 # through the existing APIs. Once those issues have been resolved then they 964 # will become mandatory. 965 # 966 # This field records the last build release in which they are optional. It 967 # defaults to None which indicates that the module was never optional. 968 # 969 # TODO(b/238203992): remove the following warning once all modules can be 970 # treated as optional at build time. 971 # 972 # DO NOT use this attr for anything other than controlling whether the 973 # generated snapshot uses its own Soong config variable or the common one. 974 # That is because this is being temporarily used to force Permission to have 975 # its own Soong config variable even though Permission is not actually 976 # optional at runtime on a GMS capable device. 977 # 978 # b/238203992 will make all modules have their own Soong config variable by 979 # default at which point this will no longer be needed on Permission and so 980 # it can be used to indicate that a module is optional at runtime. 981 last_optional_release: typing.Optional[BuildRelease] = None 982 983 # The short name for the module. 984 # 985 # Defaults to the last part of the apex name. 986 short_name: str = "" 987 988 # Additional transformations 989 additional_transformations: list[FileTransformation] = None 990 991 # The module key of SdkModule Enum defined in 992 # packages/modules/common/proto/sdk.proto. 993 module_proto_key: str = "" 994 995 def __post_init__(self): 996 # If short_name is not set then set it to the last component of the apex 997 # name. 998 if not self.short_name: 999 short_name = self.apex.rsplit(".", 1)[-1] 1000 object.__setattr__(self, "short_name", short_name) 1001 1002 def is_bundled(self): 1003 """Returns true for bundled modules. See BundledMainlineModule.""" 1004 return False 1005 1006 def transformations(self, build_release, sdk_type): 1007 """Returns the transformations to apply to this module's snapshot(s).""" 1008 transformations = [] 1009 1010 config_var = self.configVar 1011 1012 # If the module is optional then it needs its own Soong config 1013 # variable to allow it to be managed separately from other modules. 1014 if self.last_optional_release: 1015 config_var = ConfigVar( 1016 namespace=f"{self.short_name}_module", 1017 name="source_build", 1018 ) 1019 1020 prefer_handling = build_release.preferHandling 1021 if prefer_handling == PreferHandling.SOONG_CONFIG: 1022 sdk_type_prefix = sdk_type.configModuleTypePrefix 1023 config_module_type_prefix = \ 1024 f"{self.short_name}{sdk_type_prefix}_prebuilt_" 1025 inserter = SoongConfigBoilerplateInserter( 1026 "Android.bp", 1027 configVar=config_var, 1028 configModuleTypePrefix=config_module_type_prefix) 1029 transformations.append(inserter) 1030 elif prefer_handling == PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY: 1031 transformation = UseSourceConfigVarTransformation( 1032 "Android.bp", configVar=config_var) 1033 transformations.append(transformation) 1034 elif prefer_handling == PreferHandling.USE_NO_PREFER_PROPERTY: 1035 transformation = UseNoPreferPropertyTransformation( 1036 "Android.bp", configVar=config_var 1037 ) 1038 transformations.append(transformation) 1039 elif prefer_handling == PreferHandling.USE_NO_PREFER_PROPERTY: 1040 transformation = UseNoPreferPropertyTransformation( 1041 "Android.bp", configVar=config_var 1042 ) 1043 transformations.append(transformation) 1044 1045 if self.additional_transformations and build_release > R: 1046 transformations.extend(self.additional_transformations) 1047 1048 return transformations 1049 1050 def is_required_for(self, target_build_release): 1051 """True if this module is required for the target build release.""" 1052 return self.first_release <= target_build_release 1053 1054 1055@dataclasses.dataclass(frozen=True) 1056class BundledMainlineModule(MainlineModule): 1057 """Represents a bundled Mainline module or a platform SDK for module use. 1058 1059 A bundled module is always preloaded into the platform images. 1060 """ 1061 1062 # Defaults to the latest build, i.e. the build on which this script is run 1063 # as bundled modules are, by definition, only needed in this build. 1064 first_release: BuildRelease = LATEST 1065 1066 def is_bundled(self): 1067 return True 1068 1069 def transformations(self, build_release, sdk_type): 1070 # Bundled modules are only used on thin branches where the corresponding 1071 # sources are absent, so skip transformations and keep the default 1072 # `prefer: false`. 1073 return [] 1074 1075 1076# List of mainline modules. 1077MAINLINE_MODULES = [ 1078 MainlineModule( 1079 apex="com.android.adservices", 1080 sdks=["adservices-module-sdk"], 1081 first_release=Tiramisu, 1082 last_optional_release=LATEST, 1083 module_proto_key="AD_SERVICES", 1084 ), 1085 MainlineModule( 1086 apex="com.android.appsearch", 1087 sdks=["appsearch-sdk"], 1088 first_release=Tiramisu, 1089 last_optional_release=LATEST, 1090 module_proto_key="APPSEARCH", 1091 ), 1092 MainlineModule( 1093 apex="com.android.art", 1094 sdks=[ 1095 "art-module-sdk", 1096 "art-module-test-exports", 1097 "art-module-host-exports", 1098 ], 1099 first_release=S, 1100 # Override the config... fields. 1101 configVar=ConfigVar( 1102 namespace="art_module", 1103 name="source_build", 1104 ), 1105 module_proto_key="ART", 1106 ), 1107 MainlineModule( 1108 apex="com.android.btservices", 1109 sdks=["btservices-module-sdk"], 1110 first_release=UpsideDownCake, 1111 # Bluetooth has always been and is still optional. 1112 last_optional_release=LATEST, 1113 module_proto_key="", 1114 ), 1115 MainlineModule( 1116 apex="com.android.configinfrastructure", 1117 sdks=["configinfrastructure-sdk"], 1118 first_release=UpsideDownCake, 1119 last_optional_release=LATEST, 1120 module_proto_key="CONFIG_INFRASTRUCTURE", 1121 ), 1122 MainlineModule( 1123 apex="com.android.conscrypt", 1124 sdks=[ 1125 "conscrypt-module-sdk", 1126 "conscrypt-module-test-exports", 1127 "conscrypt-module-host-exports", 1128 ], 1129 first_release=Q, 1130 # No conscrypt java_sdk_library modules are exported to the R snapshot. 1131 # Conscrypt was updatable in R but the generate_ml_bundle.sh does not 1132 # appear to generate a snapshot for it. 1133 for_r_build=None, 1134 last_optional_release=LATEST, 1135 module_proto_key="CONSCRYPT", 1136 ), 1137 MainlineModule( 1138 apex="com.android.devicelock", 1139 sdks=["devicelock-module-sdk"], 1140 first_release=UpsideDownCake, 1141 # Treat DeviceLock as optional at build time 1142 # TODO(b/238203992): remove once all modules are optional at build time. 1143 last_optional_release=LATEST, 1144 module_proto_key="", 1145 ), 1146 MainlineModule( 1147 apex="com.android.healthfitness", 1148 sdks=["healthfitness-module-sdk"], 1149 first_release=UpsideDownCake, 1150 last_optional_release=LATEST, 1151 module_proto_key="HEALTH_FITNESS", 1152 ), 1153 MainlineModule( 1154 apex="com.android.ipsec", 1155 sdks=["ipsec-module-sdk"], 1156 first_release=R, 1157 for_r_build=ForRBuild(sdk_libraries=[ 1158 SdkLibrary( 1159 name="android.net.ipsec.ike", 1160 shared_library=True, 1161 ), 1162 ]), 1163 last_optional_release=LATEST, 1164 module_proto_key="IPSEC", 1165 ), 1166 MainlineModule( 1167 apex="com.android.media", 1168 sdks=["media-module-sdk"], 1169 first_release=R, 1170 for_r_build=ForRBuild(sdk_libraries=[ 1171 SdkLibrary(name="framework-media"), 1172 ]), 1173 last_optional_release=LATEST, 1174 module_proto_key="MEDIA", 1175 ), 1176 MainlineModule( 1177 apex="com.android.mediaprovider", 1178 sdks=["mediaprovider-module-sdk"], 1179 first_release=R, 1180 for_r_build=ForRBuild(sdk_libraries=[ 1181 SdkLibrary(name="framework-mediaprovider"), 1182 ]), 1183 # MP is a mandatory mainline module but in some cases (b/294190883) this 1184 # needs to be optional for Android Go on T. GTS tests might be needed to 1185 # to check the specific condition mentioned in the bug. 1186 last_optional_release=LATEST, 1187 module_proto_key="MEDIA_PROVIDER", 1188 ), 1189 MainlineModule( 1190 apex="com.android.ondevicepersonalization", 1191 sdks=["ondevicepersonalization-module-sdk"], 1192 first_release=Tiramisu, 1193 last_optional_release=LATEST, 1194 module_proto_key="ON_DEVICE_PERSONALIZATION", 1195 ), 1196 MainlineModule( 1197 apex="com.android.permission", 1198 sdks=["permission-module-sdk"], 1199 first_release=R, 1200 for_r_build=ForRBuild(sdk_libraries=[ 1201 SdkLibrary(name="framework-permission"), 1202 # framework-permission-s is not needed on R as it contains classes 1203 # that are provided in R by non-updatable parts of the 1204 # bootclasspath. 1205 ]), 1206 # Although Permission is not, and has never been, optional for GMS 1207 # capable devices it does need to be treated as optional at build time 1208 # when building non-GMS devices. 1209 # TODO(b/238203992): remove once all modules are optional at build time. 1210 last_optional_release=LATEST, 1211 module_proto_key="PERMISSIONS", 1212 ), 1213 MainlineModule( 1214 apex="com.android.rkpd", 1215 sdks=["rkpd-sdk"], 1216 first_release=UpsideDownCake, 1217 # Rkpd has always been and is still optional. 1218 last_optional_release=LATEST, 1219 module_proto_key="", 1220 ), 1221 MainlineModule( 1222 apex="com.android.scheduling", 1223 sdks=["scheduling-sdk"], 1224 first_release=S, 1225 last_optional_release=LATEST, 1226 module_proto_key="SCHEDULING", 1227 ), 1228 MainlineModule( 1229 apex="com.android.sdkext", 1230 sdks=["sdkextensions-sdk"], 1231 first_release=R, 1232 for_r_build=ForRBuild(sdk_libraries=[ 1233 SdkLibrary(name="framework-sdkextensions"), 1234 ]), 1235 last_optional_release=LATEST, 1236 module_proto_key="SDK_EXTENSIONS", 1237 ), 1238 MainlineModule( 1239 apex="com.android.os.statsd", 1240 sdks=["statsd-module-sdk"], 1241 first_release=R, 1242 for_r_build=ForRBuild(sdk_libraries=[ 1243 SdkLibrary(name="framework-statsd"), 1244 ]), 1245 last_optional_release=LATEST, 1246 module_proto_key="STATSD", 1247 ), 1248 MainlineModule( 1249 apex="com.android.tethering", 1250 sdks=["tethering-module-sdk"], 1251 first_release=R, 1252 for_r_build=ForRBuild(sdk_libraries=[ 1253 SdkLibrary(name="framework-tethering"), 1254 ]), 1255 last_optional_release=LATEST, 1256 module_proto_key="TETHERING", 1257 ), 1258 MainlineModule( 1259 apex="com.android.uwb", 1260 sdks=["uwb-module-sdk"], 1261 first_release=Tiramisu, 1262 # Uwb has always been and is still optional. 1263 last_optional_release=LATEST, 1264 module_proto_key="", 1265 ), 1266 MainlineModule( 1267 apex="com.android.wifi", 1268 sdks=["wifi-module-sdk"], 1269 first_release=R, 1270 for_r_build=ForRBuild(sdk_libraries=[ 1271 SdkLibrary(name="framework-wifi"), 1272 ]), 1273 # Wifi has always been and is still optional. 1274 last_optional_release=LATEST, 1275 module_proto_key="", 1276 ), 1277] 1278 1279# List of Mainline modules that currently are never built unbundled. They must 1280# not specify first_release, and they don't have com.google.android 1281# counterparts. 1282BUNDLED_MAINLINE_MODULES = [ 1283 BundledMainlineModule( 1284 apex="com.android.i18n", 1285 sdks=[ 1286 "i18n-module-sdk", 1287 "i18n-module-test-exports", 1288 "i18n-module-host-exports", 1289 ], 1290 ), 1291 BundledMainlineModule( 1292 apex="com.android.runtime", 1293 sdks=[ 1294 "runtime-module-host-exports", 1295 "runtime-module-sdk", 1296 ], 1297 ), 1298 BundledMainlineModule( 1299 apex="com.android.tzdata", 1300 sdks=["tzdata-module-test-exports"], 1301 ), 1302] 1303 1304# List of platform SDKs for Mainline module use. 1305PLATFORM_SDKS_FOR_MAINLINE = [ 1306 BundledMainlineModule( 1307 apex="platform-mainline", 1308 sdks=[ 1309 "platform-mainline-sdk", 1310 "platform-mainline-test-exports", 1311 ], 1312 ), 1313] 1314 1315 1316@dataclasses.dataclass 1317class SdkDistProducer: 1318 """Produces the DIST_DIR/mainline-sdks and DIST_DIR/stubs directories. 1319 1320 Builds SDK snapshots for mainline modules and then copies them into the 1321 DIST_DIR/mainline-sdks directory. Also extracts the sdk_library txt, jar and 1322 srcjar files from each SDK snapshot and copies them into the DIST_DIR/stubs 1323 directory. 1324 """ 1325 1326 # Used to run subprocesses for this. 1327 subprocess_runner: SubprocessRunner 1328 1329 # Builds sdk snapshots 1330 snapshot_builder: SnapshotBuilder 1331 1332 # The DIST_DIR environment variable. 1333 dist_dir: str = "uninitialized-dist" 1334 1335 # The path to this script. It may be inserted into files that are 1336 # transformed to document where the changes came from. 1337 script: str = sys.argv[0] 1338 1339 # The path to the mainline-sdks dist directory for unbundled modules. 1340 # 1341 # Initialized in __post_init__(). 1342 mainline_sdks_dir: str = dataclasses.field(init=False) 1343 1344 # The path to the mainline-sdks dist directory for bundled modules and 1345 # platform SDKs. 1346 # 1347 # Initialized in __post_init__(). 1348 bundled_mainline_sdks_dir: str = dataclasses.field(init=False) 1349 1350 def __post_init__(self): 1351 self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") 1352 self.bundled_mainline_sdks_dir = os.path.join(self.dist_dir, 1353 "bundled-mainline-sdks") 1354 1355 def prepare(self): 1356 pass 1357 1358 def produce_dist(self, modules, build_releases): 1359 # Prepare the dist directory for the sdks. 1360 self.prepare() 1361 1362 # Group build releases so that those with the same Soong environment are 1363 # run consecutively to avoid having to regenerate ninja files. 1364 grouped_by_env = defaultdict(list) 1365 for build_release in build_releases: 1366 grouped_by_env[str(build_release.soong_env)].append(build_release) 1367 ordered = [br for _, group in grouped_by_env.items() for br in group] 1368 1369 for build_release in ordered: 1370 # Only build modules that are required for this build release. 1371 filtered_modules = [ 1372 m for m in modules if m.is_required_for(build_release) 1373 ] 1374 if filtered_modules: 1375 print(f"Building SDK snapshots for {build_release.name}" 1376 f" build release") 1377 build_release.creator(build_release, self, filtered_modules) 1378 1379 def product_dist_for_build_r(self, build_release, modules): 1380 # Although we only need a subset of the files that a java_sdk_library 1381 # adds to an sdk snapshot generating the whole snapshot is the simplest 1382 # way to ensure that all the necessary files are produced. 1383 1384 # Filter out any modules that do not provide sdk for R. 1385 modules = [m for m in modules if m.for_r_build] 1386 1387 snapshot_dir = self.snapshot_builder.build_snapshots_for_build_r( 1388 build_release, modules) 1389 self.populate_unbundled_dist(build_release, modules, snapshot_dir) 1390 1391 def produce_unbundled_dist_for_build_release(self, build_release, modules): 1392 modules = [m for m in modules if not m.is_bundled()] 1393 snapshots_dir = self.snapshot_builder.build_snapshots( 1394 build_release, modules) 1395 if build_release.generate_gantry_metadata_and_api_diff: 1396 target_dict = self.snapshot_builder.build_sdk_scope_targets( 1397 build_release, modules) 1398 self.snapshot_builder.build_snapshot_gantry_metadata_and_api_diff( 1399 modules, target_dict, snapshots_dir) 1400 self.populate_unbundled_dist(build_release, modules, snapshots_dir) 1401 return snapshots_dir 1402 1403 def produce_bundled_dist_for_build_release(self, build_release, modules): 1404 modules = [m for m in modules if m.is_bundled()] 1405 if modules: 1406 snapshots_dir = self.snapshot_builder.build_snapshots( 1407 build_release, modules) 1408 self.populate_bundled_dist(build_release, modules, snapshots_dir) 1409 1410 def dist_sdk_snapshot_gantry_metadata_and_api_diff(self, sdk_dist_dir, sdk, 1411 module, snapshots_dir): 1412 """Copy the sdk snapshot api diff file to a dist directory.""" 1413 sdk_type = sdk_type_from_name(sdk) 1414 if not sdk_type.providesApis: 1415 return 1416 1417 sdk_dist_module_subdir = os.path.join(sdk_dist_dir, module.apex) 1418 sdk_dist_subdir = os.path.join(sdk_dist_module_subdir, "sdk") 1419 os.makedirs(sdk_dist_subdir, exist_ok=True) 1420 sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 1421 shutil.copy(sdk_api_diff_path, sdk_dist_subdir) 1422 1423 sdk_gantry_metadata_json_path = sdk_snapshot_gantry_metadata_json_file( 1424 snapshots_dir, sdk) 1425 sdk_dist_gantry_metadata_json_path = os.path.join( 1426 sdk_dist_module_subdir, "gantry-metadata.json") 1427 shutil.copy(sdk_gantry_metadata_json_path, 1428 sdk_dist_gantry_metadata_json_path) 1429 1430 def dist_generate_sdk_supported_modules_file(self, modules): 1431 sdk_modules_file = os.path.join(self.dist_dir, "sdk-modules.txt") 1432 os.makedirs(os.path.dirname(sdk_modules_file), exist_ok=True) 1433 with open(sdk_modules_file, "w", encoding="utf8") as file: 1434 for module in modules: 1435 if module in MAINLINE_MODULES: 1436 file.write(aosp_to_google_name(module.apex) + "\n") 1437 1438 def generate_mainline_modules_info_file(self, modules, root_dir): 1439 mainline_modules_info_file = os.path.join( 1440 self.dist_dir, "mainline-modules-info.json" 1441 ) 1442 os.makedirs(os.path.dirname(mainline_modules_info_file), exist_ok=True) 1443 mainline_modules_info_dict = {} 1444 for module in modules: 1445 if module not in MAINLINE_MODULES: 1446 continue 1447 module_name = aosp_to_google_name(module.apex) 1448 mainline_modules_info_dict[module_name] = dict() 1449 mainline_modules_info_dict[module_name]["module_sdk_project"] = ( 1450 module_sdk_project_for_module(module_name, root_dir) 1451 ) 1452 mainline_modules_info_dict[module_name][ 1453 "module_proto_key" 1454 ] = module.module_proto_key 1455 # The first sdk in the list is the name to use. 1456 mainline_modules_info_dict[module_name]["sdk_name"] = module.sdks[0] 1457 1458 with open(mainline_modules_info_file, "w", encoding="utf8") as file: 1459 json.dump(mainline_modules_info_dict, file, indent=4) 1460 1461 def populate_unbundled_dist(self, build_release, modules, snapshots_dir): 1462 build_release_dist_dir = os.path.join(self.mainline_sdks_dir, 1463 build_release.sub_dir) 1464 for module in modules: 1465 for sdk in module.sdks: 1466 sdk_dist_dir = os.path.join(build_release_dist_dir, SDK_VERSION) 1467 if build_release.generate_gantry_metadata_and_api_diff: 1468 self.dist_sdk_snapshot_gantry_metadata_and_api_diff( 1469 sdk_dist_dir, sdk, module, snapshots_dir) 1470 self.populate_dist_snapshot(build_release, module, sdk, 1471 sdk_dist_dir, snapshots_dir) 1472 1473 def populate_bundled_dist(self, build_release, modules, snapshots_dir): 1474 sdk_dist_dir = self.bundled_mainline_sdks_dir 1475 for module in modules: 1476 for sdk in module.sdks: 1477 self.populate_dist_snapshot(build_release, module, sdk, 1478 sdk_dist_dir, snapshots_dir) 1479 1480 def populate_dist_snapshot(self, build_release, module, sdk, sdk_dist_dir, 1481 snapshots_dir): 1482 sdk_type = sdk_type_from_name(sdk) 1483 subdir = sdk_type.name 1484 1485 sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, subdir) 1486 sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk) 1487 sdk_type = sdk_type_from_name(sdk) 1488 transformations = module.transformations(build_release, sdk_type) 1489 self.dist_sdk_snapshot_zip( 1490 build_release, sdk_path, sdk_dist_subdir, transformations) 1491 1492 def dist_sdk_snapshot_zip( 1493 self, build_release, src_sdk_zip, sdk_dist_dir, transformations): 1494 """Copy the sdk snapshot zip file to a dist directory. 1495 1496 If no transformations are provided then this simply copies the show sdk 1497 snapshot zip file to the dist dir. However, if transformations are 1498 provided then the files to be transformed are extracted from the 1499 snapshot zip file, they are transformed to files in a separate directory 1500 and then a new zip file is created in the dist directory with the 1501 original files replaced by the newly transformed files. build_release is 1502 provided for transformations if it is needed. 1503 """ 1504 os.makedirs(sdk_dist_dir, exist_ok=True) 1505 dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip)) 1506 print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}") 1507 1508 # If no transformations are provided then just copy the zip file 1509 # directly. 1510 if len(transformations) == 0: 1511 shutil.copy(src_sdk_zip, sdk_dist_dir) 1512 return 1513 1514 with tempfile.TemporaryDirectory() as tmp_dir: 1515 # Create a single pattern that will match any of the paths provided 1516 # in the transformations. 1517 pattern = "|".join( 1518 [f"({re.escape(t.path)})" for t in transformations]) 1519 1520 # Extract the matching files from the zip into the temporary 1521 # directory. 1522 extract_matching_files_from_zip(src_sdk_zip, tmp_dir, pattern) 1523 1524 # Apply the transformations to the extracted files in situ. 1525 apply_transformations(self, tmp_dir, transformations, build_release) 1526 1527 # Replace the original entries in the zip with the transformed 1528 # files. 1529 paths = [transformation.path for transformation in transformations] 1530 copy_zip_and_replace(self, src_sdk_zip, dest_sdk_zip, tmp_dir, 1531 paths) 1532 1533 1534def print_command(env, cmd): 1535 print(" ".join([f"{name}={value}" for name, value in env.items()] + cmd)) 1536 1537 1538def sdk_library_files_pattern(*, scope_pattern=r"[^/]+", name_pattern=r"[^/]+"): 1539 """Return a pattern to match sdk_library related files in an sdk snapshot""" 1540 return rf"sdk_library/{scope_pattern}/{name_pattern}\.(txt|jar|srcjar)" 1541 1542 1543def extract_matching_files_from_zip(zip_path, dest_dir, pattern): 1544 """Extracts files from a zip file into a destination directory. 1545 1546 The extracted files are those that match the specified regular expression 1547 pattern. 1548 """ 1549 os.makedirs(dest_dir, exist_ok=True) 1550 with zipfile.ZipFile(zip_path) as zip_file: 1551 for filename in zip_file.namelist(): 1552 if re.match(pattern, filename): 1553 print(f" extracting {filename}") 1554 zip_file.extract(filename, dest_dir) 1555 1556 1557def copy_zip_and_replace(producer, src_zip_path, dest_zip_path, src_dir, paths): 1558 """Copies a zip replacing some of its contents in the process. 1559 1560 The files to replace are specified by the paths parameter and are relative 1561 to the src_dir. 1562 """ 1563 # Get the absolute paths of the source and dest zip files so that they are 1564 # not affected by a change of directory. 1565 abs_src_zip_path = os.path.abspath(src_zip_path) 1566 abs_dest_zip_path = os.path.abspath(dest_zip_path) 1567 1568 # Make sure that all the paths being added to the zip file have a fixed 1569 # timestamp so that the contents of the zip file do not depend on when this 1570 # script is run, only the inputs. 1571 set_default_timestamp(src_dir, paths) 1572 1573 producer.subprocess_runner.run( 1574 ["zip", "-q", abs_src_zip_path, "--out", abs_dest_zip_path] + paths, 1575 # Change into the source directory before running zip. 1576 cwd=src_dir) 1577 1578 1579def apply_transformations(producer, tmp_dir, transformations, build_release): 1580 for transformation in transformations: 1581 path = os.path.join(tmp_dir, transformation.path) 1582 1583 # Record the timestamp of the file. 1584 modified = os.path.getmtime(path) 1585 1586 # Transform the file. 1587 transformation.apply(producer, path, build_release) 1588 1589 # Reset the timestamp of the file to the original timestamp before the 1590 # transformation was applied. 1591 os.utime(path, (modified, modified)) 1592 1593 1594def create_producer(tool_path, skip_allowed_deps_check): 1595 # Variables initialized from environment variables that are set by the 1596 # calling mainline_modules_sdks.sh. 1597 out_dir = os.environ["OUT_DIR"] 1598 dist_dir = os.environ["DIST_DIR"] 1599 1600 top_dir = os.environ["ANDROID_BUILD_TOP"] 1601 tool_path = os.path.relpath(tool_path, top_dir) 1602 tool_path = tool_path.replace(".py", ".sh") 1603 1604 subprocess_runner = SubprocessRunner() 1605 snapshot_builder = SnapshotBuilder( 1606 tool_path=tool_path, 1607 subprocess_runner=subprocess_runner, 1608 out_dir=out_dir, 1609 skip_allowed_deps_check=skip_allowed_deps_check, 1610 ) 1611 return SdkDistProducer( 1612 subprocess_runner=subprocess_runner, 1613 snapshot_builder=snapshot_builder, 1614 dist_dir=dist_dir, 1615 ) 1616 1617 1618def aosp_to_google(module): 1619 """Transform an AOSP module into a Google module""" 1620 new_apex = aosp_to_google_name(module.apex) 1621 # Create a copy of the AOSP module with the internal specific APEX name. 1622 return dataclasses.replace(module, apex=new_apex) 1623 1624 1625def aosp_to_google_name(name): 1626 """Transform an AOSP module name into a Google module name""" 1627 return name.replace("com.android.", "com.google.android.") 1628 1629 1630def google_to_aosp_name(name): 1631 """Transform a Google module name into an AOSP module name""" 1632 return name.replace("com.google.android.", "com.android.") 1633 1634 1635@dataclasses.dataclass(frozen=True) 1636class SdkType: 1637 name: str 1638 1639 configModuleTypePrefix: str 1640 1641 providesApis: bool = False 1642 1643 1644Sdk = SdkType( 1645 name="sdk", 1646 configModuleTypePrefix="", 1647 providesApis=True, 1648) 1649HostExports = SdkType( 1650 name="host-exports", 1651 configModuleTypePrefix="_host_exports", 1652) 1653TestExports = SdkType( 1654 name="test-exports", 1655 configModuleTypePrefix="_test_exports", 1656) 1657 1658 1659def sdk_type_from_name(name): 1660 if name.endswith("-sdk"): 1661 return Sdk 1662 if name.endswith("-host-exports"): 1663 return HostExports 1664 if name.endswith("-test-exports"): 1665 return TestExports 1666 1667 raise Exception(f"{name} is not a valid sdk name, expected it to end" 1668 f" with -(sdk|host-exports|test-exports)") 1669 1670 1671def filter_modules(modules, target_build_apps): 1672 if target_build_apps: 1673 target_build_apps = target_build_apps.split() 1674 return [m for m in modules if m.apex in target_build_apps] 1675 return modules 1676 1677 1678def main(args): 1679 """Program entry point.""" 1680 if not os.path.exists("build/make/core/Makefile"): 1681 sys.exit("This script must be run from the top of the tree.") 1682 1683 args_parser = argparse.ArgumentParser( 1684 description="Build snapshot zips for consumption by Gantry.") 1685 args_parser.add_argument( 1686 "--tool-path", 1687 help="The path to this tool.", 1688 default="unspecified", 1689 ) 1690 args_parser.add_argument( 1691 "--build-release", 1692 action="append", 1693 choices=[br.name for br in ALL_BUILD_RELEASES], 1694 help="A target build for which snapshots are required. " 1695 "If it is \"latest\" then Mainline module SDKs from platform and " 1696 "bundled modules are included.", 1697 ) 1698 args_parser.add_argument( 1699 "--build-platform-sdks-for-mainline", 1700 action="store_true", 1701 help="Also build the platform SDKs for Mainline modules. " 1702 "Defaults to true when TARGET_BUILD_APPS is not set. " 1703 "Applicable only if the \"latest\" build release is built.", 1704 ) 1705 args_parser.add_argument( 1706 "--skip-allowed-deps-check", 1707 action="store_true", 1708 help="Skip apex-allowed-deps-check.", 1709 ) 1710 args = args_parser.parse_args(args) 1711 1712 build_releases = ALL_BUILD_RELEASES 1713 if args.build_release: 1714 selected_build_releases = {b.lower() for b in args.build_release} 1715 build_releases = [ 1716 b for b in build_releases 1717 if b.name.lower() in selected_build_releases 1718 ] 1719 1720 target_build_apps = os.environ.get("TARGET_BUILD_APPS") 1721 modules = filter_modules(MAINLINE_MODULES + BUNDLED_MAINLINE_MODULES, 1722 target_build_apps) 1723 1724 # Also build the platform Mainline SDKs either if no specific modules are 1725 # requested or if --build-platform-sdks-for-mainline is given. 1726 if not target_build_apps or args.build_platform_sdks_for_mainline: 1727 modules += PLATFORM_SDKS_FOR_MAINLINE 1728 1729 producer = create_producer(args.tool_path, args.skip_allowed_deps_check) 1730 producer.dist_generate_sdk_supported_modules_file(modules) 1731 producer.generate_mainline_modules_info_file( 1732 modules, os.environ["ANDROID_BUILD_TOP"] 1733 ) 1734 producer.produce_dist(modules, build_releases) 1735 1736 1737if __name__ == "__main__": 1738 main(sys.argv[1:]) 1739