#!/usr/bin/env -S python3 -B # # Copyright (C) 2021 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Downloads ART Module prebuilts and creates CLs to update them in git.""" import argparse import collections import os import re import subprocess import sys import tempfile # Prebuilt description used in commit message PREBUILT_DESCR = "ART Module" # fetch_artifact branch and target BRANCH = "aosp-master-art" TARGET = "aosp_art_module" ARCHES = ["arm", "arm64", "x86", "x86_64"] # Where to install the APEX packages PACKAGE_PATH = "packages/modules/ArtPrebuilt" # Where to install the SDKs and module exports SDK_PATH = "prebuilts/module_sdk/art" SDK_VERSION = "current" # Paths to git projects to prepare CLs in GIT_PROJECT_ROOTS = [PACKAGE_PATH, SDK_PATH] SCRIPT_PATH = PACKAGE_PATH + "/update-art-module-prebuilts.py" InstallEntry = collections.namedtuple("InstallEntry", [ # Artifact path in the build, passed to fetch_target "source_path", # Local install path "install_path", # True if this is a module SDK, to be skipped by --skip-module-sdk. "module_sdk", # True if the entry is a zip file that should be unzipped to install_path "install_unzipped", ]) def install_apex_entries(apex_name): res = [] for arch in ARCHES: res.append(InstallEntry( os.path.join(arch, apex_name + ".apex"), os.path.join(PACKAGE_PATH, apex_name + "-" + arch + ".apex"), module_sdk=False, install_unzipped=False)) return res def install_sdk_entries(mainline_sdk_name, sdk_dir): return [InstallEntry( os.path.join("mainline-sdks", mainline_sdk_name + "-" + SDK_VERSION + ".zip"), os.path.join(SDK_PATH, SDK_VERSION, sdk_dir), module_sdk=True, install_unzipped=True)] install_entries = ( install_apex_entries("com.android.art") + install_apex_entries("com.android.art.debug") + install_sdk_entries("art-module-sdk", "sdk") + install_sdk_entries("art-module-host-exports", "host-exports") + install_sdk_entries("art-module-test-exports", "test-exports") ) def rewrite_bp_for_art_module_source_build(bp_path): """Rewrites an Android.bp file to conditionally prefer prebuilts.""" print("Rewriting {} for SOONG_CONFIG_art_module_source_build use." .format(bp_path)) bp_file = open(bp_path, "r+") # TODO(b/174997203): Remove this when we have a proper way to control prefer # flags in Mainline modules. header_lines = [] for line in bp_file: line = line.rstrip("\n") if not line.startswith("//"): break header_lines.append(line) art_module_types = set() content_lines = [] for line in bp_file: line = line.rstrip("\n") module_header = re.match("([a-z0-9_]+) +{$", line) if not module_header: content_lines.append(line) else: # Iterate over one Soong module. module_start = line soong_config_clause = False module_content = [] for module_line in bp_file: module_line = module_line.rstrip("\n") if module_line == "}": break if module_line == " prefer: false,": module_content.extend([ (" // Do not prefer prebuilt if " "SOONG_CONFIG_art_module_source_build is true."), " prefer: true,", " soong_config_variables: {", " source_build: {", " prefer: false,", " },", " },"]) soong_config_clause = True else: module_content.append(module_line) if soong_config_clause: module_type = "art_prebuilt_" + module_header.group(1) module_start = module_type + " {" art_module_types.add(module_type) content_lines.append(module_start) content_lines.extend(module_content) content_lines.append("}") header_lines.extend( ["", "// Soong config variable stanza added by {}.".format(SCRIPT_PATH), "soong_config_module_type_import {", " from: \"prebuilts/module_sdk/art/SoongConfig.bp\",", " module_types: ["] + [" \"{}\",".format(art_module) for art_module in sorted(art_module_types)] + [" ],", "}", ""]) bp_file.seek(0) bp_file.truncate() bp_file.write("\n".join(header_lines + content_lines)) bp_file.close() def check_call(cmd, **kwargs): """Proxy for subprocess.check_call with logging.""" msg = " ".join(cmd) if isinstance(cmd, list) else cmd if "cwd" in kwargs: msg = "In " + kwargs["cwd"] + ": " + msg print(msg) subprocess.check_call(cmd, **kwargs) def fetch_artifact(branch, target, build, fetch_pattern, local_dir): """Fetches artifact from the build server.""" fetch_artifact_path = "/google/data/ro/projects/android/fetch_artifact" cmd = [fetch_artifact_path, "--branch", branch, "--target", target, "--bid", build, fetch_pattern] check_call(cmd, cwd=local_dir) def start_branch(branch_name, git_dirs): """Creates a new repo branch in the given projects.""" check_call(["repo", "start", branch_name] + git_dirs) # In case the branch already exists we reset it to upstream, to get a clean # update CL. for git_dir in git_dirs: check_call(["git", "reset", "--hard", "@{upstream}"], cwd=git_dir) def upload_branch(git_root, branch_name): """Uploads the CLs in the given branch in the given project.""" # Set the branch as topic to bundle with the CLs in other git projects (if # any). check_call(["repo", "upload", "-t", "--br=" + branch_name, git_root]) def remove_files(git_root, subpaths, stage_removals): """Removes files in the work tree, optionally staging them in git.""" if stage_removals: check_call(["git", "rm", "-qrf", "--ignore-unmatch"] + subpaths, cwd=git_root) # Need a plain rm afterwards even if git rm was executed, because git won't # remove directories if they have non-git files in them. check_call(["rm", "-rf"] + subpaths, cwd=git_root) def commit(git_root, prebuilt_descr, branch, target, build, add_paths, bug_number): """Commits the new prebuilts.""" check_call(["git", "add"] + add_paths, cwd=git_root) if build: message = ( "Update {prebuilt_descr} prebuilts to build {build}.\n\n" "Taken from branch {branch}, target {target}." .format(prebuilt_descr=prebuilt_descr, branch=branch, target=target, build=build)) else: message = ( "DO NOT SUBMIT: Update {prebuilt_descr} prebuilts from local build." .format(prebuilt_descr=prebuilt_descr)) message += ("\n\nCL prepared by {}." "\n\nTest: Presubmits".format(SCRIPT_PATH)) if bug_number: message += ("\nBug: {}".format(bug_number)) msg_fd, msg_path = tempfile.mkstemp() with os.fdopen(msg_fd, "w") as f: f.write(message) # Do a diff first to skip the commit without error if there are no changes to # commit. check_call("git diff-index --quiet --cached HEAD -- || " "git commit -F " + msg_path, shell=True, cwd=git_root) os.unlink(msg_path) def install_entry(build, local_dist, entry): """Installs one file specified by entry.""" install_dir, install_file = os.path.split(entry.install_path) if install_dir and not os.path.exists(install_dir): os.makedirs(install_dir) if build: fetch_artifact(BRANCH, TARGET, build, entry.source_path, install_dir) else: check_call(["cp", os.path.join(local_dist, entry.source_path), install_dir]) source_file = os.path.basename(entry.source_path) if entry.install_unzipped: check_call(["mkdir", install_file], cwd=install_dir) # Add -DD to not extract timestamps that may confuse the build system. check_call(["unzip", "-DD", source_file, "-d", install_file], cwd=install_dir) check_call(["rm", source_file], cwd=install_dir) elif source_file != install_file: check_call(["mv", source_file, install_file], cwd=install_dir) def install_paths_per_git_root(roots, paths): """Partitions the given paths into subpaths within the given roots. Args: roots: List of root paths. paths: List of paths relative to the same directory as the root paths. Returns: A dict mapping each root to the subpaths under it. It's an error if some path doesn't go into any root. """ res = collections.defaultdict(list) for path in paths: found = False for root in roots: if path.startswith(root + "/"): res[root].append(path[len(root) + 1:]) found = True break if not found: sys.exit("Install path {} is not in any of the git roots: {}" .format(path, " ".join(roots))) return res def get_args(): """Parses and returns command line arguments.""" parser = argparse.ArgumentParser( epilog="Either --build or --local-dist is required.") parser.add_argument("--build", metavar="NUMBER", help="Build number to fetch from branch {}, target {}" .format(BRANCH, TARGET)) parser.add_argument("--local-dist", metavar="PATH", help="Take prebuilts from this local dist dir instead of " "using fetch_artifact") parser.add_argument("--skip-apex", action="store_true", help="Do not fetch .apex files.") parser.add_argument("--skip-module-sdk", action="store_true", help="Do not fetch and unpack sdk and module_export zips.") parser.add_argument("--skip-cls", action="store_true", help="Do not create branches or git commits") parser.add_argument("--bug", metavar="NUMBER", help="Add a 'Bug' line with this number to commit " "messages.") parser.add_argument("--upload", action="store_true", help="Upload the CLs to Gerrit") args = parser.parse_args() if ((not args.build and not args.local_dist) or (args.build and args.local_dist)): sys.exit(parser.format_help()) return args def main(): """Program entry point.""" args = get_args() if any(path for path in GIT_PROJECT_ROOTS if not os.path.exists(path)): sys.exit("This script must be run in the root of the Android build tree.") entries = install_entries if args.skip_apex: entries = [entry for entry in entries if entry.module_sdk] if args.skip_module_sdk: entries = [entry for entry in entries if not entry.module_sdk] if not entries: sys.exit("Both APEXes and SDKs skipped - nothing to do.") install_paths = [entry.install_path for entry in entries] install_paths_per_root = install_paths_per_git_root( GIT_PROJECT_ROOTS, install_paths) branch_name = PREBUILT_DESCR.lower().replace(" ", "-") + "-update" if args.build: branch_name += "-" + args.build if not args.skip_cls: git_paths = list(install_paths_per_root.keys()) start_branch(branch_name, git_paths) for git_root, subpaths in install_paths_per_root.items(): remove_files(git_root, subpaths, not args.skip_cls) for entry in entries: install_entry(args.build, args.local_dist, entry) # Postprocess the Android.bp files in the SDK snapshot to control prefer flags # on the prebuilts through SOONG_CONFIG_art_module_source_build. # TODO(b/174997203): Replace this with a better way to control prefer flags on # Mainline module prebuilts. for entry in entries: if entry.install_unzipped: bp_path = os.path.join(entry.install_path, "Android.bp") if os.path.exists(bp_path): rewrite_bp_for_art_module_source_build(bp_path) if not args.skip_cls: for git_root, subpaths in install_paths_per_root.items(): commit(git_root, PREBUILT_DESCR, BRANCH, TARGET, args.build, subpaths, args.bug) if args.upload: # Don't upload all projects in a single repo upload call, because that # makes it pop up an interactive editor. for git_root in install_paths_per_root: upload_branch(git_root, branch_name) if __name__ == "__main__": main()