1// Copyright 2020 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18	"bytes"
19	"fmt"
20	"io/ioutil"
21	"os"
22	"path/filepath"
23	"strings"
24
25	"android/soong/bazel"
26	"android/soong/shared"
27	"android/soong/ui/metrics"
28)
29
30func getBazelInfo(ctx Context, config Config, bazelExecutable string, bazelEnv map[string]string, query string) string {
31	infoCmd := Command(ctx, config, "bazel", bazelExecutable)
32
33	if extraStartupArgs, ok := infoCmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok {
34		infoCmd.Args = append(infoCmd.Args, strings.Fields(extraStartupArgs)...)
35	}
36
37	// Obtain the output directory path in the execution root.
38	infoCmd.Args = append(infoCmd.Args,
39		"info",
40		query,
41	)
42
43	for k, v := range bazelEnv {
44		infoCmd.Environment.Set(k, v)
45	}
46
47	infoCmd.Dir = filepath.Join(config.OutDir(), "..")
48
49	queryResult := strings.TrimSpace(string(infoCmd.OutputOrFatal()))
50	return queryResult
51}
52
53// Main entry point to construct the Bazel build command line, environment
54// variables and post-processing steps (e.g. converge output directories)
55func runBazel(ctx Context, config Config) {
56	ctx.BeginTrace(metrics.RunBazel, "bazel")
57	defer ctx.EndTrace()
58
59	// "droid" is the default ninja target.
60	// TODO(b/160568333): stop hardcoding 'droid' to support building any
61	// Ninja target.
62	outputGroups := "droid"
63	if len(config.ninjaArgs) > 0 {
64		// At this stage, the residue slice of args passed to ninja
65		// are the ninja targets to build, which can correspond directly
66		// to ninja_build's output_groups.
67		outputGroups = strings.Join(config.ninjaArgs, ",")
68	}
69
70	// Environment variables are the primary mechanism to pass information from
71	// soong_ui configuration or context to Bazel.
72	bazelEnv := make(map[string]string)
73
74	// Use *_NINJA variables to pass the root-relative path of the combined,
75	// kati-generated, soong-generated, and packaging Ninja files to Bazel.
76	// Bazel reads these from the lunch() repository rule.
77	bazelEnv["COMBINED_NINJA"] = config.CombinedNinjaFile()
78	bazelEnv["KATI_NINJA"] = config.KatiBuildNinjaFile()
79	bazelEnv["PACKAGE_NINJA"] = config.KatiPackageNinjaFile()
80	bazelEnv["SOONG_NINJA"] = config.SoongNinjaFile()
81
82	// NOTE: When Bazel is used, config.DistDir() is rigged to return a fake distdir under config.OutDir()
83	// This is to ensure that Bazel can actually write there. See config.go for more details.
84	bazelEnv["DIST_DIR"] = config.DistDir()
85
86	bazelEnv["SHELL"] = "/bin/bash"
87
88	// `tools/bazel` is the default entry point for executing Bazel in the AOSP
89	// source tree.
90	bazelExecutable := filepath.Join("tools", "bazel")
91	cmd := Command(ctx, config, "bazel", bazelExecutable)
92
93	// Append custom startup flags to the Bazel command. Startup flags affect
94	// the Bazel server itself, and any changes to these flags would incur a
95	// restart of the server, losing much of the in-memory incrementality.
96	if extraStartupArgs, ok := cmd.Environment.Get("BAZEL_STARTUP_ARGS"); ok {
97		cmd.Args = append(cmd.Args, strings.Fields(extraStartupArgs)...)
98	}
99
100	// Start constructing the `build` command.
101	actionName := bazel.BazelNinjaExecRunName
102	cmd.Args = append(cmd.Args,
103		"build",
104		// Use output_groups to select the set of outputs to produce from a
105		// ninja_build target.
106		"--output_groups="+outputGroups,
107		// Generate a performance profile
108		"--profile="+filepath.Join(shared.BazelMetricsFilename(config, actionName)),
109		"--slim_profile=true",
110	)
111
112	if config.UseRBE() {
113		for _, envVar := range []string{
114			// RBE client
115			"RBE_compare",
116			"RBE_exec_strategy",
117			"RBE_invocation_id",
118			"RBE_log_dir",
119			"RBE_num_retries_if_mismatched",
120			"RBE_platform",
121			"RBE_remote_accept_cache",
122			"RBE_remote_update_cache",
123			"RBE_server_address",
124			// TODO: remove old FLAG_ variables.
125			"FLAG_compare",
126			"FLAG_exec_root",
127			"FLAG_exec_strategy",
128			"FLAG_invocation_id",
129			"FLAG_log_dir",
130			"FLAG_platform",
131			"FLAG_remote_accept_cache",
132			"FLAG_remote_update_cache",
133			"FLAG_server_address",
134		} {
135			cmd.Args = append(cmd.Args,
136				"--action_env="+envVar)
137		}
138
139		// We need to calculate --RBE_exec_root ourselves
140		ctx.Println("Getting Bazel execution_root...")
141		cmd.Args = append(cmd.Args, "--action_env=RBE_exec_root="+getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "execution_root"))
142	}
143
144	// Ensure that the PATH environment variable value used in the action
145	// environment is the restricted set computed from soong_ui, and not a
146	// user-provided one, for hermeticity reasons.
147	if pathEnvValue, ok := config.environ.Get("PATH"); ok {
148		cmd.Environment.Set("PATH", pathEnvValue)
149		cmd.Args = append(cmd.Args, "--action_env=PATH="+pathEnvValue)
150	}
151
152	// Allow Bazel actions to see the SHELL variable (passed to Bazel above)
153	cmd.Args = append(cmd.Args, "--action_env=SHELL")
154
155	// Append custom build flags to the Bazel command. Changes to these flags
156	// may invalidate Bazel's analysis cache.
157	// These should be appended as the final args, so that they take precedence.
158	if extraBuildArgs, ok := cmd.Environment.Get("BAZEL_BUILD_ARGS"); ok {
159		cmd.Args = append(cmd.Args, strings.Fields(extraBuildArgs)...)
160	}
161
162	// Append the label of the default ninja_build target.
163	cmd.Args = append(cmd.Args,
164		"//:"+config.TargetProduct()+"-"+config.TargetBuildVariant(),
165	)
166
167	// Execute the command at the root of the directory.
168	cmd.Dir = filepath.Join(config.OutDir(), "..")
169
170	for k, v := range bazelEnv {
171		cmd.Environment.Set(k, v)
172	}
173
174	// Make a human-readable version of the bazelEnv map
175	bazelEnvStringBuffer := new(bytes.Buffer)
176	for k, v := range bazelEnv {
177		fmt.Fprintf(bazelEnvStringBuffer, "%s=%s ", k, v)
178	}
179
180	// Print the implicit command line
181	ctx.Println("Bazel implicit command line: " + strings.Join(cmd.Environment.Environ(), " ") + " " + cmd.Cmd.String() + "\n")
182
183	// Print the explicit command line too
184	ctx.Println("Bazel explicit command line: " + bazelEnvStringBuffer.String() + cmd.Cmd.String() + "\n")
185
186	// Execute the build command.
187	cmd.RunAndStreamOrFatal()
188
189	// Post-processing steps start here. Once the Bazel build completes, the
190	// output files are still stored in the execution root, not in $OUT_DIR.
191	// Ensure that the $OUT_DIR contains the expected set of files by symlinking
192	// the files from the execution root's output direction into $OUT_DIR.
193
194	ctx.Println("Getting Bazel output_path...")
195	outputBasePath := getBazelInfo(ctx, config, bazelExecutable, bazelEnv, "output_path")
196	// TODO: Don't hardcode out/ as the bazel output directory. This is
197	// currently hardcoded as ninja_build.output_root.
198	bazelNinjaBuildOutputRoot := filepath.Join(outputBasePath, "..", "out")
199
200	ctx.Println("Populating output directory...")
201	populateOutdir(ctx, config, bazelNinjaBuildOutputRoot, ".")
202}
203
204// For all files F recursively under rootPath/relativePath, creates symlinks
205// such that OutDir/F resolves to rootPath/F via symlinks.
206// NOTE: For distdir paths we rename files instead of creating symlinks, so that the distdir is independent.
207func populateOutdir(ctx Context, config Config, rootPath string, relativePath string) {
208	destDir := filepath.Join(rootPath, relativePath)
209	os.MkdirAll(destDir, 0755)
210	files, err := ioutil.ReadDir(destDir)
211	if err != nil {
212		ctx.Fatal(err)
213	}
214
215	for _, f := range files {
216		// The original Bazel file path
217		destPath := filepath.Join(destDir, f.Name())
218
219		// The desired Soong file path
220		srcPath := filepath.Join(config.OutDir(), relativePath, f.Name())
221
222		destLstatResult, destLstatErr := os.Lstat(destPath)
223		if destLstatErr != nil {
224			ctx.Fatalf("Unable to Lstat dest %s: %s", destPath, destLstatErr)
225		}
226
227		srcLstatResult, srcLstatErr := os.Lstat(srcPath)
228
229		if srcLstatErr == nil {
230			if srcLstatResult.IsDir() && destLstatResult.IsDir() {
231				// src and dest are both existing dirs - recurse on the dest dir contents...
232				populateOutdir(ctx, config, rootPath, filepath.Join(relativePath, f.Name()))
233			} else {
234				// Ignore other pre-existing src files (could be pre-existing files, directories, symlinks, ...)
235				// This can arise for files which are generated under OutDir outside of soong_build, such as .bootstrap files.
236				// FIXME: This might cause a problem later e.g. if a symlink in the build graph changes...
237			}
238		} else {
239			if !os.IsNotExist(srcLstatErr) {
240				ctx.Fatalf("Unable to Lstat src %s: %s", srcPath, srcLstatErr)
241			}
242
243			if strings.Contains(destDir, config.DistDir()) {
244				// We need to make a "real" file/dir instead of making a symlink (because the distdir can't have symlinks)
245				// Rename instead of copy in order to save disk space.
246				if err := os.Rename(destPath, srcPath); err != nil {
247					ctx.Fatalf("Unable to rename %s -> %s due to error %s", srcPath, destPath, err)
248				}
249			} else {
250				// src does not exist, so try to create a src -> dest symlink (i.e. a Soong path -> Bazel path symlink)
251				if err := os.Symlink(destPath, srcPath); err != nil {
252					ctx.Fatalf("Unable to create symlink %s -> %s due to error %s", srcPath, destPath, err)
253				}
254			}
255		}
256	}
257}
258