1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18	"fmt"
19	"os"
20	"path/filepath"
21	"sort"
22	"strconv"
23	"strings"
24	"time"
25
26	"android/soong/ui/metrics"
27	"android/soong/ui/status"
28)
29
30// Constructs and runs the Ninja command line with a restricted set of
31// environment variables. It's important to restrict the environment Ninja runs
32// for hermeticity reasons, and to avoid spurious rebuilds.
33func runNinjaForBuild(ctx Context, config Config) {
34	ctx.BeginTrace(metrics.PrimaryNinja, "ninja")
35	defer ctx.EndTrace()
36
37	// Sets up the FIFO status updater that reads the Ninja protobuf output, and
38	// translates it to the soong_ui status output, displaying real-time
39	// progress of the build.
40	fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
41	nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
42	defer nr.Close()
43
44	executable := config.PrebuiltBuildTool("ninja")
45	args := []string{
46		"-d", "keepdepfile",
47		"-d", "keeprsp",
48		"-d", "stats",
49		"--frontend_file", fifo,
50	}
51
52	args = append(args, config.NinjaArgs()...)
53
54	var parallel int
55	if config.UseRemoteBuild() {
56		parallel = config.RemoteParallel()
57	} else {
58		parallel = config.Parallel()
59	}
60	args = append(args, "-j", strconv.Itoa(parallel))
61	if config.keepGoing != 1 {
62		args = append(args, "-k", strconv.Itoa(config.keepGoing))
63	}
64
65	args = append(args, "-f", config.CombinedNinjaFile())
66
67	args = append(args,
68		"-o", "usesphonyoutputs=yes",
69		"-w", "dupbuild=err",
70		"-w", "missingdepfile=err")
71
72	cmd := Command(ctx, config, "ninja", executable, args...)
73
74	// Set up the nsjail sandbox Ninja runs in.
75	cmd.Sandbox = ninjaSandbox
76	if config.HasKatiSuffix() {
77		// Reads and executes a shell script from Kati that sets/unsets the
78		// environment Ninja runs in.
79		cmd.Environment.AppendFromKati(config.KatiEnvFile())
80	}
81
82	// Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been
83	// used in the past to specify extra ninja arguments.
84	if extra, ok := cmd.Environment.Get("NINJA_ARGS"); ok {
85		cmd.Args = append(cmd.Args, strings.Fields(extra)...)
86	}
87	if extra, ok := cmd.Environment.Get("NINJA_EXTRA_ARGS"); ok {
88		cmd.Args = append(cmd.Args, strings.Fields(extra)...)
89	}
90
91	ninjaHeartbeatDuration := time.Minute * 5
92	// Get the ninja heartbeat interval from the environment before it's filtered away later.
93	if overrideText, ok := cmd.Environment.Get("NINJA_HEARTBEAT_INTERVAL"); ok {
94		// For example, "1m"
95		overrideDuration, err := time.ParseDuration(overrideText)
96		if err == nil && overrideDuration.Seconds() > 0 {
97			ninjaHeartbeatDuration = overrideDuration
98		}
99	}
100
101	// Filter the environment, as ninja does not rebuild files when environment
102	// variables change.
103	//
104	// Anything listed here must not change the output of rules/actions when the
105	// value changes, otherwise incremental builds may be unsafe. Vars
106	// explicitly set to stable values elsewhere in soong_ui are fine.
107	//
108	// For the majority of cases, either Soong or the makefiles should be
109	// replicating any necessary environment variables in the command line of
110	// each action that needs it.
111	if cmd.Environment.IsEnvTrue("ALLOW_NINJA_ENV") {
112		ctx.Println("Allowing all environment variables during ninja; incremental builds may be unsafe.")
113	} else {
114		cmd.Environment.Allow(append([]string{
115			// Set the path to a symbolizer (e.g. llvm-symbolizer) so ASAN-based
116			// tools can symbolize crashes.
117			"ASAN_SYMBOLIZER_PATH",
118			"HOME",
119			"JAVA_HOME",
120			"LANG",
121			"LC_MESSAGES",
122			"OUT_DIR",
123			"PATH",
124			"PWD",
125			// https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
126			"PYTHONDONTWRITEBYTECODE",
127			"TMPDIR",
128			"USER",
129
130			// TODO: remove these carefully
131			// Options for the address sanitizer.
132			"ASAN_OPTIONS",
133			// The list of Android app modules to be built in an unbundled manner.
134			"TARGET_BUILD_APPS",
135			// The variant of the product being built. e.g. eng, userdebug, debug.
136			"TARGET_BUILD_VARIANT",
137			// The product name of the product being built, e.g. aosp_arm, aosp_flame.
138			"TARGET_PRODUCT",
139			// b/147197813 - used by art-check-debug-apex-gen
140			"EMMA_INSTRUMENT_FRAMEWORK",
141
142			// RBE client
143			"RBE_compare",
144			"RBE_exec_root",
145			"RBE_exec_strategy",
146			"RBE_invocation_id",
147			"RBE_log_dir",
148			"RBE_num_retries_if_mismatched",
149			"RBE_platform",
150			"RBE_remote_accept_cache",
151			"RBE_remote_update_cache",
152			"RBE_server_address",
153			// TODO: remove old FLAG_ variables.
154			"FLAG_compare",
155			"FLAG_exec_root",
156			"FLAG_exec_strategy",
157			"FLAG_invocation_id",
158			"FLAG_log_dir",
159			"FLAG_platform",
160			"FLAG_remote_accept_cache",
161			"FLAG_remote_update_cache",
162			"FLAG_server_address",
163
164			// ccache settings
165			"CCACHE_COMPILERCHECK",
166			"CCACHE_SLOPPINESS",
167			"CCACHE_BASEDIR",
168			"CCACHE_CPP2",
169			"CCACHE_DIR",
170		}, config.BuildBrokenNinjaUsesEnvVars()...)...)
171	}
172
173	cmd.Environment.Set("DIST_DIR", config.DistDir())
174	cmd.Environment.Set("SHELL", "/bin/bash")
175
176	// Print the environment variables that Ninja is operating in.
177	ctx.Verboseln("Ninja environment: ")
178	envVars := cmd.Environment.Environ()
179	sort.Strings(envVars)
180	for _, envVar := range envVars {
181		ctx.Verbosef("  %s", envVar)
182	}
183
184	// Poll the Ninja log for updates regularly based on the heartbeat
185	// frequency. If it isn't updated enough, then we want to surface the
186	// possibility that Ninja is stuck, to the user.
187	done := make(chan struct{})
188	defer close(done)
189	ticker := time.NewTicker(ninjaHeartbeatDuration)
190	defer ticker.Stop()
191	ninjaChecker := &ninjaStucknessChecker{
192		logPath: filepath.Join(config.OutDir(), ".ninja_log"),
193	}
194	go func() {
195		for {
196			select {
197			case <-ticker.C:
198				ninjaChecker.check(ctx, config)
199			case <-done:
200				return
201			}
202		}
203	}()
204
205	ctx.Status.Status("Starting ninja...")
206	cmd.RunAndStreamOrFatal()
207}
208
209// A simple struct for checking if Ninja gets stuck, using timestamps.
210type ninjaStucknessChecker struct {
211	logPath     string
212	prevModTime time.Time
213}
214
215// Check that a file has been modified since the last time it was checked. If
216// the mod time hasn't changed, then assume that Ninja got stuck, and print
217// diagnostics for debugging.
218func (c *ninjaStucknessChecker) check(ctx Context, config Config) {
219	info, err := os.Stat(c.logPath)
220	var newModTime time.Time
221	if err == nil {
222		newModTime = info.ModTime()
223	}
224	if newModTime == c.prevModTime {
225		// The Ninja file hasn't been modified since the last time it was
226		// checked, so Ninja could be stuck. Output some diagnostics.
227		ctx.Verbosef("ninja may be stuck; last update to %v was %v. dumping process tree...", c.logPath, newModTime)
228
229		// The "pstree" command doesn't exist on Mac, but "pstree" on Linux
230		// gives more convenient output than "ps" So, we try pstree first, and
231		// ps second
232		commandText := fmt.Sprintf("pstree -pal %v || ps -ef", os.Getpid())
233
234		cmd := Command(ctx, config, "dump process tree", "bash", "-c", commandText)
235		output := cmd.CombinedOutputOrFatal()
236		ctx.Verbose(string(output))
237
238		ctx.Verbosef("done\n")
239	}
240	c.prevModTime = newModTime
241}
242