1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18	"io/ioutil"
19	"os"
20	"path/filepath"
21	"strconv"
22
23	"android/soong/shared"
24	"github.com/google/blueprint/deptools"
25
26	soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
27	"github.com/google/blueprint"
28	"github.com/google/blueprint/bootstrap"
29
30	"github.com/golang/protobuf/proto"
31	"github.com/google/blueprint/microfactory"
32
33	"android/soong/ui/metrics"
34	"android/soong/ui/status"
35)
36
37const (
38	availableEnvFile = "soong.environment.available"
39	usedEnvFile      = "soong.environment.used"
40)
41
42func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
43	data, err := shared.EnvFileContents(envDeps)
44	if err != nil {
45		return err
46	}
47
48	return ioutil.WriteFile(envFile, data, 0644)
49}
50
51// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
52//
53// However, the execution of <builddir>/build.ninja happens later in
54// build/soong/ui/build/build.go#Build()
55//
56// We want to rely on as few prebuilts as possible, so we need to bootstrap
57// Soong. The process is as follows:
58//
59// 1. We use "Microfactory", a simple tool to compile Go code, to build
60//    first itself, then soong_ui from soong_ui.bash. This binary contains
61//    parts of soong_build that are needed to build itself.
62// 2. This simplified version of soong_build then reads the Blueprint files
63//    that describe itself and emits .bootstrap/build.ninja that describes
64//    how to build its full version and use that to produce the final Ninja
65//    file Soong emits.
66// 3. soong_ui executes .bootstrap/build.ninja
67//
68// (After this, Kati is executed to parse the Makefiles, but that's not part of
69// bootstrapping Soong)
70
71// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
72// probably be nicer to use a flag in bootstrap.Args instead.
73type BlueprintConfig struct {
74	srcDir           string
75	buildDir         string
76	ninjaBuildDir    string
77	debugCompilation bool
78}
79
80func (c BlueprintConfig) SrcDir() string {
81	return "."
82}
83
84func (c BlueprintConfig) BuildDir() string {
85	return c.buildDir
86}
87
88func (c BlueprintConfig) NinjaBuildDir() string {
89	return c.ninjaBuildDir
90}
91
92func (c BlueprintConfig) DebugCompilation() bool {
93	return c.debugCompilation
94}
95
96func environmentArgs(config Config, suffix string) []string {
97	return []string{
98		"--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
99		"--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
100	}
101}
102func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
103	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
104	defer ctx.EndTrace()
105
106	var args bootstrap.Args
107
108	mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
109	globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
110	bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
111	bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
112
113	args.RunGoTests = !config.skipSoongTests
114	args.UseValidations = true // Use validations to depend on tests
115	args.BuildDir = config.SoongOutDir()
116	args.NinjaBuildDir = config.OutDir()
117	args.TopFile = "Android.bp"
118	args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
119	args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
120	args.GlobFile = globFile
121	args.GeneratingPrimaryBuilder = true
122	args.EmptyNinjaFile = config.EmptyNinjaFile()
123
124	args.DelveListen = os.Getenv("SOONG_DELVE")
125	if args.DelveListen != "" {
126		args.DelvePath = shared.ResolveDelveBinary()
127	}
128
129	commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
130	bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
131	mainSoongBuildInputs := []string{"Android.bp"}
132
133	if integratedBp2Build {
134		mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
135	}
136
137	soongBuildArgs := make([]string, 0)
138	soongBuildArgs = append(soongBuildArgs, commonArgs...)
139	soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
140	soongBuildArgs = append(soongBuildArgs, "Android.bp")
141
142	mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
143		Inputs:  mainSoongBuildInputs,
144		Outputs: []string{mainNinjaFile},
145		Args:    soongBuildArgs,
146	}
147
148	if integratedBp2Build {
149		bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
150		bp2buildArgs = append(bp2buildArgs, commonArgs...)
151		bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
152		bp2buildArgs = append(bp2buildArgs, "Android.bp")
153
154		bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
155			Inputs:  []string{"Android.bp"},
156			Outputs: []string{bp2BuildMarkerFile},
157			Args:    bp2buildArgs,
158		}
159		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
160			bp2buildInvocation,
161			mainSoongBuildInvocation,
162		}
163	} else {
164		args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
165	}
166
167	blueprintCtx := blueprint.NewContext()
168	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
169	blueprintConfig := BlueprintConfig{
170		srcDir:           os.Getenv("TOP"),
171		buildDir:         config.SoongOutDir(),
172		ninjaBuildDir:    config.OutDir(),
173		debugCompilation: os.Getenv("SOONG_DELVE") != "",
174	}
175
176	bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
177	err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
178	if err != nil {
179		ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
180	}
181}
182
183func checkEnvironmentFile(currentEnv *Environment, envFile string) {
184	getenv := func(k string) string {
185		v, _ := currentEnv.Get(k)
186		return v
187	}
188	if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
189		os.Remove(envFile)
190	}
191}
192
193func runSoong(ctx Context, config Config) {
194	ctx.BeginTrace(metrics.RunSoong, "soong")
195	defer ctx.EndTrace()
196
197	// We have two environment files: .available is the one with every variable,
198	// .used with the ones that were actually used. The latter is used to
199	// determine whether Soong needs to be re-run since why re-run it if only
200	// unused variables were changed?
201	envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
202
203	for _, n := range []string{".bootstrap", ".minibootstrap"} {
204		dir := filepath.Join(config.SoongOutDir(), n)
205		if err := os.MkdirAll(dir, 0755); err != nil {
206			ctx.Fatalf("Cannot mkdir " + dir)
207		}
208	}
209
210	buildMode := config.bazelBuildMode()
211	integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
212
213	// This is done unconditionally, but does not take a measurable amount of time
214	bootstrapBlueprint(ctx, config, integratedBp2Build)
215
216	soongBuildEnv := config.Environment().Copy()
217	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
218	// For Bazel mixed builds.
219	soongBuildEnv.Set("BAZEL_PATH", "./tools/bazel")
220	soongBuildEnv.Set("BAZEL_HOME", filepath.Join(config.BazelOutDir(), "bazelhome"))
221	soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output"))
222	soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
223	soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
224
225	// For Soong bootstrapping tests
226	if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
227		soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
228	}
229
230	err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
231	if err != nil {
232		ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
233	}
234
235	func() {
236		ctx.BeginTrace(metrics.RunSoong, "environment check")
237		defer ctx.EndTrace()
238
239		soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
240		checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
241
242		if integratedBp2Build {
243			bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
244			checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
245		}
246	}()
247
248	var cfg microfactory.Config
249	cfg.Map("github.com/google/blueprint", "build/blueprint")
250
251	cfg.TrimPath = absPath(ctx, ".")
252
253	func() {
254		ctx.BeginTrace(metrics.RunSoong, "bpglob")
255		defer ctx.EndTrace()
256
257		bpglob := filepath.Join(config.SoongOutDir(), ".minibootstrap/bpglob")
258		if _, err := microfactory.Build(&cfg, bpglob, "github.com/google/blueprint/bootstrap/bpglob"); err != nil {
259			ctx.Fatalln("Failed to build bpglob:", err)
260		}
261	}()
262
263	ninja := func(name, file string) {
264		ctx.BeginTrace(metrics.RunSoong, name)
265		defer ctx.EndTrace()
266
267		fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
268		nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
269		defer nr.Close()
270
271		cmd := Command(ctx, config, "soong "+name,
272			config.PrebuiltBuildTool("ninja"),
273			"-d", "keepdepfile",
274			"-d", "stats",
275			"-o", "usesphonyoutputs=yes",
276			"-o", "preremoveoutputs=yes",
277			"-w", "dupbuild=err",
278			"-w", "outputdir=err",
279			"-w", "missingoutfile=err",
280			"-j", strconv.Itoa(config.Parallel()),
281			"--frontend_file", fifo,
282			"-f", filepath.Join(config.SoongOutDir(), file))
283
284		var ninjaEnv Environment
285
286		// This is currently how the command line to invoke soong_build finds the
287		// root of the source tree and the output root
288		ninjaEnv.Set("TOP", os.Getenv("TOP"))
289
290		cmd.Environment = &ninjaEnv
291		cmd.Sandbox = soongSandbox
292		cmd.RunAndStreamOrFatal()
293	}
294	// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
295	ninja("bootstrap", ".bootstrap/build.ninja")
296
297	var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
298	if shouldCollectBuildSoongMetrics(config) {
299		soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
300		logSoongBuildMetrics(ctx, soongBuildMetrics)
301	}
302
303	distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
304
305	if !config.SkipKati() {
306		distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
307		distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
308	}
309
310	if shouldCollectBuildSoongMetrics(config) && ctx.Metrics != nil {
311		ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
312	}
313}
314
315func shouldCollectBuildSoongMetrics(config Config) bool {
316	// Do not collect metrics protobuf if the soong_build binary ran as the bp2build converter.
317	return config.bazelBuildMode() != generateBuildFiles
318}
319
320func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {
321	soongBuildMetricsFile := filepath.Join(config.OutDir(), "soong", "soong_build_metrics.pb")
322	buf, err := ioutil.ReadFile(soongBuildMetricsFile)
323	if err != nil {
324		ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
325	}
326	soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}
327	err = proto.Unmarshal(buf, soongBuildMetrics)
328	if err != nil {
329		ctx.Fatalf("Failed to unmarshal %s: %s", soongBuildMetricsFile, err)
330	}
331	return soongBuildMetrics
332}
333
334func logSoongBuildMetrics(ctx Context, metrics *soong_metrics_proto.SoongBuildMetrics) {
335	ctx.Verbosef("soong_build metrics:")
336	ctx.Verbosef(" modules: %v", metrics.GetModules())
337	ctx.Verbosef(" variants: %v", metrics.GetVariants())
338	ctx.Verbosef(" max heap size: %v MB", metrics.GetMaxHeapSize()/1e6)
339	ctx.Verbosef(" total allocation count: %v", metrics.GetTotalAllocCount())
340	ctx.Verbosef(" total allocation size: %v MB", metrics.GetTotalAllocSize()/1e6)
341
342}
343