1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package main 16 17import ( 18 "flag" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "path/filepath" 23 "strings" 24 "time" 25 26 "android/soong/bp2build" 27 "android/soong/shared" 28 29 "github.com/google/blueprint/bootstrap" 30 "github.com/google/blueprint/deptools" 31 32 "android/soong/android" 33) 34 35var ( 36 topDir string 37 outDir string 38 availableEnvFile string 39 usedEnvFile string 40 41 delveListen string 42 delvePath string 43 44 docFile string 45 bazelQueryViewDir string 46 bp2buildMarker string 47) 48 49func init() { 50 // Flags that make sense in every mode 51 flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") 52 flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)") 53 flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables") 54 flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables") 55 56 // Debug flags 57 flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging") 58 flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set") 59 60 // Flags representing various modes soong_build can run in 61 flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") 62 flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") 63 flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") 64} 65 66func newNameResolver(config android.Config) *android.NameResolver { 67 namespacePathsToExport := make(map[string]bool) 68 69 for _, namespaceName := range config.ExportedNamespaces() { 70 namespacePathsToExport[namespaceName] = true 71 } 72 73 namespacePathsToExport["."] = true // always export the root namespace 74 75 exportFilter := func(namespace *android.Namespace) bool { 76 return namespacePathsToExport[namespace.Path] 77 } 78 79 return android.NewNameResolver(exportFilter) 80} 81 82func newContext(configuration android.Config, prepareBuildActions bool) *android.Context { 83 ctx := android.NewContext(configuration) 84 ctx.Register() 85 if !prepareBuildActions { 86 configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) 87 } 88 ctx.SetNameInterface(newNameResolver(configuration)) 89 ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) 90 return ctx 91} 92 93func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Config { 94 configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineArgs.ModuleListFile, availableEnv) 95 if err != nil { 96 fmt.Fprintf(os.Stderr, "%s", err) 97 os.Exit(1) 98 } 99 return configuration 100} 101 102// Bazel-enabled mode. Soong runs in two passes. 103// First pass: Analyze the build tree, but only store all bazel commands 104// needed to correctly evaluate the tree in the second pass. 105// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite 106// the incorrect results from the first pass, and file I/O is expensive. 107func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { 108 var firstArgs, secondArgs bootstrap.Args 109 110 firstArgs = bootstrap.CmdlineArgs 111 configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja) 112 bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration) 113 114 // Invoke bazel commands and save results for second pass. 115 if err := configuration.BazelContext.InvokeBazel(); err != nil { 116 fmt.Fprintf(os.Stderr, "%s", err) 117 os.Exit(1) 118 } 119 // Second pass: Full analysis, using the bazel command results. Output ninja file. 120 secondConfig, err := android.ConfigForAdditionalRun(configuration) 121 if err != nil { 122 fmt.Fprintf(os.Stderr, "%s", err) 123 os.Exit(1) 124 } 125 secondCtx := newContext(secondConfig, true) 126 secondArgs = bootstrap.CmdlineArgs 127 ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig) 128 ninjaDeps = append(ninjaDeps, extraNinjaDeps...) 129 err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps) 130 if err != nil { 131 fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err) 132 os.Exit(1) 133 } 134} 135 136// Run the code-generation phase to convert BazelTargetModules to BUILD files. 137func runQueryView(configuration android.Config, ctx *android.Context) { 138 codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView) 139 absoluteQueryViewDir := shared.JoinPath(topDir, bazelQueryViewDir) 140 if err := createBazelQueryView(codegenContext, absoluteQueryViewDir); err != nil { 141 fmt.Fprintf(os.Stderr, "%s", err) 142 os.Exit(1) 143 } 144} 145 146func runSoongDocs(configuration android.Config) { 147 ctx := newContext(configuration, false) 148 soongDocsArgs := bootstrap.CmdlineArgs 149 bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration) 150 if err := writeDocs(ctx, configuration, docFile); err != nil { 151 fmt.Fprintf(os.Stderr, "%s", err) 152 os.Exit(1) 153 } 154} 155 156func writeMetrics(configuration android.Config) { 157 metricsFile := filepath.Join(configuration.BuildDir(), "soong_build_metrics.pb") 158 err := android.WriteMetrics(configuration, metricsFile) 159 if err != nil { 160 fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err) 161 os.Exit(1) 162 } 163} 164 165func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) { 166 f, err := os.Create(path) 167 if err != nil { 168 fmt.Fprintf(os.Stderr, "%s", err) 169 os.Exit(1) 170 } 171 172 defer f.Close() 173 ctx.Context.PrintJSONGraph(f) 174 writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir()) 175} 176 177func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { 178 bazelConversionRequested := bp2buildMarker != "" 179 mixedModeBuild := configuration.BazelContext.BazelEnabled() 180 generateQueryView := bazelQueryViewDir != "" 181 jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH") 182 183 blueprintArgs := bootstrap.CmdlineArgs 184 prepareBuildActions := !generateQueryView && jsonModuleFile == "" 185 if bazelConversionRequested { 186 // Run the alternate pipeline of bp2build mutators and singleton to convert 187 // Blueprint to BUILD files before everything else. 188 runBp2Build(configuration, extraNinjaDeps) 189 if bp2buildMarker != "" { 190 return bp2buildMarker 191 } else { 192 return bootstrap.CmdlineArgs.OutFile 193 } 194 } 195 196 ctx := newContext(configuration, prepareBuildActions) 197 if mixedModeBuild { 198 runMixedModeBuild(configuration, ctx, extraNinjaDeps) 199 } else { 200 ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration) 201 ninjaDeps = append(ninjaDeps, extraNinjaDeps...) 202 err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps) 203 if err != nil { 204 fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err) 205 os.Exit(1) 206 } 207 } 208 209 // Convert the Soong module graph into Bazel BUILD files. 210 if generateQueryView { 211 runQueryView(configuration, ctx) 212 return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie 213 } 214 215 if jsonModuleFile != "" { 216 writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps) 217 return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie 218 } 219 220 writeMetrics(configuration) 221 return bootstrap.CmdlineArgs.OutFile 222} 223 224// soong_ui dumps the available environment variables to 225// soong.environment.available . Then soong_build itself is run with an empty 226// environment so that the only way environment variables can be accessed is 227// using Config, which tracks access to them. 228 229// At the end of the build, a file called soong.environment.used is written 230// containing the current value of all used environment variables. The next 231// time soong_ui is run, it checks whether any environment variables that was 232// used had changed and if so, it deletes soong.environment.used to cause a 233// rebuild. 234// 235// The dependency of build.ninja on soong.environment.used is declared in 236// build.ninja.d 237func parseAvailableEnv() map[string]string { 238 if availableEnvFile == "" { 239 fmt.Fprintf(os.Stderr, "--available_env not set\n") 240 os.Exit(1) 241 } 242 243 result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile)) 244 if err != nil { 245 fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err) 246 os.Exit(1) 247 } 248 249 return result 250} 251 252func main() { 253 flag.Parse() 254 255 shared.ReexecWithDelveMaybe(delveListen, delvePath) 256 android.InitSandbox(topDir) 257 258 availableEnv := parseAvailableEnv() 259 260 // The top-level Blueprints file is passed as the first argument. 261 srcDir := filepath.Dir(flag.Arg(0)) 262 configuration := newConfig(srcDir, outDir, availableEnv) 263 extraNinjaDeps := []string{ 264 configuration.ProductVariablesFileName, 265 usedEnvFile, 266 } 267 268 if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { 269 configuration.SetAllowMissingDependencies() 270 } 271 272 if shared.IsDebugging() { 273 // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is 274 // enabled even if it completed successfully. 275 extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve")) 276 } 277 278 if docFile != "" { 279 // We don't write an used variables file when generating documentation 280 // because that is done from within the actual builds as a Ninja action and 281 // thus it would overwrite the actual used variables file so this is 282 // special-cased. 283 // TODO: Fix this by not passing --used_env to the soong_docs invocation 284 runSoongDocs(configuration) 285 return 286 } 287 288 finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) 289 writeUsedEnvironmentFile(configuration, finalOutputFile) 290} 291 292func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) { 293 if usedEnvFile == "" { 294 return 295 } 296 297 path := shared.JoinPath(topDir, usedEnvFile) 298 data, err := shared.EnvFileContents(configuration.EnvDeps()) 299 if err != nil { 300 fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) 301 os.Exit(1) 302 } 303 304 err = ioutil.WriteFile(path, data, 0666) 305 if err != nil { 306 fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) 307 os.Exit(1) 308 } 309 310 // Touch the output file so that it's not older than the file we just 311 // wrote. We can't write the environment file earlier because one an access 312 // new environment variables while writing it. 313 touch(shared.JoinPath(topDir, finalOutputFile)) 314} 315 316// Workarounds to support running bp2build in a clean AOSP checkout with no 317// prior builds, and exiting early as soon as the BUILD files get generated, 318// therefore not creating build.ninja files that soong_ui and callers of 319// soong_build expects. 320// 321// These files are: build.ninja and build.ninja.d. Since Kati hasn't been 322// ran as well, and `nothing` is defined in a .mk file, there isn't a ninja 323// target called `nothing`, so we manually create it here. 324func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) { 325 extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ") 326 327 ninjaFileName := "build.ninja" 328 ninjaFile := shared.JoinPath(topDir, buildDir, ninjaFileName) 329 ninjaFileD := shared.JoinPath(topDir, buildDir, ninjaFileName) 330 // A workaround to create the 'nothing' ninja target so `m nothing` works, 331 // since bp2build runs without Kati, and the 'nothing' target is declared in 332 // a Makefile. 333 ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n phony_output = true\n"), 0666) 334 ioutil.WriteFile(ninjaFileD, 335 []byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFileName, extraNinjaDepsString)), 336 0666) 337} 338 339func touch(path string) { 340 f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) 341 if err != nil { 342 fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) 343 os.Exit(1) 344 } 345 346 err = f.Close() 347 if err != nil { 348 fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) 349 os.Exit(1) 350 } 351 352 currentTime := time.Now().Local() 353 err = os.Chtimes(path, currentTime, currentTime) 354 if err != nil { 355 fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err) 356 os.Exit(1) 357 } 358} 359 360// Run Soong in the bp2build mode. This creates a standalone context that registers 361// an alternate pipeline of mutators and singletons specifically for generating 362// Bazel BUILD files instead of Ninja files. 363func runBp2Build(configuration android.Config, extraNinjaDeps []string) { 364 // Register an alternate set of singletons and mutators for bazel 365 // conversion for Bazel conversion. 366 bp2buildCtx := android.NewContext(configuration) 367 368 // Propagate "allow misssing dependencies" bit. This is normally set in 369 // newContext(), but we create bp2buildCtx without calling that method. 370 bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) 371 bp2buildCtx.SetNameInterface(newNameResolver(configuration)) 372 bp2buildCtx.RegisterForBazelConversion() 373 374 // The bp2build process is a purely functional process that only depends on 375 // Android.bp files. It must not depend on the values of per-build product 376 // configurations or variables, since those will generate different BUILD 377 // files based on how the user has configured their tree. 378 bp2buildCtx.SetModuleListFile(bootstrap.CmdlineArgs.ModuleListFile) 379 modulePaths, err := bp2buildCtx.ListModulePaths(configuration.SrcDir()) 380 if err != nil { 381 panic(err) 382 } 383 384 extraNinjaDeps = append(extraNinjaDeps, modulePaths...) 385 386 // No need to generate Ninja build rules/statements from Modules and Singletons. 387 configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) 388 389 // Run the loading and analysis pipeline to prepare the graph of regular 390 // Modules parsed from Android.bp files, and the BazelTargetModules mapped 391 // from the regular Modules. 392 blueprintArgs := bootstrap.CmdlineArgs 393 ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration) 394 ninjaDeps = append(ninjaDeps, extraNinjaDeps...) 395 396 ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...) 397 398 // Run the code-generation phase to convert BazelTargetModules to BUILD files 399 // and print conversion metrics to the user. 400 codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) 401 metrics := bp2build.Codegen(codegenContext) 402 403 generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build") 404 workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace") 405 406 excludes := []string{ 407 "bazel-bin", 408 "bazel-genfiles", 409 "bazel-out", 410 "bazel-testlogs", 411 "bazel-" + filepath.Base(topDir), 412 } 413 414 if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' { 415 excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir) 416 } 417 418 symlinkForestDeps := bp2build.PlantSymlinkForest( 419 topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes) 420 421 // Only report metrics when in bp2build mode. The metrics aren't relevant 422 // for queryview, since that's a total repo-wide conversion and there's a 423 // 1:1 mapping for each module. 424 metrics.Print() 425 426 ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) 427 ninjaDeps = append(ninjaDeps, symlinkForestDeps...) 428 429 depFile := bp2buildMarker + ".d" 430 err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps) 431 if err != nil { 432 fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err) 433 os.Exit(1) 434 } 435 436 if bp2buildMarker != "" { 437 touch(shared.JoinPath(topDir, bp2buildMarker)) 438 } else { 439 writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir()) 440 } 441} 442