1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "android/soong/finder" 19 "android/soong/finder/fs" 20 "android/soong/ui/logger" 21 "bytes" 22 "io/ioutil" 23 "os" 24 "path/filepath" 25 "strings" 26 27 "android/soong/ui/metrics" 28) 29 30// This file provides an interface to the Finder type for soong_ui. Finder is 31// used to recursively traverse the source tree to gather paths of files, such 32// as Android.bp or Android.mk, and store the lists/database of paths in files 33// under `$OUT_DIR/.module_paths`. This directory can also be dist'd. 34 35// NewSourceFinder returns a new Finder configured to search for source files. 36// Callers of NewSourceFinder should call <f.Shutdown()> when done 37func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { 38 ctx.BeginTrace(metrics.RunSetupTool, "find modules") 39 defer ctx.EndTrace() 40 41 // Set up the working directory for the Finder. 42 dir, err := os.Getwd() 43 if err != nil { 44 ctx.Fatalf("No working directory for module-finder: %v", err.Error()) 45 } 46 filesystem := fs.OsFs 47 48 // .out-dir and .find-ignore are markers for Finder to ignore siblings and 49 // subdirectories of the directory Finder finds them in, hence stopping the 50 // search recursively down those branches. It's possible that these files 51 // are in the root directory, and if they are, then the subsequent error 52 // messages are very confusing, so check for that here. 53 pruneFiles := []string{".out-dir", ".find-ignore"} 54 for _, name := range pruneFiles { 55 prunePath := filepath.Join(dir, name) 56 _, statErr := filesystem.Lstat(prunePath) 57 if statErr == nil { 58 ctx.Fatalf("%v must not exist", prunePath) 59 } 60 } 61 62 // Set up configuration parameters for the Finder cache. 63 cacheParams := finder.CacheParams{ 64 WorkingDirectory: dir, 65 RootDirs: []string{"."}, 66 ExcludeDirs: []string{".git", ".repo"}, 67 PruneFiles: pruneFiles, 68 IncludeFiles: []string{ 69 // Kati build definitions. 70 "Android.mk", 71 // Product configuration files. 72 "AndroidProducts.mk", 73 // General Soong build definitions, using the Blueprint syntax. 74 "Android.bp", 75 // build/blueprint build definitions, using the Blueprint syntax. 76 "Blueprints", 77 // Bazel build definitions. 78 "BUILD.bazel", 79 // Kati clean definitions. 80 "CleanSpec.mk", 81 // Ownership definition. 82 "OWNERS", 83 // Test configuration for modules in directories that contain this 84 // file. 85 "TEST_MAPPING", 86 // Bazel top-level file to mark a directory as a Bazel workspace. 87 "WORKSPACE", 88 }, 89 // Bazel Starlark configuration files. 90 IncludeSuffixes: []string{".bzl"}, 91 } 92 dumpDir := config.FileListDir() 93 f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), 94 filepath.Join(dumpDir, "files.db")) 95 if err != nil { 96 ctx.Fatalf("Could not create module-finder: %v", err) 97 } 98 return f 99} 100 101// Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree. 102func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { 103 matches := []string{} 104 for _, foundName := range entries.FileNames { 105 if foundName == "BUILD.bazel" || foundName == "WORKSPACE" || strings.HasSuffix(foundName, ".bzl") { 106 matches = append(matches, foundName) 107 } 108 } 109 return entries.DirNames, matches 110} 111 112// FindSources searches for source files known to <f> and writes them to the filesystem for 113// use later. 114func FindSources(ctx Context, config Config, f *finder.Finder) { 115 // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder 116 // if a caller such as multiproduct_kati wants to share one Finder among several builds 117 dumpDir := config.FileListDir() 118 os.MkdirAll(dumpDir, 0777) 119 120 // Stop searching a subdirectory recursively after finding an Android.mk. 121 androidMks := f.FindFirstNamedAt(".", "Android.mk") 122 err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list")) 123 if err != nil { 124 ctx.Fatalf("Could not export module list: %v", err) 125 } 126 127 // Stop searching a subdirectory recursively after finding a CleanSpec.mk. 128 cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") 129 err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) 130 if err != nil { 131 ctx.Fatalf("Could not export module list: %v", err) 132 } 133 134 // Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories. 135 androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk") 136 androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...) 137 androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...) 138 err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list")) 139 if err != nil { 140 ctx.Fatalf("Could not export product list: %v", err) 141 } 142 143 // Recursively look for all Bazel related files. 144 bazelFiles := f.FindMatching(".", findBazelFiles) 145 err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list")) 146 if err != nil { 147 ctx.Fatalf("Could not export bazel BUILD list: %v", err) 148 } 149 150 // Recursively look for all OWNERS files. 151 owners := f.FindNamedAt(".", "OWNERS") 152 err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list")) 153 if err != nil { 154 ctx.Fatalf("Could not find OWNERS: %v", err) 155 } 156 157 // Recursively look for all TEST_MAPPING files. 158 testMappings := f.FindNamedAt(".", "TEST_MAPPING") 159 err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list")) 160 if err != nil { 161 ctx.Fatalf("Could not find TEST_MAPPING: %v", err) 162 } 163 164 // Recursively look for all Android.bp files 165 androidBps := f.FindNamedAt(".", "Android.bp") 166 // The files are named "Blueprints" only in the build/blueprint directory. 167 androidBps = append(androidBps, f.FindNamedAt("build/blueprint", "Blueprints")...) 168 if len(androidBps) == 0 { 169 ctx.Fatalf("No Android.bp found") 170 } 171 err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list")) 172 if err != nil { 173 ctx.Fatalf("Could not find modules: %v", err) 174 } 175 176 if config.Dist() { 177 f.WaitForDbDump() 178 // Dist the files.db plain text database. 179 distFile(ctx, config, f.DbPath, "module_paths") 180 } 181} 182 183// Write the .list files to disk. 184func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) { 185 desiredText := strings.Join(list, "\n") 186 desiredBytes := []byte(desiredText) 187 actualBytes, readErr := ioutil.ReadFile(filePath) 188 if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) { 189 err = ioutil.WriteFile(filePath, desiredBytes, 0777) 190 if err != nil { 191 return err 192 } 193 } 194 195 distFile(ctx, config, filePath, "module_paths") 196 197 return nil 198} 199