1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18	"android/soong/finder"
19	"android/soong/finder/fs"
20	"android/soong/ui/logger"
21	"bytes"
22	"io/ioutil"
23	"os"
24	"path/filepath"
25	"strings"
26
27	"android/soong/ui/metrics"
28)
29
30// This file provides an interface to the Finder type for soong_ui. Finder is
31// used to recursively traverse the source tree to gather paths of files, such
32// as Android.bp or Android.mk, and store the lists/database of paths in files
33// under `$OUT_DIR/.module_paths`. This directory can also be dist'd.
34
35// NewSourceFinder returns a new Finder configured to search for source files.
36// Callers of NewSourceFinder should call <f.Shutdown()> when done
37func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
38	ctx.BeginTrace(metrics.RunSetupTool, "find modules")
39	defer ctx.EndTrace()
40
41	// Set up the working directory for the Finder.
42	dir, err := os.Getwd()
43	if err != nil {
44		ctx.Fatalf("No working directory for module-finder: %v", err.Error())
45	}
46	filesystem := fs.OsFs
47
48	// .out-dir and .find-ignore are markers for Finder to ignore siblings and
49	// subdirectories of the directory Finder finds them in, hence stopping the
50	// search recursively down those branches. It's possible that these files
51	// are in the root directory, and if they are, then the subsequent error
52	// messages are very confusing, so check for that here.
53	pruneFiles := []string{".out-dir", ".find-ignore"}
54	for _, name := range pruneFiles {
55		prunePath := filepath.Join(dir, name)
56		_, statErr := filesystem.Lstat(prunePath)
57		if statErr == nil {
58			ctx.Fatalf("%v must not exist", prunePath)
59		}
60	}
61
62	// Set up configuration parameters for the Finder cache.
63	cacheParams := finder.CacheParams{
64		WorkingDirectory: dir,
65		RootDirs:         []string{"."},
66		ExcludeDirs:      []string{".git", ".repo"},
67		PruneFiles:       pruneFiles,
68		IncludeFiles: []string{
69			// Kati build definitions.
70			"Android.mk",
71			// Product configuration files.
72			"AndroidProducts.mk",
73			// General Soong build definitions, using the Blueprint syntax.
74			"Android.bp",
75			// build/blueprint build definitions, using the Blueprint syntax.
76			"Blueprints",
77			// Bazel build definitions.
78			"BUILD.bazel",
79			// Kati clean definitions.
80			"CleanSpec.mk",
81			// Ownership definition.
82			"OWNERS",
83			// Test configuration for modules in directories that contain this
84			// file.
85			"TEST_MAPPING",
86			// Bazel top-level file to mark a directory as a Bazel workspace.
87			"WORKSPACE",
88		},
89		// Bazel Starlark configuration files.
90		IncludeSuffixes: []string{".bzl"},
91	}
92	dumpDir := config.FileListDir()
93	f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
94		filepath.Join(dumpDir, "files.db"))
95	if err != nil {
96		ctx.Fatalf("Could not create module-finder: %v", err)
97	}
98	return f
99}
100
101// Finds the list of Bazel-related files (BUILD, WORKSPACE and Starlark) in the tree.
102func findBazelFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
103	matches := []string{}
104	for _, foundName := range entries.FileNames {
105		if foundName == "BUILD.bazel" || foundName == "WORKSPACE" || strings.HasSuffix(foundName, ".bzl") {
106			matches = append(matches, foundName)
107		}
108	}
109	return entries.DirNames, matches
110}
111
112// FindSources searches for source files known to <f> and writes them to the filesystem for
113// use later.
114func FindSources(ctx Context, config Config, f *finder.Finder) {
115	// note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
116	// if a caller such as multiproduct_kati wants to share one Finder among several builds
117	dumpDir := config.FileListDir()
118	os.MkdirAll(dumpDir, 0777)
119
120	// Stop searching a subdirectory recursively after finding an Android.mk.
121	androidMks := f.FindFirstNamedAt(".", "Android.mk")
122	err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list"))
123	if err != nil {
124		ctx.Fatalf("Could not export module list: %v", err)
125	}
126
127	// Stop searching a subdirectory recursively after finding a CleanSpec.mk.
128	cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
129	err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
130	if err != nil {
131		ctx.Fatalf("Could not export module list: %v", err)
132	}
133
134	// Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories.
135	androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk")
136	androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...)
137	androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...)
138	err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list"))
139	if err != nil {
140		ctx.Fatalf("Could not export product list: %v", err)
141	}
142
143	// Recursively look for all Bazel related files.
144	bazelFiles := f.FindMatching(".", findBazelFiles)
145	err = dumpListToFile(ctx, config, bazelFiles, filepath.Join(dumpDir, "bazel.list"))
146	if err != nil {
147		ctx.Fatalf("Could not export bazel BUILD list: %v", err)
148	}
149
150	// Recursively look for all OWNERS files.
151	owners := f.FindNamedAt(".", "OWNERS")
152	err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))
153	if err != nil {
154		ctx.Fatalf("Could not find OWNERS: %v", err)
155	}
156
157	// Recursively look for all TEST_MAPPING files.
158	testMappings := f.FindNamedAt(".", "TEST_MAPPING")
159	err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
160	if err != nil {
161		ctx.Fatalf("Could not find TEST_MAPPING: %v", err)
162	}
163
164	// Recursively look for all Android.bp files
165	androidBps := f.FindNamedAt(".", "Android.bp")
166	// The files are named "Blueprints" only in the build/blueprint directory.
167	androidBps = append(androidBps, f.FindNamedAt("build/blueprint", "Blueprints")...)
168	if len(androidBps) == 0 {
169		ctx.Fatalf("No Android.bp found")
170	}
171	err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list"))
172	if err != nil {
173		ctx.Fatalf("Could not find modules: %v", err)
174	}
175
176	if config.Dist() {
177		f.WaitForDbDump()
178		// Dist the files.db plain text database.
179		distFile(ctx, config, f.DbPath, "module_paths")
180	}
181}
182
183// Write the .list files to disk.
184func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) {
185	desiredText := strings.Join(list, "\n")
186	desiredBytes := []byte(desiredText)
187	actualBytes, readErr := ioutil.ReadFile(filePath)
188	if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
189		err = ioutil.WriteFile(filePath, desiredBytes, 0777)
190		if err != nil {
191			return err
192		}
193	}
194
195	distFile(ctx, config, filePath, "module_paths")
196
197	return nil
198}
199