1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18	"context"
19	"flag"
20	"fmt"
21	"io"
22	"io/ioutil"
23	"os"
24	"path/filepath"
25	"runtime"
26	"strings"
27	"sync"
28	"syscall"
29	"time"
30
31	"android/soong/finder"
32	"android/soong/ui/build"
33	"android/soong/ui/logger"
34	"android/soong/ui/status"
35	"android/soong/ui/terminal"
36	"android/soong/ui/tracer"
37	"android/soong/zip"
38)
39
40var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]")
41
42var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
43var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)")
44
45var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
46var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
47
48var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
49var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
50
51var buildVariant = flag.String("variant", "eng", "build variant to use")
52
53var shardCount = flag.Int("shard-count", 1, "split the products into multiple shards (to spread the build onto multiple machines, etc)")
54var shard = flag.Int("shard", 1, "1-indexed shard to execute")
55
56var skipProducts multipleStringArg
57var includeProducts multipleStringArg
58
59func init() {
60	flag.Var(&skipProducts, "skip-products", "comma-separated list of products to skip (known failures, etc)")
61	flag.Var(&includeProducts, "products", "comma-separated list of products to build")
62}
63
64// multipleStringArg is a flag.Value that takes comma separated lists and converts them to a
65// []string.  The argument can be passed multiple times to append more values.
66type multipleStringArg []string
67
68func (m *multipleStringArg) String() string {
69	return strings.Join(*m, `, `)
70}
71
72func (m *multipleStringArg) Set(s string) error {
73	*m = append(*m, strings.Split(s, ",")...)
74	return nil
75}
76
77const errorLeadingLines = 20
78const errorTrailingLines = 20
79
80func errMsgFromLog(filename string) string {
81	if filename == "" {
82		return ""
83	}
84
85	data, err := ioutil.ReadFile(filename)
86	if err != nil {
87		return ""
88	}
89
90	lines := strings.Split(strings.TrimSpace(string(data)), "\n")
91	if len(lines) > errorLeadingLines+errorTrailingLines+1 {
92		lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...",
93			len(lines)-errorLeadingLines-errorTrailingLines)
94
95		lines = append(lines[:errorLeadingLines+1],
96			lines[len(lines)-errorTrailingLines:]...)
97	}
98	var buf strings.Builder
99	for _, line := range lines {
100		buf.WriteString("> ")
101		buf.WriteString(line)
102		buf.WriteString("\n")
103	}
104	return buf.String()
105}
106
107// TODO(b/70370883): This tool uses a lot of open files -- over the default
108// soft limit of 1024 on some systems. So bump up to the hard limit until I fix
109// the algorithm.
110func setMaxFiles(log logger.Logger) {
111	var limits syscall.Rlimit
112
113	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits)
114	if err != nil {
115		log.Println("Failed to get file limit:", err)
116		return
117	}
118
119	log.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max)
120	if limits.Cur == limits.Max {
121		return
122	}
123
124	limits.Cur = limits.Max
125	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
126	if err != nil {
127		log.Println("Failed to increase file limit:", err)
128	}
129}
130
131func inList(str string, list []string) bool {
132	for _, other := range list {
133		if str == other {
134			return true
135		}
136	}
137	return false
138}
139
140func copyFile(from, to string) error {
141	fromFile, err := os.Open(from)
142	if err != nil {
143		return err
144	}
145	defer fromFile.Close()
146
147	toFile, err := os.Create(to)
148	if err != nil {
149		return err
150	}
151	defer toFile.Close()
152
153	_, err = io.Copy(toFile, fromFile)
154	return err
155}
156
157type mpContext struct {
158	Context context.Context
159	Logger  logger.Logger
160	Status  status.ToolStatus
161	Tracer  tracer.Tracer
162	Finder  *finder.Finder
163	Config  build.Config
164
165	LogsDir string
166}
167
168func main() {
169	stdio := terminal.StdioImpl{}
170
171	output := terminal.NewStatusOutput(stdio.Stdout(), "", false,
172		build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"))
173
174	log := logger.New(output)
175	defer log.Cleanup()
176
177	flag.Parse()
178
179	ctx, cancel := context.WithCancel(context.Background())
180	defer cancel()
181
182	trace := tracer.New(log)
183	defer trace.Close()
184
185	stat := &status.Status{}
186	defer stat.Finish()
187	stat.AddOutput(output)
188
189	var failures failureCount
190	stat.AddOutput(&failures)
191
192	build.SetupSignals(log, cancel, func() {
193		trace.Close()
194		log.Cleanup()
195		stat.Finish()
196	})
197
198	buildCtx := build.Context{ContextImpl: &build.ContextImpl{
199		Context: ctx,
200		Logger:  log,
201		Tracer:  trace,
202		Writer:  output,
203		Status:  stat,
204	}}
205
206	args := ""
207	if *alternateResultDir {
208		args = "dist"
209	}
210	config := build.NewConfig(buildCtx, args)
211	if *outDir == "" {
212		name := "multiproduct"
213		if !*incremental {
214			name += "-" + time.Now().Format("20060102150405")
215		}
216
217		*outDir = filepath.Join(config.OutDir(), name)
218
219		// Ensure the empty files exist in the output directory
220		// containing our output directory too. This is mostly for
221		// safety, but also triggers the ninja_build file so that our
222		// build servers know that they can parse the output as if it
223		// was ninja output.
224		build.SetupOutDir(buildCtx, config)
225
226		if err := os.MkdirAll(*outDir, 0777); err != nil {
227			log.Fatalf("Failed to create tempdir: %v", err)
228		}
229	}
230	config.Environment().Set("OUT_DIR", *outDir)
231	log.Println("Output directory:", *outDir)
232
233	logsDir := filepath.Join(config.OutDir(), "logs")
234	os.MkdirAll(logsDir, 0777)
235
236	build.SetupOutDir(buildCtx, config)
237
238	os.MkdirAll(config.LogsDir(), 0777)
239	log.SetOutput(filepath.Join(config.LogsDir(), "soong.log"))
240	trace.SetOutput(filepath.Join(config.LogsDir(), "build.trace"))
241
242	var jobs = *numJobs
243	if jobs < 1 {
244		jobs = runtime.NumCPU() / 4
245
246		ramGb := int(config.TotalRAM() / 1024 / 1024 / 1024)
247		if ramJobs := ramGb / 30; ramGb > 0 && jobs > ramJobs {
248			jobs = ramJobs
249		}
250
251		if jobs < 1 {
252			jobs = 1
253		}
254	}
255	log.Verbosef("Using %d parallel jobs", jobs)
256
257	setMaxFiles(log)
258
259	finder := build.NewSourceFinder(buildCtx, config)
260	defer finder.Shutdown()
261
262	build.FindSources(buildCtx, config, finder)
263
264	vars, err := build.DumpMakeVars(buildCtx, config, nil, []string{"all_named_products"})
265	if err != nil {
266		log.Fatal(err)
267	}
268	var productsList []string
269	allProducts := strings.Fields(vars["all_named_products"])
270
271	if len(includeProducts) > 0 {
272		var missingProducts []string
273		for _, product := range includeProducts {
274			if inList(product, allProducts) {
275				productsList = append(productsList, product)
276			} else {
277				missingProducts = append(missingProducts, product)
278			}
279		}
280		if len(missingProducts) > 0 {
281			log.Fatalf("Products don't exist: %s\n", missingProducts)
282		}
283	} else {
284		productsList = allProducts
285	}
286
287	finalProductsList := make([]string, 0, len(productsList))
288	skipProduct := func(p string) bool {
289		for _, s := range skipProducts {
290			if p == s {
291				return true
292			}
293		}
294		return false
295	}
296	for _, product := range productsList {
297		if !skipProduct(product) {
298			finalProductsList = append(finalProductsList, product)
299		} else {
300			log.Verbose("Skipping: ", product)
301		}
302	}
303
304	if *shard < 1 {
305		log.Fatalf("--shard value must be >= 1, not %d\n", *shard)
306	} else if *shardCount < 1 {
307		log.Fatalf("--shard-count value must be >= 1, not %d\n", *shardCount)
308	} else if *shard > *shardCount {
309		log.Fatalf("--shard (%d) must not be greater than --shard-count (%d)\n", *shard,
310			*shardCount)
311	} else if *shardCount > 1 {
312		finalProductsList = splitList(finalProductsList, *shardCount)[*shard-1]
313	}
314
315	log.Verbose("Got product list: ", finalProductsList)
316
317	s := buildCtx.Status.StartTool()
318	s.SetTotalActions(len(finalProductsList))
319
320	mpCtx := &mpContext{
321		Context: ctx,
322		Logger:  log,
323		Status:  s,
324		Tracer:  trace,
325
326		Finder: finder,
327		Config: config,
328
329		LogsDir: logsDir,
330	}
331
332	products := make(chan string, len(productsList))
333	go func() {
334		defer close(products)
335		for _, product := range finalProductsList {
336			products <- product
337		}
338	}()
339
340	var wg sync.WaitGroup
341	for i := 0; i < jobs; i++ {
342		wg.Add(1)
343		go func() {
344			defer wg.Done()
345			for {
346				select {
347				case product := <-products:
348					if product == "" {
349						return
350					}
351					buildProduct(mpCtx, product)
352				}
353			}
354		}()
355	}
356	wg.Wait()
357
358	if *alternateResultDir {
359		args := zip.ZipArgs{
360			FileArgs: []zip.FileArg{
361				{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
362			},
363			OutputFilePath:   filepath.Join(config.RealDistDir(), "logs.zip"),
364			NumParallelJobs:  runtime.NumCPU(),
365			CompressionLevel: 5,
366		}
367		if err := zip.Zip(args); err != nil {
368			log.Fatalf("Error zipping logs: %v", err)
369		}
370	}
371
372	s.Finish()
373
374	if failures == 1 {
375		log.Fatal("1 failure")
376	} else if failures > 1 {
377		log.Fatalf("%d failures", failures)
378	} else {
379		fmt.Fprintln(output, "Success")
380	}
381}
382
383func buildProduct(mpctx *mpContext, product string) {
384	var stdLog string
385
386	outDir := filepath.Join(mpctx.Config.OutDir(), product)
387	logsDir := filepath.Join(mpctx.LogsDir, product)
388
389	if err := os.MkdirAll(outDir, 0777); err != nil {
390		mpctx.Logger.Fatalf("Error creating out directory: %v", err)
391	}
392	if err := os.MkdirAll(logsDir, 0777); err != nil {
393		mpctx.Logger.Fatalf("Error creating log directory: %v", err)
394	}
395
396	stdLog = filepath.Join(logsDir, "std.log")
397	f, err := os.Create(stdLog)
398	if err != nil {
399		mpctx.Logger.Fatalf("Error creating std.log: %v", err)
400	}
401	defer f.Close()
402
403	log := logger.New(f)
404	defer log.Cleanup()
405	log.SetOutput(filepath.Join(logsDir, "soong.log"))
406
407	action := &status.Action{
408		Description: product,
409		Outputs:     []string{product},
410	}
411	mpctx.Status.StartAction(action)
412	defer logger.Recover(func(err error) {
413		mpctx.Status.FinishAction(status.ActionResult{
414			Action: action,
415			Error:  err,
416			Output: errMsgFromLog(stdLog),
417		})
418	})
419
420	ctx := build.Context{ContextImpl: &build.ContextImpl{
421		Context: mpctx.Context,
422		Logger:  log,
423		Tracer:  mpctx.Tracer,
424		Writer:  f,
425		Thread:  mpctx.Tracer.NewThread(product),
426		Status:  &status.Status{},
427	}}
428	ctx.Status.AddOutput(terminal.NewStatusOutput(ctx.Writer, "", false,
429		build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD")))
430
431	args := append([]string(nil), flag.Args()...)
432	args = append(args, "--skip-soong-tests")
433	config := build.NewConfig(ctx, args...)
434	config.Environment().Set("OUT_DIR", outDir)
435	if !*keepArtifacts {
436		config.SetEmptyNinjaFile(true)
437	}
438	build.FindSources(ctx, config, mpctx.Finder)
439	config.Lunch(ctx, product, *buildVariant)
440
441	defer func() {
442		if *keepArtifacts {
443			args := zip.ZipArgs{
444				FileArgs: []zip.FileArg{
445					{
446						GlobDir:             outDir,
447						SourcePrefixToStrip: outDir,
448					},
449				},
450				OutputFilePath:   filepath.Join(mpctx.Config.OutDir(), product+".zip"),
451				NumParallelJobs:  runtime.NumCPU(),
452				CompressionLevel: 5,
453			}
454			if err := zip.Zip(args); err != nil {
455				log.Fatalf("Error zipping artifacts: %v", err)
456			}
457		}
458		if !*incremental {
459			os.RemoveAll(outDir)
460		}
461	}()
462
463	config.SetSkipNinja(true)
464
465	buildWhat := build.RunProductConfig
466	if !*onlyConfig {
467		buildWhat |= build.RunSoong
468		if !*onlySoong {
469			buildWhat |= build.RunKati
470		}
471	}
472
473	before := time.Now()
474	build.Build(ctx, config)
475
476	// Save std_full.log if Kati re-read the makefiles
477	if buildWhat&build.RunKati != 0 {
478		if after, err := os.Stat(config.KatiBuildNinjaFile()); err == nil && after.ModTime().After(before) {
479			err := copyFile(stdLog, filepath.Join(filepath.Dir(stdLog), "std_full.log"))
480			if err != nil {
481				log.Fatalf("Error copying log file: %s", err)
482			}
483		}
484	}
485
486	mpctx.Status.FinishAction(status.ActionResult{
487		Action: action,
488	})
489}
490
491type failureCount int
492
493func (f *failureCount) StartAction(action *status.Action, counts status.Counts) {}
494
495func (f *failureCount) FinishAction(result status.ActionResult, counts status.Counts) {
496	if result.Error != nil {
497		*f += 1
498	}
499}
500
501func (f *failureCount) Message(level status.MsgLevel, message string) {
502	if level >= status.ErrorLvl {
503		*f += 1
504	}
505}
506
507func (f *failureCount) Flush() {}
508
509func (f *failureCount) Write(p []byte) (int, error) {
510	// discard writes
511	return len(p), nil
512}
513
514func splitList(list []string, shardCount int) (ret [][]string) {
515	each := len(list) / shardCount
516	extra := len(list) % shardCount
517	for i := 0; i < shardCount; i++ {
518		count := each
519		if extra > 0 {
520			count += 1
521			extra -= 1
522		}
523		ret = append(ret, list[:count])
524		list = list[count:]
525	}
526	return
527}
528