1 // Copyright 2019 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // This executable runs a series of build commands to test and benchmark some critical user journeys. 16 package main 17 18 import ( 19 "context" 20 "fmt" 21 "os" 22 "path/filepath" 23 "strconv" 24 "strings" 25 "time" 26 27 "android/soong/ui/build" 28 "android/soong/ui/logger" 29 "android/soong/ui/metrics" 30 "android/soong/ui/status" 31 "android/soong/ui/terminal" 32 "android/soong/ui/tracer" 33 ) 34 35 type Test struct { 36 name string 37 args []string 38 before func() error 39 40 results TestResults 41 } 42 43 type TestResults struct { 44 metrics *metrics.Metrics 45 err error 46 } 47 48 // Run runs a single build command. It emulates the "m" command line by calling into Soong UI directly. 49 func (t *Test) Run(logsDir string) { 50 output := terminal.NewStatusOutput(os.Stdout, "", false, false) 51 52 log := logger.New(output) 53 defer log.Cleanup() 54 55 ctx, cancel := context.WithCancel(context.Background()) 56 defer cancel() 57 58 trace := tracer.New(log) 59 defer trace.Close() 60 61 met := metrics.New() 62 63 stat := &status.Status{} 64 defer stat.Finish() 65 stat.AddOutput(output) 66 stat.AddOutput(trace.StatusTracer()) 67 68 build.SetupSignals(log, cancel, func() { 69 trace.Close() 70 log.Cleanup() 71 stat.Finish() 72 }) 73 74 buildCtx := build.Context{ContextImpl: &build.ContextImpl{ 75 Context: ctx, 76 Logger: log, 77 Metrics: met, 78 Tracer: trace, 79 Writer: output, 80 Status: stat, 81 }} 82 83 defer logger.Recover(func(err error) { 84 t.results.err = err 85 }) 86 87 config := build.NewConfig(buildCtx, t.args...) 88 build.SetupOutDir(buildCtx, config) 89 90 os.MkdirAll(logsDir, 0777) 91 log.SetOutput(filepath.Join(logsDir, "soong.log")) 92 trace.SetOutput(filepath.Join(logsDir, "build.trace")) 93 stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) 94 stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) 95 stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error"))) 96 stat.AddOutput(status.NewCriticalPath(log)) 97 98 defer met.Dump(filepath.Join(logsDir, "soong_metrics")) 99 100 if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { 101 if !strings.HasSuffix(start, "N") { 102 if start_time, err := strconv.ParseUint(start, 10, 64); err == nil { 103 log.Verbosef("Took %dms to start up.", 104 time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds()) 105 buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano())) 106 } 107 } 108 109 if executable, err := os.Executable(); err == nil { 110 trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace")) 111 } 112 } 113 114 f := build.NewSourceFinder(buildCtx, config) 115 defer f.Shutdown() 116 build.FindSources(buildCtx, config, f) 117 118 build.Build(buildCtx, config) 119 120 t.results.metrics = met 121 } 122 123 // Touch the Intent.java file to cause a rebuild of the frameworks to monitor the 124 // incremental build speed as mentioned b/152046247. Intent.java file was chosen 125 // as it is a key component of the framework and is often modified. 126 func touchIntentFile() error { 127 const intentFileName = "frameworks/base/core/java/android/content/Intent.java" 128 currentTime := time.Now().Local() 129 return os.Chtimes(intentFileName, currentTime, currentTime) 130 } 131 132 func main() { 133 outDir := os.Getenv("OUT_DIR") 134 if outDir == "" { 135 outDir = "out" 136 } 137 138 cujDir := filepath.Join(outDir, "cuj_tests") 139 140 // Use a subdirectory for the out directory for the tests to keep them isolated. 141 os.Setenv("OUT_DIR", filepath.Join(cujDir, "out")) 142 143 // Each of these tests is run in sequence without resetting the output tree. The state of the output tree will 144 // affect each successive test. To maintain the validity of the benchmarks across changes, care must be taken 145 // to avoid changing the state of the tree when a test is run. This is most easily accomplished by adding tests 146 // at the end. 147 tests := []Test{ 148 { 149 // Reset the out directory to get reproducible results. 150 name: "clean", 151 args: []string{"clean"}, 152 }, 153 { 154 // Parse the build files. 155 name: "nothing", 156 args: []string{"nothing"}, 157 }, 158 { 159 // Parse the build files again to monitor issues like globs rerunning. 160 name: "nothing_rebuild", 161 args: []string{"nothing"}, 162 }, 163 { 164 // Parse the build files again, this should always be very short. 165 name: "nothing_rebuild_twice", 166 args: []string{"nothing"}, 167 }, 168 { 169 // Build the framework as a common developer task and one that keeps getting longer. 170 name: "framework", 171 args: []string{"framework"}, 172 }, 173 { 174 // Build the framework again to make sure it doesn't rebuild anything. 175 name: "framework_rebuild", 176 args: []string{"framework"}, 177 }, 178 { 179 // Build the framework again to make sure it doesn't rebuild anything even if it did the second time. 180 name: "framework_rebuild_twice", 181 args: []string{"framework"}, 182 }, 183 { 184 // Scenario major_inc_build (b/152046247): tracking build speed of major incremental build. 185 name: "major_inc_build_droid", 186 args: []string{"droid"}, 187 }, 188 { 189 name: "major_inc_build_framework_minus_apex_after_droid_build", 190 args: []string{"framework-minus-apex"}, 191 before: touchIntentFile, 192 }, 193 { 194 name: "major_inc_build_framework_after_droid_build", 195 args: []string{"framework"}, 196 before: touchIntentFile, 197 }, 198 { 199 name: "major_inc_build_sync_after_droid_build", 200 args: []string{"sync"}, 201 before: touchIntentFile, 202 }, 203 { 204 name: "major_inc_build_droid_rebuild", 205 args: []string{"droid"}, 206 before: touchIntentFile, 207 }, 208 { 209 name: "major_inc_build_update_api_after_droid_rebuild", 210 args: []string{"update-api"}, 211 before: touchIntentFile, 212 }, 213 } 214 215 cujMetrics := metrics.NewCriticalUserJourneysMetrics() 216 defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb")) 217 218 for i, t := range tests { 219 logsSubDir := fmt.Sprintf("%02d_%s", i, t.name) 220 logsDir := filepath.Join(cujDir, "logs", logsSubDir) 221 if t.before != nil { 222 if err := t.before(); err != nil { 223 fmt.Printf("error running before function on test %q: %v\n", t.name, err) 224 break 225 } 226 } 227 t.Run(logsDir) 228 if t.results.err != nil { 229 fmt.Printf("error running test %q: %s\n", t.name, t.results.err) 230 break 231 } 232 if t.results.metrics != nil { 233 cujMetrics.Add(t.name, t.results.metrics) 234 } 235 } 236 } 237