1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package blueprint 16 17import ( 18 "bytes" 19 "errors" 20 "fmt" 21 "io" 22 "io/ioutil" 23 "os" 24 "path/filepath" 25 "reflect" 26 "runtime" 27 "sort" 28 "strings" 29 "sync" 30 "sync/atomic" 31 "text/scanner" 32 "text/template" 33 34 "github.com/google/blueprint/parser" 35 "github.com/google/blueprint/pathtools" 36 "github.com/google/blueprint/proptools" 37) 38 39var ErrBuildActionsNotReady = errors.New("build actions are not ready") 40 41const maxErrors = 10 42const MockModuleListFile = "bplist" 43 44// A Context contains all the state needed to parse a set of Blueprints files 45// and generate a Ninja file. The process of generating a Ninja file proceeds 46// through a series of four phases. Each phase corresponds with a some methods 47// on the Context object 48// 49// Phase Methods 50// ------------ ------------------------------------------- 51// 1. Registration RegisterModuleType, RegisterSingletonType 52// 53// 2. Parse ParseBlueprintsFiles, Parse 54// 55// 3. Generate ResolveDependencies, PrepareBuildActions 56// 57// 4. Write WriteBuildFile 58// 59// The registration phase prepares the context to process Blueprints files 60// containing various types of modules. The parse phase reads in one or more 61// Blueprints files and validates their contents against the module types that 62// have been registered. The generate phase then analyzes the parsed Blueprints 63// contents to create an internal representation for the build actions that must 64// be performed. This phase also performs validation of the module dependencies 65// and property values defined in the parsed Blueprints files. Finally, the 66// write phase generates the Ninja manifest text based on the generated build 67// actions. 68type Context struct { 69 // set at instantiation 70 moduleFactories map[string]ModuleFactory 71 nameInterface NameInterface 72 moduleGroups []*moduleGroup 73 moduleInfo map[Module]*moduleInfo 74 modulesSorted []*moduleInfo 75 preSingletonInfo []*singletonInfo 76 singletonInfo []*singletonInfo 77 mutatorInfo []*mutatorInfo 78 earlyMutatorInfo []*mutatorInfo 79 variantMutatorNames []string 80 81 depsModified uint32 // positive if a mutator modified the dependencies 82 83 dependenciesReady bool // set to true on a successful ResolveDependencies 84 buildActionsReady bool // set to true on a successful PrepareBuildActions 85 86 // set by SetIgnoreUnknownModuleTypes 87 ignoreUnknownModuleTypes bool 88 89 // set by SetAllowMissingDependencies 90 allowMissingDependencies bool 91 92 // set during PrepareBuildActions 93 pkgNames map[*packageContext]string 94 liveGlobals *liveTracker 95 globalVariables map[Variable]*ninjaString 96 globalPools map[Pool]*poolDef 97 globalRules map[Rule]*ruleDef 98 99 // set during PrepareBuildActions 100 ninjaBuildDir *ninjaString // The builddir special Ninja variable 101 requiredNinjaMajor int // For the ninja_required_version variable 102 requiredNinjaMinor int // For the ninja_required_version variable 103 requiredNinjaMicro int // For the ninja_required_version variable 104 105 // set lazily by sortedModuleGroups 106 cachedSortedModuleGroups []*moduleGroup 107 108 globs map[string]GlobPath 109 globLock sync.Mutex 110 111 fs pathtools.FileSystem 112 moduleListFile string 113} 114 115// An Error describes a problem that was encountered that is related to a 116// particular location in a Blueprints file. 117type BlueprintError struct { 118 Err error // the error that occurred 119 Pos scanner.Position // the relevant Blueprints file location 120} 121 122// A ModuleError describes a problem that was encountered that is related to a 123// particular module in a Blueprints file 124type ModuleError struct { 125 BlueprintError 126 module *moduleInfo 127} 128 129// A PropertyError describes a problem that was encountered that is related to a 130// particular property in a Blueprints file 131type PropertyError struct { 132 ModuleError 133 property string 134} 135 136func (e *BlueprintError) Error() string { 137 return fmt.Sprintf("%s: %s", e.Pos, e.Err) 138} 139 140func (e *ModuleError) Error() string { 141 return fmt.Sprintf("%s: %s: %s", e.Pos, e.module, e.Err) 142} 143 144func (e *PropertyError) Error() string { 145 return fmt.Sprintf("%s: %s: %s: %s", e.Pos, e.module, e.property, e.Err) 146} 147 148type localBuildActions struct { 149 variables []*localVariable 150 rules []*localRule 151 buildDefs []*buildDef 152} 153 154type moduleGroup struct { 155 name string 156 ninjaName string 157 158 modules []*moduleInfo 159 160 namespace Namespace 161} 162 163type moduleInfo struct { 164 // set during Parse 165 typeName string 166 factory ModuleFactory 167 relBlueprintsFile string 168 pos scanner.Position 169 propertyPos map[string]scanner.Position 170 171 variantName string 172 variant variationMap 173 dependencyVariant variationMap 174 175 logicModule Module 176 group *moduleGroup 177 properties []interface{} 178 179 // set during ResolveDependencies 180 directDeps []depInfo 181 missingDeps []string 182 183 // set during updateDependencies 184 reverseDeps []*moduleInfo 185 forwardDeps []*moduleInfo 186 187 // used by parallelVisitAllBottomUp 188 waitingCount int 189 190 // set during each runMutator 191 splitModules []*moduleInfo 192 193 // set during PrepareBuildActions 194 actionDefs localBuildActions 195} 196 197type depInfo struct { 198 module *moduleInfo 199 tag DependencyTag 200} 201 202func (module *moduleInfo) Name() string { 203 return module.group.name 204} 205 206func (module *moduleInfo) String() string { 207 s := fmt.Sprintf("module %q", module.Name()) 208 if module.variantName != "" { 209 s += fmt.Sprintf(" variant %q", module.variantName) 210 } 211 return s 212} 213 214func (module *moduleInfo) namespace() Namespace { 215 return module.group.namespace 216} 217 218// A Variation is a way that a variant of a module differs from other variants of the same module. 219// For example, two variants of the same module might have Variation{"arch","arm"} and 220// Variation{"arch","arm64"} 221type Variation struct { 222 // Mutator is the axis on which this variation applies, i.e. "arch" or "link" 223 Mutator string 224 // Variation is the name of the variation on the axis, i.e. "arm" or "arm64" for arch, or 225 // "shared" or "static" for link. 226 Variation string 227} 228 229// A variationMap stores a map of Mutator to Variation to specify a variant of a module. 230type variationMap map[string]string 231 232func (vm variationMap) clone() variationMap { 233 newVm := make(variationMap) 234 for k, v := range vm { 235 newVm[k] = v 236 } 237 238 return newVm 239} 240 241// Compare this variationMap to another one. Returns true if the every entry in this map 242// is either the same in the other map or doesn't exist in the other map. 243func (vm variationMap) subset(other variationMap) bool { 244 for k, v1 := range vm { 245 if v2, ok := other[k]; ok && v1 != v2 { 246 return false 247 } 248 } 249 return true 250} 251 252func (vm variationMap) equal(other variationMap) bool { 253 return reflect.DeepEqual(vm, other) 254} 255 256type singletonInfo struct { 257 // set during RegisterSingletonType 258 factory SingletonFactory 259 singleton Singleton 260 name string 261 262 // set during PrepareBuildActions 263 actionDefs localBuildActions 264} 265 266type mutatorInfo struct { 267 // set during RegisterMutator 268 topDownMutator TopDownMutator 269 bottomUpMutator BottomUpMutator 270 name string 271 parallel bool 272} 273 274func newContext() *Context { 275 return &Context{ 276 moduleFactories: make(map[string]ModuleFactory), 277 nameInterface: NewSimpleNameInterface(), 278 moduleInfo: make(map[Module]*moduleInfo), 279 globs: make(map[string]GlobPath), 280 fs: pathtools.OsFs, 281 ninjaBuildDir: nil, 282 requiredNinjaMajor: 1, 283 requiredNinjaMinor: 7, 284 requiredNinjaMicro: 0, 285 } 286} 287 288// NewContext creates a new Context object. The created context initially has 289// no module or singleton factories registered, so the RegisterModuleFactory and 290// RegisterSingletonFactory methods must be called before it can do anything 291// useful. 292func NewContext() *Context { 293 ctx := newContext() 294 295 ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator) 296 297 return ctx 298} 299 300// A ModuleFactory function creates a new Module object. See the 301// Context.RegisterModuleType method for details about how a registered 302// ModuleFactory is used by a Context. 303type ModuleFactory func() (m Module, propertyStructs []interface{}) 304 305// RegisterModuleType associates a module type name (which can appear in a 306// Blueprints file) with a Module factory function. When the given module type 307// name is encountered in a Blueprints file during parsing, the Module factory 308// is invoked to instantiate a new Module object to handle the build action 309// generation for the module. If a Mutator splits a module into multiple variants, 310// the factory is invoked again to create a new Module for each variant. 311// 312// The module type names given here must be unique for the context. The factory 313// function should be a named function so that its package and name can be 314// included in the generated Ninja file for debugging purposes. 315// 316// The factory function returns two values. The first is the newly created 317// Module object. The second is a slice of pointers to that Module object's 318// properties structs. Each properties struct is examined when parsing a module 319// definition of this type in a Blueprints file. Exported fields of the 320// properties structs are automatically set to the property values specified in 321// the Blueprints file. The properties struct field names determine the name of 322// the Blueprints file properties that are used - the Blueprints property name 323// matches that of the properties struct field name with the first letter 324// converted to lower-case. 325// 326// The fields of the properties struct must be either []string, a string, or 327// bool. The Context will panic if a Module gets instantiated with a properties 328// struct containing a field that is not one these supported types. 329// 330// Any properties that appear in the Blueprints files that are not built-in 331// module properties (such as "name" and "deps") and do not have a corresponding 332// field in the returned module properties struct result in an error during the 333// Context's parse phase. 334// 335// As an example, the follow code: 336// 337// type myModule struct { 338// properties struct { 339// Foo string 340// Bar []string 341// } 342// } 343// 344// func NewMyModule() (blueprint.Module, []interface{}) { 345// module := new(myModule) 346// properties := &module.properties 347// return module, []interface{}{properties} 348// } 349// 350// func main() { 351// ctx := blueprint.NewContext() 352// ctx.RegisterModuleType("my_module", NewMyModule) 353// // ... 354// } 355// 356// would support parsing a module defined in a Blueprints file as follows: 357// 358// my_module { 359// name: "myName", 360// foo: "my foo string", 361// bar: ["my", "bar", "strings"], 362// } 363// 364// The factory function may be called from multiple goroutines. Any accesses 365// to global variables must be synchronized. 366func (c *Context) RegisterModuleType(name string, factory ModuleFactory) { 367 if _, present := c.moduleFactories[name]; present { 368 panic(errors.New("module type name is already registered")) 369 } 370 c.moduleFactories[name] = factory 371} 372 373// A SingletonFactory function creates a new Singleton object. See the 374// Context.RegisterSingletonType method for details about how a registered 375// SingletonFactory is used by a Context. 376type SingletonFactory func() Singleton 377 378// RegisterSingletonType registers a singleton type that will be invoked to 379// generate build actions. Each registered singleton type is instantiated and 380// and invoked exactly once as part of the generate phase. Each registered 381// singleton is invoked in registration order. 382// 383// The singleton type names given here must be unique for the context. The 384// factory function should be a named function so that its package and name can 385// be included in the generated Ninja file for debugging purposes. 386func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) { 387 for _, s := range c.singletonInfo { 388 if s.name == name { 389 panic(errors.New("singleton name is already registered")) 390 } 391 } 392 393 c.singletonInfo = append(c.singletonInfo, &singletonInfo{ 394 factory: factory, 395 singleton: factory(), 396 name: name, 397 }) 398} 399 400// RegisterPreSingletonType registers a presingleton type that will be invoked to 401// generate build actions before any Blueprint files have been read. Each registered 402// presingleton type is instantiated and invoked exactly once at the beginning of the 403// parse phase. Each registered presingleton is invoked in registration order. 404// 405// The presingleton type names given here must be unique for the context. The 406// factory function should be a named function so that its package and name can 407// be included in the generated Ninja file for debugging purposes. 408func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory) { 409 for _, s := range c.preSingletonInfo { 410 if s.name == name { 411 panic(errors.New("presingleton name is already registered")) 412 } 413 } 414 415 c.preSingletonInfo = append(c.preSingletonInfo, &singletonInfo{ 416 factory: factory, 417 singleton: factory(), 418 name: name, 419 }) 420} 421 422func (c *Context) SetNameInterface(i NameInterface) { 423 c.nameInterface = i 424} 425 426func singletonPkgPath(singleton Singleton) string { 427 typ := reflect.TypeOf(singleton) 428 for typ.Kind() == reflect.Ptr { 429 typ = typ.Elem() 430 } 431 return typ.PkgPath() 432} 433 434func singletonTypeName(singleton Singleton) string { 435 typ := reflect.TypeOf(singleton) 436 for typ.Kind() == reflect.Ptr { 437 typ = typ.Elem() 438 } 439 return typ.PkgPath() + "." + typ.Name() 440} 441 442// RegisterTopDownMutator registers a mutator that will be invoked to propagate dependency info 443// top-down between Modules. Each registered mutator is invoked in registration order (mixing 444// TopDownMutators and BottomUpMutators) once per Module, and the invocation on any module will 445// have returned before it is in invoked on any of its dependencies. 446// 447// The mutator type names given here must be unique to all top down mutators in 448// the Context. 449// 450// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 451// parallel while maintaining ordering. 452func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) MutatorHandle { 453 for _, m := range c.mutatorInfo { 454 if m.name == name && m.topDownMutator != nil { 455 panic(fmt.Errorf("mutator name %s is already registered", name)) 456 } 457 } 458 459 info := &mutatorInfo{ 460 topDownMutator: mutator, 461 name: name, 462 } 463 464 c.mutatorInfo = append(c.mutatorInfo, info) 465 466 return info 467} 468 469// RegisterBottomUpMutator registers a mutator that will be invoked to split Modules into variants. 470// Each registered mutator is invoked in registration order (mixing TopDownMutators and 471// BottomUpMutators) once per Module, will not be invoked on a module until the invocations on all 472// of the modules dependencies have returned. 473// 474// The mutator type names given here must be unique to all bottom up or early 475// mutators in the Context. 476// 477// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 478// parallel while maintaining ordering. 479func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) MutatorHandle { 480 for _, m := range c.variantMutatorNames { 481 if m == name { 482 panic(fmt.Errorf("mutator name %s is already registered", name)) 483 } 484 } 485 486 info := &mutatorInfo{ 487 bottomUpMutator: mutator, 488 name: name, 489 } 490 c.mutatorInfo = append(c.mutatorInfo, info) 491 492 c.variantMutatorNames = append(c.variantMutatorNames, name) 493 494 return info 495} 496 497type MutatorHandle interface { 498 // Set the mutator to visit modules in parallel while maintaining ordering. Calling any 499 // method on the mutator context is thread-safe, but the mutator must handle synchronization 500 // for any modifications to global state or any modules outside the one it was invoked on. 501 Parallel() MutatorHandle 502} 503 504func (mutator *mutatorInfo) Parallel() MutatorHandle { 505 mutator.parallel = true 506 return mutator 507} 508 509// RegisterEarlyMutator registers a mutator that will be invoked to split 510// Modules into multiple variant Modules before any dependencies have been 511// created. Each registered mutator is invoked in registration order once 512// per Module (including each variant from previous early mutators). Module 513// order is unpredictable. 514// 515// In order for dependencies to be satisifed in a later pass, all dependencies 516// of a module either must have an identical variant or must have no variations. 517// 518// The mutator type names given here must be unique to all bottom up or early 519// mutators in the Context. 520// 521// Deprecated, use a BottomUpMutator instead. The only difference between 522// EarlyMutator and BottomUpMutator is that EarlyMutator runs before the 523// deprecated DynamicDependencies. 524func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) { 525 for _, m := range c.variantMutatorNames { 526 if m == name { 527 panic(fmt.Errorf("mutator name %s is already registered", name)) 528 } 529 } 530 531 c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{ 532 bottomUpMutator: func(mctx BottomUpMutatorContext) { 533 mutator(mctx) 534 }, 535 name: name, 536 }) 537 538 c.variantMutatorNames = append(c.variantMutatorNames, name) 539} 540 541// SetIgnoreUnknownModuleTypes sets the behavior of the context in the case 542// where it encounters an unknown module type while parsing Blueprints files. By 543// default, the context will report unknown module types as an error. If this 544// method is called with ignoreUnknownModuleTypes set to true then the context 545// will silently ignore unknown module types. 546// 547// This method should generally not be used. It exists to facilitate the 548// bootstrapping process. 549func (c *Context) SetIgnoreUnknownModuleTypes(ignoreUnknownModuleTypes bool) { 550 c.ignoreUnknownModuleTypes = ignoreUnknownModuleTypes 551} 552 553// SetAllowMissingDependencies changes the behavior of Blueprint to ignore 554// unresolved dependencies. If the module's GenerateBuildActions calls 555// ModuleContext.GetMissingDependencies Blueprint will not emit any errors 556// for missing dependencies. 557func (c *Context) SetAllowMissingDependencies(allowMissingDependencies bool) { 558 c.allowMissingDependencies = allowMissingDependencies 559} 560 561func (c *Context) SetModuleListFile(listFile string) { 562 c.moduleListFile = listFile 563} 564 565func (c *Context) ListModulePaths(baseDir string) (paths []string, err error) { 566 reader, err := c.fs.Open(c.moduleListFile) 567 if err != nil { 568 return nil, err 569 } 570 bytes, err := ioutil.ReadAll(reader) 571 if err != nil { 572 return nil, err 573 } 574 text := string(bytes) 575 576 text = strings.Trim(text, "\n") 577 lines := strings.Split(text, "\n") 578 for i := range lines { 579 lines[i] = filepath.Join(baseDir, lines[i]) 580 } 581 582 return lines, nil 583} 584 585// a fileParseContext tells the status of parsing a particular file 586type fileParseContext struct { 587 // name of file 588 fileName string 589 590 // scope to use when resolving variables 591 Scope *parser.Scope 592 593 // pointer to the one in the parent directory 594 parent *fileParseContext 595 596 // is closed once FileHandler has completed for this file 597 doneVisiting chan struct{} 598} 599 600func (c *Context) ParseBlueprintsFiles(rootFile string) (deps []string, errs []error) { 601 baseDir := filepath.Dir(rootFile) 602 pathsToParse, err := c.ListModulePaths(baseDir) 603 if err != nil { 604 return nil, []error{err} 605 } 606 return c.ParseFileList(baseDir, pathsToParse) 607} 608 609// ParseBlueprintsFiles parses a set of Blueprints files starting with the file 610// at rootFile. When it encounters a Blueprints file with a set of subdirs 611// listed it recursively parses any Blueprints files found in those 612// subdirectories. 613// 614// If no errors are encountered while parsing the files, the list of paths on 615// which the future output will depend is returned. This list will include both 616// Blueprints file paths as well as directory paths for cases where wildcard 617// subdirs are found. 618func (c *Context) ParseFileList(rootDir string, filePaths []string) (deps []string, 619 errs []error) { 620 621 if len(filePaths) < 1 { 622 return nil, []error{fmt.Errorf("no paths provided to parse")} 623 } 624 625 c.dependenciesReady = false 626 627 moduleCh := make(chan *moduleInfo) 628 errsCh := make(chan []error) 629 doneCh := make(chan struct{}) 630 var numErrs uint32 631 var numGoroutines int32 632 633 // handler must be reentrant 634 handleOneFile := func(file *parser.File) { 635 if atomic.LoadUint32(&numErrs) > maxErrors { 636 return 637 } 638 639 for _, def := range file.Defs { 640 var module *moduleInfo 641 var errs []error 642 switch def := def.(type) { 643 case *parser.Module: 644 module, errs = c.processModuleDef(def, file.Name) 645 case *parser.Assignment: 646 // Already handled via Scope object 647 default: 648 panic("unknown definition type") 649 } 650 651 if len(errs) > 0 { 652 atomic.AddUint32(&numErrs, uint32(len(errs))) 653 errsCh <- errs 654 } else if module != nil { 655 moduleCh <- module 656 } 657 } 658 } 659 660 atomic.AddInt32(&numGoroutines, 1) 661 go func() { 662 var errs []error 663 deps, errs = c.WalkBlueprintsFiles(rootDir, filePaths, handleOneFile) 664 if len(errs) > 0 { 665 errsCh <- errs 666 } 667 doneCh <- struct{}{} 668 }() 669 670loop: 671 for { 672 select { 673 case newErrs := <-errsCh: 674 errs = append(errs, newErrs...) 675 case module := <-moduleCh: 676 newErrs := c.addModule(module) 677 if len(newErrs) > 0 { 678 errs = append(errs, newErrs...) 679 } 680 case <-doneCh: 681 n := atomic.AddInt32(&numGoroutines, -1) 682 if n == 0 { 683 break loop 684 } 685 } 686 } 687 688 return deps, errs 689} 690 691type FileHandler func(*parser.File) 692 693// WalkBlueprintsFiles walks a set of Blueprints files starting with the given filepaths, 694// calling the given file handler on each 695// 696// When WalkBlueprintsFiles encounters a Blueprints file with a set of subdirs listed, 697// it recursively parses any Blueprints files found in those subdirectories. 698// 699// If any of the file paths is an ancestor directory of any other of file path, the ancestor 700// will be parsed and visited first. 701// 702// the file handler will be called from a goroutine, so it must be reentrant. 703// 704// If no errors are encountered while parsing the files, the list of paths on 705// which the future output will depend is returned. This list will include both 706// Blueprints file paths as well as directory paths for cases where wildcard 707// subdirs are found. 708// 709// visitor will be called asynchronously, and will only be called once visitor for each 710// ancestor directory has completed. 711// 712// WalkBlueprintsFiles will not return until all calls to visitor have returned. 713func (c *Context) WalkBlueprintsFiles(rootDir string, filePaths []string, 714 visitor FileHandler) (deps []string, errs []error) { 715 716 // make a mapping from ancestors to their descendants to facilitate parsing ancestors first 717 descendantsMap, err := findBlueprintDescendants(filePaths) 718 if err != nil { 719 panic(err.Error()) 720 return nil, []error{err} 721 } 722 blueprintsSet := make(map[string]bool) 723 724 // Channels to receive data back from openAndParse goroutines 725 blueprintsCh := make(chan fileParseContext) 726 errsCh := make(chan []error) 727 depsCh := make(chan string) 728 729 // Channel to notify main loop that a openAndParse goroutine has finished 730 doneParsingCh := make(chan fileParseContext) 731 732 // Number of outstanding goroutines to wait for 733 activeCount := 0 734 var pending []fileParseContext 735 tooManyErrors := false 736 737 // Limit concurrent calls to parseBlueprintFiles to 200 738 // Darwin has a default limit of 256 open files 739 maxActiveCount := 200 740 741 // count the number of pending calls to visitor() 742 visitorWaitGroup := sync.WaitGroup{} 743 744 startParseBlueprintsFile := func(blueprint fileParseContext) { 745 if blueprintsSet[blueprint.fileName] { 746 return 747 } 748 blueprintsSet[blueprint.fileName] = true 749 activeCount++ 750 deps = append(deps, blueprint.fileName) 751 visitorWaitGroup.Add(1) 752 go func() { 753 file, blueprints, deps, errs := c.openAndParse(blueprint.fileName, blueprint.Scope, rootDir, 754 &blueprint) 755 if len(errs) > 0 { 756 errsCh <- errs 757 } 758 for _, blueprint := range blueprints { 759 blueprintsCh <- blueprint 760 } 761 for _, dep := range deps { 762 depsCh <- dep 763 } 764 doneParsingCh <- blueprint 765 766 if blueprint.parent != nil && blueprint.parent.doneVisiting != nil { 767 // wait for visitor() of parent to complete 768 <-blueprint.parent.doneVisiting 769 } 770 771 if len(errs) == 0 { 772 // process this file 773 visitor(file) 774 } 775 if blueprint.doneVisiting != nil { 776 close(blueprint.doneVisiting) 777 } 778 visitorWaitGroup.Done() 779 }() 780 } 781 782 foundParseableBlueprint := func(blueprint fileParseContext) { 783 if activeCount >= maxActiveCount { 784 pending = append(pending, blueprint) 785 } else { 786 startParseBlueprintsFile(blueprint) 787 } 788 } 789 790 startParseDescendants := func(blueprint fileParseContext) { 791 descendants, hasDescendants := descendantsMap[blueprint.fileName] 792 if hasDescendants { 793 for _, descendant := range descendants { 794 foundParseableBlueprint(fileParseContext{descendant, parser.NewScope(blueprint.Scope), &blueprint, make(chan struct{})}) 795 } 796 } 797 } 798 799 // begin parsing any files that have no ancestors 800 startParseDescendants(fileParseContext{"", parser.NewScope(nil), nil, nil}) 801 802loop: 803 for { 804 if len(errs) > maxErrors { 805 tooManyErrors = true 806 } 807 808 select { 809 case newErrs := <-errsCh: 810 errs = append(errs, newErrs...) 811 case dep := <-depsCh: 812 deps = append(deps, dep) 813 case blueprint := <-blueprintsCh: 814 if tooManyErrors { 815 continue 816 } 817 foundParseableBlueprint(blueprint) 818 case blueprint := <-doneParsingCh: 819 activeCount-- 820 if !tooManyErrors { 821 startParseDescendants(blueprint) 822 } 823 if activeCount < maxActiveCount && len(pending) > 0 { 824 // start to process the next one from the queue 825 next := pending[len(pending)-1] 826 pending = pending[:len(pending)-1] 827 startParseBlueprintsFile(next) 828 } 829 if activeCount == 0 { 830 break loop 831 } 832 } 833 } 834 835 sort.Strings(deps) 836 837 // wait for every visitor() to complete 838 visitorWaitGroup.Wait() 839 840 return 841} 842 843// MockFileSystem causes the Context to replace all reads with accesses to the provided map of 844// filenames to contents stored as a byte slice. 845func (c *Context) MockFileSystem(files map[string][]byte) { 846 // look for a module list file 847 _, ok := files[MockModuleListFile] 848 if !ok { 849 // no module list file specified; find every file named Blueprints 850 pathsToParse := []string{} 851 for candidate := range files { 852 if filepath.Base(candidate) == "Blueprints" { 853 pathsToParse = append(pathsToParse, candidate) 854 } 855 } 856 if len(pathsToParse) < 1 { 857 panic(fmt.Sprintf("No Blueprints files found in mock filesystem: %v\n", files)) 858 } 859 // put the list of Blueprints files into a list file 860 files[MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n")) 861 } 862 c.SetModuleListFile(MockModuleListFile) 863 864 // mock the filesystem 865 c.fs = pathtools.MockFs(files) 866} 867 868// openAndParse opens and parses a single Blueprints file, and returns the results 869func (c *Context) openAndParse(filename string, scope *parser.Scope, rootDir string, 870 parent *fileParseContext) (file *parser.File, 871 subBlueprints []fileParseContext, deps []string, errs []error) { 872 873 f, err := c.fs.Open(filename) 874 if err != nil { 875 // couldn't open the file; see if we can provide a clearer error than "could not open file" 876 stats, statErr := c.fs.Lstat(filename) 877 if statErr == nil { 878 isSymlink := stats.Mode()&os.ModeSymlink != 0 879 if isSymlink { 880 err = fmt.Errorf("could not open symlink %v : %v", filename, err) 881 target, readlinkErr := os.Readlink(filename) 882 if readlinkErr == nil { 883 _, targetStatsErr := c.fs.Lstat(target) 884 if targetStatsErr != nil { 885 err = fmt.Errorf("could not open symlink %v; its target (%v) cannot be opened", filename, target) 886 } 887 } 888 } else { 889 err = fmt.Errorf("%v exists but could not be opened: %v", filename, err) 890 } 891 } 892 return nil, nil, nil, []error{err} 893 } 894 895 func() { 896 defer func() { 897 err = f.Close() 898 if err != nil { 899 errs = append(errs, err) 900 } 901 }() 902 file, subBlueprints, errs = c.parseOne(rootDir, filename, f, scope, parent) 903 }() 904 905 if len(errs) > 0 { 906 return nil, nil, nil, errs 907 } 908 909 for _, b := range subBlueprints { 910 deps = append(deps, b.fileName) 911 } 912 913 return file, subBlueprints, deps, nil 914} 915 916// parseOne parses a single Blueprints file from the given reader, creating Module 917// objects for each of the module definitions encountered. If the Blueprints 918// file contains an assignment to the "subdirs" variable, then the 919// subdirectories listed are searched for Blueprints files returned in the 920// subBlueprints return value. If the Blueprints file contains an assignment 921// to the "build" variable, then the file listed are returned in the 922// subBlueprints return value. 923// 924// rootDir specifies the path to the root directory of the source tree, while 925// filename specifies the path to the Blueprints file. These paths are used for 926// error reporting and for determining the module's directory. 927func (c *Context) parseOne(rootDir, filename string, reader io.Reader, 928 scope *parser.Scope, parent *fileParseContext) (file *parser.File, subBlueprints []fileParseContext, errs []error) { 929 930 relBlueprintsFile, err := filepath.Rel(rootDir, filename) 931 if err != nil { 932 return nil, nil, []error{err} 933 } 934 935 scope.Remove("subdirs") 936 scope.Remove("optional_subdirs") 937 scope.Remove("build") 938 file, errs = parser.ParseAndEval(filename, reader, scope) 939 if len(errs) > 0 { 940 for i, err := range errs { 941 if parseErr, ok := err.(*parser.ParseError); ok { 942 err = &BlueprintError{ 943 Err: parseErr.Err, 944 Pos: parseErr.Pos, 945 } 946 errs[i] = err 947 } 948 } 949 950 // If there were any parse errors don't bother trying to interpret the 951 // result. 952 return nil, nil, errs 953 } 954 file.Name = relBlueprintsFile 955 956 build, buildPos, err := getLocalStringListFromScope(scope, "build") 957 if err != nil { 958 errs = append(errs, err) 959 } 960 for _, buildEntry := range build { 961 if strings.Contains(buildEntry, "/") { 962 errs = append(errs, &BlueprintError{ 963 Err: fmt.Errorf("illegal value %v. The '/' character is not permitted", buildEntry), 964 Pos: buildPos, 965 }) 966 } 967 } 968 969 subBlueprintsName, _, err := getStringFromScope(scope, "subname") 970 if err != nil { 971 errs = append(errs, err) 972 } 973 974 if subBlueprintsName == "" { 975 subBlueprintsName = "Blueprints" 976 } 977 978 var blueprints []string 979 980 newBlueprints, newErrs := c.findBuildBlueprints(filepath.Dir(filename), build, buildPos) 981 blueprints = append(blueprints, newBlueprints...) 982 errs = append(errs, newErrs...) 983 984 subBlueprintsAndScope := make([]fileParseContext, len(blueprints)) 985 for i, b := range blueprints { 986 subBlueprintsAndScope[i] = fileParseContext{b, parser.NewScope(scope), parent, make(chan struct{})} 987 } 988 return file, subBlueprintsAndScope, errs 989} 990 991func (c *Context) findBuildBlueprints(dir string, build []string, 992 buildPos scanner.Position) ([]string, []error) { 993 994 var blueprints []string 995 var errs []error 996 997 for _, file := range build { 998 pattern := filepath.Join(dir, file) 999 var matches []string 1000 var err error 1001 1002 matches, err = c.glob(pattern, nil) 1003 1004 if err != nil { 1005 errs = append(errs, &BlueprintError{ 1006 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1007 Pos: buildPos, 1008 }) 1009 continue 1010 } 1011 1012 if len(matches) == 0 { 1013 errs = append(errs, &BlueprintError{ 1014 Err: fmt.Errorf("%q: not found", pattern), 1015 Pos: buildPos, 1016 }) 1017 } 1018 1019 for _, foundBlueprints := range matches { 1020 if strings.HasSuffix(foundBlueprints, "/") { 1021 errs = append(errs, &BlueprintError{ 1022 Err: fmt.Errorf("%q: is a directory", foundBlueprints), 1023 Pos: buildPos, 1024 }) 1025 } 1026 blueprints = append(blueprints, foundBlueprints) 1027 } 1028 } 1029 1030 return blueprints, errs 1031} 1032 1033func (c *Context) findSubdirBlueprints(dir string, subdirs []string, subdirsPos scanner.Position, 1034 subBlueprintsName string, optional bool) ([]string, []error) { 1035 1036 var blueprints []string 1037 var errs []error 1038 1039 for _, subdir := range subdirs { 1040 pattern := filepath.Join(dir, subdir, subBlueprintsName) 1041 var matches []string 1042 var err error 1043 1044 matches, err = c.glob(pattern, nil) 1045 1046 if err != nil { 1047 errs = append(errs, &BlueprintError{ 1048 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1049 Pos: subdirsPos, 1050 }) 1051 continue 1052 } 1053 1054 if len(matches) == 0 && !optional { 1055 errs = append(errs, &BlueprintError{ 1056 Err: fmt.Errorf("%q: not found", pattern), 1057 Pos: subdirsPos, 1058 }) 1059 } 1060 1061 for _, subBlueprints := range matches { 1062 if strings.HasSuffix(subBlueprints, "/") { 1063 errs = append(errs, &BlueprintError{ 1064 Err: fmt.Errorf("%q: is a directory", subBlueprints), 1065 Pos: subdirsPos, 1066 }) 1067 } 1068 blueprints = append(blueprints, subBlueprints) 1069 } 1070 } 1071 1072 return blueprints, errs 1073} 1074 1075func getLocalStringListFromScope(scope *parser.Scope, v string) ([]string, scanner.Position, error) { 1076 if assignment, local := scope.Get(v); assignment == nil || !local { 1077 return nil, scanner.Position{}, nil 1078 } else { 1079 switch value := assignment.Value.Eval().(type) { 1080 case *parser.List: 1081 ret := make([]string, 0, len(value.Values)) 1082 1083 for _, listValue := range value.Values { 1084 s, ok := listValue.(*parser.String) 1085 if !ok { 1086 // The parser should not produce this. 1087 panic("non-string value found in list") 1088 } 1089 1090 ret = append(ret, s.Value) 1091 } 1092 1093 return ret, assignment.EqualsPos, nil 1094 case *parser.Bool, *parser.String: 1095 return nil, scanner.Position{}, &BlueprintError{ 1096 Err: fmt.Errorf("%q must be a list of strings", v), 1097 Pos: assignment.EqualsPos, 1098 } 1099 default: 1100 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1101 } 1102 } 1103} 1104 1105func getStringFromScope(scope *parser.Scope, v string) (string, scanner.Position, error) { 1106 if assignment, _ := scope.Get(v); assignment == nil { 1107 return "", scanner.Position{}, nil 1108 } else { 1109 switch value := assignment.Value.Eval().(type) { 1110 case *parser.String: 1111 return value.Value, assignment.EqualsPos, nil 1112 case *parser.Bool, *parser.List: 1113 return "", scanner.Position{}, &BlueprintError{ 1114 Err: fmt.Errorf("%q must be a string", v), 1115 Pos: assignment.EqualsPos, 1116 } 1117 default: 1118 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1119 } 1120 } 1121} 1122 1123// Clones a build logic module by calling the factory method for its module type, and then cloning 1124// property values. Any values stored in the module object that are not stored in properties 1125// structs will be lost. 1126func (c *Context) cloneLogicModule(origModule *moduleInfo) (Module, []interface{}) { 1127 newLogicModule, newProperties := origModule.factory() 1128 1129 if len(newProperties) != len(origModule.properties) { 1130 panic("mismatched properties array length in " + origModule.Name()) 1131 } 1132 1133 for i := range newProperties { 1134 dst := reflect.ValueOf(newProperties[i]).Elem() 1135 src := reflect.ValueOf(origModule.properties[i]).Elem() 1136 1137 proptools.CopyProperties(dst, src) 1138 } 1139 1140 return newLogicModule, newProperties 1141} 1142 1143func (c *Context) createVariations(origModule *moduleInfo, mutatorName string, 1144 variationNames []string) ([]*moduleInfo, []error) { 1145 1146 if len(variationNames) == 0 { 1147 panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q", 1148 mutatorName, origModule.Name())) 1149 } 1150 1151 newModules := []*moduleInfo{} 1152 1153 var errs []error 1154 1155 for i, variationName := range variationNames { 1156 var newLogicModule Module 1157 var newProperties []interface{} 1158 1159 if i == 0 { 1160 // Reuse the existing module for the first new variant 1161 // This both saves creating a new module, and causes the insertion in c.moduleInfo below 1162 // with logicModule as the key to replace the original entry in c.moduleInfo 1163 newLogicModule, newProperties = origModule.logicModule, origModule.properties 1164 } else { 1165 newLogicModule, newProperties = c.cloneLogicModule(origModule) 1166 } 1167 1168 newVariant := origModule.variant.clone() 1169 newVariant[mutatorName] = variationName 1170 1171 m := *origModule 1172 newModule := &m 1173 newModule.directDeps = append([]depInfo{}, origModule.directDeps...) 1174 newModule.logicModule = newLogicModule 1175 newModule.variant = newVariant 1176 newModule.dependencyVariant = origModule.dependencyVariant.clone() 1177 newModule.properties = newProperties 1178 1179 if variationName != "" { 1180 if newModule.variantName == "" { 1181 newModule.variantName = variationName 1182 } else { 1183 newModule.variantName += "_" + variationName 1184 } 1185 } 1186 1187 newModules = append(newModules, newModule) 1188 1189 newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName) 1190 if len(newErrs) > 0 { 1191 errs = append(errs, newErrs...) 1192 } 1193 } 1194 1195 // Mark original variant as invalid. Modules that depend on this module will still 1196 // depend on origModule, but we'll fix it when the mutator is called on them. 1197 origModule.logicModule = nil 1198 origModule.splitModules = newModules 1199 1200 atomic.AddUint32(&c.depsModified, 1) 1201 1202 return newModules, errs 1203} 1204 1205func (c *Context) convertDepsToVariation(module *moduleInfo, 1206 mutatorName, variationName string) (errs []error) { 1207 1208 for i, dep := range module.directDeps { 1209 if dep.module.logicModule == nil { 1210 var newDep *moduleInfo 1211 for _, m := range dep.module.splitModules { 1212 if m.variant[mutatorName] == variationName { 1213 newDep = m 1214 break 1215 } 1216 } 1217 if newDep == nil { 1218 errs = append(errs, &BlueprintError{ 1219 Err: fmt.Errorf("failed to find variation %q for module %q needed by %q", 1220 variationName, dep.module.Name(), module.Name()), 1221 Pos: module.pos, 1222 }) 1223 continue 1224 } 1225 module.directDeps[i].module = newDep 1226 } 1227 } 1228 1229 return errs 1230} 1231 1232func (c *Context) prettyPrintVariant(variant variationMap) string { 1233 names := make([]string, 0, len(variant)) 1234 for _, m := range c.variantMutatorNames { 1235 if v, ok := variant[m]; ok { 1236 names = append(names, m+":"+v) 1237 } 1238 } 1239 1240 return strings.Join(names, ", ") 1241} 1242 1243func (c *Context) newModule(factory ModuleFactory) *moduleInfo { 1244 logicModule, properties := factory() 1245 1246 module := &moduleInfo{ 1247 logicModule: logicModule, 1248 factory: factory, 1249 } 1250 1251 module.properties = properties 1252 1253 return module 1254} 1255 1256func (c *Context) processModuleDef(moduleDef *parser.Module, 1257 relBlueprintsFile string) (*moduleInfo, []error) { 1258 1259 factory, ok := c.moduleFactories[moduleDef.Type] 1260 if !ok { 1261 if c.ignoreUnknownModuleTypes { 1262 return nil, nil 1263 } 1264 1265 return nil, []error{ 1266 &BlueprintError{ 1267 Err: fmt.Errorf("unrecognized module type %q", moduleDef.Type), 1268 Pos: moduleDef.TypePos, 1269 }, 1270 } 1271 } 1272 1273 module := c.newModule(factory) 1274 module.typeName = moduleDef.Type 1275 1276 module.relBlueprintsFile = relBlueprintsFile 1277 1278 propertyMap, errs := unpackProperties(moduleDef.Properties, module.properties...) 1279 if len(errs) > 0 { 1280 return nil, errs 1281 } 1282 1283 module.pos = moduleDef.TypePos 1284 module.propertyPos = make(map[string]scanner.Position) 1285 for name, propertyDef := range propertyMap { 1286 module.propertyPos[name] = propertyDef.ColonPos 1287 } 1288 1289 return module, nil 1290} 1291 1292func (c *Context) addModule(module *moduleInfo) []error { 1293 name := module.logicModule.Name() 1294 c.moduleInfo[module.logicModule] = module 1295 1296 group := &moduleGroup{ 1297 name: name, 1298 modules: []*moduleInfo{module}, 1299 } 1300 module.group = group 1301 namespace, errs := c.nameInterface.NewModule( 1302 newNamespaceContext(module), 1303 ModuleGroup{moduleGroup: group}, 1304 module.logicModule) 1305 if len(errs) > 0 { 1306 for i := range errs { 1307 errs[i] = &BlueprintError{Err: errs[i], Pos: module.pos} 1308 } 1309 return errs 1310 } 1311 group.namespace = namespace 1312 1313 c.moduleGroups = append(c.moduleGroups, group) 1314 1315 return nil 1316} 1317 1318// ResolveDependencies checks that the dependencies specified by all of the 1319// modules defined in the parsed Blueprints files are valid. This means that 1320// the modules depended upon are defined and that no circular dependencies 1321// exist. 1322func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) { 1323 c.liveGlobals = newLiveTracker(config) 1324 1325 deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals) 1326 if len(errs) > 0 { 1327 return nil, errs 1328 } 1329 1330 errs = c.updateDependencies() 1331 if len(errs) > 0 { 1332 return nil, errs 1333 } 1334 1335 mutatorDeps, errs := c.runMutators(config) 1336 if len(errs) > 0 { 1337 return nil, errs 1338 } 1339 deps = append(deps, mutatorDeps...) 1340 1341 c.cloneModules() 1342 1343 c.dependenciesReady = true 1344 return deps, nil 1345} 1346 1347// Default dependencies handling. If the module implements the (deprecated) 1348// DynamicDependerModule interface then this set consists of the union of those 1349// module names returned by its DynamicDependencies method and those added by calling 1350// AddDependencies or AddVariationDependencies on DynamicDependencyModuleContext. 1351func blueprintDepsMutator(ctx BottomUpMutatorContext) { 1352 if dynamicDepender, ok := ctx.Module().(DynamicDependerModule); ok { 1353 func() { 1354 defer func() { 1355 if r := recover(); r != nil { 1356 ctx.error(newPanicErrorf(r, "DynamicDependencies for %s", ctx.moduleInfo())) 1357 } 1358 }() 1359 dynamicDeps := dynamicDepender.DynamicDependencies(ctx) 1360 1361 if ctx.Failed() { 1362 return 1363 } 1364 1365 ctx.AddDependency(ctx.Module(), nil, dynamicDeps...) 1366 }() 1367 } 1368} 1369 1370// findMatchingVariant searches the moduleGroup for a module with the same variant as module, 1371// and returns the matching module, or nil if one is not found. 1372func (c *Context) findMatchingVariant(module *moduleInfo, possible []*moduleInfo) *moduleInfo { 1373 if len(possible) == 1 { 1374 return possible[0] 1375 } else { 1376 for _, m := range possible { 1377 if m.variant.equal(module.dependencyVariant) { 1378 return m 1379 } 1380 } 1381 } 1382 1383 return nil 1384} 1385 1386func (c *Context) addDependency(module *moduleInfo, tag DependencyTag, depName string) []error { 1387 if _, ok := tag.(BaseDependencyTag); ok { 1388 panic("BaseDependencyTag is not allowed to be used directly!") 1389 } 1390 1391 if depName == module.Name() { 1392 return []error{&BlueprintError{ 1393 Err: fmt.Errorf("%q depends on itself", depName), 1394 Pos: module.pos, 1395 }} 1396 } 1397 1398 possibleDeps := c.modulesFromName(depName, module.namespace()) 1399 if possibleDeps == nil { 1400 return c.discoveredMissingDependencies(module, depName) 1401 } 1402 1403 if m := c.findMatchingVariant(module, possibleDeps); m != nil { 1404 for _, dep := range module.directDeps { 1405 if m == dep.module { 1406 // TODO(ccross): what if adding a dependency with a different tag? 1407 return nil 1408 } 1409 } 1410 module.directDeps = append(module.directDeps, depInfo{m, tag}) 1411 atomic.AddUint32(&c.depsModified, 1) 1412 return nil 1413 } 1414 1415 variants := make([]string, len(possibleDeps)) 1416 for i, mod := range possibleDeps { 1417 variants[i] = c.prettyPrintVariant(mod.variant) 1418 } 1419 sort.Strings(variants) 1420 1421 return []error{&BlueprintError{ 1422 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1423 depName, module.Name(), 1424 c.prettyPrintVariant(module.dependencyVariant), 1425 strings.Join(variants, "\n ")), 1426 Pos: module.pos, 1427 }} 1428} 1429 1430func (c *Context) findReverseDependency(module *moduleInfo, destName string) (*moduleInfo, []error) { 1431 if destName == module.Name() { 1432 return nil, []error{&BlueprintError{ 1433 Err: fmt.Errorf("%q depends on itself", destName), 1434 Pos: module.pos, 1435 }} 1436 } 1437 1438 possibleDeps := c.modulesFromName(destName, module.namespace()) 1439 if possibleDeps == nil { 1440 return nil, []error{&BlueprintError{ 1441 Err: fmt.Errorf("%q has a reverse dependency on undefined module %q", 1442 module.Name(), destName), 1443 Pos: module.pos, 1444 }} 1445 } 1446 1447 if m := c.findMatchingVariant(module, possibleDeps); m != nil { 1448 return m, nil 1449 } 1450 1451 variants := make([]string, len(possibleDeps)) 1452 for i, mod := range possibleDeps { 1453 variants[i] = c.prettyPrintVariant(mod.variant) 1454 } 1455 sort.Strings(variants) 1456 1457 return nil, []error{&BlueprintError{ 1458 Err: fmt.Errorf("reverse dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1459 destName, module.Name(), 1460 c.prettyPrintVariant(module.dependencyVariant), 1461 strings.Join(variants, "\n ")), 1462 Pos: module.pos, 1463 }} 1464} 1465 1466func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation, 1467 tag DependencyTag, depName string, far bool) []error { 1468 if _, ok := tag.(BaseDependencyTag); ok { 1469 panic("BaseDependencyTag is not allowed to be used directly!") 1470 } 1471 1472 possibleDeps := c.modulesFromName(depName, module.namespace()) 1473 if possibleDeps == nil { 1474 return c.discoveredMissingDependencies(module, depName) 1475 } 1476 1477 // We can't just append variant.Variant to module.dependencyVariants.variantName and 1478 // compare the strings because the result won't be in mutator registration order. 1479 // Create a new map instead, and then deep compare the maps. 1480 var newVariant variationMap 1481 if !far { 1482 newVariant = module.dependencyVariant.clone() 1483 } else { 1484 newVariant = make(variationMap) 1485 } 1486 for _, v := range variations { 1487 newVariant[v.Mutator] = v.Variation 1488 } 1489 1490 for _, m := range possibleDeps { 1491 var found bool 1492 if far { 1493 found = m.variant.subset(newVariant) 1494 } else { 1495 found = m.variant.equal(newVariant) 1496 } 1497 if found { 1498 if module == m { 1499 return []error{&BlueprintError{ 1500 Err: fmt.Errorf("%q depends on itself", depName), 1501 Pos: module.pos, 1502 }} 1503 } 1504 // AddVariationDependency allows adding a dependency on itself, but only if 1505 // that module is earlier in the module list than this one, since we always 1506 // run GenerateBuildActions in order for the variants of a module 1507 if m.group == module.group && beforeInModuleList(module, m, module.group.modules) { 1508 return []error{&BlueprintError{ 1509 Err: fmt.Errorf("%q depends on later version of itself", depName), 1510 Pos: module.pos, 1511 }} 1512 } 1513 module.directDeps = append(module.directDeps, depInfo{m, tag}) 1514 atomic.AddUint32(&c.depsModified, 1) 1515 return nil 1516 } 1517 } 1518 1519 variants := make([]string, len(possibleDeps)) 1520 for i, mod := range possibleDeps { 1521 variants[i] = c.prettyPrintVariant(mod.variant) 1522 } 1523 sort.Strings(variants) 1524 1525 return []error{&BlueprintError{ 1526 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1527 depName, module.Name(), 1528 c.prettyPrintVariant(newVariant), 1529 strings.Join(variants, "\n ")), 1530 Pos: module.pos, 1531 }} 1532} 1533 1534func (c *Context) addInterVariantDependency(origModule *moduleInfo, tag DependencyTag, 1535 from, to Module) { 1536 if _, ok := tag.(BaseDependencyTag); ok { 1537 panic("BaseDependencyTag is not allowed to be used directly!") 1538 } 1539 1540 var fromInfo, toInfo *moduleInfo 1541 for _, m := range origModule.splitModules { 1542 if m.logicModule == from { 1543 fromInfo = m 1544 } 1545 if m.logicModule == to { 1546 toInfo = m 1547 if fromInfo != nil { 1548 panic(fmt.Errorf("%q depends on later version of itself", origModule.Name())) 1549 } 1550 } 1551 } 1552 1553 if fromInfo == nil || toInfo == nil { 1554 panic(fmt.Errorf("AddInterVariantDependency called for module %q on invalid variant", 1555 origModule.Name())) 1556 } 1557 1558 fromInfo.directDeps = append(fromInfo.directDeps, depInfo{toInfo, tag}) 1559 atomic.AddUint32(&c.depsModified, 1) 1560} 1561 1562// findBlueprintDescendants returns a map linking parent Blueprints files to child Blueprints files 1563// For example, if paths = []string{"a/b/c/Android.bp", "a/Blueprints"}, 1564// then descendants = {"":[]string{"a/Blueprints"}, "a/Blueprints":[]string{"a/b/c/Android.bp"}} 1565func findBlueprintDescendants(paths []string) (descendants map[string][]string, err error) { 1566 // make mapping from dir path to file path 1567 filesByDir := make(map[string]string, len(paths)) 1568 for _, path := range paths { 1569 dir := filepath.Dir(path) 1570 _, alreadyFound := filesByDir[dir] 1571 if alreadyFound { 1572 return nil, fmt.Errorf("Found two Blueprint files in directory %v : %v and %v", dir, filesByDir[dir], path) 1573 } 1574 filesByDir[dir] = path 1575 } 1576 1577 findAncestor := func(childFile string) (ancestor string) { 1578 prevAncestorDir := filepath.Dir(childFile) 1579 for { 1580 ancestorDir := filepath.Dir(prevAncestorDir) 1581 if ancestorDir == prevAncestorDir { 1582 // reached the root dir without any matches; assign this as a descendant of "" 1583 return "" 1584 } 1585 1586 ancestorFile, ancestorExists := filesByDir[ancestorDir] 1587 if ancestorExists { 1588 return ancestorFile 1589 } 1590 prevAncestorDir = ancestorDir 1591 } 1592 } 1593 // generate the descendants map 1594 descendants = make(map[string][]string, len(filesByDir)) 1595 for _, childFile := range filesByDir { 1596 ancestorFile := findAncestor(childFile) 1597 descendants[ancestorFile] = append(descendants[ancestorFile], childFile) 1598 } 1599 return descendants, nil 1600} 1601 1602type visitOrderer interface { 1603 // returns the number of modules that this module needs to wait for 1604 waitCount(module *moduleInfo) int 1605 // returns the list of modules that are waiting for this module 1606 propagate(module *moduleInfo) []*moduleInfo 1607 // visit modules in order 1608 visit(modules []*moduleInfo, visit func(*moduleInfo) bool) 1609} 1610 1611type unorderedVisitorImpl struct{} 1612 1613func (unorderedVisitorImpl) waitCount(module *moduleInfo) int { 1614 return 0 1615} 1616 1617func (unorderedVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1618 return nil 1619} 1620 1621func (unorderedVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1622 for _, module := range modules { 1623 if visit(module) { 1624 return 1625 } 1626 } 1627} 1628 1629type bottomUpVisitorImpl struct{} 1630 1631func (bottomUpVisitorImpl) waitCount(module *moduleInfo) int { 1632 return len(module.forwardDeps) 1633} 1634 1635func (bottomUpVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1636 return module.reverseDeps 1637} 1638 1639func (bottomUpVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1640 for _, module := range modules { 1641 if visit(module) { 1642 return 1643 } 1644 } 1645} 1646 1647type topDownVisitorImpl struct{} 1648 1649func (topDownVisitorImpl) waitCount(module *moduleInfo) int { 1650 return len(module.reverseDeps) 1651} 1652 1653func (topDownVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1654 return module.forwardDeps 1655} 1656 1657func (topDownVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1658 for i := 0; i < len(modules); i++ { 1659 module := modules[len(modules)-1-i] 1660 if visit(module) { 1661 return 1662 } 1663 } 1664} 1665 1666var ( 1667 bottomUpVisitor bottomUpVisitorImpl 1668 topDownVisitor topDownVisitorImpl 1669) 1670 1671// Calls visit on each module, guaranteeing that visit is not called on a module until visit on all 1672// of its dependencies has finished. 1673func (c *Context) parallelVisit(order visitOrderer, visit func(group *moduleInfo) bool) { 1674 doneCh := make(chan *moduleInfo) 1675 cancelCh := make(chan bool) 1676 count := 0 1677 cancel := false 1678 var backlog []*moduleInfo 1679 const limit = 1000 1680 1681 for _, module := range c.modulesSorted { 1682 module.waitingCount = order.waitCount(module) 1683 } 1684 1685 visitOne := func(module *moduleInfo) { 1686 if count < limit { 1687 count++ 1688 go func() { 1689 ret := visit(module) 1690 if ret { 1691 cancelCh <- true 1692 } 1693 doneCh <- module 1694 }() 1695 } else { 1696 backlog = append(backlog, module) 1697 } 1698 } 1699 1700 for _, module := range c.modulesSorted { 1701 if module.waitingCount == 0 { 1702 visitOne(module) 1703 } 1704 } 1705 1706 for count > 0 || len(backlog) > 0 { 1707 select { 1708 case <-cancelCh: 1709 cancel = true 1710 backlog = nil 1711 case doneModule := <-doneCh: 1712 count-- 1713 if !cancel { 1714 for count < limit && len(backlog) > 0 { 1715 toVisit := backlog[0] 1716 backlog = backlog[1:] 1717 visitOne(toVisit) 1718 } 1719 for _, module := range order.propagate(doneModule) { 1720 module.waitingCount-- 1721 if module.waitingCount == 0 { 1722 visitOne(module) 1723 } 1724 } 1725 } 1726 } 1727 } 1728} 1729 1730// updateDependencies recursively walks the module dependency graph and updates 1731// additional fields based on the dependencies. It builds a sorted list of modules 1732// such that dependencies of a module always appear first, and populates reverse 1733// dependency links and counts of total dependencies. It also reports errors when 1734// it encounters dependency cycles. This should called after resolveDependencies, 1735// as well as after any mutator pass has called addDependency 1736func (c *Context) updateDependencies() (errs []error) { 1737 visited := make(map[*moduleInfo]bool) // modules that were already checked 1738 checking := make(map[*moduleInfo]bool) // modules actively being checked 1739 1740 sorted := make([]*moduleInfo, 0, len(c.moduleInfo)) 1741 1742 var check func(group *moduleInfo) []*moduleInfo 1743 1744 cycleError := func(cycle []*moduleInfo) { 1745 // We are the "start" of the cycle, so we're responsible 1746 // for generating the errors. The cycle list is in 1747 // reverse order because all the 'check' calls append 1748 // their own module to the list. 1749 errs = append(errs, &BlueprintError{ 1750 Err: fmt.Errorf("encountered dependency cycle:"), 1751 Pos: cycle[len(cycle)-1].pos, 1752 }) 1753 1754 // Iterate backwards through the cycle list. 1755 curModule := cycle[0] 1756 for i := len(cycle) - 1; i >= 0; i-- { 1757 nextModule := cycle[i] 1758 errs = append(errs, &BlueprintError{ 1759 Err: fmt.Errorf(" %q depends on %q", 1760 curModule.Name(), 1761 nextModule.Name()), 1762 Pos: curModule.pos, 1763 }) 1764 curModule = nextModule 1765 } 1766 } 1767 1768 check = func(module *moduleInfo) []*moduleInfo { 1769 visited[module] = true 1770 checking[module] = true 1771 defer delete(checking, module) 1772 1773 deps := make(map[*moduleInfo]bool) 1774 1775 // Add an implicit dependency ordering on all earlier modules in the same module group 1776 for _, dep := range module.group.modules { 1777 if dep == module { 1778 break 1779 } 1780 deps[dep] = true 1781 } 1782 1783 for _, dep := range module.directDeps { 1784 deps[dep.module] = true 1785 } 1786 1787 module.reverseDeps = []*moduleInfo{} 1788 module.forwardDeps = []*moduleInfo{} 1789 1790 for dep := range deps { 1791 if checking[dep] { 1792 // This is a cycle. 1793 return []*moduleInfo{dep, module} 1794 } 1795 1796 if !visited[dep] { 1797 cycle := check(dep) 1798 if cycle != nil { 1799 if cycle[0] == module { 1800 // We are the "start" of the cycle, so we're responsible 1801 // for generating the errors. The cycle list is in 1802 // reverse order because all the 'check' calls append 1803 // their own module to the list. 1804 cycleError(cycle) 1805 1806 // We can continue processing this module's children to 1807 // find more cycles. Since all the modules that were 1808 // part of the found cycle were marked as visited we 1809 // won't run into that cycle again. 1810 } else { 1811 // We're not the "start" of the cycle, so we just append 1812 // our module to the list and return it. 1813 return append(cycle, module) 1814 } 1815 } 1816 } 1817 1818 module.forwardDeps = append(module.forwardDeps, dep) 1819 dep.reverseDeps = append(dep.reverseDeps, module) 1820 } 1821 1822 sorted = append(sorted, module) 1823 1824 return nil 1825 } 1826 1827 for _, module := range c.moduleInfo { 1828 if !visited[module] { 1829 cycle := check(module) 1830 if cycle != nil { 1831 if cycle[len(cycle)-1] != module { 1832 panic("inconceivable!") 1833 } 1834 cycleError(cycle) 1835 } 1836 } 1837 } 1838 1839 c.modulesSorted = sorted 1840 1841 return 1842} 1843 1844// PrepareBuildActions generates an internal representation of all the build 1845// actions that need to be performed. This process involves invoking the 1846// GenerateBuildActions method on each of the Module objects created during the 1847// parse phase and then on each of the registered Singleton objects. 1848// 1849// If the ResolveDependencies method has not already been called it is called 1850// automatically by this method. 1851// 1852// The config argument is made available to all of the Module and Singleton 1853// objects via the Config method on the ModuleContext and SingletonContext 1854// objects passed to GenerateBuildActions. It is also passed to the functions 1855// specified via PoolFunc, RuleFunc, and VariableFunc so that they can compute 1856// config-specific values. 1857// 1858// The returned deps is a list of the ninja files dependencies that were added 1859// by the modules and singletons via the ModuleContext.AddNinjaFileDeps(), 1860// SingletonContext.AddNinjaFileDeps(), and PackageContext.AddNinjaFileDeps() 1861// methods. 1862func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) { 1863 c.buildActionsReady = false 1864 1865 if !c.dependenciesReady { 1866 extraDeps, errs := c.ResolveDependencies(config) 1867 if len(errs) > 0 { 1868 return nil, errs 1869 } 1870 deps = append(deps, extraDeps...) 1871 } 1872 1873 depsModules, errs := c.generateModuleBuildActions(config, c.liveGlobals) 1874 if len(errs) > 0 { 1875 return nil, errs 1876 } 1877 1878 depsSingletons, errs := c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals) 1879 if len(errs) > 0 { 1880 return nil, errs 1881 } 1882 1883 deps = append(deps, depsModules...) 1884 deps = append(deps, depsSingletons...) 1885 1886 if c.ninjaBuildDir != nil { 1887 c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir) 1888 } 1889 1890 pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals) 1891 1892 deps = append(deps, depsPackages...) 1893 1894 // This will panic if it finds a problem since it's a programming error. 1895 c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames) 1896 1897 c.pkgNames = pkgNames 1898 c.globalVariables = c.liveGlobals.variables 1899 c.globalPools = c.liveGlobals.pools 1900 c.globalRules = c.liveGlobals.rules 1901 1902 c.buildActionsReady = true 1903 1904 return deps, nil 1905} 1906 1907func (c *Context) runMutators(config interface{}) (deps []string, errs []error) { 1908 var mutators []*mutatorInfo 1909 1910 mutators = append(mutators, c.earlyMutatorInfo...) 1911 mutators = append(mutators, c.mutatorInfo...) 1912 1913 for _, mutator := range mutators { 1914 var newDeps []string 1915 if mutator.topDownMutator != nil { 1916 newDeps, errs = c.runMutator(config, mutator, topDownMutator) 1917 } else if mutator.bottomUpMutator != nil { 1918 newDeps, errs = c.runMutator(config, mutator, bottomUpMutator) 1919 } else { 1920 panic("no mutator set on " + mutator.name) 1921 } 1922 if len(errs) > 0 { 1923 return nil, errs 1924 } 1925 deps = append(deps, newDeps...) 1926 } 1927 1928 return deps, nil 1929} 1930 1931type mutatorDirection interface { 1932 run(mutator *mutatorInfo, ctx *mutatorContext) 1933 orderer() visitOrderer 1934 fmt.Stringer 1935} 1936 1937type bottomUpMutatorImpl struct{} 1938 1939func (bottomUpMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 1940 mutator.bottomUpMutator(ctx) 1941} 1942 1943func (bottomUpMutatorImpl) orderer() visitOrderer { 1944 return bottomUpVisitor 1945} 1946 1947func (bottomUpMutatorImpl) String() string { 1948 return "bottom up mutator" 1949} 1950 1951type topDownMutatorImpl struct{} 1952 1953func (topDownMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 1954 mutator.topDownMutator(ctx) 1955} 1956 1957func (topDownMutatorImpl) orderer() visitOrderer { 1958 return topDownVisitor 1959} 1960 1961func (topDownMutatorImpl) String() string { 1962 return "top down mutator" 1963} 1964 1965var ( 1966 topDownMutator topDownMutatorImpl 1967 bottomUpMutator bottomUpMutatorImpl 1968) 1969 1970type reverseDep struct { 1971 module *moduleInfo 1972 dep depInfo 1973} 1974 1975func (c *Context) runMutator(config interface{}, mutator *mutatorInfo, 1976 direction mutatorDirection) (deps []string, errs []error) { 1977 1978 newModuleInfo := make(map[Module]*moduleInfo) 1979 for k, v := range c.moduleInfo { 1980 newModuleInfo[k] = v 1981 } 1982 1983 type globalStateChange struct { 1984 reverse []reverseDep 1985 rename []rename 1986 replace []replace 1987 newModules []*moduleInfo 1988 deps []string 1989 } 1990 1991 reverseDeps := make(map[*moduleInfo][]depInfo) 1992 var rename []rename 1993 var replace []replace 1994 var newModules []*moduleInfo 1995 1996 errsCh := make(chan []error) 1997 globalStateCh := make(chan globalStateChange) 1998 newVariationsCh := make(chan []*moduleInfo) 1999 done := make(chan bool) 2000 2001 c.depsModified = 0 2002 2003 visit := func(module *moduleInfo) bool { 2004 if module.splitModules != nil { 2005 panic("split module found in sorted module list") 2006 } 2007 2008 mctx := &mutatorContext{ 2009 baseModuleContext: baseModuleContext{ 2010 context: c, 2011 config: config, 2012 module: module, 2013 }, 2014 name: mutator.name, 2015 } 2016 2017 func() { 2018 defer func() { 2019 if r := recover(); r != nil { 2020 in := fmt.Sprintf("%s %q for %s", direction, mutator.name, module) 2021 if err, ok := r.(panicError); ok { 2022 err.addIn(in) 2023 mctx.error(err) 2024 } else { 2025 mctx.error(newPanicErrorf(r, in)) 2026 } 2027 } 2028 }() 2029 direction.run(mutator, mctx) 2030 }() 2031 2032 if len(mctx.errs) > 0 { 2033 errsCh <- mctx.errs 2034 return true 2035 } 2036 2037 if len(mctx.newVariations) > 0 { 2038 newVariationsCh <- mctx.newVariations 2039 } 2040 2041 if len(mctx.reverseDeps) > 0 || len(mctx.replace) > 0 || len(mctx.rename) > 0 || len(mctx.newModules) > 0 { 2042 globalStateCh <- globalStateChange{ 2043 reverse: mctx.reverseDeps, 2044 replace: mctx.replace, 2045 rename: mctx.rename, 2046 newModules: mctx.newModules, 2047 deps: mctx.ninjaFileDeps, 2048 } 2049 } 2050 2051 return false 2052 } 2053 2054 // Process errs and reverseDeps in a single goroutine 2055 go func() { 2056 for { 2057 select { 2058 case newErrs := <-errsCh: 2059 errs = append(errs, newErrs...) 2060 case globalStateChange := <-globalStateCh: 2061 for _, r := range globalStateChange.reverse { 2062 reverseDeps[r.module] = append(reverseDeps[r.module], r.dep) 2063 } 2064 replace = append(replace, globalStateChange.replace...) 2065 rename = append(rename, globalStateChange.rename...) 2066 newModules = append(newModules, globalStateChange.newModules...) 2067 deps = append(deps, globalStateChange.deps...) 2068 case newVariations := <-newVariationsCh: 2069 for _, m := range newVariations { 2070 newModuleInfo[m.logicModule] = m 2071 } 2072 case <-done: 2073 return 2074 } 2075 } 2076 }() 2077 2078 if mutator.parallel { 2079 c.parallelVisit(direction.orderer(), visit) 2080 } else { 2081 direction.orderer().visit(c.modulesSorted, visit) 2082 } 2083 2084 done <- true 2085 2086 if len(errs) > 0 { 2087 return nil, errs 2088 } 2089 2090 c.moduleInfo = newModuleInfo 2091 2092 for _, group := range c.moduleGroups { 2093 for i := 0; i < len(group.modules); i++ { 2094 module := group.modules[i] 2095 2096 // Update module group to contain newly split variants 2097 if module.splitModules != nil { 2098 group.modules, i = spliceModules(group.modules, i, module.splitModules) 2099 } 2100 2101 // Fix up any remaining dependencies on modules that were split into variants 2102 // by replacing them with the first variant 2103 for j, dep := range module.directDeps { 2104 if dep.module.logicModule == nil { 2105 module.directDeps[j].module = dep.module.splitModules[0] 2106 } 2107 } 2108 } 2109 } 2110 2111 for module, deps := range reverseDeps { 2112 sort.Sort(depSorter(deps)) 2113 module.directDeps = append(module.directDeps, deps...) 2114 c.depsModified++ 2115 } 2116 2117 for _, module := range newModules { 2118 errs = c.addModule(module) 2119 if len(errs) > 0 { 2120 return nil, errs 2121 } 2122 atomic.AddUint32(&c.depsModified, 1) 2123 } 2124 2125 errs = c.handleRenames(rename) 2126 if len(errs) > 0 { 2127 return nil, errs 2128 } 2129 2130 errs = c.handleReplacements(replace) 2131 if len(errs) > 0 { 2132 return nil, errs 2133 } 2134 2135 if c.depsModified > 0 { 2136 errs = c.updateDependencies() 2137 if len(errs) > 0 { 2138 return nil, errs 2139 } 2140 } 2141 2142 return deps, errs 2143} 2144 2145// Replaces every build logic module with a clone of itself. Prevents introducing problems where 2146// a mutator sets a non-property member variable on a module, which works until a later mutator 2147// creates variants of that module. 2148func (c *Context) cloneModules() { 2149 type update struct { 2150 orig Module 2151 clone *moduleInfo 2152 } 2153 ch := make(chan update) 2154 doneCh := make(chan bool) 2155 go func() { 2156 c.parallelVisit(unorderedVisitorImpl{}, func(m *moduleInfo) bool { 2157 origLogicModule := m.logicModule 2158 m.logicModule, m.properties = c.cloneLogicModule(m) 2159 ch <- update{origLogicModule, m} 2160 return false 2161 }) 2162 doneCh <- true 2163 }() 2164 2165 done := false 2166 for !done { 2167 select { 2168 case <-doneCh: 2169 done = true 2170 case update := <-ch: 2171 delete(c.moduleInfo, update.orig) 2172 c.moduleInfo[update.clone.logicModule] = update.clone 2173 } 2174 } 2175} 2176 2177// Removes modules[i] from the list and inserts newModules... where it was located, returning 2178// the new slice and the index of the last inserted element 2179func spliceModules(modules []*moduleInfo, i int, newModules []*moduleInfo) ([]*moduleInfo, int) { 2180 spliceSize := len(newModules) 2181 newLen := len(modules) + spliceSize - 1 2182 var dest []*moduleInfo 2183 if cap(modules) >= len(modules)-1+len(newModules) { 2184 // We can fit the splice in the existing capacity, do everything in place 2185 dest = modules[:newLen] 2186 } else { 2187 dest = make([]*moduleInfo, newLen) 2188 copy(dest, modules[:i]) 2189 } 2190 2191 // Move the end of the slice over by spliceSize-1 2192 copy(dest[i+spliceSize:], modules[i+1:]) 2193 2194 // Copy the new modules into the slice 2195 copy(dest[i:], newModules) 2196 2197 return dest, i + spliceSize - 1 2198} 2199 2200func (c *Context) generateModuleBuildActions(config interface{}, 2201 liveGlobals *liveTracker) ([]string, []error) { 2202 2203 var deps []string 2204 var errs []error 2205 2206 cancelCh := make(chan struct{}) 2207 errsCh := make(chan []error) 2208 depsCh := make(chan []string) 2209 2210 go func() { 2211 for { 2212 select { 2213 case <-cancelCh: 2214 close(cancelCh) 2215 return 2216 case newErrs := <-errsCh: 2217 errs = append(errs, newErrs...) 2218 case newDeps := <-depsCh: 2219 deps = append(deps, newDeps...) 2220 2221 } 2222 } 2223 }() 2224 2225 c.parallelVisit(bottomUpVisitor, func(module *moduleInfo) bool { 2226 2227 uniqueName := c.nameInterface.UniqueName(newNamespaceContext(module), module.group.name) 2228 sanitizedName := toNinjaName(uniqueName) 2229 2230 prefix := moduleNamespacePrefix(sanitizedName + "_" + module.variantName) 2231 2232 // The parent scope of the moduleContext's local scope gets overridden to be that of the 2233 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2234 // just set it to nil. 2235 scope := newLocalScope(nil, prefix) 2236 2237 mctx := &moduleContext{ 2238 baseModuleContext: baseModuleContext{ 2239 context: c, 2240 config: config, 2241 module: module, 2242 }, 2243 scope: scope, 2244 handledMissingDeps: module.missingDeps == nil, 2245 } 2246 2247 func() { 2248 defer func() { 2249 if r := recover(); r != nil { 2250 in := fmt.Sprintf("GenerateBuildActions for %s", module) 2251 if err, ok := r.(panicError); ok { 2252 err.addIn(in) 2253 mctx.error(err) 2254 } else { 2255 mctx.error(newPanicErrorf(r, in)) 2256 } 2257 } 2258 }() 2259 mctx.module.logicModule.GenerateBuildActions(mctx) 2260 }() 2261 2262 if len(mctx.errs) > 0 { 2263 errsCh <- mctx.errs 2264 return true 2265 } 2266 2267 if module.missingDeps != nil && !mctx.handledMissingDeps { 2268 var errs []error 2269 for _, depName := range module.missingDeps { 2270 errs = append(errs, c.missingDependencyError(module, depName)) 2271 } 2272 errsCh <- errs 2273 return true 2274 } 2275 2276 depsCh <- mctx.ninjaFileDeps 2277 2278 newErrs := c.processLocalBuildActions(&module.actionDefs, 2279 &mctx.actionDefs, liveGlobals) 2280 if len(newErrs) > 0 { 2281 errsCh <- newErrs 2282 return true 2283 } 2284 return false 2285 }) 2286 2287 cancelCh <- struct{}{} 2288 <-cancelCh 2289 2290 return deps, errs 2291} 2292 2293func (c *Context) generateSingletonBuildActions(config interface{}, 2294 singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { 2295 2296 var deps []string 2297 var errs []error 2298 2299 for _, info := range singletons { 2300 // The parent scope of the singletonContext's local scope gets overridden to be that of the 2301 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2302 // just set it to nil. 2303 scope := newLocalScope(nil, singletonNamespacePrefix(info.name)) 2304 2305 sctx := &singletonContext{ 2306 context: c, 2307 config: config, 2308 scope: scope, 2309 globals: liveGlobals, 2310 } 2311 2312 func() { 2313 defer func() { 2314 if r := recover(); r != nil { 2315 in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name) 2316 if err, ok := r.(panicError); ok { 2317 err.addIn(in) 2318 sctx.error(err) 2319 } else { 2320 sctx.error(newPanicErrorf(r, in)) 2321 } 2322 } 2323 }() 2324 info.singleton.GenerateBuildActions(sctx) 2325 }() 2326 2327 if len(sctx.errs) > 0 { 2328 errs = append(errs, sctx.errs...) 2329 if len(errs) > maxErrors { 2330 break 2331 } 2332 continue 2333 } 2334 2335 deps = append(deps, sctx.ninjaFileDeps...) 2336 2337 newErrs := c.processLocalBuildActions(&info.actionDefs, 2338 &sctx.actionDefs, liveGlobals) 2339 errs = append(errs, newErrs...) 2340 if len(errs) > maxErrors { 2341 break 2342 } 2343 } 2344 2345 return deps, errs 2346} 2347 2348func (c *Context) processLocalBuildActions(out, in *localBuildActions, 2349 liveGlobals *liveTracker) []error { 2350 2351 var errs []error 2352 2353 // First we go through and add everything referenced by the module's 2354 // buildDefs to the live globals set. This will end up adding the live 2355 // locals to the set as well, but we'll take them out after. 2356 for _, def := range in.buildDefs { 2357 err := liveGlobals.AddBuildDefDeps(def) 2358 if err != nil { 2359 errs = append(errs, err) 2360 } 2361 } 2362 2363 if len(errs) > 0 { 2364 return errs 2365 } 2366 2367 out.buildDefs = append(out.buildDefs, in.buildDefs...) 2368 2369 // We use the now-incorrect set of live "globals" to determine which local 2370 // definitions are live. As we go through copying those live locals to the 2371 // moduleGroup we remove them from the live globals set. 2372 for _, v := range in.variables { 2373 isLive := liveGlobals.RemoveVariableIfLive(v) 2374 if isLive { 2375 out.variables = append(out.variables, v) 2376 } 2377 } 2378 2379 for _, r := range in.rules { 2380 isLive := liveGlobals.RemoveRuleIfLive(r) 2381 if isLive { 2382 out.rules = append(out.rules, r) 2383 } 2384 } 2385 2386 return nil 2387} 2388 2389func (c *Context) walkDeps(topModule *moduleInfo, 2390 visitDown func(depInfo, *moduleInfo) bool, visitUp func(depInfo, *moduleInfo)) { 2391 2392 visited := make(map[*moduleInfo]bool) 2393 var visiting *moduleInfo 2394 2395 defer func() { 2396 if r := recover(); r != nil { 2397 panic(newPanicErrorf(r, "WalkDeps(%s, %s, %s) for dependency %s", 2398 topModule, funcName(visitDown), funcName(visitUp), visiting)) 2399 } 2400 }() 2401 2402 var walk func(module *moduleInfo) 2403 walk = func(module *moduleInfo) { 2404 for _, dep := range module.directDeps { 2405 if !visited[dep.module] { 2406 visited[dep.module] = true 2407 visiting = dep.module 2408 recurse := true 2409 if visitDown != nil { 2410 recurse = visitDown(dep, module) 2411 } 2412 if recurse { 2413 walk(dep.module) 2414 } 2415 if visitUp != nil { 2416 visitUp(dep, module) 2417 } 2418 } 2419 } 2420 } 2421 2422 walk(topModule) 2423} 2424 2425type replace struct { 2426 from, to *moduleInfo 2427} 2428 2429type rename struct { 2430 group *moduleGroup 2431 name string 2432} 2433 2434func (c *Context) moduleMatchingVariant(module *moduleInfo, name string) *moduleInfo { 2435 targets := c.modulesFromName(name, module.namespace()) 2436 2437 if targets == nil { 2438 return nil 2439 } 2440 2441 for _, m := range targets { 2442 if module.variantName == m.variantName { 2443 return m 2444 } 2445 } 2446 2447 return nil 2448} 2449 2450func (c *Context) handleRenames(renames []rename) []error { 2451 var errs []error 2452 for _, rename := range renames { 2453 group, name := rename.group, rename.name 2454 if name == group.name || len(group.modules) < 1 { 2455 continue 2456 } 2457 2458 errs = append(errs, c.nameInterface.Rename(group.name, rename.name, group.namespace)...) 2459 } 2460 2461 return errs 2462} 2463 2464func (c *Context) handleReplacements(replacements []replace) []error { 2465 var errs []error 2466 for _, replace := range replacements { 2467 for _, m := range replace.from.reverseDeps { 2468 for i, d := range m.directDeps { 2469 if d.module == replace.from { 2470 m.directDeps[i].module = replace.to 2471 } 2472 } 2473 } 2474 2475 atomic.AddUint32(&c.depsModified, 1) 2476 } 2477 2478 return errs 2479} 2480 2481func (c *Context) discoveredMissingDependencies(module *moduleInfo, depName string) (errs []error) { 2482 if c.allowMissingDependencies { 2483 module.missingDeps = append(module.missingDeps, depName) 2484 return nil 2485 } 2486 return []error{c.missingDependencyError(module, depName)} 2487} 2488 2489func (c *Context) missingDependencyError(module *moduleInfo, depName string) (errs error) { 2490 err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName) 2491 2492 return &BlueprintError{ 2493 Err: err, 2494 Pos: module.pos, 2495 } 2496} 2497 2498func (c *Context) modulesFromName(name string, namespace Namespace) []*moduleInfo { 2499 group, exists := c.nameInterface.ModuleFromName(name, namespace) 2500 if exists { 2501 return group.modules 2502 } 2503 return nil 2504} 2505 2506func (c *Context) sortedModuleGroups() []*moduleGroup { 2507 if c.cachedSortedModuleGroups == nil { 2508 unwrap := func(wrappers []ModuleGroup) []*moduleGroup { 2509 result := make([]*moduleGroup, 0, len(wrappers)) 2510 for _, group := range wrappers { 2511 result = append(result, group.moduleGroup) 2512 } 2513 return result 2514 } 2515 2516 c.cachedSortedModuleGroups = unwrap(c.nameInterface.AllModules()) 2517 } 2518 2519 return c.cachedSortedModuleGroups 2520} 2521 2522func (c *Context) visitAllModules(visit func(Module)) { 2523 var module *moduleInfo 2524 2525 defer func() { 2526 if r := recover(); r != nil { 2527 panic(newPanicErrorf(r, "VisitAllModules(%s) for %s", 2528 funcName(visit), module)) 2529 } 2530 }() 2531 2532 for _, moduleGroup := range c.sortedModuleGroups() { 2533 for _, module = range moduleGroup.modules { 2534 visit(module.logicModule) 2535 } 2536 } 2537} 2538 2539func (c *Context) visitAllModulesIf(pred func(Module) bool, 2540 visit func(Module)) { 2541 2542 var module *moduleInfo 2543 2544 defer func() { 2545 if r := recover(); r != nil { 2546 panic(newPanicErrorf(r, "VisitAllModulesIf(%s, %s) for %s", 2547 funcName(pred), funcName(visit), module)) 2548 } 2549 }() 2550 2551 for _, moduleGroup := range c.sortedModuleGroups() { 2552 for _, module := range moduleGroup.modules { 2553 if pred(module.logicModule) { 2554 visit(module.logicModule) 2555 } 2556 } 2557 } 2558} 2559 2560func (c *Context) visitAllModuleVariants(module *moduleInfo, 2561 visit func(Module)) { 2562 2563 var variant *moduleInfo 2564 2565 defer func() { 2566 if r := recover(); r != nil { 2567 panic(newPanicErrorf(r, "VisitAllModuleVariants(%s, %s) for %s", 2568 module, funcName(visit), variant)) 2569 } 2570 }() 2571 2572 for _, variant = range module.group.modules { 2573 visit(variant.logicModule) 2574 } 2575} 2576 2577func (c *Context) requireNinjaVersion(major, minor, micro int) { 2578 if major != 1 { 2579 panic("ninja version with major version != 1 not supported") 2580 } 2581 if c.requiredNinjaMinor < minor { 2582 c.requiredNinjaMinor = minor 2583 c.requiredNinjaMicro = micro 2584 } 2585 if c.requiredNinjaMinor == minor && c.requiredNinjaMicro < micro { 2586 c.requiredNinjaMicro = micro 2587 } 2588} 2589 2590func (c *Context) setNinjaBuildDir(value *ninjaString) { 2591 if c.ninjaBuildDir == nil { 2592 c.ninjaBuildDir = value 2593 } 2594} 2595 2596func (c *Context) makeUniquePackageNames( 2597 liveGlobals *liveTracker) (map[*packageContext]string, []string) { 2598 2599 pkgs := make(map[string]*packageContext) 2600 pkgNames := make(map[*packageContext]string) 2601 longPkgNames := make(map[*packageContext]bool) 2602 2603 processPackage := func(pctx *packageContext) { 2604 if pctx == nil { 2605 // This is a built-in rule and has no package. 2606 return 2607 } 2608 if _, ok := pkgNames[pctx]; ok { 2609 // We've already processed this package. 2610 return 2611 } 2612 2613 otherPkg, present := pkgs[pctx.shortName] 2614 if present { 2615 // Short name collision. Both this package and the one that's 2616 // already there need to use their full names. We leave the short 2617 // name in pkgNames for now so future collisions still get caught. 2618 longPkgNames[pctx] = true 2619 longPkgNames[otherPkg] = true 2620 } else { 2621 // No collision so far. Tentatively set the package's name to be 2622 // its short name. 2623 pkgNames[pctx] = pctx.shortName 2624 pkgs[pctx.shortName] = pctx 2625 } 2626 } 2627 2628 // We try to give all packages their short name, but when we get collisions 2629 // we need to use the full unique package name. 2630 for v, _ := range liveGlobals.variables { 2631 processPackage(v.packageContext()) 2632 } 2633 for p, _ := range liveGlobals.pools { 2634 processPackage(p.packageContext()) 2635 } 2636 for r, _ := range liveGlobals.rules { 2637 processPackage(r.packageContext()) 2638 } 2639 2640 // Add the packages that had collisions using their full unique names. This 2641 // will overwrite any short names that were added in the previous step. 2642 for pctx := range longPkgNames { 2643 pkgNames[pctx] = pctx.fullName 2644 } 2645 2646 // Create deps list from calls to PackageContext.AddNinjaFileDeps 2647 deps := []string{} 2648 for _, pkg := range pkgs { 2649 deps = append(deps, pkg.ninjaFileDeps...) 2650 } 2651 2652 return pkgNames, deps 2653} 2654 2655func (c *Context) checkForVariableReferenceCycles( 2656 variables map[Variable]*ninjaString, pkgNames map[*packageContext]string) { 2657 2658 visited := make(map[Variable]bool) // variables that were already checked 2659 checking := make(map[Variable]bool) // variables actively being checked 2660 2661 var check func(v Variable) []Variable 2662 2663 check = func(v Variable) []Variable { 2664 visited[v] = true 2665 checking[v] = true 2666 defer delete(checking, v) 2667 2668 value := variables[v] 2669 for _, dep := range value.variables { 2670 if checking[dep] { 2671 // This is a cycle. 2672 return []Variable{dep, v} 2673 } 2674 2675 if !visited[dep] { 2676 cycle := check(dep) 2677 if cycle != nil { 2678 if cycle[0] == v { 2679 // We are the "start" of the cycle, so we're responsible 2680 // for generating the errors. The cycle list is in 2681 // reverse order because all the 'check' calls append 2682 // their own module to the list. 2683 msgs := []string{"detected variable reference cycle:"} 2684 2685 // Iterate backwards through the cycle list. 2686 curName := v.fullName(pkgNames) 2687 curValue := value.Value(pkgNames) 2688 for i := len(cycle) - 1; i >= 0; i-- { 2689 next := cycle[i] 2690 nextName := next.fullName(pkgNames) 2691 nextValue := variables[next].Value(pkgNames) 2692 2693 msgs = append(msgs, fmt.Sprintf( 2694 " %q depends on %q", curName, nextName)) 2695 msgs = append(msgs, fmt.Sprintf( 2696 " [%s = %s]", curName, curValue)) 2697 2698 curName = nextName 2699 curValue = nextValue 2700 } 2701 2702 // Variable reference cycles are a programming error, 2703 // not the fault of the Blueprint file authors. 2704 panic(strings.Join(msgs, "\n")) 2705 } else { 2706 // We're not the "start" of the cycle, so we just append 2707 // our module to the list and return it. 2708 return append(cycle, v) 2709 } 2710 } 2711 } 2712 } 2713 2714 return nil 2715 } 2716 2717 for v := range variables { 2718 if !visited[v] { 2719 cycle := check(v) 2720 if cycle != nil { 2721 panic("inconceivable!") 2722 } 2723 } 2724 } 2725} 2726 2727// AllTargets returns a map all the build target names to the rule used to build 2728// them. This is the same information that is output by running 'ninja -t 2729// targets all'. If this is called before PrepareBuildActions successfully 2730// completes then ErrbuildActionsNotReady is returned. 2731func (c *Context) AllTargets() (map[string]string, error) { 2732 if !c.buildActionsReady { 2733 return nil, ErrBuildActionsNotReady 2734 } 2735 2736 targets := map[string]string{} 2737 2738 // Collect all the module build targets. 2739 for _, module := range c.moduleInfo { 2740 for _, buildDef := range module.actionDefs.buildDefs { 2741 ruleName := buildDef.Rule.fullName(c.pkgNames) 2742 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2743 outputValue, err := output.Eval(c.globalVariables) 2744 if err != nil { 2745 return nil, err 2746 } 2747 targets[outputValue] = ruleName 2748 } 2749 } 2750 } 2751 2752 // Collect all the singleton build targets. 2753 for _, info := range c.singletonInfo { 2754 for _, buildDef := range info.actionDefs.buildDefs { 2755 ruleName := buildDef.Rule.fullName(c.pkgNames) 2756 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2757 outputValue, err := output.Eval(c.globalVariables) 2758 if err != nil { 2759 return nil, err 2760 } 2761 targets[outputValue] = ruleName 2762 } 2763 } 2764 } 2765 2766 return targets, nil 2767} 2768 2769func (c *Context) NinjaBuildDir() (string, error) { 2770 if c.ninjaBuildDir != nil { 2771 return c.ninjaBuildDir.Eval(c.globalVariables) 2772 } else { 2773 return "", nil 2774 } 2775} 2776 2777// ModuleTypePropertyStructs returns a mapping from module type name to a list of pointers to 2778// property structs returned by the factory for that module type. 2779func (c *Context) ModuleTypePropertyStructs() map[string][]interface{} { 2780 ret := make(map[string][]interface{}) 2781 for moduleType, factory := range c.moduleFactories { 2782 _, ret[moduleType] = factory() 2783 } 2784 2785 return ret 2786} 2787 2788func (c *Context) ModuleName(logicModule Module) string { 2789 module := c.moduleInfo[logicModule] 2790 return module.Name() 2791} 2792 2793func (c *Context) ModulePath(logicModule Module) string { 2794 module := c.moduleInfo[logicModule] 2795 return module.relBlueprintsFile 2796} 2797 2798func (c *Context) ModuleDir(logicModule Module) string { 2799 return filepath.Dir(c.ModulePath(logicModule)) 2800} 2801 2802func (c *Context) ModuleSubDir(logicModule Module) string { 2803 module := c.moduleInfo[logicModule] 2804 return module.variantName 2805} 2806 2807func (c *Context) ModuleType(logicModule Module) string { 2808 module := c.moduleInfo[logicModule] 2809 return module.typeName 2810} 2811 2812func (c *Context) BlueprintFile(logicModule Module) string { 2813 module := c.moduleInfo[logicModule] 2814 return module.relBlueprintsFile 2815} 2816 2817func (c *Context) ModuleErrorf(logicModule Module, format string, 2818 args ...interface{}) error { 2819 2820 module := c.moduleInfo[logicModule] 2821 return &BlueprintError{ 2822 Err: fmt.Errorf(format, args...), 2823 Pos: module.pos, 2824 } 2825} 2826 2827func (c *Context) VisitAllModules(visit func(Module)) { 2828 c.visitAllModules(visit) 2829} 2830 2831func (c *Context) VisitAllModulesIf(pred func(Module) bool, 2832 visit func(Module)) { 2833 2834 c.visitAllModulesIf(pred, visit) 2835} 2836 2837func (c *Context) VisitDirectDeps(module Module, visit func(Module)) { 2838 topModule := c.moduleInfo[module] 2839 2840 var visiting *moduleInfo 2841 2842 defer func() { 2843 if r := recover(); r != nil { 2844 panic(newPanicErrorf(r, "VisitDirectDeps(%s, %s) for dependency %s", 2845 topModule, funcName(visit), visiting)) 2846 } 2847 }() 2848 2849 for _, dep := range topModule.directDeps { 2850 visiting = dep.module 2851 visit(dep.module.logicModule) 2852 } 2853} 2854 2855func (c *Context) VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) { 2856 topModule := c.moduleInfo[module] 2857 2858 var visiting *moduleInfo 2859 2860 defer func() { 2861 if r := recover(); r != nil { 2862 panic(newPanicErrorf(r, "VisitDirectDepsIf(%s, %s, %s) for dependency %s", 2863 topModule, funcName(pred), funcName(visit), visiting)) 2864 } 2865 }() 2866 2867 for _, dep := range topModule.directDeps { 2868 visiting = dep.module 2869 if pred(dep.module.logicModule) { 2870 visit(dep.module.logicModule) 2871 } 2872 } 2873} 2874 2875func (c *Context) VisitDepsDepthFirst(module Module, visit func(Module)) { 2876 topModule := c.moduleInfo[module] 2877 2878 var visiting *moduleInfo 2879 2880 defer func() { 2881 if r := recover(); r != nil { 2882 panic(newPanicErrorf(r, "VisitDepsDepthFirst(%s, %s) for dependency %s", 2883 topModule, funcName(visit), visiting)) 2884 } 2885 }() 2886 2887 c.walkDeps(topModule, nil, func(dep depInfo, parent *moduleInfo) { 2888 visiting = dep.module 2889 visit(dep.module.logicModule) 2890 }) 2891} 2892 2893func (c *Context) VisitDepsDepthFirstIf(module Module, pred func(Module) bool, visit func(Module)) { 2894 topModule := c.moduleInfo[module] 2895 2896 var visiting *moduleInfo 2897 2898 defer func() { 2899 if r := recover(); r != nil { 2900 panic(newPanicErrorf(r, "VisitDepsDepthFirstIf(%s, %s, %s) for dependency %s", 2901 topModule, funcName(pred), funcName(visit), visiting)) 2902 } 2903 }() 2904 2905 c.walkDeps(topModule, nil, func(dep depInfo, parent *moduleInfo) { 2906 if pred(dep.module.logicModule) { 2907 visiting = dep.module 2908 visit(dep.module.logicModule) 2909 } 2910 }) 2911} 2912 2913func (c *Context) PrimaryModule(module Module) Module { 2914 return c.moduleInfo[module].group.modules[0].logicModule 2915} 2916 2917func (c *Context) FinalModule(module Module) Module { 2918 modules := c.moduleInfo[module].group.modules 2919 return modules[len(modules)-1].logicModule 2920} 2921 2922func (c *Context) VisitAllModuleVariants(module Module, 2923 visit func(Module)) { 2924 2925 c.visitAllModuleVariants(c.moduleInfo[module], visit) 2926} 2927 2928// WriteBuildFile writes the Ninja manifeset text for the generated build 2929// actions to w. If this is called before PrepareBuildActions successfully 2930// completes then ErrBuildActionsNotReady is returned. 2931func (c *Context) WriteBuildFile(w io.Writer) error { 2932 if !c.buildActionsReady { 2933 return ErrBuildActionsNotReady 2934 } 2935 2936 nw := newNinjaWriter(w) 2937 2938 err := c.writeBuildFileHeader(nw) 2939 if err != nil { 2940 return err 2941 } 2942 2943 err = c.writeNinjaRequiredVersion(nw) 2944 if err != nil { 2945 return err 2946 } 2947 2948 // TODO: Group the globals by package. 2949 2950 err = c.writeGlobalVariables(nw) 2951 if err != nil { 2952 return err 2953 } 2954 2955 err = c.writeGlobalPools(nw) 2956 if err != nil { 2957 return err 2958 } 2959 2960 err = c.writeBuildDir(nw) 2961 if err != nil { 2962 return err 2963 } 2964 2965 err = c.writeGlobalRules(nw) 2966 if err != nil { 2967 return err 2968 } 2969 2970 err = c.writeAllModuleActions(nw) 2971 if err != nil { 2972 return err 2973 } 2974 2975 err = c.writeAllSingletonActions(nw) 2976 if err != nil { 2977 return err 2978 } 2979 2980 return nil 2981} 2982 2983type pkgAssociation struct { 2984 PkgName string 2985 PkgPath string 2986} 2987 2988type pkgAssociationSorter struct { 2989 pkgs []pkgAssociation 2990} 2991 2992func (s *pkgAssociationSorter) Len() int { 2993 return len(s.pkgs) 2994} 2995 2996func (s *pkgAssociationSorter) Less(i, j int) bool { 2997 iName := s.pkgs[i].PkgName 2998 jName := s.pkgs[j].PkgName 2999 return iName < jName 3000} 3001 3002func (s *pkgAssociationSorter) Swap(i, j int) { 3003 s.pkgs[i], s.pkgs[j] = s.pkgs[j], s.pkgs[i] 3004} 3005 3006func (c *Context) writeBuildFileHeader(nw *ninjaWriter) error { 3007 headerTemplate := template.New("fileHeader") 3008 _, err := headerTemplate.Parse(fileHeaderTemplate) 3009 if err != nil { 3010 // This is a programming error. 3011 panic(err) 3012 } 3013 3014 var pkgs []pkgAssociation 3015 maxNameLen := 0 3016 for pkg, name := range c.pkgNames { 3017 pkgs = append(pkgs, pkgAssociation{ 3018 PkgName: name, 3019 PkgPath: pkg.pkgPath, 3020 }) 3021 if len(name) > maxNameLen { 3022 maxNameLen = len(name) 3023 } 3024 } 3025 3026 for i := range pkgs { 3027 pkgs[i].PkgName += strings.Repeat(" ", maxNameLen-len(pkgs[i].PkgName)) 3028 } 3029 3030 sort.Sort(&pkgAssociationSorter{pkgs}) 3031 3032 params := map[string]interface{}{ 3033 "Pkgs": pkgs, 3034 } 3035 3036 buf := bytes.NewBuffer(nil) 3037 err = headerTemplate.Execute(buf, params) 3038 if err != nil { 3039 return err 3040 } 3041 3042 return nw.Comment(buf.String()) 3043} 3044 3045func (c *Context) writeNinjaRequiredVersion(nw *ninjaWriter) error { 3046 value := fmt.Sprintf("%d.%d.%d", c.requiredNinjaMajor, c.requiredNinjaMinor, 3047 c.requiredNinjaMicro) 3048 3049 err := nw.Assign("ninja_required_version", value) 3050 if err != nil { 3051 return err 3052 } 3053 3054 return nw.BlankLine() 3055} 3056 3057func (c *Context) writeBuildDir(nw *ninjaWriter) error { 3058 if c.ninjaBuildDir != nil { 3059 err := nw.Assign("builddir", c.ninjaBuildDir.Value(c.pkgNames)) 3060 if err != nil { 3061 return err 3062 } 3063 3064 err = nw.BlankLine() 3065 if err != nil { 3066 return err 3067 } 3068 } 3069 return nil 3070} 3071 3072type globalEntity interface { 3073 fullName(pkgNames map[*packageContext]string) string 3074} 3075 3076type globalEntitySorter struct { 3077 pkgNames map[*packageContext]string 3078 entities []globalEntity 3079} 3080 3081func (s *globalEntitySorter) Len() int { 3082 return len(s.entities) 3083} 3084 3085func (s *globalEntitySorter) Less(i, j int) bool { 3086 iName := s.entities[i].fullName(s.pkgNames) 3087 jName := s.entities[j].fullName(s.pkgNames) 3088 return iName < jName 3089} 3090 3091func (s *globalEntitySorter) Swap(i, j int) { 3092 s.entities[i], s.entities[j] = s.entities[j], s.entities[i] 3093} 3094 3095func (c *Context) writeGlobalVariables(nw *ninjaWriter) error { 3096 visited := make(map[Variable]bool) 3097 3098 var walk func(v Variable) error 3099 walk = func(v Variable) error { 3100 visited[v] = true 3101 3102 // First visit variables on which this variable depends. 3103 value := c.globalVariables[v] 3104 for _, dep := range value.variables { 3105 if !visited[dep] { 3106 err := walk(dep) 3107 if err != nil { 3108 return err 3109 } 3110 } 3111 } 3112 3113 err := nw.Assign(v.fullName(c.pkgNames), value.Value(c.pkgNames)) 3114 if err != nil { 3115 return err 3116 } 3117 3118 err = nw.BlankLine() 3119 if err != nil { 3120 return err 3121 } 3122 3123 return nil 3124 } 3125 3126 globalVariables := make([]globalEntity, 0, len(c.globalVariables)) 3127 for variable := range c.globalVariables { 3128 globalVariables = append(globalVariables, variable) 3129 } 3130 3131 sort.Sort(&globalEntitySorter{c.pkgNames, globalVariables}) 3132 3133 for _, entity := range globalVariables { 3134 v := entity.(Variable) 3135 if !visited[v] { 3136 err := walk(v) 3137 if err != nil { 3138 return nil 3139 } 3140 } 3141 } 3142 3143 return nil 3144} 3145 3146func (c *Context) writeGlobalPools(nw *ninjaWriter) error { 3147 globalPools := make([]globalEntity, 0, len(c.globalPools)) 3148 for pool := range c.globalPools { 3149 globalPools = append(globalPools, pool) 3150 } 3151 3152 sort.Sort(&globalEntitySorter{c.pkgNames, globalPools}) 3153 3154 for _, entity := range globalPools { 3155 pool := entity.(Pool) 3156 name := pool.fullName(c.pkgNames) 3157 def := c.globalPools[pool] 3158 err := def.WriteTo(nw, name) 3159 if err != nil { 3160 return err 3161 } 3162 3163 err = nw.BlankLine() 3164 if err != nil { 3165 return err 3166 } 3167 } 3168 3169 return nil 3170} 3171 3172func (c *Context) writeGlobalRules(nw *ninjaWriter) error { 3173 globalRules := make([]globalEntity, 0, len(c.globalRules)) 3174 for rule := range c.globalRules { 3175 globalRules = append(globalRules, rule) 3176 } 3177 3178 sort.Sort(&globalEntitySorter{c.pkgNames, globalRules}) 3179 3180 for _, entity := range globalRules { 3181 rule := entity.(Rule) 3182 name := rule.fullName(c.pkgNames) 3183 def := c.globalRules[rule] 3184 err := def.WriteTo(nw, name, c.pkgNames) 3185 if err != nil { 3186 return err 3187 } 3188 3189 err = nw.BlankLine() 3190 if err != nil { 3191 return err 3192 } 3193 } 3194 3195 return nil 3196} 3197 3198type depSorter []depInfo 3199 3200func (s depSorter) Len() int { 3201 return len(s) 3202} 3203 3204func (s depSorter) Less(i, j int) bool { 3205 iName := s[i].module.Name() 3206 jName := s[j].module.Name() 3207 if iName == jName { 3208 iName = s[i].module.variantName 3209 jName = s[j].module.variantName 3210 } 3211 return iName < jName 3212} 3213 3214func (s depSorter) Swap(i, j int) { 3215 s[i], s[j] = s[j], s[i] 3216} 3217 3218type moduleSorter struct { 3219 modules []*moduleInfo 3220 nameInterface NameInterface 3221} 3222 3223func (s moduleSorter) Len() int { 3224 return len(s.modules) 3225} 3226 3227func (s moduleSorter) Less(i, j int) bool { 3228 iMod := s.modules[i] 3229 jMod := s.modules[j] 3230 iName := s.nameInterface.UniqueName(newNamespaceContext(iMod), iMod.group.name) 3231 jName := s.nameInterface.UniqueName(newNamespaceContext(jMod), jMod.group.name) 3232 if iName == jName { 3233 iName = s.modules[i].variantName 3234 jName = s.modules[j].variantName 3235 } 3236 3237 if iName == jName { 3238 panic(fmt.Sprintf("duplicate module name: %s: %#v and %#v\n", iName, iMod, jMod)) 3239 } 3240 return iName < jName 3241} 3242 3243func (s moduleSorter) Swap(i, j int) { 3244 s.modules[i], s.modules[j] = s.modules[j], s.modules[i] 3245} 3246 3247func (c *Context) writeAllModuleActions(nw *ninjaWriter) error { 3248 headerTemplate := template.New("moduleHeader") 3249 _, err := headerTemplate.Parse(moduleHeaderTemplate) 3250 if err != nil { 3251 // This is a programming error. 3252 panic(err) 3253 } 3254 3255 modules := make([]*moduleInfo, 0, len(c.moduleInfo)) 3256 for _, module := range c.moduleInfo { 3257 modules = append(modules, module) 3258 } 3259 sort.Sort(moduleSorter{modules, c.nameInterface}) 3260 3261 buf := bytes.NewBuffer(nil) 3262 3263 for _, module := range modules { 3264 if len(module.actionDefs.variables)+len(module.actionDefs.rules)+len(module.actionDefs.buildDefs) == 0 { 3265 continue 3266 } 3267 3268 buf.Reset() 3269 3270 // In order to make the bootstrap build manifest independent of the 3271 // build dir we need to output the Blueprints file locations in the 3272 // comments as paths relative to the source directory. 3273 relPos := module.pos 3274 relPos.Filename = module.relBlueprintsFile 3275 3276 // Get the name and location of the factory function for the module. 3277 factoryFunc := runtime.FuncForPC(reflect.ValueOf(module.factory).Pointer()) 3278 factoryName := factoryFunc.Name() 3279 3280 infoMap := map[string]interface{}{ 3281 "name": module.Name(), 3282 "typeName": module.typeName, 3283 "goFactory": factoryName, 3284 "pos": relPos, 3285 "variant": module.variantName, 3286 } 3287 err = headerTemplate.Execute(buf, infoMap) 3288 if err != nil { 3289 return err 3290 } 3291 3292 err = nw.Comment(buf.String()) 3293 if err != nil { 3294 return err 3295 } 3296 3297 err = nw.BlankLine() 3298 if err != nil { 3299 return err 3300 } 3301 3302 err = c.writeLocalBuildActions(nw, &module.actionDefs) 3303 if err != nil { 3304 return err 3305 } 3306 3307 err = nw.BlankLine() 3308 if err != nil { 3309 return err 3310 } 3311 } 3312 3313 return nil 3314} 3315 3316func (c *Context) writeAllSingletonActions(nw *ninjaWriter) error { 3317 headerTemplate := template.New("singletonHeader") 3318 _, err := headerTemplate.Parse(singletonHeaderTemplate) 3319 if err != nil { 3320 // This is a programming error. 3321 panic(err) 3322 } 3323 3324 buf := bytes.NewBuffer(nil) 3325 3326 for _, info := range c.singletonInfo { 3327 if len(info.actionDefs.variables)+len(info.actionDefs.rules)+len(info.actionDefs.buildDefs) == 0 { 3328 continue 3329 } 3330 3331 // Get the name of the factory function for the module. 3332 factory := info.factory 3333 factoryFunc := runtime.FuncForPC(reflect.ValueOf(factory).Pointer()) 3334 factoryName := factoryFunc.Name() 3335 3336 buf.Reset() 3337 infoMap := map[string]interface{}{ 3338 "name": info.name, 3339 "goFactory": factoryName, 3340 } 3341 err = headerTemplate.Execute(buf, infoMap) 3342 if err != nil { 3343 return err 3344 } 3345 3346 err = nw.Comment(buf.String()) 3347 if err != nil { 3348 return err 3349 } 3350 3351 err = nw.BlankLine() 3352 if err != nil { 3353 return err 3354 } 3355 3356 err = c.writeLocalBuildActions(nw, &info.actionDefs) 3357 if err != nil { 3358 return err 3359 } 3360 3361 err = nw.BlankLine() 3362 if err != nil { 3363 return err 3364 } 3365 } 3366 3367 return nil 3368} 3369 3370func (c *Context) writeLocalBuildActions(nw *ninjaWriter, 3371 defs *localBuildActions) error { 3372 3373 // Write the local variable assignments. 3374 for _, v := range defs.variables { 3375 // A localVariable doesn't need the package names or config to 3376 // determine its name or value. 3377 name := v.fullName(nil) 3378 value, err := v.value(nil) 3379 if err != nil { 3380 panic(err) 3381 } 3382 err = nw.Assign(name, value.Value(c.pkgNames)) 3383 if err != nil { 3384 return err 3385 } 3386 } 3387 3388 if len(defs.variables) > 0 { 3389 err := nw.BlankLine() 3390 if err != nil { 3391 return err 3392 } 3393 } 3394 3395 // Write the local rules. 3396 for _, r := range defs.rules { 3397 // A localRule doesn't need the package names or config to determine 3398 // its name or definition. 3399 name := r.fullName(nil) 3400 def, err := r.def(nil) 3401 if err != nil { 3402 panic(err) 3403 } 3404 3405 err = def.WriteTo(nw, name, c.pkgNames) 3406 if err != nil { 3407 return err 3408 } 3409 3410 err = nw.BlankLine() 3411 if err != nil { 3412 return err 3413 } 3414 } 3415 3416 // Write the build definitions. 3417 for _, buildDef := range defs.buildDefs { 3418 err := buildDef.WriteTo(nw, c.pkgNames) 3419 if err != nil { 3420 return err 3421 } 3422 3423 if len(buildDef.Args) > 0 { 3424 err = nw.BlankLine() 3425 if err != nil { 3426 return err 3427 } 3428 } 3429 } 3430 3431 return nil 3432} 3433 3434func beforeInModuleList(a, b *moduleInfo, list []*moduleInfo) bool { 3435 found := false 3436 if a == b { 3437 return false 3438 } 3439 for _, l := range list { 3440 if l == a { 3441 found = true 3442 } else if l == b { 3443 return found 3444 } 3445 } 3446 3447 missing := a 3448 if found { 3449 missing = b 3450 } 3451 panic(fmt.Errorf("element %v not found in list %v", missing, list)) 3452} 3453 3454type panicError struct { 3455 panic interface{} 3456 stack []byte 3457 in string 3458} 3459 3460func newPanicErrorf(panic interface{}, in string, a ...interface{}) error { 3461 buf := make([]byte, 4096) 3462 count := runtime.Stack(buf, false) 3463 return panicError{ 3464 panic: panic, 3465 in: fmt.Sprintf(in, a...), 3466 stack: buf[:count], 3467 } 3468} 3469 3470func (p panicError) Error() string { 3471 return fmt.Sprintf("panic in %s\n%s\n%s\n", p.in, p.panic, p.stack) 3472} 3473 3474func (p *panicError) addIn(in string) { 3475 p.in += " in " + in 3476} 3477 3478func funcName(f interface{}) string { 3479 return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() 3480} 3481 3482var fileHeaderTemplate = `****************************************************************************** 3483*** This file is generated and should not be edited *** 3484****************************************************************************** 3485{{if .Pkgs}} 3486This file contains variables, rules, and pools with name prefixes indicating 3487they were generated by the following Go packages: 3488{{range .Pkgs}} 3489 {{.PkgName}} [from Go package {{.PkgPath}}]{{end}}{{end}} 3490 3491` 3492 3493var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3494Module: {{.name}} 3495Variant: {{.variant}} 3496Type: {{.typeName}} 3497Factory: {{.goFactory}} 3498Defined: {{.pos}} 3499` 3500 3501var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3502Singleton: {{.name}} 3503Factory: {{.goFactory}} 3504` 3505