// Copyright 2014 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package parser import ( "errors" "fmt" "io" "sort" "strconv" "strings" "text/scanner" ) var errTooManyErrors = errors.New("too many errors") const maxErrors = 1 type ParseError struct { Err error Pos scanner.Position } func (e *ParseError) Error() string { return fmt.Sprintf("%s: %s", e.Pos, e.Err) } type File struct { Name string Defs []Definition Comments []Comment } func parse(p *parser) (file *File, errs []error) { defer func() { if r := recover(); r != nil { if r == errTooManyErrors { errs = p.errors return } panic(r) } }() defs := p.parseDefinitions() p.accept(scanner.EOF) errs = p.errors comments := p.comments return &File{ Name: p.scanner.Filename, Defs: defs, Comments: comments, }, errs } func ParseAndEval(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { p := newParser(r, scope) p.eval = true p.scanner.Filename = filename return parse(p) } func Parse(filename string, r io.Reader, scope *Scope) (file *File, errs []error) { p := newParser(r, scope) p.scanner.Filename = filename return parse(p) } type parser struct { scanner scanner.Scanner tok rune errors []error scope *Scope comments []Comment eval bool } func newParser(r io.Reader, scope *Scope) *parser { p := &parser{} p.scope = scope p.scanner.Init(r) p.scanner.Error = func(sc *scanner.Scanner, msg string) { p.errorf(msg) } p.scanner.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments p.next() return p } func (p *parser) error(err error) { pos := p.scanner.Position if !pos.IsValid() { pos = p.scanner.Pos() } err = &ParseError{ Err: err, Pos: pos, } p.errors = append(p.errors, err) if len(p.errors) >= maxErrors { panic(errTooManyErrors) } } func (p *parser) errorf(format string, args ...interface{}) { p.error(fmt.Errorf(format, args...)) } func (p *parser) accept(toks ...rune) bool { for _, tok := range toks { if p.tok != tok { p.errorf("expected %s, found %s", scanner.TokenString(tok), scanner.TokenString(p.tok)) return false } p.next() } return true } func (p *parser) next() { if p.tok != scanner.EOF { p.tok = p.scanner.Scan() for p.tok == scanner.Comment { lines := strings.Split(p.scanner.TokenText(), "\n") p.comments = append(p.comments, Comment{lines, p.scanner.Position}) p.tok = p.scanner.Scan() } } return } func (p *parser) parseDefinitions() (defs []Definition) { for { switch p.tok { case scanner.Ident: ident := p.scanner.TokenText() pos := p.scanner.Position p.accept(scanner.Ident) switch p.tok { case '+': p.accept('+') defs = append(defs, p.parseAssignment(ident, pos, "+=")) case '=': defs = append(defs, p.parseAssignment(ident, pos, "=")) case '{', '(': defs = append(defs, p.parseModule(ident, pos)) default: p.errorf("expected \"=\" or \"+=\" or \"{\" or \"(\", found %s", scanner.TokenString(p.tok)) } case scanner.EOF: return default: p.errorf("expected assignment or module definition, found %s", scanner.TokenString(p.tok)) return } } } func (p *parser) parseAssignment(name string, namePos scanner.Position, assigner string) (assignment *Assignment) { assignment = new(Assignment) pos := p.scanner.Position if !p.accept('=') { return } value := p.parseExpression() assignment.Name = Ident{name, namePos} assignment.Value = value assignment.OrigValue = value assignment.Pos = pos assignment.Assigner = assigner if p.scope != nil { if assigner == "+=" { if old, local := p.scope.Get(assignment.Name.Name); old == nil { p.errorf("modified non-existent variable %q with +=", assignment.Name.Name) } else if !local { p.errorf("modified non-local variable %q with +=", assignment.Name.Name) } else if old.Referenced { p.errorf("modified variable %q with += after referencing", assignment.Name.Name) } else { val, err := p.evaluateOperator(old.Value, assignment.Value, '+', assignment.Pos) if err != nil { p.error(err) } else { old.Value = val } } } else { err := p.scope.Add(assignment) if err != nil { p.error(err) } } } return } func (p *parser) parseModule(typ string, typPos scanner.Position) (module *Module) { module = new(Module) compat := false lbracePos := p.scanner.Position if p.tok == '{' { compat = true } if !p.accept(p.tok) { return } properties := p.parsePropertyList(true, compat) rbracePos := p.scanner.Position if !compat { p.accept(')') } else { p.accept('}') } module.Type = Ident{typ, typPos} module.Properties = properties module.LbracePos = lbracePos module.RbracePos = rbracePos return } func (p *parser) parsePropertyList(isModule, compat bool) (properties []*Property) { for p.tok == scanner.Ident { property := p.parseProperty(isModule, compat) properties = append(properties, property) if p.tok != ',' { // There was no comma, so the list is done. break } p.accept(',') } return } func (p *parser) parseProperty(isModule, compat bool) (property *Property) { property = new(Property) name := p.scanner.TokenText() namePos := p.scanner.Position p.accept(scanner.Ident) pos := p.scanner.Position if isModule { if compat && p.tok == ':' { p.accept(':') } else { if !p.accept('=') { return } } } else { if !p.accept(':') { return } } value := p.parseExpression() property.Name = Ident{name, namePos} property.Value = value property.Pos = pos return } func (p *parser) parseExpression() (value Value) { value = p.parseValue() switch p.tok { case '+': return p.parseOperator(value) default: return value } } func (p *parser) evaluateOperator(value1, value2 Value, operator rune, pos scanner.Position) (Value, error) { value := Value{} if p.eval { if value1.Type != value2.Type { return Value{}, fmt.Errorf("mismatched type in operator %c: %s != %s", operator, value1.Type, value2.Type) } value = value1 value.Variable = "" switch operator { case '+': switch value1.Type { case String: value.StringValue = value1.StringValue + value2.StringValue case List: value.ListValue = append([]Value{}, value1.ListValue...) value.ListValue = append(value.ListValue, value2.ListValue...) case Map: var err error value.MapValue, err = p.addMaps(value.MapValue, value2.MapValue, pos) if err != nil { return Value{}, err } default: return Value{}, fmt.Errorf("operator %c not supported on type %s", operator, value1.Type) } default: panic("unknown operator " + string(operator)) } } value.Expression = &Expression{ Args: [2]Value{value1, value2}, Operator: operator, Pos: pos, } return value, nil } func (p *parser) addMaps(map1, map2 []*Property, pos scanner.Position) ([]*Property, error) { ret := make([]*Property, 0, len(map1)) inMap1 := make(map[string]*Property) inMap2 := make(map[string]*Property) inBoth := make(map[string]*Property) for _, prop1 := range map1 { inMap1[prop1.Name.Name] = prop1 } for _, prop2 := range map2 { inMap2[prop2.Name.Name] = prop2 if _, ok := inMap1[prop2.Name.Name]; ok { inBoth[prop2.Name.Name] = prop2 } } for _, prop1 := range map1 { if prop2, ok := inBoth[prop1.Name.Name]; ok { var err error newProp := *prop1 newProp.Value, err = p.evaluateOperator(prop1.Value, prop2.Value, '+', pos) if err != nil { return nil, err } ret = append(ret, &newProp) } else { ret = append(ret, prop1) } } for _, prop2 := range map2 { if _, ok := inBoth[prop2.Name.Name]; !ok { ret = append(ret, prop2) } } return ret, nil } func (p *parser) parseOperator(value1 Value) Value { operator := p.tok pos := p.scanner.Position p.accept(operator) value2 := p.parseExpression() value, err := p.evaluateOperator(value1, value2, operator, pos) if err != nil { p.error(err) return Value{} } return value } func (p *parser) parseValue() (value Value) { switch p.tok { case scanner.Ident: return p.parseVariable() case scanner.String: return p.parseStringValue() case '[': return p.parseListValue() case '{': return p.parseMapValue() default: p.errorf("expected bool, list, or string value; found %s", scanner.TokenString(p.tok)) return } } func (p *parser) parseVariable() (value Value) { switch text := p.scanner.TokenText(); text { case "true": value.Type = Bool value.BoolValue = true case "false": value.Type = Bool value.BoolValue = false default: variable := p.scanner.TokenText() if p.eval { if assignment, local := p.scope.Get(variable); assignment == nil { p.errorf("variable %q is not set", variable) } else { if local { assignment.Referenced = true } value = assignment.Value } } value.Variable = variable } value.Pos = p.scanner.Position p.accept(scanner.Ident) return } func (p *parser) parseStringValue() (value Value) { value.Type = String value.Pos = p.scanner.Position str, err := strconv.Unquote(p.scanner.TokenText()) if err != nil { p.errorf("couldn't parse string: %s", err) return } value.StringValue = str p.accept(scanner.String) return } func (p *parser) parseListValue() (value Value) { value.Type = List value.Pos = p.scanner.Position if !p.accept('[') { return } var elements []Value for p.tok != ']' { element := p.parseExpression() if p.eval && element.Type != String { p.errorf("Expected string in list, found %s", element.String()) return } elements = append(elements, element) if p.tok != ',' { // There was no comma, so the list is done. break } p.accept(',') } value.ListValue = elements value.EndPos = p.scanner.Position p.accept(']') return } func (p *parser) parseMapValue() (value Value) { value.Type = Map value.Pos = p.scanner.Position if !p.accept('{') { return } properties := p.parsePropertyList(false, false) value.MapValue = properties value.EndPos = p.scanner.Position p.accept('}') return } type Expression struct { Args [2]Value Operator rune Pos scanner.Position } func (e *Expression) Copy() *Expression { ret := *e ret.Args[0] = e.Args[0].Copy() ret.Args[1] = e.Args[1].Copy() return &ret } func (e *Expression) String() string { return fmt.Sprintf("(%s %c %s)@%d:%s", e.Args[0].String(), e.Operator, e.Args[1].String(), e.Pos.Offset, e.Pos) } type ValueType int const ( Bool ValueType = iota String List Map ) func (p ValueType) String() string { switch p { case Bool: return "bool" case String: return "string" case List: return "list" case Map: return "map" default: panic(fmt.Errorf("unknown value type: %d", p)) } } type Definition interface { String() string definitionTag() } type Assignment struct { Name Ident Value Value OrigValue Value Pos scanner.Position Assigner string Referenced bool } func (a *Assignment) String() string { return fmt.Sprintf("%s@%d:%s %s %s", a.Name, a.Pos.Offset, a.Pos, a.Assigner, a.Value) } func (a *Assignment) definitionTag() {} type Module struct { Type Ident Properties []*Property LbracePos scanner.Position RbracePos scanner.Position } func (m *Module) Copy() *Module { ret := *m ret.Properties = make([]*Property, len(m.Properties)) for i := range m.Properties { ret.Properties[i] = m.Properties[i].Copy() } return &ret } func (m *Module) String() string { propertyStrings := make([]string, len(m.Properties)) for i, property := range m.Properties { propertyStrings[i] = property.String() } return fmt.Sprintf("%s@%d:%s-%d:%s{%s}", m.Type, m.LbracePos.Offset, m.LbracePos, m.RbracePos.Offset, m.RbracePos, strings.Join(propertyStrings, ", ")) } func (m *Module) definitionTag() {} type Property struct { Name Ident Value Value Pos scanner.Position } func (p *Property) Copy() *Property { ret := *p ret.Value = p.Value.Copy() return &ret } func (p *Property) String() string { return fmt.Sprintf("%s@%d:%s: %s", p.Name, p.Pos.Offset, p.Pos, p.Value) } type Ident struct { Name string Pos scanner.Position } func (i Ident) String() string { return fmt.Sprintf("%s@%d:%s", i.Name, i.Pos.Offset, i.Pos) } type Value struct { Type ValueType BoolValue bool StringValue string ListValue []Value MapValue []*Property Expression *Expression Variable string Pos scanner.Position EndPos scanner.Position } func (p Value) Copy() Value { ret := p if p.MapValue != nil { ret.MapValue = make([]*Property, len(p.MapValue)) for i := range p.MapValue { ret.MapValue[i] = p.MapValue[i].Copy() } } if p.ListValue != nil { ret.ListValue = make([]Value, len(p.ListValue)) for i := range p.ListValue { ret.ListValue[i] = p.ListValue[i].Copy() } } if p.Expression != nil { ret.Expression = p.Expression.Copy() } return ret } func (p Value) String() string { var s string if p.Variable != "" { s += p.Variable + " = " } if p.Expression != nil { s += p.Expression.String() } switch p.Type { case Bool: s += fmt.Sprintf("%t@%d:%s", p.BoolValue, p.Pos.Offset, p.Pos) case String: s += fmt.Sprintf("%q@%d:%s", p.StringValue, p.Pos.Offset, p.Pos) case List: valueStrings := make([]string, len(p.ListValue)) for i, value := range p.ListValue { valueStrings[i] = value.String() } s += fmt.Sprintf("@%d:%s-%d:%s[%s]", p.Pos.Offset, p.Pos, p.EndPos.Offset, p.EndPos, strings.Join(valueStrings, ", ")) case Map: propertyStrings := make([]string, len(p.MapValue)) for i, property := range p.MapValue { propertyStrings[i] = property.String() } s += fmt.Sprintf("@%d:%s-%d:%s{%s}", p.Pos.Offset, p.Pos, p.EndPos.Offset, p.EndPos, strings.Join(propertyStrings, ", ")) default: panic(fmt.Errorf("bad property type: %d", p.Type)) } return s } type Scope struct { vars map[string]*Assignment inheritedVars map[string]*Assignment } func NewScope(s *Scope) *Scope { newScope := &Scope{ vars: make(map[string]*Assignment), inheritedVars: make(map[string]*Assignment), } if s != nil { for k, v := range s.vars { newScope.inheritedVars[k] = v } for k, v := range s.inheritedVars { newScope.inheritedVars[k] = v } } return newScope } func (s *Scope) Add(assignment *Assignment) error { if old, ok := s.vars[assignment.Name.Name]; ok { return fmt.Errorf("variable already set, previous assignment: %s", old) } if old, ok := s.inheritedVars[assignment.Name.Name]; ok { return fmt.Errorf("variable already set in inherited scope, previous assignment: %s", old) } s.vars[assignment.Name.Name] = assignment return nil } func (s *Scope) Remove(name string) { delete(s.vars, name) delete(s.inheritedVars, name) } func (s *Scope) Get(name string) (*Assignment, bool) { if a, ok := s.vars[name]; ok { return a, true } if a, ok := s.inheritedVars[name]; ok { return a, false } return nil, false } func (s *Scope) String() string { vars := []string{} for k := range s.vars { vars = append(vars, k) } for k := range s.inheritedVars { vars = append(vars, k) } sort.Strings(vars) ret := []string{} for _, v := range vars { if assignment, ok := s.vars[v]; ok { ret = append(ret, assignment.String()) } else { ret = append(ret, s.inheritedVars[v].String()) } } return strings.Join(ret, "\n") } type Comment struct { Comment []string Pos scanner.Position } func (c Comment) String() string { l := 0 for _, comment := range c.Comment { l += len(comment) + 1 } buf := make([]byte, 0, l) for _, comment := range c.Comment { buf = append(buf, comment...) buf = append(buf, '\n') } return string(buf) } // Return the text of the comment with // or /* and */ stripped func (c Comment) Text() string { l := 0 for _, comment := range c.Comment { l += len(comment) + 1 } buf := make([]byte, 0, l) blockComment := false if strings.HasPrefix(c.Comment[0], "/*") { blockComment = true } for i, comment := range c.Comment { if blockComment { if i == 0 { comment = strings.TrimPrefix(comment, "/*") } if i == len(c.Comment)-1 { comment = strings.TrimSuffix(comment, "*/") } } else { comment = strings.TrimPrefix(comment, "//") } buf = append(buf, comment...) buf = append(buf, '\n') } return string(buf) } // Return the line number that the comment ends on func (c Comment) EndLine() int { return c.Pos.Line + len(c.Comment) - 1 }