+23

cue/parser/doc.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // Package parser implements a parser for CUE source files. Input may be 16 16 + // provided in a variety of forms (see the various Parse* functions); the output 17 17 + // is an abstract syntax tree (AST) representing the CUE source. The parser is 18 18 + // invoked through one of the Parse* functions. 19 19 + // 20 20 + // The parser accepts a larger language than is syntactically permitted by the 21 21 + // CUE spec, for simplicity, and for improved robustness in the presence of 22 22 + // syntax errors. 23 23 + package parser // import "cuelang.org/go/cue/parser"

+198

cue/parser/error_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // This file implements a parser test harness. The files in the testdata 16 16 + // directory are parsed and the errors reported are compared against the 17 17 + // error messages expected in the test files. The test files must end in 18 18 + // .src rather than .go so that they are not disturbed by gofmt runs. 19 19 + // 20 20 + // Expected errors are indicated in the test files by putting a comment 21 21 + // of the form /* ERROR "rx" */ immediately following an offending 22 22 + // The harness will verify that an error matching the regular expression 23 23 + // rx is reported at that source position. 24 24 + // 25 25 + // For instance, the following test file indicates that a "not declared" 26 26 + // error should be reported for the undeclared variable x: 27 27 + // 28 28 + // package p 29 29 + // { 30 30 + // a = x /* ERROR "not declared" */ + 1 31 31 + // } 32 32 + 33 33 + package parser 34 34 + 35 35 + import ( 36 36 + "io/ioutil" 37 37 + "path/filepath" 38 38 + "regexp" 39 39 + "strings" 40 40 + "testing" 41 41 + 42 42 + "cuelang.org/go/cue/errors" 43 43 + "cuelang.org/go/cue/scanner" 44 44 + "cuelang.org/go/cue/token" 45 45 + ) 46 46 + 47 47 + const testdata = "testdata" 48 48 + 49 49 + // getFile assumes that each filename occurs at most once 50 50 + func getFile(fset *token.FileSet, filename string) (info *token.File) { 51 51 + fset.Iterate(func(f *token.File) bool { 52 52 + if f.Name() == filename { 53 53 + if info != nil { 54 54 + panic(filename + " used multiple times") 55 55 + } 56 56 + info = f 57 57 + } 58 58 + return true 59 59 + }) 60 60 + return info 61 61 + } 62 62 + 63 63 + func getPos(fset *token.FileSet, filename string, offset int) token.Pos { 64 64 + if f := getFile(fset, filename); f != nil { 65 65 + return f.Pos(offset, 0) 66 66 + } 67 67 + return token.NoPos 68 68 + } 69 69 + 70 70 + // ERROR comments must be of the form /* ERROR "rx" */ and rx is 71 71 + // a regular expression that matches the expected error message. 72 72 + // The special form /* ERROR HERE "rx" */ must be used for error 73 73 + // messages that appear immediately after a token, rather than at 74 74 + // a token's position. 75 75 + // 76 76 + var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE)? *"([^"]*)" *\*/$`) 77 77 + 78 78 + // expectedErrors collects the regular expressions of ERROR comments found 79 79 + // in files and returns them as a map of error positions to error messages. 80 80 + // 81 81 + func expectedErrors(t *testing.T, fset *token.FileSet, filename string, src []byte) map[token.Pos]string { 82 82 + errors := make(map[token.Pos]string) 83 83 + 84 84 + var s scanner.Scanner 85 85 + // file was parsed already - do not add it again to the file 86 86 + // set otherwise the position information returned here will 87 87 + // not match the position information collected by the parser 88 88 + s.Init(getFile(fset, filename), src, nil, scanner.ScanComments) 89 89 + var prev token.Pos // position of last non-comment, non-semicolon token 90 90 + var here token.Pos // position immediately after the token at position prev 91 91 + 92 92 + for { 93 93 + pos, tok, lit := s.Scan() 94 94 + pos = pos.WithRel(0) 95 95 + switch tok { 96 96 + case token.EOF: 97 97 + return errors 98 98 + case token.COMMENT: 99 99 + s := errRx.FindStringSubmatch(lit) 100 100 + if len(s) == 3 { 101 101 + pos := prev 102 102 + if s[1] == "HERE" { 103 103 + pos = here 104 104 + } 105 105 + errors[pos] = string(s[2]) 106 106 + } 107 107 + default: 108 108 + prev = pos 109 109 + var l int // token length 110 110 + if tok.IsLiteral() { 111 111 + l = len(lit) 112 112 + } else { 113 113 + l = len(tok.String()) 114 114 + } 115 115 + here = prev + token.Pos(l) 116 116 + } 117 117 + } 118 118 + } 119 119 + 120 120 + // compareErrors compares the map of expected error messages with the list 121 121 + // of found errors and reports discrepancies. 122 122 + // 123 123 + func compareErrors(t *testing.T, fset *token.FileSet, expected map[token.Pos]string, found errors.List) { 124 124 + t.Helper() 125 125 + for _, error := range found { 126 126 + // error.Pos is a Position, but we want 127 127 + // a Pos so we can do a map lookup 128 128 + ePos := error.Position() 129 129 + eMsg := error.Error() 130 130 + pos := getPos(fset, ePos.Filename, ePos.Offset).WithRel(0) 131 131 + if msg, found := expected[pos]; found { 132 132 + // we expect a message at pos; check if it matches 133 133 + rx, err := regexp.Compile(msg) 134 134 + if err != nil { 135 135 + t.Errorf("%s: %v", ePos, err) 136 136 + continue 137 137 + } 138 138 + if match := rx.MatchString(eMsg); !match { 139 139 + t.Errorf("%s: %q does not match %q", ePos, eMsg, msg) 140 140 + continue 141 141 + } 142 142 + // we have a match - eliminate this error 143 143 + delete(expected, pos) 144 144 + } else { 145 145 + // To keep in mind when analyzing failed test output: 146 146 + // If the same error position occurs multiple times in errors, 147 147 + // this message will be triggered (because the first error at 148 148 + // the position removes this position from the expected errors). 149 149 + t.Errorf("%s: unexpected error: -%q-", ePos, eMsg) 150 150 + } 151 151 + } 152 152 + 153 153 + // there should be no expected errors left 154 154 + if len(expected) > 0 { 155 155 + t.Errorf("%d errors not reported:", len(expected)) 156 156 + for pos, msg := range expected { 157 157 + t.Errorf("%s: -%q-\n", fset.Position(pos), msg) 158 158 + } 159 159 + } 160 160 + } 161 161 + 162 162 + func checkErrors(t *testing.T, filename string, input interface{}) { 163 163 + t.Helper() 164 164 + src, err := readSource(filename, input) 165 165 + if err != nil { 166 166 + t.Error(err) 167 167 + return 168 168 + } 169 169 + 170 170 + fset := token.NewFileSet() 171 171 + _, err = ParseFile(fset, filename, src, DeclarationErrors, AllErrors, ParseLambdas) 172 172 + found, ok := err.(errors.List) 173 173 + if err != nil && !ok { 174 174 + t.Error(err) 175 175 + return 176 176 + } 177 177 + found.RemoveMultiples() 178 178 + 179 179 + // we are expecting the following errors 180 180 + // (collect these after parsing a file so that it is found in the file set) 181 181 + expected := expectedErrors(t, fset, filename, src) 182 182 + 183 183 + // verify errors returned by the parser 184 184 + compareErrors(t, fset, expected, found) 185 185 + } 186 186 + 187 187 + func TestErrors(t *testing.T) { 188 188 + list, err := ioutil.ReadDir(testdata) 189 189 + if err != nil { 190 190 + t.Fatal(err) 191 191 + } 192 192 + for _, fi := range list { 193 193 + name := fi.Name() 194 194 + if !fi.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".src") { 195 195 + checkErrors(t, filepath.Join(testdata, name), nil) 196 196 + } 197 197 + } 198 198 + }

+41

cue/parser/example_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser_test 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + 20 20 + "cuelang.org/go/cue/parser" 21 21 + "cuelang.org/go/cue/token" 22 22 + ) 23 23 + 24 24 + func ExampleParseFile() { 25 25 + fset := token.NewFileSet() // positions are relative to fset 26 26 + 27 27 + // Parse the file containing this very example 28 28 + // but stop after processing the imports. 29 29 + f, err := parser.ParseFile(fset, "testdata/test.cue", nil) 30 30 + if err != nil { 31 31 + fmt.Println(err) 32 32 + return 33 33 + } 34 34 + 35 35 + // Print the imports from the file's AST. 36 36 + for _, s := range f.Imports { 37 37 + fmt.Println(s.Path.Value) 38 38 + } 39 39 + // Output: 40 40 + // "math" 41 41 + }

+169

cue/parser/import.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "sort" 19 19 + "strconv" 20 20 + 21 21 + "cuelang.org/go/cue/ast" 22 22 + "cuelang.org/go/cue/token" 23 23 + ) 24 24 + 25 25 + // sortImports sorts runs of consecutive import lines in import blocks in f. 26 26 + // It also removes duplicate imports when it is possible to do so without data loss. 27 27 + func sortImports(fset *token.FileSet, f *ast.File) { 28 28 + for _, d := range f.Decls { 29 29 + d, ok := d.(*ast.ImportDecl) 30 30 + if !ok { 31 31 + // Not an import declaration, so we're done. 32 32 + // Imports are always first. 33 33 + break 34 34 + } 35 35 + 36 36 + if !d.Lparen.IsValid() { 37 37 + // Not a block: sorted by default. 38 38 + continue 39 39 + } 40 40 + 41 41 + // Identify and sort runs of specs on successive lines. 42 42 + i := 0 43 43 + specs := d.Specs[:0] 44 44 + for j, s := range d.Specs { 45 45 + if j > i && fset.Position(s.Pos()).Line > 1+fset.Position(d.Specs[j-1].End()).Line { 46 46 + // j begins a new run. End this one. 47 47 + specs = append(specs, sortSpecs(fset, f, d.Specs[i:j])...) 48 48 + i = j 49 49 + } 50 50 + } 51 51 + specs = append(specs, sortSpecs(fset, f, d.Specs[i:])...) 52 52 + d.Specs = specs 53 53 + 54 54 + // Deduping can leave a blank line before the rparen; clean that up. 55 55 + if len(d.Specs) > 0 { 56 56 + lastSpec := d.Specs[len(d.Specs)-1] 57 57 + lastLine := fset.Position(lastSpec.Pos()).Line 58 58 + rParenLine := fset.Position(d.Rparen).Line 59 59 + for rParenLine > lastLine+1 { 60 60 + rParenLine-- 61 61 + fset.File(d.Rparen).MergeLine(rParenLine) 62 62 + } 63 63 + } 64 64 + } 65 65 + } 66 66 + 67 67 + func importPath(s *ast.ImportSpec) string { 68 68 + t, err := strconv.Unquote(s.Path.Value) 69 69 + if err == nil { 70 70 + return t 71 71 + } 72 72 + return "" 73 73 + } 74 74 + 75 75 + func importName(s *ast.ImportSpec) string { 76 76 + n := s.Name 77 77 + if n == nil { 78 78 + return "" 79 79 + } 80 80 + return n.Name 81 81 + } 82 82 + 83 83 + func importComment(s *ast.ImportSpec) string { 84 84 + for _, c := range s.Comments() { 85 85 + if c.Line { 86 86 + return c.Text() 87 87 + } 88 88 + } 89 89 + return "" 90 90 + } 91 91 + 92 92 + // collapse indicates whether prev may be removed, leaving only next. 93 93 + func collapse(prev, next *ast.ImportSpec) bool { 94 94 + if importPath(next) != importPath(prev) || importName(next) != importName(prev) { 95 95 + return false 96 96 + } 97 97 + for _, c := range prev.Comments() { 98 98 + if !c.Doc { 99 99 + return false 100 100 + } 101 101 + } 102 102 + return true 103 103 + } 104 104 + 105 105 + type posSpan struct { 106 106 + Start token.Pos 107 107 + End token.Pos 108 108 + } 109 109 + 110 110 + func sortSpecs(fset *token.FileSet, f *ast.File, specs []*ast.ImportSpec) []*ast.ImportSpec { 111 111 + // Can't short-circuit here even if specs are already sorted, 112 112 + // since they might yet need deduplication. 113 113 + // A lone import, however, may be safely ignored. 114 114 + if len(specs) <= 1 { 115 115 + return specs 116 116 + } 117 117 + 118 118 + // Record positions for specs. 119 119 + pos := make([]posSpan, len(specs)) 120 120 + for i, s := range specs { 121 121 + pos[i] = posSpan{s.Pos(), s.End()} 122 122 + } 123 123 + 124 124 + // Sort the import specs by import path. 125 125 + // Remove duplicates, when possible without data loss. 126 126 + // Reassign the import paths to have the same position sequence. 127 127 + // Reassign each comment to abut the end of its spec. 128 128 + // Sort the comments by new position. 129 129 + sort.Sort(byImportSpec(specs)) 130 130 + 131 131 + // Dedup. Thanks to our sorting, we can just consider 132 132 + // adjacent pairs of imports. 133 133 + deduped := specs[:0] 134 134 + for i, s := range specs { 135 135 + if i == len(specs)-1 || !collapse(s, specs[i+1]) { 136 136 + deduped = append(deduped, s) 137 137 + } else { 138 138 + p := s.Pos() 139 139 + fset.File(p).MergeLine(fset.Position(p).Line) 140 140 + } 141 141 + } 142 142 + specs = deduped 143 143 + 144 144 + return specs 145 145 + } 146 146 + 147 147 + type byImportSpec []*ast.ImportSpec 148 148 + 149 149 + func (x byImportSpec) Len() int { return len(x) } 150 150 + func (x byImportSpec) Swap(i, j int) { x[i], x[j] = x[j], x[i] } 151 151 + func (x byImportSpec) Less(i, j int) bool { 152 152 + ipath := importPath(x[i]) 153 153 + jpath := importPath(x[j]) 154 154 + if ipath != jpath { 155 155 + return ipath < jpath 156 156 + } 157 157 + iname := importName(x[i]) 158 158 + jname := importName(x[j]) 159 159 + if iname != jname { 160 160 + return iname < jname 161 161 + } 162 162 + return importComment(x[i]) < importComment(x[j]) 163 163 + } 164 164 + 165 165 + type byCommentPos []*ast.CommentGroup 166 166 + 167 167 + func (x byCommentPos) Len() int { return len(x) } 168 168 + func (x byCommentPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] } 169 169 + func (x byCommentPos) Less(i, j int) bool { return x[i].Pos() < x[j].Pos() }

+250

cue/parser/interface.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // This file contains the exported entry points for invoking the 16 16 + 17 17 + package parser 18 18 + 19 19 + import ( 20 20 + "bytes" 21 21 + "fmt" 22 22 + "io" 23 23 + "io/ioutil" 24 24 + 25 25 + "cuelang.org/go/cue/ast" 26 26 + "cuelang.org/go/cue/token" 27 27 + ) 28 28 + 29 29 + // If src != nil, readSource converts src to a []byte if possible; 30 30 + // otherwise it returns an error. If src == nil, readSource returns 31 31 + // the result of reading the file specified by filename. 32 32 + // 33 33 + func readSource(filename string, src interface{}) ([]byte, error) { 34 34 + if src != nil { 35 35 + switch s := src.(type) { 36 36 + case string: 37 37 + return []byte(s), nil 38 38 + case []byte: 39 39 + return s, nil 40 40 + case *bytes.Buffer: 41 41 + // is io.Reader, but src is already available in []byte form 42 42 + if s != nil { 43 43 + return s.Bytes(), nil 44 44 + } 45 45 + case io.Reader: 46 46 + var buf bytes.Buffer 47 47 + if _, err := io.Copy(&buf, s); err != nil { 48 48 + return nil, err 49 49 + } 50 50 + return buf.Bytes(), nil 51 51 + } 52 52 + return nil, fmt.Errorf("invalid source type %T", src) 53 53 + } 54 54 + return ioutil.ReadFile(filename) 55 55 + } 56 56 + 57 57 + type Option func(p *parser) 58 58 + 59 59 + var ( 60 60 + // PackageClauseOnly causes parsing to stop after the package clause. 61 61 + PackageClauseOnly Option = packageClauseOnly 62 62 + packageClauseOnly = func(p *parser) { 63 63 + p.mode |= packageClauseOnlyMode 64 64 + } 65 65 + 66 66 + // ImportsOnly causes parsing to stop parsing after the import declarations. 67 67 + ImportsOnly Option = importsOnly 68 68 + importsOnly = func(p *parser) { 69 69 + p.mode |= importsOnlyMode 70 70 + } 71 71 + 72 72 + // ParseComments causes comments to be parsed. 73 73 + ParseComments Option = parseComments 74 74 + parseComments = func(p *parser) { 75 75 + p.mode |= parseCommentsMode 76 76 + } 77 77 + 78 78 + // ParseLambdas enables parsing of Lambdas. By default these are disabled. 79 79 + // 80 80 + // NOTE: this option is for internal use only and can be made unavailable at 81 81 + // any time. 82 82 + ParseLambdas Option = parseLambdas 83 83 + parseLambdas = func(p *parser) { 84 84 + p.mode |= parseLambdasMode 85 85 + } 86 86 + 87 87 + // Trace causes parsing to print a trace of parsed productions. 88 88 + Trace Option = traceOpt 89 89 + traceOpt = func(p *parser) { 90 90 + p.mode |= traceMode 91 91 + } 92 92 + 93 93 + // DeclarationErrors causes parsing to report declaration errors. 94 94 + DeclarationErrors Option = declarationErrors 95 95 + declarationErrors = func(p *parser) { 96 96 + p.mode |= declarationErrorsMode 97 97 + } 98 98 + 99 99 + // AllErrors causes all errors to be reported (not just the first 10 on different lines). 100 100 + AllErrors Option = allErrors 101 101 + allErrors = func(p *parser) { 102 102 + p.mode |= allErrorsMode 103 103 + } 104 104 + 105 105 + // AllowPartial allows the parser to be used on a prefix buffer. 106 106 + AllowPartial Option = allowPartial 107 107 + allowPartial = func(p *parser) { 108 108 + p.mode |= partialMode 109 109 + } 110 110 + ) 111 111 + 112 112 + // A mode value is a set of flags (or 0). 113 113 + // They control the amount of source code parsed and other optional 114 114 + // parser functionality. 115 115 + type mode uint 116 116 + 117 117 + const ( 118 118 + packageClauseOnlyMode mode = 1 << iota // stop parsing after package clause 119 119 + importsOnlyMode // stop parsing after import declarations 120 120 + parseCommentsMode // parse comments and add them to AST 121 121 + parseLambdasMode 122 122 + partialMode 123 123 + traceMode // print a trace of parsed productions 124 124 + declarationErrorsMode // report declaration errors 125 125 + allErrorsMode // report all errors (not just the first 10 on different lines) 126 126 + ) 127 127 + 128 128 + // ParseFile parses the source code of a single CUE source file and returns 129 129 + // the corresponding File node. The source code may be provided via 130 130 + // the filename of the source file, or via the src parameter. 131 131 + // 132 132 + // If src != nil, ParseFile parses the source from src and the filename is 133 133 + // only used when recording position information. The type of the argument 134 134 + // for the src parameter must be string, []byte, or io.Reader. 135 135 + // If src == nil, ParseFile parses the file specified by filename. 136 136 + // 137 137 + // The mode parameter controls the amount of source text parsed and other 138 138 + // optional parser functionality. Position information is recorded in the 139 139 + // file set fset, which must not be nil. 140 140 + // 141 141 + // If the source couldn't be read, the returned AST is nil and the error 142 142 + // indicates the specific failure. If the source was read but syntax 143 143 + // errors were found, the result is a partial AST (with Bad* nodes 144 144 + // representing the fragments of erroneous source code). Multiple errors 145 145 + // are returned via a ErrorList which is sorted by file position. 146 146 + func ParseFile(p *token.FileSet, filename string, src interface{}, mode ...Option) (f *ast.File, err error) { 147 147 + if p == nil { 148 148 + panic("ParseFile: no file.FileSet provided (fset == nil)") 149 149 + } 150 150 + 151 151 + // get source 152 152 + text, err := readSource(filename, src) 153 153 + if err != nil { 154 154 + return nil, err 155 155 + } 156 156 + 157 157 + var pp parser 158 158 + defer func() { 159 159 + if e := recover(); e != nil { 160 160 + // resume same panic if it's not a bailout 161 161 + if _, ok := e.(bailout); !ok { 162 162 + panic(e) 163 163 + } 164 164 + } 165 165 + 166 166 + // set result values 167 167 + if f == nil { 168 168 + // source is not a valid Go source file - satisfy 169 169 + // ParseFile API and return a valid (but) empty 170 170 + // *File 171 171 + f = &ast.File{ 172 172 + Name: new(ast.Ident), 173 173 + // Scope: NewScope(nil), 174 174 + } 175 175 + } 176 176 + 177 177 + pp.errors.Sort() 178 178 + err = pp.errors.Err() 179 179 + }() 180 180 + 181 181 + // parse source 182 182 + pp.init(p, filename, text, mode) 183 183 + f = pp.parseFile() 184 184 + if f == nil { 185 185 + return nil, pp.errors 186 186 + } 187 187 + f.Filename = filename 188 188 + resolve(f, pp.error) 189 189 + 190 190 + return 191 191 + } 192 192 + 193 193 + // ParseExpr is a convenience function for parsing an expression. 194 194 + // The arguments have the same meaning as for Parse, but the source must 195 195 + // be a valid CUE (type or value) expression. Specifically, fset must not 196 196 + // be nil. 197 197 + func ParseExpr(fset *token.FileSet, filename string, src interface{}, mode ...Option) (ast.Expr, error) { 198 198 + if fset == nil { 199 199 + panic("ParseExprFrom: no file.FileSet provided (fset == nil)") 200 200 + } 201 201 + 202 202 + // get source 203 203 + text, err := readSource(filename, src) 204 204 + if err != nil { 205 205 + return nil, err 206 206 + } 207 207 + 208 208 + var p parser 209 209 + defer func() { 210 210 + if e := recover(); e != nil { 211 211 + // resume same panic if it's not a bailout 212 212 + if _, ok := e.(bailout); !ok { 213 213 + panic(e) 214 214 + } 215 215 + } 216 216 + p.errors.Sort() 217 217 + err = p.errors.Err() 218 218 + }() 219 219 + 220 220 + // parse expr 221 221 + p.init(fset, filename, text, mode) 222 222 + // Set up pkg-level scopes to avoid nil-pointer errors. 223 223 + // This is not needed for a correct expression x as the 224 224 + // parser will be ok with a nil topScope, but be cautious 225 225 + // in case of an erroneous x. 226 226 + e := p.parseRHS() 227 227 + 228 228 + // If a comma was inserted, consume it; 229 229 + // report an error if there's more tokens. 230 230 + if p.tok == token.COMMA && p.lit == "\n" { 231 231 + p.next() 232 232 + } 233 233 + if p.mode&partialMode == 0 { 234 234 + p.expect(token.EOF) 235 235 + } 236 236 + 237 237 + if p.errors.Len() > 0 { 238 238 + p.errors.Sort() 239 239 + return nil, p.errors.Err() 240 240 + } 241 241 + 242 242 + return e, nil 243 243 + } 244 244 + 245 245 + // parseExprString is a convenience function for obtaining the AST of an 246 246 + // expression x. The position information recorded in the AST is undefined. The 247 247 + // filename used in error messages is the empty string. 248 248 + func parseExprString(x string) (ast.Expr, error) { 249 249 + return ParseExpr(token.NewFileSet(), "", []byte(x)) 250 250 + }

+126

cue/parser/interface_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "reflect" 19 19 + "testing" 20 20 + 21 21 + "cuelang.org/go/cue/ast" 22 22 + "cuelang.org/go/cue/token" 23 23 + ) 24 24 + 25 25 + func Test_readSource(t *testing.T) { 26 26 + type args struct { 27 27 + filename string 28 28 + src interface{} 29 29 + } 30 30 + tests := []struct { 31 31 + name string 32 32 + args args 33 33 + want []byte 34 34 + wantErr bool 35 35 + }{ 36 36 + // TODO: Add test cases. 37 37 + } 38 38 + for _, tt := range tests { 39 39 + got, err := readSource(tt.args.filename, tt.args.src) 40 40 + if (err != nil) != tt.wantErr { 41 41 + t.Errorf("%q. readSource() error = %v, wantErr %v", tt.name, err, tt.wantErr) 42 42 + continue 43 43 + } 44 44 + if !reflect.DeepEqual(got, tt.want) { 45 45 + t.Errorf("%q. readSource() = %v, want %v", tt.name, got, tt.want) 46 46 + } 47 47 + } 48 48 + } 49 49 + 50 50 + func TestParseFile(t *testing.T) { 51 51 + type args struct { 52 52 + fset *token.FileSet 53 53 + filename string 54 54 + src interface{} 55 55 + options []Option 56 56 + } 57 57 + tests := []struct { 58 58 + name string 59 59 + args args 60 60 + wantF *ast.File 61 61 + wantErr bool 62 62 + }{ 63 63 + // TODO: Add test cases. 64 64 + } 65 65 + for _, tt := range tests { 66 66 + gotF, err := ParseFile(tt.args.fset, tt.args.filename, tt.args.src, tt.args.options...) 67 67 + if (err != nil) != tt.wantErr { 68 68 + t.Errorf("%q. ParseFile() error = %v, wantErr %v", tt.name, err, tt.wantErr) 69 69 + continue 70 70 + } 71 71 + if !reflect.DeepEqual(gotF, tt.wantF) { 72 72 + t.Errorf("%q. ParseFile() = %v, want %v", tt.name, gotF, tt.wantF) 73 73 + } 74 74 + } 75 75 + } 76 76 + 77 77 + func TestParseExprFrom(t *testing.T) { 78 78 + type args struct { 79 79 + fset *token.FileSet 80 80 + filename string 81 81 + src interface{} 82 82 + mode Option 83 83 + } 84 84 + tests := []struct { 85 85 + name string 86 86 + args args 87 87 + want ast.Expr 88 88 + wantErr bool 89 89 + }{ 90 90 + // TODO: Add test cases. 91 91 + } 92 92 + for _, tt := range tests { 93 93 + got, err := ParseExpr(tt.args.fset, tt.args.filename, tt.args.src, tt.args.mode) 94 94 + if (err != nil) != tt.wantErr { 95 95 + t.Errorf("%q. ParseExprFrom() error = %v, wantErr %v", tt.name, err, tt.wantErr) 96 96 + continue 97 97 + } 98 98 + if !reflect.DeepEqual(got, tt.want) { 99 99 + t.Errorf("%q. ParseExprFrom() = %v, want %v", tt.name, got, tt.want) 100 100 + } 101 101 + } 102 102 + } 103 103 + 104 104 + func TestParseExprString(t *testing.T) { 105 105 + type args struct { 106 106 + x string 107 107 + } 108 108 + tests := []struct { 109 109 + name string 110 110 + args args 111 111 + want ast.Expr 112 112 + wantErr bool 113 113 + }{ 114 114 + // TODO: Add test cases. 115 115 + } 116 116 + for _, tt := range tests { 117 117 + got, err := parseExprString(tt.args.x) 118 118 + if (err != nil) != tt.wantErr { 119 119 + t.Errorf("%q. ParseExpr() error = %v, wantErr %v", tt.name, err, tt.wantErr) 120 120 + continue 121 121 + } 122 122 + if !reflect.DeepEqual(got, tt.want) { 123 123 + t.Errorf("%q. ParseExpr() = %v, want %v", tt.name, got, tt.want) 124 124 + } 125 125 + } 126 126 + }

+1446

cue/parser/parser.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + "strconv" 20 20 + "strings" 21 21 + "unicode" 22 22 + 23 23 + "cuelang.org/go/cue/ast" 24 24 + "cuelang.org/go/cue/errors" 25 25 + "cuelang.org/go/cue/scanner" 26 26 + "cuelang.org/go/cue/token" 27 27 + ) 28 28 + 29 29 + // The parser structure holds the parser's internal state. 30 30 + type parser struct { 31 31 + file *token.File 32 32 + errors errors.List 33 33 + scanner scanner.Scanner 34 34 + 35 35 + // Tracing/debugging 36 36 + mode mode // parsing mode 37 37 + trace bool // == (mode & Trace != 0) 38 38 + indent int // indentation used for tracing output 39 39 + 40 40 + // Comments 41 41 + leadComment *ast.CommentGroup 42 42 + comments *commentState 43 43 + 44 44 + // Next token 45 45 + pos token.Pos // token position 46 46 + tok token.Token // one token look-ahead 47 47 + lit string // token literal 48 48 + 49 49 + // Error recovery 50 50 + // (used to limit the number of calls to syncXXX functions 51 51 + // w/o making scanning progress - avoids potential endless 52 52 + // loops across multiple parser functions during error recovery) 53 53 + syncPos token.Pos // last synchronization position 54 54 + syncCnt int // number of calls to syncXXX without progress 55 55 + 56 56 + // Non-syntactic parser control 57 57 + exprLev int // < 0: in control clause, >= 0: in expression 58 58 + 59 59 + imports []*ast.ImportSpec // list of imports 60 60 + 61 61 + } 62 62 + 63 63 + func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode []Option) { 64 64 + p.file = fset.AddFile(filename, -1, len(src)) 65 65 + for _, f := range mode { 66 66 + f(p) 67 67 + } 68 68 + var m scanner.Mode 69 69 + if p.mode&parseCommentsMode != 0 { 70 70 + m = scanner.ScanComments 71 71 + } 72 72 + eh := func(pos token.Position, msg string) { p.errors.AddNew(pos, msg) } 73 73 + p.scanner.Init(p.file, src, eh, m) 74 74 + 75 75 + p.trace = p.mode&traceMode != 0 // for convenience (p.trace is used frequently) 76 76 + 77 77 + p.comments = &commentState{pos: -1} 78 78 + 79 79 + p.next() 80 80 + } 81 81 + 82 82 + type commentList struct { 83 83 + taken bool // for validation 84 84 + attachTail bool 85 85 + head *ast.CommentGroup 86 86 + last *ast.CommentGroup 87 87 + } 88 88 + 89 89 + type commentState struct { 90 90 + parent *commentState 91 91 + pos int8 92 92 + groups []*ast.CommentGroup 93 93 + 94 94 + // lists are not attached to nodes themselves. Enclosed expressions may 95 95 + // miss a comment due to commas and line termination. closeLists ensures 96 96 + // that comments will be passed to someone. 97 97 + isList int 98 98 + lastChild ast.Node 99 99 + lastPos int8 100 100 + } 101 101 + 102 102 + // openComments reserves the next doc comment for the caller and flushes 103 103 + func (p *parser) openComments() *commentState { 104 104 + if c := p.comments; c != nil && c.isList > 0 { 105 105 + if c.lastChild != nil { 106 106 + for _, cg := range c.groups { 107 107 + cg.Position = c.lastPos 108 108 + c.lastChild.AddComment(cg) 109 109 + } 110 110 + c.groups = nil 111 111 + } 112 112 + c.lastChild = nil 113 113 + } 114 114 + c := &commentState{ 115 115 + parent: p.comments, 116 116 + groups: []*ast.CommentGroup{p.leadComment}, 117 117 + } 118 118 + p.comments = c 119 119 + p.leadComment = nil 120 120 + return c 121 121 + } 122 122 + 123 123 + // openList is used to treat a list of comments as a single comment 124 124 + // position in a production. 125 125 + func (p *parser) openList() { 126 126 + if p.comments.isList > 0 { 127 127 + p.comments.isList++ 128 128 + return 129 129 + } 130 130 + c := &commentState{ 131 131 + parent: p.comments, 132 132 + isList: 1, 133 133 + } 134 134 + p.comments = c 135 135 + } 136 136 + 137 137 + func (c *commentState) add(g *ast.CommentGroup) { 138 138 + g.Position = c.pos 139 139 + c.groups = append(c.groups, g) 140 140 + } 141 141 + 142 142 + func (p *parser) closeList() { 143 143 + c := p.comments 144 144 + if c.lastChild != nil { 145 145 + for _, cg := range c.groups { 146 146 + cg.Position = c.lastPos 147 147 + c.lastChild.AddComment(cg) 148 148 + } 149 149 + c.groups = nil 150 150 + } 151 151 + switch c.isList--; { 152 152 + case c.isList < 0: 153 153 + panic("unmatched close list") 154 154 + case c.isList == 0: 155 155 + parent := c.parent 156 156 + parent.groups = append(parent.groups, c.groups...) 157 157 + parent.pos++ 158 158 + p.comments = parent 159 159 + } 160 160 + } 161 161 + 162 162 + func (c *commentState) closeNode(p *parser, n ast.Node) ast.Node { 163 163 + if p.comments != c { 164 164 + panic("unmatched comments") 165 165 + } 166 166 + p.comments = c.parent 167 167 + if c.parent != nil { 168 168 + c.parent.lastChild = n 169 169 + c.parent.lastPos = c.pos 170 170 + c.parent.pos++ 171 171 + } 172 172 + for _, cg := range c.groups { 173 173 + if n != nil { 174 174 + n.AddComment(cg) 175 175 + } 176 176 + } 177 177 + c.groups = nil 178 178 + return n 179 179 + } 180 180 + 181 181 + func (c *commentState) closeExpr(p *parser, n ast.Expr) ast.Expr { 182 182 + c.closeNode(p, n) 183 183 + return n 184 184 + } 185 185 + 186 186 + func (c *commentState) closeClause(p *parser, n ast.Clause) ast.Clause { 187 187 + c.closeNode(p, n) 188 188 + return n 189 189 + } 190 190 + 191 191 + // ---------------------------------------------------------------------------- 192 192 + // Parsing support 193 193 + 194 194 + func (p *parser) printTrace(a ...interface{}) { 195 195 + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " 196 196 + const n = len(dots) 197 197 + pos := p.file.Position(p.pos) 198 198 + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) 199 199 + i := 2 * p.indent 200 200 + for i > n { 201 201 + fmt.Print(dots) 202 202 + i -= n 203 203 + } 204 204 + // i <= n 205 205 + fmt.Print(dots[0:i]) 206 206 + fmt.Println(a...) 207 207 + } 208 208 + 209 209 + func trace(p *parser, msg string) *parser { 210 210 + p.printTrace(msg, "(") 211 211 + p.indent++ 212 212 + return p 213 213 + } 214 214 + 215 215 + // Usage pattern: defer un(trace(p, "...")) 216 216 + func un(p *parser) { 217 217 + p.indent-- 218 218 + p.printTrace(")") 219 219 + } 220 220 + 221 221 + // Advance to the next 222 222 + func (p *parser) next0() { 223 223 + // Because of one-token look-ahead, print the previous token 224 224 + // when tracing as it provides a more readable output. The 225 225 + // very first token (!p.pos.IsValid()) is not initialized 226 226 + // (it is ILLEGAL), so don't print it . 227 227 + if p.trace && p.pos.IsValid() { 228 228 + s := p.tok.String() 229 229 + switch { 230 230 + case p.tok.IsLiteral(): 231 231 + p.printTrace(s, p.lit) 232 232 + case p.tok.IsOperator(), p.tok.IsKeyword(): 233 233 + p.printTrace("\"" + s + "\"") 234 234 + default: 235 235 + p.printTrace(s) 236 236 + } 237 237 + } 238 238 + 239 239 + p.pos, p.tok, p.lit = p.scanner.Scan() 240 240 + } 241 241 + 242 242 + // Consume a comment and return it and the line on which it ends. 243 243 + func (p *parser) consumeComment() (comment *ast.Comment, endline int) { 244 244 + // /*-style comments may end on a different line than where they start. 245 245 + // Scan the comment for '\n' chars and adjust endline accordingly. 246 246 + endline = p.file.Line(p.pos) 247 247 + if p.lit[1] == '*' { 248 248 + // don't use range here - no need to decode Unicode code points 249 249 + for i := 0; i < len(p.lit); i++ { 250 250 + if p.lit[i] == '\n' { 251 251 + endline++ 252 252 + } 253 253 + } 254 254 + } 255 255 + 256 256 + comment = &ast.Comment{Slash: p.pos, Text: p.lit} 257 257 + p.next0() 258 258 + 259 259 + return 260 260 + } 261 261 + 262 262 + // Consume a group of adjacent comments, add it to the parser's 263 263 + // comments list, and return it together with the line at which 264 264 + // the last comment in the group ends. A non-comment token or n 265 265 + // empty lines terminate a comment group. 266 266 + func (p *parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { 267 267 + var list []*ast.Comment 268 268 + endline = p.file.Line(p.pos) 269 269 + for p.tok == token.COMMENT && p.file.Line(p.pos) <= endline+n { 270 270 + var comment *ast.Comment 271 271 + comment, endline = p.consumeComment() 272 272 + list = append(list, comment) 273 273 + } 274 274 + 275 275 + cg := &ast.CommentGroup{List: list} 276 276 + comments = cg 277 277 + return 278 278 + } 279 279 + 280 280 + // Advance to the next non-comment In the process, collect 281 281 + // any comment groups encountered, and refield the last lead and 282 282 + // and line comments. 283 283 + // 284 284 + // A lead comment is a comment group that starts and ends in a 285 285 + // line without any other tokens and that is followed by a non-comment 286 286 + // token on the line immediately after the comment group. 287 287 + // 288 288 + // A line comment is a comment group that follows a non-comment 289 289 + // token on the same line, and that has no tokens after it on the line 290 290 + // where it ends. 291 291 + // 292 292 + // Lead and line comments may be considered documentation that is 293 293 + // stored in the AST. 294 294 + func (p *parser) next() { 295 295 + // A leadComment may not be consumed if it leads an inner token of a node. 296 296 + if p.leadComment != nil { 297 297 + p.comments.add(p.leadComment) 298 298 + } 299 299 + p.leadComment = nil 300 300 + prev := p.pos 301 301 + p.next0() 302 302 + p.comments.pos++ 303 303 + 304 304 + if p.tok == token.COMMENT { 305 305 + var comment *ast.CommentGroup 306 306 + var endline int 307 307 + 308 308 + if p.file.Line(p.pos) == p.file.Line(prev) { 309 309 + // The comment is on same line as the previous token; it 310 310 + // cannot be a lead comment but may be a line comment. 311 311 + comment, endline = p.consumeCommentGroup(0) 312 312 + if p.file.Line(p.pos) != endline { 313 313 + // The next token is on a different line, thus 314 314 + // the last comment group is a line comment. 315 315 + comment.Line = true 316 316 + } 317 317 + } 318 318 + 319 319 + // consume successor comments, if any 320 320 + endline = -1 321 321 + for p.tok == token.COMMENT { 322 322 + if comment != nil { 323 323 + p.comments.add(comment) 324 324 + } 325 325 + comment, endline = p.consumeCommentGroup(1) 326 326 + } 327 327 + 328 328 + if endline+1 == p.file.Line(p.pos) && p.tok != token.EOF { 329 329 + // The next token is following on the line immediately after the 330 330 + // comment group, thus the last comment group is a lead comment. 331 331 + comment.Doc = true 332 332 + p.leadComment = comment 333 333 + } else { 334 334 + p.comments.add(comment) 335 335 + } 336 336 + } 337 337 + } 338 338 + 339 339 + // A bailout panic is raised to indicate early termination. 340 340 + type bailout struct{} 341 341 + 342 342 + func (p *parser) error(pos token.Pos, msg string) { 343 343 + ePos := p.file.Position(pos) 344 344 + 345 345 + // If AllErrors is not set, discard errors reported on the same line 346 346 + // as the last recorded error and stop parsing if there are more than 347 347 + // 10 errors. 348 348 + if p.mode&allErrorsMode == 0 { 349 349 + n := len(p.errors) 350 350 + if n > 0 && p.errors[n-1].Position().Line == ePos.Line { 351 351 + return // discard - likely a spurious error 352 352 + } 353 353 + if n > 10 { 354 354 + panic(bailout{}) 355 355 + } 356 356 + } 357 357 + 358 358 + p.errors.AddNew(ePos, msg) 359 359 + } 360 360 + 361 361 + func (p *parser) errorExpected(pos token.Pos, msg string) { 362 362 + msg = "expected " + msg 363 363 + if pos == p.pos { 364 364 + // the error happened at the current position; 365 365 + // make the error message more specific 366 366 + if p.tok == token.COMMA && p.lit == "\n" { 367 367 + msg += ", found newline" 368 368 + } else { 369 369 + msg += ", found '" + p.tok.String() + "'" 370 370 + if p.tok.IsLiteral() { 371 371 + msg += " " + p.lit 372 372 + } 373 373 + } 374 374 + } 375 375 + p.error(pos, msg) 376 376 + } 377 377 + 378 378 + func (p *parser) expect(tok token.Token) token.Pos { 379 379 + pos := p.pos 380 380 + if p.tok != tok { 381 381 + p.errorExpected(pos, "'"+tok.String()+"'") 382 382 + } 383 383 + p.next() // make progress 384 384 + return pos 385 385 + } 386 386 + 387 387 + // expectClosing is like expect but provides a better error message 388 388 + // for the common case of a missing comma before a newline. 389 389 + func (p *parser) expectClosing(tok token.Token, context string) token.Pos { 390 390 + if p.tok != tok && p.tok == token.COMMA && p.lit == "\n" { 391 391 + p.error(p.pos, "missing ',' before newline in "+context) 392 392 + p.next() 393 393 + } 394 394 + return p.expect(tok) 395 395 + } 396 396 + 397 397 + func (p *parser) expectComma() { 398 398 + // semicolon is optional before a closing ')', ']', '}', or newline 399 399 + if p.tok != token.RPAREN && p.tok != token.RBRACE && p.tok != token.EOF { 400 400 + switch p.tok { 401 401 + case token.COMMA: 402 402 + p.next() 403 403 + default: 404 404 + p.errorExpected(p.pos, "','") 405 405 + syncExpr(p) 406 406 + } 407 407 + } 408 408 + } 409 409 + 410 410 + func (p *parser) atComma(context string, follow ...token.Token) bool { 411 411 + if p.tok == token.COMMA { 412 412 + return true 413 413 + } 414 414 + for _, t := range follow { 415 415 + if p.tok == t { 416 416 + return false 417 417 + } 418 418 + } 419 419 + msg := "missing ','" 420 420 + // TODO: find a way to detect crossing lines now we don't have a semi. 421 421 + if p.lit == "\n" { 422 422 + msg += " before newline" 423 423 + } 424 424 + p.error(p.pos, msg+" in "+context) 425 425 + return true // "insert" comma and continue 426 426 + } 427 427 + 428 428 + func assert(cond bool, msg string) { 429 429 + if !cond { 430 430 + panic("lacelang/parser internal error: " + msg) 431 431 + } 432 432 + } 433 433 + 434 434 + // syncExpr advances to the next field in a field list. 435 435 + // Used for synchronization after an error. 436 436 + func syncExpr(p *parser) { 437 437 + for { 438 438 + switch p.tok { 439 439 + case token.COMMA: 440 440 + // Return only if parser made some progress since last 441 441 + // sync or if it has not reached 10 sync calls without 442 442 + // progress. Otherwise consume at least one token to 443 443 + // avoid an endless parser loop (it is possible that 444 444 + // both parseOperand and parseStmt call syncStmt and 445 445 + // correctly do not advance, thus the need for the 446 446 + // invocation limit p.syncCnt). 447 447 + if p.pos == p.syncPos && p.syncCnt < 10 { 448 448 + p.syncCnt++ 449 449 + return 450 450 + } 451 451 + if p.pos > p.syncPos { 452 452 + p.syncPos = p.pos 453 453 + p.syncCnt = 0 454 454 + return 455 455 + } 456 456 + // Reaching here indicates a parser bug, likely an 457 457 + // incorrect token list in this function, but it only 458 458 + // leads to skipping of possibly correct code if a 459 459 + // previous error is present, and thus is preferred 460 460 + // over a non-terminating parse. 461 461 + case token.EOF: 462 462 + return 463 463 + } 464 464 + p.next() 465 465 + } 466 466 + } 467 467 + 468 468 + // safePos returns a valid file position for a given position: If pos 469 469 + // is valid to begin with, safePos returns pos. If pos is out-of-range, 470 470 + // safePos returns the EOF position. 471 471 + // 472 472 + // This is hack to work around "artificial" end positions in the AST which 473 473 + // are computed by adding 1 to (presumably valid) token positions. If the 474 474 + // token positions are invalid due to parse errors, the resulting end position 475 475 + // may be past the file's EOF position, which would lead to panics if used 476 476 + // later on. 477 477 + func (p *parser) safePos(pos token.Pos) (res token.Pos) { 478 478 + defer func() { 479 479 + if recover() != nil { 480 480 + res = token.Pos(p.file.Base() + p.file.Size()) // EOF position 481 481 + } 482 482 + }() 483 483 + _ = p.file.Offset(pos) // trigger a panic if position is out-of-range 484 484 + return pos 485 485 + } 486 486 + 487 487 + // ---------------------------------------------------------------------------- 488 488 + // Identifiers 489 489 + 490 490 + func (p *parser) parseIdent() *ast.Ident { 491 491 + c := p.openComments() 492 492 + pos := p.pos 493 493 + name := "_" 494 494 + if p.tok == token.IDENT { 495 495 + name = p.lit 496 496 + p.next() 497 497 + } else { 498 498 + p.expect(token.IDENT) // use expect() error handling 499 499 + } 500 500 + ident := &ast.Ident{NamePos: pos, Name: name} 501 501 + c.closeNode(p, ident) 502 502 + return ident 503 503 + } 504 504 + 505 505 + // ---------------------------------------------------------------------------- 506 506 + // Expressions 507 507 + 508 508 + // parseOperand returns an expression. 509 509 + // Callers must verify the result. 510 510 + func (p *parser) parseOperand() (expr ast.Expr) { 511 511 + if p.trace { 512 512 + defer un(trace(p, "Operand")) 513 513 + } 514 514 + 515 515 + switch p.tok { 516 516 + case token.IDENT: 517 517 + return p.parseIdent() 518 518 + 519 519 + case token.LBRACE: 520 520 + return p.parseStruct() 521 521 + 522 522 + case token.LBRACK: 523 523 + return p.parseList() 524 524 + 525 525 + case token.BOTTOM: 526 526 + c := p.openComments() 527 527 + x := &ast.BottomLit{Bottom: p.pos} 528 528 + p.next() 529 529 + return c.closeExpr(p, x) 530 530 + 531 531 + case token.NULL, token.TRUE, token.FALSE, token.INT, token.FLOAT, token.STRING: 532 532 + c := p.openComments() 533 533 + x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} 534 534 + p.next() 535 535 + return c.closeExpr(p, x) 536 536 + 537 537 + case token.INTERPOLATION: 538 538 + return p.parseInterpolation() 539 539 + 540 540 + case token.LPAREN: 541 541 + c := p.openComments() 542 542 + defer func() { c.closeNode(p, expr) }() 543 543 + lparen := p.pos 544 544 + p.next() 545 545 + if p.tok == token.RPAREN && p.mode&parseLambdasMode != 0 { 546 546 + c.pos = 2 547 547 + rparen := p.expect(token.RPAREN) 548 548 + p.expect(token.LAMBDA) 549 549 + return &ast.LambdaExpr{ 550 550 + Lparen: lparen, 551 551 + Rparen: rparen, 552 552 + Expr: p.parseRHS(), 553 553 + } 554 554 + } 555 555 + p.exprLev++ 556 556 + p.openList() 557 557 + x := p.parseRHS() // types may be parenthesized: (some type) 558 558 + var params []*ast.Field 559 559 + ident, ok := x.(*ast.Ident) 560 560 + if ok && (p.tok == token.COLON || p.tok == token.COMMA) && p.mode&parseLambdasMode != 0 { 561 561 + params = p.parseParams(ident, token.RPAREN) 562 562 + } 563 563 + p.closeList() 564 564 + p.exprLev-- 565 565 + rparen := p.expect(token.RPAREN) 566 566 + if p.tok == token.LAMBDA || params != nil && p.mode&parseLambdasMode != 0 { 567 567 + p.expect(token.LAMBDA) 568 568 + if params == nil { 569 569 + m := &ast.Field{Label: ident} 570 570 + params = append(params, m) 571 571 + } 572 572 + return &ast.LambdaExpr{ 573 573 + Lparen: lparen, 574 574 + Params: params, 575 575 + Rparen: rparen, 576 576 + Expr: p.parseRHS(), 577 577 + } 578 578 + } 579 579 + return &ast.ParenExpr{ 580 580 + Lparen: lparen, 581 581 + X: x, 582 582 + Rparen: rparen} 583 583 + } 584 584 + 585 585 + // we have an error 586 586 + c := p.openComments() 587 587 + pos := p.pos 588 588 + p.errorExpected(pos, "operand") 589 589 + syncExpr(p) 590 590 + return c.closeExpr(p, &ast.BadExpr{From: pos, To: p.pos}) 591 591 + } 592 592 + 593 593 + func (p *parser) parseParams(ident *ast.Ident, follow token.Token) (params []*ast.Field) { 594 594 + for { 595 595 + c := p.openComments() 596 596 + if ident == nil { 597 597 + ident = p.parseIdent() 598 598 + } 599 599 + m := &ast.Field{Label: ident} 600 600 + if p.tok == token.COLON { 601 601 + m.Colon = p.expect(token.COLON) 602 602 + m.Value = p.parseRHS() 603 603 + } 604 604 + hasComma := p.tok == token.COMMA 605 605 + if hasComma { 606 606 + p.expect(token.COMMA) 607 607 + } 608 608 + c.closeNode(p, m) 609 609 + params = append(params, m) 610 610 + if !hasComma || p.tok == follow || p.tok == token.EOF { 611 611 + break 612 612 + } 613 613 + ident = nil 614 614 + } 615 615 + return params 616 616 + } 617 617 + 618 618 + func (p *parser) parseIndexOrSlice(x ast.Expr) (expr ast.Expr) { 619 619 + if p.trace { 620 620 + defer un(trace(p, "IndexOrSlice")) 621 621 + } 622 622 + 623 623 + c := p.openComments() 624 624 + defer func() { c.closeNode(p, expr) }() 625 625 + c.pos = 1 626 626 + 627 627 + const N = 2 628 628 + lbrack := p.expect(token.LBRACK) 629 629 + 630 630 + p.exprLev++ 631 631 + var index [N]ast.Expr 632 632 + var colons [N - 1]token.Pos 633 633 + if p.tok != token.COLON { 634 634 + index[0] = p.parseRHS() 635 635 + } 636 636 + nColons := 0 637 637 + for p.tok == token.COLON && nColons < len(colons) { 638 638 + colons[nColons] = p.pos 639 639 + nColons++ 640 640 + p.next() 641 641 + if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { 642 642 + index[nColons] = p.parseRHS() 643 643 + } 644 644 + } 645 645 + p.exprLev-- 646 646 + rbrack := p.expect(token.RBRACK) 647 647 + 648 648 + if nColons > 0 { 649 649 + return &ast.SliceExpr{ 650 650 + X: x, 651 651 + Lbrack: lbrack, 652 652 + Low: index[0], 653 653 + High: index[1], 654 654 + Rbrack: rbrack} 655 655 + } 656 656 + 657 657 + return &ast.IndexExpr{ 658 658 + X: x, 659 659 + Lbrack: lbrack, 660 660 + Index: index[0], 661 661 + Rbrack: rbrack} 662 662 + } 663 663 + 664 664 + func (p *parser) parseCallOrConversion(fun ast.Expr) (expr *ast.CallExpr) { 665 665 + if p.trace { 666 666 + defer un(trace(p, "CallOrConversion")) 667 667 + } 668 668 + c := p.openComments() 669 669 + defer func() { c.closeNode(p, expr) }() 670 670 + 671 671 + lparen := p.expect(token.LPAREN) 672 672 + p.exprLev++ 673 673 + var list []ast.Expr 674 674 + for p.tok != token.RPAREN && p.tok != token.EOF { 675 675 + list = append(list, p.parseRHS()) // builtins may expect a type: make(some type, ...) 676 676 + if !p.atComma("argument list", token.RPAREN) { 677 677 + break 678 678 + } 679 679 + p.next() 680 680 + } 681 681 + p.exprLev-- 682 682 + rparen := p.expectClosing(token.RPAREN, "argument list") 683 683 + 684 684 + return &ast.CallExpr{ 685 685 + Fun: fun, 686 686 + Lparen: lparen, 687 687 + Args: list, 688 688 + Rparen: rparen} 689 689 + } 690 690 + 691 691 + func (p *parser) parseFieldList(allowEmit bool) (list []ast.Decl) { 692 692 + if p.trace { 693 693 + defer un(trace(p, "FieldList")) 694 694 + } 695 695 + origEmit := allowEmit 696 696 + p.openList() 697 697 + defer p.closeList() 698 698 + 699 699 + for p.tok != token.RBRACE && p.tok != token.EOF { 700 700 + d := p.parseField(allowEmit) 701 701 + if e, ok := d.(*ast.EmitDecl); ok { 702 702 + if origEmit && !allowEmit { 703 703 + p.error(p.pos, "only one emit allowed at top level") 704 704 + } 705 705 + if !origEmit || !allowEmit { 706 706 + d = &ast.BadDecl{From: e.Pos(), To: e.End()} 707 707 + for _, cg := range e.Comments() { 708 708 + d.AddComment(cg) 709 709 + } 710 710 + } 711 711 + // uncomment to only allow one emit per top-level 712 712 + // allowEmit = false 713 713 + } 714 714 + list = append(list, d) 715 715 + } 716 716 + return 717 717 + } 718 718 + func (p *parser) parseField(allowEmit bool) (decl ast.Decl) { 719 719 + if p.trace { 720 720 + defer un(trace(p, "Field")) 721 721 + } 722 722 + 723 723 + c := p.openComments() 724 724 + defer func() { c.closeNode(p, decl) }() 725 725 + 726 726 + pos := p.pos 727 727 + 728 728 + this := &ast.Field{Label: nil} 729 729 + m := this 730 730 + 731 731 + for i := 0; ; i++ { 732 732 + tok := p.tok 733 733 + 734 734 + expr, ok := p.parseLabel(m) 735 735 + 736 736 + if !ok { 737 737 + if !allowEmit { 738 738 + p.error(pos, "expected label, found "+tok.String()) 739 739 + } 740 740 + if expr == nil { 741 741 + expr = p.parseExpr() 742 742 + } 743 743 + e := &ast.EmitDecl{Expr: expr} 744 744 + if p.atComma("file", token.RBRACE) { 745 745 + p.next() 746 746 + } 747 747 + return e 748 748 + } 749 749 + 750 750 + if i == 0 && tok == token.IDENT { 751 751 + ident := expr.(*ast.Ident) 752 752 + switch p.tok { 753 753 + case token.BIND: 754 754 + pos := p.pos 755 755 + p.expect(token.BIND) 756 756 + ref := p.parseRHS() 757 757 + if p.atComma("struct literal", token.RBRACE) { // TODO: may be EOF 758 758 + p.next() 759 759 + } 760 760 + return &ast.Alias{Ident: ident, Equal: pos, Expr: ref} 761 761 + 762 762 + case token.LPAREN: 763 763 + var value ast.Expr 764 764 + if p.mode&parseLambdasMode != 0 { 765 765 + c.pos = 2 766 766 + // TODO: Only allow LambdaExpr after non-quoted identifier. 767 767 + value = p.parseOperand() 768 768 + if _, ok := unparen(value).(*ast.LambdaExpr); !ok { 769 769 + p.error(value.Pos(), "expected lambda expression") 770 770 + } 771 771 + } 772 772 + if p.atComma("struct literal", token.RBRACE) { // TODO: may be EOF 773 773 + p.next() 774 774 + } 775 775 + return &ast.Field{Label: ident, Value: value} 776 776 + } 777 777 + } 778 778 + 779 779 + if p.tok == token.COLON { 780 780 + break 781 781 + } 782 782 + 783 783 + switch p.tok { 784 784 + default: 785 785 + if !allowEmit || p.tok != token.COMMA { 786 786 + p.errorExpected(p.pos, "label or ':'") 787 787 + } 788 788 + switch tok { 789 789 + case token.IDENT, token.LBRACK, token.STRING, token.INTERPOLATION, token.NULL, token.TRUE, token.FALSE: 790 790 + if p.tok == token.COMMA { 791 791 + p.expectComma() 792 792 + return &ast.EmitDecl{Expr: expr} 793 793 + } 794 794 + } 795 795 + return &ast.BadDecl{From: pos, To: p.pos} 796 796 + 797 797 + case token.IDENT, token.STRING, token.LSS, token.INTERPOLATION, token.LBRACK: 798 798 + field := &ast.Field{} 799 799 + m.Value = &ast.StructLit{Elts: []ast.Decl{field}} 800 800 + m = field 801 801 + } 802 802 + 803 803 + allowEmit = false 804 804 + } 805 805 + 806 806 + this.Colon = p.pos 807 807 + p.expect(token.COLON) 808 808 + m.Value = p.parseRHS() 809 809 + 810 810 + decl = this 811 811 + var arrow token.Pos 812 812 + switch p.tok { 813 813 + case token.ARROW: 814 814 + arrow = p.expect(token.ARROW) 815 815 + fallthrough 816 816 + 817 817 + case token.FOR, token.IF: 818 818 + clauses := p.parseComprehensionClauses() 819 819 + return &ast.ComprehensionDecl{ 820 820 + Field: this, 821 821 + Select: arrow, 822 822 + Clauses: clauses, 823 823 + } 824 824 + } 825 825 + 826 826 + if p.atComma("struct literal", token.RBRACE) { // TODO: may be EOF 827 827 + p.next() 828 828 + } 829 829 + 830 830 + return decl 831 831 + } 832 832 + 833 833 + func (p *parser) parseLabel(f *ast.Field) (expr ast.Expr, ok bool) { 834 834 + switch p.tok { 835 835 + case token.IDENT: 836 836 + ident := p.parseIdent() 837 837 + f.Label = ident 838 838 + expr = ident 839 839 + 840 840 + case token.STRING: 841 841 + // JSON compatibility. 842 842 + 843 843 + expr = p.parseOperand() 844 844 + f.Label = expr.(ast.Label) 845 845 + 846 846 + case token.INTERPOLATION: 847 847 + expr = p.parseInterpolation() 848 848 + f.Label = expr.(ast.Label) 849 849 + 850 850 + case token.NULL, token.TRUE, token.FALSE: 851 851 + // Keywords that represent operands. 852 852 + 853 853 + // Allowing keywords to be used as a labels should not interfere with 854 854 + // generating good errors: any keyword can only appear on the RHS of a 855 855 + // field (after a ':'), whereas labels always appear on the LHS. 856 856 + ident := &ast.BasicLit{ 857 857 + Kind: p.tok, 858 858 + ValuePos: p.pos, 859 859 + Value: p.lit, 860 860 + } 861 861 + p.next() 862 862 + f.Label = ident 863 863 + expr = ident 864 864 + 865 865 + case token.IF, token.FOR, token.IN, token.LET: 866 866 + // Keywords representing clauses. 867 867 + f.Label = &ast.Ident{ 868 868 + NamePos: p.pos, 869 869 + Name: p.lit, 870 870 + } 871 871 + p.next() 872 872 + 873 873 + case token.LSS: // element templates 874 874 + pos := p.pos 875 875 + c := p.openComments() 876 876 + p.next() 877 877 + ident := p.parseIdent() 878 878 + gtr := p.pos 879 879 + if p.tok != token.GTR { 880 880 + p.expect(token.GTR) 881 881 + } 882 882 + p.next() 883 883 + label := &ast.TemplateLabel{Langle: pos, Ident: ident, Rangle: gtr} 884 884 + c.closeNode(p, label) 885 885 + f.Label = label 886 886 + 887 887 + case token.LBRACK: 888 888 + expr = p.parseList() 889 889 + list, ok := expr.(*ast.ListLit) 890 890 + if ok && len(list.Elts) == 1 && list.Ellipsis == token.NoPos { 891 891 + f.Label = &ast.ExprLabel{ 892 892 + Lbrack: list.Lbrack, 893 893 + Label: list.Elts[0], 894 894 + Rbrack: list.Rbrack, 895 895 + } 896 896 + break 897 897 + } 898 898 + 899 899 + fallthrough 900 900 + default: 901 901 + return expr, false 902 902 + } 903 903 + return expr, true 904 904 + } 905 905 + 906 906 + func (p *parser) parseStruct() (expr ast.Expr) { 907 907 + c := p.openComments() 908 908 + defer func() { c.closeNode(p, expr) }() 909 909 + 910 910 + lbrace := p.expect(token.LBRACE) 911 911 + 912 912 + if p.trace { 913 913 + defer un(trace(p, "StructLit")) 914 914 + } 915 915 + 916 916 + elts := p.parseStructBody() 917 917 + rbrace := p.expectClosing(token.RBRACE, "struct literal") 918 918 + return &ast.StructLit{ 919 919 + Lbrace: lbrace, 920 920 + Elts: elts, 921 921 + Rbrace: rbrace, 922 922 + } 923 923 + } 924 924 + 925 925 + func (p *parser) parseStructBody() []ast.Decl { 926 926 + if p.trace { 927 927 + defer un(trace(p, "StructBody")) 928 928 + } 929 929 + 930 930 + p.exprLev++ 931 931 + var elts []ast.Decl 932 932 + if p.tok != token.RBRACE { 933 933 + elts = p.parseFieldList(false) 934 934 + } 935 935 + p.exprLev-- 936 936 + 937 937 + return elts 938 938 + } 939 939 + 940 940 + func isClauseStart(tok token.Token) bool { 941 941 + return tok == token.FOR || tok == token.IF // || tok == LET 942 942 + } 943 943 + 944 944 + func (p *parser) parseComprehensionClauses() (clauses []ast.Clause) { 945 945 + // TODO: reuse Template spec, which is possible if it doesn't check the 946 946 + // first is an identifier. 947 947 + for { 948 948 + if p.tok == token.COMMA { 949 949 + p.next() 950 950 + } 951 951 + switch p.tok { 952 952 + case token.FOR: 953 953 + c := p.openComments() 954 954 + forPos := p.expect(token.FOR) 955 955 + var key, value *ast.Ident 956 956 + var colon token.Pos 957 957 + value = p.parseIdent() 958 958 + if p.tok == token.COMMA { 959 959 + colon = p.expect(token.COMMA) 960 960 + key = value 961 961 + value = p.parseIdent() 962 962 + } 963 963 + c.pos = 4 964 964 + // params := p.parseParams(nil, ARROW) 965 965 + clauses = append(clauses, c.closeClause(p, &ast.ForClause{ 966 966 + For: forPos, 967 967 + Key: key, 968 968 + Colon: colon, 969 969 + Value: value, 970 970 + In: p.expect(token.IN), 971 971 + Source: p.parseExpr(), 972 972 + })) 973 973 + 974 974 + case token.IF: 975 975 + c := p.openComments() 976 976 + clauses = append(clauses, c.closeClause(p, &ast.IfClause{ 977 977 + If: p.expect(token.IF), 978 978 + Condition: p.parseExpr(), 979 979 + })) 980 980 + 981 981 + // TODO: case LET: 982 982 + default: 983 983 + return clauses 984 984 + } 985 985 + } 986 986 + } 987 987 + 988 988 + func (p *parser) parseList() (expr ast.Expr) { 989 989 + c := p.openComments() 990 990 + defer func() { c.closeNode(p, expr) }() 991 991 + 992 992 + lbrack := p.expect(token.LBRACK) 993 993 + 994 994 + if p.trace { 995 995 + defer un(trace(p, "ListLiteral")) 996 996 + } 997 997 + 998 998 + elts := p.parseListElements() 999 999 + 1000 1000 + if clauses := p.parseComprehensionClauses(); clauses != nil { 1001 1001 + var expr ast.Expr 1002 1002 + if len(elts) != 1 { 1003 1003 + p.error(lbrack+1, "list comprehension must have exactly one element") 1004 1004 + } 1005 1005 + if len(elts) > 0 { 1006 1006 + expr = elts[0] 1007 1007 + } 1008 1008 + rbrack := p.expectClosing(token.RBRACK, "list comprehension") 1009 1009 + 1010 1010 + return &ast.ListComprehension{ 1011 1011 + Lbrack: lbrack, 1012 1012 + Expr: expr, 1013 1013 + Clauses: clauses, 1014 1014 + Rbrack: rbrack, 1015 1015 + } 1016 1016 + } 1017 1017 + 1018 1018 + ellipsis := token.NoPos 1019 1019 + typ := ast.Expr(nil) 1020 1020 + if p.tok == token.ELLIPSIS { 1021 1021 + ellipsis = p.pos 1022 1022 + p.next() 1023 1023 + if p.tok != token.COMMA && p.tok != token.RBRACK { 1024 1024 + typ = p.parseRHS() 1025 1025 + } 1026 1026 + if p.atComma("list literal", token.RBRACK) { 1027 1027 + p.next() 1028 1028 + } 1029 1029 + } 1030 1030 + 1031 1031 + rbrack := p.expectClosing(token.RBRACK, "list literal") 1032 1032 + return &ast.ListLit{ 1033 1033 + Lbrack: lbrack, 1034 1034 + Elts: elts, 1035 1035 + Ellipsis: ellipsis, 1036 1036 + Type: typ, 1037 1037 + Rbrack: rbrack} 1038 1038 + } 1039 1039 + 1040 1040 + func (p *parser) parseListElements() (list []ast.Expr) { 1041 1041 + if p.trace { 1042 1042 + defer un(trace(p, "ListElements")) 1043 1043 + } 1044 1044 + p.openList() 1045 1045 + defer p.closeList() 1046 1046 + 1047 1047 + for p.tok != token.RBRACK && p.tok != token.ELLIPSIS && p.tok != token.EOF { 1048 1048 + list = append(list, p.parseListElement()) 1049 1049 + // Enforce there is an explicit comma. We could also allow the 1050 1050 + // omission of commas in lists, but this gives rise to some ambiguities 1051 1051 + // with list comprehensions. 1052 1052 + if p.tok == token.COMMA && p.lit != "," { 1053 1053 + p.next() 1054 1054 + // Allow missing comma for last element, though, to be compliant 1055 1055 + // with JSON. 1056 1056 + if p.tok == token.RBRACK || p.tok == token.FOR || p.tok == token.IF { 1057 1057 + break 1058 1058 + } 1059 1059 + p.error(p.pos, "missing ',' before newline in list literal") 1060 1060 + } else if !p.atComma("list literal", token.RBRACK, token.FOR, token.IF) { 1061 1061 + break 1062 1062 + } 1063 1063 + p.next() 1064 1064 + } 1065 1065 + 1066 1066 + return 1067 1067 + } 1068 1068 + 1069 1069 + func (p *parser) parseListElement() (expr ast.Expr) { 1070 1070 + if p.trace { 1071 1071 + defer un(trace(p, "ListElement")) 1072 1072 + } 1073 1073 + c := p.openComments() 1074 1074 + defer func() { c.closeNode(p, expr) }() 1075 1075 + 1076 1076 + e := p.parseRHS() 1077 1077 + switch p.tok { 1078 1078 + case token.ELLIPSIS: 1079 1079 + return &ast.Ellipsis{Ellipsis: p.expect(token.ELLIPSIS), Elt: e} 1080 1080 + } 1081 1081 + return e 1082 1082 + } 1083 1083 + 1084 1084 + // checkExpr checks that x is an expression (and not a type). 1085 1085 + func (p *parser) checkExpr(x ast.Expr) ast.Expr { 1086 1086 + switch unparen(x).(type) { 1087 1087 + case *ast.BadExpr: 1088 1088 + case *ast.BottomLit: 1089 1089 + case *ast.Ident: 1090 1090 + case *ast.BasicLit: 1091 1091 + case *ast.Interpolation: 1092 1092 + case *ast.StructLit: 1093 1093 + case *ast.ListLit: 1094 1094 + case *ast.LambdaExpr: 1095 1095 + case *ast.ListComprehension: 1096 1096 + case *ast.ParenExpr: 1097 1097 + panic("unreachable") 1098 1098 + case *ast.SelectorExpr: 1099 1099 + case *ast.IndexExpr: 1100 1100 + case *ast.SliceExpr: 1101 1101 + case *ast.CallExpr: 1102 1102 + case *ast.UnaryExpr: 1103 1103 + case *ast.BinaryExpr: 1104 1104 + default: 1105 1105 + // all other nodes are not proper expressions 1106 1106 + p.errorExpected(x.Pos(), "expression") 1107 1107 + x = &ast.BadExpr{ 1108 1108 + From: x.Pos(), To: p.safePos(x.End()), 1109 1109 + } 1110 1110 + } 1111 1111 + return x 1112 1112 + } 1113 1113 + 1114 1114 + // If x is of the form (T), unparen returns unparen(T), otherwise it returns x. 1115 1115 + func unparen(x ast.Expr) ast.Expr { 1116 1116 + if p, isParen := x.(*ast.ParenExpr); isParen { 1117 1117 + x = unparen(p.X) 1118 1118 + } 1119 1119 + return x 1120 1120 + } 1121 1121 + 1122 1122 + // If lhs is set and the result is an identifier, it is not resolved. 1123 1123 + func (p *parser) parsePrimaryExpr() ast.Expr { 1124 1124 + if p.trace { 1125 1125 + defer un(trace(p, "PrimaryExpr")) 1126 1126 + } 1127 1127 + 1128 1128 + x := p.parseOperand() 1129 1129 + 1130 1130 + L: 1131 1131 + for { 1132 1132 + switch p.tok { 1133 1133 + case token.PERIOD: 1134 1134 + c := p.openComments() 1135 1135 + c.pos = 1 1136 1136 + p.next() 1137 1137 + switch p.tok { 1138 1138 + case token.IDENT: 1139 1139 + x = &ast.SelectorExpr{ 1140 1140 + X: p.checkExpr(x), 1141 1141 + Sel: p.parseIdent(), 1142 1142 + } 1143 1143 + default: 1144 1144 + pos := p.pos 1145 1145 + p.errorExpected(pos, "selector") 1146 1146 + p.next() // make progress 1147 1147 + x = &ast.SelectorExpr{X: x, Sel: &ast.Ident{NamePos: pos, Name: "_"}} 1148 1148 + } 1149 1149 + c.closeNode(p, x) 1150 1150 + case token.LBRACK: 1151 1151 + x = p.parseIndexOrSlice(p.checkExpr(x)) 1152 1152 + case token.LPAREN: 1153 1153 + x = p.parseCallOrConversion(p.checkExpr(x)) 1154 1154 + default: 1155 1155 + break L 1156 1156 + } 1157 1157 + } 1158 1158 + 1159 1159 + return x 1160 1160 + } 1161 1161 + 1162 1162 + // If lhs is set and the result is an identifier, it is not resolved. 1163 1163 + func (p *parser) parseUnaryExpr() ast.Expr { 1164 1164 + if p.trace { 1165 1165 + defer un(trace(p, "UnaryExpr")) 1166 1166 + } 1167 1167 + 1168 1168 + switch p.tok { 1169 1169 + case token.ADD, token.SUB, token.NOT: 1170 1170 + pos, op := p.pos, p.tok 1171 1171 + c := p.openComments() 1172 1172 + p.next() 1173 1173 + return c.closeExpr(p, &ast.UnaryExpr{ 1174 1174 + OpPos: pos, 1175 1175 + Op: op, 1176 1176 + X: p.checkExpr(p.parseUnaryExpr()), 1177 1177 + }) 1178 1178 + } 1179 1179 + 1180 1180 + return p.parsePrimaryExpr() 1181 1181 + } 1182 1182 + 1183 1183 + func (p *parser) tokPrec() (token.Token, int) { 1184 1184 + tok := p.tok 1185 1185 + if tok == token.IDENT { 1186 1186 + switch p.lit { 1187 1187 + case "quo": 1188 1188 + return token.IQUO, 7 1189 1189 + case "rem": 1190 1190 + return token.IREM, 7 1191 1191 + case "div": 1192 1192 + return token.IDIV, 7 1193 1193 + case "mod": 1194 1194 + return token.IMOD, 7 1195 1195 + default: 1196 1196 + return tok, 0 1197 1197 + } 1198 1198 + } 1199 1199 + return tok, tok.Precedence() 1200 1200 + } 1201 1201 + 1202 1202 + // If lhs is set and the result is an identifier, it is not resolved. 1203 1203 + func (p *parser) parseBinaryExpr(prec1 int) ast.Expr { 1204 1204 + if p.trace { 1205 1205 + defer un(trace(p, "BinaryExpr")) 1206 1206 + } 1207 1207 + p.openList() 1208 1208 + defer p.closeList() 1209 1209 + 1210 1210 + x := p.parseUnaryExpr() 1211 1211 + 1212 1212 + for { 1213 1213 + op, prec := p.tokPrec() 1214 1214 + if prec < prec1 { 1215 1215 + return x 1216 1216 + } 1217 1217 + c := p.openComments() 1218 1218 + c.pos = 1 1219 1219 + pos := p.expect(p.tok) 1220 1220 + x = c.closeExpr(p, &ast.BinaryExpr{ 1221 1221 + X: p.checkExpr(x), 1222 1222 + OpPos: pos, 1223 1223 + Op: op, 1224 1224 + Y: p.checkExpr(p.parseBinaryExpr(prec + 1))}) 1225 1225 + } 1226 1226 + } 1227 1227 + 1228 1228 + func (p *parser) parseInterpolation() (expr ast.Expr) { 1229 1229 + c := p.openComments() 1230 1230 + defer func() { c.closeNode(p, expr) }() 1231 1231 + 1232 1232 + p.openList() 1233 1233 + defer p.closeList() 1234 1234 + 1235 1235 + cc := p.openComments() 1236 1236 + 1237 1237 + lit := p.lit 1238 1238 + p.next() 1239 1239 + last := &ast.BasicLit{ValuePos: p.pos, Kind: token.STRING, Value: lit} 1240 1240 + exprs := []ast.Expr{last} 1241 1241 + 1242 1242 + quote := rune(lit[0]) 1243 1243 + numQuotes := 1 1244 1244 + if len(lit) > 2 && lit[0] == lit[1] { 1245 1245 + numQuotes = 3 1246 1246 + } 1247 1247 + 1248 1248 + for p.tok == token.LPAREN { 1249 1249 + c.pos = 1 1250 1250 + p.expect(token.LPAREN) 1251 1251 + cc.closeExpr(p, last) 1252 1252 + 1253 1253 + exprs = append(exprs, p.parseExpr()) 1254 1254 + 1255 1255 + cc = p.openComments() 1256 1256 + if p.tok != token.RPAREN { 1257 1257 + p.error(p.pos, "expected ')' for string interpolation") 1258 1258 + } 1259 1259 + lit = p.scanner.ResumeInterpolation(quote, numQuotes) 1260 1260 + p.next() 1261 1261 + last = &ast.BasicLit{ 1262 1262 + ValuePos: p.pos, 1263 1263 + Kind: token.STRING, 1264 1264 + Value: lit, 1265 1265 + } 1266 1266 + exprs = append(exprs, last) 1267 1267 + } 1268 1268 + cc.closeExpr(p, last) 1269 1269 + return &ast.Interpolation{Elts: exprs} 1270 1270 + } 1271 1271 + 1272 1272 + // Callers must check the result (using checkExpr), depending on context. 1273 1273 + func (p *parser) parseExpr() ast.Expr { 1274 1274 + if p.trace { 1275 1275 + defer un(trace(p, "Expression")) 1276 1276 + } 1277 1277 + 1278 1278 + return p.parseBinaryExpr(token.LowestPrec + 1) 1279 1279 + } 1280 1280 + 1281 1281 + func (p *parser) parseRHS() ast.Expr { 1282 1282 + x := p.checkExpr(p.parseExpr()) 1283 1283 + return x 1284 1284 + } 1285 1285 + 1286 1286 + func (p *parser) parseCallExpr(callType string) *ast.CallExpr { 1287 1287 + x := p.parseRHS() // could be a conversion: (some type)(x) 1288 1288 + if call, isCall := x.(*ast.CallExpr); isCall { 1289 1289 + return call 1290 1290 + } 1291 1291 + if _, isBad := x.(*ast.BadExpr); !isBad { 1292 1292 + // only report error if it's a new one 1293 1293 + p.error(p.safePos(x.End()), fmt.Sprintf("function must be invoked in %s statement", callType)) 1294 1294 + } 1295 1295 + return nil 1296 1296 + } 1297 1297 + 1298 1298 + // ---------------------------------------------------------------------------- 1299 1299 + // Declarations 1300 1300 + 1301 1301 + type parseSpecFunction func(iota int) *ast.ImportSpec 1302 1302 + 1303 1303 + func isValidImport(lit string) bool { 1304 1304 + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" 1305 1305 + s, _ := strconv.Unquote(lit) // go/scanner returns a legal string literal 1306 1306 + for _, r := range s { 1307 1307 + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { 1308 1308 + return false 1309 1309 + } 1310 1310 + } 1311 1311 + return s != "" 1312 1312 + } 1313 1313 + 1314 1314 + func (p *parser) parseImportSpec(_ int) *ast.ImportSpec { 1315 1315 + if p.trace { 1316 1316 + defer un(trace(p, "ImportSpec")) 1317 1317 + } 1318 1318 + 1319 1319 + c := p.openComments() 1320 1320 + 1321 1321 + var ident *ast.Ident 1322 1322 + switch p.tok { 1323 1323 + case token.PERIOD: 1324 1324 + ident = &ast.Ident{NamePos: p.pos, Name: "."} 1325 1325 + p.next() 1326 1326 + case token.IDENT: 1327 1327 + ident = p.parseIdent() 1328 1328 + } 1329 1329 + 1330 1330 + pos := p.pos 1331 1331 + var path string 1332 1332 + if p.tok == token.STRING { 1333 1333 + path = p.lit 1334 1334 + if !isValidImport(path) { 1335 1335 + p.error(pos, "invalid import path: "+path) 1336 1336 + } 1337 1337 + p.next() 1338 1338 + p.expectComma() // call before accessing p.linecomment 1339 1339 + } else { 1340 1340 + p.expect(token.STRING) // use expect() error handling 1341 1341 + if p.tok == token.COMMA { 1342 1342 + p.expectComma() // call before accessing p.linecomment 1343 1343 + } 1344 1344 + } 1345 1345 + // collect imports 1346 1346 + spec := &ast.ImportSpec{ 1347 1347 + Name: ident, 1348 1348 + Path: &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: path}, 1349 1349 + } 1350 1350 + c.closeNode(p, spec) 1351 1351 + p.imports = append(p.imports, spec) 1352 1352 + 1353 1353 + return spec 1354 1354 + } 1355 1355 + 1356 1356 + func (p *parser) parseImports() *ast.ImportDecl { 1357 1357 + if p.trace { 1358 1358 + defer un(trace(p, "Imports")) 1359 1359 + } 1360 1360 + c := p.openComments() 1361 1361 + 1362 1362 + ident := p.parseIdent() 1363 1363 + var lparen, rparen token.Pos 1364 1364 + var list []*ast.ImportSpec 1365 1365 + if p.tok == token.LPAREN { 1366 1366 + lparen = p.pos 1367 1367 + p.next() 1368 1368 + p.openList() 1369 1369 + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { 1370 1370 + list = append(list, p.parseImportSpec(iota)) 1371 1371 + } 1372 1372 + p.closeList() 1373 1373 + rparen = p.expect(token.RPAREN) 1374 1374 + p.expectComma() 1375 1375 + } else { 1376 1376 + list = append(list, p.parseImportSpec(0)) 1377 1377 + } 1378 1378 + 1379 1379 + d := &ast.ImportDecl{ 1380 1380 + Import: ident.Pos(), 1381 1381 + Lparen: lparen, 1382 1382 + Specs: list, 1383 1383 + Rparen: rparen, 1384 1384 + } 1385 1385 + c.closeNode(p, d) 1386 1386 + return d 1387 1387 + } 1388 1388 + 1389 1389 + // ---------------------------------------------------------------------------- 1390 1390 + // Source files 1391 1391 + 1392 1392 + func (p *parser) parseFile() *ast.File { 1393 1393 + if p.trace { 1394 1394 + defer un(trace(p, "File")) 1395 1395 + } 1396 1396 + 1397 1397 + c := p.comments 1398 1398 + 1399 1399 + // Don't bother parsing the rest if we had errors scanning the first 1400 1400 + // Likely not a Go source file at all. 1401 1401 + if p.errors.Len() != 0 { 1402 1402 + return nil 1403 1403 + } 1404 1404 + 1405 1405 + // The package clause is not a declaration: it does not appear in any 1406 1406 + // scope. 1407 1407 + pos := p.pos 1408 1408 + var name *ast.Ident 1409 1409 + if p.tok == token.IDENT && p.lit == "package" { 1410 1410 + p.expect(token.IDENT) 1411 1411 + name = p.parseIdent() 1412 1412 + if name.Name == "_" && p.mode&declarationErrorsMode != 0 { 1413 1413 + p.error(p.pos, "invalid package name _") 1414 1414 + } 1415 1415 + p.expectComma() 1416 1416 + } else { 1417 1417 + pos = token.NoPos 1418 1418 + } 1419 1419 + c.pos = 3 1420 1420 + 1421 1421 + p.openList() 1422 1422 + var decls []ast.Decl 1423 1423 + if p.mode&packageClauseOnlyMode == 0 { 1424 1424 + // import decls 1425 1425 + for p.tok == token.IDENT && p.lit == "import" { 1426 1426 + decls = append(decls, p.parseImports()) 1427 1427 + } 1428 1428 + 1429 1429 + if p.mode&importsOnlyMode == 0 { 1430 1430 + // rest of package decls 1431 1431 + // TODO: loop and allow multiple expressions. 1432 1432 + decls = append(decls, p.parseFieldList(true)...) 1433 1433 + p.expect(token.EOF) 1434 1434 + } 1435 1435 + } 1436 1436 + p.closeList() 1437 1437 + 1438 1438 + f := &ast.File{ 1439 1439 + Package: pos, 1440 1440 + Name: name, 1441 1441 + Imports: p.imports, 1442 1442 + Decls: decls, 1443 1443 + } 1444 1444 + c.closeNode(p, f) 1445 1445 + return f 1446 1446 + }

+543

cue/parser/parser_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "bytes" 19 19 + "fmt" 20 20 + "strings" 21 21 + "testing" 22 22 + 23 23 + "cuelang.org/go/cue/ast" 24 24 + "cuelang.org/go/cue/token" 25 25 + ) 26 26 + 27 27 + func TestParse(t *testing.T) { 28 28 + testCases := []struct{ desc, in, out string }{{ 29 29 + "empty file", "", "", 30 30 + }, { 31 31 + "empty struct", "{}", "{}", 32 32 + }, { 33 33 + "empty structs", "{},{},", "{}, {}", 34 34 + }, { 35 35 + "empty structs; elided comma", "{}\n{}", "{}, {}", 36 36 + }, { 37 37 + "basic lits", `"a","b", 3,3.4,5,2_3`, `"a", "b", 3, 3.4, 5, 2_3`, 38 38 + }, { 39 39 + "keyword basic lits", `true,false,null`, `true, false, null`, 40 40 + }, { 41 41 + "keywords as labels", 42 42 + `if: 0, for: 1, in: 2, where: 3, div: 4, quo: 5`, 43 43 + `if: 0, for: 1, in: 2, where: 3, div: 4, quo: 5`, 44 44 + }, { 45 45 + "json", 46 46 + `{ 47 47 + "a": 1, 48 48 + "b": "2", 49 49 + "c": 3 50 50 + }`, 51 51 + `{"a": 1, "b": "2", "c": 3}`, 52 52 + }, { 53 53 + "json:extra comma", 54 54 + `{ 55 55 + "a": 1, 56 56 + "b": "2", 57 57 + "c": 3, 58 58 + }`, 59 59 + `{"a": 1, "b": "2", "c": 3}`, 60 60 + }, { 61 61 + "json:simplified", 62 62 + `{ 63 63 + a: 1 64 64 + b: "2" 65 65 + c: 3 66 66 + }`, 67 67 + `{a: 1, b: "2", c: 3}`, 68 68 + }, { 69 69 + "not emitted", 70 70 + `a: true 71 71 + b: "2" 72 72 + c: 3 73 73 + `, 74 74 + `a: true, b: "2", c: 3`, 75 75 + }, { 76 76 + "emitted refrencing non-emitted", 77 77 + `a: 1 78 78 + b: "2" 79 79 + c: 3 80 80 + { name: b, total: a + b }`, 81 81 + `a: 1, b: "2", c: 3, {name: b, total: a+b}`, 82 82 + }, { 83 83 + "package file", 84 84 + `package k8s 85 85 + {} 86 86 + `, 87 87 + `package k8s, {}`, 88 88 + }, { 89 89 + "imports group", 90 90 + `package k8s 91 91 + 92 92 + import ( 93 93 + a "foo" 94 94 + "bar/baz" 95 95 + . "model" 96 96 + ) 97 97 + `, 98 98 + `package k8s, import ( a "foo", "bar/baz", . "model" )`, 99 99 + }, { 100 100 + "imports single", 101 101 + `package k8s 102 102 + 103 103 + import a "foo" 104 104 + import "bar/baz" 105 105 + import . "model" 106 106 + `, 107 107 + `package k8s, import a "foo", import "bar/baz", import . "model"`, 108 108 + }, { 109 109 + "collapsed fields", 110 110 + `a b c: 1 111 111 + // job foo { bar: 1 } // TODO error after foo 112 112 + job "foo": { bar: 1 } 113 113 + `, 114 114 + `a: {b: {c: 1}}, job: {"foo": {bar: 1}}`, 115 115 + }, { 116 116 + "identifiers", 117 117 + `// $_: 1, 118 118 + a: {b: {c: d}} 119 119 + c: a 120 120 + d: a.b 121 121 + // e: a."b" // TODO: is an error 122 122 + e: a.b.c 123 123 + "f": f, 124 124 + <X>: X 125 125 + `, 126 126 + "a: {b: {c: d}}, c: a, d: a.b, e: a.b.c, \"f\": f, <X>: X", 127 127 + }, { 128 128 + "expressions", 129 129 + ` a: (2 + 3) * 5 130 130 + b: (2 + 3) + 4 131 131 + c: 2 + 3 + 4 132 132 + d: -1 133 133 + e: !foo 134 134 + f: _|_ 135 135 + `, 136 136 + "a: (2+3)*5, b: (2+3)+4, c: 2+3+4, d: -1, e: !foo, f: _|_", 137 137 + }, { 138 138 + "pseudo keyword expressions", 139 139 + ` a: (2 div 3) mod 5 140 140 + b: (2 quo 3) rem 4 141 141 + c: 2 div 3 div 4 142 142 + `, 143 143 + "a: (2 div 3) mod 5, b: (2 quo 3) rem 4, c: 2 div 3 div 4", 144 144 + }, { 145 145 + "ranges", 146 146 + ` a: 1..2 147 147 + b: 2.0 .. 40.0 148 148 + c: "a".."b" 149 149 + v: (1..2)..(5..10) 150 150 + w: 1..2..3 151 151 + d: 3T..5M 152 152 + `, 153 153 + "a: 1..2, b: 2.0..40.0, c: \"a\"..\"b\", v: (1..2)..(5..10), w: 1..2..3, d: 3T..5M", 154 154 + }, { 155 155 + "indices", 156 156 + `{ 157 157 + a: b[2] 158 158 + b: c[1:2] 159 159 + c: "asdf" 160 160 + d: c ["a"] 161 161 + }`, 162 162 + `{a: b[2], b: c[1:2], c: "asdf", d: c["a"]}`, 163 163 + }, { 164 164 + "lambdas", 165 165 + `{ 166 166 + a(P, Q, r: R) -> { p: P, q: Q } 167 167 + b: a(4002, "s") 168 168 + }`, 169 169 + `{a: (P: _,Q: _,r: R,) -> {p: P, q: Q}, b: a(4002, "s")}`, // c(C): {d(D): {}}}`, 170 170 + }, { 171 171 + "calls", 172 172 + `{ 173 173 + a: b(a.b, c.d) 174 174 + b: a.b(c) 175 175 + }`, 176 176 + `{a: b(a.b, c.d), b: a.b(c)}`, 177 177 + }, { 178 178 + "lists", 179 179 + `{ 180 180 + a: [ 1, 2, 3, b..., c... ] 181 181 + b: [ 1, 2, 3, ], 182 182 + c: [ 1, 183 183 + 2, 184 184 + 3 185 185 + ], 186 186 + d: [ 1+2, 2, 4,] 187 187 + }`, 188 188 + `{a: [1, 2, 3, b..., c...], b: [1, 2, 3], c: [1, 2, 3], d: [1+2, 2, 4]}`, 189 189 + }, { 190 190 + "list types", 191 191 + `{ 192 192 + a: 4*[int] 193 193 + b: 0..5*[ {a: 5} ] 194 194 + c1: [...int] 195 195 + c2: [...] 196 196 + c3: [1, 2, ...int,] 197 197 + }`, 198 198 + `{a: 4*[int], b: 0..5*[{a: 5}], c1: [...int], c2: [...], c3: [1, 2, ...int]}`, 199 199 + }, { 200 200 + "list comprehensions", 201 201 + `{ 202 202 + y: [1,2,3] 203 203 + b: [ x for x in y if x == 1 ], 204 204 + }`, 205 205 + `{y: [1, 2, 3], b: [x for x in y if x==1 ]}`, 206 206 + }, { 207 207 + "field comprehensions", 208 208 + `{ 209 209 + y: { a: 1, b: 2} 210 210 + a: { "\(k)": v for k, v in y if v > 2 } 211 211 + }`, 212 212 + `{y: {a: 1, b: 2}, a: {"\(k)": v for k: v in y if v>2 }}`, 213 213 + }, { 214 214 + "duplicates allowed", 215 215 + `{ 216 216 + a b: 3 217 217 + a: { b: 3 } 218 218 + }`, 219 219 + "{a: {b: 3}, a: {b: 3}}", 220 220 + }, { 221 221 + "templates", 222 222 + `{ 223 223 + <foo>: { a: int } 224 224 + a: { a: 1 } 225 225 + }`, 226 226 + "{<foo>: {a: int}, a: {a: 1}}", 227 227 + }, { 228 228 + "foo", 229 229 + `[ 230 230 + [1], 231 231 + [1, 2], 232 232 + [1, 2, 3], 233 233 + ]`, 234 234 + "[[1], [1, 2], [1, 2, 3]]", 235 235 + }, { 236 236 + "interpolation", 237 237 + `a: "foo \(ident)" 238 238 + b: "bar \(bar) $$$ " 239 239 + c: "nest \( { a: "\( nest ) "}.a ) \(5)" 240 240 + m1: """ 241 241 + multi \(bar) 242 242 + """ 243 243 + m2: ''' 244 244 + \(bar) multi 245 245 + '''`, 246 246 + `a: "foo \(ident)", b: "bar \(bar) $$$ ", c: "nest \({a: "\(nest) "}.a) \(5)", ` + "m1: \"\"\"\n\t\t\t multi \\(bar)\n\t\t\t \"\"\", m2: '''\n\t\t\t \\(bar) multi\n\t\t\t '''", 247 247 + }, { 248 248 + "file comments", 249 249 + `// foo 250 250 + 251 251 + // uni 252 252 + package foo // uniline 253 253 + 254 254 + // file.1 255 255 + // file.2 256 256 + 257 257 + `, 258 258 + "<[0// foo] [d0// uni] [l3// uniline] [3// file.1 // file.2] package foo, >", 259 259 + }, { 260 260 + "line comments", 261 261 + `// doc 262 262 + a: 5 // line 263 263 + b: 6 // lineb 264 264 + // next 265 265 + `, // next is followed by EOF. Ensure it doesn't move to file. 266 266 + "<[d0// doc] [l4// line] a: 5>, " + 267 267 + "<[l4// lineb] [4// next] b: 6>", 268 268 + }, { 269 269 + "alt comments", 270 270 + `// a ... 271 271 + a: 5 // line a 272 272 + 273 273 + // about a 274 274 + 275 275 + // b ... 276 276 + b: // lineb 277 277 + 6 278 278 + 279 279 + // about b 280 280 + 281 281 + c: 7 282 282 + 283 283 + // about c 284 284 + 285 285 + `, 286 286 + "<[d0// a ...] [l4// line a] [4// about a] a: 5>, " + 287 287 + "<[d0// b ...] [l2// lineb] [4// about b] b: 6>, " + 288 288 + "<[4// about c] c: 7>", 289 289 + }, { 290 290 + "expr comments", 291 291 + ` 292 292 + a: 2 + // 2 + 293 293 + 3 + // 3 + 294 294 + 4 // 4 295 295 + l1( // sig 296 296 + ) -> // arrow 297 297 + 4 // expr 298 298 + l2(a // la 299 299 + ) -> // arrow 300 300 + a // l2 301 301 + l3( 302 302 + // param a 303 303 + a : // la 304 304 + 305 305 + // int 306 306 + int // lint 307 307 + ) -> // larrow 308 308 + a + 1 309 309 + `, 310 310 + "<[l4// 4] a: <[l2// 3 +] <[l2// 2 +] 2+3>+4>>, " + 311 311 + "<[l4// expr] l1: <[l1// sig] [l4// arrow] () -> 4>>, " + 312 312 + "<[l4// l2] l2: <[l4// arrow] (<[l1// la] a: _>,) -> a>>, " + 313 313 + "l3: <[l4// larrow] (<[l1// la] [l3// lint] <[d0// param a] a>: <[d0// int] int>>,) -> a+1>", 314 314 + }, { 315 315 + "composit comments", 316 316 + `a : { 317 317 + a: 1, b: 2, c: 3, d: 4 318 318 + // end 319 319 + } 320 320 + b: [ 321 321 + 1, 2, 3, 4, 5, 322 322 + // end 323 323 + ] 324 324 + c: [ 1, 2, 3, 4, // here 325 325 + 5, 6, 7, 8 // and here 326 326 + ] 327 327 + d: { 328 328 + a: /* 8 */ 1 // Hello 329 329 + // Doc 330 330 + b: 2 331 331 + } 332 332 + e1: [ 333 333 + // comment in list body 334 334 + ] 335 335 + e2: { 336 336 + // comment in struct body 337 337 + } 338 338 + `, 339 339 + "a: <[d2// end] {a: 1, b: 2, c: 3, d: 4}>, " + 340 340 + "b: <[d2// end] [1, 2, 3, 4, 5]>, " + 341 341 + "c: [1, 2, 3, <[l1// here] 4>, 5, 6, 7, <[l1// and here] 8>], " + 342 342 + "d: {<[2/* 8 */] [l4// Hello] a: 1>, <[d0// Doc] b: 2>}, " + 343 343 + "e1: <[d2// comment in list body] []>, " + 344 344 + "e2: <[d1// comment in struct body] {}>", 345 345 + }, { 346 346 + "emit comments", 347 347 + `// a comment at the beginning of the file 348 348 + 349 349 + // a second comment 350 350 + 351 351 + // comment 352 352 + a: 5 353 353 + 354 354 + {} 355 355 + 356 356 + // a comment at the end of the file 357 357 + `, 358 358 + "<[0// a comment at the beginning of the file] [0// a second comment] <[d0// comment] a: 5>, <[2// a comment at the end of the file] {}>>", 359 359 + }} 360 360 + for _, tc := range testCases { 361 361 + t.Run(tc.desc, func(t *testing.T) { 362 362 + fset := token.NewFileSet() 363 363 + mode := []Option{AllErrors, ParseLambdas} 364 364 + if strings.Contains(tc.desc, "comments") { 365 365 + mode = append(mode, ParseComments) 366 366 + } 367 367 + f, err := ParseFile(fset, "input", tc.in, mode...) 368 368 + if err != nil { 369 369 + t.Errorf("unexpected error: %v", err) 370 370 + } 371 371 + if got := debugStr(f); got != tc.out { 372 372 + t.Errorf("\ngot %q;\nwant %q", got, tc.out) 373 373 + } 374 374 + }) 375 375 + } 376 376 + } 377 377 + 378 378 + func TestParseExpr(t *testing.T) { 379 379 + // just kicking the tires: 380 380 + // a valid arithmetic expression 381 381 + src := "a + b" 382 382 + x, err := parseExprString(src) 383 383 + if err != nil { 384 384 + t.Errorf("ParseExpr(%q): %v", src, err) 385 385 + } 386 386 + // sanity check 387 387 + if _, ok := x.(*ast.BinaryExpr); !ok { 388 388 + t.Errorf("ParseExpr(%q): got %T, want *BinaryExpr", src, x) 389 389 + } 390 390 + 391 391 + // an invalid expression 392 392 + src = "a + *" 393 393 + if _, err := parseExprString(src); err == nil { 394 394 + t.Errorf("ParseExpr(%q): got no error", src) 395 395 + } 396 396 + 397 397 + // a comma is not permitted unless automatically inserted 398 398 + src = "a + b\n" 399 399 + if _, err := parseExprString(src); err != nil { 400 400 + t.Errorf("ParseExpr(%q): got error %s", src, err) 401 401 + } 402 402 + src = "a + b;" 403 403 + if _, err := parseExprString(src); err == nil { 404 404 + t.Errorf("ParseExpr(%q): got no error", src) 405 405 + } 406 406 + 407 407 + // various other stuff following a valid expression 408 408 + const validExpr = "a + b" 409 409 + const anything = "dh3*#D)#_" 410 410 + for _, c := range "!)]};," { 411 411 + src := validExpr + string(c) + anything 412 412 + if _, err := parseExprString(src); err == nil { 413 413 + t.Errorf("ParseExpr(%q): got no error", src) 414 414 + } 415 415 + } 416 416 + 417 417 + // ParseExpr must not crash 418 418 + for _, src := range valids { 419 419 + parseExprString(src) 420 420 + } 421 421 + } 422 422 + 423 423 + func TestImports(t *testing.T) { 424 424 + var imports = map[string]bool{ 425 425 + `"a"`: true, 426 426 + `"a/b"`: true, 427 427 + `"a.b"`: true, 428 428 + `"m\x61th"`: true, 429 429 + `"greek/αβ"`: true, 430 430 + `""`: false, 431 431 + 432 432 + // Each of these pairs tests both `` vs "" strings 433 433 + // and also use of invalid characters spelled out as 434 434 + // escape sequences and written directly. 435 435 + // For example `"\x00"` tests import "\x00" 436 436 + // while "`\x00`" tests import `<actual-NUL-byte>`. 437 437 + "`a`": true, 438 438 + `"\x00"`: false, 439 439 + "`\x00`": false, 440 440 + `"\x7f"`: false, 441 441 + "`\x7f`": false, 442 442 + `"a!"`: false, 443 443 + "`a!`": false, 444 444 + `"a b"`: false, 445 445 + "`a b`": false, 446 446 + `"a\\b"`: false, 447 447 + "`a\\b`": false, 448 448 + "\"`a`\"": false, 449 449 + "`\"a\"`": false, 450 450 + `"\x80\x80"`: false, 451 451 + "`\x80\x80`": false, 452 452 + `"\xFFFD"`: false, 453 453 + "`\xFFFD`": false, 454 454 + } 455 455 + for path, isValid := range imports { 456 456 + t.Run(path, func(t *testing.T) { 457 457 + src := fmt.Sprintf("package p, import %s", path) 458 458 + _, err := ParseFile(token.NewFileSet(), "", src) 459 459 + switch { 460 460 + case err != nil && isValid: 461 461 + t.Errorf("ParseFile(%s): got %v; expected no error", src, err) 462 462 + case err == nil && !isValid: 463 463 + t.Errorf("ParseFile(%s): got no error; expected one", src) 464 464 + } 465 465 + }) 466 466 + } 467 467 + } 468 468 + 469 469 + func labelName(l ast.Label) string { 470 470 + name, _ := ast.LabelName(l) 471 471 + return name 472 472 + } 473 473 + 474 474 + func getField(file *ast.File, fieldname string) *ast.Field { 475 475 + get := func(elts []ast.Decl, name string) *ast.Field { 476 476 + for _, s := range elts { 477 477 + if s, ok := s.(*ast.Field); ok && labelName(s.Label) == name { 478 478 + return s 479 479 + } 480 480 + } 481 481 + return nil 482 482 + } 483 483 + elts := file.Decls 484 484 + var m *ast.Field 485 485 + for _, p := range strings.Split(fieldname, ".") { 486 486 + m = get(elts, p) 487 487 + if v, ok := m.Value.(*ast.StructLit); ok { 488 488 + elts = v.Elts 489 489 + } else { 490 490 + break 491 491 + } 492 492 + } 493 493 + return m 494 494 + } 495 495 + 496 496 + // Don't use CommentGroup.Text() - we want to see exact comment text. 497 497 + func commentText(c *ast.CommentGroup) string { 498 498 + var buf bytes.Buffer 499 499 + if c != nil { 500 500 + for _, c := range c.List { 501 501 + buf.WriteString(c.Text) 502 502 + } 503 503 + } 504 504 + return buf.String() 505 505 + } 506 506 + 507 507 + // TestIncompleteSelection ensures that an incomplete selector 508 508 + // expression is parsed as a (blank) *SelectorExpr, not a 509 509 + // *BadExpr. 510 510 + func TestIncompleteSelection(t *testing.T) { 511 511 + for _, src := range []string{ 512 512 + "{ a: fmt. }", // at end of object 513 513 + "{ a: fmt.\n\"a\": x }", // not at end of struct 514 514 + } { 515 515 + t.Run("", func(t *testing.T) { 516 516 + fset := token.NewFileSet() 517 517 + f, err := ParseFile(fset, "", src) 518 518 + if err == nil { 519 519 + t.Fatalf("ParseFile(%s) succeeded unexpectedly", src) 520 520 + } 521 521 + 522 522 + const wantErr = "expected selector" 523 523 + if !strings.Contains(err.Error(), wantErr) { 524 524 + t.Errorf("ParseFile returned wrong error %q, want %q", err, wantErr) 525 525 + } 526 526 + 527 527 + var sel *ast.SelectorExpr 528 528 + ast.Walk(f, func(n ast.Node) bool { 529 529 + if n, ok := n.(*ast.SelectorExpr); ok { 530 530 + sel = n 531 531 + } 532 532 + return true 533 533 + }, nil) 534 534 + if sel == nil { 535 535 + t.Fatalf("found no *SelectorExpr: %#v %s", f.Decls[0], debugStr(f)) 536 536 + } 537 537 + const wantSel = "&{{<nil>} fmt _}" 538 538 + if fmt.Sprint(sel) != wantSel { 539 539 + t.Fatalf("found selector %v, want %s", sel, wantSel) 540 540 + } 541 541 + }) 542 542 + } 543 543 + }

+41

cue/parser/performance_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "io/ioutil" 19 19 + "testing" 20 20 + 21 21 + "cuelang.org/go/cue/token" 22 22 + ) 23 23 + 24 24 + var src = readFile("testdata/commas.src") 25 25 + 26 26 + func readFile(filename string) []byte { 27 27 + data, err := ioutil.ReadFile(filename) 28 28 + if err != nil { 29 29 + panic(err) 30 30 + } 31 31 + return data 32 32 + } 33 33 + 34 34 + func BenchmarkParse(b *testing.B) { 35 35 + b.SetBytes(int64(len(src))) 36 36 + for i := 0; i < b.N; i++ { 37 37 + if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments); err != nil { 38 38 + b.Fatalf("benchmark failed due to parse error: %s", err) 39 39 + } 40 40 + } 41 41 + }

+302

cue/parser/print.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + "strconv" 20 20 + "strings" 21 21 + 22 22 + "cuelang.org/go/cue/ast" 23 23 + "cuelang.org/go/cue/token" 24 24 + "cuelang.org/go/internal" 25 25 + ) 26 26 + 27 27 + func init() { 28 28 + internal.DebugStr = debugStr 29 29 + } 30 30 + 31 31 + func debugStr(x interface{}) (out string) { 32 32 + if n, ok := x.(ast.Node); ok { 33 33 + comments := "" 34 34 + for _, g := range n.Comments() { 35 35 + comments += debugStr(g) 36 36 + } 37 37 + if comments != "" { 38 38 + defer func() { out = "<" + comments + out + ">" }() 39 39 + } 40 40 + } 41 41 + switch v := x.(type) { 42 42 + case *ast.File: 43 43 + out := "" 44 44 + if v.Name != nil { 45 45 + out += "package " 46 46 + out += debugStr(v.Name) 47 47 + out += ", " 48 48 + } 49 49 + out += debugStr(v.Decls) 50 50 + return out 51 51 + 52 52 + case *ast.Alias: 53 53 + out := debugStr(v.Ident) 54 54 + out += " = " 55 55 + out += debugStr(v.Expr) 56 56 + return out 57 57 + 58 58 + case *ast.BottomLit: 59 59 + return "_|_" 60 60 + 61 61 + case *ast.BasicLit: 62 62 + return v.Value 63 63 + 64 64 + case *ast.Interpolation: 65 65 + for _, e := range v.Elts { 66 66 + out += debugStr(e) 67 67 + } 68 68 + return out 69 69 + 70 70 + case *ast.EmitDecl: 71 71 + // out := "<" 72 72 + out += debugStr(v.Expr) 73 73 + // out += ">" 74 74 + return out 75 75 + 76 76 + case *ast.ImportDecl: 77 77 + out := "import " 78 78 + if v.Lparen != token.NoPos { 79 79 + out += "( " 80 80 + out += debugStr(v.Specs) 81 81 + out += " )" 82 82 + } else { 83 83 + out += debugStr(v.Specs) 84 84 + } 85 85 + return out 86 86 + 87 87 + case *ast.ComprehensionDecl: 88 88 + out := debugStr(v.Field) 89 89 + out += " " 90 90 + out += debugStr(v.Clauses) 91 91 + return out 92 92 + 93 93 + case *ast.StructLit: 94 94 + out := "{" 95 95 + out += debugStr(v.Elts) 96 96 + out += "}" 97 97 + return out 98 98 + 99 99 + case *ast.ListLit: 100 100 + out := "[" 101 101 + out += debugStr(v.Elts) 102 102 + if v.Ellipsis != token.NoPos || v.Type != nil { 103 103 + if out != "[" { 104 104 + out += ", " 105 105 + } 106 106 + out += "..." 107 107 + if v.Type != nil { 108 108 + out += debugStr(v.Type) 109 109 + } 110 110 + } 111 111 + out += "]" 112 112 + return out 113 113 + 114 114 + case *ast.ListComprehension: 115 115 + out := "[" 116 116 + out += debugStr(v.Expr) 117 117 + out += " " 118 118 + out += debugStr(v.Clauses) 119 119 + out += "]" 120 120 + return out 121 121 + 122 122 + case *ast.ForClause: 123 123 + out := "for " 124 124 + if v.Key != nil { 125 125 + out += debugStr(v.Key) 126 126 + out += ": " 127 127 + } 128 128 + out += debugStr(v.Value) 129 129 + out += " in " 130 130 + out += debugStr(v.Source) 131 131 + return out 132 132 + 133 133 + case *ast.IfClause: 134 134 + out := "if " 135 135 + out += debugStr(v.Condition) 136 136 + return out 137 137 + 138 138 + case *ast.Field: 139 139 + out := debugStr(v.Label) 140 140 + if v.Value != nil { 141 141 + out += ": " 142 142 + out += debugStr(v.Value) 143 143 + } 144 144 + return out 145 145 + 146 146 + case *ast.LambdaExpr: 147 147 + out := "(" 148 148 + for _, m := range v.Params { 149 149 + out += debugStr(m) 150 150 + out += "," 151 151 + } 152 152 + out += ") -> " 153 153 + out += debugStr(v.Expr) 154 154 + return out 155 155 + 156 156 + case *ast.Ident: 157 157 + return v.Name 158 158 + 159 159 + case *ast.ExprLabel: 160 160 + out := "[" 161 161 + out += debugStr(v.Label) 162 162 + out += "]" 163 163 + return out 164 164 + 165 165 + case *ast.TemplateLabel: 166 166 + out := "<" 167 167 + out += debugStr(v.Ident) 168 168 + out += ">" 169 169 + return out 170 170 + 171 171 + case *ast.SelectorExpr: 172 172 + return debugStr(v.X) + "." + debugStr(v.Sel) 173 173 + 174 174 + case *ast.CallExpr: 175 175 + out := debugStr(v.Fun) 176 176 + out += "(" 177 177 + out += debugStr(v.Args) 178 178 + out += ")" 179 179 + return out 180 180 + 181 181 + case *ast.Ellipsis: 182 182 + return debugStr(v.Elt) + "..." 183 183 + 184 184 + case *ast.ParenExpr: 185 185 + out := "(" 186 186 + out += debugStr(v.X) 187 187 + out += ")" 188 188 + return out 189 189 + 190 190 + case *ast.UnaryExpr: 191 191 + return v.Op.String() + debugStr(v.X) 192 192 + 193 193 + case *ast.BinaryExpr: 194 194 + out := debugStr(v.X) 195 195 + op := v.Op.String() 196 196 + if 'a' <= op[0] && op[0] <= 'z' { 197 197 + op = fmt.Sprintf(" %s ", op) 198 198 + } 199 199 + out += op 200 200 + out += debugStr(v.Y) 201 201 + return out 202 202 + 203 203 + case []*ast.CommentGroup: 204 204 + var a []string 205 205 + for _, c := range v { 206 206 + a = append(a, debugStr(c)) 207 207 + } 208 208 + return strings.Join(a, "\n") 209 209 + 210 210 + case *ast.CommentGroup: 211 211 + str := "[" 212 212 + if v.Doc { 213 213 + str += "d" 214 214 + } 215 215 + if v.Line { 216 216 + str += "l" 217 217 + } 218 218 + str += strconv.Itoa(int(v.Position)) 219 219 + var a = []string{} 220 220 + for _, c := range v.List { 221 221 + a = append(a, c.Text) 222 222 + } 223 223 + return str + strings.Join(a, " ") + "] " 224 224 + 225 225 + case *ast.IndexExpr: 226 226 + out := debugStr(v.X) 227 227 + out += "[" 228 228 + out += debugStr(v.Index) 229 229 + out += "]" 230 230 + return out 231 231 + 232 232 + case *ast.SliceExpr: 233 233 + out := debugStr(v.X) 234 234 + out += "[" 235 235 + out += debugStr(v.Low) 236 236 + out += ":" 237 237 + out += debugStr(v.High) 238 238 + out += "]" 239 239 + return out 240 240 + 241 241 + case *ast.ImportSpec: 242 242 + out := "" 243 243 + if v.Name != nil { 244 244 + out += debugStr(v.Name) 245 245 + out += " " 246 246 + } 247 247 + out += debugStr(v.Path) 248 248 + return out 249 249 + 250 250 + case []ast.Decl: 251 251 + if len(v) == 0 { 252 252 + return "" 253 253 + } 254 254 + out := "" 255 255 + for _, d := range v { 256 256 + out += debugStr(d) 257 257 + out += sep 258 258 + } 259 259 + return out[:len(out)-len(sep)] 260 260 + 261 261 + case []ast.Clause: 262 262 + if len(v) == 0 { 263 263 + return "" 264 264 + } 265 265 + out := "" 266 266 + for _, c := range v { 267 267 + out += debugStr(c) 268 268 + out += " " 269 269 + } 270 270 + return out 271 271 + 272 272 + case []ast.Expr: 273 273 + if len(v) == 0 { 274 274 + return "" 275 275 + } 276 276 + out := "" 277 277 + for _, d := range v { 278 278 + out += debugStr(d) 279 279 + out += sep 280 280 + } 281 281 + return out[:len(out)-len(sep)] 282 282 + 283 283 + case []*ast.ImportSpec: 284 284 + if len(v) == 0 { 285 285 + return "" 286 286 + } 287 287 + out := "" 288 288 + for _, d := range v { 289 289 + out += debugStr(d) 290 290 + out += sep 291 291 + } 292 292 + return out[:len(out)-len(sep)] 293 293 + 294 294 + default: 295 295 + if v == nil { 296 296 + return "" 297 297 + } 298 298 + return fmt.Sprintf("<%T>", x) 299 299 + } 300 300 + } 301 301 + 302 302 + const sep = ", "

+209

cue/parser/resolve.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // This file implements scopes and the objects they contain. 16 16 + 17 17 + package parser 18 18 + 19 19 + import ( 20 20 + "bytes" 21 21 + "fmt" 22 22 + 23 23 + "cuelang.org/go/cue/ast" 24 24 + "cuelang.org/go/cue/token" 25 25 + ) 26 26 + 27 27 + // resolve resolves all identifiers in a file. Unresolved identifiers are 28 28 + // recorded in Unresolved. 29 29 + func resolve(f *ast.File, errFn func(pos token.Pos, msg string)) { 30 30 + walk(&scope{errFn: errFn}, f) 31 31 + } 32 32 + 33 33 + // A Scope maintains the set of named language entities declared 34 34 + // in the scope and a link to the immediately surrounding (outer) 35 35 + // scope. 36 36 + // 37 37 + type scope struct { 38 38 + file *ast.File 39 39 + outer *scope 40 40 + node ast.Node 41 41 + index map[string]ast.Node 42 42 + 43 43 + errFn func(p token.Pos, msg string) 44 44 + } 45 45 + 46 46 + func newScope(f *ast.File, outer *scope, node ast.Node, decls []ast.Decl) *scope { 47 47 + const n = 4 // initial scope capacity 48 48 + s := &scope{ 49 49 + file: f, 50 50 + outer: outer, 51 51 + node: node, 52 52 + index: make(map[string]ast.Node, n), 53 53 + errFn: outer.errFn, 54 54 + } 55 55 + for _, d := range decls { 56 56 + switch x := d.(type) { 57 57 + case *ast.Field: 58 58 + if name, ok := ast.LabelName(x.Label); ok { 59 59 + s.insert(name, x.Value) 60 60 + } 61 61 + case *ast.Alias: 62 62 + name := x.Ident.Name 63 63 + s.insert(name, x) 64 64 + // Handle imports 65 65 + } 66 66 + } 67 67 + return s 68 68 + } 69 69 + 70 70 + func (s *scope) insert(name string, n ast.Node) { 71 71 + if _, existing := s.lookup(name); existing != nil { 72 72 + _, isAlias1 := n.(*ast.Alias) 73 73 + _, isAlias2 := existing.(*ast.Alias) 74 74 + if isAlias1 != isAlias2 { 75 75 + s.errFn(n.Pos(), "cannot have alias and non-alias with the same name") 76 76 + return 77 77 + } else if isAlias1 || isAlias2 { 78 78 + s.errFn(n.Pos(), "cannot have two aliases with the same name in the same scope") 79 79 + return 80 80 + } 81 81 + } 82 82 + s.index[name] = n 83 83 + } 84 84 + 85 85 + func (s *scope) lookup(name string) (obj, node ast.Node) { 86 86 + last := s 87 87 + for s != nil { 88 88 + if n, ok := s.index[name]; ok { 89 89 + if last.node == n { 90 90 + return nil, n 91 91 + } 92 92 + return s.node, n 93 93 + } 94 94 + s, last = s.outer, s 95 95 + } 96 96 + return nil, nil 97 97 + } 98 98 + 99 99 + func (s *scope) After(n ast.Node) {} 100 100 + func (s *scope) Before(n ast.Node) (w visitor) { 101 101 + switch x := n.(type) { 102 102 + case *ast.File: 103 103 + s := newScope(x, s, x, x.Decls) 104 104 + // Support imports. 105 105 + for _, d := range x.Decls { 106 106 + walk(s, d) 107 107 + } 108 108 + return nil 109 109 + 110 110 + case *ast.StructLit: 111 111 + return newScope(s.file, s, x, x.Elts) 112 112 + 113 113 + case *ast.ComprehensionDecl: 114 114 + s = scopeClauses(s, x.Clauses) 115 115 + 116 116 + case *ast.ListComprehension: 117 117 + s = scopeClauses(s, x.Clauses) 118 118 + 119 119 + case *ast.Field: 120 120 + switch label := x.Label.(type) { 121 121 + case *ast.Interpolation: 122 122 + walk(s, label) 123 123 + case *ast.ExprLabel: 124 124 + walk(s, x.Label) 125 125 + case *ast.TemplateLabel: 126 126 + s := newScope(s.file, s, x, nil) 127 127 + name, _ := ast.LabelName(label) 128 128 + s.insert(name, x.Label) // Field used for entire lambda. 129 129 + walk(s, x.Value) 130 130 + return nil 131 131 + } 132 132 + // Disallow referring to the current LHS name (this applies recursively) 133 133 + if x.Value != nil { 134 134 + walk(s, x.Value) 135 135 + } 136 136 + return nil 137 137 + 138 138 + case *ast.Alias: 139 139 + // Disallow referring to the current LHS name. 140 140 + name := x.Ident.Name 141 141 + saved := s.index[name] 142 142 + delete(s.index, name) // The same name may still appear in another scope 143 143 + 144 144 + if x.Expr != nil { 145 145 + walk(s, x.Expr) 146 146 + } 147 147 + s.index[name] = saved 148 148 + return nil 149 149 + 150 150 + case *ast.ImportSpec: 151 151 + return nil 152 152 + 153 153 + case *ast.SelectorExpr: 154 154 + walk(s, x.X) 155 155 + return nil 156 156 + 157 157 + case *ast.LambdaExpr: 158 158 + s = newScope(s.file, s, x, nil) 159 159 + for _, p := range x.Params { 160 160 + name, _ := ast.LabelName(p.Label) 161 161 + s.insert(name, p) 162 162 + if p.Value == nil { 163 163 + // TODO: make this optional 164 164 + p.Value = ast.NewIdent("_") 165 165 + s.insert(name, p) 166 166 + } 167 167 + } 168 168 + 169 169 + case *ast.Ident: 170 170 + if obj, node := s.lookup(x.Name); node != nil { 171 171 + x.Node = node 172 172 + x.Scope = obj 173 173 + } else { 174 174 + s.file.Unresolved = append(s.file.Unresolved, x) 175 175 + } 176 176 + return nil 177 177 + } 178 178 + return s 179 179 + } 180 180 + 181 181 + func scopeClauses(s *scope, clauses []ast.Clause) *scope { 182 182 + for _, c := range clauses { 183 183 + if f, ok := c.(*ast.ForClause); ok { // TODO(let): support let clause 184 184 + walk(s, f.Source) 185 185 + s = newScope(s.file, s, f, nil) 186 186 + if f.Key != nil { 187 187 + s.insert(f.Key.Name, f.Key) 188 188 + } 189 189 + s.insert(f.Value.Name, f.Value) 190 190 + } else { 191 191 + walk(s, c) 192 192 + } 193 193 + } 194 194 + return s 195 195 + } 196 196 + 197 197 + // Debugging support 198 198 + func (s *scope) String() string { 199 199 + var buf bytes.Buffer 200 200 + fmt.Fprintf(&buf, "scope %p {", s) 201 201 + if s != nil && len(s.index) > 0 { 202 202 + fmt.Fprintln(&buf) 203 203 + for name := range s.index { 204 204 + fmt.Fprintf(&buf, "\t%v\n", name) 205 205 + } 206 206 + } 207 207 + fmt.Fprintf(&buf, "}\n") 208 208 + return buf.String() 209 209 + }

+50

cue/parser/short_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // This file contains test cases for short valid and invalid programs. 16 16 + 17 17 + package parser 18 18 + 19 19 + import "testing" 20 20 + 21 21 + var valids = []string{ 22 22 + "\n", 23 23 + `{}`, 24 24 + `{ foo: "fmt", bar: () -> { baz: fmt.Service("Hello, World!") }, }`, 25 25 + `{ <Name>: foo }`, 26 26 + `{ a: 3 }`, 27 27 + } 28 28 + 29 29 + func TestValid(t *testing.T) { 30 30 + for _, src := range valids { 31 31 + t.Run(src, func(t *testing.T) { 32 32 + checkErrors(t, src, src) 33 33 + }) 34 34 + } 35 35 + } 36 36 + 37 37 + func TestInvalid(t *testing.T) { 38 38 + invalids := []string{ 39 39 + `foo !/* ERROR "expected label or ':', found '!'" */`, 40 40 + // `foo: /* ERROR "expected operand, found '}'" */}`, // TODO: wrong position 41 41 + `{ <Name 42 42 + /* ERROR "expected '>', found newline" */ >: foo }`, 43 43 + // TODO: 44 44 + // `{ </* ERROR "expected identifier, found newline" */ 45 45 + // Name>: foo }`, 46 46 + } 47 47 + for _, src := range invalids { 48 48 + checkErrors(t, src, src) 49 49 + } 50 50 + }

+35

cue/parser/testdata/commas.src

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // Test case for error messages/parser synchronization 16 16 + // after missing commas. 17 17 + package foo 18 18 + 19 19 + import "path/to/pkg" 20 20 + import name "path/to/pkg" 21 21 + import . "path/to/pkg" 22 22 + import /* ERROR "expected 'STRING', found newline" */ 23 23 + import err /* ERROR "expected 'STRING', found newline" */ 24 24 + 25 25 + foo: [ 26 26 + 0 // legal JSON 27 27 + ] 28 28 + 29 29 + bar: [ 30 30 + 0, 31 31 + 1, 32 32 + 2, 33 33 + 3 34 34 + ] 35 35 +

+5

cue/parser/testdata/test.cue

reviewed

··· 1 1 + 2 2 + import "math" 3 3 + 4 4 + foo: 1 5 5 + bar: "baz"

+281

cue/parser/walk.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package parser 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + 20 20 + "cuelang.org/go/cue/ast" 21 21 + "cuelang.org/go/cue/token" 22 22 + ) 23 23 + 24 24 + // TODO: use ast.Walk or adopt that version to allow visitors. 25 25 + 26 26 + // A visitor's before method is invoked for each node encountered by Walk. 27 27 + // If the result visitor w is not nil, Walk visits each of the children 28 28 + // of node with the visitor w, followed by a call of w.After. 29 29 + type visitor interface { 30 30 + Before(node ast.Node) (w visitor) 31 31 + After(node ast.Node) 32 32 + } 33 33 + 34 34 + // Helper functions for common node lists. They may be empty. 35 35 + 36 36 + func walkIdentList(v visitor, list []*ast.Ident) { 37 37 + for _, x := range list { 38 38 + walk(v, x) 39 39 + } 40 40 + } 41 41 + 42 42 + func walkExprList(v visitor, list []ast.Expr) { 43 43 + for _, x := range list { 44 44 + walk(v, x) 45 45 + } 46 46 + } 47 47 + 48 48 + func walkDeclList(v visitor, list []ast.Decl) { 49 49 + for _, x := range list { 50 50 + walk(v, x) 51 51 + } 52 52 + } 53 53 + 54 54 + // walk traverses an AST in depth-first order: It starts by calling 55 55 + // v.Visit(node); node must not be nil. If the visitor w returned by 56 56 + // v.Visit(node) is not nil, walk is invoked recursively with visitor 57 57 + // w for each of the non-nil children of node, followed by a call of 58 58 + // w.Visit(nil). 59 59 + // 60 60 + func walk(v visitor, node ast.Node) { 61 61 + if v = v.Before(node); v == nil { 62 62 + return 63 63 + } 64 64 + 65 65 + // TODO: record the comment groups and interleave with the values like for 66 66 + // parsing and printing? 67 67 + for _, c := range node.Comments() { 68 68 + walk(v, c) 69 69 + } 70 70 + 71 71 + // walk children 72 72 + // (the order of the cases matches the order 73 73 + // of the corresponding node types in go) 74 74 + switch n := node.(type) { 75 75 + // Comments and fields 76 76 + case *ast.Comment: 77 77 + // nothing to do 78 78 + 79 79 + case *ast.CommentGroup: 80 80 + for _, c := range n.List { 81 81 + walk(v, c) 82 82 + } 83 83 + 84 84 + case *ast.Field: 85 85 + walk(v, n.Label) 86 86 + if n.Value != nil { 87 87 + walk(v, n.Value) 88 88 + } 89 89 + 90 90 + case *ast.LambdaExpr: 91 91 + for _, p := range n.Params { 92 92 + walk(v, p) 93 93 + } 94 94 + walk(v, n.Expr) 95 95 + 96 96 + case *ast.StructLit: 97 97 + for _, f := range n.Elts { 98 98 + walk(v, f) 99 99 + } 100 100 + 101 101 + // Expressions 102 102 + case *ast.BottomLit, *ast.BadExpr, *ast.Ident, *ast.BasicLit: 103 103 + // nothing to do 104 104 + 105 105 + case *ast.ExprLabel: 106 106 + walk(v, n.Label) 107 107 + 108 108 + case *ast.TemplateLabel: 109 109 + walk(v, n.Ident) 110 110 + 111 111 + case *ast.Interpolation: 112 112 + for _, e := range n.Elts { 113 113 + walk(v, e) 114 114 + } 115 115 + 116 116 + case *ast.Ellipsis: 117 117 + if n.Elt != nil { 118 118 + walk(v, n.Elt) 119 119 + } 120 120 + 121 121 + case *ast.ListLit: 122 122 + walkExprList(v, n.Elts) 123 123 + if n.Type != nil { 124 124 + walk(v, n.Type) 125 125 + } 126 126 + 127 127 + case *ast.ParenExpr: 128 128 + walk(v, n.X) 129 129 + 130 130 + case *ast.SelectorExpr: 131 131 + walk(v, n.X) 132 132 + walk(v, n.Sel) 133 133 + 134 134 + case *ast.IndexExpr: 135 135 + walk(v, n.X) 136 136 + walk(v, n.Index) 137 137 + 138 138 + case *ast.SliceExpr: 139 139 + walk(v, n.X) 140 140 + if n.Low != nil { 141 141 + walk(v, n.Low) 142 142 + } 143 143 + if n.High != nil { 144 144 + walk(v, n.High) 145 145 + } 146 146 + 147 147 + case *ast.CallExpr: 148 148 + walk(v, n.Fun) 149 149 + walkExprList(v, n.Args) 150 150 + 151 151 + case *ast.UnaryExpr: 152 152 + walk(v, n.X) 153 153 + 154 154 + case *ast.BinaryExpr: 155 155 + walk(v, n.X) 156 156 + walk(v, n.Y) 157 157 + 158 158 + // Declarations 159 159 + case *ast.ImportSpec: 160 160 + if n.Name != nil { 161 161 + walk(v, n.Name) 162 162 + } 163 163 + walk(v, n.Path) 164 164 + 165 165 + case *ast.BadDecl: 166 166 + // nothing to do 167 167 + 168 168 + case *ast.ImportDecl: 169 169 + for _, s := range n.Specs { 170 170 + walk(v, s) 171 171 + } 172 172 + 173 173 + case *ast.EmitDecl: 174 174 + walk(v, n.Expr) 175 175 + 176 176 + case *ast.Alias: 177 177 + walk(v, n.Ident) 178 178 + walk(v, n.Expr) 179 179 + 180 180 + case *ast.ComprehensionDecl: 181 181 + walk(v, n.Field) 182 182 + for _, c := range n.Clauses { 183 183 + walk(v, c) 184 184 + } 185 185 + 186 186 + // Files and packages 187 187 + case *ast.File: 188 188 + if n.Name != nil { 189 189 + walk(v, n.Name) 190 190 + } 191 191 + walkDeclList(v, n.Decls) 192 192 + // don't walk n.Comments - they have been 193 193 + // visited already through the individual 194 194 + // nodes 195 195 + 196 196 + case *ast.ListComprehension: 197 197 + walk(v, n.Expr) 198 198 + for _, c := range n.Clauses { 199 199 + walk(v, c) 200 200 + } 201 201 + 202 202 + case *ast.ForClause: 203 203 + if n.Key != nil { 204 204 + walk(v, n.Key) 205 205 + } 206 206 + walk(v, n.Value) 207 207 + walk(v, n.Source) 208 208 + 209 209 + case *ast.IfClause: 210 210 + walk(v, n.Condition) 211 211 + 212 212 + default: 213 213 + panic(fmt.Sprintf("Walk: unexpected node type %T", n)) 214 214 + } 215 215 + 216 216 + v.After(node) 217 217 + } 218 218 + 219 219 + type inspector struct { 220 220 + before func(ast.Node) bool 221 221 + after func(ast.Node) 222 222 + 223 223 + commentStack []commentFrame 224 224 + current commentFrame 225 225 + } 226 226 + 227 227 + type commentFrame struct { 228 228 + cg []*ast.CommentGroup 229 229 + pos int8 230 230 + } 231 231 + 232 232 + func (f *inspector) Before(node ast.Node) visitor { 233 233 + if f.before == nil || f.before(node) { 234 234 + f.commentStack = append(f.commentStack, f.current) 235 235 + f.current = commentFrame{cg: node.Comments()} 236 236 + f.visitComments(f.current.pos) 237 237 + return f 238 238 + } 239 239 + return nil 240 240 + } 241 241 + 242 242 + func (f *inspector) After(node ast.Node) { 243 243 + f.visitComments(127) 244 244 + p := len(f.commentStack) - 1 245 245 + f.current = f.commentStack[p] 246 246 + f.commentStack = f.commentStack[:p] 247 247 + f.current.pos++ 248 248 + if f.after != nil { 249 249 + f.after(node) 250 250 + } 251 251 + } 252 252 + 253 253 + func (f *inspector) Token(t token.Token) { 254 254 + f.current.pos++ 255 255 + } 256 256 + 257 257 + func (f *inspector) setPos(i int8) { 258 258 + f.current.pos = i 259 259 + } 260 260 + 261 261 + func (f *inspector) visitComments(pos int8) { 262 262 + c := &f.current 263 263 + for ; len(c.cg) > 0; c.cg = c.cg[1:] { 264 264 + cg := c.cg[0] 265 265 + if cg.Position == pos { 266 266 + continue 267 267 + } 268 268 + if f.before == nil || f.before(cg) { 269 269 + for _, c := range cg.List { 270 270 + if f.before == nil || f.before(c) { 271 271 + if f.after != nil { 272 272 + f.after(c) 273 273 + } 274 274 + } 275 275 + } 276 276 + if f.after != nil { 277 277 + f.after(cg) 278 278 + } 279 279 + } 280 280 + } 281 281 + }

+20

internal/internal.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package internal // import "cuelang.org/go/internal" 16 16 + 17 17 + // TODO: refactor packages as to make this package unnecessary. 18 18 + 19 19 + // DebugStr prints a syntax node. 20 20 + var DebugStr func(x interface{}) string