+564

cue/token/position.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package token 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + "sort" 20 20 + "sync" 21 21 + ) 22 22 + 23 23 + // ----------------------------------------------------------------------------- 24 24 + // Positions 25 25 + 26 26 + // Position describes an arbitrary source position 27 27 + // including the file, line, and column location. 28 28 + // A Position is valid if the line number is > 0. 29 29 + type Position struct { 30 30 + Filename string // filename, if any 31 31 + Offset int // offset, starting at 0 32 32 + Line int // line number, starting at 1 33 33 + Column int // column number, starting at 1 (byte count) 34 34 + // RelPos Pos // relative position information 35 35 + } 36 36 + 37 37 + // IsValid reports whether the position is valid. 38 38 + func (pos *Position) IsValid() bool { return pos.Line > 0 } 39 39 + 40 40 + // String returns a string in one of several forms: 41 41 + // 42 42 + // file:line:column valid position with file name 43 43 + // line:column valid position without file name 44 44 + // file invalid position with file name 45 45 + // - invalid position without file name 46 46 + // 47 47 + func (pos Position) String() string { 48 48 + s := pos.Filename 49 49 + if pos.IsValid() { 50 50 + if s != "" { 51 51 + s += ":" 52 52 + } 53 53 + s += fmt.Sprintf("%d:%d", pos.Line, pos.Column) 54 54 + } 55 55 + if s == "" { 56 56 + s = "-" 57 57 + } 58 58 + return s 59 59 + } 60 60 + 61 61 + // Pos is a compact encoding of a source position within a file set, as well as 62 62 + // relative positioning information. It can be converted into a Position for a 63 63 + // more convenient, but much larger, representation. 64 64 + // 65 65 + // The Pos value for a given file is a number in the range [base, base+size], 66 66 + // where base and size are specified when adding the file to the file set via 67 67 + // AddFile. 68 68 + // 69 69 + // To create the Pos value for a specific source offset (measured in bytes), 70 70 + // first add the respective file to the current file set using FileSet.AddFile 71 71 + // and then call File.Pos(offset) for that file. Given a Pos value p for a 72 72 + // specific file set fset, the corresponding Position value is obtained by 73 73 + // calling fset.Position(p). 74 74 + // 75 75 + // Pos values can be compared directly with the usual comparison operators: If 76 76 + // two Pos values p and q are in the same file, comparing p and q is equivalent 77 77 + // to comparing the respective source file offsets. If p and q are in different 78 78 + // files, p < q is true if the file implied by p was added to the respective 79 79 + // file set before the file implied by cue. 80 80 + type Pos int 81 81 + 82 82 + // NoPos is the zero value for Pos; there is no file and line information 83 83 + // associated with it, and NoPos().IsValid() is false. NoPos is always 84 84 + // smaller than any other Pos value. The corresponding Position value 85 85 + // for NoPos is the zero value for Position. 86 86 + const NoPos Pos = 0 87 87 + 88 88 + // RelPos indicates the relative position of token to the previous token. 89 89 + type RelPos int 90 90 + 91 91 + const ( 92 92 + // NoRelPos indicates no relative position is specified. 93 93 + NoRelPos RelPos = iota 94 94 + 95 95 + // Elided indicates that the token for which this position is defined is 96 96 + // not rendered at all. 97 97 + Elided 98 98 + 99 99 + // NoSpace indicates there is no whitespace after this token. 100 100 + NoSpace 101 101 + 102 102 + // Blank means there is horizontal space after this token. 103 103 + Blank 104 104 + 105 105 + // Newline means there is a single newline after this token. 106 106 + Newline 107 107 + 108 108 + // NewSection means there are two or more newlines after this token. 109 109 + NewSection 110 110 + 111 111 + relMask Pos = 0xf 112 112 + relShift = 4 113 113 + ) 114 114 + 115 115 + var relNames = []string{ 116 116 + "invalid", "elided", "nospace", "blank", "newline", "section", 117 117 + } 118 118 + 119 119 + func (p RelPos) String() string { return relNames[p] } 120 120 + 121 121 + // HasRelPos repors whether p has a relative position. 122 122 + func (p Pos) HasRelPos() bool { 123 123 + return p&relMask != 0 124 124 + 125 125 + } 126 126 + 127 127 + func (p Pos) Add(n int) Pos { 128 128 + return p + toPos(index(n)) 129 129 + } 130 130 + 131 131 + // IsValid reports whether the position is valid. 132 132 + func (p Pos) IsValid() bool { 133 133 + return p != NoPos 134 134 + } 135 135 + 136 136 + // IsNewline reports whether the relative information suggests this node should 137 137 + // be printed on a new lien. 138 138 + func (p Pos) IsNewline() bool { 139 139 + return p.RelPos() >= Newline 140 140 + } 141 141 + 142 142 + func (p Pos) WithRel(rel RelPos) Pos { 143 143 + return p&^relMask | Pos(rel) 144 144 + } 145 145 + 146 146 + func (p Pos) RelPos() RelPos { 147 147 + return RelPos(p & relMask) 148 148 + } 149 149 + 150 150 + func (p Pos) index() index { 151 151 + return index(p) >> relShift 152 152 + } 153 153 + 154 154 + func toPos(x index) Pos { 155 155 + return (Pos(x) << relShift) 156 156 + } 157 157 + 158 158 + // ----------------------------------------------------------------------------- 159 159 + // File 160 160 + 161 161 + type index int 162 162 + 163 163 + // A File is a handle for a file belonging to a FileSet. 164 164 + // A File has a name, size, and line offset table. 165 165 + type File struct { 166 166 + set *FileSet 167 167 + name string // file name as provided to AddFile 168 168 + base index // Pos index range for this file is [base...base+size] 169 169 + size index // file size as provided to AddFile 170 170 + 171 171 + // lines and infos are protected by set.mutex 172 172 + lines []index // lines contains the offset of the first character for each line (the first entry is always 0) 173 173 + infos []lineInfo 174 174 + } 175 175 + 176 176 + // Name returns the file name of file f as registered with AddFile. 177 177 + func (f *File) Name() string { 178 178 + return f.name 179 179 + } 180 180 + 181 181 + // Base returns the base offset of file f as registered with AddFile. 182 182 + func (f *File) Base() int { 183 183 + return int(f.base) 184 184 + } 185 185 + 186 186 + // Size returns the size of file f as registered with AddFile. 187 187 + func (f *File) Size() int { 188 188 + return int(f.size) 189 189 + } 190 190 + 191 191 + // LineCount returns the number of lines in file f. 192 192 + func (f *File) LineCount() int { 193 193 + f.set.mutex.RLock() 194 194 + n := len(f.lines) 195 195 + f.set.mutex.RUnlock() 196 196 + return n 197 197 + } 198 198 + 199 199 + // AddLine adds the line offset for a new line. 200 200 + // The line offset must be larger than the offset for the previous line 201 201 + // and smaller than the file size; otherwise the line offset is ignored. 202 202 + // 203 203 + func (f *File) AddLine(offset int) { 204 204 + x := index(offset) 205 205 + f.set.mutex.Lock() 206 206 + if i := len(f.lines); (i == 0 || f.lines[i-1] < x) && x < f.size { 207 207 + f.lines = append(f.lines, x) 208 208 + } 209 209 + f.set.mutex.Unlock() 210 210 + } 211 211 + 212 212 + // MergeLine merges a line with the following line. It is akin to replacing 213 213 + // the newline character at the end of the line with a space (to not change the 214 214 + // remaining offsets). To obtain the line number, consult e.g. Position.Line. 215 215 + // MergeLine will panic if given an invalid line number. 216 216 + // 217 217 + func (f *File) MergeLine(line int) { 218 218 + if line <= 0 { 219 219 + panic("illegal line number (line numbering starts at 1)") 220 220 + } 221 221 + f.set.mutex.Lock() 222 222 + defer f.set.mutex.Unlock() 223 223 + if line >= len(f.lines) { 224 224 + panic("illegal line number") 225 225 + } 226 226 + // To merge the line numbered <line> with the line numbered <line+1>, 227 227 + // we need to remove the entry in lines corresponding to the line 228 228 + // numbered <line+1>. The entry in lines corresponding to the line 229 229 + // numbered <line+1> is located at index <line>, since indices in lines 230 230 + // are 0-based and line numbers are 1-based. 231 231 + copy(f.lines[line:], f.lines[line+1:]) 232 232 + f.lines = f.lines[:len(f.lines)-1] 233 233 + } 234 234 + 235 235 + // SetLines sets the line offsets for a file and reports whether it succeeded. 236 236 + // The line offsets are the offsets of the first character of each line; 237 237 + // for instance for the content "ab\nc\n" the line offsets are {0, 3}. 238 238 + // An empty file has an empty line offset table. 239 239 + // Each line offset must be larger than the offset for the previous line 240 240 + // and smaller than the file size; otherwise SetLines fails and returns 241 241 + // false. 242 242 + // Callers must not mutate the provided slice after SetLines returns. 243 243 + // 244 244 + func (f *File) SetLines(lines []int) bool { 245 245 + // verify validity of lines table 246 246 + size := f.size 247 247 + for i, offset := range lines { 248 248 + if i > 0 && offset <= lines[i-1] || size <= index(offset) { 249 249 + return false 250 250 + } 251 251 + } 252 252 + 253 253 + // set lines table 254 254 + f.set.mutex.Lock() 255 255 + f.lines = f.lines[:0] 256 256 + for _, l := range lines { 257 257 + f.lines = append(f.lines, index(l)) 258 258 + } 259 259 + f.set.mutex.Unlock() 260 260 + return true 261 261 + } 262 262 + 263 263 + // SetLinesForContent sets the line offsets for the given file content. 264 264 + // It ignores position-altering //line comments. 265 265 + func (f *File) SetLinesForContent(content []byte) { 266 266 + var lines []index 267 267 + line := index(0) 268 268 + for offset, b := range content { 269 269 + if line >= 0 { 270 270 + lines = append(lines, line) 271 271 + } 272 272 + line = -1 273 273 + if b == '\n' { 274 274 + line = index(offset) + 1 275 275 + } 276 276 + } 277 277 + 278 278 + // set lines table 279 279 + f.set.mutex.Lock() 280 280 + f.lines = lines 281 281 + f.set.mutex.Unlock() 282 282 + } 283 283 + 284 284 + // A lineInfo object describes alternative file and line number 285 285 + // information (such as provided via a //line comment in a .go 286 286 + // file) for a given file offset. 287 287 + type lineInfo struct { 288 288 + // fields are exported to make them accessible to gob 289 289 + Offset int 290 290 + Filename string 291 291 + Line int 292 292 + } 293 293 + 294 294 + // AddLineInfo adds alternative file and line number information for 295 295 + // a given file offset. The offset must be larger than the offset for 296 296 + // the previously added alternative line info and smaller than the 297 297 + // file size; otherwise the information is ignored. 298 298 + // 299 299 + // AddLineInfo is typically used to register alternative position 300 300 + // information for //line filename:line comments in source files. 301 301 + // 302 302 + func (f *File) AddLineInfo(offset int, filename string, line int) { 303 303 + x := index(offset) 304 304 + f.set.mutex.Lock() 305 305 + if i := len(f.infos); i == 0 || index(f.infos[i-1].Offset) < x && x < f.size { 306 306 + f.infos = append(f.infos, lineInfo{offset, filename, line}) 307 307 + } 308 308 + f.set.mutex.Unlock() 309 309 + } 310 310 + 311 311 + // Pos returns the Pos value for the given file offset; 312 312 + // the offset must be <= f.Size(). 313 313 + // f.Pos(f.Offset(p)) == p. 314 314 + // 315 315 + func (f *File) Pos(offset int, rel RelPos) Pos { 316 316 + if index(offset) > f.size { 317 317 + panic("illegal file offset") 318 318 + } 319 319 + return toPos(f.base+index(offset)) + Pos(rel) 320 320 + } 321 321 + 322 322 + // Offset returns the offset for the given file position p; 323 323 + // p must be a valid Pos value in that file. 324 324 + // f.Offset(f.Pos(offset)) == offset. 325 325 + // 326 326 + func (f *File) Offset(p Pos) int { 327 327 + x := p.index() 328 328 + if x < f.base || x > f.base+index(f.size) { 329 329 + panic("illegal Pos value") 330 330 + } 331 331 + return int(x - f.base) 332 332 + } 333 333 + 334 334 + // Line returns the line number for the given file position p; 335 335 + // p must be a Pos value in that file or NoPos. 336 336 + // 337 337 + func (f *File) Line(p Pos) int { 338 338 + return f.Position(p).Line 339 339 + } 340 340 + 341 341 + func searchLineInfos(a []lineInfo, x int) int { 342 342 + return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1 343 343 + } 344 344 + 345 345 + // unpack returns the filename and line and column number for a file offset. 346 346 + // If adjusted is set, unpack will return the filename and line information 347 347 + // possibly adjusted by //line comments; otherwise those comments are ignored. 348 348 + // 349 349 + func (f *File) unpack(offset index, adjusted bool) (filename string, line, column int) { 350 350 + filename = f.name 351 351 + if i := searchInts(f.lines, offset); i >= 0 { 352 352 + line, column = int(i+1), int(offset-f.lines[i]+1) 353 353 + } 354 354 + if adjusted && len(f.infos) > 0 { 355 355 + // almost no files have extra line infos 356 356 + if i := searchLineInfos(f.infos, int(offset)); i >= 0 { 357 357 + alt := &f.infos[i] 358 358 + filename = alt.Filename 359 359 + if i := searchInts(f.lines, index(alt.Offset)); i >= 0 { 360 360 + line += alt.Line - i - 1 361 361 + } 362 362 + } 363 363 + } 364 364 + return 365 365 + } 366 366 + 367 367 + func (f *File) position(p Pos, adjusted bool) (pos Position) { 368 368 + offset := p.index() - f.base 369 369 + pos.Offset = int(offset) 370 370 + pos.Filename, pos.Line, pos.Column = f.unpack(offset, adjusted) 371 371 + return 372 372 + } 373 373 + 374 374 + // PositionFor returns the Position value for the given file position p. 375 375 + // If adjusted is set, the position may be adjusted by position-altering 376 376 + // //line comments; otherwise those comments are ignored. 377 377 + // p must be a Pos value in f or NoPos. 378 378 + // 379 379 + func (f *File) PositionFor(p Pos, adjusted bool) (pos Position) { 380 380 + x := p.index() 381 381 + if p != NoPos { 382 382 + if x < f.base || x > f.base+f.size { 383 383 + panic("illegal Pos value") 384 384 + } 385 385 + pos = f.position(p, adjusted) 386 386 + } 387 387 + return 388 388 + } 389 389 + 390 390 + // Position returns the Position value for the given file position p. 391 391 + // Calling f.Position(p) is equivalent to calling f.PositionFor(p, true). 392 392 + // 393 393 + func (f *File) Position(p Pos) (pos Position) { 394 394 + return f.PositionFor(p, true) 395 395 + } 396 396 + 397 397 + // A FileSet represents a set of source files. 398 398 + // Methods of file sets are synchronized; multiple goroutines 399 399 + // may invoke them concurrently. 400 400 + type FileSet struct { 401 401 + mutex sync.RWMutex // protects the file set 402 402 + base int // base offset for the next file 403 403 + files []*File // list of files in the order added to the set 404 404 + last *File // cache of last file looked up 405 405 + } 406 406 + 407 407 + // NewFileSet creates a new file set. 408 408 + func NewFileSet() *FileSet { 409 409 + return &FileSet{ 410 410 + base: 1, // 0 == NoPos 411 411 + } 412 412 + } 413 413 + 414 414 + // Base returns the minimum base offset that must be provided to 415 415 + // AddFile when adding the next file. 416 416 + func (s *FileSet) Base() int { 417 417 + s.mutex.RLock() 418 418 + b := s.base 419 419 + s.mutex.RUnlock() 420 420 + return b 421 421 + 422 422 + } 423 423 + 424 424 + // AddFile adds a new file with a given filename, base offset, and file size 425 425 + // to the file set s and returns the file. Multiple files may have the same 426 426 + // name. The base offset must not be smaller than the FileSet's Base(), and 427 427 + // size must not be negative. As a special case, if a negative base is provided, 428 428 + // the current value of the FileSet's Base() is used instead. 429 429 + // 430 430 + // Adding the file will set the file set's Base() value to base + size + 1 431 431 + // as the minimum base value for the next file. The following relationship 432 432 + // exists between a Pos value p for a given file offset offs: 433 433 + // 434 434 + // int(p) = base + offs 435 435 + // 436 436 + // with offs in the range [0, size] and thus p in the range [base, base+size]. 437 437 + // For convenience, File.Pos may be used to create file-specific position 438 438 + // values from a file offset. 439 439 + func (s *FileSet) AddFile(filename string, base, size int) *File { 440 440 + s.mutex.Lock() 441 441 + defer s.mutex.Unlock() 442 442 + if base < 0 { 443 443 + base = s.base 444 444 + } 445 445 + if base < s.base || size < 0 { 446 446 + panic("illegal base or size") 447 447 + } 448 448 + // base >= s.base && size >= 0 449 449 + f := &File{s, filename, index(base), index(size), []index{0}, nil} 450 450 + base += size + 1 // +1 because EOF also has a position 451 451 + if base < 0 { 452 452 + panic("token.Pos offset overflow (> 2G of source code in file set)") 453 453 + } 454 454 + // add the file to the file set 455 455 + s.base = base 456 456 + s.files = append(s.files, f) 457 457 + s.last = f 458 458 + return f 459 459 + } 460 460 + 461 461 + // Iterate calls f for the files in the file set in the order they were added 462 462 + // until f returns false. 463 463 + // 464 464 + func (s *FileSet) Iterate(f func(*File) bool) { 465 465 + for i := 0; ; i++ { 466 466 + var file *File 467 467 + s.mutex.RLock() 468 468 + if i < len(s.files) { 469 469 + file = s.files[i] 470 470 + } 471 471 + s.mutex.RUnlock() 472 472 + if file == nil || !f(file) { 473 473 + break 474 474 + } 475 475 + } 476 476 + } 477 477 + 478 478 + func searchFiles(a []*File, x index) int { 479 479 + return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 480 480 + } 481 481 + 482 482 + func (s *FileSet) file(p Pos) *File { 483 483 + x := p.index() 484 484 + s.mutex.RLock() 485 485 + // common case: p is in last file 486 486 + if f := s.last; f != nil && f.base <= x && x <= f.base+f.size { 487 487 + s.mutex.RUnlock() 488 488 + return f 489 489 + } 490 490 + // p is not in last file - search all files 491 491 + if i := searchFiles(s.files, x); i >= 0 { 492 492 + f := s.files[i] 493 493 + // f.base <= int(p) by definition of searchFiles 494 494 + if x <= f.base+f.size { 495 495 + s.mutex.RUnlock() 496 496 + s.mutex.Lock() 497 497 + s.last = f // race is ok - s.last is only a cache 498 498 + s.mutex.Unlock() 499 499 + return f 500 500 + } 501 501 + } 502 502 + s.mutex.RUnlock() 503 503 + return nil 504 504 + } 505 505 + 506 506 + // File returns the file that contains the position p. 507 507 + // If no such file is found (for instance for p == NoPos), 508 508 + // the result is nil. 509 509 + // 510 510 + func (s *FileSet) File(p Pos) (f *File) { 511 511 + if p.index() != 0 { 512 512 + f = s.file(p) 513 513 + } 514 514 + return 515 515 + } 516 516 + 517 517 + // PositionFor converts a Pos p in the fileset into a Position value. 518 518 + // If adjusted is set, the position may be adjusted by position-altering 519 519 + // //line comments; otherwise those comments are ignored. 520 520 + // p must be a Pos value in s or NoPos. 521 521 + // 522 522 + func (s *FileSet) PositionFor(p Pos, adjusted bool) (pos Position) { 523 523 + if p.index() != 0 { 524 524 + if f := s.file(p); f != nil { 525 525 + s.mutex.RLock() 526 526 + pos = f.position(p, adjusted) 527 527 + s.mutex.RUnlock() 528 528 + } 529 529 + } 530 530 + return 531 531 + } 532 532 + 533 533 + // Position converts a Pos p in the fileset into a Position value. 534 534 + // Calling s.Position(p) is equivalent to calling s.PositionFor(p, true). 535 535 + // 536 536 + func (s *FileSet) Position(p Pos) (pos Position) { 537 537 + return s.PositionFor(p, true) 538 538 + } 539 539 + 540 540 + // ----------------------------------------------------------------------------- 541 541 + // Helper functions 542 542 + 543 543 + func searchInts(a []index, x index) int { 544 544 + // This function body is a manually inlined version of: 545 545 + // 546 546 + // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 547 547 + // 548 548 + // With better compiler optimizations, this may not be needed in the 549 549 + // future, but at the moment this change improves the go/printer 550 550 + // benchmark performance by ~30%. This has a direct impact on the 551 551 + // speed of gofmt and thus seems worthwhile (2011-04-29). 552 552 + // TODO(gri): Remove this when compilers have caught up. 553 553 + i, j := 0, len(a) 554 554 + for i < j { 555 555 + h := i + (j-i)/2 // avoid overflow when computing h 556 556 + // i ≤ h < j 557 557 + if a[h] <= x { 558 558 + i = h + 1 559 559 + } else { 560 560 + j = h 561 561 + } 562 562 + } 563 563 + return i - 1 564 564 + }

+336

cue/token/position_test.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + package token 16 16 + 17 17 + import ( 18 18 + "fmt" 19 19 + "math/rand" 20 20 + "sync" 21 21 + "testing" 22 22 + ) 23 23 + 24 24 + func checkPos(t *testing.T, msg string, got, want Position) { 25 25 + if got.Filename != want.Filename { 26 26 + t.Errorf("%s: got filename = %q; want %q", msg, got.Filename, want.Filename) 27 27 + } 28 28 + if got.Offset != want.Offset { 29 29 + t.Errorf("%s: got offset = %d; want %d", msg, got.Offset, want.Offset) 30 30 + } 31 31 + if got.Line != want.Line { 32 32 + t.Errorf("%s: got line = %d; want %d", msg, got.Line, want.Line) 33 33 + } 34 34 + if got.Column != want.Column { 35 35 + t.Errorf("%s: got column = %d; want %d", msg, got.Column, want.Column) 36 36 + } 37 37 + } 38 38 + 39 39 + func TestNoPos(t *testing.T) { 40 40 + if NoPos.IsValid() { 41 41 + t.Errorf("NoPos should not be valid") 42 42 + } 43 43 + var fset *FileSet 44 44 + checkPos(t, "nil NoPos", fset.Position(NoPos), Position{}) 45 45 + fset = NewFileSet() 46 46 + checkPos(t, "fset NoPos", fset.Position(NoPos), Position{}) 47 47 + } 48 48 + 49 49 + var tests = []struct { 50 50 + filename string 51 51 + source []byte // may be nil 52 52 + size int 53 53 + lines []int 54 54 + }{ 55 55 + {"a", []byte{}, 0, []int{}}, 56 56 + {"b", []byte("01234"), 5, []int{0}}, 57 57 + {"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}}, 58 58 + {"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}}, 59 59 + {"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}}, 60 60 + {"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}}, 61 61 + {"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}}, 62 62 + {"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}}, 63 63 + } 64 64 + 65 65 + func linecol(lines []int, offs int) (int, int) { 66 66 + prevLineOffs := 0 67 67 + for line, lineOffs := range lines { 68 68 + if offs < lineOffs { 69 69 + return line, offs - prevLineOffs + 1 70 70 + } 71 71 + prevLineOffs = lineOffs 72 72 + } 73 73 + return len(lines), offs - prevLineOffs + 1 74 74 + } 75 75 + 76 76 + func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) { 77 77 + for offs := 0; offs < f.Size(); offs++ { 78 78 + p := f.Pos(offs, 0) 79 79 + offs2 := f.Offset(p) 80 80 + if offs2 != offs { 81 81 + t.Errorf("%s, Offset: got offset %d; want %d", f.Name(), offs2, offs) 82 82 + } 83 83 + line, col := linecol(lines, offs) 84 84 + msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) 85 85 + checkPos(t, msg, f.Position(f.Pos(offs, 0)), Position{f.Name(), offs, line, col}) 86 86 + checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col}) 87 87 + } 88 88 + } 89 89 + 90 90 + func makeTestSource(size int, lines []int) []byte { 91 91 + src := make([]byte, size) 92 92 + for _, offs := range lines { 93 93 + if offs > 0 { 94 94 + src[offs-1] = '\n' 95 95 + } 96 96 + } 97 97 + return src 98 98 + } 99 99 + 100 100 + func TestPositions(t *testing.T) { 101 101 + const delta = 7 // a non-zero base offset increment 102 102 + fset := NewFileSet() 103 103 + for _, test := range tests { 104 104 + // verify consistency of test case 105 105 + if test.source != nil && len(test.source) != test.size { 106 106 + t.Errorf("%s: inconsistent test case: got file size %d; want %d", test.filename, len(test.source), test.size) 107 107 + } 108 108 + 109 109 + // add file and verify name and size 110 110 + f := fset.AddFile(test.filename, fset.Base()+delta, test.size) 111 111 + if f.Name() != test.filename { 112 112 + t.Errorf("got filename %q; want %q", f.Name(), test.filename) 113 113 + } 114 114 + if f.Size() != test.size { 115 115 + t.Errorf("%s: got file size %d; want %d", f.Name(), f.Size(), test.size) 116 116 + } 117 117 + if fset.File(f.Pos(0, 0)) != f { 118 118 + t.Errorf("%s: f.Pos(0, 0) was not found in f", f.Name()) 119 119 + } 120 120 + 121 121 + // add lines individually and verify all positions 122 122 + for i, offset := range test.lines { 123 123 + f.AddLine(offset) 124 124 + if f.LineCount() != i+1 { 125 125 + t.Errorf("%s, AddLine: got line count %d; want %d", f.Name(), f.LineCount(), i+1) 126 126 + } 127 127 + // adding the same offset again should be ignored 128 128 + f.AddLine(offset) 129 129 + if f.LineCount() != i+1 { 130 130 + t.Errorf("%s, AddLine: got unchanged line count %d; want %d", f.Name(), f.LineCount(), i+1) 131 131 + } 132 132 + verifyPositions(t, fset, f, test.lines[0:i+1]) 133 133 + } 134 134 + 135 135 + // add lines with SetLines and verify all positions 136 136 + if ok := f.SetLines(test.lines); !ok { 137 137 + t.Errorf("%s: SetLines failed", f.Name()) 138 138 + } 139 139 + if f.LineCount() != len(test.lines) { 140 140 + t.Errorf("%s, SetLines: got line count %d; want %d", f.Name(), f.LineCount(), len(test.lines)) 141 141 + } 142 142 + verifyPositions(t, fset, f, test.lines) 143 143 + 144 144 + // add lines with SetLinesForContent and verify all positions 145 145 + src := test.source 146 146 + if src == nil { 147 147 + // no test source available - create one from scratch 148 148 + src = makeTestSource(test.size, test.lines) 149 149 + } 150 150 + f.SetLinesForContent(src) 151 151 + if f.LineCount() != len(test.lines) { 152 152 + t.Errorf("%s, SetLinesForContent: got line count %d; want %d", f.Name(), f.LineCount(), len(test.lines)) 153 153 + } 154 154 + verifyPositions(t, fset, f, test.lines) 155 155 + } 156 156 + } 157 157 + 158 158 + func TestLineInfo(t *testing.T) { 159 159 + fset := NewFileSet() 160 160 + f := fset.AddFile("foo", fset.Base(), 500) 161 161 + lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401} 162 162 + // add lines individually and provide alternative line information 163 163 + for _, offs := range lines { 164 164 + f.AddLine(offs) 165 165 + f.AddLineInfo(offs, "bar", 42) 166 166 + } 167 167 + // verify positions for all offsets 168 168 + for offs := 0; offs <= f.Size(); offs++ { 169 169 + p := f.Pos(offs, 0) 170 170 + _, col := linecol(lines, offs) 171 171 + msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) 172 172 + checkPos(t, msg, f.Position(f.Pos(offs, 0)), Position{"bar", offs, 42, col}) 173 173 + checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col}) 174 174 + } 175 175 + } 176 176 + 177 177 + func TestFiles(t *testing.T) { 178 178 + fset := NewFileSet() 179 179 + for i, test := range tests { 180 180 + base := fset.Base() 181 181 + if i%2 == 1 { 182 182 + // Setting a negative base is equivalent to 183 183 + // fset.Base(), so test some of each. 184 184 + base = -1 185 185 + } 186 186 + fset.AddFile(test.filename, base, test.size) 187 187 + j := 0 188 188 + fset.Iterate(func(f *File) bool { 189 189 + if f.Name() != tests[j].filename { 190 190 + t.Errorf("got filename = %s; want %s", f.Name(), tests[j].filename) 191 191 + } 192 192 + j++ 193 193 + return true 194 194 + }) 195 195 + if j != i+1 { 196 196 + t.Errorf("got %d files; want %d", j, i+1) 197 197 + } 198 198 + } 199 199 + } 200 200 + 201 201 + // FileSet.File should return nil if Pos is past the end of the FileSet. 202 202 + func TestFileSetPastEnd(t *testing.T) { 203 203 + fset := NewFileSet() 204 204 + for _, test := range tests { 205 205 + fset.AddFile(test.filename, fset.Base(), test.size) 206 206 + } 207 207 + if f := fset.File(toPos(index(fset.Base()))); f != nil { 208 208 + t.Errorf("got %v, want nil", f) 209 209 + } 210 210 + } 211 211 + 212 212 + func TestFileSetCacheUnlikely(t *testing.T) { 213 213 + fset := NewFileSet() 214 214 + offsets := make(map[string]index) 215 215 + for _, test := range tests { 216 216 + offsets[test.filename] = index(fset.Base()) 217 217 + fset.AddFile(test.filename, fset.Base(), test.size) 218 218 + } 219 219 + for file, pos := range offsets { 220 220 + f := fset.File(toPos(pos)) 221 221 + if f.Name() != file { 222 222 + t.Errorf("got %q at position %d, want %q", f.Name(), pos, file) 223 223 + } 224 224 + } 225 225 + } 226 226 + 227 227 + // issue 4345. Test that concurrent use of FileSet.Pos does not trigger a 228 228 + // race in the FileSet position cache. 229 229 + func TestFileSetRace(t *testing.T) { 230 230 + fset := NewFileSet() 231 231 + for i := 0; i < 100; i++ { 232 232 + fset.AddFile(fmt.Sprintf("file-%d", i), fset.Base(), 1031) 233 233 + } 234 234 + max := int32(fset.Base()) 235 235 + var stop sync.WaitGroup 236 236 + r := rand.New(rand.NewSource(7)) 237 237 + for i := 0; i < 2; i++ { 238 238 + r := rand.New(rand.NewSource(r.Int63())) 239 239 + stop.Add(1) 240 240 + go func() { 241 241 + for i := 0; i < 1000; i++ { 242 242 + fset.Position(Pos(r.Int31n(max))) 243 243 + } 244 244 + stop.Done() 245 245 + }() 246 246 + } 247 247 + stop.Wait() 248 248 + } 249 249 + 250 250 + // issue 16548. Test that concurrent use of File.AddLine and FileSet.PositionFor 251 251 + // does not trigger a race in the FileSet position cache. 252 252 + func TestFileSetRace2(t *testing.T) { 253 253 + const N = 1e3 254 254 + var ( 255 255 + fset = NewFileSet() 256 256 + file = fset.AddFile("", -1, N) 257 257 + ch = make(chan int, 2) 258 258 + ) 259 259 + 260 260 + go func() { 261 261 + for i := 0; i < N; i++ { 262 262 + file.AddLine(i) 263 263 + } 264 264 + ch <- 1 265 265 + }() 266 266 + 267 267 + go func() { 268 268 + pos := file.Pos(0, 0) 269 269 + for i := 0; i < N; i++ { 270 270 + fset.PositionFor(pos, false) 271 271 + } 272 272 + ch <- 1 273 273 + }() 274 274 + 275 275 + <-ch 276 276 + <-ch 277 277 + } 278 278 + 279 279 + func TestPositionFor(t *testing.T) { 280 280 + src := []byte(` 281 281 + foo 282 282 + b 283 283 + ar 284 284 + //line :100 285 285 + foobar 286 286 + //line bar:3 287 287 + done 288 288 + `) 289 289 + 290 290 + const filename = "foo" 291 291 + fset := NewFileSet() 292 292 + f := fset.AddFile(filename, fset.Base(), len(src)) 293 293 + f.SetLinesForContent(src) 294 294 + 295 295 + // verify position info 296 296 + for i, offs := range f.lines { 297 297 + got1 := f.PositionFor(f.Pos(int(offs), 0), false) 298 298 + got2 := f.PositionFor(f.Pos(int(offs), 0), true) 299 299 + got3 := f.Position(f.Pos(int(offs), 0)) 300 300 + want := Position{filename, int(offs), i + 1, 1} 301 301 + checkPos(t, "1. PositionFor unadjusted", got1, want) 302 302 + checkPos(t, "1. PositionFor adjusted", got2, want) 303 303 + checkPos(t, "1. Position", got3, want) 304 304 + } 305 305 + 306 306 + // manually add //line info on lines l1, l2 307 307 + const l1, l2 = 5, 7 308 308 + f.AddLineInfo(int(f.lines[l1-1]), "", 100) 309 309 + f.AddLineInfo(int(f.lines[l2-1]), "bar", 3) 310 310 + 311 311 + // unadjusted position info must remain unchanged 312 312 + for i, offs := range f.lines { 313 313 + got1 := f.PositionFor(f.Pos(int(offs), 0), false) 314 314 + want := Position{filename, int(offs), i + 1, 1} 315 315 + checkPos(t, "2. PositionFor unadjusted", got1, want) 316 316 + } 317 317 + 318 318 + // adjusted position info should have changed 319 319 + for i, offs := range f.lines { 320 320 + got2 := f.PositionFor(f.Pos(int(offs), 0), true) 321 321 + got3 := f.Position(f.Pos(int(offs), 0)) 322 322 + want := Position{filename, int(offs), i + 1, 1} 323 323 + // manually compute wanted filename and line 324 324 + line := want.Line 325 325 + if i+1 >= l1 { 326 326 + want.Filename = "" 327 327 + want.Line = line - l1 + 100 328 328 + } 329 329 + if i+1 >= l2 { 330 330 + want.Filename = "bar" 331 331 + want.Line = line - l2 + 3 332 332 + } 333 333 + checkPos(t, "3. PositionFor adjusted", got2, want) 334 334 + checkPos(t, "3. Position", got3, want) 335 335 + } 336 336 + }

+263

cue/token/token.go

reviewed

··· 1 1 + // Copyright 2018 The CUE Authors 2 2 + // 3 3 + // Licensed under the Apache License, Version 2.0 (the "License"); 4 4 + // you may not use this file except in compliance with the License. 5 5 + // You may obtain a copy of the License at 6 6 + // 7 7 + // http://www.apache.org/licenses/LICENSE-2.0 8 8 + // 9 9 + // Unless required by applicable law or agreed to in writing, software 10 10 + // distributed under the License is distributed on an "AS IS" BASIS, 11 11 + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 12 + // See the License for the specific language governing permissions and 13 13 + // limitations under the License. 14 14 + 15 15 + // Package token defines constants representing the lexical tokens of the Go 16 16 + // programming language and basic operations on tokens (printing, predicates). 17 17 + package token // import "cuelang.org/go/cue/token" 18 18 + 19 19 + import "strconv" 20 20 + 21 21 + // Token is the set of lexical tokens of the CUE configuration language. 22 22 + type Token int 23 23 + 24 24 + // The list of tokens. 25 25 + const ( 26 26 + // Special tokens 27 27 + ILLEGAL Token = iota 28 28 + EOF 29 29 + COMMENT 30 30 + 31 31 + literalBeg 32 32 + // Identifiers and basic type literals 33 33 + // (these tokens stand for classes of literals) 34 34 + IDENT // main, _tmp 35 35 + INT // 12_345Mi, 0700, 0xdeadbeef, 1.2M 36 36 + FLOAT // 123.45, 37 37 + // DURATION // 3m4s TODO 38 38 + STRING // "abc" 39 39 + INTERPOLATION // a part of a template string, e.g. `"age: \(` 40 40 + BOTTOM // _|_ 41 41 + 42 42 + literalEnd 43 43 + 44 44 + operatorBeg 45 45 + // Operators and delimiters 46 46 + ADD // + 47 47 + SUB // - 48 48 + MUL // * 49 49 + POW // ^ 50 50 + QUO // / 51 51 + REM // % 52 52 + 53 53 + IQUO // quo 54 54 + IREM // rem 55 55 + IDIV // div 56 56 + IMOD // mod 57 57 + 58 58 + UNIFY // & 59 59 + DISJUNCTION // | 60 60 + 61 61 + LAND // && 62 62 + LOR // || 63 63 + 64 64 + BIND // = 65 65 + EQL // == 66 66 + LSS // < 67 67 + GTR // > 68 68 + NOT // ! 69 69 + ARROW // <- 70 70 + LAMBDA // -> 71 71 + 72 72 + NEQ // != 73 73 + LEQ // <= 74 74 + GEQ // >= 75 75 + 76 76 + LPAREN // ( 77 77 + LBRACK // [ 78 78 + LBRACE // { 79 79 + COMMA // , 80 80 + PERIOD // . 81 81 + RANGE // .. 82 82 + ELLIPSIS // ... 83 83 + 84 84 + RPAREN // ) 85 85 + RBRACK // ] 86 86 + RBRACE // } 87 87 + SEMICOLON // : 88 88 + COLON // : 89 89 + operatorEnd 90 90 + 91 91 + keywordBeg 92 92 + 93 93 + IF 94 94 + FOR 95 95 + IN 96 96 + LET 97 97 + 98 98 + TRUE 99 99 + FALSE 100 100 + NULL 101 101 + 102 102 + keywordEnd 103 103 + ) 104 104 + 105 105 + var tokens = [...]string{ 106 106 + ILLEGAL: "ILLEGAL", 107 107 + 108 108 + EOF: "EOF", 109 109 + COMMENT: "COMMENT", 110 110 + 111 111 + IDENT: "IDENT", 112 112 + INT: "INT", 113 113 + FLOAT: "FLOAT", 114 114 + // DURATION: "DURATION", // TODO 115 115 + STRING: "STRING", 116 116 + INTERPOLATION: "INTERPOLATION", 117 117 + 118 118 + ADD: "+", 119 119 + SUB: "-", 120 120 + MUL: "*", 121 121 + POW: "^", 122 122 + QUO: "/", 123 123 + REM: "%", 124 124 + 125 125 + IQUO: "quo", 126 126 + IREM: "rem", 127 127 + IDIV: "div", 128 128 + IMOD: "mod", 129 129 + 130 130 + UNIFY: "&", 131 131 + DISJUNCTION: "|", 132 132 + 133 133 + LAND: "&&", 134 134 + LOR: "||", 135 135 + 136 136 + BIND: "=", 137 137 + EQL: "==", 138 138 + LSS: "<", 139 139 + GTR: ">", 140 140 + NOT: "!", 141 141 + ARROW: "<-", 142 142 + LAMBDA: "->", 143 143 + 144 144 + NEQ: "!=", 145 145 + LEQ: "<=", 146 146 + GEQ: ">=", 147 147 + 148 148 + LPAREN: "(", 149 149 + LBRACK: "[", 150 150 + LBRACE: "{", 151 151 + COMMA: ",", 152 152 + PERIOD: ".", 153 153 + RANGE: "..", 154 154 + ELLIPSIS: "...", 155 155 + 156 156 + RPAREN: ")", 157 157 + RBRACK: "]", 158 158 + RBRACE: "}", 159 159 + SEMICOLON: ";", 160 160 + COLON: ":", 161 161 + 162 162 + BOTTOM: "_|_", 163 163 + 164 164 + FALSE: "false", 165 165 + TRUE: "true", 166 166 + NULL: "null", 167 167 + 168 168 + FOR: "for", 169 169 + IF: "if", 170 170 + IN: "in", 171 171 + LET: "let", 172 172 + } 173 173 + 174 174 + // String returns the string corresponding to the token tok. 175 175 + // For operators, delimiters, and keywords the string is the actual 176 176 + // token character sequence (e.g., for the token ADD, the string is 177 177 + // "+"). For all other tokens the string corresponds to the token 178 178 + // constant name (e.g. for the token IDENT, the string is "IDENT"). 179 179 + func (tok Token) String() string { 180 180 + s := "" 181 181 + if 0 <= tok && tok < Token(len(tokens)) { 182 182 + s = tokens[tok] 183 183 + } 184 184 + if s == "" { 185 185 + s = "token(" + strconv.Itoa(int(tok)) + ")" 186 186 + } 187 187 + return s 188 188 + } 189 189 + 190 190 + // A set of constants for precedence-based expression parsing. 191 191 + // Non-operators have lowest precedence, followed by operators 192 192 + // starting with precedence 1 up to unary operators. The highest 193 193 + // precedence serves as "catch-all" precedence for selector, 194 194 + // indexing, and other operator and delimiter tokens. 195 195 + const ( 196 196 + LowestPrec = lowestPrec 197 197 + UnaryPrec = unaryPrec 198 198 + HighestPrec = highestPrec 199 199 + ) 200 200 + 201 201 + const ( 202 202 + lowestPrec = 0 // non-operators 203 203 + unaryPrec = 8 204 204 + highestPrec = 9 205 205 + ) 206 206 + 207 207 + // Precedence returns the operator precedence of the binary 208 208 + // operator op. If op is not a binary operator, the result 209 209 + // is LowestPrecedence. 210 210 + // 211 211 + func (tok Token) Precedence() int { 212 212 + switch tok { 213 213 + case DISJUNCTION: 214 214 + return 1 215 215 + case UNIFY: 216 216 + return 2 217 217 + case LOR: 218 218 + return 3 219 219 + case LAND: 220 220 + return 4 221 221 + case EQL, NEQ, LSS, LEQ, GTR, GEQ: 222 222 + return 5 223 223 + case ADD, SUB: 224 224 + return 6 225 225 + case MUL, QUO, REM, IDIV, IMOD, IQUO, IREM: 226 226 + return 7 227 227 + case RANGE: 228 228 + return 8 229 229 + } 230 230 + return lowestPrec 231 231 + } 232 232 + 233 233 + var keywords map[string]Token 234 234 + 235 235 + func init() { 236 236 + keywords = make(map[string]Token) 237 237 + for i := keywordBeg + 1; i < keywordEnd; i++ { 238 238 + keywords[tokens[i]] = i 239 239 + } 240 240 + } 241 241 + 242 242 + // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). 243 243 + // 244 244 + func Lookup(ident string) Token { 245 245 + if tok, isKeyword := keywords[ident]; isKeyword { 246 246 + return tok 247 247 + } 248 248 + return IDENT 249 249 + } 250 250 + 251 251 + // Predicates 252 252 + 253 253 + // IsLiteral returns true for tokens corresponding to identifiers 254 254 + // and basic type literals; it returns false otherwise. 255 255 + func (tok Token) IsLiteral() bool { return literalBeg < tok && tok < literalEnd } 256 256 + 257 257 + // IsOperator returns true for tokens corresponding to operators and 258 258 + // delimiters; it returns false otherwise. 259 259 + func (tok Token) IsOperator() bool { return operatorBeg < tok && tok < operatorEnd } 260 260 + 261 261 + // IsKeyword returns true for tokens corresponding to keywords; 262 262 + // it returns false otherwise. 263 263 + func (tok Token) IsKeyword() bool { return keywordBeg < tok && tok < keywordEnd }