internal/lsp: model embedded files as packages

+265

cmd/cue/cmd/integration/workspace/embedded_test.go

··· 1 + package workspace 2 + 3 + import ( 4 + "testing" 5 + 6 + "cuelang.org/go/internal/golangorgx/gopls/protocol" 7 + . "cuelang.org/go/internal/golangorgx/gopls/test/integration" 8 + "github.com/go-quicktest/qt" 9 + ) 10 + 11 + func TestEmbedSimple(t *testing.T) { 12 + const files = ` 13 + -- cue.mod/module.cue -- 14 + module: "mod.example/x" 15 + language: version: "v0.16.0" 16 + 17 + -- a.cue -- 18 + @extern(embed) 19 + package a 20 + 21 + out: _ @embed(file=data/data.json) 22 + -- data/data.json -- 23 + {"field": true} 24 + ` 25 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 26 + rootURI := env.Sandbox.Workdir.RootURI() 27 + 28 + env.OpenFile("a.cue") 29 + env.Await( 30 + env.DoneWithOpen(), 31 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 32 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 33 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 34 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 35 + ) 36 + }) 37 + } 38 + 39 + func TestEmbedMissingExtern(t *testing.T) { 40 + const files = ` 41 + -- cue.mod/module.cue -- 42 + module: "mod.example/x" 43 + language: version: "v0.16.0" 44 + 45 + -- a.cue -- 46 + package a 47 + 48 + out: _ @embed(file=data/data.json) 49 + -- data/data.json -- 50 + {"field": true} 51 + ` 52 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 53 + rootURI := env.Sandbox.Workdir.RootURI() 54 + 55 + env.OpenFile("a.cue") 56 + env.Await( 57 + env.DoneWithOpen(), 58 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 59 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 60 + NoLogMatching(protocol.Debug, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 61 + ) 62 + }) 63 + } 64 + 65 + func TestEmbedMissingFile(t *testing.T) { 66 + const files = ` 67 + -- cue.mod/module.cue -- 68 + module: "mod.example/x" 69 + language: version: "v0.16.0" 70 + 71 + -- a.cue -- 72 + @extern(embed) 73 + package a 74 + 75 + out: _ @embed(file=data/missing.json) 76 + ` 77 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 78 + rootURI := env.Sandbox.Workdir.RootURI() 79 + 80 + env.OpenFile("a.cue") 81 + env.Await( 82 + env.DoneWithOpen(), 83 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 84 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 85 + NoLogMatching(protocol.Debug, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 86 + ) 87 + }) 88 + } 89 + 90 + func TestEmbedLateFile(t *testing.T) { 91 + const files = ` 92 + -- cue.mod/module.cue -- 93 + module: "mod.example/x" 94 + language: version: "v0.16.0" 95 + 96 + -- a.cue -- 97 + @extern(embed) 98 + package a 99 + 100 + out: _ @embed(file=data/late.json) 101 + ` 102 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 103 + rootURI := env.Sandbox.Workdir.RootURI() 104 + 105 + env.OpenFile("a.cue") 106 + env.Await( 107 + env.DoneWithOpen(), 108 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 109 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 110 + NoLogMatching(protocol.Debug, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 111 + ) 112 + env.CreateBuffer("data/late.json", `{"field": 5}`) 113 + // embeds are "upstream", so we do reload the downstream a.cue - 114 + // it's the same as if one of its imports had changed. 115 + env.Await( 116 + env.DoneWithOpen(), 117 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 118 + LogExactf(protocol.Debug, 2, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 119 + ) 120 + }) 121 + } 122 + 123 + func TestEmbedDeleteFile(t *testing.T) { 124 + const files = ` 125 + -- cue.mod/module.cue -- 126 + module: "mod.example/x" 127 + language: version: "v0.16.0" 128 + 129 + -- a.cue -- 130 + @extern(embed) 131 + package a 132 + 133 + out: _ @embed(file=data/data.json) 134 + -- data/data.json -- 135 + {"field": true} 136 + ` 137 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 138 + rootURI := env.Sandbox.Workdir.RootURI() 139 + 140 + env.OpenFile("a.cue") 141 + env.Await( 142 + env.DoneWithOpen(), 143 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 144 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 145 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 146 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 147 + ) 148 + err := env.Sandbox.Workdir.RemoveFile(env.Ctx, "data/data.json") 149 + qt.Assert(t, qt.IsNil(err)) 150 + env.CheckForFileChanges() 151 + // We should see a delete of the embed pkg, and a reload of the x@v0:a 152 + env.Await( 153 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Deleted`, rootURI), 154 + LogExactf(protocol.Debug, 2, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 155 + ) 156 + }) 157 + } 158 + 159 + func TestEmbedMissingGlob(t *testing.T) { 160 + const files = ` 161 + -- cue.mod/module.cue -- 162 + module: "mod.example/x" 163 + language: version: "v0.16.0" 164 + 165 + -- a.cue -- 166 + @extern(embed) 167 + package a 168 + 169 + out: _ @embed(glob=data/*.json) 170 + ` 171 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 172 + rootURI := env.Sandbox.Workdir.RootURI() 173 + 174 + env.OpenFile("a.cue") 175 + env.Await( 176 + env.DoneWithOpen(), 177 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 178 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 179 + NoLogMatching(protocol.Debug, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 180 + ) 181 + }) 182 + } 183 + 184 + func TestEmbedLateGlob(t *testing.T) { 185 + const files = ` 186 + -- cue.mod/module.cue -- 187 + module: "mod.example/x" 188 + language: version: "v0.16.0" 189 + 190 + -- a.cue -- 191 + @extern(embed) 192 + package a 193 + 194 + out: _ @embed(glob=data/*.json) 195 + ` 196 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 197 + rootURI := env.Sandbox.Workdir.RootURI() 198 + 199 + env.OpenFile("a.cue") 200 + env.Await( 201 + env.DoneWithOpen(), 202 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 203 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 204 + NoLogMatching(protocol.Debug, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 205 + ) 206 + env.CreateBuffer("data/late1.json", `{"field": 5}`) 207 + // embeds are "upstream", so we do reload the downstream a.cue - 208 + // it's the same as if one of its imports had changed. 209 + env.Await( 210 + env.DoneWithOpen(), 211 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 212 + LogExactf(protocol.Debug, 2, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 213 + ) 214 + 215 + env.CreateBuffer("data/late2.json", `{"field": 6}`) 216 + // We will create+load the late2.json package, and reload a.cue, but we won't reload late1.json. 217 + env.Await( 218 + env.DoneWithOpen(), 219 + LogMatching(protocol.Debug, 2, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 220 + LogExactf(protocol.Debug, 3, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 221 + ) 222 + }) 223 + } 224 + 225 + func TestEmbedDeleteGlob(t *testing.T) { 226 + const files = ` 227 + -- cue.mod/module.cue -- 228 + module: "mod.example/x" 229 + language: version: "v0.16.0" 230 + 231 + -- a.cue -- 232 + @extern(embed) 233 + package a 234 + 235 + out: _ @embed(glob=data/*.json) 236 + -- data/data1.json -- 237 + {"field": true} 238 + -- data/data2.json -- 239 + {"field": true} 240 + ` 241 + WithOptions(RootURIAsDefaultFolder()).Run(t, files, func(t *testing.T, env *Env) { 242 + rootURI := env.Sandbox.Workdir.RootURI() 243 + 244 + env.OpenFile("a.cue") 245 + env.Await( 246 + env.DoneWithOpen(), 247 + LogExactf(protocol.Debug, 1, false, "Module dir=%v module=mod.example/x@v0 Loading packages [mod.example/x@v0:a]", rootURI), 248 + LogExactf(protocol.Debug, 1, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 249 + // Both embedded files/packages get loaded 250 + LogMatching(protocol.Debug, 2, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Created`, rootURI), 251 + LogMatching(protocol.Debug, 2, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 252 + ) 253 + err := env.Sandbox.Workdir.RemoveFile(env.Ctx, "data/data1.json") 254 + qt.Assert(t, qt.IsNil(err)) 255 + env.CheckForFileChanges() 256 + env.Await( 257 + // One embedded file gets deleted. 258 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Deleted`, rootURI), 259 + // The survivor does *not* get reloaded (no need). 260 + LogMatching(protocol.Debug, 2, false, `Package dirs=\[%v/data\] importPath=mod\.example/x/data@v0:_.+ Reloaded`, rootURI), 261 + // And the downstream will get reloaded. 262 + LogExactf(protocol.Debug, 2, false, "Package dirs=[%v] importPath=mod.example/x@v0:a Reloaded", rootURI), 263 + ) 264 + }) 265 + }

+2 -3

cmd/cue/cmd/integration/workspace/standalone_test.go

··· 1 1 package workspace 2 2 3 3 import ( 4 - "fmt" 5 4 "testing" 6 5 7 6 "cuelang.org/go/internal/golangorgx/gopls/protocol" ··· 89 88 // If a file is missing a package declaration then we add 90 89 // one. So if there is a valid module then such files will 91 90 // not be treated as standalone. 92 - LogMatching(protocol.Debug, fmt.Sprintf(`Package dirs=\[%v/a\] importPath=cue\.example\.net/a@v0:_.+ Created`, rootURI), 1, false), 93 - LogMatching(protocol.Debug, fmt.Sprintf(`Package dirs=\[%v/a\] importPath=cue\.example\.net/a@v0:_.+ Reloaded`, rootURI), 1, false), 91 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/a\] importPath=cue\.example\.net/a@v0:_.+ Created`, rootURI), 92 + LogMatching(protocol.Debug, 1, false, `Package dirs=\[%v/a\] importPath=cue\.example\.net/a@v0:_.+ Reloaded`, rootURI), 94 93 ) 95 94 }) 96 95 })

+153 -1

cue/interpreter/embed/embed.go

··· 100 100 "strings" 101 101 102 102 "cuelang.org/go/cue" 103 + "cuelang.org/go/cue/ast" 103 104 "cuelang.org/go/cue/build" 104 105 "cuelang.org/go/cue/errors" 105 106 "cuelang.org/go/cue/token" ··· 129 130 } 130 131 131 132 func (i interpreter) Kind() string { 132 - return "embed" 133 + return EmbedKind 133 134 } 135 + 136 + const EmbedKind = "embed" 134 137 135 138 // NewCompiler returns a compiler that can decode and embed files that exist 136 139 // within a CUE module. ··· 430 433 _, v := value.ToInternal(val) 431 434 return v, nil 432 435 } 436 + 437 + // EmbeddedPaths validates the provided attributes as embed 438 + // attributes, returning a slice of [Embed] structs for attributes 439 + // that were successfully validated, and errors for those which were 440 + // not. The filepath should be the filepath of the file from which 441 + // these attributes were extracted, and relative to whatever root is 442 + // going to be used in calls to [Embed.Matches] and [Embed.FindAll]. 443 + func EmbeddedPaths(filepath string, attrsByField map[*ast.Field]*internal.Attr) ([]*Embed, errors.Error) { 444 + if len(attrsByField) == 0 { 445 + return nil, nil 446 + } 447 + var errs errors.Error 448 + embeds := make([]*Embed, 0, len(attrsByField)) 449 + for field, attr := range attrsByField { 450 + if attr.Err != nil { 451 + errs = errors.Append(errs, attr.Err) 452 + continue 453 + } 454 + file, glob, typ, allowEmptyGlob, err := validateAttr(attr) 455 + if err != nil { 456 + errs = errors.Append(errs, err) 457 + continue 458 + } 459 + embed := &Embed{ 460 + Field: field, 461 + Attribute: attr, 462 + FilePath: filepath, 463 + Type: typ, 464 + } 465 + if file != "" { 466 + embed.interpreter = &embeddedFile{ 467 + filepath: file, 468 + } 469 + embeds = append(embeds, embed) 470 + } else if glob != "" { 471 + embed.interpreter = &embeddedGlob{ 472 + glob: glob, 473 + allowEmptyGlob: allowEmptyGlob, 474 + } 475 + embeds = append(embeds, embed) 476 + } 477 + } 478 + return embeds, errs 479 + } 480 + 481 + type Embed struct { 482 + Field *ast.Field 483 + Attribute *internal.Attr 484 + FilePath string 485 + Type string 486 + interpreter embedInterpreter 487 + } 488 + 489 + // Matches reports whether the provided filepath is matched by this 490 + // [Embed] attribute. The filepath should be relative to the same root 491 + // as the filepath provided to [EmbeddedPaths]. E.g. if in 492 + // `/wibble/foo/bar.cue` you have `@embed(filename=a/b.json)`, and 493 + // `foo/bar.cue` is the filepath passed to [EmbeddedPaths], then 494 + // [Embed.Matches] will return true if called with `foo/a/b.json`. 495 + func (e *Embed) Matches(filepath string) bool { 496 + return e.interpreter.matches(e, filepath) 497 + } 498 + 499 + // FindAll uses the provided fs to report all the filepaths that 500 + // match this [Embed] attribute. The fs must be relative to the same 501 + // root as the filepath provided to [EmbeddedPaths]. I.e. for the 502 + // filepath provided to [EmbeddedPaths], iofs.Stat(fs, filepath) 503 + // should be accessing the same file which contained this [Embed] 504 + // attribute. 505 + func (e *Embed) FindAll(fs iofs.FS) ([]string, error) { 506 + return e.interpreter.findAll(e, fs) 507 + } 508 + 509 + // IsGlob reports whether this [Embed] attribute represents a glob 510 + // embedding. 511 + func (e *Embed) IsGlob() bool { 512 + _, isGlob := e.interpreter.(*embeddedGlob) 513 + return isGlob 514 + } 515 + 516 + type embedInterpreter interface { 517 + // NB: All filepaths (including any within the Embed) are 518 + // considered relative to the same root. 519 + 520 + matches(e *Embed, filepath string) bool 521 + findAll(e *Embed, fs iofs.FS) ([]string, error) 522 + } 523 + 524 + type embeddedFile struct { 525 + filepath string 526 + } 527 + 528 + func (ef *embeddedFile) matches(e *Embed, filepath string) bool { 529 + dir := path.Dir(e.FilePath) 530 + return filepath == path.Join(dir, ef.filepath) 531 + } 532 + 533 + func (ef *embeddedFile) findAll(e *Embed, fs iofs.FS) ([]string, error) { 534 + dir := path.Dir(e.FilePath) 535 + filepath := path.Join(dir, ef.filepath) 536 + info, err := iofs.Stat(fs, filepath) 537 + if err != nil { 538 + return nil, errors.Wrapf(err, e.Attribute.Pos, "failed to stat %s: %v", filepath, err) 539 + } 540 + if info.IsDir() { 541 + return nil, errors.Newf(e.Attribute.Pos, "%v is a directory", filepath) 542 + } 543 + return []string{filepath}, nil 544 + } 545 + 546 + type embeddedGlob struct { 547 + glob string 548 + allowEmptyGlob bool 549 + } 550 + 551 + func (eg *embeddedGlob) matches(e *Embed, filepath string) bool { 552 + dir := path.Dir(e.FilePath) 553 + if dir != "." { 554 + wasCut := false 555 + filepath, wasCut = strings.CutPrefix(filepath, dir+"/") 556 + if !wasCut { 557 + return false 558 + } 559 + } 560 + result, err := pkgpath.Match(eg.glob, filepath, pkgpath.Unix) 561 + if !result || err != nil { 562 + return false 563 + } 564 + return len(filterFsGlobResults(eg.glob, filepath)) == 1 565 + } 566 + 567 + func (eg *embeddedGlob) findAll(e *Embed, fs iofs.FS) ([]string, error) { 568 + dir := path.Dir(e.FilePath) 569 + fs, err := iofs.Sub(fs, dir) 570 + if err != nil { 571 + return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) 572 + } 573 + filepaths, err := fsGlob(fs, eg.glob) 574 + if err != nil { 575 + return nil, errors.Wrapf(err, e.Attribute.Pos, "%v", err) 576 + } 577 + if !eg.allowEmptyGlob && len(filepaths) == 0 { 578 + return nil, errors.Newf(e.Attribute.Pos, "no matches for glob pattern %q", eg.glob) 579 + } 580 + for i, filepath := range filepaths { 581 + filepaths[i] = path.Join(dir, filepath) 582 + } 583 + return filepaths, nil 584 + }

+65

internal/core/runtime/extern.go

··· 291 291 return errs 292 292 } 293 293 294 + // ExtractFieldAttrsByKind finds all the attributes of the given kind 295 + // in the given AST, parsing their bodies into [internal.Attr]. 296 + func ExtractFieldAttrsByKind(file *ast.File, kind string) (attrsByField map[*ast.Field]*internal.Attr, errs errors.Error) { 297 + k, _, decs, err := findExternFileAttr(file) 298 + if err != nil || len(decs) == 0 || k != kind { 299 + return nil, err 300 + } 301 + 302 + var fieldStack []*ast.Field 303 + 304 + ast.Walk(&ast.File{Decls: decs}, func(n ast.Node) bool { 305 + switch n := n.(type) { 306 + case *ast.Field: 307 + fieldStack = append(fieldStack, n) 308 + 309 + case *ast.Attribute: 310 + pos := n.Pos() 311 + k, body := n.Split() 312 + 313 + // Support old-style and new-style extern attributes. 314 + if k != "extern" && k != kind { 315 + break 316 + } 317 + 318 + lastFieldIdx := len(fieldStack) - 1 319 + if lastFieldIdx < 0 { 320 + errs = errors.Append(errs, errors.Newf(pos, "@%s attribute not associated with field", kind)) 321 + return true 322 + } 323 + 324 + f := fieldStack[lastFieldIdx] 325 + 326 + _, _, err := ast.LabelName(f.Label) 327 + if err != nil { 328 + b, _ := format.Node(f.Label) 329 + errs = errors.Append(errs, errors.Newf(pos, "external attribute has non-concrete label %s", b)) 330 + break 331 + } 332 + 333 + if attrsByField == nil { 334 + attrsByField = make(map[*ast.Field]*internal.Attr) 335 + } 336 + if _, found := attrsByField[f]; found { 337 + errs = errors.Append(errs, errors.Newf(pos, "duplicate @%s attributes", k)) 338 + break 339 + } 340 + 341 + attrParsed := internal.ParseAttrBody(pos, body) 342 + attrsByField[f] = &attrParsed 343 + 344 + return false 345 + } 346 + 347 + return true 348 + 349 + }, func(n ast.Node) { 350 + switch n.(type) { 351 + case *ast.Field: 352 + fieldStack = fieldStack[:len(fieldStack)-1] 353 + } 354 + }) 355 + 356 + return attrsByField, errs 357 + } 358 + 294 359 func (d *externDecorator) decorateConjunct(e adt.Elem, scope *adt.Vertex) { 295 360 w := walk.Visitor{Before: func(n adt.Node) bool { 296 361 return d.processADTNode(n, scope)

+9 -7

internal/golangorgx/gopls/test/integration/expectation.go

··· 526 526 // Logs are asynchronous to other LSP messages, so this expectation should not 527 527 // be used with combinators such as OnceMet or AfterChange that assert on 528 528 // ordering with respect to other operations. 529 - func LogMatching(typ protocol.MessageType, re string, count int, atLeast bool) Expectation { 529 + func LogMatching(typ protocol.MessageType, count int, atLeast bool, formatRe string, args ...any) Expectation { 530 + re := fmt.Sprintf(formatRe, args...) 530 531 rec, err := regexp.Compile(re) 531 532 if err != nil { 532 533 panic(err) ··· 564 565 // fmt.Sprintf(format, args...) a certain number of times. 565 566 // 566 567 // It is a convenience wrapper around [LogMatching], using 567 - // [regexp.QuoteMeta] and fmt.Sprintf. Note the string/regexp is not 568 - // anchored to the beginning or end of any log message, so this 569 - // behaves like [strings.Contains] rather than ==. 568 + // [regexp.QuoteMeta] on the format argument only. Note the 569 + // string/regexp is not anchored to the beginning or end of any log 570 + // message, so this behaves like [strings.Contains] rather than ==. 570 571 func LogExactf(typ protocol.MessageType, count int, atLeast bool, format string, args ...any) Expectation { 571 - return LogMatching(typ, regexp.QuoteMeta(fmt.Sprintf(format, args...)), count, atLeast) 572 + return LogMatching(typ, count, atLeast, regexp.QuoteMeta(format), args...) 572 573 } 573 574 574 575 // NoLogMatching asserts that the client has not received a log message 575 576 // of type typ matching the regexp re. If re is an empty string, any log 576 577 // message is considered a match. 577 - func NoLogMatching(typ protocol.MessageType, re string) Expectation { 578 + func NoLogMatching(typ protocol.MessageType, formatRe string, args ...any) Expectation { 578 579 var r *regexp.Regexp 580 + re := fmt.Sprintf(formatRe, args...) 579 581 if re != "" { 580 582 var err error 581 583 r, err = regexp.Compile(re) ··· 603 605 // NoLogExactf is a convenience wrapper around [NoLogMatching], in the 604 606 // same way that [LogExactf] is a wrapper around [LogMatching]. 605 607 func NoLogExactf(typ protocol.MessageType, format string, args ...any) Expectation { 606 - return NoLogMatching(typ, regexp.QuoteMeta(fmt.Sprintf(format, args...))) 608 + return NoLogMatching(typ, regexp.QuoteMeta(format), args...) 607 609 } 608 610 609 611 // FileWatchMatching expects that a file registration matches re.

+2 -5

internal/lsp/cache/folder.go

··· 56 56 // the given patterns map and reports whether this folder requires 57 57 // subdirectories to be watched explicitly. 58 58 func (wf *WorkspaceFolder) FileWatchingGlobPatterns(patterns map[protocol.RelativePattern]struct{}) bool { 59 - const watchCueFiles = "**/*.cue" 59 + const watchCueFiles = "**/*.{cue,json,yaml,yml}" 60 60 61 - patterns[protocol.RelativePattern{ 62 - BaseURI: wf.dir, 63 - Pattern: watchCueFiles, 64 - }] = struct{}{} 61 + patterns[protocol.RelativePattern{Pattern: watchCueFiles}] = struct{}{} 65 62 66 63 return wf.watchSubdirs() 67 64 }

+28 -1

internal/lsp/cache/module.go

··· 272 272 prefix := ip.Path + "/" 273 273 for _, pkg := range m.packages { 274 274 pkgIp := pkg.importPath 275 - if pkgIp.Qualifier == ip.Qualifier && strings.HasPrefix(pkgIp.Path, prefix) { 275 + if pkg.isCue && pkgIp.Qualifier == ip.Qualifier && strings.HasPrefix(pkgIp.Path, prefix) { 276 276 if !yield(pkg) { 277 277 return 278 + } 279 + } 280 + } 281 + } 282 + } 283 + 284 + // ascendantCuePackages returns all existing loaded Cue packages 285 + // within this module that would be able to embed the given 286 + // package. This requires that their "leaf" directory encloses the 287 + // "leaf" directory of the given pkg. 288 + // 289 + // This method only returns existing packages; it does not create any 290 + // new packages. 291 + func (m *Module) ascendantCuePackages(pkg *Package) iter.Seq[*Package] { 292 + return func(yield func(*Package) bool) { 293 + for _, p := range m.packages { 294 + if p == pkg || !p.isCue { 295 + continue 296 + } 297 + for _, pDir := range p.dirURIs { 298 + for _, pkgDir := range pkg.dirURIs { 299 + if pDir.Encloses(pkgDir) { 300 + if !yield(p) { 301 + return 302 + } 303 + break 304 + } 278 305 } 279 306 } 280 307 }

+183 -2

internal/lsp/cache/package.go

··· 18 18 "errors" 19 19 "fmt" 20 20 "slices" 21 + "strings" 21 22 22 23 "cuelang.org/go/cue/ast" 24 + "cuelang.org/go/cue/interpreter/embed" 25 + "cuelang.org/go/cue/token" 26 + "cuelang.org/go/internal/core/runtime" 23 27 "cuelang.org/go/internal/golangorgx/gopls/protocol" 24 28 "cuelang.org/go/internal/lsp/eval" 25 29 "cuelang.org/go/internal/mod/modpkgload" ··· 48 52 activeFilesAndDirs(files map[protocol.DocumentURI][]packageOrModule, dirs map[protocol.DocumentURI]struct{}) 49 53 } 50 54 55 + // embedding couples an [embed.Embed] embed attribute with the results 56 + // of its expansion. 57 + type embedding struct { 58 + *embed.Embed 59 + results []*embeddingResult 60 + } 61 + 62 + type embeddingResult struct { 63 + // fileUri is immutable. 64 + fileUri protocol.DocumentURI 65 + // pkg may be nil for example if an embed attribute specifies a 66 + // file which does not exist. It may later be updated to non-nil. 67 + pkg *Package 68 + } 69 + 51 70 // Package models a single CUE package within a CUE module. 52 71 type Package struct { 53 72 // immutable fields: all set at construction only ··· 74 93 // imports contains the packages that are imported by this package. 75 94 imports []*Package 76 95 96 + // embeddedBy contains the packages that directly embed this package. 97 + embeddedBy []*Package 98 + 99 + // embeddings contain the expansion of embed attributes extracted 100 + // from the files within this package. 101 + embeddings map[token.Pos]*embedding 102 + 77 103 // isDirty means that the package needs reloading. 78 104 isDirty bool 79 105 80 106 // eval for the files in this package. This is updated 81 107 // whenever the package status transitions to splendid. 82 108 eval *eval.Evaluator 109 + 110 + // isCue records whether this package is made of regular ".cue" 111 + // files. If false, this is a package that could be embedded. 112 + isCue bool 83 113 84 114 // files contains the files that make up this package. 85 115 files map[protocol.DocumentURI]*File ··· 135 165 pkg.isDirty = true 136 166 } 137 167 168 + // matchesUnknownEmbedding reports whether the current pkg should 169 + // embed, but does not currently, the given fileUri. 170 + // 171 + // For example, if this pkg has an embed that uses a glob, and some 172 + // time after this pkg is loaded, new file is created which also 173 + // matches the glob, then this will method would return true. 174 + func (pkg *Package) matchesUnknownEmbedding(fileUri protocol.DocumentURI) bool { 175 + filepath, wasCut := strings.CutPrefix(string(fileUri), string(pkg.module.rootURI)+"/") 176 + if !wasCut { 177 + return false 178 + } 179 + for _, embedding := range pkg.embeddings { 180 + if embedding.Matches(filepath) { 181 + for _, embedded := range embedding.results { 182 + if embedded.fileUri == fileUri { 183 + return false 184 + } 185 + } 186 + return true 187 + } 188 + } 189 + return false 190 + } 191 + 138 192 // resetEval resets the package's own evaluator. If recursive is true, 139 193 // it also resets the evaluators of every upstream and downstream 140 194 // package that could possibly contain objects (frames, navigables ··· 156 210 worklist = append(worklist[1:], importedPkg.imports...) 157 211 } 158 212 159 - // downstream - packages that import us 160 - worklist = pkg.importedBy 213 + // upstream - files/pkgs we embed - no need for transitive 214 + // treatment because an embedded pkg cannot embed or import 215 + // anything. 216 + for _, embedding := range pkg.embeddings { 217 + for _, embedded := range embedding.results { 218 + if embeddedPkg := embedded.pkg; embeddedPkg != nil { 219 + embeddedPkg.resetEval(false) 220 + } 221 + } 222 + } 223 + 224 + // downstream - packages that import (or embed) us 225 + worklist = append(pkg.importedBy, pkg.embeddedBy...) 161 226 for len(worklist) > 0 { 162 227 pkg := worklist[0] 163 228 pkg.resetEval(false) ··· 183 248 importedPkg.RemoveImportedBy(pkg) 184 249 } 185 250 251 + // upstream - files/pkgs we embed 252 + for _, embedding := range pkg.embeddings { 253 + for _, embedded := range embedding.results { 254 + if embeddedPkg := embedded.pkg; embeddedPkg != nil { 255 + embeddedPkg.RemoveEmbeddedBy(pkg) 256 + } 257 + } 258 + } 259 + 186 260 w.debugLogf("%v Deleted", pkg) 187 261 pkg.files = nil 188 262 pkg.isDirty = false ··· 218 292 for _, importedPkg := range pkg.imports { 219 293 importedPkg.RemoveImportedBy(pkg) 220 294 } 295 + 296 + for _, embedding := range pkg.embeddings { 297 + for _, embedded := range embedding.results { 298 + if embeddedPkg := embedded.pkg; embeddedPkg != nil { 299 + embeddedPkg.RemoveEmbeddedBy(pkg) 300 + } 301 + } 302 + } 303 + 221 304 pkg.imports = pkg.imports[:0] 222 305 223 306 importCanonicalisation := make(map[string]ast.ImportPath) ··· 242 325 modpkgFiles := modpkg.Files() 243 326 evalASTs := make([]*ast.File, len(modpkgFiles)) 244 327 filesSet := make(map[protocol.DocumentURI]*File, len(modpkgFiles)) 328 + isCue := true 329 + var embeddings map[token.Pos]*embedding 245 330 246 331 for i, modpkgFile := range modpkgFiles { 247 332 evalASTs[i] = modpkgFile.Syntax 248 333 fileUri := m.rootURI + protocol.DocumentURI("/"+modpkgFile.FilePath) 249 334 delete(m.dirtyFiles, fileUri) 335 + 336 + isCue = isCue && strings.HasSuffix(string(fileUri), ".cue") 250 337 251 338 file := w.ensureFile(fileUri) 252 339 filesSet[fileUri] = file ··· 258 345 errs = append(errs, modpkgFile.SyntaxError) 259 346 } 260 347 348 + if isCue && syntax != nil { 349 + // embedded files are "upstream" of us, in the same way that 350 + // imported packages are upstream. modpkgload discovers the 351 + // import graph for a given package. This includes all 352 + // (transitively) imported upstream packages, but no 353 + // downstream packages. We take the same approach for embeds, 354 + // so here we take a snapshot in time of the file system, 355 + // expanding embed attributes to file paths. Later on, 356 + // [Package.linkWithEmbeddedFiles] will be called which will 357 + // enable us to link these file paths to any packages which 358 + // have been found and loaded. 359 + // 360 + // There is one key difference between imported packages and 361 + // embedded packages: a glob embed attribute expands to 362 + // several packages, and so this can change over time as 363 + // files are added and removed from the file system. This is 364 + // not possible with imported packages: an import spec always 365 + // refers to exactly one package. 366 + attrsByField, err := runtime.ExtractFieldAttrsByKind(syntax, embed.EmbedKind) 367 + if err != nil { 368 + errs = append(errs, err) 369 + } 370 + embeddedPaths, err := embed.EmbeddedPaths(modpkgFile.FilePath, attrsByField) 371 + if err != nil { 372 + errs = append(errs, err) 373 + } 374 + if len(embeddedPaths) > 0 { 375 + if embeddings == nil { 376 + embeddings = make(map[token.Pos]*embedding) 377 + } 378 + fs := w.overlayFS.IoFS(m.rootURI.Path()) 379 + for _, embed := range embeddedPaths { 380 + embedding := &embedding{Embed: embed} 381 + embeddings[embed.Attribute.Pos] = embedding 382 + matches, err := embed.FindAll(fs) 383 + if err != nil { 384 + errs = append(errs, err) 385 + continue 386 + } 387 + for _, filePath := range matches { 388 + embedding.results = append(embedding.results, &embeddingResult{ 389 + fileUri: m.rootURI + "/" + protocol.DocumentURI(filePath), 390 + }) 391 + } 392 + } 393 + } 394 + } 395 + 261 396 file.ensureUser(pkg, errs...) 262 397 w.standalone.deleteFile(fileUri) 263 398 } 399 + pkg.isCue = isCue 400 + pkg.embeddings = embeddings 264 401 265 402 for fileUri, file := range oldFilesSet { 266 403 if _, found := filesSet[fileUri]; !found { ··· 285 422 return nil 286 423 } 287 424 425 + // linkWithEmbeddedFiles works through the pkg's embeddings, 426 + // populating existing expansions of embed attributes with their 427 + // corresponding package, on a best-effort basis. 428 + func (pkg *Package) linkWithEmbeddedFiles() { 429 + activeFiles, _ := pkg.module.workspace.activeFilesAndDirs() 430 + 431 + for _, embedding := range pkg.embeddings { 432 + for _, embedded := range embedding.results { 433 + fileUri := embedded.fileUri 434 + if embedded.pkg != nil { 435 + continue 436 + } 437 + 438 + remotePkgs := activeFiles[fileUri] 439 + if len(remotePkgs) == 0 { 440 + continue 441 + } 442 + if l := len(remotePkgs); l > 1 { 443 + panic(fmt.Sprintf("Invariant failure: embedded file %q has %d packages. Must only have 1.", fileUri, l)) 444 + } 445 + remotePkg := remotePkgs[0].(*Package) 446 + embedded.pkg = remotePkg 447 + remotePkg.EnsureEmbeddedBy(pkg) 448 + } 449 + } 450 + } 451 + 288 452 // forPackage is a callback for the evaluator. See 289 453 // [eval.Config.ForPackage] 290 454 func (pkg *Package) forPackage(importPath ast.ImportPath) *eval.Evaluator { ··· 326 490 return p == importer 327 491 }) 328 492 } 493 + 494 + // EnsureEmbeddedBy ensures that embedder is recorded as a user of 495 + // this package. This method is idempotent. 496 + func (pkg *Package) EnsureEmbeddedBy(embedder *Package) { 497 + if slices.Contains(pkg.embeddedBy, embedder) { 498 + return 499 + } 500 + pkg.embeddedBy = append(pkg.embeddedBy, embedder) 501 + } 502 + 503 + // RemoveEmbeddedBy ensures that embedder is not recorded as a user of 504 + // this package. This method is idempotent. 505 + func (pkg *Package) RemoveEmbeddedBy(embedder *Package) { 506 + pkg.embeddedBy = slices.DeleteFunc(pkg.embeddedBy, func(p *Package) bool { 507 + return p == embedder 508 + }) 509 + }

+88 -17

internal/lsp/cache/workspace.go

··· 224 224 for _, wfDir := range wfDirs { 225 225 // NB: a.Encloses(b) returns true if a == b 226 226 if wfDir.Encloses(activeDir) { 227 - patterns[protocol.RelativePattern{Pattern: protocol.Pattern(activeDir)}] = struct{}{} 227 + patterns[protocol.RelativePattern{Pattern: filepath.ToSlash(activeDir.Path())}] = struct{}{} 228 228 break 229 229 } 230 230 } ··· 864 864 downstream.markDirty() 865 865 repeatReload = true 866 866 } 867 + for _, downstream := range pkg.embeddedBy { 868 + downstream.markDirty() 869 + repeatReload = true 870 + } 867 871 } 868 872 869 873 // Note that there's a potential memory leak here: we might load a ··· 938 942 pkg := pkgModPkg.pkg 939 943 m := pkg.module 940 944 941 - if len(pkg.dirURIs) != 1 { 942 - // old module system: ancestor imports not supported. 943 - continue 944 - } 945 - for descendentPkg := range m.descendantCuePackages(pkg.importPath) { 946 - key.importPath = descendentPkg.importPath 947 - if _, loaded := processedPkgs[key]; loaded { 945 + if pkg.isCue { 946 + // embeddings are "upstream" packages. The package will 947 + // have expanded every embed attribute to a set of 948 + // fileUris. We now need to try to find a package for each 949 + // such fileUri. 950 + for _, embedding := range pkg.embeddings { 951 + for _, embedded := range embedding.results { 952 + if embedded.pkg != nil { 953 + continue 954 + } 955 + ip, dirUris, err := m.FindImportPathForFile(embedded.fileUri) 956 + if err != nil || ip == nil || len(dirUris) == 0 { 957 + continue 958 + } 959 + key.importPath = *ip 960 + if _, loaded := processedPkgs[key]; loaded { 961 + // If we've just loaded the embedded pkg, 962 + // assuming the load succeeded, the embedded pkg 963 + // won't be dirty, so nothing to do here. 964 + continue 965 + } 966 + pkg := m.EnsurePackage(*ip, dirUris) 967 + // If pkg is dirty, it's probably because it's only 968 + // just been created, so it'll need fully loading. 969 + repeatReload = repeatReload || pkg.isDirty 970 + } 971 + } 972 + if len(pkg.dirURIs) != 1 { 973 + // old module system: ancestor imports not supported. 948 974 continue 949 975 } 950 - // We can be here if: two packages already exist where 951 - // one is the ancestor of the other, and a new file 952 - // appears which adds to the ancestor 953 - // package. Processing that new file will mark the 954 - // ancestor as dirty, but not the descendent. So we 955 - // deal with that situation here. 956 - for _, file := range pkg.files { 957 - descendentPkg.markFileDirty(file.uri) 976 + for descendentPkg := range m.descendantCuePackages(pkg.importPath) { 977 + key.importPath = descendentPkg.importPath 978 + if _, loaded := processedPkgs[key]; loaded { 979 + continue 980 + } 981 + // We can be here if: two packages already exist where 982 + // one is the ancestor of the other, and a new file 983 + // appears which adds to the ancestor 984 + // package. Processing that new file will mark the 985 + // ancestor as dirty, but not the descendent. So we 986 + // deal with that situation here. 987 + for _, file := range pkg.files { 988 + descendentPkg.markFileDirty(file.uri) 989 + } 990 + repeatReload = true 991 + } 992 + 993 + } else { 994 + // pkg is a non-cue pkg. We look through all cue pkgs 995 + // which could possibly embed pkg. 996 + for ascendantPkg := range m.ascendantCuePackages(pkg) { 997 + key.importPath = ascendantPkg.importPath 998 + if _, loaded := processedPkgs[key]; loaded { 999 + // If we've just loaded it, it'll find pkg in the 1000 + // call to linkWithEmbeddedFiles that happens in a 1001 + // moment. 1002 + continue 1003 + } 1004 + for fileUri := range pkg.files { 1005 + if ascendantPkg.matchesUnknownEmbedding(fileUri) { 1006 + // ascendantPkg has not just been reloaded, and 1007 + // it should embed pkg, but it doesn't. So it 1008 + // needs to be reloaded. This happens when pkg is 1009 + // created by a new (e.g. json-) file appearing, 1010 + // and ascendantPkg contains a suitable glob 1011 + // embed. This is where embed attributes (with 1012 + // globs) are very different to import specs: a 1013 + // glob embed attribute (essentially) represents 1014 + // a set of packages to embed, and that set can 1015 + // change as files are added to, or deleted from, 1016 + // the file system. 1017 + ascendantPkg.markDirty() 1018 + repeatReload = true 1019 + break 1020 + } 1021 + } 958 1022 } 959 - repeatReload = true 960 1023 } 961 1024 } 962 1025 963 1026 if !repeatReload { 964 1027 break 1028 + } 1029 + } 1030 + 1031 + // Finally, ask all packages to look up and link with any packages 1032 + // that they embed. 1033 + for _, m := range modules { 1034 + for _, pkg := range m.packages { 1035 + pkg.linkWithEmbeddedFiles() 965 1036 } 966 1037 } 967 1038 }

Configure Feed

Configure Feed