this repo has no description
start work on glot, a lexicon helper tool
+545
+12
cmd/glot/README.md
+12
cmd/glot/README.md
···
1
+
2
+
glot: AT Lexicon Utility
3
+
========================
4
+
5
+
This is a developer tool for working with Lexicon schemas:
6
+
7
+
- code generation for API clients, servers, and record schemas
8
+
- publishing schemas to and synchronizing from the AT network
9
+
- diffing, linting, and verifying schema evolution rules
10
+
11
+
This project is a work in progress (not much is implemented). This will
12
+
probably become a stand-alone git repository, and the name is likely to change.
+491
cmd/glot/lint.go
+491
cmd/glot/lint.go
···
1
+
package main
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"encoding/json"
7
+
"errors"
8
+
"slices"
9
+
"fmt"
10
+
"io/fs"
11
+
"log/slog"
12
+
"os"
13
+
"path"
14
+
"path/filepath"
15
+
"regexp"
16
+
17
+
"github.com/bluesky-social/indigo/atproto/lexicon"
18
+
"github.com/bluesky-social/indigo/atproto/syntax"
19
+
20
+
"github.com/urfave/cli/v3"
21
+
)
22
+
23
+
var (
24
+
// NOTE: not actually using these? and should replace with library if we did
25
+
ColorGreen = "\033[32m"
26
+
ColorYellow = "\033[33m"
27
+
ColorReset = "\033[0m"
28
+
29
+
// internal error used to set non-zero return code (but not print separately)
30
+
ErrLintFailures = errors.New("linting issues detected")
31
+
)
32
+
33
+
var cmdLint = &cli.Command{
34
+
Name: "lint",
35
+
Usage: "check schema style",
36
+
ArgsUsage: `<file-or-dir>*`,
37
+
Flags: []cli.Flag{
38
+
&cli.StringFlag{
39
+
Name: "lexicons-dir",
40
+
Value: "./lexicons/",
41
+
Usage: "base directory for project Lexicon files",
42
+
Sources: cli.EnvVars("LEXICONS_DIR"),
43
+
},
44
+
&cli.BoolFlag{
45
+
Name: "json",
46
+
Usage: "output structured JSON",
47
+
},
48
+
},
49
+
Action: runLint,
50
+
}
51
+
52
+
func runLint(ctx context.Context, cmd *cli.Command) error {
53
+
paths := cmd.Args().Slice()
54
+
if !cmd.Args().Present() {
55
+
paths = []string{cmd.String("lexicons-dir")}
56
+
_, err := os.Stat(paths[0])
57
+
if err != nil {
58
+
return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err)
59
+
}
60
+
}
61
+
62
+
// TODO: load up entire directory in to a catalog? or have a "linter" struct?
63
+
64
+
slog.Debug("starting lint run")
65
+
anyFailures := false
66
+
for _, p := range paths {
67
+
finfo, err := os.Stat(p)
68
+
if err != nil {
69
+
return fmt.Errorf("failed loading %s: %w", p, err)
70
+
}
71
+
if finfo.IsDir() {
72
+
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
73
+
if d.IsDir() || path.Ext(fp) != ".json" {
74
+
return nil
75
+
}
76
+
err = lintFilePath(ctx, cmd, fp)
77
+
if err == ErrLintFailures {
78
+
anyFailures = true
79
+
return nil
80
+
}
81
+
return err
82
+
}); err != nil {
83
+
return err
84
+
}
85
+
continue
86
+
}
87
+
if err := lintFilePath(ctx, cmd, p); err != nil {
88
+
if err == ErrLintFailures {
89
+
anyFailures = true
90
+
} else {
91
+
return err
92
+
}
93
+
}
94
+
}
95
+
if anyFailures {
96
+
return ErrLintFailures
97
+
}
98
+
return nil
99
+
}
100
+
101
+
func lintFilePath(ctx context.Context, cmd *cli.Command, p string) error {
102
+
b, err := os.ReadFile(p)
103
+
if err != nil {
104
+
return err
105
+
}
106
+
107
+
// parse file regularly
108
+
// TODO: use json/v2 when available for case-sensitivity
109
+
var sf lexicon.SchemaFile
110
+
if err := json.Unmarshal(b, &sf); err != nil {
111
+
return err
112
+
}
113
+
if err := sf.FinishParse(); err != nil {
114
+
return err
115
+
}
116
+
117
+
issues := lintSchemaFile(p, sf)
118
+
119
+
// check for unknown fields (more strict, as a lint/warning)
120
+
var unknownSF lexicon.SchemaFile
121
+
dec := json.NewDecoder(bytes.NewReader(b))
122
+
dec.DisallowUnknownFields()
123
+
if err := dec.Decode(&unknownSF); err != nil {
124
+
issues = append(issues, LintIssue{
125
+
FilePath: p,
126
+
NSID: syntax.NSID(sf.ID),
127
+
LintLevel: "warn",
128
+
LintName: "unexpected-field",
129
+
LintDescription: "schema JSON contains unexpected data",
130
+
Message: err.Error(),
131
+
})
132
+
}
133
+
134
+
if cmd.Bool("json") {
135
+
for _, iss := range issues {
136
+
b, err := json.Marshal(iss)
137
+
if err != nil {
138
+
return nil
139
+
}
140
+
fmt.Println(string(b))
141
+
}
142
+
} else {
143
+
if len(issues) == 0 {
144
+
fmt.Printf(" 🟢 %s\n", p)
145
+
} else {
146
+
fmt.Printf(" 🟡 %s\n", p)
147
+
for _, iss := range issues {
148
+
fmt.Printf(" [%s]: %s\n", iss.LintName, iss.Message)
149
+
}
150
+
}
151
+
}
152
+
if len(issues) > 0 {
153
+
return ErrLintFailures
154
+
}
155
+
return nil
156
+
}
157
+
158
+
type LintIssue struct {
159
+
FilePath string `json:"file-path,omitempty"`
160
+
NSID syntax.NSID `json:"nsid,omitempty"`
161
+
LintLevel string `json:"lint-level,omitempty"`
162
+
LintName string `json:"lint-name,omitempty"`
163
+
LintDescription string `json:"lint-description,omitempty"`
164
+
Message string `json:"message,omitempty"`
165
+
}
166
+
167
+
var schemaNameRegex = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9]{0,255})?$`)
168
+
169
+
func checkSchemaName(raw string) error {
170
+
if raw == "" {
171
+
return errors.New("empty name")
172
+
}
173
+
if len(raw) > 255 {
174
+
return errors.New("name is too long (255 chars max)")
175
+
}
176
+
if !schemaNameRegex.MatchString(raw) {
177
+
return errors.New("name doesn't match recommended syntax/characters")
178
+
}
179
+
return nil
180
+
}
181
+
182
+
183
+
func lintSchemaFile(p string, sf lexicon.SchemaFile) []LintIssue {
184
+
issues := []LintIssue{}
185
+
186
+
nsid, err := syntax.ParseNSID(sf.ID)
187
+
if err != nil {
188
+
issues = append(issues, LintIssue{
189
+
FilePath: p,
190
+
NSID: syntax.NSID(sf.ID),
191
+
LintLevel: "error",
192
+
LintName: "invalid-nsid",
193
+
LintDescription: "schema file declares NSID with invalid syntax",
194
+
Message: fmt.Sprintf("NSID string: %s", sf.ID),
195
+
})
196
+
}
197
+
if nsid == "" {
198
+
nsid = syntax.NSID(sf.ID)
199
+
}
200
+
if sf.Lexicon != 1 {
201
+
issues = append(issues, LintIssue{
202
+
FilePath: p,
203
+
NSID: nsid,
204
+
LintLevel: "error",
205
+
LintName: "lexicon-version",
206
+
LintDescription: "unsupported Lexicon language version",
207
+
Message: fmt.Sprintf("found version: %d", sf.Lexicon),
208
+
})
209
+
return issues
210
+
}
211
+
212
+
for defname, def := range sf.Defs {
213
+
defiss := lintSchemaDef(p, nsid, defname, def)
214
+
if len(defiss) > 0 {
215
+
issues = append(issues, defiss...)
216
+
}
217
+
}
218
+
219
+
return issues
220
+
}
221
+
222
+
func lintSchemaDef(p string, nsid syntax.NSID, defname string, def lexicon.SchemaDef) []LintIssue {
223
+
issues := []LintIssue{}
224
+
225
+
// missing description issue, in case it is needed
226
+
missingDesc := func() LintIssue {
227
+
return LintIssue{
228
+
FilePath: p,
229
+
NSID: nsid,
230
+
LintLevel: "warn",
231
+
LintName: "missing-primary-description",
232
+
LintDescription: "primary types (record, query, procedure, subscription, permission-set) should include a description",
233
+
Message: "primary type missing a description",
234
+
}
235
+
}
236
+
237
+
if err := def.CheckSchema(); err != nil {
238
+
issues = append(issues, LintIssue{
239
+
FilePath: p,
240
+
NSID: nsid,
241
+
LintLevel: "error",
242
+
LintName: "lexicon-schema",
243
+
LintDescription: "basic structure schema checks (additional errors may be collapsed)",
244
+
Message: err.Error(),
245
+
})
246
+
}
247
+
248
+
if err := checkSchemaName(defname); err != nil {
249
+
issues = append(issues, LintIssue{
250
+
FilePath: p,
251
+
NSID: nsid,
252
+
LintLevel: "warn",
253
+
LintName: "def-name-syntax",
254
+
LintDescription: "definition name does not follow syntax guidance",
255
+
Message: fmt.Sprintf("%s: %s", err.Error(), defname),
256
+
})
257
+
}
258
+
259
+
if nsid.Name() == "defs" && defname == "main" {
260
+
issues = append(issues, LintIssue{
261
+
FilePath: p,
262
+
NSID: nsid,
263
+
LintLevel: "warn",
264
+
LintName: "defs-main-definition",
265
+
LintDescription: "defs schemas should not have a 'main'",
266
+
Message: "defs schemas should not have a 'main'",
267
+
})
268
+
}
269
+
270
+
switch def.Inner.(type) {
271
+
case lexicon.SchemaRecord, lexicon.SchemaQuery, lexicon.SchemaProcedure, lexicon.SchemaSubscription, lexicon.SchemaPermissionSet:
272
+
if defname != "main" {
273
+
issues = append(issues, LintIssue{
274
+
FilePath: p,
275
+
NSID: nsid,
276
+
LintLevel: "error",
277
+
LintName: "non-main-primary",
278
+
LintDescription: "primary types (record, query, procedure, subscription, permission-set) must be 'main' definition",
279
+
Message: fmt.Sprintf("primary definition types must be 'main': %s", defname),
280
+
})
281
+
}
282
+
}
283
+
284
+
switch v := def.Inner.(type) {
285
+
case lexicon.SchemaRecord:
286
+
if v.Description == nil || *v.Description == "" {
287
+
issues = append(issues, missingDesc())
288
+
}
289
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: v.Record})
290
+
if len(reciss) > 0 {
291
+
issues = append(issues, reciss...)
292
+
}
293
+
case lexicon.SchemaQuery:
294
+
if v.Description == nil || *v.Description == "" {
295
+
issues = append(issues, missingDesc())
296
+
}
297
+
if v.Parameters != nil {
298
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Parameters})
299
+
if len(reciss) > 0 {
300
+
issues = append(issues, reciss...)
301
+
}
302
+
}
303
+
if v.Output == nil {
304
+
issues = append(issues, LintIssue{
305
+
FilePath: p,
306
+
NSID: nsid,
307
+
LintLevel: "warn",
308
+
LintName: "endpoint-output-undefined",
309
+
LintDescription: "API endpoints should define an output (even if empty)",
310
+
Message: "missing output definition",
311
+
})
312
+
} else {
313
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Output})
314
+
if len(reciss) > 0 {
315
+
issues = append(issues, reciss...)
316
+
}
317
+
}
318
+
// TODO: error names
319
+
case lexicon.SchemaProcedure:
320
+
if v.Description == nil || *v.Description == "" {
321
+
issues = append(issues, missingDesc())
322
+
}
323
+
if v.Parameters != nil {
324
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Parameters})
325
+
if len(reciss) > 0 {
326
+
issues = append(issues, reciss...)
327
+
}
328
+
}
329
+
if v.Input != nil {
330
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Input})
331
+
if len(reciss) > 0 {
332
+
issues = append(issues, reciss...)
333
+
}
334
+
}
335
+
if v.Output == nil {
336
+
issues = append(issues, LintIssue{
337
+
FilePath: p,
338
+
NSID: nsid,
339
+
LintLevel: "warn",
340
+
LintName: "endpoint-output-undefined",
341
+
LintDescription: "API endpoints should define an output (even if empty)",
342
+
Message: "missing output definition",
343
+
})
344
+
} else {
345
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Output})
346
+
if len(reciss) > 0 {
347
+
issues = append(issues, reciss...)
348
+
}
349
+
}
350
+
// TODO: error names
351
+
case lexicon.SchemaSubscription:
352
+
if v.Description == nil || *v.Description == "" {
353
+
issues = append(issues, missingDesc())
354
+
}
355
+
if v.Parameters != nil {
356
+
reciss := lintSchemaRecursive(p, nsid, lexicon.SchemaDef{Inner: *v.Parameters})
357
+
if len(reciss) > 0 {
358
+
issues = append(issues, reciss...)
359
+
}
360
+
}
361
+
if v.Message != nil {
362
+
// TODO: v.Message.Schema must be a union (CheckSchema verified this). and must only have local references (same file), and should have at least one defined
363
+
reciss := lintSchemaRecursive(p, nsid, v.Message.Schema)
364
+
if len(reciss) > 0 {
365
+
issues = append(issues, reciss...)
366
+
}
367
+
} else {
368
+
issues = append(issues, LintIssue{
369
+
FilePath: p,
370
+
NSID: nsid,
371
+
LintLevel: "warn",
372
+
LintName: "subscription-no-messages",
373
+
LintDescription: "no subscription message types defined",
374
+
Message: "no subscription message types defined",
375
+
})
376
+
}
377
+
// TODO: at least one message type
378
+
case lexicon.SchemaPermissionSet:
379
+
if v.Description == nil || *v.Description == "" {
380
+
issues = append(issues, missingDesc())
381
+
}
382
+
// TODO: translated descriptions?
383
+
if len(v.Permissions) == 0 {
384
+
issues = append(issues, LintIssue{
385
+
FilePath: p,
386
+
NSID: nsid,
387
+
LintLevel: "warn",
388
+
LintName: "permissions-no-members",
389
+
LintDescription: "permission sets should define at least one permission",
390
+
Message: "empty permission set",
391
+
})
392
+
}
393
+
for _, perm := range v.Permissions {
394
+
// TODO: any lints on permissions?
395
+
_ = perm
396
+
}
397
+
case lexicon.SchemaPermission, lexicon.SchemaNull, lexicon.SchemaBoolean, lexicon.SchemaInteger, lexicon.SchemaString, lexicon.SchemaBytes, lexicon.SchemaCIDLink, lexicon.SchemaArray, lexicon.SchemaObject, lexicon.SchemaBlob, lexicon.SchemaToken, lexicon.SchemaRef, lexicon.SchemaUnion, lexicon.SchemaUnknown:
398
+
reciss := lintSchemaRecursive(p, nsid, def)
399
+
if len(reciss) > 0 {
400
+
issues = append(issues, reciss...)
401
+
}
402
+
default:
403
+
slog.Info("no lint rules for top-level schema definition type", "type", fmt.Sprintf("%T", def.Inner))
404
+
}
405
+
return issues
406
+
}
407
+
408
+
func lintSchemaRecursive(p string, nsid syntax.NSID, def lexicon.SchemaDef) []LintIssue {
409
+
issues := []LintIssue{}
410
+
411
+
switch v := def.Inner.(type) {
412
+
case lexicon.SchemaPermission:
413
+
// TODO:
414
+
case lexicon.SchemaNull:
415
+
// pass
416
+
case lexicon.SchemaBoolean:
417
+
// TODO: default true
418
+
// TODO: both default and const
419
+
case lexicon.SchemaInteger:
420
+
// TODO: both default and const
421
+
case lexicon.SchemaString:
422
+
// TODO: no format and no max length
423
+
// TODO: format and length limits
424
+
// TODO: grapheme limit set, and maxlen either too low or not set
425
+
// TODO: very large max size
426
+
// TODO: format=handle strings within an record type
427
+
case lexicon.SchemaBytes:
428
+
// TODO: very large max size
429
+
case lexicon.SchemaCIDLink:
430
+
// pass
431
+
case lexicon.SchemaArray:
432
+
reciss := lintSchemaRecursive(p, nsid, v.Items)
433
+
if len(reciss) > 0 {
434
+
issues = append(issues, reciss...)
435
+
}
436
+
case lexicon.SchemaObject:
437
+
// NOTE: CheckSchema already verifies that nullable and required are valid against property keys
438
+
for _, propdef := range v.Properties {
439
+
reciss := lintSchemaRecursive(p, nsid, propdef)
440
+
if len(reciss) > 0 {
441
+
issues = append(issues, reciss...)
442
+
}
443
+
// TODO: property name syntax
444
+
}
445
+
for _, k := range v.Nullable {
446
+
if !slices.Contains(v.Required, k) {
447
+
issues = append(issues, LintIssue{
448
+
FilePath: p,
449
+
NSID: nsid,
450
+
LintLevel: "warn",
451
+
LintName: "nullable-and-optional",
452
+
LintDescription: "object properties should not be both optional and nullable",
453
+
Message: fmt.Sprintf("field is both nullabor and optional: %s", k),
454
+
})
455
+
}
456
+
}
457
+
case lexicon.SchemaBlob:
458
+
// pass
459
+
case lexicon.SchemaParams:
460
+
// NOTE: CheckSchema already verifies that required are valid against property keys
461
+
for _, propdef := range v.Properties {
462
+
reciss := lintSchemaRecursive(p, nsid, propdef)
463
+
if len(reciss) > 0 {
464
+
issues = append(issues, reciss...)
465
+
}
466
+
// TODO: property name syntax
467
+
}
468
+
case lexicon.SchemaToken:
469
+
// pass
470
+
case lexicon.SchemaRef:
471
+
// TODO: resolve? locally vs globally?
472
+
case lexicon.SchemaUnion:
473
+
// TODO: open vs closed?
474
+
// TODO: check that refs actually resolve?
475
+
case lexicon.SchemaUnknown:
476
+
// pass
477
+
case lexicon.SchemaBody:
478
+
if v.Schema != nil {
479
+
// NOTE: CheckSchema already verified that v.Schema is an object, ref, or union
480
+
reciss := lintSchemaRecursive(p, nsid, *v.Schema)
481
+
if len(reciss) > 0 {
482
+
issues = append(issues, reciss...)
483
+
}
484
+
}
485
+
// TODO: empty/invalid Encoding (mimetype)
486
+
default:
487
+
slog.Info("no lint rules for recursive schema type", "type", fmt.Sprintf("%T", def.Inner), "nsid", nsid)
488
+
}
489
+
490
+
return issues
491
+
}
+42
cmd/glot/main.go
+42
cmd/glot/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"os"
7
+
8
+
_ "github.com/joho/godotenv/autoload"
9
+
10
+
"github.com/earthboundkid/versioninfo/v2"
11
+
"github.com/urfave/cli/v3"
12
+
)
13
+
14
+
func main() {
15
+
err := run(os.Args)
16
+
if err == ErrLintFailures {
17
+
os.Exit(1)
18
+
} else if err != nil {
19
+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
20
+
os.Exit(-1)
21
+
}
22
+
}
23
+
24
+
func run(args []string) error {
25
+
26
+
app := cli.Command{
27
+
Name: "glot",
28
+
Usage: "AT Lexicon Tool",
29
+
Version: versioninfo.Short(),
30
+
Flags: []cli.Flag{
31
+
&cli.StringFlag{
32
+
Name: "log-level",
33
+
Usage: "log verbosity level (eg: warn, info, debug)",
34
+
Sources: cli.EnvVars("GLOT_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
35
+
},
36
+
},
37
+
}
38
+
app.Commands = []*cli.Command{
39
+
cmdLint,
40
+
}
41
+
return app.Run(context.Background(), args)
42
+
}