lexgen refactor (#1181) · alice.mosphere.at/indigo@5b368e3

+1 -1

Makefile

··· 67 67 68 68 .PHONY: lexgen 69 69 lexgen: ## Run codegen tool for lexicons (lexicon JSON to Go packages) 70 - go run ./cmd/lexgen/ --build-file cmd/lexgen/bsky.json $(LEXDIR) 70 + go run ./cmd/lexgen/ legacy --output-dir api 71 71 72 72 .PHONY: cborgen 73 73 cborgen: ## Run codegen tool for CBOR serialization

+1 -1

atproto/lexicon/testdata/catalog/procedure.json

··· 16 16 "type": "integer", 17 17 "description": "field of type integer" 18 18 }, 19 - "string": { 19 + "stringField": { 20 20 "type": "string", 21 21 "description": "field of type string" 22 22 }

+2 -2

atproto/lexicon/testdata/catalog/query.json

··· 10 10 "type": "params", 11 11 "description": "a params type", 12 12 "required": [ 13 - "string" 13 + "stringField" 14 14 ], 15 15 "properties": { 16 16 "boolean": { ··· 21 21 "type": "integer", 22 22 "description": "field of type integer" 23 23 }, 24 - "string": { 24 + "stringField": { 25 25 "type": "string", 26 26 "description": "field of type string" 27 27 },

-26

cmd/lexgen/bsky.json

··· 1 - [ 2 - { 3 - "package": "bsky", 4 - "prefix": "app.bsky", 5 - "outdir": "api/bsky", 6 - "import": "github.com/bluesky-social/indigo/api/bsky" 7 - }, 8 - { 9 - "package": "atproto", 10 - "prefix": "com.atproto", 11 - "outdir": "api/atproto", 12 - "import": "github.com/bluesky-social/indigo/api/atproto" 13 - }, 14 - { 15 - "package": "chat", 16 - "prefix": "chat.bsky", 17 - "outdir": "api/chat", 18 - "import": "github.com/bluesky-social/indigo/api/chat" 19 - }, 20 - { 21 - "package": "ozone", 22 - "prefix": "tools.ozone", 23 - "outdir": "api/ozone", 24 - "import": "github.com/bluesky-social/indigo/api/ozone" 25 - } 26 - ]

+177 -139

cmd/lexgen/main.go

··· 1 1 package main 2 2 3 3 import ( 4 - "errors" 4 + "bytes" 5 + "context" 6 + "encoding/json" 5 7 "fmt" 8 + "go/format" 6 9 "io/fs" 7 10 "os" 11 + "path" 8 12 "path/filepath" 9 - "strings" 10 13 11 - "github.com/bluesky-social/indigo/lex" 12 - "github.com/urfave/cli/v2" 13 - ) 14 - 15 - func findSchemas(dir string, out []string) ([]string, error) { 16 - err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { 17 - if err != nil { 18 - return err 19 - } 14 + _ "github.com/joho/godotenv/autoload" 20 15 21 - if info.IsDir() { 22 - return nil 23 - } 16 + "github.com/bluesky-social/indigo/atproto/lexicon" 17 + "github.com/bluesky-social/indigo/lex/lexgen" 24 18 25 - if strings.HasSuffix(path, ".json") { 26 - out = append(out, path) 27 - } 19 + "github.com/earthboundkid/versioninfo/v2" 20 + "github.com/urfave/cli/v3" 21 + "golang.org/x/tools/imports" 22 + ) 28 23 29 - return nil 30 - }) 31 - if err != nil { 32 - return out, err 24 + func main() { 25 + if err := run(os.Args); err != nil { 26 + fmt.Fprintf(os.Stderr, "error: %v\n", err) 27 + os.Exit(-1) 33 28 } 29 + } 34 30 35 - return out, nil 31 + func run(args []string) error { 36 32 37 - } 38 - 39 - // for direct .json lexicon files or directories containing lexicon .json files, get one flat list of all paths to .json files 40 - func expandArgs(args []string) ([]string, error) { 41 - var out []string 42 - for _, a := range args { 43 - st, err := os.Stat(a) 44 - if err != nil { 45 - return nil, err 46 - } 47 - if st.IsDir() { 48 - out, err = findSchemas(a, out) 49 - if err != nil { 50 - return nil, err 51 - } 52 - } else if strings.HasSuffix(a, ".json") { 53 - out = append(out, a) 54 - } 33 + app := cli.Command{ 34 + Name: "lexgen", 35 + Usage: "AT lexicon code generation for Go", 36 + //Description: "", 37 + Version: versioninfo.Short(), 38 + } 39 + app.Commands = []*cli.Command{ 40 + cmdLegacy, 41 + cmdGen, 55 42 } 56 - 57 - return out, nil 43 + return app.Run(context.Background(), args) 58 44 } 59 45 60 - func main() { 61 - app := cli.NewApp() 62 - 63 - app.Flags = []cli.Flag{ 46 + var cmdLegacy = &cli.Command{ 47 + Name: "legacy", 48 + Usage: "generate code with legacy behaviors (for indigo repo only)", 49 + ArgsUsage: `<file-or-dir>*`, 50 + Flags: []cli.Flag{ 64 51 &cli.StringFlag{ 65 - Name: "outdir", 52 + Name: "lexicons-dir", 53 + Value: "./lexicons/", 54 + Usage: "base directory for project Lexicon files", 55 + Sources: cli.EnvVars("LEXICONS_DIR"), 66 56 }, 67 - &cli.BoolFlag{ 68 - Name: "gen-server", 57 + &cli.StringFlag{ 58 + Name: "output-dir", 59 + Value: "./lexgen-output/", 60 + Usage: "base directory for output packages", 61 + Sources: cli.EnvVars("OUTPUT_DIR"), 69 62 }, 70 63 &cli.BoolFlag{ 71 - Name: "gen-handlers", 72 - }, 73 - &cli.StringSliceFlag{ 74 - Name: "types-import", 75 - }, 76 - &cli.StringSliceFlag{ 77 - Name: "external-lexicons", 64 + Name: "legacy-mode", 65 + Value: true, 78 66 }, 67 + }, 68 + Action: runGen, 69 + } 70 + 71 + var cmdGen = &cli.Command{ 72 + Name: "gen", 73 + Usage: "generate code for lexicons", 74 + ArgsUsage: `<file-or-dir>*`, 75 + Flags: []cli.Flag{ 79 76 &cli.StringFlag{ 80 - Name: "package", 81 - Value: "schemagen", 77 + Name: "lexicons-dir", 78 + Value: "./lexicons/", 79 + Usage: "base directory for project Lexicon files", 80 + Sources: cli.EnvVars("LEXICONS_DIR"), 82 81 }, 83 82 &cli.StringFlag{ 84 - Name: "build", 85 - Value: "", 83 + Name: "output-dir", 84 + Value: "./lexgen-output/", 85 + Usage: "base directory for output packages", 86 + Sources: cli.EnvVars("OUTPUT_DIR"), 86 87 }, 87 - &cli.StringFlag{ 88 - Name: "build-file", 89 - Value: "", 88 + &cli.BoolFlag{ 89 + Name: "no-imports-tidy", 90 + Usage: "skip cleanup of go imports in writen output", 90 91 }, 91 - } 92 - app.Action = func(cctx *cli.Context) error { 93 - paths, err := expandArgs(cctx.Args().Slice()) 92 + }, 93 + Action: runGen, 94 + } 95 + 96 + func collectPaths(cmd *cli.Command) ([]string, lexicon.Catalog, error) { 97 + paths := cmd.Args().Slice() 98 + if !cmd.Args().Present() { 99 + paths = []string{cmd.String("lexicons-dir")} 100 + _, err := os.Stat(paths[0]) 94 101 if err != nil { 95 - return err 102 + return nil, nil, fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err) 96 103 } 104 + } 97 105 98 - var schemas []*lex.Schema 99 - for _, arg := range paths { 100 - if strings.HasSuffix(arg, "com/atproto/temp/importRepo.json") { 101 - fmt.Printf("skipping schema: %s\n", arg) 102 - continue 103 - } 104 - s, err := lex.ReadSchema(arg) 105 - if err != nil { 106 - return fmt.Errorf("failed to read file %q: %w", arg, err) 107 - } 108 - 109 - schemas = append(schemas, s) 106 + // load all directories 107 + cat := lexicon.NewBaseCatalog() 108 + lexDir := cmd.String("lexicons-dir") 109 + ldinfo, err := os.Stat(lexDir) 110 + if err == nil && ldinfo.IsDir() { 111 + if err := cat.LoadDirectory(lexDir); err != nil { 112 + return nil, nil, err 110 113 } 114 + } 111 115 112 - externalPaths, err := expandArgs(cctx.StringSlice("external-lexicons")) 116 + filePaths := []string{} 117 + 118 + for _, p := range paths { 119 + finfo, err := os.Stat(p) 113 120 if err != nil { 114 - return err 121 + return nil, nil, fmt.Errorf("failed loading %s: %w", p, err) 115 122 } 116 - var externalSchemas []*lex.Schema 117 - for _, arg := range externalPaths { 118 - s, err := lex.ReadSchema(arg) 119 - if err != nil { 120 - return fmt.Errorf("failed to read file %q: %w", arg, err) 123 + if finfo.IsDir() { 124 + if p != cmd.String("lexicons-dir") { 125 + // HACK: load first directory 126 + if err := cat.LoadDirectory(p); err != nil { 127 + return nil, nil, err 128 + } 121 129 } 130 + if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 131 + if d.IsDir() || path.Ext(fp) != ".json" { 132 + return nil 133 + } 134 + filePaths = append(filePaths, fp) 135 + return nil 136 + }); err != nil { 137 + return nil, nil, err 138 + } 139 + continue 140 + } 141 + filePaths = append(filePaths, p) 142 + } 143 + return filePaths, &cat, nil 144 + } 122 145 123 - externalSchemas = append(externalSchemas, s) 124 - } 146 + func runGen(ctx context.Context, cmd *cli.Command) error { 147 + 148 + filePaths, cat, err := collectPaths(cmd) 149 + if err != nil { 150 + return err 151 + } 125 152 126 - buildLiteral := cctx.String("build") 127 - buildPath := cctx.String("build-file") 128 - var packages []lex.Package 129 - if buildLiteral != "" { 130 - if buildPath != "" { 131 - return errors.New("must not set both --build and --build-file") 132 - } 133 - packages, err = lex.ParsePackages([]byte(buildLiteral)) 134 - if err != nil { 135 - return fmt.Errorf("--build error, %w", err) 136 - } 137 - if len(packages) == 0 { 138 - return errors.New("--build must specify at least one Package{}") 139 - } 140 - } else if buildPath != "" { 141 - blob, err := os.ReadFile(buildPath) 142 - if err != nil { 143 - return fmt.Errorf("--build-file error, %w", err) 144 - } 145 - packages, err = lex.ParsePackages(blob) 146 - if err != nil { 147 - return fmt.Errorf("--build-file error, %w", err) 148 - } 149 - if len(packages) == 0 { 150 - return errors.New("--build-file must specify at least one Package{}") 151 - } 152 - } else { 153 - return errors.New("need exactly one of --build or --build-file") 153 + for _, p := range filePaths { 154 + if err := genFile(ctx, cmd, cat, p); err != nil { 155 + return err 154 156 } 157 + } 158 + return nil 159 + } 155 160 156 - if cctx.Bool("gen-server") { 157 - pkgname := cctx.String("package") 158 - outdir := cctx.String("outdir") 159 - if outdir == "" { 160 - return fmt.Errorf("must specify output directory (--outdir)") 161 - } 162 - defmap := lex.BuildExtDefMap(append(schemas, externalSchemas...), packages) 163 - _ = defmap 161 + func genFile(ctx context.Context, cmd *cli.Command, cat lexicon.Catalog, p string) error { 162 + b, err := os.ReadFile(p) 163 + if err != nil { 164 + return fmt.Errorf("failed to read lexicon schema from disk (%s): %w", p, err) 165 + } 164 166 165 - paths := cctx.StringSlice("types-import") 166 - importmap := make(map[string]string) 167 - for _, p := range paths { 168 - parts := strings.Split(p, ":") 169 - importmap[parts[0]] = parts[1] 170 - } 167 + // parse file regularly 168 + // NOTE: use json/v2 when it stabilizes for case-sensitivity 169 + var sf lexicon.SchemaFile 171 170 172 - handlers := cctx.Bool("gen-handlers") 171 + err = json.Unmarshal(b, &sf) 172 + if err == nil { 173 + err = sf.FinishParse() 174 + } 175 + if err != nil { 176 + return fmt.Errorf("failed to parse lexicon schema from disk (%s): %w", p, err) 177 + } 173 178 174 - if err := lex.CreateHandlerStub(pkgname, importmap, outdir, schemas, handlers); err != nil { 175 - return err 176 - } 179 + flat, err := lexgen.FlattenSchemaFile(&sf) 180 + if err != nil { 181 + return fmt.Errorf("internal codegen flattening error (%s): %w", p, err) 182 + } 183 + 184 + cfg := lexgen.NewGenConfig() 185 + if cmd.Bool("legacy-mode") { 186 + cfg = lexgen.LegacyConfig() 187 + } 177 188 178 - } else { 179 - return lex.Run(schemas, externalSchemas, packages) 180 - } 189 + buf := new(bytes.Buffer) 190 + gen := lexgen.CodeGenerator{ 191 + Config: cfg, 192 + Lex: flat, 193 + Cat: cat, 194 + Out: buf, 195 + } 196 + if err := gen.WriteLexicon(); err != nil { 197 + return fmt.Errorf("failed to format codegen output (%s): %w", p, err) 198 + } 181 199 182 - return nil 200 + outPath := path.Join(cmd.String("output-dir"), gen.PkgName(), gen.FileName()) 201 + if err := os.MkdirAll(path.Dir(outPath), 0755); err != nil { 202 + return err 183 203 } 184 204 185 - app.RunAndExitOnError() 205 + if !cmd.Bool("no-imports-tidy") { 206 + // NOTE: processing imports per file gets slow if imports are missing 207 + fmtOpts := imports.Options{ 208 + Comments: true, 209 + TabIndent: false, 210 + TabWidth: 4, 211 + } 212 + formatted, err := imports.Process(outPath, buf.Bytes(), &fmtOpts) 213 + if err != nil { 214 + return fmt.Errorf("failed to format codegen output (%s): %w", p, err) 215 + } 216 + return os.WriteFile(outPath, formatted, 0644) 217 + } else { 218 + formatted, err := format.Source(buf.Bytes()) 219 + if err != nil { 220 + return fmt.Errorf("failed to format codegen output (%s): %w", p, err) 221 + } 222 + return os.WriteFile(outPath, formatted, 0644) 223 + } 186 224 }

+856

lex/lexgen/codegen.go

··· 1 + package lexgen 2 + 3 + import ( 4 + "fmt" 5 + "io" 6 + "log/slog" 7 + "sort" 8 + "strings" 9 + 10 + "github.com/bluesky-social/indigo/atproto/lexicon" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + ) 13 + 14 + // Configuration for [CodeGenerator] output 15 + type GenConfig struct { 16 + RegisterLexiconTypeID bool 17 + PackageMappings map[string]string 18 + // one of: "type-decoder", "map-string-any", "json-raw-message" 19 + UnknownType string 20 + WarningText string 21 + LegacyMode bool 22 + } 23 + 24 + func NewGenConfig() *GenConfig { 25 + return &GenConfig{ 26 + UnknownType: "map-string-any", 27 + WarningText: "Code generated by indigo lexgen tool. DO NOT EDIT MANUALLY.", 28 + } 29 + } 30 + 31 + func LegacyConfig() *GenConfig { 32 + return &GenConfig{ 33 + RegisterLexiconTypeID: true, 34 + UnknownType: "type-decoder", 35 + WarningText: "Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.", 36 + LegacyMode: true, 37 + } 38 + } 39 + 40 + // Core implementation of Go code generation for a single Lexicon schema file (multiple definitions), building on pre-parsed [FlatLexicon] 41 + type CodeGenerator struct { 42 + Config *GenConfig 43 + Lex *FlatLexicon 44 + Cat lexicon.Catalog 45 + Out io.Writer 46 + } 47 + 48 + // Outputs Go source code to the "Out" [io.Writer]. 49 + func (gen *CodeGenerator) WriteLexicon() error { 50 + 51 + if gen.Config.WarningText != "" { 52 + fmt.Fprintf(gen.Out, "// %s\n\n", gen.Config.WarningText) 53 + } 54 + fmt.Fprintf(gen.Out, "// Lexicon schema: %s\n\n", gen.Lex.NSID) 55 + fmt.Fprintf(gen.Out, "package %s\n\n", gen.PkgName()) 56 + fmt.Fprintln(gen.Out, "import (") 57 + for dep, _ := range gen.deps() { 58 + fmt.Fprintf(gen.Out, " %s\n", dep) 59 + } 60 + fmt.Fprint(gen.Out, ")\n\n") 61 + 62 + for _, ft := range gen.Lex.Types { 63 + slog.Info("generating type", "nsid", gen.Lex.NSID, "def", ft.DefName, "path", ft.Path, "type", ft.Type) 64 + if err := gen.WriteType(&ft); err != nil { 65 + return err 66 + } 67 + } 68 + return nil 69 + } 70 + 71 + func (gen *CodeGenerator) PkgName() string { 72 + n := nsidPkgName(gen.Lex.NSID) 73 + if gen.Config.LegacyMode { 74 + switch n { 75 + case "appbsky": 76 + return "bsky" 77 + case "comatproto": 78 + return "atproto" 79 + case "toolsozone": 80 + return "ozone" 81 + case "chatbsky": 82 + return "chat" 83 + } 84 + } 85 + return n 86 + } 87 + 88 + func (gen *CodeGenerator) baseName() string { 89 + // TODO: memoize this value? this method gets called a lot 90 + return nsidBaseName(gen.Lex.NSID) 91 + } 92 + 93 + func (gen *CodeGenerator) FileName() string { 94 + return nsidFileName(gen.Lex.NSID) + ".go" 95 + } 96 + 97 + func (gen *CodeGenerator) deps() map[string]bool { 98 + d := map[string]bool{ 99 + "\"context\"": true, 100 + "\"fmt\"": true, 101 + "\"io\"": true, 102 + "\"bytes\"": true, 103 + "\"encoding/json\"": true, 104 + "lexutil \"github.com/bluesky-social/indigo/lex/util\"": true, 105 + "cbg \"github.com/whyrusleeping/cbor-gen\"": true, 106 + } 107 + 108 + for ext, _ := range gen.Lex.ExternalRefs { 109 + // TODO: replace this with configurable/extensible mappings 110 + if strings.HasPrefix(ext, "com.atproto.") { 111 + d["comatproto \"github.com/bluesky-social/indigo/api/atproto\""] = true 112 + } else if strings.HasPrefix(ext, "app.bsky.") { 113 + d["appbsky \"github.com/bluesky-social/indigo/api/bsky\""] = true 114 + } else if strings.HasPrefix(ext, "tools.ozone.") { 115 + d["toolsozone \"github.com/bluesky-social/indigo/api/ozone\""] = true 116 + } else if strings.HasPrefix(ext, "chat.bsky.") { 117 + d["chatbsky \"github.com/bluesky-social/indigo/api/chat\""] = true 118 + } else { 119 + // TODO: configurable mappings; and return error if none found 120 + slog.Error("unhandled external reference", "ref", ext) 121 + } 122 + } 123 + return d 124 + } 125 + 126 + func (gen *CodeGenerator) WriteType(ft *FlatType) error { 127 + 128 + switch v := ft.Schema.Inner.(type) { 129 + case lexicon.SchemaRecord: 130 + if gen.Config.RegisterLexiconTypeID { 131 + fmt.Fprintf(gen.Out, "func init() {\n") 132 + fmt.Fprintf(gen.Out, " lexutil.RegisterType(\"%s\", &%s{})", gen.Lex.NSID, gen.baseName()) 133 + fmt.Fprintf(gen.Out, "}\n\n") 134 + } 135 + // HACK: insert record-level description in to object if nil 136 + if v.Description != nil && v.Record.Description == nil { 137 + v.Record.Description = v.Description 138 + } 139 + if err := gen.writeStruct(ft, &v.Record); err != nil { 140 + return err 141 + } 142 + case lexicon.SchemaQuery: 143 + return gen.writeEndpoint(ft, defDescription(ft.Schema), v.Parameters, v.Output, nil, false) 144 + case lexicon.SchemaProcedure: 145 + return gen.writeEndpoint(ft, defDescription(ft.Schema), v.Parameters, v.Output, v.Input, true) 146 + case lexicon.SchemaSubscription: 147 + // pass; we only generate message types, not overall subscription 148 + case lexicon.SchemaPermissionSet, lexicon.SchemaPermission: 149 + // pass for Go codegen 150 + case lexicon.SchemaToken: 151 + // TODO: pass for now; could be a var/const? 152 + case lexicon.SchemaString, lexicon.SchemaInteger, lexicon.SchemaBoolean, lexicon.SchemaUnknown: 153 + // skip 154 + case lexicon.SchemaObject: 155 + if gen.Config.RegisterLexiconTypeID && ft.DefName == "main" && len(ft.Path) == 0 { 156 + fmt.Fprintf(gen.Out, "func init() {\n") 157 + fmt.Fprintf(gen.Out, " lexutil.RegisterType(\"%s#main\", &%s{})", gen.Lex.NSID, gen.baseName()) 158 + fmt.Fprintf(gen.Out, "}\n\n") 159 + } 160 + if err := gen.writeStruct(ft, &v); err != nil { 161 + return err 162 + } 163 + case lexicon.SchemaUnion: 164 + return gen.writeUnion(ft, &v) 165 + case lexicon.SchemaRef: 166 + // skip for now. could be an alias type? 167 + default: 168 + return fmt.Errorf("unhandled schema type for codegen: %T", ft.Schema.Inner) 169 + } 170 + 171 + return nil 172 + } 173 + 174 + func isRequired(required []string, fname string) bool { 175 + for _, k := range required { 176 + if k == fname { 177 + return true 178 + } 179 + } 180 + return false 181 + } 182 + 183 + func (gen *CodeGenerator) fieldType(fname string, def *lexicon.SchemaDef, optional bool) (string, error) { 184 + // NOTE: SchemaObject and SchemaUnion should be handled outside this function; as well as arrays of those types also count 185 + // TODO: another pass to check for type completeness 186 + switch v := def.Inner.(type) { 187 + case lexicon.SchemaNull: 188 + // NOTE: using "any" as a generic 'nil' type 189 + return "any", nil 190 + case lexicon.SchemaBoolean: 191 + if optional { 192 + return "*bool", nil 193 + } else { 194 + return "bool", nil 195 + } 196 + case lexicon.SchemaInteger: 197 + if optional { 198 + return "*int64", nil 199 + } else { 200 + return "int64", nil 201 + } 202 + case lexicon.SchemaString: 203 + if optional { 204 + return "*string", nil 205 + } else { 206 + return "string", nil 207 + } 208 + case lexicon.SchemaBytes: 209 + // NOTE: not using a pointer for optional 210 + return "lexutil.LexBytes", nil 211 + case lexicon.SchemaCIDLink: 212 + if optional { 213 + return "*lexutil.LexLink", nil 214 + } else { 215 + return "lexutil.LexLink", nil 216 + } 217 + case lexicon.SchemaBlob: 218 + if optional || gen.Config.LegacyMode { 219 + return "*lexutil.LexBlob", nil 220 + } else { 221 + return "lexutil.LexBlob", nil 222 + } 223 + case lexicon.SchemaArray: 224 + t, err := gen.fieldType(fname, &v.Items, false) 225 + if err != nil { 226 + return "", err 227 + } 228 + // NOTE: not using a pointer for optional 229 + return "[]" + t, nil 230 + case lexicon.SchemaUnknown: 231 + switch gen.Config.UnknownType { 232 + case "type-decoder": 233 + if gen.Config.LegacyMode && (fname == "didDoc" || fname == "plcOp" || fname == "meta" || fname == "debug") { 234 + if optional { 235 + return "*interface{}", nil 236 + } else { 237 + return "interface{}", nil 238 + } 239 + } 240 + return "*lexutil.LexiconTypeDecoder", nil 241 + case "json-raw-message": 242 + if optional { 243 + return "*json.RawMessage", nil 244 + } else { 245 + return "json.RawMessage", nil 246 + } 247 + case "map-string-any": 248 + return "map[string]any", nil 249 + default: 250 + return "map[string]any", nil 251 + } 252 + case lexicon.SchemaRef: 253 + ptr := "" 254 + if optional { 255 + ptr = "*" 256 + } 257 + 258 + // check for local references to concrete types first 259 + if strings.HasPrefix(v.Ref, "#") { 260 + dt, ok := gen.Lex.Defs[v.Ref[1:]] 261 + if !ok { 262 + return "", fmt.Errorf("broken self-reference: %s", v.Ref) 263 + } 264 + switch dt.Type { 265 + case "string": 266 + if gen.Config.LegacyMode { 267 + ptr = "*" 268 + } 269 + return ptr + "string", nil 270 + case "integer": 271 + return ptr + "int64", nil 272 + case "boolean": 273 + return ptr + "bool", nil 274 + // TODO: "unknown", "ref", "token", etc 275 + case "array": 276 + // TODO: more completeness here (eg, non-object types) 277 + structPtr := "" 278 + if gen.Config.LegacyMode { 279 + structPtr = "*" 280 + } 281 + return fmt.Sprintf("[]%s%s_%s_Elem", structPtr, gen.baseName(), strings.Title(v.Ref[1:])), nil 282 + default: // presumed "object", "union" 283 + if gen.Config.LegacyMode { 284 + ptr = "*" 285 + } 286 + if v.Ref == "#main" { 287 + return ptr + gen.baseName(), nil 288 + } 289 + return fmt.Sprintf("%s%s_%s", ptr, gen.baseName(), strings.Title(v.Ref[1:])), nil 290 + } 291 + } 292 + 293 + // external reference 294 + t, err := gen.externalRefType(v.Ref) 295 + if err != nil { 296 + return "", err 297 + } 298 + if gen.Config.LegacyMode { 299 + ptr = "*" 300 + } 301 + return ptr + t, nil 302 + default: 303 + return "", fmt.Errorf("unhandled schema type in struct field: %T", def.Inner) 304 + } 305 + } 306 + 307 + func (gen *CodeGenerator) externalRefType(ref string) (string, error) { 308 + s, err := gen.Cat.Resolve(ref) 309 + if err != nil { 310 + return "", fmt.Errorf("could not resolve lexicon reference (%s): %w", ref, err) 311 + } 312 + 313 + switch s.Def.(type) { 314 + case lexicon.SchemaString: 315 + return "string", nil 316 + // TODO: other concrete types and special-cases types, like arrays 317 + } 318 + 319 + parts := strings.SplitN(ref, "#", 3) 320 + if len(parts) > 2 { 321 + return "", fmt.Errorf("failed to parse external ref: %s", ref) 322 + } 323 + nsid, err := syntax.ParseNSID(parts[0]) 324 + if err != nil { 325 + return "", fmt.Errorf("failed to parse external ref NSID (%s): %w", ref, err) 326 + } 327 + 328 + // check if this is actually in the same package (which might not mean the same NSID authority) 329 + if nsidPkgName(nsid) == nsidPkgName(gen.Lex.NSID) { 330 + if len(parts) == 1 || parts[1] == "main" { 331 + return nsidBaseName(nsid), nil 332 + } else { 333 + return fmt.Sprintf("%s_%s", nsidBaseName(nsid), strings.Title(parts[1])), nil 334 + } 335 + } 336 + 337 + if len(parts) == 1 || parts[1] == "main" { 338 + return fmt.Sprintf("%s.%s", nsidPkgName(nsid), nsidBaseName(nsid)), nil 339 + } else { 340 + return fmt.Sprintf("%s.%s_%s", nsidPkgName(nsid), nsidBaseName(nsid), strings.Title(parts[1])), nil 341 + } 342 + } 343 + 344 + func (gen *CodeGenerator) writeStruct(ft *FlatType, obj *lexicon.SchemaObject) error { 345 + 346 + name := gen.baseName() 347 + if ft.DefName != "main" { 348 + name += "_" + strings.Title(ft.DefName) 349 + } 350 + for _, sub := range ft.Path { 351 + name += "_" + strings.Title(sub) 352 + } 353 + 354 + if ft.DefName != "main" && len(ft.Path) == 0 { 355 + fmt.Fprintf(gen.Out, "// %s is a \"%s\" in the %s schema.\n", name, ft.DefName, gen.Lex.NSID) 356 + if obj.Description != nil { 357 + fmt.Fprintln(gen.Out, "//") 358 + } 359 + } 360 + if gen.Lex.Defs[ft.DefName].Type == "procedure" && len(ft.Path) == 1 && ft.Path[0] == "input" { 361 + // TODO: "request body" 362 + fmt.Fprintf(gen.Out, "// %s is the input argument to a %s call.\n", name, gen.Lex.NSID) 363 + } 364 + if (gen.Lex.Defs[ft.DefName].Type == "query" || gen.Lex.Defs[ft.DefName].Type == "procedure") && len(ft.Path) == 1 && ft.Path[0] == "output" { 365 + // TODO: "response body" 366 + fmt.Fprintf(gen.Out, "// %s is the output of a %s call.\n", name, gen.Lex.NSID) 367 + } 368 + skipDesc := false 369 + if gen.Config.LegacyMode && ft.Type == "record" { 370 + skipDesc = true 371 + } 372 + if obj.Description != nil && !skipDesc { 373 + for _, l := range strings.Split(*obj.Description, "\n") { 374 + fmt.Fprintf(gen.Out, "// %s\n", l) 375 + } 376 + } 377 + fmt.Fprintf(gen.Out, "type %s struct {\n", name) 378 + 379 + // iterate field in sorted order 380 + fieldNames := []string{} 381 + for fname := range obj.Properties { 382 + fieldNames = append(fieldNames, fname) 383 + } 384 + sort.Strings(fieldNames) 385 + 386 + // if this is a def-level struct, write out type decoder 387 + skipType := false 388 + if gen.Config.LegacyMode { 389 + // TODO: skip $type for all defs in subscription. this isn't robust! 390 + switch gen.Lex.MainType() { 391 + case "subscription": 392 + skipType = true 393 + } 394 + } 395 + if len(ft.Path) == 0 && !skipType { 396 + // TODO: can skip in some more situations? 397 + fullName := gen.Lex.NSID.String() 398 + if ft.DefName != "main" { 399 + fullName += "#" + ft.DefName 400 + } 401 + omitempty := "" 402 + if gen.Config.LegacyMode && gen.Lex.NSID.String() == "com.atproto.repo.strongRef" { 403 + omitempty = ",omitempty" 404 + } 405 + fmt.Fprintf(gen.Out, " LexiconTypeID string `json:\"$type%s\" cborgen:\"$type,const=%s%s\"`\n", omitempty, fullName, omitempty) 406 + } 407 + 408 + for _, fname := range fieldNames { 409 + field := obj.Properties[fname] 410 + optional := false 411 + omitempty := "" 412 + if obj.IsNullable(fname) || !isRequired(obj.Required, fname) { 413 + optional = true 414 + omitempty = ",omitempty" 415 + } 416 + 417 + var t string 418 + var err error 419 + 420 + switch v := field.Inner.(type) { 421 + case lexicon.SchemaObject, lexicon.SchemaUnion: 422 + t = name + "_" + strings.Title(fname) 423 + if optional || gen.Config.LegacyMode { 424 + t = "*" + t 425 + } 426 + case lexicon.SchemaArray: 427 + switch v.Items.Inner.(type) { 428 + case lexicon.SchemaObject, lexicon.SchemaUnion: 429 + elemPtr := "" 430 + if gen.Config.LegacyMode { 431 + elemPtr = "*" 432 + } 433 + // NOTE: not using ptr for optional 434 + t = fmt.Sprintf("[]%s%s_%s_Elem", elemPtr, name, strings.Title(fname)) 435 + default: 436 + t, err = gen.fieldType(fname, &field, optional) 437 + if err != nil { 438 + return err 439 + } 440 + } 441 + default: 442 + t, err = gen.fieldType(fname, &field, optional) 443 + if err != nil { 444 + return err 445 + } 446 + } 447 + 448 + cborExtra := "" 449 + // HACK: copied from legacy code for now 450 + if gen.Lex.NSID.String() == "com.atproto.label.defs" && name == "LabelDefs_SelfLabels" && fname == "values" { 451 + cborExtra = ",preservenil" 452 + } 453 + 454 + desc := defDescription(&field) 455 + if desc != "" { 456 + fmt.Fprintf(gen.Out, " // %s: %s\n", fname, desc) 457 + } 458 + fmt.Fprintf(gen.Out, " %s %s", strings.ReplaceAll(strings.Title(fname), "-", ""), t) 459 + fmt.Fprintf(gen.Out, " `json:\"%s%s\" cborgen:\"%s%s%s\"`\n", fname, omitempty, fname, omitempty, cborExtra) 460 + } 461 + fmt.Fprintf(gen.Out, "}\n\n") 462 + 463 + return nil 464 + } 465 + 466 + type unionRef struct { 467 + FieldName string 468 + TypeName string 469 + LexName string 470 + } 471 + 472 + func (gen *CodeGenerator) writeUnion(ft *FlatType, union *lexicon.SchemaUnion) error { 473 + 474 + name := gen.baseName() 475 + if ft.DefName != "main" { 476 + name += "_" + strings.Title(ft.DefName) 477 + } 478 + for _, sub := range ft.Path { 479 + name += "_" + strings.Title(sub) 480 + } 481 + 482 + unionRefs := map[string]unionRef{} 483 + refNames := []string{} 484 + for _, ref := range union.Refs { 485 + r := unionRef{ 486 + LexName: ref, 487 + } 488 + 489 + if strings.HasPrefix(ref, "#") { 490 + r.LexName = gen.Lex.NSID.String() + ref 491 + n := gen.baseName() 492 + if ref != "#main" { 493 + n += "_" + strings.Title(ref[1:]) 494 + } 495 + r.FieldName = n 496 + r.TypeName = n 497 + } else { 498 + n, err := gen.externalRefType(ref) 499 + if err != nil { 500 + return err 501 + } 502 + r.FieldName = n 503 + r.TypeName = n 504 + if strings.Contains(n, ".") { 505 + parts := strings.SplitN(n, ".", 2) 506 + r.FieldName = parts[1] 507 + } 508 + } 509 + refNames = append(refNames, r.FieldName) 510 + unionRefs[r.FieldName] = r 511 + } 512 + if !gen.Config.LegacyMode { 513 + sort.Strings(refNames) 514 + } 515 + 516 + // first print out the union struct type 517 + if union.Description != nil { 518 + for _, l := range strings.Split(*union.Description, "\n") { 519 + fmt.Fprintf(gen.Out, "// %s\n", l) 520 + } 521 + } 522 + fmt.Fprintf(gen.Out, "type %s struct {\n", name) 523 + for _, rname := range refNames { 524 + ref := unionRefs[rname] 525 + fmt.Fprintf(gen.Out, " %s *%s\n", ref.FieldName, ref.TypeName) 526 + } 527 + fmt.Fprintf(gen.Out, "}\n\n") 528 + 529 + // ... then MarshalJSON 530 + fmt.Fprintf(gen.Out, "func (t *%s) MarshalJSON() ([]byte, error) {\n", name) 531 + for _, rname := range refNames { 532 + ref := unionRefs[rname] 533 + fmt.Fprintf(gen.Out, " if t.%s != nil {\n", ref.FieldName) 534 + fmt.Fprintf(gen.Out, " t.%s.LexiconTypeID = \"%s\"\n", ref.FieldName, ref.LexName) 535 + fmt.Fprintf(gen.Out, " return json.Marshal(t.%s)\n", ref.FieldName) 536 + fmt.Fprintf(gen.Out, " }\n") 537 + } 538 + fmt.Fprintf(gen.Out, " return nil, fmt.Errorf(\"can not marshal empty union as JSON\")") 539 + fmt.Fprintf(gen.Out, "}\n\n") 540 + 541 + // ... then UnmarshalJSON 542 + fmt.Fprintf(gen.Out, "func (t *%s) UnmarshalJSON(b []byte) error {\n", name) 543 + fmt.Fprintf(gen.Out, " typ, err := lexutil.TypeExtract(b)\n") 544 + fmt.Fprintf(gen.Out, " if err != nil {\n") 545 + fmt.Fprintf(gen.Out, " return err\n") 546 + fmt.Fprintf(gen.Out, " }\n\n") 547 + fmt.Fprintf(gen.Out, " switch typ {\n") 548 + for _, rname := range refNames { 549 + ref := unionRefs[rname] 550 + fmt.Fprintf(gen.Out, " case \"%s\":\n", ref.LexName) 551 + fmt.Fprintf(gen.Out, " t.%s = new(%s)\n", ref.FieldName, ref.TypeName) 552 + fmt.Fprintf(gen.Out, " return json.Unmarshal(b, t.%s)\n", ref.FieldName) 553 + } 554 + fmt.Fprintf(gen.Out, " default:\n") 555 + if union.Closed != nil && *union.Closed { 556 + // TODO: better error message 557 + fmt.Fprintf(gen.Out, " return fmt.Errorf(\"closed unions must match a listed schema\")\n") 558 + } else { 559 + fmt.Fprintf(gen.Out, " return nil\n") 560 + } 561 + fmt.Fprintf(gen.Out, " }\n") 562 + fmt.Fprintf(gen.Out, "}\n\n") 563 + 564 + // only import CBOR marshalling of unions in legacy mode 565 + if !gen.Config.LegacyMode { 566 + return nil 567 + } 568 + 569 + switch gen.Lex.MainType() { 570 + case "record", "subscription": 571 + // no-op 572 + case "object": 573 + // hacks for legacy serialization 574 + nsid := gen.Lex.NSID.String() 575 + if !(nsid == "app.bsky.richtext.facet" || (nsid == "app.bsky.embed.recordWithMedia" && ft.DefName == "main")) { 576 + return nil 577 + } 578 + default: 579 + return nil 580 + } 581 + 582 + // ... then MarshalCBOR 583 + fmt.Fprintf(gen.Out, "func (t *%s) MarshalCBOR(w io.Writer) error {\n\n", name) 584 + fmt.Fprintf(gen.Out, " if t == nil {\n") 585 + fmt.Fprintf(gen.Out, " _, err := w.Write(cbg.CborNull)\n") 586 + fmt.Fprintf(gen.Out, " return err") 587 + fmt.Fprintf(gen.Out, " }\n") 588 + for _, rname := range refNames { 589 + ref := unionRefs[rname] 590 + fmt.Fprintf(gen.Out, " if t.%s != nil {\n", ref.FieldName) 591 + fmt.Fprintf(gen.Out, " return t.%s.MarshalCBOR(w)\n", ref.FieldName) 592 + fmt.Fprintf(gen.Out, " }\n") 593 + } 594 + fmt.Fprintf(gen.Out, " return fmt.Errorf(\"can not marshal empty union as CBOR\")") 595 + fmt.Fprintf(gen.Out, "}\n\n") 596 + 597 + // ... then UnmarshalCBOR 598 + fmt.Fprintf(gen.Out, "func (t *%s) UnmarshalCBOR(r io.Reader) error {\n", name) 599 + fmt.Fprintf(gen.Out, " typ, b, err := lexutil.CborTypeExtractReader(r)\n") 600 + fmt.Fprintf(gen.Out, " if err != nil {\n") 601 + fmt.Fprintf(gen.Out, " return err\n") 602 + fmt.Fprintf(gen.Out, " }\n\n") 603 + fmt.Fprintf(gen.Out, " switch typ {\n") 604 + for _, rname := range refNames { 605 + ref := unionRefs[rname] 606 + fmt.Fprintf(gen.Out, " case \"%s\":\n", ref.LexName) 607 + fmt.Fprintf(gen.Out, " t.%s = new(%s)\n", ref.FieldName, ref.TypeName) 608 + fmt.Fprintf(gen.Out, " return t.%s.UnmarshalCBOR(bytes.NewReader(b))\n", ref.FieldName) 609 + } 610 + fmt.Fprintf(gen.Out, " default:\n") 611 + fmt.Fprintf(gen.Out, " return nil\n") 612 + fmt.Fprintf(gen.Out, " }\n") 613 + fmt.Fprintf(gen.Out, "}\n\n") 614 + 615 + return nil 616 + } 617 + 618 + func (gen *CodeGenerator) writeEndpoint(ft *FlatType, desc string, params *lexicon.SchemaParams, output, input *lexicon.SchemaBody, isProcedure bool) error { 619 + name := gen.baseName() 620 + 621 + fmt.Fprintf(gen.Out, "// %s calls the XRPC method \"%s\".\n", name, gen.Lex.NSID) 622 + if desc != "" && !gen.Config.LegacyMode { 623 + fmt.Fprintln(gen.Out, "//") 624 + for _, l := range strings.Split(desc, "\n") { 625 + fmt.Fprintf(gen.Out, "// %s\n", l) 626 + } 627 + } 628 + 629 + outputBytes := false 630 + outputStruct := "" 631 + if output != nil && output.Schema != nil { 632 + switch v := output.Schema.Inner.(type) { 633 + case lexicon.SchemaObject, lexicon.SchemaUnion: 634 + outputStruct = name + "_Output" 635 + case lexicon.SchemaRef: 636 + if strings.HasPrefix(v.Ref, "#") { 637 + // local reference 638 + outputStruct = fmt.Sprintf("%s_%s", gen.baseName(), strings.Title(v.Ref[1:])) 639 + } else { 640 + // external reference 641 + t, err := gen.externalRefType(v.Ref) 642 + if err != nil { 643 + return err 644 + } 645 + outputStruct = t 646 + } 647 + default: 648 + return fmt.Errorf("unsupported endpoint output schema def type: %T", output.Schema.Inner) 649 + } 650 + } else if output != nil && output.Encoding != "" { 651 + outputBytes = true 652 + } 653 + 654 + paramNames := []string{} 655 + if params != nil { 656 + for name := range params.Properties { 657 + paramNames = append(paramNames, name) 658 + } 659 + } 660 + sort.Strings(paramNames) 661 + 662 + args := []string{"ctx context.Context", "c lexutil.LexClient"} 663 + reqParams := []string{} 664 + optParams := []string{} 665 + if len(paramNames) > 0 { 666 + fmt.Fprintln(gen.Out, "//") 667 + for _, name := range paramNames { 668 + param := params.Properties[name] 669 + ptr := "*" 670 + if isRequired(params.Required, name) { 671 + ptr = "" 672 + reqParams = append(reqParams, name) 673 + } else { 674 + optParams = append(optParams, name) 675 + } 676 + switch v := param.Inner.(type) { 677 + case lexicon.SchemaBoolean: 678 + if v.Description != nil && *v.Description != "" { 679 + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) 680 + } 681 + if gen.Config.LegacyMode { 682 + ptr = "" 683 + } 684 + args = append(args, fmt.Sprintf("%s %sbool", name, ptr)) 685 + case lexicon.SchemaInteger: 686 + if v.Description != nil && *v.Description != "" { 687 + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) 688 + } 689 + if gen.Config.LegacyMode { 690 + ptr = "" 691 + } 692 + args = append(args, fmt.Sprintf("%s %sint64", name, ptr)) 693 + case lexicon.SchemaString: 694 + if v.Description != nil && *v.Description != "" { 695 + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) 696 + } 697 + args = append(args, fmt.Sprintf("%s string", name)) 698 + case lexicon.SchemaUnknown: 699 + if v.Description != nil && *v.Description != "" { 700 + fmt.Fprintf(gen.Out, "// %s: %s\n", name, *v.Description) 701 + } 702 + args = append(args, fmt.Sprintf("%s any", name)) 703 + case lexicon.SchemaArray: 704 + if v.Description != nil && *v.Description != "" { 705 + suffix := "[]" 706 + if gen.Config.LegacyMode { 707 + suffix = "" 708 + } 709 + fmt.Fprintf(gen.Out, "// %s%s: %s\n", name, suffix, *v.Description) 710 + } 711 + switch v.Items.Inner.(type) { 712 + case lexicon.SchemaBoolean: 713 + args = append(args, fmt.Sprintf("%s []bool", name)) 714 + case lexicon.SchemaInteger: 715 + args = append(args, fmt.Sprintf("%s []int64", name)) 716 + case lexicon.SchemaString: 717 + args = append(args, fmt.Sprintf("%s []string", name)) 718 + default: 719 + return fmt.Errorf("unsupported parameter array type: %T", param.Inner) 720 + } 721 + default: 722 + return fmt.Errorf("unsupported parameter type: %T", param.Inner) 723 + } 724 + } 725 + } 726 + 727 + inputArg := "nil" 728 + inputEncoding := "" 729 + inputStruct := "" 730 + if isProcedure && input != nil && input.Schema != nil { 731 + inputArg = "input" 732 + inputEncoding = input.Encoding 733 + switch v := input.Schema.Inner.(type) { 734 + case lexicon.SchemaObject, lexicon.SchemaUnion: 735 + inputStruct = name + "_Input" 736 + case lexicon.SchemaRef: 737 + if strings.HasPrefix(v.Ref, "#") { 738 + // local reference 739 + inputStruct = fmt.Sprintf("%s_%s", gen.baseName(), strings.Title(v.Ref[1:])) 740 + } else { 741 + // external reference 742 + t, err := gen.externalRefType(v.Ref) 743 + if err != nil { 744 + return err 745 + } 746 + inputStruct = t 747 + } 748 + } 749 + args = append(args, fmt.Sprintf("input *%s", inputStruct)) 750 + } else if isProcedure && input != nil && input.Encoding != "" { 751 + inputArg = "input" 752 + inputEncoding = input.Encoding 753 + args = append(args, "input io.Reader") 754 + } 755 + 756 + doOutParam := "" 757 + returnType := "" 758 + fmt.Fprintf(gen.Out, "func %s(%s) ", name, strings.Join(args, ", ")) 759 + if outputStruct != "" { 760 + fmt.Fprintf(gen.Out, "(*%s, error) {\n", outputStruct) 761 + fmt.Fprintf(gen.Out, " var out %s\n", outputStruct) 762 + if !gen.Config.LegacyMode { 763 + fmt.Fprintln(gen.Out, "") 764 + } 765 + doOutParam = "&out" 766 + returnType = "&out" 767 + } else if outputBytes { 768 + fmt.Fprintf(gen.Out, "([]byte, error) {\n") 769 + fmt.Fprintf(gen.Out, " buf := new(bytes.Buffer)\n\n") 770 + doOutParam = "buf" 771 + returnType = "buf.Bytes()" 772 + } else { 773 + fmt.Fprintf(gen.Out, "error {\n") 774 + doOutParam = "nil" 775 + } 776 + paramsArg := "nil" 777 + if params != nil && len(params.Properties) > 0 { 778 + paramsArg = "params" 779 + if gen.Config.LegacyMode { 780 + fmt.Fprintln(gen.Out, "") 781 + } 782 + // TODO: switch to map[string]any 783 + fmt.Fprintf(gen.Out, " params := map[string]interface{}{}\n") 784 + } 785 + for _, name := range optParams { 786 + param := params.Properties[name] 787 + switch param.Inner.(type) { 788 + case lexicon.SchemaString: 789 + fmt.Fprintf(gen.Out, " if %s != \"\" {\n", name) 790 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 791 + fmt.Fprintf(gen.Out, " }\n") 792 + case lexicon.SchemaArray: 793 + fmt.Fprintf(gen.Out, " if len(%s) != 0 {\n", name) 794 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 795 + fmt.Fprintf(gen.Out, " }\n") 796 + case lexicon.SchemaUnknown: 797 + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) 798 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 799 + fmt.Fprintf(gen.Out, " }\n") 800 + case lexicon.SchemaInteger: 801 + if gen.Config.LegacyMode { 802 + fmt.Fprintf(gen.Out, " if %s != 0 {\n", name) 803 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 804 + fmt.Fprintf(gen.Out, " }\n") 805 + } else { 806 + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) 807 + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) 808 + fmt.Fprintf(gen.Out, " }\n") 809 + } 810 + case lexicon.SchemaBoolean: 811 + if gen.Config.LegacyMode { 812 + fmt.Fprintf(gen.Out, " if %s {\n", name) 813 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 814 + fmt.Fprintf(gen.Out, " }\n") 815 + } else { 816 + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) 817 + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) 818 + fmt.Fprintf(gen.Out, " }\n") 819 + } 820 + default: 821 + fmt.Fprintf(gen.Out, " if %s != nil {\n", name) 822 + fmt.Fprintf(gen.Out, " params[\"%s\"] = *%s\n", name, name) 823 + fmt.Fprintf(gen.Out, " }\n") 824 + } 825 + } 826 + for _, name := range reqParams { 827 + fmt.Fprintf(gen.Out, " params[\"%s\"] = %s\n", name, name) 828 + } 829 + if !gen.Config.LegacyMode { 830 + fmt.Fprintln(gen.Out, "") 831 + } 832 + 833 + method := "lexutil.Query" 834 + if isProcedure { 835 + method = "lexutil.Procedure" 836 + } 837 + 838 + fmt.Fprintf(gen.Out, " if err := c.LexDo(ctx, %s, \"%s\", \"%s\", %s, %s, %s); err != nil {\n", method, inputEncoding, gen.Lex.NSID, paramsArg, inputArg, doOutParam) 839 + if returnType != "" { 840 + fmt.Fprintf(gen.Out, " return nil, err\n") 841 + } else { 842 + fmt.Fprintf(gen.Out, " return err\n") 843 + } 844 + fmt.Fprintf(gen.Out, " }\n") 845 + if gen.Config.LegacyMode { 846 + fmt.Fprintln(gen.Out, "") 847 + } 848 + if returnType != "" { 849 + fmt.Fprintf(gen.Out, " return %s, nil\n", returnType) 850 + } else { 851 + fmt.Fprintf(gen.Out, " return nil\n") 852 + } 853 + fmt.Fprintf(gen.Out, "}\n\n") 854 + 855 + return nil 856 + }

+17

lex/lexgen/doc.go

··· 1 + /* 2 + Package implementing Go code generation for lexicon schemas. 3 + 4 + Used by the 'lexgen' CLI tool to output Go structs and client API helpers based on Lexicon schemas. This package currently includes a "legacy" mode to stay as close as possible to the previous code generation output. 5 + 6 + WARNING: this package is still a work in progress. Both the package API and the generated code are likely to change, possibly in backwards-incompatible ways. 7 + 8 + # Package Structure 9 + 10 + The package works in two steps: 11 + 12 + - "flattening" parses a full lexicon schema file and copies nested type definitions in to a top-level array 13 + - code generation outputs a single Go source code file corresponding to a flattened lexicon schema file 14 + 15 + Wrapping code is expected to handle code formatting and fixing imports (which mostly means removing unused imports). 16 + */ 17 + package lexgen

+247

lex/lexgen/flatten.go

··· 1 + package lexgen 2 + 3 + import ( 4 + "fmt" 5 + "slices" 6 + "sort" 7 + "strings" 8 + 9 + "github.com/bluesky-social/indigo/atproto/lexicon" 10 + "github.com/bluesky-social/indigo/atproto/syntax" 11 + ) 12 + 13 + // Intermediate representation of a complete lexicon schema file, containing one or more definitions. 14 + type FlatLexicon struct { 15 + NSID syntax.NSID 16 + Description *string 17 + ExternalRefs map[string]bool // NSID with optional ref 18 + Defs map[string]FlatDef 19 + Types []FlatType 20 + } 21 + 22 + // Minimal context about an individual top-level schema definition: just the short name and schema type. 23 + type FlatDef struct { 24 + Name string 25 + Type string 26 + } 27 + 28 + // An individual "type definition", which is a small unit of schema definition that corresponds to a named unit of generated code. For example, a struct or API endpoint. 29 + type FlatType struct { 30 + // the short name of the schema def that this type is under 31 + DefName string 32 + Path []string 33 + Type string 34 + Schema *lexicon.SchemaDef 35 + } 36 + 37 + func FlattenSchemaFile(sf *lexicon.SchemaFile) (*FlatLexicon, error) { 38 + nsid, err := syntax.ParseNSID(sf.ID) 39 + if err != nil { 40 + return nil, err 41 + } 42 + 43 + fl := FlatLexicon{ 44 + NSID: nsid, 45 + Description: sf.Description, 46 + ExternalRefs: map[string]bool{}, 47 + Defs: map[string]FlatDef{}, 48 + Types: []FlatType{}, 49 + } 50 + 51 + // iterate defs in sorted order; except "main" is always first if present 52 + defNames := []string{} 53 + hasMain := false 54 + for name := range sf.Defs { 55 + if name == "main" { 56 + hasMain = true 57 + continue 58 + } 59 + defNames = append(defNames, name) 60 + } 61 + sort.Strings(defNames) 62 + if hasMain { 63 + defNames = append([]string{"main"}, defNames...) 64 + } 65 + 66 + for _, name := range defNames { 67 + def := sf.Defs[name] 68 + if err := fl.flattenDef(name, &def); err != nil { 69 + return nil, err 70 + } 71 + } 72 + 73 + return &fl, nil 74 + } 75 + 76 + func (fl *FlatLexicon) flattenDef(name string, def *lexicon.SchemaDef) error { 77 + 78 + t, err := defType(def) 79 + if err != nil { 80 + return err 81 + } 82 + 83 + fd := FlatDef{ 84 + Name: name, 85 + Type: t, 86 + } 87 + fl.Defs[name] = fd 88 + 89 + return fl.flattenType(&fd, []string{}, def) 90 + } 91 + 92 + func (fl *FlatLexicon) flattenType(fd *FlatDef, tpath []string, def *lexicon.SchemaDef) error { 93 + 94 + t, err := defType(def) 95 + if err != nil { 96 + return err 97 + } 98 + 99 + ft := FlatType{ 100 + DefName: fd.Name, 101 + Path: slices.Clone(tpath), 102 + Type: t, 103 + Schema: def, 104 + } 105 + 106 + switch v := def.Inner.(type) { 107 + case lexicon.SchemaRecord: 108 + fl.Types = append(fl.Types, ft) 109 + if err := fl.flattenObject(fd, tpath, &v.Record); err != nil { 110 + return err 111 + } 112 + case lexicon.SchemaQuery: 113 + // v.Properties: only boolean, integer, string, or unknown are allowed, so recursion not really needed? 114 + if v.Output != nil && v.Output.Schema != nil { 115 + tp := slices.Clone(tpath) 116 + tp = append(tp, "output") 117 + if err := fl.flattenType(fd, tp, v.Output.Schema); err != nil { 118 + return err 119 + } 120 + } 121 + fl.Types = append(fl.Types, ft) 122 + case lexicon.SchemaProcedure: 123 + // v.Properties: same as above 124 + if v.Input != nil && v.Input.Schema != nil { 125 + tp := slices.Clone(tpath) 126 + tp = append(tp, "input") 127 + if err := fl.flattenType(fd, tp, v.Input.Schema); err != nil { 128 + return err 129 + } 130 + } 131 + if v.Output != nil && v.Output.Schema != nil { 132 + tp := slices.Clone(tpath) 133 + tp = append(tp, "output") 134 + if err := fl.flattenType(fd, tp, v.Output.Schema); err != nil { 135 + return err 136 + } 137 + } 138 + fl.Types = append(fl.Types, ft) 139 + case lexicon.SchemaSubscription: 140 + // v.Properties: same as above 141 + if v.Message != nil { 142 + switch vv := v.Message.Schema.Inner.(type) { 143 + case lexicon.SchemaUnion: 144 + for _, ref := range vv.Refs { 145 + if !strings.HasPrefix(ref, "#") { 146 + fl.ExternalRefs[strings.TrimSuffix(ref, "#main")] = true 147 + } 148 + } 149 + default: 150 + return fmt.Errorf("subscription with non-union message schema: %T", v.Message.Schema.Inner) 151 + } 152 + } 153 + fl.Types = append(fl.Types, ft) 154 + case lexicon.SchemaObject: 155 + fl.Types = append(fl.Types, ft) 156 + if err := fl.flattenObject(fd, tpath, &v); err != nil { 157 + return err 158 + } 159 + case lexicon.SchemaRef: 160 + if !strings.HasPrefix(v.Ref, "#") { 161 + fl.ExternalRefs[strings.TrimSuffix(v.Ref, "#main")] = true 162 + } 163 + fl.Types = append(fl.Types, ft) 164 + case lexicon.SchemaUnion: 165 + for _, ref := range v.Refs { 166 + if !strings.HasPrefix(ref, "#") { 167 + fl.ExternalRefs[strings.TrimSuffix(ref, "#main")] = true 168 + } 169 + } 170 + fl.Types = append(fl.Types, ft) 171 + case lexicon.SchemaArray: 172 + // flatten the inner item 173 + tp := slices.Clone(tpath) 174 + tp = append(tp, "elem") 175 + if err := fl.flattenType(fd, tpath, &v.Items); err != nil { 176 + return err 177 + } 178 + // don't emit the array itself 179 + return nil 180 + case lexicon.SchemaString, lexicon.SchemaNull, lexicon.SchemaInteger, lexicon.SchemaBoolean, lexicon.SchemaUnknown, lexicon.SchemaBytes: 181 + // don't emit 182 + // NOTE: might want to emit some string "knownValue" lists in the future? 183 + case lexicon.SchemaCIDLink, lexicon.SchemaBlob: 184 + // don't emit 185 + case lexicon.SchemaToken: 186 + // pass-through (emit) 187 + fl.Types = append(fl.Types, ft) 188 + case lexicon.SchemaPermissionSet, lexicon.SchemaPermission: 189 + // pass-through (emit) 190 + fl.Types = append(fl.Types, ft) 191 + default: 192 + return fmt.Errorf("unsupported def type for flattening (%s): %T", fd.Name, def.Inner) 193 + } 194 + 195 + return nil 196 + } 197 + 198 + func (fl *FlatLexicon) flattenObject(fd *FlatDef, tpath []string, obj *lexicon.SchemaObject) error { 199 + 200 + keys := []string{} 201 + for n := range obj.Properties { 202 + keys = append(keys, n) 203 + } 204 + sort.Strings(keys) 205 + 206 + for _, fname := range keys { 207 + field := obj.Properties[fname] 208 + tp := slices.Clone(tpath) 209 + tp = append(tp, fname) 210 + switch v := field.Inner.(type) { 211 + case lexicon.SchemaNull, lexicon.SchemaBoolean, lexicon.SchemaInteger, lexicon.SchemaString, lexicon.SchemaBytes: 212 + // no-op 213 + case lexicon.SchemaCIDLink, lexicon.SchemaBlob, lexicon.SchemaUnknown: 214 + // no-op, but maybe set a flag on def? 215 + case lexicon.SchemaArray: 216 + tp = append(tp, "elem") 217 + if err := fl.flattenType(fd, tp, &v.Items); err != nil { 218 + return err 219 + } 220 + case lexicon.SchemaObject: 221 + if err := fl.flattenType(fd, tp, &field); err != nil { 222 + return err 223 + } 224 + case lexicon.SchemaRef: 225 + if !strings.HasPrefix(v.Ref, "#") { 226 + // remove any #main suffix 227 + fl.ExternalRefs[strings.TrimSuffix(v.Ref, "#main")] = true 228 + } 229 + case lexicon.SchemaUnion: 230 + if err := fl.flattenType(fd, tp, &field); err != nil { 231 + return err 232 + } 233 + default: 234 + return fmt.Errorf("unsupported field type for object flattening: %T", field.Inner) 235 + } 236 + } 237 + return nil 238 + } 239 + 240 + // Returns the type of any "#main" definition in this file (or else an empty string) 241 + func (fl *FlatLexicon) MainType() string { 242 + main, ok := fl.Defs["main"] 243 + if !ok { 244 + return "" 245 + } 246 + return main.Type 247 + }

+160

lex/lexgen/util.go

··· 1 + package lexgen 2 + 3 + import ( 4 + "fmt" 5 + "slices" 6 + "strings" 7 + 8 + "github.com/bluesky-social/indigo/atproto/lexicon" 9 + "github.com/bluesky-social/indigo/atproto/syntax" 10 + 11 + "golang.org/x/net/publicsuffix" 12 + ) 13 + 14 + func defType(sd *lexicon.SchemaDef) (string, error) { 15 + switch sd.Inner.(type) { 16 + case lexicon.SchemaRecord: 17 + return "record", nil 18 + case lexicon.SchemaQuery: 19 + return "query", nil 20 + case lexicon.SchemaProcedure: 21 + return "procedure", nil 22 + case lexicon.SchemaSubscription: 23 + return "subscription", nil 24 + case lexicon.SchemaPermissionSet: 25 + return "permission-set", nil 26 + case lexicon.SchemaPermission: 27 + return "permission", nil 28 + case lexicon.SchemaNull: 29 + return "null", nil 30 + case lexicon.SchemaBoolean: 31 + return "boolean", nil 32 + case lexicon.SchemaInteger: 33 + return "integer", nil 34 + case lexicon.SchemaString: 35 + return "string", nil 36 + case lexicon.SchemaBytes: 37 + return "bytes", nil 38 + case lexicon.SchemaCIDLink: 39 + return "cid-link", nil 40 + case lexicon.SchemaArray: 41 + return "array", nil 42 + case lexicon.SchemaObject: 43 + return "object", nil 44 + case lexicon.SchemaBlob: 45 + return "blob", nil 46 + case lexicon.SchemaParams: 47 + return "params", nil 48 + case lexicon.SchemaToken: 49 + return "token", nil 50 + case lexicon.SchemaRef: 51 + return "ref", nil 52 + case lexicon.SchemaUnion: 53 + return "union", nil 54 + case lexicon.SchemaUnknown: 55 + return "unknown", nil 56 + default: 57 + return "", fmt.Errorf("unhandled schema type: %T", sd.Inner) 58 + } 59 + } 60 + 61 + func defDescription(sd *lexicon.SchemaDef) string { 62 + var desc *string 63 + 64 + switch v := sd.Inner.(type) { 65 + case lexicon.SchemaRecord: 66 + desc = v.Description 67 + case lexicon.SchemaQuery: 68 + desc = v.Description 69 + case lexicon.SchemaProcedure: 70 + desc = v.Description 71 + case lexicon.SchemaSubscription: 72 + desc = v.Description 73 + case lexicon.SchemaPermissionSet: 74 + // TODO: extract *some* description? 75 + case lexicon.SchemaPermission: 76 + desc = v.Description 77 + case lexicon.SchemaNull: 78 + desc = v.Description 79 + case lexicon.SchemaBoolean: 80 + desc = v.Description 81 + case lexicon.SchemaInteger: 82 + desc = v.Description 83 + case lexicon.SchemaString: 84 + desc = v.Description 85 + case lexicon.SchemaBytes: 86 + desc = v.Description 87 + case lexicon.SchemaCIDLink: 88 + desc = v.Description 89 + case lexicon.SchemaArray: 90 + desc = v.Description 91 + case lexicon.SchemaObject: 92 + desc = v.Description 93 + case lexicon.SchemaBlob: 94 + desc = v.Description 95 + case lexicon.SchemaParams: 96 + desc = v.Description 97 + case lexicon.SchemaToken: 98 + desc = v.Description 99 + case lexicon.SchemaRef: 100 + desc = v.Description 101 + case lexicon.SchemaUnion: 102 + desc = v.Description 103 + case lexicon.SchemaUnknown: 104 + desc = v.Description 105 + } 106 + if desc != nil && *desc != "" { 107 + return *desc 108 + } 109 + return "" 110 + } 111 + 112 + func isCompoundDef(sd *lexicon.SchemaDef) bool { 113 + switch sd.Inner.(type) { 114 + case lexicon.SchemaRecord, lexicon.SchemaQuery, lexicon.SchemaProcedure, lexicon.SchemaSubscription, lexicon.SchemaArray, lexicon.SchemaObject, lexicon.SchemaUnion: 115 + return true 116 + default: 117 + return false 118 + } 119 + } 120 + 121 + func nsidPkgName(nsid syntax.NSID) string { 122 + domain := strings.ToLower(nsid.Authority()) 123 + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) 124 + if err != nil { 125 + return "FAIL" 126 + } 127 + parts := strings.Split(reg, ".") 128 + slices.Reverse(parts) 129 + 130 + return strings.Join(parts, "") 131 + } 132 + 133 + func nsidBaseName(nsid syntax.NSID) string { 134 + domain := strings.ToLower(nsid.Authority()) 135 + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) 136 + if err != nil { 137 + return "FAIL" 138 + } 139 + rem := domain[0 : len(domain)-len(reg)] 140 + parts := strings.Split(rem, ".") 141 + slices.Reverse(parts) 142 + parts = append(parts, nsid.Name()) 143 + for i := range parts { 144 + parts[i] = strings.Title(parts[i]) 145 + } 146 + return strings.Join(parts, "") 147 + } 148 + 149 + func nsidFileName(nsid syntax.NSID) string { 150 + domain := strings.ToLower(nsid.Authority()) 151 + reg, err := publicsuffix.EffectiveTLDPlusOne(domain) 152 + if err != nil { 153 + return "FAIL" 154 + } 155 + rem := domain[0 : len(domain)-len(reg)] 156 + parts := strings.Split(rem, ".") 157 + slices.Reverse(parts) 158 + parts = append(parts, nsid.Name()) 159 + return strings.Join(parts, "") 160 + }

+25

lex/lexgen/util_test.go

··· 1 + package lexgen 2 + 3 + import ( 4 + "testing" 5 + 6 + "github.com/bluesky-social/indigo/atproto/syntax" 7 + 8 + "github.com/stretchr/testify/assert" 9 + ) 10 + 11 + func TestNSIDNames(t *testing.T) { 12 + assert := assert.New(t) 13 + 14 + testVectors := [][]string{ 15 + {"app.bsky.feed.post", "appbsky", "FeedPost"}, 16 + {"com.atproto.admin.deleteAccount", "comatproto", "AdminDeleteAccount"}, 17 + {"uk.ac.school.lab.COOL.project", "ukacschool", "LabCoolProject"}, 18 + } 19 + 20 + for _, vec := range testVectors { 21 + nsid := syntax.NSID(vec[0]) 22 + assert.Equal(vec[1], nsidPkgName(nsid)) 23 + assert.Equal(vec[2], nsidBaseName(nsid)) 24 + } 25 + }

Configure Feed

Configure Feed