this repo has no description
basic pull functionality
+210
+1
cmd/glot/main.go
+1
cmd/glot/main.go
+209
cmd/glot/pull.go
+209
cmd/glot/pull.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"log/slog"
8
+
"os"
9
+
"path"
10
+
"strings"
11
+
12
+
"github.com/bluesky-social/indigo/api/agnostic"
13
+
"github.com/bluesky-social/indigo/atproto/client"
14
+
"github.com/bluesky-social/indigo/atproto/identity"
15
+
"github.com/bluesky-social/indigo/atproto/lexicon"
16
+
"github.com/bluesky-social/indigo/atproto/syntax"
17
+
"tangled.sh/bnewbold.net/cobalt/atproto/netclient"
18
+
19
+
"github.com/urfave/cli/v3"
20
+
)
21
+
22
+
var (
23
+
schemaNSID = syntax.NSID("com.atproto.lexicon.schema")
24
+
)
25
+
26
+
var cmdPull = &cli.Command{
27
+
Name: "pull",
28
+
Usage: "fetch (or update) lexicon schemas to local directory",
29
+
ArgsUsage: `<nsid-pattern>+`,
30
+
Flags: []cli.Flag{
31
+
&cli.StringFlag{
32
+
Name: "lexicons-dir",
33
+
Value: "./lexicons/",
34
+
Usage: "base directory for project Lexicon files",
35
+
Sources: cli.EnvVars("LEXICONS_DIR"),
36
+
},
37
+
&cli.BoolFlag{
38
+
Name: "update",
39
+
Aliases: []string{"u"},
40
+
Usage: "overwrite any existing local files",
41
+
},
42
+
// TODO: output-dir (instead of auto)
43
+
},
44
+
Action: runPull,
45
+
}
46
+
47
+
// Checks if a string is a valid NSID group pattern, which is a partial NSID ending in '.' or '.*'
48
+
func ParseNSIDGroup(raw string) (string, error) {
49
+
if strings.HasSuffix(raw, ".*") {
50
+
raw = raw[:len(raw)-1]
51
+
}
52
+
if !strings.HasSuffix(raw, ".") {
53
+
return "", fmt.Errorf("not an NSID group pattern")
54
+
}
55
+
_, err := syntax.ParseNSID(raw + "name")
56
+
if err != nil {
57
+
return "", fmt.Errorf("not an NSID group pattern")
58
+
}
59
+
return raw, nil
60
+
}
61
+
62
+
func runPull(ctx context.Context, cmd *cli.Command) error {
63
+
if !cmd.Args().Present() {
64
+
return fmt.Errorf("no NSID patterns specified")
65
+
}
66
+
67
+
for _, p := range cmd.Args().Slice() {
68
+
69
+
group, err := ParseNSIDGroup(p)
70
+
if nil == err {
71
+
if err := pullLexiconGroup(ctx, cmd, group); err != nil {
72
+
return err
73
+
}
74
+
continue
75
+
}
76
+
77
+
nsid, err := syntax.ParseNSID(p)
78
+
if err != nil {
79
+
return fmt.Errorf("invalid Lexicon NSID pattern: %s", p)
80
+
}
81
+
if err := pullLexicon(ctx, cmd, nsid); err != nil {
82
+
return err
83
+
}
84
+
}
85
+
return nil
86
+
}
87
+
88
+
func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string {
89
+
base := cmd.String("lexicons-dir")
90
+
sub := strings.ReplaceAll(nsid.String(), ".", "/")
91
+
return path.Join(base, sub+".json")
92
+
}
93
+
94
+
func pullLexicon(ctx context.Context, cmd *cli.Command, nsid syntax.NSID) error {
95
+
96
+
fpath := pathForNSID(cmd, nsid)
97
+
if !cmd.Bool("update") {
98
+
_, err := os.Stat(fpath)
99
+
if err == nil {
100
+
return fmt.Errorf("output file already exists: %s", fpath)
101
+
}
102
+
}
103
+
104
+
// TODO: common net client
105
+
netc := netclient.NewNetClient()
106
+
dir := identity.BaseDirectory{}
107
+
did, err := dir.ResolveNSID(ctx, nsid)
108
+
if err != nil {
109
+
return fmt.Errorf("failed to resolve NSID %s: %w", nsid, err)
110
+
}
111
+
112
+
var rec json.RawMessage
113
+
cid, err := netc.GetRecord(ctx, did, schemaNSID, syntax.RecordKey(nsid), &rec)
114
+
if err != nil {
115
+
return err
116
+
}
117
+
slog.Info("fetched NSID schema record", "nsid", nsid, "cid", cid)
118
+
119
+
return writeLexiconFile(ctx, cmd, nsid, fpath, rec)
120
+
}
121
+
122
+
func writeLexiconFile(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, fpath string, rec json.RawMessage) error {
123
+
124
+
var sf lexicon.SchemaFile
125
+
err := json.Unmarshal(rec, &sf)
126
+
if err == nil {
127
+
err = sf.FinishParse()
128
+
}
129
+
if err != nil {
130
+
return fmt.Errorf("schema record syntax invalid (%s): %w", nsid, err)
131
+
}
132
+
133
+
// ensure (nested) directory exists
134
+
if err := os.MkdirAll(path.Dir(fpath), 0755); err != nil {
135
+
return err
136
+
}
137
+
138
+
b, err := json.MarshalIndent(rec, "", " ")
139
+
if err != nil {
140
+
return err
141
+
}
142
+
b = append(b, '\n')
143
+
144
+
if err := os.WriteFile(fpath, b, 0666); err != nil {
145
+
return err
146
+
}
147
+
148
+
slog.Info("wrote NSID schema record to disk", "nsid", nsid, "path", fpath)
149
+
return nil
150
+
}
151
+
152
+
func pullLexiconGroup(ctx context.Context, cmd *cli.Command, group string) error {
153
+
154
+
// TODO: netclient support for listing records
155
+
dir := identity.BaseDirectory{}
156
+
did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name"))
157
+
if err != nil {
158
+
return err
159
+
}
160
+
ident, err := dir.LookupDID(ctx, did)
161
+
if err != nil {
162
+
return err
163
+
}
164
+
c := client.NewAPIClient(ident.PDSEndpoint())
165
+
166
+
cursor := ""
167
+
for {
168
+
// collection string, cursor string, limit int64, repo string, reverse bool
169
+
resp, err := agnostic.RepoListRecords(ctx, c, "com.atproto.lexicon.schema", cursor, 100, ident.DID.String(), false)
170
+
if err != nil {
171
+
return err
172
+
}
173
+
for _, rec := range resp.Records {
174
+
aturi, err := syntax.ParseATURI(rec.Uri)
175
+
if err != nil {
176
+
return err
177
+
}
178
+
nsid, err := syntax.ParseNSID(aturi.RecordKey().String())
179
+
if err != nil {
180
+
slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey())
181
+
continue
182
+
}
183
+
if !strings.HasPrefix(nsid.String(), group) {
184
+
// ignoring other NSIDs
185
+
continue
186
+
}
187
+
if rec.Value == nil {
188
+
return fmt.Errorf("missing record value: %s", nsid)
189
+
}
190
+
191
+
fpath := pathForNSID(cmd, nsid)
192
+
if !cmd.Bool("update") {
193
+
_, err := os.Stat(fpath)
194
+
if err == nil {
195
+
return fmt.Errorf("output file already exists: %s", fpath)
196
+
}
197
+
}
198
+
if err := writeLexiconFile(ctx, cmd, nsid, fpath, *rec.Value); err != nil {
199
+
return nil
200
+
}
201
+
}
202
+
if resp.Cursor != nil && *resp.Cursor != "" {
203
+
cursor = *resp.Cursor
204
+
} else {
205
+
break
206
+
}
207
+
}
208
+
return nil
209
+
}