this repo has no description
13
fork

Configure Feed

Select the types of activity you want to include in your feed.

basic pull functionality

+210
+1
cmd/glot/main.go
··· 37 37 } 38 38 app.Commands = []*cli.Command{ 39 39 cmdLint, 40 + cmdPull, 40 41 } 41 42 return app.Run(context.Background(), args) 42 43 }
+209
cmd/glot/pull.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "encoding/json" 6 + "fmt" 7 + "log/slog" 8 + "os" 9 + "path" 10 + "strings" 11 + 12 + "github.com/bluesky-social/indigo/api/agnostic" 13 + "github.com/bluesky-social/indigo/atproto/client" 14 + "github.com/bluesky-social/indigo/atproto/identity" 15 + "github.com/bluesky-social/indigo/atproto/lexicon" 16 + "github.com/bluesky-social/indigo/atproto/syntax" 17 + "tangled.sh/bnewbold.net/cobalt/atproto/netclient" 18 + 19 + "github.com/urfave/cli/v3" 20 + ) 21 + 22 + var ( 23 + schemaNSID = syntax.NSID("com.atproto.lexicon.schema") 24 + ) 25 + 26 + var cmdPull = &cli.Command{ 27 + Name: "pull", 28 + Usage: "fetch (or update) lexicon schemas to local directory", 29 + ArgsUsage: `<nsid-pattern>+`, 30 + Flags: []cli.Flag{ 31 + &cli.StringFlag{ 32 + Name: "lexicons-dir", 33 + Value: "./lexicons/", 34 + Usage: "base directory for project Lexicon files", 35 + Sources: cli.EnvVars("LEXICONS_DIR"), 36 + }, 37 + &cli.BoolFlag{ 38 + Name: "update", 39 + Aliases: []string{"u"}, 40 + Usage: "overwrite any existing local files", 41 + }, 42 + // TODO: output-dir (instead of auto) 43 + }, 44 + Action: runPull, 45 + } 46 + 47 + // Checks if a string is a valid NSID group pattern, which is a partial NSID ending in '.' or '.*' 48 + func ParseNSIDGroup(raw string) (string, error) { 49 + if strings.HasSuffix(raw, ".*") { 50 + raw = raw[:len(raw)-1] 51 + } 52 + if !strings.HasSuffix(raw, ".") { 53 + return "", fmt.Errorf("not an NSID group pattern") 54 + } 55 + _, err := syntax.ParseNSID(raw + "name") 56 + if err != nil { 57 + return "", fmt.Errorf("not an NSID group pattern") 58 + } 59 + return raw, nil 60 + } 61 + 62 + func runPull(ctx context.Context, cmd *cli.Command) error { 63 + if !cmd.Args().Present() { 64 + return fmt.Errorf("no NSID patterns specified") 65 + } 66 + 67 + for _, p := range cmd.Args().Slice() { 68 + 69 + group, err := ParseNSIDGroup(p) 70 + if nil == err { 71 + if err := pullLexiconGroup(ctx, cmd, group); err != nil { 72 + return err 73 + } 74 + continue 75 + } 76 + 77 + nsid, err := syntax.ParseNSID(p) 78 + if err != nil { 79 + return fmt.Errorf("invalid Lexicon NSID pattern: %s", p) 80 + } 81 + if err := pullLexicon(ctx, cmd, nsid); err != nil { 82 + return err 83 + } 84 + } 85 + return nil 86 + } 87 + 88 + func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string { 89 + base := cmd.String("lexicons-dir") 90 + sub := strings.ReplaceAll(nsid.String(), ".", "/") 91 + return path.Join(base, sub+".json") 92 + } 93 + 94 + func pullLexicon(ctx context.Context, cmd *cli.Command, nsid syntax.NSID) error { 95 + 96 + fpath := pathForNSID(cmd, nsid) 97 + if !cmd.Bool("update") { 98 + _, err := os.Stat(fpath) 99 + if err == nil { 100 + return fmt.Errorf("output file already exists: %s", fpath) 101 + } 102 + } 103 + 104 + // TODO: common net client 105 + netc := netclient.NewNetClient() 106 + dir := identity.BaseDirectory{} 107 + did, err := dir.ResolveNSID(ctx, nsid) 108 + if err != nil { 109 + return fmt.Errorf("failed to resolve NSID %s: %w", nsid, err) 110 + } 111 + 112 + var rec json.RawMessage 113 + cid, err := netc.GetRecord(ctx, did, schemaNSID, syntax.RecordKey(nsid), &rec) 114 + if err != nil { 115 + return err 116 + } 117 + slog.Info("fetched NSID schema record", "nsid", nsid, "cid", cid) 118 + 119 + return writeLexiconFile(ctx, cmd, nsid, fpath, rec) 120 + } 121 + 122 + func writeLexiconFile(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, fpath string, rec json.RawMessage) error { 123 + 124 + var sf lexicon.SchemaFile 125 + err := json.Unmarshal(rec, &sf) 126 + if err == nil { 127 + err = sf.FinishParse() 128 + } 129 + if err != nil { 130 + return fmt.Errorf("schema record syntax invalid (%s): %w", nsid, err) 131 + } 132 + 133 + // ensure (nested) directory exists 134 + if err := os.MkdirAll(path.Dir(fpath), 0755); err != nil { 135 + return err 136 + } 137 + 138 + b, err := json.MarshalIndent(rec, "", " ") 139 + if err != nil { 140 + return err 141 + } 142 + b = append(b, '\n') 143 + 144 + if err := os.WriteFile(fpath, b, 0666); err != nil { 145 + return err 146 + } 147 + 148 + slog.Info("wrote NSID schema record to disk", "nsid", nsid, "path", fpath) 149 + return nil 150 + } 151 + 152 + func pullLexiconGroup(ctx context.Context, cmd *cli.Command, group string) error { 153 + 154 + // TODO: netclient support for listing records 155 + dir := identity.BaseDirectory{} 156 + did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name")) 157 + if err != nil { 158 + return err 159 + } 160 + ident, err := dir.LookupDID(ctx, did) 161 + if err != nil { 162 + return err 163 + } 164 + c := client.NewAPIClient(ident.PDSEndpoint()) 165 + 166 + cursor := "" 167 + for { 168 + // collection string, cursor string, limit int64, repo string, reverse bool 169 + resp, err := agnostic.RepoListRecords(ctx, c, "com.atproto.lexicon.schema", cursor, 100, ident.DID.String(), false) 170 + if err != nil { 171 + return err 172 + } 173 + for _, rec := range resp.Records { 174 + aturi, err := syntax.ParseATURI(rec.Uri) 175 + if err != nil { 176 + return err 177 + } 178 + nsid, err := syntax.ParseNSID(aturi.RecordKey().String()) 179 + if err != nil { 180 + slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey()) 181 + continue 182 + } 183 + if !strings.HasPrefix(nsid.String(), group) { 184 + // ignoring other NSIDs 185 + continue 186 + } 187 + if rec.Value == nil { 188 + return fmt.Errorf("missing record value: %s", nsid) 189 + } 190 + 191 + fpath := pathForNSID(cmd, nsid) 192 + if !cmd.Bool("update") { 193 + _, err := os.Stat(fpath) 194 + if err == nil { 195 + return fmt.Errorf("output file already exists: %s", fpath) 196 + } 197 + } 198 + if err := writeLexiconFile(ctx, cmd, nsid, fpath, *rec.Value); err != nil { 199 + return nil 200 + } 201 + } 202 + if resp.Cursor != nil && *resp.Cursor != "" { 203 + cursor = *resp.Cursor 204 + } else { 205 + break 206 + } 207 + } 208 + return nil 209 + }