this repo has no description
goat relay helpers (mostly admin)
+577
-1
+1
-1
cmd/goat/firehose.go
+1
-1
cmd/goat/firehose.go
+145
cmd/goat/relay.go
+145
cmd/goat/relay.go
···
1
+
package main
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
7
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
8
+
"github.com/bluesky-social/indigo/xrpc"
9
+
10
+
"github.com/urfave/cli/v2"
11
+
)
12
+
13
+
var cmdRelay = &cli.Command{
14
+
Name: "relay",
15
+
Usage: "sub-commands for relays",
16
+
Flags: []cli.Flag{
17
+
&cli.StringFlag{
18
+
Name: "relay-host",
19
+
Usage: "method, hostname, and port of Relay instance",
20
+
Value: "https://bsky.network",
21
+
EnvVars: []string{"ATP_RELAY_HOST", "RELAY_HOST"},
22
+
},
23
+
},
24
+
Subcommands: []*cli.Command{
25
+
&cli.Command{
26
+
Name: "account",
27
+
Usage: "sub-commands for accounts/repos on relay",
28
+
Subcommands: []*cli.Command{
29
+
&cli.Command{
30
+
Name: "list",
31
+
Usage: "enumerate all accounts",
32
+
Flags: []cli.Flag{
33
+
&cli.StringFlag{
34
+
Name: "collection",
35
+
Aliases: []string{"c"},
36
+
Usage: "collection (NSID) to match",
37
+
},
38
+
&cli.BoolFlag{
39
+
Name: "json",
40
+
Usage: "print output as JSON lines",
41
+
},
42
+
},
43
+
Action: runRelayAccountList,
44
+
},
45
+
},
46
+
},
47
+
&cli.Command{
48
+
Name: "host",
49
+
Usage: "sub-commands for upstream hosts (eg, PDS)",
50
+
Subcommands: []*cli.Command{
51
+
&cli.Command{
52
+
Name: "add",
53
+
Usage: "request crawl of upstream host (eg, PDS)",
54
+
ArgsUsage: `<hostname>`,
55
+
Action: runRelayHostAdd,
56
+
},
57
+
},
58
+
},
59
+
cmdRelayAdmin,
60
+
},
61
+
}
62
+
63
+
func runRelayAccountList(cctx *cli.Context) error {
64
+
ctx := cctx.Context
65
+
66
+
if cctx.Args().Len() > 0 {
67
+
return fmt.Errorf("unexpected arguments")
68
+
}
69
+
70
+
client := xrpc.Client{
71
+
Host: cctx.String("relay-host"),
72
+
}
73
+
74
+
collection := cctx.String("collection")
75
+
cursor := ""
76
+
var size int64 = 500
77
+
for {
78
+
if collection != "" {
79
+
resp, err := comatproto.SyncListReposByCollection(ctx, &client, collection, cursor, size)
80
+
if err != nil {
81
+
return err
82
+
}
83
+
for _, r := range resp.Repos {
84
+
fmt.Println(r.Did)
85
+
}
86
+
87
+
if resp.Cursor == nil || *resp.Cursor == "" {
88
+
break
89
+
}
90
+
cursor = *resp.Cursor
91
+
} else {
92
+
resp, err := comatproto.SyncListRepos(ctx, &client, cursor, size)
93
+
if err != nil {
94
+
return err
95
+
}
96
+
97
+
for _, r := range resp.Repos {
98
+
if cctx.Bool("json") {
99
+
b, err := json.Marshal(r)
100
+
if err != nil {
101
+
return err
102
+
}
103
+
fmt.Println(string(b))
104
+
} else {
105
+
status := "unknown"
106
+
if r.Active != nil && *r.Active == true {
107
+
status = "active"
108
+
} else if r.Status != nil {
109
+
status = *r.Status
110
+
}
111
+
fmt.Printf("%s\t%s\t%s\n", r.Did, status, r.Rev)
112
+
}
113
+
}
114
+
115
+
if resp.Cursor == nil || *resp.Cursor == "" {
116
+
break
117
+
}
118
+
cursor = *resp.Cursor
119
+
}
120
+
}
121
+
return nil
122
+
}
123
+
124
+
func runRelayHostAdd(cctx *cli.Context) error {
125
+
ctx := cctx.Context
126
+
127
+
hostname := cctx.Args().First()
128
+
if hostname == "" {
129
+
return fmt.Errorf("need to provide hostname as argument")
130
+
}
131
+
if cctx.Args().Len() != 1 {
132
+
return fmt.Errorf("unexpected arguments")
133
+
}
134
+
135
+
client := xrpc.Client{
136
+
Host: cctx.String("relay-host"),
137
+
}
138
+
139
+
err := comatproto.SyncRequestCrawl(ctx, &client, &comatproto.SyncRequestCrawl_Input{Hostname: hostname})
140
+
if err != nil {
141
+
return err
142
+
}
143
+
fmt.Println("success")
144
+
return nil
145
+
}
+431
cmd/goat/relay_admin.go
+431
cmd/goat/relay_admin.go
···
1
+
package main
2
+
3
+
import (
4
+
"bytes"
5
+
"encoding/json"
6
+
"fmt"
7
+
"io"
8
+
"log/slog"
9
+
"net/http"
10
+
"net/url"
11
+
12
+
"github.com/carlmjohnson/versioninfo"
13
+
"github.com/urfave/cli/v2"
14
+
)
15
+
16
+
var cmdRelayAdmin = &cli.Command{
17
+
Name: "admin",
18
+
Usage: "sub-comands for relay administration",
19
+
Flags: []cli.Flag{
20
+
&cli.StringFlag{
21
+
Name: "admin-token",
22
+
Required: true,
23
+
Usage: "relay admin auth token",
24
+
EnvVars: []string{"ATP_AUTH_ADMIN_PASSWORD", "RELAY_ADMIN_TOKEN"},
25
+
},
26
+
},
27
+
Subcommands: []*cli.Command{
28
+
&cli.Command{
29
+
Name: "account",
30
+
Usage: "sub-commands for managing accounts",
31
+
Subcommands: []*cli.Command{
32
+
&cli.Command{
33
+
Name: "takedown",
34
+
Usage: "takedown a single account on relay",
35
+
Flags: []cli.Flag{
36
+
&cli.StringFlag{
37
+
Name: "collection",
38
+
Aliases: []string{"c"},
39
+
Usage: "collection (NSID) to match",
40
+
},
41
+
&cli.BoolFlag{
42
+
Name: "reverse",
43
+
Usage: "un-takedown",
44
+
},
45
+
},
46
+
Action: runRelayAdminAccountTakedown,
47
+
},
48
+
&cli.Command{
49
+
Name: "list",
50
+
Aliases: []string{"ls"},
51
+
Usage: "enumerate accounts (eg, takendown)",
52
+
Action: runRelayAdminAccountList,
53
+
},
54
+
},
55
+
},
56
+
&cli.Command{
57
+
Name: "host",
58
+
Usage: "sub-commands for upstream hosts (eg, PDS)",
59
+
Subcommands: []*cli.Command{
60
+
&cli.Command{
61
+
Name: "add",
62
+
Usage: "request crawl of upstream host (eg, PDS)",
63
+
ArgsUsage: `<hostname>`,
64
+
Action: runRelayAdminHostAdd,
65
+
},
66
+
&cli.Command{
67
+
Name: "block",
68
+
Usage: "request crawl of upstream host (eg, PDS)",
69
+
ArgsUsage: `<hostname>`,
70
+
Flags: []cli.Flag{
71
+
&cli.BoolFlag{
72
+
Name: "reverse",
73
+
Usage: "un-takedown",
74
+
},
75
+
},
76
+
Action: runRelayAdminHostBlock,
77
+
},
78
+
&cli.Command{
79
+
Name: "list",
80
+
Aliases: []string{"ls"},
81
+
Usage: "enumerate hosts crawled by relay",
82
+
Action: runRelayAdminHostList,
83
+
},
84
+
&cli.Command{
85
+
Name: "config",
86
+
Usage: "update rate-limits per host",
87
+
ArgsUsage: `<hostname>`,
88
+
Flags: []cli.Flag{
89
+
&cli.IntFlag{
90
+
Name: "per-second",
91
+
},
92
+
&cli.IntFlag{
93
+
Name: "per-hour",
94
+
},
95
+
&cli.IntFlag{
96
+
Name: "per-day",
97
+
},
98
+
&cli.IntFlag{
99
+
Name: "repo-limit",
100
+
},
101
+
},
102
+
Action: runRelayAdminHostConfig,
103
+
},
104
+
},
105
+
},
106
+
&cli.Command{
107
+
Name: "domain",
108
+
Usage: "sub-commands for domain-level config",
109
+
Subcommands: []*cli.Command{
110
+
&cli.Command{
111
+
Name: "ban",
112
+
Usage: "ban an entire domain name from being crawled",
113
+
ArgsUsage: `<domain>`,
114
+
Flags: []cli.Flag{
115
+
&cli.BoolFlag{
116
+
Name: "reverse",
117
+
Usage: "un-takedown",
118
+
},
119
+
},
120
+
Action: runRelayAdminDomainBan,
121
+
},
122
+
&cli.Command{
123
+
Name: "list",
124
+
Aliases: []string{"ls"},
125
+
Usage: "enumerate domains with configs (eg, bans)",
126
+
Action: runRelayAdminDomainList,
127
+
},
128
+
},
129
+
},
130
+
&cli.Command{
131
+
Name: "consumer",
132
+
Usage: "sub-commands for consumers",
133
+
Subcommands: []*cli.Command{
134
+
&cli.Command{
135
+
Name: "list",
136
+
Aliases: []string{"ls"},
137
+
Usage: "enumerate consumers",
138
+
Action: runRelayAdminConsumerList,
139
+
},
140
+
},
141
+
},
142
+
},
143
+
}
144
+
145
+
type RelayAdminClient struct {
146
+
Host string
147
+
BearerToken string
148
+
}
149
+
150
+
func (c *RelayAdminClient) Do(method, path string, params map[string]string, body map[string]any) ([]byte, error) {
151
+
u, err := url.Parse(c.Host)
152
+
if err != nil {
153
+
return nil, err
154
+
}
155
+
u.Path = path
156
+
q := u.Query()
157
+
for k, v := range params {
158
+
q.Add(k, v)
159
+
}
160
+
u.RawQuery = q.Encode()
161
+
162
+
var buf *bytes.Buffer
163
+
if body != nil {
164
+
b, err := json.Marshal(body)
165
+
if err != nil {
166
+
return nil, err
167
+
}
168
+
buf = bytes.NewBuffer(b)
169
+
}
170
+
171
+
var req *http.Request
172
+
if buf != nil {
173
+
req, err = http.NewRequest(method, u.String(), buf)
174
+
} else {
175
+
req, err = http.NewRequest(method, u.String(), nil)
176
+
}
177
+
if err != nil {
178
+
return nil, err
179
+
}
180
+
req.Header.Set("Authorization", "Bearer "+c.BearerToken)
181
+
req.Header.Set("User-Agent", fmt.Sprintf("goat/"+versioninfo.Short()))
182
+
if buf != nil {
183
+
req.Header.Set("Content-Type", "application/json")
184
+
}
185
+
186
+
resp, err := http.DefaultClient.Do(req)
187
+
if err != nil {
188
+
return nil, err
189
+
}
190
+
191
+
defer resp.Body.Close()
192
+
respBytes, err := io.ReadAll(resp.Body)
193
+
if err != nil {
194
+
return nil, err
195
+
}
196
+
if resp.StatusCode != http.StatusOK {
197
+
slog.Warn("relay HTTP error", "statusCode", resp.StatusCode, "body", string(respBytes))
198
+
return nil, fmt.Errorf("relay HTTP request failed: %d", resp.StatusCode)
199
+
}
200
+
return respBytes, nil
201
+
}
202
+
203
+
func NewRelayAdminClient(cctx *cli.Context) *RelayAdminClient {
204
+
// TODO: password-style admin auth
205
+
//headers["Authorization"] = "Basic "+base64.StdEncoding.EncodeToString([]byte("admin:"+*c.AdminToken)))
206
+
client := RelayAdminClient{
207
+
Host: cctx.String("relay-host"),
208
+
BearerToken: cctx.String("admin-token"),
209
+
}
210
+
return &client
211
+
}
212
+
213
+
func runRelayAdminAccountTakedown(cctx *cli.Context) error {
214
+
ctx := cctx.Context
215
+
216
+
username := cctx.Args().First()
217
+
if username == "" {
218
+
return fmt.Errorf("need to provide username as an argument")
219
+
}
220
+
ident, err := resolveIdent(ctx, username)
221
+
if err != nil {
222
+
return err
223
+
}
224
+
225
+
client := NewRelayAdminClient(cctx)
226
+
227
+
path := "/admin/repo/takeDown"
228
+
if cctx.Bool("reverse") {
229
+
path = "/admin/repo/reverseTakedown"
230
+
}
231
+
232
+
body := map[string]any{
233
+
"did": ident.DID.String(),
234
+
}
235
+
_, err = client.Do("POST", path, nil, body)
236
+
if err != nil {
237
+
return err
238
+
}
239
+
return nil
240
+
}
241
+
242
+
func runRelayAdminAccountList(cctx *cli.Context) error {
243
+
client := NewRelayAdminClient(cctx)
244
+
path := "/admin/repo/takedowns"
245
+
params := map[string]string{
246
+
"cursor": "",
247
+
"size": "500",
248
+
}
249
+
for {
250
+
respBytes, err := client.Do("GET", path, params, nil)
251
+
if err != nil {
252
+
return err
253
+
}
254
+
var resp map[string]any
255
+
if err := json.Unmarshal(respBytes, &resp); err != nil {
256
+
return err
257
+
}
258
+
for _, d := range resp["dids"].([]any) {
259
+
fmt.Println(d)
260
+
}
261
+
cursor, ok := resp["cursor"]
262
+
if !ok || cursor == "" {
263
+
break
264
+
}
265
+
params["cursor"] = cursor.(string)
266
+
}
267
+
return nil
268
+
}
269
+
270
+
func runRelayAdminHostAdd(cctx *cli.Context) error {
271
+
272
+
hostname := cctx.Args().First()
273
+
if hostname == "" {
274
+
return fmt.Errorf("need to provide hostname as an argument")
275
+
}
276
+
277
+
client := NewRelayAdminClient(cctx)
278
+
path := "/admin/pds/requestCrawl"
279
+
body := map[string]any{
280
+
"hostname": hostname,
281
+
}
282
+
_, err := client.Do("POST", path, nil, body)
283
+
if err != nil {
284
+
return err
285
+
}
286
+
return nil
287
+
}
288
+
289
+
func runRelayAdminHostBlock(cctx *cli.Context) error {
290
+
291
+
hostname := cctx.Args().First()
292
+
if hostname == "" {
293
+
return fmt.Errorf("need to provide hostname as an argument")
294
+
}
295
+
296
+
client := NewRelayAdminClient(cctx)
297
+
298
+
path := "/admin/pds/block"
299
+
if cctx.Bool("reverse") {
300
+
path = "/admin/pds/unblock"
301
+
}
302
+
303
+
params := map[string]string{
304
+
"host": hostname,
305
+
}
306
+
_, err := client.Do("POST", path, params, nil)
307
+
if err != nil {
308
+
return err
309
+
}
310
+
return nil
311
+
}
312
+
313
+
func runRelayAdminHostList(cctx *cli.Context) error {
314
+
client := NewRelayAdminClient(cctx)
315
+
path := "/admin/pds/list"
316
+
317
+
respBytes, err := client.Do("GET", path, nil, nil)
318
+
if err != nil {
319
+
return err
320
+
}
321
+
var rows []map[string]any
322
+
if err := json.Unmarshal(respBytes, &rows); err != nil {
323
+
return err
324
+
}
325
+
for _, r := range rows {
326
+
b, err := json.Marshal(r)
327
+
if err != nil {
328
+
return nil
329
+
}
330
+
fmt.Println(string(b))
331
+
}
332
+
return nil
333
+
}
334
+
335
+
func runRelayAdminHostConfig(cctx *cli.Context) error {
336
+
337
+
hostname := cctx.Args().First()
338
+
if hostname == "" {
339
+
return fmt.Errorf("need to provide hostname as an argument")
340
+
}
341
+
342
+
client := NewRelayAdminClient(cctx)
343
+
344
+
path := "/admin/pds/changeLimits"
345
+
346
+
body := map[string]any{
347
+
"host": hostname,
348
+
}
349
+
if cctx.IsSet("per-second") {
350
+
body["per_second"] = cctx.Int("per-second")
351
+
}
352
+
if cctx.IsSet("per-hour") {
353
+
body["per_hour"] = cctx.Int("per-hour")
354
+
}
355
+
if cctx.IsSet("per-day") {
356
+
body["per_day"] = cctx.Int("per-day")
357
+
}
358
+
if cctx.IsSet("repo-limit") {
359
+
body["repo_limit"] = cctx.Int("repo-limit")
360
+
}
361
+
362
+
_, err := client.Do("POST", path, nil, body)
363
+
if err != nil {
364
+
return err
365
+
}
366
+
return nil
367
+
}
368
+
369
+
func runRelayAdminDomainBan(cctx *cli.Context) error {
370
+
371
+
domain := cctx.Args().First()
372
+
if domain == "" {
373
+
return fmt.Errorf("need to provide domain as an argument")
374
+
}
375
+
376
+
client := NewRelayAdminClient(cctx)
377
+
378
+
path := "/admin/subs/banDomain"
379
+
if cctx.Bool("reverse") {
380
+
path = "/admin/subs/unbanDomain"
381
+
}
382
+
383
+
body := map[string]any{
384
+
"domain": domain,
385
+
}
386
+
_, err := client.Do("POST", path, nil, body)
387
+
if err != nil {
388
+
return err
389
+
}
390
+
return nil
391
+
}
392
+
393
+
func runRelayAdminDomainList(cctx *cli.Context) error {
394
+
client := NewRelayAdminClient(cctx)
395
+
path := "/admin/subs/listDomainBans"
396
+
397
+
respBytes, err := client.Do("GET", path, nil, nil)
398
+
if err != nil {
399
+
return err
400
+
}
401
+
var resp map[string]any
402
+
if err := json.Unmarshal(respBytes, &resp); err != nil {
403
+
return err
404
+
}
405
+
for _, d := range resp["banned_domains"].([]any) {
406
+
fmt.Println(d)
407
+
}
408
+
return nil
409
+
}
410
+
411
+
func runRelayAdminConsumerList(cctx *cli.Context) error {
412
+
client := NewRelayAdminClient(cctx)
413
+
path := "/admin/consumers/list"
414
+
415
+
respBytes, err := client.Do("GET", path, nil, nil)
416
+
if err != nil {
417
+
return err
418
+
}
419
+
var rows []map[string]any
420
+
if err := json.Unmarshal(respBytes, &rows); err != nil {
421
+
return err
422
+
}
423
+
for _, r := range rows {
424
+
b, err := json.Marshal(r)
425
+
if err != nil {
426
+
return nil
427
+
}
428
+
fmt.Println(string(b))
429
+
}
430
+
return nil
431
+
}