this repo has no description
deps cleanup (#1173)
- switches a bunch of stuff from urfave/cli/v2 to v3 (but not
everything)
- switches everything to maintained version of versioninfo
- tidies a bunch of imports
+876
-782
+20
-22
atproto/atclient/cmd/atp-client-demo/main.go
+20
-22
atproto/atclient/cmd/atp-client-demo/main.go
···
12
12
"github.com/bluesky-social/indigo/atproto/identity"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
14
15
-
"github.com/urfave/cli/v2"
15
+
"github.com/urfave/cli/v3"
16
16
)
17
17
18
18
func main() {
19
-
app := cli.App{
19
+
app := cli.Command{
20
20
Name: "atp-client-demo",
21
21
Usage: "dev helper for atproto/client SDK",
22
22
Commands: []*cli.Command{
···
117
117
}
118
118
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
119
119
slog.SetDefault(slog.New(h))
120
-
app.RunAndExitOnError()
120
+
if err := app.Run(context.Background(), os.Args); err != nil {
121
+
slog.Error("command failed", "error", err)
122
+
os.Exit(-1)
123
+
}
121
124
}
122
125
123
126
func getFeed(ctx context.Context, c *atclient.APIClient) error {
···
156
159
return nil
157
160
}
158
161
159
-
func runGetFeedPublic(cctx *cli.Context) error {
160
-
ctx := cctx.Context
162
+
func runGetFeedPublic(ctx context.Context, cmd *cli.Command) error {
161
163
162
164
c := atclient.APIClient{
163
-
Host: cctx.String("host"),
165
+
Host: cmd.String("host"),
164
166
}
165
167
166
168
return getFeed(ctx, &c)
167
169
}
168
170
169
-
func runListRecordsPublic(cctx *cli.Context) error {
170
-
ctx := cctx.Context
171
+
func runListRecordsPublic(ctx context.Context, cmd *cli.Command) error {
171
172
172
173
c := atclient.APIClient{
173
-
Host: cctx.String("host"),
174
+
Host: cmd.String("host"),
174
175
}
175
176
176
177
return listRecords(ctx, &c)
177
178
}
178
179
179
-
func runLoginAuth(cctx *cli.Context) error {
180
-
ctx := cctx.Context
180
+
func runLoginAuth(ctx context.Context, cmd *cli.Command) error {
181
181
182
-
atid, err := syntax.ParseAtIdentifier(cctx.String("username"))
182
+
atid, err := syntax.ParseAtIdentifier(cmd.String("username"))
183
183
if err != nil {
184
184
return err
185
185
}
186
186
187
187
dir := identity.DefaultDirectory()
188
188
189
-
c, err := atclient.LoginWithPassword(ctx, dir, *atid, cctx.String("password"), "", nil)
189
+
c, err := atclient.LoginWithPassword(ctx, dir, *atid, cmd.String("password"), "", nil)
190
190
if err != nil {
191
191
return err
192
192
}
···
205
205
return nil
206
206
}
207
207
208
-
func runGetFeedAuth(cctx *cli.Context) error {
209
-
ctx := cctx.Context
208
+
func runGetFeedAuth(ctx context.Context, cmd *cli.Command) error {
210
209
211
-
atid, err := syntax.ParseAtIdentifier(cctx.String("username"))
210
+
atid, err := syntax.ParseAtIdentifier(cmd.String("username"))
212
211
if err != nil {
213
212
return err
214
213
}
215
214
216
215
dir := identity.DefaultDirectory()
217
216
218
-
c, err := atclient.LoginWithPassword(ctx, dir, *atid, cctx.String("password"), "", nil)
217
+
c, err := atclient.LoginWithPassword(ctx, dir, *atid, cmd.String("password"), "", nil)
219
218
if err != nil {
220
219
return err
221
220
}
222
-
c = c.WithService(cctx.String("appview"))
221
+
c = c.WithService(cmd.String("appview"))
223
222
224
223
return getFeed(ctx, c)
225
224
}
226
225
227
-
func runLookupAdmin(cctx *cli.Context) error {
228
-
ctx := cctx.Context
226
+
func runLookupAdmin(ctx context.Context, cmd *cli.Command) error {
229
227
230
-
c := atclient.NewAdminClient(cctx.String("host"), cctx.String("admin-password"))
228
+
c := atclient.NewAdminClient(cmd.String("host"), cmd.String("admin-password"))
231
229
232
230
var d json.RawMessage
233
231
params := map[string]any{
234
-
"did": cctx.String("did"),
232
+
"did": cmd.String("did"),
235
233
}
236
234
if err := c.Get(ctx, "com.atproto.admin.getAccountInfo", params, &d); err != nil {
237
235
return err
+9
-5
atproto/atcrypto/cmd/atp-crypto/main.go
+9
-5
atproto/atcrypto/cmd/atp-crypto/main.go
···
1
1
package main
2
2
3
3
import (
4
+
"context"
4
5
"fmt"
5
6
"log/slog"
6
7
"os"
7
8
8
9
"github.com/bluesky-social/indigo/atproto/atcrypto"
9
10
10
-
"github.com/urfave/cli/v2"
11
+
"github.com/urfave/cli/v3"
11
12
)
12
13
13
14
func main() {
14
-
app := cli.App{
15
+
app := cli.Command{
15
16
Name: "atp-crypto",
16
17
Usage: "informal debugging CLI tool for atproto key and cryptography",
17
18
}
···
35
36
}
36
37
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
37
38
slog.SetDefault(slog.New(h))
38
-
app.RunAndExitOnError()
39
+
if err := app.Run(context.Background(), os.Args); err != nil {
40
+
slog.Error("command failed", "error", err)
41
+
os.Exit(-1)
42
+
}
39
43
}
40
44
41
-
func runGenerate(cctx *cli.Context) error {
42
-
if cctx.Bool("k256") {
45
+
func runGenerate(ctx context.Context, cmd *cli.Command) error {
46
+
if cmd.Bool("k256") {
43
47
priv, err := atcrypto.GeneratePrivateKeyK256()
44
48
if err != nil {
45
49
return err
+18
-14
atproto/auth/oauth/cmd/oauth-web-demo/main.go
+18
-14
atproto/auth/oauth/cmd/oauth-web-demo/main.go
···
1
1
package main
2
2
3
3
import (
4
-
_ "embed"
4
+
"context"
5
5
"encoding/json"
6
6
"fmt"
7
7
"html/template"
···
10
10
"os"
11
11
"slices"
12
12
13
+
_ "embed"
13
14
_ "github.com/joho/godotenv/autoload"
14
15
15
16
"github.com/bluesky-social/indigo/atproto/atcrypto"
···
18
19
"github.com/bluesky-social/indigo/atproto/syntax"
19
20
20
21
"github.com/gorilla/sessions"
21
-
"github.com/urfave/cli/v2"
22
+
"github.com/urfave/cli/v3"
22
23
)
23
24
24
25
func main() {
25
-
app := cli.App{
26
+
app := cli.Command{
26
27
Name: "oauth-web-demo",
27
28
Usage: "atproto OAuth web server demo",
28
29
Action: runServer,
···
31
32
Name: "session-secret",
32
33
Usage: "random string/token used for session cookie security",
33
34
Required: true,
34
-
EnvVars: []string{"SESSION_SECRET"},
35
+
Sources: cli.EnvVars("SESSION_SECRET"),
35
36
},
36
37
&cli.StringFlag{
37
38
Name: "hostname",
38
39
Usage: "public host name for this client (if not localhost dev mode)",
39
-
EnvVars: []string{"CLIENT_HOSTNAME"},
40
+
Sources: cli.EnvVars("CLIENT_HOSTNAME"),
40
41
},
41
42
&cli.StringFlag{
42
43
Name: "client-secret-key",
43
44
Usage: "confidential client secret key. should be P-256 private key in multibase encoding",
44
-
EnvVars: []string{"CLIENT_SECRET_KEY"},
45
+
Sources: cli.EnvVars("CLIENT_SECRET_KEY"),
45
46
},
46
47
&cli.StringFlag{
47
48
Name: "client-secret-key-id",
48
49
Usage: "key id for client-secret-key",
49
50
Value: "primary",
50
-
EnvVars: []string{"CLIENT_SECRET_KEY_ID"},
51
+
Sources: cli.EnvVars("CLIENT_SECRET_KEY_ID"),
51
52
},
52
53
},
53
54
}
54
55
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
55
56
slog.SetDefault(slog.New(h))
56
-
app.RunAndExitOnError()
57
+
if err := app.Run(context.Background(), os.Args); err != nil {
58
+
slog.Error("command failed", "error", err)
59
+
os.Exit(-1)
60
+
}
57
61
}
58
62
59
63
type Server struct {
···
77
81
var tmplPostText string
78
82
var tmplPost = template.Must(template.Must(template.New("post.html").Parse(tmplBaseText)).Parse(tmplPostText))
79
83
80
-
func runServer(cctx *cli.Context) error {
84
+
func runServer(ctx context.Context, cmd *cli.Command) error {
81
85
82
86
// the 'account:email' scope is requested only as a demo of users not granting a permission during auth flow
83
87
scopes := []string{"atproto", "repo:app.bsky.feed.post?action=create", "account:email"}
84
88
bind := ":8080"
85
89
86
90
var config oauth.ClientConfig
87
-
hostname := cctx.String("hostname")
91
+
hostname := cmd.String("hostname")
88
92
if hostname == "" {
89
93
config = oauth.NewLocalhostConfig(
90
94
fmt.Sprintf("http://127.0.0.1%s/oauth/callback", bind),
···
100
104
}
101
105
102
106
// If a client secret key is provided (as a multibase string), turn this in to a confidential client
103
-
if cctx.String("client-secret-key") != "" && hostname != "" {
104
-
priv, err := atcrypto.ParsePrivateMultibase(cctx.String("client-secret-key"))
107
+
if cmd.String("client-secret-key") != "" && hostname != "" {
108
+
priv, err := atcrypto.ParsePrivateMultibase(cmd.String("client-secret-key"))
105
109
if err != nil {
106
110
return err
107
111
}
108
-
if err := config.SetClientSecret(priv, cctx.String("client-secret-key-id")); err != nil {
112
+
if err := config.SetClientSecret(priv, cmd.String("client-secret-key-id")); err != nil {
109
113
return err
110
114
}
111
115
slog.Info("configuring confidential OAuth client")
···
114
118
oauthClient := oauth.NewClientApp(&config, oauth.NewMemStore())
115
119
116
120
srv := Server{
117
-
CookieStore: sessions.NewCookieStore([]byte(cctx.String("session-secret"))),
121
+
CookieStore: sessions.NewCookieStore([]byte(cmd.String("session-secret"))),
118
122
Dir: identity.DefaultDirectory(),
119
123
OAuth: oauthClient,
120
124
}
+1
-1
atproto/identity/apidir/apidir.go
+1
-1
atproto/identity/apidir/apidir.go
···
12
12
"github.com/bluesky-social/indigo/atproto/identity"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
14
15
-
"github.com/carlmjohnson/versioninfo"
15
+
"github.com/earthboundkid/versioninfo/v2"
16
16
)
17
17
18
18
// Does HTTP requests to an identity server, using standard Lexicon endpoints
+12
-12
atproto/identity/cmd/atp-id/main.go
+12
-12
atproto/identity/cmd/atp-id/main.go
···
10
10
"github.com/bluesky-social/indigo/atproto/identity"
11
11
"github.com/bluesky-social/indigo/atproto/syntax"
12
12
13
-
"github.com/urfave/cli/v2"
13
+
"github.com/urfave/cli/v3"
14
14
)
15
15
16
16
func main() {
17
-
app := cli.App{
17
+
app := cli.Command{
18
18
Name: "atp-id",
19
19
Usage: "informal debugging CLI tool for atproto identities",
20
20
}
···
40
40
}
41
41
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
42
42
slog.SetDefault(slog.New(h))
43
-
app.RunAndExitOnError()
43
+
if err := app.Run(context.Background(), os.Args); err != nil {
44
+
slog.Error("command failed", "error", err)
45
+
os.Exit(-1)
46
+
}
44
47
}
45
48
46
-
func runLookup(cctx *cli.Context) error {
47
-
ctx := context.Background()
48
-
s := cctx.Args().First()
49
+
func runLookup(ctx context.Context, cmd *cli.Command) error {
50
+
s := cmd.Args().First()
49
51
if s == "" {
50
52
return fmt.Errorf("need to provide identifier as an argument")
51
53
}
···
65
67
return nil
66
68
}
67
69
68
-
func runResolveHandle(cctx *cli.Context) error {
69
-
ctx := context.Background()
70
-
s := cctx.Args().First()
70
+
func runResolveHandle(ctx context.Context, cmd *cli.Command) error {
71
+
s := cmd.Args().First()
71
72
if s == "" {
72
73
return fmt.Errorf("need to provide handle as an argument")
73
74
}
···
87
88
return nil
88
89
}
89
90
90
-
func runResolveDID(cctx *cli.Context) error {
91
-
ctx := context.Background()
92
-
s := cctx.Args().First()
91
+
func runResolveDID(ctx context.Context, cmd *cli.Command) error {
92
+
s := cmd.Args().First()
93
93
if s == "" {
94
94
fmt.Println("need to provide DID as an argument")
95
95
os.Exit(-1)
+1
-1
atproto/identity/directory.go
+1
-1
atproto/identity/directory.go
+13
-9
atproto/lexicon/cmd/lextool/main.go
+13
-9
atproto/lexicon/cmd/lextool/main.go
···
1
1
package main
2
2
3
3
import (
4
+
"context"
4
5
"encoding/json"
5
6
"fmt"
6
7
"io"
···
9
10
10
11
"github.com/bluesky-social/indigo/atproto/lexicon"
11
12
12
-
"github.com/urfave/cli/v2"
13
+
"github.com/urfave/cli/v3"
13
14
)
14
15
15
16
func main() {
16
-
app := cli.App{
17
+
app := cli.Command{
17
18
Name: "lex-tool",
18
19
Usage: "informal debugging CLI tool for atproto lexicons",
19
20
}
···
41
42
}
42
43
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
43
44
slog.SetDefault(slog.New(h))
44
-
app.RunAndExitOnError()
45
+
if err := app.Run(context.Background(), os.Args); err != nil {
46
+
slog.Error("command failed", "error", err)
47
+
os.Exit(-1)
48
+
}
45
49
}
46
50
47
-
func runParseSchema(cctx *cli.Context) error {
48
-
p := cctx.Args().First()
51
+
func runParseSchema(ctx context.Context, cmd *cli.Command) error {
52
+
p := cmd.Args().First()
49
53
if p == "" {
50
54
return fmt.Errorf("need to provide path to a schema file as an argument")
51
55
}
···
73
77
return nil
74
78
}
75
79
76
-
func runLoadDirectory(cctx *cli.Context) error {
77
-
p := cctx.Args().First()
80
+
func runLoadDirectory(ctx context.Context, cmd *cli.Command) error {
81
+
p := cmd.Args().First()
78
82
if p == "" {
79
83
return fmt.Errorf("need to provide directory path as an argument")
80
84
}
···
89
93
return nil
90
94
}
91
95
92
-
func runResolve(cctx *cli.Context) error {
93
-
ref := cctx.Args().First()
96
+
func runResolve(ctx context.Context, cmd *cli.Command) error {
97
+
ref := cmd.Args().First()
94
98
if ref == "" {
95
99
return fmt.Errorf("need to provide NSID as an argument")
96
100
}
+3
-4
atproto/lexicon/cmd/lextool/net.go
+3
-4
atproto/lexicon/cmd/lextool/net.go
···
12
12
"github.com/bluesky-social/indigo/atproto/lexicon"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
14
15
-
"github.com/urfave/cli/v2"
15
+
"github.com/urfave/cli/v3"
16
16
)
17
17
18
-
func runValidateRecord(cctx *cli.Context) error {
19
-
ctx := context.Background()
20
-
args := cctx.Args().Slice()
18
+
func runValidateRecord(ctx context.Context, cmd *cli.Command) error {
19
+
args := cmd.Args().Slice()
21
20
if len(args) != 2 {
22
21
return fmt.Errorf("expected two args (catalog path and AT-URI)")
23
22
}
+5
-6
atproto/repo/cmd/repo-tool/firehose.go
+5
-6
atproto/repo/cmd/repo-tool/firehose.go
···
14
14
"github.com/bluesky-social/indigo/events"
15
15
"github.com/bluesky-social/indigo/events/schedulers/parallel"
16
16
17
-
"github.com/carlmjohnson/versioninfo"
17
+
"github.com/earthboundkid/versioninfo/v2"
18
18
"github.com/gorilla/websocket"
19
-
"github.com/urfave/cli/v2"
19
+
"github.com/urfave/cli/v3"
20
20
)
21
21
22
22
// write out error cases as JSON files to disk, for use in regression tests
23
23
var CAPTURE_TEST_CASES = false
24
24
25
-
func runVerifyFirehose(cctx *cli.Context) error {
26
-
ctx := context.Background()
25
+
func runVerifyFirehose(ctx context.Context, cmd *cli.Command) error {
27
26
28
-
slog.SetDefault(configLogger(cctx, os.Stdout))
27
+
slog.SetDefault(configLogger(ctx, cmd, os.Stdout))
29
28
30
-
relayHost := cctx.String("relay-host")
29
+
relayHost := cmd.String("relay-host")
31
30
32
31
dialer := websocket.DefaultDialer
33
32
u, err := url.Parse(relayHost)
+14
-13
atproto/repo/cmd/repo-tool/main.go
+14
-13
atproto/repo/cmd/repo-tool/main.go
···
12
12
"github.com/bluesky-social/indigo/atproto/repo"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
14
15
-
"github.com/urfave/cli/v2"
15
+
"github.com/urfave/cli/v3"
16
16
)
17
17
18
18
func main() {
19
-
app := cli.App{
19
+
app := cli.Command{
20
20
Name: "repo-tool",
21
21
Usage: "development tool for atproto MST trees, CAR files, etc",
22
22
Flags: []cli.Flag{
23
23
&cli.StringFlag{
24
24
Name: "log-level",
25
25
Usage: "log verbosity level (eg: warn, info, debug)",
26
-
EnvVars: []string{"BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
26
+
Sources: cli.EnvVars("BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
27
27
},
28
28
},
29
29
}
···
49
49
Name: "relay-host",
50
50
Usage: "method, hostname, and port of Relay instance (websocket)",
51
51
Value: "wss://bsky.network",
52
-
EnvVars: []string{"ATP_RELAY_HOST"},
52
+
Sources: cli.EnvVars("ATP_RELAY_HOST"),
53
53
},
54
54
},
55
55
},
56
56
}
57
57
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
58
58
slog.SetDefault(slog.New(h))
59
-
app.RunAndExitOnError()
59
+
if err := app.Run(context.Background(), os.Args); err != nil {
60
+
slog.Error("command failed", "error", err)
61
+
os.Exit(-1)
62
+
}
60
63
}
61
64
62
-
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
65
+
func configLogger(ctx context.Context, cmd *cli.Command, writer io.Writer) *slog.Logger {
63
66
var level slog.Level
64
-
switch strings.ToLower(cctx.String("log-level")) {
67
+
switch strings.ToLower(cmd.String("log-level")) {
65
68
case "error":
66
69
level = slog.LevelError
67
70
case "warn":
···
80
83
return logger
81
84
}
82
85
83
-
func runVerifyCarMst(cctx *cli.Context) error {
84
-
ctx := context.Background()
85
-
p := cctx.Args().First()
86
+
func runVerifyCarMst(ctx context.Context, cmd *cli.Command) error {
87
+
p := cmd.Args().First()
86
88
if p == "" {
87
89
return fmt.Errorf("need to provide path to CAR file")
88
90
}
···
110
112
return nil
111
113
}
112
114
113
-
func runVerifyCarSignature(cctx *cli.Context) error {
114
-
ctx := context.Background()
115
+
func runVerifyCarSignature(ctx context.Context, cmd *cli.Command) error {
115
116
dir := identity.DefaultDirectory()
116
117
117
-
p := cctx.Args().First()
118
+
p := cmd.Args().First()
118
119
if p == "" {
119
120
return fmt.Errorf("need to provide path to CAR file")
120
121
}
+11
-7
atproto/syntax/cmd/atp-syntax/main.go
+11
-7
atproto/syntax/cmd/atp-syntax/main.go
···
1
1
package main
2
2
3
3
import (
4
+
"context"
4
5
"fmt"
5
6
"log/slog"
6
7
"os"
7
8
8
9
"github.com/bluesky-social/indigo/atproto/syntax"
9
10
10
-
"github.com/urfave/cli/v2"
11
+
"github.com/urfave/cli/v3"
11
12
)
12
13
13
14
func main() {
14
-
app := cli.App{
15
+
app := cli.Command{
15
16
Name: "atp-syntax",
16
17
Usage: "informal debugging CLI tool for atproto syntax (identifiers)",
17
18
}
···
31
32
}
32
33
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
33
34
slog.SetDefault(slog.New(h))
34
-
app.RunAndExitOnError()
35
+
if err := app.Run(context.Background(), os.Args); err != nil {
36
+
slog.Error("command failed", "error", err)
37
+
os.Exit(-1)
38
+
}
35
39
}
36
40
37
-
func runParseTID(cctx *cli.Context) error {
38
-
s := cctx.Args().First()
41
+
func runParseTID(ctx context.Context, cmd *cli.Command) error {
42
+
s := cmd.Args().First()
39
43
if s == "" {
40
44
return fmt.Errorf("need to provide identifier as an argument")
41
45
}
···
50
54
return nil
51
55
}
52
56
53
-
func runParseDID(cctx *cli.Context) error {
54
-
s := cctx.Args().First()
57
+
func runParseDID(ctx context.Context, cmd *cli.Command) error {
58
+
s := cmd.Args().First()
55
59
if s == "" {
56
60
return fmt.Errorf("need to provide identifier as an argument")
57
61
}
+3
-3
automod/consumer/firehose.go
+3
-3
automod/consumer/firehose.go
···
13
13
comatproto "github.com/bluesky-social/indigo/api/atproto"
14
14
"github.com/bluesky-social/indigo/atproto/syntax"
15
15
"github.com/bluesky-social/indigo/automod"
16
+
"github.com/bluesky-social/indigo/events"
16
17
"github.com/bluesky-social/indigo/events/schedulers/autoscaling"
17
18
"github.com/bluesky-social/indigo/events/schedulers/parallel"
18
19
lexutil "github.com/bluesky-social/indigo/lex/util"
19
-
20
-
"github.com/bluesky-social/indigo/events"
21
20
"github.com/bluesky-social/indigo/repo"
22
21
"github.com/bluesky-social/indigo/repomgr"
23
-
"github.com/carlmjohnson/versioninfo"
22
+
23
+
"github.com/earthboundkid/versioninfo/v2"
24
24
"github.com/gorilla/websocket"
25
25
"github.com/redis/go-redis/v9"
26
26
)
+1
-1
automod/engine/blobs.go
+1
-1
automod/engine/blobs.go
+11
-9
automod/keyword/cmd/kw-cli/main.go
+11
-9
automod/keyword/cmd/kw-cli/main.go
···
10
10
"github.com/bluesky-social/indigo/automod/keyword"
11
11
"github.com/bluesky-social/indigo/automod/setstore"
12
12
13
-
"github.com/urfave/cli/v2"
13
+
"github.com/urfave/cli/v3"
14
14
)
15
15
16
16
func main() {
17
-
app := cli.App{
17
+
app := cli.Command{
18
18
Name: "kw-cli",
19
19
Usage: "informal debugging CLI tool for keyword matching",
20
20
}
···
48
48
}
49
49
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
50
50
slog.SetDefault(slog.New(h))
51
-
app.RunAndExitOnError()
51
+
if err := app.Run(context.Background(), os.Args); err != nil {
52
+
slog.Error("command failed", "error", err)
53
+
os.Exit(-1)
54
+
}
52
55
}
53
56
54
-
func runFuzzy(cctx *cli.Context) error {
57
+
func runFuzzy(ctx context.Context, cmd *cli.Command) error {
55
58
scanner := bufio.NewScanner(os.Stdin)
56
59
for scanner.Scan() {
57
60
line := scanner.Text()
···
63
66
return nil
64
67
}
65
68
66
-
func runTokens(cctx *cli.Context) error {
67
-
ctx := context.Background()
69
+
func runTokens(ctx context.Context, cmd *cli.Command) error {
68
70
sets := setstore.NewMemSetStore()
69
-
if err := sets.LoadFromFileJSON(cctx.String("json-set-file")); err != nil {
71
+
if err := sets.LoadFromFileJSON(cmd.String("json-set-file")); err != nil {
70
72
return err
71
73
}
72
-
setName := cctx.String("set-name")
73
-
identMode := cctx.Bool("identifiers")
74
+
setName := cmd.String("set-name")
75
+
identMode := cmd.Bool("identifiers")
74
76
scanner := bufio.NewScanner(os.Stdin)
75
77
for scanner.Scan() {
76
78
line := scanner.Text()
+1
-1
automod/visual/abyss_client.go
+1
-1
automod/visual/abyss_client.go
+1
-1
automod/visual/hiveai_client.go
+1
-1
automod/visual/hiveai_client.go
+3
-3
cmd/beemo/firehose_consumer.go
+3
-3
cmd/beemo/firehose_consumer.go
···
11
11
comatproto "github.com/bluesky-social/indigo/api/atproto"
12
12
appbsky "github.com/bluesky-social/indigo/api/bsky"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/bluesky-social/indigo/events"
14
15
"github.com/bluesky-social/indigo/events/schedulers/parallel"
15
16
lexutil "github.com/bluesky-social/indigo/lex/util"
16
-
17
-
"github.com/bluesky-social/indigo/events"
18
17
"github.com/bluesky-social/indigo/repo"
19
18
"github.com/bluesky-social/indigo/repomgr"
20
-
"github.com/carlmjohnson/versioninfo"
19
+
20
+
"github.com/earthboundkid/versioninfo/v2"
21
21
"github.com/gorilla/websocket"
22
22
)
23
23
+18
-17
cmd/beemo/main.go
+18
-17
cmd/beemo/main.go
···
4
4
package main
5
5
6
6
import (
7
+
"context"
7
8
"io"
8
9
"log/slog"
9
10
"os"
···
12
13
_ "github.com/joho/godotenv/autoload"
13
14
_ "go.uber.org/automaxprocs"
14
15
15
-
"github.com/carlmjohnson/versioninfo"
16
-
"github.com/urfave/cli/v2"
16
+
"github.com/earthboundkid/versioninfo/v2"
17
+
"github.com/urfave/cli/v3"
17
18
)
18
19
19
20
func main() {
···
25
26
26
27
func run(args []string) error {
27
28
28
-
app := cli.App{
29
+
app := cli.Command{
29
30
Name: "beemo",
30
31
Usage: "bluesky moderation reporting bot",
31
32
Version: versioninfo.Short(),
···
35
36
&cli.StringFlag{
36
37
Name: "log-level",
37
38
Usage: "log verbosity level (eg: warn, info, debug)",
38
-
EnvVars: []string{"BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
39
+
Sources: cli.EnvVars("BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
39
40
},
40
41
&cli.StringFlag{
41
42
Name: "slack-webhook-url",
42
43
// eg: https://hooks.slack.com/services/X1234
43
44
Usage: "full URL of slack webhook",
44
45
Required: true,
45
-
EnvVars: []string{"SLACK_WEBHOOK_URL"},
46
+
Sources: cli.EnvVars("SLACK_WEBHOOK_URL"),
46
47
},
47
48
}
48
49
app.Commands = []*cli.Command{
···
55
56
Name: "pds-host",
56
57
Usage: "method, hostname, and port of PDS instance",
57
58
Value: "http://localhost:4849",
58
-
EnvVars: []string{"ATP_PDS_HOST"},
59
+
Sources: cli.EnvVars("ATP_PDS_HOST"),
59
60
},
60
61
&cli.StringFlag{
61
62
Name: "admin-host",
62
63
Usage: "method, hostname, and port of admin interface (eg, Ozone), for direct links",
63
64
Value: "http://localhost:3000",
64
-
EnvVars: []string{"ATP_ADMIN_HOST"},
65
+
Sources: cli.EnvVars("ATP_ADMIN_HOST"),
65
66
},
66
67
&cli.IntFlag{
67
68
Name: "poll-period",
68
69
Usage: "API poll period in seconds",
69
70
Value: 30,
70
-
EnvVars: []string{"POLL_PERIOD"},
71
+
Sources: cli.EnvVars("POLL_PERIOD"),
71
72
},
72
73
&cli.StringFlag{
73
74
Name: "handle",
74
75
Usage: "for PDS login",
75
76
Required: true,
76
-
EnvVars: []string{"ATP_AUTH_HANDLE"},
77
+
Sources: cli.EnvVars("ATP_AUTH_HANDLE"),
77
78
},
78
79
&cli.StringFlag{
79
80
Name: "password",
80
81
Usage: "for PDS login",
81
82
Required: true,
82
-
EnvVars: []string{"ATP_AUTH_PASSWORD"},
83
+
Sources: cli.EnvVars("ATP_AUTH_PASSWORD"),
83
84
},
84
85
&cli.StringFlag{
85
86
Name: "admin-password",
86
87
Usage: "admin authentication password for PDS",
87
88
Required: true,
88
-
EnvVars: []string{"ATP_AUTH_ADMIN_PASSWORD"},
89
+
Sources: cli.EnvVars("ATP_AUTH_ADMIN_PASSWORD"),
89
90
},
90
91
},
91
92
},
···
98
99
Name: "relay-host",
99
100
Usage: "method, hostname, and port of Relay instance (websocket)",
100
101
Value: "wss://bsky.network",
101
-
EnvVars: []string{"ATP_RELAY_HOST"},
102
+
Sources: cli.EnvVars("ATP_RELAY_HOST"),
102
103
},
103
104
&cli.StringFlag{
104
105
Name: "mention-dids",
105
106
Usage: "DIDs to look for in mentions (comma-separated)",
106
107
Required: true,
107
-
EnvVars: []string{"BEEMO_MENTION_DIDS"},
108
+
Sources: cli.EnvVars("BEEMO_MENTION_DIDS"),
108
109
},
109
110
&cli.IntFlag{
110
111
Name: "minimum-words",
111
112
Usage: "minimum length of post text (word count; zero for no minimum)",
112
113
Value: 0,
113
-
EnvVars: []string{"BEEMO_MINIMUM_WORDS"},
114
+
Sources: cli.EnvVars("BEEMO_MINIMUM_WORDS"),
114
115
},
115
116
},
116
117
},
117
118
}
118
-
return app.Run(args)
119
+
return app.Run(context.Background(), args)
119
120
}
120
121
121
-
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
122
+
func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger {
122
123
var level slog.Level
123
-
switch strings.ToLower(cctx.String("log-level")) {
124
+
switch strings.ToLower(cmd.String("log-level")) {
124
125
case "error":
125
126
level = slog.LevelError
126
127
case "warn":
+7
-8
cmd/beemo/notify_mentions.go
+7
-8
cmd/beemo/notify_mentions.go
···
11
11
"github.com/bluesky-social/indigo/atproto/identity"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
13
14
-
"github.com/urfave/cli/v2"
14
+
"github.com/urfave/cli/v3"
15
15
)
16
16
17
17
type MentionChecker struct {
···
61
61
return nil
62
62
}
63
63
64
-
func notifyMentions(cctx *cli.Context) error {
65
-
ctx := context.Background()
66
-
logger := configLogger(cctx, os.Stdout)
67
-
relayHost := cctx.String("relay-host")
68
-
minimumWords := cctx.Int("minimum-words")
64
+
func notifyMentions(ctx context.Context, cmd *cli.Command) error {
65
+
logger := configLogger(cmd, os.Stdout)
66
+
relayHost := cmd.String("relay-host")
67
+
minimumWords := cmd.Int("minimum-words")
69
68
70
69
mentionDIDs := []syntax.DID{}
71
-
for _, raw := range strings.Split(cctx.String("mention-dids"), ",") {
70
+
for _, raw := range strings.Split(cmd.String("mention-dids"), ",") {
72
71
did, err := syntax.ParseDID(raw)
73
72
if err != nil {
74
73
return err
···
77
76
}
78
77
79
78
checker := MentionChecker{
80
-
slackWebhookURL: cctx.String("slack-webhook-url"),
79
+
slackWebhookURL: cmd.String("slack-webhook-url"),
81
80
mentionDIDs: mentionDIDs,
82
81
logger: logger,
83
82
directory: identity.DefaultDirectory(),
+12
-13
cmd/beemo/notify_reports.go
+12
-13
cmd/beemo/notify_reports.go
···
12
12
"github.com/bluesky-social/indigo/util"
13
13
"github.com/bluesky-social/indigo/xrpc"
14
14
15
-
"github.com/urfave/cli/v2"
15
+
"github.com/urfave/cli/v3"
16
16
)
17
17
18
-
func pollNewReports(cctx *cli.Context) error {
19
-
ctx := context.Background()
20
-
logger := configLogger(cctx, os.Stdout)
21
-
slackWebhookURL := cctx.String("slack-webhook-url")
18
+
func pollNewReports(ctx context.Context, cmd *cli.Command) error {
19
+
logger := configLogger(cmd, os.Stdout)
20
+
slackWebhookURL := cmd.String("slack-webhook-url")
22
21
23
22
// record last-seen report timestamp
24
23
since := time.Now()
25
24
// NOTE: uncomment this for testing
26
25
//since = time.Now().Add(time.Duration(-12) * time.Hour)
27
-
period := time.Duration(cctx.Int("poll-period")) * time.Second
26
+
period := time.Duration(cmd.Int("poll-period")) * time.Second
28
27
29
28
// create a new session
30
29
xrpcc := &xrpc.Client{
31
30
Client: util.RobustHTTPClient(),
32
-
Host: cctx.String("pds-host"),
33
-
Auth: &xrpc.AuthInfo{Handle: cctx.String("handle")},
31
+
Host: cmd.String("pds-host"),
32
+
Auth: &xrpc.AuthInfo{Handle: cmd.String("handle")},
34
33
}
35
34
36
35
auth, err := comatproto.ServerCreateSession(ctx, xrpcc, &comatproto.ServerCreateSession_Input{
37
36
Identifier: xrpcc.Auth.Handle,
38
-
Password: cctx.String("password"),
37
+
Password: cmd.String("password"),
39
38
})
40
39
if err != nil {
41
40
return err
···
45
44
xrpcc.Auth.Did = auth.Did
46
45
xrpcc.Auth.Handle = auth.Handle
47
46
48
-
adminToken := cctx.String("admin-password")
47
+
adminToken := cmd.String("admin-password")
49
48
if len(adminToken) > 0 {
50
49
xrpcc.AdminToken = &adminToken
51
50
}
···
70
69
// query just new reports (regardless of resolution state)
71
70
var limit int64 = 50
72
71
me, err := toolsozone.ModerationQueryEvents(
73
-
cctx.Context,
72
+
ctx,
74
73
xrpcc,
75
74
nil, // addedLabels []string
76
75
nil, // addedTags []string
···
113
112
// ok, we found a "new" report, need to notify
114
113
msg := fmt.Sprintf("⚠️ New report at `%s` ⚠️\n", evt.CreatedAt)
115
114
msg += fmt.Sprintf("report id: `%d`\t", evt.Id)
116
-
msg += fmt.Sprintf("instance: `%s`\n", cctx.String("pds-host"))
115
+
msg += fmt.Sprintf("instance: `%s`\n", cmd.String("pds-host"))
117
116
msg += fmt.Sprintf("reasonType: `%s`\t", shortType)
118
-
msg += fmt.Sprintf("Admin: %s/reports/%d\n", cctx.String("admin-host"), evt.Id)
117
+
msg += fmt.Sprintf("Admin: %s/reports/%d\n", cmd.String("admin-host"), evt.Id)
119
118
//msg += fmt.Sprintf("reportedByDid: `%s`\n", report.ReportedByDid)
120
119
logger.Info("found new report, notifying slack", "report", report)
121
120
err := sendSlackMsg(ctx, msg, slackWebhookURL)
+5
-5
cmd/bigsky/main.go
+5
-5
cmd/bigsky/main.go
···
5
5
"fmt"
6
6
"log/slog"
7
7
"net/http"
8
-
_ "net/http/pprof"
9
8
"net/url"
10
9
"os"
11
10
"os/signal"
···
13
12
"strings"
14
13
"syscall"
15
14
"time"
15
+
16
+
_ "github.com/joho/godotenv/autoload"
17
+
_ "go.uber.org/automaxprocs"
18
+
_ "net/http/pprof"
16
19
17
20
libbgs "github.com/bluesky-social/indigo/bgs"
18
21
"github.com/bluesky-social/indigo/carstore"
···
28
31
"github.com/bluesky-social/indigo/util/cliutil"
29
32
"github.com/bluesky-social/indigo/xrpc"
30
33
31
-
_ "github.com/joho/godotenv/autoload"
32
-
_ "go.uber.org/automaxprocs"
33
-
34
-
"github.com/carlmjohnson/versioninfo"
34
+
"github.com/earthboundkid/versioninfo/v2"
35
35
"github.com/urfave/cli/v2"
36
36
"go.opentelemetry.io/otel"
37
37
"go.opentelemetry.io/otel/attribute"
+2
-2
cmd/bluepages/firehose.go
+2
-2
cmd/bluepages/firehose.go
···
10
10
11
11
comatproto "github.com/bluesky-social/indigo/api/atproto"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/bluesky-social/indigo/events"
13
14
"github.com/bluesky-social/indigo/events/schedulers/parallel"
14
15
15
-
"github.com/bluesky-social/indigo/events"
16
-
"github.com/carlmjohnson/versioninfo"
16
+
"github.com/earthboundkid/versioninfo/v2"
17
17
"github.com/gorilla/websocket"
18
18
"github.com/redis/go-redis/v9"
19
19
)
+48
-52
cmd/bluepages/main.go
+48
-52
cmd/bluepages/main.go
···
6
6
"fmt"
7
7
"io"
8
8
"log/slog"
9
-
_ "net/http/pprof"
10
9
"os"
11
10
"runtime"
12
11
"strings"
12
+
13
+
_ "github.com/joho/godotenv/autoload"
14
+
_ "net/http/pprof"
13
15
14
16
"github.com/bluesky-social/indigo/atproto/identity/apidir"
15
17
"github.com/bluesky-social/indigo/atproto/syntax"
16
18
17
-
"github.com/carlmjohnson/versioninfo"
18
-
_ "github.com/joho/godotenv/autoload"
19
-
"github.com/urfave/cli/v2"
19
+
"github.com/earthboundkid/versioninfo/v2"
20
+
"github.com/urfave/cli/v3"
20
21
)
21
22
22
23
func main() {
···
28
29
29
30
func run(args []string) error {
30
31
31
-
app := cli.App{
32
+
app := cli.Command{
32
33
Name: "bluepages",
33
34
Usage: "atproto identity directory",
34
35
Version: versioninfo.Short(),
···
37
38
Name: "atp-relay-host",
38
39
Usage: "hostname and port of Relay to subscribe to",
39
40
Value: "wss://bsky.network",
40
-
EnvVars: []string{"ATP_RELAY_HOST", "ATP_BGS_HOST"},
41
+
Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"),
41
42
},
42
43
&cli.StringFlag{
43
44
Name: "atp-plc-host",
44
45
Usage: "method, hostname, and port of PLC registry",
45
46
Value: "https://plc.directory",
46
-
EnvVars: []string{"ATP_PLC_HOST"},
47
+
Sources: cli.EnvVars("ATP_PLC_HOST"),
47
48
},
48
49
&cli.IntFlag{
49
50
Name: "plc-rate-limit",
50
51
Usage: "max number of requests per second to PLC registry",
51
52
Value: 300,
52
-
EnvVars: []string{"BLUEPAGES_PLC_RATE_LIMIT"},
53
+
Sources: cli.EnvVars("BLUEPAGES_PLC_RATE_LIMIT"),
53
54
},
54
55
&cli.StringFlag{
55
56
Name: "redis-url",
56
57
Usage: "redis connection URL: redis://<user>:<pass>@<hostname>:6379/<db>",
57
58
Value: "redis://localhost:6379/0",
58
-
EnvVars: []string{"BLUEPAGES_REDIS_URL"},
59
+
Sources: cli.EnvVars("BLUEPAGES_REDIS_URL"),
59
60
},
60
61
&cli.StringFlag{
61
62
Name: "log-level",
62
63
Usage: "log verbosity level (eg: warn, info, debug)",
63
-
EnvVars: []string{"BLUEPAGES_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
64
+
Sources: cli.EnvVars("BLUEPAGES_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
64
65
},
65
66
},
66
67
Commands: []*cli.Command{
···
74
75
Usage: "Specify the local IP/port to bind to",
75
76
Required: false,
76
77
Value: ":6600",
77
-
EnvVars: []string{"BLUEPAGES_BIND"},
78
+
Sources: cli.EnvVars("BLUEPAGES_BIND"),
78
79
},
79
80
&cli.StringFlag{
80
81
Name: "metrics-listen",
81
82
Usage: "IP or address, and port, to listen on for metrics APIs",
82
83
Value: ":3989",
83
-
EnvVars: []string{"BLUEPAGES_METRICS_LISTEN"},
84
+
Sources: cli.EnvVars("BLUEPAGES_METRICS_LISTEN"),
84
85
},
85
86
&cli.BoolFlag{
86
87
Name: "disable-firehose-consumer",
87
88
Usage: "don't consume #identity events from firehose",
88
-
EnvVars: []string{"BLUEPAGES_DISABLE_FIREHOSE_CONSUMER"},
89
+
Sources: cli.EnvVars("BLUEPAGES_DISABLE_FIREHOSE_CONSUMER"),
89
90
},
90
91
&cli.BoolFlag{
91
92
Name: "disable-refresh",
92
93
Usage: "disable the refreshIdentity API endpoint",
93
-
EnvVars: []string{"BLUEPAGES_DISABLE_REFRESH"},
94
+
Sources: cli.EnvVars("BLUEPAGES_DISABLE_REFRESH"),
94
95
},
95
96
&cli.IntFlag{
96
97
Name: "firehose-parallelism",
97
98
Usage: "number of concurrent firehose workers",
98
99
Value: 4,
99
-
EnvVars: []string{"BLUEPAGES_FIREHOSE_PARALLELISM"},
100
+
Sources: cli.EnvVars("BLUEPAGES_FIREHOSE_PARALLELISM"),
100
101
},
101
102
},
102
103
},
···
110
111
Name: "host",
111
112
Usage: "bluepages server to send request to",
112
113
Value: "http://localhost:6600",
113
-
EnvVars: []string{"BLUEPAGES_HOST"},
114
+
Sources: cli.EnvVars("BLUEPAGES_HOST"),
114
115
},
115
116
},
116
117
},
···
124
125
Name: "host",
125
126
Usage: "bluepages server to send request to",
126
127
Value: "http://localhost:6600",
127
-
EnvVars: []string{"BLUEPAGES_HOST"},
128
+
Sources: cli.EnvVars("BLUEPAGES_HOST"),
128
129
},
129
130
},
130
131
},
···
138
139
Name: "host",
139
140
Usage: "bluepages server to send request to",
140
141
Value: "http://localhost:6600",
141
-
EnvVars: []string{"BLUEPAGES_HOST"},
142
+
Sources: cli.EnvVars("BLUEPAGES_HOST"),
142
143
},
143
144
},
144
145
},
···
152
153
Name: "host",
153
154
Usage: "bluepages server to send request to",
154
155
Value: "http://localhost:6600",
155
-
EnvVars: []string{"BLUEPAGES_HOST"},
156
+
Sources: cli.EnvVars("BLUEPAGES_HOST"),
156
157
},
157
158
},
158
159
},
159
160
},
160
161
}
161
162
162
-
return app.Run(args)
163
+
return app.Run(context.Background(), args)
163
164
}
164
165
165
-
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
166
+
func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger {
166
167
var level slog.Level
167
-
switch strings.ToLower(cctx.String("log-level")) {
168
+
switch strings.ToLower(cmd.String("log-level")) {
168
169
case "error":
169
170
level = slog.LevelError
170
171
case "warn":
···
183
184
return logger
184
185
}
185
186
186
-
func configClient(cctx *cli.Context) apidir.APIDirectory {
187
-
return apidir.NewAPIDirectory(cctx.String("host"))
187
+
func configClient(cmd *cli.Command) apidir.APIDirectory {
188
+
return apidir.NewAPIDirectory(cmd.String("host"))
188
189
}
189
190
190
-
func runServeCmd(cctx *cli.Context) error {
191
-
logger := configLogger(cctx, os.Stdout)
192
-
ctx := context.Background()
191
+
func runServeCmd(ctx context.Context, cmd *cli.Command) error {
192
+
logger := configLogger(cmd, os.Stdout)
193
193
194
194
srv, err := NewServer(
195
195
Config{
196
196
Logger: logger,
197
-
Bind: cctx.String("bind"),
198
-
RedisURL: cctx.String("redis-url"),
199
-
PLCHost: cctx.String("atp-plc-host"),
200
-
PLCRateLimit: cctx.Int("plc-rate-limit"),
201
-
DisableRefresh: cctx.Bool("disable-refresh"),
197
+
Bind: cmd.String("bind"),
198
+
RedisURL: cmd.String("redis-url"),
199
+
PLCHost: cmd.String("atp-plc-host"),
200
+
PLCRateLimit: cmd.Int("plc-rate-limit"),
201
+
DisableRefresh: cmd.Bool("disable-refresh"),
202
202
},
203
203
)
204
204
if err != nil {
205
205
return fmt.Errorf("failed to construct server: %v", err)
206
206
}
207
207
208
-
if !cctx.Bool("disable-firehose-consumer") {
208
+
if !cmd.Bool("disable-firehose-consumer") {
209
209
go func() {
210
-
firehoseHost := cctx.String("atp-relay-host")
211
-
firehoseParallelism := cctx.Int("firehose-parallelism")
210
+
firehoseHost := cmd.String("atp-relay-host")
211
+
firehoseParallelism := cmd.Int("firehose-parallelism")
212
212
if err := srv.RunFirehoseConsumer(ctx, firehoseHost, firehoseParallelism); err != nil {
213
213
slog.Error("firehose consumer thread failed", "err", err)
214
214
// NOTE: not crashing or halting process here
···
227
227
// TODO: what is this tuning for? just cargo-culted it
228
228
runtime.SetBlockProfileRate(10)
229
229
runtime.SetMutexProfileFraction(10)
230
-
if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil {
230
+
if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil {
231
231
slog.Error("failed to start metrics endpoint", "error", err)
232
232
// NOTE: not crashing or halting process here
233
233
}
···
236
236
return srv.RunAPI()
237
237
}
238
238
239
-
func runResolveHandleCmd(cctx *cli.Context) error {
240
-
ctx := context.Background()
241
-
dir := configClient(cctx)
239
+
func runResolveHandleCmd(ctx context.Context, cmd *cli.Command) error {
240
+
dir := configClient(cmd)
242
241
243
-
s := cctx.Args().First()
242
+
s := cmd.Args().First()
244
243
if s == "" {
245
244
return fmt.Errorf("need to provide identifier for resolution")
246
245
}
···
257
256
return nil
258
257
}
259
258
260
-
func runResolveDIDCmd(cctx *cli.Context) error {
261
-
ctx := context.Background()
262
-
dir := configClient(cctx)
259
+
func runResolveDIDCmd(ctx context.Context, cmd *cli.Command) error {
260
+
dir := configClient(cmd)
263
261
264
-
s := cctx.Args().First()
262
+
s := cmd.Args().First()
265
263
if s == "" {
266
264
return fmt.Errorf("need to provide identifier for resolution")
267
265
}
···
282
280
return nil
283
281
}
284
282
285
-
func runLookupCmd(cctx *cli.Context) error {
286
-
ctx := context.Background()
287
-
dir := configClient(cctx)
283
+
func runLookupCmd(ctx context.Context, cmd *cli.Command) error {
284
+
dir := configClient(cmd)
288
285
289
-
s := cctx.Args().First()
286
+
s := cmd.Args().First()
290
287
if s == "" {
291
288
return fmt.Errorf("need to provide identifier for resolution")
292
289
}
···
308
305
return nil
309
306
}
310
307
311
-
func runRefreshCmd(cctx *cli.Context) error {
312
-
ctx := context.Background()
313
-
dir := configClient(cctx)
308
+
func runRefreshCmd(ctx context.Context, cmd *cli.Command) error {
309
+
dir := configClient(cmd)
314
310
315
-
s := cctx.Args().First()
311
+
s := cmd.Args().First()
316
312
if s == "" {
317
313
return fmt.Errorf("need to provide identifier for resolution")
318
314
}
+34
-34
cmd/collectiondir/collectiondir.go
+34
-34
cmd/collectiondir/collectiondir.go
···
4
4
"bufio"
5
5
"bytes"
6
6
"compress/gzip"
7
+
"context"
7
8
"encoding/csv"
8
9
"encoding/json"
9
10
"errors"
···
17
18
"strings"
18
19
"time"
19
20
20
-
"github.com/carlmjohnson/versioninfo"
21
-
"github.com/urfave/cli/v2"
21
+
"github.com/earthboundkid/versioninfo/v2"
22
+
"github.com/urfave/cli/v3"
22
23
)
23
24
24
25
func main() {
25
-
app := cli.App{
26
+
app := cli.Command{
26
27
Name: "collectiondir",
27
28
Usage: "collection directory service",
28
29
Version: versioninfo.Short(),
···
40
41
adminCrawlCmd,
41
42
},
42
43
}
43
-
err := app.Run(os.Args)
44
-
if err != nil {
44
+
if err := app.Run(context.Background(), os.Args); err != nil {
45
45
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
46
46
os.Exit(1)
47
47
}
···
57
57
Required: true,
58
58
},
59
59
},
60
-
Action: func(cctx *cli.Context) error {
60
+
Action: func(ctx context.Context, cmd *cli.Command) error {
61
61
logLevel := slog.LevelInfo
62
-
if cctx.Bool("verbose") {
62
+
if cmd.Bool("verbose") {
63
63
logLevel = slog.LevelDebug
64
64
}
65
65
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
66
66
slog.SetDefault(log)
67
-
pebblePath := cctx.String("pebble")
67
+
pebblePath := cmd.String("pebble")
68
68
var db PebbleCollectionDirectory
69
69
db.log = log
70
70
err := db.Open(pebblePath)
···
94
94
Required: true,
95
95
},
96
96
},
97
-
Action: func(cctx *cli.Context) error {
97
+
Action: func(ctx context.Context, cmd *cli.Command) error {
98
98
logLevel := slog.LevelInfo
99
-
if cctx.Bool("verbose") {
99
+
if cmd.Bool("verbose") {
100
100
logLevel = slog.LevelDebug
101
101
}
102
102
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
103
103
slog.SetDefault(log)
104
-
pebblePath := cctx.String("pebble")
104
+
pebblePath := cmd.String("pebble")
105
105
var db PebbleCollectionDirectory
106
106
db.log = log
107
107
err := db.Open(pebblePath)
···
112
112
113
113
rows := make(chan CollectionDidTime, 100)
114
114
go func() {
115
-
err := db.ReadAllPrimary(cctx.Context, rows)
115
+
err := db.ReadAllPrimary(ctx, rows)
116
116
if err != nil {
117
117
log.Error("db read", "path", pebblePath, "err", err)
118
118
}
···
153
153
Required: true,
154
154
},
155
155
},
156
-
Action: func(cctx *cli.Context) error {
156
+
Action: func(ctx context.Context, cmd *cli.Command) error {
157
157
logLevel := slog.LevelInfo
158
-
if cctx.Bool("verbose") {
158
+
if cmd.Bool("verbose") {
159
159
logLevel = slog.LevelDebug
160
160
}
161
161
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
162
162
slog.SetDefault(log)
163
-
pebblePath := cctx.String("pebble")
163
+
pebblePath := cmd.String("pebble")
164
164
var db PebbleCollectionDirectory
165
165
db.log = log
166
166
err := db.Open(pebblePath)
···
168
168
return err
169
169
}
170
170
defer db.Close()
171
-
csvPath := cctx.String("csv")
171
+
csvPath := cmd.String("csv")
172
172
var fin io.Reader
173
173
if csvPath == "-" {
174
174
fin = os.Stdin
···
231
231
Name: "url",
232
232
Usage: "host:port of collectiondir server",
233
233
Required: true,
234
-
EnvVars: []string{"COLLECTIONDIR_URL"},
234
+
Sources: cli.EnvVars("COLLECTIONDIR_URL"),
235
235
},
236
236
&cli.StringFlag{
237
237
Name: "auth",
238
238
Usage: "Auth token for admin api",
239
-
EnvVars: []string{"ADMIN_AUTH"},
239
+
Sources: cli.EnvVars("ADMIN_AUTH"),
240
240
},
241
241
},
242
-
Action: func(cctx *cli.Context) error {
242
+
Action: func(ctx context.Context, cmd *cli.Command) error {
243
243
logLevel := slog.LevelInfo
244
-
if cctx.Bool("verbose") {
244
+
if cmd.Bool("verbose") {
245
245
logLevel = slog.LevelDebug
246
246
}
247
247
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
···
249
249
250
250
var hostList []string
251
251
252
-
serverUrl, err := url.Parse(cctx.String("url"))
252
+
serverUrl, err := url.Parse(cmd.String("url"))
253
253
if err != nil {
254
254
var e2 error
255
255
// try to fixup a bare host:port which can confuse url.Parse
256
-
serverUrl, e2 = url.Parse("http://" + cctx.String("url"))
256
+
serverUrl, e2 = url.Parse("http://" + cmd.String("url"))
257
257
if e2 != nil {
258
258
return fmt.Errorf("could not parse url, %w", err)
259
259
}
260
260
}
261
261
requestCrawlUrl := serverUrl.JoinPath("/admin/pds/requestCrawl")
262
262
263
-
if cctx.IsSet("list") {
264
-
fin, err := os.Open(cctx.String("list"))
263
+
if cmd.IsSet("list") {
264
+
fin, err := os.Open(cmd.String("list"))
265
265
if err != nil {
266
-
return fmt.Errorf("%s: could not open, %w", cctx.String("list"), err)
266
+
return fmt.Errorf("%s: could not open, %w", cmd.String("list"), err)
267
267
}
268
268
defer fin.Close()
269
269
bufin := bufio.NewScanner(fin)
···
272
272
}
273
273
err = bufin.Err()
274
274
if err != nil {
275
-
return fmt.Errorf("%s: error reading, %w", cctx.String("list"), err)
275
+
return fmt.Errorf("%s: error reading, %w", cmd.String("list"), err)
276
276
}
277
-
} else if cctx.IsSet("csv") {
278
-
fin, err := os.Open(cctx.String("csv"))
277
+
} else if cmd.IsSet("csv") {
278
+
fin, err := os.Open(cmd.String("csv"))
279
279
if err != nil {
280
-
return fmt.Errorf("%s: could not open, %w", cctx.String("csv"), err)
280
+
return fmt.Errorf("%s: could not open, %w", cmd.String("csv"), err)
281
281
}
282
282
defer fin.Close()
283
283
data, err := csv.NewReader(fin).ReadAll()
284
284
if err != nil {
285
-
return fmt.Errorf("%s: could not read, %w", cctx.String("csv"), err)
285
+
return fmt.Errorf("%s: could not read, %w", cmd.String("csv"), err)
286
286
}
287
287
if len(data) < 2 {
288
-
return fmt.Errorf("%s: empty CSV file", cctx.String("csv"))
288
+
return fmt.Errorf("%s: empty CSV file", cmd.String("csv"))
289
289
}
290
290
headerRow := data[0]
291
291
hostCol := -1
···
297
297
}
298
298
}
299
299
if hostCol < 0 {
300
-
return fmt.Errorf("%s: header missing 'host' or 'hostname'", cctx.String("csv"))
300
+
return fmt.Errorf("%s: header missing 'host' or 'hostname'", cmd.String("csv"))
301
301
}
302
302
for _, row := range data[1:] {
303
303
hostList = append(hostList, row[hostCol])
···
310
310
311
311
client := http.Client{Timeout: 1 * time.Second}
312
312
var headers http.Header = make(http.Header)
313
-
if cctx.IsSet("auth") {
314
-
headers.Add("Authorization", "Bearer "+cctx.String("auth"))
313
+
if cmd.IsSet("auth") {
314
+
headers.Add("Authorization", "Bearer "+cmd.String("auth"))
315
315
}
316
316
headers.Add("Content-Type", "application/json")
317
317
var response *http.Response
+8
-8
cmd/collectiondir/crawl.go
+8
-8
cmd/collectiondir/crawl.go
···
15
15
"github.com/bluesky-social/indigo/util"
16
16
"github.com/bluesky-social/indigo/xrpc"
17
17
18
-
"github.com/urfave/cli/v2"
18
+
"github.com/urfave/cli/v3"
19
19
"golang.org/x/time/rate"
20
20
)
21
21
···
55
55
&cli.StringFlag{
56
56
Name: "ratelimit-header",
57
57
Usage: "secret for friend PDSes",
58
-
EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"},
58
+
Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"),
59
59
},
60
60
},
61
-
Action: func(cctx *cli.Context) error {
61
+
Action: func(ctx context.Context, cmd *cli.Command) error {
62
62
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
63
63
ctx, cancel := context.WithCancel(context.Background())
64
64
defer cancel()
65
-
hostname := cctx.String("host")
65
+
hostname := cmd.String("host")
66
66
hosturl, err := url.Parse(hostname)
67
67
if err != nil {
68
68
hosturl = new(url.URL)
···
73
73
Host: hosturl.String(),
74
74
Client: util.RobustHTTPClient(),
75
75
}
76
-
if cctx.IsSet("ratelimit-header") {
76
+
if cmd.IsSet("ratelimit-header") {
77
77
rpcClient.Headers = map[string]string{
78
-
"x-ratelimit-bypass": cctx.String("ratelimit-header"),
78
+
"x-ratelimit-bypass": cmd.String("ratelimit-header"),
79
79
}
80
80
}
81
81
log.Info("will crawl", "url", rpcClient.Host)
82
-
csvOutPath := cctx.String("csv-out")
82
+
csvOutPath := cmd.String("csv-out")
83
83
var fout io.Writer = os.Stdout
84
84
if csvOutPath != "" {
85
85
if csvOutPath == "-" {
···
91
91
}
92
92
}
93
93
}
94
-
qps := cctx.Float64("qps")
94
+
qps := cmd.Float64("qps")
95
95
results := make(chan DidCollection, 100)
96
96
defer close(results)
97
97
go DidCollectionsToCsv(fout, results)
+28
-28
cmd/collectiondir/serve.go
+28
-28
cmd/collectiondir/serve.go
···
33
33
"github.com/labstack/echo/v4"
34
34
"github.com/labstack/echo/v4/middleware"
35
35
"github.com/prometheus/client_golang/prometheus/promhttp"
36
-
"github.com/urfave/cli/v2"
36
+
"github.com/urfave/cli/v3"
37
37
)
38
38
39
39
var serveCmd = &cli.Command{
···
42
42
&cli.StringFlag{
43
43
Name: "api-listen",
44
44
Value: ":2510",
45
-
EnvVars: []string{"COLLECTIONS_API_LISTEN"},
45
+
Sources: cli.EnvVars("COLLECTIONS_API_LISTEN"),
46
46
},
47
47
&cli.StringFlag{
48
48
Name: "metrics-listen",
49
49
Value: ":2511",
50
-
EnvVars: []string{"COLLECTIONS_METRICS_LISTEN"},
50
+
Sources: cli.EnvVars("COLLECTIONS_METRICS_LISTEN"),
51
51
},
52
52
&cli.StringFlag{
53
53
Name: "pebble",
···
62
62
&cli.StringFlag{
63
63
Name: "upstream",
64
64
Usage: "URL, e.g. wss://bsky.network",
65
-
EnvVars: []string{"COLLECTIONS_UPSTREAM"},
65
+
Sources: cli.EnvVars("COLLECTIONS_UPSTREAM"),
66
66
},
67
67
&cli.StringFlag{
68
68
Name: "admin-token",
69
69
Usage: "admin authentication",
70
-
EnvVars: []string{"COLLECTIONS_ADMIN_TOKEN"},
70
+
Sources: cli.EnvVars("COLLECTIONS_ADMIN_TOKEN"),
71
71
},
72
72
&cli.Float64Flag{
73
73
Name: "crawl-qps",
···
77
77
&cli.StringFlag{
78
78
Name: "ratelimit-header",
79
79
Usage: "secret for friend PDSes",
80
-
EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"},
80
+
Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"),
81
81
},
82
82
&cli.Uint64Flag{
83
83
Name: "clist-min-dids",
84
84
Usage: "filter collection list to >= N dids",
85
85
Value: 5,
86
-
EnvVars: []string{"COLLECTIONS_CLIST_MIN_DIDS"},
86
+
Sources: cli.EnvVars("COLLECTIONS_CLIST_MIN_DIDS"),
87
87
},
88
88
&cli.IntFlag{
89
89
Name: "max-did-collections",
90
90
Usage: "stop recording new collections per did after it has >= this many collections",
91
91
Value: 1000,
92
-
EnvVars: []string{"COLLECTIONS_MAX_DID_COLLECTIONS"},
92
+
Sources: cli.EnvVars("COLLECTIONS_MAX_DID_COLLECTIONS"),
93
93
},
94
94
&cli.StringFlag{
95
95
Name: "sets-json-path",
96
96
Usage: "file path of JSON file containing static word sets",
97
-
EnvVars: []string{"HEPA_SETS_JSON_PATH", "COLLECTIONS_SETS_JSON_PATH"},
97
+
Sources: cli.EnvVars("HEPA_SETS_JSON_PATH", "COLLECTIONS_SETS_JSON_PATH"),
98
98
},
99
99
&cli.BoolFlag{
100
100
Name: "verbose",
101
101
},
102
102
},
103
-
Action: func(cctx *cli.Context) error {
103
+
Action: func(ctx context.Context, cmd *cli.Command) error {
104
104
var server collectionServer
105
-
return server.run(cctx)
105
+
return server.run(ctx, cmd)
106
106
},
107
107
}
108
108
···
165
165
stats *CrawlStats
166
166
}
167
167
168
-
func (cs *collectionServer) run(cctx *cli.Context) error {
168
+
func (cs *collectionServer) run(ctx context.Context, cmd *cli.Command) error {
169
169
signals := make(chan os.Signal, 1)
170
170
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
171
171
cs.shutdown = make(chan struct{})
172
172
level := slog.LevelInfo
173
-
if cctx.Bool("verbose") {
173
+
if cmd.Bool("verbose") {
174
174
level = slog.LevelDebug
175
175
}
176
176
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))
177
177
slog.SetDefault(log)
178
178
179
-
if cctx.IsSet("ratelimit-header") {
180
-
cs.ratelimitHeader = cctx.String("ratelimit-header")
179
+
if cmd.IsSet("ratelimit-header") {
180
+
cs.ratelimitHeader = cmd.String("ratelimit-header")
181
181
}
182
-
if cctx.IsSet("sets-json-path") {
183
-
badwords, err := loadBadwords(cctx.String("sets-json-path"))
182
+
if cmd.IsSet("sets-json-path") {
183
+
badwords, err := loadBadwords(cmd.String("sets-json-path"))
184
184
if err != nil {
185
185
return err
186
186
}
187
187
cs.badwords = badwords
188
188
}
189
-
cs.MinDidsForCollectionList = cctx.Uint64("clist-min-dids")
190
-
cs.MaxDidCollections = cctx.Int("max-did-collections")
189
+
cs.MinDidsForCollectionList = cmd.Uint64("clist-min-dids")
190
+
cs.MaxDidCollections = cmd.Int("max-did-collections")
191
191
cs.ingestFirehose = make(chan DidCollection, 1000)
192
192
cs.ingestCrawl = make(chan DidCollection, 1000)
193
193
var err error
···
196
196
return fmt.Errorf("lru init, %w", err)
197
197
}
198
198
cs.log = log
199
-
cs.ctx = cctx.Context
200
-
cs.AdminToken = cctx.String("admin-token")
199
+
cs.ctx = ctx
200
+
cs.AdminToken = cmd.String("admin-token")
201
201
cs.ExepctedAuthHeader = "Bearer " + cs.AdminToken
202
202
cs.wg.Add(1)
203
203
go cs.ingestReceiver()
204
-
pebblePath := cctx.String("pebble")
204
+
pebblePath := cmd.String("pebble")
205
205
cs.pcd = &PebbleCollectionDirectory{
206
206
log: cs.log,
207
207
}
···
209
209
if err != nil {
210
210
return fmt.Errorf("%s: failed to open pebble db: %w", pebblePath, err)
211
211
}
212
-
cs.dauDirectoryDir = cctx.String("dau-directory")
212
+
cs.dauDirectoryDir = cmd.String("dau-directory")
213
213
if cs.dauDirectoryDir != "" {
214
214
err := cs.openDau()
215
215
if err != nil {
···
218
218
}
219
219
cs.statsCacheFresh.L = &cs.statsCacheLock
220
220
221
-
apiServerEcho, err := cs.createApiServer(cctx.Context, cctx.String("api-listen"))
221
+
apiServerEcho, err := cs.createApiServer(ctx, cmd.String("api-listen"))
222
222
if err != nil {
223
223
return err
224
224
}
225
225
cs.wg.Add(1)
226
-
go func() { cs.StartApiServer(cctx.Context, apiServerEcho) }()
226
+
go func() { cs.StartApiServer(ctx, apiServerEcho) }()
227
227
228
-
cs.createMetricsServer(cctx.String("metrics-listen"))
228
+
cs.createMetricsServer(cmd.String("metrics-listen"))
229
229
cs.wg.Add(1)
230
-
go func() { cs.StartMetricsServer(cctx.Context) }()
230
+
go func() { cs.StartMetricsServer(ctx) }()
231
231
232
-
upstream := cctx.String("upstream")
232
+
upstream := cmd.String("upstream")
233
233
if upstream != "" {
234
234
fh := Firehose{
235
235
Log: log,
+94
-46
cmd/fakermaker/main.go
+94
-46
cmd/fakermaker/main.go
···
11
11
"os"
12
12
"runtime"
13
13
14
+
_ "github.com/joho/godotenv/autoload"
15
+
_ "go.uber.org/automaxprocs"
16
+
14
17
comatproto "github.com/bluesky-social/indigo/api/atproto"
15
18
"github.com/bluesky-social/indigo/fakedata"
16
19
"github.com/bluesky-social/indigo/util/cliutil"
20
+
"github.com/bluesky-social/indigo/xrpc"
17
21
18
-
_ "github.com/joho/godotenv/autoload"
19
-
_ "go.uber.org/automaxprocs"
20
-
21
-
"github.com/carlmjohnson/versioninfo"
22
-
"github.com/urfave/cli/v2"
22
+
"github.com/earthboundkid/versioninfo/v2"
23
+
"github.com/urfave/cli/v3"
23
24
"golang.org/x/sync/errgroup"
24
25
)
25
26
···
29
30
30
31
func run(args []string) {
31
32
32
-
app := cli.App{
33
+
app := cli.Command{
33
34
Name: "fakermaker",
34
35
Usage: "bluesky fake account/content generator",
35
36
Version: versioninfo.Short(),
···
40
41
Name: "pds-host",
41
42
Usage: "method, hostname, and port of PDS instance",
42
43
Value: "http://localhost:4849",
43
-
EnvVars: []string{"ATP_PDS_HOST"},
44
+
Sources: cli.EnvVars("ATP_PDS_HOST"),
44
45
},
45
46
&cli.StringFlag{
46
47
Name: "admin-password",
47
48
Usage: "admin authentication password for PDS",
48
49
Required: true,
49
-
EnvVars: []string{"ATP_AUTH_ADMIN_PASSWORD"},
50
+
Sources: cli.EnvVars("ATP_AUTH_ADMIN_PASSWORD"),
50
51
},
51
52
&cli.IntFlag{
52
53
Name: "jobs",
···
196
197
},
197
198
}
198
199
all := fakedata.MeasureIterations("entire command")
199
-
app.RunAndExitOnError()
200
+
if err := app.Run(context.Background(), os.Args); err != nil {
201
+
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
202
+
os.Exit(1)
203
+
}
200
204
all(1)
201
205
}
202
206
203
207
// registers fake accounts with PDS, and spits out JSON-lines to stdout with auth info
204
-
func genAccounts(cctx *cli.Context) error {
208
+
func genAccounts(ctx context.Context, cmd *cli.Command) error {
205
209
206
210
// establish atproto client, with admin token for auth
207
-
xrpcc, err := cliutil.GetXrpcClient(cctx, false)
211
+
xrpcc, err := getXrpcClient(cmd, false)
208
212
if err != nil {
209
213
return err
210
214
}
211
-
adminToken := cctx.String("admin-password")
215
+
adminToken := cmd.String("admin-password")
212
216
if len(adminToken) > 0 {
213
217
xrpcc.AdminToken = &adminToken
214
218
}
215
219
216
-
countTotal := cctx.Int("count")
217
-
countCelebrities := cctx.Int("count-celebrities")
218
-
domainSuffix := cctx.String("domain-suffix")
220
+
countTotal := cmd.Int("count")
221
+
countCelebrities := cmd.Int("count-celebrities")
222
+
domainSuffix := cmd.String("domain-suffix")
219
223
if countCelebrities > countTotal {
220
224
return fmt.Errorf("more celebrities than total accounts!")
221
225
}
222
226
countRegulars := countTotal - countCelebrities
223
227
224
228
var inviteCode *string = nil
225
-
if cctx.Bool("use-invite-code") {
229
+
if cmd.Bool("use-invite-code") {
226
230
resp, err := comatproto.ServerCreateInviteCodes(context.TODO(), xrpcc, &comatproto.ServerCreateInviteCodes_Input{
227
231
UseCount: int64(countTotal),
228
232
ForAccounts: nil,
···
268
272
return nil
269
273
}
270
274
271
-
func genProfiles(cctx *cli.Context) error {
272
-
catalog, err := fakedata.ReadAccountCatalog(cctx.String("catalog"))
275
+
func genProfiles(ctx context.Context, cmd *cli.Command) error {
276
+
catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog"))
273
277
if err != nil {
274
278
return err
275
279
}
276
280
277
-
pdsHost := cctx.String("pds-host")
278
-
genAvatar := !cctx.Bool("no-avatars")
279
-
genBanner := !cctx.Bool("no-banners")
280
-
jobs := cctx.Int("jobs")
281
+
pdsHost := cmd.String("pds-host")
282
+
genAvatar := !cmd.Bool("no-avatars")
283
+
genBanner := !cmd.Bool("no-banners")
284
+
jobs := cmd.Int("jobs")
281
285
282
286
accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars))
283
287
eg := new(errgroup.Group)
···
303
307
return eg.Wait()
304
308
}
305
309
306
-
func genGraph(cctx *cli.Context) error {
307
-
catalog, err := fakedata.ReadAccountCatalog(cctx.String("catalog"))
310
+
func genGraph(ctx context.Context, cmd *cli.Command) error {
311
+
catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog"))
308
312
if err != nil {
309
313
return err
310
314
}
311
315
312
-
pdsHost := cctx.String("pds-host")
313
-
maxFollows := cctx.Int("max-follows")
314
-
maxMutes := cctx.Int("max-mutes")
315
-
jobs := cctx.Int("jobs")
316
+
pdsHost := cmd.String("pds-host")
317
+
maxFollows := cmd.Int("max-follows")
318
+
maxMutes := cmd.Int("max-mutes")
319
+
jobs := cmd.Int("jobs")
316
320
317
321
accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars))
318
322
eg := new(errgroup.Group)
···
338
342
return eg.Wait()
339
343
}
340
344
341
-
func genPosts(cctx *cli.Context) error {
342
-
catalog, err := fakedata.ReadAccountCatalog(cctx.String("catalog"))
345
+
func genPosts(ctx context.Context, cmd *cli.Command) error {
346
+
catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog"))
343
347
if err != nil {
344
348
return err
345
349
}
346
350
347
-
pdsHost := cctx.String("pds-host")
348
-
maxPosts := cctx.Int("max-posts")
349
-
fracImage := cctx.Float64("frac-image")
350
-
fracMention := cctx.Float64("frac-mention")
351
-
jobs := cctx.Int("jobs")
351
+
pdsHost := cmd.String("pds-host")
352
+
maxPosts := cmd.Int("max-posts")
353
+
fracImage := cmd.Float64("frac-image")
354
+
fracMention := cmd.Float64("frac-mention")
355
+
jobs := cmd.Int("jobs")
352
356
353
357
accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars))
354
358
eg := new(errgroup.Group)
···
374
378
return eg.Wait()
375
379
}
376
380
377
-
func genInteractions(cctx *cli.Context) error {
378
-
catalog, err := fakedata.ReadAccountCatalog(cctx.String("catalog"))
381
+
func genInteractions(ctx context.Context, cmd *cli.Command) error {
382
+
catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog"))
379
383
if err != nil {
380
384
return err
381
385
}
382
386
383
-
pdsHost := cctx.String("pds-host")
384
-
fracLike := cctx.Float64("frac-like")
385
-
fracRepost := cctx.Float64("frac-repost")
386
-
fracReply := cctx.Float64("frac-reply")
387
-
jobs := cctx.Int("jobs")
387
+
pdsHost := cmd.String("pds-host")
388
+
fracLike := cmd.Float64("frac-like")
389
+
fracRepost := cmd.Float64("frac-repost")
390
+
fracReply := cmd.Float64("frac-reply")
391
+
jobs := cmd.Int("jobs")
388
392
389
393
accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars))
390
394
eg := new(errgroup.Group)
···
412
416
return eg.Wait()
413
417
}
414
418
415
-
func runBrowsing(cctx *cli.Context) error {
416
-
catalog, err := fakedata.ReadAccountCatalog(cctx.String("catalog"))
419
+
func runBrowsing(ctx context.Context, cmd *cli.Command) error {
420
+
catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog"))
417
421
if err != nil {
418
422
return err
419
423
}
420
424
421
-
pdsHost := cctx.String("pds-host")
422
-
jobs := cctx.Int("jobs")
425
+
pdsHost := cmd.String("pds-host")
426
+
jobs := cmd.Int("jobs")
423
427
424
428
accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars))
425
429
eg := new(errgroup.Group)
···
444
448
close(accChan)
445
449
return eg.Wait()
446
450
}
451
+
452
+
func getXrpcClient(cmd *cli.Command, authreq bool) (*xrpc.Client, error) {
453
+
h := "http://localhost:4989"
454
+
if pdsurl := cmd.String("pds-host"); pdsurl != "" {
455
+
h = pdsurl
456
+
}
457
+
458
+
auth, err := loadAuthFromEnv(cmd, authreq)
459
+
if err != nil {
460
+
return nil, fmt.Errorf("loading auth: %w", err)
461
+
}
462
+
463
+
return &xrpc.Client{
464
+
Client: cliutil.NewHttpClient(),
465
+
Host: h,
466
+
Auth: auth,
467
+
}, nil
468
+
}
469
+
470
+
func loadAuthFromEnv(cmd *cli.Command, req bool) (*xrpc.AuthInfo, error) {
471
+
if a := cmd.String("auth"); a != "" {
472
+
if ai, err := cliutil.ReadAuth(a); err != nil && req {
473
+
return nil, err
474
+
} else {
475
+
return ai, nil
476
+
}
477
+
}
478
+
479
+
val := os.Getenv("ATP_AUTH_FILE")
480
+
if val == "" {
481
+
if req {
482
+
return nil, fmt.Errorf("no auth env present, ATP_AUTH_FILE not set")
483
+
}
484
+
485
+
return nil, nil
486
+
}
487
+
488
+
var auth xrpc.AuthInfo
489
+
if err := json.Unmarshal([]byte(val), &auth); err != nil {
490
+
return nil, err
491
+
}
492
+
493
+
return &auth, nil
494
+
}
+1
-1
cmd/gosky/account.go
+1
-1
cmd/gosky/account.go
+2
-1
cmd/gosky/admin.go
+2
-1
cmd/gosky/admin.go
+2
-1
cmd/gosky/bgs.go
+2
-1
cmd/gosky/bgs.go
+1
-1
cmd/gosky/bsky.go
+1
-1
cmd/gosky/bsky.go
+1
-1
cmd/gosky/car.go
+1
-1
cmd/gosky/car.go
+1
-1
cmd/gosky/debug.go
+1
-1
cmd/gosky/debug.go
+1
-1
cmd/gosky/did.go
+1
-1
cmd/gosky/did.go
+1
-1
cmd/gosky/handle.go
+1
-1
cmd/gosky/handle.go
+5
-6
cmd/gosky/main.go
+5
-6
cmd/gosky/main.go
···
15
15
"syscall"
16
16
"time"
17
17
18
+
_ "github.com/joho/godotenv/autoload"
19
+
18
20
"github.com/bluesky-social/indigo/api/atproto"
19
21
comatproto "github.com/bluesky-social/indigo/api/atproto"
20
22
"github.com/bluesky-social/indigo/api/bsky"
···
28
30
"github.com/bluesky-social/indigo/util"
29
31
"github.com/bluesky-social/indigo/util/cliutil"
30
32
"github.com/bluesky-social/indigo/xrpc"
31
-
"golang.org/x/time/rate"
32
33
34
+
"github.com/earthboundkid/versioninfo/v2"
33
35
"github.com/gorilla/websocket"
34
36
lru "github.com/hashicorp/golang-lru/v2"
35
37
"github.com/ipfs/go-cid"
36
38
"github.com/ipfs/go-datastore"
37
39
blockstore "github.com/ipfs/go-ipfs-blockstore"
38
40
"github.com/ipld/go-car"
39
-
40
-
_ "github.com/joho/godotenv/autoload"
41
-
42
-
"github.com/carlmjohnson/versioninfo"
43
41
"github.com/polydawn/refmt/cbor"
44
42
rejson "github.com/polydawn/refmt/json"
45
43
"github.com/polydawn/refmt/shared"
46
-
cli "github.com/urfave/cli/v2"
44
+
"github.com/urfave/cli/v2"
45
+
"golang.org/x/time/rate"
47
46
)
48
47
49
48
var log = slog.Default().With("system", "gosky")
+2
-1
cmd/gosky/streamdiff.go
+2
-1
cmd/gosky/streamdiff.go
···
8
8
comatproto "github.com/bluesky-social/indigo/api/atproto"
9
9
"github.com/bluesky-social/indigo/events"
10
10
"github.com/bluesky-social/indigo/events/schedulers/sequential"
11
+
11
12
"github.com/gorilla/websocket"
12
-
cli "github.com/urfave/cli/v2"
13
+
"github.com/urfave/cli/v2"
13
14
)
14
15
15
16
// TODO: WIP - turns out to be more complicated than i initially thought
+102
-105
cmd/hepa/main.go
+102
-105
cmd/hepa/main.go
···
7
7
"io"
8
8
"log/slog"
9
9
"net/http"
10
-
_ "net/http/pprof"
11
10
"os"
12
11
"runtime"
13
12
"strings"
14
13
"time"
14
+
15
+
_ "github.com/joho/godotenv/autoload"
16
+
_ "net/http/pprof"
15
17
16
18
"github.com/bluesky-social/indigo/atproto/identity"
17
19
"github.com/bluesky-social/indigo/atproto/identity/redisdir"
···
19
21
"github.com/bluesky-social/indigo/automod/capture"
20
22
"github.com/bluesky-social/indigo/automod/consumer"
21
23
22
-
"github.com/carlmjohnson/versioninfo"
23
-
_ "github.com/joho/godotenv/autoload"
24
-
cli "github.com/urfave/cli/v2"
24
+
"github.com/earthboundkid/versioninfo/v2"
25
+
"github.com/urfave/cli/v3"
25
26
"golang.org/x/time/rate"
26
27
)
27
28
···
34
35
35
36
func run(args []string) error {
36
37
37
-
app := cli.App{
38
+
app := cli.Command{
38
39
Name: "hepa",
39
40
Usage: "automod daemon (cleans the atmosphere)",
40
41
Version: versioninfo.Short(),
···
45
46
Name: "atp-relay-host",
46
47
Usage: "hostname and port of Relay to subscribe to",
47
48
Value: "wss://bsky.network",
48
-
EnvVars: []string{"ATP_RELAY_HOST", "ATP_BGS_HOST"},
49
+
Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"),
49
50
},
50
51
&cli.StringFlag{
51
52
Name: "atp-plc-host",
52
53
Usage: "method, hostname, and port of PLC registry",
53
54
Value: "https://plc.directory",
54
-
EnvVars: []string{"ATP_PLC_HOST"},
55
+
Sources: cli.EnvVars("ATP_PLC_HOST"),
55
56
},
56
57
&cli.StringFlag{
57
58
Name: "atp-bsky-host",
58
59
Usage: "method, hostname, and port of bsky API (appview) service. does not use auth",
59
60
Value: "https://public.api.bsky.app",
60
-
EnvVars: []string{"ATP_BSKY_HOST"},
61
+
Sources: cli.EnvVars("ATP_BSKY_HOST"),
61
62
},
62
63
&cli.StringFlag{
63
64
Name: "atp-ozone-host",
64
65
Usage: "method, hostname, and port of ozone instance. requires ozone-admin-token as well",
65
66
Value: "https://mod.bsky.app",
66
-
EnvVars: []string{"ATP_OZONE_HOST", "ATP_MOD_HOST"},
67
+
Sources: cli.EnvVars("ATP_OZONE_HOST", "ATP_MOD_HOST"),
67
68
},
68
69
&cli.StringFlag{
69
70
Name: "ozone-did",
70
71
Usage: "DID of account to attribute ozone actions to",
71
-
EnvVars: []string{"HEPA_OZONE_DID"},
72
+
Sources: cli.EnvVars("HEPA_OZONE_DID"),
72
73
},
73
74
&cli.StringFlag{
74
75
Name: "ozone-admin-token",
75
76
Usage: "admin authentication password for mod service",
76
-
EnvVars: []string{"HEPA_OZONE_AUTH_ADMIN_TOKEN", "HEPA_MOD_AUTH_ADMIN_TOKEN"},
77
+
Sources: cli.EnvVars("HEPA_OZONE_AUTH_ADMIN_TOKEN", "HEPA_MOD_AUTH_ADMIN_TOKEN"),
77
78
},
78
79
&cli.StringFlag{
79
80
Name: "atp-pds-host",
80
81
Usage: "method, hostname, and port of PDS (or entryway) for admin account info; uses admin auth",
81
82
Value: "https://bsky.social",
82
-
EnvVars: []string{"ATP_PDS_HOST"},
83
+
Sources: cli.EnvVars("ATP_PDS_HOST"),
83
84
},
84
85
&cli.StringFlag{
85
86
Name: "pds-admin-token",
86
87
Usage: "admin authentication password for PDS (or entryway)",
87
-
EnvVars: []string{"HEPA_PDS_AUTH_ADMIN_TOKEN"},
88
+
Sources: cli.EnvVars("HEPA_PDS_AUTH_ADMIN_TOKEN"),
88
89
},
89
90
&cli.StringFlag{
90
91
Name: "redis-url",
91
92
Usage: "redis connection URL",
92
93
// redis://<user>:<pass>@localhost:6379/<db>
93
94
// redis://localhost:6379/0
94
-
EnvVars: []string{"HEPA_REDIS_URL"},
95
+
Sources: cli.EnvVars("HEPA_REDIS_URL"),
95
96
},
96
97
&cli.IntFlag{
97
98
Name: "plc-rate-limit",
98
99
Usage: "max number of requests per second to PLC registry",
99
100
Value: 100,
100
-
EnvVars: []string{"HEPA_PLC_RATE_LIMIT"},
101
+
Sources: cli.EnvVars("HEPA_PLC_RATE_LIMIT"),
101
102
},
102
103
&cli.StringFlag{
103
104
Name: "sets-json-path",
104
105
Usage: "file path of JSON file containing static sets",
105
-
EnvVars: []string{"HEPA_SETS_JSON_PATH"},
106
+
Sources: cli.EnvVars("HEPA_SETS_JSON_PATH"),
106
107
},
107
108
&cli.StringFlag{
108
109
Name: "hiveai-api-token",
109
110
Usage: "API token for Hive AI image auto-labeling",
110
-
EnvVars: []string{"HIVEAI_API_TOKEN"},
111
+
Sources: cli.EnvVars("HIVEAI_API_TOKEN"),
111
112
},
112
113
&cli.StringFlag{
113
114
Name: "abyss-host",
114
115
Usage: "host for abusive image scanning API (scheme, host, port)",
115
-
EnvVars: []string{"ABYSS_HOST"},
116
+
Sources: cli.EnvVars("ABYSS_HOST"),
116
117
},
117
118
&cli.StringFlag{
118
119
Name: "abyss-password",
119
120
Usage: "admin auth password for abyss API",
120
-
EnvVars: []string{"ABYSS_PASSWORD"},
121
+
Sources: cli.EnvVars("ABYSS_PASSWORD"),
121
122
},
122
123
&cli.StringFlag{
123
124
Name: "ruleset",
124
125
Usage: "which ruleset config to use: default, no-blobs, only-blobs",
125
-
EnvVars: []string{"HEPA_RULESET"},
126
+
Sources: cli.EnvVars("HEPA_RULESET"),
126
127
},
127
128
&cli.StringFlag{
128
129
Name: "log-level",
129
130
Usage: "log verbosity level (eg: warn, info, debug)",
130
-
EnvVars: []string{"HEPA_LOG_LEVEL", "LOG_LEVEL"},
131
+
Sources: cli.EnvVars("HEPA_LOG_LEVEL", "LOG_LEVEL"),
131
132
},
132
133
&cli.StringFlag{
133
134
Name: "ratelimit-bypass",
134
135
Usage: "HTTP header to bypass ratelimits",
135
-
EnvVars: []string{"HEPA_RATELIMIT_BYPASS", "RATELIMIT_BYPASS"},
136
+
Sources: cli.EnvVars("HEPA_RATELIMIT_BYPASS", "RATELIMIT_BYPASS"),
136
137
},
137
138
&cli.IntFlag{
138
139
Name: "firehose-parallelism",
139
140
Usage: "force a fixed number of parallel firehose workers. default (or 0) for auto-scaling; 200 works for a large instance",
140
-
EnvVars: []string{"HEPA_FIREHOSE_PARALLELISM"},
141
+
Sources: cli.EnvVars("HEPA_FIREHOSE_PARALLELISM"),
141
142
},
142
143
&cli.StringFlag{
143
144
Name: "prescreen-host",
144
145
Usage: "hostname of prescreen server",
145
-
EnvVars: []string{"HEPA_PRESCREEN_HOST"},
146
+
Sources: cli.EnvVars("HEPA_PRESCREEN_HOST"),
146
147
},
147
148
&cli.StringFlag{
148
149
Name: "prescreen-token",
149
150
Usage: "secret token for prescreen server",
150
-
EnvVars: []string{"HEPA_PRESCREEN_TOKEN"},
151
+
Sources: cli.EnvVars("HEPA_PRESCREEN_TOKEN"),
151
152
},
152
153
&cli.DurationFlag{
153
154
Name: "report-dupe-period",
154
155
Usage: "time period within which automod will not re-report an account for the same reasonType",
155
-
EnvVars: []string{"HEPA_REPORT_DUPE_PERIOD"},
156
+
Sources: cli.EnvVars("HEPA_REPORT_DUPE_PERIOD"),
156
157
Value: 1 * 24 * time.Hour,
157
158
},
158
159
&cli.IntFlag{
159
160
Name: "quota-mod-report-day",
160
161
Usage: "number of reports automod can file per day, for all subjects and types combined (circuit breaker)",
161
-
EnvVars: []string{"HEPA_QUOTA_MOD_REPORT_DAY"},
162
+
Sources: cli.EnvVars("HEPA_QUOTA_MOD_REPORT_DAY"),
162
163
Value: 10000,
163
164
},
164
165
&cli.IntFlag{
165
166
Name: "quota-mod-takedown-day",
166
167
Usage: "number of takedowns automod can action per day, for all subjects combined (circuit breaker)",
167
-
EnvVars: []string{"HEPA_QUOTA_MOD_TAKEDOWN_DAY"},
168
+
Sources: cli.EnvVars("HEPA_QUOTA_MOD_TAKEDOWN_DAY"),
168
169
Value: 200,
169
170
},
170
171
&cli.IntFlag{
171
172
Name: "quota-mod-action-day",
172
173
Usage: "number of misc actions automod can do per day, for all subjects combined (circuit breaker)",
173
-
EnvVars: []string{"HEPA_QUOTA_MOD_ACTION_DAY"},
174
+
Sources: cli.EnvVars("HEPA_QUOTA_MOD_ACTION_DAY"),
174
175
Value: 2000,
175
176
},
176
177
&cli.DurationFlag{
177
178
Name: "record-event-timeout",
178
179
Usage: "total processing time for record events (including setup, rules, and persisting)",
179
-
EnvVars: []string{"HEPA_RECORD_EVENT_TIMEOUT"},
180
+
Sources: cli.EnvVars("HEPA_RECORD_EVENT_TIMEOUT"),
180
181
Value: 30 * time.Second,
181
182
},
182
183
&cli.DurationFlag{
183
184
Name: "identity-event-timeout",
184
185
Usage: "total processing time for identity and account events (including setup, rules, and persisting)",
185
-
EnvVars: []string{"HEPA_IDENTITY_EVENT_TIMEOUT"},
186
+
Sources: cli.EnvVars("HEPA_IDENTITY_EVENT_TIMEOUT"),
186
187
Value: 10 * time.Second,
187
188
},
188
189
&cli.DurationFlag{
189
190
Name: "ozone-event-timeout",
190
191
Usage: "total processing time for ozone events (including setup, rules, and persisting)",
191
-
EnvVars: []string{"HEPA_OZONE_EVENT_TIMEOUT"},
192
+
Sources: cli.EnvVars("HEPA_OZONE_EVENT_TIMEOUT"),
192
193
Value: 30 * time.Second,
193
194
},
194
195
}
···
200
201
captureRecentCmd,
201
202
}
202
203
203
-
return app.Run(args)
204
+
return app.Run(context.Background(), args)
204
205
}
205
206
206
-
func configDirectory(cctx *cli.Context) (identity.Directory, error) {
207
+
func configDirectory(cmd *cli.Command) (identity.Directory, error) {
207
208
baseDir := identity.BaseDirectory{
208
-
PLCURL: cctx.String("atp-plc-host"),
209
+
PLCURL: cmd.String("atp-plc-host"),
209
210
HTTPClient: http.Client{
210
211
Timeout: time.Second * 15,
211
212
},
212
-
PLCLimiter: rate.NewLimiter(rate.Limit(cctx.Int("plc-rate-limit")), 1),
213
+
PLCLimiter: rate.NewLimiter(rate.Limit(cmd.Int("plc-rate-limit")), 1),
213
214
TryAuthoritativeDNS: true,
214
215
SkipDNSDomainSuffixes: []string{".bsky.social", ".staging.bsky.dev"},
215
216
}
216
217
var dir identity.Directory
217
-
if cctx.String("redis-url") != "" {
218
-
rdir, err := redisdir.NewRedisDirectory(&baseDir, cctx.String("redis-url"), time.Hour*24, time.Minute*2, time.Minute*5, 10_000)
218
+
if cmd.String("redis-url") != "" {
219
+
rdir, err := redisdir.NewRedisDirectory(&baseDir, cmd.String("redis-url"), time.Hour*24, time.Minute*2, time.Minute*5, 10_000)
219
220
if err != nil {
220
221
return nil, err
221
222
}
···
227
228
return dir, nil
228
229
}
229
230
230
-
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
231
+
func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger {
231
232
var level slog.Level
232
-
switch strings.ToLower(cctx.String("log-level")) {
233
+
switch strings.ToLower(cmd.String("log-level")) {
233
234
case "error":
234
235
level = slog.LevelError
235
236
case "warn":
···
256
257
Name: "metrics-listen",
257
258
Usage: "IP or address, and port, to listen on for metrics APIs",
258
259
Value: ":3989",
259
-
EnvVars: []string{"HEPA_METRICS_LISTEN"},
260
+
Sources: cli.EnvVars("HEPA_METRICS_LISTEN"),
260
261
},
261
262
&cli.StringFlag{
262
263
Name: "slack-webhook-url",
263
264
// eg: https://hooks.slack.com/services/X1234
264
265
Usage: "full URL of slack webhook",
265
-
EnvVars: []string{"SLACK_WEBHOOK_URL"},
266
+
Sources: cli.EnvVars("SLACK_WEBHOOK_URL"),
266
267
},
267
268
},
268
-
Action: func(cctx *cli.Context) error {
269
-
ctx := context.Background()
270
-
logger := configLogger(cctx, os.Stdout)
269
+
Action: func(ctx context.Context, cmd *cli.Command) error {
270
+
logger := configLogger(cmd, os.Stdout)
271
271
configOTEL("hepa")
272
272
273
-
dir, err := configDirectory(cctx)
273
+
dir, err := configDirectory(cmd)
274
274
if err != nil {
275
275
return fmt.Errorf("failed to configure identity directory: %v", err)
276
276
}
···
279
279
dir,
280
280
Config{
281
281
Logger: logger,
282
-
BskyHost: cctx.String("atp-bsky-host"),
283
-
OzoneHost: cctx.String("atp-ozone-host"),
284
-
OzoneDID: cctx.String("ozone-did"),
285
-
OzoneAdminToken: cctx.String("ozone-admin-token"),
286
-
PDSHost: cctx.String("atp-pds-host"),
287
-
PDSAdminToken: cctx.String("pds-admin-token"),
288
-
SetsFileJSON: cctx.String("sets-json-path"),
289
-
RedisURL: cctx.String("redis-url"),
290
-
SlackWebhookURL: cctx.String("slack-webhook-url"),
291
-
HiveAPIToken: cctx.String("hiveai-api-token"),
292
-
AbyssHost: cctx.String("abyss-host"),
293
-
AbyssPassword: cctx.String("abyss-password"),
294
-
RatelimitBypass: cctx.String("ratelimit-bypass"),
295
-
RulesetName: cctx.String("ruleset"),
296
-
PreScreenHost: cctx.String("prescreen-host"),
297
-
PreScreenToken: cctx.String("prescreen-token"),
298
-
ReportDupePeriod: cctx.Duration("report-dupe-period"),
299
-
QuotaModReportDay: cctx.Int("quota-mod-report-day"),
300
-
QuotaModTakedownDay: cctx.Int("quota-mod-takedown-day"),
301
-
QuotaModActionDay: cctx.Int("quota-mod-action-day"),
302
-
RecordEventTimeout: cctx.Duration("record-event-timeout"),
303
-
IdentityEventTimeout: cctx.Duration("identity-event-timeout"),
304
-
OzoneEventTimeout: cctx.Duration("ozone-event-timeout"),
282
+
BskyHost: cmd.String("atp-bsky-host"),
283
+
OzoneHost: cmd.String("atp-ozone-host"),
284
+
OzoneDID: cmd.String("ozone-did"),
285
+
OzoneAdminToken: cmd.String("ozone-admin-token"),
286
+
PDSHost: cmd.String("atp-pds-host"),
287
+
PDSAdminToken: cmd.String("pds-admin-token"),
288
+
SetsFileJSON: cmd.String("sets-json-path"),
289
+
RedisURL: cmd.String("redis-url"),
290
+
SlackWebhookURL: cmd.String("slack-webhook-url"),
291
+
HiveAPIToken: cmd.String("hiveai-api-token"),
292
+
AbyssHost: cmd.String("abyss-host"),
293
+
AbyssPassword: cmd.String("abyss-password"),
294
+
RatelimitBypass: cmd.String("ratelimit-bypass"),
295
+
RulesetName: cmd.String("ruleset"),
296
+
PreScreenHost: cmd.String("prescreen-host"),
297
+
PreScreenToken: cmd.String("prescreen-token"),
298
+
ReportDupePeriod: cmd.Duration("report-dupe-period"),
299
+
QuotaModReportDay: cmd.Int("quota-mod-report-day"),
300
+
QuotaModTakedownDay: cmd.Int("quota-mod-takedown-day"),
301
+
QuotaModActionDay: cmd.Int("quota-mod-action-day"),
302
+
RecordEventTimeout: cmd.Duration("record-event-timeout"),
303
+
IdentityEventTimeout: cmd.Duration("identity-event-timeout"),
304
+
OzoneEventTimeout: cmd.Duration("ozone-event-timeout"),
305
305
},
306
306
)
307
307
if err != nil {
···
334
334
go func() {
335
335
runtime.SetBlockProfileRate(10)
336
336
runtime.SetMutexProfileFraction(10)
337
-
if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil {
337
+
if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil {
338
338
slog.Error("failed to start metrics endpoint", "error", err)
339
339
panic(fmt.Errorf("failed to start metrics endpoint: %w", err))
340
340
}
341
341
}()
342
342
343
343
// firehose event consumer (note this is actually mandatory)
344
-
relayHost := cctx.String("atp-relay-host")
344
+
relayHost := cmd.String("atp-relay-host")
345
345
if relayHost != "" {
346
346
fc := consumer.FirehoseConsumer{
347
347
Engine: srv.Engine,
348
348
Logger: logger.With("subsystem", "firehose-consumer"),
349
-
Host: cctx.String("atp-relay-host"),
350
-
Parallelism: cctx.Int("firehose-parallelism"),
349
+
Host: cmd.String("atp-relay-host"),
350
+
Parallelism: cmd.Int("firehose-parallelism"),
351
351
RedisClient: srv.RedisClient,
352
352
}
353
353
···
367
367
}
368
368
369
369
// for simple commands, not long-running daemons
370
-
func configEphemeralServer(cctx *cli.Context) (*Server, error) {
370
+
func configEphemeralServer(cmd *cli.Command) (*Server, error) {
371
371
// NOTE: using stderr not stdout because some commands print to stdout
372
-
logger := configLogger(cctx, os.Stderr)
372
+
logger := configLogger(cmd, os.Stderr)
373
373
374
-
dir, err := configDirectory(cctx)
374
+
dir, err := configDirectory(cmd)
375
375
if err != nil {
376
376
return nil, err
377
377
}
···
380
380
dir,
381
381
Config{
382
382
Logger: logger,
383
-
BskyHost: cctx.String("atp-bsky-host"),
384
-
OzoneHost: cctx.String("atp-ozone-host"),
385
-
OzoneDID: cctx.String("ozone-did"),
386
-
OzoneAdminToken: cctx.String("ozone-admin-token"),
387
-
PDSHost: cctx.String("atp-pds-host"),
388
-
PDSAdminToken: cctx.String("pds-admin-token"),
389
-
SetsFileJSON: cctx.String("sets-json-path"),
390
-
RedisURL: cctx.String("redis-url"),
391
-
HiveAPIToken: cctx.String("hiveai-api-token"),
392
-
AbyssHost: cctx.String("abyss-host"),
393
-
AbyssPassword: cctx.String("abyss-password"),
394
-
RatelimitBypass: cctx.String("ratelimit-bypass"),
395
-
RulesetName: cctx.String("ruleset"),
396
-
PreScreenHost: cctx.String("prescreen-host"),
397
-
PreScreenToken: cctx.String("prescreen-token"),
383
+
BskyHost: cmd.String("atp-bsky-host"),
384
+
OzoneHost: cmd.String("atp-ozone-host"),
385
+
OzoneDID: cmd.String("ozone-did"),
386
+
OzoneAdminToken: cmd.String("ozone-admin-token"),
387
+
PDSHost: cmd.String("atp-pds-host"),
388
+
PDSAdminToken: cmd.String("pds-admin-token"),
389
+
SetsFileJSON: cmd.String("sets-json-path"),
390
+
RedisURL: cmd.String("redis-url"),
391
+
HiveAPIToken: cmd.String("hiveai-api-token"),
392
+
AbyssHost: cmd.String("abyss-host"),
393
+
AbyssPassword: cmd.String("abyss-password"),
394
+
RatelimitBypass: cmd.String("ratelimit-bypass"),
395
+
RulesetName: cmd.String("ruleset"),
396
+
PreScreenHost: cmd.String("prescreen-host"),
397
+
PreScreenToken: cmd.String("prescreen-token"),
398
398
},
399
399
)
400
400
}
···
404
404
Usage: "process a single record in isolation",
405
405
ArgsUsage: `<at-uri>`,
406
406
Flags: []cli.Flag{},
407
-
Action: func(cctx *cli.Context) error {
408
-
ctx := context.Background()
409
-
uriArg := cctx.Args().First()
407
+
Action: func(ctx context.Context, cmd *cli.Command) error {
408
+
uriArg := cmd.Args().First()
410
409
if uriArg == "" {
411
410
return fmt.Errorf("expected a single AT-URI argument")
412
411
}
···
415
414
return fmt.Errorf("not a valid AT-URI: %v", err)
416
415
}
417
416
418
-
srv, err := configEphemeralServer(cctx)
417
+
srv, err := configEphemeralServer(cmd)
419
418
if err != nil {
420
419
return err
421
420
}
···
435
434
Value: 20,
436
435
},
437
436
},
438
-
Action: func(cctx *cli.Context) error {
439
-
ctx := context.Background()
440
-
idArg := cctx.Args().First()
437
+
Action: func(ctx context.Context, cmd *cli.Command) error {
438
+
idArg := cmd.Args().First()
441
439
if idArg == "" {
442
440
return fmt.Errorf("expected a single AT identifier (handle or DID) argument")
443
441
}
···
446
444
return fmt.Errorf("not a valid handle or DID: %v", err)
447
445
}
448
446
449
-
srv, err := configEphemeralServer(cctx)
447
+
srv, err := configEphemeralServer(cmd)
450
448
if err != nil {
451
449
return err
452
450
}
453
451
454
-
return capture.FetchAndProcessRecent(ctx, srv.Engine, *atid, cctx.Int("limit"))
452
+
return capture.FetchAndProcessRecent(ctx, srv.Engine, *atid, cmd.Int("limit"))
455
453
},
456
454
}
457
455
···
466
464
Value: 20,
467
465
},
468
466
},
469
-
Action: func(cctx *cli.Context) error {
470
-
ctx := context.Background()
471
-
idArg := cctx.Args().First()
467
+
Action: func(ctx context.Context, cmd *cli.Command) error {
468
+
idArg := cmd.Args().First()
472
469
if idArg == "" {
473
470
return fmt.Errorf("expected a single AT identifier (handle or DID) argument")
474
471
}
···
477
474
return fmt.Errorf("not a valid handle or DID: %v", err)
478
475
}
479
476
480
-
srv, err := configEphemeralServer(cctx)
477
+
srv, err := configEphemeralServer(cmd)
481
478
if err != nil {
482
479
return err
483
480
}
484
481
485
-
cap, err := capture.CaptureRecent(ctx, srv.Engine, *atid, cctx.Int("limit"))
482
+
cap, err := capture.CaptureRecent(ctx, srv.Engine, *atid, cmd.Int("limit"))
486
483
if err != nil {
487
484
return err
488
485
}
+2
-2
cmd/lexgen/main.go
+2
-2
cmd/lexgen/main.go
+21
-23
cmd/netsync/main.go
+21
-23
cmd/netsync/main.go
···
19
19
"syscall"
20
20
"time"
21
21
22
+
_ "github.com/joho/godotenv/autoload"
23
+
22
24
"github.com/bluesky-social/indigo/atproto/atdata"
23
25
"github.com/bluesky-social/indigo/repo"
26
+
27
+
"github.com/earthboundkid/versioninfo/v2"
24
28
"github.com/ipfs/go-cid"
25
-
_ "github.com/joho/godotenv/autoload"
26
29
"github.com/prometheus/client_golang/prometheus"
27
30
"github.com/prometheus/client_golang/prometheus/promauto"
28
31
"github.com/prometheus/client_golang/prometheus/promhttp"
29
-
32
+
"github.com/urfave/cli/v3"
30
33
"golang.org/x/time/rate"
31
-
32
-
"github.com/carlmjohnson/versioninfo"
33
-
"github.com/urfave/cli/v2"
34
34
)
35
35
36
36
func main() {
37
-
app := cli.App{
37
+
app := cli.Command{
38
38
Name: "netsync",
39
39
Usage: "atproto network cloning tool",
40
40
Version: versioninfo.Short(),
···
80
80
Name: "magic-header-key",
81
81
Usage: "header key to send with checkout request",
82
82
Value: "",
83
-
EnvVars: []string{"MAGIC_HEADER_KEY"},
83
+
Sources: cli.EnvVars("MAGIC_HEADER_KEY"),
84
84
},
85
85
&cli.StringFlag{
86
86
Name: "magic-header-val",
87
87
Usage: "header value to send with checkout request",
88
88
Value: "",
89
-
EnvVars: []string{"MAGIC_HEADER_VAL"},
89
+
Sources: cli.EnvVars("MAGIC_HEADER_VAL"),
90
90
},
91
91
}
92
92
···
94
94
{
95
95
Name: "retry",
96
96
Usage: "requeue failed repos",
97
-
Action: func(cctx *cli.Context) error {
97
+
Action: func(ctx context.Context, cmd *cli.Command) error {
98
98
state := &NetsyncState{
99
-
StatePath: cctx.String("state-file"),
99
+
StatePath: cmd.String("state-file"),
100
100
}
101
101
102
102
err := state.Resume()
···
123
123
124
124
app.Action = Netsync
125
125
126
-
err := app.Run(os.Args)
127
-
if err != nil {
126
+
if err := app.Run(context.Background(), os.Args); err != nil {
128
127
log.Fatal(err)
129
128
}
130
129
}
···
260
259
delete(s.EnqueuedRepos, repo)
261
260
}
262
261
263
-
func Netsync(cctx *cli.Context) error {
264
-
ctx := cctx.Context
262
+
func Netsync(ctx context.Context, cmd *cli.Command) error {
265
263
ctx, cancel := context.WithCancel(ctx)
266
264
defer cancel()
267
265
···
270
268
slog.SetDefault(slog.New(logger.Handler()))
271
269
272
270
state := &NetsyncState{
273
-
StatePath: cctx.String("state-file"),
274
-
CheckoutPath: cctx.String("checkout-path"),
271
+
StatePath: cmd.String("state-file"),
272
+
CheckoutPath: cmd.String("checkout-path"),
275
273
276
-
outDir: cctx.String("out-dir"),
277
-
workerCount: cctx.Int("worker-count"),
278
-
limiter: rate.NewLimiter(rate.Limit(cctx.Float64("checkout-limit")), 1),
279
-
magicHeaderKey: cctx.String("magic-header-key"),
280
-
magicHeaderVal: cctx.String("magic-header-val"),
274
+
outDir: cmd.String("out-dir"),
275
+
workerCount: cmd.Int("worker-count"),
276
+
limiter: rate.NewLimiter(rate.Limit(cmd.Float64("checkout-limit")), 1),
277
+
magicHeaderKey: cmd.String("magic-header-key"),
278
+
magicHeaderVal: cmd.String("magic-header-val"),
281
279
282
280
exit: make(chan struct{}),
283
281
wg: sync.WaitGroup{},
···
317
315
318
316
if err != nil {
319
317
// Read repo list
320
-
repoListFile, err := os.Open(cctx.String("repo-list"))
318
+
repoListFile, err := os.Open(cmd.String("repo-list"))
321
319
if err != nil {
322
320
return err
323
321
}
···
341
339
mux.Handle("/metrics", promhttp.Handler())
342
340
343
341
metricsServer := &http.Server{
344
-
Addr: fmt.Sprintf(":%d", cctx.Int("port")),
342
+
Addr: fmt.Sprintf(":%d", cmd.Int("port")),
345
343
Handler: mux,
346
344
}
347
345
+77
-77
cmd/palomar/main.go
+77
-77
cmd/palomar/main.go
···
13
13
"time"
14
14
15
15
_ "github.com/joho/godotenv/autoload"
16
+
17
+
"github.com/bluesky-social/indigo/atproto/identity"
18
+
"github.com/bluesky-social/indigo/search"
19
+
"github.com/bluesky-social/indigo/util/cliutil"
20
+
21
+
"github.com/earthboundkid/versioninfo/v2"
22
+
es "github.com/opensearch-project/opensearch-go/v2"
23
+
"github.com/urfave/cli/v3"
16
24
"go.opentelemetry.io/otel"
17
25
"go.opentelemetry.io/otel/attribute"
18
26
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
···
20
28
tracesdk "go.opentelemetry.io/otel/sdk/trace"
21
29
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
22
30
"golang.org/x/time/rate"
23
-
24
-
"github.com/bluesky-social/indigo/atproto/identity"
25
-
"github.com/bluesky-social/indigo/search"
26
-
"github.com/bluesky-social/indigo/util/cliutil"
27
-
28
-
"github.com/carlmjohnson/versioninfo"
29
-
es "github.com/opensearch-project/opensearch-go/v2"
30
-
cli "github.com/urfave/cli/v2"
31
31
)
32
32
33
33
func main() {
···
39
39
40
40
func run(args []string) error {
41
41
42
-
app := cli.App{
42
+
app := cli.Command{
43
43
Name: "palomar",
44
44
Usage: "search indexing and query service (using ES or OS)",
45
45
Version: versioninfo.Short(),
···
49
49
&cli.StringFlag{
50
50
Name: "elastic-cert-file",
51
51
Usage: "certificate file path",
52
-
EnvVars: []string{"ES_CERT_FILE", "ELASTIC_CERT_FILE"},
52
+
Sources: cli.EnvVars("ES_CERT_FILE", "ELASTIC_CERT_FILE"),
53
53
},
54
54
&cli.BoolFlag{
55
55
Name: "elastic-insecure-ssl",
56
56
Usage: "if true, disable SSL cert validation",
57
-
EnvVars: []string{"ES_INSECURE_SSL"},
57
+
Sources: cli.EnvVars("ES_INSECURE_SSL"),
58
58
},
59
59
&cli.StringFlag{
60
60
Name: "elastic-username",
61
61
Usage: "elasticsearch username",
62
62
Value: "admin",
63
-
EnvVars: []string{"ES_USERNAME", "ELASTIC_USERNAME"},
63
+
Sources: cli.EnvVars("ES_USERNAME", "ELASTIC_USERNAME"),
64
64
},
65
65
&cli.StringFlag{
66
66
Name: "elastic-password",
67
67
Usage: "elasticsearch password",
68
68
Value: "0penSearch-Pal0mar",
69
-
EnvVars: []string{"ES_PASSWORD", "ELASTIC_PASSWORD"},
69
+
Sources: cli.EnvVars("ES_PASSWORD", "ELASTIC_PASSWORD"),
70
70
},
71
71
&cli.StringFlag{
72
72
Name: "elastic-hosts",
73
73
Usage: "elasticsearch hosts (schema/host/port)",
74
74
Value: "http://localhost:9200",
75
-
EnvVars: []string{"ES_HOSTS", "ELASTIC_HOSTS", "OPENSEARCH_URL", "ELASTICSEARCH_URL"},
75
+
Sources: cli.EnvVars("ES_HOSTS", "ELASTIC_HOSTS", "OPENSEARCH_URL", "ELASTICSEARCH_URL"),
76
76
},
77
77
&cli.StringFlag{
78
78
Name: "es-post-index",
79
79
Usage: "ES index for 'post' documents",
80
80
Value: "palomar_post",
81
-
EnvVars: []string{"ES_POST_INDEX"},
81
+
Sources: cli.EnvVars("ES_POST_INDEX"),
82
82
},
83
83
&cli.StringFlag{
84
84
Name: "es-profile-index",
85
85
Usage: "ES index for 'profile' documents",
86
86
Value: "palomar_profile",
87
-
EnvVars: []string{"ES_PROFILE_INDEX"},
87
+
Sources: cli.EnvVars("ES_PROFILE_INDEX"),
88
88
},
89
89
&cli.StringFlag{
90
90
Name: "atp-relay-host",
91
91
Usage: "hostname and port of Relay to subscribe to",
92
92
Value: "wss://bsky.network",
93
-
EnvVars: []string{"ATP_RELAY_HOST", "ATP_BGS_HOST"},
93
+
Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"),
94
94
},
95
95
&cli.StringFlag{
96
96
Name: "atp-plc-host",
97
97
Usage: "method, hostname, and port of PLC registry",
98
98
Value: "https://plc.directory",
99
-
EnvVars: []string{"ATP_PLC_HOST"},
99
+
Sources: cli.EnvVars("ATP_PLC_HOST"),
100
100
},
101
101
&cli.IntFlag{
102
102
Name: "max-metadb-connections",
103
-
EnvVars: []string{"MAX_METADB_CONNECTIONS"},
103
+
Sources: cli.EnvVars("MAX_METADB_CONNECTIONS"),
104
104
Value: 40,
105
105
},
106
106
&cli.StringFlag{
107
107
Name: "log-level",
108
108
Usage: "log level (debug, info, warn, error)",
109
109
Value: "info",
110
-
EnvVars: []string{"GOLOG_LOG_LEVEL", "LOG_LEVEL"},
110
+
Sources: cli.EnvVars("GOLOG_LOG_LEVEL", "LOG_LEVEL"),
111
111
},
112
112
}
113
113
···
118
118
searchProfileCmd,
119
119
}
120
120
121
-
return app.Run(args)
121
+
return app.Run(context.Background(), args)
122
122
}
123
123
124
124
var runCmd = &cli.Command{
···
128
128
&cli.StringFlag{
129
129
Name: "database-url",
130
130
Value: "sqlite://data/palomar/search.db",
131
-
EnvVars: []string{"DATABASE_URL"},
131
+
Sources: cli.EnvVars("DATABASE_URL"),
132
132
},
133
133
&cli.BoolFlag{
134
134
Name: "readonly",
135
-
EnvVars: []string{"PALOMAR_READONLY", "READONLY"},
135
+
Sources: cli.EnvVars("PALOMAR_READONLY", "READONLY"),
136
136
},
137
137
&cli.StringFlag{
138
138
Name: "bind",
139
139
Usage: "IP or address, and port, to listen on for HTTP APIs",
140
140
Value: ":3999",
141
-
EnvVars: []string{"PALOMAR_BIND"},
141
+
Sources: cli.EnvVars("PALOMAR_BIND"),
142
142
},
143
143
&cli.StringFlag{
144
144
Name: "metrics-listen",
145
145
Usage: "IP or address, and port, to listen on for metrics APIs",
146
146
Value: ":3998",
147
-
EnvVars: []string{"PALOMAR_METRICS_LISTEN"},
147
+
Sources: cli.EnvVars("PALOMAR_METRICS_LISTEN"),
148
148
},
149
149
&cli.IntFlag{
150
150
Name: "relay-sync-rate-limit",
151
151
Usage: "max repo sync (checkout) requests per second to upstream (Relay)",
152
152
Value: 8,
153
-
EnvVars: []string{"PALOMAR_RELAY_SYNC_RATE_LIMIT", "PALOMAR_BGS_SYNC_RATE_LIMIT"},
153
+
Sources: cli.EnvVars("PALOMAR_RELAY_SYNC_RATE_LIMIT", "PALOMAR_BGS_SYNC_RATE_LIMIT"),
154
154
},
155
155
&cli.IntFlag{
156
156
Name: "index-max-concurrency",
157
157
Usage: "max number of concurrent index requests (HTTP POST) to search index",
158
158
Value: 20,
159
-
EnvVars: []string{"PALOMAR_INDEX_MAX_CONCURRENCY"},
159
+
Sources: cli.EnvVars("PALOMAR_INDEX_MAX_CONCURRENCY"),
160
160
},
161
161
&cli.IntFlag{
162
162
Name: "indexing-rate-limit",
163
163
Usage: "max number of documents per second to index",
164
164
Value: 50_000,
165
-
EnvVars: []string{"PALOMAR_INDEXING_RATE_LIMIT"},
165
+
Sources: cli.EnvVars("PALOMAR_INDEXING_RATE_LIMIT"),
166
166
},
167
167
&cli.IntFlag{
168
168
Name: "plc-rate-limit",
169
169
Usage: "max number of requests per second to PLC registry",
170
170
Value: 100,
171
-
EnvVars: []string{"PALOMAR_PLC_RATE_LIMIT"},
171
+
Sources: cli.EnvVars("PALOMAR_PLC_RATE_LIMIT"),
172
172
},
173
173
&cli.BoolFlag{
174
174
Name: "discover-repos",
175
175
Usage: "if true, discover repositories from the Relay",
176
-
EnvVars: []string{"PALOMAR_DISCOVER_REPOS"},
176
+
Sources: cli.EnvVars("PALOMAR_DISCOVER_REPOS"),
177
177
Value: false,
178
178
},
179
179
&cli.StringFlag{
180
180
Name: "pagerank-file",
181
-
EnvVars: []string{"PAGERANK_FILE"},
181
+
Sources: cli.EnvVars("PAGERANK_FILE"),
182
182
},
183
183
&cli.StringFlag{
184
184
Name: "bulk-posts-file",
185
-
EnvVars: []string{"BULK_POSTS_FILE"},
185
+
Sources: cli.EnvVars("BULK_POSTS_FILE"),
186
186
},
187
187
&cli.StringFlag{
188
188
Name: "bulk-profiles-file",
189
-
EnvVars: []string{"BULK_PROFILES_FILE"},
189
+
Sources: cli.EnvVars("BULK_PROFILES_FILE"),
190
190
},
191
191
},
192
-
Action: func(cctx *cli.Context) error {
192
+
Action: func(ctx context.Context, cmd *cli.Command) error {
193
193
logLevel := slog.LevelInfo
194
-
switch cctx.String("log-level") {
194
+
switch cmd.String("log-level") {
195
195
case "debug":
196
196
logLevel = slog.LevelDebug
197
197
case "info":
···
208
208
}))
209
209
slog.SetDefault(logger)
210
210
211
-
readonly := cctx.Bool("readonly")
211
+
readonly := cmd.Bool("readonly")
212
212
213
213
// Enable OTLP HTTP exporter
214
214
// For relevant environment variables:
···
245
245
otel.SetTracerProvider(tp)
246
246
}
247
247
248
-
escli, err := createEsClient(cctx)
248
+
escli, err := createEsClient(cmd)
249
249
if err != nil {
250
250
return fmt.Errorf("failed to get elasticsearch: %w", err)
251
251
}
252
252
253
253
base := identity.BaseDirectory{
254
-
PLCURL: cctx.String("atp-plc-host"),
254
+
PLCURL: cmd.String("atp-plc-host"),
255
255
HTTPClient: http.Client{
256
256
Timeout: time.Second * 15,
257
257
},
258
-
PLCLimiter: rate.NewLimiter(rate.Limit(cctx.Int("plc-rate-limit")), 1),
258
+
PLCLimiter: rate.NewLimiter(rate.Limit(cmd.Int("plc-rate-limit")), 1),
259
259
TryAuthoritativeDNS: true,
260
260
SkipDNSDomainSuffixes: []string{".bsky.social"},
261
261
}
···
263
263
264
264
apiConfig := search.ServerConfig{
265
265
Logger: logger,
266
-
ProfileIndex: cctx.String("es-profile-index"),
267
-
PostIndex: cctx.String("es-post-index"),
266
+
ProfileIndex: cmd.String("es-profile-index"),
267
+
PostIndex: cmd.String("es-post-index"),
268
268
}
269
269
270
270
srv, err := search.NewServer(escli, &dir, apiConfig)
···
274
274
275
275
// Configure the indexer if we're not in readonly mode
276
276
if !readonly {
277
-
db, err := cliutil.SetupDatabase(cctx.String("database-url"), cctx.Int("max-metadb-connections"))
277
+
db, err := cliutil.SetupDatabase(cmd.String("database-url"), cmd.Int("max-metadb-connections"))
278
278
if err != nil {
279
279
return fmt.Errorf("failed to set up database: %w", err)
280
280
}
281
281
282
282
indexerConfig := search.IndexerConfig{
283
-
RelayHost: cctx.String("atp-relay-host"),
284
-
ProfileIndex: cctx.String("es-profile-index"),
285
-
PostIndex: cctx.String("es-post-index"),
283
+
RelayHost: cmd.String("atp-relay-host"),
284
+
ProfileIndex: cmd.String("es-profile-index"),
285
+
PostIndex: cmd.String("es-post-index"),
286
286
Logger: logger,
287
-
RelaySyncRateLimit: cctx.Int("relay-sync-rate-limit"),
288
-
IndexMaxConcurrency: cctx.Int("index-max-concurrency"),
289
-
DiscoverRepos: cctx.Bool("discover-repos"),
290
-
IndexingRateLimit: cctx.Int("indexing-rate-limit"),
287
+
RelaySyncRateLimit: cmd.Int("relay-sync-rate-limit"),
288
+
IndexMaxConcurrency: cmd.Int("index-max-concurrency"),
289
+
DiscoverRepos: cmd.Bool("discover-repos"),
290
+
IndexingRateLimit: cmd.Int("indexing-rate-limit"),
291
291
}
292
292
293
293
idx, err := search.NewIndexer(db, escli, &dir, indexerConfig)
···
299
299
}
300
300
301
301
go func() {
302
-
if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil {
302
+
if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil {
303
303
slog.Error("failed to start metrics endpoint", "error", err)
304
304
panic(fmt.Errorf("failed to start metrics endpoint: %w", err))
305
305
}
306
306
}()
307
307
308
308
go func() {
309
-
srv.RunAPI(cctx.String("bind"))
309
+
srv.RunAPI(cmd.String("bind"))
310
310
}()
311
311
312
312
// If we're in readonly mode, just block forever
313
313
if readonly {
314
314
select {}
315
-
} else if cctx.String("pagerank-file") != "" && srv.Indexer != nil {
315
+
} else if cmd.String("pagerank-file") != "" && srv.Indexer != nil {
316
316
// If we're not in readonly mode, and we have a pagerank file, update pageranks
317
317
ctx := context.Background()
318
-
if err := srv.Indexer.BulkIndexPageranks(ctx, cctx.String("pagerank-file")); err != nil {
318
+
if err := srv.Indexer.BulkIndexPageranks(ctx, cmd.String("pagerank-file")); err != nil {
319
319
return fmt.Errorf("failed to update pageranks: %w", err)
320
320
}
321
-
} else if cctx.String("bulk-posts-file") != "" && srv.Indexer != nil {
321
+
} else if cmd.String("bulk-posts-file") != "" && srv.Indexer != nil {
322
322
// If we're not in readonly mode, and we have a bulk posts file, index posts
323
323
ctx := context.Background()
324
-
if err := srv.Indexer.BulkIndexPosts(ctx, cctx.String("bulk-posts-file")); err != nil {
324
+
if err := srv.Indexer.BulkIndexPosts(ctx, cmd.String("bulk-posts-file")); err != nil {
325
325
return fmt.Errorf("failed to bulk index posts: %w", err)
326
326
}
327
-
} else if cctx.String("bulk-profiles-file") != "" && srv.Indexer != nil {
327
+
} else if cmd.String("bulk-profiles-file") != "" && srv.Indexer != nil {
328
328
// If we're not in readonly mode, and we have a bulk profiles file, index profiles
329
329
ctx := context.Background()
330
-
if err := srv.Indexer.BulkIndexProfiles(ctx, cctx.String("bulk-profiles-file")); err != nil {
330
+
if err := srv.Indexer.BulkIndexProfiles(ctx, cmd.String("bulk-profiles-file")); err != nil {
331
331
return fmt.Errorf("failed to bulk index profiles: %w", err)
332
332
}
333
333
} else if srv.Indexer != nil {
···
348
348
var elasticCheckCmd = &cli.Command{
349
349
Name: "elastic-check",
350
350
Flags: []cli.Flag{},
351
-
Action: func(cctx *cli.Context) error {
352
-
escli, err := createEsClient(cctx)
351
+
Action: func(ctx context.Context, cmd *cli.Command) error {
352
+
escli, err := createEsClient(cmd)
353
353
if err != nil {
354
354
return err
355
355
}
···
365
365
}
366
366
slog.Info("opensearch client connected", "client_info", inf)
367
367
368
-
resp, err := escli.Indices.Exists([]string{cctx.String("es-profile-index"), cctx.String("es-post-index")})
368
+
resp, err := escli.Indices.Exists([]string{cmd.String("es-profile-index"), cmd.String("es-post-index")})
369
369
if err != nil {
370
370
return fmt.Errorf("failed to check index existence: %w", err)
371
371
}
···
392
392
var searchPostCmd = &cli.Command{
393
393
Name: "search-post",
394
394
Usage: "run a simple query against posts index",
395
-
Action: func(cctx *cli.Context) error {
396
-
escli, err := createEsClient(cctx)
395
+
Action: func(ctx context.Context, cmd *cli.Command) error {
396
+
escli, err := createEsClient(cmd)
397
397
if err != nil {
398
398
return err
399
399
}
···
401
401
context.Background(),
402
402
identity.DefaultDirectory(), // TODO: parse PLC arg
403
403
escli,
404
-
cctx.String("es-post-index"),
404
+
cmd.String("es-post-index"),
405
405
&search.PostSearchParams{
406
-
Query: strings.Join(cctx.Args().Slice(), " "),
406
+
Query: strings.Join(cmd.Args().Slice(), " "),
407
407
Offset: 0,
408
408
Size: 20,
409
409
},
···
424
424
Name: "typeahead",
425
425
},
426
426
},
427
-
Action: func(cctx *cli.Context) error {
428
-
escli, err := createEsClient(cctx)
427
+
Action: func(ctx context.Context, cmd *cli.Command) error {
428
+
escli, err := createEsClient(cmd)
429
429
if err != nil {
430
430
return err
431
431
}
432
-
if cctx.Bool("typeahead") {
432
+
if cmd.Bool("typeahead") {
433
433
res, err := search.DoSearchProfilesTypeahead(
434
434
context.Background(),
435
435
escli,
436
-
cctx.String("es-profile-index"),
436
+
cmd.String("es-profile-index"),
437
437
&search.ActorSearchParams{
438
-
Query: strings.Join(cctx.Args().Slice(), " "),
438
+
Query: strings.Join(cmd.Args().Slice(), " "),
439
439
Size: 10,
440
440
},
441
441
)
···
448
448
context.Background(),
449
449
identity.DefaultDirectory(), // TODO: parse PLC arg
450
450
escli,
451
-
cctx.String("es-profile-index"),
451
+
cmd.String("es-profile-index"),
452
452
&search.ActorSearchParams{
453
-
Query: strings.Join(cctx.Args().Slice(), " "),
453
+
Query: strings.Join(cmd.Args().Slice(), " "),
454
454
Offset: 0,
455
455
Size: 20,
456
456
},
···
464
464
},
465
465
}
466
466
467
-
func createEsClient(cctx *cli.Context) (*es.Client, error) {
467
+
func createEsClient(cmd *cli.Command) (*es.Client, error) {
468
468
469
469
addrs := []string{}
470
-
if hosts := cctx.String("elastic-hosts"); hosts != "" {
470
+
if hosts := cmd.String("elastic-hosts"); hosts != "" {
471
471
addrs = strings.Split(hosts, ",")
472
472
}
473
473
474
-
certfi := cctx.String("elastic-cert-file")
474
+
certfi := cmd.String("elastic-cert-file")
475
475
var cert []byte
476
476
if certfi != "" {
477
477
b, err := os.ReadFile(certfi)
···
482
482
cert = b
483
483
}
484
484
485
-
insecure := cctx.Bool("elastic-insecure-ssl")
485
+
insecure := cmd.Bool("elastic-insecure-ssl")
486
486
487
487
cfg := es.Config{
488
488
Addresses: addrs,
489
-
Username: cctx.String("elastic-username"),
490
-
Password: cctx.String("elastic-password"),
489
+
Username: cmd.String("elastic-username"),
490
+
Password: cmd.String("elastic-password"),
491
491
CACert: cert,
492
492
Transport: &http.Transport{
493
493
MaxIdleConnsPerHost: 20,
+14
-18
cmd/querycheck/main.go
+14
-18
cmd/querycheck/main.go
···
5
5
"fmt"
6
6
"log"
7
7
"log/slog"
8
+
"net/http"
8
9
"os"
9
10
"os/signal"
10
11
"sync"
11
12
"syscall"
12
13
13
-
"net/http"
14
14
_ "net/http/pprof"
15
15
16
16
"github.com/bluesky-social/indigo/querycheck"
17
17
"github.com/bluesky-social/indigo/util/tracing"
18
+
19
+
"github.com/earthboundkid/versioninfo/v2"
18
20
"github.com/labstack/echo-contrib/pprof"
19
21
"github.com/labstack/echo/v4"
20
-
21
22
"github.com/labstack/echo/v4/middleware"
22
-
23
23
"github.com/prometheus/client_golang/prometheus/promhttp"
24
+
"github.com/urfave/cli/v3"
24
25
"go.opentelemetry.io/otel/trace"
25
-
26
-
"github.com/carlmjohnson/versioninfo"
27
-
"github.com/urfave/cli/v2"
28
26
)
29
27
30
28
func main() {
31
-
app := cli.App{
29
+
app := cli.Command{
32
30
Name: "querycheck",
33
31
Usage: "a postgresql query plan checker",
34
32
Version: versioninfo.Short(),
···
39
37
Name: "postgres-url",
40
38
Usage: "postgres url for storing events",
41
39
Value: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable",
42
-
EnvVars: []string{"POSTGRES_URL"},
40
+
Sources: cli.EnvVars("POSTGRES_URL"),
43
41
},
44
42
&cli.IntFlag{
45
43
Name: "port",
46
44
Usage: "port to serve metrics on",
47
45
Value: 8080,
48
-
EnvVars: []string{"PORT"},
46
+
Sources: cli.EnvVars("PORT"),
49
47
},
50
48
&cli.StringFlag{
51
49
Name: "auth-token",
52
50
Usage: "auth token for accessing the querycheck api",
53
51
Value: "",
54
-
EnvVars: []string{"AUTH_TOKEN"},
52
+
Sources: cli.EnvVars("AUTH_TOKEN"),
55
53
},
56
54
}
57
55
58
56
app.Action = Querycheck
59
57
60
-
err := app.Run(os.Args)
61
-
if err != nil {
58
+
if err := app.Run(context.Background(), os.Args); err != nil {
62
59
log.Fatal(err)
63
60
}
64
61
}
···
66
63
var tracer trace.Tracer
67
64
68
65
// Querycheck is the main function for querycheck
69
-
func Querycheck(cctx *cli.Context) error {
70
-
ctx := cctx.Context
66
+
func Querycheck(ctx context.Context, cmd *cli.Command) error {
71
67
ctx, cancel := context.WithCancel(ctx)
72
68
defer cancel()
73
69
···
110
106
e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig))
111
107
112
108
// Start the query checker
113
-
querychecker, err := querycheck.NewQuerychecker(ctx, cctx.String("postgres-url"))
109
+
querychecker, err := querycheck.NewQuerychecker(ctx, cmd.String("postgres-url"))
114
110
if err != nil {
115
111
log.Fatalf("failed to create querychecker: %+v\n", err)
116
112
}
···
131
127
132
128
e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
133
129
return func(c echo.Context) error {
134
-
if cctx.String("auth-token") != "" && c.Request().Header.Get("Authorization") != cctx.String("auth-token") {
130
+
if cmd.String("auth-token") != "" && c.Request().Header.Get("Authorization") != cmd.String("auth-token") {
135
131
return c.String(http.StatusUnauthorized, "unauthorized")
136
132
}
137
133
return next(c)
···
147
143
// Start the metrics server
148
144
wg.Add(1)
149
145
go func() {
150
-
logger.Info("starting metrics serverd", "port", cctx.Int("port"))
151
-
if err := e.Start(fmt.Sprintf(":%d", cctx.Int("port"))); err != nil {
146
+
logger.Info("starting metrics serverd", "port", cmd.Int("port"))
147
+
if err := e.Start(fmt.Sprintf(":%d", cmd.Int("port"))); err != nil {
152
148
logger.Error("failed to start metrics server", "err", err)
153
149
}
154
150
wg.Done()
+32
-32
cmd/rainbow/main.go
+32
-32
cmd/rainbow/main.go
···
17
17
"github.com/bluesky-social/indigo/splitter"
18
18
"github.com/bluesky-social/indigo/util/svcutil"
19
19
20
-
"github.com/carlmjohnson/versioninfo"
21
-
"github.com/urfave/cli/v2"
20
+
"github.com/earthboundkid/versioninfo/v2"
21
+
"github.com/urfave/cli/v3"
22
22
"go.opentelemetry.io/otel"
23
23
"go.opentelemetry.io/otel/attribute"
24
24
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
···
36
36
37
37
func run(args []string) error {
38
38
39
-
app := cli.App{
39
+
app := cli.Command{
40
40
Name: "rainbow",
41
41
Usage: "atproto firehose fan-out daemon",
42
42
Version: versioninfo.Short(),
···
47
47
&cli.StringFlag{
48
48
Name: "log-level",
49
49
Usage: "log verbosity level (eg: warn, info, debug)",
50
-
EnvVars: []string{"RAINBOW_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
50
+
Sources: cli.EnvVars("RAINBOW_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
51
51
},
52
52
&cli.StringFlag{
53
53
Name: "upstream-host",
54
54
Value: "http://localhost:2470",
55
55
Usage: "URL (schema and hostname, no path) of the upstream host (eg, relay)",
56
-
EnvVars: []string{"ATP_RELAY_HOST", "RAINBOW_RELAY_HOST"},
56
+
Sources: cli.EnvVars("ATP_RELAY_HOST", "RAINBOW_RELAY_HOST"),
57
57
},
58
58
&cli.StringFlag{
59
59
Name: "persist-db",
60
60
Value: "./rainbow.db",
61
61
Usage: "path to persistence db",
62
-
EnvVars: []string{"RAINBOW_DB_PATH"},
62
+
Sources: cli.EnvVars("RAINBOW_DB_PATH"),
63
63
},
64
64
&cli.StringFlag{
65
65
Name: "cursor-file",
66
66
Value: "./rainbow-cursor",
67
67
Usage: "write upstream cursor number to this file",
68
-
EnvVars: []string{"RAINBOW_CURSOR_PATH"},
68
+
Sources: cli.EnvVars("RAINBOW_CURSOR_PATH"),
69
69
},
70
70
&cli.StringFlag{
71
71
Name: "api-listen",
72
72
Value: ":2480",
73
-
EnvVars: []string{"RAINBOW_API_LISTEN"},
73
+
Sources: cli.EnvVars("RAINBOW_API_LISTEN"),
74
74
},
75
75
&cli.StringFlag{
76
76
Name: "metrics-listen",
77
77
Value: ":2481",
78
-
EnvVars: []string{"RAINBOW_METRICS_LISTEN", "SPLITTER_METRICS_LISTEN"},
78
+
Sources: cli.EnvVars("RAINBOW_METRICS_LISTEN", "SPLITTER_METRICS_LISTEN"),
79
79
},
80
80
&cli.Float64Flag{
81
81
Name: "persist-hours",
82
82
Value: 24 * 3,
83
-
EnvVars: []string{"RAINBOW_PERSIST_HOURS", "SPLITTER_PERSIST_HOURS"},
83
+
Sources: cli.EnvVars("RAINBOW_PERSIST_HOURS", "SPLITTER_PERSIST_HOURS"),
84
84
Usage: "hours to buffer (float, may be fractional)",
85
85
},
86
86
&cli.Int64Flag{
87
87
Name: "persist-bytes",
88
88
Value: 0,
89
89
Usage: "max bytes target for event cache, 0 to disable size target trimming",
90
-
EnvVars: []string{"RAINBOW_PERSIST_BYTES", "SPLITTER_PERSIST_BYTES"},
90
+
Sources: cli.EnvVars("RAINBOW_PERSIST_BYTES", "SPLITTER_PERSIST_BYTES"),
91
91
},
92
92
&cli.StringSliceFlag{
93
93
// TODO: better name for this argument
94
94
Name: "next-crawler",
95
95
Usage: "forward POST requestCrawl to these hosts (schema and host, no path) in addition to upstream-host. Comma-separated or multiple flags",
96
-
EnvVars: []string{"RAINBOW_NEXT_CRAWLER", "RELAY_NEXT_CRAWLER"},
96
+
Sources: cli.EnvVars("RAINBOW_NEXT_CRAWLER", "RELAY_NEXT_CRAWLER"),
97
97
},
98
98
&cli.StringFlag{
99
99
Name: "collectiondir-host",
100
100
Value: "http://localhost:2510",
101
101
Usage: "host (schema and hostname, no path) of upstream collectiondir instance, for com.atproto.sync.listReposByCollection",
102
-
EnvVars: []string{"RAINBOW_COLLECTIONDIR_HOST"},
102
+
Sources: cli.EnvVars("RAINBOW_COLLECTIONDIR_HOST"),
103
103
},
104
104
&cli.StringFlag{
105
105
Name: "env",
106
106
Usage: "operating environment (eg, 'prod', 'test')",
107
107
Value: "dev",
108
-
EnvVars: []string{"ENVIRONMENT"},
108
+
Sources: cli.EnvVars("ENVIRONMENT"),
109
109
},
110
110
&cli.BoolFlag{
111
111
Name: "enable-otel-otlp",
112
112
Usage: "enables OTEL OTLP exporter endpoint",
113
-
EnvVars: []string{"RAINBOW_ENABLE_OTEL_OTLP", "ENABLE_OTEL_OTLP"},
113
+
Sources: cli.EnvVars("RAINBOW_ENABLE_OTEL_OTLP", "ENABLE_OTEL_OTLP"),
114
114
},
115
115
&cli.StringFlag{
116
116
Name: "otel-otlp-endpoint",
117
117
Usage: "OTEL traces export endpoint",
118
118
Value: "http://localhost:4318",
119
-
EnvVars: []string{"OTEL_EXPORTER_OTLP_ENDPOINT"},
119
+
Sources: cli.EnvVars("OTEL_EXPORTER_OTLP_ENDPOINT"),
120
120
},
121
121
}
122
122
123
-
return app.Run(args)
123
+
return app.Run(context.Background(), args)
124
124
}
125
125
126
-
func runSplitter(cctx *cli.Context) error {
126
+
func runSplitter(ctx context.Context, cmd *cli.Command) error {
127
127
// Trap SIGINT to trigger a shutdown.
128
128
signals := make(chan os.Signal, 1)
129
129
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
130
130
131
-
logger := svcutil.ConfigLogger(cctx, os.Stdout).With("system", "rainbow")
131
+
logger := svcutil.ConfigLogger(cmd, os.Stdout).With("system", "rainbow")
132
132
133
133
// Enable OTLP HTTP exporter
134
134
// For relevant environment variables:
135
135
// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables
136
-
if cctx.Bool("enable-otel-otlp") {
137
-
ep := cctx.String("otel-otlp-endpoint")
136
+
if cmd.Bool("enable-otel-otlp") {
137
+
ep := cmd.String("otel-otlp-endpoint")
138
138
logger.Info("setting up trace exporter", "endpoint", ep)
139
139
ctx, cancel := context.WithCancel(context.Background())
140
140
defer cancel()
···
152
152
}
153
153
}()
154
154
155
-
env := cctx.String("env")
155
+
env := cmd.String("env")
156
156
tp := tracesdk.NewTracerProvider(
157
157
tracesdk.WithBatcher(exp),
158
158
tracesdk.WithResource(resource.NewWithAttributes(
···
166
166
otel.SetTracerProvider(tp)
167
167
}
168
168
169
-
persistPath := cctx.String("persist-db")
170
-
upstreamHost := cctx.String("upstream-host")
171
-
collectionDirHost := cctx.String("collectiondir-host")
172
-
nextCrawlers := cctx.StringSlice("next-crawler")
169
+
persistPath := cmd.String("persist-db")
170
+
upstreamHost := cmd.String("upstream-host")
171
+
collectionDirHost := cmd.String("collectiondir-host")
172
+
nextCrawlers := cmd.StringSlice("next-crawler")
173
173
174
174
var spl *splitter.Splitter
175
175
var err error
···
177
177
logger.Info("building splitter with storage at", "path", persistPath)
178
178
ppopts := pebblepersist.PebblePersistOptions{
179
179
DbPath: persistPath,
180
-
PersistDuration: time.Duration(float64(time.Hour) * cctx.Float64("persist-hours")),
180
+
PersistDuration: time.Duration(float64(time.Hour) * cmd.Float64("persist-hours")),
181
181
GCPeriod: 5 * time.Minute,
182
-
MaxBytes: uint64(cctx.Int64("persist-bytes")),
182
+
MaxBytes: uint64(cmd.Int64("persist-bytes")),
183
183
}
184
184
conf := splitter.SplitterConfig{
185
185
UpstreamHost: upstreamHost,
186
186
CollectionDirHost: collectionDirHost,
187
-
CursorFile: cctx.String("cursor-file"),
187
+
CursorFile: cmd.String("cursor-file"),
188
188
PebbleOptions: &ppopts,
189
189
UserAgent: fmt.Sprintf("rainbow/%s (atproto-relay)", versioninfo.Short()),
190
190
}
···
194
194
conf := splitter.SplitterConfig{
195
195
UpstreamHost: upstreamHost,
196
196
CollectionDirHost: collectionDirHost,
197
-
CursorFile: cctx.String("cursor-file"),
197
+
CursorFile: cmd.String("cursor-file"),
198
198
}
199
199
spl, err = splitter.NewSplitter(conf, nextCrawlers)
200
200
}
···
205
205
}
206
206
207
207
// set up metrics endpoint
208
-
metricsListen := cctx.String("metrics-listen")
208
+
metricsListen := cmd.String("metrics-listen")
209
209
go func() {
210
210
if err := spl.StartMetrics(metricsListen); err != nil {
211
211
logger.Error("failed to start metrics endpoint", "err", err)
···
216
216
runErr := make(chan error, 1)
217
217
218
218
go func() {
219
-
err := spl.StartAPI(cctx.String("api-listen"))
219
+
err := spl.StartAPI(cmd.String("api-listen"))
220
220
runErr <- err
221
221
}()
222
222
+52
-52
cmd/relay/main.go
+52
-52
cmd/relay/main.go
···
1
1
package main
2
2
3
3
import (
4
+
"context"
4
5
"crypto/rand"
5
6
"encoding/base64"
6
7
"fmt"
···
22
23
"github.com/bluesky-social/indigo/cmd/relay/stream/persist/diskpersist"
23
24
"github.com/bluesky-social/indigo/util/cliutil"
24
25
25
-
"github.com/carlmjohnson/versioninfo"
26
-
"github.com/urfave/cli/v2"
26
+
"github.com/earthboundkid/versioninfo/v2"
27
+
"github.com/urfave/cli/v3"
27
28
"gorm.io/plugin/opentelemetry/tracing"
28
29
)
29
30
···
36
37
37
38
func run(args []string) error {
38
39
39
-
app := cli.App{
40
+
app := cli.Command{
40
41
Name: "relay",
41
42
Usage: "atproto relay daemon",
42
43
Version: versioninfo.Short(),
···
45
46
&cli.StringSliceFlag{
46
47
Name: "admin-password",
47
48
Usage: "secret password/token for accessing admin endpoints (multiple values allowed)",
48
-
EnvVars: []string{"RELAY_ADMIN_PASSWORD", "RELAY_ADMIN_KEY"},
49
+
Sources: cli.EnvVars("RELAY_ADMIN_PASSWORD", "RELAY_ADMIN_KEY"),
49
50
},
50
51
&cli.StringFlag{
51
52
Name: "plc-host",
52
53
Usage: "method, hostname, and port of PLC registry",
53
54
Value: "https://plc.directory",
54
-
EnvVars: []string{"RELAY_PLC_HOST", "ATP_PLC_HOST"},
55
+
Sources: cli.EnvVars("RELAY_PLC_HOST", "ATP_PLC_HOST"),
55
56
},
56
57
&cli.StringFlag{
57
58
Name: "log-level",
58
59
Usage: "log verbosity level (eg: warn, info, debug)",
59
-
EnvVars: []string{"RELAY_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
60
+
Sources: cli.EnvVars("RELAY_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"),
60
61
},
61
62
}
62
63
app.Commands = []*cli.Command{
···
69
70
Name: "db-url",
70
71
Usage: "database connection string for relay database",
71
72
Value: "sqlite://data/relay/relay.sqlite",
72
-
EnvVars: []string{"DATABASE_URL"},
73
+
Sources: cli.EnvVars("DATABASE_URL"),
73
74
},
74
75
&cli.IntFlag{
75
76
Name: "max-db-conn",
76
77
Usage: "limit on size of database connection pool",
77
-
EnvVars: []string{"MAX_DB_CONNECTIONS", "MAX_METADB_CONNECTIONS"},
78
+
Sources: cli.EnvVars("MAX_DB_CONNECTIONS", "MAX_METADB_CONNECTIONS"),
78
79
Value: 40,
79
80
},
80
81
&cli.StringFlag{
81
82
Name: "bind",
82
83
Usage: "IP or address, and port, to listen on for HTTP APIs (including firehose)",
83
84
Value: ":2470",
84
-
EnvVars: []string{"RELAY_API_BIND", "RELAY_API_LISTEN"},
85
+
Sources: cli.EnvVars("RELAY_API_BIND", "RELAY_API_LISTEN"),
85
86
},
86
87
&cli.StringFlag{
87
88
Name: "persist-dir",
88
89
Usage: "local folder to store firehose playback files",
89
90
Value: "data/relay/persist",
90
-
EnvVars: []string{"RELAY_PERSIST_DIR", "RELAY_PERSISTER_DIR"},
91
+
Sources: cli.EnvVars("RELAY_PERSIST_DIR", "RELAY_PERSISTER_DIR"),
91
92
},
92
93
&cli.DurationFlag{
93
94
Name: "replay-window",
94
95
Usage: "retention duration for firehose playback",
95
-
EnvVars: []string{"RELAY_REPLAY_WINDOW", "RELAY_EVENT_PLAYBACK_TTL"},
96
+
Sources: cli.EnvVars("RELAY_REPLAY_WINDOW", "RELAY_EVENT_PLAYBACK_TTL"),
96
97
Value: 72 * time.Hour,
97
98
},
98
99
&cli.IntFlag{
99
100
Name: "host-concurrency",
100
101
Usage: "number of concurrent worker routines per upstream host",
101
-
EnvVars: []string{"RELAY_HOST_CONCURRENCY", "RELAY_CONCURRENCY_PER_PDS"},
102
+
Sources: cli.EnvVars("RELAY_HOST_CONCURRENCY", "RELAY_CONCURRENCY_PER_PDS"),
102
103
Value: 40,
103
104
},
104
105
&cli.IntFlag{
105
106
Name: "default-account-limit",
106
107
Value: 100,
107
108
Usage: "max number of active accounts for new upstream hosts",
108
-
EnvVars: []string{"RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"},
109
+
Sources: cli.EnvVars("RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"),
109
110
},
110
111
&cli.IntFlag{
111
112
Name: "new-hosts-per-day-limit",
112
113
Value: 50,
113
114
Usage: "max number of new upstream hosts subscribed per day via public requestCrawl",
114
-
EnvVars: []string{"RELAY_NEW_HOSTS_PER_DAY_LIMIT"},
115
+
Sources: cli.EnvVars("RELAY_NEW_HOSTS_PER_DAY_LIMIT"),
115
116
},
116
117
&cli.IntFlag{
117
118
Name: "ident-cache-size",
118
119
Value: 5_000_000,
119
120
Usage: "size of in-process identity cache (eg, DID docs)",
120
-
EnvVars: []string{"RELAY_IDENT_CACHE_SIZE", "RELAY_DID_CACHE_SIZE"},
121
+
Sources: cli.EnvVars("RELAY_IDENT_CACHE_SIZE", "RELAY_DID_CACHE_SIZE"),
121
122
},
122
123
&cli.BoolFlag{
123
124
Name: "disable-request-crawl",
124
125
Usage: "don't process public (un-authenticated) com.atproto.sync.requestCrawl",
125
-
EnvVars: []string{"RELAY_DISABLE_REQUEST_CRAWL"},
126
+
Sources: cli.EnvVars("RELAY_DISABLE_REQUEST_CRAWL"),
126
127
},
127
128
&cli.BoolFlag{
128
129
Name: "allow-insecure-hosts",
129
130
Usage: "enables subscription to non-SSL hosts via requestCrawl",
130
-
EnvVars: []string{"RELAY_ALLOW_INSECURE_HOSTS"},
131
+
Sources: cli.EnvVars("RELAY_ALLOW_INSECURE_HOSTS"),
131
132
},
132
133
&cli.BoolFlag{
133
134
Name: "lenient-sync-validation",
134
135
Usage: "when messages fail atproto 'Sync 1.1' validation, just log, don't drop",
135
-
EnvVars: []string{"RELAY_LENIENT_SYNC_VALIDATION"},
136
+
Sources: cli.EnvVars("RELAY_LENIENT_SYNC_VALIDATION"),
136
137
},
137
138
&cli.IntFlag{
138
139
Name: "initial-seq-number",
139
140
Usage: "when initializing output firehose, start with this sequence number",
140
141
Value: 1,
141
-
EnvVars: []string{"RELAY_INITIAL_SEQ_NUMBER"},
142
+
Sources: cli.EnvVars("RELAY_INITIAL_SEQ_NUMBER"),
142
143
},
143
144
&cli.StringSliceFlag{
144
145
Name: "sibling-relays",
145
146
Usage: "servers (eg https://example.com) to forward admin state changes to; multiple allowed",
146
-
EnvVars: []string{"RELAY_SIBLING_RELAYS"},
147
+
Sources: cli.EnvVars("RELAY_SIBLING_RELAYS"),
147
148
},
148
149
&cli.StringSliceFlag{
149
150
Name: "trusted-domains",
150
151
Usage: "domain names which mark trusted hosts; use wildcard prefix to match suffixes",
151
-
Value: cli.NewStringSlice("*.host.bsky.network"),
152
-
EnvVars: []string{"RELAY_TRUSTED_DOMAINS"},
152
+
Value: []string{"*.host.bsky.network"},
153
+
Sources: cli.EnvVars("RELAY_TRUSTED_DOMAINS"),
153
154
},
154
155
&cli.StringFlag{
155
156
Name: "env",
156
157
Value: "dev",
157
-
EnvVars: []string{"ENVIRONMENT"},
158
+
Sources: cli.EnvVars("ENVIRONMENT"),
158
159
Usage: "declared hosting environment (prod, qa, etc); used in metrics",
159
160
},
160
161
&cli.BoolFlag{
···
170
171
Name: "metrics-listen",
171
172
Usage: "IP or address, and port, to listen on for prometheus metrics",
172
173
Value: ":2471",
173
-
EnvVars: []string{"RELAY_METRICS_LISTEN"},
174
+
Sources: cli.EnvVars("RELAY_METRICS_LISTEN"),
174
175
},
175
176
&cli.StringFlag{
176
177
Name: "otel-exporter-otlp-endpoint",
177
178
Value: "http://localhost:4328",
178
-
EnvVars: []string{"OTEL_EXPORTER_OTLP_ENDPOINT"},
179
+
Sources: cli.EnvVars("OTEL_EXPORTER_OTLP_ENDPOINT"),
179
180
},
180
181
},
181
182
},
182
183
// additional commands defined in pull.go
183
184
cmdPullHosts,
184
185
}
185
-
return app.Run(os.Args)
186
+
return app.Run(context.Background(), args)
186
187
187
188
}
188
189
189
-
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
190
+
func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger {
190
191
var level slog.Level
191
-
switch strings.ToLower(cctx.String("log-level")) {
192
+
switch strings.ToLower(cmd.String("log-level")) {
192
193
case "error":
193
194
level = slog.LevelError
194
195
case "warn":
···
207
208
return logger
208
209
}
209
210
210
-
func runRelay(cctx *cli.Context) error {
211
-
ctx := cctx.Context
212
-
logger := configLogger(cctx, os.Stdout)
211
+
func runRelay(ctx context.Context, cmd *cli.Command) error {
212
+
logger := configLogger(cmd, os.Stdout)
213
213
214
214
// Trap SIGINT to trigger a shutdown.
215
215
signals := make(chan os.Signal, 1)
216
216
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
217
217
218
-
dburl := cctx.String("db-url")
219
-
maxConn := cctx.Int("max-db-conn")
218
+
dburl := cmd.String("db-url")
219
+
maxConn := cmd.Int("max-db-conn")
220
220
logger.Info("configuring database", "url", dburl, "maxConn", maxConn)
221
221
db, err := cliutil.SetupDatabase(dburl, maxConn)
222
222
if err != nil {
···
228
228
SkipHandleVerification: true,
229
229
SkipDNSDomainSuffixes: []string{".bsky.social"},
230
230
TryAuthoritativeDNS: true,
231
-
PLCURL: cctx.String("plc-host"),
231
+
PLCURL: cmd.String("plc-host"),
232
232
}
233
-
dir := identity.NewCacheDirectory(&baseDir, cctx.Int("ident-cache-size"), time.Hour*24, time.Minute*2, time.Minute*5)
233
+
dir := identity.NewCacheDirectory(&baseDir, cmd.Int("ident-cache-size"), time.Hour*24, time.Minute*2, time.Minute*5)
234
234
235
-
persistDir := cctx.String("persist-dir")
235
+
persistDir := cmd.String("persist-dir")
236
236
if err := os.MkdirAll(persistDir, os.ModePerm); err != nil {
237
237
return err
238
238
}
239
239
persitConfig := diskpersist.DefaultDiskPersistOptions()
240
-
persitConfig.Retention = cctx.Duration("replay-window")
241
-
persitConfig.InitialSeq = cctx.Int64("initial-seq-number")
240
+
persitConfig.Retention = cmd.Duration("replay-window")
241
+
persitConfig.InitialSeq = cmd.Int64("initial-seq-number")
242
242
logger.Info("setting up disk persister", "dir", persistDir, "replayWindow", persitConfig.Retention)
243
243
persister, err := diskpersist.NewDiskPersistence(persistDir, "", db, persitConfig)
244
244
if err != nil {
···
247
247
248
248
relayConfig := relay.DefaultRelayConfig()
249
249
relayConfig.UserAgent = fmt.Sprintf("indigo-relay/%s (atproto-relay)", versioninfo.Short())
250
-
relayConfig.ConcurrencyPerHost = cctx.Int("host-concurrency")
251
-
relayConfig.DefaultRepoLimit = cctx.Int64("default-account-limit")
252
-
relayConfig.HostPerDayLimit = cctx.Int64("new-hosts-per-day-limit")
253
-
relayConfig.TrustedDomains = cctx.StringSlice("trusted-domains")
254
-
relayConfig.LenientSyncValidation = cctx.Bool("lenient-sync-validation")
250
+
relayConfig.ConcurrencyPerHost = cmd.Int("host-concurrency")
251
+
relayConfig.DefaultRepoLimit = cmd.Int64("default-account-limit")
252
+
relayConfig.HostPerDayLimit = cmd.Int64("new-hosts-per-day-limit")
253
+
relayConfig.TrustedDomains = cmd.StringSlice("trusted-domains")
254
+
relayConfig.LenientSyncValidation = cmd.Bool("lenient-sync-validation")
255
255
256
256
svcConfig := DefaultServiceConfig()
257
-
svcConfig.AllowInsecureHosts = cctx.Bool("allow-insecure-hosts")
258
-
svcConfig.DisableRequestCrawl = cctx.Bool("disable-request-crawl")
259
-
svcConfig.SiblingRelayHosts = cctx.StringSlice("sibling-relays")
257
+
svcConfig.AllowInsecureHosts = cmd.Bool("allow-insecure-hosts")
258
+
svcConfig.DisableRequestCrawl = cmd.Bool("disable-request-crawl")
259
+
svcConfig.SiblingRelayHosts = cmd.StringSlice("sibling-relays")
260
260
if len(svcConfig.SiblingRelayHosts) > 0 {
261
261
logger.Info("sibling relay hosts configured for admin state forwarding", "servers", svcConfig.SiblingRelayHosts)
262
262
}
263
-
if cctx.IsSet("admin-password") {
264
-
svcConfig.AdminPasswords = cctx.StringSlice("admin-password")
263
+
if cmd.IsSet("admin-password") {
264
+
svcConfig.AdminPasswords = cmd.StringSlice("admin-password")
265
265
} else {
266
266
var rblob [10]byte
267
267
_, _ = rand.Read(rblob[:])
···
285
285
286
286
// start metrics endpoint
287
287
go func() {
288
-
if err := svc.StartMetrics(cctx.String("metrics-listen")); err != nil {
288
+
if err := svc.StartMetrics(cmd.String("metrics-listen")); err != nil {
289
289
logger.Error("failed to start metrics endpoint", "err", err)
290
290
os.Exit(1)
291
291
}
292
292
}()
293
293
294
294
// start observability/tracing (OTEL and jaeger)
295
-
if err := setupOTEL(cctx); err != nil {
295
+
if err := setupOTEL(cmd); err != nil {
296
296
return err
297
297
}
298
-
if cctx.Bool("enable-db-tracing") {
298
+
if cmd.Bool("enable-db-tracing") {
299
299
if err := db.Use(tracing.NewPlugin()); err != nil {
300
300
return err
301
301
}
···
308
308
309
309
svcErr := make(chan error, 1)
310
310
go func() {
311
-
err := svc.StartAPI(cctx.String("bind"))
311
+
err := svc.StartAPI(cmd.String("bind"))
312
312
svcErr <- err
313
313
}()
314
314
+6
-6
cmd/relay/otel.go
+6
-6
cmd/relay/otel.go
···
6
6
"os"
7
7
"time"
8
8
9
-
"github.com/urfave/cli/v2"
9
+
"github.com/urfave/cli/v3"
10
10
"go.opentelemetry.io/otel"
11
11
"go.opentelemetry.io/otel/attribute"
12
12
"go.opentelemetry.io/otel/exporters/jaeger"
···
16
16
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
17
17
)
18
18
19
-
func setupOTEL(cctx *cli.Context) error {
19
+
func setupOTEL(cmd *cli.Command) error {
20
20
21
-
env := cctx.String("env")
21
+
env := cmd.String("env")
22
22
if env == "" {
23
23
env = "dev"
24
24
}
25
25
26
-
if cctx.Bool("enable-jaeger-tracing") {
26
+
if cmd.Bool("enable-jaeger-tracing") {
27
27
jaegerUrl := "http://localhost:14268/api/traces"
28
28
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl)))
29
29
if err != nil {
···
50
50
// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables
51
51
// At a minimum, you need to set
52
52
// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
53
-
if cctx.Bool("enable-otel-tracing") {
54
-
ep := cctx.String("otel-exporter-otlp-endpoint")
53
+
if cmd.Bool("enable-otel-tracing") {
54
+
ep := cmd.String("otel-exporter-otlp-endpoint")
55
55
slog.Info("setting up trace exporter", "endpoint", ep)
56
56
ctx, cancel := context.WithCancel(context.Background())
57
57
defer cancel()
+17
-17
cmd/relay/pull.go
+17
-17
cmd/relay/pull.go
···
1
1
package main
2
2
3
3
import (
4
+
"context"
4
5
"errors"
5
6
"fmt"
6
7
···
11
12
"github.com/bluesky-social/indigo/util/cliutil"
12
13
"github.com/bluesky-social/indigo/xrpc"
13
14
14
-
"github.com/urfave/cli/v2"
15
+
"github.com/urfave/cli/v3"
15
16
)
16
17
17
18
var cmdPullHosts = &cli.Command{
···
23
24
Name: "relay-host",
24
25
Usage: "method, hostname, and port of relay to pull from",
25
26
Value: "https://bsky.network",
26
-
EnvVars: []string{"RELAY_HOST"},
27
+
Sources: cli.EnvVars("RELAY_HOST"),
27
28
},
28
29
&cli.StringFlag{
29
30
Name: "db-url",
30
31
Usage: "database connection string for relay database",
31
32
Value: "sqlite://data/relay/relay.sqlite",
32
-
EnvVars: []string{"DATABASE_URL"},
33
+
Sources: cli.EnvVars("DATABASE_URL"),
33
34
},
34
35
&cli.IntFlag{
35
36
Name: "default-account-limit",
36
37
Value: 100,
37
38
Usage: "max number of active accounts for new upstream hosts",
38
-
EnvVars: []string{"RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"},
39
+
Sources: cli.EnvVars("RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"),
39
40
},
40
41
&cli.IntFlag{
41
42
Name: "batch-size",
42
43
Value: 500,
43
44
Usage: "host many hosts to pull at a time",
44
-
EnvVars: []string{"RELAY_PULL_HOSTS_BATCH_SIZE"},
45
+
Sources: cli.EnvVars("RELAY_PULL_HOSTS_BATCH_SIZE"),
45
46
},
46
47
&cli.StringSliceFlag{
47
48
Name: "trusted-domains",
48
49
Usage: "domain names which mark trusted hosts; use wildcard prefix to match suffixes",
49
-
Value: cli.NewStringSlice("*.host.bsky.network"),
50
-
EnvVars: []string{"RELAY_TRUSTED_DOMAINS"},
50
+
Value: []string{"*.host.bsky.network"},
51
+
Sources: cli.EnvVars("RELAY_TRUSTED_DOMAINS"),
51
52
},
52
53
&cli.BoolFlag{
53
54
Name: "skip-host-checks",
54
55
Usage: "don't run describeServer requests to see if host is a PDS before adding",
55
-
EnvVars: []string{"RELAY_SKIP_HOST_CHECKS"},
56
+
Sources: cli.EnvVars("RELAY_SKIP_HOST_CHECKS"),
56
57
},
57
58
},
58
59
}
59
60
60
-
func runPullHosts(cctx *cli.Context) error {
61
-
ctx := cctx.Context
61
+
func runPullHosts(ctx context.Context, cmd *cli.Command) error {
62
62
63
-
if cctx.Args().Len() > 0 {
63
+
if cmd.Args().Len() > 0 {
64
64
return fmt.Errorf("unexpected arguments")
65
65
}
66
66
67
67
client := xrpc.Client{
68
-
Host: cctx.String("relay-host"),
68
+
Host: cmd.String("relay-host"),
69
69
}
70
70
71
-
skipHostChecks := cctx.Bool("skip-host-checks")
71
+
skipHostChecks := cmd.Bool("skip-host-checks")
72
72
73
73
dir := identity.DefaultDirectory()
74
74
75
-
dburl := cctx.String("db-url")
75
+
dburl := cmd.String("db-url")
76
76
db, err := cliutil.SetupDatabase(dburl, 10)
77
77
if err != nil {
78
78
return err
79
79
}
80
80
81
81
relayConfig := relay.DefaultRelayConfig()
82
-
relayConfig.DefaultRepoLimit = cctx.Int64("default-account-limit")
83
-
relayConfig.TrustedDomains = cctx.StringSlice("trusted-domains")
82
+
relayConfig.DefaultRepoLimit = cmd.Int64("default-account-limit")
83
+
relayConfig.TrustedDomains = cmd.StringSlice("trusted-domains")
84
84
85
85
// NOTE: setting evtmgr to nil
86
86
r, err := relay.NewRelay(db, nil, dir, relayConfig)
···
91
91
checker := relay.NewHostClient(relayConfig.UserAgent)
92
92
93
93
cursor := ""
94
-
size := cctx.Int64("batch-size")
94
+
size := cmd.Int64("batch-size")
95
95
for {
96
96
resp, err := comatproto.SyncListHosts(ctx, &client, cursor, size)
97
97
if err != nil {
+16
-17
cmd/sonar/main.go
+16
-17
cmd/sonar/main.go
···
13
13
"syscall"
14
14
"time"
15
15
16
+
_ "go.uber.org/automaxprocs"
17
+
16
18
"github.com/bluesky-social/indigo/events"
17
19
"github.com/bluesky-social/indigo/events/schedulers/sequential"
20
+
21
+
"github.com/earthboundkid/versioninfo/v2"
18
22
"github.com/gorilla/websocket"
19
23
"github.com/prometheus/client_golang/prometheus/promhttp"
20
-
_ "go.uber.org/automaxprocs"
21
-
22
-
"github.com/carlmjohnson/versioninfo"
23
-
"github.com/urfave/cli/v2"
24
+
"github.com/urfave/cli/v3"
24
25
)
25
26
26
27
func main() {
27
-
app := cli.App{
28
+
app := cli.Command{
28
29
Name: "sonar",
29
30
Usage: "atproto firehose monitoring tool",
30
31
Version: versioninfo.Short(),
···
35
36
Name: "ws-url",
36
37
Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint",
37
38
Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos",
38
-
EnvVars: []string{"SONAR_WS_URL"},
39
+
Sources: cli.EnvVars("SONAR_WS_URL"),
39
40
},
40
41
&cli.StringFlag{
41
42
Name: "log-level",
42
43
Usage: "log level",
43
44
Value: "info",
44
-
EnvVars: []string{"SONAR_LOG_LEVEL"},
45
+
Sources: cli.EnvVars("SONAR_LOG_LEVEL"),
45
46
},
46
47
&cli.IntFlag{
47
48
Name: "port",
48
49
Usage: "listen port for metrics server",
49
50
Value: 8345,
50
-
EnvVars: []string{"SONAR_PORT"},
51
+
Sources: cli.EnvVars("SONAR_PORT"),
51
52
},
52
53
&cli.IntFlag{
53
54
Name: "max-queue-size",
···
58
59
Name: "cursor-file",
59
60
Usage: "path to cursor file",
60
61
Value: "sonar_cursor.json",
61
-
EnvVars: []string{"SONAR_CURSOR_FILE"},
62
+
Sources: cli.EnvVars("SONAR_CURSOR_FILE"),
62
63
},
63
64
}
64
65
65
66
app.Action = runSonar
66
67
67
-
err := app.Run(os.Args)
68
-
if err != nil {
68
+
if err := app.Run(context.Background(), os.Args); err != nil {
69
69
log.Fatal(err)
70
70
}
71
71
}
72
72
73
-
func runSonar(cctx *cli.Context) error {
74
-
ctx := cctx.Context
73
+
func runSonar(ctx context.Context, cmd *cli.Command) error {
75
74
ctx, cancel := context.WithCancel(ctx)
76
75
defer cancel()
77
76
···
88
87
logger = logger.With("source", "sonar_main")
89
88
logger.Info("starting sonar")
90
89
91
-
u, err := url.Parse(cctx.String("ws-url"))
90
+
u, err := url.Parse(cmd.String("ws-url"))
92
91
if err != nil {
93
92
log.Fatalf("failed to parse ws-url: %+v", err)
94
93
}
95
94
96
-
s, err := NewSonar(logger, cctx.String("cursor-file"), u.String())
95
+
s, err := NewSonar(logger, cmd.String("cursor-file"), u.String())
97
96
if err != nil {
98
97
log.Fatalf("failed to create sonar: %+v", err)
99
98
}
···
161
160
mux.Handle("/metrics", promhttp.Handler())
162
161
163
162
metricServer := &http.Server{
164
-
Addr: fmt.Sprintf(":%d", cctx.Int("port")),
163
+
Addr: fmt.Sprintf(":%d", cmd.Int("port")),
165
164
Handler: mux,
166
165
}
167
166
···
171
170
defer wg.Done()
172
171
logger = logger.With("source", "metrics_server")
173
172
174
-
logger.Info("metrics server listening", "port", cctx.Int("port"))
173
+
logger.Info("metrics server listening", "port", cmd.Int("port"))
175
174
176
175
if err := metricServer.ListenAndServe(); err != http.ErrServerClosed {
177
176
log.Fatalf("failed to start metrics server: %+v", err)
+65
-21
cmd/stress/main.go
+65
-21
cmd/stress/main.go
···
4
4
"context"
5
5
"crypto/rand"
6
6
"encoding/hex"
7
+
"encoding/json"
7
8
"fmt"
8
9
"os"
9
10
"sync"
10
11
"time"
12
+
13
+
_ "github.com/joho/godotenv/autoload"
11
14
12
15
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
16
appbsky "github.com/bluesky-social/indigo/api/bsky"
···
18
21
"github.com/bluesky-social/indigo/util/cliutil"
19
22
"github.com/bluesky-social/indigo/xrpc"
20
23
24
+
"github.com/earthboundkid/versioninfo/v2"
21
25
"github.com/ipfs/go-cid"
22
26
"github.com/ipfs/go-datastore"
23
27
blockstore "github.com/ipfs/go-ipfs-blockstore"
24
28
cbor "github.com/ipfs/go-ipld-cbor"
25
-
26
-
_ "github.com/joho/godotenv/autoload"
27
-
28
-
"github.com/carlmjohnson/versioninfo"
29
29
"github.com/ipld/go-car"
30
-
cli "github.com/urfave/cli/v2"
30
+
"github.com/urfave/cli/v3"
31
31
)
32
32
33
33
func main() {
···
35
35
}
36
36
37
37
func run(args []string) {
38
-
app := cli.App{
38
+
app := cli.Command{
39
39
Name: "stress",
40
40
Usage: "load generation tool for PDS instances",
41
41
Version: versioninfo.Short(),
···
46
46
genRepoCmd,
47
47
}
48
48
49
-
app.RunAndExitOnError()
49
+
if err := app.Run(context.Background(), args); err != nil {
50
+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
51
+
os.Exit(-1)
52
+
}
50
53
}
51
54
52
55
var postingCmd = &cli.Command{
···
67
70
Name: "pds-host",
68
71
Usage: "method, hostname, and port of PDS instance",
69
72
Value: "http://localhost:4849",
70
-
EnvVars: []string{"ATP_PDS_HOST"},
73
+
Sources: cli.EnvVars("ATP_PDS_HOST"),
71
74
},
72
75
&cli.StringFlag{
73
76
Name: "invite",
74
77
},
75
78
},
76
-
Action: func(cctx *cli.Context) error {
77
-
xrpcc, err := cliutil.GetXrpcClient(cctx, false)
79
+
Action: func(ctx context.Context, cmd *cli.Command) error {
80
+
xrpcc, err := getXrpcClient(cmd, false)
78
81
if err != nil {
79
82
return err
80
83
}
81
84
82
-
count := cctx.Int("count")
83
-
concurrent := cctx.Int("concurrent")
84
-
quiet := cctx.Bool("quiet")
85
-
ctx := context.TODO()
85
+
count := cmd.Int("count")
86
+
concurrent := cmd.Int("concurrent")
87
+
quiet := cmd.Bool("quiet")
86
88
87
89
buf := make([]byte, 6)
88
90
rand.Read(buf)
89
91
id := hex.EncodeToString(buf)
90
92
91
93
var invite *string
92
-
if inv := cctx.String("invite"); inv != "" {
94
+
if inv := cmd.String("invite"); inv != "" {
93
95
invite = &inv
94
96
}
95
97
···
166
168
Name: "pds-host",
167
169
Usage: "method, hostname, and port of PDS instance",
168
170
Value: "http://localhost:4849",
169
-
EnvVars: []string{"ATP_PDS_HOST"},
171
+
Sources: cli.EnvVars("ATP_PDS_HOST"),
170
172
},
171
173
},
172
174
ArgsUsage: "<car-file-path>",
173
-
Action: func(cctx *cli.Context) error {
174
-
fname := cctx.Args().First()
175
+
Action: func(ctx context.Context, cmd *cli.Command) error {
176
+
fname := cmd.Args().First()
175
177
if fname == "" {
176
178
return cli.Exit("must provide car file path", 127)
177
179
}
178
180
179
-
l := cctx.Int("len")
181
+
l := cmd.Int("len")
180
182
181
183
membs := blockstore.NewBlockstore(datastore.NewMapDatastore())
182
-
183
-
ctx := context.Background()
184
184
185
185
r := repo.NewRepo(ctx, "did:plc:foobar", membs)
186
186
···
225
225
return nil
226
226
},
227
227
}
228
+
229
+
func getXrpcClient(cmd *cli.Command, authreq bool) (*xrpc.Client, error) {
230
+
h := "http://localhost:4989"
231
+
if pdsurl := cmd.String("pds-host"); pdsurl != "" {
232
+
h = pdsurl
233
+
}
234
+
235
+
auth, err := loadAuthFromEnv(cmd, authreq)
236
+
if err != nil {
237
+
return nil, fmt.Errorf("loading auth: %w", err)
238
+
}
239
+
240
+
return &xrpc.Client{
241
+
Client: cliutil.NewHttpClient(),
242
+
Host: h,
243
+
Auth: auth,
244
+
}, nil
245
+
}
246
+
247
+
func loadAuthFromEnv(cmd *cli.Command, req bool) (*xrpc.AuthInfo, error) {
248
+
if a := cmd.String("auth"); a != "" {
249
+
if ai, err := cliutil.ReadAuth(a); err != nil && req {
250
+
return nil, err
251
+
} else {
252
+
return ai, nil
253
+
}
254
+
}
255
+
256
+
val := os.Getenv("ATP_AUTH_FILE")
257
+
if val == "" {
258
+
if req {
259
+
return nil, fmt.Errorf("no auth env present, ATP_AUTH_FILE not set")
260
+
}
261
+
262
+
return nil, nil
263
+
}
264
+
265
+
var auth xrpc.AuthInfo
266
+
if err := json.Unmarshal([]byte(val), &auth); err != nil {
267
+
return nil, err
268
+
}
269
+
270
+
return &auth, nil
271
+
}
+47
-51
cmd/supercollider/main.go
+47
-51
cmd/supercollider/main.go
···
7
7
"log"
8
8
"log/slog"
9
9
"net"
10
+
"net/http"
10
11
"os"
11
12
"os/signal"
12
13
"path/filepath"
···
15
16
"syscall"
16
17
"time"
17
18
18
-
"net/http"
19
+
_ "go.uber.org/automaxprocs"
19
20
_ "net/http/pprof"
20
21
21
22
"github.com/bluesky-social/indigo/api/atproto"
···
26
27
"github.com/bluesky-social/indigo/events"
27
28
"github.com/bluesky-social/indigo/events/yolopersist"
28
29
"github.com/bluesky-social/indigo/indexer"
30
+
lexutil "github.com/bluesky-social/indigo/lex/util"
29
31
"github.com/bluesky-social/indigo/models"
30
32
"github.com/bluesky-social/indigo/plc"
33
+
"github.com/bluesky-social/indigo/repomgr"
34
+
"github.com/bluesky-social/indigo/util"
35
+
31
36
petname "github.com/dustinkirkland/golang-petname"
37
+
"github.com/earthboundkid/versioninfo/v2"
38
+
"github.com/gorilla/websocket"
32
39
"github.com/icrowley/fake"
33
40
"github.com/labstack/echo-contrib/pprof"
34
-
"github.com/urfave/cli/v2"
35
-
godid "github.com/whyrusleeping/go-did"
36
-
"golang.org/x/crypto/acme/autocert"
37
-
"golang.org/x/time/rate"
38
-
39
-
lexutil "github.com/bluesky-social/indigo/lex/util"
40
-
"github.com/bluesky-social/indigo/repomgr"
41
-
"github.com/bluesky-social/indigo/util"
42
-
"github.com/gorilla/websocket"
43
41
"github.com/labstack/echo/v4"
44
42
"github.com/labstack/echo/v4/middleware"
45
43
"github.com/prometheus/client_golang/prometheus"
46
44
"github.com/prometheus/client_golang/prometheus/promauto"
47
45
"github.com/prometheus/client_golang/prometheus/promhttp"
48
-
_ "go.uber.org/automaxprocs"
46
+
"github.com/urfave/cli/v3"
47
+
cbg "github.com/whyrusleeping/cbor-gen"
48
+
godid "github.com/whyrusleeping/go-did"
49
+
"golang.org/x/crypto/acme/autocert"
50
+
"golang.org/x/time/rate"
49
51
"gorm.io/driver/sqlite"
50
52
"gorm.io/gorm"
51
-
52
-
"github.com/carlmjohnson/versioninfo"
53
-
cbg "github.com/whyrusleeping/cbor-gen"
54
53
)
55
54
56
55
var eventsGeneratedCounter = promauto.NewCounter(prometheus.CounterOpts{
···
85
84
ctx, cancel := context.WithCancel(ctx)
86
85
defer cancel()
87
86
88
-
app := cli.App{
87
+
app := cli.Command{
89
88
Name: "supercollider",
90
89
Usage: "atproto event noise-maker for Relay load testing",
91
90
Version: versioninfo.Short(),
···
96
95
Name: "hostname",
97
96
Usage: "hostname of this server (forward *.hostname DNS records to this server)",
98
97
Value: "supercollider.jazco.io",
99
-
EnvVars: []string{"SUPERCOLLIDER_HOST"},
98
+
Sources: cli.EnvVars("SUPERCOLLIDER_HOST"),
100
99
},
101
100
&cli.BoolFlag{
102
101
Name: "use-ssl",
103
102
Usage: "listen on port 443 and use SSL (needs to be run as root and have external DNS setup)",
104
103
Value: false,
105
-
EnvVars: []string{"SUPERCOLLIDER_USE_SSL"},
104
+
Sources: cli.EnvVars("SUPERCOLLIDER_USE_SSL"),
106
105
},
107
106
&cli.IntFlag{
108
107
Name: "port",
109
108
Usage: "port for the HTTP(S) server to listen on (defaults to 80 if not using SSL, 443 if using SSL)",
110
-
EnvVars: []string{"SUPERCOLLIDER_PORT"},
109
+
Sources: cli.EnvVars("SUPERCOLLIDER_PORT"),
111
110
},
112
111
113
112
&cli.StringFlag{
114
113
Name: "key-file",
115
114
Usage: "file to store the private key used to sign events",
116
115
Value: "key.raw",
117
-
EnvVars: []string{"KEY_FILE"},
116
+
Sources: cli.EnvVars("KEY_FILE"),
118
117
},
119
118
}
120
119
···
128
127
Name: "num-users",
129
128
Usage: "number of fake users to produce events for",
130
129
Value: 100,
131
-
EnvVars: []string{"NUM_USERS"},
130
+
Sources: cli.EnvVars("NUM_USERS"),
132
131
},
133
132
&cli.IntFlag{
134
133
Name: "total-events",
135
134
Usage: "total number of events to generate",
136
135
Value: 1_000_000,
137
-
EnvVars: []string{"TOTAL_EVENTS"},
136
+
Sources: cli.EnvVars("TOTAL_EVENTS"),
138
137
},
139
138
&cli.StringFlag{
140
139
Name: "output-file",
141
140
Usage: "output file for the generated events",
142
141
Value: "events_out.cbor",
143
-
EnvVars: []string{"OUTPUT_FILE"},
142
+
Sources: cli.EnvVars("OUTPUT_FILE"),
144
143
},
145
144
}, app.Flags...),
146
145
},
···
153
152
Name: "events-per-second",
154
153
Usage: "maximum number of events to generate per second",
155
154
Value: 300,
156
-
EnvVars: []string{"EVENTS_PER_SECOND"},
155
+
Sources: cli.EnvVars("EVENTS_PER_SECOND"),
157
156
},
158
157
&cli.StringFlag{
159
158
Name: "input-file",
160
159
Usage: "input file for the generated events (if set, will read events from this file instead of generating them)",
161
160
Value: "events_in.cbor",
162
-
EnvVars: []string{"INPUT_FILE"},
161
+
Sources: cli.EnvVars("INPUT_FILE"),
163
162
},
164
163
}, app.Flags...),
165
164
},
166
165
}
167
166
168
-
err := app.Run(os.Args)
169
-
if err != nil {
167
+
if err := app.Run(context.Background(), os.Args); err != nil {
170
168
log.Fatal(err)
171
169
}
172
170
}
173
171
174
-
func Reload(cctx *cli.Context) error {
175
-
ctx := cctx.Context
172
+
func Reload(ctx context.Context, cmd *cli.Command) error {
176
173
ctx, cancel := context.WithCancel(ctx)
177
174
defer cancel()
178
175
···
204
201
205
202
logger.Info("Starting Supercollider in Reload Mode")
206
203
logger.Info(fmt.Sprintf("Generating %d total events and writing them to %s",
207
-
cctx.Int("total-events"), cctx.String("output-file")))
204
+
cmd.Int("total-events"), cmd.String("output-file")))
208
205
209
206
em := events.NewEventManager(yolopersist.NewYoloPersister())
210
207
211
208
// Try to read the key from disk
212
-
keyBytes, err := os.ReadFile(cctx.String("key-file"))
209
+
keyBytes, err := os.ReadFile(cmd.String("key-file"))
213
210
if err != nil {
214
211
logger.Warn("failed to read key from disk, creating new key", "err", err.Error())
215
212
}
···
224
221
if err != nil {
225
222
log.Fatalf("failed to serialize privkey: %+v\n", err)
226
223
}
227
-
err = os.WriteFile(cctx.String("key-file"), rawKey, 0644)
224
+
err = os.WriteFile(cmd.String("key-file"), rawKey, 0644)
228
225
if err != nil {
229
226
log.Fatalf("failed to write privkey to disk: %+v\n", err)
230
227
}
···
248
245
249
246
// Initialize fake account DIDs
250
247
dids := []string{}
251
-
for i := 0; i < cctx.Int("num-users"); i++ {
252
-
did := fmt.Sprintf("did:web:%s.%s", petname.Generate(4, "-"), cctx.String("hostname"))
248
+
for i := 0; i < cmd.Int("num-users"); i++ {
249
+
did := fmt.Sprintf("did:web:%s.%s", petname.Generate(4, "-"), cmd.String("hostname"))
253
250
dids = append(dids, did)
254
251
}
255
252
256
253
// Instantiate Server
257
254
s := &Server{
258
255
Logger: logger,
259
-
EnableSSL: cctx.Bool("use-ssl"),
260
-
Host: cctx.String("hostname"),
256
+
EnableSSL: cmd.Bool("use-ssl"),
257
+
Host: cmd.String("hostname"),
261
258
262
259
RepoManager: repoman,
263
260
MultibaseKey: *vMethod.PublicKeyMultibase,
264
261
Dids: dids,
265
262
266
263
Events: em,
267
-
TotalDesiredEvents: cctx.Int("total-events"),
264
+
TotalDesiredEvents: cmd.Int("total-events"),
268
265
}
269
266
270
267
repoman.SetEventHandler(s.HandleRepoEvent, false)
···
282
279
})
283
280
e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
284
281
285
-
port := cctx.Int("port")
282
+
port := cmd.Int("port")
286
283
if port == 0 {
287
-
if cctx.Bool("use-ssl") {
284
+
if cmd.Bool("use-ssl") {
288
285
port = 443
289
286
} else {
290
287
port = 80
···
296
293
// Start a loop to subscribe to events and write them to a file
297
294
go func() {
298
295
defer wg.Done()
299
-
outFile := cctx.String("output-file")
296
+
outFile := cmd.String("output-file")
300
297
f, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY, 0644)
301
298
if err != nil {
302
299
log.Fatalf("failed to open output file: %+v\n", err)
···
368
365
369
366
listenAddress := fmt.Sprintf(":%d", port)
370
367
go func() {
371
-
if cctx.Bool("use-ssl") {
368
+
if cmd.Bool("use-ssl") {
372
369
err = e.StartAutoTLS(listenAddress)
373
370
} else {
374
371
err = e.Start(listenAddress)
···
384
381
return nil
385
382
}
386
383
387
-
func Fire(cctx *cli.Context) error {
388
-
ctx := cctx.Context
384
+
func Fire(ctx context.Context, cmd *cli.Command) error {
389
385
ctx, cancel := context.WithCancel(ctx)
390
386
defer cancel()
391
387
···
418
414
logger.Info("Starting Supercollider in Fire Mode")
419
415
420
416
// Try to read the key from disk
421
-
keyBytes, err := os.ReadFile(cctx.String("key-file"))
417
+
keyBytes, err := os.ReadFile(cmd.String("key-file"))
422
418
if err != nil {
423
419
logger.Warn("failed to read key from disk, creating new key", "err", err.Error())
424
420
}
···
433
429
if err != nil {
434
430
log.Fatalf("failed to serialize privkey: %+v\n", err)
435
431
}
436
-
err = os.WriteFile(cctx.String("key-file"), rawKey, 0644)
432
+
err = os.WriteFile(cmd.String("key-file"), rawKey, 0644)
437
433
if err != nil {
438
434
log.Fatalf("failed to write privkey to disk: %+v\n", err)
439
435
}
···
452
448
// Instantiate Server
453
449
s := &Server{
454
450
Logger: logger,
455
-
EnableSSL: cctx.Bool("use-ssl"),
456
-
Host: cctx.String("hostname"),
451
+
EnableSSL: cmd.Bool("use-ssl"),
452
+
Host: cmd.String("hostname"),
457
453
MultibaseKey: *vMethod.PublicKeyMultibase,
458
-
MaxEventsPerSecond: cctx.Int("events-per-second"),
459
-
PlaybackFile: cctx.String("input-file"),
454
+
MaxEventsPerSecond: cmd.Int("events-per-second"),
455
+
PlaybackFile: cmd.String("input-file"),
460
456
}
461
457
462
458
// HTTP Server setup and Middleware Plumbing
···
500
496
e.GET("/xrpc/com.atproto.sync.subscribeRepos", s.HandleSubscribeRepos)
501
497
e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
502
498
503
-
port := cctx.Int("port")
499
+
port := cmd.Int("port")
504
500
if port == 0 {
505
-
if cctx.Bool("use-ssl") {
501
+
if cmd.Bool("use-ssl") {
506
502
port = 443
507
503
} else {
508
504
port = 80
···
511
507
512
508
listenAddress := fmt.Sprintf(":%d", port)
513
509
go func() {
514
-
if cctx.Bool("use-ssl") {
510
+
if cmd.Bool("use-ssl") {
515
511
err = e.StartAutoTLS(listenAddress)
516
512
} else {
517
513
err = e.Start(listenAddress)
+1
-1
fakedata/accounts.go
+1
-1
fakedata/accounts.go
+3
-2
go.mod
+3
-2
go.mod
···
10
10
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
11
11
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
12
12
github.com/brianvoe/gofakeit/v6 v6.25.0
13
-
github.com/carlmjohnson/versioninfo v0.22.5
14
13
github.com/cockroachdb/pebble v1.1.2
15
14
github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2
15
+
github.com/earthboundkid/versioninfo/v2 v2.24.1
16
16
github.com/go-redis/cache/v9 v9.0.0
17
17
github.com/gocql/gocql v1.7.0
18
18
github.com/golang-jwt/jwt v3.2.2+incompatible
···
53
53
github.com/redis/go-redis/v9 v9.3.0
54
54
github.com/rivo/uniseg v0.1.0
55
55
github.com/samber/slog-echo v1.8.0
56
-
github.com/stretchr/testify v1.9.0
56
+
github.com/stretchr/testify v1.10.0
57
57
github.com/urfave/cli/v2 v2.25.7
58
+
github.com/urfave/cli/v3 v3.4.1
58
59
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
59
60
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6
60
61
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b
+6
-4
go.sum
+6
-4
go.sum
···
39
39
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
40
40
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
41
41
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
42
-
github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
43
-
github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
44
42
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
45
43
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
46
44
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
···
81
79
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
82
80
github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE=
83
81
github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
82
+
github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg=
83
+
github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw=
84
84
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
85
85
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
86
86
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
···
470
470
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
471
471
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
472
472
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
473
-
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
474
-
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
473
+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
474
+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
475
475
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
476
476
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
477
477
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
478
+
github.com/urfave/cli/v3 v3.4.1 h1:1M9UOCy5bLmGnuu1yn3t3CB4rG79Rtoxuv1sPhnm6qM=
479
+
github.com/urfave/cli/v3 v3.4.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo=
478
480
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
479
481
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
480
482
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
+7
-7
search/firehose.go
+7
-7
search/firehose.go
···
10
10
"time"
11
11
12
12
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
-
bsky "github.com/bluesky-social/indigo/api/bsky"
13
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
14
14
"github.com/bluesky-social/indigo/atproto/syntax"
15
15
"github.com/bluesky-social/indigo/backfill"
16
16
"github.com/bluesky-social/indigo/events"
17
17
"github.com/bluesky-social/indigo/events/schedulers/autoscaling"
18
18
lexutil "github.com/bluesky-social/indigo/lex/util"
19
19
"github.com/bluesky-social/indigo/repo"
20
-
typegen "github.com/whyrusleeping/cbor-gen"
21
20
22
-
"github.com/carlmjohnson/versioninfo"
21
+
"github.com/earthboundkid/versioninfo/v2"
23
22
"github.com/gorilla/websocket"
24
23
"github.com/ipfs/go-cid"
24
+
typegen "github.com/whyrusleeping/cbor-gen"
25
25
)
26
26
27
27
func (idx *Indexer) getLastCursor() (int64, error) {
···
219
219
}
220
220
221
221
switch rec := rec.(type) {
222
-
case *bsky.FeedPost:
222
+
case *appbsky.FeedPost:
223
223
rkey, err := syntax.ParseTID(parts[1])
224
224
if err != nil {
225
225
logger.Warn("skipping post record with non-TID rkey")
···
236
236
// Send the job to the bulk indexer
237
237
idx.postQueue <- &job
238
238
postsIndexed.Inc()
239
-
case *bsky.ActorProfile:
239
+
case *appbsky.ActorProfile:
240
240
if parts[1] != "self" {
241
241
return nil
242
242
}
···
330
330
}
331
331
332
332
switch rec := rec.(type) {
333
-
case *bsky.FeedPost:
333
+
case *appbsky.FeedPost:
334
334
rkey, err := syntax.ParseTID(parts[1])
335
335
if err != nil {
336
336
logger.Warn("skipping post record with non-TID rkey")
···
346
346
347
347
// Send the job to the bulk indexer
348
348
idx.postQueue <- &job
349
-
case *bsky.ActorProfile:
349
+
case *appbsky.ActorProfile:
350
350
if parts[1] != "self" {
351
351
return nil
352
352
}
+3
-3
search/server.go
+3
-3
search/server.go
···
9
9
"os"
10
10
"strings"
11
11
12
+
_ "net/http/pprof" // For pprof in the metrics server
13
+
12
14
"github.com/bluesky-social/indigo/atproto/identity"
13
15
14
-
"github.com/carlmjohnson/versioninfo"
16
+
"github.com/earthboundkid/versioninfo/v2"
15
17
"github.com/labstack/echo/v4"
16
18
"github.com/labstack/echo/v4/middleware"
17
19
es "github.com/opensearch-project/opensearch-go/v2"
18
20
"github.com/prometheus/client_golang/prometheus/promhttp"
19
21
slogecho "github.com/samber/slog-echo"
20
22
"go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho"
21
-
22
-
_ "net/http/pprof" // For pprof in the metrics server
23
23
)
24
24
25
25
type LastSeq struct {
+3
-3
util/svcutil/logger.go
+3
-3
util/svcutil/logger.go
···
5
5
"log/slog"
6
6
"strings"
7
7
8
-
"github.com/urfave/cli/v2"
8
+
"github.com/urfave/cli/v3"
9
9
)
10
10
11
-
func ConfigLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
11
+
func ConfigLogger(cmd *cli.Command, writer io.Writer) *slog.Logger {
12
12
var level slog.Level
13
-
switch strings.ToLower(cctx.String("log-level")) {
13
+
switch strings.ToLower(cmd.String("log-level")) {
14
14
case "error":
15
15
level = slog.LevelError
16
16
case "warn":