this repo has no description
feat: initial commit
authored by
Till Klampaeckel
and committed by
Tangled
b13dd062
5b967e6c
+2846
-1
+21
.gitignore
+21
.gitignore
···
1
+
# SQLite database files (user data, never commit)
2
+
/firehose.db
3
+
/firehose.db-shm
4
+
/firehose.db-wal
5
+
/firehose.db.bak.*
6
+
7
+
# Local backup artifacts from scripts/migrate.sh
8
+
/*.bak.*
9
+
10
+
# One-offs
11
+
/scripts/migrate.sh
12
+
/NOTES.md
13
+
14
+
# Compiled binary
15
+
/bin/
16
+
17
+
# ent codegen output: keep only the hand-written schemas and generate.go;
18
+
# everything else under /ent is regenerated by `go generate ./ent`.
19
+
/ent/*
20
+
!/ent/schema/
21
+
!/ent/generate.go
+19
.tangled/workflows/ci.yml
+19
.tangled/workflows/ci.yml
···
1
+
when:
2
+
- event: ["push", "pull_request"]
3
+
branch: main
4
+
5
+
engine: nixery
6
+
7
+
dependencies:
8
+
nixpkgs:
9
+
- go
10
+
11
+
steps:
12
+
- name: regenerate ent client
13
+
command: make generate
14
+
15
+
- name: vet
16
+
command: make vet
17
+
18
+
- name: build
19
+
command: make build
+22
Makefile
+22
Makefile
···
1
+
DB ?= firehose.db
2
+
LISTEN ?= :8080
3
+
4
+
.PHONY: run-dev build generate vet tidy clean
5
+
6
+
run-dev: generate
7
+
go run ./cmd/firehose --db $(DB) --listen $(LISTEN) --debug
8
+
9
+
build: generate
10
+
go build -o bin/firehose ./cmd/firehose
11
+
12
+
generate:
13
+
go generate ./ent/...
14
+
15
+
vet:
16
+
go vet ./...
17
+
18
+
tidy:
19
+
go mod tidy
20
+
21
+
clean:
22
+
rm -rf bin $(DB) $(DB)-shm $(DB)-wal
+30
-1
README.md
+30
-1
README.md
···
1
1
# tangled-repo-firehose
2
2
3
-
A little experiment with jetstream to visualize repositories and stargazers on [tangled.org](https://tangled.org).
3
+
A little experiment with jetstream to visualize repositories and stargazers on [tangled.org](https://tangled.org).
4
+
5
+
## What is it?
6
+
7
+
I was trying to find popular repositories on tangled.org, but so far, no such method or lexicon exists. Which led me down to, how do you figure out what people are doing right now.
8
+
9
+
And here we go.
10
+
11
+
If you want to see a demo, check here: [https://bsky.app/profile/chown.de/post/3mkvet2xfis2q](https://bsky.app/profile/chown.de/post/3mkvet2xfis2q).
12
+
13
+
### How does it work?
14
+
15
+
> **Caveat**
16
+
> There's (currently) no atproto operation to enumerate all users or repositories, so coverage grows from what the firehose shows live plus various walks across DIDs. Star counts will not match `tangled.org`.
17
+
18
+
The app subscribes to Bluesky's jetstream and extracts `sh.tangled.*`. From there, background workers will resolve DIDs to handles, fetch language stats from knots, discover unseen repos referenced by stars, and backfill historical stars per known DID.
19
+
20
+
Persistence is SQLite via [entgo.io](https://entgo.io).
21
+
22
+
### Run it
23
+
24
+
```bash
25
+
make run-dev # localhost:8080
26
+
```
27
+
28
+
Inspect:
29
+
30
+
- `/`: the repositories (alternative: `/repos.json`)
31
+
- `/knots`: known knots (and some status)
32
+
- `/handles`: resolved handles
+229
cmd/firehose/main.go
+229
cmd/firehose/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"log/slog"
7
+
"net/http"
8
+
"net/mail"
9
+
"os"
10
+
"os/signal"
11
+
"syscall"
12
+
"time"
13
+
14
+
"tangled.sh/chown.de/tangled-repo-firehose/firehose"
15
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
16
+
"tangled.sh/chown.de/tangled-repo-firehose/web"
17
+
18
+
"github.com/bluesky-social/indigo/atproto/identity"
19
+
"github.com/urfave/cli/v3"
20
+
)
21
+
22
+
var (
23
+
logger *slog.Logger
24
+
)
25
+
26
+
func main() {
27
+
app := cli.Command{
28
+
Authors: []any{
29
+
mail.Address{Name: "Till Klampaeckel"},
30
+
},
31
+
Flags: []cli.Flag{
32
+
&cli.StringFlag{
33
+
Name: "db",
34
+
Value: "firehose.db",
35
+
Usage: "SQLite database path",
36
+
},
37
+
&cli.StringFlag{
38
+
Name: "jetstream",
39
+
Value: "wss://jetstream2.us-east.bsky.network/subscribe",
40
+
Usage: "Jetstream WS endpoint",
41
+
},
42
+
&cli.IntFlag{
43
+
Name: "lang-queue",
44
+
Value: 256,
45
+
Usage: "Languages enrichment queue size",
46
+
},
47
+
&cli.BoolFlag{
48
+
Name: "debug",
49
+
Value: false,
50
+
Usage: "Enable debug logging",
51
+
},
52
+
&cli.StringFlag{
53
+
Name: "listen",
54
+
Value: ":8080",
55
+
Usage: "HTTP listen address (empty to disable)",
56
+
},
57
+
&cli.BoolFlag{
58
+
Name: "migrate-dry-run",
59
+
Value: false,
60
+
Usage: "Print the SQL auto-migration would run, then exit (no DDL applied)",
61
+
},
62
+
},
63
+
Before: func(ctx context.Context, c *cli.Command) (context.Context, error) {
64
+
level := slog.LevelInfo
65
+
if c.Bool("debug") {
66
+
level = slog.LevelDebug
67
+
}
68
+
logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))
69
+
return ctx, nil
70
+
},
71
+
Action: func(ctx context.Context, c *cli.Command) error {
72
+
if c.Bool("migrate-dry-run") {
73
+
return store.MigrationPlan(ctx, c.String("db"), os.Stdout)
74
+
}
75
+
s, err := store.Open(c.String("db"))
76
+
if err != nil {
77
+
logger.Error("open store", "err", err, "path", c.String("db"))
78
+
return err
79
+
}
80
+
defer s.Close()
81
+
82
+
ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
83
+
defer cancel()
84
+
85
+
cur, err := s.GetCursor(ctx)
86
+
if err != nil {
87
+
logger.Error("get cursor", "err", err)
88
+
return err
89
+
}
90
+
if cur == 0 {
91
+
logger.Info("no cursor saved, starting from now")
92
+
} else {
93
+
logger.Info("resuming", "cursor", cur, "lag", time.Since(time.UnixMicro(cur)).Round(time.Second))
94
+
}
95
+
96
+
dir := identity.DefaultDirectory()
97
+
98
+
langs := firehose.NewLanguagesWorker(s, logger.With("component", "languages"), c.Int("lang-queue"))
99
+
go langs.Run(ctx)
100
+
101
+
handles := firehose.NewHandlesWorker(s, dir, logger.With("component", "handles"), c.Int("lang-queue"))
102
+
go handles.Run(ctx)
103
+
104
+
discovery := firehose.NewDiscoveryWorker(s, dir, logger.With("component", "discovery"), langs, handles, c.Int("lang-queue"))
105
+
go discovery.Run(ctx)
106
+
107
+
starsBackfill := firehose.NewStarsBackfillWorker(s, dir, logger.With("component", "stars-backfill"), discovery, c.Int("lang-queue"))
108
+
discovery.SetStarsBackfill(starsBackfill)
109
+
go starsBackfill.Run(ctx)
110
+
111
+
// Periodic backfill — fires immediately on boot and every 6h after.
112
+
// Combined with the worker-level SeenTTL, stale items naturally
113
+
// re-enter the queue without a daemon restart.
114
+
const backfillInterval = 6 * time.Hour
115
+
go runBackfillLoop(ctx, backfillInterval, func() {
116
+
enqueueLanguageBackfill(ctx, s, langs, logger)
117
+
})
118
+
go runBackfillLoop(ctx, backfillInterval, func() {
119
+
enqueueHandleBackfill(ctx, s, handles, logger)
120
+
})
121
+
go runBackfillLoop(ctx, backfillInterval, func() {
122
+
enqueueStarsBackfill(ctx, s, starsBackfill, logger)
123
+
})
124
+
125
+
if addr := c.String("listen"); addr != "" {
126
+
go runHTTP(ctx, addr, s, logger.With("component", "http"))
127
+
}
128
+
129
+
consumer := &firehose.Consumer{
130
+
Endpoint: c.String("jetstream"),
131
+
WantedCollections: []string{"sh.tangled.repo", "sh.tangled.feed.star"},
132
+
Handler: firehose.Handler(s, langs, handles, discovery, starsBackfill, logger.With("component", "handler")),
133
+
Logger: logger.With("component", "jetstream"),
134
+
}
135
+
if err := consumer.Run(ctx, cur); err != nil && ctx.Err() == nil {
136
+
return err
137
+
}
138
+
logger.Info("shutdown")
139
+
return nil
140
+
},
141
+
}
142
+
143
+
if err := app.Run(context.Background(), os.Args); err != nil {
144
+
slog.Error("demon crashed", slog.Any("err", err))
145
+
os.Exit(1)
146
+
}
147
+
}
148
+
149
+
// runBackfillLoop fires fn immediately, then every interval until ctx is
150
+
// cancelled. Worker-level SeenTTL deduplicates redundant work between ticks.
151
+
func runBackfillLoop(ctx context.Context, interval time.Duration, fn func()) {
152
+
fn()
153
+
ticker := time.NewTicker(interval)
154
+
defer ticker.Stop()
155
+
for {
156
+
select {
157
+
case <-ctx.Done():
158
+
return
159
+
case <-ticker.C:
160
+
fn()
161
+
}
162
+
}
163
+
}
164
+
165
+
func runHTTP(ctx context.Context, addr string, s *store.Store, logger *slog.Logger) {
166
+
srv := &http.Server{
167
+
Addr: addr,
168
+
Handler: web.Handler(s),
169
+
ReadHeaderTimeout: 5 * time.Second,
170
+
}
171
+
go func() {
172
+
<-ctx.Done()
173
+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
174
+
defer cancel()
175
+
_ = srv.Shutdown(shutdownCtx)
176
+
}()
177
+
logger.Info("listening", "addr", addr)
178
+
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
179
+
logger.Error("http", "err", err)
180
+
}
181
+
}
182
+
183
+
func enqueueLanguageBackfill(ctx context.Context, s *store.Store, langs *firehose.LanguagesWorker, logger *slog.Logger) {
184
+
repos, err := s.ReposNeedingLanguages(ctx, 7*24*time.Hour, 1000)
185
+
if err != nil {
186
+
logger.Warn("backfill query", "err", err)
187
+
return
188
+
}
189
+
if len(repos) == 0 {
190
+
return
191
+
}
192
+
logger.Info("language backfill", "count", len(repos))
193
+
for _, r := range repos {
194
+
if r.Knot == "" {
195
+
continue
196
+
}
197
+
langs.Enqueue(r.AtURI, r.Knot, r.DID, r.Name)
198
+
}
199
+
}
200
+
201
+
func enqueueHandleBackfill(ctx context.Context, s *store.Store, handles *firehose.HandlesWorker, logger *slog.Logger) {
202
+
dids, err := s.DIDsNeedingHandles(ctx, 7*24*time.Hour, 1000)
203
+
if err != nil {
204
+
logger.Warn("handle backfill query", "err", err)
205
+
return
206
+
}
207
+
if len(dids) == 0 {
208
+
return
209
+
}
210
+
logger.Info("handle backfill", "count", len(dids))
211
+
for _, did := range dids {
212
+
handles.Enqueue(did)
213
+
}
214
+
}
215
+
216
+
func enqueueStarsBackfill(ctx context.Context, s *store.Store, w *firehose.StarsBackfillWorker, logger *slog.Logger) {
217
+
dids, err := s.AllKnownDIDs(ctx, 10000)
218
+
if err != nil {
219
+
logger.Warn("stars backfill query", "err", err)
220
+
return
221
+
}
222
+
if len(dids) == 0 {
223
+
return
224
+
}
225
+
logger.Info("stars backfill", "count", len(dids))
226
+
for _, did := range dids {
227
+
w.Enqueue(did)
228
+
}
229
+
}
+3
ent/generate.go
+3
ent/generate.go
+30
ent/schema/cursor.go
+30
ent/schema/cursor.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect/entsql"
6
+
"entgo.io/ent/schema"
7
+
"entgo.io/ent/schema/field"
8
+
)
9
+
10
+
// Cursor is the single-row Jetstream resume cursor table. The application
11
+
// always upserts id=1 (matching the original CHECK(id=1) invariant). ent
12
+
// can't model CHECK constraints, so the constraint goes away on first
13
+
// migration; the data semantic is preserved by the application.
14
+
type Cursor struct {
15
+
ent.Schema
16
+
}
17
+
18
+
func (Cursor) Annotations() []schema.Annotation {
19
+
return []schema.Annotation{
20
+
entsql.Annotation{Table: "cursor"},
21
+
}
22
+
}
23
+
24
+
func (Cursor) Fields() []ent.Field {
25
+
return []ent.Field{
26
+
field.Int("id").SchemaType(sqliteInteger),
27
+
field.Int64("cursor").SchemaType(sqliteInteger),
28
+
field.Int64("updated").SchemaType(sqliteInteger),
29
+
}
30
+
}
+26
ent/schema/discoveryfailed.go
+26
ent/schema/discoveryfailed.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect/entsql"
6
+
"entgo.io/ent/schema"
7
+
"entgo.io/ent/schema/field"
8
+
)
9
+
10
+
type DiscoveryFailed struct {
11
+
ent.Schema
12
+
}
13
+
14
+
func (DiscoveryFailed) Annotations() []schema.Annotation {
15
+
return []schema.Annotation{
16
+
entsql.Annotation{Table: "discovery_failed"},
17
+
}
18
+
}
19
+
20
+
func (DiscoveryFailed) Fields() []ent.Field {
21
+
return []ent.Field{
22
+
field.String("id").StorageKey("at_uri").Immutable().SchemaType(sqliteText),
23
+
field.Int64("failed_at").SchemaType(sqliteInteger),
24
+
field.String("reason").Optional().Nillable().SchemaType(sqliteText),
25
+
}
26
+
}
+33
ent/schema/handle.go
+33
ent/schema/handle.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect/entsql"
6
+
"entgo.io/ent/schema"
7
+
"entgo.io/ent/schema/field"
8
+
"entgo.io/ent/schema/index"
9
+
)
10
+
11
+
type Handle struct {
12
+
ent.Schema
13
+
}
14
+
15
+
func (Handle) Annotations() []schema.Annotation {
16
+
return []schema.Annotation{
17
+
entsql.Annotation{Table: "handles"},
18
+
}
19
+
}
20
+
21
+
func (Handle) Fields() []ent.Field {
22
+
return []ent.Field{
23
+
field.String("id").StorageKey("did").Immutable().SchemaType(sqliteText),
24
+
field.String("handle").SchemaType(sqliteText),
25
+
field.Int64("refreshed_at").SchemaType(sqliteInteger),
26
+
}
27
+
}
28
+
29
+
func (Handle) Indexes() []ent.Index {
30
+
return []ent.Index{
31
+
index.Fields("handle").StorageKey("idx_handles_handle"),
32
+
}
33
+
}
+33
ent/schema/knot.go
+33
ent/schema/knot.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect/entsql"
6
+
"entgo.io/ent/schema"
7
+
"entgo.io/ent/schema/field"
8
+
)
9
+
10
+
type Knot struct {
11
+
ent.Schema
12
+
}
13
+
14
+
func (Knot) Annotations() []schema.Annotation {
15
+
return []schema.Annotation{
16
+
entsql.Annotation{Table: "knots"},
17
+
}
18
+
}
19
+
20
+
func (Knot) Fields() []ent.Field {
21
+
return []ent.Field{
22
+
field.String("id").StorageKey("host").Immutable().SchemaType(sqliteText),
23
+
field.Int64("first_seen").SchemaType(sqliteInteger),
24
+
field.Int64("last_seen").SchemaType(sqliteInteger),
25
+
field.Int64("last_ok_at").Optional().Nillable().SchemaType(sqliteInteger),
26
+
field.Int64("last_error_at").Optional().Nillable().SchemaType(sqliteInteger),
27
+
field.String("last_error").Optional().Nillable().SchemaType(sqliteText),
28
+
field.Int("consecutive_errors").Default(0).SchemaType(sqliteInteger),
29
+
// `disabled` is INTEGER 0/1 in the existing DB. Using field.Int avoids
30
+
// ent emitting a `bool` type that Atlas would flag as a diff.
31
+
field.Int("disabled").Default(0).SchemaType(sqliteInteger),
32
+
}
33
+
}
+55
ent/schema/repo.go
+55
ent/schema/repo.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect"
6
+
"entgo.io/ent/dialect/entsql"
7
+
"entgo.io/ent/schema"
8
+
"entgo.io/ent/schema/field"
9
+
"entgo.io/ent/schema/index"
10
+
)
11
+
12
+
// Match the existing column types verbatim so Atlas's textual diff stops
13
+
// flagging them. SQLite is type-affinity-based so the values themselves are
14
+
// stored identically; this is purely about matching the CREATE TABLE text.
15
+
var (
16
+
sqliteText = map[string]string{dialect.SQLite: "TEXT"}
17
+
sqliteInteger = map[string]string{dialect.SQLite: "INTEGER"}
18
+
)
19
+
20
+
type Repo struct {
21
+
ent.Schema
22
+
}
23
+
24
+
func (Repo) Annotations() []schema.Annotation {
25
+
return []schema.Annotation{
26
+
entsql.Annotation{Table: "repos"},
27
+
}
28
+
}
29
+
30
+
func (Repo) Fields() []ent.Field {
31
+
return []ent.Field{
32
+
field.String("id").StorageKey("at_uri").Immutable().SchemaType(sqliteText),
33
+
field.String("did").SchemaType(sqliteText),
34
+
field.String("rkey").SchemaType(sqliteText),
35
+
field.String("name").SchemaType(sqliteText),
36
+
field.String("knot").SchemaType(sqliteText),
37
+
field.String("description").Optional().Nillable().SchemaType(sqliteText),
38
+
field.String("topics").Optional().Nillable().SchemaType(sqliteText),
39
+
field.String("website").Optional().Nillable().SchemaType(sqliteText),
40
+
field.String("source").Optional().Nillable().SchemaType(sqliteText),
41
+
field.String("spindle").Optional().Nillable().SchemaType(sqliteText),
42
+
field.String("repo_did").Optional().Nillable().SchemaType(sqliteText),
43
+
field.Int64("created_at").SchemaType(sqliteInteger),
44
+
field.Int64("seen_at").SchemaType(sqliteInteger),
45
+
field.String("languages").Optional().Nillable().SchemaType(sqliteText),
46
+
field.Int64("language_at").Optional().Nillable().SchemaType(sqliteInteger),
47
+
}
48
+
}
49
+
50
+
func (Repo) Indexes() []ent.Index {
51
+
return []ent.Index{
52
+
index.Fields("did", "name").StorageKey("idx_repos_did_name"),
53
+
index.Fields("created_at").StorageKey("idx_repos_created_at"),
54
+
}
55
+
}
+33
ent/schema/star.go
+33
ent/schema/star.go
···
1
+
package schema
2
+
3
+
import (
4
+
"entgo.io/ent"
5
+
"entgo.io/ent/dialect/entsql"
6
+
"entgo.io/ent/schema"
7
+
"entgo.io/ent/schema/field"
8
+
"entgo.io/ent/schema/index"
9
+
)
10
+
11
+
type Star struct {
12
+
ent.Schema
13
+
}
14
+
15
+
func (Star) Annotations() []schema.Annotation {
16
+
return []schema.Annotation{
17
+
entsql.Annotation{Table: "stars"},
18
+
}
19
+
}
20
+
21
+
func (Star) Fields() []ent.Field {
22
+
return []ent.Field{
23
+
field.String("id").StorageKey("at_uri").Immutable().SchemaType(sqliteText),
24
+
field.String("subject").SchemaType(sqliteText),
25
+
field.Int64("created_at").SchemaType(sqliteInteger),
26
+
}
27
+
}
28
+
29
+
func (Star) Indexes() []ent.Index {
30
+
return []ent.Index{
31
+
index.Fields("subject").StorageKey("idx_stars_subject"),
32
+
}
33
+
}
+209
firehose/discover.go
+209
firehose/discover.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"fmt"
8
+
"log/slog"
9
+
"net/http"
10
+
"time"
11
+
12
+
"github.com/bluesky-social/indigo/atproto/identity"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/bluesky-social/indigo/xrpc"
15
+
16
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
17
+
)
18
+
19
+
// recordNotFoundRetry is how long we trust the discovery_failed entry before
20
+
// re-attempting — gives a deleted record a chance to be re-created.
21
+
const recordNotFoundRetry = 7 * 24 * time.Hour
22
+
23
+
// DiscoveryWorker fetches sh.tangled.repo records we don't have in the index
24
+
// — typically because a star event referenced a repo we've never seen. It
25
+
// resolves the owner's PDS via the identity directory and queries
26
+
// com.atproto.repo.getRecord, then chains into handles, languages, and
27
+
// stars-backfill enrichers.
28
+
type DiscoveryWorker struct {
29
+
*Worker[string]
30
+
store *store.Store
31
+
dir identity.Directory
32
+
httpClient *http.Client
33
+
langs LanguagesEnqueuer
34
+
handles HandlesEnqueuer
35
+
starsBackfill StarsBackfillEnqueuer
36
+
}
37
+
38
+
func NewDiscoveryWorker(s *store.Store, dir identity.Directory, logger *slog.Logger, langs LanguagesEnqueuer, handles HandlesEnqueuer, queueSize int) *DiscoveryWorker {
39
+
w := &DiscoveryWorker{
40
+
store: s,
41
+
dir: dir,
42
+
httpClient: &http.Client{Timeout: 20 * time.Second},
43
+
langs: langs,
44
+
handles: handles,
45
+
}
46
+
w.Worker = NewWorker(WorkerConfig[string]{
47
+
Name: "discovery",
48
+
Logger: logger,
49
+
Process: w.process,
50
+
QueueSize: queueSize,
51
+
SeenTTL: 6 * time.Hour,
52
+
MinDelay: 250 * time.Millisecond,
53
+
})
54
+
return w
55
+
}
56
+
57
+
// SetStarsBackfill wires the back-edge so newly-discovered repo owners get
58
+
// their historical stars pulled immediately. Late-bound to break the
59
+
// construction cycle.
60
+
func (w *DiscoveryWorker) SetStarsBackfill(sb StarsBackfillEnqueuer) {
61
+
w.starsBackfill = sb
62
+
}
63
+
64
+
// EnqueueAtURI accepts a repo at-uri (typically the subject of a star).
65
+
// Non-blocking: drops on full queue. Use from live firehose handlers.
66
+
func (w *DiscoveryWorker) EnqueueAtURI(uri string) {
67
+
if !validRepoAtURI(uri) {
68
+
return
69
+
}
70
+
w.Worker.Enqueue(uri)
71
+
}
72
+
73
+
// EnqueueAtURIWait blocks until accepted or ctx is cancelled. Use from chained
74
+
// producers (e.g. stars-backfill bursting hundreds of subjects).
75
+
func (w *DiscoveryWorker) EnqueueAtURIWait(ctx context.Context, uri string) {
76
+
if !validRepoAtURI(uri) {
77
+
return
78
+
}
79
+
w.Worker.EnqueueWait(ctx, uri)
80
+
}
81
+
82
+
func validRepoAtURI(uri string) bool {
83
+
a, err := syntax.ParseATURI(uri)
84
+
return err == nil && a.Collection().String() == "sh.tangled.repo"
85
+
}
86
+
87
+
func (w *DiscoveryWorker) process(ctx context.Context, atURI string) error {
88
+
if has, err := w.store.HasRepo(ctx, atURI); err != nil {
89
+
return err
90
+
} else if has {
91
+
return nil
92
+
}
93
+
if failed, err := w.store.DiscoveryFailedRecently(ctx, atURI, recordNotFoundRetry); err != nil {
94
+
return err
95
+
} else if failed {
96
+
return nil
97
+
}
98
+
99
+
a, err := syntax.ParseATURI(atURI)
100
+
if err != nil {
101
+
return err
102
+
}
103
+
did, err := a.Authority().AsDID()
104
+
if err != nil {
105
+
return fmt.Errorf("authority not a DID: %w", err)
106
+
}
107
+
rkey := a.RecordKey().String()
108
+
109
+
ident, err := w.dir.LookupDID(ctx, did)
110
+
if err != nil {
111
+
return fmt.Errorf("identity lookup: %w", err)
112
+
}
113
+
pds := ident.PDSEndpoint()
114
+
if pds == "" {
115
+
return fmt.Errorf("no PDS endpoint for %s", did)
116
+
}
117
+
118
+
rec, err := w.fetchRepoRecord(ctx, pds, did.String(), rkey)
119
+
if err != nil {
120
+
if isRecordNotFound(err) {
121
+
// Deleted upstream — persist so we don't re-attempt for a week.
122
+
return w.store.MarkDiscoveryFailed(ctx, atURI, err.Error())
123
+
}
124
+
return err
125
+
}
126
+
127
+
createdAt, err := time.Parse(time.RFC3339Nano, rec.CreatedAt)
128
+
if err != nil {
129
+
createdAt = time.Now()
130
+
}
131
+
r := store.Repo{
132
+
AtURI: atURI,
133
+
DID: did.String(),
134
+
Rkey: rkey,
135
+
Name: rec.Name,
136
+
Knot: rec.Knot,
137
+
Description: rec.Description,
138
+
Topics: rec.Topics,
139
+
Website: rec.Website,
140
+
Source: rec.Source,
141
+
Spindle: rec.Spindle,
142
+
RepoDID: rec.RepoDID,
143
+
CreatedAt: createdAt,
144
+
SeenAt: time.Now(),
145
+
}
146
+
if err := w.store.UpsertRepo(ctx, r); err != nil {
147
+
return err
148
+
}
149
+
if err := w.store.UpsertKnot(ctx, rec.Knot); err != nil {
150
+
w.logger.Warn("upsert knot", "err", err, "knot", rec.Knot)
151
+
}
152
+
w.logger.Info("discovered", "did", did, "name", rec.Name, "knot", rec.Knot)
153
+
154
+
if w.langs != nil && rec.Knot != "" {
155
+
w.langs.Enqueue(atURI, rec.Knot, did.String(), rec.Name)
156
+
}
157
+
if w.handles != nil {
158
+
w.handles.Enqueue(did.String())
159
+
}
160
+
if w.starsBackfill != nil {
161
+
w.starsBackfill.Enqueue(did.String())
162
+
}
163
+
return nil
164
+
}
165
+
166
+
// isRecordNotFound returns true for XRPC responses that indicate the record
167
+
// has been deleted upstream. Bluesky PDSes return either 404 or 400 depending
168
+
// on the implementation — both with the "RecordNotFound" error name.
169
+
func isRecordNotFound(err error) bool {
170
+
var xe *xrpc.Error
171
+
if !errors.As(err, &xe) {
172
+
return false
173
+
}
174
+
if xe.StatusCode == http.StatusNotFound {
175
+
return true
176
+
}
177
+
if xe.StatusCode == http.StatusBadRequest {
178
+
var xrpcErr *xrpc.XRPCError
179
+
if errors.As(err, &xrpcErr) && xrpcErr.ErrStr == "RecordNotFound" {
180
+
return true
181
+
}
182
+
}
183
+
return false
184
+
}
185
+
186
+
func (w *DiscoveryWorker) fetchRepoRecord(ctx context.Context, pds, did, rkey string) (*repoRecord, error) {
187
+
xc := &xrpc.Client{Client: w.httpClient, Host: pds}
188
+
var out struct {
189
+
URI string `json:"uri"`
190
+
CID string `json:"cid"`
191
+
Value json.RawMessage `json:"value"`
192
+
}
193
+
err := xc.Do(ctx, xrpc.Query, "", "com.atproto.repo.getRecord", map[string]any{
194
+
"repo": did,
195
+
"collection": "sh.tangled.repo",
196
+
"rkey": rkey,
197
+
}, nil, &out)
198
+
if err != nil {
199
+
return nil, err
200
+
}
201
+
var rec repoRecord
202
+
if err := json.Unmarshal(out.Value, &rec); err != nil {
203
+
return nil, err
204
+
}
205
+
if rec.Name == "" || rec.Knot == "" {
206
+
return nil, fmt.Errorf("record missing required fields")
207
+
}
208
+
return &rec, nil
209
+
}
+167
firehose/handler.go
+167
firehose/handler.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"log/slog"
8
+
"time"
9
+
10
+
"github.com/bluesky-social/jetstream/pkg/models"
11
+
12
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
13
+
)
14
+
15
+
// repoRecord is the on-wire shape of an sh.tangled.repo record.
16
+
type repoRecord struct {
17
+
Name string `json:"name"`
18
+
Knot string `json:"knot"`
19
+
Description string `json:"description,omitempty"`
20
+
Website string `json:"website,omitempty"`
21
+
Source string `json:"source,omitempty"`
22
+
Spindle string `json:"spindle,omitempty"`
23
+
Topics []string `json:"topics,omitempty"`
24
+
RepoDID string `json:"repoDid,omitempty"`
25
+
CreatedAt string `json:"createdAt"`
26
+
}
27
+
28
+
// starRecord is the on-wire shape of an sh.tangled.feed.star record.
29
+
type starRecord struct {
30
+
Subject string `json:"subject"`
31
+
CreatedAt string `json:"createdAt"`
32
+
}
33
+
34
+
// LanguagesEnqueuer accepts (atURI, knot, did, name) for async language
35
+
// enrichment. *LanguagesWorker satisfies this; pass nil to disable enrichment.
36
+
type LanguagesEnqueuer interface {
37
+
Enqueue(atURI, knot, did, name string)
38
+
}
39
+
40
+
// HandlesEnqueuer accepts a DID for async handle resolution. *HandlesWorker
41
+
// satisfies this; pass nil to disable.
42
+
type HandlesEnqueuer interface {
43
+
Enqueue(did string)
44
+
}
45
+
46
+
// DiscoveryEnqueuer accepts a repo at-uri for async discovery (PDS fetch).
47
+
// *DiscoveryWorker satisfies this; pass nil to disable. Live firehose handlers
48
+
// use EnqueueAtURI (non-blocking, drops on full); chained producers use
49
+
// EnqueueAtURIWait (blocks for backpressure).
50
+
type DiscoveryEnqueuer interface {
51
+
EnqueueAtURI(uri string)
52
+
EnqueueAtURIWait(ctx context.Context, uri string)
53
+
}
54
+
55
+
// StarsBackfillEnqueuer accepts a DID whose historical stars should be
56
+
// pulled. *StarsBackfillWorker satisfies this; pass nil to disable.
57
+
type StarsBackfillEnqueuer interface {
58
+
Enqueue(did string)
59
+
}
60
+
61
+
// Handler routes commit events to per-collection handlers. Non-commit events
62
+
// and unknown collections still advance the cursor so they aren't replayed on
63
+
// restart.
64
+
func Handler(s *store.Store, langs LanguagesEnqueuer, handles HandlesEnqueuer, discovery DiscoveryEnqueuer, starsBackfill StarsBackfillEnqueuer, logger *slog.Logger) HandlerFunc {
65
+
return func(ctx context.Context, ev *models.Event) error {
66
+
if ev.Kind != "commit" || ev.Commit == nil {
67
+
return s.AdvanceCursor(ctx, ev.TimeUS)
68
+
}
69
+
switch ev.Commit.Collection {
70
+
case "sh.tangled.repo":
71
+
return handleRepo(ctx, s, langs, handles, starsBackfill, logger, ev)
72
+
case "sh.tangled.feed.star":
73
+
return handleStar(ctx, s, discovery, starsBackfill, logger, ev)
74
+
default:
75
+
return s.AdvanceCursor(ctx, ev.TimeUS)
76
+
}
77
+
}
78
+
}
79
+
80
+
func handleRepo(ctx context.Context, s *store.Store, langs LanguagesEnqueuer, handles HandlesEnqueuer, starsBackfill StarsBackfillEnqueuer, logger *slog.Logger, ev *models.Event) error {
81
+
atURI := fmt.Sprintf("at://%s/%s/%s", ev.Did, ev.Commit.Collection, ev.Commit.RKey)
82
+
switch ev.Commit.Operation {
83
+
case "create", "update":
84
+
var rec repoRecord
85
+
if err := json.Unmarshal(ev.Commit.Record, &rec); err != nil {
86
+
return fmt.Errorf("decode repo record: %w", err)
87
+
}
88
+
createdAt, err := time.Parse(time.RFC3339Nano, rec.CreatedAt)
89
+
if err != nil {
90
+
createdAt = time.UnixMicro(ev.TimeUS)
91
+
}
92
+
r := store.Repo{
93
+
AtURI: atURI,
94
+
DID: ev.Did,
95
+
Rkey: ev.Commit.RKey, //nolint:misspell
96
+
Name: rec.Name,
97
+
Knot: rec.Knot,
98
+
Description: rec.Description,
99
+
Topics: rec.Topics,
100
+
Website: rec.Website,
101
+
Source: rec.Source,
102
+
Spindle: rec.Spindle,
103
+
RepoDID: rec.RepoDID,
104
+
CreatedAt: createdAt,
105
+
SeenAt: time.Now(),
106
+
}
107
+
if err := s.ApplyRepoUpsert(ctx, r, ev.TimeUS); err != nil {
108
+
return err
109
+
}
110
+
if err := s.UpsertKnot(ctx, rec.Knot); err != nil {
111
+
logger.Warn("upsert knot", "err", err, "knot", rec.Knot)
112
+
}
113
+
logger.Info("repo "+ev.Commit.Operation, "did", ev.Did, "name", rec.Name, "knot", rec.Knot)
114
+
if langs != nil && rec.Knot != "" {
115
+
langs.Enqueue(atURI, rec.Knot, ev.Did, rec.Name)
116
+
}
117
+
if handles != nil {
118
+
handles.Enqueue(ev.Did)
119
+
}
120
+
if starsBackfill != nil {
121
+
starsBackfill.Enqueue(ev.Did)
122
+
}
123
+
return nil
124
+
case "delete":
125
+
logger.Info("repo delete", "at_uri", atURI)
126
+
return s.ApplyRepoDelete(ctx, atURI, ev.TimeUS)
127
+
default:
128
+
return s.AdvanceCursor(ctx, ev.TimeUS)
129
+
}
130
+
}
131
+
132
+
func handleStar(ctx context.Context, s *store.Store, discovery DiscoveryEnqueuer, starsBackfill StarsBackfillEnqueuer, logger *slog.Logger, ev *models.Event) error {
133
+
atURI := fmt.Sprintf("at://%s/%s/%s", ev.Did, ev.Commit.Collection, ev.Commit.RKey)
134
+
switch ev.Commit.Operation {
135
+
case "create", "update":
136
+
var rec starRecord
137
+
if err := json.Unmarshal(ev.Commit.Record, &rec); err != nil {
138
+
return fmt.Errorf("decode star record: %w", err)
139
+
}
140
+
if rec.Subject == "" {
141
+
return s.AdvanceCursor(ctx, ev.TimeUS)
142
+
}
143
+
createdAt, err := time.Parse(time.RFC3339Nano, rec.CreatedAt)
144
+
if err != nil {
145
+
createdAt = time.UnixMicro(ev.TimeUS)
146
+
}
147
+
if err := s.ApplyStarCreate(ctx, atURI, rec.Subject, createdAt, ev.TimeUS); err != nil {
148
+
return err
149
+
}
150
+
logger.Info("star", "by", ev.Did, "subject", rec.Subject)
151
+
// If the starred repo isn't in our index yet, queue discovery. The
152
+
// worker checks HasRepo before fetching, so duplicates are cheap.
153
+
if discovery != nil {
154
+
discovery.EnqueueAtURI(rec.Subject)
155
+
}
156
+
// First time we see this user — also pull their historical stars.
157
+
if starsBackfill != nil {
158
+
starsBackfill.Enqueue(ev.Did)
159
+
}
160
+
return nil
161
+
case "delete":
162
+
logger.Info("unstar", "at_uri", atURI)
163
+
return s.ApplyStarDelete(ctx, atURI, ev.TimeUS)
164
+
default:
165
+
return s.AdvanceCursor(ctx, ev.TimeUS)
166
+
}
167
+
}
+54
firehose/handles.go
+54
firehose/handles.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"time"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/identity"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
12
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
13
+
)
14
+
15
+
// HandlesWorker resolves DID → handle via indigo's identity.Directory.
16
+
// Bidirectional verification is intentionally bypassed by reading the
17
+
// declared handle from the DID document directly; an aggregator just needs a
18
+
// human-friendly label, and the directory still gives us caching.
19
+
type HandlesWorker struct {
20
+
*Worker[string]
21
+
store *store.Store
22
+
dir identity.Directory
23
+
}
24
+
25
+
func NewHandlesWorker(s *store.Store, dir identity.Directory, logger *slog.Logger, queueSize int) *HandlesWorker {
26
+
w := &HandlesWorker{store: s, dir: dir}
27
+
w.Worker = NewWorker(WorkerConfig[string]{
28
+
Name: "handles",
29
+
Logger: logger,
30
+
Process: w.process,
31
+
QueueSize: queueSize,
32
+
SeenTTL: 6 * time.Hour,
33
+
MinDelay: 250 * time.Millisecond,
34
+
})
35
+
return w
36
+
}
37
+
38
+
func (w *HandlesWorker) Enqueue(did string) { w.Worker.Enqueue(did) }
39
+
40
+
func (w *HandlesWorker) process(ctx context.Context, did string) error {
41
+
d, err := syntax.ParseDID(did)
42
+
if err != nil {
43
+
return err
44
+
}
45
+
ident, err := w.dir.LookupDID(ctx, d)
46
+
if err != nil {
47
+
return err
48
+
}
49
+
handle, err := ident.DeclaredHandle()
50
+
if err != nil {
51
+
return fmt.Errorf("no declared handle: %w", err)
52
+
}
53
+
return w.store.UpsertHandle(ctx, did, handle.String())
54
+
}
+100
firehose/jetstream.go
+100
firehose/jetstream.go
···
1
+
// Package firehose drives a Jetstream WebSocket subscription via the official
2
+
// bluesky-social/jetstream client and dispatches events to a handler.
3
+
package firehose
4
+
5
+
import (
6
+
"context"
7
+
"fmt"
8
+
"log/slog"
9
+
"math/rand/v2"
10
+
"sync/atomic"
11
+
"time"
12
+
13
+
"github.com/bluesky-social/jetstream/pkg/client"
14
+
"github.com/bluesky-social/jetstream/pkg/client/schedulers/sequential"
15
+
"github.com/bluesky-social/jetstream/pkg/models"
16
+
)
17
+
18
+
// HandlerFunc applies a Jetstream event. Errors are logged but don't tear down
19
+
// the stream.
20
+
type HandlerFunc func(ctx context.Context, ev *models.Event) error
21
+
22
+
type Consumer struct {
23
+
Endpoint string
24
+
WantedCollections []string
25
+
Handler HandlerFunc
26
+
Logger *slog.Logger
27
+
MaxBackoff time.Duration // default 60s
28
+
}
29
+
30
+
// Run runs the consumer until ctx is canceled. Reconnects on disconnect with
31
+
// jittered exponential backoff. The official client handles the WS dance —
32
+
// this wrapper only adds reconnect, cursor tracking, and a heartbeat log.
33
+
func (c *Consumer) Run(ctx context.Context, startCursor int64) error {
34
+
if c.MaxBackoff == 0 {
35
+
c.MaxBackoff = 60 * time.Second
36
+
}
37
+
38
+
var cursor atomic.Int64
39
+
cursor.Store(startCursor)
40
+
41
+
cfg := client.DefaultClientConfig()
42
+
cfg.WebsocketURL = c.Endpoint
43
+
cfg.WantedCollections = c.WantedCollections
44
+
45
+
sched := sequential.NewScheduler("firehose", c.Logger, func(ctx context.Context, ev *models.Event) error {
46
+
if ev.TimeUS > cursor.Load() {
47
+
cursor.Store(ev.TimeUS)
48
+
}
49
+
return c.Handler(ctx, ev)
50
+
})
51
+
52
+
cli, err := client.NewClient(cfg, c.Logger, sched)
53
+
if err != nil {
54
+
return fmt.Errorf("create jetstream client: %w", err)
55
+
}
56
+
57
+
heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx)
58
+
defer cancelHeartbeat()
59
+
go func() {
60
+
t := time.NewTicker(30 * time.Second)
61
+
defer t.Stop()
62
+
for {
63
+
select {
64
+
case <-heartbeatCtx.Done():
65
+
return
66
+
case <-t.C:
67
+
c.Logger.Info("heartbeat",
68
+
"events", cli.EventsRead.Load(),
69
+
"bytes", cli.BytesRead.Load(),
70
+
"cursor", cursor.Load())
71
+
}
72
+
}
73
+
}()
74
+
75
+
backoff := time.Second
76
+
for {
77
+
if err := ctx.Err(); err != nil {
78
+
return err
79
+
}
80
+
var resume *int64
81
+
if v := cursor.Load(); v > 0 {
82
+
resume = &v
83
+
}
84
+
err := cli.ConnectAndRead(ctx, resume)
85
+
if ctx.Err() != nil {
86
+
return ctx.Err()
87
+
}
88
+
c.Logger.Warn("disconnected", "err", err, "backoff", backoff)
89
+
jitter := time.Duration(rand.Int64N(int64(backoff/2 + 1)))
90
+
select {
91
+
case <-ctx.Done():
92
+
return ctx.Err()
93
+
case <-time.After(backoff + jitter):
94
+
}
95
+
backoff *= 2
96
+
if backoff > c.MaxBackoff {
97
+
backoff = c.MaxBackoff
98
+
}
99
+
}
100
+
}
+146
firehose/languages.go
+146
firehose/languages.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"fmt"
8
+
"log/slog"
9
+
"net"
10
+
"net/http"
11
+
"net/url"
12
+
"time"
13
+
14
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
15
+
)
16
+
17
+
type langsJob struct {
18
+
atURI string
19
+
knot string
20
+
did string
21
+
name string
22
+
}
23
+
24
+
// LanguagesWorker calls sh.tangled.repo.languages on the repo's knot and
25
+
// writes the result to the languages column.
26
+
type LanguagesWorker struct {
27
+
*Worker[langsJob]
28
+
store *store.Store
29
+
client *http.Client
30
+
}
31
+
32
+
func NewLanguagesWorker(s *store.Store, logger *slog.Logger, queueSize int) *LanguagesWorker {
33
+
w := &LanguagesWorker{
34
+
store: s,
35
+
client: &http.Client{Timeout: 30 * time.Second},
36
+
}
37
+
w.Worker = NewWorker(WorkerConfig[langsJob]{
38
+
Name: "languages",
39
+
Logger: logger,
40
+
Process: w.process,
41
+
QueueSize: queueSize,
42
+
SeenTTL: 6 * time.Hour,
43
+
MinDelay: 500 * time.Millisecond,
44
+
})
45
+
return w
46
+
}
47
+
48
+
// Enqueue is the public face of the worker; it builds the internal job struct
49
+
// and pushes it through the generic worker.
50
+
func (w *LanguagesWorker) Enqueue(atURI, knot, did, name string) {
51
+
w.Worker.Enqueue(langsJob{atURI: atURI, knot: knot, did: did, name: name})
52
+
}
53
+
54
+
// Circuit breaker: after this many consecutive errors, skip the knot for the
55
+
// cooldown window before trying again.
56
+
const (
57
+
knotCircuitMinErrors = 3
58
+
knotCircuitCooldown = time.Hour
59
+
)
60
+
61
+
func (w *LanguagesWorker) process(ctx context.Context, j langsJob) error {
62
+
// Some repos register knots on localhost or private IPs (devs testing).
63
+
// Those are permanently unreachable from us — stamp empty languages.
64
+
if isUnroutable(j.knot) {
65
+
return w.store.UpdateLanguages(ctx, j.atURI, map[string]int64{})
66
+
}
67
+
// Skip if the knot has been stalling/failing recently. The circuit naturally
68
+
// half-opens after the cooldown — next call either succeeds (counter
69
+
// resets) or re-extends the open window.
70
+
if open, err := w.store.KnotCircuitOpen(ctx, j.knot, knotCircuitMinErrors, knotCircuitCooldown); err != nil {
71
+
return err
72
+
} else if open {
73
+
return nil
74
+
}
75
+
u := url.URL{
76
+
Scheme: "https",
77
+
Host: j.knot,
78
+
Path: "/xrpc/sh.tangled.repo.languages",
79
+
}
80
+
q := u.Query()
81
+
q.Set("repo", j.did+"/"+j.name)
82
+
u.RawQuery = q.Encode()
83
+
84
+
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
85
+
if err != nil {
86
+
return err
87
+
}
88
+
resp, err := w.client.Do(req)
89
+
if err != nil {
90
+
// DNS-not-found means the knot host has vanished — treat as permanent
91
+
// (write empty so the row's language_at gets stamped and we skip it
92
+
// for the 7-day re-check window).
93
+
var dnsErr *net.DNSError
94
+
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
95
+
_ = w.store.MarkKnotError(ctx, j.knot, err.Error())
96
+
return w.store.UpdateLanguages(ctx, j.atURI, map[string]int64{})
97
+
}
98
+
_ = w.store.MarkKnotError(ctx, j.knot, err.Error())
99
+
return err
100
+
}
101
+
defer resp.Body.Close()
102
+
103
+
if resp.StatusCode >= 500 {
104
+
_ = w.store.MarkKnotError(ctx, j.knot, fmt.Sprintf("status %d", resp.StatusCode))
105
+
return fmt.Errorf("status %d", resp.StatusCode)
106
+
}
107
+
if resp.StatusCode != http.StatusOK {
108
+
// 4xx — knot is reachable, just doesn't have this repo. Mark OK so
109
+
// the circuit breaker doesn't penalize the host for missing data.
110
+
_ = w.store.MarkKnotOK(ctx, j.knot)
111
+
return w.store.UpdateLanguages(ctx, j.atURI, map[string]int64{})
112
+
}
113
+
114
+
var body struct {
115
+
Ref string `json:"ref"`
116
+
Languages []struct {
117
+
Name string `json:"name"`
118
+
Size int64 `json:"size"`
119
+
} `json:"languages"`
120
+
}
121
+
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
122
+
return err
123
+
}
124
+
out := make(map[string]int64, len(body.Languages))
125
+
for _, l := range body.Languages {
126
+
out[l.Name] = l.Size
127
+
}
128
+
_ = w.store.MarkKnotOK(ctx, j.knot)
129
+
return w.store.UpdateLanguages(ctx, j.atURI, out)
130
+
}
131
+
132
+
// isUnroutable returns true for hosts we can't reach from a remote process:
133
+
// localhost, loopback IPs, RFC1918 private ranges, and unspecified addresses.
134
+
func isUnroutable(hostport string) bool {
135
+
host, _, err := net.SplitHostPort(hostport)
136
+
if err != nil {
137
+
host = hostport
138
+
}
139
+
if host == "localhost" {
140
+
return true
141
+
}
142
+
if ip := net.ParseIP(host); ip != nil {
143
+
return ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified()
144
+
}
145
+
return false
146
+
}
+126
firehose/starsbackfill.go
+126
firehose/starsbackfill.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"net/http"
8
+
"time"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/bluesky-social/indigo/xrpc"
13
+
14
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
15
+
)
16
+
17
+
// StarsBackfillWorker pulls historical sh.tangled.feed.star records from a
18
+
// user's PDS via com.atproto.repo.listRecords. Handles pagination. Chains into
19
+
// DiscoveryWorker so any backfilled star pointing at an unknown repo triggers
20
+
// a repo fetch.
21
+
type StarsBackfillWorker struct {
22
+
*Worker[string]
23
+
store *store.Store
24
+
dir identity.Directory
25
+
httpClient *http.Client
26
+
pageSize int64
27
+
discovery DiscoveryEnqueuer
28
+
}
29
+
30
+
func NewStarsBackfillWorker(s *store.Store, dir identity.Directory, logger *slog.Logger, discovery DiscoveryEnqueuer, queueSize int) *StarsBackfillWorker {
31
+
w := &StarsBackfillWorker{
32
+
store: s,
33
+
dir: dir,
34
+
httpClient: &http.Client{Timeout: 30 * time.Second},
35
+
pageSize: 100,
36
+
discovery: discovery,
37
+
}
38
+
w.Worker = NewWorker(WorkerConfig[string]{
39
+
Name: "stars-backfill",
40
+
Logger: logger,
41
+
Process: w.process,
42
+
QueueSize: queueSize,
43
+
SeenTTL: 6 * time.Hour,
44
+
MinDelay: 500 * time.Millisecond,
45
+
})
46
+
return w
47
+
}
48
+
49
+
func (w *StarsBackfillWorker) Enqueue(did string) { w.Worker.Enqueue(did) }
50
+
51
+
func (w *StarsBackfillWorker) process(ctx context.Context, did string) error {
52
+
d, err := syntax.ParseDID(did)
53
+
if err != nil {
54
+
return err
55
+
}
56
+
ident, err := w.dir.LookupDID(ctx, d)
57
+
if err != nil {
58
+
return fmt.Errorf("identity lookup: %w", err)
59
+
}
60
+
pds := ident.PDSEndpoint()
61
+
if pds == "" {
62
+
return fmt.Errorf("no PDS endpoint")
63
+
}
64
+
65
+
xc := &xrpc.Client{Client: w.httpClient, Host: pds}
66
+
cursor := ""
67
+
total := 0
68
+
for {
69
+
records, next, err := w.listStars(ctx, xc, did, cursor)
70
+
if err != nil {
71
+
return err
72
+
}
73
+
for _, r := range records {
74
+
if r.Value.Subject == "" {
75
+
continue
76
+
}
77
+
createdAt, err := time.Parse(time.RFC3339Nano, r.Value.CreatedAt)
78
+
if err != nil {
79
+
createdAt = time.Now()
80
+
}
81
+
if err := w.store.UpsertStar(ctx, r.URI, r.Value.Subject, createdAt); err != nil {
82
+
w.logger.Warn("upsert star", "err", err, "uri", r.URI)
83
+
continue
84
+
}
85
+
total++
86
+
if w.discovery != nil {
87
+
// Block on backpressure — a single user can have hundreds of
88
+
// stars and discovery is rate-limited per PDS call.
89
+
w.discovery.EnqueueAtURIWait(ctx, r.Value.Subject)
90
+
}
91
+
}
92
+
if next == "" || next == cursor || len(records) == 0 {
93
+
break
94
+
}
95
+
cursor = next
96
+
}
97
+
if total > 0 {
98
+
w.logger.Info("backfilled stars", "did", did, "count", total)
99
+
}
100
+
return nil
101
+
}
102
+
103
+
type listRecordEntry struct {
104
+
URI string `json:"uri"`
105
+
CID string `json:"cid"`
106
+
Value starRecord `json:"value"`
107
+
}
108
+
109
+
func (w *StarsBackfillWorker) listStars(ctx context.Context, xc *xrpc.Client, did, cursor string) ([]listRecordEntry, string, error) {
110
+
params := map[string]any{
111
+
"repo": did,
112
+
"collection": "sh.tangled.feed.star",
113
+
"limit": w.pageSize,
114
+
}
115
+
if cursor != "" {
116
+
params["cursor"] = cursor
117
+
}
118
+
var out struct {
119
+
Cursor string `json:"cursor"`
120
+
Records []listRecordEntry `json:"records"`
121
+
}
122
+
if err := xc.Do(ctx, xrpc.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil {
123
+
return nil, "", err
124
+
}
125
+
return out.Records, out.Cursor, nil
126
+
}
+122
firehose/worker.go
+122
firehose/worker.go
···
1
+
package firehose
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
"sync"
7
+
"time"
8
+
)
9
+
10
+
// Worker is the shared skeleton behind every async enrichment/backfill
11
+
// worker: a bounded queue, a TTL'd dedupe map, a single goroutine that
12
+
// processes items with a min delay between calls, and immediate-retry on
13
+
// process errors (the dedupe entry is dropped).
14
+
//
15
+
// SeenTTL means "skip an item that was last seen less than this ago". A short
16
+
// TTL pairs with a periodic backfill seeder — together they replace the old
17
+
// restart-required pattern: stale items naturally re-enter the queue without a
18
+
// daemon bounce.
19
+
type Worker[T comparable] struct {
20
+
name string
21
+
queue chan T
22
+
seen sync.Map // T -> time.Time
23
+
seenTTL time.Duration
24
+
minDelay time.Duration
25
+
logger *slog.Logger
26
+
process func(ctx context.Context, item T) error
27
+
}
28
+
29
+
type WorkerConfig[T comparable] struct {
30
+
Name string // logged on errors / drops
31
+
Logger *slog.Logger // required
32
+
Process func(ctx context.Context, item T) error // required
33
+
QueueSize int // default 256
34
+
SeenTTL time.Duration // 0 = forever
35
+
MinDelay time.Duration // gap between processed items
36
+
}
37
+
38
+
func NewWorker[T comparable](cfg WorkerConfig[T]) *Worker[T] {
39
+
if cfg.QueueSize <= 0 {
40
+
cfg.QueueSize = 1024
41
+
}
42
+
return &Worker[T]{
43
+
name: cfg.Name,
44
+
queue: make(chan T, cfg.QueueSize),
45
+
seenTTL: cfg.SeenTTL,
46
+
minDelay: cfg.MinDelay,
47
+
logger: cfg.Logger,
48
+
process: cfg.Process,
49
+
}
50
+
}
51
+
52
+
// Enqueue is non-blocking. Returns false if the item was deduped (within TTL)
53
+
// or the queue is full. Zero-valued items are rejected. Use this from live
54
+
// event handlers where blocking would stall the firehose.
55
+
func (w *Worker[T]) Enqueue(item T) bool {
56
+
var zero T
57
+
if item == zero {
58
+
return false
59
+
}
60
+
if !w.markSeen(item) {
61
+
return false
62
+
}
63
+
select {
64
+
case w.queue <- item:
65
+
return true
66
+
default:
67
+
w.seen.Delete(item)
68
+
w.logger.Warn("queue full, dropping", "worker", w.name, "item", item)
69
+
return false
70
+
}
71
+
}
72
+
73
+
// EnqueueWait blocks until the queue accepts the item or ctx is cancelled.
74
+
// Use this from chained producers (one worker enqueueing into another) so the
75
+
// producer paces itself to the consumer's rate instead of dropping work.
76
+
func (w *Worker[T]) EnqueueWait(ctx context.Context, item T) bool {
77
+
var zero T
78
+
if item == zero {
79
+
return false
80
+
}
81
+
if !w.markSeen(item) {
82
+
return false
83
+
}
84
+
select {
85
+
case w.queue <- item:
86
+
return true
87
+
case <-ctx.Done():
88
+
w.seen.Delete(item)
89
+
return false
90
+
}
91
+
}
92
+
93
+
func (w *Worker[T]) markSeen(item T) bool {
94
+
if v, ok := w.seen.Load(item); ok {
95
+
if w.seenTTL == 0 || time.Since(v.(time.Time)) < w.seenTTL {
96
+
return false
97
+
}
98
+
}
99
+
w.seen.Store(item, time.Now())
100
+
return true
101
+
}
102
+
103
+
func (w *Worker[T]) Run(ctx context.Context) {
104
+
for {
105
+
select {
106
+
case <-ctx.Done():
107
+
return
108
+
case item := <-w.queue:
109
+
if err := w.process(ctx, item); err != nil {
110
+
w.logger.Warn(w.name, "err", err, "item", item)
111
+
w.seen.Delete(item)
112
+
}
113
+
if w.minDelay > 0 {
114
+
select {
115
+
case <-ctx.Done():
116
+
return
117
+
case <-time.After(w.minDelay):
118
+
}
119
+
}
120
+
}
121
+
}
122
+
}
+96
go.mod
+96
go.mod
···
1
+
module tangled.sh/chown.de/tangled-repo-firehose
2
+
3
+
go 1.26
4
+
5
+
require (
6
+
entgo.io/ent v0.14.6
7
+
github.com/bluesky-social/indigo v0.0.0-20260428083920-ce62b8fce9e0
8
+
github.com/bluesky-social/jetstream v0.0.0-20260415170838-8a65de4eda28
9
+
modernc.org/sqlite v1.34.5
10
+
)
11
+
12
+
require (
13
+
ariga.io/atlas v0.36.2-0.20250730182955-2c6300d0a3e1 // indirect
14
+
github.com/agext/levenshtein v1.2.3 // indirect
15
+
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
16
+
github.com/beorn7/perks v1.0.1 // indirect
17
+
github.com/bmatcuk/doublestar v1.3.4 // indirect
18
+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
19
+
github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect
20
+
github.com/felixge/httpsnoop v1.0.4 // indirect
21
+
github.com/go-logr/logr v1.4.1 // indirect
22
+
github.com/go-logr/stdr v1.2.2 // indirect
23
+
github.com/go-openapi/inflect v0.19.0 // indirect
24
+
github.com/goccy/go-json v0.10.2 // indirect
25
+
github.com/gogo/protobuf v1.3.2 // indirect
26
+
github.com/google/go-cmp v0.6.0 // indirect
27
+
github.com/gorilla/websocket v1.5.3 // indirect
28
+
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
29
+
github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
30
+
github.com/hashicorp/golang-lru v1.0.2 // indirect
31
+
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
32
+
github.com/hashicorp/hcl/v2 v2.18.1 // indirect
33
+
github.com/ipfs/bbloom v0.0.4 // indirect
34
+
github.com/ipfs/go-block-format v0.2.0 // indirect
35
+
github.com/ipfs/go-cid v0.4.1 // indirect
36
+
github.com/ipfs/go-datastore v0.6.0 // indirect
37
+
github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
38
+
github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
39
+
github.com/ipfs/go-ipfs-util v0.0.3 // indirect
40
+
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
41
+
github.com/ipfs/go-ipld-format v0.6.0 // indirect
42
+
github.com/ipfs/go-log v1.0.5 // indirect
43
+
github.com/ipfs/go-log/v2 v2.5.1 // indirect
44
+
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
45
+
github.com/jbenet/goprocess v0.1.4 // indirect
46
+
github.com/klauspost/compress v1.17.9 // indirect
47
+
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
48
+
github.com/minio/sha256-simd v1.0.1 // indirect
49
+
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
50
+
github.com/mr-tron/base58 v1.2.0 // indirect
51
+
github.com/multiformats/go-base32 v0.1.0 // indirect
52
+
github.com/multiformats/go-base36 v0.2.0 // indirect
53
+
github.com/multiformats/go-multibase v0.2.0 // indirect
54
+
github.com/multiformats/go-multihash v0.2.3 // indirect
55
+
github.com/multiformats/go-varint v0.0.7 // indirect
56
+
github.com/opentracing/opentracing-go v1.2.0 // indirect
57
+
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
58
+
github.com/prometheus/client_golang v1.19.1 // indirect
59
+
github.com/prometheus/client_model v0.6.1 // indirect
60
+
github.com/prometheus/common v0.54.0 // indirect
61
+
github.com/prometheus/procfs v0.15.1 // indirect
62
+
github.com/spaolacci/murmur3 v1.1.0 // indirect
63
+
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect
64
+
github.com/zclconf/go-cty v1.14.4 // indirect
65
+
github.com/zclconf/go-cty-yaml v1.1.0 // indirect
66
+
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
67
+
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
68
+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
69
+
go.opentelemetry.io/otel v1.21.0 // indirect
70
+
go.opentelemetry.io/otel/metric v1.21.0 // indirect
71
+
go.opentelemetry.io/otel/trace v1.21.0 // indirect
72
+
go.uber.org/atomic v1.11.0 // indirect
73
+
go.uber.org/multierr v1.11.0 // indirect
74
+
go.uber.org/zap v1.26.0 // indirect
75
+
golang.org/x/crypto v0.22.0 // indirect
76
+
golang.org/x/mod v0.24.0 // indirect
77
+
golang.org/x/text v0.21.0 // indirect
78
+
golang.org/x/time v0.5.0 // indirect
79
+
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
80
+
google.golang.org/protobuf v1.34.2 // indirect
81
+
gopkg.in/yaml.v3 v3.0.1 // indirect
82
+
lukechampine.com/blake3 v1.2.1 // indirect
83
+
)
84
+
85
+
require (
86
+
github.com/dustin/go-humanize v1.0.1 // indirect
87
+
github.com/google/uuid v1.6.0 // indirect
88
+
github.com/mattn/go-isatty v0.0.20 // indirect
89
+
github.com/ncruces/go-strftime v0.1.9 // indirect
90
+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
91
+
github.com/urfave/cli/v3 v3.8.0
92
+
golang.org/x/sys v0.30.0 // indirect
93
+
modernc.org/libc v1.55.3 // indirect
94
+
modernc.org/mathutil v1.6.0 // indirect
95
+
modernc.org/memory v1.8.0 // indirect
96
+
)
+343
go.sum
+343
go.sum
···
1
+
ariga.io/atlas v0.36.2-0.20250730182955-2c6300d0a3e1 h1:NPPfBaVZgz4LKBCIc0FbMogCjvXN+yGf7CZwotOwJo8=
2
+
ariga.io/atlas v0.36.2-0.20250730182955-2c6300d0a3e1/go.mod h1:Ex5l1xHsnWQUc3wYnrJ9gD7RUEzG76P7ZRQp8wNr0wc=
3
+
entgo.io/ent v0.14.6 h1:/f2696BpwuWAEEG6PVGWflg6+Inrpq4pRWuNlWz/Skk=
4
+
entgo.io/ent v0.14.6/go.mod h1:z46QBUdGC+BATwsedbDuREfSS0oSCV+csdEYlL4p73s=
5
+
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
6
+
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
7
+
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
8
+
github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
9
+
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
10
+
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
11
+
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
12
+
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
13
+
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
14
+
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
15
+
github.com/bluesky-social/indigo v0.0.0-20260428083920-ce62b8fce9e0 h1:N1c6zWfPBQ4hiCRqSP6cbdlsX38w2i9cLgGuruM1UyE=
16
+
github.com/bluesky-social/indigo v0.0.0-20260428083920-ce62b8fce9e0/go.mod h1:JqQkz8lrOI6YZivP38GHmtVOTtzsNToITKj1gMpU5Jo=
17
+
github.com/bluesky-social/jetstream v0.0.0-20260415170838-8a65de4eda28 h1:cQ7kasyLcEuh/Zd7g0h8kmaWz14SEjJvbeEPU9rCyx0=
18
+
github.com/bluesky-social/jetstream v0.0.0-20260415170838-8a65de4eda28/go.mod h1:1TEGvYje9ONndPpaGqW6WqyLiEvYm2iKd2GrpzLvGhg=
19
+
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
20
+
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
21
+
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
22
+
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
23
+
github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo=
24
+
github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
25
+
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
26
+
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
27
+
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
28
+
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
29
+
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
30
+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
31
+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
32
+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
33
+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
34
+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
35
+
github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg=
36
+
github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw=
37
+
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
38
+
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
39
+
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
40
+
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
41
+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
42
+
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
43
+
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
44
+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
45
+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
46
+
github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
47
+
github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
48
+
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
49
+
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
50
+
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
51
+
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
52
+
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
53
+
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
54
+
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
55
+
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
56
+
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
57
+
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
58
+
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
59
+
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
60
+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
61
+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
62
+
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
63
+
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
64
+
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
65
+
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
66
+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
67
+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
68
+
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
69
+
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
70
+
github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
71
+
github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
72
+
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
73
+
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
74
+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
75
+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
76
+
github.com/hashicorp/hcl/v2 v2.18.1 h1:6nxnOJFku1EuSawSD81fuviYUV8DxFr3fp2dUi3ZYSo=
77
+
github.com/hashicorp/hcl/v2 v2.18.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE=
78
+
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
79
+
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
80
+
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
81
+
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
82
+
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
83
+
github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
84
+
github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk=
85
+
github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8=
86
+
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
87
+
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
88
+
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
89
+
github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
90
+
github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw=
91
+
github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo=
92
+
github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
93
+
github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
94
+
github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
95
+
github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
96
+
github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U=
97
+
github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg=
98
+
github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
99
+
github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
100
+
github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
101
+
github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY=
102
+
github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
103
+
github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
104
+
github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
105
+
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
106
+
github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
107
+
github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
108
+
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
109
+
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
110
+
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
111
+
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
112
+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
113
+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
114
+
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
115
+
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
116
+
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
117
+
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
118
+
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
119
+
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
120
+
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
121
+
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
122
+
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
123
+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
124
+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
125
+
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
126
+
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
127
+
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
128
+
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
129
+
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
130
+
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
131
+
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
132
+
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
133
+
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
134
+
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
135
+
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
136
+
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
137
+
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
138
+
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
139
+
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
140
+
github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
141
+
github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
142
+
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
143
+
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
144
+
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
145
+
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
146
+
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
147
+
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
148
+
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
149
+
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
150
+
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
151
+
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
152
+
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
153
+
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
154
+
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
155
+
github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
156
+
github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM=
157
+
github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew=
158
+
github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA=
159
+
github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM=
160
+
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
161
+
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
162
+
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
163
+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
164
+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
165
+
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0=
166
+
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
167
+
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
168
+
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
169
+
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
170
+
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
171
+
github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
172
+
github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
173
+
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
174
+
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
175
+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
176
+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
177
+
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
178
+
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
179
+
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
180
+
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
181
+
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
182
+
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
183
+
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
184
+
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
185
+
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
186
+
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
187
+
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
188
+
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
189
+
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
190
+
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
191
+
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
192
+
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
193
+
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
194
+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
195
+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
196
+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
197
+
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
198
+
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
199
+
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
200
+
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
201
+
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
202
+
github.com/urfave/cli/v3 v3.8.0 h1:XqKPrm0q4P0q5JpoclYoCAv0/MIvH/jZ2umzuf8pNTI=
203
+
github.com/urfave/cli/v3 v3.8.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
204
+
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
205
+
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
206
+
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e h1:28X54ciEwwUxyHn9yrZfl5ojgF4CBNLWX7LR0rvBkf4=
207
+
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so=
208
+
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
209
+
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
210
+
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
211
+
github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
212
+
github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
213
+
github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0=
214
+
github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs=
215
+
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=
216
+
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
217
+
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q=
218
+
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I=
219
+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
220
+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
221
+
go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
222
+
go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
223
+
go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
224
+
go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
225
+
go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
226
+
go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
227
+
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
228
+
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
229
+
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
230
+
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
231
+
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
232
+
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
233
+
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
234
+
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
235
+
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
236
+
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
237
+
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
238
+
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
239
+
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
240
+
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
241
+
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
242
+
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
243
+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
244
+
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
245
+
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
246
+
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
247
+
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
248
+
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
249
+
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
250
+
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
251
+
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
252
+
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
253
+
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
254
+
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
255
+
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
256
+
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
257
+
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
258
+
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
259
+
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
260
+
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
261
+
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
262
+
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
263
+
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
264
+
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
265
+
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
266
+
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
267
+
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
268
+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
269
+
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
270
+
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
271
+
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
272
+
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
273
+
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
274
+
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
275
+
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
276
+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
277
+
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
278
+
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
279
+
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
280
+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
281
+
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
282
+
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
283
+
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
284
+
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
285
+
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
286
+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
287
+
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
288
+
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
289
+
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
290
+
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
291
+
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
292
+
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
293
+
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
294
+
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
295
+
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
296
+
golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
297
+
golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
298
+
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
299
+
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
300
+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
301
+
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
302
+
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
303
+
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
304
+
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
305
+
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
306
+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
307
+
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
308
+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
309
+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
310
+
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
311
+
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
312
+
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
313
+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
314
+
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
315
+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
316
+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
317
+
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
318
+
lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
319
+
lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
320
+
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
321
+
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
322
+
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
323
+
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
324
+
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
325
+
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
326
+
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
327
+
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
328
+
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
329
+
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
330
+
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
331
+
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
332
+
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
333
+
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
334
+
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
335
+
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
336
+
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
337
+
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
338
+
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
339
+
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
340
+
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
341
+
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
342
+
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
343
+
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+694
store/store.go
+694
store/store.go
···
1
+
// Package store is the persistence layer for the Tangled repo firehose
2
+
// aggregator. Schema is defined in ent/schema/ and generated via
3
+
// `go generate ./ent`. Auto-migration runs on Open and is additive only —
4
+
// existing data is never dropped.
5
+
package store
6
+
7
+
import (
8
+
"context"
9
+
"database/sql"
10
+
"encoding/json"
11
+
"fmt"
12
+
"io"
13
+
"time"
14
+
15
+
"entgo.io/ent/dialect"
16
+
entsql "entgo.io/ent/dialect/sql"
17
+
18
+
"tangled.sh/chown.de/tangled-repo-firehose/ent"
19
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/cursor"
20
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/discoveryfailed"
21
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/handle"
22
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/knot"
23
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/repo"
24
+
"tangled.sh/chown.de/tangled-repo-firehose/ent/star"
25
+
26
+
_ "modernc.org/sqlite"
27
+
)
28
+
29
+
// dsn assembles the modernc.org/sqlite connection string. PRAGMAs go in the
30
+
// query string so they're applied per-connection (the pool may open many).
31
+
// ent's migrator checks foreign_keys at startup so it must be set here.
32
+
func dsn(path string) string {
33
+
return path + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)"
34
+
}
35
+
36
+
// MigrationPlan writes the SQL that auto-migration would run to w without
37
+
// applying any DDL. Used by --migrate-dry-run.
38
+
func MigrationPlan(ctx context.Context, path string, w io.Writer) error {
39
+
db, err := sql.Open("sqlite", dsn(path))
40
+
if err != nil {
41
+
return err
42
+
}
43
+
defer db.Close()
44
+
drv := entsql.OpenDB(dialect.SQLite, db)
45
+
client := ent.NewClient(ent.Driver(drv))
46
+
defer client.Close()
47
+
return client.Schema.WriteTo(ctx, w)
48
+
}
49
+
50
+
51
+
// Repo mirrors a sh.tangled.repo record plus enrichment columns.
52
+
type Repo struct {
53
+
AtURI string
54
+
DID string
55
+
Rkey string
56
+
Name string
57
+
Knot string
58
+
Description string
59
+
Topics []string
60
+
Website string
61
+
Source string
62
+
Spindle string
63
+
RepoDID string
64
+
CreatedAt time.Time
65
+
SeenAt time.Time
66
+
Languages map[string]int64 // nil if not yet fetched
67
+
LanguageAt *time.Time
68
+
Handle string // owner's handle, "" until resolved
69
+
StarCount int // counts only stars seen since subscription started
70
+
}
71
+
72
+
// Primary returns the language with the most bytes, or "" if unknown.
73
+
func (r Repo) Primary() string {
74
+
var best string
75
+
var bestN int64
76
+
for k, v := range r.Languages {
77
+
if v > bestN {
78
+
bestN, best = v, k
79
+
}
80
+
}
81
+
return best
82
+
}
83
+
84
+
type Store struct {
85
+
client *ent.Client
86
+
db *sql.DB // raw handle for the few queries ent doesn't model cleanly
87
+
}
88
+
89
+
func Open(path string) (*Store, error) {
90
+
db, err := sql.Open("sqlite", dsn(path))
91
+
if err != nil {
92
+
return nil, err
93
+
}
94
+
drv := entsql.OpenDB(dialect.SQLite, db)
95
+
client := ent.NewClient(ent.Driver(drv))
96
+
if err := client.Schema.Create(context.Background()); err != nil {
97
+
client.Close()
98
+
return nil, fmt.Errorf("auto-migrate: %w", err)
99
+
}
100
+
return &Store{client: client, db: db}, nil
101
+
}
102
+
103
+
func (s *Store) Close() error { return s.client.Close() }
104
+
105
+
// GetCursor returns the persisted Jetstream cursor (unix microseconds), or 0
106
+
// if none has been recorded yet.
107
+
func (s *Store) GetCursor(ctx context.Context) (int64, error) {
108
+
row, err := s.client.Cursor.Query().Where(cursor.IDEQ(1)).Only(ctx)
109
+
if ent.IsNotFound(err) {
110
+
return 0, nil
111
+
}
112
+
if err != nil {
113
+
return 0, err
114
+
}
115
+
return row.Cursor, nil
116
+
}
117
+
118
+
// ApplyRepoUpsert writes (or replaces) a repo row and advances the cursor in
119
+
// a single transaction. Languages column is intentionally untouched so async
120
+
// language enrichment isn't clobbered by a record update.
121
+
func (s *Store) ApplyRepoUpsert(ctx context.Context, r Repo, cur int64) error {
122
+
return withTx(ctx, s.client, func(tx *ent.Tx) error {
123
+
if err := upsertRepo(ctx, tx, r); err != nil {
124
+
return err
125
+
}
126
+
return advanceCursor(ctx, tx, cur)
127
+
})
128
+
}
129
+
130
+
// ApplyRepoDelete removes a repo and advances the cursor in a single txn.
131
+
func (s *Store) ApplyRepoDelete(ctx context.Context, atURI string, cur int64) error {
132
+
return withTx(ctx, s.client, func(tx *ent.Tx) error {
133
+
if _, err := tx.Repo.Delete().Where(repo.IDEQ(atURI)).Exec(ctx); err != nil {
134
+
return err
135
+
}
136
+
return advanceCursor(ctx, tx, cur)
137
+
})
138
+
}
139
+
140
+
// AdvanceCursor moves the cursor forward without applying any record. Used
141
+
// for non-tracked event kinds so we don't replay them on restart.
142
+
func (s *Store) AdvanceCursor(ctx context.Context, cur int64) error {
143
+
return withTx(ctx, s.client, func(tx *ent.Tx) error {
144
+
return advanceCursor(ctx, tx, cur)
145
+
})
146
+
}
147
+
148
+
func advanceCursor(ctx context.Context, tx *ent.Tx, cur int64) error {
149
+
// Never move the cursor backward — out-of-order replays would otherwise
150
+
// rewind progress.
151
+
existing, err := tx.Cursor.Query().Where(cursor.IDEQ(1)).Only(ctx)
152
+
now := time.Now().UnixMilli()
153
+
if ent.IsNotFound(err) {
154
+
return tx.Cursor.Create().SetID(1).SetCursor(cur).SetUpdated(now).Exec(ctx)
155
+
}
156
+
if err != nil {
157
+
return err
158
+
}
159
+
if cur <= existing.Cursor {
160
+
return nil
161
+
}
162
+
return tx.Cursor.UpdateOneID(1).SetCursor(cur).SetUpdated(now).Exec(ctx)
163
+
}
164
+
165
+
// UpsertRepo writes (or replaces) a repo row outside the firehose path —
166
+
// used by discovery when a star references a repo we haven't seen yet. Does
167
+
// NOT touch the cursor.
168
+
func (s *Store) UpsertRepo(ctx context.Context, r Repo) error {
169
+
tx, err := s.client.Tx(ctx)
170
+
if err != nil {
171
+
return err
172
+
}
173
+
defer func() { _ = tx.Rollback() }()
174
+
if err := upsertRepo(ctx, tx, r); err != nil {
175
+
return err
176
+
}
177
+
return tx.Commit()
178
+
}
179
+
180
+
func upsertRepo(ctx context.Context, tx *ent.Tx, r Repo) error {
181
+
topicsJSON, err := json.Marshal(r.Topics)
182
+
if err != nil {
183
+
return err
184
+
}
185
+
topics := string(topicsJSON)
186
+
create := tx.Repo.Create().
187
+
SetID(r.AtURI).
188
+
SetDid(r.DID).
189
+
SetRkey(r.Rkey).
190
+
SetName(r.Name).
191
+
SetKnot(r.Knot).
192
+
SetCreatedAt(r.CreatedAt.UnixMilli()).
193
+
SetSeenAt(r.SeenAt.UnixMilli()).
194
+
SetTopics(topics).
195
+
SetDescription(r.Description).
196
+
SetWebsite(r.Website).
197
+
SetSource(r.Source).
198
+
SetSpindle(r.Spindle).
199
+
SetRepoDid(r.RepoDID)
200
+
return create.OnConflict().
201
+
UpdateNewValues().
202
+
Exec(ctx)
203
+
}
204
+
205
+
// HasRepo reports whether a repo at-uri already exists in the index.
206
+
func (s *Store) HasRepo(ctx context.Context, atURI string) (bool, error) {
207
+
return s.client.Repo.Query().Where(repo.IDEQ(atURI)).Exist(ctx)
208
+
}
209
+
210
+
// MarkDiscoveryFailed records that a PDS returned RecordNotFound for this
211
+
// at-uri so we don't keep retrying every backfill cycle.
212
+
func (s *Store) MarkDiscoveryFailed(ctx context.Context, atURI, reason string) error {
213
+
return s.client.DiscoveryFailed.Create().
214
+
SetID(atURI).
215
+
SetFailedAt(time.Now().UnixMilli()).
216
+
SetReason(reason).
217
+
OnConflict().
218
+
UpdateNewValues().
219
+
Exec(ctx)
220
+
}
221
+
222
+
// DiscoveryFailedRecently reports whether a previous discovery attempt failed
223
+
// within the given window — used to skip known-deleted records on retry.
224
+
func (s *Store) DiscoveryFailedRecently(ctx context.Context, atURI string, maxAge time.Duration) (bool, error) {
225
+
cutoff := time.Now().Add(-maxAge).UnixMilli()
226
+
return s.client.DiscoveryFailed.Query().
227
+
Where(discoveryfailed.IDEQ(atURI), discoveryfailed.FailedAtGTE(cutoff)).
228
+
Exist(ctx)
229
+
}
230
+
231
+
// ApplyStarCreate inserts a star (idempotent on at_uri) and advances the
232
+
// cursor in a single transaction.
233
+
func (s *Store) ApplyStarCreate(ctx context.Context, atURI, subject string, createdAt time.Time, cur int64) error {
234
+
return withTx(ctx, s.client, func(tx *ent.Tx) error {
235
+
err := tx.Star.Create().
236
+
SetID(atURI).
237
+
SetSubject(subject).
238
+
SetCreatedAt(createdAt.UnixMilli()).
239
+
OnConflict().
240
+
DoNothing().
241
+
Exec(ctx)
242
+
// ent's DoNothing returns sql.ErrNoRows when nothing was inserted —
243
+
// safe to ignore; the row already exists.
244
+
if err != nil && err != sql.ErrNoRows {
245
+
return err
246
+
}
247
+
return advanceCursor(ctx, tx, cur)
248
+
})
249
+
}
250
+
251
+
// ApplyStarDelete removes a star row and advances the cursor in a single txn.
252
+
func (s *Store) ApplyStarDelete(ctx context.Context, atURI string, cur int64) error {
253
+
return withTx(ctx, s.client, func(tx *ent.Tx) error {
254
+
if _, err := tx.Star.Delete().Where(star.IDEQ(atURI)).Exec(ctx); err != nil {
255
+
return err
256
+
}
257
+
return advanceCursor(ctx, tx, cur)
258
+
})
259
+
}
260
+
261
+
// StarCount returns the number of star records held for a repo's at-uri.
262
+
func (s *Store) StarCount(ctx context.Context, repoAtURI string) (int, error) {
263
+
return s.client.Star.Query().Where(star.SubjectEQ(repoAtURI)).Count(ctx)
264
+
}
265
+
266
+
// UpsertStar inserts a star outside the firehose path — used by the stars
267
+
// backfill worker. Idempotent on at-uri; does not advance the cursor.
268
+
func (s *Store) UpsertStar(ctx context.Context, atURI, subject string, createdAt time.Time) error {
269
+
err := s.client.Star.Create().
270
+
SetID(atURI).
271
+
SetSubject(subject).
272
+
SetCreatedAt(createdAt.UnixMilli()).
273
+
OnConflict().
274
+
DoNothing().
275
+
Exec(ctx)
276
+
if err != nil && err != sql.ErrNoRows {
277
+
return err
278
+
}
279
+
return nil
280
+
}
281
+
282
+
// AllKnownDIDs returns every DID we've encountered: repo owners plus star
283
+
// authors parsed from existing star at-uris. Uses raw SQL because ent has no
284
+
// clean expression for substring + UNION.
285
+
func (s *Store) AllKnownDIDs(ctx context.Context, limit int) ([]string, error) {
286
+
rows, err := s.db.QueryContext(ctx, `
287
+
SELECT did FROM repos
288
+
UNION
289
+
SELECT substr(at_uri, 6, instr(substr(at_uri, 6), '/') - 1) AS did
290
+
FROM stars
291
+
WHERE substr(at_uri, 6, instr(substr(at_uri, 6), '/') - 1) != ''
292
+
LIMIT ?
293
+
`, limit)
294
+
if err != nil {
295
+
return nil, err
296
+
}
297
+
defer rows.Close()
298
+
var out []string
299
+
for rows.Next() {
300
+
var did string
301
+
if err := rows.Scan(&did); err != nil {
302
+
return nil, err
303
+
}
304
+
if did != "" {
305
+
out = append(out, did)
306
+
}
307
+
}
308
+
return out, rows.Err()
309
+
}
310
+
311
+
// UpsertHandle stores or refreshes a DID→handle mapping.
312
+
func (s *Store) UpsertHandle(ctx context.Context, did, h string) error {
313
+
return s.client.Handle.Create().
314
+
SetID(did).
315
+
SetHandle(h).
316
+
SetRefreshedAt(time.Now().UnixMilli()).
317
+
OnConflict().
318
+
UpdateNewValues().
319
+
Exec(ctx)
320
+
}
321
+
322
+
// Handle returns the cached handle for a DID, or "" if unknown.
323
+
func (s *Store) Handle(ctx context.Context, did string) (string, error) {
324
+
row, err := s.client.Handle.Query().Where(handle.IDEQ(did)).Only(ctx)
325
+
if ent.IsNotFound(err) {
326
+
return "", nil
327
+
}
328
+
if err != nil {
329
+
return "", err
330
+
}
331
+
return row.Handle, nil
332
+
}
333
+
334
+
// DIDsNeedingHandles returns owner DIDs from repos that have no cached handle
335
+
// yet, or whose cached handle is older than maxAge.
336
+
func (s *Store) DIDsNeedingHandles(ctx context.Context, maxAge time.Duration, limit int) ([]string, error) {
337
+
cutoff := time.Now().Add(-maxAge).UnixMilli()
338
+
rows, err := s.db.QueryContext(ctx, `
339
+
SELECT DISTINCT r.did
340
+
FROM repos r
341
+
LEFT JOIN handles h ON h.did = r.did
342
+
WHERE h.did IS NULL OR h.refreshed_at < ?
343
+
LIMIT ?
344
+
`, cutoff, limit)
345
+
if err != nil {
346
+
return nil, err
347
+
}
348
+
defer rows.Close()
349
+
var out []string
350
+
for rows.Next() {
351
+
var did string
352
+
if err := rows.Scan(&did); err != nil {
353
+
return nil, err
354
+
}
355
+
out = append(out, did)
356
+
}
357
+
return out, rows.Err()
358
+
}
359
+
360
+
// UpdateLanguages writes the languages JSON map for an existing repo.
361
+
func (s *Store) UpdateLanguages(ctx context.Context, atURI string, langs map[string]int64) error {
362
+
raw, err := json.Marshal(langs)
363
+
if err != nil {
364
+
return err
365
+
}
366
+
return s.client.Repo.UpdateOneID(atURI).
367
+
SetLanguages(string(raw)).
368
+
SetLanguageAt(time.Now().UnixMilli()).
369
+
Exec(ctx)
370
+
}
371
+
372
+
// ReposNeedingLanguages returns repos whose language enrichment is missing or
373
+
// older than maxAge.
374
+
func (s *Store) ReposNeedingLanguages(ctx context.Context, maxAge time.Duration, limit int) ([]Repo, error) {
375
+
cutoff := time.Now().Add(-maxAge).UnixMilli()
376
+
rows, err := s.db.QueryContext(ctx, `
377
+
SELECT at_uri, did, rkey, name, knot
378
+
FROM repos
379
+
WHERE language_at IS NULL OR language_at < ?
380
+
ORDER BY seen_at DESC
381
+
LIMIT ?
382
+
`, cutoff, limit)
383
+
if err != nil {
384
+
return nil, err
385
+
}
386
+
defer rows.Close()
387
+
var out []Repo
388
+
for rows.Next() {
389
+
var r Repo
390
+
if err := rows.Scan(&r.AtURI, &r.DID, &r.Rkey, &r.Name, &r.Knot); err != nil {
391
+
return nil, err
392
+
}
393
+
out = append(out, r)
394
+
}
395
+
return out, rows.Err()
396
+
}
397
+
398
+
// RecentRepos returns repos created since `since`, optionally filtered by
399
+
// primary language. Pass limit <= 0 for no limit. Uses raw SQL for the
400
+
// scalar star-count subquery and handle JOIN — both awkward in ent.
401
+
func (s *Store) RecentRepos(ctx context.Context, language string, since time.Time, limit int) ([]Repo, error) {
402
+
candidateLimit := -1
403
+
if limit > 0 {
404
+
candidateLimit = limit
405
+
if language != "" {
406
+
candidateLimit = limit * 5
407
+
}
408
+
}
409
+
rows, err := s.db.QueryContext(ctx, `
410
+
SELECT r.at_uri, r.did, r.rkey, r.name, r.knot, r.description, r.topics,
411
+
r.website, r.source, r.spindle, r.repo_did, r.created_at, r.seen_at,
412
+
r.languages, r.language_at, COALESCE(h.handle, ''),
413
+
(SELECT COUNT(*) FROM stars s WHERE s.subject = r.at_uri) AS star_count
414
+
FROM repos r
415
+
LEFT JOIN handles h ON h.did = r.did
416
+
WHERE r.created_at >= ?
417
+
AND (? = '' OR r.languages IS NOT NULL)
418
+
ORDER BY r.created_at DESC
419
+
LIMIT ?
420
+
`, since.UnixMilli(), language, candidateLimit)
421
+
if err != nil {
422
+
return nil, err
423
+
}
424
+
defer rows.Close()
425
+
426
+
var out []Repo
427
+
for rows.Next() {
428
+
r, err := scanRepo(rows)
429
+
if err != nil {
430
+
return nil, err
431
+
}
432
+
if language != "" && r.Primary() != language {
433
+
continue
434
+
}
435
+
out = append(out, r)
436
+
if limit > 0 && len(out) >= limit {
437
+
break
438
+
}
439
+
}
440
+
return out, rows.Err()
441
+
}
442
+
443
+
// KnotEntry is one row in the knots view.
444
+
type KnotEntry struct {
445
+
Host string
446
+
FirstSeen time.Time
447
+
LastSeen time.Time
448
+
LastOK *time.Time
449
+
LastError *time.Time
450
+
LastErrorMsg string
451
+
ConsecutiveErrors int
452
+
Disabled bool
453
+
Repos int
454
+
}
455
+
456
+
// Knots returns every knot host with bookkeeping plus a count of repos
457
+
// hosted on it.
458
+
func (s *Store) Knots(ctx context.Context) ([]KnotEntry, error) {
459
+
knots, err := s.client.Knot.Query().
460
+
Order(ent.Desc(knot.FieldLastSeen)).
461
+
All(ctx)
462
+
if err != nil {
463
+
return nil, err
464
+
}
465
+
out := make([]KnotEntry, 0, len(knots))
466
+
for _, k := range knots {
467
+
repos, err := s.client.Repo.Query().Where(repo.KnotEQ(k.ID)).Count(ctx)
468
+
if err != nil {
469
+
return nil, err
470
+
}
471
+
entry := KnotEntry{
472
+
Host: k.ID,
473
+
FirstSeen: time.UnixMilli(k.FirstSeen),
474
+
LastSeen: time.UnixMilli(k.LastSeen),
475
+
ConsecutiveErrors: k.ConsecutiveErrors,
476
+
Disabled: k.Disabled != 0,
477
+
Repos: repos,
478
+
}
479
+
if k.LastOkAt != nil {
480
+
t := time.UnixMilli(*k.LastOkAt)
481
+
entry.LastOK = &t
482
+
}
483
+
if k.LastErrorAt != nil {
484
+
t := time.UnixMilli(*k.LastErrorAt)
485
+
entry.LastError = &t
486
+
}
487
+
if k.LastError != nil {
488
+
entry.LastErrorMsg = *k.LastError
489
+
}
490
+
out = append(out, entry)
491
+
}
492
+
// Sort by repos desc, then host — done in Go since it's a derived count.
493
+
for i := range out {
494
+
for j := i + 1; j < len(out); j++ {
495
+
if out[j].Repos > out[i].Repos || (out[j].Repos == out[i].Repos && out[j].Host < out[i].Host) {
496
+
out[i], out[j] = out[j], out[i]
497
+
}
498
+
}
499
+
}
500
+
return out, nil
501
+
}
502
+
503
+
// UpsertKnot records that we've seen this host, refreshing last_seen.
504
+
func (s *Store) UpsertKnot(ctx context.Context, host string) error {
505
+
if host == "" {
506
+
return nil
507
+
}
508
+
now := time.Now().UnixMilli()
509
+
return s.client.Knot.Create().
510
+
SetID(host).
511
+
SetFirstSeen(now).
512
+
SetLastSeen(now).
513
+
OnConflict().
514
+
Update(func(u *ent.KnotUpsert) {
515
+
u.SetLastSeen(now)
516
+
}).
517
+
Exec(ctx)
518
+
}
519
+
520
+
// MarkKnotOK resets consecutive_errors on a successful call.
521
+
func (s *Store) MarkKnotOK(ctx context.Context, host string) error {
522
+
if host == "" {
523
+
return nil
524
+
}
525
+
now := time.Now().UnixMilli()
526
+
return s.client.Knot.Create().
527
+
SetID(host).
528
+
SetFirstSeen(now).
529
+
SetLastSeen(now).
530
+
SetLastOkAt(now).
531
+
OnConflict().
532
+
Update(func(u *ent.KnotUpsert) {
533
+
u.SetLastSeen(now)
534
+
u.SetLastOkAt(now)
535
+
u.SetConsecutiveErrors(0)
536
+
}).
537
+
Exec(ctx)
538
+
}
539
+
540
+
// MarkKnotError records a failure and increments consecutive_errors.
541
+
func (s *Store) MarkKnotError(ctx context.Context, host, reason string) error {
542
+
if host == "" {
543
+
return nil
544
+
}
545
+
if len(reason) > 500 {
546
+
reason = reason[:500]
547
+
}
548
+
now := time.Now().UnixMilli()
549
+
// ent has no SQL-side "consecutive_errors + 1" expression in upsert, so
550
+
// we drop to raw SQL for the increment. Insert path uses ent.
551
+
_, err := s.db.ExecContext(ctx, `
552
+
INSERT INTO knots (host, first_seen, last_seen, last_error_at, last_error, consecutive_errors, disabled)
553
+
VALUES (?, ?, ?, ?, ?, 1, 0)
554
+
ON CONFLICT(host) DO UPDATE SET
555
+
last_error_at = excluded.last_error_at,
556
+
last_error = excluded.last_error,
557
+
last_seen = excluded.last_seen,
558
+
consecutive_errors = consecutive_errors + 1
559
+
`, host, now, now, now, reason)
560
+
return err
561
+
}
562
+
563
+
// KnotCircuitOpen reports whether further calls to a host should be skipped.
564
+
func (s *Store) KnotCircuitOpen(ctx context.Context, host string, minErrors int, cooldown time.Duration) (bool, error) {
565
+
if host == "" {
566
+
return false, nil
567
+
}
568
+
k, err := s.client.Knot.Query().Where(knot.IDEQ(host)).Only(ctx)
569
+
if ent.IsNotFound(err) {
570
+
return false, nil
571
+
}
572
+
if err != nil {
573
+
return false, err
574
+
}
575
+
if k.Disabled != 0 {
576
+
return true, nil
577
+
}
578
+
if k.ConsecutiveErrors < minErrors || k.LastErrorAt == nil {
579
+
return false, nil
580
+
}
581
+
return time.UnixMilli(*k.LastErrorAt).Add(cooldown).After(time.Now()), nil
582
+
}
583
+
584
+
// SetKnotDisabled toggles the knot's disabled flag.
585
+
func (s *Store) SetKnotDisabled(ctx context.Context, host string, disabled bool) error {
586
+
if host == "" {
587
+
return nil
588
+
}
589
+
v := 0
590
+
if disabled {
591
+
v = 1
592
+
}
593
+
return s.client.Knot.UpdateOneID(host).SetDisabled(v).Exec(ctx)
594
+
}
595
+
596
+
// HandleEntry is one row in the handles view.
597
+
type HandleEntry struct {
598
+
DID string
599
+
Handle string
600
+
RefreshedAt time.Time
601
+
Repos int
602
+
}
603
+
604
+
// Handles returns every resolved DID→handle mapping with repo count. Pass
605
+
// limit <= 0 for no limit.
606
+
func (s *Store) Handles(ctx context.Context, limit int) ([]HandleEntry, error) {
607
+
sqlLimit := -1
608
+
if limit > 0 {
609
+
sqlLimit = limit
610
+
}
611
+
rows, err := s.db.QueryContext(ctx, `
612
+
SELECT h.did, h.handle, h.refreshed_at,
613
+
(SELECT COUNT(*) FROM repos r WHERE r.did = h.did) AS repos
614
+
FROM handles h
615
+
ORDER BY repos DESC, h.handle
616
+
LIMIT ?
617
+
`, sqlLimit)
618
+
if err != nil {
619
+
return nil, err
620
+
}
621
+
defer rows.Close()
622
+
var out []HandleEntry
623
+
for rows.Next() {
624
+
var (
625
+
h HandleEntry
626
+
refreshed int64
627
+
)
628
+
if err := rows.Scan(&h.DID, &h.Handle, &refreshed, &h.Repos); err != nil {
629
+
return nil, err
630
+
}
631
+
h.RefreshedAt = time.UnixMilli(refreshed)
632
+
out = append(out, h)
633
+
}
634
+
return out, rows.Err()
635
+
}
636
+
637
+
// scanRepo reads a row from the SQL projection used by RecentRepos.
638
+
func scanRepo(rows *sql.Rows) (Repo, error) {
639
+
var (
640
+
r Repo
641
+
description sql.NullString
642
+
topicsRaw sql.NullString
643
+
website sql.NullString
644
+
source sql.NullString
645
+
spindle sql.NullString
646
+
repoDID sql.NullString
647
+
languages sql.NullString
648
+
languageAt sql.NullInt64
649
+
createdAt int64
650
+
seenAt int64
651
+
)
652
+
if err := rows.Scan(&r.AtURI, &r.DID, &r.Rkey, &r.Name, &r.Knot,
653
+
&description, &topicsRaw, &website, &source, &spindle, &repoDID,
654
+
&createdAt, &seenAt, &languages, &languageAt, &r.Handle, &r.StarCount); err != nil {
655
+
return r, err
656
+
}
657
+
r.Description = description.String
658
+
r.Website = website.String
659
+
r.Source = source.String
660
+
r.Spindle = spindle.String
661
+
r.RepoDID = repoDID.String
662
+
r.CreatedAt = time.UnixMilli(createdAt)
663
+
r.SeenAt = time.UnixMilli(seenAt)
664
+
if topicsRaw.Valid && topicsRaw.String != "" && topicsRaw.String != "null" {
665
+
_ = json.Unmarshal([]byte(topicsRaw.String), &r.Topics)
666
+
}
667
+
if languages.Valid && languages.String != "" {
668
+
_ = json.Unmarshal([]byte(languages.String), &r.Languages)
669
+
}
670
+
if languageAt.Valid {
671
+
t := time.UnixMilli(languageAt.Int64)
672
+
r.LanguageAt = &t
673
+
}
674
+
return r, nil
675
+
}
676
+
677
+
// withTx runs fn in a transaction, committing on success or rolling back on error.
678
+
func withTx(ctx context.Context, client *ent.Client, fn func(*ent.Tx) error) error {
679
+
tx, err := client.Tx(ctx)
680
+
if err != nil {
681
+
return err
682
+
}
683
+
defer func() {
684
+
if r := recover(); r != nil {
685
+
_ = tx.Rollback()
686
+
panic(r)
687
+
}
688
+
}()
689
+
if err := fn(tx); err != nil {
690
+
_ = tx.Rollback()
691
+
return err
692
+
}
693
+
return tx.Commit()
694
+
}
+126
web/server.go
+126
web/server.go
···
1
+
// Package web exposes a tiny HTTP frontend over the store.
2
+
package web
3
+
4
+
import (
5
+
"embed"
6
+
"encoding/json"
7
+
"html/template"
8
+
"net/http"
9
+
"time"
10
+
11
+
"tangled.sh/chown.de/tangled-repo-firehose/store"
12
+
)
13
+
14
+
//go:embed templates/*.html
15
+
var tmplFS embed.FS
16
+
17
+
var tmpl = template.Must(template.ParseFS(tmplFS, "templates/*.html"))
18
+
19
+
func Handler(s *store.Store) http.Handler {
20
+
mux := http.NewServeMux()
21
+
mux.HandleFunc("GET /{$}", reposHTML(s))
22
+
mux.HandleFunc("GET /repos.json", reposJSON(s))
23
+
mux.HandleFunc("GET /knots", knotsHTML(s))
24
+
mux.HandleFunc("POST /knots/toggle", knotToggle(s))
25
+
mux.HandleFunc("GET /handles", handlesHTML(s))
26
+
return mux
27
+
}
28
+
29
+
type query struct {
30
+
Language string
31
+
SinceRaw string // empty = no time filter
32
+
Threshold time.Time // zero = no time filter
33
+
}
34
+
35
+
func parseQuery(r *http.Request) query {
36
+
q := query{
37
+
Language: r.URL.Query().Get("language"),
38
+
SinceRaw: r.URL.Query().Get("since"),
39
+
}
40
+
if q.SinceRaw != "" {
41
+
if d, err := time.ParseDuration(q.SinceRaw); err == nil {
42
+
q.Threshold = time.Now().Add(-d)
43
+
}
44
+
}
45
+
return q
46
+
}
47
+
48
+
func reposJSON(s *store.Store) http.HandlerFunc {
49
+
return func(w http.ResponseWriter, r *http.Request) {
50
+
q := parseQuery(r)
51
+
repos, err := s.RecentRepos(r.Context(), q.Language, q.Threshold, 0)
52
+
if err != nil {
53
+
http.Error(w, err.Error(), http.StatusInternalServerError)
54
+
return
55
+
}
56
+
w.Header().Set("Content-Type", "application/json")
57
+
_ = json.NewEncoder(w).Encode(repos)
58
+
}
59
+
}
60
+
61
+
func reposHTML(s *store.Store) http.HandlerFunc {
62
+
return func(w http.ResponseWriter, r *http.Request) {
63
+
q := parseQuery(r)
64
+
repos, err := s.RecentRepos(r.Context(), q.Language, q.Threshold, 0)
65
+
if err != nil {
66
+
http.Error(w, err.Error(), http.StatusInternalServerError)
67
+
return
68
+
}
69
+
render(w, "repos", struct {
70
+
Active string
71
+
Query query
72
+
Repos []store.Repo
73
+
}{"repos", q, repos})
74
+
}
75
+
}
76
+
77
+
func knotsHTML(s *store.Store) http.HandlerFunc {
78
+
return func(w http.ResponseWriter, r *http.Request) {
79
+
knots, err := s.Knots(r.Context())
80
+
if err != nil {
81
+
http.Error(w, err.Error(), http.StatusInternalServerError)
82
+
return
83
+
}
84
+
render(w, "knots", struct {
85
+
Active string
86
+
Knots []store.KnotEntry
87
+
}{"knots", knots})
88
+
}
89
+
}
90
+
91
+
func knotToggle(s *store.Store) http.HandlerFunc {
92
+
return func(w http.ResponseWriter, r *http.Request) {
93
+
host := r.FormValue("host")
94
+
if host == "" {
95
+
http.Error(w, "missing host", http.StatusBadRequest)
96
+
return
97
+
}
98
+
disable := r.FormValue("disable") == "1"
99
+
if err := s.SetKnotDisabled(r.Context(), host, disable); err != nil {
100
+
http.Error(w, err.Error(), http.StatusInternalServerError)
101
+
return
102
+
}
103
+
http.Redirect(w, r, "/knots", http.StatusSeeOther)
104
+
}
105
+
}
106
+
107
+
func handlesHTML(s *store.Store) http.HandlerFunc {
108
+
return func(w http.ResponseWriter, r *http.Request) {
109
+
handles, err := s.Handles(r.Context(), 0)
110
+
if err != nil {
111
+
http.Error(w, err.Error(), http.StatusInternalServerError)
112
+
return
113
+
}
114
+
render(w, "handles", struct {
115
+
Active string
116
+
Handles []store.HandleEntry
117
+
}{"handles", handles})
118
+
}
119
+
}
120
+
121
+
func render(w http.ResponseWriter, name string, data any) {
122
+
w.Header().Set("Content-Type", "text/html; charset=utf-8")
123
+
if err := tmpl.ExecuteTemplate(w, name, data); err != nil {
124
+
http.Error(w, err.Error(), http.StatusInternalServerError)
125
+
}
126
+
}
+20
web/templates/handles.html
+20
web/templates/handles.html
···
1
+
{{define "handles"}}{{template "head" .}}
2
+
<p class="summary">{{len .Handles}} resolved handles</p>
3
+
{{if .Handles}}
4
+
<table>
5
+
<thead><tr><th>handle</th><th>did</th><th>repos</th><th>refreshed</th></tr></thead>
6
+
<tbody>
7
+
{{range .Handles}}
8
+
<tr>
9
+
<td class="handle"><a href="https://tangled.org/{{.Handle}}" target="_blank" rel="noopener">{{.Handle}}</a></td>
10
+
<td class="did">{{.DID}}</td>
11
+
<td class="num{{if .Repos}} has{{end}}">{{.Repos}}</td>
12
+
<td class="created">{{.RefreshedAt.UTC.Format "2006-01-02 15:04"}}</td>
13
+
</tr>
14
+
{{end}}
15
+
</tbody>
16
+
</table>
17
+
{{else}}
18
+
<div class="empty">no handles yet</div>
19
+
{{end}}
20
+
{{template "foot" .}}{{end}}
+31
web/templates/knots.html
+31
web/templates/knots.html
···
1
+
{{define "knots"}}{{template "head" .}}
2
+
<p class="summary">{{len .Knots}} knots</p>
3
+
{{if .Knots}}
4
+
<table>
5
+
<thead><tr>
6
+
<th>knot</th><th>repos</th><th>last ok</th><th>last error</th><th>err count</th><th>state</th><th></th>
7
+
</tr></thead>
8
+
<tbody>
9
+
{{range .Knots}}
10
+
<tr {{if .Disabled}}style="opacity:.5"{{end}}>
11
+
<td class="knot">{{.Host}}</td>
12
+
<td class="num has">{{.Repos}}</td>
13
+
<td class="created">{{if .LastOK}}{{.LastOK.UTC.Format "2006-01-02 15:04"}}{{end}}</td>
14
+
<td class="created" title="{{.LastErrorMsg}}">{{if .LastError}}{{.LastError.UTC.Format "2006-01-02 15:04"}}{{end}}</td>
15
+
<td class="num{{if .ConsecutiveErrors}} has{{end}}">{{.ConsecutiveErrors}}</td>
16
+
<td>{{if .Disabled}}disabled{{else if .ConsecutiveErrors}}faulty{{else if .LastOK}}ok{{else}}untested{{end}}</td>
17
+
<td>
18
+
<form method="post" action="/knots/toggle" style="margin:0">
19
+
<input type="hidden" name="host" value="{{.Host}}">
20
+
<input type="hidden" name="disable" value="{{if .Disabled}}0{{else}}1{{end}}">
21
+
<button>{{if .Disabled}}enable{{else}}disable{{end}}</button>
22
+
</form>
23
+
</td>
24
+
</tr>
25
+
{{end}}
26
+
</tbody>
27
+
</table>
28
+
{{else}}
29
+
<div class="empty">no knots yet</div>
30
+
{{end}}
31
+
{{template "foot" .}}{{end}}
+45
web/templates/layout.html
+45
web/templates/layout.html
···
1
+
{{define "head"}}<!doctype html>
2
+
<html lang="en">
3
+
<head>
4
+
<meta charset="utf-8">
5
+
<title>tangled repo firehose</title>
6
+
<style>
7
+
:root { color-scheme: light dark; }
8
+
body { font: 14px ui-monospace, SFMono-Regular, Menlo, monospace; max-width: 1200px; margin: 2em auto; padding: 0 1em; }
9
+
h1 { font-size: 1.4em; margin: 0 0 .25em; }
10
+
nav { display: flex; gap: 1em; margin: 1em 0; padding-bottom: .5em; border-bottom: 1px solid color-mix(in srgb, currentColor 15%, transparent); }
11
+
nav a { color: inherit; text-decoration: none; opacity: .6; }
12
+
nav a:hover { opacity: 1; }
13
+
nav a.active { font-weight: 700; opacity: 1; }
14
+
form { display: flex; gap: .75em; align-items: end; margin: 1em 0; flex-wrap: wrap; }
15
+
label { display: flex; flex-direction: column; font-size: 12px; color: #888; }
16
+
input, button { font: inherit; padding: .35em .5em; }
17
+
.summary { color: #888; margin-bottom: .5em; }
18
+
table { border-collapse: collapse; width: 100%; }
19
+
th, td { border-bottom: 1px solid color-mix(in srgb, currentColor 15%, transparent); padding: .4em .6em; text-align: left; vertical-align: top; }
20
+
th { font-weight: 600; }
21
+
td.created { white-space: nowrap; color: #888; }
22
+
td.handle { font-weight: 600; }
23
+
td.handle a, td.knot a, .link a { color: inherit; text-decoration: none; border-bottom: 1px dotted currentColor; }
24
+
td.handle a:hover, td.knot a:hover, .link a:hover { border-bottom-style: solid; }
25
+
td.knot, td.did { color: #888; }
26
+
td.lang { color: #6a6; }
27
+
td.num { text-align: right; font-variant-numeric: tabular-nums; color: #888; }
28
+
td.num.has { color: inherit; }
29
+
td.desc { color: #aaa; max-width: 480px; word-break: break-word; }
30
+
.empty { padding: 3em; text-align: center; color: #888; border: 1px dashed currentColor; opacity: .4; }
31
+
</style>
32
+
</head>
33
+
<body>
34
+
<h1>tangled repo firehose</h1>
35
+
<nav>
36
+
<a href="/" {{if eq .Active "repos"}}class="active"{{end}}>repos</a>
37
+
<a href="/knots" {{if eq .Active "knots"}}class="active"{{end}}>knots</a>
38
+
<a href="/handles" {{if eq .Active "handles"}}class="active"{{end}}>handles</a>
39
+
</nav>
40
+
{{end}}
41
+
42
+
{{define "foot"}}
43
+
</body>
44
+
</html>
45
+
{{end}}
+33
web/templates/repos.html
+33
web/templates/repos.html
···
1
+
{{define "repos"}}{{template "head" .}}
2
+
<form method="get">
3
+
<label>language<input name="language" value="{{.Query.Language}}" placeholder="any"></label>
4
+
<label>since<input name="since" value="{{.Query.SinceRaw}}" placeholder="e.g. 24h"></label>
5
+
<button>filter</button>
6
+
</form>
7
+
<p class="summary">{{len .Repos}} repos · <a href="/repos.json?language={{.Query.Language}}&since={{.Query.SinceRaw}}">json</a></p>
8
+
{{if .Repos}}
9
+
<table>
10
+
<thead><tr><th>created</th><th>repo</th><th>knot</th><th>language</th><th>★</th><th>description</th></tr></thead>
11
+
<tbody>
12
+
{{range .Repos}}
13
+
<tr>
14
+
<td class="created">{{.CreatedAt.UTC.Format "2006-01-02 15:04"}}</td>
15
+
<td class="handle">
16
+
{{- if .Handle -}}
17
+
<a href="https://tangled.org/{{.Handle}}/{{.Name}}" target="_blank" rel="noopener">{{.Handle}}/{{.Name}}</a>
18
+
{{- else -}}
19
+
{{.DID}}/{{.Name}}
20
+
{{- end -}}
21
+
</td>
22
+
<td class="knot">{{.Knot}}</td>
23
+
<td class="lang">{{.Primary}}</td>
24
+
<td class="num{{if .StarCount}} has{{end}}">{{.StarCount}}</td>
25
+
<td class="desc">{{.Description}}</td>
26
+
</tr>
27
+
{{end}}
28
+
</tbody>
29
+
</table>
30
+
{{else}}
31
+
<div class="empty">no repos yet</div>
32
+
{{end}}
33
+
{{template "foot" .}}{{end}}