this repo has no description
automod: rules engine framework (#434)
`automod` is a package to help write moderation automation "rules" to
catch things like spam and flag accounts or content for human review (or
potentially even auto-action).
`hepa` is a simple service which consumes from a BGS firehose and pushes
events through `automod`.
This is an early draft of the rules engine, some notes:
- rules are written in golang here. we might end up using a DSL or
scripting language of some sort eventually
- the framework and some example rules are here in a public repo. we'll
obviously keep some rules and configuration private in the long run
- intention is to use redis for caching and state persistence. will
likely end up with additional datastores
+2758
-13
+52
.github/workflows/container-hepa-aws.yaml
+52
.github/workflows/container-hepa-aws.yaml
···
1
+
name: container-hepa-aws
2
+
on: [push]
3
+
env:
4
+
REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
5
+
USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}
6
+
PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }}
7
+
# github.repository as <account>/<repo>
8
+
IMAGE_NAME: hepa
9
+
10
+
jobs:
11
+
container-hepa-aws:
12
+
if: github.repository == 'bluesky-social/indigo'
13
+
runs-on: ubuntu-latest
14
+
permissions:
15
+
contents: read
16
+
packages: write
17
+
id-token: write
18
+
19
+
steps:
20
+
- name: Checkout repository
21
+
uses: actions/checkout@v3
22
+
23
+
- name: Setup Docker buildx
24
+
uses: docker/setup-buildx-action@v1
25
+
26
+
- name: Log into registry ${{ env.REGISTRY }}
27
+
uses: docker/login-action@v2
28
+
with:
29
+
registry: ${{ env.REGISTRY }}
30
+
username: ${{ env.USERNAME }}
31
+
password: ${{ env.PASSWORD }}
32
+
33
+
- name: Extract Docker metadata
34
+
id: meta
35
+
uses: docker/metadata-action@v4
36
+
with:
37
+
images: |
38
+
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
39
+
tags: |
40
+
type=sha,enable=true,priority=100,prefix=,suffix=,format=long
41
+
42
+
- name: Build and push Docker image
43
+
id: build-and-push
44
+
uses: docker/build-push-action@v4
45
+
with:
46
+
context: .
47
+
file: ./cmd/hepa/Dockerfile
48
+
push: ${{ github.event_name != 'pull_request' }}
49
+
tags: ${{ steps.meta.outputs.tags }}
50
+
labels: ${{ steps.meta.outputs.labels }}
51
+
cache-from: type=gha
52
+
cache-to: type=gha,mode=max
+54
.github/workflows/container-hepa-ghcr.yaml
+54
.github/workflows/container-hepa-ghcr.yaml
···
1
+
name: container-hepa-ghcr
2
+
on:
3
+
push:
4
+
branches:
5
+
- main
6
+
- bnewbold/automod
7
+
env:
8
+
REGISTRY: ghcr.io
9
+
# github.repository as <account>/<repo>
10
+
IMAGE_NAME: ${{ github.repository }}
11
+
12
+
jobs:
13
+
container-hepa-ghcr:
14
+
if: github.repository == 'bluesky-social/indigo'
15
+
runs-on: ubuntu-latest
16
+
permissions:
17
+
contents: read
18
+
packages: write
19
+
id-token: write
20
+
21
+
steps:
22
+
- name: Checkout repository
23
+
uses: actions/checkout@v3
24
+
25
+
- name: Setup Docker buildx
26
+
uses: docker/setup-buildx-action@v1
27
+
28
+
- name: Log into registry ${{ env.REGISTRY }}
29
+
uses: docker/login-action@v2
30
+
with:
31
+
registry: ${{ env.REGISTRY }}
32
+
username: ${{ github.actor }}
33
+
password: ${{ secrets.GITHUB_TOKEN }}
34
+
35
+
- name: Extract Docker metadata
36
+
id: meta
37
+
uses: docker/metadata-action@v4
38
+
with:
39
+
images: |
40
+
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
41
+
tags: |
42
+
type=sha,enable=true,priority=100,prefix=hepa:,suffix=,format=long
43
+
44
+
- name: Build and push Docker image
45
+
id: build-and-push
46
+
uses: docker/build-push-action@v4
47
+
with:
48
+
context: .
49
+
file: ./cmd/hepa/Dockerfile
50
+
push: ${{ github.event_name != 'pull_request' }}
51
+
tags: ${{ steps.meta.outputs.tags }}
52
+
labels: ${{ steps.meta.outputs.labels }}
53
+
cache-from: type=gha
54
+
cache-to: type=gha,mode=max
+2
HACKING.md
+2
HACKING.md
···
13
13
- `cmd/fakermaker`: helper to generate fake accounts and content for testing
14
14
- `cmd/supercollider`: event stream load generation tool
15
15
- `cmd/sonar`: event stream monitoring tool
16
+
- `cmd/hepa`: auto-moderation rule engine service
16
17
- `gen`: dev tool to run CBOR type codegen
17
18
18
19
Packages:
···
23
24
- `atproto/crypto`: crytographic helpers (signing, key generation and serialization)
24
25
- `atproto/syntax`: string types and parsers for identifiers, datetimes, etc
25
26
- `atproto/identity`: DID and handle resolution
27
+
- `automod`: moderation and anti-spam rules engine
26
28
- `bgs`: server implementation for crawling, etc
27
29
- `carstore`: library for storing repo data in CAR files on disk, plus a metadata SQL db
28
30
- `events`: types, codegen CBOR helpers, and persistence for event feeds
+1
Makefile
+1
Makefile
+7
-8
api/atproto/servercreateSession.go
+7
-8
api/atproto/servercreateSession.go
···
7
7
import (
8
8
"context"
9
9
10
-
"github.com/bluesky-social/indigo/lex/util"
11
10
"github.com/bluesky-social/indigo/xrpc"
12
11
)
13
12
···
20
19
21
20
// ServerCreateSession_Output is the output of a com.atproto.server.createSession call.
22
21
type ServerCreateSession_Output struct {
23
-
AccessJwt string `json:"accessJwt" cborgen:"accessJwt"`
24
-
Did string `json:"did" cborgen:"did"`
25
-
DidDoc *util.LexiconTypeDecoder `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"`
26
-
Email *string `json:"email,omitempty" cborgen:"email,omitempty"`
27
-
EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"`
28
-
Handle string `json:"handle" cborgen:"handle"`
29
-
RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"`
22
+
AccessJwt string `json:"accessJwt" cborgen:"accessJwt"`
23
+
Did string `json:"did" cborgen:"did"`
24
+
//DidDoc *util.LexiconTypeDecoder `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"`
25
+
Email *string `json:"email,omitempty" cborgen:"email,omitempty"`
26
+
EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"`
27
+
Handle string `json:"handle" cborgen:"handle"`
28
+
RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"`
30
29
}
31
30
32
31
// ServerCreateSession calls the XRPC method "com.atproto.server.createSession".
+4
-4
atproto/syntax/aturi.go
+4
-4
atproto/syntax/aturi.go
···
62
62
// Returns path segment, without leading slash, as would be used in an atproto repository key. Or empty string if there is no path.
63
63
func (n ATURI) Path() string {
64
64
parts := strings.SplitN(string(n), "/", 5)
65
-
if len(parts) < 3 {
65
+
if len(parts) < 4 {
66
66
// something has gone wrong (would not validate)
67
67
return ""
68
68
}
69
-
if len(parts) == 3 {
70
-
return parts[2]
69
+
if len(parts) == 4 {
70
+
return parts[3]
71
71
}
72
-
return parts[2] + "/" + parts[3]
72
+
return parts[3] + "/" + parts[4]
73
73
}
74
74
75
75
// Returns a valid NSID if there is one in the appropriate part of the path, otherwise empty.
+26
-1
atproto/syntax/aturi_test.go
+26
-1
atproto/syntax/aturi_test.go
···
20
20
if len(line) == 0 || line[0] == '#' {
21
21
continue
22
22
}
23
-
_, err := ParseATURI(line)
23
+
aturi, err := ParseATURI(line)
24
24
if err != nil {
25
25
fmt.Println("FAILED, GOOD: " + line)
26
26
}
27
27
assert.NoError(err)
28
+
29
+
// check that Path() is working
30
+
col := aturi.Collection()
31
+
rkey := aturi.RecordKey()
32
+
if rkey != "" {
33
+
assert.Equal(col.String()+"/"+rkey.String(), aturi.Path())
34
+
} else if col != "" {
35
+
assert.Equal(col.String(), aturi.Path())
36
+
}
28
37
}
29
38
assert.NoError(scanner.Err())
30
39
}
···
67
76
rkey := uri.RecordKey()
68
77
assert.Equal(parts[3], rkey.String())
69
78
}
79
+
}
70
80
81
+
func TestATURIPath(t *testing.T) {
82
+
assert := assert.New(t)
83
+
84
+
uri1, err := ParseATURI("at://did:abc:123/io.nsid.someFunc/record-key")
85
+
assert.NoError(err)
86
+
assert.Equal("io.nsid.someFunc/record-key", uri1.Path())
87
+
88
+
uri2, err := ParseATURI("at://did:abc:123/io.nsid.someFunc")
89
+
assert.NoError(err)
90
+
assert.Equal("io.nsid.someFunc", uri2.Path())
91
+
92
+
uri3, err := ParseATURI("at://did:abc:123")
93
+
assert.NoError(err)
94
+
assert.Equal("", uri3.Path())
71
95
}
72
96
73
97
func TestATURINormalize(t *testing.T) {
···
93
117
_ = bad.RecordKey()
94
118
_ = bad.Normalize()
95
119
_ = bad.String()
120
+
_ = bad.Path()
96
121
}
97
122
}
+23
automod/README.md
+23
automod/README.md
···
1
+
indigo/automod
2
+
==============
3
+
4
+
This package (`github.com/bluesky-social/indigo/automod`) contains a "rules engine" to augment human moderators in the atproto network. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". A lot of what this package does is collect and maintain caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information.
5
+
6
+
A primary design goal is to have a flexible framework to allow new rules to be written and deployed rapidly in response to new patterns of spam and abuse.
7
+
8
+
Some example rules are included in the `automod/rules` package, but the expectation is that some real-world rules will be kept secret.
9
+
10
+
Code for subscribing to a firehose is not included here; see `cmd/hepa` for a complete service built on this library.
11
+
12
+
13
+
## Design
14
+
15
+
Prior art and inspiration:
16
+
17
+
* The [SQRL language](https://sqrl-lang.github.io/sqrl/) and runtime was originally developed by an industry vendor named Smyte, then acquired by Twitter, with some core Javascript components released open source in 2023. The SQRL documentation is extensive and describes many of the design trade-offs and features specific to rules engines. Bluesky considered adopting SQRL but decided to start with a simpler runtime with rules in a known language (golang).
18
+
19
+
* Reddit's [automod system](https://www.reddit.com/wiki/automoderator/) is simple an accessible for non-technical sub-reddit community moderators. Discord has a large ecosystem of bots which can help communities manage some moderation tasks, in particular mitigating spam and brigading.
20
+
21
+
* Facebook's FXL and Haxl rule languages have been in use for over a decade. The 2012 paper ["The Facebook Immune System"](https://css.csail.mit.edu/6.858/2012/readings/facebook-immune.pdf) gives a good overview of design goals and how a rules engine fits in to a an overall anti-spam/anti-abuse pipeline.
22
+
23
+
* Email anti-spam systems like SpamAssassin and rspamd have been modular and configurable for several decades.
+126
automod/account_meta.go
+126
automod/account_meta.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"time"
8
+
9
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
10
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
11
+
"github.com/bluesky-social/indigo/atproto/identity"
12
+
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
)
14
+
15
+
type ProfileSummary struct {
16
+
HasAvatar bool
17
+
Description *string
18
+
DisplayName *string
19
+
}
20
+
21
+
type AccountPrivate struct {
22
+
Email string
23
+
EmailConfirmed bool
24
+
IndexedAt time.Time
25
+
}
26
+
27
+
// information about a repo/account/identity, always pre-populated and relevant to many rules
28
+
type AccountMeta struct {
29
+
Identity *identity.Identity
30
+
Profile ProfileSummary
31
+
Private *AccountPrivate
32
+
AccountLabels []string
33
+
FollowersCount int64
34
+
FollowsCount int64
35
+
PostsCount int64
36
+
}
37
+
38
+
func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) {
39
+
40
+
// wipe parsed public key; it's a waste of space and can't serialize
41
+
ident.ParsedPublicKey = nil
42
+
43
+
// fallback in case client wasn't configured (eg, testing)
44
+
if e.BskyClient == nil {
45
+
e.Logger.Warn("skipping account meta hydration")
46
+
am := AccountMeta{
47
+
Identity: ident,
48
+
Profile: ProfileSummary{},
49
+
}
50
+
return &am, nil
51
+
}
52
+
53
+
existing, err := e.Cache.Get(ctx, "acct", ident.DID.String())
54
+
if err != nil {
55
+
return nil, err
56
+
}
57
+
if existing != "" {
58
+
var am AccountMeta
59
+
err := json.Unmarshal([]byte(existing), &am)
60
+
if err != nil {
61
+
return nil, fmt.Errorf("parsing AccountMeta from cache: %v", err)
62
+
}
63
+
am.Identity = ident
64
+
return &am, nil
65
+
}
66
+
67
+
// fetch account metadata
68
+
pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
69
+
if err != nil {
70
+
return nil, err
71
+
}
72
+
73
+
var labels []string
74
+
for _, lbl := range pv.Labels {
75
+
labels = append(labels, lbl.Val)
76
+
}
77
+
78
+
am := AccountMeta{
79
+
Identity: ident,
80
+
Profile: ProfileSummary{
81
+
HasAvatar: pv.Avatar != nil,
82
+
Description: pv.Description,
83
+
DisplayName: pv.DisplayName,
84
+
},
85
+
AccountLabels: dedupeStrings(labels),
86
+
}
87
+
if pv.PostsCount != nil {
88
+
am.PostsCount = *pv.PostsCount
89
+
}
90
+
if pv.FollowersCount != nil {
91
+
am.FollowersCount = *pv.FollowersCount
92
+
}
93
+
if pv.FollowsCount != nil {
94
+
am.FollowsCount = *pv.FollowsCount
95
+
}
96
+
97
+
if e.AdminClient != nil {
98
+
pv, err := comatproto.AdminGetAccountInfo(ctx, e.AdminClient, ident.DID.String())
99
+
if err != nil {
100
+
return nil, err
101
+
}
102
+
ap := AccountPrivate{}
103
+
if pv.Email != nil && *pv.Email != "" {
104
+
ap.Email = *pv.Email
105
+
}
106
+
if pv.EmailConfirmedAt != nil && *pv.EmailConfirmedAt != "" {
107
+
ap.EmailConfirmed = true
108
+
}
109
+
ts, err := syntax.ParseDatetimeTime(pv.IndexedAt)
110
+
if err != nil {
111
+
return nil, err
112
+
}
113
+
ap.IndexedAt = ts
114
+
am.Private = &ap
115
+
}
116
+
117
+
val, err := json.Marshal(&am)
118
+
if err != nil {
119
+
return nil, err
120
+
}
121
+
122
+
if err := e.Cache.Set(ctx, "acct", ident.DID.String(), string(val)); err != nil {
123
+
return nil, err
124
+
}
125
+
return &am, nil
126
+
}
+36
automod/cachestore.go
+36
automod/cachestore.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/hashicorp/golang-lru/v2/expirable"
8
+
)
9
+
10
+
type CacheStore interface {
11
+
Get(ctx context.Context, name, key string) (string, error)
12
+
Set(ctx context.Context, name, key string, val string) error
13
+
}
14
+
15
+
type MemCacheStore struct {
16
+
Data *expirable.LRU[string, string]
17
+
}
18
+
19
+
func NewMemCacheStore(capacity int, ttl time.Duration) MemCacheStore {
20
+
return MemCacheStore{
21
+
Data: expirable.NewLRU[string, string](capacity, nil, ttl),
22
+
}
23
+
}
24
+
25
+
func (s MemCacheStore) Get(ctx context.Context, name, key string) (string, error) {
26
+
v, ok := s.Data.Get(name + "/" + key)
27
+
if !ok {
28
+
return "", nil
29
+
}
30
+
return v, nil
31
+
}
32
+
33
+
func (s MemCacheStore) Set(ctx context.Context, name, key string, val string) error {
34
+
s.Data.Add(name+"/"+key, val)
35
+
return nil
36
+
}
+68
automod/countstore.go
+68
automod/countstore.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"time"
8
+
)
9
+
10
+
const (
11
+
PeriodTotal = "total"
12
+
PeriodDay = "day"
13
+
PeriodHour = "hour"
14
+
)
15
+
16
+
type CountStore interface {
17
+
GetCount(ctx context.Context, name, val, period string) (int, error)
18
+
Increment(ctx context.Context, name, val string) error
19
+
// TODO: batch increment method
20
+
}
21
+
22
+
// TODO: this implementation isn't race-safe (yet)!
23
+
type MemCountStore struct {
24
+
Counts map[string]int
25
+
}
26
+
27
+
func NewMemCountStore() MemCountStore {
28
+
return MemCountStore{
29
+
Counts: make(map[string]int),
30
+
}
31
+
}
32
+
33
+
func PeriodBucket(name, val, period string) string {
34
+
switch period {
35
+
case PeriodTotal:
36
+
return fmt.Sprintf("%s/%s", name, val)
37
+
case PeriodDay:
38
+
t := time.Now().UTC().Format(time.DateOnly)
39
+
return fmt.Sprintf("%s/%s/%s", name, val, t)
40
+
case PeriodHour:
41
+
t := time.Now().UTC().Format(time.RFC3339)[0:13]
42
+
return fmt.Sprintf("%s/%s/%s", name, val, t)
43
+
default:
44
+
slog.Warn("unhandled counter period", "period", period)
45
+
return fmt.Sprintf("%s/%s", name, val)
46
+
}
47
+
}
48
+
49
+
func (s MemCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) {
50
+
v, ok := s.Counts[PeriodBucket(name, val, period)]
51
+
if !ok {
52
+
return 0, nil
53
+
}
54
+
return v, nil
55
+
}
56
+
57
+
func (s MemCountStore) Increment(ctx context.Context, name, val string) error {
58
+
for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} {
59
+
k := PeriodBucket(name, val, p)
60
+
v, ok := s.Counts[k]
61
+
if !ok {
62
+
v = 0
63
+
}
64
+
v = v + 1
65
+
s.Counts[k] = v
66
+
}
67
+
return nil
68
+
}
+6
automod/doc.go
+6
automod/doc.go
···
1
+
// Auto-Moderation rules engine for anti-spam and other moderation tasks.
2
+
//
3
+
// This package (`github.com/bluesky-social/indigo/automod`) contains a "rules engine" to augment human moderators in the atproto network. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". A lot of what this package does is collect and maintain caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information.
4
+
//
5
+
// See `automod/README.md` for more background, and `cmd/hepa` for a daemon built on this package.
6
+
package automod
+166
automod/engine.go
+166
automod/engine.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"strings"
8
+
9
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/bluesky-social/indigo/xrpc"
13
+
)
14
+
15
+
// runtime for executing rules, managing state, and recording moderation actions.
16
+
//
17
+
// TODO: careful when initializing: several fields should not be null or zero, even though they are pointer type.
18
+
type Engine struct {
19
+
Logger *slog.Logger
20
+
Directory identity.Directory
21
+
Rules RuleSet
22
+
Counters CountStore
23
+
Sets SetStore
24
+
Cache CacheStore
25
+
RelayClient *xrpc.Client
26
+
BskyClient *xrpc.Client
27
+
// used to persist moderation actions in mod service (optional)
28
+
AdminClient *xrpc.Client
29
+
}
30
+
31
+
func (e *Engine) ProcessIdentityEvent(ctx context.Context, t string, did syntax.DID) error {
32
+
// similar to an HTTP server, we want to recover any panics from rule execution
33
+
defer func() {
34
+
if r := recover(); r != nil {
35
+
e.Logger.Error("automod event execution exception", "err", r)
36
+
}
37
+
}()
38
+
39
+
ident, err := e.Directory.LookupDID(ctx, did)
40
+
if err != nil {
41
+
return fmt.Errorf("resolving identity: %w", err)
42
+
}
43
+
if ident == nil {
44
+
return fmt.Errorf("identity not found for did: %s", did.String())
45
+
}
46
+
47
+
am, err := e.GetAccountMeta(ctx, ident)
48
+
if err != nil {
49
+
return err
50
+
}
51
+
evt := IdentityEvent{
52
+
RepoEvent{
53
+
Engine: e,
54
+
Logger: e.Logger.With("did", am.Identity.DID),
55
+
Account: *am,
56
+
},
57
+
}
58
+
if err := e.Rules.CallIdentityRules(&evt); err != nil {
59
+
return err
60
+
}
61
+
if evt.Err != nil {
62
+
return evt.Err
63
+
}
64
+
evt.CanonicalLogLine()
65
+
if err := evt.PersistActions(ctx); err != nil {
66
+
return err
67
+
}
68
+
return nil
69
+
}
70
+
71
+
func (e *Engine) ProcessRecord(ctx context.Context, did syntax.DID, path, recCID string, rec any) error {
72
+
// similar to an HTTP server, we want to recover any panics from rule execution
73
+
defer func() {
74
+
if r := recover(); r != nil {
75
+
e.Logger.Error("automod event execution exception", "err", r)
76
+
}
77
+
}()
78
+
79
+
ident, err := e.Directory.LookupDID(ctx, did)
80
+
if err != nil {
81
+
return fmt.Errorf("resolving identity: %w", err)
82
+
}
83
+
if ident == nil {
84
+
return fmt.Errorf("identity not found for did: %s", did.String())
85
+
}
86
+
87
+
am, err := e.GetAccountMeta(ctx, ident)
88
+
if err != nil {
89
+
return err
90
+
}
91
+
evt := e.NewRecordEvent(*am, path, recCID, rec)
92
+
e.Logger.Debug("processing record", "did", ident.DID, "path", path)
93
+
if err := e.Rules.CallRecordRules(&evt); err != nil {
94
+
return err
95
+
}
96
+
if evt.Err != nil {
97
+
return evt.Err
98
+
}
99
+
evt.CanonicalLogLine()
100
+
if err := evt.PersistActions(ctx); err != nil {
101
+
return err
102
+
}
103
+
if err := evt.PersistCounters(ctx); err != nil {
104
+
return err
105
+
}
106
+
107
+
return nil
108
+
}
109
+
110
+
func (e *Engine) FetchAndProcessRecord(ctx context.Context, uri string) error {
111
+
// resolve URI, identity, and record
112
+
aturi, err := syntax.ParseATURI(uri)
113
+
if err != nil {
114
+
return fmt.Errorf("parsing AT-URI argument: %v", err)
115
+
}
116
+
if aturi.RecordKey() == "" {
117
+
return fmt.Errorf("need a full, not partial, AT-URI: %s", uri)
118
+
}
119
+
ident, err := e.Directory.Lookup(ctx, aturi.Authority())
120
+
if err != nil {
121
+
return fmt.Errorf("resolving AT-URI authority: %v", err)
122
+
}
123
+
pdsURL := ident.PDSEndpoint()
124
+
if pdsURL == "" {
125
+
return fmt.Errorf("could not resolve PDS endpoint for AT-URI account: %s", ident.DID.String())
126
+
}
127
+
pdsClient := xrpc.Client{Host: ident.PDSEndpoint()}
128
+
129
+
e.Logger.Info("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String())
130
+
out, err := comatproto.RepoGetRecord(ctx, &pdsClient, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String())
131
+
if err != nil {
132
+
return fmt.Errorf("fetching record from Relay (%s): %v", aturi, err)
133
+
}
134
+
if out.Cid == nil {
135
+
return fmt.Errorf("expected a CID in getRecord response")
136
+
}
137
+
return e.ProcessRecord(ctx, ident.DID, aturi.Path(), *out.Cid, out.Value.Val)
138
+
}
139
+
140
+
func (e *Engine) NewRecordEvent(am AccountMeta, path, recCID string, rec any) RecordEvent {
141
+
parts := strings.SplitN(path, "/", 2)
142
+
return RecordEvent{
143
+
RepoEvent{
144
+
Engine: e,
145
+
Logger: e.Logger.With("did", am.Identity.DID, "collection", parts[0], "rkey", parts[1]),
146
+
Account: am,
147
+
},
148
+
rec,
149
+
parts[0],
150
+
parts[1],
151
+
recCID,
152
+
[]string{},
153
+
false,
154
+
[]ModReport{},
155
+
[]string{},
156
+
}
157
+
}
158
+
159
+
func (e *Engine) GetCount(name, val, period string) (int, error) {
160
+
return e.Counters.GetCount(context.TODO(), name, val, period)
161
+
}
162
+
163
+
// checks if `val` is an element of set `name`
164
+
func (e *Engine) InSet(name, val string) (bool, error) {
165
+
return e.Sets.InSet(context.TODO(), name, val)
166
+
}
+82
automod/engine_test.go
+82
automod/engine_test.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
"testing"
7
+
8
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
9
+
"github.com/bluesky-social/indigo/atproto/identity"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
12
+
"github.com/stretchr/testify/assert"
13
+
)
14
+
15
+
func simpleRule(evt *RecordEvent, post *appbsky.FeedPost) error {
16
+
for _, tag := range post.Tags {
17
+
if evt.InSet("banned-hashtags", tag) {
18
+
evt.AddRecordLabel("bad-hashtag")
19
+
break
20
+
}
21
+
}
22
+
for _, facet := range post.Facets {
23
+
for _, feat := range facet.Features {
24
+
if feat.RichtextFacet_Tag != nil {
25
+
tag := feat.RichtextFacet_Tag.Tag
26
+
if evt.InSet("banned-hashtags", tag) {
27
+
evt.AddRecordLabel("bad-hashtag")
28
+
break
29
+
}
30
+
}
31
+
}
32
+
}
33
+
return nil
34
+
}
35
+
36
+
func engineFixture() Engine {
37
+
rules := RuleSet{
38
+
PostRules: []PostRuleFunc{
39
+
simpleRule,
40
+
},
41
+
}
42
+
sets := NewMemSetStore()
43
+
sets.Sets["banned-hashtags"] = make(map[string]bool)
44
+
sets.Sets["banned-hashtags"]["slur"] = true
45
+
dir := identity.NewMockDirectory()
46
+
id1 := identity.Identity{
47
+
DID: syntax.DID("did:plc:abc111"),
48
+
Handle: syntax.Handle("handle.example.com"),
49
+
}
50
+
dir.Insert(id1)
51
+
engine := Engine{
52
+
Logger: slog.Default(),
53
+
Directory: &dir,
54
+
Counters: NewMemCountStore(),
55
+
Sets: sets,
56
+
Rules: rules,
57
+
}
58
+
return engine
59
+
}
60
+
61
+
func TestEngineBasics(t *testing.T) {
62
+
assert := assert.New(t)
63
+
ctx := context.Background()
64
+
65
+
engine := engineFixture()
66
+
id1 := identity.Identity{
67
+
DID: syntax.DID("did:plc:abc111"),
68
+
Handle: syntax.Handle("handle.example.com"),
69
+
}
70
+
path := "app.bsky.feed.post/abc123"
71
+
cid1 := "cid123"
72
+
p1 := appbsky.FeedPost{
73
+
Text: "some post blah",
74
+
}
75
+
assert.NoError(engine.ProcessRecord(ctx, id1.DID, path, cid1, &p1))
76
+
77
+
p2 := appbsky.FeedPost{
78
+
Text: "some post blah",
79
+
Tags: []string{"one", "slur"},
80
+
}
81
+
assert.NoError(engine.ProcessRecord(ctx, id1.DID, path, cid1, &p2))
82
+
}
+263
automod/event.go
+263
automod/event.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
8
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
9
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
10
+
)
11
+
12
+
type ModReport struct {
13
+
ReasonType string
14
+
Comment string
15
+
}
16
+
17
+
type CounterRef struct {
18
+
Name string
19
+
Val string
20
+
}
21
+
22
+
// base type for events specific to an account, usually derived from a repo event stream message (one such message may result in multiple `RepoEvent`)
23
+
//
24
+
// events are both containers for data about the event itself (similar to an HTTP request type); aggregate results and state (counters, mod actions) to be persisted after all rules are run; and act as an API for additional network reads and operations.
25
+
type RepoEvent struct {
26
+
Engine *Engine
27
+
Err error
28
+
Logger *slog.Logger
29
+
Account AccountMeta
30
+
CounterIncrements []CounterRef
31
+
AccountLabels []string
32
+
AccountFlags []string
33
+
AccountReports []ModReport
34
+
AccountTakedown bool
35
+
}
36
+
37
+
func (e *RepoEvent) GetCount(name, val, period string) int {
38
+
v, err := e.Engine.GetCount(name, val, period)
39
+
if err != nil {
40
+
e.Err = err
41
+
return 0
42
+
}
43
+
return v
44
+
}
45
+
46
+
func (e *RepoEvent) InSet(name, val string) bool {
47
+
v, err := e.Engine.InSet(name, val)
48
+
if err != nil {
49
+
e.Err = err
50
+
return false
51
+
}
52
+
return v
53
+
}
54
+
55
+
func (e *RepoEvent) Increment(name, val string) {
56
+
e.CounterIncrements = append(e.CounterIncrements, CounterRef{Name: name, Val: val})
57
+
}
58
+
59
+
func (e *RepoEvent) TakedownAccount() {
60
+
e.AccountTakedown = true
61
+
}
62
+
63
+
func (e *RepoEvent) AddAccountLabel(val string) {
64
+
e.AccountLabels = append(e.AccountLabels, val)
65
+
}
66
+
67
+
func (e *RepoEvent) AddAccountFlag(val string) {
68
+
e.AccountFlags = append(e.AccountFlags, val)
69
+
}
70
+
71
+
func (e *RepoEvent) ReportAccount(reason, comment string) {
72
+
e.AccountReports = append(e.AccountReports, ModReport{ReasonType: reason, Comment: comment})
73
+
}
74
+
75
+
func (e *RepoEvent) PersistAccountActions(ctx context.Context) error {
76
+
if e.Engine.AdminClient == nil {
77
+
return nil
78
+
}
79
+
xrpcc := e.Engine.AdminClient
80
+
if len(e.AccountLabels) > 0 {
81
+
_, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{
82
+
Action: "com.atproto.admin.defs#flag",
83
+
CreateLabelVals: dedupeStrings(e.AccountLabels),
84
+
Reason: "automod",
85
+
CreatedBy: xrpcc.Auth.Did,
86
+
Subject: &comatproto.AdminTakeModerationAction_Input_Subject{
87
+
AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{
88
+
Did: e.Account.Identity.DID.String(),
89
+
},
90
+
},
91
+
})
92
+
if err != nil {
93
+
return err
94
+
}
95
+
}
96
+
// TODO: AccountFlags
97
+
for _, mr := range e.AccountReports {
98
+
_, err := comatproto.ModerationCreateReport(ctx, xrpcc, &comatproto.ModerationCreateReport_Input{
99
+
ReasonType: &mr.ReasonType,
100
+
Reason: &mr.Comment,
101
+
Subject: &comatproto.ModerationCreateReport_Input_Subject{
102
+
AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{
103
+
Did: e.Account.Identity.DID.String(),
104
+
},
105
+
},
106
+
})
107
+
if err != nil {
108
+
return err
109
+
}
110
+
}
111
+
if e.AccountTakedown {
112
+
_, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{
113
+
Action: "com.atproto.admin.defs#takedown",
114
+
Reason: "automod",
115
+
CreatedBy: xrpcc.Auth.Did,
116
+
Subject: &comatproto.AdminTakeModerationAction_Input_Subject{
117
+
AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{
118
+
Did: e.Account.Identity.DID.String(),
119
+
},
120
+
},
121
+
})
122
+
if err != nil {
123
+
return err
124
+
}
125
+
}
126
+
return nil
127
+
}
128
+
129
+
func (e *RepoEvent) PersistActions(ctx context.Context) error {
130
+
return e.PersistAccountActions(ctx)
131
+
}
132
+
133
+
func (e *RepoEvent) PersistCounters(ctx context.Context) error {
134
+
// TODO: dedupe this array
135
+
for _, ref := range e.CounterIncrements {
136
+
err := e.Engine.Counters.Increment(ctx, ref.Name, ref.Val)
137
+
if err != nil {
138
+
return err
139
+
}
140
+
}
141
+
return nil
142
+
}
143
+
144
+
func (e *RepoEvent) CanonicalLogLine() {
145
+
e.Logger.Info("canonical-event-line",
146
+
"accountLabels", e.AccountLabels,
147
+
"accountFlags", e.AccountFlags,
148
+
"accountTakedown", e.AccountTakedown,
149
+
"accountReports", len(e.AccountReports),
150
+
)
151
+
}
152
+
153
+
type IdentityEvent struct {
154
+
RepoEvent
155
+
}
156
+
157
+
type RecordEvent struct {
158
+
RepoEvent
159
+
160
+
Record any
161
+
Collection string
162
+
RecordKey string
163
+
CID string
164
+
RecordLabels []string
165
+
RecordTakedown bool
166
+
RecordReports []ModReport
167
+
RecordFlags []string
168
+
// TODO: commit metadata
169
+
}
170
+
171
+
func (e *RecordEvent) TakedownRecord() {
172
+
e.RecordTakedown = true
173
+
}
174
+
175
+
func (e *RecordEvent) AddRecordLabel(val string) {
176
+
e.RecordLabels = append(e.RecordLabels, val)
177
+
}
178
+
179
+
func (e *RecordEvent) AddRecordFlag(val string) {
180
+
e.RecordFlags = append(e.RecordFlags, val)
181
+
}
182
+
183
+
func (e *RecordEvent) ReportRecord(reason, comment string) {
184
+
e.RecordReports = append(e.RecordReports, ModReport{ReasonType: reason, Comment: comment})
185
+
}
186
+
187
+
func (e *RecordEvent) PersistRecordActions(ctx context.Context) error {
188
+
if e.Engine.AdminClient == nil {
189
+
return nil
190
+
}
191
+
strongRef := comatproto.RepoStrongRef{
192
+
Cid: e.CID,
193
+
Uri: fmt.Sprintf("at://%s/%s/%s", e.Account.Identity.DID, e.Collection, e.RecordKey),
194
+
}
195
+
xrpcc := e.Engine.AdminClient
196
+
if len(e.RecordLabels) > 0 {
197
+
// TODO: this does an action, not just create labels; will update after event refactor
198
+
_, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{
199
+
Action: "com.atproto.admin.defs#flag",
200
+
CreateLabelVals: dedupeStrings(e.RecordLabels),
201
+
Reason: "automod",
202
+
CreatedBy: xrpcc.Auth.Did,
203
+
Subject: &comatproto.AdminTakeModerationAction_Input_Subject{
204
+
RepoStrongRef: &strongRef,
205
+
},
206
+
})
207
+
if err != nil {
208
+
return err
209
+
}
210
+
}
211
+
// TODO: AccountFlags
212
+
for _, mr := range e.RecordReports {
213
+
_, err := comatproto.ModerationCreateReport(ctx, xrpcc, &comatproto.ModerationCreateReport_Input{
214
+
ReasonType: &mr.ReasonType,
215
+
Reason: &mr.Comment,
216
+
Subject: &comatproto.ModerationCreateReport_Input_Subject{
217
+
RepoStrongRef: &strongRef,
218
+
},
219
+
})
220
+
if err != nil {
221
+
return err
222
+
}
223
+
}
224
+
if e.RecordTakedown {
225
+
_, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{
226
+
Action: "com.atproto.admin.defs#takedown",
227
+
Reason: "automod",
228
+
CreatedBy: xrpcc.Auth.Did,
229
+
Subject: &comatproto.AdminTakeModerationAction_Input_Subject{
230
+
RepoStrongRef: &strongRef,
231
+
},
232
+
})
233
+
if err != nil {
234
+
return err
235
+
}
236
+
}
237
+
return nil
238
+
}
239
+
240
+
func (e *RecordEvent) PersistActions(ctx context.Context) error {
241
+
if err := e.PersistAccountActions(ctx); err != nil {
242
+
return err
243
+
}
244
+
return e.PersistRecordActions(ctx)
245
+
}
246
+
247
+
func (e *RecordEvent) CanonicalLogLine() {
248
+
e.Logger.Info("canonical-event-line",
249
+
"accountLabels", e.AccountLabels,
250
+
"accountFlags", e.AccountFlags,
251
+
"accountTakedown", e.AccountTakedown,
252
+
"accountReports", len(e.AccountReports),
253
+
"recordLabels", e.RecordLabels,
254
+
"recordFlags", e.RecordFlags,
255
+
"recordTakedown", e.RecordTakedown,
256
+
"recordReports", len(e.RecordReports),
257
+
)
258
+
}
259
+
260
+
type IdentityRuleFunc = func(evt *IdentityEvent) error
261
+
type RecordRuleFunc = func(evt *RecordEvent) error
262
+
type PostRuleFunc = func(evt *RecordEvent, post *appbsky.FeedPost) error
263
+
type ProfileRuleFunc = func(evt *RecordEvent, profile *appbsky.ActorProfile) error
+63
automod/redis_cache.go
+63
automod/redis_cache.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/go-redis/cache/v9"
8
+
"github.com/redis/go-redis/v9"
9
+
)
10
+
11
+
type RedisCacheStore struct {
12
+
Data *cache.Cache
13
+
TTL time.Duration
14
+
}
15
+
16
+
var _ CacheStore = (*RedisCacheStore)(nil)
17
+
18
+
func NewRedisCacheStore(redisURL string, ttl time.Duration) (*RedisCacheStore, error) {
19
+
opt, err := redis.ParseURL(redisURL)
20
+
if err != nil {
21
+
return nil, err
22
+
}
23
+
rdb := redis.NewClient(opt)
24
+
// check redis connection
25
+
_, err = rdb.Ping(context.TODO()).Result()
26
+
if err != nil {
27
+
return nil, err
28
+
}
29
+
data := cache.New(&cache.Options{
30
+
Redis: rdb,
31
+
LocalCache: cache.NewTinyLFU(10_000, ttl),
32
+
})
33
+
return &RedisCacheStore{
34
+
Data: data,
35
+
TTL: ttl,
36
+
}, nil
37
+
}
38
+
39
+
func redisCacheKey(name, key string) string {
40
+
return "cache/" + name + "/" + key
41
+
}
42
+
43
+
func (s RedisCacheStore) Get(ctx context.Context, name, key string) (string, error) {
44
+
var val string
45
+
err := s.Data.Get(ctx, redisCacheKey(name, key), &val)
46
+
if err == cache.ErrCacheMiss {
47
+
return "", nil
48
+
}
49
+
if err != nil {
50
+
return "", err
51
+
}
52
+
return val, nil
53
+
}
54
+
55
+
func (s RedisCacheStore) Set(ctx context.Context, name, key string, val string) error {
56
+
s.Data.Set(&cache.Item{
57
+
Ctx: ctx,
58
+
Key: redisCacheKey(name, key),
59
+
Value: val,
60
+
TTL: s.TTL,
61
+
})
62
+
return nil
63
+
}
+65
automod/redis_counters.go
+65
automod/redis_counters.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/redis/go-redis/v9"
8
+
)
9
+
10
+
var redisCountPrefix string = "count/"
11
+
12
+
type RedisCountStore struct {
13
+
Client *redis.Client
14
+
}
15
+
16
+
func NewRedisCountStore(redisURL string) (*RedisCountStore, error) {
17
+
opt, err := redis.ParseURL(redisURL)
18
+
if err != nil {
19
+
return nil, err
20
+
}
21
+
rdb := redis.NewClient(opt)
22
+
// check redis connection
23
+
_, err = rdb.Ping(context.TODO()).Result()
24
+
if err != nil {
25
+
return nil, err
26
+
}
27
+
rcs := RedisCountStore{
28
+
Client: rdb,
29
+
}
30
+
return &rcs, nil
31
+
}
32
+
33
+
func (s *RedisCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) {
34
+
key := redisCountPrefix + PeriodBucket(name, val, period)
35
+
c, err := s.Client.Get(ctx, key).Int()
36
+
if err == redis.Nil {
37
+
return 0, nil
38
+
} else if err != nil {
39
+
return 0, err
40
+
}
41
+
return c, nil
42
+
}
43
+
44
+
func (s *RedisCountStore) Increment(ctx context.Context, name, val string) error {
45
+
46
+
var key string
47
+
48
+
// increment multiple counters in a single redis round-trip
49
+
multi := s.Client.Pipeline()
50
+
51
+
key = redisCountPrefix + PeriodBucket(name, val, PeriodHour)
52
+
multi.Incr(ctx, key)
53
+
multi.Expire(ctx, key, 2*time.Hour)
54
+
55
+
key = redisCountPrefix + PeriodBucket(name, val, PeriodDay)
56
+
multi.Incr(ctx, key)
57
+
multi.Expire(ctx, key, 48*time.Hour)
58
+
59
+
key = redisCountPrefix + PeriodBucket(name, val, PeriodTotal)
60
+
multi.Incr(ctx, key)
61
+
// no expiration for total
62
+
63
+
_, err := multi.Exec(ctx)
64
+
return err
65
+
}
+354
automod/redis_directory.go
+354
automod/redis_directory.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"sync"
7
+
"time"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/identity"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
12
+
"github.com/go-redis/cache/v9"
13
+
"github.com/prometheus/client_golang/prometheus"
14
+
"github.com/prometheus/client_golang/prometheus/promauto"
15
+
"github.com/redis/go-redis/v9"
16
+
)
17
+
18
+
var redisDirPrefix string = "dir/"
19
+
20
+
// uses redis as a cache for identity lookups. includes a local cache layer as well, for hot keys
21
+
type RedisDirectory struct {
22
+
Inner identity.Directory
23
+
ErrTTL time.Duration
24
+
HitTTL time.Duration
25
+
26
+
handleCache *cache.Cache
27
+
identityCache *cache.Cache
28
+
didLookupChans sync.Map
29
+
handleLookupChans sync.Map
30
+
}
31
+
32
+
type HandleEntry struct {
33
+
Updated time.Time
34
+
DID syntax.DID
35
+
Err error
36
+
}
37
+
38
+
type IdentityEntry struct {
39
+
Updated time.Time
40
+
Identity *identity.Identity
41
+
Err error
42
+
}
43
+
44
+
var _ identity.Directory = (*RedisDirectory)(nil)
45
+
46
+
func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL time.Duration) (*RedisDirectory, error) {
47
+
opt, err := redis.ParseURL(redisURL)
48
+
if err != nil {
49
+
return nil, err
50
+
}
51
+
rdb := redis.NewClient(opt)
52
+
// check redis connection
53
+
_, err = rdb.Ping(context.TODO()).Result()
54
+
if err != nil {
55
+
return nil, err
56
+
}
57
+
handleCache := cache.New(&cache.Options{
58
+
Redis: rdb,
59
+
LocalCache: cache.NewTinyLFU(10_000, hitTTL),
60
+
})
61
+
identityCache := cache.New(&cache.Options{
62
+
Redis: rdb,
63
+
LocalCache: cache.NewTinyLFU(10_000, hitTTL),
64
+
})
65
+
return &RedisDirectory{
66
+
Inner: inner,
67
+
ErrTTL: errTTL,
68
+
HitTTL: hitTTL,
69
+
handleCache: handleCache,
70
+
identityCache: identityCache,
71
+
}, nil
72
+
}
73
+
74
+
func (d *RedisDirectory) IsHandleStale(e *HandleEntry) bool {
75
+
if e.Err != nil && time.Since(e.Updated) > d.ErrTTL {
76
+
return true
77
+
}
78
+
return false
79
+
}
80
+
81
+
func (d *RedisDirectory) IsIdentityStale(e *IdentityEntry) bool {
82
+
if e.Err != nil && time.Since(e.Updated) > d.ErrTTL {
83
+
return true
84
+
}
85
+
return false
86
+
}
87
+
88
+
func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*HandleEntry, error) {
89
+
ident, err := d.Inner.LookupHandle(ctx, h)
90
+
if err != nil {
91
+
he := HandleEntry{
92
+
Updated: time.Now(),
93
+
DID: "",
94
+
Err: err,
95
+
}
96
+
err = d.handleCache.Set(&cache.Item{
97
+
Ctx: ctx,
98
+
Key: redisDirPrefix + h.String(),
99
+
Value: he,
100
+
TTL: d.ErrTTL,
101
+
})
102
+
if err != nil {
103
+
return nil, err
104
+
}
105
+
return &he, nil
106
+
}
107
+
108
+
ident.ParsedPublicKey = nil
109
+
entry := IdentityEntry{
110
+
Updated: time.Now(),
111
+
Identity: ident,
112
+
Err: nil,
113
+
}
114
+
he := HandleEntry{
115
+
Updated: time.Now(),
116
+
DID: ident.DID,
117
+
Err: nil,
118
+
}
119
+
120
+
err = d.identityCache.Set(&cache.Item{
121
+
Ctx: ctx,
122
+
Key: redisDirPrefix + ident.DID.String(),
123
+
Value: entry,
124
+
TTL: d.HitTTL,
125
+
})
126
+
if err != nil {
127
+
return nil, err
128
+
}
129
+
err = d.handleCache.Set(&cache.Item{
130
+
Ctx: ctx,
131
+
Key: redisDirPrefix + h.String(),
132
+
Value: he,
133
+
TTL: d.HitTTL,
134
+
})
135
+
if err != nil {
136
+
return nil, err
137
+
}
138
+
return &he, nil
139
+
}
140
+
141
+
func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) {
142
+
var entry HandleEntry
143
+
err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), &entry)
144
+
if err != nil && err != cache.ErrCacheMiss {
145
+
return "", err
146
+
}
147
+
if err != cache.ErrCacheMiss && !d.IsHandleStale(&entry) {
148
+
handleCacheHits.Inc()
149
+
return entry.DID, entry.Err
150
+
}
151
+
handleCacheMisses.Inc()
152
+
153
+
// Coalesce multiple requests for the same Handle
154
+
res := make(chan struct{})
155
+
val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res)
156
+
if loaded {
157
+
handleRequestsCoalesced.Inc()
158
+
// Wait for the result from the pending request
159
+
select {
160
+
case <-val.(chan struct{}):
161
+
// The result should now be in the cache
162
+
err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), entry)
163
+
if err != nil && err != cache.ErrCacheMiss {
164
+
return "", err
165
+
}
166
+
if err != cache.ErrCacheMiss && !d.IsHandleStale(&entry) {
167
+
return entry.DID, entry.Err
168
+
}
169
+
return "", fmt.Errorf("identity not found in cache after coalesce returned")
170
+
case <-ctx.Done():
171
+
return "", ctx.Err()
172
+
}
173
+
}
174
+
175
+
var did syntax.DID
176
+
// Update the Handle Entry from PLC and cache the result
177
+
newEntry, err := d.updateHandle(ctx, h)
178
+
if err == nil && newEntry != nil {
179
+
did = newEntry.DID
180
+
}
181
+
// Cleanup the coalesce map and close the results channel
182
+
d.handleLookupChans.Delete(h.String())
183
+
// Callers waiting will now get the result from the cache
184
+
close(res)
185
+
186
+
return did, err
187
+
}
188
+
189
+
func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*IdentityEntry, error) {
190
+
ident, err := d.Inner.LookupDID(ctx, did)
191
+
// wipe parsed public key; it's a waste of space and can't serialize
192
+
if nil == err {
193
+
ident.ParsedPublicKey = nil
194
+
}
195
+
// persist the identity lookup error, instead of processing it immediately
196
+
entry := IdentityEntry{
197
+
Updated: time.Now(),
198
+
Identity: ident,
199
+
Err: err,
200
+
}
201
+
var he *HandleEntry
202
+
// if *not* an error, then also update the handle cache
203
+
if nil == err && !ident.Handle.IsInvalidHandle() {
204
+
he = &HandleEntry{
205
+
Updated: time.Now(),
206
+
DID: did,
207
+
Err: nil,
208
+
}
209
+
}
210
+
211
+
err = d.identityCache.Set(&cache.Item{
212
+
Ctx: ctx,
213
+
Key: redisDirPrefix + did.String(),
214
+
Value: entry,
215
+
TTL: d.HitTTL,
216
+
})
217
+
if err != nil {
218
+
return nil, err
219
+
}
220
+
if he != nil {
221
+
err = d.handleCache.Set(&cache.Item{
222
+
Ctx: ctx,
223
+
Key: redisDirPrefix + ident.Handle.String(),
224
+
Value: *he,
225
+
TTL: d.HitTTL,
226
+
})
227
+
if err != nil {
228
+
return nil, err
229
+
}
230
+
}
231
+
return &entry, nil
232
+
}
233
+
234
+
func (d *RedisDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identity.Identity, error) {
235
+
var entry IdentityEntry
236
+
err := d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry)
237
+
if err != nil && err != cache.ErrCacheMiss {
238
+
return nil, err
239
+
}
240
+
if err != cache.ErrCacheMiss && !d.IsIdentityStale(&entry) {
241
+
identityCacheHits.Inc()
242
+
return entry.Identity, entry.Err
243
+
}
244
+
identityCacheMisses.Inc()
245
+
246
+
// Coalesce multiple requests for the same DID
247
+
res := make(chan struct{})
248
+
val, loaded := d.didLookupChans.LoadOrStore(did.String(), res)
249
+
if loaded {
250
+
identityRequestsCoalesced.Inc()
251
+
// Wait for the result from the pending request
252
+
select {
253
+
case <-val.(chan struct{}):
254
+
// The result should now be in the cache
255
+
err = d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry)
256
+
if err != nil && err != cache.ErrCacheMiss {
257
+
return nil, err
258
+
}
259
+
if err != cache.ErrCacheMiss && !d.IsIdentityStale(&entry) {
260
+
return entry.Identity, entry.Err
261
+
}
262
+
return nil, fmt.Errorf("identity not found in cache after coalesce returned")
263
+
case <-ctx.Done():
264
+
return nil, ctx.Err()
265
+
}
266
+
}
267
+
268
+
var doc *identity.Identity
269
+
// Update the Identity Entry from PLC and cache the result
270
+
newEntry, err := d.updateDID(ctx, did)
271
+
if err == nil && newEntry != nil {
272
+
doc = newEntry.Identity
273
+
}
274
+
// Cleanup the coalesce map and close the results channel
275
+
d.didLookupChans.Delete(did.String())
276
+
// Callers waiting will now get the result from the cache
277
+
close(res)
278
+
279
+
return doc, err
280
+
}
281
+
282
+
func (d *RedisDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*identity.Identity, error) {
283
+
did, err := d.ResolveHandle(ctx, h)
284
+
if err != nil {
285
+
return nil, err
286
+
}
287
+
ident, err := d.LookupDID(ctx, did)
288
+
if err != nil {
289
+
return nil, err
290
+
}
291
+
292
+
declared, err := ident.DeclaredHandle()
293
+
if err != nil {
294
+
return nil, err
295
+
}
296
+
if declared != h {
297
+
return nil, fmt.Errorf("handle does not match that declared in DID document")
298
+
}
299
+
return ident, nil
300
+
}
301
+
302
+
func (d *RedisDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*identity.Identity, error) {
303
+
handle, err := a.AsHandle()
304
+
if nil == err { // if not an error, is a handle
305
+
return d.LookupHandle(ctx, handle)
306
+
}
307
+
did, err := a.AsDID()
308
+
if nil == err { // if not an error, is a DID
309
+
return d.LookupDID(ctx, did)
310
+
}
311
+
return nil, fmt.Errorf("at-identifier neither a Handle nor a DID")
312
+
}
313
+
314
+
func (d *RedisDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error {
315
+
handle, err := a.AsHandle()
316
+
if nil == err { // if not an error, is a handle
317
+
return d.handleCache.Delete(ctx, handle.String())
318
+
}
319
+
did, err := a.AsDID()
320
+
if nil == err { // if not an error, is a DID
321
+
return d.identityCache.Delete(ctx, did.String())
322
+
}
323
+
return fmt.Errorf("at-identifier neither a Handle nor a DID")
324
+
}
325
+
326
+
var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{
327
+
Name: "atproto_redis_directory_handle_cache_hits",
328
+
Help: "Number of cache hits for ATProto handle lookups",
329
+
})
330
+
331
+
var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{
332
+
Name: "atproto_redis_directory_handle_cache_misses",
333
+
Help: "Number of cache misses for ATProto handle lookups",
334
+
})
335
+
336
+
var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{
337
+
Name: "atproto_redis_directory_identity_cache_hits",
338
+
Help: "Number of cache hits for ATProto identity lookups",
339
+
})
340
+
341
+
var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{
342
+
Name: "atproto_redis_directory_identity_cache_misses",
343
+
Help: "Number of cache misses for ATProto identity lookups",
344
+
})
345
+
346
+
var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{
347
+
Name: "atproto_redis_directory_identity_requests_coalesced",
348
+
Help: "Number of identity requests coalesced",
349
+
})
350
+
351
+
var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{
352
+
Name: "atproto_redis_directory_handle_requests_coalesced",
353
+
Help: "Number of handle requests coalesced",
354
+
})
+19
automod/rules/all.go
+19
automod/rules/all.go
···
1
+
package rules
2
+
3
+
import (
4
+
"github.com/bluesky-social/indigo/automod"
5
+
)
6
+
7
+
func DefaultRules() automod.RuleSet {
8
+
rules := automod.RuleSet{
9
+
PostRules: []automod.PostRuleFunc{
10
+
MisleadingURLPostRule,
11
+
MisleadingMentionPostRule,
12
+
ReplyCountPostRule,
13
+
BanHashtagsPostRule,
14
+
AccountDemoPostRule,
15
+
AccountPrivateDemoPostRule,
16
+
},
17
+
}
18
+
return rules
19
+
}
+6
automod/rules/example_sets.json
+6
automod/rules/example_sets.json
+44
automod/rules/fixture_test.go
+44
automod/rules/fixture_test.go
···
1
+
package rules
2
+
3
+
import (
4
+
"log/slog"
5
+
6
+
"github.com/bluesky-social/indigo/atproto/identity"
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
"github.com/bluesky-social/indigo/automod"
9
+
"github.com/bluesky-social/indigo/xrpc"
10
+
)
11
+
12
+
func engineFixture() automod.Engine {
13
+
rules := automod.RuleSet{
14
+
PostRules: []automod.PostRuleFunc{
15
+
BanHashtagsPostRule,
16
+
},
17
+
}
18
+
sets := automod.NewMemSetStore()
19
+
sets.Sets["banned-hashtags"] = make(map[string]bool)
20
+
sets.Sets["banned-hashtags"]["slur"] = true
21
+
dir := identity.NewMockDirectory()
22
+
id1 := identity.Identity{
23
+
DID: syntax.DID("did:plc:abc111"),
24
+
Handle: syntax.Handle("handle.example.com"),
25
+
}
26
+
id2 := identity.Identity{
27
+
DID: syntax.DID("did:plc:abc222"),
28
+
Handle: syntax.Handle("imposter.example.com"),
29
+
}
30
+
dir.Insert(id1)
31
+
dir.Insert(id2)
32
+
adminc := xrpc.Client{
33
+
Host: "http://dummy.local",
34
+
}
35
+
engine := automod.Engine{
36
+
Logger: slog.Default(),
37
+
Directory: &dir,
38
+
Counters: automod.NewMemCountStore(),
39
+
Sets: sets,
40
+
Rules: rules,
41
+
AdminClient: &adminc,
42
+
}
43
+
return engine
44
+
}
+78
automod/rules/helpers.go
+78
automod/rules/helpers.go
···
1
+
package rules
2
+
3
+
import (
4
+
"fmt"
5
+
6
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
7
+
)
8
+
9
+
func dedupeStrings(in []string) []string {
10
+
var out []string
11
+
seen := make(map[string]bool)
12
+
for _, v := range in {
13
+
if !seen[v] {
14
+
out = append(out, v)
15
+
seen[v] = true
16
+
}
17
+
}
18
+
return out
19
+
}
20
+
21
+
func ExtractHashtags(post *appbsky.FeedPost) []string {
22
+
var tags []string
23
+
for _, tag := range post.Tags {
24
+
tags = append(tags, tag)
25
+
}
26
+
for _, facet := range post.Facets {
27
+
for _, feat := range facet.Features {
28
+
if feat.RichtextFacet_Tag != nil {
29
+
tags = append(tags, feat.RichtextFacet_Tag.Tag)
30
+
}
31
+
}
32
+
}
33
+
return dedupeStrings(tags)
34
+
}
35
+
36
+
type PostFacet struct {
37
+
Text string
38
+
URL *string
39
+
DID *string
40
+
Tag *string
41
+
}
42
+
43
+
func ExtractFacets(post *appbsky.FeedPost) ([]PostFacet, error) {
44
+
var out []PostFacet
45
+
46
+
for _, facet := range post.Facets {
47
+
for _, feat := range facet.Features {
48
+
if int(facet.Index.ByteEnd) > len([]byte(post.Text)) || facet.Index.ByteStart > facet.Index.ByteEnd {
49
+
return nil, fmt.Errorf("invalid facet byte range")
50
+
}
51
+
52
+
txt := string([]byte(post.Text)[facet.Index.ByteStart:facet.Index.ByteEnd])
53
+
if txt == "" {
54
+
return nil, fmt.Errorf("empty facet text")
55
+
}
56
+
57
+
if feat.RichtextFacet_Link != nil {
58
+
out = append(out, PostFacet{
59
+
Text: txt,
60
+
URL: &feat.RichtextFacet_Link.Uri,
61
+
})
62
+
}
63
+
if feat.RichtextFacet_Tag != nil {
64
+
out = append(out, PostFacet{
65
+
Text: txt,
66
+
Tag: &feat.RichtextFacet_Tag.Tag,
67
+
})
68
+
}
69
+
if feat.RichtextFacet_Mention != nil {
70
+
out = append(out, PostFacet{
71
+
Text: txt,
72
+
DID: &feat.RichtextFacet_Mention.Did,
73
+
})
74
+
}
75
+
}
76
+
}
77
+
return out, nil
78
+
}
+94
automod/rules/misleading.go
+94
automod/rules/misleading.go
···
1
+
package rules
2
+
3
+
import (
4
+
"context"
5
+
"net/url"
6
+
"strings"
7
+
8
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"github.com/bluesky-social/indigo/automod"
11
+
)
12
+
13
+
func MisleadingURLPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error {
14
+
facets, err := ExtractFacets(post)
15
+
if err != nil {
16
+
evt.Logger.Warn("invalid facets", "err", err)
17
+
evt.AddRecordFlag("invalid") // TODO: or some other "this record is corrupt" indicator?
18
+
return nil
19
+
}
20
+
for _, facet := range facets {
21
+
if facet.URL != nil {
22
+
linkURL, err := url.Parse(*facet.URL)
23
+
if err != nil {
24
+
evt.Logger.Warn("invalid link metadata URL", "url", facet.URL)
25
+
continue
26
+
}
27
+
28
+
// basic text string pre-cleanups
29
+
text := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(facet.Text), "..."))
30
+
// if really not a domain, just skipp
31
+
if !strings.Contains(text, ".") {
32
+
continue
33
+
}
34
+
// try to fix any missing method in the text
35
+
if !strings.Contains(text, "://") {
36
+
text = "https://" + text
37
+
}
38
+
39
+
// try parsing as a full URL (with whitespace trimmed)
40
+
textURL, err := url.Parse(text)
41
+
if err != nil {
42
+
evt.Logger.Warn("invalid link text URL", "url", facet.Text)
43
+
continue
44
+
}
45
+
46
+
// for now just compare domains to handle the most obvious cases
47
+
// this public code will obviously get discovered and bypassed. this doesn't earn you any security cred!
48
+
if linkURL.Host != textURL.Host && linkURL.Host != "www."+linkURL.Host {
49
+
evt.Logger.Warn("misleading mismatched domains", "linkHost", linkURL.Host, "textHost", textURL.Host, "text", facet.Text)
50
+
evt.AddRecordFlag("misleading")
51
+
}
52
+
}
53
+
}
54
+
return nil
55
+
}
56
+
57
+
func MisleadingMentionPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error {
58
+
// TODO: do we really need to route context around? probably
59
+
ctx := context.TODO()
60
+
facets, err := ExtractFacets(post)
61
+
if err != nil {
62
+
evt.Logger.Warn("invalid facets", "err", err)
63
+
evt.AddRecordFlag("invalid") // TODO: or some other "this record is corrupt" indicator?
64
+
return nil
65
+
}
66
+
for _, facet := range facets {
67
+
if facet.DID != nil {
68
+
txt := facet.Text
69
+
if txt[0] == '@' {
70
+
txt = txt[1:]
71
+
}
72
+
handle, err := syntax.ParseHandle(txt)
73
+
if err != nil {
74
+
evt.Logger.Warn("mention was not a valid handle", "text", txt)
75
+
continue
76
+
}
77
+
78
+
mentioned, err := evt.Engine.Directory.LookupHandle(ctx, handle)
79
+
if err != nil {
80
+
evt.Logger.Warn("could not resolve handle", "handle", handle)
81
+
evt.AddRecordFlag("misleading")
82
+
break
83
+
}
84
+
85
+
// TODO: check if mentioned DID was recently updated? might be a caching issue
86
+
if mentioned.DID.String() != *facet.DID {
87
+
evt.Logger.Warn("misleading mention", "text", txt, "did", facet.DID)
88
+
evt.AddRecordFlag("misleading")
89
+
continue
90
+
}
91
+
}
92
+
}
93
+
return nil
94
+
}
+82
automod/rules/misleading_test.go
+82
automod/rules/misleading_test.go
···
1
+
package rules
2
+
3
+
import (
4
+
"testing"
5
+
6
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
7
+
"github.com/bluesky-social/indigo/atproto/identity"
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
"github.com/bluesky-social/indigo/automod"
10
+
11
+
"github.com/stretchr/testify/assert"
12
+
)
13
+
14
+
func TestMisleadingURLPostRule(t *testing.T) {
15
+
assert := assert.New(t)
16
+
17
+
engine := engineFixture()
18
+
am1 := automod.AccountMeta{
19
+
Identity: &identity.Identity{
20
+
DID: syntax.DID("did:plc:abc111"),
21
+
Handle: syntax.Handle("handle.example.com"),
22
+
},
23
+
}
24
+
path := "app.bsky.feed.post/abc123"
25
+
cid1 := "cid123"
26
+
p1 := appbsky.FeedPost{
27
+
Text: "https://safe.com/ is very reputable",
28
+
Facets: []*appbsky.RichtextFacet{
29
+
&appbsky.RichtextFacet{
30
+
Features: []*appbsky.RichtextFacet_Features_Elem{
31
+
&appbsky.RichtextFacet_Features_Elem{
32
+
RichtextFacet_Link: &appbsky.RichtextFacet_Link{
33
+
Uri: "https://evil.com",
34
+
},
35
+
},
36
+
},
37
+
Index: &appbsky.RichtextFacet_ByteSlice{
38
+
ByteStart: 0,
39
+
ByteEnd: 16,
40
+
},
41
+
},
42
+
},
43
+
}
44
+
evt1 := engine.NewRecordEvent(am1, path, cid1, &p1)
45
+
assert.NoError(MisleadingURLPostRule(&evt1, &p1))
46
+
assert.NotEmpty(evt1.RecordFlags)
47
+
}
48
+
49
+
func TestMisleadingMentionPostRule(t *testing.T) {
50
+
assert := assert.New(t)
51
+
52
+
engine := engineFixture()
53
+
am1 := automod.AccountMeta{
54
+
Identity: &identity.Identity{
55
+
DID: syntax.DID("did:plc:abc111"),
56
+
Handle: syntax.Handle("handle.example.com"),
57
+
},
58
+
}
59
+
path := "app.bsky.feed.post/abc123"
60
+
cid1 := "cid123"
61
+
p1 := appbsky.FeedPost{
62
+
Text: "@handle.example.com is a friend",
63
+
Facets: []*appbsky.RichtextFacet{
64
+
&appbsky.RichtextFacet{
65
+
Features: []*appbsky.RichtextFacet_Features_Elem{
66
+
&appbsky.RichtextFacet_Features_Elem{
67
+
RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{
68
+
Did: "did:plc:abc222",
69
+
},
70
+
},
71
+
},
72
+
Index: &appbsky.RichtextFacet_ByteSlice{
73
+
ByteStart: 1,
74
+
ByteEnd: 19,
75
+
},
76
+
},
77
+
},
78
+
}
79
+
evt1 := engine.NewRecordEvent(am1, path, cid1, &p1)
80
+
assert.NoError(MisleadingMentionPostRule(&evt1, &p1))
81
+
assert.NotEmpty(evt1.RecordFlags)
82
+
}
+18
automod/rules/private.go
+18
automod/rules/private.go
···
1
+
package rules
2
+
3
+
import (
4
+
"strings"
5
+
6
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
7
+
"github.com/bluesky-social/indigo/automod"
8
+
)
9
+
10
+
// dummy rule. this leaks PII (account email) in logs and should never be used in real life
11
+
func AccountPrivateDemoPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error {
12
+
if evt.Account.Private != nil {
13
+
if strings.HasSuffix(evt.Account.Private.Email, "@blueskyweb.xyz") {
14
+
evt.Logger.Info("hello dev!", "email", evt.Account.Private.Email)
15
+
}
16
+
}
17
+
return nil
18
+
}
+14
automod/rules/profile.go
+14
automod/rules/profile.go
···
1
+
package rules
2
+
3
+
import (
4
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
5
+
"github.com/bluesky-social/indigo/automod"
6
+
)
7
+
8
+
// this is a dummy rule to demonstrate accessing account metadata (eg, profile) from within post handler
9
+
func AccountDemoPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error {
10
+
if evt.Account.Profile.Description != nil && len(post.Text) > 5 && *evt.Account.Profile.Description == post.Text {
11
+
evt.AddRecordFlag("own-profile-description")
12
+
}
13
+
return nil
14
+
}
+17
automod/rules/replies.go
+17
automod/rules/replies.go
···
1
+
package rules
2
+
3
+
import (
4
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
5
+
"github.com/bluesky-social/indigo/automod"
6
+
)
7
+
8
+
func ReplyCountPostRule(evt *automod.RecordEvent, post *appbsky.FeedPost) error {
9
+
if post.Reply != nil {
10
+
did := evt.Account.Identity.DID.String()
11
+
if evt.GetCount("reply", did, automod.PeriodDay) > 3 {
12
+
evt.AddAccountFlag("frequent-replier")
13
+
}
14
+
evt.Increment("reply", did)
15
+
}
16
+
return nil
17
+
}
+72
automod/ruleset.go
+72
automod/ruleset.go
···
1
+
package automod
2
+
3
+
import (
4
+
"fmt"
5
+
6
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
7
+
)
8
+
9
+
type RuleSet struct {
10
+
PostRules []PostRuleFunc
11
+
ProfileRules []ProfileRuleFunc
12
+
RecordRules []RecordRuleFunc
13
+
IdentityRules []IdentityRuleFunc
14
+
}
15
+
16
+
func (r *RuleSet) CallRecordRules(evt *RecordEvent) error {
17
+
// first the generic rules
18
+
for _, f := range r.RecordRules {
19
+
err := f(evt)
20
+
if err != nil {
21
+
return err
22
+
}
23
+
if evt.Err != nil {
24
+
return evt.Err
25
+
}
26
+
}
27
+
// then any record-type-specific rules
28
+
switch evt.Collection {
29
+
case "app.bsky.feed.post":
30
+
post, ok := evt.Record.(*appbsky.FeedPost)
31
+
if !ok {
32
+
return fmt.Errorf("mismatch between collection (%s) and type", evt.Collection)
33
+
}
34
+
for _, f := range r.PostRules {
35
+
err := f(evt, post)
36
+
if err != nil {
37
+
return err
38
+
}
39
+
if evt.Err != nil {
40
+
return evt.Err
41
+
}
42
+
}
43
+
case "app.bsky.actor.profile":
44
+
profile, ok := evt.Record.(*appbsky.ActorProfile)
45
+
if !ok {
46
+
return fmt.Errorf("mismatch between collection (%s) and type", evt.Collection)
47
+
}
48
+
for _, f := range r.ProfileRules {
49
+
err := f(evt, profile)
50
+
if err != nil {
51
+
return err
52
+
}
53
+
if evt.Err != nil {
54
+
return evt.Err
55
+
}
56
+
}
57
+
}
58
+
return nil
59
+
}
60
+
61
+
func (r *RuleSet) CallIdentityRules(evt *IdentityEvent) error {
62
+
for _, f := range r.IdentityRules {
63
+
err := f(evt)
64
+
if err != nil {
65
+
return err
66
+
}
67
+
if evt.Err != nil {
68
+
return evt.Err
69
+
}
70
+
}
71
+
return nil
72
+
}
+61
automod/setstore.go
+61
automod/setstore.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"io"
7
+
"os"
8
+
)
9
+
10
+
type SetStore interface {
11
+
InSet(ctx context.Context, name, val string) (bool, error)
12
+
}
13
+
14
+
// TODO: this implementation isn't race-safe (yet)!
15
+
type MemSetStore struct {
16
+
Sets map[string]map[string]bool
17
+
}
18
+
19
+
func NewMemSetStore() MemSetStore {
20
+
return MemSetStore{
21
+
Sets: make(map[string]map[string]bool),
22
+
}
23
+
}
24
+
25
+
func (s MemSetStore) InSet(ctx context.Context, name, val string) (bool, error) {
26
+
set, ok := s.Sets[name]
27
+
if !ok {
28
+
// NOTE: currently returns false when entire set isn't found
29
+
return false, nil
30
+
}
31
+
_, ok = set[val]
32
+
return ok, nil
33
+
}
34
+
35
+
func (s *MemSetStore) LoadFromFileJSON(p string) error {
36
+
37
+
f, err := os.Open(p)
38
+
if err != nil {
39
+
return err
40
+
}
41
+
defer func() { _ = f.Close() }()
42
+
43
+
raw, err := io.ReadAll(f)
44
+
if err != nil {
45
+
return err
46
+
}
47
+
48
+
var rules map[string][]string
49
+
if err := json.Unmarshal(raw, &rules); err != nil {
50
+
return err
51
+
}
52
+
53
+
for name, l := range rules {
54
+
m := make(map[string]bool, len(l))
55
+
for _, val := range l {
56
+
m[val] = true
57
+
}
58
+
s.Sets[name] = m
59
+
}
60
+
return nil
61
+
}
+13
automod/util.go
+13
automod/util.go
+37
cmd/hepa/Dockerfile
+37
cmd/hepa/Dockerfile
···
1
+
# Run this dockerfile from the top level of the indigo git repository like:
2
+
#
3
+
# podman build -f ./cmd/hepa/Dockerfile -t hepa .
4
+
5
+
### Compile stage
6
+
FROM golang:1.21-alpine3.18 AS build-env
7
+
RUN apk add --no-cache build-base make git
8
+
9
+
ADD . /dockerbuild
10
+
WORKDIR /dockerbuild
11
+
12
+
# timezone data for alpine builds
13
+
ENV GOEXPERIMENT=loopvar
14
+
RUN GIT_VERSION=$(git describe --tags --long --always) && \
15
+
go build -tags timetzdata -o /hepa ./cmd/hepa
16
+
17
+
### Run stage
18
+
FROM alpine:3.18
19
+
20
+
RUN apk add --no-cache --update dumb-init ca-certificates
21
+
ENTRYPOINT ["dumb-init", "--"]
22
+
23
+
WORKDIR /
24
+
RUN mkdir -p data/hepa
25
+
COPY --from=build-env /hepa /
26
+
27
+
# small things to make golang binaries work well under alpine
28
+
ENV GODEBUG=netdns=go
29
+
ENV TZ=Etc/UTC
30
+
31
+
EXPOSE 2210
32
+
33
+
CMD ["/hepa"]
34
+
35
+
LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo
36
+
LABEL org.opencontainers.image.description="ATP Auto-Moderation Service (hepa)"
37
+
LABEL org.opencontainers.image.licenses=MIT
+22
cmd/hepa/README.md
+22
cmd/hepa/README.md
···
1
+
2
+
hepa
3
+
====
4
+
5
+
This is a simple auto-moderation daemon which wraps the automod package.
6
+
7
+
The name is a reference to HEPA air filters, which help keep the local atmosphere clean and healthy for humans.
8
+
9
+
Available commands, flags, and config are documented in the usage (`--help`).
10
+
11
+
Current features and design decisions:
12
+
13
+
- all state (counters) and caches stored in Redis
14
+
- consumes from Relay firehose; no backfill functionality yet
15
+
- which rules are included configured at compile time
16
+
- admin access to fetch private account metadata, and to persist moderation actions, is optional. it is possible for anybody to run a `hepa` instance
17
+
18
+
This is not a "labeling service" per say, in that it pushes labels in to an existing moderation service, and doesn't provide API endpoints or label streams. see `labelmaker` for a self-contained labeling service.
19
+
20
+
Performance is generally slow when first starting up, because account-level metadata is being fetched (and cached) for every firehose event. After the caches have "warmed up", events are processed faster.
21
+
22
+
See the `automod` package's README for more documentation.
+130
cmd/hepa/consumer.go
+130
cmd/hepa/consumer.go
···
1
+
package main
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"net/http"
8
+
"net/url"
9
+
10
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
"github.com/bluesky-social/indigo/events/schedulers/autoscaling"
13
+
lexutil "github.com/bluesky-social/indigo/lex/util"
14
+
15
+
"github.com/bluesky-social/indigo/events"
16
+
"github.com/bluesky-social/indigo/repo"
17
+
"github.com/bluesky-social/indigo/repomgr"
18
+
"github.com/carlmjohnson/versioninfo"
19
+
"github.com/gorilla/websocket"
20
+
)
21
+
22
+
func (s *Server) RunConsumer(ctx context.Context) error {
23
+
24
+
// TODO: persist cursor in a database or local disk
25
+
cur, err := s.ReadLastCursor(ctx)
26
+
if err != nil {
27
+
return err
28
+
}
29
+
30
+
dialer := websocket.DefaultDialer
31
+
u, err := url.Parse(s.bgshost)
32
+
if err != nil {
33
+
return fmt.Errorf("invalid bgshost URI: %w", err)
34
+
}
35
+
u.Path = "xrpc/com.atproto.sync.subscribeRepos"
36
+
if cur != 0 {
37
+
u.RawQuery = fmt.Sprintf("cursor=%d", cur)
38
+
}
39
+
s.logger.Info("subscribing to repo event stream", "upstream", s.bgshost, "cursor", cur)
40
+
con, _, err := dialer.Dial(u.String(), http.Header{
41
+
"User-Agent": []string{fmt.Sprintf("hepa/%s", versioninfo.Short())},
42
+
})
43
+
if err != nil {
44
+
return fmt.Errorf("subscribing to firehose failed (dialing): %w", err)
45
+
}
46
+
47
+
rsc := &events.RepoStreamCallbacks{
48
+
RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error {
49
+
s.lastSeq = evt.Seq
50
+
return s.HandleRepoCommit(ctx, evt)
51
+
},
52
+
RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error {
53
+
s.lastSeq = evt.Seq
54
+
did, err := syntax.ParseDID(evt.Did)
55
+
if err != nil {
56
+
s.logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err)
57
+
return nil
58
+
}
59
+
if err := s.engine.ProcessIdentityEvent(ctx, "handle", did); err != nil {
60
+
s.logger.Error("processing handle update failed", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err)
61
+
}
62
+
return nil
63
+
},
64
+
// TODO: other event callbacks as needed
65
+
}
66
+
67
+
// start at higher parallelism (somewhat arbitrary)
68
+
scaleSettings := autoscaling.DefaultAutoscaleSettings()
69
+
scaleSettings.Concurrency = 6
70
+
return events.HandleRepoStream(
71
+
ctx, con, autoscaling.NewScheduler(
72
+
scaleSettings,
73
+
s.bgshost,
74
+
rsc.EventHandler,
75
+
),
76
+
)
77
+
}
78
+
79
+
// NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better.
80
+
func (s *Server) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error {
81
+
82
+
logger := s.logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq)
83
+
logger.Debug("received commit event")
84
+
85
+
if evt.TooBig {
86
+
logger.Warn("skipping tooBig events for now")
87
+
return nil
88
+
}
89
+
90
+
did, err := syntax.ParseDID(evt.Repo)
91
+
if err != nil {
92
+
logger.Error("bad DID syntax in event", "err", err)
93
+
return nil
94
+
}
95
+
96
+
rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks))
97
+
if err != nil {
98
+
logger.Error("failed to read repo from car", "err", err)
99
+
return nil
100
+
}
101
+
102
+
for _, op := range evt.Ops {
103
+
logger = logger.With("eventKind", op.Action, "path", op.Path)
104
+
105
+
ek := repomgr.EventKind(op.Action)
106
+
switch ek {
107
+
case repomgr.EvtKindCreateRecord:
108
+
// read the record from blocks, and verify CID
109
+
rc, rec, err := rr.GetRecord(ctx, op.Path)
110
+
if err != nil {
111
+
logger.Error("reading record from event blocks (CAR)", "err", err)
112
+
break
113
+
}
114
+
if op.Cid == nil || lexutil.LexLink(rc) != *op.Cid {
115
+
logger.Error("mismatch between commit op CID and record block", "recordCID", rc, "opCID", op.Cid)
116
+
break
117
+
}
118
+
119
+
err = s.engine.ProcessRecord(ctx, did, op.Path, op.Cid.String(), rec)
120
+
if err != nil {
121
+
logger.Error("engine failed to process record", "err", err)
122
+
continue
123
+
}
124
+
default:
125
+
// TODO: other event types: update, delete
126
+
}
127
+
}
128
+
129
+
return nil
130
+
}
+234
cmd/hepa/main.go
+234
cmd/hepa/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"net/http"
8
+
"os"
9
+
"time"
10
+
11
+
"github.com/bluesky-social/indigo/atproto/identity"
12
+
"github.com/bluesky-social/indigo/automod"
13
+
14
+
"github.com/carlmjohnson/versioninfo"
15
+
_ "github.com/joho/godotenv/autoload"
16
+
cli "github.com/urfave/cli/v2"
17
+
"golang.org/x/time/rate"
18
+
)
19
+
20
+
func main() {
21
+
if err := run(os.Args); err != nil {
22
+
slog.Error("exiting", "err", err)
23
+
os.Exit(-1)
24
+
}
25
+
}
26
+
27
+
func run(args []string) error {
28
+
29
+
app := cli.App{
30
+
Name: "hepa",
31
+
Usage: "automod daemon (cleans the atmosphere)",
32
+
Version: versioninfo.Short(),
33
+
}
34
+
35
+
app.Flags = []cli.Flag{
36
+
&cli.StringFlag{
37
+
Name: "atp-bgs-host",
38
+
Usage: "hostname and port of BGS to subscribe to",
39
+
Value: "wss://bsky.network",
40
+
EnvVars: []string{"ATP_BGS_HOST"},
41
+
},
42
+
&cli.StringFlag{
43
+
Name: "atp-plc-host",
44
+
Usage: "method, hostname, and port of PLC registry",
45
+
Value: "https://plc.directory",
46
+
EnvVars: []string{"ATP_PLC_HOST"},
47
+
},
48
+
&cli.StringFlag{
49
+
Name: "atp-mod-host",
50
+
Usage: "method, hostname, and port of moderation service",
51
+
Value: "https://api.bsky.app",
52
+
EnvVars: []string{"ATP_MOD_HOST"},
53
+
},
54
+
&cli.StringFlag{
55
+
Name: "atp-bsky-host",
56
+
Usage: "method, hostname, and port of bsky API (appview) service",
57
+
Value: "https://api.bsky.app",
58
+
EnvVars: []string{"ATP_BSKY_HOST"},
59
+
},
60
+
&cli.StringFlag{
61
+
Name: "redis-url",
62
+
Usage: "redis connection URL",
63
+
// redis://<user>:<pass>@localhost:6379/<db>
64
+
// redis://localhost:6379/0
65
+
EnvVars: []string{"HEPA_REDIS_URL"},
66
+
},
67
+
&cli.StringFlag{
68
+
Name: "mod-handle",
69
+
Usage: "for mod service login",
70
+
EnvVars: []string{"HEPA_MOD_AUTH_HANDLE"},
71
+
},
72
+
&cli.StringFlag{
73
+
Name: "mod-password",
74
+
Usage: "for mod service login",
75
+
EnvVars: []string{"HEPA_MOD_AUTH_PASSWORD"},
76
+
},
77
+
&cli.StringFlag{
78
+
Name: "mod-admin-token",
79
+
Usage: "admin authentication password for mod service",
80
+
EnvVars: []string{"HEPA_MOD_AUTH_ADMIN_TOKEN"},
81
+
},
82
+
&cli.IntFlag{
83
+
Name: "plc-rate-limit",
84
+
Usage: "max number of requests per second to PLC registry",
85
+
Value: 100,
86
+
EnvVars: []string{"HEPA_PLC_RATE_LIMIT"},
87
+
},
88
+
&cli.StringFlag{
89
+
Name: "sets-json-path",
90
+
Usage: "file path of JSON file containing static sets",
91
+
EnvVars: []string{"HEPA_SETS_JSON_PATH"},
92
+
},
93
+
}
94
+
95
+
app.Commands = []*cli.Command{
96
+
runCmd,
97
+
processRecordCmd,
98
+
}
99
+
100
+
return app.Run(args)
101
+
}
102
+
103
+
func configDirectory(cctx *cli.Context) (identity.Directory, error) {
104
+
baseDir := identity.BaseDirectory{
105
+
PLCURL: cctx.String("atp-plc-host"),
106
+
HTTPClient: http.Client{
107
+
Timeout: time.Second * 15,
108
+
},
109
+
PLCLimiter: rate.NewLimiter(rate.Limit(cctx.Int("plc-rate-limit")), 1),
110
+
TryAuthoritativeDNS: true,
111
+
SkipDNSDomainSuffixes: []string{".bsky.social", ".staging.bsky.dev"},
112
+
}
113
+
var dir identity.Directory
114
+
if cctx.String("redis-url") != "" {
115
+
rdir, err := automod.NewRedisDirectory(&baseDir, cctx.String("redis-url"), time.Hour*24, time.Minute*2)
116
+
if err != nil {
117
+
return nil, err
118
+
}
119
+
dir = rdir
120
+
} else {
121
+
cdir := identity.NewCacheDirectory(&baseDir, 1_500_000, time.Hour*24, time.Minute*2)
122
+
dir = &cdir
123
+
}
124
+
return dir, nil
125
+
}
126
+
127
+
var runCmd = &cli.Command{
128
+
Name: "run",
129
+
Usage: "run the hepa daemon",
130
+
Flags: []cli.Flag{
131
+
&cli.StringFlag{
132
+
Name: "metrics-listen",
133
+
Usage: "IP or address, and port, to listen on for metrics APIs",
134
+
Value: ":3989",
135
+
EnvVars: []string{"HEPA_METRICS_LISTEN"},
136
+
},
137
+
},
138
+
Action: func(cctx *cli.Context) error {
139
+
ctx := context.Background()
140
+
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
141
+
Level: slog.LevelInfo,
142
+
}))
143
+
slog.SetDefault(logger)
144
+
145
+
configOTEL("hepa")
146
+
147
+
dir, err := configDirectory(cctx)
148
+
if err != nil {
149
+
return err
150
+
}
151
+
152
+
srv, err := NewServer(
153
+
dir,
154
+
Config{
155
+
BGSHost: cctx.String("atp-bgs-host"),
156
+
BskyHost: cctx.String("atp-bsky-host"),
157
+
Logger: logger,
158
+
ModHost: cctx.String("atp-mod-host"),
159
+
ModAdminToken: cctx.String("mod-admin-token"),
160
+
ModUsername: cctx.String("mod-handle"),
161
+
ModPassword: cctx.String("mod-password"),
162
+
SetsFileJSON: cctx.String("sets-json-path"),
163
+
RedisURL: cctx.String("redis-url"),
164
+
},
165
+
)
166
+
if err != nil {
167
+
return err
168
+
}
169
+
170
+
// prometheus HTTP endpoint: /metrics
171
+
go func() {
172
+
if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil {
173
+
slog.Error("failed to start metrics endpoint", "error", err)
174
+
panic(fmt.Errorf("failed to start metrics endpoint: %w", err))
175
+
}
176
+
}()
177
+
178
+
go func() {
179
+
if err := srv.RunPersistCursor(ctx); err != nil {
180
+
slog.Error("cursor routine failed", "err", err)
181
+
}
182
+
}()
183
+
184
+
// the main service loop
185
+
if err := srv.RunConsumer(ctx); err != nil {
186
+
return fmt.Errorf("failure consuming and processing firehose: %w", err)
187
+
}
188
+
return nil
189
+
},
190
+
}
191
+
192
+
var processRecordCmd = &cli.Command{
193
+
Name: "process-record",
194
+
Usage: "process a single record in isolation",
195
+
ArgsUsage: `<at-uri>`,
196
+
Flags: []cli.Flag{},
197
+
Action: func(cctx *cli.Context) error {
198
+
uri := cctx.Args().First()
199
+
if uri == "" {
200
+
return fmt.Errorf("expected a single AT-URI argument")
201
+
}
202
+
203
+
ctx := context.Background()
204
+
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
205
+
Level: slog.LevelInfo,
206
+
}))
207
+
slog.SetDefault(logger)
208
+
209
+
dir, err := configDirectory(cctx)
210
+
if err != nil {
211
+
return err
212
+
}
213
+
214
+
srv, err := NewServer(
215
+
dir,
216
+
Config{
217
+
BGSHost: cctx.String("atp-bgs-host"),
218
+
BskyHost: cctx.String("atp-bsky-host"),
219
+
Logger: logger,
220
+
ModHost: cctx.String("atp-mod-host"),
221
+
ModAdminToken: cctx.String("mod-admin-token"),
222
+
ModUsername: cctx.String("mod-handle"),
223
+
ModPassword: cctx.String("mod-password"),
224
+
SetsFileJSON: cctx.String("sets-json-path"),
225
+
RedisURL: cctx.String("redis-url"),
226
+
},
227
+
)
228
+
if err != nil {
229
+
return err
230
+
}
231
+
232
+
return srv.engine.FetchAndProcessRecord(ctx, uri)
233
+
},
234
+
}
+56
cmd/hepa/otel.go
+56
cmd/hepa/otel.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"log"
6
+
"log/slog"
7
+
"os"
8
+
"time"
9
+
10
+
"go.opentelemetry.io/otel"
11
+
"go.opentelemetry.io/otel/attribute"
12
+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
13
+
"go.opentelemetry.io/otel/sdk/resource"
14
+
tracesdk "go.opentelemetry.io/otel/sdk/trace"
15
+
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
16
+
)
17
+
18
+
var tracer = otel.Tracer("hepa")
19
+
20
+
// Enable OTLP HTTP exporter
21
+
// For relevant environment variables:
22
+
// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables
23
+
// At a minimum, you need to set
24
+
// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
25
+
// TODO: this should be in cliutil or something
26
+
func configOTEL(serviceName string) {
27
+
if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" {
28
+
slog.Info("setting up trace exporter", "endpoint", ep)
29
+
ctx, cancel := context.WithCancel(context.Background())
30
+
defer cancel()
31
+
32
+
exp, err := otlptracehttp.New(ctx)
33
+
if err != nil {
34
+
log.Fatal("failed to create trace exporter", "error", err)
35
+
}
36
+
defer func() {
37
+
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
38
+
defer cancel()
39
+
if err := exp.Shutdown(ctx); err != nil {
40
+
slog.Error("failed to shutdown trace exporter", "error", err)
41
+
}
42
+
}()
43
+
44
+
tp := tracesdk.NewTracerProvider(
45
+
tracesdk.WithBatcher(exp),
46
+
tracesdk.WithResource(resource.NewWithAttributes(
47
+
semconv.SchemaURL,
48
+
semconv.ServiceNameKey.String(serviceName),
49
+
attribute.String("env", os.Getenv("ENVIRONMENT")), // DataDog
50
+
attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others
51
+
attribute.Int64("ID", 1),
52
+
)),
53
+
)
54
+
otel.SetTracerProvider(tp)
55
+
}
56
+
}
+207
cmd/hepa/server.go
+207
cmd/hepa/server.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"net/http"
8
+
"os"
9
+
"strings"
10
+
"time"
11
+
12
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
+
"github.com/bluesky-social/indigo/atproto/identity"
14
+
"github.com/bluesky-social/indigo/automod"
15
+
"github.com/bluesky-social/indigo/automod/rules"
16
+
"github.com/bluesky-social/indigo/util"
17
+
"github.com/bluesky-social/indigo/xrpc"
18
+
19
+
"github.com/prometheus/client_golang/prometheus/promhttp"
20
+
"github.com/redis/go-redis/v9"
21
+
)
22
+
23
+
type Server struct {
24
+
bgshost string
25
+
logger *slog.Logger
26
+
engine *automod.Engine
27
+
rdb *redis.Client
28
+
lastSeq int64
29
+
}
30
+
31
+
type Config struct {
32
+
BGSHost string
33
+
BskyHost string
34
+
ModHost string
35
+
ModAdminToken string
36
+
ModUsername string
37
+
ModPassword string
38
+
SetsFileJSON string
39
+
RedisURL string
40
+
Logger *slog.Logger
41
+
}
42
+
43
+
func NewServer(dir identity.Directory, config Config) (*Server, error) {
44
+
logger := config.Logger
45
+
if logger == nil {
46
+
logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
47
+
Level: slog.LevelInfo,
48
+
}))
49
+
}
50
+
51
+
bgsws := config.BGSHost
52
+
if !strings.HasPrefix(bgsws, "ws") {
53
+
return nil, fmt.Errorf("specified bgs host must include 'ws://' or 'wss://'")
54
+
}
55
+
56
+
// TODO: this isn't a very robust way to handle a peristent client
57
+
var xrpcc *xrpc.Client
58
+
if config.ModAdminToken != "" {
59
+
xrpcc = &xrpc.Client{
60
+
Client: util.RobustHTTPClient(),
61
+
Host: config.ModHost,
62
+
AdminToken: &config.ModAdminToken,
63
+
Auth: &xrpc.AuthInfo{},
64
+
}
65
+
66
+
auth, err := comatproto.ServerCreateSession(context.TODO(), xrpcc, &comatproto.ServerCreateSession_Input{
67
+
Identifier: config.ModUsername,
68
+
Password: config.ModPassword,
69
+
})
70
+
if err != nil {
71
+
return nil, err
72
+
}
73
+
xrpcc.Auth.AccessJwt = auth.AccessJwt
74
+
xrpcc.Auth.RefreshJwt = auth.RefreshJwt
75
+
xrpcc.Auth.Did = auth.Did
76
+
xrpcc.Auth.Handle = auth.Handle
77
+
}
78
+
79
+
sets := automod.NewMemSetStore()
80
+
if config.SetsFileJSON != "" {
81
+
if err := sets.LoadFromFileJSON(config.SetsFileJSON); err != nil {
82
+
return nil, err
83
+
} else {
84
+
logger.Info("loaded set config from JSON", "path", config.SetsFileJSON)
85
+
}
86
+
}
87
+
88
+
var counters automod.CountStore
89
+
var cache automod.CacheStore
90
+
var rdb *redis.Client
91
+
if config.RedisURL != "" {
92
+
// generic client, for cursor state
93
+
opt, err := redis.ParseURL(config.RedisURL)
94
+
if err != nil {
95
+
return nil, err
96
+
}
97
+
rdb = redis.NewClient(opt)
98
+
// check redis connection
99
+
_, err = rdb.Ping(context.TODO()).Result()
100
+
if err != nil {
101
+
return nil, err
102
+
}
103
+
104
+
cnt, err := automod.NewRedisCountStore(config.RedisURL)
105
+
if err != nil {
106
+
return nil, err
107
+
}
108
+
counters = cnt
109
+
110
+
csh, err := automod.NewRedisCacheStore(config.RedisURL, 30*time.Minute)
111
+
if err != nil {
112
+
return nil, err
113
+
}
114
+
cache = csh
115
+
} else {
116
+
counters = automod.NewMemCountStore()
117
+
cache = automod.NewMemCacheStore(5_000, 30*time.Minute)
118
+
}
119
+
120
+
engine := automod.Engine{
121
+
Logger: logger,
122
+
Directory: dir,
123
+
Counters: counters,
124
+
Sets: sets,
125
+
Cache: cache,
126
+
Rules: rules.DefaultRules(),
127
+
AdminClient: xrpcc,
128
+
BskyClient: &xrpc.Client{
129
+
Client: util.RobustHTTPClient(),
130
+
Host: config.BskyHost,
131
+
},
132
+
}
133
+
134
+
s := &Server{
135
+
bgshost: config.BGSHost,
136
+
logger: logger,
137
+
engine: &engine,
138
+
rdb: rdb,
139
+
}
140
+
141
+
return s, nil
142
+
}
143
+
144
+
func (s *Server) RunMetrics(listen string) error {
145
+
http.Handle("/metrics", promhttp.Handler())
146
+
return http.ListenAndServe(listen, nil)
147
+
}
148
+
149
+
var cursorKey = "hepa/seq"
150
+
151
+
func (s *Server) ReadLastCursor(ctx context.Context) (int64, error) {
152
+
// if redis isn't configured, just skip
153
+
if s.rdb == nil {
154
+
s.logger.Info("redis not configured, skipping cursor read")
155
+
return 0, nil
156
+
}
157
+
158
+
val, err := s.rdb.Get(ctx, cursorKey).Int64()
159
+
if err == redis.Nil {
160
+
s.logger.Info("no pre-existing cursor in redis")
161
+
return 0, nil
162
+
}
163
+
s.logger.Info("successfully found prior subscription cursor seq in redis", "seq", val)
164
+
return val, err
165
+
}
166
+
167
+
func (s *Server) PersistCursor(ctx context.Context) error {
168
+
// if redis isn't configured, just skip
169
+
if s.rdb == nil {
170
+
return nil
171
+
}
172
+
if s.lastSeq <= 0 {
173
+
return nil
174
+
}
175
+
err := s.rdb.Set(ctx, cursorKey, s.lastSeq, 14*24*time.Hour).Err()
176
+
return err
177
+
}
178
+
179
+
// this method runs in a loop, persisting the current cursor state every 5 seconds
180
+
func (s *Server) RunPersistCursor(ctx context.Context) error {
181
+
182
+
// if redis isn't configured, just skip
183
+
if s.rdb == nil {
184
+
return nil
185
+
}
186
+
ticker := time.NewTicker(5 * time.Second)
187
+
for {
188
+
select {
189
+
case <-ctx.Done():
190
+
if s.lastSeq >= 1 {
191
+
s.logger.Info("persisting final cursor seq value", "seq", s.lastSeq)
192
+
err := s.PersistCursor(ctx)
193
+
if err != nil {
194
+
s.logger.Error("failed to persist cursor", "err", err, "seq", s.lastSeq)
195
+
}
196
+
}
197
+
return nil
198
+
case <-ticker.C:
199
+
if s.lastSeq >= 1 {
200
+
err := s.PersistCursor(ctx)
201
+
if err != nil {
202
+
s.logger.Error("failed to persist cursor", "err", err, "seq", s.lastSeq)
203
+
}
204
+
}
205
+
}
206
+
}
207
+
}
+10
go.mod
+10
go.mod
···
9
9
github.com/carlmjohnson/versioninfo v0.22.5
10
10
github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49
11
11
github.com/flosch/pongo2/v6 v6.0.0
12
+
github.com/go-redis/cache/v9 v9.0.0
12
13
github.com/goccy/go-json v0.10.2
13
14
github.com/gocql/gocql v1.6.0
14
15
github.com/golang-jwt/jwt v3.2.2+incompatible
···
47
48
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f
48
49
github.com/prometheus/client_golang v1.14.0
49
50
github.com/prometheus/client_model v0.3.0
51
+
github.com/redis/go-redis/v9 v9.3.0
50
52
github.com/rivo/uniseg v0.1.0
51
53
github.com/samber/slog-echo v1.2.1
52
54
github.com/scylladb/gocqlx/v2 v2.8.1-0.20230309105046-dec046bd85e6
···
75
77
gorm.io/driver/sqlite v1.5.0
76
78
gorm.io/gorm v1.25.1
77
79
gorm.io/plugin/opentelemetry v0.1.3
80
+
)
81
+
82
+
require (
83
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
84
+
github.com/klauspost/compress v1.13.6 // indirect
85
+
github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
86
+
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
87
+
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
78
88
)
79
89
80
90
require (
+90
go.sum
+90
go.sum
···
71
71
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
72
72
github.com/brianvoe/gofakeit/v6 v6.20.2 h1:FLloufuC7NcbHqDzVQ42CG9AKryS1gAGCRt8nQRsW+Y=
73
73
github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
74
+
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
75
+
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
76
+
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
77
+
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
74
78
github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
75
79
github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
76
80
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
···
99
103
github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
100
104
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
101
105
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
106
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
107
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
102
108
github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 h1:6SNWi8VxQeCSwmLuTbEvJd7xvPmdS//zvMBWweZLgck=
103
109
github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49/go.mod h1:V+Qd57rJe8gd4eiGzZyg4h54VLHmYVVw54iMnlAMrF8=
104
110
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
···
112
118
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
113
119
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
114
120
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
121
+
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
122
+
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
123
+
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
115
124
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
116
125
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
117
126
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
···
127
136
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
128
137
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
129
138
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
139
+
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
130
140
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
131
141
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
132
142
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
133
143
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
144
+
github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0=
145
+
github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI=
134
146
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
147
+
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
135
148
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
136
149
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
137
150
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
···
206
219
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
207
220
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
208
221
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
222
+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
209
223
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
210
224
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
211
225
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
···
239
253
github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno=
240
254
github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM=
241
255
github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
256
+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
242
257
github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
243
258
github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
244
259
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
260
+
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
245
261
github.com/icrowley/fake v0.0.0-20221112152111-d7b7e2276db2 h1:qU3v73XG4QAqCPHA4HOpfC1EfUvtLIDvQK4mNQ0LvgI=
246
262
github.com/icrowley/fake v0.0.0-20221112152111-d7b7e2276db2/go.mod h1:dQ6TM/OGAe+cMws81eTe4Btv1dKxfPZ2CX+YaAFAPN4=
247
263
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
···
367
383
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
368
384
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
369
385
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
386
+
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
387
+
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
370
388
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
371
389
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
372
390
github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
···
493
511
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
494
512
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
495
513
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
514
+
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
515
+
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
516
+
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
517
+
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
518
+
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
519
+
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
520
+
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
521
+
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
522
+
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
523
+
github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
524
+
github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
525
+
github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
526
+
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
527
+
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
528
+
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
529
+
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
530
+
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
531
+
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
532
+
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
533
+
github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
534
+
github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
535
+
github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
536
+
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
537
+
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
538
+
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
539
+
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
496
540
github.com/opensearch-project/opensearch-go/v2 v2.2.0 h1:6RicCBiqboSVtLMjSiKgVQIsND4I3sxELg9uwWe/TKM=
497
541
github.com/opensearch-project/opensearch-go/v2 v2.2.0/go.mod h1:R8NTTQMmfSRsmZdfEn2o9ZSuSXn0WTHPYhzgl7LCFLY=
498
542
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
···
544
588
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
545
589
github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0=
546
590
github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=
591
+
github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA=
592
+
github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
593
+
github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
547
594
github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
548
595
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
549
596
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
···
584
631
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
585
632
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
586
633
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
634
+
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
587
635
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
588
636
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
589
637
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
···
600
648
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
601
649
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
602
650
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
651
+
github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI=
652
+
github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q=
653
+
github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc=
654
+
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
655
+
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
656
+
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
603
657
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
604
658
github.com/warpfork/go-testmark v0.11.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
605
659
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
···
620
674
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
621
675
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
622
676
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
677
+
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
623
678
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
624
679
gitlab.com/yawning/secp256k1-voi v0.0.0-20230815035612-a7264edccf80 h1:+Hti+G65Kc88hK0GFQ6NzzncsOmoqxmlXaxM1+FPPqM=
625
680
gitlab.com/yawning/secp256k1-voi v0.0.0-20230815035612-a7264edccf80/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
···
686
741
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
687
742
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
688
743
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
744
+
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
689
745
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
690
746
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
691
747
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
···
723
779
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
724
780
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
725
781
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
782
+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
726
783
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
784
+
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
785
+
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
727
786
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
728
787
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
729
788
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
730
789
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
731
790
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
791
+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
732
792
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
733
793
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
734
794
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
···
751
811
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
752
812
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
753
813
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
814
+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
754
815
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
755
816
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
756
817
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
···
759
820
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
760
821
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
761
822
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
823
+
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
762
824
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
825
+
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
763
826
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
764
827
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
828
+
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
765
829
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
766
830
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
831
+
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
832
+
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
833
+
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
767
834
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
768
835
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
769
836
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
···
793
860
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
794
861
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
795
862
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
863
+
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
796
864
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
797
865
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
798
866
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
···
804
872
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
805
873
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
806
874
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
875
+
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
807
876
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
877
+
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
808
878
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
879
+
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
809
880
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
810
881
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
811
882
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
···
826
897
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
827
898
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
828
899
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
900
+
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
829
901
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
830
902
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
831
903
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
···
835
907
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
836
908
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
837
909
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
910
+
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
838
911
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
839
912
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
840
913
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
914
+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
841
915
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
916
+
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
842
917
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
843
918
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
844
919
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
845
920
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
846
921
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
847
922
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
923
+
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
924
+
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
925
+
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
848
926
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
849
927
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
850
928
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
···
855
933
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
856
934
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
857
935
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
936
+
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
937
+
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
938
+
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
858
939
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
859
940
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
860
941
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
···
866
947
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
867
948
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
868
949
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
950
+
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
951
+
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
869
952
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
870
953
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
871
954
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
···
921
1004
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
922
1005
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
923
1006
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
1007
+
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
924
1008
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
925
1009
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
1010
+
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
926
1011
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
1012
+
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
1013
+
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
927
1014
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
928
1015
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
929
1016
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
···
1028
1115
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
1029
1116
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
1030
1117
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
1118
+
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
1031
1119
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
1032
1120
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
1121
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
1122
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
1033
1123
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
1034
1124
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
1035
1125
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=