this repo has no description
automod: add generic caching, and hydrate some account meta
+310
-41
+98
automod/account_meta.go
+98
automod/account_meta.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"time"
8
+
9
+
appbsky "github.com/bluesky-social/indigo/api/bsky"
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
)
12
+
13
+
type ProfileSummary struct {
14
+
HasAvatar bool
15
+
Description *string
16
+
DisplayName *string
17
+
}
18
+
19
+
type AccountPrivate struct {
20
+
Email string
21
+
EmailConfirmed bool
22
+
}
23
+
24
+
// information about a repo/account/identity, always pre-populated and relevant to many rules
25
+
type AccountMeta struct {
26
+
Identity *identity.Identity
27
+
Profile ProfileSummary
28
+
Private *AccountPrivate
29
+
AccountLabels []string
30
+
FollowersCount int64
31
+
FollowsCount int64
32
+
PostsCount int64
33
+
IndexedAt *time.Time
34
+
}
35
+
36
+
func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) {
37
+
38
+
// wipe parsed public key; it's a waste of space and can't serialize
39
+
ident.ParsedPublicKey = nil
40
+
41
+
existing, err := e.Cache.Get(ctx, "acct", ident.DID.String())
42
+
if err != nil {
43
+
return nil, err
44
+
}
45
+
if existing != "" {
46
+
var am AccountMeta
47
+
err := json.Unmarshal([]byte(existing), &am)
48
+
if err != nil {
49
+
return nil, fmt.Errorf("parsing AccountMeta from cache: %v", err)
50
+
}
51
+
am.Identity = ident
52
+
return &am, nil
53
+
}
54
+
55
+
// fetch account metadata
56
+
pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
57
+
if err != nil {
58
+
return nil, err
59
+
}
60
+
61
+
var labels []string
62
+
for _, lbl := range pv.Labels {
63
+
labels = append(labels, lbl.Val)
64
+
}
65
+
66
+
am := AccountMeta{
67
+
Identity: ident,
68
+
Profile: ProfileSummary{
69
+
HasAvatar: pv.Avatar != nil,
70
+
Description: pv.Description,
71
+
DisplayName: pv.DisplayName,
72
+
},
73
+
AccountLabels: dedupeStrings(labels),
74
+
}
75
+
if pv.PostsCount != nil {
76
+
am.PostsCount = *pv.PostsCount
77
+
}
78
+
if pv.FollowersCount != nil {
79
+
am.FollowersCount = *pv.FollowersCount
80
+
}
81
+
if pv.FollowsCount != nil {
82
+
am.FollowsCount = *pv.FollowsCount
83
+
}
84
+
85
+
if e.AdminClient != nil {
86
+
// XXX: get admin-level info (email, indexed at, etc). requires lexgen update
87
+
}
88
+
89
+
val, err := json.Marshal(&am)
90
+
if err != nil {
91
+
return nil, err
92
+
}
93
+
94
+
if err := e.Cache.Set(ctx, "acct", ident.DID.String(), string(val)); err != nil {
95
+
return nil, err
96
+
}
97
+
return &am, nil
98
+
}
+36
automod/cachestore.go
+36
automod/cachestore.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/hashicorp/golang-lru/v2/expirable"
8
+
)
9
+
10
+
type CacheStore interface {
11
+
Get(ctx context.Context, name, key string) (string, error)
12
+
Set(ctx context.Context, name, key string, val string) error
13
+
}
14
+
15
+
type MemCacheStore struct {
16
+
Data *expirable.LRU[string, string]
17
+
}
18
+
19
+
func NewMemCacheStore(capacity int, ttl time.Duration) MemCacheStore {
20
+
return MemCacheStore{
21
+
Data: expirable.NewLRU[string, string](capacity, nil, ttl),
22
+
}
23
+
}
24
+
25
+
func (s MemCacheStore) Get(ctx context.Context, name, key string) (string, error) {
26
+
v, ok := s.Data.Get(name + "/" + key)
27
+
if !ok {
28
+
return "", nil
29
+
}
30
+
return v, nil
31
+
}
32
+
33
+
func (s MemCacheStore) Set(ctx context.Context, name, key string, val string) error {
34
+
s.Data.Add(name+"/"+key, val)
35
+
return nil
36
+
}
+30
-14
automod/engine.go
+30
-14
automod/engine.go
···
16
16
//
17
17
// TODO: careful when initializing: several fields should not be null or zero, even though they are pointer type.
18
18
type Engine struct {
19
-
Logger *slog.Logger
20
-
Directory identity.Directory
21
-
Rules RuleSet
22
-
Counters CountStore
23
-
Sets SetStore
19
+
Logger *slog.Logger
20
+
Directory identity.Directory
21
+
Rules RuleSet
22
+
Counters CountStore
23
+
Sets SetStore
24
+
Cache CacheStore
25
+
BskyClient *xrpc.Client
24
26
// used to persist moderation actions in mod service (optional)
25
27
AdminClient *xrpc.Client
26
28
}
···
41
43
return fmt.Errorf("identity not found for did: %s", did.String())
42
44
}
43
45
46
+
am, err := e.GetAccountMeta(ctx, ident)
47
+
if err != nil {
48
+
return err
49
+
}
44
50
evt := IdentityEvent{
45
51
Event{
46
52
Engine: e,
47
-
Account: AccountMeta{Identity: ident},
53
+
Account: *am,
48
54
},
49
55
}
50
56
if err := e.Rules.CallIdentityRules(&evt); err != nil {
···
62
68
63
69
func (e *Engine) ProcessRecord(ctx context.Context, did syntax.DID, path, recCID string, rec any) error {
64
70
// similar to an HTTP server, we want to recover any panics from rule execution
71
+
/* XXX
65
72
defer func() {
66
73
if r := recover(); r != nil {
67
74
e.Logger.Error("automod event execution exception", "err", r)
68
75
}
69
76
}()
77
+
*/
70
78
71
79
ident, err := e.Directory.LookupDID(ctx, did)
72
80
if err != nil {
···
83
91
if !ok {
84
92
return fmt.Errorf("mismatch between collection (%s) and type", collection)
85
93
}
86
-
evt := e.NewPostEvent(ident, path, recCID, post)
94
+
am, err := e.GetAccountMeta(ctx, ident)
95
+
if err != nil {
96
+
return err
97
+
}
98
+
evt := e.NewPostEvent(*am, path, recCID, post)
87
99
e.Logger.Debug("processing post", "did", ident.DID, "path", path)
88
100
if err := e.Rules.CallPostRules(&evt); err != nil {
89
101
return err
···
99
111
return err
100
112
}
101
113
default:
102
-
evt := e.NewRecordEvent(ident, path, recCID, rec)
114
+
am, err := e.GetAccountMeta(ctx, ident)
115
+
if err != nil {
116
+
return err
117
+
}
118
+
evt := e.NewRecordEvent(*am, path, recCID, rec)
103
119
e.Logger.Debug("processing record", "did", ident.DID, "path", path)
104
120
if err := e.Rules.CallRecordRules(&evt); err != nil {
105
121
return err
···
119
135
return nil
120
136
}
121
137
122
-
func (e *Engine) NewPostEvent(ident *identity.Identity, path, recCID string, post *appbsky.FeedPost) PostEvent {
138
+
func (e *Engine) NewPostEvent(am AccountMeta, path, recCID string, post *appbsky.FeedPost) PostEvent {
123
139
parts := strings.SplitN(path, "/", 2)
124
140
return PostEvent{
125
141
RecordEvent{
126
142
Event{
127
143
Engine: e,
128
-
Logger: e.Logger.With("did", ident.DID, "collection", parts[0], "rkey", parts[1]),
129
-
Account: AccountMeta{Identity: ident},
144
+
Logger: e.Logger.With("did", am.Identity.DID, "collection", parts[0], "rkey", parts[1]),
145
+
Account: am,
130
146
},
131
147
parts[0],
132
148
parts[1],
···
140
156
}
141
157
}
142
158
143
-
func (e *Engine) NewRecordEvent(ident *identity.Identity, path, recCID string, rec any) RecordEvent {
159
+
func (e *Engine) NewRecordEvent(am AccountMeta, path, recCID string, rec any) RecordEvent {
144
160
parts := strings.SplitN(path, "/", 2)
145
161
return RecordEvent{
146
162
Event{
147
163
Engine: e,
148
-
Logger: e.Logger.With("did", ident.DID, "collection", parts[0], "rkey", parts[1]),
149
-
Account: AccountMeta{Identity: ident},
164
+
Logger: e.Logger.With("did", am.Identity.DID, "collection", parts[0], "rkey", parts[1]),
165
+
Account: am,
150
166
},
151
167
parts[0],
152
168
parts[1],
+1
-8
automod/event.go
+1
-8
automod/event.go
···
7
7
8
8
comatproto "github.com/bluesky-social/indigo/api/atproto"
9
9
appbsky "github.com/bluesky-social/indigo/api/bsky"
10
-
"github.com/bluesky-social/indigo/atproto/identity"
11
10
)
12
11
13
12
type ModReport struct {
14
13
ReasonType string
15
14
Comment string
16
-
}
17
-
18
-
// information about a repo/account/identity, always pre-populated and relevant to many rules
19
-
type AccountMeta struct {
20
-
Identity *identity.Identity
21
-
// TODO: createdAt / age
22
15
}
23
16
24
17
type CounterRef struct {
···
84
77
xrpcc := e.Engine.AdminClient
85
78
if len(e.AccountLabels) > 0 {
86
79
_, err := comatproto.AdminTakeModerationAction(ctx, xrpcc, &comatproto.AdminTakeModerationAction_Input{
87
-
Action: "com.atproto.admin.defs#createLabels",
80
+
Action: "com.atproto.admin.defs#flag",
88
81
CreateLabelVals: dedupeStrings(e.AccountLabels),
89
82
Reason: "automod",
90
83
CreatedBy: xrpcc.Auth.Did,
+63
automod/redis_cache.go
+63
automod/redis_cache.go
···
1
+
package automod
2
+
3
+
import (
4
+
"context"
5
+
"time"
6
+
7
+
"github.com/go-redis/cache/v9"
8
+
"github.com/redis/go-redis/v9"
9
+
)
10
+
11
+
type RedisCacheStore struct {
12
+
Data *cache.Cache
13
+
TTL time.Duration
14
+
}
15
+
16
+
var _ CacheStore = (*RedisCacheStore)(nil)
17
+
18
+
func NewRedisCacheStore(redisURL string, ttl time.Duration) (*RedisCacheStore, error) {
19
+
opt, err := redis.ParseURL(redisURL)
20
+
if err != nil {
21
+
return nil, err
22
+
}
23
+
rdb := redis.NewClient(opt)
24
+
// check redis connection
25
+
_, err = rdb.Ping(context.TODO()).Result()
26
+
if err != nil {
27
+
return nil, err
28
+
}
29
+
data := cache.New(&cache.Options{
30
+
Redis: rdb,
31
+
LocalCache: cache.NewTinyLFU(10_000, ttl),
32
+
})
33
+
return &RedisCacheStore{
34
+
Data: data,
35
+
TTL: ttl,
36
+
}, nil
37
+
}
38
+
39
+
func redisCacheKey(name, key string) string {
40
+
return "cache/" + name + "/" + key
41
+
}
42
+
43
+
func (s RedisCacheStore) Get(ctx context.Context, name, key string) (string, error) {
44
+
var val string
45
+
err := s.Data.Get(ctx, redisCacheKey(name, key), &val)
46
+
if err == cache.ErrCacheMiss {
47
+
return "", nil
48
+
}
49
+
if err != nil {
50
+
return "", err
51
+
}
52
+
return val, nil
53
+
}
54
+
55
+
func (s RedisCacheStore) Set(ctx context.Context, name, key string, val string) error {
56
+
s.Data.Set(&cache.Item{
57
+
Ctx: ctx,
58
+
Key: redisCacheKey(name, key),
59
+
Value: val,
60
+
TTL: s.TTL,
61
+
})
62
+
return nil
63
+
}
+22
-6
automod/redis_directory.go
+22
-6
automod/redis_directory.go
···
93
93
DID: "",
94
94
Err: err,
95
95
}
96
-
d.handleCache.Set(&cache.Item{
96
+
err = d.handleCache.Set(&cache.Item{
97
97
Ctx: ctx,
98
98
Key: redisDirPrefix + h.String(),
99
99
Value: he,
100
100
TTL: d.ErrTTL,
101
101
})
102
+
if err != nil {
103
+
return nil, err
104
+
}
102
105
return &he, nil
103
106
}
104
107
108
+
ident.ParsedPublicKey = nil
105
109
entry := IdentityEntry{
106
110
Updated: time.Now(),
107
111
Identity: ident,
···
113
117
Err: nil,
114
118
}
115
119
116
-
d.identityCache.Set(&cache.Item{
120
+
err = d.identityCache.Set(&cache.Item{
117
121
Ctx: ctx,
118
122
Key: redisDirPrefix + ident.DID.String(),
119
123
Value: entry,
120
124
TTL: d.HitTTL,
121
125
})
122
-
d.handleCache.Set(&cache.Item{
126
+
if err != nil {
127
+
return nil, err
128
+
}
129
+
err = d.handleCache.Set(&cache.Item{
123
130
Ctx: ctx,
124
131
Key: redisDirPrefix + h.String(),
125
132
Value: he,
126
133
TTL: d.HitTTL,
127
134
})
135
+
if err != nil {
136
+
return nil, err
137
+
}
128
138
return &he, nil
129
139
}
130
140
···
178
188
179
189
func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) (*IdentityEntry, error) {
180
190
ident, err := d.Inner.LookupDID(ctx, did)
181
-
// wipe parsed public key; it's a waste of space
191
+
// wipe parsed public key; it's a waste of space and can't serialize
182
192
if nil == err {
183
193
ident.ParsedPublicKey = nil
184
194
}
···
198
208
}
199
209
}
200
210
201
-
d.identityCache.Set(&cache.Item{
211
+
err = d.identityCache.Set(&cache.Item{
202
212
Ctx: ctx,
203
213
Key: redisDirPrefix + did.String(),
204
214
Value: entry,
205
215
TTL: d.HitTTL,
206
216
})
217
+
if err != nil {
218
+
return nil, err
219
+
}
207
220
if he != nil {
208
-
d.handleCache.Set(&cache.Item{
221
+
err = d.handleCache.Set(&cache.Item{
209
222
Ctx: ctx,
210
223
Key: redisDirPrefix + ident.Handle.String(),
211
224
Value: *he,
212
225
TTL: d.HitTTL,
213
226
})
227
+
if err != nil {
228
+
return nil, err
229
+
}
214
230
}
215
231
return &entry, nil
216
232
}
+1
automod/rules/all.go
+1
automod/rules/all.go
+13
-8
automod/rules/misleading_test.go
+13
-8
automod/rules/misleading_test.go
···
6
6
appbsky "github.com/bluesky-social/indigo/api/bsky"
7
7
"github.com/bluesky-social/indigo/atproto/identity"
8
8
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
"github.com/bluesky-social/indigo/automod"
9
10
10
11
"github.com/stretchr/testify/assert"
11
12
)
···
14
15
assert := assert.New(t)
15
16
16
17
engine := engineFixture()
17
-
id1 := identity.Identity{
18
-
DID: syntax.DID("did:plc:abc111"),
19
-
Handle: syntax.Handle("handle.example.com"),
18
+
am1 := automod.AccountMeta{
19
+
Identity: &identity.Identity{
20
+
DID: syntax.DID("did:plc:abc111"),
21
+
Handle: syntax.Handle("handle.example.com"),
22
+
},
20
23
}
21
24
path := "app.bsky.feed.post/abc123"
22
25
cid1 := "cid123"
···
38
41
},
39
42
},
40
43
}
41
-
evt1 := engine.NewPostEvent(&id1, path, cid1, &p1)
44
+
evt1 := engine.NewPostEvent(am1, path, cid1, &p1)
42
45
assert.NoError(MisleadingURLPostRule(&evt1))
43
46
assert.NotEmpty(evt1.RecordLabels)
44
47
}
···
47
50
assert := assert.New(t)
48
51
49
52
engine := engineFixture()
50
-
id1 := identity.Identity{
51
-
DID: syntax.DID("did:plc:abc111"),
52
-
Handle: syntax.Handle("handle.example.com"),
53
+
am1 := automod.AccountMeta{
54
+
Identity: &identity.Identity{
55
+
DID: syntax.DID("did:plc:abc111"),
56
+
Handle: syntax.Handle("handle.example.com"),
57
+
},
53
58
}
54
59
path := "app.bsky.feed.post/abc123"
55
60
cid1 := "cid123"
···
71
76
},
72
77
},
73
78
}
74
-
evt1 := engine.NewPostEvent(&id1, path, cid1, &p1)
79
+
evt1 := engine.NewPostEvent(am1, path, cid1, &p1)
75
80
assert.NoError(MisleadingMentionPostRule(&evt1))
76
81
assert.NotEmpty(evt1.RecordLabels)
77
82
}
+13
automod/rules/profile.go
+13
automod/rules/profile.go
···
1
+
package rules
2
+
3
+
import (
4
+
"github.com/bluesky-social/indigo/automod"
5
+
)
6
+
7
+
// this is a dummy rule to demonstrate accessing account metadata (eg, profile) from within post handler
8
+
func AccountDemoPostRule(evt *automod.PostEvent) error {
9
+
if evt.Account.Profile.Description != nil && len(evt.Post.Text) > 5 && *evt.Account.Profile.Description == evt.Post.Text {
10
+
evt.AddRecordFlag("own-profile-description")
11
+
}
12
+
return nil
13
+
}
+7
cmd/hepa/main.go
+7
cmd/hepa/main.go
···
51
51
Value: "https://api.bsky.app",
52
52
EnvVars: []string{"ATP_MOD_HOST"},
53
53
},
54
+
&cli.StringFlag{
55
+
Name: "atp-bsky-host",
56
+
Usage: "method, hostname, and port of bsky API (appview) service",
57
+
Value: "https://api.bsky.app",
58
+
EnvVars: []string{"ATP_BSKY_HOST"},
59
+
},
54
60
}
55
61
56
62
app.Commands = []*cli.Command{
···
138
144
dir,
139
145
Config{
140
146
BGSHost: cctx.String("atp-bgs-host"),
147
+
BskyHost: cctx.String("atp-bsky-host"),
141
148
Logger: logger,
142
149
ModHost: cctx.String("atp-mod-host"),
143
150
ModAdminToken: cctx.String("mod-admin-token"),
+18
cmd/hepa/server.go
+18
cmd/hepa/server.go
···
7
7
"net/http"
8
8
"os"
9
9
"strings"
10
+
"time"
10
11
11
12
comatproto "github.com/bluesky-social/indigo/api/atproto"
12
13
"github.com/bluesky-social/indigo/atproto/identity"
···
26
27
27
28
type Config struct {
28
29
BGSHost string
30
+
BskyHost string
29
31
ModHost string
30
32
ModAdminToken string
31
33
ModUsername string
···
91
93
counters = automod.NewMemCountStore()
92
94
}
93
95
96
+
var cache automod.CacheStore
97
+
if config.RedisURL != "" {
98
+
c, err := automod.NewRedisCacheStore(config.RedisURL, 30*time.Minute)
99
+
if err != nil {
100
+
return nil, err
101
+
}
102
+
cache = c
103
+
} else {
104
+
cache = automod.NewMemCacheStore(5_000, 30*time.Minute)
105
+
}
106
+
94
107
engine := automod.Engine{
95
108
Logger: logger,
96
109
Directory: dir,
97
110
Counters: counters,
98
111
Sets: sets,
112
+
Cache: cache,
99
113
Rules: rules.DefaultRules(),
100
114
AdminClient: xrpcc,
115
+
BskyClient: &xrpc.Client{
116
+
Client: util.RobustHTTPClient(),
117
+
Host: config.BskyHost,
118
+
},
101
119
}
102
120
103
121
s := &Server{