this repo has no description
automod: refactor identity and account event processing (#781)
@foysalit: this is pulling out some of the engine-layer refactors from
https://github.com/bluesky-social/indigo/pull/708
+113
-53
+9
-1
automod/capture/testing.go
+9
-1
automod/capture/testing.go
···
7
7
"io"
8
8
"os"
9
9
10
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
10
11
"github.com/bluesky-social/indigo/atproto/identity"
11
12
"github.com/bluesky-social/indigo/atproto/syntax"
12
13
"github.com/bluesky-social/indigo/automod"
···
38
39
ctx := context.Background()
39
40
40
41
did := capture.AccountMeta.Identity.DID
42
+
handle := capture.AccountMeta.Identity.Handle.String()
41
43
dir := identity.NewMockDirectory()
42
44
dir.Insert(*capture.AccountMeta.Identity)
43
45
eng.Directory = &dir
44
46
45
47
// initial identity rules
46
-
eng.ProcessIdentityEvent(ctx, "new", did)
48
+
identEvent := comatproto.SyncSubscribeRepos_Identity{
49
+
Did: did.String(),
50
+
Handle: &handle,
51
+
Seq: 12345,
52
+
Time: syntax.DatetimeNow().String(),
53
+
}
54
+
eng.ProcessIdentityEvent(ctx, identEvent)
47
55
48
56
// all the post rules
49
57
for _, pr := range capture.PostRecords {
+10
-49
automod/consumer/firehose.go
+10
-49
automod/consumer/firehose.go
···
80
80
},
81
81
RepoIdentity: func(evt *comatproto.SyncSubscribeRepos_Identity) error {
82
82
atomic.StoreInt64(&fc.lastSeq, evt.Seq)
83
-
did, err := syntax.ParseDID(evt.Did)
84
-
if err != nil {
85
-
fc.Logger.Error("bad DID in RepoIdentity event", "did", evt.Did, "seq", evt.Seq, "err", err)
86
-
return nil
87
-
}
88
-
if err := fc.Engine.ProcessIdentityEvent(ctx, "identity", did); err != nil {
83
+
if err := fc.Engine.ProcessIdentityEvent(ctx, *evt); err != nil {
89
84
fc.Logger.Error("processing repo identity failed", "did", evt.Did, "seq", evt.Seq, "err", err)
90
85
}
91
86
return nil
92
87
},
93
88
RepoAccount: func(evt *comatproto.SyncSubscribeRepos_Account) error {
94
89
atomic.StoreInt64(&fc.lastSeq, evt.Seq)
95
-
did, err := syntax.ParseDID(evt.Did)
96
-
if err != nil {
97
-
fc.Logger.Error("bad DID in RepoAccount event", "did", evt.Did, "seq", evt.Seq, "err", err)
98
-
return nil
99
-
}
100
-
if err := fc.Engine.ProcessIdentityEvent(ctx, "account", did); err != nil {
90
+
if err := fc.Engine.ProcessAccountEvent(ctx, *evt); err != nil {
101
91
fc.Logger.Error("processing repo account failed", "did", evt.Did, "seq", evt.Seq, "err", err)
102
92
}
103
93
return nil
104
94
},
105
-
// TODO: deprecated
106
-
RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error {
107
-
atomic.StoreInt64(&fc.lastSeq, evt.Seq)
108
-
did, err := syntax.ParseDID(evt.Did)
109
-
if err != nil {
110
-
fc.Logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err)
111
-
return nil
112
-
}
113
-
if err := fc.Engine.ProcessIdentityEvent(ctx, "handle", did); err != nil {
114
-
fc.Logger.Error("processing handle update failed", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err)
115
-
}
116
-
return nil
117
-
},
118
-
// TODO: deprecated
119
-
RepoTombstone: func(evt *comatproto.SyncSubscribeRepos_Tombstone) error {
120
-
atomic.StoreInt64(&fc.lastSeq, evt.Seq)
121
-
did, err := syntax.ParseDID(evt.Did)
122
-
if err != nil {
123
-
fc.Logger.Error("bad DID in RepoTombstone event", "did", evt.Did, "seq", evt.Seq, "err", err)
124
-
return nil
125
-
}
126
-
if err := fc.Engine.ProcessIdentityEvent(ctx, "tombstone", did); err != nil {
127
-
fc.Logger.Error("processing repo tombstone failed", "did", evt.Did, "seq", evt.Seq, "err", err)
128
-
}
129
-
return nil
130
-
},
95
+
// NOTE: no longer process #handle events
96
+
// NOTE: no longer process #tombstone events
131
97
}
132
98
133
99
var scheduler events.Scheduler
···
176
142
return nil
177
143
}
178
144
179
-
// empty commit is a special case, temporarily, basically indicates "new account"
180
-
if len(evt.Ops) == 0 {
181
-
if err := fc.Engine.ProcessIdentityEvent(ctx, "create", did); err != nil {
182
-
fc.Logger.Error("processing handle update failed", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq, "err", err)
183
-
}
184
-
}
185
-
186
145
for _, op := range evt.Ops {
187
146
logger = logger.With("eventKind", op.Action, "path", op.Path)
188
147
collection, rkey, err := splitRepoPath(op.Path)
···
215
174
break
216
175
}
217
176
recCID := syntax.CID(op.Cid.String())
218
-
err = fc.Engine.ProcessRecordOp(ctx, automod.RecordOp{
177
+
op := automod.RecordOp{
219
178
Action: action,
220
179
DID: did,
221
180
Collection: collection,
222
181
RecordKey: rkey,
223
182
CID: &recCID,
224
183
RecordCBOR: *recCBOR,
225
-
})
184
+
}
185
+
err = fc.Engine.ProcessRecordOp(ctx, op)
226
186
if err != nil {
227
187
logger.Error("engine failed to process record", "err", err)
228
188
continue
229
189
}
230
190
case repomgr.EvtKindDeleteRecord:
231
-
err = fc.Engine.ProcessRecordOp(ctx, automod.RecordOp{
191
+
op := automod.RecordOp{
232
192
Action: automod.DeleteOp,
233
193
DID: did,
234
194
Collection: collection,
235
195
RecordKey: rkey,
236
196
CID: nil,
237
197
RecordCBOR: nil,
238
-
})
198
+
}
199
+
err = fc.Engine.ProcessRecordOp(ctx, op)
239
200
if err != nil {
240
201
logger.Error("engine failed to process record", "err", err)
241
202
continue
+81
-3
automod/engine/engine.go
+81
-3
automod/engine/engine.go
···
7
7
"net/http"
8
8
"time"
9
9
10
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
10
11
"github.com/bluesky-social/indigo/atproto/identity"
11
12
"github.com/bluesky-social/indigo/atproto/syntax"
12
13
"github.com/bluesky-social/indigo/automod/cachestore"
···
53
54
SkipAccountMeta bool
54
55
}
55
56
56
-
// Entrypoint for external code pushing arbitrary identity events in to the engine.
57
+
// Entrypoint for external code pushing #identity events in to the engine.
57
58
//
58
59
// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel.
59
-
func (eng *Engine) ProcessIdentityEvent(ctx context.Context, typ string, did syntax.DID) error {
60
+
func (eng *Engine) ProcessIdentityEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Identity) error {
60
61
eventProcessCount.WithLabelValues("identity").Inc()
61
62
start := time.Now()
62
63
defer func() {
···
64
65
eventProcessDuration.WithLabelValues("identity").Observe(duration.Seconds())
65
66
}()
66
67
68
+
did, err := syntax.ParseDID(evt.Did)
69
+
if err != nil {
70
+
return fmt.Errorf("bad DID in repo #identity event (%s): %w", evt.Did, err)
71
+
}
72
+
67
73
// similar to an HTTP server, we want to recover any panics from rule execution
68
74
defer func() {
69
75
if r := recover(); r != nil {
70
-
eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", typ)
76
+
eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "identity")
71
77
eventErrorCount.WithLabelValues("identity").Inc()
72
78
}
73
79
}()
···
78
84
if err := eng.PurgeAccountCaches(ctx, did); err != nil {
79
85
eng.Logger.Error("failed to purge identity cache; identity rule may not run correctly", "err", err)
80
86
}
87
+
// TODO(bnewbold): if it was a tombstone, this might fail
81
88
ident, err := eng.Directory.LookupDID(ctx, did)
82
89
if err != nil {
83
90
eventErrorCount.WithLabelValues("identity").Inc()
···
114
121
if err := eng.persistCounters(ctx, ac.effects); err != nil {
115
122
eventErrorCount.WithLabelValues("identity").Inc()
116
123
return fmt.Errorf("failed to persist counters for identity event: %w", err)
124
+
}
125
+
return nil
126
+
}
127
+
128
+
// Entrypoint for external code pushing #account events in to the engine.
129
+
//
130
+
// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel.
131
+
func (eng *Engine) ProcessAccountEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Account) error {
132
+
eventProcessCount.WithLabelValues("account").Inc()
133
+
start := time.Now()
134
+
defer func() {
135
+
duration := time.Since(start)
136
+
eventProcessDuration.WithLabelValues("account").Observe(duration.Seconds())
137
+
}()
138
+
139
+
did, err := syntax.ParseDID(evt.Did)
140
+
if err != nil {
141
+
return fmt.Errorf("bad DID in repo #account event (%s): %w", evt.Did, err)
142
+
}
143
+
144
+
// similar to an HTTP server, we want to recover any panics from rule execution
145
+
defer func() {
146
+
if r := recover(); r != nil {
147
+
eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "account")
148
+
eventErrorCount.WithLabelValues("account").Inc()
149
+
}
150
+
}()
151
+
ctx, cancel := context.WithTimeout(ctx, identityEventTimeout)
152
+
defer cancel()
153
+
154
+
// first purge any caches; we need to re-resolve from scratch on account updates
155
+
if err := eng.PurgeAccountCaches(ctx, did); err != nil {
156
+
eng.Logger.Error("failed to purge account cache; account rule may not run correctly", "err", err)
157
+
}
158
+
// TODO(bnewbold): if it was a tombstone, this might fail
159
+
ident, err := eng.Directory.LookupDID(ctx, did)
160
+
if err != nil {
161
+
eventErrorCount.WithLabelValues("account").Inc()
162
+
return fmt.Errorf("resolving identity: %w", err)
163
+
}
164
+
if ident == nil {
165
+
eventErrorCount.WithLabelValues("account").Inc()
166
+
return fmt.Errorf("identity not found for DID: %s", did.String())
167
+
}
168
+
169
+
var am *AccountMeta
170
+
if !eng.Config.SkipAccountMeta {
171
+
am, err = eng.GetAccountMeta(ctx, ident)
172
+
if err != nil {
173
+
eventErrorCount.WithLabelValues("identity").Inc()
174
+
return fmt.Errorf("failed to fetch account metadata: %w", err)
175
+
}
176
+
} else {
177
+
am = &AccountMeta{
178
+
Identity: ident,
179
+
Profile: ProfileSummary{},
180
+
}
181
+
}
182
+
ac := NewAccountContext(ctx, eng, *am)
183
+
if err := eng.Rules.CallAccountRules(&ac); err != nil {
184
+
eventErrorCount.WithLabelValues("account").Inc()
185
+
return fmt.Errorf("rule execution failed: %w", err)
186
+
}
187
+
eng.CanonicalLogLineAccount(&ac)
188
+
if err := eng.persistAccountModActions(&ac); err != nil {
189
+
eventErrorCount.WithLabelValues("account").Inc()
190
+
return fmt.Errorf("failed to persist actions for account event: %w", err)
191
+
}
192
+
if err := eng.persistCounters(ctx, ac.effects); err != nil {
193
+
eventErrorCount.WithLabelValues("account").Inc()
194
+
return fmt.Errorf("failed to persist counters for account event: %w", err)
117
195
}
118
196
return nil
119
197
}
+12
automod/engine/ruleset.go
+12
automod/engine/ruleset.go
···
16
16
RecordRules []RecordRuleFunc
17
17
RecordDeleteRules []RecordRuleFunc
18
18
IdentityRules []IdentityRuleFunc
19
+
AccountRules []AccountRuleFunc
19
20
BlobRules []BlobRuleFunc
20
21
NotificationRules []NotificationRuleFunc
21
22
OzoneEventRules []OzoneEventRuleFunc
···
84
85
err := f(c)
85
86
if err != nil {
86
87
c.Logger.Error("identity rule execution failed", "err", err)
88
+
}
89
+
}
90
+
return nil
91
+
}
92
+
93
+
// Executes rules for account update events.
94
+
func (r *RuleSet) CallAccountRules(c *AccountContext) error {
95
+
for _, f := range r.AccountRules {
96
+
err := f(c)
97
+
if err != nil {
98
+
c.Logger.Error("account rule execution failed", "err", err)
87
99
}
88
100
}
89
101
return nil
+1
automod/engine/ruletypes.go
+1
automod/engine/ruletypes.go
···
6
6
)
7
7
8
8
type IdentityRuleFunc = func(c *AccountContext) error
9
+
type AccountRuleFunc = func(c *AccountContext) error
9
10
type RecordRuleFunc = func(c *RecordContext) error
10
11
type PostRuleFunc = func(c *RecordContext, post *appbsky.FeedPost) error
11
12
type ProfileRuleFunc = func(c *RecordContext, profile *appbsky.ActorProfile) error