this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

automod: refactor identity and account event processing (#781)

@foysalit: this is pulling out some of the engine-layer refactors from
https://github.com/bluesky-social/indigo/pull/708

authored by

bnewbold and committed by
GitHub
296087d0 3ff74054

+113 -53
+9 -1
automod/capture/testing.go
··· 7 7 "io" 8 8 "os" 9 9 10 + comatproto "github.com/bluesky-social/indigo/api/atproto" 10 11 "github.com/bluesky-social/indigo/atproto/identity" 11 12 "github.com/bluesky-social/indigo/atproto/syntax" 12 13 "github.com/bluesky-social/indigo/automod" ··· 38 39 ctx := context.Background() 39 40 40 41 did := capture.AccountMeta.Identity.DID 42 + handle := capture.AccountMeta.Identity.Handle.String() 41 43 dir := identity.NewMockDirectory() 42 44 dir.Insert(*capture.AccountMeta.Identity) 43 45 eng.Directory = &dir 44 46 45 47 // initial identity rules 46 - eng.ProcessIdentityEvent(ctx, "new", did) 48 + identEvent := comatproto.SyncSubscribeRepos_Identity{ 49 + Did: did.String(), 50 + Handle: &handle, 51 + Seq: 12345, 52 + Time: syntax.DatetimeNow().String(), 53 + } 54 + eng.ProcessIdentityEvent(ctx, identEvent) 47 55 48 56 // all the post rules 49 57 for _, pr := range capture.PostRecords {
+10 -49
automod/consumer/firehose.go
··· 80 80 }, 81 81 RepoIdentity: func(evt *comatproto.SyncSubscribeRepos_Identity) error { 82 82 atomic.StoreInt64(&fc.lastSeq, evt.Seq) 83 - did, err := syntax.ParseDID(evt.Did) 84 - if err != nil { 85 - fc.Logger.Error("bad DID in RepoIdentity event", "did", evt.Did, "seq", evt.Seq, "err", err) 86 - return nil 87 - } 88 - if err := fc.Engine.ProcessIdentityEvent(ctx, "identity", did); err != nil { 83 + if err := fc.Engine.ProcessIdentityEvent(ctx, *evt); err != nil { 89 84 fc.Logger.Error("processing repo identity failed", "did", evt.Did, "seq", evt.Seq, "err", err) 90 85 } 91 86 return nil 92 87 }, 93 88 RepoAccount: func(evt *comatproto.SyncSubscribeRepos_Account) error { 94 89 atomic.StoreInt64(&fc.lastSeq, evt.Seq) 95 - did, err := syntax.ParseDID(evt.Did) 96 - if err != nil { 97 - fc.Logger.Error("bad DID in RepoAccount event", "did", evt.Did, "seq", evt.Seq, "err", err) 98 - return nil 99 - } 100 - if err := fc.Engine.ProcessIdentityEvent(ctx, "account", did); err != nil { 90 + if err := fc.Engine.ProcessAccountEvent(ctx, *evt); err != nil { 101 91 fc.Logger.Error("processing repo account failed", "did", evt.Did, "seq", evt.Seq, "err", err) 102 92 } 103 93 return nil 104 94 }, 105 - // TODO: deprecated 106 - RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error { 107 - atomic.StoreInt64(&fc.lastSeq, evt.Seq) 108 - did, err := syntax.ParseDID(evt.Did) 109 - if err != nil { 110 - fc.Logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) 111 - return nil 112 - } 113 - if err := fc.Engine.ProcessIdentityEvent(ctx, "handle", did); err != nil { 114 - fc.Logger.Error("processing handle update failed", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) 115 - } 116 - return nil 117 - }, 118 - // TODO: deprecated 119 - RepoTombstone: func(evt *comatproto.SyncSubscribeRepos_Tombstone) error { 120 - atomic.StoreInt64(&fc.lastSeq, evt.Seq) 121 - did, err := syntax.ParseDID(evt.Did) 122 - if err != nil { 123 - fc.Logger.Error("bad DID in RepoTombstone event", "did", evt.Did, "seq", evt.Seq, "err", err) 124 - return nil 125 - } 126 - if err := fc.Engine.ProcessIdentityEvent(ctx, "tombstone", did); err != nil { 127 - fc.Logger.Error("processing repo tombstone failed", "did", evt.Did, "seq", evt.Seq, "err", err) 128 - } 129 - return nil 130 - }, 95 + // NOTE: no longer process #handle events 96 + // NOTE: no longer process #tombstone events 131 97 } 132 98 133 99 var scheduler events.Scheduler ··· 176 142 return nil 177 143 } 178 144 179 - // empty commit is a special case, temporarily, basically indicates "new account" 180 - if len(evt.Ops) == 0 { 181 - if err := fc.Engine.ProcessIdentityEvent(ctx, "create", did); err != nil { 182 - fc.Logger.Error("processing handle update failed", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq, "err", err) 183 - } 184 - } 185 - 186 145 for _, op := range evt.Ops { 187 146 logger = logger.With("eventKind", op.Action, "path", op.Path) 188 147 collection, rkey, err := splitRepoPath(op.Path) ··· 215 174 break 216 175 } 217 176 recCID := syntax.CID(op.Cid.String()) 218 - err = fc.Engine.ProcessRecordOp(ctx, automod.RecordOp{ 177 + op := automod.RecordOp{ 219 178 Action: action, 220 179 DID: did, 221 180 Collection: collection, 222 181 RecordKey: rkey, 223 182 CID: &recCID, 224 183 RecordCBOR: *recCBOR, 225 - }) 184 + } 185 + err = fc.Engine.ProcessRecordOp(ctx, op) 226 186 if err != nil { 227 187 logger.Error("engine failed to process record", "err", err) 228 188 continue 229 189 } 230 190 case repomgr.EvtKindDeleteRecord: 231 - err = fc.Engine.ProcessRecordOp(ctx, automod.RecordOp{ 191 + op := automod.RecordOp{ 232 192 Action: automod.DeleteOp, 233 193 DID: did, 234 194 Collection: collection, 235 195 RecordKey: rkey, 236 196 CID: nil, 237 197 RecordCBOR: nil, 238 - }) 198 + } 199 + err = fc.Engine.ProcessRecordOp(ctx, op) 239 200 if err != nil { 240 201 logger.Error("engine failed to process record", "err", err) 241 202 continue
+81 -3
automod/engine/engine.go
··· 7 7 "net/http" 8 8 "time" 9 9 10 + comatproto "github.com/bluesky-social/indigo/api/atproto" 10 11 "github.com/bluesky-social/indigo/atproto/identity" 11 12 "github.com/bluesky-social/indigo/atproto/syntax" 12 13 "github.com/bluesky-social/indigo/automod/cachestore" ··· 53 54 SkipAccountMeta bool 54 55 } 55 56 56 - // Entrypoint for external code pushing arbitrary identity events in to the engine. 57 + // Entrypoint for external code pushing #identity events in to the engine. 57 58 // 58 59 // This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. 59 - func (eng *Engine) ProcessIdentityEvent(ctx context.Context, typ string, did syntax.DID) error { 60 + func (eng *Engine) ProcessIdentityEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Identity) error { 60 61 eventProcessCount.WithLabelValues("identity").Inc() 61 62 start := time.Now() 62 63 defer func() { ··· 64 65 eventProcessDuration.WithLabelValues("identity").Observe(duration.Seconds()) 65 66 }() 66 67 68 + did, err := syntax.ParseDID(evt.Did) 69 + if err != nil { 70 + return fmt.Errorf("bad DID in repo #identity event (%s): %w", evt.Did, err) 71 + } 72 + 67 73 // similar to an HTTP server, we want to recover any panics from rule execution 68 74 defer func() { 69 75 if r := recover(); r != nil { 70 - eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", typ) 76 + eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "identity") 71 77 eventErrorCount.WithLabelValues("identity").Inc() 72 78 } 73 79 }() ··· 78 84 if err := eng.PurgeAccountCaches(ctx, did); err != nil { 79 85 eng.Logger.Error("failed to purge identity cache; identity rule may not run correctly", "err", err) 80 86 } 87 + // TODO(bnewbold): if it was a tombstone, this might fail 81 88 ident, err := eng.Directory.LookupDID(ctx, did) 82 89 if err != nil { 83 90 eventErrorCount.WithLabelValues("identity").Inc() ··· 114 121 if err := eng.persistCounters(ctx, ac.effects); err != nil { 115 122 eventErrorCount.WithLabelValues("identity").Inc() 116 123 return fmt.Errorf("failed to persist counters for identity event: %w", err) 124 + } 125 + return nil 126 + } 127 + 128 + // Entrypoint for external code pushing #account events in to the engine. 129 + // 130 + // This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. 131 + func (eng *Engine) ProcessAccountEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Account) error { 132 + eventProcessCount.WithLabelValues("account").Inc() 133 + start := time.Now() 134 + defer func() { 135 + duration := time.Since(start) 136 + eventProcessDuration.WithLabelValues("account").Observe(duration.Seconds()) 137 + }() 138 + 139 + did, err := syntax.ParseDID(evt.Did) 140 + if err != nil { 141 + return fmt.Errorf("bad DID in repo #account event (%s): %w", evt.Did, err) 142 + } 143 + 144 + // similar to an HTTP server, we want to recover any panics from rule execution 145 + defer func() { 146 + if r := recover(); r != nil { 147 + eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "account") 148 + eventErrorCount.WithLabelValues("account").Inc() 149 + } 150 + }() 151 + ctx, cancel := context.WithTimeout(ctx, identityEventTimeout) 152 + defer cancel() 153 + 154 + // first purge any caches; we need to re-resolve from scratch on account updates 155 + if err := eng.PurgeAccountCaches(ctx, did); err != nil { 156 + eng.Logger.Error("failed to purge account cache; account rule may not run correctly", "err", err) 157 + } 158 + // TODO(bnewbold): if it was a tombstone, this might fail 159 + ident, err := eng.Directory.LookupDID(ctx, did) 160 + if err != nil { 161 + eventErrorCount.WithLabelValues("account").Inc() 162 + return fmt.Errorf("resolving identity: %w", err) 163 + } 164 + if ident == nil { 165 + eventErrorCount.WithLabelValues("account").Inc() 166 + return fmt.Errorf("identity not found for DID: %s", did.String()) 167 + } 168 + 169 + var am *AccountMeta 170 + if !eng.Config.SkipAccountMeta { 171 + am, err = eng.GetAccountMeta(ctx, ident) 172 + if err != nil { 173 + eventErrorCount.WithLabelValues("identity").Inc() 174 + return fmt.Errorf("failed to fetch account metadata: %w", err) 175 + } 176 + } else { 177 + am = &AccountMeta{ 178 + Identity: ident, 179 + Profile: ProfileSummary{}, 180 + } 181 + } 182 + ac := NewAccountContext(ctx, eng, *am) 183 + if err := eng.Rules.CallAccountRules(&ac); err != nil { 184 + eventErrorCount.WithLabelValues("account").Inc() 185 + return fmt.Errorf("rule execution failed: %w", err) 186 + } 187 + eng.CanonicalLogLineAccount(&ac) 188 + if err := eng.persistAccountModActions(&ac); err != nil { 189 + eventErrorCount.WithLabelValues("account").Inc() 190 + return fmt.Errorf("failed to persist actions for account event: %w", err) 191 + } 192 + if err := eng.persistCounters(ctx, ac.effects); err != nil { 193 + eventErrorCount.WithLabelValues("account").Inc() 194 + return fmt.Errorf("failed to persist counters for account event: %w", err) 117 195 } 118 196 return nil 119 197 }
+12
automod/engine/ruleset.go
··· 16 16 RecordRules []RecordRuleFunc 17 17 RecordDeleteRules []RecordRuleFunc 18 18 IdentityRules []IdentityRuleFunc 19 + AccountRules []AccountRuleFunc 19 20 BlobRules []BlobRuleFunc 20 21 NotificationRules []NotificationRuleFunc 21 22 OzoneEventRules []OzoneEventRuleFunc ··· 84 85 err := f(c) 85 86 if err != nil { 86 87 c.Logger.Error("identity rule execution failed", "err", err) 88 + } 89 + } 90 + return nil 91 + } 92 + 93 + // Executes rules for account update events. 94 + func (r *RuleSet) CallAccountRules(c *AccountContext) error { 95 + for _, f := range r.AccountRules { 96 + err := f(c) 97 + if err != nil { 98 + c.Logger.Error("account rule execution failed", "err", err) 87 99 } 88 100 } 89 101 return nil
+1
automod/engine/ruletypes.go
··· 6 6 ) 7 7 8 8 type IdentityRuleFunc = func(c *AccountContext) error 9 + type AccountRuleFunc = func(c *AccountContext) error 9 10 type RecordRuleFunc = func(c *RecordContext) error 10 11 type PostRuleFunc = func(c *RecordContext, post *appbsky.FeedPost) error 11 12 type ProfileRuleFunc = func(c *RecordContext, profile *appbsky.ActorProfile) error