this repo has no description
automod: fix automod account age (#713)
these are cleanups related to an earlier bug when rolling out account
creation timestamp handling in automod
+171
-82
+1
-1
automod/engine/account_meta.go
+1
-1
automod/engine/account_meta.go
+2
-2
automod/engine/engine_ozone.go
+2
-2
automod/engine/engine_ozone.go
···
110
110
return nil, err
111
111
}
112
112
if creatorIdent == nil {
113
-
return nil, fmt.Errorf("identity not found for DID: %s", evt.CreatedBy)
113
+
return nil, fmt.Errorf("identity not found for creator DID: %s", evt.CreatedBy)
114
114
}
115
115
creatorMeta, err := eng.GetAccountMeta(ctx, creatorIdent)
116
116
if err != nil {
···
122
122
return nil, err
123
123
}
124
124
if subjectIdent == nil {
125
-
return nil, fmt.Errorf("identity not found for DID: %s", evt.SubjectDID)
125
+
return nil, fmt.Errorf("identity not found for subject DID: %s", evt.SubjectDID)
126
126
}
127
127
accountMeta, err := eng.GetAccountMeta(ctx, subjectIdent)
128
128
if err != nil {
+27
-10
automod/engine/fetch_account_meta.go
+27
-10
automod/engine/fetch_account_meta.go
···
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
+
"errors"
6
7
"fmt"
8
+
"time"
7
9
8
10
comatproto "github.com/bluesky-social/indigo/api/atproto"
9
11
appbsky "github.com/bluesky-social/indigo/api/bsky"
10
12
toolsozone "github.com/bluesky-social/indigo/api/ozone"
11
13
"github.com/bluesky-social/indigo/atproto/identity"
12
14
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/bluesky-social/indigo/xrpc"
13
16
)
17
+
18
+
var newAccountRetryDuration = 3 * 1000 * time.Millisecond
14
19
15
20
// Helper to hydrate metadata about an account from several sources: PDS (if access), mod service (if access), public identity resolution
16
21
func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) {
···
56
61
57
62
// fetch account metadata from AppView
58
63
pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
64
+
// most common cause of this is a race between automod and ozone/appview for new accounts. just sleep a couple seconds and retry!
65
+
var xrpcError *xrpc.Error
66
+
if err != nil && errors.As(err, &xrpcError) && (xrpcError.StatusCode == 400 || xrpcError.StatusCode == 404) {
67
+
logger.Info("account profile lookup initially failed (from bsky appview), will retry", "err", err, "sleepDuration", newAccountRetryDuration)
68
+
time.Sleep(newAccountRetryDuration)
69
+
pv, err = appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String())
70
+
}
59
71
if err != nil {
60
-
logger.Warn("account profile lookup failed", "err", err)
72
+
logger.Warn("account profile lookup failed (from bsky appview)", "err", err)
61
73
return &am, nil
62
74
}
63
75
···
110
122
if rd.EmailConfirmedAt != nil && *rd.EmailConfirmedAt != "" {
111
123
ap.EmailConfirmed = true
112
124
}
113
-
ts, err := syntax.ParseDatetimeTime(rd.IndexedAt)
114
-
if err != nil {
115
-
return nil, fmt.Errorf("bad account IndexedAt: %w", err)
116
-
}
117
-
ap.IndexedAt = ts
125
+
// TODO: ozone doesn't really return good account "created at", just just leave that field nil
126
+
ap.IndexedAt = nil
118
127
if rd.DeactivatedAt != nil {
119
128
am.Deactivated = true
120
129
}
···
126
135
}
127
136
am.Private = &ap
128
137
}
129
-
} else if e.AdminClient != nil {
130
-
// fall back to PDS/entryway fetching; less metadata available
138
+
}
139
+
// fall back to PDS/entryway fetching; less metadata available
140
+
if am.Private == nil && e.AdminClient != nil {
131
141
pv, err := comatproto.AdminGetAccountInfo(ctx, e.AdminClient, ident.DID.String())
132
142
if err != nil {
133
143
logger.Warn("failed to fetch private account metadata from PDS/entryway", "err", err)
···
141
151
}
142
152
ts, err := syntax.ParseDatetimeTime(pv.IndexedAt)
143
153
if err != nil {
144
-
return nil, fmt.Errorf("bad account IndexedAt: %w", err)
154
+
return nil, fmt.Errorf("bad entryway account IndexedAt: %w", err)
145
155
}
146
-
ap.IndexedAt = ts
156
+
ap.IndexedAt = &ts
147
157
am.Private = &ap
158
+
if am.CreatedAt == nil {
159
+
am.CreatedAt = &ts
160
+
}
148
161
}
162
+
}
163
+
164
+
if am.CreatedAt == nil {
165
+
logger.Warn("account metadata missing CreatedAt time")
149
166
}
150
167
151
168
val, err := json.Marshal(&am)
+1
-1
automod/engine/persist.go
+1
-1
automod/engine/persist.go
···
170
170
rv, err := toolsozone.ModerationGetRecord(ctx, eng.OzoneClient, c.RecordOp.CID.String(), c.RecordOp.ATURI().String())
171
171
if err != nil {
172
172
// NOTE: there is a frequent 4xx error here from Ozone because this record has not been indexed yet
173
-
c.Logger.Warn("failed to fetch private record metadata", "err", err)
173
+
c.Logger.Warn("failed to fetch private record metadata from Ozone", "err", err)
174
174
} else {
175
175
var existingLabels []string
176
176
var negLabels []string
+2
automod/pkg.go
+2
automod/pkg.go
+25
-13
automod/rules/harassment.go
+25
-13
automod/rules/harassment.go
···
14
14
15
15
// looks for new accounts, which interact with frequently-harassed accounts, and report them for review
16
16
func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
17
-
if c.Account.Private == nil || c.Account.Identity == nil {
18
-
return nil
19
-
}
20
-
// TODO: helper for account age; and use public info for this (not private)
21
-
age := time.Since(c.Account.Private.IndexedAt)
22
-
if age > 7*24*time.Hour {
17
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) {
23
18
return nil
24
19
}
25
20
···
40
35
}
41
36
interactionDIDs = append(interactionDIDs, parentURI.Authority().String())
42
37
}
43
-
// TODO: quote-posts; any other interactions?
38
+
// quote posts
39
+
if post.Embed != nil && post.Embed.EmbedRecord != nil && post.Embed.EmbedRecord.Record != nil {
40
+
uri, err := syntax.ParseATURI(post.Embed.EmbedRecord.Record.Uri)
41
+
if err != nil {
42
+
c.Logger.Warn("invalid AT-URI in post embed record (quote-post)", "uri", post.Embed.EmbedRecord.Record.Uri)
43
+
} else {
44
+
interactionDIDs = append(interactionDIDs, uri.Authority().String())
45
+
}
46
+
}
44
47
if len(interactionDIDs) == 0 {
45
48
return nil
46
49
}
47
50
51
+
// more than a handful of followers or posts from author account? skip
52
+
if c.Account.FollowersCount > 10 || c.Account.PostsCount > 10 {
53
+
return nil
54
+
}
55
+
postCount := c.GetCount("post", c.Account.Identity.DID.String(), countstore.PeriodTotal)
56
+
if postCount > 20 {
57
+
return nil
58
+
}
59
+
48
60
interactionDIDs = dedupeStrings(interactionDIDs)
49
61
for _, d := range interactionDIDs {
50
62
did, err := syntax.ParseDID(d)
···
83
95
}
84
96
85
97
//c.AddRecordFlag("interaction-harassed-target")
98
+
var privCreatedAt *time.Time
99
+
if c.Account.Private != nil && c.Account.Private.IndexedAt != nil {
100
+
privCreatedAt = c.Account.Private.IndexedAt
101
+
}
102
+
c.Logger.Warn("possible harassment", "targetDID", did, "author", c.Account.Identity.DID, "accountCreated", c.Account.CreatedAt, "privateAccountCreated", privCreatedAt)
86
103
c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("possible harassment of known target account: %s (also labeled; remove label if this isn't harassment)", did))
87
104
c.AddAccountLabel("!hide")
88
105
c.Notify("slack")
···
95
112
96
113
// looks for new accounts, which frequently post the same type of content
97
114
func HarassmentTrivialPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
98
-
if c.Account.Private == nil || c.Account.Identity == nil {
99
-
return nil
100
-
}
101
-
// TODO: helper for account age; and use public info for this (not private)
102
-
age := time.Since(c.Account.Private.IndexedAt)
103
-
if age > 7*24*time.Hour {
115
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) {
104
116
return nil
105
117
}
106
118
+43
automod/rules/helpers.go
+43
automod/rules/helpers.go
···
3
3
import (
4
4
"fmt"
5
5
"regexp"
6
+
"time"
6
7
7
8
appbsky "github.com/bluesky-social/indigo/api/bsky"
8
9
"github.com/bluesky-social/indigo/atproto/syntax"
···
240
241
}
241
242
return false
242
243
}
244
+
245
+
// no accounts exist before this time
246
+
var atprotoAccountEpoch = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
247
+
248
+
// returns true if account creation timestamp is plausible: not-nil, not in distant past, not in the future
249
+
func plausibleAccountCreation(when *time.Time) bool {
250
+
if when == nil {
251
+
return false
252
+
}
253
+
// this is mostly to check for misconfigurations or null values (eg, UNIX epoch zero means "unknown" not actually 1970)
254
+
if !when.After(atprotoAccountEpoch) {
255
+
return false
256
+
}
257
+
// a timestamp in the future would also indicate some misconfiguration
258
+
if when.After(time.Now().Add(time.Hour)) {
259
+
return false
260
+
}
261
+
return true
262
+
}
263
+
264
+
// checks if account was created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false'
265
+
func AccountIsYoungerThan(c *automod.AccountContext, age time.Duration) bool {
266
+
// TODO: consider swapping priority order here (and below)
267
+
if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) {
268
+
return time.Since(*c.Account.CreatedAt) < age
269
+
}
270
+
if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) {
271
+
return time.Since(*c.Account.Private.IndexedAt) < age
272
+
}
273
+
return false
274
+
}
275
+
276
+
// checks if account was *not* created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false'
277
+
func AccountIsOlderThan(c *automod.AccountContext, age time.Duration) bool {
278
+
if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) {
279
+
return time.Since(*c.Account.CreatedAt) >= age
280
+
}
281
+
if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) {
282
+
return time.Since(*c.Account.Private.IndexedAt) >= age
283
+
}
284
+
return false
285
+
}
+54
automod/rules/helpers_test.go
+54
automod/rules/helpers_test.go
···
2
2
3
3
import (
4
4
"testing"
5
+
"time"
5
6
7
+
"github.com/bluesky-social/indigo/atproto/identity"
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
"github.com/bluesky-social/indigo/automod"
6
10
"github.com/bluesky-social/indigo/automod/keyword"
7
11
"github.com/stretchr/testify/assert"
8
12
)
···
61
65
// hashing function should be consistent over time
62
66
assert.Equal("4e6f69c0e3d10992", HashOfString("dummy-value"))
63
67
}
68
+
69
+
func TestAccountIsYoungerThan(t *testing.T) {
70
+
assert := assert.New(t)
71
+
72
+
am := automod.AccountMeta{
73
+
Identity: &identity.Identity{
74
+
DID: syntax.DID("did:plc:abc111"),
75
+
Handle: syntax.Handle("handle.example.com"),
76
+
},
77
+
Profile: automod.ProfileSummary{},
78
+
Private: nil,
79
+
}
80
+
now := time.Now()
81
+
ac := automod.AccountContext{
82
+
Account: am,
83
+
}
84
+
assert.False(AccountIsYoungerThan(&ac, time.Hour))
85
+
assert.False(AccountIsOlderThan(&ac, time.Hour))
86
+
87
+
ac.Account.CreatedAt = &now
88
+
assert.True(AccountIsYoungerThan(&ac, time.Hour))
89
+
assert.False(AccountIsOlderThan(&ac, time.Hour))
90
+
91
+
yesterday := time.Now().Add(-1 * time.Hour * 24)
92
+
ac.Account.CreatedAt = &yesterday
93
+
assert.False(AccountIsYoungerThan(&ac, time.Hour))
94
+
assert.True(AccountIsOlderThan(&ac, time.Hour))
95
+
96
+
old := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
97
+
ac.Account.CreatedAt = &old
98
+
assert.False(AccountIsYoungerThan(&ac, time.Hour))
99
+
assert.False(AccountIsYoungerThan(&ac, time.Hour*24*365*100))
100
+
assert.False(AccountIsOlderThan(&ac, time.Hour))
101
+
assert.False(AccountIsOlderThan(&ac, time.Hour*24*365*100))
102
+
103
+
future := time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC)
104
+
ac.Account.CreatedAt = &future
105
+
assert.False(AccountIsYoungerThan(&ac, time.Hour))
106
+
assert.False(AccountIsOlderThan(&ac, time.Hour))
107
+
108
+
ac.Account.CreatedAt = nil
109
+
ac.Account.Private = &automod.AccountPrivate{
110
+
Email: "account@example.com",
111
+
IndexedAt: &yesterday,
112
+
}
113
+
assert.True(AccountIsYoungerThan(&ac, 48*time.Hour))
114
+
assert.False(AccountIsYoungerThan(&ac, time.Hour))
115
+
assert.True(AccountIsOlderThan(&ac, time.Hour))
116
+
assert.False(AccountIsOlderThan(&ac, 48*time.Hour))
117
+
}
+1
-6
automod/rules/identity.go
+1
-6
automod/rules/identity.go
···
11
11
12
12
// triggers on first identity event for an account (DID)
13
13
func NewAccountRule(c *automod.AccountContext) error {
14
-
// need access to IndexedAt for this rule
15
-
if c.Account.Private == nil || c.Account.Identity == nil {
14
+
if c.Account.Identity == nil || !AccountIsYoungerThan(c, 4*time.Hour) {
16
15
return nil
17
16
}
18
17
19
18
did := c.Account.Identity.DID.String()
20
-
age := time.Since(c.Account.Private.IndexedAt)
21
-
if age > 4*time.Hour {
22
-
return nil
23
-
}
24
19
exists := c.GetCount("acct/exists", did, countstore.PeriodTotal)
25
20
if exists == 0 {
26
21
c.Logger.Info("new account")
+2
-7
automod/rules/mentions.go
+2
-7
automod/rules/mentions.go
···
47
47
var _ automod.PostRuleFunc = YoungAccountDistinctMentionsRule
48
48
49
49
func YoungAccountDistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
50
-
51
-
// only young posting accounts
52
-
if c.Account.Private != nil {
53
-
age := time.Since(c.Account.Private.IndexedAt)
54
-
if age > 2*7*24*time.Hour {
55
-
return nil
56
-
}
50
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) {
51
+
return nil
57
52
}
58
53
59
54
// parse out all the mentions
+1
-10
automod/rules/nostr.go
+1
-10
automod/rules/nostr.go
···
13
13
14
14
// looks for new accounts, which frequently post the same type of content
15
15
func NostrSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
16
-
if c.Account.Identity == nil {
16
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) {
17
17
return nil
18
-
}
19
-
20
-
// often don't have private metadata for these accounts right after creation
21
-
if c.Account.Private != nil {
22
-
// TODO: helper for account age; and use public info for this (not private)
23
-
age := time.Since(c.Account.Private.IndexedAt)
24
-
if age > 2*24*time.Hour {
25
-
return nil
26
-
}
27
18
}
28
19
29
20
// is this a bridged nostr account? if not, bail out
+1
-6
automod/rules/promo.go
+1
-6
automod/rules/promo.go
···
17
17
//
18
18
// this rule depends on ReplyCountPostRule() to set counts
19
19
func AggressivePromotionRule(c *automod.RecordContext, post *appbsky.FeedPost) error {
20
-
if c.Account.Private == nil || c.Account.Identity == nil {
21
-
return nil
22
-
}
23
-
// TODO: helper for account age
24
-
age := time.Since(c.Account.Private.IndexedAt)
25
-
if age > 7*24*time.Hour {
20
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) {
26
21
return nil
27
22
}
28
23
if post.Reply == nil || IsSelfThread(c, post) {
+1
-7
automod/rules/quick.go
+1
-7
automod/rules/quick.go
···
46
46
var _ automod.IdentityRuleFunc = NewAccountBotEmailRule
47
47
48
48
func NewAccountBotEmailRule(c *automod.AccountContext) error {
49
-
// need access to IndexedAt for this rule
50
-
if c.Account.Private == nil || c.Account.Identity == nil {
51
-
return nil
52
-
}
53
-
54
-
age := time.Since(c.Account.Private.IndexedAt)
55
-
if age > 1*time.Hour {
49
+
if c.Account.Identity == nil || !AccountIsYoungerThan(c, 1*time.Hour) {
56
50
return nil
57
51
}
58
52
+4
-10
automod/rules/replies.go
+4
-10
automod/rules/replies.go
···
52
52
if utf8.RuneCountInString(post.Text) <= 10 {
53
53
return nil
54
54
}
55
-
if c.Account.Private != nil {
56
-
age := time.Since(c.Account.Private.IndexedAt)
57
-
if age > 2*7*24*time.Hour {
58
-
return nil
59
-
}
55
+
if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) {
56
+
return nil
60
57
}
61
58
62
59
// don't count if there is a follow-back relationship
···
92
89
if utf8.RuneCountInString(post.Text) <= 10 {
93
90
return nil
94
91
}
95
-
if c.Account.Private != nil {
96
-
age := time.Since(c.Account.Private.IndexedAt)
97
-
if age > 2*7*24*time.Hour {
98
-
return nil
99
-
}
92
+
if AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) {
93
+
return nil
100
94
}
101
95
102
96
// don't count if there is a follow-back relationship
+4
-7
automod/rules/reposts.go
+4
-7
automod/rules/reposts.go
···
17
17
18
18
// looks for accounts which do frequent reposts
19
19
func TooManyRepostRule(c *automod.RecordContext) error {
20
+
// Don't bother checking reposts from accounts older than 30 days
21
+
if c.Account.Identity == nil || !AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) {
22
+
return nil
23
+
}
20
24
21
25
did := c.Account.Identity.DID.String()
22
-
// Don't bother checking reposts from accounts older than 30 days
23
-
if c.Account.Private != nil {
24
-
age := time.Since(c.Account.Private.IndexedAt)
25
-
if age > 30*24*time.Hour {
26
-
return nil
27
-
}
28
-
}
29
26
30
27
// Special case for newsmast bridge feeds
31
28
handle := c.Account.Identity.Handle.String()
+2
-2
cmd/hepa/server.go
+2
-2
cmd/hepa/server.go
···
147
147
}
148
148
counters = cnt
149
149
150
-
csh, err := cachestore.NewRedisCacheStore(config.RedisURL, 30*time.Minute)
150
+
csh, err := cachestore.NewRedisCacheStore(config.RedisURL, 6*time.Hour)
151
151
if err != nil {
152
152
return nil, fmt.Errorf("initializing redis cachestore: %v", err)
153
153
}
···
160
160
flags = flg
161
161
} else {
162
162
counters = countstore.NewMemCountStore()
163
-
cache = cachestore.NewMemCacheStore(5_000, 30*time.Minute)
163
+
cache = cachestore.NewMemCacheStore(5_000, 1*time.Hour)
164
164
flags = flagstore.NewMemFlagStore()
165
165
}
166
166