this repo has no description
automod: quick fix to remove duplicate implementation of countstore. (#493)
authored by
Eric Myhre
and committed by
GitHub
5d2ac2e4
4ee97204
+19
-112
+2
-1
automod/action_dedupe_test.go
+2
-1
automod/action_dedupe_test.go
···
7
7
appbsky "github.com/bluesky-social/indigo/api/bsky"
8
8
"github.com/bluesky-social/indigo/atproto/identity"
9
9
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"github.com/bluesky-social/indigo/automod/countstore"
10
11
11
12
"github.com/stretchr/testify/assert"
12
13
)
···
39
40
assert.NoError(engine.ProcessRecord(ctx, id1.DID, path, cid1, &p1))
40
41
}
41
42
42
-
reports, err := engine.GetCount("automod-quota", "report", PeriodDay)
43
+
reports, err := engine.GetCount("automod-quota", "report", countstore.PeriodDay)
43
44
assert.NoError(err)
44
45
assert.Equal(1, reports)
45
46
}
+3
-2
automod/capture_test.go
+3
-2
automod/capture_test.go
···
3
3
import (
4
4
"testing"
5
5
6
+
"github.com/bluesky-social/indigo/automod/countstore"
6
7
"github.com/stretchr/testify/assert"
7
8
)
8
9
···
12
13
engine := EngineTestFixture()
13
14
capture := MustLoadCapture("testdata/capture_atprotocom.json")
14
15
assert.NoError(ProcessCaptureRules(&engine, capture))
15
-
c, err := engine.GetCount("automod-quota", "report", PeriodDay)
16
+
c, err := engine.GetCount("automod-quota", "report", countstore.PeriodDay)
16
17
assert.NoError(err)
17
18
assert.Equal(0, c)
18
-
c, err = engine.GetCount("automod-quota", "takedown", PeriodDay)
19
+
c, err = engine.GetCount("automod-quota", "takedown", countstore.PeriodDay)
19
20
assert.NoError(err)
20
21
assert.Equal(0, c)
21
22
}
+5
-4
automod/circuit_breaker_test.go
+5
-4
automod/circuit_breaker_test.go
···
8
8
appbsky "github.com/bluesky-social/indigo/api/bsky"
9
9
"github.com/bluesky-social/indigo/atproto/identity"
10
10
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
"github.com/bluesky-social/indigo/automod/countstore"
11
12
12
13
"github.com/stretchr/testify/assert"
13
14
)
···
49
50
assert.NoError(engine.ProcessRecord(ctx, ident.DID, path, cid1, &p1))
50
51
}
51
52
52
-
takedowns, err := engine.GetCount("automod-quota", "takedown", PeriodDay)
53
+
takedowns, err := engine.GetCount("automod-quota", "takedown", countstore.PeriodDay)
53
54
assert.NoError(err)
54
55
assert.Equal(QuotaModTakedownDay, takedowns)
55
56
56
-
reports, err := engine.GetCount("automod-quota", "report", PeriodDay)
57
+
reports, err := engine.GetCount("automod-quota", "report", countstore.PeriodDay)
57
58
assert.NoError(err)
58
59
assert.Equal(0, reports)
59
60
}
···
84
85
assert.NoError(engine.ProcessRecord(ctx, ident.DID, path, cid1, &p1))
85
86
}
86
87
87
-
takedowns, err := engine.GetCount("automod-quota", "takedown", PeriodDay)
88
+
takedowns, err := engine.GetCount("automod-quota", "takedown", countstore.PeriodDay)
88
89
assert.NoError(err)
89
90
assert.Equal(0, takedowns)
90
91
91
-
reports, err := engine.GetCount("automod-quota", "report", PeriodDay)
92
+
reports, err := engine.GetCount("automod-quota", "report", countstore.PeriodDay)
92
93
assert.NoError(err)
93
94
assert.Equal(QuotaModReportDay, reports)
94
95
}
-99
automod/countstore.go
-99
automod/countstore.go
···
1
-
package automod
2
-
3
-
import (
4
-
"context"
5
-
"fmt"
6
-
"log/slog"
7
-
"time"
8
-
)
9
-
10
-
const (
11
-
PeriodTotal = "total"
12
-
PeriodDay = "day"
13
-
PeriodHour = "hour"
14
-
)
15
-
16
-
type CountStore interface {
17
-
GetCount(ctx context.Context, name, val, period string) (int, error)
18
-
Increment(ctx context.Context, name, val string) error
19
-
IncrementPeriod(ctx context.Context, name, val, period string) error
20
-
// TODO: batch increment method
21
-
GetCountDistinct(ctx context.Context, name, bucket, period string) (int, error)
22
-
IncrementDistinct(ctx context.Context, name, bucket, val string) error
23
-
}
24
-
25
-
// TODO: this implementation isn't race-safe (yet)!
26
-
type MemCountStore struct {
27
-
Counts map[string]int
28
-
DistinctCounts map[string]map[string]bool
29
-
}
30
-
31
-
func NewMemCountStore() MemCountStore {
32
-
return MemCountStore{
33
-
Counts: make(map[string]int),
34
-
DistinctCounts: make(map[string]map[string]bool),
35
-
}
36
-
}
37
-
38
-
func PeriodBucket(name, val, period string) string {
39
-
switch period {
40
-
case PeriodTotal:
41
-
return fmt.Sprintf("%s/%s", name, val)
42
-
case PeriodDay:
43
-
t := time.Now().UTC().Format(time.DateOnly)
44
-
return fmt.Sprintf("%s/%s/%s", name, val, t)
45
-
case PeriodHour:
46
-
t := time.Now().UTC().Format(time.RFC3339)[0:13]
47
-
return fmt.Sprintf("%s/%s/%s", name, val, t)
48
-
default:
49
-
slog.Warn("unhandled counter period", "period", period)
50
-
return fmt.Sprintf("%s/%s", name, val)
51
-
}
52
-
}
53
-
54
-
func (s MemCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) {
55
-
v, ok := s.Counts[PeriodBucket(name, val, period)]
56
-
if !ok {
57
-
return 0, nil
58
-
}
59
-
return v, nil
60
-
}
61
-
62
-
func (s MemCountStore) Increment(ctx context.Context, name, val string) error {
63
-
for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} {
64
-
s.IncrementPeriod(ctx, name, val, p)
65
-
}
66
-
return nil
67
-
}
68
-
69
-
func (s MemCountStore) IncrementPeriod(ctx context.Context, name, val, period string) error {
70
-
k := PeriodBucket(name, val, period)
71
-
v, ok := s.Counts[k]
72
-
if !ok {
73
-
v = 0
74
-
}
75
-
v = v + 1
76
-
s.Counts[k] = v
77
-
return nil
78
-
}
79
-
80
-
func (s MemCountStore) GetCountDistinct(ctx context.Context, name, bucket, period string) (int, error) {
81
-
v, ok := s.DistinctCounts[PeriodBucket(name, bucket, period)]
82
-
if !ok {
83
-
return 0, nil
84
-
}
85
-
return len(v), nil
86
-
}
87
-
88
-
func (s MemCountStore) IncrementDistinct(ctx context.Context, name, bucket, val string) error {
89
-
for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} {
90
-
k := PeriodBucket(name, bucket, p)
91
-
m, ok := s.DistinctCounts[k]
92
-
if !ok {
93
-
m = make(map[string]bool)
94
-
}
95
-
m[val] = true
96
-
s.DistinctCounts[k] = m
97
-
}
98
-
return nil
99
-
}
+4
-3
automod/event.go
+4
-3
automod/event.go
···
10
10
comatproto "github.com/bluesky-social/indigo/api/atproto"
11
11
appbsky "github.com/bluesky-social/indigo/api/bsky"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/bluesky-social/indigo/automod/countstore"
13
14
"github.com/bluesky-social/indigo/xrpc"
14
15
)
15
16
···
205
206
newReports := []ModReport{}
206
207
for _, r := range reports {
207
208
counterName := "automod-account-report-" + reasonShortName(r.ReasonType)
208
-
existing := evt.GetCount(counterName, evt.Account.Identity.DID.String(), PeriodDay)
209
+
existing := evt.GetCount(counterName, evt.Account.Identity.DID.String(), countstore.PeriodDay)
209
210
if existing > 0 {
210
211
evt.Logger.Debug("skipping account report due to counter", "existing", existing, "reason", reasonShortName(r.ReasonType))
211
212
} else {
···
220
221
if len(reports) == 0 {
221
222
return []ModReport{}
222
223
}
223
-
if evt.GetCount("automod-quota", "report", PeriodDay) >= QuotaModReportDay {
224
+
if evt.GetCount("automod-quota", "report", countstore.PeriodDay) >= QuotaModReportDay {
224
225
evt.Logger.Warn("CIRCUIT BREAKER: automod reports")
225
226
return []ModReport{}
226
227
}
···
232
233
if !takedown {
233
234
return takedown
234
235
}
235
-
if evt.GetCount("automod-quota", "takedown", PeriodDay) >= QuotaModTakedownDay {
236
+
if evt.GetCount("automod-quota", "takedown", countstore.PeriodDay) >= QuotaModTakedownDay {
236
237
evt.Logger.Warn("CIRCUIT BREAKER: automod takedowns")
237
238
return false
238
239
}
+2
-1
automod/rules/mentions.go
+2
-1
automod/rules/mentions.go
···
3
3
import (
4
4
appbsky "github.com/bluesky-social/indigo/api/bsky"
5
5
"github.com/bluesky-social/indigo/automod"
6
+
"github.com/bluesky-social/indigo/automod/countstore"
6
7
)
7
8
8
9
var _ automod.PostRuleFunc = DistinctMentionsRule
···
30
31
if !newMentions {
31
32
return nil
32
33
}
33
-
if mentionHourlyThreshold <= evt.GetCountDistinct("mentions", did, automod.PeriodHour) {
34
+
if mentionHourlyThreshold <= evt.GetCountDistinct("mentions", did, countstore.PeriodHour) {
34
35
evt.AddAccountFlag("high-distinct-mentions")
35
36
}
36
37
+1
-1
automod/rules/replies.go
+1
-1
automod/rules/replies.go
···
44
44
}
45
45
46
46
// increment first. use a specific period (IncrementPeriod()) to reduce the number of counters (one per unique post text)
47
-
period := automod.PeriodDay
47
+
period := countstore.PeriodDay
48
48
bucket := evt.Account.Identity.DID.String() + "/" + HashOfString(post.Text)
49
49
evt.IncrementPeriod("reply-text", bucket, period)
50
50
+2
-1
automod/testing.go
+2
-1
automod/testing.go
···
11
11
appbsky "github.com/bluesky-social/indigo/api/bsky"
12
12
"github.com/bluesky-social/indigo/atproto/identity"
13
13
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/bluesky-social/indigo/automod/countstore"
14
15
)
15
16
16
17
func simpleRule(evt *RecordEvent, post *appbsky.FeedPost) error {
···
54
55
engine := Engine{
55
56
Logger: slog.Default(),
56
57
Directory: &dir,
57
-
Counters: NewMemCountStore(),
58
+
Counters: countstore.NewMemCountStore(),
58
59
Sets: sets,
59
60
Flags: flags,
60
61
Cache: cache,