Mirror of https://github.com/roostorg/coop
github.com/roostorg/coop
1import { readFileSync } from 'fs';
2import { dirname, join } from 'path';
3import fc from 'fast-check';
4import yaml from 'js-yaml';
5import _ from 'lodash';
6
7import { RuleAlarmStatus } from '../moderationConfigService/index.js';
8import getRuleAlarmStatus from './getRuleAlarmStatus.js';
9
10const { sum } = _;
11
12const sampleArbitrary = fc
13 .tuple(fc.nat(), fc.nat())
14 .map(([a, b]) => ({ passes: a, runs: a + b || 1 }));
15
16const samplesPassRate = (samples: { passes: number; runs: number }[]) =>
17 sum(samples.map((it) => it.passes)) / sum(samples.map((it) => it.runs));
18
19const __dirname = dirname(new URL(import.meta.url).pathname);
20const tableDump = yaml.load(
21 // eslint-disable-next-line security/detect-non-literal-fs-filename
22 readFileSync(
23 join(__dirname, '../../test/stubs/rule_pass_sample_data.yaml'),
24 'utf-8',
25 ),
26) as {
27 [ruleId: string]: { passes: number; runs: number; pass_rate: number }[];
28};
29
30describe('getRuleAlarmStatus', () => {
31 // Again, we don't really know what the 'right' behavior is, so these tests
32 // just verify some lose constrainsts as a sanity check on the current logic,
33 // and to prevent regressions.
34 test('should return false when no executions passed', () => {
35 // This could be a property test, but it doesn't really need to be.
36 const sampleData = [
37 { passes: 0, runs: 15783 },
38 { passes: 0, runs: 18528 },
39 { passes: 0, runs: 17253 },
40 { passes: 0, runs: 15372 },
41 { passes: 0, runs: 9759 },
42 ];
43
44 expect([RuleAlarmStatus.OK, RuleAlarmStatus.INSUFFICIENT_DATA]).toContain(
45 getRuleAlarmStatus(sampleData),
46 );
47 });
48
49 test('should return false if the pass rate in the most recent period is lower than the historical average', () => {
50 fc.assert(
51 fc.property(fc.array(sampleArbitrary, { minLength: 25 }), (samples) => {
52 // Construct a new sample with a mean just below the generated ones.
53 const samplePassRate = samplesPassRate(samples);
54
55 const latestPeriod = {
56 passes: 1,
57 runs: Math.ceil(1 / samplePassRate) + 1,
58 };
59
60 expect([
61 RuleAlarmStatus.OK,
62 RuleAlarmStatus.INSUFFICIENT_DATA,
63 ]).toContain(getRuleAlarmStatus([latestPeriod, ...samples]));
64 }),
65 );
66 });
67
68 test('should produce plausible results given sample data', () => {
69 const results = Object.fromEntries(
70 Object.entries(tableDump).map(([ruleId, ruleData]) => {
71 return [ruleId, getRuleAlarmStatus(ruleData)];
72 }),
73 );
74
75 // Everything from this table dump of real data isn't anomalous.
76 expect(results).toMatchInlineSnapshot(`
77 {
78 "060ba6f64ab": "OK",
79 "07b248e6c5b": "OK",
80 "0bec4897302": "OK",
81 "2bf679d4520": "OK",
82 "2fc6ec48b68": "OK",
83 "4fb36ec8fb0": "OK",
84 "67b4a7ff206": "OK",
85 "682bf679d45": "OK",
86 "772be50f82a": "OK",
87 "7b248e6c5bd": "OK",
88 "7b4a7ff2064": "OK",
89 "8060ba6f64a": "OK",
90 "82bf679d452": "OK",
91 "878060ba6f6": "OK",
92 "a0140eb5fa0": "OK",
93 "b1fd90d4b09": "OK",
94 "b4a7ff2064d": "OK",
95 "ba9fb0cf3f8": "OK",
96 "bec48973022": "OK",
97 "c682bf679d4": "OK",
98 "ce549bbaf40": "OK",
99 "e549bbaf40a": "OK",
100 "e6884fe7426": "OK",
101 }
102 `);
103 });
104
105 test('should report some anomalies if the pass rate is 1%', () => {
106 // The media pass rate across all rules in our sample data way, way under
107 // 1%, so this is a huge increase that should very often get flagged.
108 const results = Object.fromEntries(
109 Object.entries(tableDump).map(([ruleId, [lastPeriod, ...rest]]) => {
110 return [
111 ruleId,
112 [
113 // log true pass rate from before we modified it.
114 lastPeriod.passes / lastPeriod.runs,
115 getRuleAlarmStatus([
116 { ...lastPeriod, passes: Math.floor(lastPeriod.runs * 0.01) },
117 ...rest,
118 ]),
119 ],
120 ];
121 }),
122 );
123
124 expect(results).toMatchInlineSnapshot(`
125 {
126 "060ba6f64ab": [
127 0.0010648007301490721,
128 "ALARM",
129 ],
130 "07b248e6c5b": [
131 0.0019014298752662003,
132 "ALARM",
133 ],
134 "0bec4897302": [
135 0,
136 "ALARM",
137 ],
138 "2bf679d4520": [
139 0.000152114390021296,
140 "ALARM",
141 ],
142 "2fc6ec48b68": [
143 0,
144 "ALARM",
145 ],
146 "4fb36ec8fb0": [
147 0.0027380590203833284,
148 "ALARM",
149 ],
150 "67b4a7ff206": [
151 0.000076057195010648,
152 "ALARM",
153 ],
154 "682bf679d45": [
155 0.00007599939200486396,
156 "ALARM",
157 ],
158 "772be50f82a": [
159 0.000076057195010648,
160 "ALARM",
161 ],
162 "7b248e6c5bd": [
163 0.009507149376331,
164 "OK",
165 ],
166 "7b4a7ff2064": [
167 0.0038028597505324006,
168 "ALARM",
169 ],
170 "8060ba6f64a": [
171 0.0007605719501064801,
172 "ALARM",
173 ],
174 "82bf679d452": [
175 0.00022817158503194403,
176 "ALARM",
177 ],
178 "878060ba6f6": [
179 0.0063127471858837846,
180 "ALARM",
181 ],
182 "a0140eb5fa0": [
183 0,
184 "ALARM",
185 ],
186 "b1fd90d4b09": [
187 0,
188 "ALARM",
189 ],
190 "b4a7ff2064d": [
191 0.006991944064447485,
192 "ALARM",
193 ],
194 "ba9fb0cf3f8": [
195 0.001823985408116735,
196 "ALARM",
197 ],
198 "bec48973022": [
199 0.0007605719501064801,
200 "ALARM",
201 ],
202 "c682bf679d4": [
203 0,
204 "ALARM",
205 ],
206 "ce549bbaf40": [
207 0.0015972010952236082,
208 "ALARM",
209 ],
210 "e549bbaf40a": [
211 0.0006845147550958321,
212 "ALARM",
213 ],
214 "e6884fe7426": [
215 0,
216 "ALARM",
217 ],
218 }
219 `);
220 });
221});