Webhooks for the AT Protocol
airglow.run
atproto
atprotocol
automation
webhook
feat: events rate limiting
+1166
-9
+45
app/routes/api/automations/[rkey].test.ts
+45
app/routes/api/automations/[rkey].test.ts
···
211
211
expect(action.verified).toBe(true);
212
212
});
213
213
214
+
it("re-enabling a rate-limited automation clears disabled fields and resets the window", async () => {
215
+
// Simulate a prior auto-disable: inactive with a rate-limit reason.
216
+
const disabledAt = new Date("2024-01-01T00:00:00.000Z");
217
+
await db
218
+
.update(automations)
219
+
.set({
220
+
active: false,
221
+
disabledReason: "rate_limit:second",
222
+
disabledAt,
223
+
rateLimitResetAt: null,
224
+
})
225
+
.where(eq(automations.uri, TEST_AUTO.uri));
226
+
227
+
const before = Date.now();
228
+
const res = await app.request(patchReq("rk1", { active: true }));
229
+
expect(res.status).toBe(200);
230
+
const after = Date.now();
231
+
232
+
const auto = await db.query.automations.findFirst();
233
+
expect(auto!.active).toBe(true);
234
+
expect(auto!.disabledReason).toBeNull();
235
+
expect(auto!.disabledAt).toBeNull();
236
+
// rateLimitResetAt is set to the re-enable moment so old log rows drop
237
+
// out of the rolling-window count.
238
+
expect(auto!.rateLimitResetAt).not.toBeNull();
239
+
const resetMs = auto!.rateLimitResetAt!.getTime();
240
+
expect(resetMs).toBeGreaterThanOrEqual(before);
241
+
expect(resetMs).toBeLessThanOrEqual(after);
242
+
});
243
+
244
+
it("editing an already-active automation does not clear the rate-limit reset marker", async () => {
245
+
// Pretend the user re-enabled once already: rateLimitResetAt is set.
246
+
const priorReset = new Date("2024-06-01T00:00:00.000Z");
247
+
await db
248
+
.update(automations)
249
+
.set({ rateLimitResetAt: priorReset })
250
+
.where(eq(automations.uri, TEST_AUTO.uri));
251
+
252
+
const res = await app.request(patchReq("rk1", { name: "Renamed" }));
253
+
expect(res.status).toBe(200);
254
+
255
+
const auto = await db.query.automations.findFirst();
256
+
expect(auto!.rateLimitResetAt?.getTime()).toBe(priorReset.getTime());
257
+
});
258
+
214
259
it("reactivation succeeds even when verification fails", async () => {
215
260
await db.update(automations).set({ active: false }).where(eq(automations.uri, TEST_AUTO.uri));
216
261
+5
-1
app/routes/api/automations/[rkey].ts
+5
-1
app/routes/api/automations/[rkey].ts
···
502
502
return c.json({ error: "Failed to update automation on PDS" }, 502);
503
503
}
504
504
505
-
// Update local index
505
+
// Update local index. Re-enabling clears any prior auto-disable reason so
506
+
// the dashboard stops showing the "rate-limited" state and a fresh rolling
507
+
// window starts counting forward from here.
506
508
const now = new Date();
509
+
const reEnabling = active && !auto.active;
507
510
await db
508
511
.update(automations)
509
512
.set({
···
517
520
active,
518
521
dryRun,
519
522
indexedAt: now,
523
+
...(reEnabling ? { disabledReason: null, disabledAt: null, rateLimitResetAt: now } : {}),
520
524
})
521
525
.where(eq(automations.uri, auto.uri));
522
526
+54
-4
app/routes/dashboard/automations/[rkey].tsx
+54
-4
app/routes/dashboard/automations/[rkey].tsx
···
1
1
import { createRoute } from "honox/factory";
2
2
import { eq, and, desc } from "drizzle-orm";
3
-
import { ArrowLeft, Copy, ExternalLink, Pencil, Filter, Database, Zap } from "../../../icons.js";
3
+
import {
4
+
ArrowLeft,
5
+
Copy,
6
+
ExternalLink,
7
+
Pencil,
8
+
Filter,
9
+
Database,
10
+
Zap,
11
+
Activity,
12
+
} from "../../../icons.js";
13
+
import { getRateLimitCounts } from "@/jetstream/rate-limit.js";
4
14
import {
5
15
opLabels,
6
16
actionTypeLabels,
···
65
75
});
66
76
const hasMore = rawLogs.length > 50;
67
77
const logs = rawLogs.slice(0, 50);
78
+
79
+
const rateLimitCounts = await getRateLimitCounts(auto);
80
+
const rateLimited = auto.disabledReason?.startsWith("rate_limit") ?? false;
68
81
69
82
const needsScopeUpgrade = !scopeCoversActions(user.scope, auto.actions);
70
83
const upgradeScope = needsScopeUpgrade ? computeRequiredScope(auto.actions) : null;
···
81
94
<ArrowLeft size={14} /> Back
82
95
</Button>
83
96
<span data-automation-status>
84
-
<Badge variant={!auto.active ? "neutral" : auto.dryRun ? "warning" : "success"}>
85
-
{!auto.active ? "Inactive" : auto.dryRun ? "Dry Run" : "Active"}
86
-
</Badge>
97
+
{rateLimited ? (
98
+
<Badge variant="error">Rate limited</Badge>
99
+
) : (
100
+
<Badge variant={!auto.active ? "neutral" : auto.dryRun ? "warning" : "success"}>
101
+
{!auto.active ? "Inactive" : auto.dryRun ? "Dry Run" : "Active"}
102
+
</Badge>
103
+
)}
87
104
</span>
88
105
<Button href={`/dashboard/automations/${rkey}/edit`} variant="secondary" size="sm">
89
106
<Pencil size={14} /> Edit
···
111
128
</Alert>
112
129
)}
113
130
131
+
{rateLimited && (
132
+
<Alert variant="error">
133
+
This automation was auto-disabled after exceeding its rate limit
134
+
{auto.disabledAt && ` on ${auto.disabledAt.toLocaleString()}`}. Edit it to re-enable —
135
+
the rolling window will reset.
136
+
</Alert>
137
+
)}
138
+
114
139
<Stack gap={6}>
115
140
<Card variant="flat">
116
141
<DescriptionList>
···
159
184
</dd>
160
185
</DescriptionList>
161
186
</details>
187
+
</Card>
188
+
189
+
<Card variant="flat">
190
+
<Stack gap={3}>
191
+
<h3 class={inlineCluster}>
192
+
<Activity size={18} /> Rate limits
193
+
</h3>
194
+
<ul class={plainList}>
195
+
{rateLimitCounts.map((rl) => {
196
+
const ratio = rl.limit === 0 ? 0 : rl.count / rl.limit;
197
+
const variant = ratio >= 1 ? "error" : ratio >= 0.8 ? "warning" : "neutral";
198
+
return (
199
+
<li key={rl.window} class={inlineCluster}>
200
+
<span>per {rl.window}</span>
201
+
<Badge variant={variant}>
202
+
{rl.count} / {rl.limit}
203
+
</Badge>
204
+
</li>
205
+
);
206
+
})}
207
+
</ul>
208
+
<p class={textMuted}>
209
+
Hitting any window auto-disables the automation. Dry-run fires do not count.
210
+
</p>
211
+
</Stack>
162
212
</Card>
163
213
164
214
{auto.conditions.length > 0 && (
+9
-3
app/routes/dashboard/index.tsx
+9
-3
app/routes/dashboard/index.tsx
···
99
99
))}
100
100
</td>
101
101
<td>
102
-
<Badge variant={!auto.active ? "neutral" : auto.dryRun ? "warning" : "success"}>
103
-
{!auto.active ? "Inactive" : auto.dryRun ? "Dry Run" : "Active"}
104
-
</Badge>
102
+
{!auto.active && auto.disabledReason?.startsWith("rate_limit") ? (
103
+
<Badge variant="error">Rate limited</Badge>
104
+
) : (
105
+
<Badge
106
+
variant={!auto.active ? "neutral" : auto.dryRun ? "warning" : "success"}
107
+
>
108
+
{!auto.active ? "Inactive" : auto.dryRun ? "Dry Run" : "Active"}
109
+
</Badge>
110
+
)}
105
111
</td>
106
112
<td>
107
113
<Button href={`/dashboard/automations/${auto.rkey}`} variant="ghost" size="sm">
+3
lib/db/migrations/0008_volatile_thundra.sql
+3
lib/db/migrations/0008_volatile_thundra.sql
+587
lib/db/migrations/meta/0008_snapshot.json
+587
lib/db/migrations/meta/0008_snapshot.json
···
1
+
{
2
+
"version": "6",
3
+
"dialect": "sqlite",
4
+
"id": "4db119a5-e24c-4466-8dda-8a454ba9918b",
5
+
"prevId": "be7ee96c-dd9b-4de3-b74e-0b9139455f3f",
6
+
"tables": {
7
+
"automations": {
8
+
"name": "automations",
9
+
"columns": {
10
+
"uri": {
11
+
"name": "uri",
12
+
"type": "text",
13
+
"primaryKey": true,
14
+
"notNull": true,
15
+
"autoincrement": false
16
+
},
17
+
"did": {
18
+
"name": "did",
19
+
"type": "text",
20
+
"primaryKey": false,
21
+
"notNull": true,
22
+
"autoincrement": false
23
+
},
24
+
"rkey": {
25
+
"name": "rkey",
26
+
"type": "text",
27
+
"primaryKey": false,
28
+
"notNull": true,
29
+
"autoincrement": false
30
+
},
31
+
"name": {
32
+
"name": "name",
33
+
"type": "text",
34
+
"primaryKey": false,
35
+
"notNull": true,
36
+
"autoincrement": false
37
+
},
38
+
"description": {
39
+
"name": "description",
40
+
"type": "text",
41
+
"primaryKey": false,
42
+
"notNull": false,
43
+
"autoincrement": false
44
+
},
45
+
"lexicon": {
46
+
"name": "lexicon",
47
+
"type": "text",
48
+
"primaryKey": false,
49
+
"notNull": true,
50
+
"autoincrement": false
51
+
},
52
+
"operation": {
53
+
"name": "operation",
54
+
"type": "text",
55
+
"primaryKey": false,
56
+
"notNull": true,
57
+
"autoincrement": false,
58
+
"default": "'[\"create\"]'"
59
+
},
60
+
"actions": {
61
+
"name": "actions",
62
+
"type": "text",
63
+
"primaryKey": false,
64
+
"notNull": true,
65
+
"autoincrement": false,
66
+
"default": "'[]'"
67
+
},
68
+
"fetches": {
69
+
"name": "fetches",
70
+
"type": "text",
71
+
"primaryKey": false,
72
+
"notNull": true,
73
+
"autoincrement": false,
74
+
"default": "'[]'"
75
+
},
76
+
"conditions": {
77
+
"name": "conditions",
78
+
"type": "text",
79
+
"primaryKey": false,
80
+
"notNull": true,
81
+
"autoincrement": false,
82
+
"default": "'[]'"
83
+
},
84
+
"wanted_dids": {
85
+
"name": "wanted_dids",
86
+
"type": "text",
87
+
"primaryKey": false,
88
+
"notNull": true,
89
+
"autoincrement": false,
90
+
"default": "'[]'"
91
+
},
92
+
"active": {
93
+
"name": "active",
94
+
"type": "integer",
95
+
"primaryKey": false,
96
+
"notNull": true,
97
+
"autoincrement": false,
98
+
"default": false
99
+
},
100
+
"dry_run": {
101
+
"name": "dry_run",
102
+
"type": "integer",
103
+
"primaryKey": false,
104
+
"notNull": true,
105
+
"autoincrement": false,
106
+
"default": false
107
+
},
108
+
"disabled_reason": {
109
+
"name": "disabled_reason",
110
+
"type": "text",
111
+
"primaryKey": false,
112
+
"notNull": false,
113
+
"autoincrement": false
114
+
},
115
+
"disabled_at": {
116
+
"name": "disabled_at",
117
+
"type": "integer",
118
+
"primaryKey": false,
119
+
"notNull": false,
120
+
"autoincrement": false
121
+
},
122
+
"rate_limit_reset_at": {
123
+
"name": "rate_limit_reset_at",
124
+
"type": "integer",
125
+
"primaryKey": false,
126
+
"notNull": false,
127
+
"autoincrement": false
128
+
},
129
+
"indexed_at": {
130
+
"name": "indexed_at",
131
+
"type": "integer",
132
+
"primaryKey": false,
133
+
"notNull": true,
134
+
"autoincrement": false
135
+
}
136
+
},
137
+
"indexes": {
138
+
"automations_did_idx": {
139
+
"name": "automations_did_idx",
140
+
"columns": [
141
+
"did"
142
+
],
143
+
"isUnique": false
144
+
},
145
+
"automations_active_indexed_at_idx": {
146
+
"name": "automations_active_indexed_at_idx",
147
+
"columns": [
148
+
"active",
149
+
"indexed_at"
150
+
],
151
+
"isUnique": false
152
+
}
153
+
},
154
+
"foreignKeys": {},
155
+
"compositePrimaryKeys": {},
156
+
"uniqueConstraints": {},
157
+
"checkConstraints": {}
158
+
},
159
+
"delivery_logs": {
160
+
"name": "delivery_logs",
161
+
"columns": {
162
+
"id": {
163
+
"name": "id",
164
+
"type": "integer",
165
+
"primaryKey": true,
166
+
"notNull": true,
167
+
"autoincrement": true
168
+
},
169
+
"automation_uri": {
170
+
"name": "automation_uri",
171
+
"type": "text",
172
+
"primaryKey": false,
173
+
"notNull": true,
174
+
"autoincrement": false
175
+
},
176
+
"action_index": {
177
+
"name": "action_index",
178
+
"type": "integer",
179
+
"primaryKey": false,
180
+
"notNull": true,
181
+
"autoincrement": false,
182
+
"default": 0
183
+
},
184
+
"event_time_us": {
185
+
"name": "event_time_us",
186
+
"type": "integer",
187
+
"primaryKey": false,
188
+
"notNull": true,
189
+
"autoincrement": false
190
+
},
191
+
"payload": {
192
+
"name": "payload",
193
+
"type": "text",
194
+
"primaryKey": false,
195
+
"notNull": false,
196
+
"autoincrement": false
197
+
},
198
+
"status_code": {
199
+
"name": "status_code",
200
+
"type": "integer",
201
+
"primaryKey": false,
202
+
"notNull": false,
203
+
"autoincrement": false
204
+
},
205
+
"message": {
206
+
"name": "message",
207
+
"type": "text",
208
+
"primaryKey": false,
209
+
"notNull": false,
210
+
"autoincrement": false
211
+
},
212
+
"error": {
213
+
"name": "error",
214
+
"type": "text",
215
+
"primaryKey": false,
216
+
"notNull": false,
217
+
"autoincrement": false
218
+
},
219
+
"dry_run": {
220
+
"name": "dry_run",
221
+
"type": "integer",
222
+
"primaryKey": false,
223
+
"notNull": true,
224
+
"autoincrement": false,
225
+
"default": false
226
+
},
227
+
"attempt": {
228
+
"name": "attempt",
229
+
"type": "integer",
230
+
"primaryKey": false,
231
+
"notNull": true,
232
+
"autoincrement": false,
233
+
"default": 1
234
+
},
235
+
"created_at": {
236
+
"name": "created_at",
237
+
"type": "integer",
238
+
"primaryKey": false,
239
+
"notNull": true,
240
+
"autoincrement": false
241
+
}
242
+
},
243
+
"indexes": {
244
+
"delivery_logs_automation_uri_id_idx": {
245
+
"name": "delivery_logs_automation_uri_id_idx",
246
+
"columns": [
247
+
"automation_uri",
248
+
"id"
249
+
],
250
+
"isUnique": false
251
+
}
252
+
},
253
+
"foreignKeys": {
254
+
"delivery_logs_automation_uri_automations_uri_fk": {
255
+
"name": "delivery_logs_automation_uri_automations_uri_fk",
256
+
"tableFrom": "delivery_logs",
257
+
"tableTo": "automations",
258
+
"columnsFrom": [
259
+
"automation_uri"
260
+
],
261
+
"columnsTo": [
262
+
"uri"
263
+
],
264
+
"onDelete": "cascade",
265
+
"onUpdate": "no action"
266
+
}
267
+
},
268
+
"compositePrimaryKeys": {},
269
+
"uniqueConstraints": {},
270
+
"checkConstraints": {}
271
+
},
272
+
"favicon_cache": {
273
+
"name": "favicon_cache",
274
+
"columns": {
275
+
"domain": {
276
+
"name": "domain",
277
+
"type": "text",
278
+
"primaryKey": true,
279
+
"notNull": true,
280
+
"autoincrement": false
281
+
},
282
+
"data": {
283
+
"name": "data",
284
+
"type": "text",
285
+
"primaryKey": false,
286
+
"notNull": true,
287
+
"autoincrement": false
288
+
},
289
+
"content_type": {
290
+
"name": "content_type",
291
+
"type": "text",
292
+
"primaryKey": false,
293
+
"notNull": true,
294
+
"autoincrement": false
295
+
},
296
+
"fetched_at": {
297
+
"name": "fetched_at",
298
+
"type": "integer",
299
+
"primaryKey": false,
300
+
"notNull": true,
301
+
"autoincrement": false
302
+
}
303
+
},
304
+
"indexes": {},
305
+
"foreignKeys": {},
306
+
"compositePrimaryKeys": {},
307
+
"uniqueConstraints": {},
308
+
"checkConstraints": {}
309
+
},
310
+
"lexicon_cache": {
311
+
"name": "lexicon_cache",
312
+
"columns": {
313
+
"nsid": {
314
+
"name": "nsid",
315
+
"type": "text",
316
+
"primaryKey": true,
317
+
"notNull": true,
318
+
"autoincrement": false
319
+
},
320
+
"schema": {
321
+
"name": "schema",
322
+
"type": "text",
323
+
"primaryKey": false,
324
+
"notNull": true,
325
+
"autoincrement": false
326
+
},
327
+
"fetched_at": {
328
+
"name": "fetched_at",
329
+
"type": "integer",
330
+
"primaryKey": false,
331
+
"notNull": true,
332
+
"autoincrement": false
333
+
}
334
+
},
335
+
"indexes": {},
336
+
"foreignKeys": {},
337
+
"compositePrimaryKeys": {},
338
+
"uniqueConstraints": {},
339
+
"checkConstraints": {}
340
+
},
341
+
"oauth_sessions": {
342
+
"name": "oauth_sessions",
343
+
"columns": {
344
+
"key": {
345
+
"name": "key",
346
+
"type": "text",
347
+
"primaryKey": true,
348
+
"notNull": true,
349
+
"autoincrement": false
350
+
},
351
+
"value": {
352
+
"name": "value",
353
+
"type": "text",
354
+
"primaryKey": false,
355
+
"notNull": true,
356
+
"autoincrement": false
357
+
},
358
+
"expires_at": {
359
+
"name": "expires_at",
360
+
"type": "integer",
361
+
"primaryKey": false,
362
+
"notNull": false,
363
+
"autoincrement": false
364
+
}
365
+
},
366
+
"indexes": {},
367
+
"foreignKeys": {},
368
+
"compositePrimaryKeys": {},
369
+
"uniqueConstraints": {},
370
+
"checkConstraints": {}
371
+
},
372
+
"oauth_states": {
373
+
"name": "oauth_states",
374
+
"columns": {
375
+
"key": {
376
+
"name": "key",
377
+
"type": "text",
378
+
"primaryKey": true,
379
+
"notNull": true,
380
+
"autoincrement": false
381
+
},
382
+
"value": {
383
+
"name": "value",
384
+
"type": "text",
385
+
"primaryKey": false,
386
+
"notNull": true,
387
+
"autoincrement": false
388
+
},
389
+
"expires_at": {
390
+
"name": "expires_at",
391
+
"type": "integer",
392
+
"primaryKey": false,
393
+
"notNull": false,
394
+
"autoincrement": false
395
+
}
396
+
},
397
+
"indexes": {},
398
+
"foreignKeys": {},
399
+
"compositePrimaryKeys": {},
400
+
"uniqueConstraints": {},
401
+
"checkConstraints": {}
402
+
},
403
+
"secret_events": {
404
+
"name": "secret_events",
405
+
"columns": {
406
+
"id": {
407
+
"name": "id",
408
+
"type": "integer",
409
+
"primaryKey": true,
410
+
"notNull": true,
411
+
"autoincrement": true
412
+
},
413
+
"did": {
414
+
"name": "did",
415
+
"type": "text",
416
+
"primaryKey": false,
417
+
"notNull": true,
418
+
"autoincrement": false
419
+
},
420
+
"name": {
421
+
"name": "name",
422
+
"type": "text",
423
+
"primaryKey": false,
424
+
"notNull": true,
425
+
"autoincrement": false
426
+
},
427
+
"action": {
428
+
"name": "action",
429
+
"type": "text",
430
+
"primaryKey": false,
431
+
"notNull": true,
432
+
"autoincrement": false
433
+
},
434
+
"created_at": {
435
+
"name": "created_at",
436
+
"type": "integer",
437
+
"primaryKey": false,
438
+
"notNull": true,
439
+
"autoincrement": false
440
+
}
441
+
},
442
+
"indexes": {},
443
+
"foreignKeys": {},
444
+
"compositePrimaryKeys": {},
445
+
"uniqueConstraints": {},
446
+
"checkConstraints": {}
447
+
},
448
+
"user_secrets": {
449
+
"name": "user_secrets",
450
+
"columns": {
451
+
"id": {
452
+
"name": "id",
453
+
"type": "integer",
454
+
"primaryKey": true,
455
+
"notNull": true,
456
+
"autoincrement": true
457
+
},
458
+
"did": {
459
+
"name": "did",
460
+
"type": "text",
461
+
"primaryKey": false,
462
+
"notNull": true,
463
+
"autoincrement": false
464
+
},
465
+
"name": {
466
+
"name": "name",
467
+
"type": "text",
468
+
"primaryKey": false,
469
+
"notNull": true,
470
+
"autoincrement": false
471
+
},
472
+
"encrypted_value": {
473
+
"name": "encrypted_value",
474
+
"type": "blob",
475
+
"primaryKey": false,
476
+
"notNull": true,
477
+
"autoincrement": false
478
+
},
479
+
"created_at": {
480
+
"name": "created_at",
481
+
"type": "integer",
482
+
"primaryKey": false,
483
+
"notNull": true,
484
+
"autoincrement": false
485
+
},
486
+
"updated_at": {
487
+
"name": "updated_at",
488
+
"type": "integer",
489
+
"primaryKey": false,
490
+
"notNull": true,
491
+
"autoincrement": false
492
+
}
493
+
},
494
+
"indexes": {
495
+
"user_secrets_did_name_unique": {
496
+
"name": "user_secrets_did_name_unique",
497
+
"columns": [
498
+
"did",
499
+
"name"
500
+
],
501
+
"isUnique": true
502
+
}
503
+
},
504
+
"foreignKeys": {
505
+
"user_secrets_did_users_did_fk": {
506
+
"name": "user_secrets_did_users_did_fk",
507
+
"tableFrom": "user_secrets",
508
+
"tableTo": "users",
509
+
"columnsFrom": [
510
+
"did"
511
+
],
512
+
"columnsTo": [
513
+
"did"
514
+
],
515
+
"onDelete": "cascade",
516
+
"onUpdate": "no action"
517
+
}
518
+
},
519
+
"compositePrimaryKeys": {},
520
+
"uniqueConstraints": {},
521
+
"checkConstraints": {}
522
+
},
523
+
"users": {
524
+
"name": "users",
525
+
"columns": {
526
+
"id": {
527
+
"name": "id",
528
+
"type": "integer",
529
+
"primaryKey": true,
530
+
"notNull": true,
531
+
"autoincrement": true
532
+
},
533
+
"did": {
534
+
"name": "did",
535
+
"type": "text",
536
+
"primaryKey": false,
537
+
"notNull": true,
538
+
"autoincrement": false
539
+
},
540
+
"handle": {
541
+
"name": "handle",
542
+
"type": "text",
543
+
"primaryKey": false,
544
+
"notNull": true,
545
+
"autoincrement": false
546
+
},
547
+
"scope": {
548
+
"name": "scope",
549
+
"type": "text",
550
+
"primaryKey": false,
551
+
"notNull": false,
552
+
"autoincrement": false
553
+
},
554
+
"created_at": {
555
+
"name": "created_at",
556
+
"type": "integer",
557
+
"primaryKey": false,
558
+
"notNull": true,
559
+
"autoincrement": false
560
+
}
561
+
},
562
+
"indexes": {
563
+
"users_did_unique": {
564
+
"name": "users_did_unique",
565
+
"columns": [
566
+
"did"
567
+
],
568
+
"isUnique": true
569
+
}
570
+
},
571
+
"foreignKeys": {},
572
+
"compositePrimaryKeys": {},
573
+
"uniqueConstraints": {},
574
+
"checkConstraints": {}
575
+
}
576
+
},
577
+
"views": {},
578
+
"enums": {},
579
+
"_meta": {
580
+
"schemas": {},
581
+
"tables": {},
582
+
"columns": {}
583
+
},
584
+
"internal": {
585
+
"indexes": {}
586
+
}
587
+
}
+7
lib/db/migrations/meta/_journal.json
+7
lib/db/migrations/meta/_journal.json
+9
lib/db/schema.ts
+9
lib/db/schema.ts
···
134
134
wantedDids: text("wanted_dids", { mode: "json" }).notNull().$type<string[]>().default([]),
135
135
active: integer("active", { mode: "boolean" }).notNull().default(false),
136
136
dryRun: integer("dry_run", { mode: "boolean" }).notNull().default(false),
137
+
// Populated when the engine auto-disables the automation (e.g. rate-limit
138
+
// breach). Cleared when the user re-enables. Format: "rate_limit:<window>".
139
+
disabledReason: text("disabled_reason"),
140
+
disabledAt: integer("disabled_at", { mode: "timestamp_ms" }),
141
+
// Cut-off for rate-limit counting. Delivery log rows before this timestamp
142
+
// are ignored by the rolling-window check. Set to the re-enable moment so
143
+
// a manual re-enable after a rate-limit trip doesn't instantly re-trip on
144
+
// the same logs. Null on automations that have never been rate-limited.
145
+
rateLimitResetAt: integer("rate_limit_reset_at", { mode: "timestamp_ms" }),
137
146
indexedAt: integer("indexed_at", { mode: "timestamp_ms" }).notNull(),
138
147
},
139
148
(table) => [
+97
lib/jetstream/handler.test.ts
+97
lib/jetstream/handler.test.ts
···
12
12
13
13
vi.mock("@/webhooks/dispatcher.js", () => ({
14
14
dispatch: vi.fn(),
15
+
buildPayload: vi.fn(() => ({ event: "dry-run" })),
15
16
}));
16
17
17
18
vi.mock("@/actions/executor.js", () => ({
···
28
29
29
30
vi.mock("@/actions/fetcher.js", () => ({
30
31
resolveFetches: vi.fn(),
32
+
}));
33
+
34
+
vi.mock("./rate-limit.js", () => ({
35
+
checkRateLimit: vi.fn().mockResolvedValue(null),
36
+
disableForRateLimit: vi.fn().mockResolvedValue(true),
37
+
}));
38
+
39
+
vi.mock("./consumer.js", () => ({
40
+
notifyAutomationChange: vi.fn(),
31
41
}));
32
42
33
43
import { handleMatchedEvent } from "./handler.js";
···
36
46
import { executeBskyPost } from "../actions/bsky-post.js";
37
47
import { executePatchRecord } from "../actions/patch-record.js";
38
48
import { resolveFetches } from "../actions/fetcher.js";
49
+
import { checkRateLimit, disableForRateLimit } from "./rate-limit.js";
50
+
import { notifyAutomationChange } from "./consumer.js";
39
51
import {
40
52
makeMatch,
41
53
makeWebhookAction,
···
57
69
const mockExecuteBskyPost = vi.mocked(executeBskyPost);
58
70
const mockExecutePatchRecord = vi.mocked(executePatchRecord);
59
71
const mockResolveFetches = vi.mocked(resolveFetches);
72
+
const mockCheckRateLimit = vi.mocked(checkRateLimit);
73
+
const mockDisableForRateLimit = vi.mocked(disableForRateLimit);
74
+
const mockNotifyAutomationChange = vi.mocked(notifyAutomationChange);
60
75
61
76
describe("handleMatchedEvent", () => {
62
77
beforeEach(() => {
···
65
80
mockExecuteBskyPost.mockReset().mockResolvedValue(okWithUri);
66
81
mockExecutePatchRecord.mockReset().mockResolvedValue(okWithUri);
67
82
mockResolveFetches.mockReset();
83
+
mockCheckRateLimit.mockReset().mockResolvedValue(null);
84
+
mockDisableForRateLimit.mockReset().mockResolvedValue(true);
85
+
mockNotifyAutomationChange.mockReset();
86
+
mockInsertValues.mockClear();
68
87
});
69
88
70
89
it("dispatches a webhook action", async () => {
···
306
325
// So we verify the record action result is present but webhook result is not
307
326
const finalCtx = mockExecuteAction.mock.calls[0]![2]!;
308
327
expect(finalCtx).not.toHaveProperty("action1");
328
+
});
329
+
330
+
describe("rate limiting", () => {
331
+
it("disables the automation and skips actions when a window is breached", async () => {
332
+
mockCheckRateLimit.mockResolvedValueOnce({ window: "second", count: 10, limit: 10 });
333
+
334
+
const match = makeMatch({
335
+
automation: { actions: [makeWebhookAction()], fetches: [] },
336
+
});
337
+
338
+
await handleMatchedEvent(match);
339
+
340
+
expect(mockCheckRateLimit).toHaveBeenCalledWith(match.automation);
341
+
expect(mockDisableForRateLimit).toHaveBeenCalledWith(match.automation.uri, {
342
+
window: "second",
343
+
count: 10,
344
+
limit: 10,
345
+
});
346
+
expect(mockNotifyAutomationChange).toHaveBeenCalledOnce();
347
+
expect(mockDispatch).not.toHaveBeenCalled();
348
+
expect(mockResolveFetches).not.toHaveBeenCalled();
349
+
350
+
// A delivery log entry was written explaining why the automation stopped.
351
+
expect(mockInsertValues).toHaveBeenCalledOnce();
352
+
const logged = mockInsertValues.mock.calls[0]![0] as { error: string; dryRun: boolean };
353
+
expect(logged.error).toContain("Rate limit exceeded");
354
+
expect(logged.error).toContain("second");
355
+
expect(logged.dryRun).toBe(false);
356
+
});
357
+
358
+
it("does not log or notify when another handler already disabled the automation", async () => {
359
+
// Concurrent path: checkRateLimit sees a breach (stale read from before
360
+
// the re-partition), but disableForRateLimit returns false because the
361
+
// first handler already flipped `active` to false. We must not spam the
362
+
// delivery log or re-notify.
363
+
mockCheckRateLimit.mockResolvedValueOnce({ window: "second", count: 10, limit: 10 });
364
+
mockDisableForRateLimit.mockResolvedValueOnce(false);
365
+
366
+
const match = makeMatch({
367
+
automation: { actions: [makeWebhookAction()], fetches: [] },
368
+
});
369
+
370
+
await handleMatchedEvent(match);
371
+
372
+
expect(mockDisableForRateLimit).toHaveBeenCalledOnce();
373
+
expect(mockNotifyAutomationChange).not.toHaveBeenCalled();
374
+
expect(mockInsertValues).not.toHaveBeenCalled();
375
+
expect(mockDispatch).not.toHaveBeenCalled();
376
+
});
377
+
378
+
it("does not rate-limit dry-run automations", async () => {
379
+
// The rate-limit mock would return a breach if consulted — but for
380
+
// dry-run it should never be called in the first place.
381
+
mockCheckRateLimit.mockResolvedValue({ window: "minute", count: 100, limit: 100 });
382
+
383
+
const match = makeMatch({
384
+
automation: { actions: [makeWebhookAction()], fetches: [], dryRun: true },
385
+
});
386
+
387
+
await handleMatchedEvent(match);
388
+
389
+
expect(mockCheckRateLimit).not.toHaveBeenCalled();
390
+
expect(mockDisableForRateLimit).not.toHaveBeenCalled();
391
+
});
392
+
393
+
it("runs actions normally when no window is breached", async () => {
394
+
mockCheckRateLimit.mockResolvedValueOnce(null);
395
+
396
+
const match = makeMatch({
397
+
automation: { actions: [makeWebhookAction()], fetches: [] },
398
+
});
399
+
400
+
await handleMatchedEvent(match);
401
+
402
+
expect(mockCheckRateLimit).toHaveBeenCalledOnce();
403
+
expect(mockDisableForRateLimit).not.toHaveBeenCalled();
404
+
expect(mockDispatch).toHaveBeenCalledOnce();
405
+
});
309
406
});
310
407
});
+46
-1
lib/jetstream/handler.ts
+46
-1
lib/jetstream/handler.ts
···
10
10
import { resolveFetches } from "../actions/fetcher.js";
11
11
import { renderTemplate, renderTextTemplate, type FetchContext } from "../actions/template.js";
12
12
import { parseAtUri } from "../pds/resolver.js";
13
-
import type { MatchedEvent } from "./consumer.js";
13
+
import { notifyAutomationChange, type MatchedEvent } from "./consumer.js";
14
+
import { checkRateLimit, disableForRateLimit, type RateLimitBreach } from "./rate-limit.js";
14
15
15
16
/** Handle a matched Jetstream event: resolve fetches, then dispatch all actions. */
16
17
export async function handleMatchedEvent(match: MatchedEvent) {
18
+
// Rate-limit gate. Runs before fetches so a breached automation stops
19
+
// spending PDS resources entirely. Dry-run fires never count toward limits
20
+
// (checkRateLimit filters them out), so dry-run automations pass freely.
21
+
// Skip the check when the automation is already in dry-run — no point
22
+
// disabling something that can't cause damage.
23
+
if (!match.automation.dryRun) {
24
+
const breach = await checkRateLimit(match.automation);
25
+
if (breach) {
26
+
await handleRateLimitBreach(match, breach);
27
+
return;
28
+
}
29
+
}
30
+
17
31
let fetchContext: FetchContext = {};
18
32
if (match.automation.fetches.length > 0) {
19
33
const result = await resolveFetches(
···
214
228
attempt: 1,
215
229
createdAt: new Date(),
216
230
});
231
+
}
232
+
233
+
/**
234
+
* Disable the automation, write a log entry so the user sees why, and notify
235
+
* the Jetstream manager to drop it from the subscription so no more events are
236
+
* routed to it.
237
+
*
238
+
* `disableForRateLimit` is guarded on `active = true`, so only the first of
239
+
* several concurrent breach handlers (in-flight events beating the re-partition)
240
+
* actually flips the row — only that caller writes the disable log entry and
241
+
* notifies. Later callers no-op silently. The log entry uses `dryRun: false`
242
+
* so it shows up in the real delivery log.
243
+
*/
244
+
async function handleRateLimitBreach(match: MatchedEvent, breach: RateLimitBreach) {
245
+
const msg = `Rate limit exceeded: ${breach.count} actions in last ${breach.window}. Automation disabled.`;
246
+
const justDisabled = await disableForRateLimit(match.automation.uri, breach);
247
+
if (!justDisabled) return;
248
+
console.warn(`[rate-limit] ${match.automation.uri}: ${msg}`);
249
+
await db.insert(deliveryLogs).values({
250
+
automationUri: match.automation.uri,
251
+
actionIndex: 0,
252
+
eventTimeUs: match.event.time_us,
253
+
payload: null,
254
+
statusCode: null,
255
+
message: null,
256
+
error: msg,
257
+
dryRun: false,
258
+
attempt: 1,
259
+
createdAt: new Date(),
260
+
});
261
+
notifyAutomationChange();
217
262
}
218
263
219
264
async function logDrySkip(match: MatchedEvent, skippedBy: string | undefined) {
+162
lib/jetstream/rate-limit.test.ts
+162
lib/jetstream/rate-limit.test.ts
···
1
+
import { describe, it, expect, vi, beforeEach } from "vitest";
2
+
import { eq } from "drizzle-orm";
3
+
4
+
vi.mock("@/config.js", () => ({
5
+
config: {
6
+
databasePath: ":memory:",
7
+
jetstreamUrl: "wss://jetstream.test/subscribe",
8
+
publicUrl: "https://airglow.test",
9
+
nsidAllowlist: [],
10
+
nsidBlocklist: [],
11
+
nsidRequireDids: [],
12
+
},
13
+
}));
14
+
15
+
vi.mock("@/db/index.js", async () => {
16
+
const { createTestDb } = await import("../test/db.js");
17
+
return { db: createTestDb() };
18
+
});
19
+
20
+
import {
21
+
checkRateLimit,
22
+
disableForRateLimit,
23
+
getRateLimitCounts,
24
+
RATE_LIMIT_WINDOWS,
25
+
} from "./rate-limit.js";
26
+
import { db } from "../db/index.js";
27
+
import { automations, deliveryLogs } from "../db/schema.js";
28
+
import { makeAutomation } from "../test/fixtures.js";
29
+
30
+
const URI = "at://did:plc:test/run.airglow.automation/rl";
31
+
const NOW = 1_800_000_000_000; // fixed reference "now"
32
+
33
+
async function seedAutomation() {
34
+
const auto = makeAutomation({ uri: URI, rkey: "rl" });
35
+
await db.insert(automations).values(auto);
36
+
}
37
+
38
+
/** Re-read the automation row so callers get a fresh `rateLimitResetAt` after
39
+
* updates. The rate-limit API takes the row directly (no hidden DB reads). */
40
+
async function loadAuto() {
41
+
const row = await db.query.automations.findFirst({
42
+
where: eq(automations.uri, URI),
43
+
});
44
+
if (!row) throw new Error("seedAutomation must be called first");
45
+
return row;
46
+
}
47
+
48
+
async function insertLogs(count: number, offsetsMs: number[], dryRun = false) {
49
+
const values = offsetsMs.slice(0, count).map((offset) => ({
50
+
automationUri: URI,
51
+
actionIndex: 0,
52
+
eventTimeUs: NOW * 1000,
53
+
dryRun,
54
+
attempt: 1,
55
+
createdAt: new Date(NOW - offset),
56
+
}));
57
+
if (values.length > 0) await db.insert(deliveryLogs).values(values);
58
+
}
59
+
60
+
describe("rate-limit", () => {
61
+
beforeEach(async () => {
62
+
await db.delete(deliveryLogs);
63
+
await db.delete(automations);
64
+
await seedAutomation();
65
+
});
66
+
67
+
it("returns null when well below every window limit", async () => {
68
+
await insertLogs(3, [100, 200, 300]);
69
+
const breach = await checkRateLimit(await loadAuto(), NOW);
70
+
expect(breach).toBeNull();
71
+
});
72
+
73
+
it("flags the second window when 10+ actions land within 1s", async () => {
74
+
// 10 logs within the last 900ms — at-or-above the 10/sec limit.
75
+
await insertLogs(
76
+
10,
77
+
Array.from({ length: 10 }, (_, i) => 100 + i * 80),
78
+
);
79
+
const breach = await checkRateLimit(await loadAuto(), NOW);
80
+
expect(breach).toEqual({ window: "second", count: 10, limit: 10 });
81
+
});
82
+
83
+
it("flags the minute window when 100 actions spread over 30s", async () => {
84
+
// Space them so only 1 lands in the last second (below 10), but 100 land
85
+
// in the last 30s (at the 100/min limit).
86
+
const offsets = Array.from({ length: 100 }, (_, i) => 1_500 + i * 300);
87
+
await insertLogs(100, offsets);
88
+
const breach = await checkRateLimit(await loadAuto(), NOW);
89
+
expect(breach).toEqual({ window: "minute", count: 100, limit: 100 });
90
+
});
91
+
92
+
it("flags the hour window when 500 actions spread over 50 minutes", async () => {
93
+
// Space them so neither sec nor min breaches but hour does.
94
+
const offsets = Array.from({ length: 500 }, (_, i) => 61_000 + i * 6_000);
95
+
await insertLogs(500, offsets);
96
+
const breach = await checkRateLimit(await loadAuto(), NOW);
97
+
expect(breach).toEqual({ window: "hour", count: 500, limit: 500 });
98
+
});
99
+
100
+
it("does not count dry-run fires toward the limit", async () => {
101
+
await insertLogs(
102
+
20,
103
+
Array.from({ length: 20 }, (_, i) => 100 + i * 40),
104
+
true,
105
+
);
106
+
const breach = await checkRateLimit(await loadAuto(), NOW);
107
+
expect(breach).toBeNull();
108
+
});
109
+
110
+
it("ignores log rows older than rateLimitResetAt", async () => {
111
+
// 10 old logs (from ~5s ago) that would breach the minute window if not
112
+
// filtered — but the reset cutoff was set 2s ago, so they're ignored.
113
+
await insertLogs(
114
+
10,
115
+
Array.from({ length: 10 }, (_, i) => 5_000 + i * 50),
116
+
);
117
+
await db
118
+
.update(automations)
119
+
.set({ rateLimitResetAt: new Date(NOW - 2_000) })
120
+
.where(eq(automations.uri, URI));
121
+
122
+
const breach = await checkRateLimit(await loadAuto(), NOW);
123
+
expect(breach).toBeNull();
124
+
});
125
+
126
+
it("disableForRateLimit flips active=false and returns true on transition", async () => {
127
+
const changed = await disableForRateLimit(URI, { window: "second", count: 10, limit: 10 }, NOW);
128
+
expect(changed).toBe(true);
129
+
130
+
const row = await loadAuto();
131
+
expect(row.active).toBe(false);
132
+
expect(row.disabledReason).toBe("rate_limit:second");
133
+
expect(row.disabledAt?.getTime()).toBe(NOW);
134
+
});
135
+
136
+
it("disableForRateLimit is a no-op when already inactive", async () => {
137
+
await disableForRateLimit(URI, { window: "second", count: 10, limit: 10 }, NOW);
138
+
// Second call: row is already active=false, WHERE clause filters it out.
139
+
const changed = await disableForRateLimit(
140
+
URI,
141
+
{ window: "minute", count: 100, limit: 100 },
142
+
NOW + 1_000,
143
+
);
144
+
expect(changed).toBe(false);
145
+
146
+
// The original reason + timestamp are preserved — not overwritten by the
147
+
// second caller.
148
+
const row = await loadAuto();
149
+
expect(row.disabledReason).toBe("rate_limit:second");
150
+
expect(row.disabledAt?.getTime()).toBe(NOW);
151
+
});
152
+
153
+
it("getRateLimitCounts returns one entry per window", async () => {
154
+
await insertLogs(3, [100, 30_000, 300_000]);
155
+
const counts = await getRateLimitCounts(await loadAuto(), NOW);
156
+
expect(counts).toHaveLength(RATE_LIMIT_WINDOWS.length);
157
+
expect(counts.map((c) => c.window)).toEqual(["second", "minute", "hour"]);
158
+
expect(counts[0]!.count).toBe(1); // only the 100ms-ago one
159
+
expect(counts[1]!.count).toBe(2); // 100ms and 30s ago
160
+
expect(counts[2]!.count).toBe(3); // all three within the hour
161
+
});
162
+
});
+142
lib/jetstream/rate-limit.ts
+142
lib/jetstream/rate-limit.ts
···
1
+
import { and, eq, gte } from "drizzle-orm";
2
+
import { db } from "../db/index.js";
3
+
import { automations, deliveryLogs } from "../db/schema.js";
4
+
5
+
/**
6
+
* Per-automation rate limits. Any window hitting its limit auto-disables the
7
+
* automation. Counts only real (non dry-run) action executions — dry-run fires
8
+
* log separately and never contribute to the limit.
9
+
*
10
+
* Tuned to catch runaway automations (infinite-loop mirrors, self-triggering
11
+
* actions) before they can melt the PDS, while leaving plenty of headroom for
12
+
* normal traffic on a per-user workflow.
13
+
*/
14
+
export type RateLimitWindow = {
15
+
name: "second" | "minute" | "hour";
16
+
/** Window length in milliseconds. */
17
+
ms: number;
18
+
/** Max action executions allowed within the window. */
19
+
limit: number;
20
+
};
21
+
22
+
export const RATE_LIMIT_WINDOWS: readonly RateLimitWindow[] = [
23
+
{ name: "second", ms: 1_000, limit: 10 },
24
+
{ name: "minute", ms: 60_000, limit: 100 },
25
+
{ name: "hour", ms: 3_600_000, limit: 500 },
26
+
];
27
+
28
+
const HOUR_WINDOW_MS = RATE_LIMIT_WINDOWS[RATE_LIMIT_WINDOWS.length - 1]!.ms;
29
+
30
+
export type RateLimitBreach = {
31
+
window: RateLimitWindow["name"];
32
+
count: number;
33
+
limit: number;
34
+
};
35
+
36
+
export type RateLimitCount = {
37
+
window: RateLimitWindow["name"];
38
+
count: number;
39
+
limit: number;
40
+
};
41
+
42
+
/** Minimal automation shape the rate limiter needs. Takes the row already
43
+
* loaded by the caller to avoid a redundant DB round-trip on the hot path. */
44
+
export type RateLimitAutomation = {
45
+
uri: string;
46
+
rateLimitResetAt: Date | null;
47
+
};
48
+
49
+
/** Timestamps of recent (non dry-run) log rows. Honours `rateLimitResetAt` so
50
+
* a manual re-enable after an auto-disable starts the counters from zero
51
+
* instead of immediately re-tripping on the same logs. */
52
+
async function loadRecentTimestamps(
53
+
automation: RateLimitAutomation,
54
+
now: number,
55
+
): Promise<number[]> {
56
+
const windowCutoff = now - HOUR_WINDOW_MS;
57
+
const resetCutoff = automation.rateLimitResetAt?.getTime() ?? 0;
58
+
const cutoff = new Date(Math.max(windowCutoff, resetCutoff));
59
+
const rows = await db
60
+
.select({ createdAt: deliveryLogs.createdAt })
61
+
.from(deliveryLogs)
62
+
.where(
63
+
and(
64
+
eq(deliveryLogs.automationUri, automation.uri),
65
+
eq(deliveryLogs.dryRun, false),
66
+
gte(deliveryLogs.createdAt, cutoff),
67
+
),
68
+
);
69
+
return rows.map((r) => r.createdAt.getTime());
70
+
}
71
+
72
+
function countWithin(timestamps: number[], now: number, ms: number): number {
73
+
const cutoff = now - ms;
74
+
let count = 0;
75
+
for (const t of timestamps) {
76
+
if (t >= cutoff) count++;
77
+
}
78
+
return count;
79
+
}
80
+
81
+
/**
82
+
* Check every window. Returns the first (shortest) window whose count is at or
83
+
* above its limit, or `null` if the automation is within all limits.
84
+
*
85
+
* "At or above" means the *next* action would push past — the current in-flight
86
+
* fire is the last one allowed before we disable. Callers check this *before*
87
+
* running actions, so a breach short-circuits the fire.
88
+
*
89
+
* Two near-simultaneous events can both see count=limit-1 and both execute;
90
+
* acceptable slack for the "catch runaway loops" use case.
91
+
*/
92
+
export async function checkRateLimit(
93
+
automation: RateLimitAutomation,
94
+
now: number = Date.now(),
95
+
): Promise<RateLimitBreach | null> {
96
+
const timestamps = await loadRecentTimestamps(automation, now);
97
+
for (const window of RATE_LIMIT_WINDOWS) {
98
+
const count = countWithin(timestamps, now, window.ms);
99
+
if (count >= window.limit) {
100
+
return { window: window.name, count, limit: window.limit };
101
+
}
102
+
}
103
+
return null;
104
+
}
105
+
106
+
/** Snapshot of current usage across all windows. Drives the dashboard gauges. */
107
+
export async function getRateLimitCounts(
108
+
automation: RateLimitAutomation,
109
+
now: number = Date.now(),
110
+
): Promise<RateLimitCount[]> {
111
+
const timestamps = await loadRecentTimestamps(automation, now);
112
+
return RATE_LIMIT_WINDOWS.map((w) => ({
113
+
window: w.name,
114
+
count: countWithin(timestamps, now, w.ms),
115
+
limit: w.limit,
116
+
}));
117
+
}
118
+
119
+
/**
120
+
* Flip the automation to inactive with a "rate_limit:<window>" reason. The
121
+
* update is guarded on `active = true` so concurrent breach handlers (several
122
+
* in-flight events beating the jetstream re-partition) only one of them wins.
123
+
* Returns `true` if this call actually changed the row, `false` if someone else
124
+
* got there first. Callers use the return value to avoid spamming the delivery
125
+
* log with duplicate "disabled" entries.
126
+
*/
127
+
export async function disableForRateLimit(
128
+
uri: string,
129
+
breach: RateLimitBreach,
130
+
now: number = Date.now(),
131
+
): Promise<boolean> {
132
+
const updated = await db
133
+
.update(automations)
134
+
.set({
135
+
active: false,
136
+
disabledReason: `rate_limit:${breach.window}`,
137
+
disabledAt: new Date(now),
138
+
})
139
+
.where(and(eq(automations.uri, uri), eq(automations.active, true)))
140
+
.returning({ uri: automations.uri });
141
+
return updated.length > 0;
142
+
}