A tool for tailing a labelers' firehose, rehydrating, and storing records for future analysis of moderation decisions.
Remove unused prototype files
- Delete src/agent.ts (superseded by hydration services)
- Delete src/firehose.ts (superseded by firehose/ directory)
These were reference implementations used during development.
All tests still passing.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
-182
-17
src/agent.ts
-17
src/agent.ts
···
1
-
import { setGlobalDispatcher, Agent as Agent } from "undici";
2
-
setGlobalDispatcher(new Agent({ connect: { timeout: 20_000 } }));
3
-
import { BSKY_HANDLE, BSKY_PASSWORD, PDS } from "./config.js";
4
-
import { AtpAgent } from "@atproto/api";
5
-
6
-
export const agent = new AtpAgent({
7
-
service: `https://${PDS}`,
8
-
});
9
-
export const login = () =>
10
-
agent.login({
11
-
identifier: BSKY_HANDLE,
12
-
password: BSKY_PASSWORD,
13
-
});
14
-
15
-
export const isLoggedIn = login()
16
-
.then(() => true)
17
-
.catch(() => false);
-165
src/firehose.ts
-165
src/firehose.ts
···
1
-
import { decode, decodeFirst } from "@atcute/cbor";
2
-
import { readFileSync, writeFileSync } from "fs";
3
-
import { WSS_URL } from "./config.js";
4
-
import { logger } from "./logger.js";
5
-
import { LabelEvent } from "./types.js";
6
-
7
-
let ws: WebSocket | null = null;
8
-
let reconnectTimeout: NodeJS.Timeout | null = null;
9
-
let reconnectAttempts = 0;
10
-
let cursor: string = "";
11
-
const MAX_RECONNECT_DELAY = 60000;
12
-
const INITIAL_RECONNECT_DELAY = 1000;
13
-
const CURSOR_FILE = "./cursor.txt";
14
-
15
-
function getReconnectDelay(): number {
16
-
const delay = Math.min(
17
-
INITIAL_RECONNECT_DELAY * Math.pow(2, reconnectAttempts),
18
-
MAX_RECONNECT_DELAY,
19
-
);
20
-
reconnectAttempts++;
21
-
return delay;
22
-
}
23
-
24
-
async function handleLabelEvent(event: LabelEvent): Promise<void> {
25
-
// Placeholder for hydration logic
26
-
logger.info({ event }, "Received label event");
27
-
}
28
-
29
-
function saveCursor(seq: string): void {
30
-
try {
31
-
cursor = seq;
32
-
writeFileSync(CURSOR_FILE, seq, "utf8");
33
-
logger.debug({ cursor: seq }, "Saved cursor");
34
-
} catch (err) {
35
-
logger.warn({ err }, "Failed to save cursor");
36
-
}
37
-
}
38
-
39
-
function loadCursor(): string {
40
-
try {
41
-
const saved = readFileSync(CURSOR_FILE, "utf8").trim();
42
-
logger.info({ cursor: saved }, "Loaded cursor from file");
43
-
return saved;
44
-
} catch (err) {
45
-
logger.info("No cursor file found, starting from live");
46
-
return "";
47
-
}
48
-
}
49
-
50
-
function parseMessage(data: any): void {
51
-
try {
52
-
let buffer: Uint8Array;
53
-
54
-
if (data instanceof ArrayBuffer) {
55
-
buffer = new Uint8Array(data);
56
-
} else if (data instanceof Uint8Array) {
57
-
buffer = data;
58
-
} else if (typeof data === "string") {
59
-
try {
60
-
const parsed = JSON.parse(data);
61
-
if (parsed.seq) {
62
-
saveCursor(parsed.seq.toString());
63
-
}
64
-
processLabels(parsed);
65
-
return;
66
-
} catch {
67
-
logger.warn("Received non-JSON string message");
68
-
return;
69
-
}
70
-
} else {
71
-
processLabels(data);
72
-
return;
73
-
}
74
-
75
-
const [header, remainder] = decodeFirst(buffer);
76
-
const [body] = decodeFirst(remainder);
77
-
78
-
if (body && typeof body === "object" && "seq" in body) {
79
-
saveCursor(body.seq.toString());
80
-
}
81
-
82
-
processLabels(body);
83
-
} catch (err) {
84
-
logger.error({ err }, "Error parsing message");
85
-
}
86
-
}
87
-
88
-
function processLabels(parsed: any): void {
89
-
if (parsed.labels && Array.isArray(parsed.labels)) {
90
-
for (const label of parsed.labels) {
91
-
handleLabelEvent(label as LabelEvent);
92
-
}
93
-
} else if (parsed.label) {
94
-
handleLabelEvent(parsed.label as LabelEvent);
95
-
} else {
96
-
logger.debug({ parsed }, "Message does not contain label data");
97
-
}
98
-
}
99
-
100
-
function connect(): void {
101
-
if (
102
-
ws &&
103
-
(ws.readyState === WebSocket.CONNECTING || ws.readyState === WebSocket.OPEN)
104
-
) {
105
-
logger.debug("WebSocket already connected or connecting");
106
-
return;
107
-
}
108
-
109
-
const url = cursor ? `${WSS_URL}?cursor=${cursor}` : WSS_URL;
110
-
logger.info({ url, cursor }, "Connecting to firehose");
111
-
112
-
ws = new WebSocket(url);
113
-
114
-
ws.addEventListener("open", () => {
115
-
logger.info("Firehose connection established");
116
-
reconnectAttempts = 0;
117
-
});
118
-
119
-
ws.addEventListener("message", (event) => {
120
-
parseMessage(event.data);
121
-
});
122
-
123
-
ws.addEventListener("error", (event) => {
124
-
logger.error({ event }, "Firehose WebSocket error");
125
-
});
126
-
127
-
ws.addEventListener("close", (event) => {
128
-
logger.warn(
129
-
{ code: event.code, reason: event.reason },
130
-
"Firehose connection closed",
131
-
);
132
-
scheduleReconnect();
133
-
});
134
-
}
135
-
136
-
function scheduleReconnect(): void {
137
-
if (reconnectTimeout) {
138
-
clearTimeout(reconnectTimeout);
139
-
}
140
-
141
-
const delay = getReconnectDelay();
142
-
logger.info({ delay, attempt: reconnectAttempts }, "Scheduling reconnect");
143
-
144
-
reconnectTimeout = setTimeout(() => {
145
-
connect();
146
-
}, delay);
147
-
}
148
-
149
-
export function startFirehose(): void {
150
-
cursor = loadCursor();
151
-
connect();
152
-
}
153
-
154
-
export function stopFirehose(): void {
155
-
if (reconnectTimeout) {
156
-
clearTimeout(reconnectTimeout);
157
-
reconnectTimeout = null;
158
-
}
159
-
160
-
if (ws) {
161
-
logger.info("Closing firehose connection");
162
-
ws.close();
163
-
ws = null;
164
-
}
165
-
}