A fullstack app for indexing standard.site documents
feat: added new jetstream-consumer package
Replaces need for tap to index documents
+427
-2
+23
-2
bun.lock
+23
-2
bun.lock
···
21
21
"vite": "^5.0.0",
22
22
},
23
23
},
24
+
"packages/jetstream-consumer": {
25
+
"name": "jetstream-consumer",
26
+
"version": "1.0.0",
27
+
"devDependencies": {
28
+
"@types/bun": "^1.3.9",
29
+
"@types/node": "^25.3.0",
30
+
},
31
+
"peerDependencies": {
32
+
"typescript": "^5",
33
+
},
34
+
},
24
35
"packages/server": {
25
36
"name": "@document-feeds/server",
26
37
"version": "1.0.0",
···
257
268
258
269
"@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="],
259
270
271
+
"@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
272
+
260
273
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
261
274
262
-
"@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="],
275
+
"@types/node": ["@types/node@25.3.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-4K3bqJpXpqfg2XKGK9bpDTc6xO/xoUP/RBWS7AtRMug6zZFaRekiLzjVtAoZMquxoAbzBvy5nxQ7veS5eYzf8A=="],
263
276
264
277
"@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="],
265
278
···
280
293
"blake3-wasm": ["blake3-wasm@2.1.5", "", {}, "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g=="],
281
294
282
295
"browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="],
296
+
297
+
"bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
283
298
284
299
"caniuse-lite": ["caniuse-lite@1.0.30001764", "", {}, "sha512-9JGuzl2M+vPL+pz70gtMF9sHdMFbY9FJaQBi186cHKH3pSzDvzoUJUPV6fqiKIMyXbud9ZLg4F3Yza1vJ1+93g=="],
285
300
···
333
348
334
349
"is-arrayish": ["is-arrayish@0.3.4", "", {}, "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA=="],
335
350
351
+
"jetstream-consumer": ["jetstream-consumer@workspace:packages/jetstream-consumer"],
352
+
336
353
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
337
354
338
355
"jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
···
411
428
412
429
"undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="],
413
430
414
-
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
431
+
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
415
432
416
433
"unenv": ["unenv@2.0.0-rc.14", "", { "dependencies": { "defu": "^6.1.4", "exsolve": "^1.0.1", "ohash": "^2.0.10", "pathe": "^2.0.3", "ufo": "^1.5.4" } }, "sha512-od496pShMen7nOy5VmVJCnq8rptd45vh6Nx/r2iPbrba6pa6p+tS2ywuIHRZ/OBvSbQZB0kWvpO9XBNVFXHD3Q=="],
417
434
···
437
454
438
455
"@cspotcode/source-map-support/@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="],
439
456
457
+
"bun-types/@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="],
458
+
440
459
"sharp/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="],
441
460
442
461
"wrangler/esbuild": ["esbuild@0.17.19", "", { "optionalDependencies": { "@esbuild/android-arm": "0.17.19", "@esbuild/android-arm64": "0.17.19", "@esbuild/android-x64": "0.17.19", "@esbuild/darwin-arm64": "0.17.19", "@esbuild/darwin-x64": "0.17.19", "@esbuild/freebsd-arm64": "0.17.19", "@esbuild/freebsd-x64": "0.17.19", "@esbuild/linux-arm": "0.17.19", "@esbuild/linux-arm64": "0.17.19", "@esbuild/linux-ia32": "0.17.19", "@esbuild/linux-loong64": "0.17.19", "@esbuild/linux-mips64el": "0.17.19", "@esbuild/linux-ppc64": "0.17.19", "@esbuild/linux-riscv64": "0.17.19", "@esbuild/linux-s390x": "0.17.19", "@esbuild/linux-x64": "0.17.19", "@esbuild/netbsd-x64": "0.17.19", "@esbuild/openbsd-x64": "0.17.19", "@esbuild/sunos-x64": "0.17.19", "@esbuild/win32-arm64": "0.17.19", "@esbuild/win32-ia32": "0.17.19", "@esbuild/win32-x64": "0.17.19" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw=="],
462
+
463
+
"bun-types/@types/node/undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
443
464
444
465
"wrangler/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.17.19", "", { "os": "android", "cpu": "arm" }, "sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A=="],
445
466
+34
packages/jetstream-consumer/.gitignore
+34
packages/jetstream-consumer/.gitignore
···
1
+
# dependencies (bun install)
2
+
node_modules
3
+
4
+
# output
5
+
out
6
+
dist
7
+
*.tgz
8
+
9
+
# code coverage
10
+
coverage
11
+
*.lcov
12
+
13
+
# logs
14
+
logs
15
+
_.log
16
+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
17
+
18
+
# dotenv environment variable files
19
+
.env
20
+
.env.development.local
21
+
.env.test.local
22
+
.env.production.local
23
+
.env.local
24
+
25
+
# caches
26
+
.eslintcache
27
+
.cache
28
+
*.tsbuildinfo
29
+
30
+
# IntelliJ based IDEs
31
+
.idea
32
+
33
+
# Finder (MacOS) folder config
34
+
.DS_Store
+106
packages/jetstream-consumer/CLAUDE.md
+106
packages/jetstream-consumer/CLAUDE.md
···
1
+
2
+
Default to using Bun instead of Node.js.
3
+
4
+
- Use `bun <file>` instead of `node <file>` or `ts-node <file>`
5
+
- Use `bun test` instead of `jest` or `vitest`
6
+
- Use `bun build <file.html|file.ts|file.css>` instead of `webpack` or `esbuild`
7
+
- Use `bun install` instead of `npm install` or `yarn install` or `pnpm install`
8
+
- Use `bun run <script>` instead of `npm run <script>` or `yarn run <script>` or `pnpm run <script>`
9
+
- Use `bunx <package> <command>` instead of `npx <package> <command>`
10
+
- Bun automatically loads .env, so don't use dotenv.
11
+
12
+
## APIs
13
+
14
+
- `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`.
15
+
- `bun:sqlite` for SQLite. Don't use `better-sqlite3`.
16
+
- `Bun.redis` for Redis. Don't use `ioredis`.
17
+
- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`.
18
+
- `WebSocket` is built-in. Don't use `ws`.
19
+
- Prefer `Bun.file` over `node:fs`'s readFile/writeFile
20
+
- Bun.$`ls` instead of execa.
21
+
22
+
## Testing
23
+
24
+
Use `bun test` to run tests.
25
+
26
+
```ts#index.test.ts
27
+
import { test, expect } from "bun:test";
28
+
29
+
test("hello world", () => {
30
+
expect(1).toBe(1);
31
+
});
32
+
```
33
+
34
+
## Frontend
35
+
36
+
Use HTML imports with `Bun.serve()`. Don't use `vite`. HTML imports fully support React, CSS, Tailwind.
37
+
38
+
Server:
39
+
40
+
```ts#index.ts
41
+
import index from "./index.html"
42
+
43
+
Bun.serve({
44
+
routes: {
45
+
"/": index,
46
+
"/api/users/:id": {
47
+
GET: (req) => {
48
+
return new Response(JSON.stringify({ id: req.params.id }));
49
+
},
50
+
},
51
+
},
52
+
// optional websocket support
53
+
websocket: {
54
+
open: (ws) => {
55
+
ws.send("Hello, world!");
56
+
},
57
+
message: (ws, message) => {
58
+
ws.send(message);
59
+
},
60
+
close: (ws) => {
61
+
// handle close
62
+
}
63
+
},
64
+
development: {
65
+
hmr: true,
66
+
console: true,
67
+
}
68
+
})
69
+
```
70
+
71
+
HTML files can import .tsx, .jsx or .js files directly and Bun's bundler will transpile & bundle automatically. `<link>` tags can point to stylesheets and Bun's CSS bundler will bundle.
72
+
73
+
```html#index.html
74
+
<html>
75
+
<body>
76
+
<h1>Hello, world!</h1>
77
+
<script type="module" src="./frontend.tsx"></script>
78
+
</body>
79
+
</html>
80
+
```
81
+
82
+
With the following `frontend.tsx`:
83
+
84
+
```tsx#frontend.tsx
85
+
import React from "react";
86
+
import { createRoot } from "react-dom/client";
87
+
88
+
// import .css files directly and it works
89
+
import './index.css';
90
+
91
+
const root = createRoot(document.body);
92
+
93
+
export default function Frontend() {
94
+
return <h1>Hello, world!</h1>;
95
+
}
96
+
97
+
root.render(<Frontend />);
98
+
```
99
+
100
+
Then, run index.ts
101
+
102
+
```sh
103
+
bun --hot ./index.ts
104
+
```
105
+
106
+
For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`.
+13
packages/jetstream-consumer/Dockerfile
+13
packages/jetstream-consumer/Dockerfile
+15
packages/jetstream-consumer/README.md
+15
packages/jetstream-consumer/README.md
···
1
+
# jetstream-consumer
2
+
3
+
To install dependencies:
4
+
5
+
```bash
6
+
bun install
7
+
```
8
+
9
+
To run:
10
+
11
+
```bash
12
+
bun run src/index.ts
13
+
```
14
+
15
+
This project was created using `bun init` in bun v1.3.5. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime.
+1
packages/jetstream-consumer/cursor.txt
+1
packages/jetstream-consumer/cursor.txt
···
1
+
1772039237880654
+14
packages/jetstream-consumer/docker-compose.yml
+14
packages/jetstream-consumer/docker-compose.yml
···
1
+
services:
2
+
jetstream-consumer:
3
+
build: .
4
+
environment:
5
+
- WEBHOOK_URL=https://example.com/webhook/tap/batch
6
+
- WEBHOOK_SECRET=
7
+
- JETSTREAM_INSTANCES=jetstream1.us-east.bsky.network,jetstream2.us-east.bsky.network,jetstream1.us-west.bsky.network,jetstream2.us-west.bsky.network
8
+
- WANTED_COLLECTIONS=site.standard.document
9
+
- CURSOR_FILE=/app/data/cursor.txt
10
+
- BATCH_INTERVAL_MS=5000
11
+
- INACTIVITY_TIMEOUT_MS=300000
12
+
volumes:
13
+
- ./data:/app/data
14
+
restart: unless-stopped
+18
packages/jetstream-consumer/package.json
+18
packages/jetstream-consumer/package.json
···
1
+
{
2
+
"name": "jetstream-consumer",
3
+
"version": "1.0.0",
4
+
"private": true,
5
+
"scripts": {
6
+
"start": "bun run src/index.ts",
7
+
"dev": "bun run src/index.ts"
8
+
},
9
+
"devDependencies": {
10
+
"@types/bun": "^1.3.9",
11
+
"@types/node": "^25.3.0"
12
+
},
13
+
"module": "src/index.ts",
14
+
"type": "module",
15
+
"peerDependencies": {
16
+
"typescript": "^5"
17
+
}
18
+
}
+190
packages/jetstream-consumer/src/index.ts
+190
packages/jetstream-consumer/src/index.ts
···
1
+
export {};
2
+
3
+
const WEBHOOK_URL = process.env.WEBHOOK_URL;
4
+
const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET;
5
+
const JETSTREAM_INSTANCES = (
6
+
process.env.JETSTREAM_INSTANCES ??
7
+
"jetstream1.us-east.bsky.network,jetstream2.us-east.bsky.network,jetstream1.us-west.bsky.network,jetstream2.us-west.bsky.network"
8
+
)
9
+
.split(",")
10
+
.map((h) => h.trim());
11
+
const WANTED_COLLECTIONS = (
12
+
process.env.WANTED_COLLECTIONS ?? "site.standard.document"
13
+
)
14
+
.split(",")
15
+
.map((c) => c.trim());
16
+
const CURSOR_FILE = process.env.CURSOR_FILE ?? "./cursor.txt";
17
+
const BATCH_INTERVAL_MS = Number(process.env.BATCH_INTERVAL_MS) || 5000;
18
+
const INACTIVITY_TIMEOUT_MS =
19
+
Number(process.env.INACTIVITY_TIMEOUT_MS) || 300_000;
20
+
21
+
const DEFAULT_CURSOR = "1772036746";
22
+
23
+
if (!WEBHOOK_URL) {
24
+
console.error("WEBHOOK_URL environment variable is required");
25
+
process.exit(1);
26
+
}
27
+
28
+
let cursor = DEFAULT_CURSOR;
29
+
let currentInstanceIndex = 0;
30
+
let ws: WebSocket | null = null;
31
+
let lastMessageTime = Date.now();
32
+
let batch: Array<{
33
+
type: string;
34
+
did: string;
35
+
collection: string;
36
+
rkey: string;
37
+
cid: string;
38
+
record?: Record<string, unknown>;
39
+
}> = [];
40
+
41
+
async function loadCursor(): Promise<void> {
42
+
try {
43
+
const file = Bun.file(CURSOR_FILE);
44
+
const text = await file.text();
45
+
const trimmed = text.trim();
46
+
if (trimmed) {
47
+
cursor = trimmed;
48
+
console.log(`Loaded cursor from file: ${cursor}`);
49
+
}
50
+
} catch {
51
+
console.log(`No cursor file found, using default: ${cursor}`);
52
+
}
53
+
}
54
+
55
+
async function saveCursor(): Promise<void> {
56
+
try {
57
+
await Bun.write(CURSOR_FILE, cursor);
58
+
} catch (err) {
59
+
console.error("Failed to save cursor:", err);
60
+
}
61
+
}
62
+
63
+
async function flushBatch(): Promise<void> {
64
+
if (batch.length === 0) return;
65
+
66
+
const events = batch.splice(0, batch.length);
67
+
console.log(`Flushing batch of ${events.length} events`);
68
+
69
+
try {
70
+
const headers: Record<string, string> = {
71
+
"Content-Type": "application/json",
72
+
};
73
+
if (WEBHOOK_SECRET) {
74
+
headers["Authorization"] = `Bearer ${WEBHOOK_SECRET}`;
75
+
}
76
+
77
+
const res = await fetch(WEBHOOK_URL!, {
78
+
method: "POST",
79
+
headers,
80
+
body: JSON.stringify(events),
81
+
});
82
+
83
+
if (!res.ok) {
84
+
console.error(
85
+
`Webhook responded with ${res.status}: ${await res.text()}`,
86
+
);
87
+
return;
88
+
}
89
+
90
+
const result = (await res.json()) as Record<string, unknown>;
91
+
console.log("Batch result:", JSON.stringify(result));
92
+
await saveCursor();
93
+
} catch (err) {
94
+
console.error("Failed to send batch:", err);
95
+
}
96
+
}
97
+
98
+
function connect(): void {
99
+
const host = JETSTREAM_INSTANCES[currentInstanceIndex];
100
+
const collections = WANTED_COLLECTIONS.map(
101
+
(c) => `wantedCollections=${c}`,
102
+
).join("&");
103
+
const url = `wss://${host}/subscribe?${collections}&cursor=${cursor}`;
104
+
105
+
console.log(`Connecting to ${host} with cursor ${cursor}`);
106
+
107
+
ws = new WebSocket(url);
108
+
109
+
ws.addEventListener("open", () => {
110
+
console.log(`Connected to ${host}`);
111
+
lastMessageTime = Date.now();
112
+
});
113
+
114
+
ws.addEventListener("message", (event) => {
115
+
lastMessageTime = Date.now();
116
+
117
+
try {
118
+
const data = JSON.parse(String(event.data));
119
+
120
+
if (data.kind !== "commit") return;
121
+
122
+
cursor = String(data.time_us);
123
+
124
+
batch.push({
125
+
type: data.commit.operation,
126
+
did: data.did,
127
+
collection: data.commit.collection,
128
+
rkey: data.commit.rkey,
129
+
cid: data.commit.cid,
130
+
record: data.commit.record,
131
+
});
132
+
} catch (err) {
133
+
console.error("Failed to parse message:", err);
134
+
}
135
+
});
136
+
137
+
ws.addEventListener("close", (event) => {
138
+
console.log(`WebSocket closed: code=${event.code} reason=${event.reason}`);
139
+
});
140
+
141
+
ws.addEventListener("error", (event) => {
142
+
console.error("WebSocket error:", event);
143
+
});
144
+
}
145
+
146
+
function switchInstance(): void {
147
+
currentInstanceIndex =
148
+
(currentInstanceIndex + 1) % JETSTREAM_INSTANCES.length;
149
+
const newHost = JETSTREAM_INSTANCES[currentInstanceIndex];
150
+
console.log(`Switching to instance: ${newHost}`);
151
+
152
+
if (ws) {
153
+
try {
154
+
ws.close();
155
+
} catch {
156
+
// ignore close errors
157
+
}
158
+
ws = null;
159
+
}
160
+
161
+
connect();
162
+
}
163
+
164
+
// --- Main ---
165
+
166
+
await loadCursor();
167
+
connect();
168
+
169
+
// Batch flush interval
170
+
setInterval(() => {
171
+
flushBatch();
172
+
}, BATCH_INTERVAL_MS);
173
+
174
+
// Inactivity check every 30s
175
+
setInterval(() => {
176
+
const elapsed = Date.now() - lastMessageTime;
177
+
if (elapsed > INACTIVITY_TIMEOUT_MS) {
178
+
console.log(
179
+
`No messages for ${Math.round(elapsed / 1000)}s, triggering failover`,
180
+
);
181
+
switchInstance();
182
+
}
183
+
}, 30_000);
184
+
185
+
console.log("Jetstream consumer started");
186
+
console.log(` Webhook URL: ${WEBHOOK_URL}`);
187
+
console.log(` Instances: ${JETSTREAM_INSTANCES.join(", ")}`);
188
+
console.log(` Collections: ${WANTED_COLLECTIONS.join(", ")}`);
189
+
console.log(` Batch interval: ${BATCH_INTERVAL_MS}ms`);
190
+
console.log(` Inactivity timeout: ${INACTIVITY_TIMEOUT_MS}ms`);
+13
packages/jetstream-consumer/tsconfig.json
+13
packages/jetstream-consumer/tsconfig.json