tracks lexicons and how many times they appeared on the jetstream
refactor: prepare for rust server
+28
-309
+2
-2
.gitignore
+2
-2
.gitignore
.npmrc
client/.npmrc
.npmrc
client/.npmrc
+4
-31
bun.lock
client/bun.lock
+4
-31
bun.lock
client/bun.lock
···
3
3
"workspaces": {
4
4
"": {
5
5
"name": "nsid-tracker",
6
-
"dependencies": {
7
-
"@atcute/jetstream": "^1.0.2",
8
-
},
9
6
"devDependencies": {
10
7
"@eslint/compat": "^1.2.5",
11
8
"@eslint/js": "^9.18.0",
9
+
"@sveltejs/adapter-static": "^3.0.8",
12
10
"@sveltejs/kit": "^2.22.0",
13
11
"@sveltejs/vite-plugin-svelte": "^6.0.0",
14
12
"@tailwindcss/vite": "^4.0.0",
···
17
15
"eslint-plugin-svelte": "^3.0.0",
18
16
"globals": "^16.0.0",
19
17
"svelte": "^5.0.0",
20
-
"svelte-adapter-bun": "^0.5.2",
21
18
"svelte-check": "^4.0.0",
22
19
"tailwindcss": "^4.0.0",
23
20
"typescript": "^5.0.0",
···
29
26
"packages": {
30
27
"@ampproject/remapping": ["@ampproject/remapping@2.3.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw=="],
31
28
32
-
"@atcute/jetstream": ["@atcute/jetstream@1.0.2", "", { "dependencies": { "@atcute/lexicons": "^1.0.2", "@badrap/valita": "^0.4.2", "@mary-ext/event-iterator": "^1.0.0", "@mary-ext/simple-event-emitter": "^1.0.0", "partysocket": "^1.1.4", "type-fest": "^4.41.0", "yocto-queue": "^1.2.1" } }, "sha512-ZtdNNxl4zq9cgUpXSL9F+AsXUZt0Zuyj0V7974D7LxdMxfTItPnMZ9dRG8GoFkkGz3+pszdsG888Ix8C0F2+mA=="],
33
-
34
-
"@atcute/lexicons": ["@atcute/lexicons@1.1.0", "", { "dependencies": { "esm-env": "^1.2.2" } }, "sha512-LFqwnria78xLYb62Ri/+WwQpUTgZp2DuyolNGIIOV1dpiKhFFFh//nscHMA6IExFLQRqWDs3tTjy7zv0h3sf1Q=="],
35
-
36
-
"@badrap/valita": ["@badrap/valita@0.4.5", "", {}, "sha512-4QwGbuhh/JesHRQj79mO/l37PvJj4l/tlAu7+S1n4h47qwaNpZ0WDvIwUGLYUsdi9uQ5UPpiG9wb1Wm3XUFBUQ=="],
37
-
38
29
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.6", "", { "os": "aix", "cpu": "ppc64" }, "sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw=="],
39
30
40
31
"@esbuild/android-arm": ["@esbuild/android-arm@0.25.6", "", { "os": "android", "cpu": "arm" }, "sha512-S8ToEOVfg++AU/bHwdksHNnyLyVM+eMVAOf6yRKFitnwnbwwPNqKr3srzFRe7nzV69RQKb5DgchIX5pt3L53xg=="],
···
125
116
126
117
"@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.29", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ=="],
127
118
128
-
"@mary-ext/event-iterator": ["@mary-ext/event-iterator@1.0.0", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-l6gCPsWJ8aRCe/s7/oCmero70kDHgIK5m4uJvYgwEYTqVxoBOIXbKr5tnkLqUHEg6mNduB4IWvms3h70Hp9ADQ=="],
129
-
130
-
"@mary-ext/simple-event-emitter": ["@mary-ext/simple-event-emitter@1.0.0", "", {}, "sha512-meA/zJZKIN1RVBNEYIbjufkUrW7/tRjHH60FjolpG1ixJKo76TB208qefQLNdOVDA7uIG0CGEDuhmMirtHKLAg=="],
131
-
132
119
"@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="],
133
120
134
121
"@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="],
···
178
165
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.45.1", "", { "os": "win32", "cpu": "x64" }, "sha512-M/fKi4sasCdM8i0aWJjCSFm2qEnYRR8AMLG2kxp6wD13+tMGA4Z1tVAuHkNRjud5SW2EM3naLuK35w9twvf6aA=="],
179
166
180
167
"@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.5", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-IwQk4yfwLdibDlrXVE04jTZYlLnwsTT2PIOQQGNLWfjavGifnk1JD1LcZjZaBTRcxZu2FfPfNLOE04DSu9lqtQ=="],
168
+
169
+
"@sveltejs/adapter-static": ["@sveltejs/adapter-static@3.0.8", "", { "peerDependencies": { "@sveltejs/kit": "^2.0.0" } }, "sha512-YaDrquRpZwfcXbnlDsSrBQNCChVOT9MGuSg+dMAyfsAa1SmiAhrA5jUYUiIMC59G92kIbY/AaQOWcBdq+lh+zg=="],
181
170
182
171
"@sveltejs/kit": ["@sveltejs/kit@2.25.1", "", { "dependencies": { "@sveltejs/acorn-typescript": "^1.0.5", "@types/cookie": "^0.6.0", "acorn": "^8.14.1", "cookie": "^0.6.0", "devalue": "^5.1.0", "esm-env": "^1.2.2", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", "sirv": "^3.0.0" }, "peerDependencies": { "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0 || ^6.0.0-next.0", "svelte": "^4.0.0 || ^5.0.0-next.0", "vite": "^5.0.3 || ^6.0.0 || ^7.0.0-beta.0" }, "bin": { "svelte-kit": "svelte-kit.js" } }, "sha512-8H+fxDEp7Xq6tLFdrGdS5fLu6ONDQQ9DgyjboXpChubuFdfH9QoFX09ypssBpyNkJNZFt9eW3yLmXIc9CesPCA=="],
183
172
···
331
320
332
321
"esutils": ["esutils@2.0.3", "", {}, "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="],
333
322
334
-
"event-target-polyfill": ["event-target-polyfill@0.0.4", "", {}, "sha512-Gs6RLjzlLRdT8X9ZipJdIZI/Y6/HhRLyq9RdDlCsnpxr/+Nn6bU2EFGuC94GjxqhM+Nmij2Vcq98yoHrU8uNFQ=="],
335
-
336
323
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
337
324
338
325
"fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="],
···
360
347
"glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="],
361
348
362
349
"globals": ["globals@16.3.0", "", {}, "sha512-bqWEnJ1Nt3neqx2q5SFfGS8r/ahumIakg3HcwtNlrVlwXIeNumWn/c7Pn/wKzGhf6SaW6H6uWXLqC30STCMchQ=="],
363
-
364
-
"globalyzer": ["globalyzer@0.1.0", "", {}, "sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q=="],
365
-
366
-
"globrex": ["globrex@0.1.2", "", {}, "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg=="],
367
350
368
351
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
369
352
···
467
450
468
451
"parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="],
469
452
470
-
"partysocket": ["partysocket@1.1.4", "", { "dependencies": { "event-target-polyfill": "^0.0.4" } }, "sha512-jXP7PFj2h5/v4UjDS8P7MZy6NJUQ7sspiFyxL4uc/+oKOL+KdtXzHnTV8INPGxBrLTXgalyG3kd12Qm7WrYc3A=="],
471
-
472
453
"path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="],
473
454
474
455
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
···
522
503
"supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
523
504
524
505
"svelte": ["svelte@5.36.8", "", { "dependencies": { "@ampproject/remapping": "^2.3.0", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "esm-env": "^1.2.1", "esrap": "^2.1.0", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-8JbZWQu96hMjH/oYQPxXW6taeC6Awl6muGHeZzJTxQx7NGRQ/J9wN1hkzRKLOlSDlbS2igiFg7p5xyTp5uXG3A=="],
525
-
526
-
"svelte-adapter-bun": ["svelte-adapter-bun@0.5.2", "", { "dependencies": { "tiny-glob": "^0.2.9" } }, "sha512-xEtFgaal6UgrCwwkSIcapO9kopoFNUYCYqyKCikdqxX9bz2TDYnrWQZ7qBnkunMxi1HOIERUCvTcebYGiarZLA=="],
527
506
528
507
"svelte-check": ["svelte-check@4.3.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "chokidar": "^4.0.1", "fdir": "^6.2.0", "picocolors": "^1.0.0", "sade": "^1.7.4" }, "peerDependencies": { "svelte": "^4.0.0 || ^5.0.0-next.0", "typescript": ">=5.0.0" }, "bin": { "svelte-check": "bin/svelte-check" } }, "sha512-Iz8dFXzBNAM7XlEIsUjUGQhbEE+Pvv9odb9+0+ITTgFWZBGeJRRYqHUUglwe2EkLD5LIsQaAc4IUJyvtKuOO5w=="],
529
508
···
535
514
536
515
"tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="],
537
516
538
-
"tiny-glob": ["tiny-glob@0.2.9", "", { "dependencies": { "globalyzer": "0.1.0", "globrex": "^0.1.2" } }, "sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg=="],
539
-
540
517
"tinyglobby": ["tinyglobby@0.2.14", "", { "dependencies": { "fdir": "^6.4.4", "picomatch": "^4.0.2" } }, "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ=="],
541
518
542
519
"to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
···
546
523
"ts-api-utils": ["ts-api-utils@2.1.0", "", { "peerDependencies": { "typescript": ">=4.8.4" } }, "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ=="],
547
524
548
525
"type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="],
549
-
550
-
"type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
551
526
552
527
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
553
528
···
571
546
572
547
"yaml": ["yaml@1.10.2", "", {}, "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg=="],
573
548
574
-
"yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="],
549
+
"yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
575
550
576
551
"zimmerframe": ["zimmerframe@1.1.2", "", {}, "sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w=="],
577
552
···
600
575
"fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="],
601
576
602
577
"micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
603
-
604
-
"p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
605
578
606
579
"@tailwindcss/oxide-wasm32-wasi/@napi-rs/wasm-runtime/@tybys/wasm-util": ["@tybys/wasm-util@0.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ=="],
607
580
+6
client/src/lib/types.ts
+6
client/src/lib/types.ts
+2
client/src/routes/+layout.ts
+2
client/src/routes/+layout.ts
eslint.config.js
client/eslint.config.js
eslint.config.js
client/eslint.config.js
+12
-15
package.json
client/package.json
+12
-15
package.json
client/package.json
···
1
1
{
2
2
"name": "nsid-tracker",
3
-
"private": true,
4
3
"version": "0.0.1",
5
-
"type": "module",
6
-
"scripts": {
7
-
"dev": "vite dev",
8
-
"build": "vite build",
9
-
"preview": "vite preview",
10
-
"prepare": "svelte-kit sync || echo ''",
11
-
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
12
-
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
13
-
"lint": "eslint ."
14
-
},
15
4
"devDependencies": {
16
5
"@eslint/compat": "^1.2.5",
17
6
"@eslint/js": "^9.18.0",
7
+
"@sveltejs/adapter-static": "^3.0.8",
18
8
"@sveltejs/kit": "^2.22.0",
19
9
"@sveltejs/vite-plugin-svelte": "^6.0.0",
20
10
"@tailwindcss/vite": "^4.0.0",
···
23
13
"eslint-plugin-svelte": "^3.0.0",
24
14
"globals": "^16.0.0",
25
15
"svelte": "^5.0.0",
26
-
"svelte-adapter-bun": "^0.5.2",
27
16
"svelte-check": "^4.0.0",
28
17
"tailwindcss": "^4.0.0",
29
18
"typescript": "^5.0.0",
30
19
"typescript-eslint": "^8.20.0",
31
20
"vite": "^7.0.4"
32
21
},
33
-
"dependencies": {
34
-
"@atcute/jetstream": "^1.0.2"
35
-
}
22
+
"private": true,
23
+
"scripts": {
24
+
"dev": "vite dev",
25
+
"build": "vite build",
26
+
"preview": "vite preview",
27
+
"prepare": "svelte-kit sync || echo ''",
28
+
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
29
+
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
30
+
"lint": "eslint ."
31
+
},
32
+
"type": "module"
36
33
}
src/app.css
client/src/app.css
src/app.css
client/src/app.css
src/app.d.ts
client/src/app.d.ts
src/app.d.ts
client/src/app.d.ts
src/app.html
client/src/app.html
src/app.html
client/src/app.html
-12
src/hooks.server.ts
-12
src/hooks.server.ts
···
1
-
import { eventTracker } from "$lib/db";
2
-
import { exit as workerExit, start } from "$lib/worker_manager.js";
3
-
4
-
const exit = () => {
5
-
workerExit();
6
-
eventTracker.exit();
7
-
setTimeout(process.exit, 2000);
8
-
};
9
-
start();
10
-
11
-
process.on("SIGINT", exit);
12
-
process.on("SIGTERM", exit);
-137
src/lib/db.ts
-137
src/lib/db.ts
···
1
-
import { Database } from "bun:sqlite";
2
-
import { env } from "process";
3
-
import type { WorkerEventData } from "./types";
4
-
5
-
export interface EventRecord {
6
-
nsid: string;
7
-
timestamp: number;
8
-
count: number;
9
-
deleted_count: number;
10
-
}
11
-
12
-
const ALL_NSID = "*";
13
-
14
-
class EventTracker {
15
-
private db: Database;
16
-
private insertNsidQuery;
17
-
private insertEventQuery;
18
-
private updateCountQuery;
19
-
private getNsidCountQuery;
20
-
21
-
// private bufferedRecords: WorkerEventData[];
22
-
23
-
constructor() {
24
-
// this.bufferedRecords = [];
25
-
this.db = new Database(env.DB_PATH ?? "events.sqlite");
26
-
// init db
27
-
this.db.run("PRAGMA journal_mode = WAL;");
28
-
// events
29
-
this.db.run(`
30
-
CREATE TABLE IF NOT EXISTS events (
31
-
nsid_idx INTEGER NOT NULL,
32
-
timestamp INTEGER NOT NULL,
33
-
deleted INTEGER NOT NULL,
34
-
PRIMARY KEY (nsid_idx, timestamp)
35
-
)
36
-
`);
37
-
// aggregated counts
38
-
this.db.run(`
39
-
CREATE TABLE IF NOT EXISTS nsid_counts (
40
-
nsid_idx INTEGER PRIMARY KEY,
41
-
count INTEGER NOT NULL,
42
-
deleted_count INTEGER NOT NULL,
43
-
last_updated INTEGER NOT NULL
44
-
)
45
-
`);
46
-
this.db.run(`
47
-
CREATE TABLE IF NOT EXISTS nsid_types (
48
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
49
-
nsid TEXT UNIQUE NOT NULL
50
-
);
51
-
`);
52
-
// compile queries
53
-
this.insertNsidQuery = this.db.query(
54
-
"INSERT OR IGNORE INTO nsid_types (nsid) VALUES (?)",
55
-
);
56
-
this.insertEventQuery = this.db.query(`
57
-
INSERT OR IGNORE INTO events (nsid_idx, timestamp, deleted)
58
-
VALUES (
59
-
(SELECT id FROM nsid_types WHERE nsid = ?),
60
-
?,
61
-
?
62
-
)
63
-
`);
64
-
this.updateCountQuery = this.db.query(`
65
-
INSERT INTO nsid_counts (nsid_idx, count, deleted_count, last_updated)
66
-
VALUES (
67
-
(SELECT id FROM nsid_types WHERE nsid = $nsid),
68
-
1 - $deleted,
69
-
$deleted,
70
-
$timestamp
71
-
)
72
-
ON CONFLICT(nsid_idx) DO UPDATE SET
73
-
count = count + (1 - $deleted),
74
-
deleted_count = deleted_count + $deleted,
75
-
last_updated = $timestamp
76
-
`);
77
-
this.getNsidCountQuery = this.db.query(`
78
-
SELECT
79
-
(SELECT nsid FROM nsid_types WHERE id = nsid_idx) as nsid,
80
-
count,
81
-
deleted_count,
82
-
last_updated as timestamp
83
-
FROM nsid_counts
84
-
ORDER BY count DESC
85
-
`);
86
-
this.insertNsidQuery.run(ALL_NSID);
87
-
}
88
-
89
-
// private recordBufferedEvents = () => {
90
-
// try {
91
-
// this.db.transaction(() => {
92
-
// for (const event of this.bufferedRecords) {
93
-
// this.insertEventQuery.run(event.nsid, event.timestamp, event.deleted);
94
-
// }
95
-
// })();
96
-
// this.bufferedRecords = [];
97
-
// } catch (e) {
98
-
// console.error(`can't commit to db: ${e}`);
99
-
// }
100
-
// };
101
-
102
-
recordEventHit = (data: WorkerEventData) => {
103
-
try {
104
-
this.db.transaction(() => {
105
-
const { nsid, timestamp, deleted } = data;
106
-
this.insertNsidQuery.run(nsid);
107
-
this.insertEventQuery.run(nsid, timestamp, deleted);
108
-
this.updateCountQuery.run({
109
-
$nsid: nsid,
110
-
$deleted: deleted,
111
-
$timestamp: timestamp,
112
-
});
113
-
this.updateCountQuery.run({
114
-
$nsid: ALL_NSID,
115
-
$deleted: deleted,
116
-
$timestamp: timestamp,
117
-
});
118
-
})();
119
-
// this.bufferedRecords.push(data);
120
-
} catch (e) {
121
-
console.error(`can't commit to db: ${e}`);
122
-
}
123
-
// commit buffered if at 10k
124
-
// if (this.bufferedRecords.length > 9999) this.recordBufferedEvents();
125
-
};
126
-
127
-
getNsidCounts = (): EventRecord[] => {
128
-
return this.getNsidCountQuery.all() as EventRecord[];
129
-
};
130
-
131
-
exit = () => {
132
-
// this.recordBufferedEvents();
133
-
this.db.close();
134
-
};
135
-
}
136
-
137
-
export const eventTracker = new EventTracker();
-6
src/lib/types.ts
-6
src/lib/types.ts
-61
src/lib/worker.ts
-61
src/lib/worker.ts
···
1
-
import { JetstreamSubscription } from "@atcute/jetstream";
2
-
import type { WorkerEventData, WorkerCommand } from "./types.js";
3
-
import * as wt from "node:worker_threads";
4
-
5
-
const port = wt.parentPort!;
6
-
let subscription: JetstreamSubscription | null = null;
7
-
8
-
const track = async () => {
9
-
subscription = new JetstreamSubscription({
10
-
url: "wss://jetstream2.us-east.bsky.network",
11
-
validateEvents: false, // trust the jetstream :3
12
-
});
13
-
14
-
for await (const event of subscription) {
15
-
if (event.kind !== "commit") {
16
-
continue;
17
-
}
18
-
19
-
const { operation, collection } = event.commit;
20
-
21
-
const data: WorkerEventData = {
22
-
nsid: collection,
23
-
timestamp: event.time_us,
24
-
deleted: operation === "delete",
25
-
};
26
-
27
-
port.postMessage(data);
28
-
}
29
-
};
30
-
31
-
const trackLoop = async () => {
32
-
if (subscription !== null) {
33
-
return;
34
-
}
35
-
36
-
let retryCount = 0;
37
-
const baseDelay = 1000; // 1 second
38
-
const maxDelay = 60000; // 60 seconds
39
-
40
-
// if the above fails we fall into here
41
-
while (true) {
42
-
try {
43
-
await track();
44
-
retryCount = 0; // Reset on success
45
-
} catch (e) {
46
-
console.log(`tracking failed: ${e}`);
47
-
48
-
const delay = Math.min(baseDelay * Math.pow(2, retryCount), maxDelay);
49
-
console.log(`retrying in ${delay}ms (attempt ${retryCount + 1})`);
50
-
51
-
await new Promise((resolve) => setTimeout(resolve, delay));
52
-
retryCount++;
53
-
}
54
-
}
55
-
};
56
-
57
-
port.on("message", (command: WorkerCommand) => {
58
-
if (command === "exit") process.exit();
59
-
});
60
-
61
-
trackLoop().catch(console.error);
-21
src/lib/worker_manager.ts
-21
src/lib/worker_manager.ts
···
1
-
import { eventTracker } from "./db.js";
2
-
import type { WorkerCommand } from "./types.js";
3
-
import * as wt from "node:worker_threads";
4
-
import workerSrc from "./worker.js?raw";
5
-
6
-
let worker: wt.Worker | null = null;
7
-
8
-
const sendCommand = (command: WorkerCommand) => {
9
-
worker?.postMessage(command);
10
-
};
11
-
12
-
export const start = () => {
13
-
worker = new wt.Worker(workerSrc, { eval: true });
14
-
worker.on("message", eventTracker.recordEventHit);
15
-
};
16
-
17
-
export const exit = () => {
18
-
if (worker === null) return;
19
-
sendCommand("exit");
20
-
worker = null;
21
-
};
src/routes/+layout.svelte
client/src/routes/+layout.svelte
src/routes/+layout.svelte
client/src/routes/+layout.svelte
-9
src/routes/+page.server.ts
-9
src/routes/+page.server.ts
+1
-1
src/routes/+page.svelte
client/src/routes/+page.svelte
+1
-1
src/routes/+page.svelte
client/src/routes/+page.svelte
-13
src/routes/api/events/+server.ts
-13
src/routes/api/events/+server.ts
···
1
-
import { json } from "@sveltejs/kit";
2
-
import { eventTracker } from "$lib/db.js";
3
-
4
-
export const GET = async () => {
5
-
try {
6
-
const events = eventTracker.getNsidCounts();
7
-
8
-
return json({ events });
9
-
} catch (error) {
10
-
console.error("error fetching events:", error);
11
-
return json({ error: "failed to fetch events" }, { status: 500 });
12
-
}
13
-
};
static/favicon.svg
client/static/favicon.svg
static/favicon.svg
client/static/favicon.svg
+1
-1
svelte.config.js
client/svelte.config.js
+1
-1
svelte.config.js
client/svelte.config.js
tsconfig.json
client/tsconfig.json
tsconfig.json
client/tsconfig.json
vite.config.ts
client/vite.config.ts
vite.config.ts
client/vite.config.ts