forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
hosting service now is using node.js for runtime, better lexicon validation across main app and microservice
+756
-253
-60
hosting-service/bun.lock
-60
hosting-service/bun.lock
···
1
-
{
2
-
"lockfileVersion": 1,
3
-
"workspaces": {
4
-
"": {
5
-
"name": "wisp-hosting-service",
6
-
"dependencies": {
7
-
"@atproto/api": "^0.13.20",
8
-
"@atproto/xrpc": "^0.6.4",
9
-
"hono": "^4.6.14",
10
-
"postgres": "^3.4.5",
11
-
},
12
-
"devDependencies": {
13
-
"@types/bun": "latest",
14
-
},
15
-
},
16
-
},
17
-
"packages": {
18
-
"@atproto/api": ["@atproto/api@0.13.35", "", { "dependencies": { "@atproto/common-web": "^0.4.0", "@atproto/lexicon": "^0.4.6", "@atproto/syntax": "^0.3.2", "@atproto/xrpc": "^0.6.8", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-vsEfBj0C333TLjDppvTdTE0IdKlXuljKSveAeI4PPx/l6eUKNnDTsYxvILtXUVzwUlTDmSRqy5O4Ryh78n1b7g=="],
19
-
20
-
"@atproto/common-web": ["@atproto/common-web@0.4.3", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-nRDINmSe4VycJzPo6fP/hEltBcULFxt9Kw7fQk6405FyAWZiTluYHlXOnU7GkQfeUK44OENG1qFTBcmCJ7e8pg=="],
21
-
22
-
"@atproto/lexicon": ["@atproto/lexicon@0.4.14", "", { "dependencies": { "@atproto/common-web": "^0.4.2", "@atproto/syntax": "^0.4.0", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "zod": "^3.23.8" } }, "sha512-jiKpmH1QER3Gvc7JVY5brwrfo+etFoe57tKPQX/SmPwjvUsFnJAow5xLIryuBaJgFAhnTZViXKs41t//pahGHQ=="],
23
-
24
-
"@atproto/syntax": ["@atproto/syntax@0.3.4", "", {}, "sha512-8CNmi5DipOLaVeSMPggMe7FCksVag0aO6XZy9WflbduTKM4dFZVCs4686UeMLfGRXX+X966XgwECHoLYrovMMg=="],
25
-
26
-
"@atproto/xrpc": ["@atproto/xrpc@0.6.12", "", { "dependencies": { "@atproto/lexicon": "^0.4.10", "zod": "^3.23.8" } }, "sha512-Ut3iISNLujlmY9Gu8sNU+SPDJDvqlVzWddU8qUr0Yae5oD4SguaUFjjhireMGhQ3M5E0KljQgDbTmnBo1kIZ3w=="],
27
-
28
-
"@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
29
-
30
-
"@types/node": ["@types/node@24.9.1", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg=="],
31
-
32
-
"@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="],
33
-
34
-
"await-lock": ["await-lock@2.2.2", "", {}, "sha512-aDczADvlvTGajTDjcjpJMqRkOF6Qdz3YbPZm/PyW6tKPkx2hlYBzxMhEywM/tU72HrVZjgl5VCdRuMlA7pZ8Gw=="],
35
-
36
-
"bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
37
-
38
-
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
39
-
40
-
"graphemer": ["graphemer@1.4.0", "", {}, "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="],
41
-
42
-
"hono": ["hono@4.10.2", "", {}, "sha512-p6fyzl+mQo6uhESLxbF5WlBOAJMDh36PljwlKtP5V1v09NxlqGru3ShK+4wKhSuhuYf8qxMmrivHOa/M7q0sMg=="],
43
-
44
-
"iso-datestring-validator": ["iso-datestring-validator@2.2.2", "", {}, "sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA=="],
45
-
46
-
"multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
47
-
48
-
"postgres": ["postgres@3.4.7", "", {}, "sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw=="],
49
-
50
-
"tlds": ["tlds@1.261.0", "", { "bin": { "tlds": "bin.js" } }, "sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA=="],
51
-
52
-
"uint8arrays": ["uint8arrays@3.0.0", "", { "dependencies": { "multiformats": "^9.4.2" } }, "sha512-HRCx0q6O9Bfbp+HHSfQQKD7wU70+lydKVt4EghkdOvlK/NlrF90z+eXV34mUd48rNvVJXwkrMSPpCATkct8fJA=="],
53
-
54
-
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
55
-
56
-
"zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
57
-
58
-
"@atproto/lexicon/@atproto/syntax": ["@atproto/syntax@0.4.1", "", {}, "sha512-CJdImtLAiFO+0z3BWTtxwk6aY5w4t8orHTMVJgkf++QRJWTxPbIFko/0hrkADB7n2EruDxDSeAgfUGehpH6ngw=="],
59
-
}
60
-
}
+13
-5
hosting-service/package.json
+13
-5
hosting-service/package.json
···
3
3
"version": "1.0.0",
4
4
"type": "module",
5
5
"scripts": {
6
-
"dev": "bun --watch src/index.ts",
7
-
"start": "bun src/index.ts"
6
+
"dev": "tsx watch src/index.ts",
7
+
"start": "node --loader tsx src/index.ts"
8
8
},
9
9
"dependencies": {
10
+
"@atproto/api": "^0.17.4",
11
+
"@atproto/identity": "^0.4.9",
12
+
"@atproto/lexicon": "^0.5.1",
13
+
"@atproto/sync": "^0.1.35",
14
+
"@atproto/xrpc": "^0.7.5",
15
+
"@hono/node-server": "^1.13.7",
10
16
"hono": "^4.6.14",
11
-
"@atproto/api": "^0.13.20",
12
-
"@atproto/xrpc": "^0.6.4",
17
+
"mime-types": "^2.1.35",
18
+
"multiformats": "^13.4.1",
13
19
"postgres": "^3.4.5"
14
20
},
15
21
"devDependencies": {
16
-
"@types/bun": "latest"
22
+
"@types/mime-types": "^2.1.4",
23
+
"@types/node": "^22.10.5",
24
+
"tsx": "^4.19.2"
17
25
}
18
26
}
+9
-9
hosting-service/src/index.ts
+9
-9
hosting-service/src/index.ts
···
1
-
import { serve } from 'bun';
2
-
import app from './server';
3
-
import { FirehoseWorker } from './lib/firehose';
1
+
import { serve } from '@hono/node-server';
2
+
import app from './server.js';
3
+
import { FirehoseWorker } from './lib/firehose.js';
4
4
import { mkdirSync, existsSync } from 'fs';
5
5
6
-
const PORT = process.env.PORT || 3001;
6
+
const PORT = process.env.PORT ? parseInt(process.env.PORT) : 3001;
7
7
const CACHE_DIR = './cache/sites';
8
8
9
9
// Ensure cache directory exists
···
40
40
Server: http://localhost:${PORT}
41
41
Health: http://localhost:${PORT}/health
42
42
Cache: ${CACHE_DIR}
43
-
Firehose: Connected to Jetstream
43
+
Firehose: Connected to Firehose
44
44
`);
45
45
46
46
// Graceful shutdown
47
-
process.on('SIGINT', () => {
47
+
process.on('SIGINT', async () => {
48
48
console.log('\n🛑 Shutting down...');
49
49
firehose.stop();
50
-
server.stop();
50
+
server.close();
51
51
process.exit(0);
52
52
});
53
53
54
-
process.on('SIGTERM', () => {
54
+
process.on('SIGTERM', async () => {
55
55
console.log('\n🛑 Shutting down...');
56
56
firehose.stop();
57
-
server.stop();
57
+
server.close();
58
58
process.exit(0);
59
59
});
+127
hosting-service/src/lexicon/lexicons.ts
+127
hosting-service/src/lexicon/lexicons.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
import {
5
+
type LexiconDoc,
6
+
Lexicons,
7
+
ValidationError,
8
+
type ValidationResult,
9
+
} from '@atproto/lexicon'
10
+
import { type $Typed, is$typed, maybe$typed } from './util.js'
11
+
12
+
export const schemaDict = {
13
+
PlaceWispFs: {
14
+
lexicon: 1,
15
+
id: 'place.wisp.fs',
16
+
defs: {
17
+
main: {
18
+
type: 'record',
19
+
description: 'Virtual filesystem manifest for a Wisp site',
20
+
record: {
21
+
type: 'object',
22
+
required: ['site', 'root', 'createdAt'],
23
+
properties: {
24
+
site: {
25
+
type: 'string',
26
+
},
27
+
root: {
28
+
type: 'ref',
29
+
ref: 'lex:place.wisp.fs#directory',
30
+
},
31
+
fileCount: {
32
+
type: 'integer',
33
+
minimum: 0,
34
+
maximum: 1000,
35
+
},
36
+
createdAt: {
37
+
type: 'string',
38
+
format: 'datetime',
39
+
},
40
+
},
41
+
},
42
+
},
43
+
file: {
44
+
type: 'object',
45
+
required: ['type', 'blob'],
46
+
properties: {
47
+
type: {
48
+
type: 'string',
49
+
const: 'file',
50
+
},
51
+
blob: {
52
+
type: 'blob',
53
+
accept: ['*/*'],
54
+
maxSize: 1000000,
55
+
description: 'Content blob ref',
56
+
},
57
+
},
58
+
},
59
+
directory: {
60
+
type: 'object',
61
+
required: ['type', 'entries'],
62
+
properties: {
63
+
type: {
64
+
type: 'string',
65
+
const: 'directory',
66
+
},
67
+
entries: {
68
+
type: 'array',
69
+
maxLength: 500,
70
+
items: {
71
+
type: 'ref',
72
+
ref: 'lex:place.wisp.fs#entry',
73
+
},
74
+
},
75
+
},
76
+
},
77
+
entry: {
78
+
type: 'object',
79
+
required: ['name', 'node'],
80
+
properties: {
81
+
name: {
82
+
type: 'string',
83
+
maxLength: 255,
84
+
},
85
+
node: {
86
+
type: 'union',
87
+
refs: ['lex:place.wisp.fs#file', 'lex:place.wisp.fs#directory'],
88
+
},
89
+
},
90
+
},
91
+
},
92
+
},
93
+
} as const satisfies Record<string, LexiconDoc>
94
+
export const schemas = Object.values(schemaDict) satisfies LexiconDoc[]
95
+
export const lexicons: Lexicons = new Lexicons(schemas)
96
+
97
+
export function validate<T extends { $type: string }>(
98
+
v: unknown,
99
+
id: string,
100
+
hash: string,
101
+
requiredType: true,
102
+
): ValidationResult<T>
103
+
export function validate<T extends { $type?: string }>(
104
+
v: unknown,
105
+
id: string,
106
+
hash: string,
107
+
requiredType?: false,
108
+
): ValidationResult<T>
109
+
export function validate(
110
+
v: unknown,
111
+
id: string,
112
+
hash: string,
113
+
requiredType?: boolean,
114
+
): ValidationResult {
115
+
return (requiredType ? is$typed : maybe$typed)(v, id, hash)
116
+
? lexicons.validate(`${id}#${hash}`, v)
117
+
: {
118
+
success: false,
119
+
error: new ValidationError(
120
+
`Must be an object with "${hash === 'main' ? id : `${id}#${hash}`}" $type property`,
121
+
),
122
+
}
123
+
}
124
+
125
+
export const ids = {
126
+
PlaceWispFs: 'place.wisp.fs',
127
+
} as const
+79
hosting-service/src/lexicon/types/place/wisp/fs.ts
+79
hosting-service/src/lexicon/types/place/wisp/fs.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
import { type ValidationResult, BlobRef } from '@atproto/lexicon'
5
+
import { CID } from 'multiformats/cid'
6
+
import { validate as _validate } from '../../../lexicons'
7
+
import { type $Typed, is$typed as _is$typed, type OmitKey } from '../../../util'
8
+
9
+
const is$typed = _is$typed,
10
+
validate = _validate
11
+
const id = 'place.wisp.fs'
12
+
13
+
export interface Record {
14
+
$type: 'place.wisp.fs'
15
+
site: string
16
+
root: Directory
17
+
fileCount?: number
18
+
createdAt: string
19
+
[k: string]: unknown
20
+
}
21
+
22
+
const hashRecord = 'main'
23
+
24
+
export function isRecord<V>(v: V) {
25
+
return is$typed(v, id, hashRecord)
26
+
}
27
+
28
+
export function validateRecord<V>(v: V) {
29
+
return validate<Record & V>(v, id, hashRecord, true)
30
+
}
31
+
32
+
export interface File {
33
+
$type?: 'place.wisp.fs#file'
34
+
type: 'file'
35
+
/** Content blob ref */
36
+
blob: BlobRef
37
+
}
38
+
39
+
const hashFile = 'file'
40
+
41
+
export function isFile<V>(v: V) {
42
+
return is$typed(v, id, hashFile)
43
+
}
44
+
45
+
export function validateFile<V>(v: V) {
46
+
return validate<File & V>(v, id, hashFile)
47
+
}
48
+
49
+
export interface Directory {
50
+
$type?: 'place.wisp.fs#directory'
51
+
type: 'directory'
52
+
entries: Entry[]
53
+
}
54
+
55
+
const hashDirectory = 'directory'
56
+
57
+
export function isDirectory<V>(v: V) {
58
+
return is$typed(v, id, hashDirectory)
59
+
}
60
+
61
+
export function validateDirectory<V>(v: V) {
62
+
return validate<Directory & V>(v, id, hashDirectory)
63
+
}
64
+
65
+
export interface Entry {
66
+
$type?: 'place.wisp.fs#entry'
67
+
name: string
68
+
node: $Typed<File> | $Typed<Directory> | { $type: string }
69
+
}
70
+
71
+
const hashEntry = 'entry'
72
+
73
+
export function isEntry<V>(v: V) {
74
+
return is$typed(v, id, hashEntry)
75
+
}
76
+
77
+
export function validateEntry<V>(v: V) {
78
+
return validate<Entry & V>(v, id, hashEntry)
79
+
}
+82
hosting-service/src/lexicon/util.ts
+82
hosting-service/src/lexicon/util.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
5
+
import { type ValidationResult } from '@atproto/lexicon'
6
+
7
+
export type OmitKey<T, K extends keyof T> = {
8
+
[K2 in keyof T as K2 extends K ? never : K2]: T[K2]
9
+
}
10
+
11
+
export type $Typed<V, T extends string = string> = V & { $type: T }
12
+
export type Un$Typed<V extends { $type?: string }> = OmitKey<V, '$type'>
13
+
14
+
export type $Type<Id extends string, Hash extends string> = Hash extends 'main'
15
+
? Id
16
+
: `${Id}#${Hash}`
17
+
18
+
function isObject<V>(v: V): v is V & object {
19
+
return v != null && typeof v === 'object'
20
+
}
21
+
22
+
function is$type<Id extends string, Hash extends string>(
23
+
$type: unknown,
24
+
id: Id,
25
+
hash: Hash,
26
+
): $type is $Type<Id, Hash> {
27
+
return hash === 'main'
28
+
? $type === id
29
+
: // $type === `${id}#${hash}`
30
+
typeof $type === 'string' &&
31
+
$type.length === id.length + 1 + hash.length &&
32
+
$type.charCodeAt(id.length) === 35 /* '#' */ &&
33
+
$type.startsWith(id) &&
34
+
$type.endsWith(hash)
35
+
}
36
+
37
+
export type $TypedObject<
38
+
V,
39
+
Id extends string,
40
+
Hash extends string,
41
+
> = V extends {
42
+
$type: $Type<Id, Hash>
43
+
}
44
+
? V
45
+
: V extends { $type?: string }
46
+
? V extends { $type?: infer T extends $Type<Id, Hash> }
47
+
? V & { $type: T }
48
+
: never
49
+
: V & { $type: $Type<Id, Hash> }
50
+
51
+
export function is$typed<V, Id extends string, Hash extends string>(
52
+
v: V,
53
+
id: Id,
54
+
hash: Hash,
55
+
): v is $TypedObject<V, Id, Hash> {
56
+
return isObject(v) && '$type' in v && is$type(v.$type, id, hash)
57
+
}
58
+
59
+
export function maybe$typed<V, Id extends string, Hash extends string>(
60
+
v: V,
61
+
id: Id,
62
+
hash: Hash,
63
+
): v is V & object & { $type?: $Type<Id, Hash> } {
64
+
return (
65
+
isObject(v) &&
66
+
('$type' in v ? v.$type === undefined || is$type(v.$type, id, hash) : true)
67
+
)
68
+
}
69
+
70
+
export type Validator<R = unknown> = (v: unknown) => ValidationResult<R>
71
+
export type ValidatorParam<V extends Validator> =
72
+
V extends Validator<infer R> ? R : never
73
+
74
+
/**
75
+
* Utility function that allows to convert a "validate*" utility function into a
76
+
* type predicate.
77
+
*/
78
+
export function asPredicate<V extends Validator>(validate: V) {
79
+
return function <T>(v: T): v is T & ValidatorParam<V> {
80
+
return validate(v).success
81
+
}
82
+
}
+70
-163
hosting-service/src/lib/firehose.ts
+70
-163
hosting-service/src/lib/firehose.ts
···
1
1
import { existsSync, rmSync } from 'fs';
2
-
import type { WispFsRecord } from './types';
3
2
import { getPdsForDid, downloadAndCacheSite, extractBlobCid, fetchSiteRecord } from './utils';
4
3
import { upsertSite } from './db';
5
4
import { safeFetch } from './safe-fetch';
5
+
import { isRecord, validateRecord } from '../lexicon/types/place/wisp/fs';
6
+
import { Firehose } from '@atproto/sync';
7
+
import { IdResolver } from '@atproto/identity';
6
8
7
9
const CACHE_DIR = './cache/sites';
8
-
const JETSTREAM_URL = 'wss://jetstream2.us-west.bsky.network/subscribe';
9
-
const RECONNECT_DELAY = 5000; // 5 seconds
10
-
const MAX_RECONNECT_DELAY = 60000; // 1 minute
11
-
12
-
interface JetstreamCommitEvent {
13
-
did: string;
14
-
time_us: number;
15
-
type: 'com' | 'identity' | 'account';
16
-
kind: 'commit';
17
-
commit: {
18
-
rev: string;
19
-
operation: 'create' | 'update' | 'delete';
20
-
collection: string;
21
-
rkey: string;
22
-
record?: any;
23
-
cid?: string;
24
-
};
25
-
}
26
-
27
-
interface JetstreamIdentityEvent {
28
-
did: string;
29
-
time_us: number;
30
-
type: 'identity';
31
-
kind: 'update';
32
-
identity: {
33
-
did: string;
34
-
handle: string;
35
-
seq: number;
36
-
time: string;
37
-
};
38
-
}
39
-
40
-
interface JetstreamAccountEvent {
41
-
did: string;
42
-
time_us: number;
43
-
type: 'account';
44
-
kind: 'update' | 'delete';
45
-
account: {
46
-
active: boolean;
47
-
did: string;
48
-
seq: number;
49
-
time: string;
50
-
};
51
-
}
52
-
53
-
type JetstreamEvent =
54
-
| JetstreamCommitEvent
55
-
| JetstreamIdentityEvent
56
-
| JetstreamAccountEvent;
57
10
58
11
export class FirehoseWorker {
59
-
private ws: WebSocket | null = null;
60
-
private reconnectAttempts = 0;
61
-
private reconnectTimeout: Timer | null = null;
12
+
private firehose: Firehose | null = null;
13
+
private idResolver: IdResolver;
62
14
private isShuttingDown = false;
63
15
private lastEventTime = Date.now();
64
16
65
17
constructor(
66
18
private logger?: (msg: string, data?: Record<string, unknown>) => void,
67
-
) {}
19
+
) {
20
+
this.idResolver = new IdResolver();
21
+
}
68
22
69
23
private log(msg: string, data?: Record<string, unknown>) {
70
24
const log = this.logger || console.log;
···
80
34
this.log('Stopping firehose worker');
81
35
this.isShuttingDown = true;
82
36
83
-
if (this.reconnectTimeout) {
84
-
clearTimeout(this.reconnectTimeout);
85
-
this.reconnectTimeout = null;
86
-
}
87
-
88
-
if (this.ws) {
89
-
this.ws.close();
90
-
this.ws = null;
37
+
if (this.firehose) {
38
+
this.firehose.destroy();
39
+
this.firehose = null;
91
40
}
92
41
}
93
42
94
43
private connect() {
95
44
if (this.isShuttingDown) return;
96
45
97
-
const url = new URL(JETSTREAM_URL);
98
-
url.searchParams.set('wantedCollections', 'place.wisp.fs');
46
+
this.log('Connecting to AT Protocol firehose');
99
47
100
-
this.log('Connecting to Jetstream', { url: url.toString() });
48
+
this.firehose = new Firehose({
49
+
idResolver: this.idResolver,
50
+
service: 'wss://bsky.network',
51
+
filterCollections: ['place.wisp.fs'],
52
+
handleEvent: async (evt) => {
53
+
this.lastEventTime = Date.now();
101
54
102
-
try {
103
-
this.ws = new WebSocket(url.toString());
55
+
// Watch for write events
56
+
if (evt.event === 'create' || evt.event === 'update') {
57
+
const record = evt.record;
104
58
105
-
this.ws.onopen = () => {
106
-
this.log('Connected to Jetstream');
107
-
this.reconnectAttempts = 0;
108
-
this.lastEventTime = Date.now();
109
-
};
59
+
// If the write is a valid place.wisp.fs record
60
+
if (
61
+
evt.collection === 'place.wisp.fs' &&
62
+
isRecord(record) &&
63
+
validateRecord(record).success
64
+
) {
65
+
this.log('Received place.wisp.fs event', {
66
+
did: evt.did,
67
+
event: evt.event,
68
+
rkey: evt.rkey,
69
+
});
110
70
111
-
this.ws.onmessage = async (event) => {
112
-
this.lastEventTime = Date.now();
113
-
114
-
try {
115
-
const data = JSON.parse(event.data as string) as JetstreamEvent;
116
-
await this.handleEvent(data);
117
-
} catch (err) {
118
-
this.log('Error processing event', {
119
-
error: err instanceof Error ? err.message : String(err),
71
+
try {
72
+
await this.handleCreateOrUpdate(evt.did, evt.rkey, record, evt.cid?.toString());
73
+
} catch (err) {
74
+
this.log('Error handling event', {
75
+
did: evt.did,
76
+
event: evt.event,
77
+
rkey: evt.rkey,
78
+
error: err instanceof Error ? err.message : String(err),
79
+
});
80
+
}
81
+
}
82
+
} else if (evt.event === 'delete' && evt.collection === 'place.wisp.fs') {
83
+
this.log('Received delete event', {
84
+
did: evt.did,
85
+
rkey: evt.rkey,
120
86
});
121
-
}
122
-
};
123
87
124
-
this.ws.onerror = (error) => {
125
-
this.log('WebSocket error', { error: String(error) });
126
-
};
127
-
128
-
this.ws.onclose = () => {
129
-
this.log('WebSocket closed');
130
-
this.ws = null;
131
-
132
-
if (!this.isShuttingDown) {
133
-
this.scheduleReconnect();
88
+
try {
89
+
await this.handleDelete(evt.did, evt.rkey);
90
+
} catch (err) {
91
+
this.log('Error handling delete', {
92
+
did: evt.did,
93
+
rkey: evt.rkey,
94
+
error: err instanceof Error ? err.message : String(err),
95
+
});
96
+
}
134
97
}
135
-
};
136
-
} catch (err) {
137
-
this.log('Failed to create WebSocket', {
138
-
error: err instanceof Error ? err.message : String(err),
139
-
});
140
-
this.scheduleReconnect();
141
-
}
142
-
}
143
-
144
-
private scheduleReconnect() {
145
-
if (this.isShuttingDown) return;
146
-
147
-
this.reconnectAttempts++;
148
-
const delay = Math.min(
149
-
RECONNECT_DELAY * Math.pow(2, this.reconnectAttempts - 1),
150
-
MAX_RECONNECT_DELAY,
151
-
);
152
-
153
-
this.log(`Scheduling reconnect attempt ${this.reconnectAttempts}`, {
154
-
delay: `${delay}ms`,
155
-
});
156
-
157
-
this.reconnectTimeout = setTimeout(() => {
158
-
this.connect();
159
-
}, delay);
160
-
}
161
-
162
-
private async handleEvent(event: JetstreamEvent) {
163
-
if (event.kind !== 'commit') return;
164
-
165
-
const commitEvent = event as JetstreamCommitEvent;
166
-
const { commit, did } = commitEvent;
167
-
168
-
if (commit.collection !== 'place.wisp.fs') return;
169
-
170
-
this.log('Received place.wisp.fs event', {
171
-
did,
172
-
operation: commit.operation,
173
-
rkey: commit.rkey,
98
+
},
99
+
onError: (err) => {
100
+
this.log('Firehose error', {
101
+
error: err instanceof Error ? err.message : String(err),
102
+
stack: err instanceof Error ? err.stack : undefined,
103
+
fullError: err,
104
+
});
105
+
console.error('Full firehose error:', err);
106
+
},
174
107
});
175
108
176
-
try {
177
-
if (commit.operation === 'create' || commit.operation === 'update') {
178
-
// Pass the CID from the event for verification
179
-
await this.handleCreateOrUpdate(did, commit.rkey, commit.record, commit.cid);
180
-
} else if (commit.operation === 'delete') {
181
-
await this.handleDelete(did, commit.rkey);
182
-
}
183
-
} catch (err) {
184
-
this.log('Error handling event', {
185
-
did,
186
-
operation: commit.operation,
187
-
rkey: commit.rkey,
188
-
error: err instanceof Error ? err.message : String(err),
189
-
});
190
-
}
109
+
this.firehose.start();
110
+
this.log('Firehose started');
191
111
}
192
112
193
113
private async handleCreateOrUpdate(did: string, site: string, record: any, eventCid?: string) {
194
114
this.log('Processing create/update', { did, site });
195
115
196
-
if (!this.validateRecord(record)) {
197
-
this.log('Invalid record structure, skipping', { did, site });
198
-
return;
199
-
}
200
-
201
-
const fsRecord = record as WispFsRecord;
116
+
// Record is already validated in handleEvent
117
+
const fsRecord = record;
202
118
203
119
const pdsEndpoint = await getPdsForDid(did);
204
120
if (!pdsEndpoint) {
···
291
207
this.log('Successfully processed delete', { did, site });
292
208
}
293
209
294
-
private validateRecord(record: any): boolean {
295
-
if (!record || typeof record !== 'object') return false;
296
-
if (record.$type !== 'place.wisp.fs') return false;
297
-
if (!record.root || typeof record.root !== 'object') return false;
298
-
if (!record.site || typeof record.site !== 'string') return false;
299
-
return true;
300
-
}
301
-
302
210
private deleteCache(did: string, site: string) {
303
211
const cacheDir = `${CACHE_DIR}/${did}/${site}`;
304
212
···
324
232
}
325
233
326
234
getHealth() {
327
-
const isConnected = this.ws !== null && this.ws.readyState === WebSocket.OPEN;
235
+
const isConnected = this.firehose !== null;
328
236
const timeSinceLastEvent = Date.now() - this.lastEventTime;
329
237
330
238
return {
331
239
connected: isConnected,
332
-
reconnectAttempts: this.reconnectAttempts,
333
240
lastEventTime: this.lastEventTime,
334
241
timeSinceLastEvent,
335
242
healthy: isConnected && timeSinceLastEvent < 300000, // 5 minutes
+15
-14
hosting-service/src/server.ts
+15
-14
hosting-service/src/server.ts
···
1
1
import { Hono } from 'hono';
2
-
import { serveStatic } from 'hono/bun';
3
2
import { getWispDomain, getCustomDomain, getCustomDomainByHash } from './lib/db';
4
3
import { resolveDid, getPdsForDid, fetchSiteRecord, downloadAndCacheSite, getCachedFilePath, isCached, sanitizePath } from './lib/utils';
5
4
import { rewriteHtmlPaths, isHtmlContent } from './lib/html-rewriter';
6
-
import { existsSync } from 'fs';
5
+
import { existsSync, readFileSync } from 'fs';
6
+
import { lookup } from 'mime-types';
7
7
8
8
const app = new Hono();
9
9
···
33
33
const cachedFile = getCachedFilePath(did, rkey, requestPath);
34
34
35
35
if (existsSync(cachedFile)) {
36
-
const file = Bun.file(cachedFile);
37
-
return new Response(file, {
36
+
const content = readFileSync(cachedFile);
37
+
const mimeType = lookup(cachedFile) || 'application/octet-stream';
38
+
return new Response(content, {
38
39
headers: {
39
-
'Content-Type': file.type || 'application/octet-stream',
40
+
'Content-Type': mimeType,
40
41
},
41
42
});
42
43
}
···
45
46
if (!requestPath.includes('.')) {
46
47
const indexFile = getCachedFilePath(did, rkey, `${requestPath}/index.html`);
47
48
if (existsSync(indexFile)) {
48
-
const file = Bun.file(indexFile);
49
-
return new Response(file, {
49
+
const content = readFileSync(indexFile);
50
+
return new Response(content, {
50
51
headers: {
51
52
'Content-Type': 'text/html; charset=utf-8',
52
53
},
···
73
74
const cachedFile = getCachedFilePath(did, rkey, requestPath);
74
75
75
76
if (existsSync(cachedFile)) {
76
-
const file = Bun.file(cachedFile);
77
+
const mimeType = lookup(cachedFile) || 'application/octet-stream';
77
78
78
79
// Check if this is HTML content that needs rewriting
79
-
if (isHtmlContent(requestPath, file.type)) {
80
-
const content = await file.text();
80
+
if (isHtmlContent(requestPath, mimeType)) {
81
+
const content = readFileSync(cachedFile, 'utf-8');
81
82
const rewritten = rewriteHtmlPaths(content, basePath);
82
83
return new Response(rewritten, {
83
84
headers: {
···
87
88
}
88
89
89
90
// Non-HTML files served with proper MIME type
90
-
return new Response(file, {
91
+
const content = readFileSync(cachedFile);
92
+
return new Response(content, {
91
93
headers: {
92
-
'Content-Type': file.type || 'application/octet-stream',
94
+
'Content-Type': mimeType,
93
95
},
94
96
});
95
97
}
···
98
100
if (!requestPath.includes('.')) {
99
101
const indexFile = getCachedFilePath(did, rkey, `${requestPath}/index.html`);
100
102
if (existsSync(indexFile)) {
101
-
const file = Bun.file(indexFile);
102
-
const content = await file.text();
103
+
const content = readFileSync(indexFile, 'utf-8');
103
104
const rewritten = rewriteHtmlPaths(content, basePath);
104
105
return new Response(rewritten, {
105
106
headers: {
-1
src/index.ts
-1
src/index.ts
+44
src/lexicons/index.ts
+44
src/lexicons/index.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
import {
5
+
type Auth,
6
+
type Options as XrpcOptions,
7
+
Server as XrpcServer,
8
+
type StreamConfigOrHandler,
9
+
type MethodConfigOrHandler,
10
+
createServer as createXrpcServer,
11
+
} from '@atproto/xrpc-server'
12
+
import { schemas } from './lexicons.js'
13
+
14
+
export function createServer(options?: XrpcOptions): Server {
15
+
return new Server(options)
16
+
}
17
+
18
+
export class Server {
19
+
xrpc: XrpcServer
20
+
place: PlaceNS
21
+
22
+
constructor(options?: XrpcOptions) {
23
+
this.xrpc = createXrpcServer(schemas, options)
24
+
this.place = new PlaceNS(this)
25
+
}
26
+
}
27
+
28
+
export class PlaceNS {
29
+
_server: Server
30
+
wisp: PlaceWispNS
31
+
32
+
constructor(server: Server) {
33
+
this._server = server
34
+
this.wisp = new PlaceWispNS(server)
35
+
}
36
+
}
37
+
38
+
export class PlaceWispNS {
39
+
_server: Server
40
+
41
+
constructor(server: Server) {
42
+
this._server = server
43
+
}
44
+
}
+127
src/lexicons/lexicons.ts
+127
src/lexicons/lexicons.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
import {
5
+
type LexiconDoc,
6
+
Lexicons,
7
+
ValidationError,
8
+
type ValidationResult,
9
+
} from '@atproto/lexicon'
10
+
import { type $Typed, is$typed, maybe$typed } from './util.js'
11
+
12
+
export const schemaDict = {
13
+
PlaceWispFs: {
14
+
lexicon: 1,
15
+
id: 'place.wisp.fs',
16
+
defs: {
17
+
main: {
18
+
type: 'record',
19
+
description: 'Virtual filesystem manifest for a Wisp site',
20
+
record: {
21
+
type: 'object',
22
+
required: ['site', 'root', 'createdAt'],
23
+
properties: {
24
+
site: {
25
+
type: 'string',
26
+
},
27
+
root: {
28
+
type: 'ref',
29
+
ref: 'lex:place.wisp.fs#directory',
30
+
},
31
+
fileCount: {
32
+
type: 'integer',
33
+
minimum: 0,
34
+
maximum: 1000,
35
+
},
36
+
createdAt: {
37
+
type: 'string',
38
+
format: 'datetime',
39
+
},
40
+
},
41
+
},
42
+
},
43
+
file: {
44
+
type: 'object',
45
+
required: ['type', 'blob'],
46
+
properties: {
47
+
type: {
48
+
type: 'string',
49
+
const: 'file',
50
+
},
51
+
blob: {
52
+
type: 'blob',
53
+
accept: ['*/*'],
54
+
maxSize: 1000000,
55
+
description: 'Content blob ref',
56
+
},
57
+
},
58
+
},
59
+
directory: {
60
+
type: 'object',
61
+
required: ['type', 'entries'],
62
+
properties: {
63
+
type: {
64
+
type: 'string',
65
+
const: 'directory',
66
+
},
67
+
entries: {
68
+
type: 'array',
69
+
maxLength: 500,
70
+
items: {
71
+
type: 'ref',
72
+
ref: 'lex:place.wisp.fs#entry',
73
+
},
74
+
},
75
+
},
76
+
},
77
+
entry: {
78
+
type: 'object',
79
+
required: ['name', 'node'],
80
+
properties: {
81
+
name: {
82
+
type: 'string',
83
+
maxLength: 255,
84
+
},
85
+
node: {
86
+
type: 'union',
87
+
refs: ['lex:place.wisp.fs#file', 'lex:place.wisp.fs#directory'],
88
+
},
89
+
},
90
+
},
91
+
},
92
+
},
93
+
} as const satisfies Record<string, LexiconDoc>
94
+
export const schemas = Object.values(schemaDict) satisfies LexiconDoc[]
95
+
export const lexicons: Lexicons = new Lexicons(schemas)
96
+
97
+
export function validate<T extends { $type: string }>(
98
+
v: unknown,
99
+
id: string,
100
+
hash: string,
101
+
requiredType: true,
102
+
): ValidationResult<T>
103
+
export function validate<T extends { $type?: string }>(
104
+
v: unknown,
105
+
id: string,
106
+
hash: string,
107
+
requiredType?: false,
108
+
): ValidationResult<T>
109
+
export function validate(
110
+
v: unknown,
111
+
id: string,
112
+
hash: string,
113
+
requiredType?: boolean,
114
+
): ValidationResult {
115
+
return (requiredType ? is$typed : maybe$typed)(v, id, hash)
116
+
? lexicons.validate(`${id}#${hash}`, v)
117
+
: {
118
+
success: false,
119
+
error: new ValidationError(
120
+
`Must be an object with "${hash === 'main' ? id : `${id}#${hash}`}" $type property`,
121
+
),
122
+
}
123
+
}
124
+
125
+
export const ids = {
126
+
PlaceWispFs: 'place.wisp.fs',
127
+
} as const
+85
src/lexicons/types/place/wisp/fs.ts
+85
src/lexicons/types/place/wisp/fs.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
import { type ValidationResult, BlobRef } from '@atproto/lexicon'
5
+
import { CID } from 'multiformats/cid'
6
+
import { validate as _validate } from '../../../lexicons'
7
+
import { type $Typed, is$typed as _is$typed, type OmitKey } from '../../../util'
8
+
9
+
const is$typed = _is$typed,
10
+
validate = _validate
11
+
const id = 'place.wisp.fs'
12
+
13
+
export interface Main {
14
+
$type: 'place.wisp.fs'
15
+
site: string
16
+
root: Directory
17
+
fileCount?: number
18
+
createdAt: string
19
+
[k: string]: unknown
20
+
}
21
+
22
+
const hashMain = 'main'
23
+
24
+
export function isMain<V>(v: V) {
25
+
return is$typed(v, id, hashMain)
26
+
}
27
+
28
+
export function validateMain<V>(v: V) {
29
+
return validate<Main & V>(v, id, hashMain, true)
30
+
}
31
+
32
+
export {
33
+
type Main as Record,
34
+
isMain as isRecord,
35
+
validateMain as validateRecord,
36
+
}
37
+
38
+
export interface File {
39
+
$type?: 'place.wisp.fs#file'
40
+
type: 'file'
41
+
/** Content blob ref */
42
+
blob: BlobRef
43
+
}
44
+
45
+
const hashFile = 'file'
46
+
47
+
export function isFile<V>(v: V) {
48
+
return is$typed(v, id, hashFile)
49
+
}
50
+
51
+
export function validateFile<V>(v: V) {
52
+
return validate<File & V>(v, id, hashFile)
53
+
}
54
+
55
+
export interface Directory {
56
+
$type?: 'place.wisp.fs#directory'
57
+
type: 'directory'
58
+
entries: Entry[]
59
+
}
60
+
61
+
const hashDirectory = 'directory'
62
+
63
+
export function isDirectory<V>(v: V) {
64
+
return is$typed(v, id, hashDirectory)
65
+
}
66
+
67
+
export function validateDirectory<V>(v: V) {
68
+
return validate<Directory & V>(v, id, hashDirectory)
69
+
}
70
+
71
+
export interface Entry {
72
+
$type?: 'place.wisp.fs#entry'
73
+
name: string
74
+
node: $Typed<File> | $Typed<Directory> | { $type: string }
75
+
}
76
+
77
+
const hashEntry = 'entry'
78
+
79
+
export function isEntry<V>(v: V) {
80
+
return is$typed(v, id, hashEntry)
81
+
}
82
+
83
+
export function validateEntry<V>(v: V) {
84
+
return validate<Entry & V>(v, id, hashEntry)
85
+
}
+82
src/lexicons/util.ts
+82
src/lexicons/util.ts
···
1
+
/**
2
+
* GENERATED CODE - DO NOT MODIFY
3
+
*/
4
+
5
+
import { type ValidationResult } from '@atproto/lexicon'
6
+
7
+
export type OmitKey<T, K extends keyof T> = {
8
+
[K2 in keyof T as K2 extends K ? never : K2]: T[K2]
9
+
}
10
+
11
+
export type $Typed<V, T extends string = string> = V & { $type: T }
12
+
export type Un$Typed<V extends { $type?: string }> = OmitKey<V, '$type'>
13
+
14
+
export type $Type<Id extends string, Hash extends string> = Hash extends 'main'
15
+
? Id
16
+
: `${Id}#${Hash}`
17
+
18
+
function isObject<V>(v: V): v is V & object {
19
+
return v != null && typeof v === 'object'
20
+
}
21
+
22
+
function is$type<Id extends string, Hash extends string>(
23
+
$type: unknown,
24
+
id: Id,
25
+
hash: Hash,
26
+
): $type is $Type<Id, Hash> {
27
+
return hash === 'main'
28
+
? $type === id
29
+
: // $type === `${id}#${hash}`
30
+
typeof $type === 'string' &&
31
+
$type.length === id.length + 1 + hash.length &&
32
+
$type.charCodeAt(id.length) === 35 /* '#' */ &&
33
+
$type.startsWith(id) &&
34
+
$type.endsWith(hash)
35
+
}
36
+
37
+
export type $TypedObject<
38
+
V,
39
+
Id extends string,
40
+
Hash extends string,
41
+
> = V extends {
42
+
$type: $Type<Id, Hash>
43
+
}
44
+
? V
45
+
: V extends { $type?: string }
46
+
? V extends { $type?: infer T extends $Type<Id, Hash> }
47
+
? V & { $type: T }
48
+
: never
49
+
: V & { $type: $Type<Id, Hash> }
50
+
51
+
export function is$typed<V, Id extends string, Hash extends string>(
52
+
v: V,
53
+
id: Id,
54
+
hash: Hash,
55
+
): v is $TypedObject<V, Id, Hash> {
56
+
return isObject(v) && '$type' in v && is$type(v.$type, id, hash)
57
+
}
58
+
59
+
export function maybe$typed<V, Id extends string, Hash extends string>(
60
+
v: V,
61
+
id: Id,
62
+
hash: Hash,
63
+
): v is V & object & { $type?: $Type<Id, Hash> } {
64
+
return (
65
+
isObject(v) &&
66
+
('$type' in v ? v.$type === undefined || is$type(v.$type, id, hash) : true)
67
+
)
68
+
}
69
+
70
+
export type Validator<R = unknown> = (v: unknown) => ValidationResult<R>
71
+
export type ValidatorParam<V extends Validator> =
72
+
V extends Validator<infer R> ? R : never
73
+
74
+
/**
75
+
* Utility function that allows to convert a "validate*" utility function into a
76
+
* type predicate.
77
+
*/
78
+
export function asPredicate<V extends Validator>(validate: V) {
79
+
return function <T>(v: T): v is T & ValidatorParam<V> {
80
+
return validate(v).success
81
+
}
82
+
}
+10
-1
src/lib/wisp-utils.ts
+10
-1
src/lib/wisp-utils.ts
···
1
1
import type { BlobRef } from "@atproto/api";
2
2
import type { Record, Directory, File, Entry } from "../lexicon/types/place/wisp/fs";
3
+
import { validateRecord } from "../lexicon/types/place/wisp/fs";
3
4
4
5
export interface UploadedFile {
5
6
name: string;
···
126
127
root: Directory,
127
128
fileCount: number
128
129
): Record {
129
-
return {
130
+
const manifest = {
130
131
$type: 'place.wisp.fs' as const,
131
132
site: siteName,
132
133
root,
133
134
fileCount,
134
135
createdAt: new Date().toISOString()
135
136
};
137
+
138
+
// Validate the manifest before returning
139
+
const validationResult = validateRecord(manifest);
140
+
if (!validationResult.success) {
141
+
throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`);
142
+
}
143
+
144
+
return manifest;
136
145
}
137
146
138
147
/**
+13
src/routes/wisp.ts
+13
src/routes/wisp.ts
···
11
11
} from '../lib/wisp-utils'
12
12
import { upsertSite } from '../lib/db'
13
13
import { logger } from '../lib/logger'
14
+
import { validateRecord } from '../lexicon/types/place/wisp/fs'
14
15
15
16
function isValidSiteName(siteName: string): boolean {
16
17
if (!siteName || typeof siteName !== 'string') return false;
···
72
73
fileCount: 0,
73
74
createdAt: new Date().toISOString()
74
75
};
76
+
77
+
// Validate the manifest
78
+
const validationResult = validateRecord(emptyManifest);
79
+
if (!validationResult.success) {
80
+
throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`);
81
+
}
75
82
76
83
// Use site name as rkey
77
84
const rkey = siteName;
···
186
193
fileCount: 0,
187
194
createdAt: new Date().toISOString()
188
195
};
196
+
197
+
// Validate the manifest
198
+
const validationResult = validateRecord(emptyManifest);
199
+
if (!validationResult.success) {
200
+
throw new Error(`Invalid manifest: ${validationResult.error?.message || 'Validation failed'}`);
201
+
}
189
202
190
203
// Use site name as rkey
191
204
const rkey = siteName;