Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
fix old sites table still being used
+52
-225
+12
-58
apps/main-app/src/lib/db.ts
+12
-58
apps/main-app/src/lib/db.ts
···
81
81
)
82
82
`
83
83
84
-
// Sites table - cache of place.wisp.fs records from PDS
84
+
// Legacy sites table. Main-app now uses site_cache as the authoritative runtime projection.
85
85
await db`
86
86
CREATE TABLE IF NOT EXISTS sites (
87
87
did TEXT NOT NULL,
···
339
339
}
340
340
341
341
export const getSitesByDid = async (did: string) => {
342
-
const rows = await db`SELECT * FROM sites WHERE did = ${did} ORDER BY created_at DESC`
342
+
const rows = await db`
343
+
SELECT
344
+
did,
345
+
rkey,
346
+
rkey AS display_name,
347
+
cached_at AS created_at,
348
+
updated_at
349
+
FROM site_cache
350
+
WHERE did = ${did}
351
+
ORDER BY cached_at DESC
352
+
`
343
353
return rows
344
-
}
345
-
346
-
export const upsertSite = async (did: string, rkey: string, displayName?: string) => {
347
-
try {
348
-
// Only set display_name if provided (not undefined/null/empty)
349
-
const cleanDisplayName = displayName?.trim() ? displayName.trim() : null
350
-
351
-
await db`
352
-
INSERT INTO sites (did, rkey, display_name, created_at, updated_at)
353
-
VALUES (${did}, ${rkey}, ${cleanDisplayName}, EXTRACT(EPOCH FROM NOW()), EXTRACT(EPOCH FROM NOW()))
354
-
ON CONFLICT (did, rkey)
355
-
DO UPDATE SET
356
-
display_name = CASE
357
-
WHEN EXCLUDED.display_name IS NOT NULL THEN EXCLUDED.display_name
358
-
ELSE sites.display_name
359
-
END,
360
-
updated_at = EXTRACT(EPOCH FROM NOW())
361
-
`
362
-
return { success: true }
363
-
} catch (err) {
364
-
console.error('Failed to upsert site', err)
365
-
return { success: false, error: err }
366
-
}
367
-
}
368
-
369
-
export const deleteSite = async (did: string, rkey: string) => {
370
-
try {
371
-
await db`DELETE FROM sites WHERE did = ${did} AND rkey = ${rkey}`
372
-
return { success: true }
373
-
} catch (err) {
374
-
console.error('Failed to delete site', err)
375
-
return { success: false, error: err }
376
-
}
377
-
}
378
-
379
-
export const upsertSiteCache = async (
380
-
did: string,
381
-
rkey: string,
382
-
recordCid: string,
383
-
fileCids: Record<string, string> = {},
384
-
) => {
385
-
try {
386
-
await db`
387
-
INSERT INTO site_cache (did, rkey, record_cid, file_cids, cached_at, updated_at)
388
-
VALUES (${did}, ${rkey}, ${recordCid}, ${JSON.stringify(fileCids)}, EXTRACT(EPOCH FROM NOW()), EXTRACT(EPOCH FROM NOW()))
389
-
ON CONFLICT (did, rkey)
390
-
DO UPDATE SET
391
-
record_cid = EXCLUDED.record_cid,
392
-
file_cids = EXCLUDED.file_cids,
393
-
updated_at = EXTRACT(EPOCH FROM NOW())
394
-
`
395
-
return { success: true }
396
-
} catch (err) {
397
-
console.error('Failed to upsert site cache', err)
398
-
return { success: false, error: err }
399
-
}
400
354
}
401
355
402
356
// Get all domains (wisp + custom) mapped to a specific site
+22
-6
apps/main-app/src/lib/migrations.ts
+22
-6
apps/main-app/src/lib/migrations.ts
···
98
98
{ silent: true },
99
99
)
100
100
101
-
// Ensure existing domain mappings only point to owned sites before adding FK constraints.
101
+
// Ensure existing domain mappings only point to owned cached sites before adding FK constraints.
102
102
await runMigration('normalize invalid domains.rkey mappings', async () => {
103
103
await db`
104
104
UPDATE domains d
···
106
106
WHERE rkey IS NOT NULL
107
107
AND NOT EXISTS (
108
108
SELECT 1
109
-
FROM sites s
109
+
FROM site_cache s
110
110
WHERE s.did = d.did
111
111
AND s.rkey = d.rkey
112
112
)
···
120
120
WHERE rkey IS NOT NULL
121
121
AND NOT EXISTS (
122
122
SELECT 1
123
-
FROM sites s
123
+
FROM site_cache s
124
124
WHERE s.did = d.did
125
125
AND s.rkey = d.rkey
126
126
)
···
135
135
{ silent: true },
136
136
)
137
137
138
-
// Enforce mapped site rkeys belong to same DID as mapped domain.
138
+
// Mapped site rkeys must refer to an existing cached site owned by the same DID.
139
+
await runMigration(
140
+
'drop legacy fk_domains_site_owner',
141
+
async () => {
142
+
await db`ALTER TABLE domains DROP CONSTRAINT IF EXISTS fk_domains_site_owner`
143
+
},
144
+
{ silent: true },
145
+
)
146
+
147
+
await runMigration(
148
+
'drop legacy fk_custom_domains_site_owner',
149
+
async () => {
150
+
await db`ALTER TABLE custom_domains DROP CONSTRAINT IF EXISTS fk_custom_domains_site_owner`
151
+
},
152
+
{ silent: true },
153
+
)
154
+
139
155
await runMigration(
140
156
'add fk_domains_site_owner',
141
157
async () => {
···
143
159
ALTER TABLE domains
144
160
ADD CONSTRAINT fk_domains_site_owner
145
161
FOREIGN KEY (did, rkey)
146
-
REFERENCES sites(did, rkey)
162
+
REFERENCES site_cache(did, rkey)
147
163
ON UPDATE CASCADE
148
164
ON DELETE SET NULL
149
165
`
···
158
174
ALTER TABLE custom_domains
159
175
ADD CONSTRAINT fk_custom_domains_site_owner
160
176
FOREIGN KEY (did, rkey)
161
-
REFERENCES sites(did, rkey)
177
+
REFERENCES site_cache(did, rkey)
162
178
ON UPDATE CASCADE
163
179
ON DELETE SET NULL
164
180
`
-84
apps/main-app/src/lib/sync-sites.ts
-84
apps/main-app/src/lib/sync-sites.ts
···
1
-
import { Agent } from '@atproto/api'
2
-
import type { OAuthSession } from '@atproto/oauth-client-node'
3
-
import { upsertSite } from './db'
4
-
5
-
/**
6
-
* Sync sites from user's PDS into the database cache
7
-
* - Fetches all place.wisp.fs records from AT Protocol repo
8
-
* - Validates record structure
9
-
* - Backfills into sites table
10
-
*/
11
-
export async function syncSitesFromPDS(
12
-
did: string,
13
-
session: OAuthSession,
14
-
): Promise<{ synced: number; errors: string[] }> {
15
-
console.log(`[Sync] Starting site sync for ${did}`)
16
-
17
-
const agent = new Agent((url, init) => session.fetchHandler(url, init))
18
-
const errors: string[] = []
19
-
let synced = 0
20
-
21
-
try {
22
-
// List all records in the place.wisp.fs collection
23
-
console.log('[Sync] Fetching place.wisp.fs records from PDS')
24
-
const records = await agent.com.atproto.repo.listRecords({
25
-
repo: did,
26
-
collection: 'place.wisp.fs',
27
-
limit: 100, // Adjust if users might have more sites
28
-
})
29
-
30
-
console.log(`[Sync] Found ${records.data.records.length} records`)
31
-
32
-
// Process each record
33
-
for (const record of records.data.records) {
34
-
try {
35
-
const { uri, value } = record
36
-
37
-
// Extract rkey from URI (at://did/collection/rkey)
38
-
const rkey = uri.split('/').pop()
39
-
if (!rkey) {
40
-
errors.push(`Invalid URI format: ${uri}`)
41
-
continue
42
-
}
43
-
44
-
// Validate record structure
45
-
if (!value || typeof value !== 'object') {
46
-
errors.push(`Invalid record value for ${rkey}`)
47
-
continue
48
-
}
49
-
50
-
const siteValue = value as any
51
-
52
-
// Check for required fields
53
-
if (siteValue.$type !== 'place.wisp.fs') {
54
-
errors.push(`Invalid $type for ${rkey}: ${siteValue.$type}`)
55
-
continue
56
-
}
57
-
58
-
if (!siteValue.site || typeof siteValue.site !== 'string') {
59
-
errors.push(`Missing or invalid site name for ${rkey}`)
60
-
continue
61
-
}
62
-
63
-
// Upsert into database
64
-
const displayName = siteValue.site
65
-
await upsertSite(did, rkey, displayName)
66
-
67
-
console.log(`[Sync] ✓ Synced site: ${displayName} (${rkey})`)
68
-
synced++
69
-
} catch (err) {
70
-
const errorMsg = `Error processing record: ${err instanceof Error ? err.message : 'Unknown error'}`
71
-
console.error(`[Sync] ${errorMsg}`)
72
-
errors.push(errorMsg)
73
-
}
74
-
}
75
-
76
-
console.log(`[Sync] Complete: ${synced} synced, ${errors.length} errors`)
77
-
return { synced, errors }
78
-
} catch (err) {
79
-
const errorMsg = `Failed to fetch records from PDS: ${err instanceof Error ? err.message : 'Unknown error'}`
80
-
console.error(`[Sync] ${errorMsg}`)
81
-
errors.push(errorMsg)
82
-
return { synced, errors }
83
-
}
84
-
}
+9
-9
apps/main-app/src/routes/admin.ts
+9
-9
apps/main-app/src/routes/admin.ts
···
139
139
140
140
try {
141
141
// Get total counts
142
-
const allSitesResult = await db`SELECT COUNT(*) as count FROM sites`
142
+
const allSitesResult = await db`SELECT COUNT(*) as count FROM site_cache`
143
143
const wispSubdomainsResult = await db`SELECT COUNT(*) as count FROM domains WHERE domain LIKE '%.wisp.place'`
144
144
const customDomainsResult = await db`SELECT COUNT(*) as count FROM custom_domains WHERE verified = true`
145
145
const siteCacheResult = await db`SELECT COUNT(*) as count FROM site_cache`
···
150
150
SELECT
151
151
s.did,
152
152
s.rkey,
153
-
s.display_name,
154
-
s.created_at,
153
+
s.rkey as display_name,
154
+
s.cached_at as created_at,
155
155
d.domain as subdomain,
156
156
cd.domain as custom_domain
157
-
FROM sites s
157
+
FROM site_cache s
158
158
LEFT JOIN domains d ON s.did = d.did AND s.rkey = d.rkey AND d.domain LIKE '%.wisp.place'
159
159
LEFT JOIN custom_domains cd ON s.did = cd.did AND s.rkey = cd.rkey AND cd.verified = true
160
-
ORDER BY s.created_at DESC
160
+
ORDER BY s.cached_at DESC
161
161
LIMIT 10
162
162
`
163
163
···
217
217
SELECT
218
218
s.did,
219
219
s.rkey,
220
-
s.display_name,
221
-
s.created_at,
220
+
s.rkey as display_name,
221
+
s.cached_at as created_at,
222
222
d.domain as subdomain,
223
223
cd.domain as custom_domain
224
-
FROM sites s
224
+
FROM site_cache s
225
225
LEFT JOIN domains d ON s.did = d.did AND s.rkey = d.rkey AND d.domain LIKE '%.wisp.place'
226
226
LEFT JOIN custom_domains cd ON s.did = cd.did AND s.rkey = cd.rkey AND cd.verified = true
227
-
ORDER BY s.created_at DESC
227
+
ORDER BY s.cached_at DESC
228
228
LIMIT ${limit} OFFSET ${offset}
229
229
`
230
230
+1
-15
apps/main-app/src/routes/auth.ts
+1
-15
apps/main-app/src/routes/auth.ts
···
2
2
import { createLogger } from '@wispplace/observability'
3
3
import { Elysia, t } from 'elysia'
4
4
import { getDomainByDid, getSitesByDid } from '../lib/db'
5
-
import { syncSitesFromPDS } from '../lib/sync-sites'
6
5
import { authenticateRequest } from '../lib/wisp-auth'
7
6
8
7
const logger = createLogger('main-app')
···
98
97
maxAge: 30 * 24 * 60 * 60, // 30 days
99
98
})
100
99
101
-
// Sync sites from PDS to database cache
102
-
logger.debug('[Auth] Syncing sites from PDS for', session.did as any)
103
-
try {
104
-
const syncResult = await syncSitesFromPDS(session.did, session)
105
-
logger.debug(`[Auth] Sync complete: ${syncResult.synced} sites synced`)
106
-
if (syncResult.errors.length > 0) {
107
-
logger.debug('[Auth] Sync errors:', syncResult.errors)
108
-
}
109
-
} catch (err) {
110
-
logger.error('[Auth] Failed to sync sites', err)
111
-
// Don't fail auth if sync fails, just log it
112
-
}
113
-
114
-
// Check if user has any sites or domain
100
+
// Check if user has any cached sites or a claimed domain
115
101
const sites = await getSitesByDid(session.did)
116
102
const domain = await getDomainByDid(session.did)
117
103
-7
apps/main-app/src/routes/site.ts
-7
apps/main-app/src/routes/site.ts
···
3
3
import { extractSubfsUris } from '@wispplace/atproto-utils'
4
4
import { createLogger } from '@wispplace/observability'
5
5
import { Elysia } from 'elysia'
6
-
import { deleteSite } from '../lib/db'
7
6
import { requireAuth } from '../lib/wisp-auth'
8
7
9
8
const logger = createLogger('main-app')
···
108
107
)
109
108
110
109
logger.info(`[Site] Deleted ${subfsUris.length} subfs records for ${rkey}`)
111
-
}
112
-
113
-
// Delete from database
114
-
const result = await deleteSite(auth.did, rkey)
115
-
if (!result.success) {
116
-
throw new Error('Failed to delete site from database')
117
110
}
118
111
119
112
logger.info(`[Site] Successfully deleted site ${rkey} for ${auth.did}`)
+4
-5
apps/main-app/src/routes/user.ts
+4
-5
apps/main-app/src/routes/user.ts
···
10
10
getSitesByDid,
11
11
isSupporter,
12
12
} from '../lib/db'
13
-
import { syncSitesFromPDS } from '../lib/sync-sites'
14
13
import { requireAuth } from '../lib/wisp-auth'
15
14
16
15
const logger = createLogger('main-app')
···
126
125
*/
127
126
.post('/sync', async ({ auth }) => {
128
127
try {
129
-
logger.debug('[User] Manual sync requested for', { did: auth.did })
130
-
const result = await syncSitesFromPDS(auth.did, auth.session)
128
+
logger.debug('[User] Manual site refresh requested; site availability is firehose-driven', { did: auth.did })
129
+
const sites = await getSitesByDid(auth.did)
131
130
132
131
return {
133
132
success: true,
134
-
synced: result.synced,
135
-
errors: result.errors,
133
+
synced: sites.length,
134
+
errors: [],
136
135
}
137
136
} catch (err) {
138
137
logger.error('[User] Sync error', err)
-28
apps/main-app/src/routes/wisp.ts
-28
apps/main-app/src/routes/wisp.ts
···
26
26
import type { Directory } from '@wispplace/lexicons/types/place/wisp/fs'
27
27
import { createLogger } from '@wispplace/observability'
28
28
import { Elysia } from 'elysia'
29
-
import { upsertSite } from '../lib/db'
30
29
import { createIgnoreMatcher, parseWispignore, shouldIgnore } from '../lib/ignore-patterns'
31
30
import {
32
31
addJobListener,
···
292
291
rkey: rkey,
293
292
record: emptyManifest,
294
293
})
295
-
296
-
await upsertSite(did, rkey, siteName)
297
-
298
-
// Cache the empty site for the hosting service
299
-
try {
300
-
await upsertSiteCache(did, rkey, record.data.cid, {})
301
-
} catch (err) {
302
-
// Don't fail the upload if caching fails
303
-
logger.warn('Failed to cache site', err as any)
304
-
}
305
294
306
295
completeUploadJob(jobId, {
307
296
success: true,
···
829
818
830
819
// First attempt: no base64 encoding
831
820
let record: Awaited<ReturnType<typeof agent.com.atproto.repo.putRecord>>
832
-
let finalBlobs = uploadedBlobs
833
821
try {
834
822
record = await buildManifestAndPut(uploadedBlobs)
835
823
} catch (err: any) {
···
877
865
}
878
866
}
879
867
880
-
finalBlobs = base64Blobs
881
868
record = await buildManifestAndPut(base64Blobs)
882
869
}
883
870
884
-
const rkey = siteName
885
-
886
-
// Store site in database cache
887
-
await upsertSite(did, rkey, siteName)
888
-
889
871
// Clean up old subfs records if we had any
890
872
if (oldSubfsUris.length > 0) {
891
873
console.log(`Cleaning up ${oldSubfsUris.length} old subfs records...`)
···
1103
1085
rkey: rkey,
1104
1086
record: emptyManifest,
1105
1087
})
1106
-
1107
-
await upsertSite(auth.did, rkey, siteName)
1108
-
1109
-
// Cache the empty site for the hosting service
1110
-
try {
1111
-
await upsertSiteCache(auth.did, rkey, record.data.cid, {})
1112
-
} catch (err) {
1113
-
// Don't fail the upload if caching fails
1114
-
logger.warn('Failed to cache site', err as any)
1115
-
}
1116
1088
1117
1089
return {
1118
1090
success: true,
-10
apps/main-app/src/routes/xrpc.ts
-10
apps/main-app/src/routes/xrpc.ts
···
21
21
claimCustomDomain,
22
22
claimDomain,
23
23
deleteCustomDomain,
24
-
deleteSite,
25
24
deleteWispDomain,
26
25
getAllWispDomains,
27
26
getCustomDomainInfo,
···
511
510
status: mapped.verified ? 'verified' : 'pendingVerification',
512
511
})
513
512
}
514
-
}
515
-
516
-
const deleted = await deleteSite(did, siteRkey)
517
-
if (!deleted.success) {
518
-
throw new XRPCError({
519
-
status: 500,
520
-
error: 'InternalServerError',
521
-
description: 'failed to delete site',
522
-
})
523
513
}
524
514
525
515
return json({
+4
-3
apps/webhook-service/src/lib/firehose.ts
+4
-3
apps/webhook-service/src/lib/firehose.ts
···
304
304
setInterval(() => {
305
305
const direct = directJetstream?.cursor
306
306
const backlink = backlinkJetstream?.cursor
307
-
const cursor = direct !== undefined && backlink !== undefined
308
-
? Math.max(direct, backlink)
309
-
: (direct ?? backlink ?? (isConnected ? Date.now() * 1000 : undefined))
307
+
const cursor =
308
+
direct !== undefined && backlink !== undefined
309
+
? Math.max(direct, backlink)
310
+
: (direct ?? backlink ?? (isConnected ? Date.now() * 1000 : undefined))
310
311
if (cursor !== undefined && cursor !== lastSavedCursor) {
311
312
lastSavedCursor = cursor
312
313
saveCursor(cursor, config.jetstreamUrl)