forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
finish work in removing old sites table codepaths
+44
-108
+20
-27
apps/firehose-service/src/index.ts
+20
-27
apps/firehose-service/src/index.ts
···
3
3
*
4
4
* Modes:
5
5
* - Normal: Watch firehose for place.wisp.fs events
6
-
* - Backfill: Process existing sites from database
7
-
* - DB Fill Only: Collect DIDs and backfill sites table (skip S3 writes)
6
+
* - Backfill: Process existing sites discovered from known DIDs
7
+
* - DB Fill Only: Legacy mode; no-op now that site_cache is the canonical projection
8
8
*/
9
9
10
10
import { serve } from '@hono/node-server'
···
14
14
import { config } from './config'
15
15
import { closeCacheInvalidationPublisher } from './lib/cache-invalidation'
16
16
import { fetchSiteRecord, handleSiteCreateOrUpdate, listSiteRecordsForDid } from './lib/cache-writer'
17
-
import { closeDatabase, getSiteCache, listAllKnownDids, listAllSiteCaches, listAllSites, upsertSite } from './lib/db'
17
+
import { closeDatabase, getSiteCache, listAllKnownDids } from './lib/db'
18
18
import { getActiveService, getCurrentSeq, getFirehoseHealth, startFirehose, stopFirehose } from './lib/firehose'
19
19
import { closeLeaderRedis, getLeaderInfo, releaseLeadership, runLeaderElection, saveCursor } from './lib/leader'
20
20
import { startRevalidateWorker, stopRevalidateWorker } from './lib/revalidate-worker'
···
80
80
/**
81
81
* Backfill phase 1+2:
82
82
* - Collect all known DIDs from DB
83
-
* - Backfill each DID's place.wisp.fs records into the sites table
83
+
* - Discover each DID's place.wisp.fs records directly from the PDS
84
84
*/
85
-
async function backfillSitesTableFromKnownDids(): Promise<void> {
85
+
async function collectSitesFromKnownDids(): Promise<Array<{ did: string; rkey: string }>> {
86
86
logger.info('Phase 1/3: Collecting known DIDs')
87
87
const dids = await listAllKnownDids()
88
88
logger.info(`Collected ${dids.length} known DIDs`)
89
89
90
90
if (dids.length === 0) {
91
-
logger.warn('No known DIDs found; skipping sites table backfill')
92
-
return
91
+
logger.warn('No known DIDs found; skipping site discovery')
92
+
return []
93
93
}
94
94
95
-
logger.info('Phase 2/3: Backfilling place.wisp.fs records into sites table')
95
+
logger.info('Phase 2/3: Discovering place.wisp.fs records from known DIDs')
96
96
97
97
let didsProcessed = 0
98
98
let didsFailed = 0
99
-
let sitesSynced = 0
99
+
let sitesDiscovered = 0
100
100
let sitesFailed = 0
101
+
const discoveredSites = new Map<string, { did: string; rkey: string }>()
101
102
102
103
const concurrency = config.backfillConcurrency
103
104
···
106
107
const records = await listSiteRecordsForDid(did)
107
108
for (const row of records) {
108
109
try {
109
-
const siteName =
110
-
typeof row.record.site === 'string' && row.record.site.length > 0 ? row.record.site : row.rkey
111
-
await upsertSite(did, row.rkey, siteName)
112
-
sitesSynced++
110
+
discoveredSites.set(`${did}:${row.rkey}`, { did, rkey: row.rkey })
111
+
sitesDiscovered++
113
112
} catch (err) {
114
-
logger.error(`[Backfill:sites] Failed to upsert site ${did}/${row.rkey}`, err)
113
+
logger.error(`[Backfill:sites] Failed to register site ${did}/${row.rkey}`, err)
115
114
sitesFailed++
116
115
}
117
116
}
···
121
120
didsFailed++
122
121
}
123
122
logger.info(
124
-
`[Backfill:sites] Progress ${didsProcessed + didsFailed}/${dids.length} DIDs (${sitesSynced} sites synced, ${sitesFailed} sites failed)`,
123
+
`[Backfill:sites] Progress ${didsProcessed + didsFailed}/${dids.length} DIDs (${sitesDiscovered} sites discovered, ${sitesFailed} sites failed)`,
125
124
)
126
125
}
127
126
···
138
137
await Promise.all(inFlight)
139
138
140
139
logger.info(
141
-
`Phase 2/3 complete: ${didsProcessed} DIDs processed, ${didsFailed} DIDs failed, ${sitesSynced} sites synced, ${sitesFailed} sites failed`,
140
+
`Phase 2/3 complete: ${didsProcessed} DIDs processed, ${didsFailed} DIDs failed, ${discoveredSites.size} unique sites discovered, ${sitesFailed} sites failed`,
142
141
)
142
+
return [...discoveredSites.values()]
143
143
}
144
144
145
145
/**
146
146
* Backfill phase 3:
147
-
* - process sites from database and backfill blobs into S3
147
+
* - process discovered sites and backfill blobs into S3
148
148
*/
149
149
async function runBackfill(): Promise<void> {
150
150
logger.info('Starting backfill mode')
···
159
159
logger.info('Forcing full file download/write for all backfilled sites')
160
160
}
161
161
162
-
await backfillSitesTableFromKnownDids()
162
+
const sites = await collectSitesFromKnownDids()
163
163
164
164
if (config.isDbFillOnly) {
165
-
logger.info('DB fill only mode complete; skipping phase 3/3 S3 backfill')
165
+
logger.info('DB fill only mode enabled; skipping phase 3/3 cache backfill')
166
166
return
167
167
}
168
168
169
169
logger.info('Phase 3/3: Backfilling site blobs into S3')
170
170
171
-
let sites = await listAllSites()
172
-
if (sites.length === 0) {
173
-
const cachedSites = await listAllSiteCaches()
174
-
sites = cachedSites.map((site) => ({ did: site.did, rkey: site.rkey }))
175
-
logger.info('Sites table empty; falling back to site_cache entries')
176
-
}
177
-
178
171
const concurrency = config.backfillConcurrency
179
-
logger.info(`Found ${sites.length} sites in database (concurrency: ${concurrency})`)
172
+
logger.info(`Found ${sites.length} sites to process (concurrency: ${concurrency})`)
180
173
181
174
let processed = 0
182
175
let skipped = 0
-4
apps/firehose-service/src/lib/cache-writer.ts
-4
apps/firehose-service/src/lib/cache-writer.ts
···
16
16
import { safeFetchBlob, safeFetchJson } from '@wispplace/safe-fetch'
17
17
import { publishCacheInvalidation } from './cache-invalidation'
18
18
import {
19
-
deleteSite,
20
19
deleteSiteCache,
21
20
deleteSiteSettingsCache,
22
21
getSiteCache,
23
22
isSupporter,
24
-
upsertSite,
25
23
upsertSiteCache,
26
24
upsertSiteSettingsCache,
27
25
} from './db'
···
826
824
// Update DB with new CIDs
827
825
logger.debug(`About to upsert site cache for ${did}/${rkey}`)
828
826
await upsertSiteCache(did, rkey, recordCid, newFileCids)
829
-
await upsertSite(did, rkey, record.site)
830
827
logger.debug(`Updated site cache for ${did}/${rkey} with record CID ${recordCid}`)
831
828
832
829
// Backfill settings if a record exists for this rkey
···
864
861
865
862
// Delete from DB
866
863
await deleteSiteCache(did, rkey)
867
-
await deleteSite(did, rkey)
868
864
869
865
// Notify hosting-service to invalidate its local caches
870
866
await publishCacheInvalidation(did, rkey, 'delete')
+1
-32
apps/firehose-service/src/lib/db.ts
+1
-32
apps/firehose-service/src/lib/db.ts
···
1
-
import type { SiteCache, SiteRecord, SiteSettingsCache } from '@wispplace/database'
1
+
import type { SiteCache, SiteSettingsCache } from '@wispplace/database'
2
2
import { createLogger } from '@wispplace/observability'
3
3
import postgres from 'postgres'
4
4
import { config } from '../config'
···
41
41
`
42
42
}
43
43
44
-
export async function listAllSites(): Promise<SiteRecord[]> {
45
-
return await sql<SiteRecord[]>`
46
-
SELECT did, rkey, display_name, created_at, updated_at
47
-
FROM sites
48
-
ORDER BY updated_at DESC
49
-
`
50
-
}
51
-
52
44
/**
53
45
* List all known DIDs from all DID-bearing tables.
54
46
* Missing tables are skipped to keep bootstrapping resilient.
55
47
*/
56
48
export async function listAllKnownDids(): Promise<string[]> {
57
49
const sources: Array<{ name: string; fetch: () => Promise<Array<{ did: string }>> }> = [
58
-
{
59
-
name: 'sites',
60
-
fetch: () => sql<Array<{ did: string }>>`
61
-
SELECT DISTINCT did
62
-
FROM sites
63
-
WHERE did IS NOT NULL AND did <> ''
64
-
`,
65
-
},
66
50
{
67
51
name: 'site_cache',
68
52
fetch: () => sql<Array<{ did: string }>>`
···
218
202
219
203
export async function deleteSiteSettingsCache(did: string, rkey: string): Promise<void> {
220
204
await sql`DELETE FROM site_settings_cache WHERE did = ${did} AND rkey = ${rkey}`
221
-
}
222
-
223
-
export async function upsertSite(did: string, rkey: string, displayName: string): Promise<void> {
224
-
await sql`
225
-
INSERT INTO sites (did, rkey, display_name, created_at, updated_at)
226
-
VALUES (${did}, ${rkey}, ${displayName}, EXTRACT(EPOCH FROM NOW()), EXTRACT(EPOCH FROM NOW()))
227
-
ON CONFLICT (did, rkey)
228
-
DO UPDATE SET
229
-
display_name = EXCLUDED.display_name,
230
-
updated_at = EXTRACT(EPOCH FROM NOW())
231
-
`
232
-
}
233
-
234
-
export async function deleteSite(did: string, rkey: string): Promise<void> {
235
-
await sql`DELETE FROM sites WHERE did = ${did} AND rkey = ${rkey}`
236
205
}
237
206
238
207
export async function isSupporter(did: string): Promise<boolean> {
-23
apps/hosting-service/src/lib/db.ts
-23
apps/hosting-service/src/lib/db.ts
···
70
70
)
71
71
}
72
72
73
-
export async function upsertSite(did: string, rkey: string, displayName?: string) {
74
-
console.log('[DB] Read-only mode: skipping upsertSite', { did, rkey, displayName })
75
-
}
76
-
77
73
/**
78
74
* Upsert site cache entry (used by on-demand caching when a site is completely missing)
79
75
*/
···
102
98
const error = err instanceof Error ? err : new Error(String(err))
103
99
console.error('[DB] upsertSiteCache error:', { did, rkey, error: error.message })
104
100
throw error
105
-
}
106
-
}
107
-
108
-
export interface SiteRecord {
109
-
did: string
110
-
rkey: string
111
-
display_name?: string
112
-
}
113
-
114
-
export async function getAllSites(): Promise<SiteRecord[]> {
115
-
try {
116
-
const result = await sql<SiteRecord[]>`
117
-
SELECT did, rkey, display_name FROM sites
118
-
ORDER BY created_at DESC
119
-
`
120
-
return result
121
-
} catch (err) {
122
-
console.error('Failed to get all sites', err)
123
-
return []
124
101
}
125
102
}
126
103
+1
apps/main-app/public/css.d.ts
+1
apps/main-app/public/css.d.ts
···
1
+
declare module '*.css'
-12
apps/main-app/src/lib/db.ts
-12
apps/main-app/src/lib/db.ts
···
81
81
)
82
82
`
83
83
84
-
// Legacy sites table. Main-app now uses site_cache as the authoritative runtime projection.
85
-
await db`
86
-
CREATE TABLE IF NOT EXISTS sites (
87
-
did TEXT NOT NULL,
88
-
rkey TEXT NOT NULL,
89
-
display_name TEXT,
90
-
created_at BIGINT DEFAULT EXTRACT(EPOCH FROM NOW()),
91
-
updated_at BIGINT DEFAULT EXTRACT(EPOCH FROM NOW()),
92
-
PRIMARY KEY (did, rkey)
93
-
)
94
-
`
95
-
96
84
// Site cache table - stores CIDs for cached sites (used by firehose/hosting services)
97
85
await db`
98
86
CREATE TABLE IF NOT EXISTS site_cache (
-5
apps/main-app/src/lib/migrations.ts
-5
apps/main-app/src/lib/migrations.ts
···
228
228
console.error('Failed to create idx_custom_domains_verified:', err)
229
229
}
230
230
}),
231
-
db`CREATE INDEX IF NOT EXISTS idx_sites_did ON sites(did)`.catch((err) => {
232
-
if (!hasAlreadyExists(err)) {
233
-
console.error('Failed to create idx_sites_did:', err)
234
-
}
235
-
}),
236
231
db`CREATE INDEX IF NOT EXISTS idx_site_cache_did ON site_cache(did)`.catch((err) => {
237
232
if (!hasAlreadyExists(err)) {
238
233
console.error('Failed to create idx_site_cache_did:', err)
+1
-1
apps/webhook-service/src/lib/db.ts
+1
-1
apps/webhook-service/src/lib/db.ts
···
288
288
const dids = new Set<string>()
289
289
290
290
const sites = await db<Array<{ did: string }>>`
291
-
SELECT DISTINCT did FROM sites WHERE did IS NOT NULL AND did <> ''
291
+
SELECT DISTINCT did FROM site_cache WHERE did IS NOT NULL AND did <> ''
292
292
`
293
293
for (const r of sites) dids.add(r.did)
294
294
+18
-1
bun.lock
+18
-1
bun.lock
···
16
16
"@oven/bun-darwin-aarch64": "^1.3.6",
17
17
"@oven/bun-linux-x64": "^1.3.6",
18
18
"@types/bun": "^1.3.5",
19
+
"@typescript/native-preview": "^7.0.0-dev.20260421.2",
19
20
},
20
21
},
21
22
"apps/firehose-service": {
···
161
162
},
162
163
"cli": {
163
164
"name": "wispctl",
164
-
"version": "1.1.0",
165
+
"version": "1.1.2",
165
166
"bin": {
166
167
"wispctl": "dist/index.js",
167
168
},
···
1135
1136
"@typescript-eslint/utils": ["@typescript-eslint/utils@8.54.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.54.0", "@typescript-eslint/types": "8.54.0", "@typescript-eslint/typescript-estree": "8.54.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-9Cnda8GS57AQakvRyG0PTejJNlA2xhvyNtEVIMlDWOOeEyBkYWhGPnfrIAnqxLMTSTo6q8g12XVjjev5l1NvMA=="],
1136
1137
1137
1138
"@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.54.0", "", { "dependencies": { "@typescript-eslint/types": "8.54.0", "eslint-visitor-keys": "^4.2.1" } }, "sha512-VFlhGSl4opC0bprJiItPQ1RfUhGDIBokcPwaFH4yiBCaNPeld/9VeXbiPO1cLyorQi1G1vL+ecBk1x8o1axORA=="],
1139
+
1140
+
"@typescript/native-preview": ["@typescript/native-preview@7.0.0-dev.20260421.2", "", { "optionalDependencies": { "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-arm": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-linux-x64": "7.0.0-dev.20260421.2", "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260421.2", "@typescript/native-preview-win32-x64": "7.0.0-dev.20260421.2" }, "bin": { "tsgo": "bin/tsgo.js" } }, "sha512-CmajHI25HpVWE9R1XFoxr+cphJPxoYD3eFioQtAvXYkMFKnLdICMS9pXre9Pybizb75ejRxjKD5/CVG055rEIg=="],
1141
+
1142
+
"@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260421.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-fHv1r3ZmVo6zxuAIFmuX3w9QxbcauoG0SsWhmDwm6VmRubLlOJIcmTtlmV3JAb9oOnq8LuzZljzT7Q39fSMQDw=="],
1143
+
1144
+
"@typescript/native-preview-darwin-x64": ["@typescript/native-preview-darwin-x64@7.0.0-dev.20260421.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-KWTR6xbW9t+JS7D5DQIzo75pqVXVWUxF9PMv/+S6xsnOjCVd6g0ixHcFpFMJMKSUQpGPr8Z5f7b8ks6LHW01jg=="],
1145
+
1146
+
"@typescript/native-preview-linux-arm": ["@typescript/native-preview-linux-arm@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "arm" }, "sha512-BWLQO3nemLDSV5PoE5GPHe1dU9Dth77Kv8/cle9Ujcp4LhPo0KincdPqFH/qKeU/xvW25mgFueflZ1nc4rKuww=="],
1147
+
1148
+
"@typescript/native-preview-linux-arm64": ["@typescript/native-preview-linux-arm64@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-VLMEuml3BhUb+jaL0TXQ4xvVODxJF+RhkI+tBWvlynsJI4khTXEiwWh+wPOJrsfBRYFRMXEu28Odl/HXkYze8w=="],
1149
+
1150
+
"@typescript/native-preview-linux-x64": ["@typescript/native-preview-linux-x64@7.0.0-dev.20260421.2", "", { "os": "linux", "cpu": "x64" }, "sha512-qUrJWTB5/wv4wnRG0TRXElAxc2kykNiRNyEIEqBbLmzDlrcvAW7RRy8MXoY1ZyTiKGMu14itZ3x9oW6+blFpRw=="],
1151
+
1152
+
"@typescript/native-preview-win32-arm64": ["@typescript/native-preview-win32-arm64@7.0.0-dev.20260421.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Rc6NsWlZmCs5YUKVzKgwoBOoRUGsPzct4BDMRX0csD1devLBBc4AbUXWKsJRbpwIAnqMO1ld4sNHEb+wXgfNHQ=="],
1153
+
1154
+
"@typescript/native-preview-win32-x64": ["@typescript/native-preview-win32-x64@7.0.0-dev.20260421.2", "", { "os": "win32", "cpu": "x64" }, "sha512-GQv1+dya1t6EqF2Cpsb+xoozovdX10JUSf6Kl/8xNkTapzmlHd+uMr+8ku3jIASTxoRGn0Mklgjj3MDKrOTuLg=="],
1138
1155
1139
1156
"@vitest/expect": ["@vitest/expect@4.0.18", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.0.18", "@vitest/utils": "4.0.18", "chai": "^6.2.1", "tinyrainbow": "^3.0.3" } }, "sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ=="],
1140
1157
+2
-1
package.json
+2
-1
package.json
+1
-2
tsconfig.json
+1
-2
tsconfig.json
···
103
103
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
104
104
"allowImportingTsExtensions": true,
105
105
"noEmit": true,
106
-
"baseUrl": ".",
107
106
"paths": {
108
107
"@server": ["./apps/main-app/src/index.ts"],
109
108
"@server/*": ["./apps/main-app/src/*"],
···
111
110
"@wispplace/*": ["./packages/@wispplace/*/src"]
112
111
}
113
112
},
114
-
"exclude": ["docs", "node_modules"]
113
+
"exclude": ["docs", "node_modules", "apps/hosting-service/cache", "**/dist", "**/node_modules"]
115
114
}