Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
e2e and prewarm hot tier
+1244
-18
+4
.dockerignore
+4
.dockerignore
+1
-1
Dockerfile
+1
-1
Dockerfile
+1
-1
apps/firehose-service/Dockerfile
+1
-1
apps/firehose-service/Dockerfile
+2
-2
apps/hosting-service/Dockerfile
+2
-2
apps/hosting-service/Dockerfile
···
1
1
# Build from monorepo root: docker build -f apps/hosting-service/Dockerfile .
2
2
3
3
# Stage 1: Compile to standalone binary
4
-
FROM oven/bun:1-alpine AS builder
4
+
FROM oven/bun:1.3.7-alpine AS builder
5
5
6
6
WORKDIR /app
7
7
···
39
39
EXPOSE 3001
40
40
41
41
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
42
-
CMD wget --no-verbose --tries=1 --spider http://localhost:3001/health || exit 1
42
+
CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3001/health || exit 1
43
43
44
44
CMD ["hosting"]
+41
apps/hosting-service/src/lib/cache-invalidation.test.ts
+41
apps/hosting-service/src/lib/cache-invalidation.test.ts
···
87
87
})
88
88
})
89
89
90
+
test('domain invalidation message parsing preserves domain keys', () => {
91
+
expect(
92
+
parseCacheInvalidationMessage(
93
+
JSON.stringify({
94
+
action: 'domain',
95
+
domain: 'example.wisp.place',
96
+
domainKind: 'wisp',
97
+
streamId: '1713811200000-6',
98
+
}),
99
+
),
100
+
).toEqual({
101
+
action: 'domain',
102
+
domain: 'example.wisp.place',
103
+
domainKind: 'wisp',
104
+
streamId: '1713811200000-6',
105
+
})
106
+
})
107
+
108
+
test('domain stream entry parsing reconstructs domain invalidation messages', () => {
109
+
expect(
110
+
parseCacheInvalidationStreamEntry('1713811200000-7', [
111
+
'action',
112
+
'domain',
113
+
'domain',
114
+
'example.com',
115
+
'domainKind',
116
+
'custom',
117
+
'customDomainId',
118
+
'abc123',
119
+
'ts',
120
+
'1713811200000',
121
+
]),
122
+
).toEqual({
123
+
action: 'domain',
124
+
domain: 'example.com',
125
+
domainKind: 'custom',
126
+
customDomainId: 'abc123',
127
+
streamId: '1713811200000-7',
128
+
})
129
+
})
130
+
90
131
test('stream ids sort by timestamp and sequence', () => {
91
132
expect(compareStreamIds('1713811200000-1', '1713811200000-2')).toBeLessThan(0)
92
133
expect(compareStreamIds('1713811200001-0', '1713811200000-999')).toBeGreaterThan(0)
+79
-11
apps/hosting-service/src/lib/cache-invalidation.ts
+79
-11
apps/hosting-service/src/lib/cache-invalidation.ts
···
15
15
import type { StorageTier } from '@wispplace/tiered-storage'
16
16
import Redis from 'ioredis'
17
17
import { cache } from './cache-manager'
18
+
import { resetSiteHtmlHotCacheWarmup } from './html-prewarm'
18
19
import { hotTier, warmTier } from './storage'
19
20
20
21
const CHANNEL = 'wisp:cache-invalidate'
···
26
27
resolve(process.env.CACHE_DIR || './cache/sites', '..', 'cache-invalidation.lastid')
27
28
const STREAM_ID_PATTERN = /^\d+-\d+$/
28
29
29
-
type CacheInvalidationAction = 'updating' | 'update' | 'delete' | 'settings'
30
+
type CacheInvalidationAction = 'updating' | 'update' | 'delete' | 'settings' | 'domain'
30
31
31
32
export interface CacheInvalidationMessage {
32
-
did: string
33
-
rkey: string
33
+
did?: string
34
+
rkey?: string
34
35
action: CacheInvalidationAction
36
+
domain?: string
37
+
domainKind?: 'wisp' | 'custom'
38
+
customDomainId?: string
35
39
token?: string
36
40
streamId?: string
37
41
}
···
171
175
}
172
176
173
177
if (
174
-
typeof parsed.did !== 'string' ||
175
-
typeof parsed.rkey !== 'string' ||
176
-
(parsed.action !== 'updating' &&
177
-
parsed.action !== 'update' &&
178
-
parsed.action !== 'delete' &&
179
-
parsed.action !== 'settings')
178
+
parsed.action !== 'updating' &&
179
+
parsed.action !== 'update' &&
180
+
parsed.action !== 'delete' &&
181
+
parsed.action !== 'settings' &&
182
+
parsed.action !== 'domain'
180
183
) {
181
184
return null
182
185
}
183
186
187
+
if (parsed.action === 'domain') {
188
+
if (typeof parsed.domain !== 'string') return null
189
+
if (parsed.domainKind !== undefined && parsed.domainKind !== 'wisp' && parsed.domainKind !== 'custom') {
190
+
return null
191
+
}
192
+
193
+
return {
194
+
action: 'domain',
195
+
domain: parsed.domain,
196
+
domainKind: parsed.domainKind,
197
+
customDomainId: typeof parsed.customDomainId === 'string' ? parsed.customDomainId : undefined,
198
+
streamId: normalizeStreamId(parsed.streamId),
199
+
}
200
+
}
201
+
202
+
if (typeof parsed.did !== 'string' || typeof parsed.rkey !== 'string') {
203
+
return null
204
+
}
205
+
184
206
return {
185
207
did: parsed.did,
186
208
rkey: parsed.rkey,
187
209
action: parsed.action,
210
+
domain: typeof parsed.domain === 'string' ? parsed.domain : undefined,
211
+
domainKind: parsed.domainKind === 'wisp' || parsed.domainKind === 'custom' ? parsed.domainKind : undefined,
212
+
customDomainId: typeof parsed.customDomainId === 'string' ? parsed.customDomainId : undefined,
188
213
token: typeof parsed.token === 'string' ? parsed.token : undefined,
189
214
streamId: normalizeStreamId(parsed.streamId),
190
215
}
···
202
227
did: payload.did,
203
228
rkey: payload.rkey,
204
229
action: payload.action,
230
+
domain: payload.domain,
231
+
domainKind: payload.domainKind,
232
+
customDomainId: payload.customDomainId,
205
233
token: payload.token,
206
234
streamId,
207
235
}),
···
234
262
235
263
if (shouldSkipReplayMessage(streamId)) {
236
264
console.log(
237
-
`[CacheInvalidation] Skipping duplicate ${action} for ${did}/${rkey} from ${source} (stream ${streamId})`,
265
+
`[CacheInvalidation] Skipping duplicate ${formatInvalidationTarget(parsed)} from ${source} (stream ${streamId})`,
238
266
)
239
267
return
240
268
}
241
269
242
270
console.log(
243
-
`[CacheInvalidation] Received ${action} for ${did}/${rkey} from ${source}${streamId ? ` (stream ${streamId})` : ''}`,
271
+
`[CacheInvalidation] Received ${formatInvalidationTarget(parsed)} from ${source}${streamId ? ` (stream ${streamId})` : ''}`,
244
272
)
245
273
274
+
if (action === 'domain') {
275
+
applyDomainCacheInvalidation(parsed)
276
+
advanceStreamCursor(streamId)
277
+
return
278
+
}
279
+
280
+
if (!did || !rkey) {
281
+
console.warn('[CacheInvalidation] Missing did/rkey for site invalidation', parsed)
282
+
advanceStreamCursor(streamId)
283
+
return
284
+
}
285
+
246
286
if (action === 'updating') {
247
287
markSiteUpdating(did, rkey, token)
248
288
advanceStreamCursor(streamId)
···
266
306
cache.delete('redirectRules', `${did}:${rkey}`)
267
307
cache.delete('settings', `${did}:${rkey}`)
268
308
cache.deletePrefix('siteFiles', `${did}:${rkey}:`)
309
+
resetSiteHtmlHotCacheWarmup(did, rkey)
269
310
advanceStreamCursor(streamId)
311
+
}
312
+
313
+
function formatInvalidationTarget(parsed: CacheInvalidationMessage): string {
314
+
if (parsed.action === 'domain') {
315
+
return `domain:${parsed.domainKind ?? 'any'}:${parsed.domain ?? '(missing)'}`
316
+
}
317
+
318
+
return `${parsed.action} for ${parsed.did}/${parsed.rkey}`
319
+
}
320
+
321
+
function applyDomainCacheInvalidation(parsed: CacheInvalidationMessage): void {
322
+
const domain = parsed.domain?.trim().toLowerCase()
323
+
if (!domain) return
324
+
325
+
if (parsed.domainKind !== 'custom') {
326
+
cache.delete('domains', domain)
327
+
}
328
+
329
+
if (parsed.domainKind !== 'wisp') {
330
+
cache.delete('customDomains', domain)
331
+
if (parsed.customDomainId) {
332
+
cache.delete('customDomains', parsed.customDomainId)
333
+
cache.delete('customDomains', `hash:${parsed.customDomainId}`)
334
+
}
335
+
}
336
+
337
+
console.log(`[CacheInvalidation] Cleared domain lookup cache for ${domain}`)
270
338
}
271
339
272
340
function enqueueCacheInvalidation(parsed: CacheInvalidationMessage, source: 'pubsub' | 'replay'): Promise<void> {
+5
apps/hosting-service/src/lib/file-serving.ts
+5
apps/hosting-service/src/lib/file-serving.ts
···
14
14
import { isSiteUpdating } from './cache-invalidation'
15
15
import { cache } from './cache-manager'
16
16
import { getSiteCache } from './db'
17
+
import { triggerSiteHtmlHotCacheWarmup } from './html-prewarm'
17
18
import { fetchAndCacheSite } from './on-demand-cache'
18
19
import { generate404Page, generateDirectoryListing, siteUpdatingResponse } from './page-generators'
19
20
import { loadRedirectRules, matchRedirectRule, parseCookies, parseQueryString } from './redirects'
···
329
330
? siteUpdatingResponse()
330
331
: new Response('Site is updating', { status: 503, headers: { 'Cache-Control': 'no-store', 'Retry-After': '5' } })
331
332
}
333
+
334
+
triggerSiteHtmlHotCacheWarmup(did, rkey)
332
335
333
336
const trace = createTrace()
334
337
···
645
648
? siteUpdatingResponse()
646
649
: new Response('Site is updating', { status: 503, headers: { 'Cache-Control': 'no-store', 'Retry-After': '5' } })
647
650
}
651
+
652
+
triggerSiteHtmlHotCacheWarmup(did, rkey)
648
653
649
654
const trace = createTrace()
650
655
+97
apps/hosting-service/src/lib/html-prewarm.test.ts
+97
apps/hosting-service/src/lib/html-prewarm.test.ts
···
1
+
import { beforeEach, describe, expect, mock, test } from 'bun:test'
2
+
3
+
const cachedKeys = new Set<string>()
4
+
const promotedKeys: string[] = []
5
+
6
+
const fakeStorage = {
7
+
async getWithMetadata(key: string) {
8
+
promotedKeys.push(key)
9
+
return {
10
+
data: new Uint8Array([1]),
11
+
metadata: {
12
+
key,
13
+
size: 1,
14
+
createdAt: new Date(),
15
+
lastAccessed: new Date(),
16
+
accessCount: 0,
17
+
checksum: 'checksum',
18
+
},
19
+
source: 'cold' as const,
20
+
}
21
+
},
22
+
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
23
+
for (const key of cachedKeys) {
24
+
if (!prefix || key.startsWith(prefix)) {
25
+
yield key
26
+
}
27
+
}
28
+
},
29
+
}
30
+
31
+
mock.module('./storage', () => ({
32
+
storage: fakeStorage,
33
+
}))
34
+
35
+
const {
36
+
resetHtmlHotCacheWarmupForTests,
37
+
resetSiteHtmlHotCacheWarmup,
38
+
triggerSiteHtmlHotCacheWarmup,
39
+
waitForSiteHtmlHotCacheWarmupForTests,
40
+
} = await import('./html-prewarm')
41
+
42
+
const DID = 'did:plc:test'
43
+
const RKEY = 'site'
44
+
45
+
describe('HTML prewarm', () => {
46
+
beforeEach(() => {
47
+
cachedKeys.clear()
48
+
promotedKeys.length = 0
49
+
resetHtmlHotCacheWarmupForTests()
50
+
})
51
+
52
+
test('loads all HTML keys for a site on first hit', async () => {
53
+
cachedKeys.add(`${DID}/${RKEY}/index.html`)
54
+
cachedKeys.add(`${DID}/${RKEY}/nested/about.htm`)
55
+
cachedKeys.add(`${DID}/${RKEY}/docs/guide.HTML`)
56
+
cachedKeys.add(`${DID}/${RKEY}/style.css`)
57
+
cachedKeys.add(`${DID}/other-site/index.html`)
58
+
59
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
60
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
61
+
62
+
expect(promotedKeys.sort()).toEqual(
63
+
[`${DID}/${RKEY}/docs/guide.HTML`, `${DID}/${RKEY}/index.html`, `${DID}/${RKEY}/nested/about.htm`].sort(),
64
+
)
65
+
66
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
67
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
68
+
69
+
expect(promotedKeys).toHaveLength(3)
70
+
})
71
+
72
+
test('reset allows a site to be prewarmed again', async () => {
73
+
cachedKeys.add(`${DID}/${RKEY}/index.html`)
74
+
75
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
76
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
77
+
expect(promotedKeys).toHaveLength(1)
78
+
79
+
resetSiteHtmlHotCacheWarmup(DID, RKEY)
80
+
81
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
82
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
83
+
expect(promotedKeys).toHaveLength(2)
84
+
})
85
+
86
+
test('site with no cached keys is not permanently marked warm', async () => {
87
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
88
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
89
+
expect(promotedKeys).toHaveLength(0)
90
+
91
+
cachedKeys.add(`${DID}/${RKEY}/index.html`)
92
+
93
+
triggerSiteHtmlHotCacheWarmup(DID, RKEY)
94
+
await waitForSiteHtmlHotCacheWarmupForTests(DID, RKEY)
95
+
expect(promotedKeys).toHaveLength(1)
96
+
})
97
+
})
+100
apps/hosting-service/src/lib/html-prewarm.ts
+100
apps/hosting-service/src/lib/html-prewarm.ts
···
1
+
import { createLogger } from '@wispplace/observability'
2
+
import { storage } from './storage'
3
+
4
+
const logger = createLogger('html-prewarm')
5
+
6
+
const warmedSites = new Set<string>()
7
+
const prewarmGeneration = new Map<string, number>()
8
+
const prewarmInFlight = new Map<string, { generation: number; promise: Promise<void> }>()
9
+
10
+
function getSiteKey(did: string, rkey: string): string {
11
+
return `${did}/${rkey}`
12
+
}
13
+
14
+
function isHtmlStorageKey(key: string): boolean {
15
+
const normalized = key.toLowerCase()
16
+
return normalized.endsWith('.html') || normalized.endsWith('.htm')
17
+
}
18
+
19
+
async function loadSiteHtmlKeysIntoHotTier(
20
+
did: string,
21
+
rkey: string,
22
+
): Promise<{ scannedKeys: number; warmedHtmlKeys: number }> {
23
+
const prefix = `${did}/${rkey}/`
24
+
let scannedKeys = 0
25
+
let warmedHtmlKeys = 0
26
+
27
+
for await (const key of storage.listKeys(prefix)) {
28
+
scannedKeys++
29
+
if (!isHtmlStorageKey(key)) continue
30
+
31
+
// getWithMetadata uses eager promotion and moves the key into hot tier.
32
+
const result = await storage.getWithMetadata(key)
33
+
if (result) {
34
+
warmedHtmlKeys++
35
+
}
36
+
}
37
+
38
+
return { scannedKeys, warmedHtmlKeys }
39
+
}
40
+
41
+
export function triggerSiteHtmlHotCacheWarmup(did: string, rkey: string): void {
42
+
const siteKey = getSiteKey(did, rkey)
43
+
if (warmedSites.has(siteKey)) return
44
+
45
+
const generation = prewarmGeneration.get(siteKey) ?? 0
46
+
const existing = prewarmInFlight.get(siteKey)
47
+
if (existing && existing.generation === generation) return
48
+
49
+
const entry = {
50
+
generation,
51
+
promise: (async () => {
52
+
try {
53
+
const { scannedKeys, warmedHtmlKeys } = await loadSiteHtmlKeysIntoHotTier(did, rkey)
54
+
const latestGeneration = prewarmGeneration.get(siteKey) ?? 0
55
+
if (latestGeneration !== generation) return
56
+
57
+
// When the site doesn't exist yet, avoid permanently marking it as warmed.
58
+
if (scannedKeys > 0) {
59
+
warmedSites.add(siteKey)
60
+
}
61
+
62
+
logger.debug(`HTML prewarm finished for ${did}/${rkey}`, {
63
+
scannedKeys,
64
+
warmedHtmlKeys,
65
+
})
66
+
} catch (err) {
67
+
logger.warn(`HTML prewarm failed for ${did}/${rkey}`, { error: err })
68
+
}
69
+
})(),
70
+
}
71
+
72
+
prewarmInFlight.set(siteKey, entry)
73
+
entry.promise.finally(() => {
74
+
const current = prewarmInFlight.get(siteKey)
75
+
if (current === entry) {
76
+
prewarmInFlight.delete(siteKey)
77
+
}
78
+
})
79
+
}
80
+
81
+
export function resetSiteHtmlHotCacheWarmup(did: string, rkey: string): void {
82
+
const siteKey = getSiteKey(did, rkey)
83
+
warmedSites.delete(siteKey)
84
+
prewarmGeneration.set(siteKey, (prewarmGeneration.get(siteKey) ?? 0) + 1)
85
+
prewarmInFlight.delete(siteKey)
86
+
}
87
+
88
+
export function resetHtmlHotCacheWarmupForTests(): void {
89
+
warmedSites.clear()
90
+
prewarmGeneration.clear()
91
+
prewarmInFlight.clear()
92
+
}
93
+
94
+
export async function waitForSiteHtmlHotCacheWarmupForTests(did: string, rkey: string): Promise<void> {
95
+
const siteKey = getSiteKey(did, rkey)
96
+
const inFlight = prewarmInFlight.get(siteKey)
97
+
if (inFlight) {
98
+
await inFlight.promise
99
+
}
100
+
}
+2
apps/hosting-service/src/server.ts
+2
apps/hosting-service/src/server.ts
+18
apps/main-app/e2e/Dockerfile
+18
apps/main-app/e2e/Dockerfile
···
1
+
FROM oven/bun:1.3.7
2
+
3
+
WORKDIR /app
4
+
5
+
ENV DEBIAN_FRONTEND=noninteractive
6
+
ENV E2E_CHROMIUM_EXECUTABLE=/usr/bin/chromium
7
+
8
+
RUN apt-get update && \
9
+
apt-get install -y --no-install-recommends ca-certificates chromium fonts-liberation && \
10
+
rm -rf /var/lib/apt/lists/*
11
+
12
+
COPY . .
13
+
14
+
RUN bun install --frozen-lockfile
15
+
16
+
WORKDIR /app/apps/main-app
17
+
18
+
CMD ["bun", "e2e/wisp-e2e.ts"]
+40
apps/main-app/e2e/README.md
+40
apps/main-app/e2e/README.md
···
1
+
# wisp.place E2E Harness
2
+
3
+
Runs the full path with Docker Compose:
4
+
5
+
1. Main app signs into a real ATProto test account through Playwright OAuth.
6
+
2. The harness claims a random `*.wisp.place` domain.
7
+
3. The harness uploads a small site through `/wisp/upload-files`.
8
+
4. The firehose service receives the `place.wisp.fs` event and writes files to MinIO/S3.
9
+
5. The hosting service serves the mapped domain, first from cold/S3, then from hot memory.
10
+
6. The harness deletes the claimed domain and verifies hosting stops serving the cached domain route.
11
+
7. The harness deletes the site record and verifies firehose plus hosting cache invalidation stops direct site serving.
12
+
13
+
Required environment:
14
+
15
+
```sh
16
+
export E2E_ATPROTO_HANDLE='test.example.com'
17
+
export E2E_ATPROTO_PASSWORD='...'
18
+
```
19
+
20
+
Run:
21
+
22
+
```sh
23
+
bun run e2e:harness
24
+
```
25
+
26
+
Tear down volumes:
27
+
28
+
```sh
29
+
bun run e2e:harness:down
30
+
```
31
+
32
+
Useful overrides:
33
+
34
+
```sh
35
+
E2E_CLEANUP=false
36
+
E2E_HEADLESS=false
37
+
E2E_TIMEOUT_MS=300000
38
+
E2E_DOMAIN_HANDLE=e2e-my-run
39
+
E2E_SITE_RKEY=e2e-my-run
40
+
```
+610
apps/main-app/e2e/wisp-e2e.ts
+610
apps/main-app/e2e/wisp-e2e.ts
···
1
+
import { setTimeout as delay } from 'node:timers/promises'
2
+
import { type Browser, chromium, type Page } from 'playwright'
3
+
4
+
type JsonResponse<T = unknown> = {
5
+
ok: boolean
6
+
status: number
7
+
data: T | null
8
+
text: string
9
+
}
10
+
11
+
type UploadStartResponse = {
12
+
success?: boolean
13
+
jobId?: string
14
+
error?: string
15
+
}
16
+
17
+
type UploadDoneResponse = {
18
+
success?: boolean
19
+
fileCount?: number
20
+
uploadedCount?: number
21
+
hasFailures?: boolean
22
+
failedFiles?: Array<{ name: string; error: string }>
23
+
}
24
+
25
+
const requiredEnv = (name: string): string => {
26
+
const value = process.env[name]
27
+
if (!value) {
28
+
throw new Error(`${name} is required`)
29
+
}
30
+
return value
31
+
}
32
+
33
+
const env = (name: string, fallback: string): string => process.env[name] || fallback
34
+
35
+
const appUrl = env('E2E_APP_URL', 'http://127.0.0.1:8000').replace(/\/$/, '')
36
+
const appUpstream = env('E2E_APP_UPSTREAM', 'http://main-app:8000').replace(/\/$/, '')
37
+
const hostingUrl = env('E2E_HOSTING_URL', 'http://hosting-service:3001').replace(/\/$/, '')
38
+
const firehoseUrl = env('E2E_FIREHOSE_URL', 'http://firehose-service:3001').replace(/\/$/, '')
39
+
const atprotoHandle = requiredEnv('E2E_ATPROTO_HANDLE')
40
+
const atprotoPassword = requiredEnv('E2E_ATPROTO_PASSWORD')
41
+
const timeoutMs = Number.parseInt(env('E2E_TIMEOUT_MS', '180000'), 10)
42
+
const cleanupEnabled = env('E2E_CLEANUP', 'true') !== 'false'
43
+
const headless = env('E2E_HEADLESS', 'true') !== 'false'
44
+
const chromiumExecutable = process.env.E2E_CHROMIUM_EXECUTABLE || undefined
45
+
46
+
const runId = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`
47
+
const domainHandle = env('E2E_DOMAIN_HANDLE', `e2e-${runId}`).toLowerCase()
48
+
const domain = `${domainHandle}.wisp.place`
49
+
const siteName = env('E2E_SITE_RKEY', `e2e-${runId}`).toLowerCase()
50
+
const marker = `wisp-e2e-${runId}`
51
+
52
+
function startLoopbackProxy() {
53
+
const listenUrl = new URL(appUrl)
54
+
const upstreamUrl = new URL(appUpstream)
55
+
56
+
const server = Bun.serve({
57
+
hostname: listenUrl.hostname,
58
+
port: Number(listenUrl.port || '80'),
59
+
async fetch(request) {
60
+
const incomingUrl = new URL(request.url)
61
+
const target = new URL(`${incomingUrl.pathname}${incomingUrl.search}`, upstreamUrl)
62
+
const headers = new Headers(request.headers)
63
+
headers.delete('host')
64
+
headers.set('x-forwarded-host', listenUrl.host)
65
+
headers.set('x-forwarded-proto', listenUrl.protocol.replace(':', ''))
66
+
67
+
return fetch(target, {
68
+
method: request.method,
69
+
headers,
70
+
body: request.method === 'GET' || request.method === 'HEAD' ? undefined : request.body,
71
+
redirect: 'manual',
72
+
})
73
+
},
74
+
})
75
+
76
+
console.log(`[e2e] Loopback proxy listening on ${appUrl} -> ${appUpstream}`)
77
+
return server
78
+
}
79
+
80
+
async function waitForHttpOk(url: string, label: string): Promise<void> {
81
+
await poll(
82
+
async () => {
83
+
try {
84
+
const response = await fetch(url)
85
+
return response.ok
86
+
} catch {
87
+
return false
88
+
}
89
+
},
90
+
{ label, timeout: timeoutMs, interval: 1000 },
91
+
)
92
+
}
93
+
94
+
async function poll(
95
+
check: () => Promise<boolean>,
96
+
options: { label: string; timeout: number; interval?: number },
97
+
): Promise<void> {
98
+
const started = Date.now()
99
+
const interval = options.interval ?? 1000
100
+
let attempts = 0
101
+
102
+
while (Date.now() - started < options.timeout) {
103
+
attempts++
104
+
if (await check()) return
105
+
if (attempts % 10 === 0) {
106
+
console.log(`[e2e] Still waiting for ${options.label} (${Math.round((Date.now() - started) / 1000)}s)`)
107
+
}
108
+
await delay(interval)
109
+
}
110
+
111
+
throw new Error(`Timed out waiting for ${options.label}`)
112
+
}
113
+
114
+
async function maybeFill(page: Page, selectors: string[], value: string): Promise<boolean> {
115
+
for (const selector of selectors) {
116
+
const locator = page.locator(selector).first()
117
+
if ((await locator.count()) === 0) continue
118
+
if (!(await locator.isVisible().catch(() => false))) continue
119
+
if (!(await locator.isEnabled().catch(() => false))) continue
120
+
121
+
const current = await locator.inputValue().catch(() => '')
122
+
if (current !== value) {
123
+
await locator.fill(value)
124
+
}
125
+
return true
126
+
}
127
+
return false
128
+
}
129
+
130
+
async function clickFirstButton(page: Page, names: RegExp[]): Promise<boolean> {
131
+
for (const name of names) {
132
+
const button = page.getByRole('button', { name }).first()
133
+
if ((await button.count()) > 0 && (await button.isVisible().catch(() => false))) {
134
+
if (await button.isEnabled().catch(() => false)) {
135
+
await button.click()
136
+
return true
137
+
}
138
+
}
139
+
}
140
+
141
+
const submit = page.locator('button[type="submit"], input[type="submit"]').first()
142
+
if ((await submit.count()) > 0 && (await submit.isVisible().catch(() => false))) {
143
+
if (await submit.isEnabled().catch(() => false)) {
144
+
await submit.click()
145
+
return true
146
+
}
147
+
}
148
+
149
+
return false
150
+
}
151
+
152
+
async function appAuthStatus(page: Page): Promise<{ authenticated: boolean; did?: string }> {
153
+
return await page.evaluate(async () => {
154
+
const response = await fetch('/api/auth/status', { credentials: 'include' })
155
+
return response.json()
156
+
})
157
+
}
158
+
159
+
async function completeAtprotoLogin(page: Page): Promise<string> {
160
+
const loginUrl = `${appUrl}/api/auth/login?login_hint=${encodeURIComponent(atprotoHandle)}`
161
+
await page.goto(loginUrl, { waitUntil: 'domcontentloaded' })
162
+
163
+
const started = Date.now()
164
+
let lastLoggedUrl = ''
165
+
166
+
while (Date.now() - started < timeoutMs) {
167
+
const currentUrl = page.url()
168
+
if (currentUrl !== lastLoggedUrl) {
169
+
console.log(`[e2e] Auth flow at ${currentUrl}`)
170
+
lastLoggedUrl = currentUrl
171
+
}
172
+
173
+
if (currentUrl.startsWith(appUrl)) {
174
+
const status: { authenticated: boolean; did?: string } = await appAuthStatus(page).catch(() => ({
175
+
authenticated: false,
176
+
}))
177
+
if (status.authenticated && status.did) {
178
+
console.log(`[e2e] Signed in as ${status.did}`)
179
+
return status.did
180
+
}
181
+
}
182
+
183
+
const twoFactorInput = page
184
+
.locator('input[name*="code" i], input[autocomplete="one-time-code"], input[inputmode="numeric"]')
185
+
.first()
186
+
if ((await twoFactorInput.count()) > 0 && (await twoFactorInput.isVisible().catch(() => false))) {
187
+
throw new Error('Two-factor auth is not supported by this harness; use a dedicated test account without 2FA')
188
+
}
189
+
190
+
const filledIdentifier = await maybeFill(
191
+
page,
192
+
[
193
+
'input[name="identifier"]',
194
+
'input[name="handle"]',
195
+
'input[name="login"]',
196
+
'input[autocomplete="username"]',
197
+
'input[type="email"]',
198
+
'input[placeholder*="handle" i]',
199
+
'input[placeholder*="email" i]',
200
+
],
201
+
atprotoHandle,
202
+
)
203
+
204
+
const filledPassword = await maybeFill(
205
+
page,
206
+
['input[type="password"]', 'input[name="password"]', 'input[autocomplete="current-password"]'],
207
+
atprotoPassword,
208
+
)
209
+
210
+
const clicked = await clickFirstButton(
211
+
page,
212
+
filledPassword
213
+
? [/log in/i, /sign in/i, /continue/i, /authorize/i, /allow/i, /accept/i]
214
+
: filledIdentifier
215
+
? [/next/i, /continue/i, /log in/i, /sign in/i]
216
+
: [/authorize/i, /allow/i, /accept/i, /continue/i, /yes/i],
217
+
)
218
+
219
+
if (!clicked) {
220
+
await page.keyboard.press('Enter').catch(() => undefined)
221
+
}
222
+
223
+
await Promise.race([page.waitForLoadState('networkidle', { timeout: 2000 }).catch(() => undefined), delay(750)])
224
+
}
225
+
226
+
throw new Error('Timed out completing ATProto OAuth login')
227
+
}
228
+
229
+
async function appJson<T>(page: Page, path: string, init?: RequestInit): Promise<JsonResponse<T>> {
230
+
return (await page.evaluate(
231
+
async ({ path, init }) => {
232
+
const response = await fetch(path, { credentials: 'include', ...init })
233
+
const text = await response.text()
234
+
let data: unknown = null
235
+
try {
236
+
data = text ? JSON.parse(text) : null
237
+
} catch {
238
+
data = null
239
+
}
240
+
return {
241
+
ok: response.ok,
242
+
status: response.status,
243
+
data,
244
+
text,
245
+
}
246
+
},
247
+
{ path, init },
248
+
)) as JsonResponse<T>
249
+
}
250
+
251
+
async function claimDomain(page: Page): Promise<void> {
252
+
const response = await appJson<{ success?: boolean; domain?: string; error?: string }>(page, '/api/domain/claim', {
253
+
method: 'POST',
254
+
headers: { 'Content-Type': 'application/json' },
255
+
body: JSON.stringify({ handle: domainHandle }),
256
+
})
257
+
258
+
if (!response.ok || !response.data?.success) {
259
+
throw new Error(`Domain claim failed (${response.status}): ${response.text}`)
260
+
}
261
+
262
+
console.log(`[e2e] Claimed ${response.data.domain}`)
263
+
}
264
+
265
+
async function uploadSite(page: Page): Promise<UploadDoneResponse> {
266
+
const html = `<!doctype html>
267
+
<html>
268
+
<head>
269
+
<meta charset="utf-8">
270
+
<title>wisp e2e</title>
271
+
<link rel="stylesheet" href="/style.css">
272
+
</head>
273
+
<body data-marker="${marker}">
274
+
<h1>${marker}</h1>
275
+
</body>
276
+
</html>`
277
+
const css = `body { font-family: monospace; color: #123; }`
278
+
279
+
const started = await page.evaluate(
280
+
async ({ siteName, html, css }) => {
281
+
const formData = new FormData()
282
+
formData.append('siteName', siteName)
283
+
formData.append('files', new File([html], 'index.html', { type: 'text/html' }))
284
+
formData.append('files', new File([css], 'style.css', { type: 'text/css' }))
285
+
286
+
const response = await fetch('/wisp/upload-files', {
287
+
method: 'POST',
288
+
body: formData,
289
+
credentials: 'include',
290
+
})
291
+
const text = await response.text()
292
+
let data: unknown
293
+
try {
294
+
data = JSON.parse(text)
295
+
} catch {
296
+
data = { error: text }
297
+
}
298
+
299
+
return { ok: response.ok, status: response.status, data, text }
300
+
},
301
+
{ siteName, html, css },
302
+
)
303
+
304
+
const data = started.data as UploadStartResponse
305
+
if (!started.ok || !data.success || !data.jobId) {
306
+
throw new Error(`Upload did not start (${started.status}): ${started.text}`)
307
+
}
308
+
309
+
console.log(`[e2e] Upload job ${data.jobId} started`)
310
+
311
+
const done = await page.evaluate(
312
+
async ({ jobId, timeoutMs }) => {
313
+
return await new Promise<UploadDoneResponse>((resolve, reject) => {
314
+
const eventSource = new EventSource(`/wisp/upload-progress/${jobId}`)
315
+
const timeout = setTimeout(() => {
316
+
eventSource.close()
317
+
reject(new Error('Timed out waiting for upload SSE'))
318
+
}, timeoutMs)
319
+
320
+
const finish = (result: UploadDoneResponse) => {
321
+
clearTimeout(timeout)
322
+
eventSource.close()
323
+
resolve(result)
324
+
}
325
+
326
+
eventSource.addEventListener('progress', (event) => {
327
+
const payload = JSON.parse(event.data)
328
+
if (payload.status === 'completed') finish(payload.result || {})
329
+
if (payload.status === 'failed') reject(new Error(payload.error || 'Upload failed'))
330
+
})
331
+
332
+
eventSource.addEventListener('done', (event) => {
333
+
finish(JSON.parse(event.data))
334
+
})
335
+
336
+
eventSource.addEventListener('error', (event) => {
337
+
const message = 'data' in event && event.data ? String(event.data) : 'Upload SSE failed'
338
+
clearTimeout(timeout)
339
+
eventSource.close()
340
+
reject(new Error(message))
341
+
})
342
+
})
343
+
},
344
+
{ jobId: data.jobId, timeoutMs },
345
+
)
346
+
347
+
if (done.hasFailures || (done.failedFiles && done.failedFiles.length > 0)) {
348
+
throw new Error(`Upload completed with failed files: ${JSON.stringify(done.failedFiles)}`)
349
+
}
350
+
351
+
console.log(`[e2e] Uploaded ${done.uploadedCount ?? done.fileCount ?? 0} files to ${siteName}`)
352
+
return done
353
+
}
354
+
355
+
async function waitForFirehoseProjection(page: Page): Promise<void> {
356
+
await poll(
357
+
async () => {
358
+
const response = await appJson<{ sites?: Array<{ rkey: string }> }>(page, '/api/user/sites')
359
+
return response.ok && Boolean(response.data?.sites?.some((site) => site.rkey === siteName))
360
+
},
361
+
{ label: `firehose projection for ${siteName}`, timeout: timeoutMs, interval: 2000 },
362
+
)
363
+
console.log(`[e2e] Firehose projected ${siteName} into site_cache`)
364
+
}
365
+
366
+
async function mapDomain(page: Page): Promise<void> {
367
+
const response = await appJson<{ success?: boolean }>(page, '/api/domain/wisp/map-site', {
368
+
method: 'POST',
369
+
headers: { 'Content-Type': 'application/json' },
370
+
body: JSON.stringify({ domain, siteRkey: siteName }),
371
+
})
372
+
373
+
if (!response.ok || !response.data?.success) {
374
+
throw new Error(`Domain mapping failed (${response.status}): ${response.text}`)
375
+
}
376
+
377
+
console.log(`[e2e] Mapped ${domain} -> ${siteName}`)
378
+
}
379
+
380
+
async function fetchHostedSite(): Promise<{ status: number; text: string; tier: string | null }> {
381
+
const response = await fetch(`${hostingUrl}/`, {
382
+
headers: {
383
+
Host: domain,
384
+
Accept: 'text/html',
385
+
'Accept-Encoding': 'identity',
386
+
},
387
+
redirect: 'manual',
388
+
})
389
+
const text = await response.text()
390
+
return {
391
+
status: response.status,
392
+
text,
393
+
tier: response.headers.get('x-cache-tier'),
394
+
}
395
+
}
396
+
397
+
async function fetchHostedDirect(did: string): Promise<{ status: number; text: string; tier: string | null }> {
398
+
const response = await fetch(`${hostingUrl}/${did}/${siteName}/`, {
399
+
headers: {
400
+
Host: 'sites.wisp.place',
401
+
Accept: 'text/html',
402
+
'Accept-Encoding': 'identity',
403
+
},
404
+
redirect: 'manual',
405
+
})
406
+
const text = await response.text()
407
+
return {
408
+
status: response.status,
409
+
text,
410
+
tier: response.headers.get('x-cache-tier'),
411
+
}
412
+
}
413
+
414
+
async function verifyHostingCacheFlow(): Promise<void> {
415
+
let first: { status: number; text: string; tier: string | null } | undefined
416
+
417
+
await poll(
418
+
async () => {
419
+
const response = await fetchHostedSite()
420
+
if (response.status === 200 && response.text.includes(marker)) {
421
+
first = response
422
+
return true
423
+
}
424
+
console.log(
425
+
`[e2e] Hosting not ready: status=${response.status} tier=${response.tier} body=${response.text.slice(0, 80)}`,
426
+
)
427
+
return false
428
+
},
429
+
{ label: `hosting response for ${domain}`, timeout: timeoutMs, interval: 2000 },
430
+
)
431
+
432
+
const firstResponse = first
433
+
if (!firstResponse) {
434
+
throw new Error('Hosting response missing after readiness poll')
435
+
}
436
+
437
+
if (firstResponse.tier !== 'cold') {
438
+
throw new Error(`Expected first hosting read to come from cold/S3, got ${firstResponse.tier || '(missing)'}`)
439
+
}
440
+
441
+
const second = await fetchHostedSite()
442
+
if (second.status !== 200 || !second.text.includes(marker)) {
443
+
throw new Error(`Second hosting read failed: status=${second.status} body=${second.text.slice(0, 120)}`)
444
+
}
445
+
446
+
if (second.tier !== 'hot') {
447
+
throw new Error(`Expected second hosting read to come from hot/memory, got ${second.tier || '(missing)'}`)
448
+
}
449
+
450
+
console.log('[e2e] Hosting served first from cold/S3 and second from hot/memory')
451
+
}
452
+
453
+
async function deleteDomainAndVerify(page: Page): Promise<void> {
454
+
const domainDelete = await appJson(page, `/api/domain/wisp/${encodeURIComponent(domain)}`, {
455
+
method: 'DELETE',
456
+
})
457
+
if (!domainDelete.ok) {
458
+
throw new Error(`Domain delete failed (${domainDelete.status}): ${domainDelete.text}`)
459
+
}
460
+
461
+
await poll(
462
+
async () => {
463
+
const response = await appJson(page, `/api/domain/registered?domain=${encodeURIComponent(domain)}`)
464
+
return response.status === 404
465
+
},
466
+
{ label: `main-app domain deletion for ${domain}`, timeout: timeoutMs, interval: 1000 },
467
+
)
468
+
469
+
await poll(
470
+
async () => {
471
+
const response = await fetchHostedSite()
472
+
if (response.status === 404 && !response.text.includes(marker)) return true
473
+
console.log(
474
+
`[e2e] Domain cache still serving or not settled: status=${response.status} tier=${response.tier} body=${response.text.slice(0, 80)}`,
475
+
)
476
+
return false
477
+
},
478
+
{ label: `hosting domain cache invalidation for ${domain}`, timeout: timeoutMs, interval: 1000 },
479
+
)
480
+
481
+
console.log(`[e2e] Deleted ${domain} and verified hosting domain cache invalidation`)
482
+
}
483
+
484
+
async function primeDirectSiteCache(did: string): Promise<void> {
485
+
await poll(
486
+
async () => {
487
+
const response = await fetchHostedDirect(did)
488
+
if (response.status === 200 && response.text.includes(marker)) return true
489
+
console.log(
490
+
`[e2e] Direct site route not ready: status=${response.status} tier=${response.tier} body=${response.text.slice(0, 80)}`,
491
+
)
492
+
return false
493
+
},
494
+
{ label: `direct hosted site route for ${did}/${siteName}`, timeout: timeoutMs, interval: 1000 },
495
+
)
496
+
497
+
const cached = await fetchHostedDirect(did)
498
+
if (cached.status !== 200 || !cached.text.includes(marker)) {
499
+
throw new Error(`Direct site cache prime failed: status=${cached.status} body=${cached.text.slice(0, 120)}`)
500
+
}
501
+
502
+
if (cached.tier !== 'hot') {
503
+
throw new Error(`Expected direct route second read to come from hot/memory, got ${cached.tier || '(missing)'}`)
504
+
}
505
+
506
+
console.log('[e2e] Primed direct site route into hosting hot cache')
507
+
}
508
+
509
+
async function deleteSiteRecordAndVerify(page: Page, did: string): Promise<void> {
510
+
const siteDelete = await appJson(page, `/api/site/${encodeURIComponent(siteName)}`, {
511
+
method: 'DELETE',
512
+
})
513
+
if (!siteDelete.ok) {
514
+
throw new Error(`Site record delete failed (${siteDelete.status}): ${siteDelete.text}`)
515
+
}
516
+
517
+
await poll(
518
+
async () => {
519
+
const response = await appJson<{ sites?: Array<{ rkey: string }> }>(page, '/api/user/sites')
520
+
return response.ok && !response.data?.sites?.some((site) => site.rkey === siteName)
521
+
},
522
+
{ label: `firehose deletion projection for ${siteName}`, timeout: timeoutMs, interval: 2000 },
523
+
)
524
+
525
+
await poll(
526
+
async () => {
527
+
const response = await fetchHostedDirect(did)
528
+
if (response.status === 404 && !response.text.includes(marker)) return true
529
+
console.log(
530
+
`[e2e] Site cache still serving or not settled: status=${response.status} tier=${response.tier} body=${response.text.slice(0, 80)}`,
531
+
)
532
+
return false
533
+
},
534
+
{ label: `hosting site storage invalidation for ${did}/${siteName}`, timeout: timeoutMs, interval: 1000 },
535
+
)
536
+
537
+
console.log('[e2e] Deleted place.wisp.fs record and verified firehose + hosting cache eviction')
538
+
}
539
+
540
+
async function cleanup(page: Page): Promise<void> {
541
+
if (!cleanupEnabled) return
542
+
543
+
console.log('[e2e] Cleaning up test records')
544
+
545
+
const domainDelete = await appJson(page, `/api/domain/wisp/${encodeURIComponent(domain)}`, {
546
+
method: 'DELETE',
547
+
})
548
+
if (!domainDelete.ok) {
549
+
console.warn(`[e2e] Domain cleanup failed (${domainDelete.status}): ${domainDelete.text}`)
550
+
}
551
+
552
+
const siteDelete = await appJson(page, `/api/site/${encodeURIComponent(siteName)}`, {
553
+
method: 'DELETE',
554
+
})
555
+
if (!siteDelete.ok) {
556
+
console.warn(`[e2e] Site cleanup failed (${siteDelete.status}): ${siteDelete.text}`)
557
+
}
558
+
}
559
+
560
+
async function main(): Promise<void> {
561
+
console.log(`[e2e] domain=${domain} site=${siteName} marker=${marker}`)
562
+
const proxy = startLoopbackProxy()
563
+
let browser: Browser | null = null
564
+
let page: Page | null = null
565
+
let needsCleanup = true
566
+
567
+
try {
568
+
await waitForHttpOk(`${appUpstream}/api/health`, 'main app upstream')
569
+
await waitForHttpOk(`${firehoseUrl}/health`, 'firehose health endpoint')
570
+
await waitForHttpOk(`${hostingUrl}/health`, 'hosting health endpoint')
571
+
572
+
browser = await chromium.launch({ headless, executablePath: chromiumExecutable })
573
+
const context = await browser.newContext({
574
+
baseURL: appUrl,
575
+
ignoreHTTPSErrors: true,
576
+
})
577
+
page = await context.newPage()
578
+
579
+
const did = await completeAtprotoLogin(page)
580
+
await page.goto(`${appUrl}/onboarding`, { waitUntil: 'domcontentloaded' })
581
+
582
+
await claimDomain(page)
583
+
await uploadSite(page)
584
+
await waitForFirehoseProjection(page)
585
+
586
+
// Let the cache-invalidation update clear settle before the first hosting read.
587
+
await delay(2000)
588
+
589
+
await mapDomain(page)
590
+
await verifyHostingCacheFlow()
591
+
await deleteDomainAndVerify(page)
592
+
await primeDirectSiteCache(did)
593
+
await deleteSiteRecordAndVerify(page, did)
594
+
595
+
needsCleanup = false
596
+
console.log('[e2e] Harness completed successfully')
597
+
} finally {
598
+
if (page && needsCleanup) {
599
+
await cleanup(page)
600
+
}
601
+
await browser?.close()
602
+
proxy.stop(true)
603
+
}
604
+
}
605
+
606
+
main().catch((error) => {
607
+
console.error('[e2e] Harness failed')
608
+
console.error(error)
609
+
process.exit(1)
610
+
})
+52
apps/main-app/src/lib/cache-invalidation.ts
+52
apps/main-app/src/lib/cache-invalidation.ts
···
1
+
import { createLogger } from '@wispplace/observability'
2
+
import { getRedisClient } from './redis'
3
+
4
+
const logger = createLogger('main-app:cache-invalidation')
5
+
const CHANNEL = 'wisp:cache-invalidate'
6
+
7
+
type DomainKind = 'wisp' | 'custom'
8
+
9
+
export async function publishDomainCacheInvalidation(
10
+
domain: string,
11
+
domainKind: DomainKind,
12
+
customDomainId?: string,
13
+
): Promise<void> {
14
+
const redis = getRedisClient()
15
+
if (!redis) return
16
+
17
+
const normalizedDomain = domain.trim().toLowerCase()
18
+
if (!normalizedDomain) return
19
+
20
+
try {
21
+
const stream = process.env.WISP_CACHE_INVALIDATION_STREAM || 'wisp:cache-invalidate-stream'
22
+
const maxLen = process.env.WISP_CACHE_INVALIDATION_STREAM_MAXLEN || '10000'
23
+
const fields = [
24
+
'action',
25
+
'domain',
26
+
'domain',
27
+
normalizedDomain,
28
+
'domainKind',
29
+
domainKind,
30
+
...(customDomainId ? ['customDomainId', customDomainId] : []),
31
+
'ts',
32
+
Date.now().toString(),
33
+
]
34
+
35
+
const streamId = (await redis.send('XADD', [stream, 'MAXLEN', '~', maxLen, '*', ...fields])) as string
36
+
const message = JSON.stringify({
37
+
action: 'domain',
38
+
domain: normalizedDomain,
39
+
domainKind,
40
+
customDomainId,
41
+
streamId,
42
+
})
43
+
await redis.publish(CHANNEL, message)
44
+
} catch (err) {
45
+
logger.warn('[CacheInvalidation] Failed to publish domain invalidation', {
46
+
domain: normalizedDomain,
47
+
domainKind,
48
+
customDomainId,
49
+
err,
50
+
})
51
+
}
52
+
}
+11
apps/main-app/src/routes/domain.ts
+11
apps/main-app/src/routes/domain.ts
···
3
3
import type { NodeOAuthClient } from '@atproto/oauth-client-node'
4
4
import { createLogger } from '@wispplace/observability'
5
5
import { Elysia } from 'elysia'
6
+
import { publishDomainCacheInvalidation } from '../lib/cache-invalidation'
6
7
import {
7
8
claimCustomDomain,
8
9
claimDomain,
···
140
141
} as any,
141
142
validate: false,
142
143
})
144
+
await publishDomainCacheInvalidation(domain, 'wisp')
143
145
144
146
return { success: true, domain }
145
147
} catch (err) {
···
188
190
} as any,
189
191
validate: false,
190
192
})
193
+
if (current) await publishDomainCacheInvalidation(current, 'wisp')
194
+
await publishDomainCacheInvalidation(domain, 'wisp')
191
195
192
196
return { success: true, domain }
193
197
} catch (err) {
···
218
222
}
219
223
220
224
if (existing && existing.did === auth.did) {
225
+
await publishDomainCacheInvalidation(domainLower, 'custom', existing.id)
221
226
return {
222
227
success: true,
223
228
id: existing.id,
···
231
236
232
237
// Store in database only
233
238
await claimCustomDomain(auth.did, domainLower, hash)
239
+
await publishDomainCacheInvalidation(domainLower, 'custom', hash)
234
240
235
241
return {
236
242
success: true,
···
264
270
265
271
// Update verification status in database
266
272
await updateCustomDomainVerification(id, result.verified)
273
+
await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id)
267
274
268
275
return {
269
276
success: true,
···
299
306
300
307
// Delete from database
301
308
await deleteCustomDomain(id)
309
+
await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id)
302
310
303
311
return { success: true }
304
312
} catch (err) {
···
335
343
336
344
// Update wisp.place domain to point to this site
337
345
await updateWispDomainSite(domainLower, siteRkey)
346
+
await publishDomainCacheInvalidation(domainLower, 'wisp')
338
347
339
348
return { success: true }
340
349
} catch (err) {
···
366
375
367
376
// Delete from database
368
377
await deleteWispDomain(domainLower)
378
+
await publishDomainCacheInvalidation(domainLower, 'wisp')
369
379
370
380
// Delete from PDS
371
381
const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
···
414
424
415
425
// Update custom domain to point to this site
416
426
await updateCustomDomainRkey(id, siteRkey)
427
+
await publishDomainCacheInvalidation(domainInfo.domain, 'custom', id)
417
428
418
429
return { success: true }
419
430
} catch (err) {
+1
-1
apps/webhook-service/Dockerfile
+1
-1
apps/webhook-service/Dockerfile
+2
-2
bun.lock
+2
-2
bun.lock
···
2131
2131
2132
2132
"@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="],
2133
2133
2134
-
"@wispplace/bun-firehose/@types/bun": ["@types/bun@1.3.12", "", { "dependencies": { "bun-types": "1.3.12" } }, "sha512-DBv81elK+/VSwXHDlnH3Qduw+KxkTIWi7TXkAeh24zpi5l0B2kUg9Ga3tb4nJaPcOFswflgi/yAvMVBPrxMB+A=="],
2134
+
"@wispplace/bun-firehose/@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="],
2135
2135
2136
2136
"@wispplace/tiered-storage/@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
2137
2137
···
2321
2321
2322
2322
"@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="],
2323
2323
2324
-
"@wispplace/bun-firehose/@types/bun/bun-types": ["bun-types@1.3.12", "", { "dependencies": { "@types/node": "*" } }, "sha512-HqOLj5PoFajAQciOMRiIZGNoKxDJSr6qigAttOX40vJuSp6DN/CxWp9s3C1Xwm4oH7ybueITwiaOcWXoYVoRkA=="],
2324
+
"@wispplace/bun-firehose/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
2325
2325
2326
2326
"@wispplace/tiered-storage/@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
2327
2327
+176
docker-compose.e2e.yml
+176
docker-compose.e2e.yml
···
1
+
services:
2
+
postgres:
3
+
image: postgres:16-alpine
4
+
environment:
5
+
POSTGRES_USER: postgres
6
+
POSTGRES_PASSWORD: postgres
7
+
POSTGRES_DB: wisp
8
+
volumes:
9
+
- e2e_postgres_data:/var/lib/postgresql/data
10
+
healthcheck:
11
+
test: ["CMD-SHELL", "pg_isready -U postgres -d wisp"]
12
+
interval: 5s
13
+
timeout: 5s
14
+
retries: 20
15
+
16
+
redis:
17
+
image: redis:7-alpine
18
+
volumes:
19
+
- e2e_redis_data:/data
20
+
healthcheck:
21
+
test: ["CMD", "redis-cli", "ping"]
22
+
interval: 5s
23
+
timeout: 5s
24
+
retries: 20
25
+
26
+
minio:
27
+
image: minio/minio:RELEASE.2025-09-07T16-13-09Z
28
+
command: server /data --console-address ":9001"
29
+
environment:
30
+
MINIO_ROOT_USER: minioadmin
31
+
MINIO_ROOT_PASSWORD: minioadmin
32
+
volumes:
33
+
- e2e_minio_data:/data
34
+
healthcheck:
35
+
test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live >/dev/null"]
36
+
interval: 5s
37
+
timeout: 5s
38
+
retries: 20
39
+
40
+
minio-create-bucket:
41
+
image: minio/mc:RELEASE.2025-08-13T08-35-41Z
42
+
depends_on:
43
+
minio:
44
+
condition: service_healthy
45
+
entrypoint:
46
+
- /bin/sh
47
+
- -c
48
+
- |
49
+
mc alias set local http://minio:9000 minioadmin minioadmin
50
+
mc mb --ignore-existing local/wisp-e2e
51
+
52
+
main-app:
53
+
build:
54
+
context: .
55
+
dockerfile: Dockerfile
56
+
depends_on:
57
+
postgres:
58
+
condition: service_healthy
59
+
redis:
60
+
condition: service_healthy
61
+
environment:
62
+
NODE_ENV: development
63
+
LOCAL_DEV: "true"
64
+
PORT: "8000"
65
+
DOMAIN: http://127.0.0.1:8000
66
+
DATABASE_URL: postgres://postgres:postgres@postgres:5432/wisp
67
+
REDIS_URL: redis://redis:6379
68
+
SKIP_ADMIN_SETUP: "true"
69
+
DISABLE_DNS_WORKER: "true"
70
+
BASE_DOMAIN: wisp.place
71
+
WISP_CACHE_INVALIDATION_STREAM: wisp:e2e:cache-invalidate-stream
72
+
healthcheck:
73
+
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:8000/api/health >/dev/null"]
74
+
interval: 5s
75
+
timeout: 5s
76
+
retries: 30
77
+
78
+
firehose-service:
79
+
build:
80
+
context: .
81
+
dockerfile: apps/firehose-service/Dockerfile
82
+
depends_on:
83
+
postgres:
84
+
condition: service_healthy
85
+
redis:
86
+
condition: service_healthy
87
+
minio-create-bucket:
88
+
condition: service_completed_successfully
89
+
environment:
90
+
NODE_ENV: production
91
+
DATABASE_URL: postgres://postgres:postgres@postgres:5432/wisp
92
+
REDIS_URL: redis://redis:6379
93
+
HEALTH_PORT: "3001"
94
+
FIREHOSE_SERVICE: ${FIREHOSE_SERVICE:-wss://bsky.network}
95
+
FIREHOSE_SERVICE_SECONDARY: ${FIREHOSE_SERVICE_SECONDARY:-}
96
+
FIREHOSE_MAX_CONCURRENCY: "5"
97
+
S3_BUCKET: wisp-e2e
98
+
S3_REGION: us-east-1
99
+
S3_ENDPOINT: http://minio:9000
100
+
S3_FORCE_PATH_STYLE: "true"
101
+
S3_PREFIX: sites/
102
+
AWS_ACCESS_KEY_ID: minioadmin
103
+
AWS_SECRET_ACCESS_KEY: minioadmin
104
+
WISP_CACHE_INVALIDATION_STREAM: wisp:e2e:cache-invalidate-stream
105
+
WISP_REVALIDATE_STREAM: wisp:e2e:revalidate
106
+
healthcheck:
107
+
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:3001/health >/dev/null"]
108
+
interval: 5s
109
+
timeout: 5s
110
+
retries: 30
111
+
112
+
hosting-service:
113
+
build:
114
+
context: .
115
+
dockerfile: apps/hosting-service/Dockerfile
116
+
depends_on:
117
+
postgres:
118
+
condition: service_healthy
119
+
redis:
120
+
condition: service_healthy
121
+
minio-create-bucket:
122
+
condition: service_completed_successfully
123
+
environment:
124
+
NODE_ENV: production
125
+
PORT: "3001"
126
+
DATABASE_URL: postgres://postgres:postgres@postgres:5432/wisp
127
+
REDIS_URL: redis://redis:6379
128
+
BASE_HOST: wisp.place
129
+
BASE_DOMAIN: wisp.place
130
+
CACHE_DIR: /cache/sites
131
+
HOT_CACHE_TTL: "300"
132
+
S3_BUCKET: wisp-e2e
133
+
S3_REGION: us-east-1
134
+
S3_ENDPOINT: http://minio:9000
135
+
S3_FORCE_PATH_STYLE: "true"
136
+
S3_PREFIX: sites/
137
+
AWS_ACCESS_KEY_ID: minioadmin
138
+
AWS_SECRET_ACCESS_KEY: minioadmin
139
+
WISP_CACHE_INVALIDATION_STREAM: wisp:e2e:cache-invalidate-stream
140
+
WISP_REVALIDATE_STREAM: wisp:e2e:revalidate
141
+
WISP_CACHE_INVALIDATION_CURSOR_FILE: /cache/cache-invalidation.lastid
142
+
volumes:
143
+
- e2e_hosting_cache:/cache
144
+
healthcheck:
145
+
test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:3001/health >/dev/null"]
146
+
interval: 5s
147
+
timeout: 5s
148
+
retries: 30
149
+
150
+
e2e-runner:
151
+
build:
152
+
context: .
153
+
dockerfile: apps/main-app/e2e/Dockerfile
154
+
depends_on:
155
+
main-app:
156
+
condition: service_healthy
157
+
firehose-service:
158
+
condition: service_healthy
159
+
hosting-service:
160
+
condition: service_healthy
161
+
environment:
162
+
E2E_ATPROTO_HANDLE: ${E2E_ATPROTO_HANDLE:?Set E2E_ATPROTO_HANDLE to the test ATProto handle}
163
+
E2E_ATPROTO_PASSWORD: ${E2E_ATPROTO_PASSWORD:?Set E2E_ATPROTO_PASSWORD to the test account password}
164
+
E2E_APP_URL: http://127.0.0.1:8000
165
+
E2E_APP_UPSTREAM: http://main-app:8000
166
+
E2E_HOSTING_URL: http://hosting-service:3001
167
+
E2E_FIREHOSE_URL: http://firehose-service:3001
168
+
E2E_CLEANUP: ${E2E_CLEANUP:-true}
169
+
E2E_HEADLESS: ${E2E_HEADLESS:-true}
170
+
E2E_TIMEOUT_MS: ${E2E_TIMEOUT_MS:-180000}
171
+
172
+
volumes:
173
+
e2e_postgres_data:
174
+
e2e_redis_data:
175
+
e2e_minio_data:
176
+
e2e_hosting_cache:
+2
package.json
+2
package.json
···
32
32
"lint": "biome check .",
33
33
"lint:fix": "biome check --write .",
34
34
"format": "biome format --write .",
35
+
"e2e:harness": "docker compose -f docker-compose.e2e.yml up --build --abort-on-container-exit --exit-code-from e2e-runner e2e-runner",
36
+
"e2e:harness:down": "docker compose -f docker-compose.e2e.yml down -v --remove-orphans",
35
37
"codegen": "./scripts/codegen.sh",
36
38
"download:aturi": "bun run scripts/download-place-wisp-fs.ts",
37
39
"publish:cli": "cd cli && bun run build && npm publish && cd ../packages/create-wisp && npm publish"