Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
wisp.place
:3
+68
-35
+20
apps/hosting-service/src/lib/file-serving.integration.test.ts
+20
apps/hosting-service/src/lib/file-serving.integration.test.ts
···
104
104
storageGetWithMetadataKeys.length = 0
105
105
siteFileCids = null
106
106
cache.clear('redirectRules')
107
+
cache.clear('siteCache')
107
108
resetHtmlHotCacheWarmupForTests()
108
109
})
109
110
···
166
167
expect(response.status).toBe(200)
167
168
expect(await response.text()).toBe('direct markdown')
168
169
expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/direct.md`)).toHaveLength(1)
170
+
})
171
+
172
+
test('skips manifest-absent storage probes before clean URL html fallback', async () => {
173
+
storeFile('modelapp.html', '<html>model app</html>')
174
+
siteFileCids = {
175
+
'modelapp.html': 'modelapp-cid',
176
+
}
177
+
178
+
const response = await serveFileInternal(DID, RKEY, 'modelapp', {
179
+
$type: 'place.wisp.settings',
180
+
directoryListing: false,
181
+
cleanUrls: true,
182
+
})
183
+
184
+
expect(response.status).toBe(200)
185
+
expect(await response.text()).toBe('<html>model app</html>')
186
+
expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp`)
187
+
expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp/index.html`)
188
+
expect(storageGetWithMetadataKeys).toContain(`${DID}/${RKEY}/modelapp.html`)
169
189
})
170
190
})
+48
-35
apps/hosting-service/src/lib/file-serving.ts
+48
-35
apps/hosting-service/src/lib/file-serving.ts
···
63
63
return `${did}/${rkey}/${normalized}`
64
64
}
65
65
66
+
function normalizeFilePath(filePath: string): string {
67
+
return filePath.startsWith('/') ? filePath.slice(1) : filePath
68
+
}
69
+
70
+
function manifestHasPath(fileCids: Record<string, string> | null, filePath: string): boolean {
71
+
return fileCids === null || fileCids[normalizeFilePath(filePath)] !== undefined
72
+
}
73
+
66
74
/**
67
75
* Fetch a per-site fallback file (SPA, custom 404, auto-detected 404 pages),
68
76
* caching null results so repeated 404 responses don't re-hit S3 for files
···
104
112
cache.set('siteFiles', cacheKey, null)
105
113
}
106
114
return result
115
+
}
116
+
117
+
async function getExpectedFileCidsForSite(
118
+
did: string,
119
+
rkey: string,
120
+
trace?: RequestTrace | null,
121
+
): Promise<Record<string, string> | null> {
122
+
const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey))
123
+
if (!siteCache) return null
124
+
return normalizeFileCids(siteCache.file_cids).value
107
125
}
108
126
109
127
function shouldServeUpdatingPage(requestHeaders?: Record<string, string>): boolean {
···
190
208
checkPath += indexFiles[0] || 'index.html'
191
209
}
192
210
193
-
const normalizedCheckPath = checkPath.startsWith('/') ? checkPath.slice(1) : checkPath
194
-
const siteCache = await span(trace, 'db:siteCache:redirectCheck', () => getSiteCache(did, rkey))
195
-
if (siteCache) {
196
-
const fileCids = normalizeFileCids(siteCache.file_cids).value
197
-
return fileCids[normalizedCheckPath] !== undefined
211
+
const fileCids = await getExpectedFileCidsForSite(did, rkey, trace)
212
+
if (fileCids !== null) {
213
+
return manifestHasPath(fileCids, checkPath)
198
214
}
199
215
200
216
const fileInStorage = await span(trace, `storage:${checkPath}`, () => getFileWithMetadata(did, rkey, checkPath))
···
446
462
447
463
const getExpectedFileCids = async (): Promise<Record<string, string> | null> => {
448
464
if (expectedFileCids !== undefined) return expectedFileCids
449
-
const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey))
450
-
if (!siteCache) {
451
-
expectedFileCids = null
452
-
return null
453
-
}
454
-
expectedFileCids = normalizeFileCids(siteCache.file_cids).value
465
+
expectedFileCids = await getExpectedFileCidsForSite(did, rkey, trace)
455
466
return expectedFileCids
467
+
}
468
+
469
+
const getExpectedFileWithMetadata = async (path: string): Promise<FileStorageResult | null> => {
470
+
const fileCids = await getExpectedFileCids()
471
+
if (!manifestHasPath(fileCids, path)) return null
472
+
return await span(trace, `storage:${path}`, () => getFileWithMetadata(did, rkey, path))
456
473
}
457
474
458
475
const markExpectedMiss = async (path: string) => {
···
485
502
if (!requestPath || !hasFileExtension(requestPath)) {
486
503
// For non-empty extensionless paths, try as a direct file first (e.g. binary downloads)
487
504
if (requestPath && !isDirectoryPathRequest) {
488
-
const directResult = await span(trace, `storage:${requestPath}`, () =>
489
-
getFileWithMetadata(did, rkey, requestPath),
490
-
)
505
+
const directResult = await getExpectedFileWithMetadata(requestPath)
491
506
if (directResult) {
492
507
return buildResponseFromStorageResult(directResult, requestPath, settings, requestHeaders)
493
508
}
···
496
511
497
512
for (const indexFile of indexFiles) {
498
513
const indexPath = requestPath ? `${requestPath}/${indexFile}` : indexFile
499
-
const result = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath))
514
+
const result = await getExpectedFileWithMetadata(indexPath)
500
515
if (result) {
501
516
return buildResponseFromStorageResult(result, indexPath, settings, requestHeaders)
502
517
}
···
531
546
const fileRequestPath: string = requestPath || indexFiles[0] || 'index.html'
532
547
533
548
// Retrieve from tiered storage
534
-
const result = await span(trace, `storage:${fileRequestPath}`, () => getFileWithMetadata(did, rkey, fileRequestPath))
549
+
const result = await getExpectedFileWithMetadata(fileRequestPath)
535
550
536
551
if (result) {
537
552
return buildResponseFromStorageResult(result, fileRequestPath, settings, requestHeaders)
···
542
557
// e.g. Astro emits `relay.md/index.html` for .md routes)
543
558
for (const indexFileName of indexFiles) {
544
559
const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName
545
-
const indexResult = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath))
560
+
const indexResult = await getExpectedFileWithMetadata(indexPath)
546
561
if (indexResult) {
547
562
return buildResponseFromStorageResult(indexResult, indexPath, settings, requestHeaders)
548
563
}
···
552
567
// Try clean URLs: /about -> /about.html
553
568
if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) {
554
569
const htmlPath = `${fileRequestPath}.html`
555
-
const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileWithMetadata(did, rkey, htmlPath))
570
+
const htmlResult = await getExpectedFileWithMetadata(htmlPath)
556
571
if (htmlResult) {
557
572
return buildResponseFromStorageResult(htmlResult, htmlPath, settings, requestHeaders)
558
573
}
···
561
576
// Also try /about/index.html
562
577
for (const indexFileName of indexFiles) {
563
578
const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName
564
-
const indexResult = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath))
579
+
const indexResult = await getExpectedFileWithMetadata(indexPath)
565
580
if (indexResult) {
566
581
return buildResponseFromStorageResult(indexResult, indexPath, settings, requestHeaders)
567
582
}
···
770
785
771
786
const getExpectedFileCids = async (): Promise<Record<string, string> | null> => {
772
787
if (expectedFileCids !== undefined) return expectedFileCids
773
-
const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey))
774
-
if (!siteCache) {
775
-
expectedFileCids = null
776
-
return null
777
-
}
778
-
expectedFileCids = normalizeFileCids(siteCache.file_cids).value
788
+
expectedFileCids = await getExpectedFileCidsForSite(did, rkey, trace)
779
789
return expectedFileCids
780
790
}
781
791
792
+
const getExpectedFileForRequest = async (path: string): Promise<FileForRequestResult | null> => {
793
+
const fileCids = await getExpectedFileCids()
794
+
const rewrittenPath = `.rewritten/${normalizeFilePath(path)}`
795
+
if (!manifestHasPath(fileCids, path) && !manifestHasPath(fileCids, rewrittenPath)) return null
796
+
return await span(trace, `storage:${path}`, () => getFileForRequest(did, rkey, path, true))
797
+
}
798
+
782
799
const markExpectedMiss = async (path: string) => {
783
800
if (expectedMissPath) return
784
801
const fileCids = await getExpectedFileCids()
···
827
844
if (!requestPath || !hasFileExtension(requestPath)) {
828
845
// For non-empty extensionless paths, try as a direct file first (e.g. binary downloads)
829
846
if (requestPath && !isDirectoryPathRequest) {
830
-
const directResult = await span(trace, `storage:${requestPath}`, () =>
831
-
getFileForRequest(did, rkey, requestPath, true),
832
-
)
847
+
const directResult = await getExpectedFileForRequest(requestPath)
833
848
if (directResult) {
834
849
return await buildResponse(directResult)
835
850
}
···
838
853
839
854
for (const indexFile of indexFiles) {
840
855
const indexPath = requestPath ? `${requestPath}/${indexFile}` : indexFile
841
-
const fileResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true))
856
+
const fileResult = await getExpectedFileForRequest(indexPath)
842
857
if (fileResult) {
843
858
return await buildResponse(fileResult)
844
859
}
···
872
887
// Try to serve as a file
873
888
const fileRequestPath: string = requestPath || indexFiles[0] || 'index.html'
874
889
875
-
const fileResult = await span(trace, `storage:${fileRequestPath}`, () =>
876
-
getFileForRequest(did, rkey, fileRequestPath, true),
877
-
)
890
+
const fileResult = await getExpectedFileForRequest(fileRequestPath)
878
891
if (fileResult) {
879
892
return await buildResponse(fileResult)
880
893
}
···
884
897
// e.g. Astro emits `relay.md/index.html` for .md routes)
885
898
for (const indexFileName of indexFiles) {
886
899
const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName
887
-
const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true))
900
+
const indexResult = await getExpectedFileForRequest(indexPath)
888
901
if (indexResult) {
889
902
return await buildResponse(indexResult)
890
903
}
···
894
907
// Try clean URLs: /about -> /about.html
895
908
if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) {
896
909
const htmlPath = `${fileRequestPath}.html`
897
-
const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileForRequest(did, rkey, htmlPath, true))
910
+
const htmlResult = await getExpectedFileForRequest(htmlPath)
898
911
if (htmlResult) {
899
912
return await buildResponse(htmlResult)
900
913
}
···
903
916
// Also try /about/index.html
904
917
for (const indexFileName of indexFiles) {
905
918
const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName
906
-
const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true))
919
+
const indexResult = await getExpectedFileForRequest(indexPath)
907
920
if (indexResult) {
908
921
return await buildResponse(indexResult)
909
922
}