Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place
87
fork

Configure Feed

Select the types of activity you want to include in your feed.

:3

+68 -35
+20
apps/hosting-service/src/lib/file-serving.integration.test.ts
··· 104 104 storageGetWithMetadataKeys.length = 0 105 105 siteFileCids = null 106 106 cache.clear('redirectRules') 107 + cache.clear('siteCache') 107 108 resetHtmlHotCacheWarmupForTests() 108 109 }) 109 110 ··· 166 167 expect(response.status).toBe(200) 167 168 expect(await response.text()).toBe('direct markdown') 168 169 expect(storageGetWithMetadataKeys.filter((key) => key === `${DID}/${RKEY}/direct.md`)).toHaveLength(1) 170 + }) 171 + 172 + test('skips manifest-absent storage probes before clean URL html fallback', async () => { 173 + storeFile('modelapp.html', '<html>model app</html>') 174 + siteFileCids = { 175 + 'modelapp.html': 'modelapp-cid', 176 + } 177 + 178 + const response = await serveFileInternal(DID, RKEY, 'modelapp', { 179 + $type: 'place.wisp.settings', 180 + directoryListing: false, 181 + cleanUrls: true, 182 + }) 183 + 184 + expect(response.status).toBe(200) 185 + expect(await response.text()).toBe('<html>model app</html>') 186 + expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp`) 187 + expect(storageGetWithMetadataKeys).not.toContain(`${DID}/${RKEY}/modelapp/index.html`) 188 + expect(storageGetWithMetadataKeys).toContain(`${DID}/${RKEY}/modelapp.html`) 169 189 }) 170 190 })
+48 -35
apps/hosting-service/src/lib/file-serving.ts
··· 63 63 return `${did}/${rkey}/${normalized}` 64 64 } 65 65 66 + function normalizeFilePath(filePath: string): string { 67 + return filePath.startsWith('/') ? filePath.slice(1) : filePath 68 + } 69 + 70 + function manifestHasPath(fileCids: Record<string, string> | null, filePath: string): boolean { 71 + return fileCids === null || fileCids[normalizeFilePath(filePath)] !== undefined 72 + } 73 + 66 74 /** 67 75 * Fetch a per-site fallback file (SPA, custom 404, auto-detected 404 pages), 68 76 * caching null results so repeated 404 responses don't re-hit S3 for files ··· 104 112 cache.set('siteFiles', cacheKey, null) 105 113 } 106 114 return result 115 + } 116 + 117 + async function getExpectedFileCidsForSite( 118 + did: string, 119 + rkey: string, 120 + trace?: RequestTrace | null, 121 + ): Promise<Record<string, string> | null> { 122 + const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey)) 123 + if (!siteCache) return null 124 + return normalizeFileCids(siteCache.file_cids).value 107 125 } 108 126 109 127 function shouldServeUpdatingPage(requestHeaders?: Record<string, string>): boolean { ··· 190 208 checkPath += indexFiles[0] || 'index.html' 191 209 } 192 210 193 - const normalizedCheckPath = checkPath.startsWith('/') ? checkPath.slice(1) : checkPath 194 - const siteCache = await span(trace, 'db:siteCache:redirectCheck', () => getSiteCache(did, rkey)) 195 - if (siteCache) { 196 - const fileCids = normalizeFileCids(siteCache.file_cids).value 197 - return fileCids[normalizedCheckPath] !== undefined 211 + const fileCids = await getExpectedFileCidsForSite(did, rkey, trace) 212 + if (fileCids !== null) { 213 + return manifestHasPath(fileCids, checkPath) 198 214 } 199 215 200 216 const fileInStorage = await span(trace, `storage:${checkPath}`, () => getFileWithMetadata(did, rkey, checkPath)) ··· 446 462 447 463 const getExpectedFileCids = async (): Promise<Record<string, string> | null> => { 448 464 if (expectedFileCids !== undefined) return expectedFileCids 449 - const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey)) 450 - if (!siteCache) { 451 - expectedFileCids = null 452 - return null 453 - } 454 - expectedFileCids = normalizeFileCids(siteCache.file_cids).value 465 + expectedFileCids = await getExpectedFileCidsForSite(did, rkey, trace) 455 466 return expectedFileCids 467 + } 468 + 469 + const getExpectedFileWithMetadata = async (path: string): Promise<FileStorageResult | null> => { 470 + const fileCids = await getExpectedFileCids() 471 + if (!manifestHasPath(fileCids, path)) return null 472 + return await span(trace, `storage:${path}`, () => getFileWithMetadata(did, rkey, path)) 456 473 } 457 474 458 475 const markExpectedMiss = async (path: string) => { ··· 485 502 if (!requestPath || !hasFileExtension(requestPath)) { 486 503 // For non-empty extensionless paths, try as a direct file first (e.g. binary downloads) 487 504 if (requestPath && !isDirectoryPathRequest) { 488 - const directResult = await span(trace, `storage:${requestPath}`, () => 489 - getFileWithMetadata(did, rkey, requestPath), 490 - ) 505 + const directResult = await getExpectedFileWithMetadata(requestPath) 491 506 if (directResult) { 492 507 return buildResponseFromStorageResult(directResult, requestPath, settings, requestHeaders) 493 508 } ··· 496 511 497 512 for (const indexFile of indexFiles) { 498 513 const indexPath = requestPath ? `${requestPath}/${indexFile}` : indexFile 499 - const result = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath)) 514 + const result = await getExpectedFileWithMetadata(indexPath) 500 515 if (result) { 501 516 return buildResponseFromStorageResult(result, indexPath, settings, requestHeaders) 502 517 } ··· 531 546 const fileRequestPath: string = requestPath || indexFiles[0] || 'index.html' 532 547 533 548 // Retrieve from tiered storage 534 - const result = await span(trace, `storage:${fileRequestPath}`, () => getFileWithMetadata(did, rkey, fileRequestPath)) 549 + const result = await getExpectedFileWithMetadata(fileRequestPath) 535 550 536 551 if (result) { 537 552 return buildResponseFromStorageResult(result, fileRequestPath, settings, requestHeaders) ··· 542 557 // e.g. Astro emits `relay.md/index.html` for .md routes) 543 558 for (const indexFileName of indexFiles) { 544 559 const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName 545 - const indexResult = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath)) 560 + const indexResult = await getExpectedFileWithMetadata(indexPath) 546 561 if (indexResult) { 547 562 return buildResponseFromStorageResult(indexResult, indexPath, settings, requestHeaders) 548 563 } ··· 552 567 // Try clean URLs: /about -> /about.html 553 568 if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { 554 569 const htmlPath = `${fileRequestPath}.html` 555 - const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileWithMetadata(did, rkey, htmlPath)) 570 + const htmlResult = await getExpectedFileWithMetadata(htmlPath) 556 571 if (htmlResult) { 557 572 return buildResponseFromStorageResult(htmlResult, htmlPath, settings, requestHeaders) 558 573 } ··· 561 576 // Also try /about/index.html 562 577 for (const indexFileName of indexFiles) { 563 578 const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName 564 - const indexResult = await span(trace, `storage:${indexPath}`, () => getFileWithMetadata(did, rkey, indexPath)) 579 + const indexResult = await getExpectedFileWithMetadata(indexPath) 565 580 if (indexResult) { 566 581 return buildResponseFromStorageResult(indexResult, indexPath, settings, requestHeaders) 567 582 } ··· 770 785 771 786 const getExpectedFileCids = async (): Promise<Record<string, string> | null> => { 772 787 if (expectedFileCids !== undefined) return expectedFileCids 773 - const siteCache = await span(trace, 'db:siteCache', () => getSiteCache(did, rkey)) 774 - if (!siteCache) { 775 - expectedFileCids = null 776 - return null 777 - } 778 - expectedFileCids = normalizeFileCids(siteCache.file_cids).value 788 + expectedFileCids = await getExpectedFileCidsForSite(did, rkey, trace) 779 789 return expectedFileCids 780 790 } 781 791 792 + const getExpectedFileForRequest = async (path: string): Promise<FileForRequestResult | null> => { 793 + const fileCids = await getExpectedFileCids() 794 + const rewrittenPath = `.rewritten/${normalizeFilePath(path)}` 795 + if (!manifestHasPath(fileCids, path) && !manifestHasPath(fileCids, rewrittenPath)) return null 796 + return await span(trace, `storage:${path}`, () => getFileForRequest(did, rkey, path, true)) 797 + } 798 + 782 799 const markExpectedMiss = async (path: string) => { 783 800 if (expectedMissPath) return 784 801 const fileCids = await getExpectedFileCids() ··· 827 844 if (!requestPath || !hasFileExtension(requestPath)) { 828 845 // For non-empty extensionless paths, try as a direct file first (e.g. binary downloads) 829 846 if (requestPath && !isDirectoryPathRequest) { 830 - const directResult = await span(trace, `storage:${requestPath}`, () => 831 - getFileForRequest(did, rkey, requestPath, true), 832 - ) 847 + const directResult = await getExpectedFileForRequest(requestPath) 833 848 if (directResult) { 834 849 return await buildResponse(directResult) 835 850 } ··· 838 853 839 854 for (const indexFile of indexFiles) { 840 855 const indexPath = requestPath ? `${requestPath}/${indexFile}` : indexFile 841 - const fileResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) 856 + const fileResult = await getExpectedFileForRequest(indexPath) 842 857 if (fileResult) { 843 858 return await buildResponse(fileResult) 844 859 } ··· 872 887 // Try to serve as a file 873 888 const fileRequestPath: string = requestPath || indexFiles[0] || 'index.html' 874 889 875 - const fileResult = await span(trace, `storage:${fileRequestPath}`, () => 876 - getFileForRequest(did, rkey, fileRequestPath, true), 877 - ) 890 + const fileResult = await getExpectedFileForRequest(fileRequestPath) 878 891 if (fileResult) { 879 892 return await buildResponse(fileResult) 880 893 } ··· 884 897 // e.g. Astro emits `relay.md/index.html` for .md routes) 885 898 for (const indexFileName of indexFiles) { 886 899 const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName 887 - const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) 900 + const indexResult = await getExpectedFileForRequest(indexPath) 888 901 if (indexResult) { 889 902 return await buildResponse(indexResult) 890 903 } ··· 894 907 // Try clean URLs: /about -> /about.html 895 908 if (settings?.cleanUrls && !hasFileExtension(fileRequestPath)) { 896 909 const htmlPath = `${fileRequestPath}.html` 897 - const htmlResult = await span(trace, `storage:${htmlPath}`, () => getFileForRequest(did, rkey, htmlPath, true)) 910 + const htmlResult = await getExpectedFileForRequest(htmlPath) 898 911 if (htmlResult) { 899 912 return await buildResponse(htmlResult) 900 913 } ··· 903 916 // Also try /about/index.html 904 917 for (const indexFileName of indexFiles) { 905 918 const indexPath = fileRequestPath ? `${fileRequestPath}/${indexFileName}` : indexFileName 906 - const indexResult = await span(trace, `storage:${indexPath}`, () => getFileForRequest(did, rkey, indexPath, true)) 919 + const indexResult = await getExpectedFileForRequest(indexPath) 907 920 if (indexResult) { 908 921 return await buildResponse(indexResult) 909 922 }