dunkirk.sh / hn-alerts
this repo has no description
3
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: improve etags for dynamic routes

Kieran Klukas c33af148 456ca69f

+163 -52
+40 -12
public/app.js
··· 98 98 let allStories = []; // Store all stories for filtering 99 99 let showVerifiedOnly = false; // Default to showing all stories 100 100 101 - // Cache for ETags and data to support conditional requests 102 - const etagCache = { 101 + // Cache for ETags and data to support conditional requests with sessionStorage persistence 102 + const etagCache = JSON.parse(sessionStorage.getItem('etagCache') || JSON.stringify({ 103 103 stories: null, 104 104 totalStories: null, 105 105 verifiedUsers: null, 106 - }; 106 + })); 107 107 108 108 // Cache for actual response data 109 - const responseCache = { 109 + const responseCache = JSON.parse(sessionStorage.getItem('responseCache') || JSON.stringify({ 110 110 stories: null, 111 111 totalStories: null, 112 112 verifiedUsers: null, 113 - }; 113 + })); 114 + 115 + // Helper function to persist cache state 116 + function persistCaches() { 117 + sessionStorage.setItem('etagCache', JSON.stringify(etagCache)); 118 + sessionStorage.setItem('responseCache', JSON.stringify(responseCache)); 119 + } 114 120 115 121 // Fetch stories data 116 122 function fetchStories() { ··· 142 148 .then((response) => { 143 149 // Store the new ETag if available 144 150 const etag = response.headers.get("ETag"); 145 - if (etag) etagCache.totalStories = etag; 151 + if (etag) { 152 + etagCache.totalStories = etag; 153 + persistCaches(); 154 + } 146 155 147 156 // If 304 Not Modified, use cached data 148 157 if (response.status === 304) { 149 158 console.log("Total stories not modified, using cached data"); 150 - return responseCache.totalStories; // Use cached data 159 + return Promise.resolve(responseCache.totalStories); // Use cached data 151 160 } 152 161 153 162 return response.json(); ··· 156 165 if (data) { 157 166 // Store in cache for future 304 responses 158 167 responseCache.totalStories = data; 168 + persistCaches(); 159 169 160 170 if (typeof data.count !== "undefined") { 161 171 totalStoriesCount = data.count; ··· 177 187 verifiedUsersOptions.headers["If-None-Match"] = etagCache.verifiedUsers; 178 188 } 179 189 190 + // Add Accept-Encoding header if browser supports it 191 + if ('Accept-Encoding' in navigator) { 192 + verifiedUsersOptions.headers["Accept-Encoding"] = "gzip, deflate, br"; 193 + } 194 + 180 195 fetch("/api/stats/verified-users", verifiedUsersOptions) 181 196 .then((response) => { 182 197 // Store the new ETag if available 183 198 const etag = response.headers.get("ETag"); 184 - if (etag) etagCache.verifiedUsers = etag; 199 + if (etag) { 200 + etagCache.verifiedUsers = etag; 201 + persistCaches(); 202 + } 185 203 186 204 // If 304 Not Modified, use cached data 187 205 if (response.status === 304) { 188 - console.log("Verified users stats not modified, using cached data"); 189 - return responseCache.verifiedUsers; // Use cached data 206 + console.log("Verified users not modified, using cached data"); 207 + return Promise.resolve(responseCache.verifiedUsers); // Use cached data 190 208 } 191 209 192 210 return response.json(); ··· 195 213 if (data) { 196 214 // Store in cache for future 304 responses 197 215 responseCache.verifiedUsers = data; 216 + persistCaches(); 198 217 199 218 verifiedUserStats = data; 200 219 updateTopStats(data); // Update UI with the new stats ··· 214 233 storiesOptions.headers["If-None-Match"] = etagCache.stories; 215 234 } 216 235 236 + // Add Accept-Encoding header if browser supports it 237 + if ('Accept-Encoding' in navigator) { 238 + storiesOptions.headers["Accept-Encoding"] = "gzip, deflate, br"; 239 + } 240 + 217 241 fetch("/api/stories", storiesOptions) 218 242 .then((response) => { 219 243 // Store the new ETag if available 220 244 const etag = response.headers.get("ETag"); 221 - if (etag) etagCache.stories = etag; 245 + if (etag) { 246 + etagCache.stories = etag; 247 + persistCaches(); 248 + } 222 249 223 250 if (!response.ok) { 224 251 // Allow 304 Not Modified 225 252 if (response.status === 304) { 226 253 console.log("Stories not modified, using cached data"); 227 - return responseCache.stories; // Use cached data 254 + return Promise.resolve(responseCache.stories); // Use cached data 228 255 } 229 256 throw new Error("Network response was not ok"); 230 257 } ··· 234 261 if (data) { 235 262 // Store in cache for future 304 responses 236 263 responseCache.stories = data; 264 + persistCaches(); 237 265 238 266 // Store all stories for filtering 239 267 allStories = data;
+65 -16
public/item.html
··· 505 505 const urlParams = new URLSearchParams(window.location.search); 506 506 const storyId = urlParams.get("id"); 507 507 508 - // Cache for snapshot data and ETags 509 - const snapshotDataCache = {}; 510 - const snapshotEtagCache = {}; 508 + // Cache for snapshot data and ETags with localStorage persistence 509 + const snapshotDataCache = JSON.parse(localStorage.getItem('snapshotDataCache') || '{}'); 510 + const snapshotEtagCache = JSON.parse(localStorage.getItem('snapshotEtagCache') || '{}'); 511 + 512 + // Helper functions for ETag management 513 + const etagManager = { 514 + save: () => { 515 + localStorage.setItem('snapshotEtagCache', JSON.stringify(snapshotEtagCache)); 516 + localStorage.setItem('snapshotDataCache', JSON.stringify(snapshotDataCache)); 517 + }, 518 + clear: () => { 519 + localStorage.removeItem('snapshotEtagCache'); 520 + localStorage.removeItem('snapshotDataCache'); 521 + Object.keys(snapshotDataCache).forEach(key => delete snapshotDataCache[key]); 522 + Object.keys(snapshotEtagCache).forEach(key => delete snapshotEtagCache[key]); 523 + } 524 + }; 511 525 512 526 // Check if we have a valid story ID 513 527 if (!storyId) { ··· 524 538 } 525 539 526 540 function fetchStoryData(storyId) { 527 - fetch(`/api/story/${storyId}`) 541 + const options = { 542 + headers: {} 543 + }; 544 + 545 + // Add If-None-Match header if we have an ETag for this story 546 + const storyKey = `story_${storyId}`; 547 + if (snapshotEtagCache[storyKey]) { 548 + options.headers["If-None-Match"] = snapshotEtagCache[storyKey]; 549 + } 550 + 551 + fetch(`/api/story/${storyId}`, options) 528 552 .then((response) => { 553 + // Store the new ETag if available 554 + const etag = response.headers.get("ETag"); 555 + if (etag) { 556 + snapshotEtagCache[storyKey] = etag; 557 + etagManager.save(); 558 + } 559 + 560 + // If 304 Not Modified, use cached data 561 + if (response.status === 304) { 562 + console.log("Story not modified, using cached data"); 563 + return snapshotDataCache[storyKey]; 564 + } 565 + 529 566 if (!response.ok) { 530 567 throw new Error(`HTTP error ${response.status}`); 531 568 } 532 - return response.json(); 569 + 570 + return response.json().then(data => { 571 + // Cache the data 572 + snapshotDataCache[storyKey] = data; 573 + etagManager.save(); 574 + return data; 575 + }); 533 576 }) 534 577 .then((story) => { 535 578 displayStoryData(story); ··· 689 732 headers: {}, 690 733 }; 691 734 692 - // Add If-None-Match header if we have an ETag for this story 693 - if (snapshotEtagCache[storyId]) { 694 - options.headers["If-None-Match"] = 695 - snapshotEtagCache[storyId]; 735 + // Create snapshot-specific cache key 736 + const snapshotKey = `snapshots_${storyId}`; 737 + 738 + // Add If-None-Match header if we have an ETag for this story's snapshots 739 + if (snapshotEtagCache[snapshotKey]) { 740 + options.headers["If-None-Match"] = snapshotEtagCache[snapshotKey]; 696 741 } 697 742 698 743 fetch(`/api/story/${storyId}/snapshots`, options) 699 744 .then((response) => { 700 745 // Store the new ETag if available 701 746 const etag = response.headers.get("ETag"); 702 - if (etag) snapshotEtagCache[storyId] = etag; 747 + if (etag) { 748 + snapshotEtagCache[snapshotKey] = etag; 749 + etagManager.save(); 750 + } 703 751 704 752 if (!response.ok) { 705 - // Allow 304 Not Modified 706 - if (response.status === 304) { 753 + // Handle 304 Not Modified - use cached data 754 + if (response.status === 304 && snapshotDataCache[snapshotKey]) { 707 755 console.log( 708 756 `Story ${storyId} snapshots not modified, using cached data`, 709 757 ); 710 758 // Use the cached data for this story ID 711 - if (snapshotDataCache[storyId]) { 712 - return snapshotDataCache[storyId]; 713 - } 759 + if (snapshotDataCache[snapshotKey]) { 760 + return snapshotDataCache[snapshotKey]; 761 + } 714 762 // If we don't have cached data, re-fetch 715 763 throw new Error( 716 764 "Cache miss on 304, re-fetching", ··· 730 778 } 731 779 732 780 // Cache the snapshots data for future use 733 - snapshotDataCache[storyId] = snapshots; 781 + snapshotDataCache[snapshotKey] = snapshots; 782 + etagManager.save(); 734 783 735 784 // Show the chart 736 785 noGraph.style.display = "none";
+33 -5
src/index.ts
··· 174 174 // Register this dynamic query for potential cache warming 175 175 queryCache.register(cacheKey, queryFn, 600); 176 176 177 + // Check client ETag before executing query 178 + const clientETag = req.headers.get("if-none-match"); 179 + 177 180 // Execute the query with caching 178 181 const data = await queryCache.get(cacheKey, queryFn, 600); 179 182 ··· 184 187 }); 185 188 } 186 189 187 - // Create response with cached headers 188 - const headers = createCacheHeaders(cacheKey, 600); 189 - const response = new Response(JSON.stringify(data), { headers }); 190 + // Create response with cached headers based on actual data content 191 + const headers = createCacheHeaders(cacheKey, 600, data); 190 192 193 + // Return 304 if client's ETag matches our data-based ETag 194 + if (clientETag && clientETag === headers.ETag) { 195 + return new Response(null, { 196 + status: 304, 197 + headers: { 198 + ETag: headers.ETag, 199 + "Cache-Control": headers["Cache-Control"] as string, 200 + }, 201 + }); 202 + } 203 + 204 + const response = new Response(JSON.stringify(data), { headers }); 191 205 return compressResponse(req, response); 192 206 } catch (error) { 193 207 if (!isProduction) { ··· 393 407 // Execute the query with caching 394 408 const data = await queryCache.get(cacheKey, queryFn, 3600); 395 409 396 - // Use cached headers for better performance 397 - const headers = createCacheHeaders(cacheKey, 3600); 410 + // Check client ETag before sending response 411 + const clientETag = req.headers.get("if-none-match"); 412 + 413 + // Create response with cached headers based on actual data 414 + const headers = createCacheHeaders(cacheKey, 3600, data); 415 + 416 + // Return 304 if client's ETag matches our data-based ETag 417 + if (clientETag && clientETag === headers.ETag) { 418 + return new Response(null, { 419 + status: 304, 420 + headers: { 421 + ETag: headers.ETag, 422 + "Cache-Control": headers["Cache-Control"] as string, 423 + }, 424 + }); 425 + } 398 426 399 427 // Create response with optimized headers 400 428 const response = new Response(JSON.stringify(data), { headers });
+25 -19
src/libs/cache.ts
··· 13 13 export function createCacheHeaders( 14 14 key: string, 15 15 maxAge = 300, 16 + data?: unknown, 16 17 ): Record<string, string> { 17 - // Generate stable ETag based on version and cache key 18 - // Only changes when version changes or when cache TTL expires (divided by TTL) 19 - const etag = `"${version}-${key}-${Math.floor(Date.now() / (maxAge * 1000))}"`; 18 + // Generate stable ETag based on version, cache key, and data hash if available 19 + let etag: string; 20 + 21 + if (data) { 22 + // Generate based on actual data content for stronger validation 23 + const dataStr = JSON.stringify(data); 24 + const dataHash = Bun.hash(dataStr).toString(36).slice(0, 12); 25 + etag = `"${version}-${key}-${dataHash}"`; 26 + } else { 27 + // Fallback to time-based for headers without data 28 + etag = `"${version}-${key}-${Math.floor(Date.now() / (maxAge * 1000))}"`; 29 + } 20 30 21 31 return { 22 32 "Content-Type": "application/json", ··· 628 638 const defaultToPriority = cacheKey === "leaderboard_stories" || isPriority; 629 639 queryCache.register(cacheKey, queryFn, ttl, defaultToPriority); 630 640 631 - // Pre-create cache headers to avoid recreating them on each request 632 - const cacheHeaders = createCacheHeaders(cacheKey, ttl); 633 - 634 641 // Prepare common response headers 635 642 const errorHeaders = { 636 643 "Content-Type": "application/json", ··· 649 656 const requestStart = isProduction ? 0 : performance.now(); 650 657 651 658 try { 652 - // Check client ETag before executing query 659 + // Get data from cache or execute query first 660 + const data = await queryCache.get(cacheKey, queryFn, ttl); 661 + 662 + // Generate data-based ETag for better validation 663 + const headers = createCacheHeaders(cacheKey, ttl, data); 664 + 665 + // Check client ETag after we have our data 653 666 const clientETag = request.headers.get("if-none-match"); 654 - const cacheHeaders = createCacheHeaders(cacheKey, ttl); 655 - 656 - // If client ETag matches, return 304 657 - if (clientETag && clientETag === cacheHeaders.ETag) { 667 + 668 + // Return 304 if client's ETag matches our data-based ETag 669 + if (clientETag && clientETag === headers.ETag) { 658 670 return new Response(null, { 659 671 status: 304, 660 672 headers: { 661 - ETag: cacheHeaders.ETag, 662 - "Cache-Control": cacheHeaders["Cache-Control"] as string, 673 + ETag: headers.ETag, 674 + "Cache-Control": headers["Cache-Control"] as string, 663 675 }, 664 676 }); 665 677 } 666 - 667 - // Get data from cache or execute query 668 - const data = await queryCache.get(cacheKey, queryFn, ttl); 669 - 670 - // Create response with proper caching headers and timing info 671 - const headers = { ...cacheHeaders }; 672 678 673 679 // Add server timing header in development 674 680 if (!isProduction && requestStart > 0) {