this repo has no description
3
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: add cache system

+375 -116
+171 -21
public/app.js
··· 98 98 let allStories = []; // Store all stories for filtering 99 99 let showVerifiedOnly = false; // Default to showing all stories 100 100 101 + // Cache for ETags and data to support conditional requests 102 + const etagCache = { 103 + stories: null, 104 + totalStories: null, 105 + verifiedUsers: null 106 + }; 107 + 108 + // Cache for actual response data 109 + const responseCache = { 110 + stories: null, 111 + totalStories: null, 112 + verifiedUsers: null 113 + }; 114 + 101 115 // Fetch stories data 102 116 function fetchStories() { 103 117 // Update last refresh time ··· 111 125 } 112 126 113 127 // Fetch total stories count first 114 - fetch(`/api/stats/total-stories?_=${Date.now()}`) 115 - .then((response) => response.json()) 128 + const totalStoriesOptions = { 129 + headers: {} 130 + }; 131 + 132 + // Add If-None-Match header if we have an ETag 133 + if (etagCache.totalStories) { 134 + totalStoriesOptions.headers['If-None-Match'] = etagCache.totalStories; 135 + } 136 + 137 + fetch('/api/stats/total-stories', totalStoriesOptions) 138 + .then((response) => { 139 + // Store the new ETag if available 140 + const etag = response.headers.get('ETag'); 141 + if (etag) etagCache.totalStories = etag; 142 + 143 + // If 304 Not Modified, use cached data 144 + if (response.status === 304) { 145 + console.log('Total stories not modified, using cached data'); 146 + return responseCache.totalStories; // Use cached data 147 + } 148 + 149 + return response.json(); 150 + }) 116 151 .then((data) => { 117 - if (data && typeof data.count !== "undefined") { 118 - totalStoriesCount = data.count; 119 - currentFrontpageCountEl.textContent = totalStoriesCount; 152 + if (data) { 153 + // Store in cache for future 304 responses 154 + responseCache.totalStories = data; 155 + 156 + if (typeof data.count !== "undefined") { 157 + totalStoriesCount = data.count; 158 + currentFrontpageCountEl.textContent = totalStoriesCount; 159 + } 120 160 } 121 161 }) 122 162 .catch((error) => { ··· 124 164 }); 125 165 126 166 // Fetch verified user stats for the top row 127 - fetch(`/api/stats/verified-users?_=${Date.now()}`) 128 - .then((response) => response.json()) 167 + const verifiedUsersOptions = { 168 + headers: {} 169 + }; 170 + 171 + // Add If-None-Match header if we have an ETag 172 + if (etagCache.verifiedUsers) { 173 + verifiedUsersOptions.headers['If-None-Match'] = etagCache.verifiedUsers; 174 + } 175 + 176 + fetch('/api/stats/verified-users', verifiedUsersOptions) 177 + .then((response) => { 178 + // Store the new ETag if available 179 + const etag = response.headers.get('ETag'); 180 + if (etag) etagCache.verifiedUsers = etag; 181 + 182 + // If 304 Not Modified, use cached data 183 + if (response.status === 304) { 184 + console.log('Verified users stats not modified, using cached data'); 185 + return responseCache.verifiedUsers; // Use cached data 186 + } 187 + 188 + return response.json(); 189 + }) 129 190 .then((data) => { 130 - verifiedUserStats = data; 131 - // Update top row stats with the new data 132 - updateTopStats(data); 191 + if (data) { 192 + // Store in cache for future 304 responses 193 + responseCache.verifiedUsers = data; 194 + 195 + verifiedUserStats = data; 196 + updateTopStats(data); // Update UI with the new stats 197 + } 133 198 }) 134 199 .catch((error) => { 135 200 console.error("Error fetching verified user stats:", error); 136 201 }); 137 202 138 - fetch(`/api/stories?_=${Date.now()}`) 203 + // Fetch stories 204 + const storiesOptions = { 205 + headers: {} 206 + }; 207 + 208 + // Add If-None-Match header if we have an ETag 209 + if (etagCache.stories) { 210 + storiesOptions.headers['If-None-Match'] = etagCache.stories; 211 + } 212 + 213 + fetch('/api/stories', storiesOptions) 139 214 .then((response) => { 215 + // Store the new ETag if available 216 + const etag = response.headers.get('ETag'); 217 + if (etag) etagCache.stories = etag; 218 + 140 219 if (!response.ok) { 220 + // Allow 304 Not Modified 221 + if (response.status === 304) { 222 + console.log('Stories not modified, using cached data'); 223 + return responseCache.stories; // Use cached data 224 + } 141 225 throw new Error("Network response was not ok"); 142 226 } 143 227 return response.json(); 144 228 }) 145 229 .then((data) => { 146 - // Store all stories for filtering 147 - allStories = data; 148 - 149 - // Apply filters and update UI 150 - applyFiltersAndUpdateUI(); 230 + if (data) { 231 + // Store in cache for future 304 responses 232 + responseCache.stories = data; 233 + 234 + // Store all stories for filtering 235 + allStories = data; 236 + // Apply filters and update UI 237 + applyFiltersAndUpdateUI(); 238 + } 151 239 }) 152 240 .catch((error) => { 153 241 storyList.innerHTML = `<div class="loading">Error loading data: ${error.message}</div>`; ··· 287 375 } 288 376 289 377 // Load story snapshots and display graph 378 + // Cache for story snapshot ETags and data 379 + const snapshotEtagCache = {}; 380 + const snapshotDataCache = {}; 381 + 290 382 function loadStoryGraph(storyId) { 291 383 if (activeStoryId === storyId) return; 292 384 activeStoryId = storyId; ··· 295 387 rankChart.style.display = "none"; 296 388 noGraph.innerHTML = '<div class="loading">Loading graph data...</div>'; 297 389 298 - fetch(`/api/story/${storyId}/snapshots?_=${Date.now()}`) 390 + const options = { 391 + headers: {} 392 + }; 393 + 394 + // Add If-None-Match header if we have an ETag for this story 395 + if (snapshotEtagCache[storyId]) { 396 + options.headers['If-None-Match'] = snapshotEtagCache[storyId]; 397 + } 398 + 399 + // Use template literal correctly since we need string interpolation 400 + fetch(`/api/story/${storyId}/snapshots`, options) 299 401 .then((response) => { 402 + // Store the new ETag if available 403 + const etag = response.headers.get('ETag'); 404 + if (etag) snapshotEtagCache[storyId] = etag; 405 + 300 406 if (!response.ok) { 407 + // Allow 304 Not Modified 408 + if (response.status === 304) { 409 + console.log(`Story ${storyId} snapshots not modified, using cached data`); 410 + // Use the cached data for this story ID 411 + if (snapshotDataCache[storyId]) { 412 + return snapshotDataCache[storyId]; 413 + } 414 + // If we don't have cached data, re-fetch 415 + throw new Error("Cache miss on 304, re-fetching"); 416 + } 301 417 throw new Error("Failed to fetch snapshot data"); 302 418 } 303 419 return response.json(); ··· 309 425 return; 310 426 } 311 427 428 + // Cache the snapshots data for future use 429 + snapshotDataCache[storyId] = snapshots; 430 + 312 431 displayGraph(snapshots); 313 432 }) 314 433 .catch((error) => { ··· 430 549 now = Date.now(); 431 550 432 551 // Get total stories from the API 433 - fetch(`/api/stats/total-stories?_=${Date.now()}`) 434 - .then((response) => response.json()) 552 + const options = { 553 + headers: {} 554 + }; 555 + 556 + // Add If-None-Match header if we have an ETag 557 + if (etagCache.totalStories) { 558 + options.headers['If-None-Match'] = etagCache.totalStories; 559 + } 560 + 561 + fetch('/api/stats/total-stories', options) 562 + .then((response) => { 563 + // Store the new ETag if available 564 + const etag = response.headers.get('ETag'); 565 + if (etag) etagCache.totalStories = etag; 566 + 567 + // If 304 Not Modified, use cached data 568 + if (response.status === 304) { 569 + console.log('Total stories not modified, using cached data for metrics'); 570 + return responseCache.totalStories; 571 + } 572 + 573 + return response.json(); 574 + }) 435 575 .then((data) => { 436 - currentFrontpageCountEl.textContent = 437 - data.count !== undefined ? data.count : allStories.length; 576 + if (data) { 577 + // Update cache 578 + responseCache.totalStories = data; 579 + 580 + if (data.count !== undefined) { 581 + currentFrontpageCountEl.textContent = data.count; 582 + } else { 583 + currentFrontpageCountEl.textContent = allStories.length; // Fallback to local data 584 + } 585 + } else { 586 + currentFrontpageCountEl.textContent = allStories.length; // Fallback to local data 587 + } 438 588 }) 439 589 .catch((error) => { 440 590 console.error("Error fetching total stories count:", error);
+35 -17
src/features/services/check_hn.ts
··· 1 1 import { CronJob } from "cron"; 2 2 import * as Sentry from "@sentry/bun"; 3 - import { db, environment } from "../../index"; 3 + import { db, environment, invalidateAllCaches } from "../../index"; 4 4 import { 5 5 users as usersTable, 6 6 stories as storiesTable, ··· 147 147 // First, filter out job stories and create a map of adjusted positions 148 148 const positionMap = new Map<number, number>(); // Maps story ID to adjusted position 149 149 let adjustedPosition = 0; 150 - 150 + 151 151 for (let i = 0; i < stories.length; i++) { 152 152 const story = stories[i]; 153 153 if (!story) continue; 154 - 154 + 155 155 // Skip jobs, but include other types (mainly 'story') 156 - if (story.type === 'job') { 157 - console.log(`[INFO] Skipping job at original position ${i + 1}, ID: ${story.id}`); 156 + if (story.type === "job") { 157 + console.log( 158 + `[INFO] Skipping job at original position ${i + 1}, ID: ${story.id}`, 159 + ); 158 160 continue; 159 161 } 160 - 162 + 161 163 // Only increment position for non-job stories 162 164 adjustedPosition++; 163 165 positionMap.set(story.id, adjustedPosition); 164 - 166 + 165 167 // Add to non-job story IDs for front page consideration 166 168 if (adjustedPosition <= TOP_STORIES_LIMIT) { 167 169 nonJobStoryIds.push(story.id); 168 170 } 169 171 } 170 - 171 - console.log(`Filtered out job stories. Have ${adjustedPosition} stories after filtering.`); 172 - console.log(`Front page contains the top ${Math.min(TOP_STORIES_LIMIT, nonJobStoryIds.length)} non-job stories`); 173 - 172 + 173 + console.log( 174 + `Filtered out job stories. Have ${adjustedPosition} stories after filtering.`, 175 + ); 176 + console.log( 177 + `Front page contains the top ${Math.min(TOP_STORIES_LIMIT, nonJobStoryIds.length)} non-job stories`, 178 + ); 179 + 174 180 // Now use the adjusted positions when processing stories 175 181 const frontPageIds = nonJobStoryIds.slice(0, TOP_STORIES_LIMIT); 176 - 182 + 177 183 // Process each story 178 184 for (let i = 0; i < stories.length; i++) { 179 185 const story = stories[i]; 180 186 if (!story) { 181 - console.log(`[WARNING] Null story at original position ${i + 1}, skipping`); 187 + console.log( 188 + `[WARNING] Null story at original position ${i + 1}, skipping`, 189 + ); 182 190 continue; 183 191 } 184 - 192 + 185 193 // Skip job stories entirely 186 - if (story.type === 'job') { 194 + if (story.type === "job") { 187 195 continue; 188 196 } 189 - 197 + 190 198 // Use the adjusted position for non-job stories 191 - const position = positionMap.get(story.id) || (i + 1); 199 + const position = positionMap.get(story.id) || i + 1; 192 200 const isOnFrontPage = position <= TOP_STORIES_LIMIT; 193 201 const isNumberOne = position === 1; 194 202 ··· 796 804 await processStories(); 797 805 console.log("Story processing completed"); 798 806 807 + // Invalidate all caches after data update 808 + invalidateAllCaches(); 809 + console.log("All query caches invalidated"); 810 + 799 811 console.log("Starting cleanup of expired stories..."); 800 812 // Clean up expired stories 801 813 await cleanupExpiredStories(); 802 814 console.log("Cleanup completed"); 815 + 816 + // Invalidate caches again after cleanup 817 + invalidateAllCaches(); 803 818 } catch (error) { 804 819 console.error("Error in checkHackerNews:", error); 805 820 Sentry.captureException(error); ··· 830 845 831 846 // Run immediately on startup 832 847 checkHackerNews(); 848 + 849 + // Initialize query cache 850 + console.log("Query cache initialized"); 833 851 834 852 console.log("HackerNews monitoring service started"); 835 853 }
+169 -78
src/index.ts
··· 7 7 import { count } from "drizzle-orm"; 8 8 import { stories } from "./libs/schema"; 9 9 10 + // Cache system for database queries 11 + type CacheItem<T> = { 12 + data: T; 13 + timestamp: number; 14 + expiresAt: number; 15 + }; 16 + 17 + class QueryCache { 18 + private cache: Map<string, CacheItem<unknown>> = new Map(); 19 + private defaultTTL: number = 60 * 5; // 5 minutes in seconds 20 + 21 + constructor(defaultTTL?: number) { 22 + if (defaultTTL) { 23 + this.defaultTTL = defaultTTL; 24 + } 25 + console.log(`Initialized query cache with ${this.defaultTTL}s TTL`); 26 + } 27 + 28 + async get<T>( 29 + key: string, 30 + queryFn: () => Promise<T>, 31 + ttl: number = this.defaultTTL 32 + ): Promise<T> { 33 + const now = Math.floor(Date.now() / 1000); 34 + const cached = this.cache.get(key); 35 + 36 + // Return cached value if it exists and is not expired 37 + if (cached && cached.expiresAt > now) { 38 + console.log(`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`); 39 + return cached.data as T; 40 + } 41 + 42 + // Execute the query 43 + console.log(`Cache miss for ${key}, fetching from database...`); 44 + const data = await queryFn(); 45 + 46 + // Cache the result 47 + this.cache.set(key, { 48 + data, 49 + timestamp: now, 50 + expiresAt: now + ttl, 51 + }); 52 + 53 + return data; 54 + } 55 + 56 + invalidate(key: string): void { 57 + if (this.cache.has(key)) { 58 + console.log(`Invalidating cache for ${key}`); 59 + this.cache.delete(key); 60 + } 61 + } 62 + 63 + invalidateAll(): void { 64 + console.log("Invalidating entire cache"); 65 + this.cache.clear(); 66 + } 67 + } 68 + 69 + // Create a global cache instance 70 + const queryCache = new QueryCache(); 71 + 10 72 const environment = process.env.NODE_ENV; 11 73 const commit = (() => { 12 74 try { ··· 69 131 "/api/stories": async () => { 70 132 try { 71 133 // Get stories that reached the front page (leaderboard) 72 - const storyAlerts = await db.query.stories.findMany({ 73 - where: (stories, { eq }) => eq(stories.isOnLeaderboard, true), 74 - orderBy: (stories, { asc }) => [asc(stories.position)], 75 - limit: 100, 76 - }); 77 - 78 - // Transform story data to match the format expected by the frontend 79 - const alerts = storyAlerts.map((story) => ({ 80 - id: story.id, 81 - title: story.title, 82 - url: story.url || `https://news.ycombinator.com/item?id=${story.id}`, 83 - rank: story.position, 84 - peakRank: story.peakPosition, 85 - points: story.score, 86 - peakPoints: story.peakScore, 87 - comments: story.descendants, 88 - timestamp: story.enteredLeaderboardAt 89 - ? new Date(story.enteredLeaderboardAt * 1000).toISOString() 90 - : new Date(story.firstSeenAt * 1000).toISOString(), 91 - by: story.by, 92 - isFromMonitoredUser: story.isFromMonitoredUser, 93 - })); 134 + const alerts = await queryCache.get( 135 + 'leaderboard_stories', 136 + async () => { 137 + const storyAlerts = await db.query.stories.findMany({ 138 + where: (stories, { eq }) => eq(stories.isOnLeaderboard, true), 139 + orderBy: (stories, { asc }) => [asc(stories.position)], 140 + limit: 100, 141 + }); 142 + 143 + // Transform story data to match the format expected by the frontend 144 + return storyAlerts.map((story) => ({ 145 + id: story.id, 146 + title: story.title, 147 + url: story.url || `https://news.ycombinator.com/item?id=${story.id}`, 148 + rank: story.position, 149 + peakRank: story.peakPosition, 150 + points: story.score, 151 + peakPoints: story.peakScore, 152 + comments: story.descendants, 153 + timestamp: story.enteredLeaderboardAt 154 + ? new Date(story.enteredLeaderboardAt * 1000).toISOString() 155 + : new Date(story.firstSeenAt * 1000).toISOString(), 156 + by: story.by, 157 + isFromMonitoredUser: story.isFromMonitoredUser, 158 + })); 159 + } 160 + ); 94 161 95 162 return new Response(JSON.stringify(alerts), { 96 163 headers: { 97 164 "Content-Type": "application/json", 98 - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate", 99 - "Pragma": "no-cache", 100 - "Expires": "0" 165 + "Cache-Control": "max-age=300", 166 + "ETag": `"${version}-stories-${Date.now()}"`, 101 167 }, 102 168 }); 103 169 } catch (error) { ··· 113 179 }, 114 180 "/api/stats/total-stories": async () => { 115 181 try { 116 - // Count all stories in the database 117 - const result = await db.select({ count: count() }).from(stories); 118 - const totalCount = Number(result[0]?.count); 182 + // Count all stories in the database using the cache 183 + const totalCount = await queryCache.get( 184 + 'total_stories_count', 185 + async () => { 186 + const result = await db.select({ count: count() }).from(stories); 187 + return Number(result[0]?.count); 188 + } 189 + ); 119 190 120 191 return new Response( 121 192 JSON.stringify({ ··· 125 196 { 126 197 headers: { 127 198 "Content-Type": "application/json", 128 - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate", 129 - "Pragma": "no-cache", 130 - "Expires": "0" 199 + "Cache-Control": "max-age=300", 200 + "ETag": `"${version}-stats-${Date.now()}"`, 131 201 }, 132 202 }, 133 203 ); ··· 144 214 }, 145 215 "/api/stats/verified-users": async () => { 146 216 try { 147 - // Get stats for verified user stories 148 - const verifiedStories = await db.query.stories.findMany({ 149 - where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true), 150 - }); 151 - // Get count of verified users in the system 152 - const verifiedUsersCount = await db.query.users 153 - .findMany({ 154 - where: (users, { eq }) => eq(users.verified, true), 155 - }) 156 - .then((users) => users.length); 157 - 158 - // Count stories on front page (rank <= 30) 159 - const frontPageCount = verifiedStories.filter( 160 - (s) => s.isOnLeaderboard, 161 - ).length; 162 - 163 - // Calculate average peak points for verified users 164 - let totalPeakPoints = 0; 165 - for (const s of verifiedStories) { 166 - if (s.peakScore) totalPeakPoints += s.peakScore; 167 - } 168 - const avgPeakPoints = verifiedStories.length 169 - ? Math.round(totalPeakPoints / verifiedStories.length) 170 - : 0; 217 + // Get stats for verified user stories using the cache 218 + const verifiedStats = await queryCache.get( 219 + 'verified_users_stats', 220 + async () => { 221 + // Get stats for verified user stories 222 + const verifiedStories = await db.query.stories.findMany({ 223 + where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true), 224 + }); 225 + // Get count of verified users in the system 226 + const verifiedUsersCount = await db.query.users 227 + .findMany({ 228 + where: (users, { eq }) => eq(users.verified, true), 229 + }) 230 + .then((users) => users.length); 231 + 232 + // Count stories on front page (rank <= 30) 233 + const frontPageCount = verifiedStories.filter( 234 + (s) => s.isOnLeaderboard, 235 + ).length; 236 + 237 + // Calculate average peak points for verified users 238 + let totalPeakPoints = 0; 239 + for (const s of verifiedStories) { 240 + if (s.peakScore) totalPeakPoints += s.peakScore; 241 + } 242 + const avgPeakPoints = verifiedStories.length 243 + ? Math.round(totalPeakPoints / verifiedStories.length) 244 + : 0; 245 + 246 + return { 247 + totalCount: verifiedUsersCount, 248 + frontPageCount: frontPageCount, 249 + avgPeakPoints: avgPeakPoints, 250 + }; 251 + } 252 + ); 171 253 172 254 return new Response( 173 255 JSON.stringify({ 174 - totalCount: verifiedUsersCount, 175 - frontPageCount: frontPageCount, 176 - avgPeakPoints: avgPeakPoints, 256 + ...verifiedStats, 177 257 timestamp: Math.floor(Date.now() / 1000), 178 258 }), 179 259 { 180 260 headers: { 181 261 "Content-Type": "application/json", 182 - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate", 183 - "Pragma": "no-cache", 184 - "Expires": "0" 262 + "Cache-Control": "max-age=300", 263 + "ETag": `"${version}-verified-${Date.now()}"`, 185 264 }, 186 265 }, 187 266 ); ··· 206 285 }); 207 286 } 208 287 209 - // Get snapshots for the story 210 - const snapshots = await db.query.leaderboardSnapshots.findMany({ 211 - where: (snapshots, { eq }) => eq(snapshots.storyId, storyId), 212 - orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)], 213 - }); 214 - 215 - // Transform snapshot data for frontend 216 - const graphData = snapshots.map((snapshot) => ({ 217 - timestamp: snapshot.timestamp, 218 - position: snapshot.position, 219 - score: snapshot.score, 220 - date: new Date(snapshot.timestamp * 1000).toISOString(), 221 - })); 288 + // Get snapshots for the story using the cache 289 + const graphData = await queryCache.get( 290 + `story_snapshots_${storyId}`, 291 + async () => { 292 + // Get snapshots for the story 293 + const snapshots = await db.query.leaderboardSnapshots.findMany({ 294 + where: (snapshots, { eq }) => eq(snapshots.storyId, storyId), 295 + orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)], 296 + }); 297 + 298 + // Transform snapshot data for frontend 299 + return snapshots.map((snapshot) => ({ 300 + timestamp: snapshot.timestamp, 301 + position: snapshot.position, 302 + score: snapshot.score, 303 + date: new Date(snapshot.timestamp * 1000).toISOString(), 304 + })); 305 + }, 306 + 3600 // Cache story snapshots for 1 hour as they change less frequently 307 + ); 222 308 223 309 return new Response(JSON.stringify(graphData), { 224 310 headers: { 225 311 "Content-Type": "application/json", 226 - "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate", 227 - "Pragma": "no-cache", 228 - "Expires": "0" 312 + "Cache-Control": "max-age=3600", 313 + "ETag": `"${version}-snapshot-${storyId}-${Date.now()}"`, 229 314 }, 230 315 }); 231 316 } catch (error) { ··· 261 346 } milliseconds on version: ${version}@${commit}!\n\n----------------------------------\n`, 262 347 ); 263 348 264 - export { slackApp, slackClient, version, name, environment, db }; 349 + // Function to invalidate all caches - call this when data is updated 350 + function invalidateAllCaches() { 351 + console.log("Invalidating all query caches"); 352 + queryCache.invalidateAll(); 353 + } 354 + 355 + export { slackApp, slackClient, version, name, environment, db, queryCache, invalidateAllCaches };