···9898 let allStories = []; // Store all stories for filtering
9999 let showVerifiedOnly = false; // Default to showing all stories
100100101101+ // Cache for ETags and data to support conditional requests
102102+ const etagCache = {
103103+ stories: null,
104104+ totalStories: null,
105105+ verifiedUsers: null
106106+ };
107107+108108+ // Cache for actual response data
109109+ const responseCache = {
110110+ stories: null,
111111+ totalStories: null,
112112+ verifiedUsers: null
113113+ };
114114+101115 // Fetch stories data
102116 function fetchStories() {
103117 // Update last refresh time
···111125 }
112126113127 // Fetch total stories count first
114114- fetch(`/api/stats/total-stories?_=${Date.now()}`)
115115- .then((response) => response.json())
128128+ const totalStoriesOptions = {
129129+ headers: {}
130130+ };
131131+132132+ // Add If-None-Match header if we have an ETag
133133+ if (etagCache.totalStories) {
134134+ totalStoriesOptions.headers['If-None-Match'] = etagCache.totalStories;
135135+ }
136136+137137+ fetch('/api/stats/total-stories', totalStoriesOptions)
138138+ .then((response) => {
139139+ // Store the new ETag if available
140140+ const etag = response.headers.get('ETag');
141141+ if (etag) etagCache.totalStories = etag;
142142+143143+ // If 304 Not Modified, use cached data
144144+ if (response.status === 304) {
145145+ console.log('Total stories not modified, using cached data');
146146+ return responseCache.totalStories; // Use cached data
147147+ }
148148+149149+ return response.json();
150150+ })
116151 .then((data) => {
117117- if (data && typeof data.count !== "undefined") {
118118- totalStoriesCount = data.count;
119119- currentFrontpageCountEl.textContent = totalStoriesCount;
152152+ if (data) {
153153+ // Store in cache for future 304 responses
154154+ responseCache.totalStories = data;
155155+156156+ if (typeof data.count !== "undefined") {
157157+ totalStoriesCount = data.count;
158158+ currentFrontpageCountEl.textContent = totalStoriesCount;
159159+ }
120160 }
121161 })
122162 .catch((error) => {
···124164 });
125165126166 // Fetch verified user stats for the top row
127127- fetch(`/api/stats/verified-users?_=${Date.now()}`)
128128- .then((response) => response.json())
167167+ const verifiedUsersOptions = {
168168+ headers: {}
169169+ };
170170+171171+ // Add If-None-Match header if we have an ETag
172172+ if (etagCache.verifiedUsers) {
173173+ verifiedUsersOptions.headers['If-None-Match'] = etagCache.verifiedUsers;
174174+ }
175175+176176+ fetch('/api/stats/verified-users', verifiedUsersOptions)
177177+ .then((response) => {
178178+ // Store the new ETag if available
179179+ const etag = response.headers.get('ETag');
180180+ if (etag) etagCache.verifiedUsers = etag;
181181+182182+ // If 304 Not Modified, use cached data
183183+ if (response.status === 304) {
184184+ console.log('Verified users stats not modified, using cached data');
185185+ return responseCache.verifiedUsers; // Use cached data
186186+ }
187187+188188+ return response.json();
189189+ })
129190 .then((data) => {
130130- verifiedUserStats = data;
131131- // Update top row stats with the new data
132132- updateTopStats(data);
191191+ if (data) {
192192+ // Store in cache for future 304 responses
193193+ responseCache.verifiedUsers = data;
194194+195195+ verifiedUserStats = data;
196196+ updateTopStats(data); // Update UI with the new stats
197197+ }
133198 })
134199 .catch((error) => {
135200 console.error("Error fetching verified user stats:", error);
136201 });
137202138138- fetch(`/api/stories?_=${Date.now()}`)
203203+ // Fetch stories
204204+ const storiesOptions = {
205205+ headers: {}
206206+ };
207207+208208+ // Add If-None-Match header if we have an ETag
209209+ if (etagCache.stories) {
210210+ storiesOptions.headers['If-None-Match'] = etagCache.stories;
211211+ }
212212+213213+ fetch('/api/stories', storiesOptions)
139214 .then((response) => {
215215+ // Store the new ETag if available
216216+ const etag = response.headers.get('ETag');
217217+ if (etag) etagCache.stories = etag;
218218+140219 if (!response.ok) {
220220+ // Allow 304 Not Modified
221221+ if (response.status === 304) {
222222+ console.log('Stories not modified, using cached data');
223223+ return responseCache.stories; // Use cached data
224224+ }
141225 throw new Error("Network response was not ok");
142226 }
143227 return response.json();
144228 })
145229 .then((data) => {
146146- // Store all stories for filtering
147147- allStories = data;
148148-149149- // Apply filters and update UI
150150- applyFiltersAndUpdateUI();
230230+ if (data) {
231231+ // Store in cache for future 304 responses
232232+ responseCache.stories = data;
233233+234234+ // Store all stories for filtering
235235+ allStories = data;
236236+ // Apply filters and update UI
237237+ applyFiltersAndUpdateUI();
238238+ }
151239 })
152240 .catch((error) => {
153241 storyList.innerHTML = `<div class="loading">Error loading data: ${error.message}</div>`;
···287375 }
288376289377 // Load story snapshots and display graph
378378+ // Cache for story snapshot ETags and data
379379+ const snapshotEtagCache = {};
380380+ const snapshotDataCache = {};
381381+290382 function loadStoryGraph(storyId) {
291383 if (activeStoryId === storyId) return;
292384 activeStoryId = storyId;
···295387 rankChart.style.display = "none";
296388 noGraph.innerHTML = '<div class="loading">Loading graph data...</div>';
297389298298- fetch(`/api/story/${storyId}/snapshots?_=${Date.now()}`)
390390+ const options = {
391391+ headers: {}
392392+ };
393393+394394+ // Add If-None-Match header if we have an ETag for this story
395395+ if (snapshotEtagCache[storyId]) {
396396+ options.headers['If-None-Match'] = snapshotEtagCache[storyId];
397397+ }
398398+399399+ // Use template literal correctly since we need string interpolation
400400+ fetch(`/api/story/${storyId}/snapshots`, options)
299401 .then((response) => {
402402+ // Store the new ETag if available
403403+ const etag = response.headers.get('ETag');
404404+ if (etag) snapshotEtagCache[storyId] = etag;
405405+300406 if (!response.ok) {
407407+ // Allow 304 Not Modified
408408+ if (response.status === 304) {
409409+ console.log(`Story ${storyId} snapshots not modified, using cached data`);
410410+ // Use the cached data for this story ID
411411+ if (snapshotDataCache[storyId]) {
412412+ return snapshotDataCache[storyId];
413413+ }
414414+ // If we don't have cached data, re-fetch
415415+ throw new Error("Cache miss on 304, re-fetching");
416416+ }
301417 throw new Error("Failed to fetch snapshot data");
302418 }
303419 return response.json();
···309425 return;
310426 }
311427428428+ // Cache the snapshots data for future use
429429+ snapshotDataCache[storyId] = snapshots;
430430+312431 displayGraph(snapshots);
313432 })
314433 .catch((error) => {
···430549 now = Date.now();
431550432551 // Get total stories from the API
433433- fetch(`/api/stats/total-stories?_=${Date.now()}`)
434434- .then((response) => response.json())
552552+ const options = {
553553+ headers: {}
554554+ };
555555+556556+ // Add If-None-Match header if we have an ETag
557557+ if (etagCache.totalStories) {
558558+ options.headers['If-None-Match'] = etagCache.totalStories;
559559+ }
560560+561561+ fetch('/api/stats/total-stories', options)
562562+ .then((response) => {
563563+ // Store the new ETag if available
564564+ const etag = response.headers.get('ETag');
565565+ if (etag) etagCache.totalStories = etag;
566566+567567+ // If 304 Not Modified, use cached data
568568+ if (response.status === 304) {
569569+ console.log('Total stories not modified, using cached data for metrics');
570570+ return responseCache.totalStories;
571571+ }
572572+573573+ return response.json();
574574+ })
435575 .then((data) => {
436436- currentFrontpageCountEl.textContent =
437437- data.count !== undefined ? data.count : allStories.length;
576576+ if (data) {
577577+ // Update cache
578578+ responseCache.totalStories = data;
579579+580580+ if (data.count !== undefined) {
581581+ currentFrontpageCountEl.textContent = data.count;
582582+ } else {
583583+ currentFrontpageCountEl.textContent = allStories.length; // Fallback to local data
584584+ }
585585+ } else {
586586+ currentFrontpageCountEl.textContent = allStories.length; // Fallback to local data
587587+ }
438588 })
439589 .catch((error) => {
440590 console.error("Error fetching total stories count:", error);
+35-17
src/features/services/check_hn.ts
···11import { CronJob } from "cron";
22import * as Sentry from "@sentry/bun";
33-import { db, environment } from "../../index";
33+import { db, environment, invalidateAllCaches } from "../../index";
44import {
55 users as usersTable,
66 stories as storiesTable,
···147147 // First, filter out job stories and create a map of adjusted positions
148148 const positionMap = new Map<number, number>(); // Maps story ID to adjusted position
149149 let adjustedPosition = 0;
150150-150150+151151 for (let i = 0; i < stories.length; i++) {
152152 const story = stories[i];
153153 if (!story) continue;
154154-154154+155155 // Skip jobs, but include other types (mainly 'story')
156156- if (story.type === 'job') {
157157- console.log(`[INFO] Skipping job at original position ${i + 1}, ID: ${story.id}`);
156156+ if (story.type === "job") {
157157+ console.log(
158158+ `[INFO] Skipping job at original position ${i + 1}, ID: ${story.id}`,
159159+ );
158160 continue;
159161 }
160160-162162+161163 // Only increment position for non-job stories
162164 adjustedPosition++;
163165 positionMap.set(story.id, adjustedPosition);
164164-166166+165167 // Add to non-job story IDs for front page consideration
166168 if (adjustedPosition <= TOP_STORIES_LIMIT) {
167169 nonJobStoryIds.push(story.id);
168170 }
169171 }
170170-171171- console.log(`Filtered out job stories. Have ${adjustedPosition} stories after filtering.`);
172172- console.log(`Front page contains the top ${Math.min(TOP_STORIES_LIMIT, nonJobStoryIds.length)} non-job stories`);
173173-172172+173173+ console.log(
174174+ `Filtered out job stories. Have ${adjustedPosition} stories after filtering.`,
175175+ );
176176+ console.log(
177177+ `Front page contains the top ${Math.min(TOP_STORIES_LIMIT, nonJobStoryIds.length)} non-job stories`,
178178+ );
179179+174180 // Now use the adjusted positions when processing stories
175181 const frontPageIds = nonJobStoryIds.slice(0, TOP_STORIES_LIMIT);
176176-182182+177183 // Process each story
178184 for (let i = 0; i < stories.length; i++) {
179185 const story = stories[i];
180186 if (!story) {
181181- console.log(`[WARNING] Null story at original position ${i + 1}, skipping`);
187187+ console.log(
188188+ `[WARNING] Null story at original position ${i + 1}, skipping`,
189189+ );
182190 continue;
183191 }
184184-192192+185193 // Skip job stories entirely
186186- if (story.type === 'job') {
194194+ if (story.type === "job") {
187195 continue;
188196 }
189189-197197+190198 // Use the adjusted position for non-job stories
191191- const position = positionMap.get(story.id) || (i + 1);
199199+ const position = positionMap.get(story.id) || i + 1;
192200 const isOnFrontPage = position <= TOP_STORIES_LIMIT;
193201 const isNumberOne = position === 1;
194202···796804 await processStories();
797805 console.log("Story processing completed");
798806807807+ // Invalidate all caches after data update
808808+ invalidateAllCaches();
809809+ console.log("All query caches invalidated");
810810+799811 console.log("Starting cleanup of expired stories...");
800812 // Clean up expired stories
801813 await cleanupExpiredStories();
802814 console.log("Cleanup completed");
815815+816816+ // Invalidate caches again after cleanup
817817+ invalidateAllCaches();
803818 } catch (error) {
804819 console.error("Error in checkHackerNews:", error);
805820 Sentry.captureException(error);
···830845831846 // Run immediately on startup
832847 checkHackerNews();
848848+849849+ // Initialize query cache
850850+ console.log("Query cache initialized");
833851834852 console.log("HackerNews monitoring service started");
835853}
+169-78
src/index.ts
···77import { count } from "drizzle-orm";
88import { stories } from "./libs/schema";
991010+// Cache system for database queries
1111+type CacheItem<T> = {
1212+ data: T;
1313+ timestamp: number;
1414+ expiresAt: number;
1515+};
1616+1717+class QueryCache {
1818+ private cache: Map<string, CacheItem<unknown>> = new Map();
1919+ private defaultTTL: number = 60 * 5; // 5 minutes in seconds
2020+2121+ constructor(defaultTTL?: number) {
2222+ if (defaultTTL) {
2323+ this.defaultTTL = defaultTTL;
2424+ }
2525+ console.log(`Initialized query cache with ${this.defaultTTL}s TTL`);
2626+ }
2727+2828+ async get<T>(
2929+ key: string,
3030+ queryFn: () => Promise<T>,
3131+ ttl: number = this.defaultTTL
3232+ ): Promise<T> {
3333+ const now = Math.floor(Date.now() / 1000);
3434+ const cached = this.cache.get(key);
3535+3636+ // Return cached value if it exists and is not expired
3737+ if (cached && cached.expiresAt > now) {
3838+ console.log(`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`);
3939+ return cached.data as T;
4040+ }
4141+4242+ // Execute the query
4343+ console.log(`Cache miss for ${key}, fetching from database...`);
4444+ const data = await queryFn();
4545+4646+ // Cache the result
4747+ this.cache.set(key, {
4848+ data,
4949+ timestamp: now,
5050+ expiresAt: now + ttl,
5151+ });
5252+5353+ return data;
5454+ }
5555+5656+ invalidate(key: string): void {
5757+ if (this.cache.has(key)) {
5858+ console.log(`Invalidating cache for ${key}`);
5959+ this.cache.delete(key);
6060+ }
6161+ }
6262+6363+ invalidateAll(): void {
6464+ console.log("Invalidating entire cache");
6565+ this.cache.clear();
6666+ }
6767+}
6868+6969+// Create a global cache instance
7070+const queryCache = new QueryCache();
7171+1072const environment = process.env.NODE_ENV;
1173const commit = (() => {
1274 try {
···69131 "/api/stories": async () => {
70132 try {
71133 // Get stories that reached the front page (leaderboard)
7272- const storyAlerts = await db.query.stories.findMany({
7373- where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
7474- orderBy: (stories, { asc }) => [asc(stories.position)],
7575- limit: 100,
7676- });
7777-7878- // Transform story data to match the format expected by the frontend
7979- const alerts = storyAlerts.map((story) => ({
8080- id: story.id,
8181- title: story.title,
8282- url: story.url || `https://news.ycombinator.com/item?id=${story.id}`,
8383- rank: story.position,
8484- peakRank: story.peakPosition,
8585- points: story.score,
8686- peakPoints: story.peakScore,
8787- comments: story.descendants,
8888- timestamp: story.enteredLeaderboardAt
8989- ? new Date(story.enteredLeaderboardAt * 1000).toISOString()
9090- : new Date(story.firstSeenAt * 1000).toISOString(),
9191- by: story.by,
9292- isFromMonitoredUser: story.isFromMonitoredUser,
9393- }));
134134+ const alerts = await queryCache.get(
135135+ 'leaderboard_stories',
136136+ async () => {
137137+ const storyAlerts = await db.query.stories.findMany({
138138+ where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
139139+ orderBy: (stories, { asc }) => [asc(stories.position)],
140140+ limit: 100,
141141+ });
142142+143143+ // Transform story data to match the format expected by the frontend
144144+ return storyAlerts.map((story) => ({
145145+ id: story.id,
146146+ title: story.title,
147147+ url: story.url || `https://news.ycombinator.com/item?id=${story.id}`,
148148+ rank: story.position,
149149+ peakRank: story.peakPosition,
150150+ points: story.score,
151151+ peakPoints: story.peakScore,
152152+ comments: story.descendants,
153153+ timestamp: story.enteredLeaderboardAt
154154+ ? new Date(story.enteredLeaderboardAt * 1000).toISOString()
155155+ : new Date(story.firstSeenAt * 1000).toISOString(),
156156+ by: story.by,
157157+ isFromMonitoredUser: story.isFromMonitoredUser,
158158+ }));
159159+ }
160160+ );
9416195162 return new Response(JSON.stringify(alerts), {
96163 headers: {
97164 "Content-Type": "application/json",
9898- "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate",
9999- "Pragma": "no-cache",
100100- "Expires": "0"
165165+ "Cache-Control": "max-age=300",
166166+ "ETag": `"${version}-stories-${Date.now()}"`,
101167 },
102168 });
103169 } catch (error) {
···113179 },
114180 "/api/stats/total-stories": async () => {
115181 try {
116116- // Count all stories in the database
117117- const result = await db.select({ count: count() }).from(stories);
118118- const totalCount = Number(result[0]?.count);
182182+ // Count all stories in the database using the cache
183183+ const totalCount = await queryCache.get(
184184+ 'total_stories_count',
185185+ async () => {
186186+ const result = await db.select({ count: count() }).from(stories);
187187+ return Number(result[0]?.count);
188188+ }
189189+ );
119190120191 return new Response(
121192 JSON.stringify({
···125196 {
126197 headers: {
127198 "Content-Type": "application/json",
128128- "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate",
129129- "Pragma": "no-cache",
130130- "Expires": "0"
199199+ "Cache-Control": "max-age=300",
200200+ "ETag": `"${version}-stats-${Date.now()}"`,
131201 },
132202 },
133203 );
···144214 },
145215 "/api/stats/verified-users": async () => {
146216 try {
147147- // Get stats for verified user stories
148148- const verifiedStories = await db.query.stories.findMany({
149149- where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true),
150150- });
151151- // Get count of verified users in the system
152152- const verifiedUsersCount = await db.query.users
153153- .findMany({
154154- where: (users, { eq }) => eq(users.verified, true),
155155- })
156156- .then((users) => users.length);
157157-158158- // Count stories on front page (rank <= 30)
159159- const frontPageCount = verifiedStories.filter(
160160- (s) => s.isOnLeaderboard,
161161- ).length;
162162-163163- // Calculate average peak points for verified users
164164- let totalPeakPoints = 0;
165165- for (const s of verifiedStories) {
166166- if (s.peakScore) totalPeakPoints += s.peakScore;
167167- }
168168- const avgPeakPoints = verifiedStories.length
169169- ? Math.round(totalPeakPoints / verifiedStories.length)
170170- : 0;
217217+ // Get stats for verified user stories using the cache
218218+ const verifiedStats = await queryCache.get(
219219+ 'verified_users_stats',
220220+ async () => {
221221+ // Get stats for verified user stories
222222+ const verifiedStories = await db.query.stories.findMany({
223223+ where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true),
224224+ });
225225+ // Get count of verified users in the system
226226+ const verifiedUsersCount = await db.query.users
227227+ .findMany({
228228+ where: (users, { eq }) => eq(users.verified, true),
229229+ })
230230+ .then((users) => users.length);
231231+232232+ // Count stories on front page (rank <= 30)
233233+ const frontPageCount = verifiedStories.filter(
234234+ (s) => s.isOnLeaderboard,
235235+ ).length;
236236+237237+ // Calculate average peak points for verified users
238238+ let totalPeakPoints = 0;
239239+ for (const s of verifiedStories) {
240240+ if (s.peakScore) totalPeakPoints += s.peakScore;
241241+ }
242242+ const avgPeakPoints = verifiedStories.length
243243+ ? Math.round(totalPeakPoints / verifiedStories.length)
244244+ : 0;
245245+246246+ return {
247247+ totalCount: verifiedUsersCount,
248248+ frontPageCount: frontPageCount,
249249+ avgPeakPoints: avgPeakPoints,
250250+ };
251251+ }
252252+ );
171253172254 return new Response(
173255 JSON.stringify({
174174- totalCount: verifiedUsersCount,
175175- frontPageCount: frontPageCount,
176176- avgPeakPoints: avgPeakPoints,
256256+ ...verifiedStats,
177257 timestamp: Math.floor(Date.now() / 1000),
178258 }),
179259 {
180260 headers: {
181261 "Content-Type": "application/json",
182182- "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate",
183183- "Pragma": "no-cache",
184184- "Expires": "0"
262262+ "Cache-Control": "max-age=300",
263263+ "ETag": `"${version}-verified-${Date.now()}"`,
185264 },
186265 },
187266 );
···206285 });
207286 }
208287209209- // Get snapshots for the story
210210- const snapshots = await db.query.leaderboardSnapshots.findMany({
211211- where: (snapshots, { eq }) => eq(snapshots.storyId, storyId),
212212- orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)],
213213- });
214214-215215- // Transform snapshot data for frontend
216216- const graphData = snapshots.map((snapshot) => ({
217217- timestamp: snapshot.timestamp,
218218- position: snapshot.position,
219219- score: snapshot.score,
220220- date: new Date(snapshot.timestamp * 1000).toISOString(),
221221- }));
288288+ // Get snapshots for the story using the cache
289289+ const graphData = await queryCache.get(
290290+ `story_snapshots_${storyId}`,
291291+ async () => {
292292+ // Get snapshots for the story
293293+ const snapshots = await db.query.leaderboardSnapshots.findMany({
294294+ where: (snapshots, { eq }) => eq(snapshots.storyId, storyId),
295295+ orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)],
296296+ });
297297+298298+ // Transform snapshot data for frontend
299299+ return snapshots.map((snapshot) => ({
300300+ timestamp: snapshot.timestamp,
301301+ position: snapshot.position,
302302+ score: snapshot.score,
303303+ date: new Date(snapshot.timestamp * 1000).toISOString(),
304304+ }));
305305+ },
306306+ 3600 // Cache story snapshots for 1 hour as they change less frequently
307307+ );
222308223309 return new Response(JSON.stringify(graphData), {
224310 headers: {
225311 "Content-Type": "application/json",
226226- "Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate",
227227- "Pragma": "no-cache",
228228- "Expires": "0"
312312+ "Cache-Control": "max-age=3600",
313313+ "ETag": `"${version}-snapshot-${storyId}-${Date.now()}"`,
229314 },
230315 });
231316 } catch (error) {
···261346 } milliseconds on version: ${version}@${commit}!\n\n----------------------------------\n`,
262347);
263348264264-export { slackApp, slackClient, version, name, environment, db };
349349+// Function to invalidate all caches - call this when data is updated
350350+function invalidateAllCaches() {
351351+ console.log("Invalidating all query caches");
352352+ queryCache.invalidateAll();
353353+}
354354+355355+export { slackApp, slackClient, version, name, environment, db, queryCache, invalidateAllCaches };