this repo has no description
feat: improve caching
+593
-291
+8
-5
src/features/services/check_hn.ts
+8
-5
src/features/services/check_hn.ts
···
804
804
await processStories();
805
805
console.log("Story processing completed");
806
806
807
-
// Invalidate all caches after data update
807
+
// Invalidate all caches and reload them after data update
808
808
invalidateAllCaches();
809
-
console.log("All query caches invalidated");
809
+
console.log("All query caches invalidated and refreshed");
810
810
811
811
console.log("Starting cleanup of expired stories...");
812
812
// Clean up expired stories
813
813
await cleanupExpiredStories();
814
814
console.log("Cleanup completed");
815
815
816
-
// Invalidate caches again after cleanup
816
+
// Invalidate caches again after cleanup and reload them
817
817
invalidateAllCaches();
818
818
} catch (error) {
819
819
console.error("Error in checkHackerNews:", error);
···
843
843
});
844
844
}
845
845
846
-
// Run immediately on startup
847
-
checkHackerNews();
846
+
// Run a few seconds after startup to give server time to initialize
847
+
setTimeout(() => {
848
+
console.log("Running initial data check...");
849
+
checkHackerNews();
850
+
}, 3000);
848
851
849
852
// Initialize query cache
850
853
console.log("Query cache initialized");
+91
-225
src/index.ts
+91
-225
src/index.ts
···
3
3
import setup from "./features";
4
4
import { db } from "./libs/db";
5
5
import { version, name } from "../package.json";
6
+
import { preloadCaches, invalidateAndRefreshCaches } from "./libs/cacheWarming";
7
+
import { queryCache, createCacheHeaders, compressResponse, createCachedEndpoint } from "./libs/cache";
6
8
import root from "../public/index.html";
7
9
import { count } from "drizzle-orm";
8
10
import { stories } from "./libs/schema";
9
11
10
-
// Cache system for database queries
11
-
type CacheItem<T> = {
12
-
data: T;
13
-
timestamp: number;
14
-
expiresAt: number;
15
-
};
16
-
17
-
class QueryCache {
18
-
private cache: Map<string, CacheItem<unknown>> = new Map();
19
-
private defaultTTL: number = 60 * 5; // 5 minutes in seconds
20
-
21
-
constructor(defaultTTL?: number) {
22
-
if (defaultTTL) {
23
-
this.defaultTTL = defaultTTL;
24
-
}
25
-
console.log(`Initialized query cache with ${this.defaultTTL}s TTL`);
26
-
}
27
-
28
-
async get<T>(
29
-
key: string,
30
-
queryFn: () => Promise<T>,
31
-
ttl: number = this.defaultTTL
32
-
): Promise<T> {
33
-
const now = Math.floor(Date.now() / 1000);
34
-
const cached = this.cache.get(key);
35
-
36
-
// Return cached value if it exists and is not expired
37
-
if (cached && cached.expiresAt > now) {
38
-
console.log(`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`);
39
-
return cached.data as T;
40
-
}
41
-
42
-
// Execute the query
43
-
console.log(`Cache miss for ${key}, fetching from database...`);
44
-
const data = await queryFn();
45
-
46
-
// Cache the result
47
-
this.cache.set(key, {
48
-
data,
49
-
timestamp: now,
50
-
expiresAt: now + ttl,
51
-
});
52
-
53
-
return data;
54
-
}
55
-
56
-
invalidate(key: string): void {
57
-
if (this.cache.has(key)) {
58
-
console.log(`Invalidating cache for ${key}`);
59
-
this.cache.delete(key);
60
-
}
61
-
}
62
-
63
-
invalidateAll(): void {
64
-
console.log("Invalidating entire cache");
65
-
this.cache.clear();
66
-
}
67
-
}
68
-
69
-
// Create a global cache instance
70
-
const queryCache = new QueryCache();
71
-
72
12
const environment = process.env.NODE_ENV;
73
13
const commit = (() => {
74
14
try {
···
111
51
console.log("📦 Loading Slack App...");
112
52
console.log("🔑 Loading environment variables...");
113
53
54
+
// Initialize Slack app
114
55
const slackApp = new SlackApp({
115
56
env: {
116
57
SLACK_BOT_TOKEN: process.env.SLACK_BOT_TOKEN as string,
···
122
63
const slackClient = slackApp.client;
123
64
124
65
await setup();
66
+
await preloadCaches();
125
67
126
68
const server = Bun.serve({
127
69
port: process.env.PORT || 3000,
128
70
reusePort: true,
129
71
routes: {
130
72
"/": root,
131
-
"/api/stories": async () => {
132
-
try {
133
-
// Get stories that reached the front page (leaderboard)
134
-
const alerts = await queryCache.get(
135
-
'leaderboard_stories',
136
-
async () => {
137
-
const storyAlerts = await db.query.stories.findMany({
138
-
where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
139
-
orderBy: (stories, { asc }) => [asc(stories.position)],
140
-
limit: 100,
141
-
});
142
-
143
-
// Transform story data to match the format expected by the frontend
144
-
return storyAlerts.map((story) => ({
145
-
id: story.id,
146
-
title: story.title,
147
-
url: story.url || `https://news.ycombinator.com/item?id=${story.id}`,
148
-
rank: story.position,
149
-
peakRank: story.peakPosition,
150
-
points: story.score,
151
-
peakPoints: story.peakScore,
152
-
comments: story.descendants,
153
-
timestamp: story.enteredLeaderboardAt
154
-
? new Date(story.enteredLeaderboardAt * 1000).toISOString()
155
-
: new Date(story.firstSeenAt * 1000).toISOString(),
156
-
by: story.by,
157
-
isFromMonitoredUser: story.isFromMonitoredUser,
158
-
}));
159
-
}
160
-
);
73
+
"/api/stories": createCachedEndpoint("leaderboard_stories", async () => {
74
+
const storyAlerts = await db.query.stories.findMany({
75
+
where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
76
+
orderBy: (stories, { asc }) => [asc(stories.position)],
77
+
limit: 100,
78
+
});
161
79
162
-
return new Response(JSON.stringify(alerts), {
163
-
headers: {
164
-
"Content-Type": "application/json",
165
-
"Cache-Control": "max-age=300",
166
-
"ETag": `"${version}-stories-${Date.now()}"`,
167
-
},
168
-
});
169
-
} catch (error) {
170
-
console.error("Failed to fetch alerts:", error);
171
-
return new Response(
172
-
JSON.stringify({ error: "Failed to fetch alerts" }),
173
-
{
174
-
status: 500,
175
-
headers: { "Content-Type": "application/json" },
176
-
},
177
-
);
178
-
}
179
-
},
180
-
"/api/stats/total-stories": async () => {
181
-
try {
182
-
// Count all stories in the database using the cache
183
-
const totalCount = await queryCache.get(
184
-
'total_stories_count',
185
-
async () => {
186
-
const result = await db.select({ count: count() }).from(stories);
187
-
return Number(result[0]?.count);
188
-
}
189
-
);
80
+
// Transform story data to match the format expected by the frontend
81
+
return storyAlerts.map((story) => ({
82
+
id: story.id,
83
+
title: story.title,
84
+
url:
85
+
story.url || `https://news.ycombinator.com/item?id=${story.id}`,
86
+
rank: story.position,
87
+
peakRank: story.peakPosition,
88
+
points: story.score,
89
+
peakPoints: story.peakScore,
90
+
comments: story.descendants,
91
+
timestamp: story.enteredLeaderboardAt
92
+
? new Date(story.enteredLeaderboardAt * 1000).toISOString()
93
+
: new Date(story.firstSeenAt * 1000).toISOString(),
94
+
by: story.by,
95
+
isFromMonitoredUser: story.isFromMonitoredUser,
96
+
}));
97
+
}, 300),
98
+
"/api/stats/total-stories": createCachedEndpoint("total_stories_count", async () => {
99
+
const result = await db.select({ count: count() }).from(stories);
100
+
return {
101
+
count: Number(result[0]?.count),
102
+
timestamp: Math.floor(Date.now() / 1000),
103
+
};
104
+
}, 300),
105
+
"/api/stats/verified-users": createCachedEndpoint("verified_users_stats", async () => {
106
+
// Get stats for verified user stories
107
+
const verifiedStories = await db.query.stories.findMany({
108
+
where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true),
109
+
});
110
+
// Get count of verified users in the system
111
+
const verifiedUsersCount = await db.query.users
112
+
.findMany({
113
+
where: (users, { eq }) => eq(users.verified, true),
114
+
})
115
+
.then((users) => users.length);
190
116
191
-
return new Response(
192
-
JSON.stringify({
193
-
count: totalCount,
194
-
timestamp: Math.floor(Date.now() / 1000),
195
-
}),
196
-
{
197
-
headers: {
198
-
"Content-Type": "application/json",
199
-
"Cache-Control": "max-age=300",
200
-
"ETag": `"${version}-stats-${Date.now()}"`,
201
-
},
202
-
},
203
-
);
204
-
} catch (error) {
205
-
console.error("Failed to count stories:", error);
206
-
return new Response(
207
-
JSON.stringify({ error: "Failed to count stories" }),
208
-
{
209
-
status: 500,
210
-
headers: { "Content-Type": "application/json" },
211
-
},
212
-
);
213
-
}
214
-
},
215
-
"/api/stats/verified-users": async () => {
216
-
try {
217
-
// Get stats for verified user stories using the cache
218
-
const verifiedStats = await queryCache.get(
219
-
'verified_users_stats',
220
-
async () => {
221
-
// Get stats for verified user stories
222
-
const verifiedStories = await db.query.stories.findMany({
223
-
where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true),
224
-
});
225
-
// Get count of verified users in the system
226
-
const verifiedUsersCount = await db.query.users
227
-
.findMany({
228
-
where: (users, { eq }) => eq(users.verified, true),
229
-
})
230
-
.then((users) => users.length);
231
-
232
-
// Count stories on front page (rank <= 30)
233
-
const frontPageCount = verifiedStories.filter(
234
-
(s) => s.isOnLeaderboard,
235
-
).length;
236
-
237
-
// Calculate average peak points for verified users
238
-
let totalPeakPoints = 0;
239
-
for (const s of verifiedStories) {
240
-
if (s.peakScore) totalPeakPoints += s.peakScore;
241
-
}
242
-
const avgPeakPoints = verifiedStories.length
243
-
? Math.round(totalPeakPoints / verifiedStories.length)
244
-
: 0;
245
-
246
-
return {
247
-
totalCount: verifiedUsersCount,
248
-
frontPageCount: frontPageCount,
249
-
avgPeakPoints: avgPeakPoints,
250
-
};
251
-
}
252
-
);
117
+
// Count stories on front page (rank <= 30)
118
+
const frontPageCount = verifiedStories.filter(
119
+
(s) => s.isOnLeaderboard,
120
+
).length;
253
121
254
-
return new Response(
255
-
JSON.stringify({
256
-
...verifiedStats,
257
-
timestamp: Math.floor(Date.now() / 1000),
258
-
}),
259
-
{
260
-
headers: {
261
-
"Content-Type": "application/json",
262
-
"Cache-Control": "max-age=300",
263
-
"ETag": `"${version}-verified-${Date.now()}"`,
264
-
},
265
-
},
266
-
);
267
-
} catch (error) {
268
-
console.error("Failed to get verified user stats:", error);
269
-
return new Response(
270
-
JSON.stringify({ error: "Failed to get verified user stats" }),
271
-
{
272
-
status: 500,
273
-
headers: { "Content-Type": "application/json" },
274
-
},
275
-
);
122
+
// Calculate average peak points for verified users
123
+
let totalPeakPoints = 0;
124
+
for (const s of verifiedStories) {
125
+
if (s.peakScore) totalPeakPoints += s.peakScore;
276
126
}
277
-
},
127
+
const avgPeakPoints = verifiedStories.length
128
+
? Math.round(totalPeakPoints / verifiedStories.length)
129
+
: 0;
130
+
131
+
return {
132
+
totalCount: verifiedUsersCount,
133
+
frontPageCount: frontPageCount,
134
+
avgPeakPoints: avgPeakPoints,
135
+
timestamp: Math.floor(Date.now() / 1000),
136
+
};
137
+
}, 300),
278
138
"/api/story/:id/snapshots": async (req) => {
279
139
try {
280
140
const storyId = Number.parseInt(req.params.id as string);
···
285
145
});
286
146
}
287
147
288
-
// Get snapshots for the story using the cache
289
-
const graphData = await queryCache.get(
148
+
// Create a cached endpoint handler dynamically based on the story ID
149
+
const handler = createCachedEndpoint(
290
150
`story_snapshots_${storyId}`,
291
151
async () => {
292
152
// Get snapshots for the story
···
294
154
where: (snapshots, { eq }) => eq(snapshots.storyId, storyId),
295
155
orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)],
296
156
});
297
-
157
+
298
158
// Transform snapshot data for frontend
299
159
return snapshots.map((snapshot) => ({
300
160
timestamp: snapshot.timestamp,
···
306
166
3600 // Cache story snapshots for 1 hour as they change less frequently
307
167
);
308
168
309
-
return new Response(JSON.stringify(graphData), {
310
-
headers: {
311
-
"Content-Type": "application/json",
312
-
"Cache-Control": "max-age=3600",
313
-
"ETag": `"${version}-snapshot-${storyId}-${Date.now()}"`,
314
-
},
315
-
});
169
+
// Execute the cached handler
170
+
return handler(req);
316
171
} catch (error) {
317
172
console.error("Failed to fetch snapshots for story:", error);
318
173
return new Response(
···
324
179
);
325
180
}
326
181
},
327
-
"/health": () => {
328
-
return new Response(JSON.stringify({ status: "ok" }), {
329
-
headers: {
182
+
"/health": (req) => {
183
+
const response = new Response(JSON.stringify({ status: "ok" }), {
184
+
headers: {
330
185
"Content-Type": "application/json",
331
-
"Cache-Control": "no-store, no-cache, must-revalidate, proxy-revalidate",
332
-
"Pragma": "no-cache",
333
-
"Expires": "0"
186
+
"Cache-Control":
187
+
"no-store, no-cache, must-revalidate, proxy-revalidate",
188
+
Pragma: "no-cache",
189
+
Expires: "0",
334
190
},
335
191
});
192
+
return compressResponse(req, response);
336
193
},
337
194
"/slack": (res: Request) => {
338
195
return slackApp.run(res);
···
346
203
} milliseconds on version: ${version}@${commit}!\n\n----------------------------------\n`,
347
204
);
348
205
349
-
// Function to invalidate all caches - call this when data is updated
206
+
// Function to invalidate all caches and refresh them - call this when data is updated
350
207
function invalidateAllCaches() {
351
-
console.log("Invalidating all query caches");
352
-
queryCache.invalidateAll();
208
+
console.log("Invalidating all query caches and refreshing data");
209
+
invalidateAndRefreshCaches();
353
210
}
354
211
355
-
export { slackApp, slackClient, version, name, environment, db, queryCache, invalidateAllCaches };
212
+
export {
213
+
slackApp,
214
+
slackClient,
215
+
version,
216
+
name,
217
+
environment,
218
+
db,
219
+
queryCache,
220
+
invalidateAllCaches,
221
+
};
+229
src/libs/cache.ts
+229
src/libs/cache.ts
···
1
+
import { version } from "../../package.json";
2
+
3
+
/**
4
+
* Creates consistent cache headers with stable ETags
5
+
* @param key Cache key for the resource
6
+
* @param maxAge Max age in seconds for the Cache-Control header
7
+
* @returns Headers object with proper caching directives
8
+
*/
9
+
export function createCacheHeaders(
10
+
key: string,
11
+
maxAge = 300,
12
+
): Record<string, string> {
13
+
// Generate stable ETag based on version and cache key
14
+
// Only changes when version changes or when cache TTL expires (divided by TTL)
15
+
const etag = `"${version}-${key}-${Math.floor(Date.now() / (maxAge * 1000))}"`;
16
+
17
+
return {
18
+
"Content-Type": "application/json",
19
+
"Cache-Control": `public, max-age=${maxAge - 10}, stale-while-revalidate=30`,
20
+
ETag: etag,
21
+
};
22
+
}
23
+
24
+
/**
25
+
* Applies compression to a Response if the client supports it
26
+
* @param request Original request to check Accept-Encoding
27
+
* @param response Response to potentially compress
28
+
* @returns Compressed response if possible, original otherwise
29
+
*/
30
+
export async function compressResponse(
31
+
request: Request,
32
+
response: Response,
33
+
): Promise<Response> {
34
+
// Only compress JSON responses
35
+
const contentType = response.headers.get("Content-Type");
36
+
if (!contentType?.includes("application/json")) {
37
+
return response;
38
+
}
39
+
40
+
// Check if client accepts compression
41
+
const acceptEncoding = request.headers.get("Accept-Encoding") || "";
42
+
if (acceptEncoding.includes("gzip")) {
43
+
// Clone the response
44
+
const body = await response.text();
45
+
46
+
// Create compressed body with Bun's built-in gzip compression
47
+
const compressedBody = Bun.gzipSync(Buffer.from(body));
48
+
49
+
// Create new response with compressed body and updated headers
50
+
return new Response(compressedBody, {
51
+
status: response.status,
52
+
headers: {
53
+
...Object.fromEntries(response.headers.entries()),
54
+
"Content-Encoding": "gzip",
55
+
"Content-Length": compressedBody.length.toString(),
56
+
},
57
+
});
58
+
}
59
+
// Bun.deflateSync uses zlib format. If we wanted to support 'deflate':
60
+
if (acceptEncoding.includes("deflate")) {
61
+
const body = await response.text();
62
+
const compressedBody = Bun.deflateSync(Buffer.from(body));
63
+
return new Response(compressedBody, {
64
+
status: response.status,
65
+
headers: {
66
+
...Object.fromEntries(response.headers.entries()),
67
+
"Content-Encoding": "deflate",
68
+
"Content-Length": compressedBody.length.toString(),
69
+
},
70
+
});
71
+
}
72
+
73
+
// Return original response if compression not supported/needed
74
+
return response;
75
+
}
76
+
77
+
// Cache system for database queries
78
+
export type CacheItem<T> = {
79
+
data: T;
80
+
timestamp: number;
81
+
expiresAt: number;
82
+
};
83
+
84
+
export class QueryCache {
85
+
private cache: Map<string, CacheItem<unknown>> = new Map();
86
+
private defaultTTL: number = 60 * 5; // 5 minutes in seconds
87
+
private prefetchQueue: Set<string> = new Set();
88
+
89
+
constructor(defaultTTL?: number) {
90
+
if (defaultTTL) {
91
+
this.defaultTTL = defaultTTL;
92
+
}
93
+
console.log(`Initialized query cache with ${this.defaultTTL}s TTL`);
94
+
}
95
+
96
+
async get<T>(
97
+
key: string,
98
+
queryFn: () => Promise<T>,
99
+
ttl: number = this.defaultTTL,
100
+
): Promise<T> {
101
+
const now = Math.floor(Date.now() / 1000);
102
+
const cached = this.cache.get(key);
103
+
104
+
// Return cached value if it exists and is not expired
105
+
if (cached && cached.expiresAt > now) {
106
+
console.log(
107
+
`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`,
108
+
);
109
+
110
+
// Prefetch if approaching expiration (last 10% of TTL)
111
+
if (cached.expiresAt - now < ttl * 0.1 && !this.prefetchQueue.has(key)) {
112
+
this.prefetch(key, queryFn, ttl);
113
+
}
114
+
115
+
return cached.data as T;
116
+
}
117
+
118
+
// Execute the query
119
+
console.log(`Cache miss for ${key}, fetching from database...`);
120
+
const data = await queryFn();
121
+
122
+
// Cache the result
123
+
this.cache.set(key, {
124
+
data,
125
+
timestamp: now,
126
+
expiresAt: now + ttl,
127
+
});
128
+
129
+
return data;
130
+
}
131
+
132
+
// Background prefetch to refresh cache before expiration
133
+
private prefetch<T>(
134
+
key: string,
135
+
queryFn: () => Promise<T>,
136
+
ttl: number,
137
+
): void {
138
+
this.prefetchQueue.add(key);
139
+
140
+
// Use setTimeout to run this outside the current request
141
+
setTimeout(async () => {
142
+
try {
143
+
console.log(`Prefetching ${key} before expiration`);
144
+
const data = await queryFn();
145
+
const now = Math.floor(Date.now() / 1000);
146
+
147
+
this.cache.set(key, {
148
+
data,
149
+
timestamp: now,
150
+
expiresAt: now + ttl,
151
+
});
152
+
153
+
console.log(`Successfully prefetched ${key}`);
154
+
} catch (error) {
155
+
console.error(`Error prefetching ${key}:`, error);
156
+
} finally {
157
+
this.prefetchQueue.delete(key);
158
+
}
159
+
}, 0);
160
+
}
161
+
162
+
invalidate(key: string): void {
163
+
if (this.cache.has(key)) {
164
+
console.log(`Invalidating cache for ${key}`);
165
+
this.cache.delete(key);
166
+
}
167
+
}
168
+
169
+
invalidateAll(): void {
170
+
console.log("Invalidating entire cache");
171
+
this.cache.clear();
172
+
}
173
+
174
+
// Get cache stats for monitoring
175
+
getStats(): { size: number; keys: string[] } {
176
+
return {
177
+
size: this.cache.size,
178
+
keys: Array.from(this.cache.keys()),
179
+
};
180
+
}
181
+
}
182
+
183
+
/**
184
+
* Factory function for creating consistent API endpoint handlers
185
+
* @param cacheKey Key for caching the response
186
+
* @param queryFn Function that performs the actual database query
187
+
* @param ttl Cache TTL in seconds
188
+
*/
189
+
export function createCachedEndpoint<T>(
190
+
cacheKey: string,
191
+
queryFn: () => Promise<T>,
192
+
ttl = 300,
193
+
) {
194
+
return async (request: Request) => {
195
+
try {
196
+
// Get data from cache or execute query
197
+
const data = await queryCache.get(cacheKey, queryFn, ttl);
198
+
199
+
// Create response with proper caching headers
200
+
const response = new Response(JSON.stringify(data), {
201
+
headers: createCacheHeaders(cacheKey, ttl),
202
+
});
203
+
204
+
// Apply compression and return
205
+
return compressResponse(request, response);
206
+
} catch (error) {
207
+
// Log the error with context
208
+
console.error(`Error in endpoint ${cacheKey}:`, error);
209
+
210
+
// Return consistent error response
211
+
return new Response(
212
+
JSON.stringify({
213
+
error: "An error occurred processing your request",
214
+
code: "INTERNAL_SERVER_ERROR",
215
+
}),
216
+
{
217
+
status: 500,
218
+
headers: { "Content-Type": "application/json" },
219
+
},
220
+
);
221
+
}
222
+
};
223
+
}
224
+
225
+
// Create a global cache instance
226
+
export const queryCache = new QueryCache();
227
+
228
+
// Import Sentry for error reporting
229
+
import * as Sentry from "@sentry/bun";
+137
src/libs/cacheWarming.ts
+137
src/libs/cacheWarming.ts
···
1
+
import * as Sentry from "@sentry/bun";
2
+
import { db } from "./db";
3
+
import { count } from "drizzle-orm";
4
+
import { stories, users } from "./schema";
5
+
import { queryCache } from "./cache";
6
+
7
+
/**
8
+
* Proactively warms the cache by loading commonly accessed data
9
+
* Call this after cron jobs update the database or at server startup
10
+
*/
11
+
export async function preloadCaches(): Promise<void> {
12
+
console.log("Preloading all caches for optimal performance...");
13
+
14
+
try {
15
+
// Load critical caches sequentially to avoid database contention
16
+
17
+
// 1. Leaderboard stories (most frequently accessed)
18
+
console.log("Preloading leaderboard stories cache...");
19
+
await queryCache.get('leaderboard_stories', async () => {
20
+
const storyAlerts = await db.query.stories.findMany({
21
+
where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
22
+
orderBy: (stories, { asc }) => [asc(stories.position)],
23
+
limit: 100,
24
+
});
25
+
26
+
// Transform for frontend
27
+
return storyAlerts.map((story) => ({
28
+
id: story.id,
29
+
title: story.title,
30
+
url: story.url || `https://news.ycombinator.com/item?id=${story.id}`,
31
+
rank: story.position,
32
+
peakRank: story.peakPosition,
33
+
points: story.score,
34
+
peakPoints: story.peakScore,
35
+
comments: story.descendants,
36
+
timestamp: story.enteredLeaderboardAt
37
+
? new Date(story.enteredLeaderboardAt * 1000).toISOString()
38
+
: new Date(story.firstSeenAt * 1000).toISOString(),
39
+
by: story.by,
40
+
isFromMonitoredUser: story.isFromMonitoredUser,
41
+
}));
42
+
});
43
+
44
+
// 2. Total stories count
45
+
console.log("Preloading story count cache...");
46
+
await queryCache.get('total_stories_count', async () => {
47
+
const result = await db.select({ count: count() }).from(stories);
48
+
return Number(result[0]?.count);
49
+
});
50
+
51
+
// 3. Verified users stats
52
+
console.log("Preloading verified users stats cache...");
53
+
await queryCache.get('verified_users_stats', async () => {
54
+
// Get stats for verified user stories
55
+
const verifiedStories = await db.query.stories.findMany({
56
+
where: (stories, { eq }) => eq(stories.isFromMonitoredUser, true),
57
+
});
58
+
59
+
// Get count of verified users in the system
60
+
const verifiedUsersCount = await db.query.users
61
+
.findMany({
62
+
where: (users, { eq }) => eq(users.verified, true),
63
+
})
64
+
.then((users) => users.length);
65
+
66
+
// Count stories on front page (rank <= 30)
67
+
const frontPageCount = verifiedStories.filter(
68
+
(s) => s.isOnLeaderboard,
69
+
).length;
70
+
71
+
// Calculate average peak points for verified users
72
+
let totalPeakPoints = 0;
73
+
for (const s of verifiedStories) {
74
+
if (s.peakScore) totalPeakPoints += s.peakScore;
75
+
}
76
+
const avgPeakPoints = verifiedStories.length
77
+
? Math.round(totalPeakPoints / verifiedStories.length)
78
+
: 0;
79
+
80
+
return {
81
+
totalCount: verifiedUsersCount,
82
+
frontPageCount: frontPageCount,
83
+
avgPeakPoints: avgPeakPoints,
84
+
};
85
+
});
86
+
87
+
// 4. Optional: Warm up top 5 story snapshots (preload most accessed story graphs)
88
+
// This is done with lower priority as it's less critical
89
+
console.log("Preloading top story snapshots (limited to 5)...");
90
+
91
+
// Get IDs of top 5 stories to warm their snapshots
92
+
const topStories = await db.query.stories.findMany({
93
+
where: (stories, { eq }) => eq(stories.isOnLeaderboard, true),
94
+
orderBy: (stories, { asc }) => [asc(stories.position)],
95
+
limit: 5, // Reduced from 20 to 5 to minimize initial load
96
+
});
97
+
98
+
// Preload snapshots for these stories sequentially
99
+
for (const story of topStories) {
100
+
await queryCache.get(`story_snapshots_${story.id}`, async () => {
101
+
// Get snapshots for the story
102
+
const snapshots = await db.query.leaderboardSnapshots.findMany({
103
+
where: (snapshots, { eq }) => eq(snapshots.storyId, story.id),
104
+
orderBy: (snapshots, { asc }) => [asc(snapshots.timestamp)],
105
+
});
106
+
107
+
// Transform snapshot data for frontend
108
+
return snapshots.map((snapshot) => ({
109
+
timestamp: snapshot.timestamp,
110
+
position: snapshot.position,
111
+
score: snapshot.score,
112
+
date: new Date(snapshot.timestamp * 1000).toISOString(),
113
+
}));
114
+
}, 3600); // Cache story snapshots for 1 hour
115
+
}
116
+
117
+
console.log("Cache preloading completed successfully");
118
+
} catch (error) {
119
+
console.error("Error during cache preloading:", error);
120
+
Sentry.captureException(error);
121
+
}
122
+
}
123
+
124
+
/**
125
+
* Invalidates all caches and then immediately reloads them
126
+
* Call this after data updates (like the HN check cron job)
127
+
*/
128
+
export function invalidateAndRefreshCaches(): void {
129
+
console.log("Invalidating all query caches and refreshing data");
130
+
queryCache.invalidateAll();
131
+
132
+
// Immediately refill the cache
133
+
preloadCaches().catch(err => {
134
+
console.error("Error during cache preloading after invalidation:", err);
135
+
Sentry.captureException(err);
136
+
});
137
+
}
+16
-2
src/libs/db.ts
+16
-2
src/libs/db.ts
···
5
5
// Use environment variable for the database path in production
6
6
const dbPath = process.env.DATABASE_PATH || "./local.db";
7
7
8
-
// Create a SQLite database instance using Bun's built-in driver
9
-
const sqlite = new Database(dbPath);
8
+
// Create a SQLite database instance using Bun's built-in driver with improved concurrency settings
9
+
const sqlite = new Database(dbPath, {
10
+
// Use WAL mode for better concurrency
11
+
readonly: false,
12
+
create: true
13
+
});
14
+
15
+
// Set a longer busy timeout to reduce "database is locked" errors
16
+
sqlite.exec("PRAGMA busy_timeout = 5000;");
17
+
18
+
// Enable Write-Ahead Logging mode for better concurrent performance
19
+
sqlite.exec("PRAGMA journal_mode = WAL;");
20
+
// Set synchronous mode for better performance (still safe in WAL mode)
21
+
sqlite.exec("PRAGMA synchronous = NORMAL;");
22
+
// Increase cache size for better performance
23
+
sqlite.exec("PRAGMA cache_size = -16000;"); // Use ~16MB of memory for cache
10
24
11
25
// Create a Drizzle instance with the database and schema
12
26
export const db = drizzle(sqlite, { schema });
+112
-59
src/libs/schema.ts
+112
-59
src/libs/schema.ts
···
1
-
import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core";
1
+
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core";
2
2
3
3
// Define the users table
4
-
export const users = sqliteTable("users", {
5
-
id: text("id").primaryKey(),
6
-
hackernewsUsername: text("hackernews_username"),
7
-
challenge: text("challenge"),
8
-
verified: integer("verified", { mode: "boolean" }).default(false).notNull(),
9
-
});
4
+
export const users = sqliteTable(
5
+
"users",
6
+
{
7
+
id: text("id").primaryKey(),
8
+
hackernewsUsername: text("hackernews_username"),
9
+
challenge: text("challenge"),
10
+
verified: integer("verified", { mode: "boolean" }).default(false).notNull(),
11
+
},
12
+
(table) => ({
13
+
// Add index on verified status for faster lookup of verified users
14
+
verifiedIdx: index("idx_users_verified").on(table.verified),
15
+
// Add index on hackernews username for quick lookups
16
+
usernameIdx: index("idx_users_username").on(table.hackernewsUsername),
17
+
}),
18
+
);
10
19
11
-
export const stories = sqliteTable("stories", {
12
-
id: integer("id").primaryKey(),
13
-
by: text("by").notNull(),
14
-
title: text("title").notNull(),
15
-
url: text("url"),
16
-
text: text("text"),
17
-
time: integer("time").notNull(),
18
-
score: integer("score"),
19
-
position: integer("position"),
20
-
descendants: integer("descendants"),
20
+
export const stories = sqliteTable(
21
+
"stories",
22
+
{
23
+
id: integer("id").primaryKey(),
24
+
by: text("by").notNull(),
25
+
title: text("title").notNull(),
26
+
url: text("url"),
27
+
text: text("text"),
28
+
time: integer("time").notNull(),
29
+
score: integer("score"),
30
+
position: integer("position"),
31
+
descendants: integer("descendants"),
21
32
22
-
// New tracking fields
23
-
firstSeenAt: integer("first_seen_at").notNull(), // When we first saw it
24
-
lastUpdatedAt: integer("last_updated_at").notNull(), // Last time we updated this record
25
-
notifiedAt: integer("notified_at"), // When first notification was sent
33
+
// New tracking fields
34
+
firstSeenAt: integer("first_seen_at").notNull(), // When we first saw it
35
+
lastUpdatedAt: integer("last_updated_at").notNull(), // Last time we updated this record
36
+
notifiedAt: integer("notified_at"), // When first notification was sent
26
37
27
-
// Notification tracking flags - avoids duplicate notifications on restart
28
-
notifiedNewStory: integer("notified_new_story", { mode: "boolean" }).default(
29
-
false,
30
-
),
31
-
notifiedFrontPage: integer("notified_front_page", {
32
-
mode: "boolean",
33
-
}).default(false),
34
-
notifiedNumberOne: integer("notified_number_one", {
35
-
mode: "boolean",
36
-
}).default(false),
38
+
// Notification tracking flags - avoids duplicate notifications on restart
39
+
notifiedNewStory: integer("notified_new_story", {
40
+
mode: "boolean",
41
+
}).default(false),
42
+
notifiedFrontPage: integer("notified_front_page", {
43
+
mode: "boolean",
44
+
}).default(false),
45
+
notifiedNumberOne: integer("notified_number_one", {
46
+
mode: "boolean",
47
+
}).default(false),
37
48
38
-
// Leaderboard tracking
39
-
isOnLeaderboard: integer("is_on_leaderboard", { mode: "boolean" }).default(
40
-
false,
41
-
),
42
-
enteredLeaderboardAt: integer("entered_leaderboard_at"),
43
-
exitedLeaderboardAt: integer("exited_leaderboard_at"),
44
-
peakPosition: integer("peak_position"),
45
-
peakPositionAt: integer("peak_position_at"),
46
-
peakScore: integer("peak_score"),
47
-
peakScoreAt: integer("peak_score_at"),
49
+
// Leaderboard tracking
50
+
isOnLeaderboard: integer("is_on_leaderboard", { mode: "boolean" }).default(
51
+
false,
52
+
),
53
+
enteredLeaderboardAt: integer("entered_leaderboard_at"),
54
+
exitedLeaderboardAt: integer("exited_leaderboard_at"),
55
+
peakPosition: integer("peak_position"),
56
+
peakPositionAt: integer("peak_position_at"),
57
+
peakScore: integer("peak_score"),
58
+
peakScoreAt: integer("peak_score_at"),
48
59
49
-
// Tracking if this is from a monitored user
50
-
isFromMonitoredUser: integer("is_from_monitored_user", {
51
-
mode: "boolean",
52
-
}).default(false),
60
+
// Tracking if this is from a monitored user
61
+
isFromMonitoredUser: integer("is_from_monitored_user", {
62
+
mode: "boolean",
63
+
}).default(false),
53
64
54
-
// Cache management - TTL field for automatic cleanup
55
-
expiresAt: integer("expires_at"), // NULL for monitored user stories (permanent)
56
-
});
65
+
// Cache management - TTL field for automatic cleanup
66
+
expiresAt: integer("expires_at"), // NULL for monitored user stories (permanent)
67
+
},
68
+
(table) => {
69
+
return {
70
+
// Add index on leaderboard status
71
+
leaderboardIdx: index("idx_stories_leaderboard").on(
72
+
table.isOnLeaderboard,
73
+
),
74
+
// Add index on monitored user stories
75
+
monitoredUserIdx: index("idx_stories_monitored_user").on(
76
+
table.isFromMonitoredUser,
77
+
),
78
+
// Add compound index for sorting leaderboard stories by position
79
+
leaderboardPosIdx: index("idx_stories_leaderboard_position").on(
80
+
table.isOnLeaderboard,
81
+
table.position,
82
+
),
83
+
// Add index on by field for user-specific queries
84
+
byIdx: index("idx_stories_by").on(table.by),
85
+
// Add index on expiration for cleanup queries
86
+
expiresIdx: index("idx_stories_expires").on(table.expiresAt),
87
+
// Add index on time for timeline queries
88
+
timeIdx: index("idx_stories_time").on(table.time),
89
+
};
90
+
},
91
+
);
57
92
58
93
// For tracking leaderboard positions
59
-
export const leaderboardSnapshots = sqliteTable("leaderboard_snapshots", {
60
-
id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }),
61
-
storyId: integer("story_id")
62
-
.notNull()
63
-
.references(() => stories.id),
64
-
timestamp: integer("timestamp").notNull(),
65
-
position: integer("position").notNull(),
66
-
score: integer("score").notNull(),
94
+
export const leaderboardSnapshots = sqliteTable(
95
+
"leaderboard_snapshots",
96
+
{
97
+
id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }),
98
+
storyId: integer("story_id")
99
+
.notNull()
100
+
.references(() => stories.id),
101
+
timestamp: integer("timestamp").notNull(),
102
+
position: integer("position").notNull(),
103
+
score: integer("score").notNull(),
67
104
68
-
// TTL for cleanup
69
-
expiresAt: integer("expires_at").notNull(),
70
-
});
105
+
// TTL for cleanup
106
+
expiresAt: integer("expires_at").notNull(),
107
+
},
108
+
(table) => {
109
+
return {
110
+
// Add index on story ID for faster lookups of a story's snapshots
111
+
storyIdx: index("idx_snapshots_story").on(table.storyId),
112
+
// Add index on timestamp for chronological queries
113
+
timestampIdx: index("idx_snapshots_timestamp").on(table.timestamp),
114
+
// Add compound index for a story's chronological snapshots
115
+
storyTimeIdx: index("idx_snapshots_story_time").on(
116
+
table.storyId,
117
+
table.timestamp,
118
+
),
119
+
// Add index on expiration for cleanup queries
120
+
expiresIdx: index("idx_snapshots_expires").on(table.expiresAt),
121
+
};
122
+
},
123
+
);