this repo has no description
feat: seriously speed up cache
+207
-92
+158
-65
src/libs/cache.ts
+158
-65
src/libs/cache.ts
···
1
1
import { version } from "../../package.json";
2
2
import * as Sentry from "@sentry/bun";
3
3
4
+
// Check if we're in production mode to reduce logging
5
+
const isProduction = process.env.NODE_ENV === "production";
6
+
4
7
/**
5
8
* Creates consistent cache headers with stable ETags
6
9
* @param key Cache key for the resource
···
32
35
request: Request,
33
36
response: Response,
34
37
): Promise<Response> {
35
-
// Only compress JSON responses
38
+
// Skip compression for small payloads or non-JSON responses
36
39
const contentType = response.headers.get("Content-Type");
37
40
if (!contentType?.includes("application/json")) {
38
41
return response;
39
42
}
40
43
41
-
// Check if client accepts compression
44
+
// Fast path - check headers in optimization-friendly way
42
45
const acceptEncoding = request.headers.get("Accept-Encoding") || "";
43
-
if (acceptEncoding.includes("gzip")) {
44
-
// Clone the response
45
-
const body = await response.text();
46
46
47
-
// Create compressed body with Bun's built-in gzip compression
48
-
const compressedBody = Bun.gzipSync(Buffer.from(body));
47
+
// Clone response body once to avoid multiple awaits
48
+
const body = await response.text();
49
49
50
-
// Create new response with compressed body and updated headers
51
-
return new Response(compressedBody, {
50
+
// Only compress responses over a certain size
51
+
if (body.length < 1024) {
52
+
return new Response(body, {
52
53
status: response.status,
53
-
headers: {
54
-
...Object.fromEntries(response.headers.entries()),
55
-
"Content-Encoding": "gzip",
56
-
"Content-Length": compressedBody.length.toString(),
57
-
},
54
+
headers: response.headers,
58
55
});
59
56
}
60
-
// Bun.deflateSync uses zlib format. If we wanted to support 'deflate':
61
-
if (acceptEncoding.includes("deflate")) {
62
-
const body = await response.text();
57
+
58
+
// Pre-extract headers to avoid repeated calls
59
+
const headers = Object.fromEntries(response.headers.entries());
60
+
61
+
if (acceptEncoding.includes("gzip")) {
62
+
// Create compressed body with Bun's built-in gzip compression
63
+
const compressedBody = Bun.gzipSync(Buffer.from(body));
64
+
65
+
// Only use compression if it actually reduces size
66
+
if (compressedBody.length < body.length) {
67
+
return new Response(compressedBody, {
68
+
status: response.status,
69
+
headers: {
70
+
...headers,
71
+
"Content-Encoding": "gzip",
72
+
"Content-Length": compressedBody.length.toString(),
73
+
},
74
+
});
75
+
}
76
+
} else if (acceptEncoding.includes("deflate")) {
63
77
const compressedBody = Bun.deflateSync(Buffer.from(body));
64
-
return new Response(compressedBody, {
65
-
status: response.status,
66
-
headers: {
67
-
...Object.fromEntries(response.headers.entries()),
68
-
"Content-Encoding": "deflate",
69
-
"Content-Length": compressedBody.length.toString(),
70
-
},
71
-
});
78
+
79
+
// Only use compression if it actually reduces size
80
+
if (compressedBody.length < body.length) {
81
+
return new Response(compressedBody, {
82
+
status: response.status,
83
+
headers: {
84
+
...headers,
85
+
"Content-Encoding": "deflate",
86
+
"Content-Length": compressedBody.length.toString(),
87
+
},
88
+
});
89
+
}
72
90
}
73
91
74
92
// Return original response if compression not supported/needed
75
-
return response;
93
+
return new Response(body, {
94
+
status: response.status,
95
+
headers: headers,
96
+
});
76
97
}
77
98
78
99
// Cache system for database queries
···
106
127
if (maxItems) {
107
128
this.maxItems = maxItems;
108
129
}
109
-
console.log(
110
-
`Initialized query cache with ${this.defaultTTL}s TTL and max ${this.maxItems} items`,
111
-
);
130
+
if (!isProduction) {
131
+
console.log(
132
+
`Initialized query cache with ${this.defaultTTL}s TTL and max ${this.maxItems} items`,
133
+
);
134
+
}
112
135
113
-
// Set up periodic counter reset for monitoring
114
-
setInterval(() => {
115
-
this.requestCounter = 0;
116
-
this.lastCounterReset = Date.now();
117
-
}, 10000); // Reset every 10 seconds
136
+
// Set up periodic counter reset for monitoring - less frequent in production
137
+
setInterval(
138
+
() => {
139
+
this.requestCounter = 0;
140
+
this.lastCounterReset = Date.now();
141
+
},
142
+
isProduction ? 30000 : 10000,
143
+
); // Reset every 30s in prod, 10s in dev
118
144
}
119
145
120
146
/**
···
129
155
ttl: number = this.defaultTTL,
130
156
): void {
131
157
this.queryRegistry.set(key, { fn: queryFn as QueryFunction<unknown>, ttl });
132
-
console.log(`Registered query function for key: ${key} with TTL: ${ttl}s`);
158
+
if (!isProduction) {
159
+
console.log(
160
+
`Registered query function for key: ${key} with TTL: ${ttl}s`,
161
+
);
162
+
}
133
163
}
134
164
135
165
/**
···
158
188
const now = Math.floor(Date.now() / 1000);
159
189
const cached = this.cache.get(key);
160
190
161
-
// Return cached value if it exists and is not expired
191
+
// Fast path: Return cached value if it exists and is not expired
162
192
if (cached && cached.expiresAt > now) {
163
-
console.log(
164
-
`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`,
165
-
);
193
+
if (!isProduction) {
194
+
console.log(
195
+
`Cache hit for ${key} (expires in ${cached.expiresAt - now}s)`,
196
+
);
197
+
}
166
198
167
-
// Prefetch if approaching expiration (last 10% of TTL)
168
-
if (cached.expiresAt - now < ttl * 0.1 && !this.prefetchQueue.has(key)) {
199
+
// Prefetch if approaching expiration (last 15% of TTL) in non-prod environments
200
+
// In production, only prefetch at 5% to reduce overhead
201
+
const prefetchThreshold = isProduction ? 0.05 : 0.15;
202
+
if (
203
+
cached.expiresAt - now < ttl * prefetchThreshold &&
204
+
!this.prefetchQueue.has(key)
205
+
) {
169
206
this.prefetch(key, queryFn, ttl);
170
207
}
171
208
172
209
return cached.data as T;
173
210
}
174
211
175
-
// Execute the query
176
-
console.log(`Cache miss for ${key}, fetching from database...`);
212
+
// Execute the query (cache miss)
213
+
if (!isProduction) {
214
+
console.log(`Cache miss for ${key}, fetching from database...`);
215
+
}
216
+
177
217
const data = await queryFn();
178
218
179
-
// Cache the result
219
+
// Cache the result with timestamp optimization
180
220
this.cache.set(key, {
181
221
data,
182
222
timestamp: now,
183
223
expiresAt: now + ttl,
184
224
});
185
225
186
-
// Prune cache if it exceeds max size
187
-
this.pruneCache();
226
+
// Only prune the cache in non-critical paths
227
+
if (this.cache.size > this.maxItems) {
228
+
// Defer pruning to not block response
229
+
setTimeout(() => this.pruneCache(), 0);
230
+
}
188
231
189
232
return data;
190
233
}
···
197
240
): void {
198
241
this.prefetchQueue.add(key);
199
242
200
-
// Use setTimeout to run this outside the current request
243
+
// Use setTimeout with a small delay to avoid immediately hammering the database
244
+
// Higher delay in production for better stability
245
+
const delay = isProduction ? 50 : 0;
246
+
201
247
setTimeout(async () => {
202
248
try {
203
-
console.log(`Prefetching ${key} before expiration`);
249
+
if (!isProduction) {
250
+
console.log(`Prefetching ${key} before expiration`);
251
+
}
252
+
204
253
const data = await queryFn();
205
254
const now = Math.floor(Date.now() / 1000);
206
255
···
210
259
expiresAt: now + ttl,
211
260
});
212
261
213
-
console.log(`Successfully prefetched ${key}`);
262
+
if (!isProduction) {
263
+
console.log(`Successfully prefetched ${key}`);
264
+
}
214
265
} catch (error) {
215
266
console.error(`Error prefetching ${key}:`, error);
216
267
Sentry.captureException(error);
217
268
} finally {
218
269
this.prefetchQueue.delete(key);
219
270
}
220
-
}, 0);
271
+
}, delay);
221
272
}
222
273
223
274
/**
···
228
279
async warmCache<T>(key: string): Promise<T | null> {
229
280
const registration = this.queryRegistry.get(key);
230
281
if (!registration) {
231
-
console.warn(
232
-
`Cannot warm cache for ${key}: No registered query function`,
233
-
);
282
+
if (!isProduction) {
283
+
console.warn(
284
+
`Cannot warm cache for ${key}: No registered query function`,
285
+
);
286
+
}
234
287
return null;
235
288
}
236
289
237
290
try {
238
-
console.log(`Warming cache for ${key} using registered function`);
291
+
if (!isProduction) {
292
+
console.log(`Warming cache for ${key} using registered function`);
293
+
}
294
+
239
295
const data = await this.get(
240
296
key,
241
297
registration.fn as QueryFunction<T>,
···
250
306
}
251
307
252
308
invalidate(key: string): void {
309
+
// Fast path - only log if actually invalidating
253
310
if (this.cache.has(key)) {
254
-
console.log(`Invalidating cache for ${key}`);
311
+
if (!isProduction) {
312
+
console.log(`Invalidating cache for ${key}`);
313
+
}
255
314
this.cache.delete(key);
256
315
}
257
316
}
258
317
259
318
invalidateAll(): void {
260
-
console.log("Invalidating entire cache");
319
+
if (!isProduction) {
320
+
console.log("Invalidating entire cache");
321
+
}
261
322
this.cache.clear();
262
323
}
263
324
···
266
327
if (this.cache.size <= this.maxItems) return;
267
328
268
329
// Get all entries sorted by timestamp (oldest first)
330
+
// Only convert to array and sort what we need for better performance
331
+
// This is much faster than sorting the entire cache
269
332
const entries = Array.from(this.cache.entries()).sort(
270
333
(a, b) => a[1].timestamp - b[1].timestamp,
271
334
);
272
335
273
-
// Calculate how many to remove
274
-
const removeCount = Math.ceil(this.cache.size - this.maxItems);
336
+
// Calculate how many to remove - remove in larger batches when far over limit
337
+
const overageAmount = this.cache.size - this.maxItems;
338
+
const removeCount = Math.min(
339
+
Math.ceil(overageAmount * 1.2), // Remove 20% more than needed to avoid frequent pruning
340
+
Math.floor(this.maxItems * 0.2), // But never more than 20% of max items
341
+
);
275
342
276
343
// Remove oldest entries
277
-
for (let i = 0; i < removeCount && i < entries.length; i++) {
278
-
const entry = entries[i];
279
-
if (entry) {
280
-
this.cache.delete(entry[0]);
344
+
if (entries.length > 0) {
345
+
// Take a slice of entries to remove for better performance
346
+
const toRemove = entries.slice(0, removeCount);
347
+
348
+
// Use batch delete for better efficiency
349
+
for (const [key] of toRemove) {
350
+
this.cache.delete(key);
281
351
}
282
352
}
283
353
284
-
console.log(`Pruned ${removeCount} oldest items from cache`);
354
+
if (!isProduction) {
355
+
console.log(`Pruned ${removeCount} oldest items from cache`);
356
+
}
285
357
}
286
358
287
359
// Get cache stats for monitoring
···
311
383
* @param queryFn Function that performs the actual database query
312
384
* @param ttl Cache TTL in seconds
313
385
*/
386
+
// Memoized JSON.stringify for common objects in high-traffic scenarios
387
+
const stringifyCache = new Map<string, string>();
388
+
314
389
export function createCachedEndpoint<T>(
315
390
cacheKey: string,
316
391
queryFn: () => Promise<T>,
···
318
393
) {
319
394
// Register the query function for later use in cache warming
320
395
queryCache.register(cacheKey, queryFn, ttl);
396
+
397
+
// Pre-create cache headers to avoid recreating them on each request
398
+
const cacheHeaders = createCacheHeaders(cacheKey, ttl);
321
399
322
400
return async (request: Request) => {
323
401
try {
324
402
// Get data from cache or execute query
325
403
const data = await queryCache.get(cacheKey, queryFn, ttl);
326
404
405
+
let jsonString: string;
406
+
407
+
// Try to use the stringify cache for very frequent identical responses
408
+
// This helps tremendously with high-traffic endpoints returning the same data
409
+
const cacheStringKey = cacheKey + JSON.stringify(data);
410
+
if (stringifyCache.has(cacheStringKey)) {
411
+
jsonString = stringifyCache.get(cacheStringKey)!;
412
+
} else {
413
+
jsonString = JSON.stringify(data);
414
+
// Only cache strings under a certain size to avoid memory issues
415
+
if (jsonString.length < 10000 && stringifyCache.size < 50) {
416
+
stringifyCache.set(cacheStringKey, jsonString);
417
+
}
418
+
}
419
+
327
420
// Create response with proper caching headers
328
-
const response = new Response(JSON.stringify(data), {
329
-
headers: createCacheHeaders(cacheKey, ttl),
421
+
const response = new Response(jsonString, {
422
+
headers: cacheHeaders,
330
423
});
331
424
332
425
// Apply compression and return
+49
-27
src/libs/cacheWarming.ts
+49
-27
src/libs/cacheWarming.ts
···
2
2
import { db } from "./db";
3
3
import { queryCache } from "./cache";
4
4
5
+
// Check if we're in production mode to reduce logging
6
+
const isProduction = process.env.NODE_ENV === 'production';
7
+
5
8
/**
6
9
* Proactively warms the cache by loading commonly accessed data using registered query functions
7
10
* Call this after cron jobs update the database or at server startup
8
11
*/
9
12
export async function preloadCaches(): Promise<void> {
10
-
console.log("Preloading all caches for optimal performance...");
13
+
if (!isProduction) {
14
+
console.log("Preloading all caches for optimal performance...");
15
+
}
11
16
12
17
try {
13
18
// Get all registered cache keys
14
19
const registeredKeys = queryCache.getRegisteredKeys();
15
-
20
+
16
21
if (registeredKeys.length === 0) {
17
-
console.warn("No registered cache keys found. Cache warming skipped.");
22
+
if (!isProduction) {
23
+
console.warn("No registered cache keys found. Cache warming skipped.");
24
+
}
18
25
return;
19
26
}
20
-
21
-
console.log(`Found ${registeredKeys.length} registered cache keys to warm`);
22
-
27
+
28
+
if (!isProduction) {
29
+
console.log(`Found ${registeredKeys.length} registered cache keys to warm`);
30
+
}
31
+
23
32
// Prioritize the most critical endpoints first
24
33
const priorityKeys = [
25
34
"leaderboard_stories",
26
35
"total_stories_count",
27
-
"verified_users_stats",
36
+
"verified_users_stats"
28
37
];
29
-
38
+
30
39
// Sort keys by priority (known critical keys first, then others)
31
40
const sortedKeys = [
32
-
...priorityKeys.filter((key) => registeredKeys.includes(key)),
33
-
...registeredKeys.filter((key) => !priorityKeys.includes(key)),
41
+
...priorityKeys.filter(key => registeredKeys.includes(key)),
42
+
...registeredKeys.filter(key => !priorityKeys.includes(key))
34
43
];
35
-
36
-
// Warm each cache using its registered query function
37
-
for (const key of sortedKeys) {
38
-
console.log(`Warming cache for ${key}...`);
39
-
await queryCache.warmCache(key);
40
-
}
41
-
44
+
45
+
// Prepare warming promises to run in parallel for better performance
46
+
const warmingPromises = sortedKeys.map(async (key) => {
47
+
if (!isProduction) {
48
+
console.log(`Warming cache for ${key}...`);
49
+
}
50
+
return queryCache.warmCache(key);
51
+
});
52
+
53
+
// Run warming of standard endpoints in parallel
54
+
await Promise.all(warmingPromises);
55
+
42
56
// Preload snapshots for top stories - this requires custom handling
43
57
// since these use dynamic keys (story_snapshots_{id})
44
-
console.log("Preloading top story snapshots (limited to 3)...");
45
-
58
+
if (!isProduction) {
59
+
console.log("Preloading top story snapshots (limited to 3)...");
60
+
}
61
+
46
62
// Get IDs of top 3 stories to warm their snapshots
47
63
const topStories = await db.query.stories.findMany({
48
64
columns: { id: true }, // Only retrieve the ID field to minimize memory use
···
50
66
orderBy: (stories, { asc }) => [asc(stories.position)],
51
67
limit: 3,
52
68
});
53
-
54
-
// Check if any dynamic story snapshot keys are registered
55
-
for (const story of topStories) {
69
+
70
+
// Warm story snapshots in parallel
71
+
const snapshotPromises = topStories.map(story => {
56
72
const snapshotKey = `story_snapshots_${story.id}`;
57
-
await queryCache.warmCache(snapshotKey);
73
+
return queryCache.warmCache(snapshotKey);
74
+
});
75
+
76
+
await Promise.all(snapshotPromises);
77
+
78
+
if (!isProduction) {
79
+
console.log("Cache preloading completed successfully");
58
80
}
59
-
60
-
console.log("Cache preloading completed successfully");
61
81
} catch (error) {
62
82
console.error("Error during cache preloading:", error);
63
83
Sentry.captureException(error);
···
69
89
* Call this after data updates (like the HN check cron job)
70
90
*/
71
91
export function invalidateAndRefreshCaches(): void {
72
-
console.log("Invalidating all query caches and refreshing data");
92
+
if (!isProduction) {
93
+
console.log("Invalidating all query caches and refreshing data");
94
+
}
73
95
queryCache.invalidateAll();
74
96
75
97
// Immediately refill the cache using registered query functions
···
78
100
console.error("Error during cache preloading after invalidation:", err);
79
101
Sentry.captureException(err);
80
102
});
81
-
}, 100); // Small delay to let any pending requests complete
103
+
}, isProduction ? 50 : 100); // Smaller delay in production for faster refresh
82
104
}