fuzzy find my records
ken.waow.tech
embeddings
pds
search
hard cap at 50k records with a friendly error
if even the 2-year time cutoff leaves more than 50k records, bail
before embedding rather than OOMing the 4GB fly machine. the error
message is user-facing and tells them to DM @zzstoatzz.io.
also: runJob now preserves a pre-set error_msg instead of blindly
overwriting it with the error code, so custom messages from
openAndWalkRepo actually reach the UI.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+25
-3
+25
-3
backend/src/indexer.zig
+25
-3
backend/src/indexer.zig
···
65
65
/// tell the user exactly what got cut.
66
66
const AUTO_CUTOFF_WINDOW_US: i64 = 2 * 365 * 24 * 60 * 60 * 1_000_000;
67
67
68
+
/// hard ceiling on post-filter record count. if even the 2-year time
69
+
/// cutoff leaves more records than this, we bail rather than OOMing
70
+
/// the 4 GB fly machine. the error message is user-facing.
71
+
const ABSOLUTE_MAX_RECORDS: usize = 50_000;
72
+
68
73
pub const Status = enum { indexing, ready, @"error" };
69
74
70
75
pub const Entry = struct {
···
261
266
pub fn runJob(job: *Job) void {
262
267
doIndex(job) catch |err| {
263
268
std.log.err("indexing job failed for {s}: {t}", .{ job.pack.handle, err });
264
-
const msg = std.fmt.allocPrint(job.pack.arena.allocator(), "{t}", .{err}) catch "unknown";
265
269
job.pack.status = .@"error";
266
-
job.pack.error_msg = msg;
270
+
// openAndWalkRepo may have already set a user-facing error_msg
271
+
// (e.g. the repo-too-large message). only overwrite with the
272
+
// generic error code if nothing was set.
273
+
if (job.pack.error_msg.len == 0) {
274
+
job.pack.error_msg = std.fmt.allocPrint(job.pack.arena.allocator(), "{t}", .{err}) catch "unknown";
275
+
}
267
276
};
268
277
}
269
278
···
315
324
);
316
325
}
317
326
318
-
return try repo_walk.walkOpened(arena, &opened, pack.did, filter);
327
+
const walked = try repo_walk.walkOpened(arena, &opened, pack.did, filter);
328
+
329
+
if (walked.records.len > ABSOLUTE_MAX_RECORDS) {
330
+
pack.error_msg = std.fmt.allocPrint(
331
+
pack.arena.allocator(),
332
+
"hey, we know you have a big PDS — everybody knows you have a big PDS — " ++
333
+
"but ken can only handle {d} records right now and yours has {d} after filtering. " ++
334
+
"DM @zzstoatzz.io on bluesky if you want to talk about it",
335
+
.{ ABSOLUTE_MAX_RECORDS, walked.records.len },
336
+
) catch "repo too large";
337
+
return error.OutOfMemory;
338
+
}
339
+
340
+
return walked;
319
341
}
320
342
321
343
fn doIndex(job: *Job) !void {