search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
fix: incremental sync on startup with tombstone-based cleanup
was doing full sync (~80 min for 12k docs) on every deploy, blocking
new content from appearing in search. the full sync existed to clean
up stale docs deleted from Turso.
now: incremental sync on startup (seconds), with tombstone queries to
handle deletions. fullSync only runs on first-ever boot (no last_sync).
tombstones table was already populated by deleteDocument/deletePublication
but never queried during sync.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+40
-5
+4
-3
backend/src/db/mod.zig
+4
-3
backend/src/db/mod.zig
···
84
84
}
85
85
86
86
fn syncLoop(turso: *Client, local: *LocalDb) void {
87
-
// full sync on startup — reconciles with Turso and cleans up stale docs
88
-
sync.fullSync(turso, local) catch |err| {
89
-
std.debug.print("sync: initial full sync failed: {}\n", .{err});
87
+
// incremental sync on startup — gets new docs + cleans tombstoned deletions
88
+
// (falls back to full sync automatically if no last_sync exists, i.e., first boot)
89
+
sync.incrementalSync(turso, local) catch |err| {
90
+
std.debug.print("sync: initial sync failed: {}\n", .{err});
90
91
};
91
92
92
93
// get sync interval from env (default 5 minutes)
+36
-2
backend/src/db/sync.zig
+36
-2
backend/src/db/sync.zig
···
302
302
) catch {};
303
303
}
304
304
305
+
// sync deletions via tombstones (deleted_at is unix timestamp integer)
306
+
var deleted: usize = 0;
307
+
tombstone: {
308
+
var since_ts_buf: [20]u8 = undefined;
309
+
const since_ts_str = std.fmt.bufPrint(&since_ts_buf, "{d}", .{since_ts}) catch break :tombstone;
310
+
311
+
var tomb_result = turso.query(
312
+
"SELECT uri, record_type FROM tombstones WHERE deleted_at >= ?",
313
+
&.{since_ts_str},
314
+
) catch |err| {
315
+
std.debug.print("sync: tombstone query failed: {}\n", .{err});
316
+
break :tombstone;
317
+
};
318
+
defer tomb_result.deinit();
319
+
320
+
if (tomb_result.rows.len > 0) {
321
+
local.lock();
322
+
defer local.unlock();
323
+
for (tomb_result.rows) |row| {
324
+
const uri = row.text(0);
325
+
const record_type = row.text(1);
326
+
if (std.mem.eql(u8, record_type, "document")) {
327
+
conn.exec("DELETE FROM documents WHERE uri = ?", .{uri}) catch {};
328
+
conn.exec("DELETE FROM documents_fts WHERE uri = ?", .{uri}) catch {};
329
+
conn.exec("DELETE FROM document_tags WHERE document_uri = ?", .{uri}) catch {};
330
+
} else if (std.mem.eql(u8, record_type, "publication")) {
331
+
conn.exec("DELETE FROM publications WHERE uri = ?", .{uri}) catch {};
332
+
conn.exec("DELETE FROM publications_fts WHERE uri = ?", .{uri}) catch {};
333
+
}
334
+
deleted += 1;
335
+
}
336
+
}
337
+
}
338
+
305
339
// periodic WAL checkpoint to prevent unbounded growth
306
340
local.lock();
307
341
conn.exec("PRAGMA wal_checkpoint(PASSIVE)", .{}) catch {};
308
342
local.unlock();
309
343
310
-
if (new_docs > 0) {
311
-
std.debug.print("sync: incremental sync added {d} new documents\n", .{new_docs});
344
+
if (new_docs > 0 or deleted > 0) {
345
+
std.debug.print("sync: incremental sync — {d} new docs, {d} tombstone deletions\n", .{ new_docs, deleted });
312
346
}
313
347
}
314
348