search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
fix: add WAL checkpointing to prevent query failures
Root cause: WAL file grew to 498MB (main db was 113MB) without ever
being checkpointed back to main database. Large WAL files cause FTS5
queries to fail with generic SQLite errors.
Changes:
- Add PRAGMA wal_checkpoint(TRUNCATE) after full sync
- Add PRAGMA wal_checkpoint(PASSIVE) after incremental sync
- Add integrity check on startup that auto-deletes corrupt db
- Store db path for potential auto-recovery
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+61
+51
backend/src/db/LocalDb.zig
+51
backend/src/db/LocalDb.zig
···
11
11
conn: ?zqlite.Conn = null,
12
12
allocator: Allocator,
13
13
is_ready: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
14
+
needs_resync: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
14
15
mutex: std.Thread.Mutex = .{},
16
+
path: []const u8 = "",
17
+
consecutive_errors: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
15
18
16
19
pub fn init(allocator: Allocator) LocalDb {
17
20
return .{ .allocator = allocator };
18
21
}
19
22
23
+
/// Check database integrity and return false if corrupt
24
+
fn checkIntegrity(self: *LocalDb) bool {
25
+
const c = self.conn orelse return false;
26
+
const row = c.row("PRAGMA integrity_check", .{}) catch return false;
27
+
if (row) |r| {
28
+
defer r.deinit();
29
+
const result = r.text(0);
30
+
if (std.mem.eql(u8, result, "ok")) {
31
+
return true;
32
+
}
33
+
std.debug.print("local db: integrity check failed: {s}\n", .{result});
34
+
return false;
35
+
}
36
+
return false;
37
+
}
38
+
39
+
/// Delete the database file and WAL/SHM files
40
+
fn deleteDbFiles(path: []const u8) void {
41
+
std.fs.cwd().deleteFile(path) catch {};
42
+
// also delete WAL and SHM files
43
+
var wal_buf: [260]u8 = undefined;
44
+
var shm_buf: [260]u8 = undefined;
45
+
if (path.len < 252) {
46
+
const wal_path = std.fmt.bufPrint(&wal_buf, "{s}-wal", .{path}) catch return;
47
+
const shm_path = std.fmt.bufPrint(&shm_buf, "{s}-shm", .{path}) catch return;
48
+
std.fs.cwd().deleteFile(wal_path) catch {};
49
+
std.fs.cwd().deleteFile(shm_path) catch {};
50
+
}
51
+
}
52
+
20
53
pub fn open(self: *LocalDb) !void {
21
54
const path_env = posix.getenv("LOCAL_DB_PATH") orelse "/data/local.db";
55
+
self.path = path_env;
22
56
57
+
try self.openDb(path_env, false);
58
+
}
59
+
60
+
fn openDb(self: *LocalDb, path_env: []const u8, is_retry: bool) !void {
23
61
// convert to null-terminated for zqlite
24
62
var path_buf: [256]u8 = undefined;
25
63
if (path_env.len >= path_buf.len) return error.PathTooLong;
···
38
76
// enable WAL for better concurrency
39
77
_ = self.conn.?.exec("PRAGMA journal_mode=WAL", .{}) catch {};
40
78
_ = self.conn.?.exec("PRAGMA busy_timeout=5000", .{}) catch {};
79
+
80
+
// check integrity - if corrupt, delete and recreate
81
+
if (!self.checkIntegrity()) {
82
+
if (is_retry) {
83
+
std.debug.print("local db: still corrupt after recreation, giving up\n", .{});
84
+
return error.DatabaseCorrupt;
85
+
}
86
+
std.debug.print("local db: corrupt, deleting and recreating\n", .{});
87
+
if (self.conn) |c| c.close();
88
+
self.conn = null;
89
+
deleteDbFiles(path_env);
90
+
return self.openDb(path_env, true);
91
+
}
41
92
42
93
try self.createSchema();
43
94
std.debug.print("local db: initialized\n", .{});
+10
backend/src/db/sync.zig
+10
backend/src/db/sync.zig
···
151
151
return err;
152
152
};
153
153
154
+
// checkpoint WAL to prevent unbounded growth
155
+
conn.exec("PRAGMA wal_checkpoint(TRUNCATE)", .{}) catch |err| {
156
+
std.debug.print("sync: wal checkpoint failed: {}\n", .{err});
157
+
};
158
+
154
159
local.setReady(true);
155
160
std.debug.print("sync: full sync complete - {d} docs, {d} pubs, {d} tags, {d} popular\n", .{ doc_count, pub_count, tag_count, popular_count });
156
161
}
···
241
246
.{ts_str},
242
247
) catch {};
243
248
}
249
+
250
+
// periodic WAL checkpoint to prevent unbounded growth
251
+
local.lock();
252
+
conn.exec("PRAGMA wal_checkpoint(PASSIVE)", .{}) catch {};
253
+
local.unlock();
244
254
245
255
if (new_docs > 0) {
246
256
std.debug.print("sync: incremental sync added {d} new documents\n", .{new_docs});