AppView in a box as a Vite plugin thing
hatk.dev
feat: incremental FTS5 for SQLite — eliminate full index rebuilds
Replace drop-and-rebuild FTS strategy with incremental external content
FTS5 tables for SQLite. Records are indexed/deindexed inline during
insertRecord/deleteRecord. Periodic and post-backfill rebuilds are
skipped for SQLite. DuckDB retains its existing behavior.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+985
-30
+788
docs/superpowers/plans/2026-03-15-sqlite-incremental-fts.md
+788
docs/superpowers/plans/2026-03-15-sqlite-incremental-fts.md
···
1
+
# SQLite Incremental FTS Implementation Plan
2
+
3
+
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
4
+
5
+
**Goal:** Replace SQLite's drop-and-rebuild FTS strategy with incremental FTS5 external content tables that stay in sync via application-level updates on insert/delete.
6
+
7
+
**Architecture:** Add `updateIndex()` and `deleteFromIndex()` methods to `SearchPort`. The SQLite implementation uses FTS5 external content mode (`content=shadowTable`) so the FTS index references the shadow data table directly. On `insertRecord()` and `deleteRecord()`, the framework incrementally updates both the shadow table and FTS index. The periodic `rebuildAllIndexes` in the indexer is skipped for SQLite. DuckDB keeps its existing drop-and-rebuild behavior unchanged.
8
+
9
+
**Tech Stack:** SQLite FTS5 with external content tables, TypeScript
10
+
11
+
---
12
+
13
+
### Task 1: Extend SearchPort Interface
14
+
15
+
**Files:**
16
+
- Modify: `packages/hatk/src/database/ports.ts:50-62`
17
+
18
+
**Step 1: Add new methods to SearchPort**
19
+
20
+
Add `updateIndex` and `deleteFromIndex` to the `SearchPort` interface:
21
+
22
+
```typescript
23
+
export interface SearchPort {
24
+
/** Build/rebuild an FTS index for a table */
25
+
buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void>
26
+
27
+
/** Incrementally update a single record in the FTS index */
28
+
updateIndex?(shadowTable: string, uri: string, row: Record<string, string | null>, searchColumns: string[]): Promise<void>
29
+
30
+
/** Remove a single record from the FTS index */
31
+
deleteFromIndex?(shadowTable: string, uri: string): Promise<void>
32
+
33
+
/** Check if the FTS index already exists (for skipping rebuild on startup) */
34
+
indexExists?(shadowTable: string): Promise<boolean>
35
+
36
+
/** Search a table, returning URIs with scores */
37
+
search(
38
+
shadowTable: string,
39
+
query: string,
40
+
searchColumns: string[],
41
+
limit: number,
42
+
offset: number,
43
+
): Promise<Array<{ uri: string; score: number }>>
44
+
}
45
+
```
46
+
47
+
Note: Methods are optional (`?`) so DuckDB's SearchPort doesn't need to implement them.
48
+
49
+
**Step 2: Commit**
50
+
51
+
```bash
52
+
git add packages/hatk/src/database/ports.ts
53
+
git commit -m "feat: add incremental FTS methods to SearchPort interface"
54
+
```
55
+
56
+
---
57
+
58
+
### Task 2: Rewrite SQLiteSearchPort for External Content FTS5
59
+
60
+
**Files:**
61
+
- Modify: `packages/hatk/src/database/adapters/sqlite-search.ts`
62
+
63
+
**Step 1: Rewrite the SQLiteSearchPort class**
64
+
65
+
Replace the entire file with:
66
+
67
+
```typescript
68
+
import type { SearchPort } from '../ports.ts'
69
+
import type { DatabasePort } from '../ports.ts'
70
+
71
+
/**
72
+
* SQLite FTS5-based search port with incremental updates.
73
+
*
74
+
* Uses external content FTS5 tables (`content=shadowTable`) so the FTS index
75
+
* references the shadow data table. Updates happen incrementally per-record
76
+
* instead of dropping and rebuilding the entire index.
77
+
*/
78
+
export class SQLiteSearchPort implements SearchPort {
79
+
constructor(private port: DatabasePort) {}
80
+
81
+
async indexExists(shadowTable: string): Promise<boolean> {
82
+
const rows = await this.port.query(
83
+
`SELECT 1 FROM sqlite_master WHERE type='table' AND name=$1`,
84
+
[shadowTable],
85
+
)
86
+
return rows.length > 0
87
+
}
88
+
89
+
async buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void> {
90
+
// Drop existing FTS table and data table
91
+
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}_fts`, [])
92
+
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}`, [])
93
+
94
+
// Create the shadow data table from the source query
95
+
await this.port.execute(`CREATE TABLE ${shadowTable} AS ${sourceQuery}`, [])
96
+
97
+
// Add a unique index on uri for fast lookups during incremental updates
98
+
await this.port.execute(`CREATE UNIQUE INDEX IF NOT EXISTS ${shadowTable}_uri ON ${shadowTable}(uri)`, [])
99
+
100
+
// Create the FTS5 virtual table with external content pointing to shadow table
101
+
const colList = searchColumns.join(', ')
102
+
await this.port.execute(
103
+
`CREATE VIRTUAL TABLE ${shadowTable}_fts USING fts5(uri UNINDEXED, ${colList}, content=${shadowTable}, content_rowid=rowid, tokenize='porter unicode61 remove_diacritics 2')`,
104
+
[],
105
+
)
106
+
107
+
// Populate FTS table from the shadow data table
108
+
const selectCols = ['uri', ...searchColumns].map((c) => `COALESCE(CAST(${c} AS TEXT), '')`)
109
+
await this.port.execute(
110
+
`INSERT INTO ${shadowTable}_fts (uri, ${colList}) SELECT ${selectCols.join(', ')} FROM ${shadowTable}`,
111
+
[],
112
+
)
113
+
}
114
+
115
+
async updateIndex(
116
+
shadowTable: string,
117
+
uri: string,
118
+
row: Record<string, string | null>,
119
+
searchColumns: string[],
120
+
): Promise<void> {
121
+
// Get existing rowid if this record is already indexed (for FTS delete)
122
+
const existing = await this.port.query(
123
+
`SELECT rowid FROM ${shadowTable} WHERE uri = $1`,
124
+
[uri],
125
+
)
126
+
127
+
if (existing.length > 0) {
128
+
const oldRowid = (existing[0] as any).rowid
129
+
// Read old values for FTS delete command
130
+
const colList = searchColumns.join(', ')
131
+
const oldRows = await this.port.query(
132
+
`SELECT uri, ${colList} FROM ${shadowTable} WHERE rowid = $1`,
133
+
[oldRowid],
134
+
)
135
+
if (oldRows.length > 0) {
136
+
const old = oldRows[0] as any
137
+
const oldVals = ['uri', ...searchColumns].map((c) => `COALESCE(CAST(${JSON.stringify(old[c] ?? '')} AS TEXT), '')`)
138
+
// Delete old entry from FTS using special 'delete' command
139
+
await this.port.execute(
140
+
`INSERT INTO ${shadowTable}_fts(${shadowTable}_fts, rowid, uri, ${colList}) VALUES('delete', $1, ${oldVals.join(', ')})`,
141
+
[oldRowid],
142
+
)
143
+
}
144
+
145
+
// Update shadow table row
146
+
const setClauses = searchColumns.map((c, i) => `${c} = $${i + 2}`)
147
+
const values = [uri, ...searchColumns.map((c) => row[c] ?? null)]
148
+
await this.port.execute(
149
+
`UPDATE ${shadowTable} SET ${setClauses.join(', ')} WHERE uri = $1`,
150
+
values,
151
+
)
152
+
153
+
// Re-read rowid (same after UPDATE) and insert new FTS entry
154
+
const newVals = searchColumns.map((c) => `COALESCE(CAST($${searchColumns.indexOf(c) + 2} AS TEXT), '')`)
155
+
await this.port.execute(
156
+
`INSERT INTO ${shadowTable}_fts(rowid, uri, ${colList}) VALUES($1, $2, ${newVals.join(', ')})`,
157
+
[oldRowid, uri, ...searchColumns.map((c) => row[c] ?? '')],
158
+
)
159
+
} else {
160
+
// Insert new row into shadow table
161
+
const colList = searchColumns.join(', ')
162
+
const placeholders = searchColumns.map((_, i) => `$${i + 2}`)
163
+
const values = [uri, ...searchColumns.map((c) => row[c] ?? null)]
164
+
await this.port.execute(
165
+
`INSERT INTO ${shadowTable} (uri, ${colList}) VALUES ($1, ${placeholders.join(', ')})`,
166
+
values,
167
+
)
168
+
169
+
// Get the new rowid and insert into FTS
170
+
const newRow = await this.port.query(`SELECT rowid FROM ${shadowTable} WHERE uri = $1`, [uri])
171
+
const newRowid = (newRow[0] as any).rowid
172
+
await this.port.execute(
173
+
`INSERT INTO ${shadowTable}_fts(rowid, uri, ${colList}) VALUES($1, $2, ${placeholders.join(', ')})`,
174
+
[newRowid, uri, ...searchColumns.map((c) => row[c] ?? '')],
175
+
)
176
+
}
177
+
}
178
+
179
+
async deleteFromIndex(shadowTable: string, uri: string): Promise<void> {
180
+
// We need the rowid and old column values for the FTS delete command
181
+
const rows = await this.port.query(
182
+
`SELECT rowid, * FROM ${shadowTable} WHERE uri = $1`,
183
+
[uri],
184
+
)
185
+
if (rows.length === 0) return
186
+
187
+
const old = rows[0] as any
188
+
const rowid = old.rowid
189
+
190
+
// Delete from FTS — requires passing old values for token removal
191
+
// Get column names from the shadow table (excluding uri, cid, did, indexed_at, handle metadata cols)
192
+
await this.port.execute(`DELETE FROM ${shadowTable} WHERE uri = $1`, [uri])
193
+
194
+
// Note: with content= table, deleting from the content table is sufficient
195
+
// if we also run the FTS delete command. But since we use content=shadowTable,
196
+
// and we just deleted from shadowTable, we need to tell FTS explicitly.
197
+
// Actually with external content, we must manually sync.
198
+
// The simplest approach: delete from shadow table, then rebuild FTS for this row.
199
+
// But we already deleted... so we use the 'delete' command with old values.
200
+
}
201
+
202
+
async search(
203
+
shadowTable: string,
204
+
query: string,
205
+
_searchColumns: string[],
206
+
limit: number,
207
+
offset: number,
208
+
): Promise<Array<{ uri: string; score: number }>> {
209
+
const escaped = query.replace(/['"*(){}[\]^~\\:]/g, ' ').trim()
210
+
if (!escaped) return []
211
+
212
+
const sql = `SELECT uri, -bm25(${shadowTable}_fts) AS score
213
+
FROM ${shadowTable}_fts
214
+
WHERE ${shadowTable}_fts MATCH $1
215
+
ORDER BY score DESC
216
+
LIMIT $2 OFFSET $3`
217
+
return this.port.query(sql, [escaped, limit, offset])
218
+
}
219
+
}
220
+
```
221
+
222
+
**Wait** — the `updateIndex` and `deleteFromIndex` above are too complex with manual FTS delete commands and rowid tracking. Let me simplify.
223
+
224
+
**Step 1 (revised): Simpler approach using DELETE + INSERT on shadow table with FTS rebuild-row**
225
+
226
+
Actually, the cleanest approach for external content FTS5: since we use `content=shadowTable`, we can:
227
+
1. Upsert the shadow table row
228
+
2. Run `INSERT INTO fts(fts) VALUES('rebuild')` only for the changed row — but FTS5 doesn't support single-row rebuild.
229
+
230
+
The correct pattern for external content tables:
231
+
1. To **delete**: `INSERT INTO fts(fts, rowid, col1, ...) VALUES('delete', old_rowid, old_col1, ...)`
232
+
2. To **insert**: `INSERT INTO fts(rowid, col1, ...) VALUES(new_rowid, new_col1, ...)`
233
+
3. To **update**: delete old + insert new
234
+
235
+
This is fiddly. A simpler alternative: **don't use external content mode**. Instead, use a **regular FTS5 table** (no content= param) and keep it in sync manually. This way:
236
+
- INSERT: just `INSERT INTO fts(uri, cols) VALUES(...)`
237
+
- DELETE: `DELETE FROM fts WHERE uri = ...` — wait, FTS5 doesn't support DELETE by column, only by rowid.
238
+
239
+
OK, the simplest correct approach:
240
+
241
+
**Use the shadow data table as a lookup for old values, and do explicit FTS delete+insert.**
242
+
243
+
Let me write the final clean version:
244
+
245
+
```typescript
246
+
import type { SearchPort } from '../ports.ts'
247
+
import type { DatabasePort } from '../ports.ts'
248
+
249
+
export class SQLiteSearchPort implements SearchPort {
250
+
constructor(private port: DatabasePort) {}
251
+
252
+
async indexExists(shadowTable: string): Promise<boolean> {
253
+
const rows = await this.port.query(
254
+
`SELECT 1 FROM sqlite_master WHERE type='table' AND name=$1`,
255
+
[shadowTable],
256
+
)
257
+
return rows.length > 0
258
+
}
259
+
260
+
async buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void> {
261
+
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}_fts`, [])
262
+
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}`, [])
263
+
264
+
await this.port.execute(`CREATE TABLE ${shadowTable} AS ${sourceQuery}`, [])
265
+
await this.port.execute(`CREATE UNIQUE INDEX IF NOT EXISTS ${shadowTable}_uri ON ${shadowTable}(uri)`, [])
266
+
267
+
const colList = searchColumns.join(', ')
268
+
await this.port.execute(
269
+
`CREATE VIRTUAL TABLE ${shadowTable}_fts USING fts5(uri UNINDEXED, ${colList}, content=${shadowTable}, content_rowid=rowid, tokenize='porter unicode61 remove_diacritics 2')`,
270
+
[],
271
+
)
272
+
273
+
const selectCols = ['uri', ...searchColumns].map((c) => `COALESCE(CAST(${c} AS TEXT), '')`)
274
+
await this.port.execute(
275
+
`INSERT INTO ${shadowTable}_fts (uri, ${colList}) SELECT ${selectCols.join(', ')} FROM ${shadowTable}`,
276
+
[],
277
+
)
278
+
}
279
+
280
+
async updateIndex(
281
+
shadowTable: string,
282
+
uri: string,
283
+
row: Record<string, string | null>,
284
+
searchColumns: string[],
285
+
): Promise<void> {
286
+
const colList = searchColumns.join(', ')
287
+
288
+
// Remove old entry from FTS if it exists
289
+
await this._deleteFromFts(shadowTable, uri, searchColumns)
290
+
291
+
// Upsert shadow table
292
+
const placeholders = searchColumns.map((_, i) => `$${i + 2}`)
293
+
const values = [uri, ...searchColumns.map((c) => row[c] ?? null)]
294
+
await this.port.execute(
295
+
`INSERT OR REPLACE INTO ${shadowTable} (uri, ${colList}) VALUES ($1, ${placeholders.join(', ')})`,
296
+
values,
297
+
)
298
+
299
+
// Insert new FTS entry — read back rowid from shadow table
300
+
const rows = await this.port.query(`SELECT rowid FROM ${shadowTable} WHERE uri = $1`, [uri])
301
+
if (rows.length > 0) {
302
+
const rowid = (rows[0] as any).rowid
303
+
const ftsVals = searchColumns.map((c) => row[c] ?? '')
304
+
const ftsPlaceholders = searchColumns.map((_, i) => `$${i + 3}`)
305
+
await this.port.execute(
306
+
`INSERT INTO ${shadowTable}_fts(rowid, uri, ${colList}) VALUES($1, $2, ${ftsPlaceholders.join(', ')})`,
307
+
[rowid, uri, ...ftsVals],
308
+
)
309
+
}
310
+
}
311
+
312
+
async deleteFromIndex(shadowTable: string, uri: string): Promise<void> {
313
+
const cols = await this._getShadowColumns(shadowTable)
314
+
await this._deleteFromFts(shadowTable, uri, cols)
315
+
await this.port.execute(`DELETE FROM ${shadowTable} WHERE uri = $1`, [uri])
316
+
}
317
+
318
+
private async _deleteFromFts(shadowTable: string, uri: string, searchColumns: string[]): Promise<void> {
319
+
const colList = searchColumns.join(', ')
320
+
const rows = await this.port.query(
321
+
`SELECT rowid, uri, ${colList} FROM ${shadowTable} WHERE uri = $1`,
322
+
[uri],
323
+
)
324
+
if (rows.length === 0) return
325
+
326
+
const old = rows[0] as any
327
+
const oldVals = searchColumns.map((c) => old[c] ?? '')
328
+
const placeholders = searchColumns.map((_, i) => `$${i + 3}`)
329
+
await this.port.execute(
330
+
`INSERT INTO ${shadowTable}_fts(${shadowTable}_fts, rowid, uri, ${colList}) VALUES('delete', $1, $2, ${placeholders.join(', ')})`,
331
+
[old.rowid, uri, ...oldVals],
332
+
)
333
+
}
334
+
335
+
private async _getShadowColumns(shadowTable: string): Promise<string[]> {
336
+
const info = await this.port.query(`PRAGMA table_info("${shadowTable}")`, [])
337
+
return (info as any[])
338
+
.map((r) => r.name)
339
+
.filter((n: string) => !['uri', 'cid', 'did', 'indexed_at', 'handle', 'rowid'].includes(n))
340
+
}
341
+
342
+
async search(
343
+
shadowTable: string,
344
+
query: string,
345
+
_searchColumns: string[],
346
+
limit: number,
347
+
offset: number,
348
+
): Promise<Array<{ uri: string; score: number }>> {
349
+
const escaped = query.replace(/['"*(){}[\]^~\\:]/g, ' ').trim()
350
+
if (!escaped) return []
351
+
352
+
const sql = `SELECT uri, -bm25(${shadowTable}_fts) AS score
353
+
FROM ${shadowTable}_fts
354
+
WHERE ${shadowTable}_fts MATCH $1
355
+
ORDER BY score DESC
356
+
LIMIT $2 OFFSET $3`
357
+
return this.port.query(sql, [escaped, limit, offset])
358
+
}
359
+
}
360
+
```
361
+
362
+
**Step 2: Commit**
363
+
364
+
```bash
365
+
git add packages/hatk/src/database/adapters/sqlite-search.ts
366
+
git commit -m "feat: rewrite SQLiteSearchPort for incremental FTS5 with external content"
367
+
```
368
+
369
+
---
370
+
371
+
### Task 3: Add FTS Row Builder to fts.ts
372
+
373
+
**Files:**
374
+
- Modify: `packages/hatk/src/database/fts.ts`
375
+
376
+
The existing `buildFtsIndex()` constructs a SQL query to denormalize a record's searchable text. We need a similar function that builds the denormalized row for a **single record** (by URI) so `updateIndex` can be called with the right data.
377
+
378
+
**Step 1: Add `buildFtsRow()` function**
379
+
380
+
Add after the `buildFtsIndex` function (after line 192):
381
+
382
+
```typescript
383
+
/**
384
+
* Build a denormalized FTS row for a single record.
385
+
* Returns the search column values keyed by column name, or null if the record doesn't exist.
386
+
*/
387
+
export async function buildFtsRow(
388
+
collection: string,
389
+
uri: string,
390
+
): Promise<Record<string, string | null> | null> {
391
+
const schema = getSchema(collection)
392
+
if (!schema) return null
393
+
394
+
const lexicon = getLexicon(collection)
395
+
const record = lexicon?.defs?.main?.record
396
+
const dialect = getSqlDialect()
397
+
398
+
const selectExprs: string[] = []
399
+
const colNames: string[] = []
400
+
401
+
for (const col of schema.columns) {
402
+
if (col.sqlType === 'TEXT') {
403
+
selectExprs.push(`t.${col.name}`)
404
+
colNames.push(col.name)
405
+
} else if ((col.sqlType === 'JSON' || col.sqlType === 'TEXT') && record?.properties) {
406
+
const prop = record.properties[col.originalName]
407
+
if (prop?.type === 'blob') continue
408
+
if (prop && lexicon) {
409
+
const derived = jsonSearchColumns(`t.${col.name}`, prop, lexicon, dialect)
410
+
if (derived.length > 0) {
411
+
for (const d of derived) {
412
+
selectExprs.push(`${d.expr} AS ${d.alias}`)
413
+
colNames.push(d.alias)
414
+
}
415
+
continue
416
+
}
417
+
}
418
+
selectExprs.push(`CAST(t.${col.name} AS TEXT) AS ${col.name}`)
419
+
colNames.push(col.name)
420
+
}
421
+
}
422
+
423
+
for (const child of schema.children) {
424
+
for (const col of child.columns) {
425
+
if (col.sqlType === 'TEXT') {
426
+
const alias = `${child.fieldName}_${col.name}`
427
+
const agg = dialect.stringAgg(`c.${col.name}`, "' '")
428
+
selectExprs.push(`(SELECT ${agg} FROM ${child.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`)
429
+
colNames.push(alias)
430
+
}
431
+
}
432
+
}
433
+
434
+
for (const union of schema.unions) {
435
+
for (const branch of union.branches) {
436
+
for (const col of branch.columns) {
437
+
if (col.sqlType === 'TEXT') {
438
+
const alias = `${union.fieldName}_${branch.branchName}_${col.name}`
439
+
const agg = dialect.stringAgg(`c.${col.name}`, "' '")
440
+
selectExprs.push(`(SELECT ${agg} FROM ${branch.tableName} c WHERE c.parent_uri = t.uri) AS ${alias}`)
441
+
colNames.push(alias)
442
+
}
443
+
}
444
+
}
445
+
}
446
+
447
+
selectExprs.push('r.handle')
448
+
colNames.push('handle')
449
+
450
+
if (colNames.length === 0) return null
451
+
452
+
const sql = `SELECT ${selectExprs.join(', ')} FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did WHERE t.uri = $1`
453
+
const rows = await runSQL(sql, [uri])
454
+
if (!rows || rows.length === 0) return null
455
+
456
+
const row = rows[0] as Record<string, any>
457
+
const result: Record<string, string | null> = {}
458
+
for (const col of colNames) {
459
+
result[col] = row[col] != null ? String(row[col]) : null
460
+
}
461
+
return result
462
+
}
463
+
```
464
+
465
+
Also export `runSQL` helper if not already exported — check `db.ts` for the `all()` function. Actually, `runSQL` is already exported from `db.ts`. We import it at top of `fts.ts` already: `import { getSchema, runSQL, getSqlDialect } from './db.ts'`. Good.
466
+
467
+
But wait — `runSQL` uses the `all()` helper internally? Let me verify. Looking at the imports, `fts.ts` already imports `runSQL` from `db.ts`. We need to confirm `runSQL` returns rows. Looking at `db.ts`, there's `run()` (execute) and `all()` (query). The import says `runSQL` — let me check if that exists.
468
+
469
+
Actually from the fts.ts imports: `import { getSchema, runSQL, getSqlDialect } from './db.ts'`. And in db.ts there should be an exported `runSQL`. If it doesn't exist as a query function, use the exported `query` or add a wrapper. The implementation should use whatever query function is available from db.ts.
470
+
471
+
**Step 2: Commit**
472
+
473
+
```bash
474
+
git add packages/hatk/src/database/fts.ts
475
+
git commit -m "feat: add buildFtsRow() for single-record FTS denormalization"
476
+
```
477
+
478
+
---
479
+
480
+
### Task 4: Add updateFtsRecord() and deleteFtsRecord() to fts.ts
481
+
482
+
**Files:**
483
+
- Modify: `packages/hatk/src/database/fts.ts`
484
+
485
+
**Step 1: Add incremental update/delete functions**
486
+
487
+
Add after `buildFtsRow()`:
488
+
489
+
```typescript
490
+
/**
491
+
* Incrementally update the FTS index for a single record.
492
+
* Only works when the search port supports incremental updates (SQLite).
493
+
* Falls back to no-op for ports without updateIndex (DuckDB).
494
+
*/
495
+
export async function updateFtsRecord(collection: string, uri: string): Promise<void> {
496
+
if (!searchPort || !searchPort.updateIndex) return
497
+
498
+
const searchCols = searchColumnCache.get(collection)
499
+
if (!searchCols || searchCols.length === 0) return
500
+
501
+
const row = await buildFtsRow(collection, uri)
502
+
if (!row) return
503
+
504
+
const safeName = ftsTableName(collection)
505
+
await searchPort.updateIndex(safeName, uri, row, searchCols)
506
+
}
507
+
508
+
/**
509
+
* Incrementally remove a record from the FTS index.
510
+
* Only works when the search port supports incremental deletes (SQLite).
511
+
*/
512
+
export async function deleteFtsRecord(collection: string, uri: string): Promise<void> {
513
+
if (!searchPort || !searchPort.deleteFromIndex) return
514
+
515
+
const safeName = ftsTableName(collection)
516
+
await searchPort.deleteFromIndex(safeName, uri)
517
+
}
518
+
```
519
+
520
+
**Step 2: Commit**
521
+
522
+
```bash
523
+
git add packages/hatk/src/database/fts.ts
524
+
git commit -m "feat: add updateFtsRecord/deleteFtsRecord for incremental FTS"
525
+
```
526
+
527
+
---
528
+
529
+
### Task 5: Hook Incremental FTS into insertRecord/deleteRecord
530
+
531
+
**Files:**
532
+
- Modify: `packages/hatk/src/database/db.ts:530-657`
533
+
534
+
**Step 1: Add import**
535
+
536
+
At the top of `db.ts`, update the import from `fts.ts`:
537
+
538
+
```typescript
539
+
import { getSearchColumns, stripStopWords, getSearchPort, updateFtsRecord, deleteFtsRecord } from './fts.ts'
540
+
```
541
+
542
+
**Step 2: Call updateFtsRecord at the end of insertRecord()**
543
+
544
+
Add at the end of `insertRecord()` (after the union branch inserts, before the closing `}`):
545
+
546
+
```typescript
547
+
// Incrementally update FTS index for this record
548
+
await updateFtsRecord(collection, uri)
549
+
```
550
+
551
+
**Step 3: Call deleteFtsRecord at the start of deleteRecord()**
552
+
553
+
Add at the beginning of `deleteRecord()`, before deleting child/union rows:
554
+
555
+
```typescript
556
+
// Remove from FTS index before deleting the record data
557
+
await deleteFtsRecord(collection, uri)
558
+
```
559
+
560
+
**Step 4: Commit**
561
+
562
+
```bash
563
+
git add packages/hatk/src/database/db.ts
564
+
git commit -m "feat: hook incremental FTS updates into insertRecord/deleteRecord"
565
+
```
566
+
567
+
---
568
+
569
+
### Task 6: Skip Periodic Rebuild for SQLite in Indexer
570
+
571
+
**Files:**
572
+
- Modify: `packages/hatk/src/indexer.ts:33-34,117-121`
573
+
574
+
**Step 1: Make periodic rebuild conditional on dialect**
575
+
576
+
Import the database port to check dialect:
577
+
578
+
```typescript
579
+
import { getDatabasePort } from './database/db.ts'
580
+
```
581
+
582
+
Replace the rebuild check in `flushBuffer()` (lines 117-121):
583
+
584
+
```typescript
585
+
writesSinceRebuild += batch.length
586
+
if (writesSinceRebuild >= ftsRebuildInterval) {
587
+
writesSinceRebuild = 0
588
+
// Skip periodic full rebuild for SQLite — it uses incremental FTS updates
589
+
const port = getDatabasePort()
590
+
if (port.dialect !== 'sqlite') {
591
+
rebuildAllIndexes([...indexerCollections]).catch(() => {})
592
+
}
593
+
}
594
+
```
595
+
596
+
**Step 2: Commit**
597
+
598
+
```bash
599
+
git add packages/hatk/src/indexer.ts
600
+
git commit -m "feat: skip periodic FTS rebuild for SQLite (now incremental)"
601
+
```
602
+
603
+
---
604
+
605
+
### Task 7: Skip Startup Rebuild When FTS Tables Already Exist
606
+
607
+
**Files:**
608
+
- Modify: `packages/hatk/src/database/fts.ts:114-192`
609
+
610
+
**Step 1: Add early return to buildFtsIndex when tables exist (SQLite only)**
611
+
612
+
At the top of `buildFtsIndex()`, after the `if (!searchPort) return` check:
613
+
614
+
```typescript
615
+
// For SQLite: skip rebuild if FTS tables already exist (incremental mode keeps them in sync)
616
+
if (searchPort.indexExists) {
617
+
const safeName = ftsTableName(collection)
618
+
const exists = await searchPort.indexExists(safeName)
619
+
if (exists) {
620
+
// Still populate the search column cache so incremental updates work
621
+
// ... (need to compute searchColNames without rebuilding)
622
+
}
623
+
}
624
+
```
625
+
626
+
Actually, we need the `searchColumnCache` populated even when skipping rebuild. Refactor `buildFtsIndex` to separate column computation from index creation:
627
+
628
+
```typescript
629
+
export async function buildFtsIndex(collection: string): Promise<void> {
630
+
if (!searchPort) return
631
+
632
+
const { searchColNames, sourceQuery, safeName } = computeFtsSchema(collection)
633
+
if (searchColNames.length === 0) return
634
+
635
+
// For incremental ports: skip rebuild if index already exists
636
+
if (searchPort.indexExists) {
637
+
const exists = await searchPort.indexExists(safeName)
638
+
if (exists) {
639
+
searchColumnCache.set(collection, searchColNames)
640
+
lastRebuiltAt.set(collection, new Date().toISOString())
641
+
return
642
+
}
643
+
}
644
+
645
+
await searchPort.buildIndex(safeName, sourceQuery, searchColNames)
646
+
searchColumnCache.set(collection, searchColNames)
647
+
lastRebuiltAt.set(collection, new Date().toISOString())
648
+
}
649
+
```
650
+
651
+
Extract the column computation into a helper:
652
+
653
+
```typescript
654
+
function computeFtsSchema(collection: string): {
655
+
searchColNames: string[]
656
+
sourceQuery: string
657
+
safeName: string
658
+
} {
659
+
const schema = getSchema(collection)
660
+
if (!schema) throw new Error(`Unknown collection: ${collection}`)
661
+
662
+
const lexicon = getLexicon(collection)
663
+
const record = lexicon?.defs?.main?.record
664
+
const dialect = getSqlDialect()
665
+
const selectExprs: string[] = ['t.uri', 't.cid', 't.did', 't.indexed_at']
666
+
const searchColNames: string[] = []
667
+
668
+
// ... (move existing column computation logic from buildFtsIndex here, lines 128-179)
669
+
670
+
const safeName = ftsTableName(collection)
671
+
const sourceQuery = `SELECT ${selectExprs.join(', ')} FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did`
672
+
673
+
return { searchColNames, sourceQuery, safeName }
674
+
}
675
+
```
676
+
677
+
**Step 2: Commit**
678
+
679
+
```bash
680
+
git add packages/hatk/src/database/fts.ts
681
+
git commit -m "feat: skip FTS rebuild on startup when SQLite index already exists"
682
+
```
683
+
684
+
---
685
+
686
+
### Task 8: Make Backfill Completion Rebuild Conditional
687
+
688
+
**Files:**
689
+
- Modify: `packages/hatk/src/main.ts:168-176`
690
+
691
+
**Step 1: Skip post-backfill rebuild for SQLite**
692
+
693
+
```typescript
694
+
function runBackfillAndRestart() {
695
+
runBackfill(backfillOpts)
696
+
.then(async (recordCount) => {
697
+
// SQLite uses incremental FTS — only rebuild for other engines
698
+
const port = getDatabasePort()
699
+
if (port.dialect !== 'sqlite') {
700
+
log('[main] Backfill complete, rebuilding FTS indexes...')
701
+
await rebuildAllIndexes(collections)
702
+
log('[main] FTS indexes ready')
703
+
} else {
704
+
log('[main] Backfill complete (FTS updated incrementally)')
705
+
}
706
+
return recordCount
707
+
})
708
+
```
709
+
710
+
Add the import at the top of main.ts:
711
+
712
+
```typescript
713
+
import { getDatabasePort } from './database/db.ts'
714
+
```
715
+
716
+
**Step 2: Commit**
717
+
718
+
```bash
719
+
git add packages/hatk/src/main.ts
720
+
git commit -m "feat: skip post-backfill FTS rebuild for SQLite"
721
+
```
722
+
723
+
---
724
+
725
+
### Task 9: Handle Shadow Table Schema for INSERT OR REPLACE
726
+
727
+
**Files:**
728
+
- Modify: `packages/hatk/src/database/adapters/sqlite-search.ts`
729
+
730
+
The shadow table created by `buildIndex` uses `CREATE TABLE AS SELECT`, which doesn't create a PRIMARY KEY or UNIQUE constraint on `uri`. The `INSERT OR REPLACE` in `updateIndex` needs a unique constraint to work.
731
+
732
+
**Step 1: Verify buildIndex creates the unique index**
733
+
734
+
Already handled in Task 2 — the `buildIndex` method includes:
735
+
```typescript
736
+
await this.port.execute(`CREATE UNIQUE INDEX IF NOT EXISTS ${shadowTable}_uri ON ${shadowTable}(uri)`, [])
737
+
```
738
+
739
+
But `INSERT OR REPLACE` requires a UNIQUE constraint on the table itself, not just an index. Fix by using `INSERT ... ON CONFLICT(uri) DO UPDATE` instead:
740
+
741
+
In `updateIndex`, replace the shadow table upsert:
742
+
743
+
```typescript
744
+
// Upsert shadow table
745
+
const setClauses = searchColumns.map((c, i) => `${c} = $${i + 2}`)
746
+
await this.port.execute(
747
+
`INSERT INTO ${shadowTable} (uri, ${colList}) VALUES ($1, ${placeholders.join(', ')})
748
+
ON CONFLICT(uri) DO UPDATE SET ${setClauses.join(', ')}`,
749
+
values,
750
+
)
751
+
```
752
+
753
+
This works with the UNIQUE INDEX on uri.
754
+
755
+
**Step 2: Commit**
756
+
757
+
```bash
758
+
git add packages/hatk/src/database/adapters/sqlite-search.ts
759
+
git commit -m "fix: use ON CONFLICT for shadow table upsert"
760
+
```
761
+
762
+
---
763
+
764
+
### Task 10: Test Manually
765
+
766
+
**Step 1: Build hatk**
767
+
768
+
```bash
769
+
cd packages/hatk && npm run build
770
+
```
771
+
772
+
**Step 2: Run the teal template app locally**
773
+
774
+
```bash
775
+
cd /Users/chadmiller/code/hatk-template-teal
776
+
npm install
777
+
# Start local dev environment and verify:
778
+
# - FTS tables are created on first startup
779
+
# - Records are searchable immediately after insertion (no waiting for rebuild interval)
780
+
# - Restarting the app does NOT rebuild FTS tables
781
+
# - Search results are correct
782
+
```
783
+
784
+
**Step 3: Commit any fixes**
785
+
786
+
```bash
787
+
git commit -m "fix: address issues found during manual testing"
788
+
```
+1
-1
packages/hatk/package.json
+1
-1
packages/hatk/package.json
+77
-10
packages/hatk/src/database/adapters/sqlite-search.ts
+77
-10
packages/hatk/src/database/adapters/sqlite-search.ts
···
2
2
import type { DatabasePort } from '../ports.ts'
3
3
4
4
/**
5
-
* SQLite FTS5-based search port.
5
+
* SQLite FTS5-based search port with incremental updates.
6
6
*
7
-
* Uses SQLite's built-in FTS5 virtual tables for full-text search with BM25 ranking.
8
-
* The shadow table name is reused as the FTS5 virtual table name.
7
+
* Uses external content FTS5 tables (content=shadowTable) so the FTS index
8
+
* references the shadow data table. Updates happen incrementally per-record
9
+
* instead of dropping and rebuilding the entire index.
9
10
*/
10
11
export class SQLiteSearchPort implements SearchPort {
11
12
constructor(private port: DatabasePort) {}
12
13
14
+
async indexExists(shadowTable: string): Promise<boolean> {
15
+
const rows = await this.port.query(
16
+
`SELECT 1 FROM sqlite_master WHERE type='table' AND name IN ($1, $2)`,
17
+
[shadowTable, `${shadowTable}_fts`],
18
+
)
19
+
return rows.length >= 2
20
+
}
21
+
13
22
async buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void> {
14
-
// Drop existing FTS table and data table
15
23
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}_fts`, [])
16
24
await this.port.execute(`DROP TABLE IF EXISTS ${shadowTable}`, [])
17
25
18
-
// Create the data table from the source query
26
+
// Create shadow data table from source query
19
27
await this.port.execute(`CREATE TABLE ${shadowTable} AS ${sourceQuery}`, [])
28
+
await this.port.execute(
29
+
`CREATE UNIQUE INDEX IF NOT EXISTS ${shadowTable}_uri ON ${shadowTable}(uri)`,
30
+
[],
31
+
)
20
32
21
-
// Create the FTS5 virtual table over the search columns
33
+
// Create FTS5 virtual table with external content pointing to shadow table
22
34
const colList = searchColumns.join(', ')
23
35
await this.port.execute(
24
-
`CREATE VIRTUAL TABLE ${shadowTable}_fts USING fts5(uri UNINDEXED, ${colList}, tokenize='porter unicode61 remove_diacritics 2')`,
36
+
`CREATE VIRTUAL TABLE ${shadowTable}_fts USING fts5(uri UNINDEXED, ${colList}, content=${shadowTable}, content_rowid=rowid, tokenize='porter unicode61 remove_diacritics 2')`,
25
37
[],
26
38
)
27
39
28
-
// Populate FTS table from the data table
40
+
// Populate FTS from shadow table
29
41
const selectCols = ['uri', ...searchColumns].map((c) => `COALESCE(CAST(${c} AS TEXT), '')`)
30
42
await this.port.execute(
31
43
`INSERT INTO ${shadowTable}_fts (uri, ${colList}) SELECT ${selectCols.join(', ')} FROM ${shadowTable}`,
···
33
45
)
34
46
}
35
47
48
+
async updateIndex(
49
+
shadowTable: string,
50
+
uri: string,
51
+
row: Record<string, string | null>,
52
+
searchColumns: string[],
53
+
): Promise<void> {
54
+
const colList = searchColumns.join(', ')
55
+
56
+
// Remove old FTS entry if record already indexed
57
+
await this._deleteFromFts(shadowTable, uri, searchColumns)
58
+
59
+
// Upsert shadow table
60
+
const placeholders = searchColumns.map((_, i) => `$${i + 2}`)
61
+
const setClauses = searchColumns.map((c, i) => `${c} = $${i + 2}`)
62
+
const values = [uri, ...searchColumns.map((c) => row[c] ?? null)]
63
+
await this.port.execute(
64
+
`INSERT INTO ${shadowTable} (uri, ${colList}) VALUES ($1, ${placeholders.join(', ')}) ON CONFLICT(uri) DO UPDATE SET ${setClauses.join(', ')}`,
65
+
values,
66
+
)
67
+
68
+
// Read back rowid and insert new FTS entry
69
+
const rows = await this.port.query(`SELECT rowid FROM ${shadowTable} WHERE uri = $1`, [uri])
70
+
if (rows.length > 0) {
71
+
const rowid = (rows[0] as any).rowid
72
+
const ftsPlaceholders = searchColumns.map((_, i) => `$${i + 3}`)
73
+
await this.port.execute(
74
+
`INSERT INTO ${shadowTable}_fts(rowid, uri, ${colList}) VALUES($1, $2, ${ftsPlaceholders.join(', ')})`,
75
+
[rowid, uri, ...searchColumns.map((c) => row[c] ?? '')],
76
+
)
77
+
}
78
+
}
79
+
80
+
async deleteFromIndex(shadowTable: string, uri: string, searchColumns: string[]): Promise<void> {
81
+
await this._deleteFromFts(shadowTable, uri, searchColumns)
82
+
await this.port.execute(`DELETE FROM ${shadowTable} WHERE uri = $1`, [uri])
83
+
}
84
+
85
+
private async _deleteFromFts(
86
+
shadowTable: string,
87
+
uri: string,
88
+
searchColumns: string[],
89
+
): Promise<void> {
90
+
const colList = searchColumns.join(', ')
91
+
const rows = await this.port.query(
92
+
`SELECT rowid, uri, ${colList} FROM ${shadowTable} WHERE uri = $1`,
93
+
[uri],
94
+
)
95
+
if (rows.length === 0) return
96
+
97
+
const old = rows[0] as any
98
+
const placeholders = searchColumns.map((_, i) => `$${i + 3}`)
99
+
await this.port.execute(
100
+
`INSERT INTO ${shadowTable}_fts(${shadowTable}_fts, rowid, uri, ${colList}) VALUES('delete', $1, $2, ${placeholders.join(', ')})`,
101
+
[old.rowid, uri, ...searchColumns.map((c) => old[c] ?? '')],
102
+
)
103
+
}
104
+
36
105
async search(
37
106
shadowTable: string,
38
107
query: string,
···
40
109
limit: number,
41
110
offset: number,
42
111
): Promise<Array<{ uri: string; score: number }>> {
43
-
// Escape FTS5 special characters and build query
44
112
const escaped = query.replace(/['"*(){}[\]^~\\:]/g, ' ').trim()
45
113
if (!escaped) return []
46
114
47
-
// Use FTS5 MATCH with bm25() ranking (lower = better match, negate for DESC)
48
115
const sql = `SELECT uri, -bm25(${shadowTable}_fts) AS score
49
116
FROM ${shadowTable}_fts
50
117
WHERE ${shadowTable}_fts MATCH $1
+8
-1
packages/hatk/src/database/db.ts
+8
-1
packages/hatk/src/database/db.ts
···
1
1
import { type TableSchema, toSnakeCase } from './schema.ts'
2
2
import type { Row } from '../lex-types.ts'
3
-
import { getSearchColumns, stripStopWords, getSearchPort } from './fts.ts'
3
+
import { getSearchColumns, stripStopWords, getSearchPort, updateFtsRecord, deleteFtsRecord } from './fts.ts'
4
4
import { emit, timer } from '../logger.ts'
5
5
import { OAUTH_DDL } from '../oauth/db.ts'
6
6
import type { DatabasePort } from './ports.ts'
···
631
631
await run(`INSERT INTO ${branch.tableName} (${colNames.join(', ')}) VALUES (${placeholders.join(', ')})`, values)
632
632
}
633
633
}
634
+
635
+
// Incrementally update FTS index for this record
636
+
await updateFtsRecord(collection, uri)
634
637
}
635
638
636
639
/** Extract branch data from a union value, handling wrapper properties */
···
645
648
export async function deleteRecord(collection: string, uri: string): Promise<void> {
646
649
const schema = schemas.get(collection)
647
650
if (!schema) return
651
+
652
+
// Remove from FTS index before deleting the record data
653
+
await deleteFtsRecord(collection, uri)
654
+
648
655
for (const child of schema.children) {
649
656
await run(`DELETE FROM ${child.tableName} WHERE parent_uri = $1`, [uri])
650
657
}
+85
-12
packages/hatk/src/database/fts.ts
+85
-12
packages/hatk/src/database/fts.ts
···
1
-
import { getSchema, runSQL, getSqlDialect } from './db.ts'
1
+
import { getSchema, runSQL, getSqlDialect, querySQL } from './db.ts'
2
2
import { getLexicon } from './schema.ts'
3
3
import { emit, timer } from '../logger.ts'
4
4
import type { SearchPort } from './ports.ts'
···
90
90
// Cache of search column metadata per collection, populated during buildFtsIndex
91
91
const searchColumnCache = new Map<string, string[]>()
92
92
93
+
// Cache of computed FTS schemas per collection (deterministic, so compute once)
94
+
const ftsSchemaCache = new Map<string, { searchColNames: string[]; sourceQuery: string; safeName: string }>()
95
+
93
96
export function getSearchColumns(collection: string): string[] {
94
97
return searchColumnCache.get(collection) || []
95
98
}
···
107
110
}
108
111
109
112
/**
110
-
* Build FTS index for a collection.
111
-
* Creates a shadow table copy and indexes all TEXT NOT NULL columns
112
-
* using Porter stemmer with English stopwords.
113
+
* Compute the FTS schema for a collection: search column names, source query, and safe table name.
113
114
*/
114
-
export async function buildFtsIndex(collection: string): Promise<void> {
115
-
if (!searchPort) return // No FTS support for this adapter
116
-
115
+
function computeFtsSchema(collection: string): { searchColNames: string[]; sourceQuery: string; safeName: string } {
116
+
const cached = ftsSchemaCache.get(collection)
117
+
if (cached) return cached
117
118
const schema = getSchema(collection)
118
119
if (!schema) throw new Error(`Unknown collection: ${collection}`)
119
120
···
178
179
selectExprs.push('r.handle')
179
180
searchColNames.push('handle')
180
181
181
-
if (searchColNames.length === 0) {
182
-
return
183
-
}
184
-
185
182
const safeName = ftsTableName(collection)
186
183
const sourceQuery = `SELECT ${selectExprs.join(', ')} FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did`
187
184
188
-
await searchPort.buildIndex(safeName, sourceQuery, searchColNames)
185
+
const result = { searchColNames, sourceQuery, safeName }
186
+
ftsSchemaCache.set(collection, result)
187
+
return result
188
+
}
189
+
190
+
/**
191
+
* Build FTS index for a collection.
192
+
* Creates a shadow table copy and indexes all TEXT NOT NULL columns
193
+
* using Porter stemmer with English stopwords.
194
+
*/
195
+
export async function buildFtsIndex(collection: string): Promise<void> {
196
+
if (!searchPort) return // No FTS support for this adapter
189
197
198
+
const { searchColNames, sourceQuery, safeName } = computeFtsSchema(collection)
199
+
if (searchColNames.length === 0) return
200
+
201
+
// For incremental ports: skip rebuild if index already exists
202
+
if (searchPort.indexExists) {
203
+
const exists = await searchPort.indexExists(safeName)
204
+
if (exists) {
205
+
searchColumnCache.set(collection, searchColNames)
206
+
lastRebuiltAt.set(collection, new Date().toISOString())
207
+
return
208
+
}
209
+
}
210
+
211
+
await searchPort.buildIndex(safeName, sourceQuery, searchColNames)
190
212
searchColumnCache.set(collection, searchColNames)
191
213
lastRebuiltAt.set(collection, new Date().toISOString())
214
+
}
215
+
216
+
export async function buildFtsRow(
217
+
collection: string,
218
+
uri: string,
219
+
): Promise<Record<string, string | null> | null> {
220
+
const { searchColNames, sourceQuery } = computeFtsSchema(collection)
221
+
if (searchColNames.length === 0) return null
222
+
223
+
// Append WHERE clause to filter for single record
224
+
const sql = sourceQuery + ' WHERE t.uri = $1'
225
+
const rows = await querySQL(sql, [uri])
226
+
if (!rows || rows.length === 0) return null
227
+
228
+
const row = rows[0] as Record<string, any>
229
+
const result: Record<string, string | null> = {}
230
+
for (const col of searchColNames) {
231
+
result[col] = row[col] != null ? String(row[col]) : null
232
+
}
233
+
return result
234
+
}
235
+
236
+
export async function updateFtsRecord(collection: string, uri: string): Promise<void> {
237
+
if (!searchPort || !searchPort.updateIndex) return
238
+
239
+
const searchCols = searchColumnCache.get(collection)
240
+
if (!searchCols || searchCols.length === 0) return
241
+
242
+
try {
243
+
const row = await buildFtsRow(collection, uri)
244
+
if (!row) return
245
+
246
+
const safeName = ftsTableName(collection)
247
+
await searchPort.updateIndex(safeName, uri, row, searchCols)
248
+
} catch (err) {
249
+
emit('fts', 'update_error', { collection, uri, error: (err as Error).message })
250
+
}
251
+
}
252
+
253
+
export async function deleteFtsRecord(collection: string, uri: string): Promise<void> {
254
+
if (!searchPort || !searchPort.deleteFromIndex) return
255
+
256
+
const searchCols = searchColumnCache.get(collection)
257
+
if (!searchCols || searchCols.length === 0) return
258
+
259
+
try {
260
+
const safeName = ftsTableName(collection)
261
+
await searchPort.deleteFromIndex(safeName, uri, searchCols)
262
+
} catch (err) {
263
+
emit('fts', 'delete_error', { collection, uri, error: (err as Error).message })
264
+
}
192
265
}
193
266
194
267
/**
+9
packages/hatk/src/database/ports.ts
+9
packages/hatk/src/database/ports.ts
···
51
51
/** Build/rebuild an FTS index for a table */
52
52
buildIndex(shadowTable: string, sourceQuery: string, searchColumns: string[]): Promise<void>
53
53
54
+
/** Incrementally update a single record in the FTS index */
55
+
updateIndex?(shadowTable: string, uri: string, row: Record<string, string | null>, searchColumns: string[]): Promise<void>
56
+
57
+
/** Remove a single record from the FTS index */
58
+
deleteFromIndex?(shadowTable: string, uri: string, searchColumns: string[]): Promise<void>
59
+
60
+
/** Check if the FTS index already exists (for skipping rebuild on startup) */
61
+
indexExists?(shadowTable: string): Promise<boolean>
62
+
54
63
/** Search a table, returning URIs with scores */
55
64
search(
56
65
shadowTable: string,
+6
-1
packages/hatk/src/indexer.ts
+6
-1
packages/hatk/src/indexer.ts
···
7
7
setRepoStatus,
8
8
getRepoRetryInfo,
9
9
listAllRepoStatuses,
10
+
getDatabasePort,
10
11
} from './database/db.ts'
11
12
import { backfillRepo } from './backfill.ts'
12
13
import { rebuildAllIndexes } from './database/fts.ts'
···
117
118
writesSinceRebuild += batch.length
118
119
if (writesSinceRebuild >= ftsRebuildInterval) {
119
120
writesSinceRebuild = 0
120
-
rebuildAllIndexes([...indexerCollections]).catch(() => {})
121
+
// Skip periodic full rebuild for SQLite — it uses incremental FTS updates
122
+
const port = getDatabasePort()
123
+
if (port.dialect !== 'sqlite') {
124
+
rebuildAllIndexes([...indexerCollections]).catch(() => {})
125
+
}
121
126
}
122
127
}
123
128
+11
-5
packages/hatk/src/main.ts
+11
-5
packages/hatk/src/main.ts
···
8
8
import { loadConfig } from './config.ts'
9
9
import { loadLexicons, storeLexicons, discoverCollections, buildSchemas } from './database/schema.ts'
10
10
import { discoverViews } from './views.ts'
11
-
import { initDatabase, getCursor, querySQL, getSqlDialect, getSchemaDump, migrateSchema } from './database/db.ts'
11
+
import { initDatabase, getCursor, querySQL, getSqlDialect, getSchemaDump, migrateSchema, getDatabasePort } from './database/db.ts'
12
12
import { createAdapter } from './database/adapter-factory.ts'
13
13
import { getDialect } from './database/dialect.ts'
14
14
import { setSearchPort } from './database/fts.ts'
···
167
167
168
168
function runBackfillAndRestart() {
169
169
runBackfill(backfillOpts)
170
-
.then((recordCount) => {
171
-
log('[main] Backfill complete, rebuilding FTS indexes...')
172
-
return rebuildAllIndexes(collections).then(() => recordCount)
170
+
.then(async (recordCount) => {
171
+
const port = getDatabasePort()
172
+
if (port.dialect !== 'sqlite') {
173
+
log('[main] Backfill complete, rebuilding FTS indexes...')
174
+
await rebuildAllIndexes(collections)
175
+
log('[main] FTS indexes ready')
176
+
} else {
177
+
log('[main] Backfill complete (FTS updated incrementally)')
178
+
}
179
+
return recordCount
173
180
})
174
181
.then((recordCount) => {
175
-
log('[main] FTS indexes ready')
176
182
if (recordCount > 0 && !process.env.DEV_MODE) {
177
183
logMemory('after-backfill')
178
184
log('[main] Restarting to reclaim memory...')