AppView in a box as a Vite plugin thing
hatk.dev
perf: remove exact match and recent rows search phases
BM25 FTS5 already indexes child table text (artist names) and with
incremental FTS the index is always current. The LIKE phases were
scanning 1.7M rows for 12-18s finding nothing.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+1
-91
+1
-91
packages/hatk/src/database/db.ts
+1
-91
packages/hatk/src/database/db.ts
···
1220
1220
const hasMore = bm25Results.length > limit
1221
1221
if (hasMore) bm25Results.pop()
1222
1222
1223
-
// Phase 2: Exact substring match — boosts phrase matches above BM25 results
1224
-
const exactMatchResults: any[] = []
1225
-
const bm25Uris = new Set(bm25Results.map((r: any) => r.uri))
1226
-
try {
1227
-
const searchParam = `%${query}%`
1228
-
let paramIdx = 1
1229
-
const ilikeConds: string[] = []
1230
-
const params: any[] = []
1231
-
1232
-
// TEXT columns — direct ILIKE/LIKE
1233
-
for (const c of textCols) {
1234
-
ilikeConds.push(`t.${c.name} ${dialect.ilike} $${paramIdx++}`)
1235
-
params.push(searchParam)
1236
-
}
1237
-
1238
-
// JSON columns — cast to text then ILIKE/LIKE
1239
-
const jsonCols = schema.columns.filter((c) => c.sqlType === 'JSON' || c.sqlType === 'TEXT')
1240
-
for (const c of jsonCols) {
1241
-
if (textCols.some((tc) => tc.name === c.name)) continue // skip already-added TEXT cols
1242
-
ilikeConds.push(`CAST(t.${c.name} AS TEXT) ${dialect.ilike} $${paramIdx++}`)
1243
-
params.push(searchParam)
1244
-
}
1245
-
1246
-
// Handle from _repos table
1247
-
ilikeConds.push(`r.handle ${dialect.ilike} $${paramIdx++}`)
1248
-
params.push(searchParam)
1249
-
1250
-
if (ilikeConds.length > 0) {
1251
-
const exactSQL = `SELECT t.* FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did
1252
-
WHERE (${ilikeConds.join(' OR ')})
1253
-
ORDER BY t.indexed_at DESC
1254
-
LIMIT $${paramIdx++}`
1255
-
params.push(limit)
1256
-
1257
-
const rows = await all(exactSQL, params)
1258
-
phasesUsed.push('exact')
1259
-
for (const row of rows) {
1260
-
if (!bm25Uris.has(row.uri)) {
1261
-
exactMatchResults.push(row)
1262
-
bm25Uris.add(row.uri)
1263
-
}
1264
-
}
1265
-
}
1266
-
} catch (err: any) {
1267
-
phaseErrors.push(`exact: ${err.message}`)
1268
-
}
1269
-
1270
-
// Merge: exact matches first, then BM25 results, capped at limit
1271
-
const mergedResults = [...exactMatchResults, ...bm25Results].slice(0, limit + (hasMore ? 1 : 0))
1272
-
// Replace bm25Results with merged for downstream phases
1273
-
bm25Results = mergedResults
1274
-
1275
-
// Phase 3: ILIKE scan of rows written since last FTS rebuild (immediate searchability)
1276
1223
const existingUris = new Set(bm25Results.map((r: any) => r.uri))
1277
1224
1278
-
const { getLastRebuiltAt } = await import('./fts.ts')
1279
-
const rebuiltAt = getLastRebuiltAt(collection)
1280
-
let recentCount = 0
1281
-
1282
-
if (rebuiltAt && bm25Results.length < limit) {
1283
-
const remaining = limit - bm25Results.length
1284
-
const searchParam = `%${query}%`
1285
-
let paramIdx = 1
1286
-
const ilikeParts = textCols.map((c) => `t.${c.name} ${dialect.ilike} $${paramIdx++}`)
1287
-
ilikeParts.push(`r.handle ${dialect.ilike} $${paramIdx++}`)
1288
-
const ilikeConds = ilikeParts.join(' OR ')
1289
-
const params: any[] = [...textCols.map(() => searchParam), searchParam]
1290
-
1291
-
const recentSQL = `SELECT t.* FROM ${schema.tableName} t LEFT JOIN _repos r ON t.did = r.did
1292
-
WHERE t.indexed_at >= $${paramIdx++} AND t.uri NOT IN (SELECT uri FROM ${safeName}) AND (${ilikeConds})
1293
-
ORDER BY t.indexed_at DESC
1294
-
LIMIT $${paramIdx++}`
1295
-
params.push(rebuiltAt, remaining + existingUris.size)
1296
-
1297
-
try {
1298
-
const recentRows = await all(recentSQL, params)
1299
-
phasesUsed.push('recent')
1300
-
for (const row of recentRows) {
1301
-
if (bm25Results.length >= limit) break
1302
-
if (!existingUris.has(row.uri)) {
1303
-
existingUris.add(row.uri)
1304
-
bm25Results.push(row)
1305
-
recentCount++
1306
-
}
1307
-
}
1308
-
} catch (err: any) {
1309
-
phaseErrors.push(`recent: ${err.message}`)
1310
-
}
1311
-
}
1312
-
1313
-
// Phase 4: Fuzzy fallback for typo tolerance (if still under limit)
1225
+
// Phase 2: Fuzzy fallback for typo tolerance (if still under limit)
1314
1226
// Only available on dialects with jaro_winkler_similarity (DuckDB)
1315
1227
let fuzzyCount = 0
1316
1228
if (fuzzy && dialect.jaroWinklerSimilarity && bm25Results.length < limit) {
···
1362
1274
collection,
1363
1275
query,
1364
1276
bm25_count: bm25Count > limit ? bm25Count - 1 : bm25Count,
1365
-
exact_count: exactMatchResults.length,
1366
-
recent_count: recentCount,
1367
1277
fuzzy_count: fuzzyCount,
1368
1278
total_results: records.length,
1369
1279
duration_ms: elapsed(),