search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
fix: match FTS5 tokenizer for all non-alphanumeric separators
buildFtsQuery only treated spaces and dots as word separators,
but FTS5's unicode61 tokenizer treats ALL non-alphanumeric as
separators. queries like "crypto-casino" returned 0 results.
now matches FTS5 behavior: any non-alphanumeric is a separator.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+19
-5
+19
-5
backend/src/search.zig
+19
-5
backend/src/search.zig
···
439
439
}
440
440
441
441
// count words and total length
442
+
// match FTS5 unicode61 tokenizer: non-alphanumeric = separator
442
443
var word_count: usize = 0;
443
444
var total_word_len: usize = 0;
444
445
var in_word = false;
445
446
for (trimmed) |c| {
446
-
const is_sep = (c == ' ' or c == '.');
447
-
if (is_sep) {
447
+
const is_alnum = (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9');
448
+
if (!is_alnum) {
448
449
in_word = false;
449
450
} else {
450
451
if (!in_word) word_count += 1;
···
460
461
const buf = try alloc.alloc(u8, total_word_len + 1);
461
462
var pos: usize = 0;
462
463
for (trimmed) |c| {
463
-
if (c != ' ' and c != '.') {
464
+
const is_alnum = (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9');
465
+
if (is_alnum) {
464
466
buf[pos] = c;
465
467
pos += 1;
466
468
}
···
479
481
in_word = false;
480
482
481
483
for (trimmed) |c| {
482
-
const is_sep = (c == ' ' or c == '.');
483
-
if (is_sep) {
484
+
const is_alnum = (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9');
485
+
if (!is_alnum) {
484
486
if (in_word) {
485
487
// end of word - add " OR " if not last
486
488
current_word += 1;
···
547
549
defer std.testing.allocator.free(result);
548
550
try std.testing.expectEqualStrings("foo OR bar*", result);
549
551
}
552
+
553
+
test "buildFtsQuery: hyphens as separators" {
554
+
const result = try buildFtsQuery(std.testing.allocator, "crypto-casino");
555
+
defer std.testing.allocator.free(result);
556
+
try std.testing.expectEqualStrings("crypto OR casino*", result);
557
+
}
558
+
559
+
test "buildFtsQuery: mixed punctuation" {
560
+
const result = try buildFtsQuery(std.testing.allocator, "don't@stop_now");
561
+
defer std.testing.allocator.free(result);
562
+
try std.testing.expectEqualStrings("don OR t OR stop OR now*", result);
563
+
}