+52

src/lib/__tests__/normalize-text.test.ts

reviewed

··· 1 1 + import { describe, expect, it } from "vitest"; 2 2 + import { stripDiacritics } from "../normalize-text"; 3 3 + 4 4 + describe("stripDiacritics", () => { 5 5 + it("strips umlauts", () => { 6 6 + expect(stripDiacritics("Jötun")).toBe("Jotun"); 7 7 + expect(stripDiacritics("ö")).toBe("o"); 8 8 + expect(stripDiacritics("ü")).toBe("u"); 9 9 + expect(stripDiacritics("ä")).toBe("a"); 10 10 + }); 11 11 + 12 12 + it("strips circumflexes", () => { 13 13 + expect(stripDiacritics("Lim-Dûl")).toBe("Lim-Dul"); 14 14 + expect(stripDiacritics("û")).toBe("u"); 15 15 + expect(stripDiacritics("ê")).toBe("e"); 16 16 + expect(stripDiacritics("â")).toBe("a"); 17 17 + }); 18 18 + 19 19 + it("strips other Latin diacritics", () => { 20 20 + expect(stripDiacritics("Dandân")).toBe("Dandan"); 21 21 + expect(stripDiacritics("café")).toBe("cafe"); 22 22 + expect(stripDiacritics("naïve")).toBe("naive"); 23 23 + expect(stripDiacritics("señor")).toBe("senor"); 24 24 + }); 25 25 + 26 26 + it("preserves non-diacritic characters", () => { 27 27 + expect(stripDiacritics("Lightning Bolt")).toBe("Lightning Bolt"); 28 28 + expect(stripDiacritics("Fire & Ice")).toBe("Fire & Ice"); 29 29 + expect(stripDiacritics("123")).toBe("123"); 30 30 + }); 31 31 + 32 32 + it("does not affect CJK characters", () => { 33 33 + expect(stripDiacritics("稲妻")).toBe("稲妻"); 34 34 + expect(stripDiacritics("日本語")).toBe("日本語"); 35 35 + }); 36 36 + 37 37 + it("does not affect Cyrillic", () => { 38 38 + expect(stripDiacritics("Москва")).toBe("Москва"); 39 39 + }); 40 40 + 41 41 + it("does not affect Arabic", () => { 42 42 + expect(stripDiacritics("العربية")).toBe("العربية"); 43 43 + }); 44 44 + 45 45 + it("handles empty string", () => { 46 46 + expect(stripDiacritics("")).toBe(""); 47 47 + }); 48 48 + 49 49 + it("handles mixed content", () => { 50 50 + expect(stripDiacritics("Jötun's Wrath")).toBe("Jotun's Wrath"); 51 51 + }); 52 52 + });

+20

src/lib/normalize-text.ts

reviewed

··· 1 1 + /** 2 2 + * Text normalization for search 3 3 + * 4 4 + * Strips diacritical marks from text for search matching (ö→o, û→u, etc.) 5 5 + * 6 6 + * IMPORTANT: This uses the Unicode "Combining Diacritical Marks" block 7 7 + * (U+0300-U+036F) which specifically targets Latin script diacritics. 8 8 + * Other scripts (Arabic, Hebrew, CJK, Cyrillic) are unaffected. 9 9 + * 10 10 + * This is appropriate for English MTG card names which may contain borrowed 11 11 + * diacritics (Jötun, Lim-Dûl, Dandân). If we ever index non-English printings 12 12 + * (using `printed_name`), we'd need to reconsider script handling. 13 13 + */ 14 14 + 15 15 + /** 16 16 + * Strip diacritical marks from text for search normalization 17 17 + */ 18 18 + export function stripDiacritics(text: string): string { 19 19 + return text.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); 20 20 + }

+36

src/lib/search/__tests__/integration.test.ts

reviewed

··· 56 56 expect(result.value.match(bolt)).toBe(true); 57 57 } 58 58 }); 59 59 + 60 60 + it("matches names with diacritics using ASCII equivalents", async () => { 61 61 + const nazgul = await cards.get("Nazgûl"); 62 62 + 63 63 + // Should match without the diacritic 64 64 + const withoutDiacritic = search("nazgul"); 65 65 + expect(withoutDiacritic.ok).toBe(true); 66 66 + if (withoutDiacritic.ok) { 67 67 + expect(withoutDiacritic.value.match(nazgul)).toBe(true); 68 68 + } 69 69 + 70 70 + // Should also match with the diacritic 71 71 + const withDiacritic = search("nazgûl"); 72 72 + expect(withDiacritic.ok).toBe(true); 73 73 + if (withDiacritic.ok) { 74 74 + expect(withDiacritic.value.match(nazgul)).toBe(true); 75 75 + } 76 76 + }); 77 77 + 78 78 + it("exact name match works with diacritics", async () => { 79 79 + const nazgul = await cards.get("Nazgûl"); 80 80 + 81 81 + // ASCII equivalent should match exactly 82 82 + const ascii = search('!"Nazgul"'); 83 83 + expect(ascii.ok).toBe(true); 84 84 + if (ascii.ok) { 85 85 + expect(ascii.value.match(nazgul)).toBe(true); 86 86 + } 87 87 + 88 88 + // With diacritic should also match 89 89 + const diacritic = search('!"Nazgûl"'); 90 90 + expect(diacritic.ok).toBe(true); 91 91 + if (diacritic.ok) { 92 92 + expect(diacritic.value.match(nazgul)).toBe(true); 93 93 + } 94 94 + }); 59 95 }); 60 96 61 97 describe("type matching", () => {

+15 -7

src/lib/search/matcher.ts

reviewed

··· 4 4 * Compiles a SearchNode AST into a function that tests cards. 5 5 */ 6 6 7 7 + import { stripDiacritics } from "../normalize-text"; 7 8 import { type CardPredicate, compileField } from "./fields"; 8 9 import type { CompileError, Result, SearchNode } from "./types"; 9 10 import { ok } from "./types"; ··· 82 83 } 83 84 84 85 /** 86 86 + * Normalize text for name comparison (lowercase + strip diacritics) 87 87 + */ 88 88 + function normalizeName(text: string): string { 89 89 + return stripDiacritics(text).toLowerCase(); 90 90 + } 91 91 + 92 92 + /** 85 93 * Compile name search - substring or regex match 86 94 */ 87 95 function compileName(value: string, pattern: RegExp | null): CardPredicate { ··· 101 109 }; 102 110 } 103 111 104 104 - // Substring match (case-insensitive) 105 105 - const lower = value.toLowerCase(); 112 112 + // Substring match (case-insensitive, diacritic-insensitive) 113 113 + const normalized = normalizeName(value); 106 114 return (card) => { 107 115 // Match against main name 108 108 - if (card.name.toLowerCase().includes(lower)) return true; 116 116 + if (normalizeName(card.name).includes(normalized)) return true; 109 117 110 118 // Match against card face names for multi-face cards 111 119 if (card.card_faces) { 112 120 for (const face of card.card_faces) { 113 113 - if (face.name.toLowerCase().includes(lower)) return true; 121 121 + if (normalizeName(face.name).includes(normalized)) return true; 114 122 } 115 123 } 116 124 ··· 122 130 * Compile exact name match 123 131 */ 124 132 function compileExactName(value: string): CardPredicate { 125 125 - const lower = value.toLowerCase(); 133 133 + const normalized = normalizeName(value); 126 134 return (card) => { 127 135 // Match against main name exactly 128 128 - if (card.name.toLowerCase() === lower) return true; 136 136 + if (normalizeName(card.name) === normalized) return true; 129 137 130 138 // Match against card face names for multi-face cards 131 139 if (card.card_faces) { 132 140 for (const face of card.card_faces) { 133 133 - if (face.name.toLowerCase() === lower) return true; 141 141 + if (normalizeName(face.name) === normalized) return true; 134 142 } 135 143 } 136 144

+15

src/workers/cards.worker.ts

reviewed

··· 9 9 import MiniSearch from "minisearch"; 10 10 import { CARD_CHUNKS, CARD_INDEXES, CARD_VOLATILE } from "../lib/card-manifest"; 11 11 import { LRUCache } from "../lib/lru-cache"; 12 12 + import { stripDiacritics } from "../lib/normalize-text"; 12 13 import type { 13 14 CardDataOutput, 14 15 ManaColor, ··· 304 305 305 306 // Build fuzzy search index 306 307 console.log("[CardsWorker] Building search index..."); 308 308 + 309 309 + // MiniSearch's default tokenizer splits on /[\n\r\p{Z}\p{P}]+/u (Unicode 310 310 + // separators and punctuation), which strips "&" from card names like 311 311 + // "Minsc & Boo". We use a negated version of the same pattern but carve out 312 312 + // "&" so processTerm can normalize it to "and". This is uglier than a 313 313 + // positive match like /[\p{L}\p{N}\p{M}]+|&/gu but correct by construction 314 314 + // (guaranteed to match exactly what the default tokenizer would, plus &). 315 315 + const SEARCH_TOKEN = /&|[^\n\r\p{Z}\p{P}]+/gu; 316 316 + 307 317 this.searchIndex = new MiniSearch<Card>({ 308 318 fields: ["name"], 309 319 storeFields: ["id", "oracle_id", "name"], 320 320 + tokenize: (text) => text.match(SEARCH_TOKEN) ?? [], 321 321 + processTerm: (term) => { 322 322 + if (term === "&") return "and"; 323 323 + return stripDiacritics(term).toLowerCase(); 324 324 + }, 310 325 searchOptions: { 311 326 prefix: true, // "bol" matches "bolt" 312 327 fuzzy: 0.3, // ~2 char tolerance