forked from
jollywhoppers.com/witchsky.app
Bluesky app fork with some witchin' additions 💫
Reduce the odds of false positives in the language suggester (close #2598) (#2700)
authored by
Paul Frazee
and committed by
GitHub
31bf286f
42fe0e16
+23
-10
+23
-9
src/view/com/composer/select-language/SuggestedLanguage.tsx
+23
-9
src/view/com/composer/select-language/SuggestedLanguage.tsx
···
23
23
const cancelIdle = globalThis.cancelIdleCallback || clearTimeout
24
24
25
25
export function SuggestedLanguage({text}: {text: string}) {
26
-
const [suggestedLanguage, setSuggestedLanguage] = useState<string>()
26
+
const [suggestedLanguage, setSuggestedLanguage] = useState<
27
+
string | undefined
28
+
>()
27
29
const langPrefs = useLanguagePrefs()
28
30
const setLangPrefs = useLanguagePrefsApi()
29
31
const pal = usePalette('default')
···
40
42
}
41
43
42
44
const idle = onIdle(() => {
43
-
// Only select languages that have a high confidence and convert to code2
44
-
const result = lande(textTrimmed).filter(
45
-
([lang, value]) => value >= 0.97 && code3ToCode2Strict(lang),
46
-
)
47
-
48
-
setSuggestedLanguage(
49
-
result.length > 0 ? code3ToCode2Strict(result[0][0]) : undefined,
50
-
)
45
+
setSuggestedLanguage(guessLanguage(textTrimmed))
51
46
})
52
47
53
48
return () => cancelIdle(idle)
···
99
94
marginBottom: 10,
100
95
},
101
96
})
97
+
98
+
/**
99
+
* This function is using the lande language model to attempt to detect the language
100
+
* We want to only make suggestions when we feel a high degree of certainty
101
+
* The magic numbers are based on debugging sessions against some test strings
102
+
*/
103
+
function guessLanguage(text: string): string | undefined {
104
+
const scores = lande(text).filter(([_lang, value]) => value >= 0.0002)
105
+
// if the model has multiple items with a score higher than 0.0002, it isn't certain enough
106
+
if (scores.length !== 1) {
107
+
return undefined
108
+
}
109
+
const [lang, value] = scores[0]
110
+
// if the model doesn't give a score of 0.97 or above, it isn't certain enough
111
+
if (value < 0.97) {
112
+
return undefined
113
+
}
114
+
return code3ToCode2Strict(lang)
115
+
}