deer social fork for personal usage. but you might see a use idk. github mirror
better language detection, inspired by https://github.com/bluesky-social/social-app/pull/8156
+26
-12
+3
src/locale/helpers.ts
+3
src/locale/helpers.ts
+23
-12
src/view/com/composer/select-language/SuggestedLanguage.tsx
+23
-12
src/view/com/composer/select-language/SuggestedLanguage.tsx
···
5
5
import {useLingui} from '@lingui/react'
6
6
import lande from 'lande'
7
7
8
-
import {code3ToCode2Strict, codeToLanguageName} from '#/locale/helpers'
8
+
import {
9
+
code2ToCode3,
10
+
code3ToCode2Strict,
11
+
codeToLanguageName,
12
+
koreanRegex,
13
+
} from '#/locale/helpers'
9
14
import {
10
15
toPostLanguages,
11
16
useLanguagePrefs,
···
33
38
>(text.length === 0 ? replyToLanguage : undefined)
34
39
const langPrefs = useLanguagePrefs()
35
40
const setLangPrefs = useLanguagePrefsApi()
41
+
const languageHistory = langPrefs.postLanguageHistory.map(language =>
42
+
code2ToCode3(language),
43
+
)
36
44
const t = useTheme()
37
45
const {_} = useLingui()
38
46
···
48
56
49
57
// Don't run the language model on small posts, the results are likely
50
58
// to be inaccurate anyway.
51
-
if (textTrimmed.length < 40) {
59
+
if (textTrimmed.length < 20) {
52
60
setSuggestedLanguage(undefined)
53
61
return
54
62
}
55
63
56
64
const idle = onIdle(() => {
57
-
setSuggestedLanguage(guessLanguage(textTrimmed))
65
+
setSuggestedLanguage(guessLanguage(textTrimmed, languageHistory))
58
66
})
59
67
60
68
return () => cancelIdle(idle)
61
-
}, [text, replyToLanguage])
69
+
}, [text, replyToLanguage, languageHistory])
62
70
63
71
if (
64
72
suggestedLanguage &&
···
114
122
* We want to only make suggestions when we feel a high degree of certainty
115
123
* The magic numbers are based on debugging sessions against some test strings
116
124
*/
117
-
function guessLanguage(text: string): string | undefined {
118
-
const scores = lande(text).filter(([_lang, value]) => value >= 0.0002)
119
-
// if the model has multiple items with a score higher than 0.0002, it isn't certain enough
125
+
function guessLanguage(
126
+
text: string,
127
+
languageHistory: string[],
128
+
): string | undefined {
129
+
let scores = lande(text).filter(
130
+
([lang, value]) =>
131
+
(value >= 0.9 || (languageHistory.includes(lang) && value >= 0.6)) &&
132
+
!(lang === 'kor' && !koreanRegex.test(text)),
133
+
)
134
+
120
135
if (scores.length !== 1) {
121
136
return undefined
122
137
}
123
-
const [lang, value] = scores[0]
124
-
// if the model doesn't give a score of 0.97 or above, it isn't certain enough
125
-
if (value < 0.97) {
126
-
return undefined
127
-
}
138
+
const [lang, _] = scores[0]
128
139
return code3ToCode2Strict(lang)
129
140
}
130
141