this repo has no description
feat: add language support
+53
-2
+1
-1
packages/remanso-cli/src/lib/config.ts
+1
-1
packages/remanso-cli/src/lib/config.ts
+33
packages/remanso-cli/src/lib/detect-languages.test.ts
+33
packages/remanso-cli/src/lib/detect-languages.test.ts
···
1
+
import { describe, expect, test } from "bun:test";
2
+
import { detectLanguage } from "./detect-languages";
3
+
4
+
describe("Detect languages", () => {
5
+
test("Detect English", () => {
6
+
const lang = detectLanguage("Hello, World!");
7
+
expect(lang).toBe("eng");
8
+
});
9
+
10
+
test("Detect French", () => {
11
+
const lang = detectLanguage("Bonjour le monde !");
12
+
expect(lang).toBe("fra");
13
+
});
14
+
15
+
test("Detect Portuguese", () => {
16
+
const lang = detectLanguage("Oi, gente! Tudo bom?");
17
+
expect(lang).toBe("por");
18
+
});
19
+
20
+
test("Detect the most used langage, here French", () => {
21
+
const lang = detectLanguage("Hello World! J'espère que vous allez bien.");
22
+
23
+
expect(lang).toBe("fra");
24
+
});
25
+
26
+
test("Detect the most used langage, here English", () => {
27
+
const lang = detectLanguage(
28
+
"Bonjour le monde! I hope, you're doing well in this incredible day!",
29
+
);
30
+
31
+
expect(lang).toBe("eng");
32
+
});
33
+
});
+15
packages/remanso-cli/src/lib/detect-languages.ts
+15
packages/remanso-cli/src/lib/detect-languages.ts
···
1
+
import lande, { type Language } from "lande";
2
+
3
+
export const DEFAULT_LANG: Language = "eng";
4
+
5
+
export const detectLanguage = (text: string): Language => {
6
+
const probabilities = lande(text);
7
+
8
+
if (!probabilities.length) {
9
+
return DEFAULT_LANG;
10
+
}
11
+
12
+
const language = probabilities[0];
13
+
14
+
return language?.[0] ?? DEFAULT_LANG;
15
+
};
+4
-1
packages/remanso-cli/src/lib/note.ts
+4
-1
packages/remanso-cli/src/lib/note.ts
···
3
3
import * as path from "node:path";
4
4
import mimeTypes from "mime-types";
5
5
import type { BlogPost, BlobObject } from "../../../cli/src/lib/types";
6
+
import { detectLanguage } from "./detect-languages";
6
7
7
8
const LEXICON = "space.remanso.note";
8
9
const MAX_CONTENT = 10000;
···
201
202
const title = titleMatch ? titleMatch[1] : post.frontmatter.title;
202
203
203
204
const { content, images } = await processNoteContent(agent, post, options);
205
+
const slicedContent = content.slice(0, MAX_CONTENT);
204
206
205
207
const record: Record<string, unknown> = {
206
208
$type: LEXICON,
207
209
title,
208
-
content: content.slice(0, MAX_CONTENT),
210
+
content: slicedContent,
209
211
createdAt: publishDate,
210
212
publishedAt: publishDate,
213
+
language: detectLanguage(title || slicedContent),
211
214
};
212
215
213
216
if (images.length > 0) {