this repo has no description
cosmetics
+5
-12
+5
-5
packages/backend/src/lib/emojiNormalization.ts
+5
-5
packages/backend/src/lib/emojiNormalization.ts
···
3
3
import { codePointToEmoji, emojiToCodePoint, lowercaseObject } from './helpers.js';
4
4
import { Emoji, EmojiVariationSequence } from './types.js';
5
5
6
-
const eVSPath = new URL('./data/emojiVariationSequences.json', import.meta.url);
7
-
const eJSONPath = new URL('./data/emoji.json', import.meta.url);
8
-
9
6
// Load and parse normalization data
10
7
// converted from: https://unicode.org/Public/emoji/12.1/emoji-variation-sequences.txt
11
8
// regex in Sublime Text form:
12
9
// find: ([0-9A-F]{4,5}) +FE0E +; +.+? style; +\# \((\d.\d)\) ([A-Z0-9\- ]+)\n[0-9A-F]{4,5} +FE0F +; +.+? style; +\# \(\d.\d\) [A-Z0-9\- ]+\n
13
10
// replace: {"code": "$1", "textStyle": "$1 FE0E", "emojiStyle": "$1 FE0F", "version": "$2", "name": "$3"},\n
11
+
const eVSPath = new URL('./data/emojiVariationSequences.json', import.meta.url);
12
+
13
+
// source: https://github.com/iamcal/emoji-data/blob/master/emoji.json
14
+
const eJSONPath = new URL('./data/emoji.json', import.meta.url);
15
+
14
16
const emojiVariationSequences: EmojiVariationSequence[] = JSON.parse(
15
17
fs.readFileSync(eVSPath, 'utf8'),
16
18
) as EmojiVariationSequence[];
···
41
43
let firstPass;
42
44
if (normalizationMap[emojiCodePoints]) {
43
45
firstPass = normalizationMap[emojiCodePoints];
44
-
// console.log(`first pass normalized: ${firstPass} (${emojiCodePoints})`);
45
46
} else {
46
47
firstPass = emojiCodePoints;
47
48
}
···
51
52
const unifiedCodePoints = nonQualifiedMap[firstPass];
52
53
if (unifiedCodePoints && unifiedCodePoints !== firstPass) {
53
54
normalizedEmoji = codePointToEmoji(unifiedCodePoints);
54
-
// console.log(`second pass normalized: ${normalizedEmoji} (${unifiedCodePoints})`);
55
55
}
56
56
57
57
return normalizedEmoji;
-7
packages/backend/src/lib/emojiStats.ts
-7
packages/backend/src/lib/emojiStats.ts
···
55
55
const emojiMatches: string[] = record.text.match(emojiRegex) ?? [];
56
56
57
57
if (emojiMatches.length > 0) {
58
-
// if (emojiMatches.includes('🏳️⚧️')) {
59
-
// console.log(`found: CID ${event.commit.cid}`);
60
-
// console.dir(emojiMatches, { depth: null });
61
-
// const normalized = batchNormalizeEmojis(emojiMatches);
62
-
// console.log('normalized:');
63
-
// console.dir(normalized, { depth: null });
64
-
// }
65
58
const stringifiedLangs = JSON.stringify(Array.from(langs));
66
59
67
60
const normalizedEmojis = JSON.stringify(batchNormalizeEmojis(emojiMatches));