this repo has no description
fix; don't use lua
+48
-30
+1
-1
packages/backend/src/index.ts
+1
-1
packages/backend/src/index.ts
+2
-5
packages/backend/src/lib/emojiNormalization.ts
+2
-5
packages/backend/src/lib/emojiNormalization.ts
···
1
1
import fs from 'fs';
2
2
3
3
import { codePointToEmoji, emojiToCodePoint, lowercaseObject } from './helpers.js';
4
+
import logger from './logger.js';
4
5
import { Emoji, EmojiVariationSequence } from './types.js';
5
6
6
7
// Load and parse normalization data
···
58
59
}
59
60
60
61
export function batchNormalizeEmojis(emojis: string[]): string[] {
61
-
const normalizationResults: string[] = [];
62
-
emojis.forEach((emoji) => {
63
-
normalizationResults.push(normalizeEmoji(emoji));
64
-
});
65
-
return normalizationResults;
62
+
return emojis.map((emoji) => normalizeEmoji(emoji));
66
63
}
+45
-24
packages/backend/src/lib/emojiStats.ts
+45
-24
packages/backend/src/lib/emojiStats.ts
···
42
42
43
43
try {
44
44
let langs = new Set<string>();
45
-
if (record.langs && Array.isArray(record.langs)) {
45
+
if (record.langs && Array.isArray(record.langs) && record.langs.length > 0) {
46
46
langs = new Set(record.langs);
47
47
} else {
48
48
langs.add('unknown');
49
49
}
50
50
51
-
const emojiMatches: string[] = record.text.match(emojiRegex) ?? [];
52
-
await db.transaction().execute(async (tx) => {
53
-
if (emojiMatches.length > 0) {
54
-
const stringifiedLangs = JSON.stringify(Array.from(langs));
51
+
if (langs.size === 0) {
52
+
logger.error('langs.size is 0, this should never happen');
53
+
process.exit(1);
54
+
}
55
55
56
-
const normalizedEmojis = JSON.stringify(batchNormalizeEmojis(emojiMatches));
56
+
const emojiMatches = record.text.match(emojiRegex) ?? [];
57
+
const normalizedEmojis = batchNormalizeEmojis(emojiMatches);
58
+
const hasEmojis = normalizedEmojis.length > 0;
57
59
58
-
await redis.evalSha(SCRIPT_SHA, {
59
-
arguments: [normalizedEmojis, stringifiedLangs],
60
-
});
61
-
62
-
logger.debug(`Emojis updated for languages: ${Array.from(langs).join(', ')}`);
63
-
incrementTotalEmojis(emojiMatches.length);
64
-
totalPostsWithEmojis.inc();
65
-
} else {
66
-
await redis.incr(POSTS_WITHOUT_EMOJIS);
67
-
totalPostsWithoutEmojis.inc();
68
-
}
69
-
70
-
await redis.incr(PROCESSED_POSTS);
71
-
incrementTotalPosts();
72
-
60
+
/* step 1: postgres */
61
+
await db.transaction().execute(async (tx) => {
62
+
// add post to db
73
63
const { id } = await tx
74
64
.insertInto('posts')
75
65
.values({
76
66
cid: cid,
77
67
did: did,
78
68
rkey: rkey,
79
-
has_emojis: emojiMatches.length > 0,
69
+
has_emojis: hasEmojis,
80
70
langs: Array.from(langs),
81
71
})
82
72
.returning('id')
83
73
.executeTakeFirstOrThrow();
84
74
85
-
for (const emoji of emojiMatches) {
75
+
// add emojis to db (if any)
76
+
for (const emoji of normalizedEmojis) {
86
77
for (const lang of langs) {
87
78
await tx
88
79
.insertInto('emojis')
···
91
82
emoji: emoji,
92
83
lang: lang,
93
84
})
94
-
.returning('id')
95
85
.executeTakeFirstOrThrow();
96
86
}
97
87
}
98
88
});
89
+
90
+
/* step 2: redis */
91
+
if (!hasEmojis) {
92
+
await redis.incr(POSTS_WITHOUT_EMOJIS);
93
+
totalPostsWithoutEmojis.inc();
94
+
} else {
95
+
// Start of Selection
96
+
const transaction = redis.multi();
97
+
transaction.incr('postsWithEmojis');
98
+
99
+
for (const emoji of normalizedEmojis) {
100
+
transaction.zIncrBy('emojiStats', 1, emoji);
101
+
transaction.incr('processedEmojis');
102
+
}
103
+
104
+
for (const emoji of normalizedEmojis) {
105
+
for (const lang of langs) {
106
+
transaction.zIncrBy(lang, 1, emoji);
107
+
transaction.zIncrBy('languageStats', 1, lang);
108
+
}
109
+
}
110
+
111
+
await transaction.exec();
112
+
113
+
incrementTotalEmojis(normalizedEmojis.length);
114
+
totalPostsWithEmojis.inc();
115
+
}
116
+
117
+
/* step 3: global metrics */
118
+
await redis.incr(PROCESSED_POSTS);
119
+
incrementTotalPosts();
99
120
} catch (error) {
100
121
logger.error(`Error processing "create" commit: ${(error as Error).message}`, { commit, record });
101
122
logger.error(`Malformed record data: ${JSON.stringify(record)}`);