this repo has no description
feat: Implement scheduled recache functionality and update configuration for recaching process
+70
-28
+4
.env.example
+4
.env.example
···
16
16
DEFAULT_CRON_SCHEDULE=0 */1 * * *
17
17
IMAGES_PER_INTERVAL=1
18
18
19
+
# Recache Cron Schedule
20
+
# How often to run the recache process (default: every day at 3:30am)
21
+
RECACHE_CRON_SCHEDULE=30 3 * * *
22
+
19
23
# Queue Configuration
20
24
# QUEUE_FILE_PATH=/path/to/custom/queue.json
21
25
+58
-28
bot/telegrambot/commands/recache.js
+58
-28
bot/telegrambot/commands/recache.js
···
1
1
const queueManager = require('../../../queue/queueManager');
2
2
const mediaCache = require('../../../utils/mediaCache');
3
+
const fs = require('fs-extra');
4
+
const path = require('path');
3
5
4
6
/**
5
-
* /recache command handler
7
+
* /recache command handler and scheduled recache logic
6
8
*/
7
9
class RecacheCommand {
8
10
constructor(bot, authHelper) {
···
18
20
return;
19
21
}
20
22
this.bot.sendMessage(chatId, 'Recaching missing images and files. This may take a while...');
21
-
try {
22
-
const queue = await queueManager.getQueue();
23
-
let recachedCount = 0;
24
-
for (let i = 0; i < queue.length; i++) {
25
-
const item = queue[i];
26
-
// Support both single and multiple images
27
-
const urls = [];
28
-
if (item.sourceImgUrl) urls.push(item.sourceImgUrl);
29
-
if (Array.isArray(item.originalImageUrls)) urls.push(...item.originalImageUrls);
30
-
if (item.downloadUrl && !urls.includes(item.downloadUrl)) urls.push(item.downloadUrl);
31
-
if (item.originalImageUrl && !urls.includes(item.originalImageUrl)) urls.push(item.originalImageUrl);
32
-
// Remove duplicates
33
-
const uniqueUrls = [...new Set(urls.filter(Boolean))];
34
-
for (const url of uniqueUrls) {
35
-
try {
36
-
const result = await mediaCache.processMediaUrl(url, item.isVideo);
37
-
// Optionally update imageUrl/imageUrls if missing or invalid
38
-
// (not overwriting if already present and valid)
39
-
// Could add logic here if needed
40
-
recachedCount++;
41
-
} catch (err) {
42
-
// Log but continue
43
-
console.error(`Failed to recache for queue item: ${item.title || item.id} (${url}):`, err.message);
23
+
const { processed, removed } = await RecacheCommand.runScheduledRecache();
24
+
this.bot.sendMessage(chatId, `Recache complete. Processed ${processed} queue items. Removed ${removed} items after 3 failed attempts.`);
25
+
});
26
+
}
27
+
28
+
/**
29
+
* Run recache for missing files, remove items after 3 failures
30
+
* Can be called from a scheduler (does not require bot instance)
31
+
* @returns {Promise<{processed: number, removed: number}>}
32
+
*/
33
+
static async runScheduledRecache() {
34
+
const queue = await queueManager.getQueue();
35
+
let processed = 0;
36
+
let removed = 0;
37
+
// Track indices to remove after loop to avoid index shifting
38
+
const indicesToRemove = [];
39
+
for (let i = 0; i < queue.length; i++) {
40
+
const item = queue[i];
41
+
// Gather all relevant URLs
42
+
const urls = [];
43
+
if (item.sourceImgUrl) urls.push(item.sourceImgUrl);
44
+
if (Array.isArray(item.originalImageUrls)) urls.push(...item.originalImageUrls);
45
+
if (item.downloadUrl && !urls.includes(item.downloadUrl)) urls.push(item.downloadUrl);
46
+
if (item.originalImageUrl && !urls.includes(item.originalImageUrl)) urls.push(item.originalImageUrl);
47
+
const uniqueUrls = [...new Set(urls.filter(Boolean))];
48
+
let allFilesExist = true;
49
+
for (const url of uniqueUrls) {
50
+
// Determine expected cache path
51
+
const hash = mediaCache.getHashedFilename(url);
52
+
const ext = mediaCache.getFileExtension(url);
53
+
const isVideo = item.isVideo;
54
+
const dir = isVideo ? mediaCache.videoDir : mediaCache.imageDir;
55
+
const filePath = path.join(dir, `${hash}${ext}`);
56
+
const exists = await fs.pathExists(filePath);
57
+
if (!exists) {
58
+
allFilesExist = false;
59
+
try {
60
+
await mediaCache.processMediaUrl(url, isVideo);
61
+
// Reset failure count on success
62
+
if (item._recacheFailures) item._recacheFailures = 0;
63
+
} catch (err) {
64
+
item._recacheFailures = (item._recacheFailures || 0) + 1;
65
+
if (item._recacheFailures >= 3) {
66
+
indicesToRemove.push(i);
67
+
break; // No need to try other URLs for this item
44
68
}
45
69
}
46
70
}
47
-
this.bot.sendMessage(chatId, `Recache complete. Processed ${queue.length} queue items. Attempted to recache ${recachedCount} files.`);
48
-
} catch (error) {
49
-
this.bot.sendMessage(chatId, `Error during recache: ${error.message}`);
50
71
}
51
-
});
72
+
processed++;
73
+
}
74
+
// Remove items in reverse order to avoid index shifting
75
+
indicesToRemove.sort((a, b) => b - a);
76
+
for (const idx of indicesToRemove) {
77
+
queue.splice(idx, 1);
78
+
removed++;
79
+
}
80
+
if (removed > 0) await queueManager.saveQueueToDisk();
81
+
return { processed, removed };
52
82
}
53
83
}
54
84
+8
recache-scheduler.js
+8
recache-scheduler.js
···
1
+
// Daily recache scheduler for missing files
2
+
const RecacheCommand = require('./bot/telegrambot/commands/recache');
3
+
4
+
(async () => {
5
+
console.log('Running scheduled recache for missing files...');
6
+
const { processed, removed } = await RecacheCommand.runScheduledRecache();
7
+
console.log(`Scheduled recache complete. Processed ${processed} items. Removed ${removed} items after 3 failures.`);
8
+
})();