+2 -2

src/constants.ts

reviewed

··· 6 6 export const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379'; 7 7 export const SQL_OUTPUT_FILE = 'dids_pds.jsonl'; 8 8 export const HEALTH_CHECK_FILE = 'pds_health.json'; 9 9 - export const SUCCESSFUL_DIDS_LOG_INTERVAL = 100_000; 9 9 + export const SUCCESSFUL_DIDS_LOG_INTERVAL = 10_000; 10 10 export const PDS_HEALTH_CHECK_CONCURRENCY = 50; 11 11 export const PDS_HEALTH_CHECK_TIMEOUT_MS = 25_000; 12 12 - export const PDS_DATA_FETCH_CONCURRENCY = 150; 12 12 + export const PDS_DATA_FETCH_CONCURRENCY = 100; 13 13 export const DIDS_TO_PROCESS = parseInt(process.argv[2], 10) || 1_000_000_000; 14 14 export const PYTHON_SERVICE_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes 15 15 export const BATCH_SIZE = 1000; // Number of postgres records per batch

+1 -1

src/stages/stage3.ts

reviewed

··· 208 208 }), 209 209 ); 210 210 211 211 - const chunkedTasks = chunkArray(tasks, 100_000); 211 211 + const chunkedTasks = chunkArray(tasks, SUCCESSFUL_DIDS_LOG_INTERVAL); 212 212 for (const chunk of chunkedTasks) { 213 213 try { 214 214 await Promise.all(chunk);