this repo has no description
fixes
+38
-44
+1
src/constants.ts
+1
src/constants.ts
+1
-1
src/index.ts
+1
-1
src/index.ts
···
1
1
import { DIDS_TO_PROCESS, METRICS_PORT } from './constants.js';
2
+
import { startMetricsServer } from './metrics.js';
2
3
import { gracefulShutdown, registerShutdownHandlers } from './shutdown.js';
3
4
import { fetchAndDumpDidsPdses } from './stages/stage1.js';
4
5
import { checkAllPDSHealth, selectAllDids } from './stages/stage2.js';
5
6
import { processDidsAndFetchData } from './stages/stage3.js';
6
7
import { DidAndPds } from './types.js';
7
-
import { startMetricsServer, stopMetricsServer } from './metrics.js';
8
8
9
9
async function main() {
10
10
// Register graceful shutdown handlers
+10
-14
src/metrics.ts
+10
-14
src/metrics.ts
···
1
1
import { monitorPgPool } from '@christiangalsterer/node-postgres-prometheus-exporter';
2
2
import express from 'express';
3
+
import { Server } from 'http';
3
4
import { Gauge, Registry, collectDefaultMetrics } from 'prom-client';
4
5
5
6
import { pool } from './postgres.js';
6
-
import { Server } from 'http';
7
7
8
8
const register = new Registry();
9
9
collectDefaultMetrics({ register });
···
47
47
48
48
export function stopMetricsServer(): Promise<void> {
49
49
return new Promise((resolve, reject) => {
50
-
if (metricsServer) {
51
-
metricsServer.close((err) => {
52
-
if (err) {
53
-
console.error('Error shutting down metrics server:', err);
54
-
reject(err);
55
-
} else {
56
-
console.log('Metrics server shut down successfully.');
57
-
resolve();
58
-
}
59
-
});
60
-
} else {
61
-
resolve();
62
-
}
50
+
metricsServer.close((err) => {
51
+
if (err) {
52
+
console.error('Error shutting down metrics server:', err);
53
+
reject(err);
54
+
} else {
55
+
console.log('Metrics server shut down successfully.');
56
+
resolve();
57
+
}
58
+
});
63
59
});
64
60
}
+4
-4
src/postgresBatchQueue.ts
+4
-4
src/postgresBatchQueue.ts
···
1
1
// src/postgresBatchQueue.ts
2
-
3
2
import { Mutex } from 'async-mutex';
4
-
import { db, closeDatabase } from './postgres.js';
3
+
4
+
import { BATCH_SIZE, BATCH_TIMEOUT_MS, MAX_FLUSH_RETRIES } from './constants.js';
5
5
import { concurrentPostgresInserts } from './metrics.js';
6
+
import { closeDatabase, db } from './postgres.js';
6
7
import { PostData } from './types.js';
7
-
import { BATCH_SIZE, BATCH_TIMEOUT_MS, MAX_FLUSH_RETRIES } from './constants.js';
8
8
9
9
export class PostgresBatchQueue {
10
10
private queue: PostData[] = [];
···
50
50
*/
51
51
private scheduleFlush(): void {
52
52
this.batchTimer = setTimeout(() => {
53
-
this.flushQueue().catch((err) => {
53
+
this.flushQueue().catch((err: unknown) => {
54
54
console.error(`Scheduled flush error: ${(err as Error).message}`);
55
55
});
56
56
}, this.batchTimeoutMs);
+3
-5
src/schema.d.ts
+3
-5
src/schema.d.ts
···
2
2
* This file was generated by kysely-codegen.
3
3
* Please do not edit it manually.
4
4
*/
5
-
6
-
import type { ColumnType } from "kysely";
5
+
import type { ColumnType } from 'kysely';
7
6
8
-
export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
9
-
? ColumnType<S, I | undefined, U>
10
-
: ColumnType<T, T | undefined, T>;
7
+
export type Generated<T> =
8
+
T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S, I | undefined, U> : ColumnType<T, T | undefined, T>;
11
9
12
10
export type Int8 = ColumnType<string, bigint | number | string>;
13
11
+8
-9
src/shutdown.ts
+8
-9
src/shutdown.ts
···
1
1
// src/shutdown.ts
2
-
2
+
import { stopMetricsServer } from './metrics.js';
3
+
import { closeDatabase } from './postgres.js';
3
4
import { postgresBatchQueue } from './postgresBatchQueue.js';
4
-
import { closeDatabase } from './postgres.js';
5
-
import { stopMetricsServer } from './metrics.js';
6
5
7
6
export async function gracefulShutdown(): Promise<void> {
8
7
console.log('Initiating graceful shutdown...');
···
21
20
22
21
// Register shutdown handlers
23
22
export function registerShutdownHandlers() {
24
-
process.on('SIGINT', gracefulShutdown);
25
-
process.on('SIGTERM', gracefulShutdown);
26
-
process.on('uncaughtException', async (err) => {
23
+
process.on('SIGINT', () => void gracefulShutdown());
24
+
process.on('SIGTERM', () => void gracefulShutdown());
25
+
process.on('uncaughtException', (err) => {
27
26
console.error('Uncaught Exception:', err);
28
-
await gracefulShutdown();
27
+
void gracefulShutdown();
29
28
});
30
-
process.on('unhandledRejection', async (reason, promise) => {
29
+
process.on('unhandledRejection', (reason, promise) => {
31
30
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
32
-
await gracefulShutdown();
31
+
void gracefulShutdown();
33
32
});
34
33
}
+11
-11
src/stages/stage2.ts
+11
-11
src/stages/stage2.ts
···
1
-
import { HEALTH_CHECK_FILE, PDS_HEALTH_CHECK_CONCURRENCY, SQL_OUTPUT_FILE } from "../constants.js";
2
-
import fs from "node:fs/promises";
3
-
import readline from "node:readline";
4
-
import pLimit from "p-limit";
5
-
import { isPDSHealthy, sanitizePDSName } from "../helpers.js";
6
-
import { PdsToDidsMap, PdsHealthStatus } from "../types.js";
1
+
import fs from 'node:fs/promises';
2
+
import readline from 'node:readline';
3
+
import pLimit from 'p-limit';
7
4
5
+
import { HEALTH_CHECK_FILE, PDS_HEALTH_CHECK_CONCURRENCY, SQL_OUTPUT_FILE } from '../constants.js';
6
+
import { isPDSHealthy, sanitizePDSName } from '../helpers.js';
7
+
import { PdsHealthStatus, PdsToDidsMap } from '../types.js';
8
8
9
9
export async function checkAllPDSHealth() {
10
10
const startTime = performance.now();
···
32
32
}
33
33
34
34
const { did, pds } = JSON.parse(line) as { did: string; pds: string };
35
-
35
+
36
36
if (!groupedByPDS[pds]) {
37
37
groupedByPDS[pds] = [];
38
38
}
···
86
86
const healthCheckEndTime = performance.now();
87
87
const healthCheckTime = (healthCheckEndTime - healthCheckStartTime) / 1000;
88
88
console.log(`Health check process took ${healthCheckTime.toFixed(2)} seconds`);
89
-
89
+
90
90
const healthyCount = Object.values(pdsHealthStatus).filter(Boolean).length;
91
91
const unhealthyCount = Object.values(pdsHealthStatus).length - healthyCount;
92
92
console.log(`Total PDS count: ${sanitizedPDSMap.size}`);
93
93
console.log(`Healthy PDS count: ${healthyCount}`);
94
94
console.log(`Unhealthy PDS count: ${unhealthyCount}`);
95
-
95
+
96
96
const endTime = performance.now();
97
97
const totalTime = (endTime - startTime) / 1000;
98
-
console.log(`Total processing time: ${totalTime.toFixed(2)} seconds`);
98
+
console.log(`Total processing time: ${totalTime.toFixed(2)} seconds`);
99
99
}
100
100
101
101
return { groupedByPDS, pdsHealthStatus };
···
123
123
console.log(`Total DIDs from unhealthy PDSes: ${totalUnhealthyDIDs}`);
124
124
125
125
// Prepare the list of DIDs to process
126
-
const allDids: { did: string; pds: string; }[] = [];
126
+
const allDids: { did: string; pds: string }[] = [];
127
127
128
128
for (const [pds, dids] of Object.entries(healthyGroupedByPDS)) {
129
129
for (const did of dids!) {