simple list of pds servers with open registration
Add smarter caching to reduce redundant work in cron jobs
ETag caching for state.json and GitHub tags API skips the full 2900-server
upsert and version update when data hasn't changed. Enrichment reuses
cached DNS/geo data from SQLite instead of re-resolving every run.
Cache-Control headers on HTML/API responses (1hr max-age). Batch size
bumped from 50 to 75.
+213
-69
+11
-4
backend/database/queries.ts
+11
-4
backend/database/queries.ts
···
1
1
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite?v=13";
2
2
import { METADATA_TABLE, SERVERS_TABLE } from "../../shared/constants.ts";
3
-
import type { PdsServer } from "../../shared/types.ts";
3
+
import type { PdsServer, ServerToEnrich } from "../../shared/types.ts";
4
4
5
5
// --- Server operations ---
6
6
···
94
94
});
95
95
}
96
96
97
-
export async function getServersToEnrich(limit: number): Promise<string[]> {
97
+
export async function getServersToEnrich(
98
+
limit: number,
99
+
): Promise<ServerToEnrich[]> {
98
100
const result = await sqlite.execute({
99
101
sql: `
100
-
SELECT url FROM ${SERVERS_TABLE}
102
+
SELECT url, ip_address, country_code, country_name FROM ${SERVERS_TABLE}
101
103
WHERE is_open = 1
102
104
ORDER BY
103
105
CASE WHEN last_enriched IS NULL THEN 0
···
110
112
`,
111
113
args: [limit],
112
114
});
113
-
return result.rows.map((r) => r.url as string);
115
+
return result.rows.map((r) => ({
116
+
url: r.url as string,
117
+
ipAddress: (r.ip_address as string) || null,
118
+
countryCode: (r.country_code as string) || null,
119
+
countryName: (r.country_name as string) || null,
120
+
}));
114
121
}
115
122
116
123
export async function getOpenServers(
+4
backend/routes/api.ts
+4
backend/routes/api.ts
+4
backend/routes/pages.ts
+4
backend/routes/pages.ts
+71
-26
backend/services/pds-enricher.ts
+71
-26
backend/services/pds-enricher.ts
···
2
2
import type {
3
3
DescribeServerResponse,
4
4
HealthResponse,
5
+
ServerToEnrich,
5
6
} from "../../shared/types.ts";
6
7
import { batchGeoLookup, resolveIp } from "./geo-resolver.ts";
7
8
···
20
21
countryName: string | null;
21
22
};
22
23
23
-
/** Enrich a batch of PDS URLs with metadata */
24
-
export async function enrichBatch(urls: string[]): Promise<EnrichmentResult[]> {
25
-
// Step 1: Fetch PDS endpoints concurrently
24
+
export type EnrichmentStats = {
25
+
cachedDns: number;
26
+
freshDns: number;
27
+
cachedGeo: number;
28
+
freshGeo: number;
29
+
};
30
+
31
+
/** Enrich a batch of PDS servers with metadata, reusing cached DNS/geo data */
32
+
export async function enrichBatch(
33
+
servers: ServerToEnrich[],
34
+
): Promise<{ results: EnrichmentResult[]; stats: EnrichmentStats }> {
35
+
const stats: EnrichmentStats = {
36
+
cachedDns: 0,
37
+
freshDns: 0,
38
+
cachedGeo: 0,
39
+
freshGeo: 0,
40
+
};
41
+
42
+
// Step 1: Fetch PDS endpoints concurrently, passing existing IP to skip DNS
26
43
const pdsResults = await Promise.allSettled(
27
-
urls.map((url) => enrichSinglePds(url)),
44
+
servers.map((s) => enrichSinglePds(s.url, s.ipAddress)),
28
45
);
29
46
30
47
const results: EnrichmentResult[] = [];
31
-
const ipMap = new Map<string, string>();
48
+
const needsGeoLookup = new Map<string, string>();
32
49
33
-
for (let i = 0; i < urls.length; i++) {
50
+
for (let i = 0; i < servers.length; i++) {
51
+
const server = servers[i];
34
52
const r = pdsResults[i];
53
+
let result: EnrichmentResult;
54
+
35
55
if (r.status === "fulfilled") {
36
-
results.push(r.value);
37
-
if (r.value.ipAddress) {
38
-
ipMap.set(r.value.url, r.value.ipAddress);
39
-
}
56
+
result = r.value;
40
57
} else {
41
-
console.error(`Enrichment failed for ${urls[i]}:`, r.reason);
42
-
// Return a minimal result so we still update last_enriched
43
-
results.push(emptyResult(urls[i]));
58
+
console.error(`Enrichment failed for ${server.url}:`, r.reason);
59
+
result = emptyResult(server.url);
60
+
}
61
+
62
+
// Track DNS cache stats
63
+
if (server.ipAddress && result.ipAddress === server.ipAddress) {
64
+
stats.cachedDns++;
65
+
} else if (result.ipAddress) {
66
+
stats.freshDns++;
67
+
}
68
+
69
+
// Determine if we need geo lookup or can carry forward cached data
70
+
if (result.ipAddress && !server.countryCode) {
71
+
// Have IP but no cached geo — need lookup
72
+
needsGeoLookup.set(result.url, result.ipAddress);
73
+
} else if (server.countryCode) {
74
+
// Carry forward cached geo data
75
+
result.countryCode = result.countryCode ?? server.countryCode;
76
+
result.countryName = result.countryName ?? server.countryName;
77
+
stats.cachedGeo++;
44
78
}
79
+
80
+
results.push(result);
45
81
}
46
82
47
-
// Step 2: Batch geo-IP lookup
48
-
const geoData = await batchGeoLookup(ipMap);
49
-
for (const result of results) {
50
-
const geo = geoData.get(result.url);
51
-
if (geo) {
52
-
result.countryCode = geo.countryCode;
53
-
result.countryName = geo.countryName;
83
+
// Step 2: Batch geo-IP lookup only for servers that need it
84
+
if (needsGeoLookup.size > 0) {
85
+
const geoData = await batchGeoLookup(needsGeoLookup);
86
+
for (const result of results) {
87
+
const geo = geoData.get(result.url);
88
+
if (geo) {
89
+
result.countryCode = geo.countryCode;
90
+
result.countryName = geo.countryName;
91
+
stats.freshGeo++;
92
+
}
54
93
}
55
94
}
56
95
57
-
return results;
96
+
return { results, stats };
58
97
}
59
98
60
-
async function enrichSinglePds(url: string): Promise<EnrichmentResult> {
99
+
async function enrichSinglePds(
100
+
url: string,
101
+
existingIp?: string | null,
102
+
): Promise<EnrichmentResult> {
61
103
const result = emptyResult(url);
62
104
63
-
// Resolve IP from hostname
64
-
const hostname = new URL(url).hostname;
65
-
result.ipAddress = await resolveIp(hostname);
105
+
// Reuse existing IP or resolve fresh
106
+
if (existingIp) {
107
+
result.ipAddress = existingIp;
108
+
} else {
109
+
const hostname = new URL(url).hostname;
110
+
result.ipAddress = await resolveIp(hostname);
111
+
}
66
112
67
113
// Fetch health, describeServer, and listRepos concurrently
68
-
// Each gets its own timeout so DNS resolve doesn't eat into fetch time
69
114
const [health, describe, userCount] = await Promise.allSettled([
70
115
fetchHealth(url, AbortSignal.timeout(PDS_REQUEST_TIMEOUT_MS)),
71
116
fetchDescribeServer(url, AbortSignal.timeout(PDS_REQUEST_TIMEOUT_MS)),
+22
-4
backend/services/pds-fetcher.ts
+22
-4
backend/services/pds-fetcher.ts
···
9
9
isOpen: boolean;
10
10
};
11
11
12
-
/** Fetch state.json and return all PDSes with their open status */
13
-
export async function fetchPdsList(): Promise<FilteredPds[]> {
14
-
const resp = await fetch(STATE_JSON_URL);
12
+
export type FetchPdsResult = {
13
+
pdsList: FilteredPds[] | null;
14
+
etag: string | null;
15
+
};
16
+
17
+
/** Fetch state.json with ETag caching. Returns null pdsList on 304. */
18
+
export async function fetchPdsList(
19
+
previousEtag?: string | null,
20
+
): Promise<FetchPdsResult> {
21
+
const headers: Record<string, string> = {};
22
+
if (previousEtag) {
23
+
headers["If-None-Match"] = previousEtag;
24
+
}
25
+
26
+
const resp = await fetch(STATE_JSON_URL, { headers });
27
+
const etag = resp.headers.get("etag");
28
+
29
+
if (resp.status === 304) {
30
+
return { pdsList: null, etag: etag ?? previousEtag ?? null };
31
+
}
32
+
15
33
if (!resp.ok) {
16
34
throw new Error(`Failed to fetch state.json: ${resp.status}`);
17
35
}
···
30
48
});
31
49
}
32
50
33
-
return results;
51
+
return { pdsList: results, etag: etag ?? null };
34
52
}
+33
-8
backend/services/version-checker.ts
+33
-8
backend/services/version-checker.ts
···
1
1
import { PDS_REPO_NAME, PDS_REPO_OWNER } from "../../shared/constants.ts";
2
2
3
-
/** Fetch the latest PDS release version from GitHub */
4
-
export async function fetchLatestPdsVersion(): Promise<string | null> {
3
+
export type FetchVersionResult = {
4
+
version: string | null;
5
+
etag: string | null;
6
+
notModified: boolean;
7
+
};
8
+
9
+
/** Fetch latest PDS version from GitHub with ETag caching. */
10
+
export async function fetchLatestPdsVersion(
11
+
previousEtag?: string | null,
12
+
): Promise<FetchVersionResult> {
5
13
const url =
6
14
`https://api.github.com/repos/${PDS_REPO_OWNER}/${PDS_REPO_NAME}/tags?per_page=5`;
7
15
8
-
const resp = await fetch(url, {
9
-
headers: { "Accept": "application/vnd.github.v3+json" },
10
-
});
16
+
const headers: Record<string, string> = {
17
+
"Accept": "application/vnd.github.v3+json",
18
+
};
19
+
if (previousEtag) {
20
+
headers["If-None-Match"] = previousEtag;
21
+
}
22
+
23
+
const resp = await fetch(url, { headers });
24
+
const etag = resp.headers.get("etag");
25
+
26
+
if (resp.status === 304) {
27
+
return {
28
+
version: null,
29
+
etag: etag ?? previousEtag ?? null,
30
+
notModified: true,
31
+
};
32
+
}
11
33
12
34
if (!resp.ok) {
13
35
console.error(`GitHub API error: ${resp.status}`);
14
-
return null;
36
+
return { version: null, etag: etag ?? null, notModified: false };
15
37
}
16
38
17
39
const tags: Array<{ name: string }> = await resp.json();
18
-
if (tags.length === 0) return null;
40
+
if (tags.length === 0) {
41
+
return { version: null, etag: etag ?? null, notModified: false };
42
+
}
19
43
20
44
// Tags are like "v0.4.74" — strip the "v" prefix to match _health output
21
-
return tags[0].name.replace(/^v/, "");
45
+
const version = tags[0].name.replace(/^v/, "");
46
+
return { version, etag: etag ?? null, notModified: false };
22
47
}
+59
-26
cron/refresh.cron.ts
+59
-26
cron/refresh.cron.ts
···
1
1
import { ENRICHMENT_BATCH_SIZE } from "../shared/constants.ts";
2
2
import { runMigrations } from "../backend/database/migrations.ts";
3
3
import {
4
+
getMetadata,
4
5
getServersToEnrich,
5
6
setMetadata,
6
7
updateEnrichment,
···
16
17
try {
17
18
await runMigrations();
18
19
19
-
// 1. Fetch and sync state.json
20
-
const pdsList = await fetchPdsList();
21
-
console.log(`Fetched ${pdsList.length} PDSes from state.json`);
20
+
// 1. Fetch and sync state.json (with ETag caching)
21
+
const previousStateEtag = await getMetadata("state_json_etag");
22
+
const { pdsList, etag: stateEtag } = await fetchPdsList(previousStateEtag);
23
+
24
+
if (stateEtag) {
25
+
await setMetadata("state_json_etag", stateEtag);
26
+
}
27
+
28
+
if (pdsList) {
29
+
console.log(
30
+
`state.json: new data (${pdsList.length} PDSes), etag=${stateEtag}`,
31
+
);
32
+
let openCount = 0;
33
+
for (const pds of pdsList) {
34
+
await upsertServer(pds.url, {
35
+
inviteCodeRequired: pds.inviteCodeRequired,
36
+
version: pds.version,
37
+
errorAt: pds.errorAt,
38
+
isOpen: pds.isOpen,
39
+
});
40
+
if (pds.isOpen) openCount++;
41
+
}
42
+
console.log(`Synced to DB: ${openCount} open, ${pdsList.length} total`);
43
+
} else {
44
+
console.log(
45
+
`state.json: 304 not modified, skipping upsert (etag=${stateEtag})`,
46
+
);
47
+
}
48
+
49
+
// 2. Fetch latest PDS version (with ETag caching)
50
+
const previousGithubEtag = await getMetadata("github_tags_etag");
51
+
const { version: latestVersion, etag: githubEtag, notModified } =
52
+
await fetchLatestPdsVersion(previousGithubEtag);
22
53
23
-
let openCount = 0;
24
-
for (const pds of pdsList) {
25
-
await upsertServer(pds.url, {
26
-
inviteCodeRequired: pds.inviteCodeRequired,
27
-
version: pds.version,
28
-
errorAt: pds.errorAt,
29
-
isOpen: pds.isOpen,
30
-
});
31
-
if (pds.isOpen) openCount++;
54
+
if (githubEtag) {
55
+
await setMetadata("github_tags_etag", githubEtag);
32
56
}
33
-
console.log(`Synced to DB: ${openCount} open, ${pdsList.length} total`);
34
57
35
-
// 2. Fetch latest PDS version
36
-
const latestVersion = await fetchLatestPdsVersion();
37
-
if (latestVersion) {
58
+
if (notModified) {
59
+
console.log(
60
+
`GitHub tags: 304 not modified (etag=${githubEtag})`,
61
+
);
62
+
} else if (latestVersion) {
38
63
await setMetadata("latest_pds_version", latestVersion);
39
-
console.log(`Latest PDS version: ${latestVersion}`);
64
+
console.log(
65
+
`GitHub tags: new version ${latestVersion} (etag=${githubEtag})`,
66
+
);
67
+
} else {
68
+
console.log("GitHub tags: no version found");
40
69
}
41
70
42
71
// 3. Enrich a batch of servers
43
72
const toEnrich = await getServersToEnrich(ENRICHMENT_BATCH_SIZE);
44
73
if (toEnrich.length > 0) {
45
-
console.log(`Enriching ${toEnrich.length} servers`);
46
-
const enriched = await enrichBatch(toEnrich);
74
+
const cachedIpCount = toEnrich.filter((s) => s.ipAddress).length;
75
+
const cachedGeoCount = toEnrich.filter((s) => s.countryCode).length;
76
+
console.log(
77
+
`Enriching ${toEnrich.length} servers (${cachedIpCount} have cached IP, ${cachedGeoCount} have cached geo)`,
78
+
);
79
+
80
+
const { results: enriched, stats } = await enrichBatch(toEnrich);
47
81
48
82
for (const data of enriched) {
49
83
await updateEnrichment(data.url, {
···
59
93
countryName: data.countryName,
60
94
ipAddress: data.ipAddress,
61
95
});
62
-
const host = new URL(data.url).hostname;
63
-
console.log(
64
-
` ${host}: users=${data.userCount ?? "?"} country=${
65
-
data.countryCode ?? "?"
66
-
} version=${data.version ?? "?"}`,
67
-
);
68
96
}
69
-
console.log(`Enrichment complete for ${enriched.length} servers`);
97
+
98
+
console.log(
99
+
`Enriched ${enriched.length} servers: ` +
100
+
`DNS ${stats.cachedDns} cached/${stats.freshDns} fresh, ` +
101
+
`geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh`,
102
+
);
70
103
}
71
104
72
105
await setMetadata("last_full_refresh", new Date().toISOString());