simple list of pds servers with open registration
Fix four cron job issues: URL normalization, GitHub API resilience, log noise, cache TTL
- Strip trailing slashes from PDS URLs to prevent double-slash in xrpc paths
- Add User-Agent header to GitHub API requests and preserve etag on errors
- Suppress individual listRepos 404 logs, show summary count instead
- Add DNS (7d) and geo (30d) cache TTL to refresh stale enrichment data
+83
-27
+3
-1
backend/database/queries.ts
+3
-1
backend/database/queries.ts
···
99
99
): Promise<ServerToEnrich[]> {
100
100
const result = await sqlite.execute({
101
101
sql: `
102
-
SELECT url, ip_address, country_code, country_name FROM ${SERVERS_TABLE}
102
+
SELECT url, ip_address, country_code, country_name, last_enriched
103
+
FROM ${SERVERS_TABLE}
103
104
WHERE is_open = 1
104
105
ORDER BY
105
106
CASE WHEN last_enriched IS NULL THEN 0
···
117
118
ipAddress: (r.ip_address as string) || null,
118
119
countryCode: (r.country_code as string) || null,
119
120
countryName: (r.country_name as string) || null,
121
+
lastEnriched: (r.last_enriched as string) || null,
120
122
}));
121
123
}
122
124
+34
-7
backend/services/pds-enricher.ts
+34
-7
backend/services/pds-enricher.ts
···
1
-
import { PDS_REQUEST_TIMEOUT_MS } from "../../shared/constants.ts";
1
+
import {
2
+
DNS_CACHE_TTL_DAYS,
3
+
GEO_CACHE_TTL_DAYS,
4
+
PDS_REQUEST_TIMEOUT_MS,
5
+
} from "../../shared/constants.ts";
2
6
import type {
3
7
DescribeServerResponse,
4
8
HealthResponse,
5
9
ServerToEnrich,
6
10
} from "../../shared/types.ts";
7
-
import { isPrivateIp, isValidPdsUrl } from "../../shared/url-validation.ts";
11
+
import {
12
+
isPrivateIp,
13
+
isValidPdsUrl,
14
+
normalizePdsUrl,
15
+
} from "../../shared/url-validation.ts";
8
16
import { batchGeoLookup, resolveIp } from "./geo-resolver.ts";
9
17
10
18
export type EnrichmentResult = {
···
29
37
freshGeo: number;
30
38
};
31
39
40
+
/** Check if a cached value is older than the given TTL in days */
41
+
function isCacheStale(lastEnriched: string | null, ttlDays: number): boolean {
42
+
if (!lastEnriched) return true;
43
+
const ageMs = Date.now() - new Date(lastEnriched).getTime();
44
+
return ageMs > ttlDays * 24 * 60 * 60 * 1000;
45
+
}
46
+
32
47
/** Enrich a batch of PDS servers with metadata, reusing cached DNS/geo data */
33
48
export async function enrichBatch(
34
49
servers: ServerToEnrich[],
···
41
56
};
42
57
43
58
// Step 1: Fetch PDS endpoints concurrently, passing existing IP to skip DNS
59
+
// Invalidate stale DNS caches by passing null instead of the cached IP
44
60
const pdsResults = await Promise.allSettled(
45
-
servers.map((s) => enrichSinglePds(s.url, s.ipAddress)),
61
+
servers.map((s) => {
62
+
const dnsStale = isCacheStale(s.lastEnriched, DNS_CACHE_TTL_DAYS);
63
+
return enrichSinglePds(s.url, dnsStale ? null : s.ipAddress);
64
+
}),
46
65
);
47
66
48
67
const results: EnrichmentResult[] = [];
···
66
85
} else if (result.ipAddress) {
67
86
stats.freshDns++;
68
87
}
88
+
89
+
const geoStale = isCacheStale(server.lastEnriched, GEO_CACHE_TTL_DAYS);
90
+
const ipChanged = result.ipAddress !== null &&
91
+
result.ipAddress !== server.ipAddress;
69
92
70
93
// Determine if we need geo lookup or can carry forward cached data
71
-
if (result.ipAddress && !server.countryCode) {
72
-
// Have IP but no cached geo — need lookup
94
+
if (result.ipAddress && (!server.countryCode || geoStale || ipChanged)) {
95
+
// Need fresh geo: no cached data, cache expired, or IP changed
73
96
needsGeoLookup.set(result.url, result.ipAddress);
74
97
} else if (server.countryCode) {
75
98
// Carry forward cached geo data
···
98
121
}
99
122
100
123
async function enrichSinglePds(
101
-
url: string,
124
+
rawUrl: string,
102
125
existingIp?: string | null,
103
126
): Promise<EnrichmentResult> {
127
+
const url = normalizePdsUrl(rawUrl);
104
128
if (!isValidPdsUrl(url)) {
105
129
throw new Error(`Refusing to enrich invalid URL: ${url}`);
106
130
}
···
186
210
{ signal },
187
211
);
188
212
if (!resp.ok) {
189
-
console.error(`listRepos ${url}: HTTP ${resp.status}`);
213
+
// 404 is expected for non-standard PDSes — only log other errors
214
+
if (resp.status !== 404) {
215
+
console.error(`listRepos ${url}: HTTP ${resp.status}`);
216
+
}
190
217
return null;
191
218
}
192
219
const data = await resp.json();
+4
-3
backend/services/pds-fetcher.ts
+4
-3
backend/services/pds-fetcher.ts
···
1
1
import { STATE_JSON_URL } from "../../shared/constants.ts";
2
2
import type { StateJson } from "../../shared/types.ts";
3
-
import { isValidPdsUrl } from "../../shared/url-validation.ts";
3
+
import { isValidPdsUrl, normalizePdsUrl } from "../../shared/url-validation.ts";
4
4
5
5
export type FilteredPds = {
6
6
url: string;
···
39
39
const results: FilteredPds[] = [];
40
40
41
41
let skipped = 0;
42
-
for (const [url, entry] of Object.entries(data.pdses)) {
43
-
if (!isValidPdsUrl(url)) {
42
+
for (const [rawUrl, entry] of Object.entries(data.pdses)) {
43
+
if (!isValidPdsUrl(rawUrl)) {
44
44
skipped++;
45
45
continue;
46
46
}
47
+
const url = normalizePdsUrl(rawUrl);
47
48
const isOpen = !entry.inviteCodeRequired && !entry.errorAt;
48
49
results.push({
49
50
url,
+17
-3
backend/services/version-checker.ts
+17
-3
backend/services/version-checker.ts
···
4
4
version: string | null;
5
5
etag: string | null;
6
6
notModified: boolean;
7
+
error: boolean;
7
8
};
8
9
9
10
/** Fetch latest PDS version from GitHub with ETag caching. */
···
15
16
16
17
const headers: Record<string, string> = {
17
18
"Accept": "application/vnd.github.v3+json",
19
+
"User-Agent": "OpenPDS/1.0 (github.com/tijs/openpds)",
18
20
};
19
21
if (previousEtag) {
20
22
headers["If-None-Match"] = previousEtag;
···
28
30
version: null,
29
31
etag: etag ?? previousEtag ?? null,
30
32
notModified: true,
33
+
error: false,
31
34
};
32
35
}
33
36
34
37
if (!resp.ok) {
35
38
console.error(`GitHub API error: ${resp.status}`);
36
-
return { version: null, etag: etag ?? null, notModified: false };
39
+
// Preserve previous etag on error so we can retry conditional request next time
40
+
return {
41
+
version: null,
42
+
etag: previousEtag ?? null,
43
+
notModified: false,
44
+
error: true,
45
+
};
37
46
}
38
47
39
48
const tags: Array<{ name: string }> = await resp.json();
40
49
if (tags.length === 0) {
41
-
return { version: null, etag: etag ?? null, notModified: false };
50
+
return {
51
+
version: null,
52
+
etag: etag ?? null,
53
+
notModified: false,
54
+
error: false,
55
+
};
42
56
}
43
57
44
58
// Tags are like "v0.4.74" — strip the "v" prefix to match _health output
45
59
const version = tags[0].name.replace(/^v/, "");
46
-
return { version, etag: etag ?? null, notModified: false };
60
+
return { version, etag: etag ?? null, notModified: false, error: false };
47
61
}
+11
-3
cron/refresh.cron.ts
+11
-3
cron/refresh.cron.ts
···
48
48
49
49
// 2. Fetch latest PDS version (with ETag caching)
50
50
const previousGithubEtag = await getMetadata("github_tags_etag");
51
-
const { version: latestVersion, etag: githubEtag, notModified } =
52
-
await fetchLatestPdsVersion(previousGithubEtag);
51
+
const {
52
+
version: latestVersion,
53
+
etag: githubEtag,
54
+
notModified,
55
+
error: githubError,
56
+
} = await fetchLatestPdsVersion(previousGithubEtag);
53
57
54
58
if (githubEtag) {
55
59
await setMetadata("github_tags_etag", githubEtag);
···
64
68
console.log(
65
69
`GitHub tags: new version ${latestVersion} (etag=${githubEtag})`,
66
70
);
71
+
} else if (githubError) {
72
+
console.log("GitHub tags: API error (using cached version)");
67
73
} else {
68
74
console.log("GitHub tags: no version found");
69
75
}
···
95
101
});
96
102
}
97
103
104
+
const noUserCount = enriched.filter((s) => s.userCount === null).length;
98
105
console.log(
99
106
`Enriched ${enriched.length} servers: ` +
100
107
`DNS ${stats.cachedDns} cached/${stats.freshDns} fresh, ` +
101
-
`geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh`,
108
+
`geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh` +
109
+
(noUserCount > 0 ? `, ${noUserCount} without user count` : ""),
102
110
);
103
111
}
104
112