simple list of pds servers with open registration
Initial commit: AT Protocol PDS directory
Server-rendered directory of open AT Protocol PDS servers.
Daily cron fetches state from atproto-scraping, enriches with
version, geo-IP, user count, and trust signals. Deployed on Val Town.
+1532
+17
.gitignore
+17
.gitignore
+329
AGENTS.md
+329
AGENTS.md
···
1
+
You are an advanced assistant specialized in generating Val Town code.
2
+
3
+
## Core Guidelines
4
+
5
+
- Ask clarifying questions when requirements are ambiguous
6
+
- Provide complete, functional solutions rather than skeleton implementations
7
+
- Test your logic against edge cases before presenting the final solution
8
+
- Ensure all code follows Val Town's specific platform requirements
9
+
- If a section of code that you're working on is getting too complex, consider
10
+
refactoring it into subcomponents
11
+
12
+
## Code Standards
13
+
14
+
- Generate code in TypeScript or TSX
15
+
- Add appropriate TypeScript types and interfaces for all data structures
16
+
- Prefer official SDKs or libraries than writing API calls directly
17
+
- Ask the user to supply API or library documentation if you are at all unsure
18
+
about it
19
+
- **Never bake in secrets into the code** - always use environment variables
20
+
- Include comments explaining complex logic (avoid commenting obvious
21
+
operations)
22
+
- Follow modern ES6+ conventions and functional programming practices if
23
+
possible
24
+
25
+
## Types of triggers
26
+
27
+
### 1. HTTP Trigger
28
+
29
+
- Create web APIs and endpoints
30
+
- Handle HTTP requests and responses
31
+
- Example structure:
32
+
33
+
```ts
34
+
export default async function (req: Request) {
35
+
return new Response("Hello World");
36
+
}
37
+
```
38
+
39
+
Files that are HTTP triggers have http in their name like `foobar.http.tsx`
40
+
41
+
### 2. Cron Triggers
42
+
43
+
- Run on a schedule
44
+
- Use cron expressions for timing
45
+
- Example structure:
46
+
47
+
```ts
48
+
export default async function () {
49
+
// Scheduled task code
50
+
}
51
+
```
52
+
53
+
Files that are Cron triggers have cron in their name like `foobar.cron.tsx`
54
+
55
+
### 3. Email Triggers
56
+
57
+
- Process incoming emails
58
+
- Handle email-based workflows
59
+
- Example structure:
60
+
61
+
```ts
62
+
export default async function (email: Email) {
63
+
// Process email
64
+
}
65
+
```
66
+
67
+
Files that are Email triggers have email in their name like `foobar.email.tsx`
68
+
69
+
## Val Town Standard Libraries
70
+
71
+
Val Town provides several hosted services and utility functions.
72
+
73
+
### Blob Storage
74
+
75
+
```ts
76
+
import { blob } from "https://esm.town/v/std/blob";
77
+
await blob.setJSON("myKey", { hello: "world" });
78
+
let blobDemo = await blob.getJSON("myKey");
79
+
let appKeys = await blob.list("app_");
80
+
await blob.delete("myKey");
81
+
```
82
+
83
+
### SQLite
84
+
85
+
```ts
86
+
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite";
87
+
const TABLE_NAME = "todo_app_users_2";
88
+
// Create table - do this before usage and change table name when modifying schema
89
+
await sqlite.execute(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (
90
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
91
+
name TEXT NOT NULL
92
+
)`);
93
+
// Query data
94
+
const result = await sqlite.execute(
95
+
`SELECT * FROM ${TABLE_NAME} WHERE id = ?`,
96
+
[1],
97
+
);
98
+
```
99
+
100
+
Note: When changing a SQLite table's schema, change the table's name (e.g., add
101
+
_2 or _3) to create a fresh table.
102
+
103
+
### OpenAI
104
+
105
+
```ts
106
+
import { OpenAI } from "https://esm.town/v/std/openai";
107
+
const openai = new OpenAI();
108
+
const completion = await openai.chat.completions.create({
109
+
messages: [
110
+
{ role: "user", content: "Say hello in a creative way" },
111
+
],
112
+
model: "gpt-4o-mini",
113
+
max_tokens: 30,
114
+
});
115
+
```
116
+
117
+
### Email
118
+
119
+
```ts
120
+
import { email } from "https://esm.town/v/std/email";
121
+
// By default emails the owner of the val
122
+
await email({
123
+
subject: "Hi",
124
+
text: "Hi",
125
+
html: "<h1>Hi</h1>",
126
+
});
127
+
```
128
+
129
+
## Val Town Utility Functions
130
+
131
+
Val Town provides several utility functions to help with common project tasks.
132
+
133
+
### Importing Utilities
134
+
135
+
Always import utilities with version pins to avoid breaking changes:
136
+
137
+
```ts
138
+
import {
139
+
parseProject,
140
+
readFile,
141
+
serveFile,
142
+
} from "https://esm.town/v/std/utils@85-main/index.ts";
143
+
```
144
+
145
+
### Available Utilities
146
+
147
+
#### **serveFile** - Serve project files with proper content types
148
+
149
+
For example, in Hono:
150
+
151
+
```ts
152
+
// serve all files in frontend/ and shared/
153
+
app.get("/frontend/*", (c) => serveFile(c.req.path, import.meta.url));
154
+
app.get("/shared/*", (c) => serveFile(c.req.path, import.meta.url));
155
+
```
156
+
157
+
#### **readFile** - Read files from within the project:
158
+
159
+
```ts
160
+
// Read a file from the project
161
+
const fileContent = await readFile("/frontend/index.html", import.meta.url);
162
+
```
163
+
164
+
#### **listFiles** - List all files in the project
165
+
166
+
```ts
167
+
const files = await listFiles(import.meta.url);
168
+
```
169
+
170
+
#### **parseProject** - Extract information about the current project from import.meta.url
171
+
172
+
This is useful for including info for linking back to a val, ie in "view source"
173
+
urls:
174
+
175
+
```ts
176
+
const projectVal = parseProject(import.meta.url);
177
+
console.log(projectVal.username); // Owner of the project
178
+
console.log(projectVal.name); // Project name
179
+
console.log(projectVal.version); // Version number
180
+
console.log(projectVal.branch); // Branch name
181
+
console.log(projectVal.links.self.project); // URL to the project page
182
+
```
183
+
184
+
However, it's _extremely importing_ to note that `parseProject` and other
185
+
Standard Library utilities ONLY RUN ON THE SERVER. If you need access to this
186
+
data on the client, run it in the server and pass it to the client by splicing
187
+
it into the HTML page or by making an API request for it.
188
+
189
+
## Val Town Platform Specifics
190
+
191
+
- **Redirects:** Use
192
+
`return new Response(null, { status: 302, headers: { Location: "/place/to/redirect" }})`
193
+
instead of `Response.redirect` which is broken
194
+
- **Images:** Avoid external images or base64 images. Use emojis, unicode
195
+
symbols, or icon fonts/libraries instead
196
+
- **AI Image:** To inline generate an AI image use:
197
+
`<img src="https://maxm-imggenurl.web.val.run/the-description-of-your-image" />`
198
+
- **Storage:** DO NOT use the Deno KV module for storage
199
+
- **Browser APIs:** DO NOT use the `alert()`, `prompt()`, or `confirm()` methods
200
+
- **Weather Data:** Use open-meteo for weather data (doesn't require API keys)
201
+
unless otherwise specified
202
+
- **View Source:** Add a view source link by importing & using
203
+
`import.meta.url.replace("ems.sh", "val.town)"` (or passing this data to the
204
+
client) and include `target="_top"` attribute
205
+
- **Error Debugging:** Add
206
+
`<script src="https://esm.town/v/std/catch"></script>` to HTML to capture
207
+
client-side errors
208
+
- **Error Handling:** Only use try...catch when there's a clear local
209
+
resolution; Avoid catches that merely log or return 500s. Let errors bubble up
210
+
with full context
211
+
- **Environment Variables:** Use `Deno.env.get('keyname')` when you need to, but
212
+
generally prefer APIs that don't require keys
213
+
- **Imports:** Use `https://esm.sh` for npm and Deno dependencies to ensure
214
+
compatibility on server and browser
215
+
- **Storage Strategy:** Only use backend storage if explicitly required; prefer
216
+
simple static client-side sites
217
+
- **React Configuration:** When using React libraries, pin versions with
218
+
`?deps=react@18.2.0,react-dom@18.2.0` and start the file with
219
+
`/** @jsxImportSource https://esm.sh/react@18.2.0 */`
220
+
- Ensure all React dependencies and sub-dependencies are pinned to the same
221
+
version
222
+
- **Styling:** Default to using TailwindCSS via
223
+
`<script src="https://cdn.twind.style" crossorigin></script>` unless otherwise
224
+
specified
225
+
226
+
## Project Structure and Design Patterns
227
+
228
+
### Recommended Directory Structure
229
+
230
+
```
231
+
├── backend/
232
+
│ ├── database/
233
+
│ │ ├── migrations.ts # Schema definitions
234
+
│ │ ├── queries.ts # DB query functions
235
+
│ │ └── README.md
236
+
│ └── routes/ # Route modules
237
+
│ ├── [route].ts
238
+
│ └── static.ts # Static file serving
239
+
│ ├── index.ts # Main entry point
240
+
│ └── README.md
241
+
├── frontend/
242
+
│ ├── components/
243
+
│ │ ├── App.tsx
244
+
│ │ └── [Component].tsx
245
+
│ ├── favicon.svg
246
+
│ ├── index.html # Main HTML template
247
+
│ ├── index.tsx # Frontend JS entry point
248
+
│ ├── README.md
249
+
│ └── style.css
250
+
├── README.md
251
+
└── shared/
252
+
├── README.md
253
+
└── utils.ts # Shared types and functions
254
+
```
255
+
256
+
### Backend (Hono) Best Practices
257
+
258
+
- Hono is the recommended API framework
259
+
- Main entry point should be `backend/index.ts`
260
+
- **Static asset serving:** Use the utility functions to read and serve project
261
+
files:
262
+
```ts
263
+
import {
264
+
readFile,
265
+
serveFile,
266
+
} from "https://esm.town/v/std/utils@85-main/index.ts";
267
+
268
+
// serve all files in frontend/ and shared/
269
+
app.get("/frontend/*", (c) => serveFile(c.req.path, import.meta.url));
270
+
app.get("/shared/*", (c) => serveFile(c.req.path, import.meta.url));
271
+
272
+
// For index.html, often you'll want to bootstrap with initial data
273
+
app.get("/", async (c) => {
274
+
let html = await readFile("/frontend/index.html", import.meta.url);
275
+
276
+
// Inject data to avoid extra round-trips
277
+
const initialData = await fetchInitialData();
278
+
const dataScript = `<script>
279
+
window.__INITIAL_DATA__ = ${JSON.stringify(initialData)};
280
+
</script>`;
281
+
282
+
html = html.replace("</head>", `${dataScript}</head>`);
283
+
return c.html(html);
284
+
});
285
+
```
286
+
- Create RESTful API routes for CRUD operations
287
+
- Always include this snippet at the top-level Hono app to re-throwing errors to
288
+
see full stack traces:
289
+
```ts
290
+
// Unwrap Hono errors to see original error details
291
+
app.onError((err, c) => {
292
+
throw err;
293
+
});
294
+
```
295
+
296
+
### Database Patterns
297
+
298
+
- Run migrations on startup or comment out for performance
299
+
- Change table names when modifying schemas rather than altering
300
+
- Export clear query functions with proper TypeScript typing
301
+
302
+
## Common Gotchas and Solutions
303
+
304
+
1. **Environment Limitations:**
305
+
- Val Town runs on Deno in a serverless context, not Node.js
306
+
- Code in `shared/` must work in both frontend and backend environments
307
+
- Cannot use `Deno` keyword in shared code
308
+
- Use `https://esm.sh` for imports that work in both environments
309
+
310
+
2. **SQLite Peculiarities:**
311
+
- Limited support for ALTER TABLE operations
312
+
- Create new tables with updated schemas and copy data when needed
313
+
- Always run table creation before querying
314
+
315
+
3. **React Configuration:**
316
+
- All React dependencies must be pinned to 18.2.0
317
+
- Always include `@jsxImportSource https://esm.sh/react@18.2.0` at the top of
318
+
React files
319
+
- Rendering issues often come from mismatched React versions
320
+
321
+
4. **File Handling:**
322
+
- Val Town only supports text files, not binary
323
+
- Use the provided utilities to read files across branches and forks
324
+
- For files in the project, use `readFile` helpers
325
+
326
+
5. **API Design:**
327
+
- `fetch` handler is the entry point for HTTP vals
328
+
- Run the Hono app with
329
+
`export default app.fetch // This is the entry point for HTTP vals`
+42
README.md
+42
README.md
···
1
+
# OpenPDS
2
+
3
+
Directory of AT Protocol PDS servers with open registration.
4
+
5
+
Pulls data from
6
+
[mary-ext/atproto-scraping](https://github.com/mary-ext/atproto-scraping),
7
+
enriches it with server metadata (version, user count, geo-location, contact
8
+
info), and presents a browsable directory.
9
+
10
+
## How it works
11
+
12
+
A daily cron job:
13
+
14
+
1. Fetches `state.json` listing ~2900 known PDSes with signup status
15
+
2. Syncs open/closed status to a SQLite database
16
+
3. Enriches a batch of 20 servers per run (health, describeServer, listRepos,
17
+
geo-IP)
18
+
4. Checks for the latest PDS version from the `bluesky-social/pds` GitHub repo
19
+
20
+
The web interface shows open servers with sortable columns, version badges, and
21
+
trust signals.
22
+
23
+
## Endpoints
24
+
25
+
- `/` — HTML directory page
26
+
- `/api/servers` — JSON API with all open servers and metadata
27
+
28
+
## Development
29
+
30
+
```bash
31
+
deno task check # type check
32
+
deno task quality # fmt + lint + check
33
+
deno task deploy # quality + vt push
34
+
```
35
+
36
+
## Data sources
37
+
38
+
- [atproto-scraping state.json](https://github.com/mary-ext/atproto-scraping) —
39
+
PDS list and signup status
40
+
- [dns.google](https://dns.google) — hostname to IP resolution
41
+
- [ip-api.com](http://ip-api.com) — IP to country geo-lookup
42
+
- [GitHub API](https://api.github.com) — latest PDS version tag
+44
backend/database/migrations.ts
+44
backend/database/migrations.ts
···
1
+
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite?v=13";
2
+
import { METADATA_TABLE, SERVERS_TABLE } from "../../shared/constants.ts";
3
+
4
+
export async function runMigrations(): Promise<void> {
5
+
await sqlite.execute(`
6
+
CREATE TABLE IF NOT EXISTS ${SERVERS_TABLE} (
7
+
url TEXT PRIMARY KEY,
8
+
first_seen TEXT NOT NULL,
9
+
last_seen TEXT NOT NULL,
10
+
last_enriched TEXT,
11
+
version TEXT,
12
+
did TEXT,
13
+
invite_code_required INTEGER NOT NULL DEFAULT 0,
14
+
phone_verification INTEGER NOT NULL DEFAULT 0,
15
+
user_domains TEXT,
16
+
contact_email TEXT,
17
+
privacy_policy TEXT,
18
+
terms_of_service TEXT,
19
+
user_count INTEGER,
20
+
country_code TEXT,
21
+
country_name TEXT,
22
+
ip_address TEXT,
23
+
is_open INTEGER NOT NULL DEFAULT 0,
24
+
error_at INTEGER
25
+
)
26
+
`);
27
+
28
+
await sqlite.execute(`
29
+
CREATE INDEX IF NOT EXISTS idx_${SERVERS_TABLE}_is_open
30
+
ON ${SERVERS_TABLE} (is_open)
31
+
`);
32
+
33
+
await sqlite.execute(`
34
+
CREATE INDEX IF NOT EXISTS idx_${SERVERS_TABLE}_last_enriched
35
+
ON ${SERVERS_TABLE} (last_enriched)
36
+
`);
37
+
38
+
await sqlite.execute(`
39
+
CREATE TABLE IF NOT EXISTS ${METADATA_TABLE} (
40
+
key TEXT PRIMARY KEY,
41
+
value TEXT NOT NULL
42
+
)
43
+
`);
44
+
}
+220
backend/database/queries.ts
+220
backend/database/queries.ts
···
1
+
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite?v=13";
2
+
import { METADATA_TABLE, SERVERS_TABLE } from "../../shared/constants.ts";
3
+
import type {
4
+
PdsServer,
5
+
SortDirection,
6
+
SortField,
7
+
} from "../../shared/types.ts";
8
+
9
+
// --- Server operations ---
10
+
11
+
export async function upsertServer(
12
+
url: string,
13
+
opts: {
14
+
inviteCodeRequired: boolean;
15
+
version: string | null;
16
+
errorAt: number | null;
17
+
isOpen: boolean;
18
+
},
19
+
): Promise<void> {
20
+
const now = new Date().toISOString();
21
+
await sqlite.execute({
22
+
sql: `
23
+
INSERT INTO ${SERVERS_TABLE} (url, first_seen, last_seen, invite_code_required, version, error_at, is_open)
24
+
VALUES (?, ?, ?, ?, ?, ?, ?)
25
+
ON CONFLICT(url) DO UPDATE SET
26
+
last_seen = ?,
27
+
invite_code_required = ?,
28
+
version = COALESCE(?, version),
29
+
error_at = ?,
30
+
is_open = ?
31
+
`,
32
+
args: [
33
+
url,
34
+
now,
35
+
now,
36
+
opts.inviteCodeRequired ? 1 : 0,
37
+
opts.version,
38
+
opts.errorAt,
39
+
opts.isOpen ? 1 : 0,
40
+
now,
41
+
opts.inviteCodeRequired ? 1 : 0,
42
+
opts.version,
43
+
opts.errorAt,
44
+
opts.isOpen ? 1 : 0,
45
+
],
46
+
});
47
+
}
48
+
49
+
export async function updateEnrichment(
50
+
url: string,
51
+
data: {
52
+
version?: string | null;
53
+
did?: string | null;
54
+
phoneVerification?: boolean;
55
+
userDomains?: string[];
56
+
contactEmail?: string | null;
57
+
privacyPolicy?: string | null;
58
+
termsOfService?: string | null;
59
+
userCount?: number | null;
60
+
countryCode?: string | null;
61
+
countryName?: string | null;
62
+
ipAddress?: string | null;
63
+
},
64
+
): Promise<void> {
65
+
const now = new Date().toISOString();
66
+
await sqlite.execute({
67
+
sql: `
68
+
UPDATE ${SERVERS_TABLE} SET
69
+
last_enriched = ?,
70
+
version = COALESCE(?, version),
71
+
did = COALESCE(?, did),
72
+
phone_verification = ?,
73
+
user_domains = ?,
74
+
contact_email = ?,
75
+
privacy_policy = ?,
76
+
terms_of_service = ?,
77
+
user_count = COALESCE(?, user_count),
78
+
country_code = COALESCE(?, country_code),
79
+
country_name = COALESCE(?, country_name),
80
+
ip_address = COALESCE(?, ip_address)
81
+
WHERE url = ?
82
+
`,
83
+
args: [
84
+
now,
85
+
data.version ?? null,
86
+
data.did ?? null,
87
+
data.phoneVerification ? 1 : 0,
88
+
data.userDomains ? JSON.stringify(data.userDomains) : null,
89
+
data.contactEmail ?? null,
90
+
data.privacyPolicy ?? null,
91
+
data.termsOfService ?? null,
92
+
data.userCount ?? null,
93
+
data.countryCode ?? null,
94
+
data.countryName ?? null,
95
+
data.ipAddress ?? null,
96
+
url,
97
+
],
98
+
});
99
+
}
100
+
101
+
export async function getServersToEnrich(limit: number): Promise<string[]> {
102
+
const result = await sqlite.execute({
103
+
sql: `
104
+
SELECT url FROM ${SERVERS_TABLE}
105
+
WHERE is_open = 1
106
+
ORDER BY last_enriched ASC NULLS FIRST
107
+
LIMIT ?
108
+
`,
109
+
args: [limit],
110
+
});
111
+
return result.rows.map((r) => r.url as string);
112
+
}
113
+
114
+
export async function getOpenServers(
115
+
sort: SortField = "users",
116
+
dir: SortDirection = "desc",
117
+
showOutdated: boolean = false,
118
+
latestVersion: string | null = null,
119
+
): Promise<PdsServer[]> {
120
+
let orderClause: string;
121
+
switch (sort) {
122
+
case "users":
123
+
orderClause = `user_count ${dir} NULLS LAST`;
124
+
break;
125
+
case "country":
126
+
orderClause = `country_name ${dir} NULLS LAST`;
127
+
break;
128
+
case "first_seen":
129
+
orderClause = `first_seen ${dir}`;
130
+
break;
131
+
case "version":
132
+
orderClause = `version ${dir} NULLS LAST`;
133
+
break;
134
+
default:
135
+
orderClause = "user_count DESC NULLS LAST";
136
+
}
137
+
138
+
let whereClause = "is_open = 1";
139
+
const args: (string | number)[] = [];
140
+
141
+
if (!showOutdated && latestVersion) {
142
+
whereClause += " AND (version = ? OR version IS NULL)";
143
+
args.push(latestVersion);
144
+
}
145
+
146
+
const result = await sqlite.execute({
147
+
sql:
148
+
`SELECT * FROM ${SERVERS_TABLE} WHERE ${whereClause} ORDER BY ${orderClause}`,
149
+
args,
150
+
});
151
+
152
+
return result.rows.map(rowToServer);
153
+
}
154
+
155
+
export async function getAllServers(): Promise<PdsServer[]> {
156
+
const result = await sqlite.execute(
157
+
`SELECT * FROM ${SERVERS_TABLE} WHERE is_open = 1 ORDER BY user_count DESC NULLS LAST`,
158
+
);
159
+
return result.rows.map(rowToServer);
160
+
}
161
+
162
+
export async function getServerCount(): Promise<
163
+
{ open: number; total: number }
164
+
> {
165
+
const result = await sqlite.execute(
166
+
`SELECT COUNT(*) as total, SUM(CASE WHEN is_open = 1 THEN 1 ELSE 0 END) as open FROM ${SERVERS_TABLE}`,
167
+
);
168
+
const row = result.rows[0];
169
+
return {
170
+
total: (row.total as number) || 0,
171
+
open: (row.open as number) || 0,
172
+
};
173
+
}
174
+
175
+
// --- Metadata operations ---
176
+
177
+
export async function getMetadata(key: string): Promise<string | null> {
178
+
const result = await sqlite.execute({
179
+
sql: `SELECT value FROM ${METADATA_TABLE} WHERE key = ?`,
180
+
args: [key],
181
+
});
182
+
return result.rows.length > 0 ? (result.rows[0].value as string) : null;
183
+
}
184
+
185
+
export async function setMetadata(key: string, value: string): Promise<void> {
186
+
await sqlite.execute({
187
+
sql: `
188
+
INSERT INTO ${METADATA_TABLE} (key, value) VALUES (?, ?)
189
+
ON CONFLICT(key) DO UPDATE SET value = ?
190
+
`,
191
+
args: [key, value, value],
192
+
});
193
+
}
194
+
195
+
// --- Helpers ---
196
+
197
+
function rowToServer(row: Record<string, unknown>): PdsServer {
198
+
return {
199
+
url: row.url as string,
200
+
first_seen: row.first_seen as string,
201
+
last_seen: row.last_seen as string,
202
+
last_enriched: (row.last_enriched as string) || null,
203
+
version: (row.version as string) || null,
204
+
did: (row.did as string) || null,
205
+
invite_code_required: Boolean(row.invite_code_required),
206
+
phone_verification: Boolean(row.phone_verification),
207
+
user_domains: row.user_domains
208
+
? JSON.parse(row.user_domains as string)
209
+
: [],
210
+
contact_email: (row.contact_email as string) || null,
211
+
privacy_policy: (row.privacy_policy as string) || null,
212
+
terms_of_service: (row.terms_of_service as string) || null,
213
+
user_count: row.user_count != null ? Number(row.user_count) : null,
214
+
country_code: (row.country_code as string) || null,
215
+
country_name: (row.country_name as string) || null,
216
+
ip_address: (row.ip_address as string) || null,
217
+
is_open: Boolean(row.is_open),
218
+
error_at: row.error_at != null ? Number(row.error_at) : null,
219
+
};
220
+
}
+24
backend/index.http.ts
+24
backend/index.http.ts
···
1
+
import { Hono } from "https://esm.sh/hono@4.4.0";
2
+
import { runMigrations } from "./database/migrations.ts";
3
+
import { pages } from "./routes/pages.ts";
4
+
import { api } from "./routes/api.ts";
5
+
6
+
const app = new Hono();
7
+
8
+
app.onError((err, _c) => {
9
+
throw err;
10
+
});
11
+
12
+
let migrationsRan = false;
13
+
app.use("*", async (_c, next) => {
14
+
if (!migrationsRan) {
15
+
await runMigrations();
16
+
migrationsRan = true;
17
+
}
18
+
await next();
19
+
});
20
+
21
+
app.route("/api", api);
22
+
app.route("/", pages);
23
+
24
+
export default app.fetch;
+28
backend/routes/api.ts
+28
backend/routes/api.ts
···
1
+
import { Hono } from "https://esm.sh/hono@4.4.0";
2
+
import {
3
+
getAllServers,
4
+
getMetadata,
5
+
getServerCount,
6
+
} from "../database/queries.ts";
7
+
8
+
const api = new Hono();
9
+
10
+
api.get("/servers", async (c) => {
11
+
const [servers, latestVersion, counts] = await Promise.all([
12
+
getAllServers(),
13
+
getMetadata("latest_pds_version"),
14
+
getServerCount(),
15
+
]);
16
+
17
+
return c.json({
18
+
meta: {
19
+
latest_pds_version: latestVersion,
20
+
total_known: counts.total,
21
+
total_open: counts.open,
22
+
updated_at: new Date().toISOString(),
23
+
},
24
+
servers,
25
+
});
26
+
});
27
+
28
+
export { api };
+334
backend/routes/pages.ts
+334
backend/routes/pages.ts
···
1
+
import { Hono } from "https://esm.sh/hono@4.4.0";
2
+
import {
3
+
getMetadata,
4
+
getOpenServers,
5
+
getServerCount,
6
+
} from "../database/queries.ts";
7
+
import { countryFlag } from "../../shared/constants.ts";
8
+
import type {
9
+
PdsServer,
10
+
SortDirection,
11
+
SortField,
12
+
} from "../../shared/types.ts";
13
+
14
+
const pages = new Hono();
15
+
16
+
pages.get("/", async (c) => {
17
+
const sort = (c.req.query("sort") || "users") as SortField;
18
+
const dir = (c.req.query("dir") || "desc") as SortDirection;
19
+
const showOutdated = c.req.query("show_outdated") === "true";
20
+
21
+
const [latestVersion, counts] = await Promise.all([
22
+
getMetadata("latest_pds_version"),
23
+
getServerCount(),
24
+
]);
25
+
26
+
const servers = await getOpenServers(sort, dir, showOutdated, latestVersion);
27
+
28
+
return c.html(renderPage(servers, {
29
+
sort,
30
+
dir,
31
+
showOutdated,
32
+
latestVersion,
33
+
openCount: counts.open,
34
+
totalCount: counts.total,
35
+
}));
36
+
});
37
+
38
+
type PageOpts = {
39
+
sort: SortField;
40
+
dir: SortDirection;
41
+
showOutdated: boolean;
42
+
latestVersion: string | null;
43
+
openCount: number;
44
+
totalCount: number;
45
+
};
46
+
47
+
function renderPage(servers: PdsServer[], opts: PageOpts): string {
48
+
const { sort, dir, showOutdated, latestVersion, openCount, totalCount } =
49
+
opts;
50
+
51
+
function sortUrl(field: SortField): string {
52
+
const newDir = sort === field && dir === "desc" ? "asc" : "desc";
53
+
const params = new URLSearchParams();
54
+
params.set("sort", field);
55
+
params.set("dir", newDir);
56
+
if (showOutdated) params.set("show_outdated", "true");
57
+
return `/?${params}`;
58
+
}
59
+
60
+
function sortIndicator(field: SortField): string {
61
+
if (sort !== field) return "";
62
+
return dir === "asc" ? " ▲" : " ▼";
63
+
}
64
+
65
+
const toggleUrl = (() => {
66
+
const params = new URLSearchParams();
67
+
params.set("sort", sort);
68
+
params.set("dir", dir);
69
+
if (!showOutdated) params.set("show_outdated", "true");
70
+
return `/?${params}`;
71
+
})();
72
+
73
+
const rows = servers.map((s) => renderRow(s, latestVersion)).join("\n");
74
+
75
+
return `<!DOCTYPE html>
76
+
<html lang="en">
77
+
<head>
78
+
<meta charset="utf-8">
79
+
<meta name="viewport" content="width=device-width, initial-scale=1">
80
+
<title>OpenPDS - AT Protocol PDS Directory</title>
81
+
<style>
82
+
:root {
83
+
--bg: #fafafa;
84
+
--fg: #1a1a1a;
85
+
--muted: #666;
86
+
--border: #e0e0e0;
87
+
--hover: #f0f0f0;
88
+
--green: #16a34a;
89
+
--amber: #d97706;
90
+
--red: #dc2626;
91
+
--blue: #2563eb;
92
+
}
93
+
@media (prefers-color-scheme: dark) {
94
+
:root {
95
+
--bg: #111;
96
+
--fg: #e5e5e5;
97
+
--muted: #999;
98
+
--border: #333;
99
+
--hover: #1a1a1a;
100
+
}
101
+
}
102
+
* { margin: 0; padding: 0; box-sizing: border-box; }
103
+
body {
104
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
105
+
background: var(--bg);
106
+
color: var(--fg);
107
+
line-height: 1.5;
108
+
padding: 2rem 1rem;
109
+
max-width: 1100px;
110
+
margin: 0 auto;
111
+
}
112
+
h1 { font-size: 1.5rem; margin-bottom: 0.25rem; }
113
+
.subtitle { color: var(--muted); margin-bottom: 1.5rem; font-size: 0.9rem; }
114
+
.stats {
115
+
display: flex;
116
+
gap: 1.5rem;
117
+
margin-bottom: 1rem;
118
+
font-size: 0.85rem;
119
+
color: var(--muted);
120
+
}
121
+
.controls {
122
+
display: flex;
123
+
justify-content: flex-end;
124
+
margin-bottom: 0.5rem;
125
+
}
126
+
.controls a {
127
+
font-size: 0.8rem;
128
+
color: var(--blue);
129
+
text-decoration: none;
130
+
}
131
+
.controls a:hover { text-decoration: underline; }
132
+
table {
133
+
width: 100%;
134
+
border-collapse: collapse;
135
+
font-size: 0.85rem;
136
+
}
137
+
th {
138
+
text-align: left;
139
+
padding: 0.5rem 0.75rem;
140
+
border-bottom: 2px solid var(--border);
141
+
font-weight: 600;
142
+
white-space: nowrap;
143
+
}
144
+
th a {
145
+
color: var(--fg);
146
+
text-decoration: none;
147
+
}
148
+
th a:hover { color: var(--blue); }
149
+
td {
150
+
padding: 0.5rem 0.75rem;
151
+
border-bottom: 1px solid var(--border);
152
+
vertical-align: top;
153
+
}
154
+
tr:hover { background: var(--hover); }
155
+
a { color: var(--blue); }
156
+
.hostname { font-weight: 500; }
157
+
.version-badge {
158
+
display: inline-block;
159
+
padding: 0.1rem 0.4rem;
160
+
border-radius: 3px;
161
+
font-size: 0.75rem;
162
+
font-family: monospace;
163
+
}
164
+
.version-current { background: #dcfce7; color: var(--green); }
165
+
.version-outdated { background: #fef3c7; color: var(--amber); }
166
+
.version-unknown { background: #f3f4f6; color: var(--muted); }
167
+
@media (prefers-color-scheme: dark) {
168
+
.version-current { background: #052e16; }
169
+
.version-outdated { background: #451a03; }
170
+
.version-unknown { background: #1f2937; }
171
+
}
172
+
.trust-icons { display: flex; gap: 0.25rem; }
173
+
.trust-icon {
174
+
width: 16px;
175
+
height: 16px;
176
+
display: inline-flex;
177
+
align-items: center;
178
+
justify-content: center;
179
+
font-size: 0.7rem;
180
+
border-radius: 2px;
181
+
background: #f3f4f6;
182
+
color: var(--muted);
183
+
text-decoration: none;
184
+
title: attr(title);
185
+
}
186
+
.trust-icon.active { background: #dcfce7; color: var(--green); }
187
+
@media (prefers-color-scheme: dark) {
188
+
.trust-icon { background: #1f2937; }
189
+
.trust-icon.active { background: #052e16; }
190
+
}
191
+
.empty { text-align: center; padding: 3rem 1rem; color: var(--muted); }
192
+
footer {
193
+
margin-top: 2rem;
194
+
padding-top: 1rem;
195
+
border-top: 1px solid var(--border);
196
+
font-size: 0.8rem;
197
+
color: var(--muted);
198
+
}
199
+
footer a { color: var(--muted); }
200
+
@media (max-width: 768px) {
201
+
body { padding: 1rem 0.5rem; }
202
+
.hide-mobile { display: none; }
203
+
td, th { padding: 0.4rem; }
204
+
}
205
+
</style>
206
+
</head>
207
+
<body>
208
+
<h1>OpenPDS</h1>
209
+
<p class="subtitle">
210
+
AT Protocol PDS servers with open registration.
211
+
Data from <a href="https://github.com/mary-ext/atproto-scraping">atproto-scraping</a>,
212
+
enriched daily.
213
+
</p>
214
+
<div class="stats">
215
+
<span>${servers.length} servers listed</span>
216
+
<span>${openCount} open of ${totalCount} known</span>
217
+
${latestVersion ? `<span>Latest PDS: ${esc(latestVersion)}</span>` : ""}
218
+
</div>
219
+
<div class="controls">
220
+
<a href="${esc(toggleUrl)}">${
221
+
showOutdated ? "Hide outdated versions" : "Show outdated versions"
222
+
}</a>
223
+
</div>
224
+
${
225
+
servers.length > 0
226
+
? `
227
+
<table>
228
+
<thead>
229
+
<tr>
230
+
<th><a href="${esc(sortUrl("users"))}">Hostname</a></th>
231
+
<th class="hide-mobile"><a href="${esc(sortUrl("country"))}">Country${
232
+
sortIndicator("country")
233
+
}</a></th>
234
+
<th><a href="${esc(sortUrl("version"))}">Version${
235
+
sortIndicator("version")
236
+
}</a></th>
237
+
<th><a href="${esc(sortUrl("users"))}">Users${
238
+
sortIndicator("users")
239
+
}</a></th>
240
+
<th class="hide-mobile"><a href="${
241
+
esc(sortUrl("first_seen"))
242
+
}">First seen${sortIndicator("first_seen")}</a></th>
243
+
<th class="hide-mobile">Trust</th>
244
+
</tr>
245
+
</thead>
246
+
<tbody>
247
+
${rows}
248
+
</tbody>
249
+
</table>`
250
+
: `<div class="empty">No servers found. Data may still be loading — check back after the first cron run.</div>`
251
+
}
252
+
<footer>
253
+
<a href="/api/servers">JSON API</a>
254
+
· Source on <a href="https://github.com/tijs">GitHub</a>
255
+
</footer>
256
+
</body>
257
+
</html>`;
258
+
}
259
+
260
+
function renderRow(s: PdsServer, latestVersion: string | null): string {
261
+
const hostname = new URL(s.url).hostname;
262
+
const flag = s.country_code ? countryFlag(s.country_code) + " " : "";
263
+
const country = s.country_name || "";
264
+
265
+
let versionClass = "version-unknown";
266
+
let versionText = "unknown";
267
+
if (s.version) {
268
+
versionText = s.version;
269
+
versionClass = s.version === latestVersion
270
+
? "version-current"
271
+
: "version-outdated";
272
+
}
273
+
274
+
const users = s.user_count != null
275
+
? (s.user_count >= 1000 ? "1000+" : String(s.user_count))
276
+
: "-";
277
+
278
+
const firstSeen = s.first_seen
279
+
? new Date(s.first_seen).toLocaleDateString("en-US", {
280
+
month: "short",
281
+
day: "numeric",
282
+
year: "numeric",
283
+
})
284
+
: "-";
285
+
286
+
const hasEmail = s.contact_email;
287
+
const hasTos = s.terms_of_service;
288
+
const hasPrivacy = s.privacy_policy;
289
+
290
+
return `<tr>
291
+
<td class="hostname"><a href="${
292
+
esc(s.url)
293
+
}" target="_blank" rel="noopener">${esc(hostname)}</a></td>
294
+
<td class="hide-mobile">${flag}${esc(country)}</td>
295
+
<td><span class="version-badge ${versionClass}">${
296
+
esc(versionText)
297
+
}</span></td>
298
+
<td>${esc(users)}</td>
299
+
<td class="hide-mobile">${esc(firstSeen)}</td>
300
+
<td class="hide-mobile">
301
+
<span class="trust-icons">
302
+
${
303
+
hasEmail
304
+
? `<span class="trust-icon active" title="Contact email">@</span>`
305
+
: `<span class="trust-icon" title="No contact email">@</span>`
306
+
}
307
+
${
308
+
hasTos
309
+
? `<a class="trust-icon active" href="${
310
+
esc(hasTos)
311
+
}" title="Terms of Service" target="_blank" rel="noopener">T</a>`
312
+
: `<span class="trust-icon" title="No Terms of Service">T</span>`
313
+
}
314
+
${
315
+
hasPrivacy
316
+
? `<a class="trust-icon active" href="${
317
+
esc(hasPrivacy)
318
+
}" title="Privacy Policy" target="_blank" rel="noopener">P</a>`
319
+
: `<span class="trust-icon" title="No Privacy Policy">P</span>`
320
+
}
321
+
</span>
322
+
</td>
323
+
</tr>`;
324
+
}
325
+
326
+
function esc(str: string): string {
327
+
return str
328
+
.replace(/&/g, "&")
329
+
.replace(/</g, "<")
330
+
.replace(/>/g, ">")
331
+
.replace(/"/g, """);
332
+
}
333
+
334
+
export { pages };
+78
backend/services/geo-resolver.ts
+78
backend/services/geo-resolver.ts
···
1
+
import type { DnsResponse, GeoIpResult } from "../../shared/types.ts";
2
+
3
+
/** Resolve hostname to IP address via DNS-over-HTTPS */
4
+
export async function resolveIp(hostname: string): Promise<string | null> {
5
+
try {
6
+
const resp = await fetch(
7
+
`https://dns.google/resolve?name=${encodeURIComponent(hostname)}&type=A`,
8
+
{ signal: AbortSignal.timeout(5000) },
9
+
);
10
+
if (!resp.ok) return null;
11
+
12
+
const data: DnsResponse = await resp.json();
13
+
if (!data.Answer || data.Answer.length === 0) return null;
14
+
15
+
// Return the last A record (usually the actual IP after CNAMEs)
16
+
return data.Answer[data.Answer.length - 1].data;
17
+
} catch {
18
+
return null;
19
+
}
20
+
}
21
+
22
+
/** Batch geo-IP lookup for up to 100 IPs */
23
+
export async function batchGeoLookup(
24
+
ipMap: Map<string, string>,
25
+
): Promise<Map<string, { countryCode: string; countryName: string }>> {
26
+
const results = new Map<
27
+
string,
28
+
{ countryCode: string; countryName: string }
29
+
>();
30
+
if (ipMap.size === 0) return results;
31
+
32
+
// ip-api.com batch endpoint: POST array of IPs
33
+
const ips = [...new Set(ipMap.values())].filter(Boolean);
34
+
if (ips.length === 0) return results;
35
+
36
+
try {
37
+
const resp = await fetch(
38
+
"http://ip-api.com/batch?fields=status,query,country,countryCode",
39
+
{
40
+
method: "POST",
41
+
headers: { "Content-Type": "application/json" },
42
+
body: JSON.stringify(ips),
43
+
signal: AbortSignal.timeout(10000),
44
+
},
45
+
);
46
+
47
+
if (!resp.ok) {
48
+
console.error(`ip-api.com batch error: ${resp.status}`);
49
+
return results;
50
+
}
51
+
52
+
const geoResults: GeoIpResult[] = await resp.json();
53
+
54
+
// Build IP → geo map
55
+
const ipToGeo = new Map<
56
+
string,
57
+
{ countryCode: string; countryName: string }
58
+
>();
59
+
for (const r of geoResults) {
60
+
if (r.status === "success") {
61
+
ipToGeo.set(r.query, {
62
+
countryCode: r.countryCode,
63
+
countryName: r.country,
64
+
});
65
+
}
66
+
}
67
+
68
+
// Map hostnames back to geo data via their IPs
69
+
for (const [hostname, ip] of ipMap) {
70
+
const geo = ipToGeo.get(ip);
71
+
if (geo) results.set(hostname, geo);
72
+
}
73
+
} catch (err) {
74
+
console.error("Geo lookup failed:", err);
75
+
}
76
+
77
+
return results;
78
+
}
+159
backend/services/pds-enricher.ts
+159
backend/services/pds-enricher.ts
···
1
+
import { PDS_REQUEST_TIMEOUT_MS } from "../../shared/constants.ts";
2
+
import type {
3
+
DescribeServerResponse,
4
+
HealthResponse,
5
+
} from "../../shared/types.ts";
6
+
import { batchGeoLookup, resolveIp } from "./geo-resolver.ts";
7
+
8
+
export type EnrichmentResult = {
9
+
url: string;
10
+
version: string | null;
11
+
did: string | null;
12
+
phoneVerification: boolean;
13
+
userDomains: string[];
14
+
contactEmail: string | null;
15
+
privacyPolicy: string | null;
16
+
termsOfService: string | null;
17
+
userCount: number | null;
18
+
ipAddress: string | null;
19
+
countryCode: string | null;
20
+
countryName: string | null;
21
+
};
22
+
23
+
/** Enrich a batch of PDS URLs with metadata */
24
+
export async function enrichBatch(urls: string[]): Promise<EnrichmentResult[]> {
25
+
// Step 1: Fetch PDS endpoints concurrently
26
+
const pdsResults = await Promise.allSettled(
27
+
urls.map((url) => enrichSinglePds(url)),
28
+
);
29
+
30
+
const results: EnrichmentResult[] = [];
31
+
const ipMap = new Map<string, string>();
32
+
33
+
for (let i = 0; i < urls.length; i++) {
34
+
const r = pdsResults[i];
35
+
if (r.status === "fulfilled") {
36
+
results.push(r.value);
37
+
if (r.value.ipAddress) {
38
+
ipMap.set(r.value.url, r.value.ipAddress);
39
+
}
40
+
} else {
41
+
console.error(`Enrichment failed for ${urls[i]}:`, r.reason);
42
+
// Return a minimal result so we still update last_enriched
43
+
results.push(emptyResult(urls[i]));
44
+
}
45
+
}
46
+
47
+
// Step 2: Batch geo-IP lookup
48
+
const geoData = await batchGeoLookup(ipMap);
49
+
for (const result of results) {
50
+
const geo = geoData.get(result.url);
51
+
if (geo) {
52
+
result.countryCode = geo.countryCode;
53
+
result.countryName = geo.countryName;
54
+
}
55
+
}
56
+
57
+
return results;
58
+
}
59
+
60
+
async function enrichSinglePds(url: string): Promise<EnrichmentResult> {
61
+
const result = emptyResult(url);
62
+
const signal = AbortSignal.timeout(PDS_REQUEST_TIMEOUT_MS);
63
+
64
+
// Resolve IP from hostname
65
+
const hostname = new URL(url).hostname;
66
+
result.ipAddress = await resolveIp(hostname);
67
+
68
+
// Fetch health, describeServer, and listRepos concurrently
69
+
const [health, describe, userCount] = await Promise.allSettled([
70
+
fetchHealth(url, signal),
71
+
fetchDescribeServer(url, signal),
72
+
fetchUserCount(url, signal),
73
+
]);
74
+
75
+
if (health.status === "fulfilled" && health.value) {
76
+
result.version = health.value.version ?? null;
77
+
}
78
+
79
+
if (describe.status === "fulfilled" && describe.value) {
80
+
const d = describe.value;
81
+
result.did = d.did;
82
+
result.userDomains = d.availableUserDomains ?? [];
83
+
result.phoneVerification = d.phoneVerificationRequired ?? false;
84
+
result.contactEmail = d.contact?.email ?? null;
85
+
result.privacyPolicy = d.links?.privacyPolicy ?? null;
86
+
result.termsOfService = d.links?.termsOfService ?? null;
87
+
}
88
+
89
+
if (userCount.status === "fulfilled") {
90
+
result.userCount = userCount.value;
91
+
}
92
+
93
+
return result;
94
+
}
95
+
96
+
async function fetchHealth(
97
+
url: string,
98
+
signal: AbortSignal,
99
+
): Promise<HealthResponse | null> {
100
+
try {
101
+
const resp = await fetch(`${url}/xrpc/_health`, { signal });
102
+
if (!resp.ok) return null;
103
+
return await resp.json();
104
+
} catch {
105
+
return null;
106
+
}
107
+
}
108
+
109
+
async function fetchDescribeServer(
110
+
url: string,
111
+
signal: AbortSignal,
112
+
): Promise<DescribeServerResponse | null> {
113
+
try {
114
+
const resp = await fetch(
115
+
`${url}/xrpc/com.atproto.server.describeServer`,
116
+
{ signal },
117
+
);
118
+
if (!resp.ok) return null;
119
+
return await resp.json();
120
+
} catch {
121
+
return null;
122
+
}
123
+
}
124
+
125
+
async function fetchUserCount(
126
+
url: string,
127
+
signal: AbortSignal,
128
+
): Promise<number | null> {
129
+
try {
130
+
const resp = await fetch(
131
+
`${url}/xrpc/com.atproto.sync.listRepos?limit=1000`,
132
+
{ signal },
133
+
);
134
+
if (!resp.ok) return null;
135
+
const data = await resp.json();
136
+
const repos: unknown[] = data.repos ?? [];
137
+
// If there's a cursor, there are more than 1000
138
+
return data.cursor ? 1000 : repos.length;
139
+
} catch {
140
+
return null;
141
+
}
142
+
}
143
+
144
+
function emptyResult(url: string): EnrichmentResult {
145
+
return {
146
+
url,
147
+
version: null,
148
+
did: null,
149
+
phoneVerification: false,
150
+
userDomains: [],
151
+
contactEmail: null,
152
+
privacyPolicy: null,
153
+
termsOfService: null,
154
+
userCount: null,
155
+
ipAddress: null,
156
+
countryCode: null,
157
+
countryName: null,
158
+
};
159
+
}
+34
backend/services/pds-fetcher.ts
+34
backend/services/pds-fetcher.ts
···
1
+
import { STATE_JSON_URL } from "../../shared/constants.ts";
2
+
import type { StateJson } from "../../shared/types.ts";
3
+
4
+
export type FilteredPds = {
5
+
url: string;
6
+
inviteCodeRequired: boolean;
7
+
version: string;
8
+
errorAt: number | null;
9
+
isOpen: boolean;
10
+
};
11
+
12
+
/** Fetch state.json and return all PDSes with their open status */
13
+
export async function fetchPdsList(): Promise<FilteredPds[]> {
14
+
const resp = await fetch(STATE_JSON_URL);
15
+
if (!resp.ok) {
16
+
throw new Error(`Failed to fetch state.json: ${resp.status}`);
17
+
}
18
+
19
+
const data: StateJson = await resp.json();
20
+
const results: FilteredPds[] = [];
21
+
22
+
for (const [url, entry] of Object.entries(data.pdses)) {
23
+
const isOpen = !entry.inviteCodeRequired && !entry.errorAt;
24
+
results.push({
25
+
url,
26
+
inviteCodeRequired: entry.inviteCodeRequired,
27
+
version: entry.version,
28
+
errorAt: entry.errorAt ?? null,
29
+
isOpen,
30
+
});
31
+
}
32
+
33
+
return results;
34
+
}
+22
backend/services/version-checker.ts
+22
backend/services/version-checker.ts
···
1
+
import { PDS_REPO_NAME, PDS_REPO_OWNER } from "../../shared/constants.ts";
2
+
3
+
/** Fetch the latest PDS release version from GitHub */
4
+
export async function fetchLatestPdsVersion(): Promise<string | null> {
5
+
const url =
6
+
`https://api.github.com/repos/${PDS_REPO_OWNER}/${PDS_REPO_NAME}/tags?per_page=5`;
7
+
8
+
const resp = await fetch(url, {
9
+
headers: { "Accept": "application/vnd.github.v3+json" },
10
+
});
11
+
12
+
if (!resp.ok) {
13
+
console.error(`GitHub API error: ${resp.status}`);
14
+
return null;
15
+
}
16
+
17
+
const tags: Array<{ name: string }> = await resp.json();
18
+
if (tags.length === 0) return null;
19
+
20
+
// Tags are like "v0.4.74" — strip the "v" prefix to match _health output
21
+
return tags[0].name.replace(/^v/, "");
22
+
}
+71
cron/refresh.cron.ts
+71
cron/refresh.cron.ts
···
1
+
import { ENRICHMENT_BATCH_SIZE } from "../shared/constants.ts";
2
+
import { runMigrations } from "../backend/database/migrations.ts";
3
+
import {
4
+
getServersToEnrich,
5
+
setMetadata,
6
+
updateEnrichment,
7
+
upsertServer,
8
+
} from "../backend/database/queries.ts";
9
+
import { fetchPdsList } from "../backend/services/pds-fetcher.ts";
10
+
import { fetchLatestPdsVersion } from "../backend/services/version-checker.ts";
11
+
import { enrichBatch } from "../backend/services/pds-enricher.ts";
12
+
13
+
export default async function () {
14
+
console.log("OpenPDS refresh started:", new Date().toISOString());
15
+
16
+
try {
17
+
await runMigrations();
18
+
19
+
// 1. Fetch and sync state.json
20
+
const pdsList = await fetchPdsList();
21
+
console.log(`Fetched ${pdsList.length} PDSes from state.json`);
22
+
23
+
let openCount = 0;
24
+
for (const pds of pdsList) {
25
+
await upsertServer(pds.url, {
26
+
inviteCodeRequired: pds.inviteCodeRequired,
27
+
version: pds.version,
28
+
errorAt: pds.errorAt,
29
+
isOpen: pds.isOpen,
30
+
});
31
+
if (pds.isOpen) openCount++;
32
+
}
33
+
console.log(`Synced to DB: ${openCount} open, ${pdsList.length} total`);
34
+
35
+
// 2. Fetch latest PDS version
36
+
const latestVersion = await fetchLatestPdsVersion();
37
+
if (latestVersion) {
38
+
await setMetadata("latest_pds_version", latestVersion);
39
+
console.log(`Latest PDS version: ${latestVersion}`);
40
+
}
41
+
42
+
// 3. Enrich a batch of servers
43
+
const toEnrich = await getServersToEnrich(ENRICHMENT_BATCH_SIZE);
44
+
if (toEnrich.length > 0) {
45
+
console.log(`Enriching ${toEnrich.length} servers`);
46
+
const enriched = await enrichBatch(toEnrich);
47
+
48
+
for (const data of enriched) {
49
+
await updateEnrichment(data.url, {
50
+
version: data.version,
51
+
did: data.did,
52
+
phoneVerification: data.phoneVerification,
53
+
userDomains: data.userDomains,
54
+
contactEmail: data.contactEmail,
55
+
privacyPolicy: data.privacyPolicy,
56
+
termsOfService: data.termsOfService,
57
+
userCount: data.userCount,
58
+
countryCode: data.countryCode,
59
+
countryName: data.countryName,
60
+
ipAddress: data.ipAddress,
61
+
});
62
+
}
63
+
console.log(`Enrichment complete for ${enriched.length} servers`);
64
+
}
65
+
66
+
await setMetadata("last_full_refresh", new Date().toISOString());
67
+
console.log("OpenPDS refresh completed");
68
+
} catch (err) {
69
+
console.error("OpenPDS refresh failed:", err);
70
+
}
71
+
}
+26
deno.json
+26
deno.json
···
1
+
{
2
+
"$schema": "https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json",
3
+
"lock": false,
4
+
"compilerOptions": {
5
+
"noImplicitAny": false,
6
+
"strict": false,
7
+
"types": ["https://www.val.town/types/valtown.d.ts"],
8
+
"lib": [
9
+
"dom",
10
+
"dom.iterable",
11
+
"dom.asynciterable",
12
+
"deno.ns",
13
+
"deno.unstable"
14
+
]
15
+
},
16
+
"lint": {
17
+
"include": ["**/*.ts"],
18
+
"rules": { "exclude": ["no-explicit-any", "no-import-prefix"] }
19
+
},
20
+
"node_modules_dir": false,
21
+
"tasks": {
22
+
"check": "deno check --allow-import backend/index.http.ts cron/refresh.cron.ts",
23
+
"test": "deno test --allow-net --allow-read --allow-import",
24
+
"deploy": "deno fmt && deno lint && deno task check && deno task test && vt push"
25
+
}
26
+
}