goes to a random website hosted on wisp.place
use place.wisp.v2.site.getDomains xrpc instead of firehose domain tracking
- Replace handleDomainEvent + queryDomainRegistered with fetchSiteDomains
that calls the getDomains xrpc directly
- SiteValue now has domains[] + lastScanned instead of domainUrl
- Custom domains now work (previously only wisp subdomains were tracked)
- Stale entries (>1h) get refreshed when picked
- Random domain selection from the domains list
- Dropped DOMAIN_COLLECTION filter, domain index, and DomainIndexValue
- Net -36 lines
+56
-92
+56
-92
main.ts
+56
-92
main.ts
···
14
14
const HYDRANT_PORT = getFreePort();
15
15
const HYDRANT_URL = `http://localhost:${HYDRANT_PORT}`;
16
16
17
-
const FS_COLLECTION = "place.wisp.fs";
18
-
const DOMAIN_COLLECTION = "place.wisp.domain";
17
+
const FS_COLLECTION = "place.wisp.fs";
19
18
20
19
type SiteValue = {
21
20
fallbackUrl: string;
22
-
domainUrl: string | null;
23
-
};
24
-
25
-
// secondary index: domain -> site key components
26
-
type DomainIndexValue = {
27
-
did: string;
28
-
siteName: string;
21
+
domains: string[];
22
+
lastScanned: number;
29
23
};
30
24
31
25
type HydrantRecord = {
···
41
35
42
36
type HydrantEvent = HydrantRecord | { readonly type: "identity" | "account" };
43
37
44
-
type DomainRegistered = {
45
-
readonly registered: true;
46
-
readonly type: "wisp" | "custom";
47
-
readonly domain: string;
48
-
readonly did: string;
49
-
readonly rkey: string | null;
50
-
};
51
-
52
-
type DomainStatus = DomainRegistered | { readonly registered: false };
53
-
54
38
const siteKey = (did: string, siteName: string) => ["sites", did, siteName] as const;
55
-
const domainKey = (domain: string) => ["domain_idx", domain] as const;
56
39
const cursorKey = () => ["cursor"] as const;
57
40
58
41
const fallbackUrl = (did: string, siteName: string): string =>
59
42
`https://sites.wisp.place/${did}/${siteName}`;
60
-
const resolveUrl = (site: SiteValue): string =>
61
-
site.domainUrl ?? site.fallbackUrl;
43
+
44
+
const resolveUrl = (site: SiteValue): string => {
45
+
if (site.domains.length > 0) {
46
+
return `https://${site.domains[Math.floor(Math.random() * site.domains.length)]}/`;
47
+
}
48
+
return site.fallbackUrl;
49
+
};
62
50
63
51
const kv = await Deno.openKv(KV_PATH);
64
52
65
53
if (CURSOR) await kv.set(cursorKey(), parseInt(CURSOR));
66
54
67
-
const allSites = async (): Promise<SiteValue[]> => {
68
-
const entries: SiteValue[] = [];
55
+
const allSiteEntries = async (): Promise<Map<Deno.KvKey, SiteValue>> => {
56
+
const map = new Map<Deno.KvKey, SiteValue>();
69
57
for await (const entry of kv.list<SiteValue>({ prefix: ["sites"] })) {
70
-
entries.push(entry.value);
58
+
map.set(entry.key, entry.value);
71
59
}
72
-
return entries;
60
+
return map;
73
61
};
74
62
75
-
const queryDomainRegistered = async (domain: string): Promise<DomainStatus | null> => {
76
-
const url = new URL(`${WISP_API}/api/domain/registered`);
77
-
url.searchParams.set("domain", domain);
63
+
const fetchSiteDomains = async (did: string, rkey: string): Promise<string[]> => {
64
+
const url = new URL(`${WISP_API}/xrpc/place.wisp.v2.site.getDomains`);
65
+
url.searchParams.set("did", did);
66
+
url.searchParams.set("rkey", rkey);
78
67
try {
79
68
const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
80
-
return res.ok ? await res.json() as DomainStatus : null;
69
+
if (!res.ok) return [];
70
+
const data = await res.json() as { domains: Array<{ domain: string; kind: string; verified: boolean }> };
71
+
return data.domains
72
+
.filter((d) => d.verified)
73
+
.map((d) => d.domain);
81
74
} catch {
82
-
return null;
75
+
return [];
83
76
}
84
77
};
85
78
···
96
89
return;
97
90
}
98
91
99
-
// preserve existing domainUrl on upsert
100
-
const existing = await kv.get<SiteValue>(key);
92
+
const domains = await fetchSiteDomains(did, rkey);
101
93
await kv.set(key, {
102
94
fallbackUrl: fallbackUrl(did, rkey),
103
-
domainUrl: existing.value?.domainUrl ?? null,
95
+
domains,
96
+
lastScanned: Date.now(),
104
97
});
105
-
console.log(`[+] fs ${action} ${did}:${rkey}`);
106
-
};
107
-
108
-
const handleDomainEvent = async (
109
-
_did: string,
110
-
rkey: string,
111
-
action: "create" | "update" | "delete",
112
-
): Promise<void> => {
113
-
// rkey is the subdomain label e.g. "alice" -> alice.wisp.place
114
-
const domain = `${rkey}.wisp.place`;
115
-
const dKey = domainKey(domain);
116
-
117
-
if (action === "delete") {
118
-
const idx = await kv.get<DomainIndexValue>(dKey);
119
-
if (idx.value) {
120
-
const sKey = siteKey(idx.value.did, idx.value.siteName);
121
-
const site = await kv.get<SiteValue>(sKey);
122
-
if (site.value) {
123
-
await kv.set(sKey, { ...site.value, domainUrl: null });
124
-
}
125
-
}
126
-
await kv.delete(dKey);
127
-
console.log(`[-] domain ${domain} unlinked`);
128
-
return;
129
-
}
130
-
131
-
const status = await queryDomainRegistered(domain);
132
-
if (!status?.registered || !status.rkey) {
133
-
console.warn(`[!] domain ${domain}: not registered, no site mapped, or api error`);
134
-
return;
135
-
}
136
-
137
-
const domainUrl = `https://${status.domain}/`;
138
-
const sKey = siteKey(status.did, status.rkey);
139
-
140
-
// update or pre-create the site row with the resolved domainUrl
141
-
const existing = await kv.get<SiteValue>(sKey);
142
-
await kv.atomic()
143
-
.set(sKey, {
144
-
fallbackUrl: existing.value?.fallbackUrl ?? fallbackUrl(status.did, status.rkey),
145
-
domainUrl,
146
-
})
147
-
.set(dKey, { did: status.did, siteName: status.rkey } satisfies DomainIndexValue)
148
-
.commit();
149
-
150
-
console.log(`[+] domain ${domain} -> ${status.did}:${status.rkey} (${status.type})`);
98
+
console.log(`[+] fs ${action} ${did}:${rkey} (${domains.length} domains)`);
151
99
};
152
100
153
101
const handleEvent = async (raw: string): Promise<void> => {
···
161
109
162
110
if (collection === FS_COLLECTION) {
163
111
await handleFsEvent(did, rkey, action);
164
-
} else if (collection === DOMAIN_COLLECTION) {
165
-
await handleDomainEvent(did, rkey, action);
166
112
}
167
113
};
168
114
···
193
139
};
194
140
195
141
const PROBE_BATCH = 10;
196
-
const pickRandomReachable = async (sites: SiteValue[]): Promise<SiteValue | null> => {
197
-
const shuffled = [...sites].sort(() => Math.random() - 0.5);
198
-
for (let i = 0; i < shuffled.length; i += PROBE_BATCH) {
199
-
const batch = shuffled.slice(i, i + PROBE_BATCH);
142
+
const STALE_MS = 60 * 60 * 1000; // 1 hour
143
+
144
+
const refreshIfStale = async (entry: { key: Deno.KvKey; value: SiteValue }): Promise<SiteValue> => {
145
+
const { key, value } = entry;
146
+
if (Date.now() - value.lastScanned < STALE_MS) return value;
147
+
148
+
// extract did and siteName from key ["sites", did, siteName]
149
+
const did = key[1] as string;
150
+
const siteName = key[2] as string;
151
+
const domains = await fetchSiteDomains(did, siteName);
152
+
const updated: SiteValue = { ...value, domains, lastScanned: Date.now() };
153
+
await kv.set(key, updated);
154
+
return updated;
155
+
};
156
+
157
+
const pickRandomReachable = async (sites: Map<Deno.KvKey, SiteValue>): Promise<SiteValue | null> => {
158
+
const entries = [...sites.entries()].sort(() => Math.random() - 0.5);
159
+
for (let i = 0; i < entries.length; i += PROBE_BATCH) {
160
+
const batch = entries.slice(i, i + PROBE_BATCH);
200
161
const results = await Promise.all(
201
-
batch.map(async (site) => ({ site, ok: await isReachable(resolveUrl(site)) }))
162
+
batch.map(async ([key, site]) => {
163
+
const refreshed = await refreshIfStale({ key, value: site });
164
+
return { site: refreshed, ok: await isReachable(resolveUrl(refreshed)) };
165
+
})
202
166
);
203
167
const found = results.find((r) => r.ok);
204
168
if (found) return found.site;
···
220
184
const { pathname } = new URL(req.url);
221
185
222
186
if (pathname === "/health") {
223
-
const sites = await allSites();
187
+
const entries = await allSiteEntries();
224
188
const data = {
225
-
total: sites.length,
226
-
withDomain: sites.filter((s) => s.domainUrl).length,
189
+
total: entries.size,
190
+
withDomain: [...entries.values()].filter((s) => s.domains.length > 0).length,
227
191
};
228
192
return Response.json(data, corsHeaders);
229
193
}
230
194
231
-
const site = await pickRandomReachable(await allSites());
195
+
const site = await pickRandomReachable(await allSiteEntries());
232
196
return site
233
197
? Response.json(site, corsHeaders)
234
198
: new Response(
···
244
208
conf("API_PORT", `${HYDRANT_PORT}`);
245
209
conf("ENABLE_CRAWLER", "true");
246
210
conf("FILTER_SIGNALS", [FS_COLLECTION]);
247
-
conf("FILTER_COLLECTIONS", [FS_COLLECTION, DOMAIN_COLLECTION].join(","));
211
+
conf("FILTER_COLLECTIONS", [FS_COLLECTION].join(","));
248
212
conf("PLC_URL", "https://plc.directory");
249
213
conf("ENABLE_DEBUG", "true");
250
214