goes to a random website hosted on wisp.place
1const WISP_API = Deno.env.get("WISP_API_URL") ?? "https://wisp.place";
2const HYDRANT_BIN = Deno.env.get("HYDRANT_BIN") ?? "hydrant";
3const PORT = parseInt(Deno.env.get("PORT") ?? "8080");
4const KV_PATH = Deno.env.get("KV_PATH") ?? "random-wisp-place.kv";
5const CURSOR = Deno.env.get("CURSOR");
6
7const getFreePort = () => {
8 const listener = Deno.listen({ port: 0 });
9 const port = (listener.addr as Deno.NetAddr).port;
10 listener.close();
11 return port;
12};
13
14const HYDRANT_PORT = getFreePort();
15const HYDRANT_URL = `http://localhost:${HYDRANT_PORT}`;
16
17const FS_COLLECTION = "place.wisp.fs";
18
19type SiteValue = {
20 fallbackUrl: string;
21 domains: string[];
22 lastScanned: number;
23};
24
25type HydrantRecord = {
26 readonly type: "record";
27 readonly id: number;
28 readonly record: {
29 readonly did: string;
30 readonly collection: string;
31 readonly rkey: string;
32 readonly action: "create" | "update" | "delete";
33 };
34};
35
36type HydrantEvent = HydrantRecord | { readonly type: "identity" | "account" };
37
38const siteKey = (did: string, siteName: string) => ["sites", did, siteName] as const;
39const cursorKey = () => ["cursor"] as const;
40
41const fallbackUrl = (did: string, siteName: string): string =>
42 `https://sites.wisp.place/${did}/${siteName}`;
43
44const resolveUrl = (site: SiteValue): string => {
45 if (site.domains.length > 0) {
46 return `https://${site.domains[Math.floor(Math.random() * site.domains.length)]}/`;
47 }
48 return site.fallbackUrl;
49};
50
51const kv = await Deno.openKv(KV_PATH);
52
53if (CURSOR) await kv.set(cursorKey(), parseInt(CURSOR));
54
55const allSiteEntries = async (): Promise<Map<Deno.KvKey, SiteValue>> => {
56 const map = new Map<Deno.KvKey, SiteValue>();
57 for await (const entry of kv.list<SiteValue>({ prefix: ["sites"] })) {
58 map.set(entry.key, entry.value);
59 }
60 return map;
61};
62
63const fetchSiteDomains = async (did: string, rkey: string): Promise<string[]> => {
64 const url = new URL(`${WISP_API}/xrpc/place.wisp.v2.site.getDomains`);
65 url.searchParams.set("did", did);
66 url.searchParams.set("rkey", rkey);
67 try {
68 const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
69 if (!res.ok) return [];
70 const data = await res.json() as { domains: Array<{ domain: string; kind: string; verified: boolean }> };
71 return data.domains
72 .filter((d) => d.verified)
73 .map((d) => d.domain);
74 } catch {
75 return [];
76 }
77};
78
79const handleFsEvent = async (
80 did: string,
81 rkey: string,
82 action: "create" | "update" | "delete",
83): Promise<void> => {
84 const key = siteKey(did, rkey);
85
86 if (action === "delete") {
87 await kv.delete(key);
88 console.log(`[-] fs ${did}:${rkey}`);
89 return;
90 }
91
92 const domains = await fetchSiteDomains(did, rkey);
93 await kv.set(key, {
94 fallbackUrl: fallbackUrl(did, rkey),
95 domains,
96 lastScanned: Date.now(),
97 });
98 console.log(`[+] fs ${action} ${did}:${rkey} (${domains.length} domains)`);
99};
100
101const handleEvent = async (raw: string): Promise<void> => {
102 let event: HydrantEvent;
103 try { event = JSON.parse(raw) as HydrantEvent; }
104 catch { return; }
105 if (event.type !== "record") return;
106
107 const { did, collection, rkey, action } = event.record;
108 await kv.set(cursorKey(), event.id);
109
110 if (collection === FS_COLLECTION) {
111 await handleFsEvent(did, rkey, action);
112 }
113};
114
115const connectToHydrant = async (cursor?: number): Promise<void> => {
116 const wsUrl = new URL(`${HYDRANT_URL.replace(/^http/, "ws")}/stream`);
117 if (cursor !== undefined) wsUrl.searchParams.set("cursor", String(cursor));
118
119 console.log(`[?] connecting to hydrant: ${wsUrl}`);
120 const ws = new WebSocket(wsUrl.toString());
121
122 ws.onopen = () => console.log("[?] hydrant stream connected");
123 ws.onmessage = ({ data }) => { handleEvent(String(data)).catch(console.error); };
124 ws.onerror = (e) => console.error("[!] ws error:", e);
125 ws.onclose = async () => {
126 const saved = (await kv.get<number>(cursorKey())).value ?? undefined;
127 console.log(`[!] ws closed (cursor=${saved ?? "none"}), reconnecting in 5s...`);
128 setTimeout(() => connectToHydrant(saved), 5_000);
129 };
130};
131
132const isReachable = async (url: string): Promise<boolean> => {
133 try {
134 const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(3_000) });
135 return res.status !== 404;
136 } catch {
137 return false;
138 }
139};
140
141const PROBE_BATCH = 10;
142const STALE_MS = 60 * 60 * 1000; // 1 hour
143
144const refreshIfStale = async (entry: { key: Deno.KvKey; value: SiteValue }): Promise<SiteValue> => {
145 const { key, value } = entry;
146 if (Date.now() - value.lastScanned < STALE_MS) return value;
147
148 // extract did and siteName from key ["sites", did, siteName]
149 const did = key[1] as string;
150 const siteName = key[2] as string;
151 const domains = await fetchSiteDomains(did, siteName);
152 const updated: SiteValue = { ...value, domains, lastScanned: Date.now() };
153 await kv.set(key, updated);
154 return updated;
155};
156
157const pickRandomReachable = async (sites: Map<Deno.KvKey, SiteValue>): Promise<SiteValue | null> => {
158 const entries = [...sites.entries()].sort(() => Math.random() - 0.5);
159 for (let i = 0; i < entries.length; i += PROBE_BATCH) {
160 const batch = entries.slice(i, i + PROBE_BATCH);
161 const results = await Promise.all(
162 batch.map(async ([key, site]) => {
163 const refreshed = await refreshIfStale({ key, value: site });
164 return { site: refreshed, ok: await isReachable(resolveUrl(refreshed)) };
165 })
166 );
167 const found = results.find((r) => r.ok);
168 if (found) return found.site;
169 }
170 return null;
171};
172
173const corsHeaders = {
174 headers: {
175 "Access-Control-Allow-Origin": "*",
176 "Access-Control-Allow-Methods": "GET",
177 }
178};
179Deno.serve({ port: PORT }, async (req) => {
180 if (req.method === "OPTIONS") {
181 return new Response(null, { status: 204, ...corsHeaders });
182 }
183
184 const { pathname } = new URL(req.url);
185
186 if (pathname === "/health") {
187 const entries = await allSiteEntries();
188 const data = {
189 total: entries.size,
190 withDomain: [...entries.values()].filter((s) => s.domains.length > 0).length,
191 };
192 return Response.json(data, corsHeaders);
193 }
194
195 const site = await pickRandomReachable(await allSiteEntries());
196 return site
197 ? Response.json(site, corsHeaders)
198 : new Response(
199 "no sites discovered yet, try again later",
200 { status: 503, ...corsHeaders },
201 );
202});
203console.log(`[?] listening on :${PORT}`);
204
205console.log(`[?] starting hydrant on :${HYDRANT_PORT}...`);
206try {
207 const conf = (name: string, value: string) => Deno.env.set(`HYDRANT_${name}`, value);
208 conf("API_PORT", `${HYDRANT_PORT}`);
209 conf("ENABLE_CRAWLER", "true");
210 conf("FILTER_SIGNALS", [FS_COLLECTION]);
211 conf("FILTER_COLLECTIONS", [FS_COLLECTION].join(","));
212 conf("PLC_URL", "https://plc.directory");
213 conf("ENABLE_DEBUG", "true");
214
215 const cmd = new Deno.Command(HYDRANT_BIN, {
216 stdout: "inherit",
217 stderr: "inherit",
218 });
219 const child = cmd.spawn();
220
221 const cleanup = () => {
222 console.log("[?] shutting down hydrant...");
223 child.kill("SIGTERM");
224 Deno.exit();
225 };
226
227 Deno.addSignalListener("SIGTERM", cleanup);
228 Deno.addSignalListener("SIGINT", cleanup);
229
230 child.status.then((status) => {
231 console.error(`[!] hydrant process exited with code ${status.code}`);
232 Deno.exit(1);
233 });
234} catch (e) {
235 console.error(`[!] failed to start hydrant: ${e.message}`);
236 Deno.exit(2);
237}
238
239const savedCursor = (await kv.get<number>(cursorKey())).value ?? undefined;
240console.log(`[?] resuming from cursor ${savedCursor ?? "start (0)"}`);
241connectToHydrant(savedCursor ?? 0);