search for standard sites pub-search.waow.tech
search zig blog atproto
11
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: add constellation — 2D semantic map of the document index

python pipeline (scripts/build-constellation) exports vectors from
turbopuffer, projects to 2D via PCA+UMAP, clusters with HDBSCAN at
two granularities, and labels via c-TF-IDF on titles.

frontend (site/constellation.{html,css,js}) renders a full-viewport
dark canvas with celestial-style radial gradients per platform,
pan/zoom, semantic zoom labels, hover tooltips, and click-through.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

+1216 -2
+1
.gitignore
··· 6 6 .zig-cache/ 7 7 zig-out/ 8 8 .loq_cache 9 + site/constellation.json
+356
scripts/build-constellation
··· 1 + #!/usr/bin/env -S uv run --script --quiet 2 + # /// script 3 + # requires-python = ">=3.12" 4 + # dependencies = ["httpx", "numpy", "scikit-learn", "umap-learn", "hdbscan", "pydantic-settings"] 5 + # /// 6 + """ 7 + Build constellation.json — 2D semantic map of the document index. 8 + 9 + Exports vectors from turbopuffer, projects to 2D via PCA+UMAP, 10 + clusters with HDBSCAN at two granularities, labels via c-TF-IDF. 11 + 12 + Usage: 13 + ./scripts/build-constellation # writes site/constellation.json 14 + ./scripts/build-constellation --output out.json 15 + """ 16 + 17 + import argparse 18 + import json 19 + import os 20 + import re 21 + import sys 22 + import time 23 + from collections import Counter 24 + from pathlib import Path 25 + 26 + import httpx 27 + import numpy as np 28 + from pydantic_settings import BaseSettings, SettingsConfigDict 29 + from sklearn.decomposition import PCA 30 + from sklearn.feature_extraction.text import TfidfVectorizer 31 + 32 + 33 + def log(msg: str) -> None: 34 + print(msg, flush=True) 35 + 36 + 37 + class Settings(BaseSettings): 38 + model_config = SettingsConfigDict( 39 + env_file=os.environ.get("ENV_FILE", ".env"), extra="ignore" 40 + ) 41 + 42 + turbopuffer_api_key: str 43 + turbopuffer_namespace: str = "leaflet-search" 44 + 45 + @classmethod 46 + def settings_customise_sources(cls, settings_cls, **kwargs): 47 + """Dotenv file wins over environment variables.""" 48 + return ( 49 + kwargs["init_settings"], 50 + kwargs["dotenv_settings"], 51 + kwargs["env_settings"], 52 + kwargs["file_secret_settings"], 53 + ) 54 + 55 + 56 + def tpuf_export(client: httpx.Client, settings: Settings) -> list[dict]: 57 + """Export all vectors from turbopuffer via paginated query.""" 58 + url = f"https://api.turbopuffer.com/v2/namespaces/{settings.turbopuffer_namespace}/query" 59 + headers = { 60 + "Authorization": f"Bearer {settings.turbopuffer_api_key}", 61 + "Content-Type": "application/json", 62 + } 63 + attrs = ["uri", "title", "platform", "base_path", "did", "vector"] 64 + all_rows = [] 65 + last_id = None 66 + 67 + while True: 68 + body: dict = { 69 + "rank_by": ["id", "asc"], 70 + "limit": 10000, 71 + "include_attributes": attrs, 72 + } 73 + if last_id is not None: 74 + body["filters"] = ["id", "Gt", last_id] 75 + 76 + resp = client.post(url, headers=headers, json=body, timeout=120) 77 + resp.raise_for_status() 78 + rows = resp.json().get("rows", []) 79 + if not rows: 80 + break 81 + 82 + all_rows.extend(rows) 83 + last_id = rows[-1]["id"] 84 + log(f" fetched {len(all_rows)} vectors so far...") 85 + 86 + if len(rows) < 10000: 87 + break 88 + 89 + return all_rows 90 + 91 + 92 + def extract_terms(title: str) -> list[str]: 93 + """Extract lowercase alphanumeric tokens from a title.""" 94 + if not title: 95 + return [] 96 + return [t for t in re.findall(r"[a-z0-9]+", title.lower()) if len(t) > 1] 97 + 98 + 99 + def cluster_labels(titles_per_cluster: dict[int, list[str]], n_terms: int = 3) -> dict[int, str]: 100 + """Compute c-TF-IDF labels for clusters from document titles.""" 101 + cluster_ids = sorted(titles_per_cluster.keys()) 102 + if not cluster_ids: 103 + return {} 104 + 105 + # build one "document" per cluster: concatenated titles 106 + docs = [] 107 + for cid in cluster_ids: 108 + docs.append(" ".join(titles_per_cluster[cid])) 109 + 110 + # TF-IDF across cluster pseudo-documents 111 + vectorizer = TfidfVectorizer( 112 + max_features=5000, 113 + stop_words="english", 114 + token_pattern=r"[a-z0-9]{2,}", 115 + lowercase=True, 116 + ) 117 + try: 118 + tfidf = vectorizer.fit_transform(docs) 119 + except ValueError: 120 + return {cid: f"cluster {cid}" for cid in cluster_ids} 121 + 122 + feature_names = vectorizer.get_feature_names_out() 123 + labels = {} 124 + for i, cid in enumerate(cluster_ids): 125 + row = tfidf[i].toarray().flatten() 126 + top_idx = row.argsort()[-n_terms:][::-1] 127 + top_terms = [feature_names[j] for j in top_idx if row[j] > 0] 128 + labels[cid] = " ".join(top_terms) if top_terms else f"cluster {cid}" 129 + 130 + return labels 131 + 132 + 133 + def assign_outliers(coords_2d: np.ndarray, labels: np.ndarray, centroids: dict[int, np.ndarray]) -> np.ndarray: 134 + """Assign outlier points (label == -1) to nearest cluster centroid.""" 135 + result = labels.copy() 136 + outlier_mask = result == -1 137 + if not outlier_mask.any() or not centroids: 138 + return result 139 + 140 + centroid_ids = sorted(centroids.keys()) 141 + centroid_arr = np.array([centroids[c] for c in centroid_ids]) 142 + outlier_coords = coords_2d[outlier_mask] 143 + 144 + # nearest centroid for each outlier 145 + dists = np.linalg.norm(outlier_coords[:, None] - centroid_arr[None, :], axis=2) 146 + nearest = dists.argmin(axis=1) 147 + result[outlier_mask] = np.array(centroid_ids)[nearest] 148 + 149 + return result 150 + 151 + 152 + def main(): 153 + parser = argparse.ArgumentParser(description="Build constellation.json") 154 + parser.add_argument( 155 + "--output", "-o", 156 + default=str(Path(__file__).resolve().parent.parent / "site" / "constellation.json"), 157 + help="Output path (default: site/constellation.json)", 158 + ) 159 + args = parser.parse_args() 160 + 161 + try: 162 + settings = Settings() # type: ignore 163 + except Exception as e: 164 + print(f"error loading settings: {e}", file=sys.stderr) 165 + print("required env vars: TURBOPUFFER_API_KEY", file=sys.stderr) 166 + sys.exit(1) 167 + 168 + t_start = time.monotonic() 169 + 170 + # --- step 1: export vectors from turbopuffer --- 171 + log("exporting vectors from turbopuffer...") 172 + client = httpx.Client() 173 + rows = tpuf_export(client, settings) 174 + client.close() 175 + log(f" got {len(rows)} documents") 176 + 177 + if len(rows) < 10: 178 + print("too few documents to build constellation", file=sys.stderr) 179 + sys.exit(1) 180 + 181 + # parse into arrays 182 + vectors = [] 183 + metadata = [] 184 + for row in rows: 185 + vec = row.get("vector") 186 + if not vec: 187 + continue 188 + vectors.append(vec) 189 + metadata.append({ 190 + "uri": row.get("uri", row.get("id", "")), 191 + "title": row.get("title", ""), 192 + "platform": row.get("platform", "other"), 193 + "basePath": row.get("base_path", ""), 194 + "did": row.get("did", ""), 195 + }) 196 + 197 + X = np.array(vectors, dtype=np.float32) 198 + log(f" {X.shape[0]} vectors, {X.shape[1]} dims") 199 + 200 + # --- step 2: dimensionality reduction --- 201 + log("PCA 1024 → 50...") 202 + n_components_pca = min(50, X.shape[0] - 1, X.shape[1]) 203 + pca = PCA(n_components=n_components_pca, random_state=42) 204 + X_pca = pca.fit_transform(X) 205 + log(f" variance explained: {pca.explained_variance_ratio_.sum():.2%}") 206 + 207 + log("UMAP 50 → 2...") 208 + import umap 209 + reducer = umap.UMAP( 210 + n_components=2, 211 + metric="cosine", 212 + n_neighbors=15, 213 + min_dist=0.1, 214 + random_state=42, 215 + ) 216 + X_2d = reducer.fit_transform(X_pca) 217 + 218 + # normalize to [-1, 1] range 219 + for dim in range(2): 220 + lo, hi = X_2d[:, dim].min(), X_2d[:, dim].max() 221 + if hi > lo: 222 + X_2d[:, dim] = 2 * (X_2d[:, dim] - lo) / (hi - lo) - 1 223 + 224 + log(f" 2D range: x=[{X_2d[:,0].min():.2f}, {X_2d[:,0].max():.2f}] y=[{X_2d[:,1].min():.2f}, {X_2d[:,1].max():.2f}]") 225 + 226 + # --- step 3: clustering --- 227 + import hdbscan 228 + 229 + log("HDBSCAN coarse (min_cluster_size=100)...") 230 + clusterer_coarse = hdbscan.HDBSCAN(min_cluster_size=100, min_samples=5) 231 + labels_coarse_raw = clusterer_coarse.fit_predict(X_2d) 232 + n_coarse = len(set(labels_coarse_raw)) - (1 if -1 in labels_coarse_raw else 0) 233 + n_outliers_coarse = (labels_coarse_raw == -1).sum() 234 + log(f" {n_coarse} clusters, {n_outliers_coarse} outliers") 235 + 236 + # compute centroids for coarse 237 + coarse_centroids = {} 238 + for cid in set(labels_coarse_raw): 239 + if cid == -1: 240 + continue 241 + mask = labels_coarse_raw == cid 242 + coarse_centroids[cid] = X_2d[mask].mean(axis=0) 243 + 244 + labels_coarse = assign_outliers(X_2d, labels_coarse_raw, coarse_centroids) 245 + # recompute centroids after outlier assignment 246 + for cid in set(labels_coarse): 247 + mask = labels_coarse == cid 248 + coarse_centroids[cid] = X_2d[mask].mean(axis=0) 249 + 250 + log("HDBSCAN fine (min_cluster_size=20)...") 251 + clusterer_fine = hdbscan.HDBSCAN(min_cluster_size=20, min_samples=3) 252 + labels_fine_raw = clusterer_fine.fit_predict(X_2d) 253 + n_fine = len(set(labels_fine_raw)) - (1 if -1 in labels_fine_raw else 0) 254 + n_outliers_fine = (labels_fine_raw == -1).sum() 255 + log(f" {n_fine} clusters, {n_outliers_fine} outliers") 256 + 257 + fine_centroids = {} 258 + for cid in set(labels_fine_raw): 259 + if cid == -1: 260 + continue 261 + mask = labels_fine_raw == cid 262 + fine_centroids[cid] = X_2d[mask].mean(axis=0) 263 + 264 + labels_fine = assign_outliers(X_2d, labels_fine_raw, fine_centroids) 265 + for cid in set(labels_fine): 266 + mask = labels_fine == cid 267 + fine_centroids[cid] = X_2d[mask].mean(axis=0) 268 + 269 + # --- step 4: labels via c-TF-IDF --- 270 + log("computing cluster labels (c-TF-IDF on titles)...") 271 + 272 + coarse_titles: dict[int, list[str]] = {} 273 + fine_titles: dict[int, list[str]] = {} 274 + for i, meta in enumerate(metadata): 275 + title = meta.get("title", "") 276 + if title: 277 + coarse_titles.setdefault(int(labels_coarse[i]), []).append(title) 278 + fine_titles.setdefault(int(labels_fine[i]), []).append(title) 279 + 280 + coarse_labels = cluster_labels(coarse_titles) 281 + fine_labels = cluster_labels(fine_titles) 282 + 283 + # map fine clusters to their parent coarse cluster (majority vote) 284 + fine_to_coarse = {} 285 + for fine_id in set(labels_fine): 286 + fine_mask = labels_fine == fine_id 287 + coarse_for_fine = labels_coarse[fine_mask] 288 + counts = Counter(coarse_for_fine) 289 + fine_to_coarse[int(fine_id)] = int(counts.most_common(1)[0][0]) 290 + 291 + log(f" coarse: {len(coarse_labels)} labels") 292 + log(f" fine: {len(fine_labels)} labels") 293 + 294 + # --- step 5: build output --- 295 + log("building output...") 296 + 297 + points = [] 298 + for i, meta in enumerate(metadata): 299 + points.append({ 300 + "x": round(float(X_2d[i, 0]), 4), 301 + "y": round(float(X_2d[i, 1]), 4), 302 + "uri": meta["uri"], 303 + "title": meta["title"], 304 + "platform": meta["platform"], 305 + "basePath": meta["basePath"], 306 + "clusterCoarse": int(labels_coarse[i]), 307 + "clusterFine": int(labels_fine[i]), 308 + }) 309 + 310 + coarse_clusters = [] 311 + for cid in sorted(coarse_centroids.keys()): 312 + count = int((labels_coarse == cid).sum()) 313 + coarse_clusters.append({ 314 + "id": int(cid), 315 + "label": coarse_labels.get(int(cid), f"cluster {cid}"), 316 + "cx": round(float(coarse_centroids[cid][0]), 4), 317 + "cy": round(float(coarse_centroids[cid][1]), 4), 318 + "count": count, 319 + }) 320 + 321 + fine_clusters = [] 322 + for cid in sorted(fine_centroids.keys()): 323 + count = int((labels_fine == cid).sum()) 324 + fine_clusters.append({ 325 + "id": int(cid), 326 + "label": fine_labels.get(int(cid), f"cluster {cid}"), 327 + "cx": round(float(fine_centroids[cid][0]), 4), 328 + "cy": round(float(fine_centroids[cid][1]), 4), 329 + "count": count, 330 + "parent": fine_to_coarse.get(int(cid), 0), 331 + }) 332 + 333 + output = { 334 + "points": points, 335 + "clusters": { 336 + "coarse": coarse_clusters, 337 + "fine": fine_clusters, 338 + }, 339 + "meta": { 340 + "generatedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), 341 + "nDocuments": len(points), 342 + }, 343 + } 344 + 345 + out_path = Path(args.output) 346 + out_path.parent.mkdir(parents=True, exist_ok=True) 347 + out_path.write_text(json.dumps(output, separators=(",", ":"))) 348 + 349 + size_kb = out_path.stat().st_size / 1024 350 + elapsed = time.monotonic() - t_start 351 + log(f"\nwrote {out_path} ({size_kb:.0f} KB, {len(points)} points)") 352 + log(f"done in {elapsed:.1f}s") 353 + 354 + 355 + if __name__ == "__main__": 356 + main()
+205
site/constellation.css
··· 1 + :root, [data-theme="dark"] { 2 + --bg: #050505; 3 + --bg-subtle: #111; 4 + --text: #ccc; 5 + --text-bright: #fff; 6 + --text-secondary: #888; 7 + --text-dim: #555; 8 + --text-muted: #444; 9 + --tooltip-bg: rgba(10, 10, 10, 0.92); 10 + --tooltip-border: #333; 11 + --overlay-bg: rgba(5, 5, 5, 0.85); 12 + } 13 + 14 + [data-theme="light"] { 15 + --bg: #f5f5f0; 16 + --bg-subtle: #eee; 17 + --text: #333; 18 + --text-bright: #111; 19 + --text-secondary: #555; 20 + --text-dim: #888; 21 + --text-muted: #999; 22 + --tooltip-bg: rgba(245, 245, 240, 0.92); 23 + --tooltip-border: #ccc; 24 + --overlay-bg: rgba(245, 245, 240, 0.85); 25 + } 26 + 27 + * { box-sizing: border-box; margin: 0; padding: 0; } 28 + 29 + html, body { 30 + width: 100%; 31 + height: 100%; 32 + overflow: hidden; 33 + font-family: monospace; 34 + background: var(--bg); 35 + color: var(--text); 36 + font-size: 13px; 37 + } 38 + 39 + canvas { 40 + display: block; 41 + width: 100%; 42 + height: 100%; 43 + cursor: grab; 44 + } 45 + 46 + canvas:active { 47 + cursor: grabbing; 48 + } 49 + 50 + /* header overlay */ 51 + .header { 52 + position: fixed; 53 + top: 0; 54 + left: 0; 55 + right: 0; 56 + padding: 12px 16px; 57 + z-index: 10; 58 + pointer-events: none; 59 + display: flex; 60 + justify-content: space-between; 61 + align-items: center; 62 + } 63 + 64 + .header > * { 65 + pointer-events: auto; 66 + } 67 + 68 + .header h1 { 69 + font-size: 12px; 70 + font-weight: normal; 71 + } 72 + 73 + .header h1 a { 74 + color: var(--text-secondary); 75 + text-decoration: none; 76 + } 77 + 78 + .header h1 a:hover { 79 + color: var(--text-bright); 80 + } 81 + 82 + .header .dim { 83 + color: var(--text-dim); 84 + } 85 + 86 + .header .nav { 87 + font-size: 11px; 88 + display: flex; 89 + gap: 12px; 90 + align-items: center; 91 + } 92 + 93 + .header .nav a { 94 + color: var(--text-dim); 95 + text-decoration: none; 96 + } 97 + 98 + .header .nav a:hover { 99 + color: var(--text-secondary); 100 + } 101 + 102 + .theme-toggle { 103 + cursor: pointer; 104 + color: var(--text-dim); 105 + user-select: none; 106 + } 107 + 108 + .theme-toggle:hover { 109 + color: var(--text-secondary); 110 + } 111 + 112 + /* tooltip */ 113 + .tooltip { 114 + display: none; 115 + position: fixed; 116 + z-index: 20; 117 + background: var(--tooltip-bg); 118 + border: 1px solid var(--tooltip-border); 119 + padding: 8px 10px; 120 + font-size: 12px; 121 + line-height: 1.5; 122 + max-width: 320px; 123 + pointer-events: none; 124 + backdrop-filter: blur(8px); 125 + } 126 + 127 + .tooltip .title { 128 + color: var(--text-bright); 129 + font-weight: bold; 130 + overflow: hidden; 131 + text-overflow: ellipsis; 132 + white-space: nowrap; 133 + } 134 + 135 + .tooltip .meta { 136 + color: var(--text-secondary); 137 + font-size: 11px; 138 + } 139 + 140 + .tooltip .platform-badge { 141 + display: inline-block; 142 + font-size: 10px; 143 + padding: 1px 5px; 144 + border-radius: 2px; 145 + margin-top: 2px; 146 + } 147 + 148 + /* loading overlay */ 149 + .loading-overlay { 150 + position: fixed; 151 + top: 0; left: 0; right: 0; bottom: 0; 152 + z-index: 30; 153 + background: var(--bg); 154 + display: flex; 155 + flex-direction: column; 156 + align-items: center; 157 + justify-content: center; 158 + gap: 12px; 159 + transition: opacity 0.4s; 160 + } 161 + 162 + .loading-overlay.hidden { 163 + opacity: 0; 164 + pointer-events: none; 165 + } 166 + 167 + .loading-overlay .spinner { 168 + color: var(--text-dim); 169 + font-size: 13px; 170 + } 171 + 172 + /* legend */ 173 + .legend { 174 + position: fixed; 175 + bottom: 12px; 176 + left: 16px; 177 + z-index: 10; 178 + font-size: 10px; 179 + color: var(--text-dim); 180 + display: flex; 181 + gap: 10px; 182 + flex-wrap: wrap; 183 + } 184 + 185 + .legend-item { 186 + display: flex; 187 + align-items: center; 188 + gap: 4px; 189 + } 190 + 191 + .legend-dot { 192 + width: 8px; 193 + height: 8px; 194 + border-radius: 50%; 195 + } 196 + 197 + /* stats */ 198 + .stats { 199 + position: fixed; 200 + bottom: 12px; 201 + right: 16px; 202 + z-index: 10; 203 + font-size: 10px; 204 + color: var(--text-muted); 205 + }
+70
site/constellation.html
··· 1 + <!DOCTYPE html> 2 + <html lang="en"> 3 + <head> 4 + <meta charset="UTF-8"> 5 + <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> 6 + <title>pub search / constellation</title> 7 + <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='2' fill='%231B7340'/><circle cx='8' cy='10' r='1.5' fill='%231B7340' opacity='.6'/><circle cx='24' cy='8' r='1' fill='%231B7340' opacity='.4'/><circle cx='22' cy='22' r='1.5' fill='%231B7340' opacity='.5'/><circle cx='6' cy='24' r='1' fill='%231B7340' opacity='.3'/></svg>"> 8 + <script> 9 + (function() { 10 + var t = localStorage.getItem('theme') || 'dark'; 11 + if (t === 'system') t = matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; 12 + document.documentElement.setAttribute('data-theme', t); 13 + })(); 14 + </script> 15 + <link rel="stylesheet" href="constellation.css"> 16 + </head> 17 + <body> 18 + <div class="loading-overlay" id="loading"> 19 + <div class="spinner">loading constellation...</div> 20 + </div> 21 + 22 + <canvas id="canvas"></canvas> 23 + 24 + <div class="header"> 25 + <h1><a href="/">pub search</a> <span class="dim">/ constellation</span></h1> 26 + <div class="nav"> 27 + <a href="/dashboard.html">stats</a> 28 + <span class="theme-toggle" id="theme-toggle"></span> 29 + </div> 30 + </div> 31 + 32 + <div class="tooltip" id="tooltip"> 33 + <div class="title" id="tooltip-title"></div> 34 + <div class="meta" id="tooltip-meta"></div> 35 + <div class="platform-badge" id="tooltip-platform"></div> 36 + </div> 37 + 38 + <div class="legend" id="legend"></div> 39 + <div class="stats" id="stats"></div> 40 + 41 + <script> 42 + var currentTheme = localStorage.getItem('theme') || 'dark'; 43 + function applyTheme(theme) { 44 + currentTheme = theme; 45 + localStorage.setItem('theme', theme); 46 + var resolved = theme; 47 + if (theme === 'system') resolved = matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; 48 + document.documentElement.setAttribute('data-theme', resolved); 49 + renderThemeToggle(); 50 + if (window.constellation) window.constellation.setDirty(); 51 + } 52 + var THEME_ICONS = { dark: '\u263E', light: '\u263C', system: '\u25D1' }; 53 + var THEME_CYCLE = ['dark', 'light', 'system']; 54 + function cycleTheme() { 55 + var next = THEME_CYCLE[(THEME_CYCLE.indexOf(currentTheme) + 1) % 3]; 56 + applyTheme(next); 57 + } 58 + function renderThemeToggle() { 59 + var el = document.getElementById('theme-toggle'); 60 + if (!el) return; 61 + el.innerHTML = '<span onclick="cycleTheme()" title="' + currentTheme + '">' + THEME_ICONS[currentTheme] + '</span>'; 62 + } 63 + matchMedia('(prefers-color-scheme: light)').addEventListener('change', function() { 64 + if (currentTheme === 'system') applyTheme('system'); 65 + }); 66 + renderThemeToggle(); 67 + </script> 68 + <script src="constellation.js"></script> 69 + </body> 70 + </html>
+582
site/constellation.js
··· 1 + (function() { 2 + 'use strict'; 3 + 4 + // --- platform colors: [core, mid, edge] triplets --- 5 + var PLATFORM_COLORS = { 6 + leaflet: { core: '#4ade80', mid: '#22c55e', edge: '#166534' }, 7 + whitewind: { core: '#60a5fa', mid: '#3b82f6', edge: '#1e3a8a' }, 8 + pckt: { core: '#fbbf24', mid: '#f59e0b', edge: '#92400e' }, 9 + offprint: { core: '#fb7185', mid: '#f43f5e', edge: '#881337' }, 10 + greengale: { core: '#2dd4bf', mid: '#14b8a6', edge: '#134e4a' }, 11 + other: { core: '#9ca3af', mid: '#6b7280', edge: '#374151' }, 12 + }; 13 + 14 + // light theme overrides (darker cores for visibility) 15 + var PLATFORM_COLORS_LIGHT = { 16 + leaflet: { core: '#16a34a', mid: '#15803d', edge: '#a7f3d0' }, 17 + whitewind: { core: '#2563eb', mid: '#1d4ed8', edge: '#bfdbfe' }, 18 + pckt: { core: '#d97706', mid: '#b45309', edge: '#fde68a' }, 19 + offprint: { core: '#e11d48', mid: '#be123c', edge: '#fecdd3' }, 20 + greengale: { core: '#0d9488', mid: '#0f766e', edge: '#99f6e4' }, 21 + other: { core: '#4b5563', mid: '#374151', edge: '#d1d5db' }, 22 + }; 23 + 24 + function getColors() { 25 + var theme = document.documentElement.getAttribute('data-theme'); 26 + return theme === 'light' ? PLATFORM_COLORS_LIGHT : PLATFORM_COLORS; 27 + } 28 + 29 + function isDark() { 30 + return document.documentElement.getAttribute('data-theme') !== 'light'; 31 + } 32 + 33 + // --- view state --- 34 + var view = { 35 + zoom: 1, 36 + panX: 0, 37 + panY: 0, 38 + minZoom: 0.5, 39 + maxZoom: 15, 40 + dirty: true, 41 + }; 42 + 43 + // --- data --- 44 + var data = null; 45 + var pointsX = null; // Float32Array 46 + var pointsY = null; 47 + var gridIndex = null; // spatial index for hover 48 + 49 + // --- canvas --- 50 + var canvas = document.getElementById('canvas'); 51 + var ctx = canvas.getContext('2d'); 52 + var dpr = window.devicePixelRatio || 1; 53 + var W, H; 54 + 55 + // --- hover state --- 56 + var hoveredIndex = -1; 57 + var hoverTimer = null; 58 + var mouseX = 0, mouseY = 0; 59 + 60 + // --- interaction state --- 61 + var dragging = false; 62 + var dragStartX, dragStartY; 63 + var dragStartPanX, dragStartPanY; 64 + var pinchStartDist = 0; 65 + var pinchStartZoom = 1; 66 + 67 + // --- gradient cache --- 68 + var gradientCache = {}; 69 + 70 + function resizeCanvas() { 71 + W = window.innerWidth; 72 + H = window.innerHeight; 73 + canvas.width = W * dpr; 74 + canvas.height = H * dpr; 75 + canvas.style.width = W + 'px'; 76 + canvas.style.height = H + 'px'; 77 + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); 78 + gradientCache = {}; 79 + view.dirty = true; 80 + } 81 + 82 + // --- coordinate transforms --- 83 + function dataToScreen(dx, dy) { 84 + var cx = W / 2; 85 + var cy = H / 2; 86 + var scale = Math.min(W, H) * 0.42 * view.zoom; 87 + return [ 88 + cx + (dx + view.panX) * scale, 89 + cy + (dy + view.panY) * scale, 90 + ]; 91 + } 92 + 93 + function screenToData(sx, sy) { 94 + var cx = W / 2; 95 + var cy = H / 2; 96 + var scale = Math.min(W, H) * 0.42 * view.zoom; 97 + return [ 98 + (sx - cx) / scale - view.panX, 99 + (sy - cy) / scale - view.panY, 100 + ]; 101 + } 102 + 103 + // --- spatial index (grid-based) --- 104 + function buildSpatialIndex() { 105 + if (!data) return; 106 + var cellSize = 0.02; // in data space 107 + gridIndex = { cellSize: cellSize, cells: {} }; 108 + for (var i = 0; i < data.points.length; i++) { 109 + var gx = Math.floor(pointsX[i] / cellSize); 110 + var gy = Math.floor(pointsY[i] / cellSize); 111 + var key = gx + ',' + gy; 112 + if (!gridIndex.cells[key]) gridIndex.cells[key] = []; 113 + gridIndex.cells[key].push(i); 114 + } 115 + } 116 + 117 + function findNearest(sx, sy, maxDist) { 118 + if (!gridIndex) return -1; 119 + var d = screenToData(sx, sy); 120 + var dx = d[0], dy = d[1]; 121 + var scale = Math.min(W, H) * 0.42 * view.zoom; 122 + var searchRadius = maxDist / scale; 123 + var cs = gridIndex.cellSize; 124 + var gxMin = Math.floor((dx - searchRadius) / cs); 125 + var gxMax = Math.floor((dx + searchRadius) / cs); 126 + var gyMin = Math.floor((dy - searchRadius) / cs); 127 + var gyMax = Math.floor((dy + searchRadius) / cs); 128 + 129 + var bestIdx = -1; 130 + var bestDist = searchRadius * searchRadius; 131 + 132 + for (var gx = gxMin; gx <= gxMax; gx++) { 133 + for (var gy = gyMin; gy <= gyMax; gy++) { 134 + var cell = gridIndex.cells[gx + ',' + gy]; 135 + if (!cell) continue; 136 + for (var k = 0; k < cell.length; k++) { 137 + var i = cell[k]; 138 + var ddx = pointsX[i] - dx; 139 + var ddy = pointsY[i] - dy; 140 + var dist2 = ddx * ddx + ddy * ddy; 141 + if (dist2 < bestDist) { 142 + bestDist = dist2; 143 + bestIdx = i; 144 + } 145 + } 146 + } 147 + } 148 + return bestIdx; 149 + } 150 + 151 + // --- rendering --- 152 + function getPointRadius(zoom) { 153 + if (zoom < 2) return 1.8; 154 + if (zoom < 5) return 1.5 + zoom * 0.3; 155 + return 2 + zoom * 0.2; 156 + } 157 + 158 + function drawPoint(x, y, r, colors, alpha) { 159 + if (r < 1.5) { 160 + // tiny points: simple filled circle 161 + ctx.globalAlpha = alpha; 162 + ctx.fillStyle = colors.mid; 163 + ctx.beginPath(); 164 + ctx.arc(x, y, r, 0, Math.PI * 2); 165 + ctx.fill(); 166 + return; 167 + } 168 + 169 + // celestial body: radial gradient 170 + var cacheKey = colors.core + '_' + Math.round(r * 10); 171 + var grad = gradientCache[cacheKey]; 172 + if (!grad) { 173 + grad = ctx.createRadialGradient(x, y, 0, x, y, r * 2.5); 174 + grad.addColorStop(0, colors.core); 175 + grad.addColorStop(0.3, colors.mid); 176 + grad.addColorStop(0.7, colors.edge); 177 + grad.addColorStop(1, 'transparent'); 178 + // don't cache position-dependent gradients for large radii 179 + } 180 + 181 + // for larger radii, always create fresh (position-dependent) 182 + grad = ctx.createRadialGradient(x, y, 0, x, y, r * 2.5); 183 + grad.addColorStop(0, colors.core); 184 + grad.addColorStop(0.3, colors.mid); 185 + grad.addColorStop(0.7, colors.edge); 186 + grad.addColorStop(1, 'transparent'); 187 + 188 + ctx.globalAlpha = alpha; 189 + ctx.fillStyle = grad; 190 + ctx.beginPath(); 191 + ctx.arc(x, y, r * 2.5, 0, Math.PI * 2); 192 + ctx.fill(); 193 + 194 + // bright core 195 + ctx.globalAlpha = alpha * 0.9; 196 + ctx.fillStyle = colors.core; 197 + ctx.beginPath(); 198 + ctx.arc(x, y, r * 0.5, 0, Math.PI * 2); 199 + ctx.fill(); 200 + } 201 + 202 + function drawClusterGlow(cx, cy, count, alpha) { 203 + var r = Math.sqrt(count) * 2; 204 + var grad = ctx.createRadialGradient(cx, cy, 0, cx, cy, r); 205 + var dark = isDark(); 206 + grad.addColorStop(0, dark ? 'rgba(255,255,255,0.03)' : 'rgba(0,0,0,0.02)'); 207 + grad.addColorStop(1, 'transparent'); 208 + ctx.globalAlpha = alpha; 209 + ctx.fillStyle = grad; 210 + ctx.beginPath(); 211 + ctx.arc(cx, cy, r, 0, Math.PI * 2); 212 + ctx.fill(); 213 + } 214 + 215 + function drawLabel(text, sx, sy, fontSize, alpha) { 216 + ctx.globalAlpha = alpha; 217 + var dark = isDark(); 218 + ctx.fillStyle = dark ? 'rgba(255,255,255,0.7)' : 'rgba(0,0,0,0.6)'; 219 + ctx.font = fontSize + 'px monospace'; 220 + ctx.textAlign = 'center'; 221 + ctx.textBaseline = 'middle'; 222 + 223 + // text shadow for readability 224 + if (dark) { 225 + ctx.shadowColor = 'rgba(0,0,0,0.8)'; 226 + ctx.shadowBlur = 4; 227 + } else { 228 + ctx.shadowColor = 'rgba(255,255,255,0.8)'; 229 + ctx.shadowBlur = 4; 230 + } 231 + ctx.fillText(text, sx, sy); 232 + ctx.shadowBlur = 0; 233 + } 234 + 235 + function render() { 236 + if (!data || !view.dirty) return; 237 + view.dirty = false; 238 + 239 + var dark = isDark(); 240 + ctx.clearRect(0, 0, W, H); 241 + 242 + // background 243 + ctx.fillStyle = dark ? '#050505' : '#f5f5f0'; 244 + ctx.fillRect(0, 0, W, H); 245 + 246 + var zoom = view.zoom; 247 + var scale = Math.min(W, H) * 0.42 * zoom; 248 + var colors = getColors(); 249 + var r = getPointRadius(zoom); 250 + 251 + // visible bounds in data space 252 + var tl = screenToData(0, 0); 253 + var br = screenToData(W, H); 254 + var pad = 0.05; 255 + var xMin = tl[0] - pad, xMax = br[0] + pad; 256 + var yMin = tl[1] - pad, yMax = br[1] + pad; 257 + 258 + // --- cluster glows (zoomed out) --- 259 + if (zoom < 4) { 260 + var clusters = zoom < 2 ? data.clusters.coarse : data.clusters.fine; 261 + for (var c = 0; c < clusters.length; c++) { 262 + var cl = clusters[c]; 263 + var sp = dataToScreen(cl.cx, cl.cy); 264 + if (sp[0] < -100 || sp[0] > W + 100 || sp[1] < -100 || sp[1] > H + 100) continue; 265 + drawClusterGlow(sp[0], sp[1], cl.count, zoom < 2 ? 0.6 : 0.3); 266 + } 267 + } 268 + 269 + // --- points --- 270 + var points = data.points; 271 + var n = points.length; 272 + ctx.globalAlpha = 1; 273 + 274 + for (var i = 0; i < n; i++) { 275 + var px = pointsX[i]; 276 + var py = pointsY[i]; 277 + if (px < xMin || px > xMax || py < yMin || py > yMax) continue; 278 + 279 + var sp = dataToScreen(px, py); 280 + var platform = points[i].platform || 'other'; 281 + var c = colors[platform] || colors.other; 282 + var alpha = (i === hoveredIndex) ? 1.0 : (zoom > 3 ? 0.85 : 0.7); 283 + drawPoint(sp[0], sp[1], (i === hoveredIndex) ? r * 2 : r, c, alpha); 284 + } 285 + 286 + // --- labels --- 287 + ctx.globalAlpha = 1; 288 + 289 + if (zoom < 2) { 290 + // coarse cluster labels 291 + var fontSize = Math.max(10, 13 / zoom); 292 + for (var c = 0; c < data.clusters.coarse.length; c++) { 293 + var cl = data.clusters.coarse[c]; 294 + var sp = dataToScreen(cl.cx, cl.cy); 295 + if (sp[0] < -50 || sp[0] > W + 50 || sp[1] < -20 || sp[1] > H + 20) continue; 296 + drawLabel(cl.label, sp[0], sp[1] - Math.sqrt(cl.count) * 1.5, fontSize, 0.8); 297 + } 298 + } else if (zoom < 5) { 299 + // fine cluster labels 300 + var fontSize = Math.max(9, 11 / (zoom * 0.5)); 301 + for (var c = 0; c < data.clusters.fine.length; c++) { 302 + var cl = data.clusters.fine[c]; 303 + if (cl.cx < xMin || cl.cx > xMax || cl.cy < yMin || cl.cy > yMax) continue; 304 + var sp = dataToScreen(cl.cx, cl.cy); 305 + if (sp[0] < -50 || sp[0] > W + 50 || sp[1] < -20 || sp[1] > H + 20) continue; 306 + drawLabel(cl.label, sp[0], sp[1] - 12, fontSize, 0.7); 307 + } 308 + } else { 309 + // individual document titles 310 + var fontSize = Math.min(12, 10 / (zoom * 0.15)); 311 + var shown = 0; 312 + var maxLabels = 60; 313 + for (var i = 0; i < n && shown < maxLabels; i++) { 314 + var px = pointsX[i]; 315 + var py = pointsY[i]; 316 + if (px < xMin || px > xMax || py < yMin || py > yMax) continue; 317 + var title = points[i].title; 318 + if (!title) continue; 319 + var sp = dataToScreen(px, py); 320 + if (sp[0] < 0 || sp[0] > W || sp[1] < 0 || sp[1] > H) continue; 321 + // truncate long titles 322 + if (title.length > 40) title = title.substring(0, 38) + '\u2026'; 323 + drawLabel(title, sp[0], sp[1] - r * 3 - 4, fontSize, 0.6); 324 + shown++; 325 + } 326 + } 327 + 328 + ctx.globalAlpha = 1; 329 + } 330 + 331 + // --- animation loop --- 332 + function loop() { 333 + render(); 334 + requestAnimationFrame(loop); 335 + } 336 + 337 + // --- interaction: mouse --- 338 + canvas.addEventListener('wheel', function(e) { 339 + e.preventDefault(); 340 + var factor = e.deltaY > 0 ? 0.9 : 1.1; 341 + var newZoom = Math.max(view.minZoom, Math.min(view.maxZoom, view.zoom * factor)); 342 + 343 + // zoom toward cursor 344 + var d = screenToData(e.clientX, e.clientY); 345 + view.zoom = newZoom; 346 + var d2 = screenToData(e.clientX, e.clientY); 347 + view.panX += d2[0] - d[0]; 348 + view.panY += d2[1] - d[1]; 349 + 350 + view.dirty = true; 351 + gradientCache = {}; 352 + }, { passive: false }); 353 + 354 + canvas.addEventListener('mousedown', function(e) { 355 + if (e.button !== 0) return; 356 + dragging = true; 357 + dragStartX = e.clientX; 358 + dragStartY = e.clientY; 359 + dragStartPanX = view.panX; 360 + dragStartPanY = view.panY; 361 + }); 362 + 363 + window.addEventListener('mousemove', function(e) { 364 + mouseX = e.clientX; 365 + mouseY = e.clientY; 366 + 367 + if (dragging) { 368 + var scale = Math.min(W, H) * 0.42 * view.zoom; 369 + view.panX = dragStartPanX + (e.clientX - dragStartX) / scale; 370 + view.panY = dragStartPanY + (e.clientY - dragStartY) / scale; 371 + view.dirty = true; 372 + hideTooltip(); 373 + return; 374 + } 375 + 376 + // hover with delay 377 + clearTimeout(hoverTimer); 378 + hoverTimer = setTimeout(function() { 379 + var idx = findNearest(mouseX, mouseY, 20); 380 + if (idx !== hoveredIndex) { 381 + hoveredIndex = idx; 382 + view.dirty = true; 383 + if (idx >= 0) { 384 + showTooltip(idx, mouseX, mouseY); 385 + } else { 386 + hideTooltip(); 387 + } 388 + } 389 + }, 100); 390 + }); 391 + 392 + window.addEventListener('mouseup', function(e) { 393 + if (dragging) { 394 + var dx = Math.abs(e.clientX - dragStartX); 395 + var dy = Math.abs(e.clientY - dragStartY); 396 + // click detection: small drag = click 397 + if (dx < 4 && dy < 4 && hoveredIndex >= 0) { 398 + var p = data.points[hoveredIndex]; 399 + var url = atUriToUrl(p.uri, p.basePath, p.platform); 400 + if (url) window.open(url, '_blank'); 401 + } 402 + dragging = false; 403 + } 404 + }); 405 + 406 + // --- interaction: touch --- 407 + var touches = {}; 408 + 409 + canvas.addEventListener('touchstart', function(e) { 410 + e.preventDefault(); 411 + for (var i = 0; i < e.changedTouches.length; i++) { 412 + var t = e.changedTouches[i]; 413 + touches[t.identifier] = { x: t.clientX, y: t.clientY }; 414 + } 415 + var ids = Object.keys(touches); 416 + if (ids.length === 1) { 417 + dragging = true; 418 + dragStartX = touches[ids[0]].x; 419 + dragStartY = touches[ids[0]].y; 420 + dragStartPanX = view.panX; 421 + dragStartPanY = view.panY; 422 + } else if (ids.length === 2) { 423 + dragging = false; 424 + var a = touches[ids[0]], b = touches[ids[1]]; 425 + pinchStartDist = Math.hypot(a.x - b.x, a.y - b.y); 426 + pinchStartZoom = view.zoom; 427 + } 428 + }, { passive: false }); 429 + 430 + canvas.addEventListener('touchmove', function(e) { 431 + e.preventDefault(); 432 + for (var i = 0; i < e.changedTouches.length; i++) { 433 + var t = e.changedTouches[i]; 434 + touches[t.identifier] = { x: t.clientX, y: t.clientY }; 435 + } 436 + var ids = Object.keys(touches); 437 + if (ids.length === 1 && dragging) { 438 + var scale = Math.min(W, H) * 0.42 * view.zoom; 439 + view.panX = dragStartPanX + (touches[ids[0]].x - dragStartX) / scale; 440 + view.panY = dragStartPanY + (touches[ids[0]].y - dragStartY) / scale; 441 + view.dirty = true; 442 + } else if (ids.length === 2) { 443 + var a = touches[ids[0]], b = touches[ids[1]]; 444 + var dist = Math.hypot(a.x - b.x, a.y - b.y); 445 + var newZoom = pinchStartZoom * (dist / pinchStartDist); 446 + view.zoom = Math.max(view.minZoom, Math.min(view.maxZoom, newZoom)); 447 + view.dirty = true; 448 + gradientCache = {}; 449 + } 450 + }, { passive: false }); 451 + 452 + canvas.addEventListener('touchend', function(e) { 453 + for (var i = 0; i < e.changedTouches.length; i++) { 454 + delete touches[e.changedTouches[i].identifier]; 455 + } 456 + if (Object.keys(touches).length === 0) { 457 + dragging = false; 458 + } 459 + }); 460 + 461 + // --- tooltip --- 462 + var tooltip = document.getElementById('tooltip'); 463 + var tooltipTitle = document.getElementById('tooltip-title'); 464 + var tooltipMeta = document.getElementById('tooltip-meta'); 465 + var tooltipPlatform = document.getElementById('tooltip-platform'); 466 + 467 + function showTooltip(idx, sx, sy) { 468 + var p = data.points[idx]; 469 + tooltipTitle.textContent = p.title || '(untitled)'; 470 + tooltipMeta.textContent = p.basePath || p.uri; 471 + tooltipPlatform.textContent = p.platform; 472 + var colors = getColors(); 473 + var c = colors[p.platform] || colors.other; 474 + tooltipPlatform.style.background = c.edge; 475 + tooltipPlatform.style.color = c.core; 476 + 477 + tooltip.style.display = 'block'; 478 + // position: avoid going off screen 479 + var tw = tooltip.offsetWidth; 480 + var th = tooltip.offsetHeight; 481 + var tx = sx + 16; 482 + var ty = sy - th - 8; 483 + if (tx + tw > W - 10) tx = sx - tw - 16; 484 + if (ty < 10) ty = sy + 16; 485 + tooltip.style.left = tx + 'px'; 486 + tooltip.style.top = ty + 'px'; 487 + 488 + canvas.style.cursor = 'pointer'; 489 + } 490 + 491 + function hideTooltip() { 492 + tooltip.style.display = 'none'; 493 + hoveredIndex = -1; 494 + canvas.style.cursor = dragging ? 'grabbing' : 'grab'; 495 + } 496 + 497 + // --- AT URI to URL --- 498 + function atUriToUrl(uri, basePath, platform) { 499 + // at://did:plc:xxx/collection/rkey 500 + var m = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/(.+)$/); 501 + if (!m) return null; 502 + var did = m[1], collection = m[2], rkey = m[3]; 503 + 504 + if (platform === 'whitewind' || collection.startsWith('com.whtwnd.')) { 505 + return 'https://whtwnd.com/' + did + '/' + rkey; 506 + } 507 + if (basePath) { 508 + return 'https://' + basePath + '/' + rkey; 509 + } 510 + // fallback: try to construct a reasonable URL 511 + return 'https://pds.pub/at/' + encodeURIComponent(uri); 512 + } 513 + 514 + // --- legend --- 515 + function renderLegend() { 516 + var el = document.getElementById('legend'); 517 + var colors = getColors(); 518 + var html = ''; 519 + var platforms = ['leaflet', 'whitewind', 'pckt', 'offprint', 'greengale', 'other']; 520 + for (var i = 0; i < platforms.length; i++) { 521 + var p = platforms[i]; 522 + var c = colors[p]; 523 + html += '<div class="legend-item"><span class="legend-dot" style="background:' + c.mid + '"></span>' + p + '</div>'; 524 + } 525 + el.innerHTML = html; 526 + } 527 + 528 + // --- load data --- 529 + function loadData() { 530 + fetch('constellation.json') 531 + .then(function(r) { 532 + if (!r.ok) throw new Error('failed to load constellation.json: ' + r.status); 533 + return r.json(); 534 + }) 535 + .then(function(d) { 536 + data = d; 537 + 538 + // build typed arrays 539 + var n = d.points.length; 540 + pointsX = new Float32Array(n); 541 + pointsY = new Float32Array(n); 542 + for (var i = 0; i < n; i++) { 543 + pointsX[i] = d.points[i].x; 544 + pointsY[i] = d.points[i].y; 545 + } 546 + 547 + buildSpatialIndex(); 548 + renderLegend(); 549 + 550 + // stats 551 + document.getElementById('stats').textContent = 552 + n.toLocaleString() + ' documents \u00B7 ' + 553 + d.clusters.coarse.length + ' regions \u00B7 ' + 554 + d.clusters.fine.length + ' clusters'; 555 + 556 + // hide loading 557 + document.getElementById('loading').classList.add('hidden'); 558 + 559 + view.dirty = true; 560 + }) 561 + .catch(function(err) { 562 + document.getElementById('loading').querySelector('.spinner').textContent = 563 + 'error: ' + err.message; 564 + console.error(err); 565 + }); 566 + } 567 + 568 + // --- init --- 569 + window.addEventListener('resize', resizeCanvas); 570 + resizeCanvas(); 571 + loadData(); 572 + loop(); 573 + 574 + // expose for theme toggle 575 + window.constellation = { 576 + setDirty: function() { 577 + gradientCache = {}; 578 + renderLegend(); 579 + view.dirty = true; 580 + } 581 + }; 582 + })();
+1 -1
site/dashboard.html
··· 98 98 </section> 99 99 100 100 <footer> 101 - <a href="/">back</a> · source on <a href="https://tangled.sh/@zzstoatzz.io/leaflet-search">tangled</a> · <span class="theme-toggle" id="theme-toggle"></span> 101 + <a href="/">back</a> · <a href="/constellation.html">constellation</a> · source on <a href="https://tangled.sh/@zzstoatzz.io/leaflet-search">tangled</a> · <span class="theme-toggle" id="theme-toggle"></span> 102 102 </footer> 103 103 </div> 104 104
+1 -1
site/index.html
··· 1594 1594 fetch(`${API_URL}/stats`) 1595 1595 .then(r => r.json()) 1596 1596 .then(data => { 1597 - statsDiv.innerHTML = `${data.documents} documents, ${data.publications} publications · <a href="/dashboard.html">stats</a>`; 1597 + statsDiv.innerHTML = `${data.documents} documents, ${data.publications} publications · <a href="/dashboard.html">stats</a> · <a href="/constellation.html">constellation</a>`; 1598 1598 const headerStats = document.getElementById('header-stats'); 1599 1599 headerStats.href = '/dashboard.html'; 1600 1600 headerStats.textContent = `[${data.documents.toLocaleString()} docs]`;