kidlisp: v2 structural index + dashboard kidlisp tab · aesthetic.computer/core@1ee2c6c

+8 -2

kidlisp-sidecar/src/ac/kidlisp/core.clj

··· 3 3 [reitit.ring :as ring] 4 4 [reitit.ring.middleware.muuntaja :as muuntaja-mw] 5 5 [muuntaja.core :as m] 6 + [ring.middleware.params :as params] 6 7 [ac.kidlisp.db :as db] 7 8 [ac.kidlisp.schema :as schema] 8 9 [ac.kidlisp.handlers :as h] ··· 33 34 ["/:code/pending-rebake" {:post (h/set-pending-rebake conn)}] 34 35 ["/:code/ipfs-media" {:post (h/set-ipfs-media conn)}] 35 36 ["/:code/atproto-rkey" {:post (h/set-atproto-rkey conn)}] 36 - ["/:code/lineage" {:get (h/lineage conn)}]] 37 + ["/:code/lineage" {:get (h/lineage conn)}] 38 + ["/:code/ast" {:get (h/get-ast conn) 39 + :post (h/set-ast conn)}] 40 + ["/structural/pieces-using" {:get (h/find-pieces-using-op conn)}]] 37 41 38 42 ;; ───── Admin surface (silo-only, read-only in v1) ───── 39 43 ["/admin" ··· 63 67 port (Integer/parseInt (env "PORT" "8891")) 64 68 conn (db/connect uri)] 65 69 (schema/ensure! conn) 66 - (http/run-server (app-router conn) {:port port :ip "127.0.0.1"}) 70 + ;; wrap-params populates :query-params (string-keyed) from the URL. 71 + (http/run-server (params/wrap-params (app-router conn)) 72 + {:port port :ip "127.0.0.1"}) 67 73 (println (str "kidlisp-sidecar listening on 127.0.0.1:" port))))

+92

kidlisp-sidecar/src/ac/kidlisp/handlers.clj

··· 376 376 (ok {:ok true})) 377 377 (not-found))))) 378 378 379 + (defn set-ast 380 + "POST /kidlisp/:code/ast 381 + body: {nodes: [{id, kind, op?, literal?, parent?, position, depth}]} 382 + Atomically replaces the piece's AST — retracts all existing nodes for 383 + this piece, then asserts the new set. `id` is a client-side integer 384 + used for parent refs inside the batch; it doesn't persist." 385 + [conn] 386 + (fn [req] 387 + (let [code (get-in req [:path-params :code]) 388 + nodes (get-in req [:body-params :nodes]) 389 + db (d/db conn)] 390 + (cond 391 + (nil? code) (bad "missing :code") 392 + (nil? nodes) (bad "missing :nodes") 393 + :else 394 + (if-let [piece-eid (d/entid db [:kidlisp/code code])] 395 + (let [old-eids (d/q '[:find [?n ...] 396 + :in $ ?p 397 + :where [?n :ast/piece ?p]] 398 + db piece-eid) 399 + retract-tx (mapv (fn [eid] [:db/retractEntity eid]) old-eids) 400 + id->tmp (into {} (for [n nodes] [(:id n) (str "n" (:id n))])) 401 + node-tx 402 + (mapv (fn [n] 403 + (cond-> {:db/id (id->tmp (:id n)) 404 + :ast/piece piece-eid 405 + :ast/kind (keyword (:kind n)) 406 + :ast/position (or (:position n) 0) 407 + :ast/depth (or (:depth n) 0)} 408 + (:op n) (assoc :ast/op (:op n)) 409 + (:literal n) (assoc :ast/literal (:literal n)) 410 + (:parent n) (assoc :ast/parent (id->tmp (:parent n))))) 411 + nodes) 412 + link-tx (mapv (fn [n] 413 + [:db/add piece-eid :kidlisp/ast-nodes (id->tmp (:id n))]) 414 + nodes) 415 + tx (vec (concat retract-tx node-tx link-tx))] 416 + (when (seq tx) (db/transact conn tx)) 417 + (ok {:ok true 418 + :nodes (count nodes) 419 + :retracted (count old-eids)})) 420 + (not-found)))))) 421 + 422 + (defn get-ast 423 + "GET /kidlisp/:code/ast — returns nodes as a flat list keyed by eid. 424 + Dashboard renders as a tree by following :parent refs on the client." 425 + [conn] 426 + (fn [req] 427 + (let [code (get-in req [:path-params :code]) 428 + db (d/db conn)] 429 + (if-let [piece-eid (d/entid db [:kidlisp/code code])] 430 + (let [node-eids (d/q '[:find [?n ...] 431 + :in $ ?p 432 + :where [?n :ast/piece ?p]] 433 + db piece-eid) 434 + nodes (->> node-eids 435 + (map #(d/entity db %)) 436 + (mapv (fn [e] 437 + {:id (:db/id e) 438 + :kind (:ast/kind e) 439 + :op (:ast/op e) 440 + :literal (:ast/literal e) 441 + :parent (get-in e [:ast/parent :db/id]) 442 + :position (:ast/position e) 443 + :depth (:ast/depth e)})) 444 + (sort-by (juxt :depth :position :id)))] 445 + (ok {:code code :nodes (vec nodes)})) 446 + (not-found))))) 447 + 448 + (defn find-pieces-using-op 449 + "GET /kidlisp/structural/pieces-using?op=wipe&limit=50 450 + Returns codes of pieces containing at least one AST node with the given op. 451 + This is the corpus-wide structural query that Mongo couldn't do." 452 + [conn] 453 + (fn [req] 454 + (let [op (get-in req [:query-params "op"]) 455 + limit (max 1 (min 1000 (Integer/parseInt 456 + (or (get-in req [:query-params "limit"]) "50")))) 457 + db (d/db conn)] 458 + (if-not (and op (seq op)) 459 + (bad "op query param required") 460 + (let [codes (d/q '[:find [?code ...] 461 + :in $ ?op 462 + :where 463 + [?n :ast/op ?op] 464 + [?n :ast/piece ?p] 465 + [?p :kidlisp/code ?code]] 466 + db op)] 467 + (ok {:op op 468 + :count (count codes) 469 + :codes (vec (take limit (sort codes)))})))))) 470 + 379 471 (defn lineage 380 472 "GET /kidlisp/:code/lineage — returns {ancestors, descendants} as 381 473 chains of {code, author}. Ancestors walks :kidlisp/forked-from up,

+76 -6

kidlisp-sidecar/src/ac/kidlisp/schema.clj

··· 227 227 :db/cardinality :db.cardinality/one} 228 228 {:db/ident :rebake/contract-version 229 229 :db/valueType :db.type/string 230 - :db/cardinality :db.cardinality/one}]) 230 + :db/cardinality :db.cardinality/one} 231 + 232 + ;; ───────── AST (v2) ───────── 233 + ;; Parsed structure of a kidlisp piece. One entity per node — flat 234 + ;; list linked via :ast/parent refs. Source of truth is the 235 + ;; JS parser in system/backend/kidlisp-ast.mjs; the sidecar just 236 + ;; persists what it's told. 237 + ;; 238 + ;; :ast/piece is indexed so "all nodes for piece X" is fast. 239 + ;; :ast/op is indexed for structural queries like "all pieces using wipe". 240 + 241 + {:db/ident :kidlisp/ast-nodes 242 + :db/valueType :db.type/ref 243 + :db/cardinality :db.cardinality/many 244 + :db/isComponent true 245 + :db/doc "Component backref: all AST nodes that belong to this 246 + piece. Retracting the piece retracts its AST."} 247 + 248 + {:db/ident :ast/piece 249 + :db/valueType :db.type/ref 250 + :db/cardinality :db.cardinality/one 251 + :db/index true 252 + :db/doc "The kidlisp piece this node belongs to. Every node 253 + carries this so corpus-wide structural queries can 254 + roll up to pieces cheaply."} 231 255 232 - (defn- schema-installed? [db] 233 - (some? (d/entid db :kidlisp/code))) 256 + {:db/ident :ast/kind 257 + :db/valueType :db.type/keyword 258 + :db/cardinality :db.cardinality/one 259 + :db/index true 260 + :db/doc ":call | :atom | :number | :string | :ref | :timing | 261 + :fade | :comment"} 262 + 263 + {:db/ident :ast/op 264 + :db/valueType :db.type/string 265 + :db/cardinality :db.cardinality/one 266 + :db/index true 267 + :db/doc "The head symbol of a :call node (e.g. \"wipe\", 268 + \"repeat\"). Absent for atoms/literals."} 269 + 270 + {:db/ident :ast/literal 271 + :db/valueType :db.type/string 272 + :db/cardinality :db.cardinality/one 273 + :db/doc "The raw literal for :atom, :number, :string, :ref, 274 + :timing, :fade, :comment nodes. Stored as string 275 + for simple text search; numbers keep their source 276 + form (\"1.5\")."} 277 + 278 + {:db/ident :ast/parent 279 + :db/valueType :db.type/ref 280 + :db/cardinality :db.cardinality/one 281 + :db/doc "Parent node ref. Nil for the piece's top-level 282 + sequence of forms."} 283 + 284 + {:db/ident :ast/position 285 + :db/valueType :db.type/long 286 + :db/cardinality :db.cardinality/one 287 + :db/doc "Zero-based index within the parent's children. 288 + Enables ordered children queries."} 289 + 290 + {:db/ident :ast/depth 291 + :db/valueType :db.type/long 292 + :db/cardinality :db.cardinality/one 293 + :db/doc "Nesting depth from the piece root. Useful for 294 + quick `wiggle-inside-repeat` style patterns — 295 + pair with op + ancestor walks."}]) 234 296 235 297 (defn ensure! 236 - "Idempotently installs schema-v1 if not already present." 298 + "Idempotently installs any schema entries not yet present. Per-attribute 299 + check so evolving the schema over time (e.g. v2 AST attrs) just adds 300 + the missing pieces on the next sidecar boot." 237 301 [conn] 238 - (when-not (schema-installed? (d/db conn)) 239 - @(d/transact conn schema-v1))) 302 + (let [db (d/db conn) 303 + missing (filter (fn [m] 304 + (when-let [ident (:db/ident m)] 305 + (nil? (d/entid db ident)))) 306 + schema-v1)] 307 + (when (seq missing) 308 + @(d/transact conn (vec missing)) 309 + (println (str "schema: installed " (count missing) " new attribute(s)")))))

+183

silo/dashboard.html

··· 317 317 <button class="tab-btn" data-tab="7">telemetry</button> 318 318 <button class="tab-btn" data-tab="8">lith</button> 319 319 <button class="tab-btn" data-tab="9">datomic</button> 320 + <button class="tab-btn" data-tab="10">kidlisp</button> 320 321 </div> 321 322 322 323 <div class="panels"> ··· 709 710 <span class="bar-dim" id="dt-query-status"></span> 710 711 </div> 711 712 <pre id="dt-query-result" style="font-size:11px;margin:0;white-space:pre-wrap"></pre> 713 + </div> 714 + </div> 715 + 716 +  717 + <div class="panel" data-panel="10"> 718 + <div style="display:flex;gap:6px;align-items:center;padding:4px 6px;border-bottom:1px solid var(--border)"> 719 + <input id="kl-search" placeholder="code or fragment..." class="browse-input" 720 + style="font-family:inherit;font-size:11px;padding:3px 6px;background:var(--bg);color:var(--fg);border:1px solid var(--border);width:180px"> 721 + <button class="btn" onclick="klLoad('hits')">by hits</button> 722 + <button class="btn" onclick="klLoad('recent')">recent</button> 723 + <span class="bar-dim" id="kl-status"></span> 724 + </div> 725 + <div style="display:grid;grid-template-columns:minmax(200px,1fr) 1.2fr 1fr;gap:6px;padding:6px;height:calc(100vh - 150px);overflow:hidden"> 726 +  727 + <div class="card" style="overflow-y:auto;padding:4px"> 728 + <div class="card-hd">pieces <b id="kl-list-count"></b></div> 729 + <div id="kl-list" style="font-size:11px">pick a sort to load</div> 730 + </div> 731 +  732 + <div class="card" style="overflow-y:auto;padding:4px"> 733 + <div class="card-hd"> 734 + <span id="kl-detail-code">—</span> 735 + <span class="bar-dim" id="kl-detail-meta"></span> 736 + </div> 737 + <pre id="kl-source" style="font-size:11px;margin:0;white-space:pre-wrap;color:var(--fg)">select a piece</pre> 738 + <div style="margin-top:6px;padding-top:4px;border-top:1px solid var(--border);font-size:10px;color:var(--fg2)" id="kl-meta"></div> 739 + </div> 740 +  741 + <div class="card" style="overflow-y:auto;padding:4px"> 742 + <div class="card-hd">structure <span class="bar-dim" style="font-size:10px">click op → corpus search</span></div> 743 + <div id="kl-tree" style="font-size:11px;font-family:inherit">—</div> 744 + <div style="margin-top:6px;padding-top:4px;border-top:1px solid var(--border)"> 745 + <div class="card-hd"> 746 + <span id="kl-struct-label">pieces using X</span> 747 + <span class="bar-dim" id="kl-struct-count"></span> 748 + </div> 749 + <div id="kl-struct-result" style="font-size:11px;max-height:240px;overflow-y:auto">click an op in the tree above</div> 750 + </div> 751 + </div> 712 752 </div> 713 753 </div> 714 754 ··· 2369 2409 if (e.target.matches('[data-tab="9"]')) { 2370 2410 loadDatomic(); 2371 2411 if (!datomicTimer) datomicTimer = setInterval(loadDatomic, 15000); 2412 + } 2413 + }); 2414 + 2415 + // ───────────────────────── kidlisp tab (v2: AST-aware) ───────────────────────── 2416 + let klPieces = []; 2417 + let klSelectedCode = null; 2418 + 2419 + function klEsc(s) { 2420 + return String(s ?? '').replace(/[&<>"']/g, (c) => ({ 2421 + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', 2422 + })[c]); 2423 + } 2424 + 2425 + async function klLoad(sort) { 2426 + document.getElementById('kl-status').textContent = 'loading...'; 2427 + try { 2428 + const qs = new URLSearchParams({ sort: sort || 'hits', limit: '200' }); 2429 + const data = await dtFetch('/api/datomic/kidlisp?' + qs.toString()); 2430 + klPieces = data.recent || []; 2431 + document.getElementById('kl-list-count').textContent = klPieces.length; 2432 + document.getElementById('kl-status').textContent = ''; 2433 + klRenderList(); 2434 + } catch (err) { 2435 + document.getElementById('kl-status').textContent = 'error: ' + err.message; 2436 + } 2437 + } 2438 + 2439 + function klRenderList() { 2440 + const filter = (document.getElementById('kl-search').value || '').toLowerCase(); 2441 + const rows = klPieces.filter((p) => { 2442 + if (!filter) return true; 2443 + return ( 2444 + (p.code || '').toLowerCase().includes(filter) || 2445 + (p.source || '').toLowerCase().includes(filter) 2446 + ); 2447 + }); 2448 + document.getElementById('kl-list').innerHTML = rows.map((p) => { 2449 + const hits = p.hits ?? 0; 2450 + const preview = (p.source || '').substring(0, 30).replace(/\n/g, ' '); 2451 + const active = p.code === klSelectedCode ? 'background:var(--hover);' : ''; 2452 + return ( 2453 + '<div style="padding:2px 4px;cursor:pointer;border-bottom:1px solid var(--border);' + active + '" ' + 2454 + 'onclick="klShow(\'' + klEsc(p.code) + '\')">' + 2455 + '<span style="color:var(--accent)">$' + klEsc(p.code) + '</span> ' + 2456 + '<span style="color:var(--fg2)">·</span> ' + 2457 + '<span style="color:var(--fg2)">' + hits + '</span> ' + 2458 + '<div style="color:var(--fg2);font-size:10px;margin-left:4px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">' + 2459 + klEsc(preview) + '</div></div>' 2460 + ); 2461 + }).join(''); 2462 + } 2463 + 2464 + document.addEventListener('input', (e) => { 2465 + if (e.target.id === 'kl-search') klRenderList(); 2466 + }); 2467 + 2468 + async function klShow(code) { 2469 + klSelectedCode = code; 2470 + klRenderList(); 2471 + document.getElementById('kl-detail-code').textContent = '$' + code; 2472 + document.getElementById('kl-source').textContent = 'loading...'; 2473 + document.getElementById('kl-tree').textContent = 'loading...'; 2474 + document.getElementById('kl-struct-result').textContent = 'click an op in the tree above'; 2475 + document.getElementById('kl-struct-count').textContent = ''; 2476 + document.getElementById('kl-struct-label').textContent = 'pieces using X'; 2477 + 2478 + try { 2479 + const piece = await dtFetch('/api/datomic/kidlisp/' + encodeURIComponent(code)); 2480 + document.getElementById('kl-source').textContent = piece.source || '(empty)'; 2481 + document.getElementById('kl-detail-meta').textContent = 2482 + (piece.user ? piece.user.substring(0, 20) + ' · ' : '') + 2483 + (piece.hits ?? 0) + ' hits'; 2484 + const meta = []; 2485 + if (piece.when) meta.push('created ' + new Date(piece.when).toLocaleDateString()); 2486 + if (piece.atproto?.rkey) meta.push('atproto: ' + piece.atproto.rkey); 2487 + if (piece.tezos?.minted) meta.push('tezos minted #' + piece.tezos.tokenId); 2488 + if (piece.keeps?.length) meta.push(piece.keeps.length + ' keep(s)'); 2489 + document.getElementById('kl-meta').textContent = meta.join(' · '); 2490 + } catch (err) { 2491 + document.getElementById('kl-source').textContent = 'error: ' + err.message; 2492 + } 2493 + 2494 + try { 2495 + const ast = await dtFetch('/api/datomic/kidlisp/' + encodeURIComponent(code) + '/ast'); 2496 + document.getElementById('kl-tree').innerHTML = klRenderTree(ast.nodes || []); 2497 + } catch (err) { 2498 + document.getElementById('kl-tree').textContent = 'no AST indexed yet — run backfill'; 2499 + } 2500 + } 2501 + 2502 + function klRenderTree(nodes) { 2503 + if (!nodes.length) return '<span style="color:var(--fg2)">(empty)</span>'; 2504 + const byId = new Map(nodes.map((n) => [n.id, n])); 2505 + const kids = new Map(); 2506 + for (const n of nodes) { 2507 + const p = n.parent ?? -1; 2508 + if (!kids.has(p)) kids.set(p, []); 2509 + kids.get(p).push(n); 2510 + } 2511 + for (const arr of kids.values()) arr.sort((a, b) => (a.position ?? 0) - (b.position ?? 0)); 2512 + 2513 + function walk(id, indent) { 2514 + const n = byId.get(id); 2515 + const pad = '  '.repeat(indent); 2516 + let label; 2517 + if (n.kind === ':call' || n.kind === 'call') { 2518 + const op = n.op || '?'; 2519 + label = pad + '<span class="kl-op" data-op="' + klEsc(op) + '" ' + 2520 + 'onclick="klStructuralSearch(\'' + klEsc(op) + '\')" ' + 2521 + 'style="color:var(--accent);cursor:pointer;text-decoration:underline dotted">(' + klEsc(op) + ')</span>'; 2522 + } else { 2523 + const kindLabel = String(n.kind || '').replace(/^:/, ''); 2524 + label = pad + '<span style="color:var(--fg2)">' + kindLabel + ':</span><span>' + 2525 + klEsc(n.literal ?? '') + '</span>'; 2526 + } 2527 + const out = [label]; 2528 + for (const c of kids.get(id) ?? []) out.push(walk(c.id, indent + 1)); 2529 + return out.join('<br>'); 2530 + } 2531 + return (kids.get(-1) ?? []).map((n) => walk(n.id, 0)).join('<br>'); 2532 + } 2533 + 2534 + async function klStructuralSearch(op) { 2535 + document.getElementById('kl-struct-label').textContent = "pieces using '" + op + "'"; 2536 + document.getElementById('kl-struct-count').textContent = 'searching...'; 2537 + document.getElementById('kl-struct-result').textContent = ''; 2538 + try { 2539 + const qs = new URLSearchParams({ op, limit: '100' }); 2540 + const data = await dtFetch('/api/datomic/structural/pieces-using?' + qs.toString()); 2541 + document.getElementById('kl-struct-count').textContent = data.count + ' pieces'; 2542 + document.getElementById('kl-struct-result').innerHTML = (data.codes || []).map((c) => 2543 + '<span style="display:inline-block;margin:1px 4px 1px 0;padding:1px 4px;background:var(--bg);border:1px solid var(--border);cursor:pointer;color:var(--accent)" ' + 2544 + 'onclick="klShow(\'' + klEsc(c) + '\')">$' + klEsc(c) + '</span>' 2545 + ).join('') || '(none)'; 2546 + } catch (err) { 2547 + document.getElementById('kl-struct-count').textContent = 'error'; 2548 + document.getElementById('kl-struct-result').textContent = err.message; 2549 + } 2550 + } 2551 + 2552 + document.addEventListener('click', e => { 2553 + if (e.target.matches('[data-tab="10"]')) { 2554 + if (!klPieces.length) klLoad('hits'); 2372 2555 } 2373 2556 }); 2374 2557

+51

silo/server.mjs

··· 1387 1387 app.get("/api/datomic/backups", requireAdmin, (req, res) => 1388 1388 datomicProxy(req, res, { method: "GET", path: "/admin/backups" })); 1389 1389 1390 + // Admin-gated proxy into the sidecar's client-secret read endpoints — 1391 + // used by the dashboard's kidlisp tab to render AST trees + run structural 1392 + // queries. Silo holds both secrets; dashboard admin auth → silo adds the 1393 + // client-secret header → sidecar serves. 1394 + const DATOMIC_SIDECAR_CLIENT_SECRET = process.env.DATOMIC_SIDECAR_CLIENT_SECRET; 1395 + 1396 + async function sidecarClientProxy(req, res, { method, path, body }) { 1397 + if (!DATOMIC_SIDECAR_CLIENT_SECRET) { 1398 + return res.status(503).json({ error: "sidecar client secret not configured on silo" }); 1399 + } 1400 + try { 1401 + const resp = await fetch(`${DATOMIC_SIDECAR_URL}${path}`, { 1402 + method, 1403 + headers: { 1404 + "content-type": "application/json", 1405 + "x-sidecar-secret": DATOMIC_SIDECAR_CLIENT_SECRET, 1406 + }, 1407 + body: body != null ? JSON.stringify(body) : undefined, 1408 + signal: AbortSignal.timeout(15000), 1409 + }); 1410 + const text = await resp.text(); 1411 + let data; 1412 + try { data = text ? JSON.parse(text) : null; } catch { data = text; } 1413 + res.status(resp.status).json(data); 1414 + } catch (err) { 1415 + res.status(502).json({ error: err.message }); 1416 + } 1417 + } 1418 + 1419 + app.get("/api/datomic/kidlisp", requireAdmin, (req, res) => { 1420 + const qs = new URLSearchParams(req.query).toString(); 1421 + sidecarClientProxy(req, res, { method: "GET", path: `/kidlisp${qs ? `?${qs}` : ""}` }); 1422 + }); 1423 + 1424 + app.get("/api/datomic/kidlisp/:code", requireAdmin, (req, res) => 1425 + sidecarClientProxy(req, res, { method: "GET", path: `/kidlisp/${encodeURIComponent(req.params.code)}` })); 1426 + 1427 + app.get("/api/datomic/kidlisp/:code/ast", requireAdmin, (req, res) => 1428 + sidecarClientProxy(req, res, { method: "GET", path: `/kidlisp/${encodeURIComponent(req.params.code)}/ast` })); 1429 + 1430 + app.get("/api/datomic/kidlisp/:code/lineage", requireAdmin, (req, res) => 1431 + sidecarClientProxy(req, res, { method: "GET", path: `/kidlisp/${encodeURIComponent(req.params.code)}/lineage` })); 1432 + 1433 + app.get("/api/datomic/structural/pieces-using", requireAdmin, (req, res) => { 1434 + const qs = new URLSearchParams(req.query).toString(); 1435 + sidecarClientProxy(req, res, { 1436 + method: "GET", 1437 + path: `/kidlisp/structural/pieces-using${qs ? `?${qs}` : ""}`, 1438 + }); 1439 + }); 1440 + 1390 1441 // ─────────────────── Datomic sidecar — public kidlisp proxy ─────────────────── 1391 1442 // 1392 1443 // Server-to-server surface for lith (which runs store-kidlisp-datomic.mjs).

+114

system/backend/backfill-kidlisp-ast.mjs

··· 1 + // Backfill AST for existing kidlisp pieces that were migrated before the 2 + // v2 structural index landed. Iterates Datomic via the sidecar's list 3 + // endpoint, extracts AST from source, POSTs to /kidlisp/:code/ast. 4 + // Idempotent — retracts-old + asserts-new on the sidecar side. 5 + // 6 + // Env: 7 + // SIDECAR_URL default http://127.0.0.1:8891 8 + // CLIENT_SECRET required 9 + // DRY_RUN "true" to count without writing 10 + // BATCH_SIZE default 200 (per /kidlisp list page) 11 + // CONCURRENCY default 4 (parallel POSTs) 12 + 13 + import { extractAst } from "./kidlisp-ast.mjs"; 14 + 15 + const SIDECAR_URL = process.env.SIDECAR_URL || "http://127.0.0.1:8891"; 16 + const CLIENT_SECRET = process.env.CLIENT_SECRET; 17 + const DRY_RUN = process.env.DRY_RUN === "true"; 18 + const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || "200", 10); 19 + const CONCURRENCY = parseInt(process.env.CONCURRENCY || "4", 10); 20 + 21 + if (!DRY_RUN && !CLIENT_SECRET) { 22 + console.error("CLIENT_SECRET is required (or DRY_RUN=true)"); 23 + process.exit(1); 24 + } 25 + 26 + const headers = () => ({ 27 + "content-type": "application/json", 28 + "x-sidecar-secret": CLIENT_SECRET, 29 + }); 30 + 31 + async function listPieces(limit, sort = "recent") { 32 + // The sidecar's /kidlisp list endpoint returns up to `limit` pieces. 33 + // For the backfill we ask for everything, sorted by created-at oldest 34 + // first so we process deterministically. 35 + const qs = new URLSearchParams({ limit: String(limit), sort }); 36 + const r = await fetch(`${SIDECAR_URL}/kidlisp?${qs.toString()}`, { headers: headers() }); 37 + if (!r.ok) throw new Error(`list failed: ${r.status} ${await r.text()}`); 38 + const body = await r.json(); 39 + return body.recent || []; 40 + } 41 + 42 + async function postAst(code, nodes) { 43 + if (DRY_RUN) return { ok: true, dryRun: true }; 44 + const r = await fetch(`${SIDECAR_URL}/kidlisp/${encodeURIComponent(code)}/ast`, { 45 + method: "POST", 46 + headers: headers(), 47 + body: JSON.stringify({ nodes }), 48 + }); 49 + if (!r.ok) { 50 + return { ok: false, status: r.status, body: await r.text().catch(() => "") }; 51 + } 52 + return { ok: true, body: await r.json().catch(() => ({})) }; 53 + } 54 + 55 + // Simple concurrent pool. 56 + async function pool(items, workerFn, size) { 57 + const queue = [...items]; 58 + const results = []; 59 + async function worker() { 60 + while (queue.length) { 61 + const it = queue.shift(); 62 + results.push(await workerFn(it)); 63 + } 64 + } 65 + await Promise.all(Array.from({ length: size }, worker)); 66 + return results; 67 + } 68 + 69 + async function main() { 70 + const started = Date.now(); 71 + console.log(`▶ AST backfill start — dryRun=${DRY_RUN} sidecar=${SIDECAR_URL}`); 72 + 73 + // Pull the whole corpus in one shot — /kidlisp supports huge limits. 74 + // If this ever hits memory limits, switch to pagination via `since`. 75 + const pieces = await listPieces(100000); 76 + console.log(` pieces to process: ${pieces.length}`); 77 + 78 + const stats = { processed: 0, indexed: 0, empty: 0, errors: 0, nodes: 0 }; 79 + let lastLog = Date.now(); 80 + 81 + await pool( 82 + pieces, 83 + async (p) => { 84 + const nodes = extractAst(p.source || ""); 85 + stats.processed++; 86 + if (!nodes.length) { 87 + stats.empty++; 88 + } else { 89 + const r = await postAst(p.code, nodes); 90 + if (r.ok) { 91 + stats.indexed++; 92 + stats.nodes += nodes.length; 93 + } else { 94 + stats.errors++; 95 + console.error(` ! ${p.code} ast failed: ${r.status} ${r.body}`); 96 + } 97 + } 98 + if (Date.now() - lastLog > 3000) { 99 + console.log(` ${stats.processed}/${pieces.length} — ${JSON.stringify(stats)}`); 100 + lastLog = Date.now(); 101 + } 102 + }, 103 + CONCURRENCY, 104 + ); 105 + 106 + const secs = ((Date.now() - started) / 1000).toFixed(1); 107 + console.log(`✓ AST backfill done in ${secs}s — ${JSON.stringify(stats)}`); 108 + if (stats.errors > 0) process.exit(1); 109 + } 110 + 111 + main().catch((err) => { 112 + console.error("✗ AST backfill fatal:", err); 113 + process.exit(1); 114 + });

+193

system/backend/kidlisp-ast.mjs

··· 1 + // KidLisp AST extractor — Node-only, dependency-free. 2 + // 3 + // Produces a flat list of nodes with parent refs, suitable for sending 4 + // to the sidecar's POST /kidlisp/:code/ast endpoint. The shape is 5 + // structural, not semantic — we don't resolve references or evaluate 6 + // anything. Good enough for corpus-wide structural queries like 7 + // "find all pieces using `wipe`" or "pieces with `repeat` at depth ≥ 2". 8 + // 9 + // This is NOT a strict decree-compliant parser. It's a lightweight 10 + // structural tokenizer aligned with the conventions in 11 + // system/public/aesthetic.computer/lib/kidlisp.mjs: 12 + // - semicolon comments 13 + // - parenthesized s-expressions with optional comma arg separators 14 + // - bare identifiers at line start → wrapped as (ident) 15 + // - $code references 16 + // - timing tokens like 1s, 2.5s., 1.5s... 17 + // - fade:red-blue gradient tokens 18 + // - string literals in "..." or '...' 19 + // - numbers (int/float, optional leading -) 20 + // 21 + // Node shape: 22 + // { id, kind, op?, literal?, parent?, position, depth } 23 + // Where `id` is batch-local integer, `parent` is another id or null. 24 + 25 + const TOKEN_RE = 26 + /\s*(;.*|[(),]|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|[^\s(),;"']+)/g; 27 + 28 + function tokenize(src) { 29 + const out = []; 30 + let m; 31 + TOKEN_RE.lastIndex = 0; 32 + while ((m = TOKEN_RE.exec(src)) !== null) { 33 + out.push(m[1]); 34 + } 35 + return out; 36 + } 37 + 38 + // Detect which pre-parse transform to apply (mirrors kidlisp.mjs). 39 + // Returns a source string where bare-word commands / comma lists have 40 + // been wrapped in parens so the recursive descent below can stay simple. 41 + function normalize(src) { 42 + // Strip line comments up-front, keeping blank lines so line-splitting stays aligned. 43 + const lines = src.split("\n").map((l) => { 44 + const i = l.indexOf(";"); 45 + return i === -1 ? l : l.substring(0, i); 46 + }); 47 + 48 + // Wrap comma-separated one-liners into parenthesized exprs. 49 + // e.g. "blue, ink rainbow" -> "(blue) (ink rainbow)" 50 + const transformed = lines.map((line) => { 51 + const trimmed = line.trim(); 52 + if (!trimmed) return ""; 53 + if (trimmed.includes(",")) { 54 + return trimmed 55 + .split(",") 56 + .map((e) => e.trim()) 57 + .filter(Boolean) 58 + .map((e) => 59 + e.startsWith("(") 60 + ? e 61 + : /^[a-zA-Z_$]/.test(e) 62 + ? `(${e})` 63 + : e, 64 + ) 65 + .join(" "); 66 + } 67 + // Bare-word command at line start: wrap 68 + if (!trimmed.startsWith("(") && /^[a-zA-Z_$]/.test(trimmed)) { 69 + return `(${trimmed})`; 70 + } 71 + return trimmed; 72 + }); 73 + 74 + return transformed.join("\n"); 75 + } 76 + 77 + // Classify a non-call atom token. 78 + function classifyAtom(tok) { 79 + if (/^\$[a-zA-Z0-9]+$/.test(tok)) return { kind: "ref", literal: tok }; 80 + if (/^\d*\.?\d+s(\.{1,3})?!?$/.test(tok)) 81 + return { kind: "timing", literal: tok }; 82 + if (tok.startsWith("fade:")) return { kind: "fade", literal: tok }; 83 + if (/^-?\d+(\.\d+)?$/.test(tok)) return { kind: "number", literal: tok }; 84 + if (/^["'].*["']$/.test(tok)) return { kind: "string", literal: tok }; 85 + return { kind: "atom", literal: tok }; 86 + } 87 + 88 + /** 89 + * Parse a kidlisp source string into a flat AST node list with parent 90 + * refs. Top-level forms are siblings under an implicit root (which is 91 + * not emitted — top-level nodes just have `parent: null`). 92 + * 93 + * @param {string} source 94 + * @returns {Array<{id:number, kind:string, op?:string, literal?:string, 95 + * parent:number|null, position:number, depth:number}>} 96 + */ 97 + export function extractAst(source) { 98 + if (!source || typeof source !== "string") return []; 99 + 100 + const normalized = normalize(source); 101 + const tokens = tokenize(normalized); 102 + 103 + const nodes = []; 104 + let nextId = 0; 105 + const push = (node) => { 106 + const id = nextId++; 107 + nodes.push({ id, ...node }); 108 + return id; 109 + }; 110 + 111 + // Recursive descent over tokens[ptr]. Returns new ptr. 112 + let ptr = 0; 113 + const siblingCounts = new Map(); // parentId(or -1) -> count 114 + 115 + function nextSiblingPos(parent) { 116 + const key = parent === null ? -1 : parent; 117 + const n = siblingCounts.get(key) ?? 0; 118 + siblingCounts.set(key, n + 1); 119 + return n; 120 + } 121 + 122 + function readExpr(parent, depth) { 123 + if (ptr >= tokens.length) return; 124 + const tok = tokens[ptr]; 125 + 126 + if (tok === "(") { 127 + ptr++; 128 + // First token inside parens is the op symbol (best effort — if 129 + // missing or itself a paren, record as unknown). 130 + let op = null; 131 + if (ptr < tokens.length && tokens[ptr] !== "(" && tokens[ptr] !== ")") { 132 + op = tokens[ptr]; 133 + ptr++; 134 + } 135 + const callId = push({ 136 + kind: "call", 137 + op: op ?? undefined, 138 + parent, 139 + position: nextSiblingPos(parent), 140 + depth, 141 + }); 142 + // Remaining tokens up to matching `)` are children 143 + while (ptr < tokens.length && tokens[ptr] !== ")") { 144 + readExpr(callId, depth + 1); 145 + } 146 + if (tokens[ptr] === ")") ptr++; 147 + return; 148 + } 149 + 150 + if (tok === ")") { 151 + // Stray close paren — skip 152 + ptr++; 153 + return; 154 + } 155 + 156 + // Atom 157 + ptr++; 158 + const cls = classifyAtom(tok); 159 + push({ 160 + kind: cls.kind, 161 + literal: cls.literal, 162 + parent, 163 + position: nextSiblingPos(parent), 164 + depth, 165 + }); 166 + } 167 + 168 + while (ptr < tokens.length) readExpr(null, 0); 169 + 170 + return nodes; 171 + } 172 + 173 + // Helper for debugging / tests: reconstruct a text tree. 174 + export function astToTree(nodes) { 175 + const byId = new Map(nodes.map((n) => [n.id, n])); 176 + const kids = new Map(); 177 + for (const n of nodes) { 178 + const p = n.parent ?? -1; 179 + if (!kids.has(p)) kids.set(p, []); 180 + kids.get(p).push(n); 181 + } 182 + for (const arr of kids.values()) arr.sort((a, b) => a.position - b.position); 183 + 184 + const lines = []; 185 + function walk(id, indent) { 186 + const n = byId.get(id); 187 + const label = n.kind === "call" ? `(${n.op ?? "?"})` : `${n.kind}:${n.literal}`; 188 + lines.push(" ".repeat(indent) + label); 189 + for (const c of kids.get(id) ?? []) walk(c.id, indent + 1); 190 + } 191 + for (const top of kids.get(-1) ?? []) walk(top.id, 0); 192 + return lines.join("\n"); 193 + }

+23

system/backend/kidlisp-sidecar.mjs

··· 118 118 if (!r.ok) throw new Error(`sidecar lineage failed: ${r.status}`); 119 119 return r.body; 120 120 }, 121 + 122 + // AST (v2): replace-all. `nodes` is the flat list from 123 + // system/backend/kidlisp-ast.mjs extractAst(). Atomic retract + assert 124 + // on the sidecar side. 125 + async setAst(code, nodes) { 126 + const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/ast`, { nodes }); 127 + if (!r.ok) throw new Error(`sidecar setAst failed: ${r.status}`); 128 + return r.body; 129 + }, 130 + 131 + async getAst(code) { 132 + const r = await req("GET", `/kidlisp/${encodeURIComponent(code)}/ast`); 133 + if (r.status === 404) return null; 134 + if (!r.ok) throw new Error(`sidecar getAst failed: ${r.status}`); 135 + return r.body; 136 + }, 137 + 138 + async piecesUsingOp(op, limit = 50) { 139 + const qs = new URLSearchParams({ op, limit: String(limit) }); 140 + const r = await req("GET", `/kidlisp/structural/pieces-using?${qs.toString()}`); 141 + if (!r.ok) throw new Error(`sidecar piecesUsingOp failed: ${r.status}`); 142 + return r.body; 143 + }, 121 144 }; 122 145 123 146 // Feature flag helper — read once per request since Netlify functions

+15

system/netlify/functions/store-kidlisp-datomic.mjs

··· 16 16 import { createMediaRecord, MediaTypes } from "../../backend/media-atproto.mjs"; 17 17 import { publishProfileEvent } from "../../backend/profile-stream.mjs"; 18 18 import { sidecar } from "../../backend/kidlisp-sidecar.mjs"; 19 + import { extractAst } from "../../backend/kidlisp-ast.mjs"; 19 20 import crypto from "crypto"; 20 21 21 22 const TEZOS_ENABLED = process.env.TEZOS_ENABLED === "true"; ··· 162 163 // sidecar returned {code, cached} — if cached (raced with another write), 163 164 // the returned code may differ from our proposed `code`. Honor it. 164 165 const finalCode = created.code; 166 + 167 + // AST index (fire-and-forget): parse source structure and POST to 168 + // sidecar so corpus-wide datalog queries can find it. If the piece 169 + // was cached (hash dedup), its AST is already stored — skip. 170 + if (!created.cached) { 171 + try { 172 + const astNodes = extractAst(source.trim()); 173 + sidecar.setAst(finalCode, astNodes).catch((err) => { 174 + console.warn("⚠️ AST index failed:", err?.message || err); 175 + }); 176 + } catch (err) { 177 + console.warn("⚠️ AST extract failed:", err?.message || err); 178 + } 179 + } 165 180 166 181 // Profile stream event (fire-and-forget) 167 182 if (profileHandle) {

Configure Feed

Configure Feed