Monorepo for Aesthetic.Computer
aesthetic.computer
kidlisp: add Datomic v1 (sidecar + silo infra + feature-flagged route)
New `kidlisp-sidecar/` Clojure service exposes a Datomic-native API for
kidlisp pieces, ATProto rkeys, IPFS bundles, mint events, and the
existing tezos/keep state. Schema marks `:kidlisp/hits` and
`:kidlisp/last-accessed` `:db/noHistory` so the tx log stays meaningful.
A read-only `/admin/*` surface (schema, stats, entity browse + history,
tx log, sandboxed datalog console, backups) is silo-only.
`silo/datomic/` holds the host-side ops: transactor systemd unit + properties
template, Postgres init, nightly pg_dump cron with 14-day retention, and
deploy/bootstrap fish scripts. Storage-access-key is aligned to the
`datomic` PG user so peer connections work without a separate role.
`silo/server.mjs` proxies `/api/datomic/*` to the sidecar with
`requireAdmin`, and `silo/dashboard.html` adds a `datomic` tab with
status, schema table, entity browser + history, tx log, and a
read-only datalog console. Sidecar is bound to 127.0.0.1; only silo
talks to it.
`system/netlify/functions/store-kidlisp-datomic.mjs` is a parallel
implementation of `store-kidlisp.mjs` that speaks the existing external
API but routes all reads/writes through the sidecar (no Mongo writes for
kidlisp). The original handler routes to it when `KIDLISP_DATOMIC=on`,
so cutover is a single env-var flip and rollback is the inverse.
`system/backend/backfill-kidlisp-to-datomic.mjs` is the one-shot Mongo
replay (idempotent via hash dedup). MIGRATION.md punchlists the 30+
remaining touch points to migrate post-cutover.
v1 brought up live on silo, backfilled 17,532 docs / 0 errors, counts
match Mongo exactly. Feature flag stays off pending cutover.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
+2903
+7
kidlisp-sidecar/.gitignore
+7
kidlisp-sidecar/.gitignore
+96
kidlisp-sidecar/deploy.fish
+96
kidlisp-sidecar/deploy.fish
···
1
+
#!/usr/bin/env fish
2
+
## Builds the kidlisp-sidecar uberjar and ships it to silo.
3
+
## Assumes silo/datomic/deploy.fish has already set up the transactor.
4
+
##
5
+
## Usage:
6
+
## fish deploy.fish Full deploy (build + ship + restart)
7
+
## fish deploy.fish --no-build Ship existing uberjar only
8
+
9
+
set RED '\033[0;31m'
10
+
set GREEN '\033[0;32m'
11
+
set YELLOW '\033[1;33m'
12
+
set NC '\033[0m'
13
+
14
+
set SCRIPT_DIR (dirname (status --current-filename))
15
+
set VAULT_DIR "$SCRIPT_DIR/../aesthetic-computer-vault"
16
+
set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa"
17
+
set SIDECAR_ENV_GPG "$VAULT_DIR/kidlisp-datomic/sidecar.env.gpg"
18
+
set SILO_HOST "silo.aesthetic.computer"
19
+
set SILO_USER "root"
20
+
set REMOTE_DIR "/opt/kidlisp-sidecar"
21
+
set JAR "$SCRIPT_DIR/target/kidlisp-sidecar.jar"
22
+
23
+
set DO_BUILD true
24
+
if contains -- --no-build $argv
25
+
set DO_BUILD false
26
+
end
27
+
28
+
## Prereqs
29
+
if not test -f $SSH_KEY
30
+
echo -e "$RED x SSH key not found: $SSH_KEY$NC"
31
+
exit 1
32
+
end
33
+
34
+
if not test -f $SIDECAR_ENV_GPG
35
+
echo -e "$RED x Vault file missing: $SIDECAR_ENV_GPG$NC"
36
+
echo -e "$YELLOW Populate via vault workflow before deploying.$NC"
37
+
exit 1
38
+
end
39
+
40
+
## Build
41
+
if test $DO_BUILD = true
42
+
echo -e "$GREEN-> Building uberjar...$NC"
43
+
cd $SCRIPT_DIR
44
+
clojure -X:uberjar
45
+
if test $status -ne 0
46
+
echo -e "$RED x Build failed.$NC"
47
+
exit 1
48
+
end
49
+
end
50
+
51
+
if not test -f $JAR
52
+
echo -e "$RED x Uberjar not found: $JAR$NC"
53
+
exit 1
54
+
end
55
+
56
+
## Decrypt env locally, upload, then wipe local copy
57
+
set TMP_ENV (mktemp)
58
+
gpg --decrypt --quiet --output $TMP_ENV $SIDECAR_ENV_GPG
59
+
if test $status -ne 0
60
+
echo -e "$RED x gpg decrypt failed.$NC"
61
+
rm -f $TMP_ENV
62
+
exit 1
63
+
end
64
+
65
+
echo -e "$GREEN-> Uploading jar + env + systemd unit...$NC"
66
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
67
+
$JAR \
68
+
$SILO_USER@$SILO_HOST:$REMOTE_DIR/kidlisp-sidecar.jar
69
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
70
+
$TMP_ENV \
71
+
$SILO_USER@$SILO_HOST:$REMOTE_DIR/.env
72
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
73
+
$SCRIPT_DIR/kidlisp-sidecar.service \
74
+
$SILO_USER@$SILO_HOST:/etc/systemd/system/kidlisp-sidecar.service
75
+
76
+
rm -f $TMP_ENV
77
+
78
+
## Permission + restart
79
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "
80
+
chown datomic:datomic $REMOTE_DIR/kidlisp-sidecar.jar $REMOTE_DIR/.env
81
+
chmod 0600 $REMOTE_DIR/.env
82
+
systemctl daemon-reload
83
+
systemctl restart kidlisp-sidecar
84
+
sleep 2
85
+
systemctl is-active kidlisp-sidecar
86
+
"
87
+
set STATUS $status
88
+
if test $STATUS -eq 0
89
+
echo -e "$GREEN Sidecar restarted OK.$NC"
90
+
else
91
+
echo -e "$RED x Sidecar failed to start. Check:$NC"
92
+
echo -e "$YELLOW ssh -i $SSH_KEY $SILO_USER@$SILO_HOST journalctl -u kidlisp-sidecar -n 50$NC"
93
+
exit 1
94
+
end
95
+
96
+
echo -e "$GREEN Done.$NC"
+35
kidlisp-sidecar/deps.edn
+35
kidlisp-sidecar/deps.edn
···
1
+
{:paths ["src" "resources"]
2
+
:deps {org.clojure/clojure {:mvn/version "1.12.0"}
3
+
4
+
;; HTTP server
5
+
metosin/reitit {:mvn/version "0.7.2"}
6
+
metosin/muuntaja {:mvn/version "0.6.11"}
7
+
metosin/jsonista {:mvn/version "0.3.13"}
8
+
http-kit/http-kit {:mvn/version "2.8.0"}
9
+
ring/ring-core {:mvn/version "1.13.0"}
10
+
11
+
;; Datomic Pro (Apache 2.0)
12
+
com.datomic/peer {:mvn/version "1.0.7180"}
13
+
14
+
;; Postgres JDBC driver — required by the peer to read from SQL
15
+
;; storage (and to register the driver with DriverManager when
16
+
;; the datomic:sql://... URI is parsed).
17
+
org.postgresql/postgresql {:mvn/version "42.7.4"}
18
+
19
+
;; Logging
20
+
org.slf4j/slf4j-simple {:mvn/version "2.0.16"}}
21
+
22
+
:aliases
23
+
{:run {:main-opts ["-m" "ac.kidlisp.core"]}
24
+
25
+
:uberjar {:deps {com.github.seancorfield/depstar {:mvn/version "2.1.303"}}
26
+
:replace-paths []
27
+
:replace-deps {com.github.clj-easy/graal-build-time
28
+
{:mvn/version "1.0.5"}}
29
+
:exec-fn hf.depstar/uberjar
30
+
:exec-args {:jar "target/kidlisp-sidecar.jar"
31
+
:aot true
32
+
:main-class ac.kidlisp.core}}
33
+
34
+
:dev {:extra-deps {nrepl/nrepl {:mvn/version "1.3.0"}}
35
+
:main-opts ["-m" "nrepl.cmdline" "--port" "7888"]}}}
+24
kidlisp-sidecar/kidlisp-sidecar.service
+24
kidlisp-sidecar/kidlisp-sidecar.service
···
1
+
[Unit]
2
+
Description=kidlisp-sidecar (Datomic bridge for Aesthetic Computer)
3
+
After=network.target datomic-transactor.service
4
+
Requires=datomic-transactor.service
5
+
6
+
[Service]
7
+
Type=simple
8
+
User=datomic
9
+
Group=datomic
10
+
WorkingDirectory=/opt/kidlisp-sidecar
11
+
EnvironmentFile=/opt/kidlisp-sidecar/.env
12
+
## Runs directly from the checked-out source via Clojure CLI. Deps are
13
+
## resolved on first start and cached in ~/.m2 for subsequent restarts.
14
+
ExecStart=/usr/local/bin/clojure -J-Xmx256m -J-Xms128m -M -m ac.kidlisp.core
15
+
Environment=HOME=/opt/kidlisp-sidecar
16
+
Restart=on-failure
17
+
RestartSec=5
18
+
StandardOutput=journal
19
+
StandardError=journal
20
+
21
+
LimitNOFILE=65536
22
+
23
+
[Install]
24
+
WantedBy=multi-user.target
+210
kidlisp-sidecar/src/ac/kidlisp/admin.clj
+210
kidlisp-sidecar/src/ac/kidlisp/admin.clj
···
1
+
(ns ac.kidlisp.admin
2
+
"Read-only admin surface consumed by the silo dashboard via the silo server.
3
+
v1 is strictly read-only: no transact, no retract, no write surface at all."
4
+
(:require [datomic.api :as d]
5
+
[clojure.edn :as edn]
6
+
[clojure.java.io :as io]
7
+
[clojure.string :as str]))
8
+
9
+
(defn- ok [body] {:status 200 :body body})
10
+
(defn- bad [msg] {:status 400 :body {:error msg}})
11
+
12
+
(defn schema
13
+
"GET /admin/schema — attribute catalog (only :kidlisp/ :keep/ :tezos/
14
+
:rebake/ :ipfs/ :user/ idents; filters out system attributes)."
15
+
[conn]
16
+
(fn [_req]
17
+
(let [db (d/db conn)
18
+
our? (fn [kw]
19
+
(contains? #{"kidlisp" "keep" "tezos" "rebake" "ipfs" "user" "mint"}
20
+
(namespace kw)))
21
+
attrs (d/q '[:find [?ident ...]
22
+
:where [?e :db/ident ?ident]]
23
+
db)
24
+
rows (for [ident attrs
25
+
:when (and (keyword? ident) (our? ident))
26
+
:let [e (d/entity db ident)]]
27
+
{:ident ident
28
+
:valueType (:db/valueType e)
29
+
:cardinality (:db/cardinality e)
30
+
:unique (:db/unique e)
31
+
:indexed (boolean (:db/index e))
32
+
:isComponent (boolean (:db/isComponent e))
33
+
:noHistory (boolean (:db/noHistory e))
34
+
:doc (:db/doc e)})]
35
+
(ok {:attributes (vec (sort-by :ident rows))}))))
36
+
37
+
(defn stats
38
+
"GET /admin/stats — entity counts per type + recent tx rate."
39
+
[conn]
40
+
(fn [_req]
41
+
(let [db (d/db conn)
42
+
kidlisp (ffirst (d/q '[:find (count ?e) :where [?e :kidlisp/code]] db))
43
+
users (ffirst (d/q '[:find (count ?e) :where [?e :user/sub]] db))
44
+
keeps (ffirst (d/q '[:find (count ?e) :where [?e :keep/token-id]] db))
45
+
tx-last-hour
46
+
(ffirst (d/q '[:find (count ?tx)
47
+
:in $ ?since
48
+
:where
49
+
[?tx :db/txInstant ?inst]
50
+
[(> ?inst ?since)]]
51
+
db
52
+
(java.util.Date. (- (System/currentTimeMillis) (* 60 60 1000)))))]
53
+
(ok {:entityCounts {:kidlisp kidlisp
54
+
:users users
55
+
:keeps keeps}
56
+
:txLastHour tx-last-hour
57
+
:basisT (d/basis-t db)}))))
58
+
59
+
(defn list-entities
60
+
"GET /admin/entities/:type?limit=&offset=
61
+
Supports :type = kidlisp | user | keep."
62
+
[conn]
63
+
(fn [req]
64
+
(let [typ (get-in req [:path-params :type])
65
+
{:strs [limit offset]} (:query-params req)
66
+
limit (max 1 (min 1000 (Integer/parseInt (or limit "50"))))
67
+
offset (max 0 (Integer/parseInt (or offset "0")))
68
+
db (d/db conn)
69
+
q (case typ
70
+
"kidlisp" '[:find [?e ...] :where [?e :kidlisp/code]]
71
+
"user" '[:find [?e ...] :where [?e :user/sub]]
72
+
"keep" '[:find [?e ...] :where [?e :keep/token-id]]
73
+
nil)]
74
+
(if-not q
75
+
(bad (str "unknown type: " typ))
76
+
(let [all (vec (sort (d/q q db)))
77
+
page (->> all (drop offset) (take limit))
78
+
rows (mapv (fn [eid]
79
+
(let [e (d/entity db eid)]
80
+
(into {:db/id eid} e)))
81
+
page)]
82
+
(ok {:type typ
83
+
:total (count all)
84
+
:offset offset
85
+
:limit limit
86
+
:items rows}))))))
87
+
88
+
(defn entity
89
+
"GET /admin/entity/:eid — current facts."
90
+
[conn]
91
+
(fn [req]
92
+
(let [eid (Long/parseLong (get-in req [:path-params :eid]))
93
+
db (d/db conn)
94
+
e (d/entity db eid)]
95
+
(if e
96
+
(ok (into {:db/id eid} e))
97
+
{:status 404 :body {:error "not-found"}}))))
98
+
99
+
(defn entity-history
100
+
"GET /admin/entity/:eid/history — all facts ever asserted about this
101
+
entity, with tx timestamp and add/retract flag."
102
+
[conn]
103
+
(fn [req]
104
+
(let [eid (Long/parseLong (get-in req [:path-params :eid]))
105
+
db (d/db conn)
106
+
rows (d/q '[:find ?a ?v ?tx ?inst ?added
107
+
:in $ ?e
108
+
:where
109
+
[?e ?a ?v ?tx ?added]
110
+
[?tx :db/txInstant ?inst]]
111
+
(d/history db) eid)
112
+
by-tx (sort-by #(nth % 3) rows)]
113
+
(ok {:eid eid
114
+
:history (mapv (fn [[a v tx inst added]]
115
+
{:attr (d/ident db a)
116
+
:value (if (keyword? v) v (str v))
117
+
:tx tx
118
+
:at inst
119
+
:added added})
120
+
by-tx)}))))
121
+
122
+
(defn tx-log
123
+
"GET /admin/tx-log?limit=&since=
124
+
Recent transactions with summary of assertions/retractions."
125
+
[conn]
126
+
(fn [req]
127
+
(let [{:strs [limit since]} (:query-params req)
128
+
limit (max 1 (min 1000 (Integer/parseInt (or limit "50"))))
129
+
db (d/db conn)
130
+
since-inst (when (seq since)
131
+
(java.util.Date/from (java.time.Instant/parse since)))
132
+
rows (d/q (if since-inst
133
+
'[:find ?tx ?inst
134
+
:in $ ?since
135
+
:where
136
+
[?tx :db/txInstant ?inst]
137
+
[(> ?inst ?since)]]
138
+
'[:find ?tx ?inst
139
+
:where
140
+
[?tx :db/txInstant ?inst]])
141
+
db (or since-inst (java.util.Date. 0)))
142
+
recent (->> rows (sort-by second #(compare %2 %1)) (take limit))]
143
+
(ok {:transactions
144
+
(mapv (fn [[tx inst]]
145
+
{:tx tx
146
+
:at inst})
147
+
recent)}))))
148
+
149
+
(defn- safe-query?
150
+
"Ultra-conservative read-only check on a datalog query edn string.
151
+
Rejects transact/retract/with/apply-tx/io-rsrc forms and anything
152
+
mentioning `:db/add` or `:db/retract`. Intentionally strict."
153
+
[q-str]
154
+
(and (string? q-str)
155
+
(not (re-find #"(?i)\b(transact|retract|with|d/with|db/add|db/retract|d/transact|io-rsrc)\b" q-str))))
156
+
157
+
(defn query
158
+
"POST /admin/query — body: {query (edn string), args (vec)}
159
+
Runs against (d/db conn) only. Results capped at 10k rows + 5s timeout."
160
+
[conn]
161
+
(fn [req]
162
+
(let [{:keys [query args]} (:body-params req)]
163
+
(if-not (safe-query? query)
164
+
(bad "query rejected (read-only policy)")
165
+
(try
166
+
(let [parsed (edn/read-string query)
167
+
db (d/db conn)
168
+
fut (future (d/q parsed db (or args [])))
169
+
result (deref fut 5000 ::timeout)]
170
+
(cond
171
+
(= result ::timeout)
172
+
(do (future-cancel fut)
173
+
{:status 408 :body {:error "query timed out (5s)"}})
174
+
175
+
:else
176
+
(ok {:count (count result)
177
+
:rows (->> result (take 10000) vec)
178
+
:truncated (> (count result) 10000)})))
179
+
(catch Throwable t
180
+
{:status 500 :body {:error (.getMessage t)}}))))))
181
+
182
+
(defn backups
183
+
"GET /admin/backups — lists pg_dump files in /var/backups/datomic."
184
+
[_conn]
185
+
(fn [_req]
186
+
(let [dir (io/file "/var/backups/datomic")
187
+
files (when (.isDirectory dir)
188
+
(->> (.listFiles dir)
189
+
(filter #(str/ends-with? (.getName ^java.io.File %) ".sql.gz"))
190
+
(sort-by #(.lastModified ^java.io.File %) >)
191
+
(take 30)
192
+
(map (fn [^java.io.File f]
193
+
{:name (.getName f)
194
+
:size (.length f)
195
+
:at (java.util.Date. (.lastModified f))}))))]
196
+
(ok {:backups (vec files)}))))
197
+
198
+
(defn health
199
+
"GET /admin/health — transactor reachable, schema present."
200
+
[conn]
201
+
(fn [_req]
202
+
(try
203
+
(let [db (d/db conn)
204
+
schema-ok? (some? (d/entid db :kidlisp/code))]
205
+
(ok {:transactor true
206
+
:schema schema-ok?
207
+
:basisT (d/basis-t db)}))
208
+
(catch Throwable t
209
+
{:status 500 :body {:transactor false
210
+
:error (.getMessage t)}}))))
+26
kidlisp-sidecar/src/ac/kidlisp/auth.clj
+26
kidlisp-sidecar/src/ac/kidlisp/auth.clj
···
1
+
(ns ac.kidlisp.auth
2
+
"Shared-secret middleware. Two separate secrets:
3
+
CLIENT_SECRET — used by AC Netlify functions hitting /kidlisp/*
4
+
ADMIN_SECRET — used only by silo server hitting /admin/*
5
+
Keeping them separate means compromising the client secret cannot also
6
+
read admin-level data (history, tx log, arbitrary datalog).")
7
+
8
+
(defn- env [k]
9
+
(System/getenv k))
10
+
11
+
(defn- check-header [req secret]
12
+
(= (get-in req [:headers "x-sidecar-secret"]) secret))
13
+
14
+
(defn client-secret-middleware []
15
+
(fn [handler]
16
+
(fn [req]
17
+
(if (check-header req (env "CLIENT_SECRET"))
18
+
(handler req)
19
+
{:status 401 :body {:error "unauthorized"}}))))
20
+
21
+
(defn admin-secret-middleware []
22
+
(fn [handler]
23
+
(fn [req]
24
+
(if (check-header req (env "ADMIN_SECRET"))
25
+
(handler req)
26
+
{:status 401 :body {:error "unauthorized"}}))))
+67
kidlisp-sidecar/src/ac/kidlisp/core.clj
+67
kidlisp-sidecar/src/ac/kidlisp/core.clj
···
1
+
(ns ac.kidlisp.core
2
+
(:require [org.httpkit.server :as http]
3
+
[reitit.ring :as ring]
4
+
[reitit.ring.middleware.muuntaja :as muuntaja-mw]
5
+
[muuntaja.core :as m]
6
+
[ac.kidlisp.db :as db]
7
+
[ac.kidlisp.schema :as schema]
8
+
[ac.kidlisp.handlers :as h]
9
+
[ac.kidlisp.admin :as admin]
10
+
[ac.kidlisp.auth :as auth])
11
+
(:gen-class))
12
+
13
+
(defn- env [k default]
14
+
(or (System/getenv k) default))
15
+
16
+
(defn- app-router [conn]
17
+
(ring/ring-handler
18
+
(ring/router
19
+
[["/health"
20
+
{:get (fn [_] {:status 200 :body {:ok true}})}]
21
+
22
+
;; ───── Public kidlisp API (called by AC backend) ─────
23
+
["/kidlisp"
24
+
{:middleware [(auth/client-secret-middleware)]}
25
+
["" {:post (h/create conn)
26
+
:get (h/list-codes conn)}]
27
+
["/lookup" {:post (h/batch-lookup conn)}]
28
+
["/stats/functions" {:get (h/stats-functions conn)}]
29
+
["/hash/:hash" {:get (h/lookup-hash conn)}]
30
+
["/:code" {:get (h/lookup-code conn)}]
31
+
["/:code/mint" {:post (h/record-mint conn)}]
32
+
["/:code/tezos-state" {:post (h/set-tezos-state conn)}]
33
+
["/:code/pending-rebake" {:post (h/set-pending-rebake conn)}]
34
+
["/:code/ipfs-media" {:post (h/set-ipfs-media conn)}]
35
+
["/:code/atproto-rkey" {:post (h/set-atproto-rkey conn)}]
36
+
["/:code/lineage" {:get (h/lineage conn)}]]
37
+
38
+
;; ───── Admin surface (silo-only, read-only in v1) ─────
39
+
["/admin"
40
+
{:middleware [(auth/admin-secret-middleware)]}
41
+
["/schema" {:get (admin/schema conn)}]
42
+
["/stats" {:get (admin/stats conn)}]
43
+
["/entities/:type" {:get (admin/list-entities conn)}]
44
+
["/entity/:eid" {:get (admin/entity conn)}]
45
+
["/entity/:eid/history" {:get (admin/entity-history conn)}]
46
+
["/tx-log" {:get (admin/tx-log conn)}]
47
+
["/query" {:post (admin/query conn)}]
48
+
["/backups" {:get (admin/backups conn)}]
49
+
["/health" {:get (admin/health conn)}]]]
50
+
51
+
;; :conflicts nil disables reitit's strict conflict detector. Our
52
+
;; overlaps are resolved either by static-segment precedence
53
+
;; (e.g. /kidlisp/hash beats /kidlisp/:code) or by HTTP method
54
+
;; (e.g. POST /kidlisp/lookup vs GET /kidlisp/:code).
55
+
{:conflicts nil
56
+
:data {:muuntaja m/instance
57
+
:middleware [muuntaja-mw/format-middleware]}})
58
+
(ring/create-default-handler)))
59
+
60
+
(defn -main [& _]
61
+
(let [uri (env "DATOMIC_URI"
62
+
"datomic:sql://kidlisp?jdbc:postgresql://localhost:5432/datomic")
63
+
port (Integer/parseInt (env "PORT" "8891"))
64
+
conn (db/connect uri)]
65
+
(schema/ensure! conn)
66
+
(http/run-server (app-router conn) {:port port :ip "127.0.0.1"})
67
+
(println (str "kidlisp-sidecar listening on 127.0.0.1:" port))))
+15
kidlisp-sidecar/src/ac/kidlisp/db.clj
+15
kidlisp-sidecar/src/ac/kidlisp/db.clj
···
1
+
(ns ac.kidlisp.db
2
+
(:require [datomic.api :as d]))
3
+
4
+
(defn connect
5
+
"Create database if missing, return a connection.
6
+
URI is the full Datomic URI including storage protocol + db name,
7
+
e.g. datomic:sql://kidlisp?jdbc:postgresql://localhost:5432/datomic"
8
+
[uri]
9
+
(d/create-database uri)
10
+
(d/connect uri))
11
+
12
+
(defn db [conn] (d/db conn))
13
+
14
+
(defn transact [conn tx-data]
15
+
@(d/transact conn tx-data))
+409
kidlisp-sidecar/src/ac/kidlisp/handlers.clj
+409
kidlisp-sidecar/src/ac/kidlisp/handlers.clj
···
1
+
(ns ac.kidlisp.handlers
2
+
"Public kidlisp endpoints called by the AC backend (Netlify functions).
3
+
The external Mongo-era response shape is preserved by the Node compat
4
+
layer (system/netlify/functions/store-kidlisp.mjs); this sidecar returns
5
+
clean, Datomic-native shapes."
6
+
(:require [datomic.api :as d]
7
+
[ac.kidlisp.db :as db]))
8
+
9
+
;; ─────────────────── helpers ───────────────────
10
+
11
+
(defn- ok [body] {:status 200 :body body})
12
+
(defn- created [body] {:status 201 :body body})
13
+
(defn- bad [msg] {:status 400 :body {:error msg}})
14
+
(defn- not-found [] {:status 404 :body {:error "not-found"}})
15
+
16
+
(defn- now-inst [] (java.util.Date.))
17
+
18
+
(defn- ->user-ref
19
+
"Upserts a user entity by sub and returns a lookup ref usable in a
20
+
following tx data map."
21
+
[sub]
22
+
(when sub [:user/sub sub]))
23
+
24
+
(defn- piece->map
25
+
"Entity -> API response map. Pulls component children as well."
26
+
[e]
27
+
(let [m (into {} e)]
28
+
(cond-> {:code (:kidlisp/code m)
29
+
:hash (:kidlisp/hash m)
30
+
:source (:kidlisp/source m)
31
+
:when (:kidlisp/created-at m)
32
+
:lastAccessed (:kidlisp/last-accessed m)
33
+
:hits (or (:kidlisp/hits m) 0)
34
+
:user (get-in m [:kidlisp/author :user/sub])
35
+
:atproto (when-let [rk (:kidlisp/atproto-rkey m)]
36
+
{:rkey rk})}
37
+
(:kidlisp/ipfs-media m)
38
+
(assoc :ipfsMedia
39
+
(let [im (:kidlisp/ipfs-media m)]
40
+
{:artifactUri (:ipfs/artifact-uri im)
41
+
:thumbnailUri (:ipfs/thumbnail-uri im)
42
+
:sourceHash (:ipfs/source-hash im)
43
+
:createdAt (:ipfs/created-at im)
44
+
:authorHandle (:ipfs/author-handle im)
45
+
:depCount (:ipfs/dep-count im)
46
+
:packDate (:ipfs/pack-date im)}))
47
+
48
+
(seq (:kidlisp/keeps m))
49
+
(assoc :keeps
50
+
(mapv (fn [k]
51
+
{:tokenId (:keep/token-id k)
52
+
:network (:keep/network k)
53
+
:txHash (:keep/tx-hash k)
54
+
:contractAddress (:keep/contract-address k)
55
+
:contractProfile (:keep/contract-profile k)
56
+
:contractVersion (:keep/contract-version k)
57
+
:keptAt (:keep/kept-at k)
58
+
:keptBy (:keep/kept-by k)
59
+
:walletAddress (:keep/wallet-address k)
60
+
:artifactUri (:keep/artifact-uri k)
61
+
:thumbnailUri (:keep/thumbnail-uri k)
62
+
:metadataUri (:keep/metadata-uri k)
63
+
:source (:keep/source k)})
64
+
(:kidlisp/keeps m)))
65
+
66
+
(:kidlisp/tezos-state m)
67
+
(assoc :tezos
68
+
(let [t (:kidlisp/tezos-state m)]
69
+
{:minted (:tezos/minted t)
70
+
:exists (:tezos/exists t)
71
+
:tokenId (:tezos/token-id t)
72
+
:txHash (:tezos/tx-hash t)
73
+
:creatorAddress (:tezos/creator-address t)
74
+
:codeHash (:tezos/code-hash t)
75
+
:network (:tezos/network t)
76
+
:mintedAt (:tezos/minted-at t)
77
+
:checkedAt (:tezos/checked-at t)
78
+
:attemptedAt (:tezos/attempted-at t)
79
+
:failedAt (:tezos/failed-at t)
80
+
:reason (:tezos/reason t)
81
+
:error (:tezos/error t)}))
82
+
83
+
(:kidlisp/pending-rebake m)
84
+
(assoc :pendingRebake
85
+
(let [r (:kidlisp/pending-rebake m)]
86
+
{:artifactUri (:rebake/artifact-uri r)
87
+
:thumbnailUri (:rebake/thumbnail-uri r)
88
+
:metadataUri (:rebake/metadata-uri r)
89
+
:createdAt (:rebake/created-at r)
90
+
:contractAddress (:rebake/contract-address r)
91
+
:contractProfile (:rebake/contract-profile r)
92
+
:contractVersion (:rebake/contract-version r)})))))
93
+
94
+
(defn- entity-by-code [db code]
95
+
(when-let [eid (d/entid db [:kidlisp/code code])]
96
+
(d/entity db eid)))
97
+
98
+
;; ─────────────────── handlers ───────────────────
99
+
100
+
(defn- parse-inst [v]
101
+
(cond
102
+
(inst? v) v
103
+
(string? v) (java.util.Date/from (java.time.Instant/parse v))
104
+
(number? v) (java.util.Date. (long v))
105
+
:else nil))
106
+
107
+
(defn create
108
+
"POST /kidlisp
109
+
body: {source, hash, code, user_sub?, forked_from?, when?, hits?}
110
+
Dedup by hash: if an entity with the given hash exists, bump hits and
111
+
return its existing code. Otherwise insert with the proposed code.
112
+
113
+
`when` and `hits` are optional — present during backfill so historical
114
+
timestamps and hit counts are preserved. Absent during normal writes,
115
+
in which case server time / hits=1 is used."
116
+
[conn]
117
+
(fn [req]
118
+
(let [{:keys [source hash code user_sub forked_from when hits]}
119
+
(:body-params req)]
120
+
(if (or (nil? source) (nil? hash) (nil? code))
121
+
(bad "source, hash, code are required")
122
+
(let [db (d/db conn)]
123
+
(if-let [existing-eid (d/entid db [:kidlisp/hash hash])]
124
+
(let [existing (d/entity db existing-eid)]
125
+
(db/transact conn
126
+
[[:db/add existing-eid :kidlisp/hits
127
+
(inc (or (:kidlisp/hits existing) 0))]
128
+
[:db/add existing-eid :kidlisp/last-accessed
129
+
(now-inst)]])
130
+
(ok {:code (:kidlisp/code existing) :cached true}))
131
+
(let [created-at (or (parse-inst when) (now-inst))
132
+
piece-tx
133
+
(cond-> {:kidlisp/code code
134
+
:kidlisp/hash hash
135
+
:kidlisp/source source
136
+
:kidlisp/created-at created-at
137
+
:kidlisp/last-accessed created-at
138
+
:kidlisp/hits (or hits 1)}
139
+
user_sub
140
+
(assoc :kidlisp/author {:user/sub user_sub})
141
+
142
+
forked_from
143
+
(assoc :kidlisp/forked-from [:kidlisp/code forked_from]))]
144
+
(db/transact conn [piece-tx])
145
+
(created {:code code :cached false}))))))))
146
+
147
+
(defn lookup-code
148
+
"GET /kidlisp/:code — returns full entity, increments hits."
149
+
[conn]
150
+
(fn [req]
151
+
(let [code (get-in req [:path-params :code])
152
+
db (d/db conn)]
153
+
(if-let [e (entity-by-code db code)]
154
+
(do
155
+
(db/transact conn
156
+
[[:db/add (:db/id e) :kidlisp/hits
157
+
(inc (or (:kidlisp/hits e) 0))]
158
+
[:db/add (:db/id e) :kidlisp/last-accessed
159
+
(now-inst)]])
160
+
(ok (piece->map e)))
161
+
(not-found)))))
162
+
163
+
(defn lookup-hash
164
+
"GET /kidlisp/hash/:hash — dedup lookup (does not increment hits)."
165
+
[conn]
166
+
(fn [req]
167
+
(let [hsh (get-in req [:path-params :hash])
168
+
db (d/db conn)]
169
+
(if-let [eid (d/entid db [:kidlisp/hash hsh])]
170
+
(ok (piece->map (d/entity db eid)))
171
+
(not-found)))))
172
+
173
+
(defn batch-lookup
174
+
"POST /kidlisp/lookup
175
+
body: {codes: [...]}
176
+
Returns {results: {code -> entity|null}, summary: {...}}
177
+
Increments hits for found codes."
178
+
[conn]
179
+
(fn [req]
180
+
(let [codes (get-in req [:body-params :codes])]
181
+
(if-not (and (sequential? codes) (seq codes))
182
+
(bad "codes must be a non-empty array")
183
+
(let [db (d/db conn)
184
+
found (into {} (for [c codes
185
+
:let [e (entity-by-code db c)]
186
+
:when e]
187
+
[c (piece->map e)]))
188
+
missing (vec (remove found codes))
189
+
tx (for [c (keys found)
190
+
:let [e (entity-by-code db c)]]
191
+
[:db/add (:db/id e) :kidlisp/hits
192
+
(inc (or (:kidlisp/hits e) 0))])]
193
+
(when (seq tx) (db/transact conn (vec tx)))
194
+
(ok {:results (merge (into {} (for [c missing] [c nil])) found)
195
+
:summary {:requested (count codes)
196
+
:found (count found)
197
+
:missing (count missing)
198
+
:foundCodes (vec (keys found))
199
+
:missingCodes missing}}))))))
200
+
201
+
(defn list-codes
202
+
"GET /kidlisp?since=...&limit=...&sort=recent|hits&handle=...&codes=...
203
+
Returns {recent: [...], count, limit}. `handle` filter matched against
204
+
the author's handle — but this sidecar does not store handles; the
205
+
Node compat layer joins @handles from Mongo server-side."
206
+
[conn]
207
+
(fn [req]
208
+
(let [{:strs [limit sort since]} (:query-params req)
209
+
limit (min 100000 (max 1 (Integer/parseInt (or limit "50"))))
210
+
db (d/db conn)
211
+
since-inst (when (seq since) (java.util.Date/from (java.time.Instant/parse since)))
212
+
q (if since-inst
213
+
'[:find [?e ...]
214
+
:in $ ?since
215
+
:where
216
+
[?e :kidlisp/created-at ?when]
217
+
[(> ?when ?since)]]
218
+
'[:find [?e ...]
219
+
:where
220
+
[?e :kidlisp/code]])
221
+
eids (if since-inst (d/q q db since-inst) (d/q q db))
222
+
pieces (->> eids
223
+
(map #(d/entity db %))
224
+
(sort-by (case sort
225
+
"hits" #(- (or (:kidlisp/hits %) 0))
226
+
#(- (.getTime ^java.util.Date (:kidlisp/created-at %)))))
227
+
(take limit)
228
+
(map piece->map))]
229
+
(ok {:recent (vec pieces)
230
+
:count (count pieces)
231
+
:limit limit}))))
232
+
233
+
(defn stats-functions
234
+
"GET /kidlisp/stats/functions?limit=5000
235
+
Corpus-wide aggregation: scans top-N pieces by hits and tallies
236
+
function-call usage. Implementation mirrors the logic in the existing
237
+
store-kidlisp.mjs handler. For v1 we return raw (code, source, hits)
238
+
tuples and let the Node compat layer do the tallying — keeps the
239
+
sidecar simple and reuses the existing JS tokenizer regex."
240
+
[conn]
241
+
(fn [req]
242
+
(let [{:strs [limit]} (:query-params req)
243
+
limit (min 100000 (Integer/parseInt (or limit "5000")))
244
+
db (d/db conn)
245
+
rows (d/q '[:find ?src ?hits
246
+
:where
247
+
[?e :kidlisp/source ?src]
248
+
[?e :kidlisp/hits ?hits]]
249
+
db)]
250
+
(ok {:docs (->> rows
251
+
(sort-by second >)
252
+
(take limit)
253
+
(map (fn [[src hits]] {:source src :hits hits})))}))))
254
+
255
+
(defn- update-piece
256
+
"Shared utility: set multi-attribute component entity on a piece by code.
257
+
`sub-attrs` is a map of sub-attribute (e.g. :keep/token-id) -> value.
258
+
`parent-attr` is the ref attribute on the piece (e.g. :kidlisp/keeps).
259
+
`cardinality` is :one or :many."
260
+
[conn code parent-attr cardinality sub-attrs]
261
+
(let [db (d/db conn)]
262
+
(if-let [piece-eid (d/entid db [:kidlisp/code code])]
263
+
(let [new-eid (d/tempid :db.part/user)
264
+
ent-map (assoc sub-attrs :db/id new-eid)
265
+
op (if (= cardinality :many)
266
+
[:db/add piece-eid parent-attr new-eid]
267
+
[:db/add piece-eid parent-attr new-eid])]
268
+
(db/transact conn [ent-map op])
269
+
true)
270
+
false)))
271
+
272
+
(defn record-mint
273
+
"POST /kidlisp/:code/mint — appends a keep record."
274
+
[conn]
275
+
(fn [req]
276
+
(let [code (get-in req [:path-params :code])
277
+
{:keys [tokenId network txHash contractAddress contractProfile
278
+
contractVersion keptAt keptBy walletAddress artifactUri
279
+
thumbnailUri metadataUri source]}
280
+
(:body-params req)]
281
+
(if (and tokenId contractAddress)
282
+
(if (update-piece conn code :kidlisp/keeps :many
283
+
(cond-> {}
284
+
tokenId (assoc :keep/token-id tokenId)
285
+
network (assoc :keep/network network)
286
+
txHash (assoc :keep/tx-hash txHash)
287
+
contractAddress (assoc :keep/contract-address contractAddress)
288
+
contractProfile (assoc :keep/contract-profile contractProfile)
289
+
contractVersion (assoc :keep/contract-version contractVersion)
290
+
keptAt (assoc :keep/kept-at (java.util.Date/from
291
+
(java.time.Instant/parse keptAt)))
292
+
keptBy (assoc :keep/kept-by keptBy)
293
+
walletAddress (assoc :keep/wallet-address walletAddress)
294
+
artifactUri (assoc :keep/artifact-uri artifactUri)
295
+
thumbnailUri (assoc :keep/thumbnail-uri thumbnailUri)
296
+
metadataUri (assoc :keep/metadata-uri metadataUri)
297
+
source (assoc :keep/source source)))
298
+
(ok {:ok true})
299
+
(not-found))
300
+
(bad "tokenId and contractAddress required")))))
301
+
302
+
(defn- merge-component
303
+
"Replaces-or-creates a cardinality/one component. Sub-attrs are keyed by
304
+
Datomic keyword; nil values are skipped."
305
+
[conn code parent-attr sub-attrs]
306
+
(let [db (d/db conn)]
307
+
(if-let [piece-eid (d/entid db [:kidlisp/code code])]
308
+
(let [cleaned (into {} (remove (comp nil? val) sub-attrs))
309
+
tx (if (seq cleaned)
310
+
[(assoc cleaned :db/id "new")
311
+
[:db/add piece-eid parent-attr "new"]]
312
+
[])]
313
+
(when (seq tx) (db/transact conn tx))
314
+
true)
315
+
false)))
316
+
317
+
(defn set-tezos-state
318
+
"POST /kidlisp/:code/tezos-state — replaces the legacy tezos summary."
319
+
[conn]
320
+
(fn [req]
321
+
(let [code (get-in req [:path-params :code])
322
+
b (:body-params req)]
323
+
(if (merge-component conn code :kidlisp/tezos-state
324
+
{:tezos/minted (:minted b)
325
+
:tezos/exists (:exists b)
326
+
:tezos/token-id (:tokenId b)
327
+
:tezos/tx-hash (:txHash b)
328
+
:tezos/creator-address (:creatorAddress b)
329
+
:tezos/code-hash (:codeHash b)
330
+
:tezos/network (:network b)
331
+
:tezos/reason (:reason b)
332
+
:tezos/error (:error b)})
333
+
(ok {:ok true})
334
+
(not-found)))))
335
+
336
+
(defn set-pending-rebake
337
+
"POST /kidlisp/:code/pending-rebake — replaces the pending rebake blob."
338
+
[conn]
339
+
(fn [req]
340
+
(let [code (get-in req [:path-params :code])
341
+
b (:body-params req)]
342
+
(if (merge-component conn code :kidlisp/pending-rebake
343
+
{:rebake/artifact-uri (:artifactUri b)
344
+
:rebake/thumbnail-uri (:thumbnailUri b)
345
+
:rebake/metadata-uri (:metadataUri b)
346
+
:rebake/contract-address (:contractAddress b)
347
+
:rebake/contract-profile (:contractProfile b)
348
+
:rebake/contract-version (:contractVersion b)})
349
+
(ok {:ok true})
350
+
(not-found)))))
351
+
352
+
(defn set-ipfs-media
353
+
"POST /kidlisp/:code/ipfs-media — replaces the IPFS bundle cache."
354
+
[conn]
355
+
(fn [req]
356
+
(let [code (get-in req [:path-params :code])
357
+
b (:body-params req)]
358
+
(if (merge-component conn code :kidlisp/ipfs-media
359
+
{:ipfs/artifact-uri (:artifactUri b)
360
+
:ipfs/thumbnail-uri (:thumbnailUri b)
361
+
:ipfs/source-hash (:sourceHash b)
362
+
:ipfs/author-handle (:authorHandle b)
363
+
:ipfs/dep-count (:depCount b)})
364
+
(ok {:ok true})
365
+
(not-found)))))
366
+
367
+
(defn set-atproto-rkey
368
+
"POST /kidlisp/:code/atproto-rkey — records the Bluesky record key."
369
+
[conn]
370
+
(fn [req]
371
+
(let [code (get-in req [:path-params :code])
372
+
rkey (get-in req [:body-params :rkey])
373
+
db (d/db conn)]
374
+
(if-let [eid (d/entid db [:kidlisp/code code])]
375
+
(do (db/transact conn [[:db/add eid :kidlisp/atproto-rkey rkey]])
376
+
(ok {:ok true}))
377
+
(not-found)))))
378
+
379
+
(defn lineage
380
+
"GET /kidlisp/:code/lineage — returns {ancestors, descendants} as
381
+
chains of {code, author}. Ancestors walks :kidlisp/forked-from up,
382
+
descendants uses a reverse lookup."
383
+
[conn]
384
+
(fn [req]
385
+
(let [code (get-in req [:path-params :code])
386
+
db (d/db conn)]
387
+
(if-let [eid (d/entid db [:kidlisp/code code])]
388
+
(let [ancestors
389
+
(loop [cur eid acc []]
390
+
(let [e (d/entity db cur)
391
+
parent (:kidlisp/forked-from e)]
392
+
(if parent
393
+
(recur (:db/id parent)
394
+
(conj acc {:code (:kidlisp/code parent)
395
+
:author (get-in parent [:kidlisp/author :user/sub])}))
396
+
acc)))
397
+
descendants
398
+
(vec (d/q '[:find ?code ?sub
399
+
:in $ ?root
400
+
:where
401
+
[?c :kidlisp/forked-from ?root]
402
+
[?c :kidlisp/code ?code]
403
+
[?c :kidlisp/author ?a]
404
+
[?a :user/sub ?sub]]
405
+
db eid))]
406
+
(ok {:root code
407
+
:ancestors ancestors
408
+
:descendants (mapv (fn [[c s]] {:code c :author s}) descendants)}))
409
+
(not-found)))))
+239
kidlisp-sidecar/src/ac/kidlisp/schema.clj
+239
kidlisp-sidecar/src/ac/kidlisp/schema.clj
···
1
+
(ns ac.kidlisp.schema
2
+
"Datomic schema for kidlisp v1. Covers every field that the existing
3
+
store-kidlisp.mjs currently persists in the Mongo `kidlisp` collection,
4
+
so that 'zero Mongo writes for kidlisp' is actually achievable.
5
+
6
+
Keep records and mint/attempt events are modeled as their own entities
7
+
referenced via :kidlisp/keeps and :kidlisp/mint-attempts. Legacy Tezos
8
+
state lives alongside for migration fidelity."
9
+
(:require [datomic.api :as d]))
10
+
11
+
(def schema-v1
12
+
[;; ───────── user ─────────
13
+
{:db/ident :user/sub
14
+
:db/valueType :db.type/string
15
+
:db/cardinality :db.cardinality/one
16
+
:db/unique :db.unique/identity
17
+
:db/doc "Auth0 subject identifier for the author."}
18
+
19
+
;; ───────── kidlisp piece ─────────
20
+
{:db/ident :kidlisp/code
21
+
:db/valueType :db.type/string
22
+
:db/cardinality :db.cardinality/one
23
+
:db/unique :db.unique/identity
24
+
:db/doc "Nanoid slug — the $code a user types."}
25
+
26
+
{:db/ident :kidlisp/hash
27
+
:db/valueType :db.type/string
28
+
:db/cardinality :db.cardinality/one
29
+
:db/unique :db.unique/identity
30
+
:db/doc "SHA-256 of trimmed source. Unique: dedup."}
31
+
32
+
{:db/ident :kidlisp/source
33
+
:db/valueType :db.type/string
34
+
:db/cardinality :db.cardinality/one
35
+
:db/doc "Source code text."}
36
+
37
+
{:db/ident :kidlisp/author
38
+
:db/valueType :db.type/ref
39
+
:db/cardinality :db.cardinality/one
40
+
:db/doc "Ref to :user/sub entity. Nullable (anonymous caches)."}
41
+
42
+
{:db/ident :kidlisp/created-at
43
+
:db/valueType :db.type/instant
44
+
:db/cardinality :db.cardinality/one
45
+
:db/index true
46
+
:db/doc "Mirrors existing Mongo `when` field."}
47
+
48
+
{:db/ident :kidlisp/last-accessed
49
+
:db/valueType :db.type/instant
50
+
:db/cardinality :db.cardinality/one
51
+
:db/noHistory true
52
+
:db/doc "Updated on every GET. :db/noHistory prevents the tx
53
+
log from filling with access events — history queries
54
+
won't see past values, only the current one. v2 will
55
+
migrate this to Redis entirely."}
56
+
57
+
{:db/ident :kidlisp/hits
58
+
:db/valueType :db.type/long
59
+
:db/cardinality :db.cardinality/one
60
+
:db/noHistory true
61
+
:db/doc "Access counter. :db/noHistory for same reason as
62
+
:kidlisp/last-accessed."}
63
+
64
+
{:db/ident :kidlisp/forked-from
65
+
:db/valueType :db.type/ref
66
+
:db/cardinality :db.cardinality/one
67
+
:db/doc "Parent piece ref — enables lineage queries."}
68
+
69
+
;; ───────── ATProto sync ─────────
70
+
{:db/ident :kidlisp/atproto-rkey
71
+
:db/valueType :db.type/string
72
+
:db/cardinality :db.cardinality/one
73
+
:db/doc "Bluesky record key after mirror-to-PDS completes."}
74
+
75
+
;; ───────── IPFS media (bundle cache) ─────────
76
+
{:db/ident :kidlisp/ipfs-media
77
+
:db/valueType :db.type/ref
78
+
:db/cardinality :db.cardinality/one
79
+
:db/isComponent true
80
+
:db/doc "Component entity: cached IPFS artifacts."}
81
+
82
+
{:db/ident :ipfs/artifact-uri
83
+
:db/valueType :db.type/string
84
+
:db/cardinality :db.cardinality/one}
85
+
{:db/ident :ipfs/thumbnail-uri
86
+
:db/valueType :db.type/string
87
+
:db/cardinality :db.cardinality/one}
88
+
{:db/ident :ipfs/source-hash
89
+
:db/valueType :db.type/string
90
+
:db/cardinality :db.cardinality/one}
91
+
{:db/ident :ipfs/created-at
92
+
:db/valueType :db.type/instant
93
+
:db/cardinality :db.cardinality/one}
94
+
{:db/ident :ipfs/author-handle
95
+
:db/valueType :db.type/string
96
+
:db/cardinality :db.cardinality/one}
97
+
{:db/ident :ipfs/dep-count
98
+
:db/valueType :db.type/long
99
+
:db/cardinality :db.cardinality/one}
100
+
{:db/ident :ipfs/pack-date
101
+
:db/valueType :db.type/instant
102
+
:db/cardinality :db.cardinality/one}
103
+
104
+
;; ───────── Keep records (plural; contract-keyed) ─────────
105
+
{:db/ident :kidlisp/keeps
106
+
:db/valueType :db.type/ref
107
+
:db/cardinality :db.cardinality/many
108
+
:db/isComponent true
109
+
:db/doc "Zero or more on-chain keep/mint records, one per
110
+
contract+tokenId+network combination."}
111
+
112
+
{:db/ident :keep/token-id
113
+
:db/valueType :db.type/long
114
+
:db/cardinality :db.cardinality/one
115
+
:db/index true}
116
+
{:db/ident :keep/network
117
+
:db/valueType :db.type/string
118
+
:db/cardinality :db.cardinality/one}
119
+
{:db/ident :keep/tx-hash
120
+
:db/valueType :db.type/string
121
+
:db/cardinality :db.cardinality/one}
122
+
{:db/ident :keep/contract-address
123
+
:db/valueType :db.type/string
124
+
:db/cardinality :db.cardinality/one
125
+
:db/index true}
126
+
{:db/ident :keep/contract-profile
127
+
:db/valueType :db.type/string
128
+
:db/cardinality :db.cardinality/one}
129
+
{:db/ident :keep/contract-version
130
+
:db/valueType :db.type/string
131
+
:db/cardinality :db.cardinality/one}
132
+
{:db/ident :keep/kept-at
133
+
:db/valueType :db.type/instant
134
+
:db/cardinality :db.cardinality/one}
135
+
{:db/ident :keep/kept-by
136
+
:db/valueType :db.type/string
137
+
:db/cardinality :db.cardinality/one}
138
+
{:db/ident :keep/wallet-address
139
+
:db/valueType :db.type/string
140
+
:db/cardinality :db.cardinality/one}
141
+
{:db/ident :keep/artifact-uri
142
+
:db/valueType :db.type/string
143
+
:db/cardinality :db.cardinality/one}
144
+
{:db/ident :keep/thumbnail-uri
145
+
:db/valueType :db.type/string
146
+
:db/cardinality :db.cardinality/one}
147
+
{:db/ident :keep/metadata-uri
148
+
:db/valueType :db.type/string
149
+
:db/cardinality :db.cardinality/one}
150
+
{:db/ident :keep/source
151
+
:db/valueType :db.type/string
152
+
:db/cardinality :db.cardinality/one
153
+
:db/doc "Origin of the record: kept | legacy_tezos | contract_keyed | unknown"}
154
+
155
+
;; ───────── Legacy Tezos summary (single-piece field) ─────────
156
+
{:db/ident :kidlisp/tezos-state
157
+
:db/valueType :db.type/ref
158
+
:db/cardinality :db.cardinality/one
159
+
:db/isComponent true
160
+
:db/doc "Mirrors the current `tezos` object on Mongo docs —
161
+
mint-attempted/exists/error/skipped summary."}
162
+
163
+
{:db/ident :tezos/minted
164
+
:db/valueType :db.type/boolean
165
+
:db/cardinality :db.cardinality/one}
166
+
{:db/ident :tezos/exists
167
+
:db/valueType :db.type/boolean
168
+
:db/cardinality :db.cardinality/one}
169
+
{:db/ident :tezos/token-id
170
+
:db/valueType :db.type/long
171
+
:db/cardinality :db.cardinality/one}
172
+
{:db/ident :tezos/tx-hash
173
+
:db/valueType :db.type/string
174
+
:db/cardinality :db.cardinality/one}
175
+
{:db/ident :tezos/creator-address
176
+
:db/valueType :db.type/string
177
+
:db/cardinality :db.cardinality/one}
178
+
{:db/ident :tezos/code-hash
179
+
:db/valueType :db.type/string
180
+
:db/cardinality :db.cardinality/one}
181
+
{:db/ident :tezos/network
182
+
:db/valueType :db.type/string
183
+
:db/cardinality :db.cardinality/one}
184
+
{:db/ident :tezos/minted-at
185
+
:db/valueType :db.type/instant
186
+
:db/cardinality :db.cardinality/one}
187
+
{:db/ident :tezos/checked-at
188
+
:db/valueType :db.type/instant
189
+
:db/cardinality :db.cardinality/one}
190
+
{:db/ident :tezos/attempted-at
191
+
:db/valueType :db.type/instant
192
+
:db/cardinality :db.cardinality/one}
193
+
{:db/ident :tezos/failed-at
194
+
:db/valueType :db.type/instant
195
+
:db/cardinality :db.cardinality/one}
196
+
{:db/ident :tezos/reason
197
+
:db/valueType :db.type/string
198
+
:db/cardinality :db.cardinality/one}
199
+
{:db/ident :tezos/error
200
+
:db/valueType :db.type/string
201
+
:db/cardinality :db.cardinality/one}
202
+
203
+
;; ───────── Pending rebake ─────────
204
+
{:db/ident :kidlisp/pending-rebake
205
+
:db/valueType :db.type/ref
206
+
:db/cardinality :db.cardinality/one
207
+
:db/isComponent true
208
+
:db/doc "State after a rebake that is not yet reflected on chain."}
209
+
210
+
{:db/ident :rebake/artifact-uri
211
+
:db/valueType :db.type/string
212
+
:db/cardinality :db.cardinality/one}
213
+
{:db/ident :rebake/thumbnail-uri
214
+
:db/valueType :db.type/string
215
+
:db/cardinality :db.cardinality/one}
216
+
{:db/ident :rebake/metadata-uri
217
+
:db/valueType :db.type/string
218
+
:db/cardinality :db.cardinality/one}
219
+
{:db/ident :rebake/created-at
220
+
:db/valueType :db.type/instant
221
+
:db/cardinality :db.cardinality/one}
222
+
{:db/ident :rebake/contract-address
223
+
:db/valueType :db.type/string
224
+
:db/cardinality :db.cardinality/one}
225
+
{:db/ident :rebake/contract-profile
226
+
:db/valueType :db.type/string
227
+
:db/cardinality :db.cardinality/one}
228
+
{:db/ident :rebake/contract-version
229
+
:db/valueType :db.type/string
230
+
:db/cardinality :db.cardinality/one}])
231
+
232
+
(defn- schema-installed? [db]
233
+
(some? (d/entid db :kidlisp/code)))
234
+
235
+
(defn ensure!
236
+
"Idempotently installs schema-v1 if not already present."
237
+
[conn]
238
+
(when-not (schema-installed? (d/db conn))
239
+
@(d/transact conn schema-v1)))
+245
silo/dashboard.html
+245
silo/dashboard.html
···
315
315
<button class="tab-btn" data-tab="6">feed</button>
316
316
<button class="tab-btn" data-tab="7">telemetry</button>
317
317
<button class="tab-btn" data-tab="8">lith</button>
318
+
<button class="tab-btn" data-tab="9">datomic</button>
318
319
</div>
319
320
320
321
<div class="panels">
···
625
626
</div>
626
627
</div>
627
628
629
+
</div>
630
+
631
+
<!-- datomic panel (read-only admin) -->
632
+
<div class="panel" data-panel="9">
633
+
<div class="overview-grid">
634
+
<div class="card">
635
+
<div class="card-hd">status <b id="dt-basisT"></b></div>
636
+
<div class="kv"><span class="k"><span class="dot" id="dt-tx-dot"></span> transactor</span><span class="v" id="dt-tx">-</span></div>
637
+
<div class="kv"><span class="k">schema</span><span class="v" id="dt-schema-ok">-</span></div>
638
+
<div class="kv"><span class="k">kidlisp entities</span><span class="v" id="dt-c-kidlisp">-</span></div>
639
+
<div class="kv"><span class="k">users</span><span class="v" id="dt-c-users">-</span></div>
640
+
<div class="kv"><span class="k">keeps</span><span class="v" id="dt-c-keeps">-</span></div>
641
+
<div class="kv"><span class="k">tx (last hour)</span><span class="v" id="dt-tx-hour">-</span></div>
642
+
</div>
643
+
<div class="card">
644
+
<div class="card-hd">backups <span class="bar-dim" style="font-size:10px">pg_dump · 14d retention</span></div>
645
+
<div id="dt-backups" style="font-size:11px;max-height:200px;overflow-y:auto">loading...</div>
646
+
</div>
647
+
</div>
648
+
649
+
<div style="display:flex;gap:4px;padding:4px 6px;border-bottom:1px solid var(--border);margin-top:6px">
650
+
<button class="sub-tab-btn active" onclick="dtSubTab(this,'dt-schema-panel')">schema</button>
651
+
<button class="sub-tab-btn" onclick="dtSubTab(this,'dt-entities-panel')">entities</button>
652
+
<button class="sub-tab-btn" onclick="dtSubTab(this,'dt-txlog-panel')">tx log</button>
653
+
<button class="sub-tab-btn" onclick="dtSubTab(this,'dt-query-panel')">query</button>
654
+
</div>
655
+
656
+
<div id="dt-schema-panel" class="sub-panel active" style="display:block;padding:6px;overflow-y:auto;max-height:calc(100vh - 380px)">
657
+
<table style="width:100%;font-size:11px;border-collapse:collapse">
658
+
<thead><tr style="color:var(--fg2);text-align:left">
659
+
<th style="padding:2px 6px">ident</th>
660
+
<th style="padding:2px 6px">type</th>
661
+
<th style="padding:2px 6px">card</th>
662
+
<th style="padding:2px 6px">flags</th>
663
+
<th style="padding:2px 6px">doc</th>
664
+
</tr></thead>
665
+
<tbody id="dt-schema-body"></tbody>
666
+
</table>
667
+
</div>
668
+
669
+
<div id="dt-entities-panel" class="sub-panel" style="display:none;padding:6px;max-height:calc(100vh - 380px);overflow-y:auto">
670
+
<div style="display:flex;gap:6px;align-items:center;margin-bottom:6px">
671
+
<span class="bar-dim">type:</span>
672
+
<button class="btn" onclick="dtLoadEntities('kidlisp')">kidlisp</button>
673
+
<button class="btn" onclick="dtLoadEntities('user')">user</button>
674
+
<button class="btn" onclick="dtLoadEntities('keep')">keep</button>
675
+
<span class="bar-dim" id="dt-ent-status" style="margin-left:8px"></span>
676
+
</div>
677
+
<div id="dt-entities-list" style="font-size:11px">pick a type</div>
678
+
<div id="dt-entity-detail" style="margin-top:8px;padding:6px;background:var(--bg);border:1px solid var(--border);display:none">
679
+
<div class="card-hd">entity <b id="dt-entity-eid"></b>
680
+
<span class="bar-right">
681
+
<button class="btn" onclick="dtLoadEntityHistory()">history</button>
682
+
<button class="btn" onclick="document.getElementById('dt-entity-detail').style.display='none'">close</button>
683
+
</span>
684
+
</div>
685
+
<pre id="dt-entity-body" style="white-space:pre-wrap;font-size:11px;margin:0"></pre>
686
+
</div>
687
+
</div>
688
+
689
+
<div id="dt-txlog-panel" class="sub-panel" style="display:none;padding:6px;max-height:calc(100vh - 380px);overflow-y:auto">
690
+
<table style="width:100%;font-size:11px;border-collapse:collapse">
691
+
<thead><tr style="color:var(--fg2);text-align:left">
692
+
<th style="padding:2px 6px">tx</th>
693
+
<th style="padding:2px 6px">at</th>
694
+
</tr></thead>
695
+
<tbody id="dt-txlog-body"></tbody>
696
+
</table>
697
+
</div>
698
+
699
+
<div id="dt-query-panel" class="sub-panel" style="display:none;padding:6px;max-height:calc(100vh - 380px);overflow-y:auto">
700
+
<div style="font-size:11px;color:var(--fg2);margin-bottom:4px">
701
+
read-only datalog · timeout 5s · capped 10k rows
702
+
</div>
703
+
<textarea id="dt-query-text" rows="6"
704
+
style="width:100%;font-family:inherit;font-size:11px;background:var(--bg);color:var(--fg);border:1px solid var(--border);padding:4px"
705
+
placeholder="[:find ?code ?hits :where [?e :kidlisp/code ?code] [?e :kidlisp/hits ?hits]]"></textarea>
706
+
<div style="margin:4px 0;display:flex;gap:4px;align-items:center">
707
+
<button class="btn" onclick="dtRunQuery()">run</button>
708
+
<span class="bar-dim" id="dt-query-status"></span>
709
+
</div>
710
+
<pre id="dt-query-result" style="font-size:11px;margin:0;white-space:pre-wrap"></pre>
711
+
</div>
628
712
</div>
629
713
630
714
</div>
···
2091
2175
if (e.target.matches('[data-tab="8"]')) {
2092
2176
loadLith();
2093
2177
if (!lithTimer) lithTimer = setInterval(loadLith, 10000);
2178
+
}
2179
+
});
2180
+
2181
+
// ───────────────────────── datomic tab ─────────────────────────
2182
+
let datomicTimer = null;
2183
+
let datomicCurrentType = null;
2184
+
2185
+
function dtAuth() {
2186
+
return accessToken ? { 'Authorization': 'Bearer ' + accessToken } : {};
2187
+
}
2188
+
2189
+
async function dtFetch(path, opts = {}) {
2190
+
const r = await fetch(path, { ...opts, headers: { ...dtAuth(), ...(opts.headers || {}) } });
2191
+
if (!r.ok) throw new Error(path + ' → ' + r.status);
2192
+
return r.json();
2193
+
}
2194
+
2195
+
function dtSubTab(btn, panelId) {
2196
+
const bar = btn.parentElement;
2197
+
bar.querySelectorAll('.sub-tab-btn').forEach(b => b.classList.remove('active'));
2198
+
btn.classList.add('active');
2199
+
const panel = btn.closest('.panel');
2200
+
panel.querySelectorAll('.sub-panel').forEach(p => { p.style.display = 'none'; });
2201
+
const target = document.getElementById(panelId);
2202
+
if (target) target.style.display = 'block';
2203
+
if (panelId === 'dt-txlog-panel') dtLoadTxLog();
2204
+
}
2205
+
2206
+
async function loadDatomic() {
2207
+
try {
2208
+
const h = await dtFetch('/api/datomic/health');
2209
+
document.getElementById('dt-tx').textContent = h.transactor ? 'ok' : 'down';
2210
+
document.getElementById('dt-tx').className = 'v ' + (h.transactor ? 'ok' : 'err');
2211
+
document.getElementById('dt-tx-dot').className = 'dot ' + (h.transactor ? 'ok' : 'err');
2212
+
document.getElementById('dt-schema-ok').textContent = h.schema ? 'installed' : 'missing';
2213
+
document.getElementById('dt-basisT').textContent = h.basisT != null ? ('t=' + h.basisT) : '';
2214
+
} catch {
2215
+
document.getElementById('dt-tx').textContent = 'unreachable';
2216
+
document.getElementById('dt-tx').className = 'v err';
2217
+
document.getElementById('dt-tx-dot').className = 'dot err';
2218
+
}
2219
+
2220
+
try {
2221
+
const s = await dtFetch('/api/datomic/stats');
2222
+
document.getElementById('dt-c-kidlisp').textContent = s.entityCounts?.kidlisp ?? 0;
2223
+
document.getElementById('dt-c-users').textContent = s.entityCounts?.users ?? 0;
2224
+
document.getElementById('dt-c-keeps').textContent = s.entityCounts?.keeps ?? 0;
2225
+
document.getElementById('dt-tx-hour').textContent = s.txLastHour ?? 0;
2226
+
} catch {}
2227
+
2228
+
try {
2229
+
const { attributes } = await dtFetch('/api/datomic/schema');
2230
+
document.getElementById('dt-schema-body').innerHTML = attributes.map(a => {
2231
+
const flags = [
2232
+
a.unique && 'unique',
2233
+
a.indexed && 'indexed',
2234
+
a.isComponent && 'component',
2235
+
a.noHistory && 'noHistory',
2236
+
].filter(Boolean).join(' ');
2237
+
return '<tr><td style="padding:2px 6px">' + (a.ident || '').toString() +
2238
+
'</td><td style="padding:2px 6px;color:var(--fg2)">' + (a.valueType || '').toString().replace(':db.type/', '') +
2239
+
'</td><td style="padding:2px 6px;color:var(--fg2)">' + (a.cardinality || '').toString().replace(':db.cardinality/', '') +
2240
+
'</td><td style="padding:2px 6px;color:var(--fg2)">' + flags +
2241
+
'</td><td style="padding:2px 6px;color:var(--fg2)">' + (a.doc || '').replace(/\n/g, ' ') + '</td></tr>';
2242
+
}).join('');
2243
+
} catch {}
2244
+
2245
+
try {
2246
+
const { backups } = await dtFetch('/api/datomic/backups');
2247
+
const fmt = (n) => (n / 1024 / 1024).toFixed(1) + ' MB';
2248
+
document.getElementById('dt-backups').innerHTML = backups.length
2249
+
? backups.map(b => '<div style="padding:1px 0"><span>' + b.name +
2250
+
'</span> <span style="color:var(--fg2)">' + fmt(b.size) + ' · ' +
2251
+
new Date(b.at).toLocaleString() + '</span></div>').join('')
2252
+
: '<div style="color:var(--fg2)">no backups yet</div>';
2253
+
} catch {
2254
+
document.getElementById('dt-backups').textContent = 'backups dir not present';
2255
+
}
2256
+
}
2257
+
2258
+
async function dtLoadEntities(type) {
2259
+
datomicCurrentType = type;
2260
+
document.getElementById('dt-ent-status').textContent = 'loading...';
2261
+
try {
2262
+
const data = await dtFetch('/api/datomic/entities/' + encodeURIComponent(type) + '?limit=100');
2263
+
document.getElementById('dt-ent-status').textContent = data.total + ' total (showing ' + data.items.length + ')';
2264
+
document.getElementById('dt-entities-list').innerHTML = data.items.map(it => {
2265
+
const label = it['kidlisp/code'] || it['user/sub'] || ('keep #' + it['keep/token-id']) || ('eid ' + it['db/id']);
2266
+
return '<div style="padding:2px 0;cursor:pointer;border-bottom:1px solid var(--border)" ' +
2267
+
'onclick="dtShowEntity(' + it['db/id'] + ')">' +
2268
+
'<span style="color:var(--accent)">' + label + '</span> ' +
2269
+
'<span style="color:var(--fg2)">eid ' + it['db/id'] + '</span></div>';
2270
+
}).join('');
2271
+
} catch (err) {
2272
+
document.getElementById('dt-ent-status').textContent = err.message;
2273
+
}
2274
+
}
2275
+
2276
+
let dtCurrentEid = null;
2277
+
async function dtShowEntity(eid) {
2278
+
dtCurrentEid = eid;
2279
+
try {
2280
+
const entity = await dtFetch('/api/datomic/entity/' + eid);
2281
+
document.getElementById('dt-entity-eid').textContent = eid;
2282
+
document.getElementById('dt-entity-body').textContent = JSON.stringify(entity, null, 2);
2283
+
document.getElementById('dt-entity-detail').style.display = 'block';
2284
+
} catch (err) {
2285
+
alert(err.message);
2286
+
}
2287
+
}
2288
+
2289
+
async function dtLoadEntityHistory() {
2290
+
if (dtCurrentEid == null) return;
2291
+
try {
2292
+
const data = await dtFetch('/api/datomic/entity/' + dtCurrentEid + '/history');
2293
+
document.getElementById('dt-entity-body').textContent = JSON.stringify(data, null, 2);
2294
+
} catch (err) {
2295
+
alert(err.message);
2296
+
}
2297
+
}
2298
+
2299
+
async function dtLoadTxLog() {
2300
+
try {
2301
+
const { transactions } = await dtFetch('/api/datomic/tx-log?limit=100');
2302
+
document.getElementById('dt-txlog-body').innerHTML = transactions.map(t =>
2303
+
'<tr><td style="padding:2px 6px">' + t.tx + '</td>' +
2304
+
'<td style="padding:2px 6px;color:var(--fg2)">' + new Date(t.at).toLocaleString() + '</td></tr>'
2305
+
).join('');
2306
+
} catch {
2307
+
document.getElementById('dt-txlog-body').innerHTML = '<tr><td colspan="2" style="color:var(--err)">unavailable</td></tr>';
2308
+
}
2309
+
}
2310
+
2311
+
async function dtRunQuery() {
2312
+
const q = document.getElementById('dt-query-text').value;
2313
+
document.getElementById('dt-query-status').textContent = 'running...';
2314
+
document.getElementById('dt-query-result').textContent = '';
2315
+
try {
2316
+
const r = await fetch('/api/datomic/query', {
2317
+
method: 'POST',
2318
+
headers: { 'content-type': 'application/json', ...dtAuth() },
2319
+
body: JSON.stringify({ query: q, args: [] }),
2320
+
});
2321
+
const data = await r.json();
2322
+
if (!r.ok) {
2323
+
document.getElementById('dt-query-status').textContent = 'error ' + r.status;
2324
+
document.getElementById('dt-query-result').textContent = JSON.stringify(data, null, 2);
2325
+
return;
2326
+
}
2327
+
document.getElementById('dt-query-status').textContent =
2328
+
data.count + ' rows' + (data.truncated ? ' (truncated)' : '');
2329
+
document.getElementById('dt-query-result').textContent = JSON.stringify(data.rows, null, 2);
2330
+
} catch (err) {
2331
+
document.getElementById('dt-query-status').textContent = err.message;
2332
+
}
2333
+
}
2334
+
2335
+
document.addEventListener('click', e => {
2336
+
if (e.target.matches('[data-tab="9"]')) {
2337
+
loadDatomic();
2338
+
if (!datomicTimer) datomicTimer = setInterval(loadDatomic, 15000);
2094
2339
}
2095
2340
});
2096
2341
+86
silo/datomic/MIGRATION.md
+86
silo/datomic/MIGRATION.md
···
1
+
# kidlisp → Datomic migration punchlist
2
+
3
+
Generated from a grep across the repo for `.collection('kidlisp')`. Each
4
+
entry is one place that reads or writes the Mongo `kidlisp` collection
5
+
and needs a decision: migrate to sidecar, keep on Mongo read-only, or
6
+
retire.
7
+
8
+
## ✅ Migrated in v1
9
+
10
+
- [`system/netlify/functions/store-kidlisp.mjs`](../../system/netlify/functions/store-kidlisp.mjs) → routes to
11
+
[`store-kidlisp-datomic.mjs`](../../system/netlify/functions/store-kidlisp-datomic.mjs) behind `KIDLISP_DATOMIC=on`.
12
+
- [`system/backend/backfill-kidlisp-to-datomic.mjs`](../../system/backend/backfill-kidlisp-to-datomic.mjs) — one-shot
13
+
Mongo → Datomic replay. Idempotent.
14
+
- [`silo/server.mjs`](../server.mjs) — `/api/datomic/*` proxy to the sidecar's admin
15
+
surface. Silo dashboard still browses Mongo `kidlisp` for historical
16
+
comparison during validation.
17
+
18
+
## 🟡 Production hot path — migrate before flipping `KIDLISP_DATOMIC=on` everywhere
19
+
20
+
These functions read/write the kidlisp collection directly. Each needs
21
+
to be rewritten to call the sidecar client in
22
+
`system/backend/kidlisp-sidecar.mjs`.
23
+
24
+
- [ ] `system/netlify/functions/kidlisp-list.mjs` — list endpoint.
25
+
- [ ] `system/netlify/functions/kidlisp-count.mjs` — count endpoint.
26
+
- [ ] `system/netlify/functions/kidlisp-keep.mjs` — keep write path.
27
+
- [ ] `system/netlify/functions/keep-prepare.mjs` — pre-mint prep writes.
28
+
- [ ] `system/netlify/functions/keep-prepare-background.mjs` — background
29
+
writer.
30
+
- [ ] `system/netlify/functions/keep-confirm.mjs` — mint confirmation
31
+
write (sets `kept` field).
32
+
- [ ] `system/netlify/functions/keep-mint.mjs` — mint execution write.
33
+
- [ ] `system/netlify/functions/keep-update.mjs` — rebake path.
34
+
- [ ] `system/netlify/functions/keep-update-confirm.mjs` — rebake
35
+
confirmation write.
36
+
- [ ] `system/netlify/functions/tv.mjs` — reads for TV feed.
37
+
- [ ] `system/netlify/functions/test-tv-hits.mjs` — test endpoint reads.
38
+
- [ ] `system/netlify/functions/metrics.mjs` — analytics reads.
39
+
- [ ] `system/netlify/functions/atproto-user-stats.mjs` — per-user reads.
40
+
- [ ] `system/netlify/functions/index.mjs` — verify kidlisp reference.
41
+
42
+
## 🟠 Other services (non-Netlify, need own sidecar client)
43
+
44
+
- [ ] `oven/grabber.mjs` — screenshot/OG image gen reads kidlisp source.
45
+
Oven has its own Mongo connection; give it a sidecar client too.
46
+
- [ ] `oven/kidlisp-mini/bundle.mjs` — bundle pipeline reads + writes
47
+
`ipfsMedia`. Biggest side-effect surface.
48
+
49
+
## 🟢 Tezos tooling (operator-run CLIs; low urgency)
50
+
51
+
- [ ] `tezos/keep-cli.mjs`
52
+
- [ ] `tezos/ac-keeps.mjs`
53
+
- [ ] `tezos/find-my-kidlisp.mjs`
54
+
55
+
## 🟢 Feed worker (Cloudflare; runs off its own schedule)
56
+
57
+
- [ ] `feed/silo-update-top100.mjs`
58
+
- [ ] `feed/create-top-kidlisp-playlist.mjs`
59
+
60
+
## 🔵 Scripts / utilities / one-offs — likely retire after cutover
61
+
62
+
- `scripts/kidlisp-db-stats.mjs` — ad-hoc stats. Can be replaced by
63
+
`GET /admin/stats` + `/api/datomic/stats`.
64
+
- `scripts/atproto/backfill-kidlisp-rkeys.mjs` — one-shot backfill,
65
+
already ran. Keep for reference, no migration needed.
66
+
- `scripts/atproto/check-fifi-kidlisp.mjs` — ad-hoc debug.
67
+
- `scripts/atproto/check-kidlisp-atproto.mjs` — ad-hoc debug.
68
+
- `scripts/atproto/sync-fifi-kidlisp.mjs` — ad-hoc sync.
69
+
- `scripts/atproto/sync-jeffrey-kidlisp.mjs` — ad-hoc sync.
70
+
- `at/scripts/atproto/backfill-kidlisp-to-atproto.mjs` — one-shot.
71
+
- `utilities/keep-cleanup.mjs` — cleanup utility; rewrite against
72
+
sidecar if still needed.
73
+
74
+
## Cutover sequence (when ready)
75
+
76
+
1. Bring Datomic + sidecar up on silo (see [README.md](README.md)).
77
+
2. Populate vault entries (already done in `aesthetic-computer-vault/kidlisp-datomic/`).
78
+
3. Run backfill from silo: `node system/backend/backfill-kidlisp-to-datomic.mjs`.
79
+
4. Verify counts match: `/api/datomic/stats` vs Mongo `kidlisp.countDocuments()`.
80
+
5. Set `KIDLISP_DATOMIC=on` in lith's `.env` and redeploy. `store-kidlisp.mjs`
81
+
is now Datomic-authoritative for create/read/update-hit/mint/etc.
82
+
6. Work through the 🟡 hot-path list above. Each file becomes a small PR.
83
+
7. When 🟡 list is empty, Mongo `kidlisp` collection is frozen (read-only
84
+
for the migrated paths; unused by new code).
85
+
8. After stable operation, work the 🟠 + 🟢 lists.
86
+
9. 🔵 list can be retired anytime — confirm no scheduled jobs invoke them.
+112
silo/datomic/README.md
+112
silo/datomic/README.md
···
1
+
# silo/datomic
2
+
3
+
Host-side infrastructure for the Datomic Pro instance that backs kidlisp (v1).
4
+
The Clojure sidecar that talks to this transactor lives at `/kidlisp-sidecar/`.
5
+
6
+
## Components on silo
7
+
8
+
- **Postgres** — Datomic storage backend. Datomic treats it as a KV blob store;
9
+
the logical schema lives inside Datomic, not in Postgres.
10
+
- **Datomic transactor** — JVM process, single node, auto-restart via systemd.
11
+
- **pg_dump backup** — nightly, retained 14 days.
12
+
13
+
## Layout
14
+
15
+
transactor/
16
+
transactor.properties.template merged with vault secrets at deploy
17
+
datomic-transactor.service systemd unit
18
+
postgres/
19
+
init.sql creates datomic db + role
20
+
backup.fish pg_dump wrapper
21
+
backup.cron cron entry (run nightly)
22
+
deploy.fish installs/updates transactor + pg config
23
+
24
+
## One-time: add silo env vars
25
+
26
+
The silo dashboard needs to know where the sidecar lives and what admin
27
+
secret to present. The sidecar's `ADMIN_SECRET` was generated into
28
+
`aesthetic-computer-vault/kidlisp-datomic/sidecar.env.gpg`. Copy that value
29
+
into silo's existing env:
30
+
31
+
cd /workspaces/aesthetic-computer/aesthetic-computer-vault
32
+
fish vault-tool.fish unlock
33
+
# ADMIN_SECRET=<hex> ← read from kidlisp-datomic/sidecar.env
34
+
# Append two lines to silo/.env:
35
+
# DATOMIC_SIDECAR_URL=http://127.0.0.1:8891
36
+
# DATOMIC_SIDECAR_ADMIN_SECRET=<hex>
37
+
fish vault-tool.fish edit silo/.env
38
+
fish vault-tool.fish lock
39
+
40
+
Silo proxies all `/api/datomic/*` requests to the sidecar with this header.
41
+
Sidecar binds to 127.0.0.1 on silo, so the URL is internal-only.
42
+
43
+
## End-to-end bring-up sequence
44
+
45
+
Run each step from your local terminal (not via Claude tool calls — gpg +
46
+
ssh keys need interactive agent access).
47
+
48
+
# 1. Unlock vault
49
+
fish aesthetic-computer-vault/vault-tool.fish unlock
50
+
51
+
# 2. Append the two sidecar env vars to silo/.env
52
+
fish silo/datomic/update-silo-env.fish
53
+
54
+
# 3. Install JVM, Postgres, create datomic user + dirs on silo
55
+
fish silo/datomic/bootstrap-silo.fish
56
+
57
+
# 4. Download Datomic Pro (Apache 2.0) and ship the jar to silo
58
+
# The bootstrap script prints exact commands if the jar is absent.
59
+
60
+
# 5. Deploy transactor config + systemd + backup cron
61
+
fish silo/datomic/deploy.fish
62
+
63
+
# 6. Build the Clojure sidecar uberjar and ship it
64
+
fish kidlisp-sidecar/deploy.fish
65
+
66
+
# 7. Redeploy silo to pick up the new env vars
67
+
fish silo/deploy.fish
68
+
69
+
# 8. Backfill Mongo kidlisp → Datomic (from silo, since sidecar is localhost)
70
+
ssh -i aesthetic-computer-vault/home/.ssh/id_rsa root@silo.aesthetic.computer \
71
+
"cd /opt/silo && CLIENT_SECRET=\$(grep CLIENT_SECRET /opt/kidlisp-sidecar/.env | cut -d= -f2) \
72
+
node /opt/ac/system/backend/backfill-kidlisp-to-datomic.mjs"
73
+
74
+
# 9. Verify the dashboard → datomic tab shows healthy transactor,
75
+
# schema installed, entity counts matching Mongo.
76
+
77
+
# 10. Flip the feature flag to cut over.
78
+
# Edit aesthetic-computer-vault/lith/.env and add: KIDLISP_DATOMIC=on
79
+
# Then: cd lith && fish deploy.fish
80
+
# After this, store-kidlisp.mjs routes all traffic to Datomic.
81
+
82
+
# 11. Lock the vault when done
83
+
fish aesthetic-computer-vault/vault-tool.fish lock
84
+
85
+
## Individual runbook steps (reference)
86
+
87
+
- **First-time software install on silo**: handled by `bootstrap-silo.fish`
88
+
(apt install temurin-21-jdk + postgresql, create datomic user, mkdir tree,
89
+
create PG role + db, verify jar present).
90
+
- **Datomic jar acquisition**: not scripted — operator downloads
91
+
`datomic-pro-<version>.zip` from https://docs.datomic.com/pro/ and scp's to
92
+
`/tmp/` on silo. Bootstrap script prints the exact unzip/symlink commands.
93
+
- **Config rollout**: `deploy.fish` renders `transactor.properties.template`
94
+
in `/dev/shm`, uploads to silo, restarts transactor via systemd.
95
+
- **Sidecar rollout**: `kidlisp-sidecar/deploy.fish` builds uberjar locally,
96
+
decrypts `sidecar.env.gpg` into `/dev/shm`, scp's both to silo, restarts.
97
+
- **Backup cron**: installed at `/etc/cron.d/datomic-backup` by `deploy.fish`,
98
+
runs 03:17 UTC daily, 14-day retention in `/var/backups/datomic/`.
99
+
100
+
## Ongoing ops
101
+
102
+
- **Logs**: `journalctl -u datomic-transactor -f`
103
+
- **Restart**: `systemctl restart datomic-transactor`
104
+
- **Backup now**: `fish /opt/datomic/backup.fish`
105
+
- **Restore**: stop transactor, `psql` the latest dump into a fresh db, restart.
106
+
107
+
## Non-goals
108
+
109
+
- No HA transactor in v1. Single node. If/when kidlisp traffic warrants it,
110
+
Datomic Pro supports standby transactors with automatic failover.
111
+
- Not moving moods/paintings/handles/chat here — see root CLAUDE.md and the
112
+
v1 plan.
+6
silo/datomic/backup.cron
+6
silo/datomic/backup.cron
···
1
+
## Datomic storage backup — installed to /etc/cron.d/datomic-backup
2
+
## 03:17 UTC daily (off-peak, outside other silo cron windows).
3
+
SHELL=/bin/bash
4
+
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
5
+
6
+
17 3 * * * postgres /usr/bin/fish /opt/datomic/backup.fish >> /var/log/datomic/backup.log 2>&1
+177
silo/datomic/bootstrap-silo.fish
+177
silo/datomic/bootstrap-silo.fish
···
1
+
#!/usr/bin/env fish
2
+
## First-time bring-up of Datomic + Postgres on silo.
3
+
## Idempotent — safe to re-run.
4
+
##
5
+
## Prereqs (run locally, from repo root):
6
+
## fish aesthetic-computer-vault/vault-tool.fish unlock
7
+
##
8
+
## What it does on silo (as root):
9
+
## 1. apt install: temurin-21-jdk, postgresql-16, unzip
10
+
## 2. Create `datomic` user + dirs (/opt/datomic, /var/log/datomic,
11
+
## /var/lib/datomic, /var/backups/datomic)
12
+
## 3. Create PG datomic role + database using vault creds
13
+
## 4. Verify Datomic jar is present (or fail with instructions)
14
+
##
15
+
## Datomic Pro jar download:
16
+
## The user must manually download datomic-pro-<version>.zip from
17
+
## https://docs.datomic.com/pro/ (now Apache 2.0) and scp it to
18
+
## /opt/datomic/ on silo BEFORE running this script. This script
19
+
## handles unzip + symlink setup.
20
+
##
21
+
## Usage:
22
+
## fish silo/datomic/bootstrap-silo.fish
23
+
## fish silo/datomic/bootstrap-silo.fish --check # dry run, capacity check only
24
+
25
+
set RED '\033[0;31m'
26
+
set GREEN '\033[0;32m'
27
+
set YELLOW '\033[1;33m'
28
+
set BLUE '\033[0;34m'
29
+
set NC '\033[0m'
30
+
31
+
set SCRIPT_DIR (dirname (status --current-filename))
32
+
set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault"
33
+
set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa"
34
+
set POSTGRES_ENV "$VAULT_DIR/kidlisp-datomic/postgres.env"
35
+
set SILO_HOST "silo.aesthetic.computer"
36
+
set SILO_USER "root"
37
+
38
+
set CHECK_ONLY false
39
+
if contains -- --check $argv
40
+
set CHECK_ONLY true
41
+
end
42
+
43
+
function die
44
+
echo -e "$RED x $argv$NC" >&2
45
+
exit 1
46
+
end
47
+
48
+
function step
49
+
echo -e "$BLUE-> $argv$NC"
50
+
end
51
+
52
+
## Prereq checks (local)
53
+
if not test -f $SSH_KEY
54
+
die "SSH key not found at $SSH_KEY. Run: fish aesthetic-computer-vault/vault-tool.fish unlock"
55
+
end
56
+
57
+
if not test -f $POSTGRES_ENV
58
+
die "Postgres env not found at $POSTGRES_ENV. Run: fish aesthetic-computer-vault/vault-tool.fish unlock"
59
+
end
60
+
61
+
## Source the decrypted postgres env
62
+
set POSTGRES_USER (grep '^POSTGRES_USER=' $POSTGRES_ENV | cut -d= -f2-)
63
+
set POSTGRES_PASSWORD (grep '^POSTGRES_PASSWORD=' $POSTGRES_ENV | cut -d= -f2-)
64
+
65
+
if test -z "$POSTGRES_USER"; or test -z "$POSTGRES_PASSWORD"
66
+
die "postgres.env missing POSTGRES_USER or POSTGRES_PASSWORD"
67
+
end
68
+
69
+
step "Testing SSH to $SILO_HOST..."
70
+
if not ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
71
+
$SILO_USER@$SILO_HOST "echo ok" >/dev/null 2>&1
72
+
die "Cannot connect to $SILO_HOST"
73
+
end
74
+
75
+
step "Capacity check on silo..."
76
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST '
77
+
echo " host: $(hostname)"
78
+
echo " ram: $(free -h | awk "NR==2 {print \$2 \" total, \" \$3 \" used, \" \$7 \" available\"}")"
79
+
echo " disk: $(df -h / | awk "NR==2 {print \$2 \" total, \" \$3 \" used, \" \$4 \" avail on /\"}")"
80
+
echo " java: $(command -v java || echo \"(not installed)\")"
81
+
echo " pg: $(command -v psql || echo \"(not installed)\")"
82
+
echo " datomic user: $(id -u datomic 2>/dev/null || echo \"(absent)\")"
83
+
'
84
+
85
+
if test $CHECK_ONLY = true
86
+
echo -e "$GREEN Check-only mode — no changes made.$NC"
87
+
exit 0
88
+
end
89
+
90
+
## Confirm
91
+
echo -e "$YELLOW About to install JDK + Postgres + configure Datomic on silo.$NC"
92
+
echo -n "Proceed? [y/N] "
93
+
read -P "" answer
94
+
if test "$answer" != "y"; and test "$answer" != "Y"
95
+
echo "Aborted."
96
+
exit 0
97
+
end
98
+
99
+
step "Installing JDK + Postgres via apt..."
100
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "
101
+
set -e
102
+
export DEBIAN_FRONTEND=noninteractive
103
+
104
+
# Temurin (Eclipse Adoptium) repo
105
+
if ! command -v java >/dev/null 2>&1; then
106
+
apt-get update -qq
107
+
apt-get install -y -qq wget gnupg ca-certificates
108
+
mkdir -p /etc/apt/keyrings
109
+
wget -qO- https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor -o /etc/apt/keyrings/adoptium.gpg
110
+
echo 'deb [signed-by=/etc/apt/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb '(. /etc/os-release && echo \$VERSION_CODENAME)' main' > /etc/apt/sources.list.d/adoptium.list
111
+
apt-get update -qq
112
+
apt-get install -y -qq temurin-21-jdk
113
+
fi
114
+
115
+
if ! command -v psql >/dev/null 2>&1; then
116
+
apt-get install -y -qq postgresql postgresql-contrib
117
+
fi
118
+
119
+
apt-get install -y -qq unzip
120
+
systemctl enable --now postgresql
121
+
"
122
+
or die "apt install failed"
123
+
124
+
step "Creating datomic user + directories..."
125
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "
126
+
set -e
127
+
if ! id -u datomic >/dev/null 2>&1; then
128
+
useradd -r -s /bin/false -d /opt/datomic datomic
129
+
fi
130
+
mkdir -p /opt/datomic /opt/datomic/config /opt/kidlisp-sidecar \
131
+
/var/log/datomic /var/lib/datomic /var/backups/datomic
132
+
chown -R datomic:datomic /opt/datomic /opt/kidlisp-sidecar \
133
+
/var/log/datomic /var/lib/datomic /var/backups/datomic
134
+
"
135
+
or die "dir setup failed"
136
+
137
+
step "Creating Postgres datomic role + database (idempotent)..."
138
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "
139
+
set -e
140
+
sudo -u postgres psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname='datomic'\" | grep -q 1 || \
141
+
sudo -u postgres psql -c \"CREATE ROLE datomic WITH LOGIN PASSWORD '$POSTGRES_PASSWORD'\"
142
+
sudo -u postgres psql -tAc \"SELECT 1 FROM pg_database WHERE datname='datomic'\" | grep -q 1 || \
143
+
sudo -u postgres psql -c \"CREATE DATABASE datomic OWNER datomic\"
144
+
sudo -u postgres psql -c \"GRANT ALL PRIVILEGES ON DATABASE datomic TO datomic\"
145
+
"
146
+
or die "postgres setup failed"
147
+
148
+
step "Checking for Datomic Pro jar on silo..."
149
+
set DATOMIC_PRESENT (ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "ls /opt/datomic/bin/transactor 2>/dev/null && echo yes || echo no" | tail -1)
150
+
if test "$DATOMIC_PRESENT" != "yes"
151
+
echo -e "$YELLOW"
152
+
echo "Datomic Pro transactor not installed at /opt/datomic/bin/transactor."
153
+
echo ""
154
+
echo "To install:"
155
+
echo " 1. Download datomic-pro-<version>.zip from https://docs.datomic.com/pro/"
156
+
echo " 2. scp -i $SSH_KEY datomic-pro-<version>.zip root@$SILO_HOST:/tmp/"
157
+
echo " 3. ssh -i $SSH_KEY root@$SILO_HOST \\"
158
+
echo " 'cd /opt/datomic && unzip -o /tmp/datomic-pro-*.zip && \\"
159
+
echo " ln -sfn datomic-pro-*/bin bin && \\"
160
+
echo " ln -sfn datomic-pro-*/lib lib && \\"
161
+
echo " chown -R datomic:datomic /opt/datomic'"
162
+
echo ""
163
+
echo "Then re-run this script to complete bring-up."
164
+
echo -e "$NC"
165
+
exit 2
166
+
end
167
+
168
+
step "Bootstrap complete."
169
+
echo -e "$GREEN"
170
+
echo "Next steps:"
171
+
echo " 1. fish silo/datomic/deploy.fish (uploads transactor config, starts service)"
172
+
echo " 2. fish kidlisp-sidecar/deploy.fish (builds + deploys sidecar)"
173
+
echo " 3. Add DATOMIC_SIDECAR_URL + DATOMIC_SIDECAR_ADMIN_SECRET to silo/.env"
174
+
echo " (see silo/datomic/README.md for values)"
175
+
echo " 4. Redeploy silo: fish silo/deploy.fish"
176
+
echo " 5. Run backfill: ssh + node system/backend/backfill-kidlisp-to-datomic.mjs"
177
+
echo -e "$NC"
+123
silo/datomic/deploy.fish
+123
silo/datomic/deploy.fish
···
1
+
#!/usr/bin/env fish
2
+
## Deploys Datomic transactor config, postgres init, and backup script to silo.
3
+
## Does NOT install JVM/Postgres/Datomic jar itself — those are one-time
4
+
## operator steps (see silo/datomic/README.md bring-up section).
5
+
##
6
+
## Usage:
7
+
## fish deploy.fish Full deploy (config + systemd + cron)
8
+
## fish deploy.fish --config Config only (no systemd reload, no restart)
9
+
10
+
set RED '\033[0;31m'
11
+
set GREEN '\033[0;32m'
12
+
set YELLOW '\033[1;33m'
13
+
set NC '\033[0m'
14
+
15
+
set SCRIPT_DIR (dirname (status --current-filename))
16
+
set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault"
17
+
set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa"
18
+
set TRANSACTOR_PROPS_GPG "$VAULT_DIR/kidlisp-datomic/transactor.properties.gpg"
19
+
set POSTGRES_ENV_GPG "$VAULT_DIR/kidlisp-datomic/postgres.env.gpg"
20
+
set SILO_HOST "silo.aesthetic.computer"
21
+
set SILO_USER "root"
22
+
set REMOTE_OPT "/opt/datomic"
23
+
24
+
set CONFIG_ONLY false
25
+
if contains -- --config $argv
26
+
set CONFIG_ONLY true
27
+
end
28
+
29
+
## Prereq checks
30
+
if not test -f $SSH_KEY
31
+
echo -e "$RED x SSH key not found: $SSH_KEY$NC"
32
+
exit 1
33
+
end
34
+
35
+
for f in $TRANSACTOR_PROPS_GPG $POSTGRES_ENV_GPG
36
+
if not test -f $f
37
+
echo -e "$RED x Vault file missing: $f$NC"
38
+
echo -e "$YELLOW Populate via vault workflow before deploying.$NC"
39
+
exit 1
40
+
end
41
+
end
42
+
43
+
echo -e "$GREEN-> Testing SSH to $SILO_HOST...$NC"
44
+
if not ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
45
+
$SILO_USER@$SILO_HOST "echo ok" &>/dev/null
46
+
echo -e "$RED x Cannot connect to $SILO_HOST$NC"
47
+
exit 1
48
+
end
49
+
50
+
## Render transactor.properties from template + vault values in-memory.
51
+
## Secrets never touch disk on the dev machine — decrypted into /dev/shm,
52
+
## scp'd to silo, then shredded.
53
+
set TPL "$SCRIPT_DIR/transactor/transactor.properties.template"
54
+
set TMP_DIR (mktemp -d /dev/shm/datomic-render-XXXXXX)
55
+
set RENDERED "$TMP_DIR/transactor.properties"
56
+
set DECRYPTED_TRANSACTOR "$TMP_DIR/transactor.values"
57
+
58
+
## Decrypt transactor.properties values
59
+
gpg --decrypt --quiet --output $DECRYPTED_TRANSACTOR $TRANSACTOR_PROPS_GPG
60
+
if test $status -ne 0
61
+
echo -e "$RED x Failed to decrypt $TRANSACTOR_PROPS_GPG$NC"
62
+
shred -u $DECRYPTED_TRANSACTOR 2>/dev/null
63
+
rmdir $TMP_DIR
64
+
exit 1
65
+
end
66
+
67
+
## Source values into current fish env (isolated subshell vars)
68
+
set -l POSTGRES_USER (grep '^POSTGRES_USER=' $DECRYPTED_TRANSACTOR | cut -d= -f2-)
69
+
set -l POSTGRES_PASSWORD (grep '^POSTGRES_PASSWORD=' $DECRYPTED_TRANSACTOR | cut -d= -f2-)
70
+
set -l STORAGE_ACCESS_SECRET (grep '^STORAGE_ACCESS_SECRET=' $DECRYPTED_TRANSACTOR | cut -d= -f2-)
71
+
72
+
## Substitute placeholders
73
+
sed -e "s|{{POSTGRES_USER}}|$POSTGRES_USER|g" \
74
+
-e "s|{{POSTGRES_PASSWORD}}|$POSTGRES_PASSWORD|g" \
75
+
-e "s|{{STORAGE_ACCESS_SECRET}}|$STORAGE_ACCESS_SECRET|g" \
76
+
$TPL > $RENDERED
77
+
78
+
echo -e "$GREEN-> Uploading rendered transactor.properties...$NC"
79
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
80
+
$RENDERED \
81
+
$SILO_USER@$SILO_HOST:/opt/datomic/config/transactor.properties
82
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST \
83
+
"chown datomic:datomic /opt/datomic/config/transactor.properties; chmod 0600 /opt/datomic/config/transactor.properties"
84
+
85
+
## Shred plaintext (including sed's temp file if any)
86
+
shred -u $RENDERED $DECRYPTED_TRANSACTOR 2>/dev/null
87
+
rmdir $TMP_DIR
88
+
89
+
## Upload non-secret files
90
+
echo -e "$GREEN-> Uploading systemd unit, init.sql, backup.fish, cron...$NC"
91
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
92
+
$SCRIPT_DIR/transactor/datomic-transactor.service \
93
+
$SILO_USER@$SILO_HOST:/etc/systemd/system/datomic-transactor.service
94
+
95
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
96
+
$SCRIPT_DIR/postgres/init.sql \
97
+
$SCRIPT_DIR/postgres/backup.fish \
98
+
$SILO_USER@$SILO_HOST:$REMOTE_OPT/
99
+
100
+
scp -i $SSH_KEY -o StrictHostKeyChecking=no \
101
+
$SCRIPT_DIR/backup.cron \
102
+
$SILO_USER@$SILO_HOST:/etc/cron.d/datomic-backup
103
+
104
+
if test $CONFIG_ONLY = false
105
+
ssh -i $SSH_KEY $SILO_USER@$SILO_HOST "
106
+
chmod +x $REMOTE_OPT/backup.fish
107
+
chmod 0644 /etc/cron.d/datomic-backup
108
+
systemctl daemon-reload
109
+
systemctl restart datomic-transactor
110
+
sleep 2
111
+
systemctl is-active datomic-transactor
112
+
"
113
+
set STATUS $status
114
+
if test $STATUS -eq 0
115
+
echo -e "$GREEN Transactor restarted OK.$NC"
116
+
else
117
+
echo -e "$RED x Transactor failed to start. Check:$NC"
118
+
echo -e "$YELLOW ssh -i $SSH_KEY $SILO_USER@$SILO_HOST journalctl -u datomic-transactor -n 50$NC"
119
+
exit 1
120
+
end
121
+
end
122
+
123
+
echo -e "$GREEN Done.$NC"
+22
silo/datomic/postgres/backup.fish
+22
silo/datomic/postgres/backup.fish
···
1
+
#!/usr/bin/env fish
2
+
## Nightly pg_dump of the Datomic storage database on silo.
3
+
## Intended to run under cron as the postgres system user.
4
+
## Installed at /opt/datomic/backup.fish via silo/datomic/deploy.fish.
5
+
6
+
set -l BACKUP_DIR /var/backups/datomic
7
+
set -l TS (date +%Y%m%d-%H%M%S)
8
+
set -l OUT "$BACKUP_DIR/datomic-$TS.sql.gz"
9
+
set -l RETAIN_DAYS 14
10
+
11
+
mkdir -p $BACKUP_DIR
12
+
13
+
pg_dump -Fc datomic | gzip -9 > $OUT
14
+
if test $status -ne 0
15
+
echo "backup failed: $OUT" >&2
16
+
exit 1
17
+
end
18
+
19
+
## Prune old dumps
20
+
find $BACKUP_DIR -type f -name 'datomic-*.sql.gz' -mtime +$RETAIN_DAYS -delete
21
+
22
+
echo "backup ok: $OUT"
+12
silo/datomic/postgres/init.sql
+12
silo/datomic/postgres/init.sql
···
1
+
-- Datomic Pro storage bootstrap on silo Postgres.
2
+
-- Run as the postgres superuser ONCE during first-time bring-up.
3
+
-- Replace :password below with the value from vault
4
+
-- (aesthetic-computer-vault/kidlisp-datomic/postgres.env.gpg).
5
+
6
+
CREATE ROLE datomic WITH LOGIN PASSWORD :'password';
7
+
CREATE DATABASE datomic OWNER datomic;
8
+
9
+
-- Datomic creates its own KV table (`datomic_kvs`) inside the `datomic`
10
+
-- database on first transactor start. No further schema needed here.
11
+
12
+
GRANT ALL PRIVILEGES ON DATABASE datomic TO datomic;
+25
silo/datomic/transactor/datomic-transactor.service
+25
silo/datomic/transactor/datomic-transactor.service
···
1
+
[Unit]
2
+
Description=Datomic Pro transactor (kidlisp)
3
+
After=network.target postgresql.service
4
+
Requires=postgresql.service
5
+
6
+
[Service]
7
+
Type=simple
8
+
User=datomic
9
+
Group=datomic
10
+
## /opt/datomic/current is a symlink to the versioned install, so upgrades
11
+
## are just `ln -sfn datomic-pro-NEW current && systemctl restart`.
12
+
WorkingDirectory=/opt/datomic/current
13
+
## JVM heap sized for silo's 3.8GB RAM with MongoDB + other services.
14
+
## Bump the -Xmx once kidlisp corpus outgrows this. The transactor
15
+
## script accepts -Xmx/-Xms positionally before the properties file.
16
+
ExecStart=/opt/datomic/current/bin/transactor -Xmx512m -Xms256m /opt/datomic/config/transactor.properties
17
+
Restart=on-failure
18
+
RestartSec=5
19
+
StandardOutput=journal
20
+
StandardError=journal
21
+
22
+
LimitNOFILE=65536
23
+
24
+
[Install]
25
+
WantedBy=multi-user.target
+37
silo/datomic/transactor/transactor.properties.template
+37
silo/datomic/transactor/transactor.properties.template
···
1
+
## Datomic Pro transactor — silo
2
+
## This file is a TEMPLATE. Values in {{ }} are filled in at deploy time
3
+
## from aesthetic-computer-vault/kidlisp-datomic/transactor.properties.gpg
4
+
## and from postgres.env.gpg.
5
+
6
+
protocol=sql
7
+
8
+
host=127.0.0.1
9
+
port=4334
10
+
11
+
## Storage: local Postgres on silo
12
+
sql-url=jdbc:postgresql://127.0.0.1:5432/datomic
13
+
sql-user={{POSTGRES_USER}}
14
+
sql-password={{POSTGRES_PASSWORD}}
15
+
sql-driver-class=org.postgresql.Driver
16
+
17
+
## Memory — tuned for silo (3.8GB total, shared with MongoDB + caddy + etc).
18
+
## These sit inside the transactor's -Xmx512m heap.
19
+
memory-index-threshold=16m
20
+
memory-index-max=128m
21
+
object-cache-max=64m
22
+
23
+
## Write guard: Datomic requires license key for Pro. Apache-2.0 build:
24
+
## leave blank.
25
+
license-key=
26
+
27
+
## Storage access key — this doubles as the PG username that peers use
28
+
## when connecting directly to storage. Must be a real PG user. We reuse
29
+
## the `datomic` PG user so there's only one credential to manage.
30
+
storage-access-key={{POSTGRES_USER}}
31
+
storage-access-secret={{STORAGE_ACCESS_SECRET}}
32
+
33
+
## Logs
34
+
log-dir=/var/log/datomic
35
+
36
+
## Data dir (peer local storage; small)
37
+
data-dir=/var/lib/datomic
+61
silo/datomic/update-silo-env.fish
+61
silo/datomic/update-silo-env.fish
···
1
+
#!/usr/bin/env fish
2
+
## Adds DATOMIC_SIDECAR_URL and DATOMIC_SIDECAR_ADMIN_SECRET to silo/.env,
3
+
## reading ADMIN_SECRET from kidlisp-datomic/sidecar.env.
4
+
##
5
+
## Safe to re-run: appends only if the vars aren't already present.
6
+
## Requires the vault to be unlocked (plaintext files on disk).
7
+
##
8
+
## Usage:
9
+
## fish aesthetic-computer-vault/vault-tool.fish unlock
10
+
## fish silo/datomic/update-silo-env.fish
11
+
## fish aesthetic-computer-vault/vault-tool.fish lock # optional
12
+
13
+
set RED '\033[0;31m'
14
+
set GREEN '\033[0;32m'
15
+
set YELLOW '\033[1;33m'
16
+
set NC '\033[0m'
17
+
18
+
set SCRIPT_DIR (dirname (status --current-filename))
19
+
set VAULT_DIR "$SCRIPT_DIR/../../aesthetic-computer-vault"
20
+
set SIDECAR_ENV "$VAULT_DIR/kidlisp-datomic/sidecar.env"
21
+
set SILO_ENV "$VAULT_DIR/silo/.env"
22
+
23
+
for f in $SIDECAR_ENV $SILO_ENV
24
+
if not test -f $f
25
+
echo -e "$RED x Not found: $f$NC"
26
+
echo -e "$YELLOW Run: fish aesthetic-computer-vault/vault-tool.fish unlock$NC"
27
+
exit 1
28
+
end
29
+
end
30
+
31
+
set ADMIN_SECRET (grep '^ADMIN_SECRET=' $SIDECAR_ENV | cut -d= -f2-)
32
+
if test -z "$ADMIN_SECRET"
33
+
echo -e "$RED x ADMIN_SECRET missing from $SIDECAR_ENV$NC"
34
+
exit 1
35
+
end
36
+
37
+
set CHANGED false
38
+
39
+
if not grep -q '^DATOMIC_SIDECAR_URL=' $SILO_ENV
40
+
echo 'DATOMIC_SIDECAR_URL=http://127.0.0.1:8891' >> $SILO_ENV
41
+
echo -e "$GREEN + DATOMIC_SIDECAR_URL added$NC"
42
+
set CHANGED true
43
+
else
44
+
echo " DATOMIC_SIDECAR_URL already present — skipping"
45
+
end
46
+
47
+
if not grep -q '^DATOMIC_SIDECAR_ADMIN_SECRET=' $SILO_ENV
48
+
echo "DATOMIC_SIDECAR_ADMIN_SECRET=$ADMIN_SECRET" >> $SILO_ENV
49
+
echo -e "$GREEN + DATOMIC_SIDECAR_ADMIN_SECRET added$NC"
50
+
set CHANGED true
51
+
else
52
+
echo " DATOMIC_SIDECAR_ADMIN_SECRET already present — skipping"
53
+
end
54
+
55
+
if test $CHANGED = true
56
+
echo -e "$GREEN Done. Remember to re-lock the vault and redeploy silo:$NC"
57
+
echo " fish aesthetic-computer-vault/vault-tool.fish lock"
58
+
echo " fish silo/deploy.fish"
59
+
else
60
+
echo -e "$GREEN Nothing to change.$NC"
61
+
end
+65
silo/server.mjs
+65
silo/server.mjs
···
1322
1322
}
1323
1323
});
1324
1324
1325
+
// ─────────────────── Datomic admin (kidlisp sidecar proxy) ───────────────────
1326
+
//
1327
+
// Dashboard → silo (/api/datomic/*) → sidecar (/admin/*) with admin secret.
1328
+
// Sidecar is bound to 127.0.0.1 on silo, so the URL is silo-internal only.
1329
+
1330
+
const DATOMIC_SIDECAR_URL = process.env.DATOMIC_SIDECAR_URL || "http://127.0.0.1:8891";
1331
+
const DATOMIC_SIDECAR_ADMIN_SECRET = process.env.DATOMIC_SIDECAR_ADMIN_SECRET;
1332
+
1333
+
async function datomicProxy(req, res, { method, path, body }) {
1334
+
if (!DATOMIC_SIDECAR_ADMIN_SECRET) {
1335
+
return res.status(503).json({ error: "Datomic sidecar not configured" });
1336
+
}
1337
+
try {
1338
+
const resp = await fetch(`${DATOMIC_SIDECAR_URL}${path}`, {
1339
+
method,
1340
+
headers: {
1341
+
"content-type": "application/json",
1342
+
"x-sidecar-secret": DATOMIC_SIDECAR_ADMIN_SECRET,
1343
+
},
1344
+
body: body != null ? JSON.stringify(body) : undefined,
1345
+
signal: AbortSignal.timeout(15000),
1346
+
});
1347
+
const text = await resp.text();
1348
+
let data;
1349
+
try { data = text ? JSON.parse(text) : null; } catch { data = text; }
1350
+
res.status(resp.status).json(data);
1351
+
} catch (err) {
1352
+
res.status(502).json({ error: err.message });
1353
+
}
1354
+
}
1355
+
1356
+
app.get("/api/datomic/health", requireAdmin, (req, res) =>
1357
+
datomicProxy(req, res, { method: "GET", path: "/admin/health" }));
1358
+
1359
+
app.get("/api/datomic/schema", requireAdmin, (req, res) =>
1360
+
datomicProxy(req, res, { method: "GET", path: "/admin/schema" }));
1361
+
1362
+
app.get("/api/datomic/stats", requireAdmin, (req, res) =>
1363
+
datomicProxy(req, res, { method: "GET", path: "/admin/stats" }));
1364
+
1365
+
app.get("/api/datomic/entities/:type", requireAdmin, (req, res) => {
1366
+
const qs = new URLSearchParams(req.query).toString();
1367
+
datomicProxy(req, res, {
1368
+
method: "GET",
1369
+
path: `/admin/entities/${encodeURIComponent(req.params.type)}${qs ? `?${qs}` : ""}`,
1370
+
});
1371
+
});
1372
+
1373
+
app.get("/api/datomic/entity/:eid", requireAdmin, (req, res) =>
1374
+
datomicProxy(req, res, { method: "GET", path: `/admin/entity/${encodeURIComponent(req.params.eid)}` }));
1375
+
1376
+
app.get("/api/datomic/entity/:eid/history", requireAdmin, (req, res) =>
1377
+
datomicProxy(req, res, { method: "GET", path: `/admin/entity/${encodeURIComponent(req.params.eid)}/history` }));
1378
+
1379
+
app.get("/api/datomic/tx-log", requireAdmin, (req, res) => {
1380
+
const qs = new URLSearchParams(req.query).toString();
1381
+
datomicProxy(req, res, { method: "GET", path: `/admin/tx-log${qs ? `?${qs}` : ""}` });
1382
+
});
1383
+
1384
+
app.post("/api/datomic/query", requireAdmin, (req, res) =>
1385
+
datomicProxy(req, res, { method: "POST", path: "/admin/query", body: req.body }));
1386
+
1387
+
app.get("/api/datomic/backups", requireAdmin, (req, res) =>
1388
+
datomicProxy(req, res, { method: "GET", path: "/admin/backups" }));
1389
+
1325
1390
app.get("/api/firehose/history", async (req, res) => {
1326
1391
if (!db) return res.json([]);
1327
1392
const limit = Math.min(parseInt(req.query.limit) || 100, 1000);
+192
system/backend/backfill-kidlisp-to-datomic.mjs
+192
system/backend/backfill-kidlisp-to-datomic.mjs
···
1
+
// Backfills the Mongo `kidlisp` collection into the Datomic sidecar.
2
+
// Idempotent: sidecar dedups by hash on POST /kidlisp. Re-runs safely.
3
+
//
4
+
// Env:
5
+
// SIDECAR_URL default http://127.0.0.1:8891
6
+
// CLIENT_SECRET required — matches sidecar.env's CLIENT_SECRET
7
+
// DRY_RUN if "true", counts but does not POST
8
+
// BATCH_SIZE default 500
9
+
// START_AFTER ISO timestamp — resume point (exclusive). Optional.
10
+
//
11
+
// Usage (from silo after sidecar is up):
12
+
// SIDECAR_URL=http://127.0.0.1:8891 \
13
+
// CLIENT_SECRET=... \
14
+
// node system/backend/backfill-kidlisp-to-datomic.mjs
15
+
16
+
import { connect } from "./database.mjs";
17
+
18
+
const SIDECAR_URL = process.env.SIDECAR_URL || "http://127.0.0.1:8891";
19
+
const CLIENT_SECRET = process.env.CLIENT_SECRET;
20
+
const DRY_RUN = process.env.DRY_RUN === "true";
21
+
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || "500", 10);
22
+
const START_AFTER = process.env.START_AFTER ? new Date(process.env.START_AFTER) : null;
23
+
24
+
if (!DRY_RUN && !CLIENT_SECRET) {
25
+
console.error("CLIENT_SECRET is required (or set DRY_RUN=true)");
26
+
process.exit(1);
27
+
}
28
+
29
+
function sidecarHeaders() {
30
+
return {
31
+
"content-type": "application/json",
32
+
"x-sidecar-secret": CLIENT_SECRET,
33
+
};
34
+
}
35
+
36
+
async function postJSON(path, body) {
37
+
if (DRY_RUN) return { ok: true, status: 200, dryRun: true };
38
+
const res = await fetch(`${SIDECAR_URL}${path}`, {
39
+
method: "POST",
40
+
headers: sidecarHeaders(),
41
+
body: JSON.stringify(body),
42
+
});
43
+
if (!res.ok) {
44
+
const text = await res.text().catch(() => "");
45
+
return { ok: false, status: res.status, body: text };
46
+
}
47
+
return { ok: true, status: res.status, body: await res.json().catch(() => ({})) };
48
+
}
49
+
50
+
function normalizeKeepForSidecar(k = {}, defaults = {}) {
51
+
const tokenId = Number(k.tokenId);
52
+
if (!Number.isInteger(tokenId) || tokenId < 0) return null;
53
+
const contractAddress = k.contractAddress || defaults.contractAddress || null;
54
+
if (!contractAddress) return null;
55
+
return {
56
+
tokenId,
57
+
network: k.network || defaults.network || "mainnet",
58
+
txHash: k.txHash || defaults.txHash || null,
59
+
contractAddress,
60
+
contractProfile: k.contractProfile || k.profile || defaults.contractProfile || null,
61
+
contractVersion: k.contractVersion || k.version || defaults.contractVersion || null,
62
+
keptAt: k.keptAt || k.mintedAt || defaults.keptAt || null,
63
+
keptBy: k.keptBy || defaults.keptBy || null,
64
+
walletAddress: k.walletAddress || k.owner || defaults.walletAddress || null,
65
+
artifactUri: k.artifactUri || defaults.artifactUri || null,
66
+
thumbnailUri: k.thumbnailUri || defaults.thumbnailUri || null,
67
+
metadataUri: k.metadataUri || defaults.metadataUri || null,
68
+
source: defaults.source || "backfill",
69
+
};
70
+
}
71
+
72
+
async function backfillDoc(doc, stats) {
73
+
const base = {
74
+
code: doc.code,
75
+
source: doc.source,
76
+
hash: doc.hash,
77
+
user_sub: doc.user || null,
78
+
when: doc.when ? new Date(doc.when).toISOString() : null,
79
+
hits: typeof doc.hits === "number" ? doc.hits : 1,
80
+
};
81
+
const create = await postJSON("/kidlisp", base);
82
+
if (!create.ok) {
83
+
stats.errors++;
84
+
console.error(` ! create failed for ${doc.code}: ${create.status} ${create.body}`);
85
+
return;
86
+
}
87
+
stats.created++;
88
+
89
+
// Keep records
90
+
const keeps = [];
91
+
if (doc.kept && typeof doc.kept === "object") {
92
+
const k = normalizeKeepForSidecar(doc.kept, { source: "kept" });
93
+
if (k) keeps.push(k);
94
+
}
95
+
if (doc.tezos?.minted) {
96
+
const k = normalizeKeepForSidecar(doc.tezos, {
97
+
source: "legacy_tezos",
98
+
contractAddress: doc.tezos.contractAddress || doc.tezos.contract || null,
99
+
keptAt: doc.tezos.mintedAt || null,
100
+
});
101
+
if (k) keeps.push(k);
102
+
}
103
+
if (doc.tezos?.contracts && typeof doc.tezos.contracts === "object") {
104
+
for (const [contractAddress, v] of Object.entries(doc.tezos.contracts)) {
105
+
if (!v || typeof v !== "object") continue;
106
+
const k = normalizeKeepForSidecar(v, { source: "contract_keyed", contractAddress });
107
+
if (k) keeps.push(k);
108
+
}
109
+
}
110
+
for (const k of keeps) {
111
+
const r = await postJSON(`/kidlisp/${doc.code}/mint`, k);
112
+
if (r.ok) stats.keeps++;
113
+
else { stats.errors++; console.error(` ! mint failed for ${doc.code}:`, r.status); }
114
+
}
115
+
116
+
// Tezos legacy summary (even when not minted — status/reason/error)
117
+
if (doc.tezos && typeof doc.tezos === "object") {
118
+
const t = doc.tezos;
119
+
const r = await postJSON(`/kidlisp/${doc.code}/tezos-state`, {
120
+
minted: !!t.minted,
121
+
exists: !!t.exists,
122
+
tokenId: t.tokenId ?? null,
123
+
txHash: t.txHash ?? null,
124
+
creatorAddress: t.creatorAddress ?? null,
125
+
codeHash: t.codeHash ?? null,
126
+
network: t.network ?? null,
127
+
reason: t.reason ?? null,
128
+
error: t.error ?? null,
129
+
});
130
+
if (r.ok) stats.tezos++;
131
+
else stats.errors++;
132
+
}
133
+
134
+
// Pending rebake
135
+
if (doc.pendingRebake && typeof doc.pendingRebake === "object") {
136
+
const r = await postJSON(`/kidlisp/${doc.code}/pending-rebake`, doc.pendingRebake);
137
+
if (r.ok) stats.pendingRebake++;
138
+
else stats.errors++;
139
+
}
140
+
141
+
// IPFS media
142
+
if (doc.ipfsMedia && typeof doc.ipfsMedia === "object") {
143
+
const r = await postJSON(`/kidlisp/${doc.code}/ipfs-media`, doc.ipfsMedia);
144
+
if (r.ok) stats.ipfs++;
145
+
else stats.errors++;
146
+
}
147
+
148
+
// ATProto rkey
149
+
if (doc.atproto?.rkey) {
150
+
const r = await postJSON(`/kidlisp/${doc.code}/atproto-rkey`, { rkey: doc.atproto.rkey });
151
+
if (r.ok) stats.atproto++;
152
+
else stats.errors++;
153
+
}
154
+
}
155
+
156
+
async function main() {
157
+
const started = Date.now();
158
+
console.log(`▶ backfill start — dryRun=${DRY_RUN} sidecar=${SIDECAR_URL}`);
159
+
const database = await connect();
160
+
const coll = database.db.collection("kidlisp");
161
+
162
+
const filter = START_AFTER ? { when: { $gt: START_AFTER } } : {};
163
+
const total = await coll.countDocuments(filter);
164
+
console.log(` docs to process: ${total}`);
165
+
166
+
const cursor = coll.find(filter).sort({ when: 1 }).batchSize(BATCH_SIZE);
167
+
168
+
const stats = {
169
+
processed: 0, created: 0, keeps: 0, tezos: 0,
170
+
pendingRebake: 0, ipfs: 0, atproto: 0, errors: 0,
171
+
};
172
+
173
+
let last = Date.now();
174
+
for await (const doc of cursor) {
175
+
await backfillDoc(doc, stats);
176
+
stats.processed++;
177
+
if (Date.now() - last > 3000) {
178
+
console.log(` ${stats.processed}/${total} — ${JSON.stringify(stats)}`);
179
+
last = Date.now();
180
+
}
181
+
}
182
+
183
+
await database.disconnect();
184
+
const secs = ((Date.now() - started) / 1000).toFixed(1);
185
+
console.log(`✓ backfill done in ${secs}s — ${JSON.stringify(stats)}`);
186
+
if (stats.errors > 0) process.exit(1);
187
+
}
188
+
189
+
main().catch((err) => {
190
+
console.error("✗ backfill fatal:", err);
191
+
process.exit(1);
192
+
});
+127
system/backend/kidlisp-sidecar.mjs
+127
system/backend/kidlisp-sidecar.mjs
···
1
+
// Thin HTTP client for the kidlisp Datomic sidecar.
2
+
// Used by system/netlify/functions/store-kidlisp.mjs behind the
3
+
// KIDLISP_DATOMIC feature flag (default off — see that file).
4
+
//
5
+
// The sidecar exposes a clean Datomic-native API; this module does not
6
+
// reshape responses. The caller is responsible for mapping sidecar
7
+
// shapes to the external Mongo-era response shape that existing clients
8
+
// depend on.
9
+
//
10
+
// Env:
11
+
// DATOMIC_SIDECAR_URL default http://127.0.0.1:8891
12
+
// DATOMIC_SIDECAR_CLIENT_SECRET required when KIDLISP_DATOMIC=on
13
+
14
+
const URL_BASE = () =>
15
+
process.env.DATOMIC_SIDECAR_URL || "http://127.0.0.1:8891";
16
+
17
+
function headers() {
18
+
const s = process.env.DATOMIC_SIDECAR_CLIENT_SECRET;
19
+
if (!s) throw new Error("DATOMIC_SIDECAR_CLIENT_SECRET not set");
20
+
return { "content-type": "application/json", "x-sidecar-secret": s };
21
+
}
22
+
23
+
async function req(method, path, body) {
24
+
const res = await fetch(`${URL_BASE()}${path}`, {
25
+
method,
26
+
headers: headers(),
27
+
body: body != null ? JSON.stringify(body) : undefined,
28
+
});
29
+
const text = await res.text();
30
+
let json = null;
31
+
try { json = text ? JSON.parse(text) : null; } catch { /* not JSON */ }
32
+
return { ok: res.ok, status: res.status, body: json ?? text };
33
+
}
34
+
35
+
export const sidecar = {
36
+
// ── public API surface used by store-kidlisp.mjs ──
37
+
38
+
async lookupByHash(hash) {
39
+
const r = await req("GET", `/kidlisp/hash/${encodeURIComponent(hash)}`);
40
+
if (r.status === 404) return null;
41
+
if (!r.ok) throw new Error(`sidecar lookupByHash failed: ${r.status}`);
42
+
return r.body;
43
+
},
44
+
45
+
async lookupByCode(code) {
46
+
const r = await req("GET", `/kidlisp/${encodeURIComponent(code)}`);
47
+
if (r.status === 404) return null;
48
+
if (!r.ok) throw new Error(`sidecar lookupByCode failed: ${r.status}`);
49
+
return r.body;
50
+
},
51
+
52
+
async batchLookup(codes) {
53
+
const r = await req("POST", `/kidlisp/lookup`, { codes });
54
+
if (!r.ok) throw new Error(`sidecar batchLookup failed: ${r.status}`);
55
+
return r.body;
56
+
},
57
+
58
+
async listCodes({ limit, sort, since } = {}) {
59
+
const qs = new URLSearchParams();
60
+
if (limit) qs.set("limit", String(limit));
61
+
if (sort) qs.set("sort", sort);
62
+
if (since) qs.set("since", since);
63
+
const r = await req("GET", `/kidlisp?${qs.toString()}`);
64
+
if (!r.ok) throw new Error(`sidecar listCodes failed: ${r.status}`);
65
+
return r.body;
66
+
},
67
+
68
+
async statsFunctions({ limit } = {}) {
69
+
const qs = new URLSearchParams();
70
+
if (limit) qs.set("limit", String(limit));
71
+
const r = await req("GET", `/kidlisp/stats/functions?${qs.toString()}`);
72
+
if (!r.ok) throw new Error(`sidecar statsFunctions failed: ${r.status}`);
73
+
return r.body;
74
+
},
75
+
76
+
// Create-or-find. Body fields:
77
+
// source, hash, code, user_sub?, forked_from?, when?, hits?
78
+
// Returns { code, cached }.
79
+
async create(piece) {
80
+
const r = await req("POST", "/kidlisp", piece);
81
+
if (!r.ok) throw new Error(`sidecar create failed: ${r.status} ${r.body}`);
82
+
return r.body;
83
+
},
84
+
85
+
async recordMint(code, keep) {
86
+
const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/mint`, keep);
87
+
if (!r.ok) throw new Error(`sidecar recordMint failed: ${r.status}`);
88
+
return r.body;
89
+
},
90
+
91
+
async setTezosState(code, state) {
92
+
const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/tezos-state`, state);
93
+
if (!r.ok) throw new Error(`sidecar setTezosState failed: ${r.status}`);
94
+
return r.body;
95
+
},
96
+
97
+
async setPendingRebake(code, rebake) {
98
+
const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/pending-rebake`, rebake);
99
+
if (!r.ok) throw new Error(`sidecar setPendingRebake failed: ${r.status}`);
100
+
return r.body;
101
+
},
102
+
103
+
async setIpfsMedia(code, media) {
104
+
const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/ipfs-media`, media);
105
+
if (!r.ok) throw new Error(`sidecar setIpfsMedia failed: ${r.status}`);
106
+
return r.body;
107
+
},
108
+
109
+
async setAtprotoRkey(code, rkey) {
110
+
const r = await req("POST", `/kidlisp/${encodeURIComponent(code)}/atproto-rkey`, { rkey });
111
+
if (!r.ok) throw new Error(`sidecar setAtprotoRkey failed: ${r.status}`);
112
+
return r.body;
113
+
},
114
+
115
+
async lineage(code) {
116
+
const r = await req("GET", `/kidlisp/${encodeURIComponent(code)}/lineage`);
117
+
if (r.status === 404) return null;
118
+
if (!r.ok) throw new Error(`sidecar lineage failed: ${r.status}`);
119
+
return r.body;
120
+
},
121
+
};
122
+
123
+
// Feature flag helper — read once per request since Netlify functions
124
+
// reuse the module across invocations.
125
+
export function kidlispDatomicEnabled() {
126
+
return process.env.KIDLISP_DATOMIC === "on";
127
+
}
+477
system/netlify/functions/store-kidlisp-datomic.mjs
+477
system/netlify/functions/store-kidlisp-datomic.mjs
···
1
+
// Datomic-backed implementation of store-kidlisp.
2
+
// Invoked by store-kidlisp.mjs when KIDLISP_DATOMIC=on.
3
+
//
4
+
// Responsibilities:
5
+
// - Speak the SAME external request/response shape as store-kidlisp.mjs
6
+
// (callers cannot tell which backend is live).
7
+
// - Translate Datomic-native sidecar responses to the Mongo-era shape.
8
+
// - Join @handles from Mongo for display names (handles stay in Mongo).
9
+
//
10
+
// Writes NEVER touch the Mongo `kidlisp` collection.
11
+
12
+
import { authorize, getHandleOrEmail } from "../../backend/authorization.mjs";
13
+
import { connect } from "../../backend/database.mjs";
14
+
import { respond } from "../../backend/http.mjs";
15
+
import { generateUniqueCode } from "../../backend/generate-short-code.mjs";
16
+
import { createMediaRecord, MediaTypes } from "../../backend/media-atproto.mjs";
17
+
import { publishProfileEvent } from "../../backend/profile-stream.mjs";
18
+
import { sidecar } from "../../backend/kidlisp-sidecar.mjs";
19
+
import crypto from "crypto";
20
+
21
+
const TEZOS_ENABLED = process.env.TEZOS_ENABLED === "true";
22
+
const NO_CACHE_HEADERS = {
23
+
"Cache-Control": "no-store, no-cache, must-revalidate, max-age=0",
24
+
Pragma: "no-cache",
25
+
Expires: "0",
26
+
};
27
+
28
+
// ── handle join (the one thing we still need Mongo for) ──
29
+
30
+
async function handlesBySub(database, subs) {
31
+
if (!subs?.length) return new Map();
32
+
const rows = await database.db
33
+
.collection("@handles")
34
+
.find({ _id: { $in: [...new Set(subs.filter(Boolean))] } })
35
+
.project({ _id: 1, handle: 1 })
36
+
.toArray();
37
+
return new Map(rows.map((r) => [r._id, `@${r.handle}`]));
38
+
}
39
+
40
+
// ── shape translators (sidecar entity → mongo-era response shape) ──
41
+
42
+
function selectPrimaryKeep(keeps, preferredContract) {
43
+
if (!Array.isArray(keeps) || keeps.length === 0) return null;
44
+
if (!preferredContract) return keeps[0];
45
+
const pref = keeps.find(
46
+
(k) =>
47
+
(k.contractAddress || "").toLowerCase() ===
48
+
preferredContract.toLowerCase()
49
+
);
50
+
return pref || keeps[0];
51
+
}
52
+
53
+
function filterKeeps(keeps, { contract, contractProfile, contractVersion }) {
54
+
if (!Array.isArray(keeps)) return [];
55
+
const c = contract?.toLowerCase() || null;
56
+
const p = contractProfile?.toLowerCase() || null;
57
+
const v = contractVersion || null;
58
+
return keeps.filter((k) => {
59
+
if (c && (k.contractAddress || "").toLowerCase() !== c) return false;
60
+
if (p && k.contractProfile && k.contractProfile !== p) return false;
61
+
if (v && k.contractVersion && k.contractVersion !== v) return false;
62
+
return true;
63
+
});
64
+
}
65
+
66
+
function toMongoShape(entity, opts = {}) {
67
+
if (!entity) return null;
68
+
const out = {
69
+
source: entity.source,
70
+
when: entity.when,
71
+
hits: entity.hits ?? 0,
72
+
user: entity.user || null,
73
+
handle: opts.handle || null,
74
+
};
75
+
if (entity.ipfsMedia) out.ipfsMedia = entity.ipfsMedia;
76
+
const keeps = filterKeeps(entity.keeps, {
77
+
contract: opts.contract,
78
+
contractProfile: opts.contractProfile,
79
+
contractVersion: opts.contractVersion,
80
+
});
81
+
if (keeps.length > 0) {
82
+
out.kept = selectPrimaryKeep(keeps, opts.contract);
83
+
if (keeps.length > 1) out.keptRecords = keeps;
84
+
}
85
+
if (entity.pendingRebake) out.pendingRebake = entity.pendingRebake;
86
+
return out;
87
+
}
88
+
89
+
// ── main handler ──
90
+
91
+
export async function handler(event, context) {
92
+
if (event.httpMethod === "OPTIONS") return respond(204, "");
93
+
94
+
let database;
95
+
try {
96
+
database = await connect();
97
+
} catch (err) {
98
+
console.error("❌ Mongo connect failed (needed for @handles join):", err.message);
99
+
return respond(503, { error: "Database temporarily unavailable" });
100
+
}
101
+
102
+
try {
103
+
if (event.httpMethod === "POST") {
104
+
const { source } = JSON.parse(event.body || "{}");
105
+
if (!source || typeof source !== "string" || source.length > 50000) {
106
+
await database.disconnect();
107
+
return respond(400, { error: "Invalid source" });
108
+
}
109
+
110
+
// Authorize (best-effort, anonymous allowed)
111
+
let user;
112
+
try {
113
+
const authPromise = authorize(event.headers);
114
+
const timeout = new Promise((_, rej) =>
115
+
setTimeout(() => rej(new Error("Auth timeout")), 3000)
116
+
);
117
+
user = await Promise.race([authPromise, timeout]);
118
+
} catch {
119
+
/* anonymous */
120
+
}
121
+
122
+
let profileHandle = null;
123
+
if (user?.sub) {
124
+
try {
125
+
const h = await getHandleOrEmail(user.sub);
126
+
if (typeof h === "string" && h.startsWith("@")) profileHandle = h;
127
+
} catch {}
128
+
}
129
+
130
+
const hash = crypto.createHash("sha256").update(source.trim()).digest("hex");
131
+
132
+
// Dedup: fast path via sidecar hash lookup
133
+
const existing = await sidecar.lookupByHash(hash);
134
+
if (existing) {
135
+
await database.disconnect();
136
+
return respond(200, { code: existing.code, cached: true });
137
+
}
138
+
139
+
// Generate a code. The Mongo `collection` previously passed to
140
+
// generateUniqueCode is used for collision checks, but with Datomic
141
+
// authoritative we check via sidecar instead. Keep the inference
142
+
// logic by passing a minimal shim.
143
+
const codeCheckShim = {
144
+
findOne: async ({ code }) => {
145
+
const hit = await sidecar.lookupByCode(code);
146
+
return hit ? { code } : null;
147
+
},
148
+
};
149
+
const code = await generateUniqueCode(codeCheckShim, {
150
+
mode: "inferred",
151
+
sourceText: source,
152
+
type: "kidlisp",
153
+
});
154
+
155
+
// Insert via sidecar
156
+
const created = await sidecar.create({
157
+
source: source.trim(),
158
+
hash,
159
+
code,
160
+
user_sub: user?.sub || null,
161
+
});
162
+
// sidecar returned {code, cached} — if cached (raced with another write),
163
+
// the returned code may differ from our proposed `code`. Honor it.
164
+
const finalCode = created.code;
165
+
166
+
// Profile stream event (fire-and-forget)
167
+
if (profileHandle) {
168
+
publishProfileEvent({
169
+
handle: profileHandle,
170
+
event: {
171
+
type: "kidlisp",
172
+
when: Date.now(),
173
+
label: `KidLisp $${finalCode}`,
174
+
ref: finalCode,
175
+
},
176
+
countsDelta: { kidlisp: 1 },
177
+
}).catch(() => {});
178
+
}
179
+
180
+
// ATProto sync (fire-and-forget). We synthesize a savedRecord shape
181
+
// matching what createMediaRecord expects.
182
+
const savedRecord = {
183
+
code: finalCode,
184
+
source: source.trim(),
185
+
hash,
186
+
when: new Date(),
187
+
user: user?.sub || null,
188
+
};
189
+
createMediaRecord(database, MediaTypes.KIDLISP, savedRecord, { userSub: user?.sub })
190
+
.then((result) => {
191
+
if (result?.rkey) {
192
+
sidecar.setAtprotoRkey(finalCode, result.rkey).catch(() => {});
193
+
}
194
+
})
195
+
.catch(() => {});
196
+
197
+
// Tezos integration (optional, fire-and-forget)
198
+
let tezosResult = null;
199
+
if (TEZOS_ENABLED && user?.sub) {
200
+
try {
201
+
const { integrateWithKidLispCache } = await import(
202
+
"../../../tezos/src/integration.js"
203
+
);
204
+
tezosResult = await integrateWithKidLispCache(
205
+
source.trim(),
206
+
user,
207
+
finalCode
208
+
);
209
+
210
+
if (tezosResult?.minted) {
211
+
await sidecar.setTezosState(finalCode, {
212
+
minted: true,
213
+
tokenId: tezosResult.tokenId,
214
+
txHash: tezosResult.txHash,
215
+
creatorAddress: tezosResult.creatorAddress,
216
+
codeHash: tezosResult.codeHash,
217
+
network: tezosResult.network,
218
+
});
219
+
} else if (tezosResult?.exists) {
220
+
await sidecar.setTezosState(finalCode, {
221
+
minted: false,
222
+
exists: true,
223
+
tokenId: tezosResult.tokenId,
224
+
codeHash: tezosResult.codeHash,
225
+
network: tezosResult.network,
226
+
reason: tezosResult.reason,
227
+
});
228
+
} else if (tezosResult) {
229
+
await sidecar.setTezosState(finalCode, {
230
+
minted: false,
231
+
exists: false,
232
+
reason: tezosResult.reason,
233
+
error: tezosResult.error,
234
+
});
235
+
}
236
+
} catch (err) {
237
+
await sidecar.setTezosState(finalCode, {
238
+
minted: false,
239
+
error: err.message,
240
+
});
241
+
}
242
+
}
243
+
244
+
await database.disconnect();
245
+
return respond(201, {
246
+
code: finalCode,
247
+
cached: false,
248
+
...(tezosResult && { tezos: tezosResult }),
249
+
});
250
+
}
251
+
252
+
if (event.httpMethod === "GET") {
253
+
const q = event.queryStringParameters || {};
254
+
const { code, codes, recent, stats } = q;
255
+
const requestedContract = q.contract || null;
256
+
const requestedContractVersion = q.contractVersion || null;
257
+
const requestedContractProfile = q.contractProfile || null;
258
+
259
+
// ── stats=functions (aggregation delegated to Node via sidecar raw docs) ──
260
+
if (stats === "functions") {
261
+
const limit = parseInt(q.limit) || 5000;
262
+
const { docs } = await sidecar.statsFunctions({ limit });
263
+
264
+
const rawCounts = {};
265
+
const weightedCounts = {};
266
+
let totalHits = 0;
267
+
268
+
const funcPattern = /\(\s*([a-zA-Z_+\-*/%?][a-zA-Z0-9_]*)/g;
269
+
const bareCommands = new Set([
270
+
"wipe","ink","line","box","circle","plot","point","flood",
271
+
"scroll","spin","zoom","blur","contrast","suck","sort",
272
+
"bake","fill","outline","stroke","nofill","nostroke",
273
+
"resolution","mask","unmask","steal","putback",
274
+
"rainbow","zebra","noise","unpan","resetSpin",
275
+
]);
276
+
const bareColors = new Set([
277
+
"red","green","blue","yellow","orange","purple","pink",
278
+
"cyan","magenta","black","white","gray","grey","brown",
279
+
"lime","navy","teal","olive","maroon","aqua","fuchsia",
280
+
"silver","gold","coral","salmon","khaki","indigo","violet",
281
+
"turquoise","tomato","crimson","lavender","beige","plum",
282
+
"orchid","tan","chocolate","sienna","peru","wheat",
283
+
"deepskyblue","hotpink","springgreen","darkslategray",
284
+
]);
285
+
286
+
for (const d of docs || []) {
287
+
const src = d.source || "";
288
+
const hits = d.hits || 1;
289
+
totalHits += hits;
290
+
const seen = new Set();
291
+
let m;
292
+
funcPattern.lastIndex = 0;
293
+
while ((m = funcPattern.exec(src)) !== null) seen.add(m[1]);
294
+
const tokens = src.split(/[,\n]/).map((t) => t.trim().split(/\s+/)[0]);
295
+
for (const t of tokens) {
296
+
if (bareCommands.has(t)) seen.add(t);
297
+
if (bareColors.has(t)) seen.add("wipe");
298
+
}
299
+
if (/\$[a-zA-Z0-9]+/.test(src)) seen.add("embed");
300
+
if (/\d+\.?\d*s[.!]?/.test(src)) seen.add("timing");
301
+
if (/fade:/.test(src)) seen.add("fade");
302
+
303
+
for (const fn of seen) {
304
+
rawCounts[fn] = (rawCounts[fn] || 0) + 1;
305
+
weightedCounts[fn] = (weightedCounts[fn] || 0) + hits;
306
+
}
307
+
}
308
+
309
+
const sorted = Object.entries(weightedCounts)
310
+
.sort((a, b) => b[1] - a[1])
311
+
.map(([name, weighted]) => ({
312
+
name,
313
+
pieces: rawCounts[name] || 0,
314
+
weighted,
315
+
}));
316
+
317
+
await database.disconnect();
318
+
return respond(200, {
319
+
functions: sorted,
320
+
total_pieces: (docs || []).length,
321
+
total_hits: totalHits,
322
+
});
323
+
}
324
+
325
+
// ── recent ──
326
+
if (recent) {
327
+
const limit = Math.min(parseInt(q.limit) || 50, 100000);
328
+
const sort = q.sort || "recent";
329
+
const since = q.since;
330
+
const filterHandle = q.handle;
331
+
332
+
const res = await sidecar.listCodes({ limit, sort, since });
333
+
const pieces = res.recent || [];
334
+
335
+
const handles = await handlesBySub(database, pieces.map((p) => p.user));
336
+
337
+
const shaped = pieces
338
+
.map((p) => {
339
+
const handle = p.user ? handles.get(p.user) : null;
340
+
return {
341
+
code: p.code,
342
+
source: p.source,
343
+
preview:
344
+
p.source && p.source.length > 40
345
+
? p.source.substring(0, 37) + "..."
346
+
: p.source,
347
+
when: p.when,
348
+
hits: p.hits,
349
+
user: p.user || null,
350
+
handle: handle || null,
351
+
...(toMongoShape(p, {
352
+
contract: requestedContract,
353
+
contractProfile: requestedContractProfile,
354
+
contractVersion: requestedContractVersion,
355
+
}) || {}),
356
+
};
357
+
})
358
+
.filter((p) => {
359
+
if (!filterHandle) return true;
360
+
const want = filterHandle.startsWith("@")
361
+
? filterHandle
362
+
: `@${filterHandle}`;
363
+
return p.handle === want;
364
+
});
365
+
366
+
await database.disconnect();
367
+
return respond(
368
+
200,
369
+
{ recent: shaped, count: shaped.length, limit },
370
+
NO_CACHE_HEADERS
371
+
);
372
+
}
373
+
374
+
// ── batch lookup ──
375
+
if (codes) {
376
+
let codeList;
377
+
try {
378
+
codeList = codes.startsWith("[")
379
+
? JSON.parse(codes)
380
+
: codes.split(",").map((c) => c.trim()).filter(Boolean);
381
+
} catch {
382
+
await database.disconnect();
383
+
return respond(400, {
384
+
error: "Invalid codes format. Use comma-separated or JSON array.",
385
+
});
386
+
}
387
+
if (!Array.isArray(codeList) || !codeList.length) {
388
+
await database.disconnect();
389
+
return respond(400, { error: "Codes must be a non-empty array" });
390
+
}
391
+
if (codeList.length > 50) {
392
+
await database.disconnect();
393
+
return respond(400, {
394
+
error: "Too many codes. Maximum 50 per request.",
395
+
});
396
+
}
397
+
398
+
const { results: rawResults } = await sidecar.batchLookup(codeList);
399
+
const foundSubs = Object.values(rawResults)
400
+
.filter(Boolean)
401
+
.map((r) => r.user)
402
+
.filter(Boolean);
403
+
const handles = await handlesBySub(database, foundSubs);
404
+
405
+
const results = {};
406
+
const found = [];
407
+
const missing = [];
408
+
for (const c of codeList) {
409
+
const r = rawResults[c];
410
+
if (r) {
411
+
results[c] = toMongoShape(r, {
412
+
handle: r.user ? handles.get(r.user) || null : null,
413
+
contract: requestedContract,
414
+
contractProfile: requestedContractProfile,
415
+
contractVersion: requestedContractVersion,
416
+
});
417
+
found.push(c);
418
+
} else {
419
+
results[c] = null;
420
+
missing.push(c);
421
+
}
422
+
}
423
+
424
+
await database.disconnect();
425
+
return respond(
426
+
200,
427
+
{
428
+
results,
429
+
summary: {
430
+
requested: codeList.length,
431
+
found: found.length,
432
+
missing: missing.length,
433
+
foundCodes: found,
434
+
missingCodes: missing,
435
+
},
436
+
},
437
+
NO_CACHE_HEADERS
438
+
);
439
+
}
440
+
441
+
// ── single-code lookup ──
442
+
if (!code) {
443
+
await database.disconnect();
444
+
return respond(400, { error: "Code or codes parameter required" });
445
+
}
446
+
447
+
const entity = await sidecar.lookupByCode(code);
448
+
if (!entity) {
449
+
await database.disconnect();
450
+
return respond(404, { error: "Not found" });
451
+
}
452
+
453
+
const handle = entity.user
454
+
? (await handlesBySub(database, [entity.user])).get(entity.user)
455
+
: null;
456
+
457
+
await database.disconnect();
458
+
return respond(
459
+
200,
460
+
toMongoShape(entity, {
461
+
handle: handle || null,
462
+
contract: requestedContract,
463
+
contractProfile: requestedContractProfile,
464
+
contractVersion: requestedContractVersion,
465
+
}),
466
+
NO_CACHE_HEADERS
467
+
);
468
+
}
469
+
470
+
await database.disconnect();
471
+
return respond(405, { error: "Method not allowed" });
472
+
} catch (err) {
473
+
console.error("❌ Datomic kidlisp error:", err);
474
+
try { await database.disconnect(); } catch {}
475
+
return respond(500, { error: "Internal server error", details: err.message });
476
+
}
477
+
}
+8
system/netlify/functions/store-kidlisp.mjs
+8
system/netlify/functions/store-kidlisp.mjs
···
7
7
import { generateUniqueCode } from "../../backend/generate-short-code.mjs";
8
8
import { createMediaRecord, MediaTypes } from "../../backend/media-atproto.mjs";
9
9
import { publishProfileEvent } from "../../backend/profile-stream.mjs";
10
+
import { kidlispDatomicEnabled } from "../../backend/kidlisp-sidecar.mjs";
11
+
import { handler as datomicHandler } from "./store-kidlisp-datomic.mjs";
10
12
import crypto from 'crypto';
11
13
12
14
// Feature flag for Tezos integration (disabled by default until integration file exists)
···
263
265
}
264
266
265
267
export async function handler(event, context) {
268
+
// Feature flag: route to the Datomic-backed implementation when on.
269
+
// Default off — existing Mongo behavior runs unchanged.
270
+
if (kidlispDatomicEnabled()) {
271
+
return datomicHandler(event, context);
272
+
}
273
+
266
274
// Log all incoming requests for debugging
267
275
console.log(`📥 Kidlisp store request: ${event.httpMethod} ${event.path || event.rawUrl || 'unknown'}`);
268
276
console.log(`📊 Headers:`, Object.keys(event.headers || {}).length > 0 ? Object.keys(event.headers) : 'none');