sans-self.org / cluster-infrastructure
this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Removed my spingle instance :(

Not Herself 6b80add5 3844fa8e

+1 -734
-3
.gitignore
··· 33 33 *.yaml.backup 34 34 k3s_kustomization_backup.yaml 35 35 36 - # Spindle binary (built externally, not checked in) 37 - spindle/spindle 38 - 39 36 # Claude 40 37 settings.local.json
-15
.tangled/workflows/test-secrets.yml
··· 1 - when: 2 - - event: ["push"] 3 - branch: ["main"] 4 - 5 - engine: "nixery" 6 - 7 - steps: 8 - - name: "Verify secret access" 9 - command: | 10 - if [ "$TEST_SECRET_VERIFICATION" = "Hello :)" ]; then 11 - echo "Secret verification passed" 12 - else 13 - echo "Secret verification failed: expected 'Hello :)', got '$TEST_SECRET_VERIFICATION'" 14 - exit 1 15 - fi
-78
Makefile
··· 25 25 clean-secrets: 26 26 rm -f $(JUICEFS_METAURL) $(TRANQUIL_DB_URL) $(TRANQUIL_VALKEY_URL) 27 27 28 - # Spindle CI runner 29 - # Full flow: build-spindle → push-spindle → update-spindle → start-spindle (first time only) 30 - # Updates: build-spindle → push-spindle → update-spindle (restarts where already active) 31 - SPINDLE_CORE ?= /tmp/tangled-core 32 - 33 - .PHONY: build-spindle push-spindle update-spindle start-spindle logs-spindle setup-openbao logs-openbao 34 - 35 - build-spindle: 36 - @test -d "$(SPINDLE_CORE)" || { echo "error: tangled core not found at $(SPINDLE_CORE)"; echo "Clone: git clone git@tangled.org:tangled.org/core.git $(SPINDLE_CORE)"; exit 1; } 37 - docker run --rm -v "$(SPINDLE_CORE)":/src -v "$(CURDIR)/spindle":/out -w /src \ 38 - -e GOARCH=arm64 -e GOOS=linux -e CGO_ENABLED=1 -e GOTOOLCHAIN=auto \ 39 - -e CC=aarch64-linux-gnu-gcc \ 40 - golang:1.25rc1 sh -c 'apt-get update -qq && apt-get install -y -qq gcc-aarch64-linux-gnu >/dev/null 2>&1 && go build -o /out/spindle ./cmd/spindle' 41 - 42 - push-spindle: spindle/spindle 43 - docker build --platform linux/arm64 -f spindle/Containerfile -t zot.sans-self.org/infra/spindle:latest spindle/ 44 - docker push zot.sans-self.org/infra/spindle:latest 45 - 46 - spindle/spindle: 47 - @echo "Binary missing. Run: make build-spindle SPINDLE_CORE=/path/to/tangled/core" 48 - @exit 1 49 - 50 - update-spindle: 51 - kubectl delete job spindle-update --ignore-not-found 52 - kubectl apply -f spindle/update-job.yaml 53 - 54 - start-spindle: 55 - kubectl delete job spindle-start --ignore-not-found 56 - kubectl apply -f spindle/start-job.yaml 57 - 58 28 # Tranquil PDS 59 29 TRANQUIL_REPO ?= /tmp/tranquil-pds 60 30 ··· 74 44 push-tranquil-frontend: 75 45 docker push zot.sans-self.org/infra/tranquil-frontend:latest 76 46 77 - logs-spindle: 78 - @NODE_NAME=$$(kubectl get configmap spindle-leader -o jsonpath='{.data.node}' 2>/dev/null) && \ 79 - NODE_IP=$$(kubectl get node $$NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}' 2>/dev/null) && \ 80 - test -n "$$NODE_IP" || { echo "error: could not find spindle leader node"; exit 1; } && \ 81 - echo "==> Streaming logs from $$NODE_NAME ($$NODE_IP)" && \ 82 - ssh -p 22222 -o StrictHostKeyChecking=no -i keypair/id_ed25519_homelab root@$$NODE_IP \ 83 - 'SPINDLE_UID=$$(id -u spindle) && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$$SPINDLE_UID/bus journalctl --user -u spindle.service -f --no-pager' 84 - 85 - logs-openbao: 86 - kubectl logs -n openbao statefulset/openbao -f 87 - 88 - # OpenBao — one-time init + AppRole setup for Spindle secrets 89 - # Initializes OpenBao, creates KV engine + AppRole, deploys proxy credentials to spindle leader node 90 - setup-openbao: 91 - @echo "==> Initializing OpenBao..." 92 - @INIT_OUTPUT=$$(kubectl exec -n openbao openbao-0 -- bao operator init -key-shares=1 -key-threshold=1 -format=json 2>/dev/null) && \ 93 - UNSEAL_KEY=$$(echo "$$INIT_OUTPUT" | python3 -c "import sys,json; print(json.load(sys.stdin)['unseal_keys_b64'][0])") && \ 94 - ROOT_TOKEN=$$(echo "$$INIT_OUTPUT" | python3 -c "import sys,json; print(json.load(sys.stdin)['root_token'])") && \ 95 - echo "==> Unsealing..." && \ 96 - kubectl exec -n openbao openbao-0 -- bao operator unseal "$$UNSEAL_KEY" >/dev/null && \ 97 - echo "==> Creating unseal Secret..." && \ 98 - kubectl create secret generic openbao-unseal -n openbao --from-literal=unseal-key="$$UNSEAL_KEY" --dry-run=client -o yaml | kubectl apply -f - && \ 99 - echo "==> Configuring KV engine + AppRole..." && \ 100 - kubectl exec -n openbao openbao-0 -- sh -c "export BAO_TOKEN=$$ROOT_TOKEN && \ 101 - bao secrets enable -path=spindle -version=2 kv && \ 102 - bao policy write spindle-policy /openbao/config/spindle-policy.hcl && \ 103 - bao auth enable approle && \ 104 - bao write auth/approle/role/spindle token_policies=spindle-policy token_ttl=1h token_max_ttl=4h bind_secret_id=true secret_id_ttl=0 secret_id_num_uses=0" && \ 105 - ROLE_ID=$$(kubectl exec -n openbao openbao-0 -- sh -c "BAO_TOKEN=$$ROOT_TOKEN bao read -field=role_id auth/approle/role/spindle/role-id") && \ 106 - SECRET_ID=$$(kubectl exec -n openbao openbao-0 -- sh -c "BAO_TOKEN=$$ROOT_TOKEN bao write -f -field=secret_id auth/approle/role/spindle/secret-id") && \ 107 - echo "==> Deploying proxy credentials to spindle leader..." && \ 108 - NODE_NAME=$$(kubectl get configmap spindle-leader -o jsonpath='{.data.node}' 2>/dev/null) && \ 109 - NODE_IP=$$(kubectl get node $$NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}' 2>/dev/null) && \ 110 - test -n "$$NODE_IP" || { echo "error: could not find spindle leader node — run make start-spindle first"; exit 1; } && \ 111 - ssh -p 22222 -o StrictHostKeyChecking=no -i keypair/id_ed25519_homelab root@$$NODE_IP " \ 112 - printf '%s' '$$ROLE_ID' > /home/spindle/.openbao/role-id && \ 113 - printf '%s' '$$SECRET_ID' > /home/spindle/.openbao/secret-id && \ 114 - chmod 600 /home/spindle/.openbao/role-id /home/spindle/.openbao/secret-id && \ 115 - chown spindle:spindle /home/spindle/.openbao/role-id /home/spindle/.openbao/secret-id && \ 116 - SPINDLE_UID=\$$(id -u spindle) && \ 117 - runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/\$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$$SPINDLE_UID/bus systemctl --user restart openbao-proxy.service && \ 118 - runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/\$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$$SPINDLE_UID/bus systemctl --user restart spindle.service \ 119 - " && \ 120 - echo "" && \ 121 - echo "==> Done. Save this root token somewhere safe:" && \ 122 - echo " $$ROOT_TOKEN" && \ 123 - echo "" && \ 124 - echo "OpenBao initialized, AppRole configured, proxy credentials deployed."
-52
README.md
··· 52 52 3. After bootstrap, fetch a fresh kubeconfig from the node — the one in tofu state will have the wrong CA. 53 53 4. JuiceFS CSI on SELinux (MicroOS) requires `sidecarPrivileged: true` in `juicefs-csi-values.yaml` under `node:`. Without it, the CSI socket has a label mismatch and sidecars can't connect. 54 54 55 - ## Spindle CI Runner 56 - 57 - Self-hosted [Spindle](https://tangled.org) runner at `spindle.sans-self.org` for pipeline execution on our knot. Runs as a systemd user service with Podman rootless — no privileged containers, no DinD. 58 - 59 - ### Architecture 60 - 61 - Spindle runs outside k8s (needs direct access to the Podman socket) but is exposed via Traefik through a selectorless k8s Service + Endpoints. A CronJob healthcheck (`spindle-healthcheck`) runs every 5 minutes: it heartbeats to a `spindle-leader` ConfigMap and keeps the Endpoints pointed at the active node. If the heartbeat goes stale (>10 min), the healthcheck auto-starts Spindle on whichever node it lands on. 62 - 63 - Node provisioning is automatic: `postinstall_exec` in kube.tf creates the spindle user, configures rootless Podman, and pulls the binary from Zot (`zot.sans-self.org/infra/spindle:latest`, anonymous pull). This runs on every new or replaced node. 64 - 65 - ### Build 66 - 67 - ```sh 68 - git clone git@tangled.org:tangled.org/core.git ~/Projects/tangled 69 - make build-spindle SPINDLE_CORE=~/Projects/tangled 70 - ``` 71 - 72 - Requires Docker (cross-compiles with `aarch64-linux-gnu-gcc` for CGo/sqlite3). 73 - 74 - ### Initial setup 75 - 76 - ```sh 77 - make build-spindle # compile ARM64 binary via Docker 78 - make push-spindle # build OCI image, push to Zot (needs docker login) 79 - make update-spindle # deploy binary to all nodes via k8s Job 80 - make start-spindle # start service on one node (run once) 81 - kubectl apply -f spindle/ingress.yaml 82 - kubectl apply -f spindle/healthcheck-cronjob.yaml 83 - ``` 84 - 85 - Then add the runner in the tangled.org UI with hostname `spindle.sans-self.org`. 86 - 87 - ### Binary updates 88 - 89 - ```sh 90 - make build-spindle # recompile from latest source 91 - make push-spindle # push new image to Zot 92 - make update-spindle # rolls out to all nodes, restarts where active 93 - ``` 94 - 95 - ### Node replacement 96 - 97 - Automatic. The healthcheck CronJob detects the stale heartbeat and starts Spindle on another node within 5 minutes. The Endpoints object is updated to route traffic to the new node. 98 - 99 - ### Operations 100 - 101 - ```sh 102 - make logs-spindle # stream journal from active node 103 - make start-spindle # manual start (if healthcheck hasn't kicked in yet) 104 - make update-spindle # redeploy binary + restart 105 - ``` 106 - 107 55 ## Backups 108 56 109 57 Daily S3 backups via CronJobs (02:00 PDS, 02:30 knot). See [RESTORE.md](RESTORE.md) for recovery procedures.
+1 -1
k8s/knot/network-policy.yaml
··· 24 24 kubernetes.io/metadata.name: traefik 25 25 ports: 26 26 - port: 22 27 - # Internal: allow Spindle (future) to reach knot events endpoint 27 + # Internal: knot pods reaching their own API (post-receive hooks → port 5444) 28 28 - from: 29 29 - namespaceSelector: 30 30 matchLabels:
-1
k8s/kustomization.yaml
··· 10 10 - registry 11 11 - alerting 12 12 - opake 13 - - openbao 14 13 15 14 generatorOptions: 16 15 disableNameSuffixHash: true
-17
k8s/openbao/kustomization.yaml
··· 1 - apiVersion: kustomize.config.k8s.io/v1beta1 2 - kind: Kustomization 3 - namespace: openbao 4 - 5 - resources: 6 - - namespace.yaml 7 - - statefulset.yaml 8 - - service.yaml 9 - 10 - configMapGenerator: 11 - - name: openbao-config 12 - namespace: openbao 13 - files: 14 - - server.hcl 15 - - spindle-policy.hcl 16 - options: 17 - disableNameSuffixHash: true
-4
k8s/openbao/namespace.yaml
··· 1 - apiVersion: v1 2 - kind: Namespace 3 - metadata: 4 - name: openbao
-11
k8s/openbao/server.hcl
··· 1 - storage "file" { 2 - path = "/openbao/data" 3 - } 4 - 5 - listener "tcp" { 6 - address = "0.0.0.0:8200" 7 - tls_disable = true 8 - } 9 - 10 - disable_mlock = true 11 - ui = false
-14
k8s/openbao/service.yaml
··· 1 - apiVersion: v1 2 - kind: Service 3 - metadata: 4 - name: openbao 5 - namespace: openbao 6 - spec: 7 - type: NodePort 8 - selector: 9 - app: openbao 10 - ports: 11 - - name: api 12 - port: 8200 13 - targetPort: 8200 14 - nodePort: 30820
-15
k8s/openbao/spindle-policy.hcl
··· 1 - path "spindle/data/*" { 2 - capabilities = ["create", "read", "update", "delete"] 3 - } 4 - 5 - path "spindle/metadata/*" { 6 - capabilities = ["list", "read", "delete", "update"] 7 - } 8 - 9 - path "spindle/" { 10 - capabilities = ["list"] 11 - } 12 - 13 - path "auth/token/lookup-self" { 14 - capabilities = ["read"] 15 - }
-89
k8s/openbao/statefulset.yaml
··· 1 - apiVersion: apps/v1 2 - kind: StatefulSet 3 - metadata: 4 - name: openbao 5 - namespace: openbao 6 - spec: 7 - serviceName: openbao 8 - replicas: 1 9 - selector: 10 - matchLabels: 11 - app: openbao 12 - template: 13 - metadata: 14 - labels: 15 - app: openbao 16 - spec: 17 - securityContext: 18 - runAsUser: 100 19 - runAsGroup: 1000 20 - fsGroup: 1000 21 - containers: 22 - - name: openbao 23 - image: openbao/openbao:2.5.1 24 - args: ["server", "-config=/openbao/config/server.hcl"] 25 - ports: 26 - - containerPort: 8200 27 - name: api 28 - env: 29 - - name: BAO_ADDR 30 - value: "http://127.0.0.1:8200" 31 - - name: SKIP_SETCAP 32 - value: "true" 33 - resources: 34 - requests: 35 - cpu: 50m 36 - memory: 64Mi 37 - limits: 38 - cpu: 200m 39 - memory: 256Mi 40 - readinessProbe: 41 - httpGet: 42 - path: /v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 43 - port: 8200 44 - initialDelaySeconds: 5 45 - periodSeconds: 10 46 - livenessProbe: 47 - httpGet: 48 - path: /v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 49 - port: 8200 50 - initialDelaySeconds: 10 51 - periodSeconds: 30 52 - lifecycle: 53 - postStart: 54 - exec: 55 - command: 56 - - /bin/sh 57 - - -c 58 - - | 59 - sleep 3 60 - if [ -f /openbao/credentials/unseal-key ]; then 61 - bao operator unseal "$(cat /openbao/credentials/unseal-key)" 2>/dev/null || true 62 - fi 63 - volumeMounts: 64 - - name: data 65 - mountPath: /openbao/data 66 - - name: config 67 - mountPath: /openbao/config 68 - readOnly: true 69 - - name: unseal-key 70 - mountPath: /openbao/credentials 71 - readOnly: true 72 - volumes: 73 - - name: config 74 - configMap: 75 - name: openbao-config 76 - - name: unseal-key 77 - secret: 78 - secretName: openbao-unseal 79 - optional: true 80 - volumeClaimTemplates: 81 - - metadata: 82 - name: data 83 - spec: 84 - accessModes: 85 - - ReadWriteOnce 86 - storageClassName: juicefs-sc 87 - resources: 88 - requests: 89 - storage: 1Gi
-32
kube.tf
··· 103 103 } 104 104 ] 105 105 106 - # Spindle CI runner — provision user + rootless Podman on every node. 107 - # Binary pulled from Zot registry (fails gracefully on first bootstrap when Zot isn't up yet). 108 - postinstall_exec = [ 109 - # User + subuid/subgid for rootless Podman 110 - "useradd --create-home --shell /bin/bash spindle 2>/dev/null || true", 111 - "chown spindle:spindle /home/spindle", 112 - "grep -q '^spindle:' /etc/subuid || usermod --add-subuids 100000-165535 spindle", 113 - "grep -q '^spindle:' /etc/subgid || usermod --add-subgids 100000-165535 spindle", 114 - "loginctl enable-linger spindle", 115 - 116 - # Directories: logs, data, systemd unit, podman config, openbao proxy 117 - "mkdir -p /var/log/spindle && chown spindle:spindle /var/log/spindle", 118 - "mkdir -p /var/lib/spindle/logs && chown -R spindle:spindle /var/lib/spindle", 119 - "mkdir -p /home/spindle/.config/systemd/user && chown -R spindle:spindle /home/spindle/.config", 120 - "mkdir -p /home/spindle/.openbao && chown spindle:spindle /home/spindle/.openbao", 121 - 122 - # Disable SELinux labels in rootless containers — kernel 6.19 + SELinux prevents 123 - # mprotect in user namespaces, causing RELRO failures in musl-based containers. 124 - # Rootless userns isolation is the primary security boundary; labels are defense-in-depth. 125 - # See: https://github.com/containers/podman/issues/27895 126 - "mkdir -p /home/spindle/.config/containers && printf '[containers]\\nlabel = false\\n' > /home/spindle/.config/containers/containers.conf && chown -R spindle:spindle /home/spindle/.config/containers", 127 - 128 - # Fix SELinux contexts on home dir (provisioning runs as root, labels end up wrong) 129 - "restorecon -R /home/spindle", 130 - 131 - # Pull spindle + openbao proxy from Zot OCI image 132 - "podman pull zot.sans-self.org/infra/spindle:latest && CID=$(podman create zot.sans-self.org/infra/spindle:latest) && podman cp $CID:/spindle /usr/local/bin/spindle && podman cp $CID:/bao /usr/local/bin/bao && podman cp $CID:/spindle.service /home/spindle/.config/systemd/user/spindle.service && podman cp $CID:/openbao-proxy.service /home/spindle/.config/systemd/user/openbao-proxy.service && podman cp $CID:/openbao-proxy.hcl /home/spindle/.openbao/proxy.hcl && podman rm $CID && chmod 755 /usr/local/bin/spindle /usr/local/bin/bao && chown -R spindle:spindle /home/spindle/.config /home/spindle/.openbao || echo 'WARN: spindle image not available, run make update-spindle after cluster is ready'", 133 - 134 - # Enable podman socket for rootless container API 135 - "sleep 2 && SPINDLE_UID=$(id -u spindle) && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$SPINDLE_UID/bus systemctl --user daemon-reload && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$SPINDLE_UID/bus systemctl --user enable --now podman.socket || true", 136 - ] 137 - 138 106 create_kubeconfig = false 139 107 } 140 108
-8
spindle/Containerfile
··· 1 - FROM openbao/openbao:2.5.1 AS openbao 2 - 3 - FROM busybox:1.37 4 - COPY spindle /spindle 5 - COPY spindle.service /spindle.service 6 - COPY --from=openbao /bin/bao /bao 7 - COPY openbao-proxy.service /openbao-proxy.service 8 - COPY openbao-proxy.hcl /openbao-proxy.hcl
-161
spindle/healthcheck-cronjob.yaml
··· 1 - apiVersion: v1 2 - kind: ServiceAccount 3 - metadata: 4 - name: spindle-healthcheck 5 - --- 6 - apiVersion: rbac.authorization.k8s.io/v1 7 - kind: Role 8 - metadata: 9 - name: spindle-healthcheck 10 - rules: 11 - - apiGroups: [""] 12 - resources: ["configmaps"] 13 - verbs: ["get", "create", "update", "patch"] 14 - - apiGroups: [""] 15 - resources: ["endpoints"] 16 - verbs: ["get", "update", "patch"] 17 - --- 18 - apiVersion: rbac.authorization.k8s.io/v1 19 - kind: ClusterRole 20 - metadata: 21 - name: spindle-healthcheck-nodes 22 - rules: 23 - - apiGroups: [""] 24 - resources: ["nodes"] 25 - verbs: ["get", "list"] 26 - --- 27 - apiVersion: rbac.authorization.k8s.io/v1 28 - kind: RoleBinding 29 - metadata: 30 - name: spindle-healthcheck 31 - subjects: 32 - - kind: ServiceAccount 33 - name: spindle-healthcheck 34 - roleRef: 35 - apiGroup: rbac.authorization.k8s.io 36 - kind: Role 37 - name: spindle-healthcheck 38 - --- 39 - apiVersion: rbac.authorization.k8s.io/v1 40 - kind: ClusterRoleBinding 41 - metadata: 42 - name: spindle-healthcheck-nodes 43 - subjects: 44 - - kind: ServiceAccount 45 - name: spindle-healthcheck 46 - namespace: default 47 - roleRef: 48 - apiGroup: rbac.authorization.k8s.io 49 - kind: ClusterRole 50 - name: spindle-healthcheck-nodes 51 - --- 52 - apiVersion: batch/v1 53 - kind: CronJob 54 - metadata: 55 - name: spindle-healthcheck 56 - spec: 57 - schedule: "*/5 * * * *" 58 - concurrencyPolicy: Forbid 59 - successfulJobsHistoryLimit: 1 60 - failedJobsHistoryLimit: 3 61 - jobTemplate: 62 - spec: 63 - template: 64 - metadata: 65 - labels: 66 - app: spindle-healthcheck 67 - spec: 68 - serviceAccountName: spindle-healthcheck 69 - hostPID: true 70 - tolerations: 71 - - operator: Exists 72 - containers: 73 - - name: check 74 - image: alpine:3.21 75 - securityContext: 76 - privileged: true 77 - env: 78 - - name: NODE_NAME 79 - valueFrom: 80 - fieldRef: 81 - fieldPath: spec.nodeName 82 - command: ["/bin/sh", "-c"] 83 - args: 84 - - | 85 - apk add --no-cache --quiet util-linux curl jq >/dev/null 2>&1 86 - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) 87 - CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt 88 - NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) 89 - K8S="https://kubernetes.default.svc" 90 - AUTH="-H \"Authorization: Bearer $TOKEN\"" 91 - 92 - kapi() { curl -sk --cacert $CA -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" "$@"; } 93 - 94 - # Check if spindle is running on THIS node 95 - ACTIVE=$(nsenter -t 1 -m -u -i -n -- bash -c ' 96 - uid=$(id -u spindle 2>/dev/null) || exit 1 97 - XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 98 - runuser -u spindle -- systemctl --user is-active --quiet spindle.service 2>/dev/null && echo "yes" || echo "no" 99 - ') 100 - 101 - update_endpoint() { 102 - local node="$1" 103 - # Get node's InternalIP 104 - NODE_IP=$(kapi "$K8S/api/v1/nodes/$node" 2>/dev/null | jq -r '.status.addresses[] | select(.type=="InternalIP") | .address') 105 - if [ -z "$NODE_IP" ] || [ "$NODE_IP" = "null" ]; then 106 - echo "WARN: could not resolve InternalIP for $node" 107 - return 1 108 - fi 109 - EPBODY="{\"apiVersion\":\"v1\",\"kind\":\"Endpoints\",\"metadata\":{\"name\":\"spindle\"},\"subsets\":[{\"addresses\":[{\"ip\":\"$NODE_IP\"}],\"ports\":[{\"port\":6555,\"protocol\":\"TCP\"}]}]}" 110 - kapi -X PUT "$K8S/api/v1/namespaces/$NS/endpoints/spindle" -d "$EPBODY" >/dev/null 2>&1 111 - echo "Endpoints updated: $node ($NODE_IP)" 112 - } 113 - 114 - update_heartbeat() { 115 - local node="$1" 116 - TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) 117 - BODY="{\"apiVersion\":\"v1\",\"kind\":\"ConfigMap\",\"metadata\":{\"name\":\"spindle-leader\"},\"data\":{\"node\":\"$node\",\"heartbeat\":\"$TIMESTAMP\"}}" 118 - kapi -X PUT "$K8S/api/v1/namespaces/$NS/configmaps/spindle-leader" -d "$BODY" >/dev/null 2>&1 || \ 119 - kapi -X POST "$K8S/api/v1/namespaces/$NS/configmaps" -d "$BODY" >/dev/null 2>&1 120 - } 121 - 122 - if [ "$ACTIVE" = "yes" ]; then 123 - update_heartbeat "$NODE_NAME" 124 - update_endpoint "$NODE_NAME" 125 - echo "Spindle active on $NODE_NAME, heartbeat + endpoint updated" 126 - exit 0 127 - fi 128 - 129 - # Not running here. Check if someone else has a recent heartbeat. 130 - LEADER_DATA=$(kapi "$K8S/api/v1/namespaces/$NS/configmaps/spindle-leader" 2>/dev/null) 131 - HEARTBEAT=$(echo "$LEADER_DATA" | jq -r '.data.heartbeat // empty') 132 - LEADER_NODE=$(echo "$LEADER_DATA" | jq -r '.data.node // empty') 133 - 134 - if [ -n "$HEARTBEAT" ]; then 135 - HEARTBEAT_EPOCH=$(date -d "$HEARTBEAT" +%s 2>/dev/null || echo 0) 136 - NOW_EPOCH=$(date +%s) 137 - AGE=$(( NOW_EPOCH - HEARTBEAT_EPOCH )) 138 - if [ "$AGE" -lt 600 ]; then 139 - echo "Spindle healthy on $LEADER_NODE (heartbeat ${AGE}s ago), nothing to do" 140 - exit 0 141 - fi 142 - echo "Stale heartbeat from $LEADER_NODE (${AGE}s ago), taking over" 143 - fi 144 - 145 - # No healthy spindle. Start it on this node. 146 - echo "Starting spindle on $NODE_NAME" 147 - nsenter -t 1 -m -u -i -n -- bash -c ' 148 - uid=$(id -u spindle 2>/dev/null) || exit 1 149 - XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 150 - runuser -u spindle -- systemctl --user enable --now podman.socket 151 - XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 152 - runuser -u spindle -- systemctl --user enable --now openbao-proxy.service 153 - XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 154 - runuser -u spindle -- systemctl --user enable --now spindle.service 155 - ' 156 - 157 - update_heartbeat "$NODE_NAME" 158 - update_endpoint "$NODE_NAME" 159 - echo "Spindle started on $NODE_NAME" 160 - restartPolicy: Never 161 - backoffLimit: 0
-43
spindle/ingress.yaml
··· 1 - apiVersion: cert-manager.io/v1 2 - kind: Certificate 3 - metadata: 4 - name: spindle-sans-self-org 5 - spec: 6 - secretName: spindle-sans-self-org-tls 7 - issuerRef: 8 - name: letsencrypt-prod 9 - kind: ClusterIssuer 10 - dnsNames: 11 - - spindle.sans-self.org 12 - --- 13 - apiVersion: v1 14 - kind: Service 15 - metadata: 16 - name: spindle 17 - spec: 18 - ports: 19 - - port: 6555 20 - targetPort: 6555 21 - protocol: TCP 22 - --- 23 - apiVersion: v1 24 - kind: Endpoints 25 - metadata: 26 - name: spindle 27 - subsets: [] 28 - --- 29 - apiVersion: traefik.io/v1alpha1 30 - kind: IngressRoute 31 - metadata: 32 - name: spindle 33 - spec: 34 - entryPoints: 35 - - websecure 36 - tls: 37 - secretName: spindle-sans-self-org-tls 38 - routes: 39 - - match: Host(`spindle.sans-self.org`) 40 - kind: Rule 41 - services: 42 - - name: spindle 43 - port: 6555
-34
spindle/openbao-proxy.hcl
··· 1 - vault { 2 - address = "http://127.0.0.1:30820" 3 - } 4 - 5 - auto_auth { 6 - method "approle" { 7 - mount_path = "auth/approle" 8 - config = { 9 - role_id_file_path = "/home/spindle/.openbao/role-id" 10 - secret_id_file_path = "/home/spindle/.openbao/secret-id" 11 - remove_secret_id_file_after_reading = false 12 - } 13 - } 14 - 15 - sink "file" { 16 - config = { 17 - path = "/home/spindle/.openbao/token" 18 - mode = 0640 19 - } 20 - } 21 - } 22 - 23 - listener "tcp" { 24 - address = "127.0.0.1:8201" 25 - tls_disable = true 26 - } 27 - 28 - api_proxy { 29 - use_auto_auth_token = true 30 - } 31 - 32 - cache { 33 - use_auto_auth_token = true 34 - }
-12
spindle/openbao-proxy.service
··· 1 - [Unit] 2 - Description=OpenBao Proxy (Spindle secrets) 3 - After=podman.socket 4 - 5 - [Service] 6 - Type=simple 7 - ExecStart=/usr/local/bin/bao proxy -config=/home/spindle/.openbao/proxy.hcl 8 - Restart=on-failure 9 - RestartSec=5 10 - 11 - [Install] 12 - WantedBy=default.target
-23
spindle/spindle.service
··· 1 - [Unit] 2 - Description=Spindle CI Runner 3 - After=openbao-proxy.service podman.socket 4 - Requires=openbao-proxy.service 5 - 6 - [Service] 7 - Type=simple 8 - Environment=DOCKER_HOST=unix:///run/user/%U/podman/podman.sock 9 - Environment=SPINDLE_SERVER_LISTEN_ADDR=0.0.0.0:6555 10 - Environment=SPINDLE_SERVER_DB_PATH=/var/lib/spindle/spindle.db 11 - Environment=SPINDLE_SERVER_HOSTNAME=spindle.sans-self.org 12 - Environment=SPINDLE_SERVER_OWNER=did:plc:wydyrngmxbcsqdvhmd7whmye 13 - Environment=SPINDLE_PIPELINES_LOG_DIR=/var/lib/spindle/logs 14 - Environment=SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=10m 15 - Environment=SPINDLE_SERVER_SECRETS_PROVIDER=openbao 16 - Environment=SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=http://127.0.0.1:8201 17 - Environment=SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=spindle 18 - ExecStart=/usr/local/bin/spindle 19 - Restart=on-failure 20 - RestartSec=5 21 - 22 - [Install] 23 - WantedBy=default.target
-35
spindle/start-job.yaml
··· 1 - apiVersion: batch/v1 2 - kind: Job 3 - metadata: 4 - name: spindle-start 5 - spec: 6 - completions: 1 7 - parallelism: 1 8 - template: 9 - metadata: 10 - labels: 11 - app: spindle-start 12 - spec: 13 - hostPID: true 14 - tolerations: 15 - - operator: Exists 16 - containers: 17 - - name: start 18 - image: alpine:3.21 19 - securityContext: 20 - privileged: true 21 - command: ["/bin/sh", "-c"] 22 - args: 23 - - | 24 - apk add --no-cache --quiet util-linux >/dev/null 2>&1 25 - nsenter -t 1 -m -u -i -n -- bash -c ' 26 - uid=$(id -u spindle) 27 - export XDG_RUNTIME_DIR=/run/user/$uid 28 - export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus 29 - runuser -u spindle -- systemctl --user enable --now podman.socket 30 - runuser -u spindle -- systemctl --user enable --now openbao-proxy.service 31 - runuser -u spindle -- systemctl --user enable --now spindle.service 32 - echo "OpenBao proxy + Spindle started" 33 - ' 34 - restartPolicy: Never 35 - backoffLimit: 1
-86
spindle/update-job.yaml
··· 1 - apiVersion: batch/v1 2 - kind: Job 3 - metadata: 4 - name: spindle-update 5 - spec: 6 - completions: 3 7 - parallelism: 3 8 - template: 9 - metadata: 10 - labels: 11 - app: spindle-update 12 - spec: 13 - hostPID: true 14 - affinity: 15 - podAntiAffinity: 16 - requiredDuringSchedulingIgnoredDuringExecution: 17 - - labelSelector: 18 - matchLabels: 19 - app: spindle-update 20 - topologyKey: kubernetes.io/hostname 21 - tolerations: 22 - - operator: Exists 23 - initContainers: 24 - - name: extract 25 - image: zot.sans-self.org/infra/spindle:latest 26 - securityContext: 27 - privileged: true 28 - command: ["/bin/sh", "-c"] 29 - args: 30 - - | 31 - cp /spindle /host-bin/spindle 32 - cp /bao /host-bin/bao 33 - cp /spindle.service /host-service/spindle.service 34 - cp /openbao-proxy.service /host-service/openbao-proxy.service 35 - cp /openbao-proxy.hcl /host-openbao/proxy.hcl 36 - volumeMounts: 37 - - name: host-bin 38 - mountPath: /host-bin 39 - - name: host-service 40 - mountPath: /host-service 41 - - name: host-openbao 42 - mountPath: /host-openbao 43 - containers: 44 - - name: deploy 45 - image: alpine:3.21 46 - securityContext: 47 - privileged: true 48 - command: ["/bin/sh", "-c"] 49 - args: 50 - - | 51 - apk add --no-cache --quiet util-linux >/dev/null 2>&1 52 - nsenter -t 1 -m -u -i -n -- bash -c ' 53 - chmod 755 /usr/local/bin/spindle /usr/local/bin/bao 54 - chown -R spindle:spindle /home/spindle/.config 55 - chown spindle:spindle /home/spindle/.openbao /home/spindle/.openbao/proxy.hcl 56 - mkdir -p /var/lib/spindle/logs && chown -R spindle:spindle /var/lib/spindle 57 - uid=$(id -u spindle 2>/dev/null) || { echo "spindle user missing"; exit 0; } 58 - export XDG_RUNTIME_DIR=/run/user/$uid 59 - export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus 60 - runuser -u spindle -- systemctl --user daemon-reload 61 - if runuser -u spindle -- systemctl --user is-active --quiet openbao-proxy.service 2>/dev/null; then 62 - runuser -u spindle -- systemctl --user restart openbao-proxy.service 63 - echo "OpenBao proxy restarted" 64 - fi 65 - if runuser -u spindle -- systemctl --user is-active --quiet spindle.service 2>/dev/null; then 66 - runuser -u spindle -- systemctl --user restart spindle.service 67 - echo "Spindle restarted" 68 - else 69 - echo "Spindle not active on this node, skipping" 70 - fi 71 - ' 72 - volumes: 73 - - name: host-bin 74 - hostPath: 75 - path: /usr/local/bin 76 - type: Directory 77 - - name: host-service 78 - hostPath: 79 - path: /home/spindle/.config/systemd/user 80 - type: DirectoryOrCreate 81 - - name: host-openbao 82 - hostPath: 83 - path: /home/spindle/.openbao 84 - type: DirectoryOrCreate 85 - restartPolicy: Never 86 - backoffLimit: 3