my prefect server setup
prefect-metrics.waow.tech
python
orchestration
1# my-prefect-server
2# required env vars: HCLOUD_TOKEN, POSTGRES_PASSWORD, AUTH_STRING, DOMAIN, LETSENCRYPT_EMAIL
3# optional env vars: GRAFANA_DOMAIN (default: prefect-metrics.waow.tech)
4
5set dotenv-load
6
7export KUBECONFIG := source_directory() / "kubeconfig.yaml"
8
9# --- dev ---
10
11# sync workspace (all members)
12sync:
13 uv sync
14
15# run a prefect CLI command against the remote server
16prefect *args:
17 PREFECT_API_URL="https://$DOMAIN/api" PREFECT_API_AUTH_STRING="$AUTH_STRING" \
18 uv run --with prefect prefect {{args}}
19
20# --- infrastructure ---
21
22# initialize terraform
23init:
24 terraform -chdir=infra init
25
26# create the hetzner server with k3s
27infra:
28 terraform -chdir=infra apply -var="hcloud_token=$HCLOUD_TOKEN"
29
30# destroy all infrastructure
31destroy:
32 terraform -chdir=infra destroy -var="hcloud_token=$HCLOUD_TOKEN"
33
34# get the server IP from terraform
35server-ip:
36 @terraform -chdir=infra output -raw server_ip
37
38# ssh into the server
39ssh:
40 ssh root@$(just server-ip)
41
42# fetch kubeconfig from the server (run after cloud-init finishes)
43kubeconfig:
44 #!/usr/bin/env bash
45 set -euo pipefail
46 IP=$(just server-ip)
47 echo "fetching kubeconfig from $IP..."
48 until ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new root@$IP test -f /run/k3s-ready 2>/dev/null; do
49 echo " waiting for k3s..."
50 sleep 5
51 done
52 scp root@$IP:/etc/rancher/k3s/k3s.yaml kubeconfig.yaml
53 if [[ "$(uname)" == "Darwin" ]]; then
54 sed -i '' "s|127.0.0.1|$IP|g" kubeconfig.yaml
55 else
56 sed -i "s|127.0.0.1|$IP|g" kubeconfig.yaml
57 fi
58 chmod 600 kubeconfig.yaml
59 echo "kubeconfig written"
60 kubectl get nodes
61
62# --- cluster ---
63
64# deploy everything to the cluster (idempotent)
65deploy:
66 #!/usr/bin/env bash
67 set -euo pipefail
68
69 helm repo add prefect https://prefecthq.github.io/prefect-helm
70 helm repo add jetstack https://charts.jetstack.io
71 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
72 helm repo update
73
74 : "${DOMAIN:?set DOMAIN}"
75 : "${AUTH_STRING:?set AUTH_STRING}"
76 : "${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}"
77 : "${LETSENCRYPT_EMAIL:?set LETSENCRYPT_EMAIL}"
78 GRAFANA_DOMAIN="${GRAFANA_DOMAIN:-prefect-metrics.waow.tech}"
79
80 echo "==> creating namespaces"
81 kubectl create namespace prefect --dry-run=client -o yaml | kubectl apply -f -
82 kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
83
84 echo "==> installing cert-manager"
85 helm upgrade --install cert-manager jetstack/cert-manager \
86 --namespace cert-manager --create-namespace \
87 --set crds.enabled=true \
88 --wait
89
90 echo "==> applying cluster issuer"
91 sed "s|LETSENCRYPT_EMAIL_PLACEHOLDER|$LETSENCRYPT_EMAIL|g" deploy/cluster-issuer.yaml \
92 | kubectl apply -f -
93
94 echo "==> creating prefect auth secret"
95 kubectl create secret generic prefect-auth \
96 --namespace prefect \
97 --from-literal=auth-string="$AUTH_STRING" \
98 --dry-run=client -o yaml | kubectl apply -f -
99
100 echo "==> installing prefect server"
101 sed "s|DOMAIN_PLACEHOLDER|$DOMAIN|g" deploy/prefect-values.yaml \
102 | helm upgrade --install prefect-server prefect/prefect-server \
103 --namespace prefect \
104 --values - \
105 --set postgresql.auth.password="$POSTGRES_PASSWORD" \
106 --wait --timeout 5m
107
108 echo "==> installing monitoring stack"
109 sed "s|GRAFANA_DOMAIN_PLACEHOLDER|$GRAFANA_DOMAIN|g" deploy/monitoring-values.yaml \
110 | helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
111 --namespace monitoring \
112 --values - \
113 --wait --timeout 5m
114
115 echo "==> applying grafana ingress"
116 sed "s|GRAFANA_DOMAIN_PLACEHOLDER|$GRAFANA_DOMAIN|g" deploy/grafana-ingress.yaml \
117 | kubectl apply -f -
118
119 echo "==> loading prefect dashboards"
120 for dashboard in deploy/dashboards/*.json; do
121 name=$(basename "$dashboard" .json | tr '.' '-')
122 kubectl create configmap "prefect-dashboard-$name" \
123 --namespace monitoring \
124 --from-file="$dashboard" \
125 --dry-run=client -o yaml \
126 | kubectl label --local -f - grafana_dashboard=1 -o yaml \
127 | kubectl apply -f -
128 done
129
130 echo "==> installing prefect exporter"
131 helm upgrade --install prometheus-prefect-exporter prefect/prometheus-prefect-exporter \
132 --namespace prefect \
133 --values deploy/exporter-values.yaml \
134 --wait --timeout 2m
135
136 echo ""
137 echo "done. point DNS:"
138 echo " $DOMAIN -> $(just server-ip)"
139 echo " $GRAFANA_DOMAIN -> $(just server-ip)"
140
141# apply the kubernetes worker
142worker:
143 kubectl apply -f deploy/worker.yaml
144
145# create the analytics hostPath + results PVC and patch the work pool
146storage: _analytics-dir
147 #!/usr/bin/env bash
148 set -euo pipefail
149 : "${DOMAIN:?set DOMAIN}"
150 : "${AUTH_STRING:?set AUTH_STRING}"
151 echo "==> creating results PVC"
152 kubectl apply -f deploy/results-pvc.yaml
153 echo "==> patching kubernetes-pool base job template"
154 PREFECT_API_URL="https://$DOMAIN/api" PREFECT_API_AUTH_STRING="$AUTH_STRING" \
155 uv run --with prefect python scripts/patch_work_pool.py
156
157_analytics-dir:
158 ssh root@$(just server-ip) "mkdir -p /var/lib/prefect-analytics"
159
160# --- operations ---
161
162# check cluster state
163status:
164 @echo "==> nodes"
165 @kubectl top nodes
166 @echo ""
167 @echo "==> pods (by memory)"
168 @kubectl top pods --all-namespaces --sort-by=memory
169 @echo ""
170 @echo "==> pods (prefect)"
171 @kubectl get pods -n prefect
172 @echo ""
173 @echo "==> pods (monitoring)"
174 @kubectl get pods -n monitoring
175
176# tail logs for a component (server, background-services, worker)
177logs component="prefect-server":
178 kubectl logs -n prefect -l app.kubernetes.io/name={{component}} -f
179
180# check prefect API health
181health:
182 #!/usr/bin/env bash
183 : "${DOMAIN:?set DOMAIN}"
184 curl -sf "https://$DOMAIN/api/health" | jq .
185
186# reload grafana dashboards from deploy/dashboards/
187dashboards:
188 #!/usr/bin/env bash
189 set -euo pipefail
190 for dashboard in deploy/dashboards/*.json; do
191 name=$(basename "$dashboard" .json | tr '.' '-')
192 kubectl create configmap "prefect-dashboard-$name" \
193 --namespace monitoring \
194 --from-file="$dashboard" \
195 --dry-run=client -o yaml \
196 | kubectl label --local -f - grafana_dashboard=1 -o yaml \
197 | kubectl apply -f -
198 echo " loaded $name"
199 done
200
201# --- analytics ---
202
203# first-time dbt setup: install deps, seed reference data, compile models
204init-analytics:
205 cd analytics && uv run dbt deps && uv run dbt seed && uv run dbt compile
206
207# --- hub ---
208
209# build the hub container image (linux/amd64 for hetzner k3s node)
210build-web:
211 docker build --platform linux/amd64 -t atcr.io/zzstoatzz.io/hub:latest web/
212
213# build and push the hub image
214push-web: build-web
215 docker push atcr.io/zzstoatzz.io/hub:latest
216
217# apply hub k8s manifests and restart the pod to pull the new image
218deploy-web:
219 kubectl apply -f deploy/hub-deployment.yaml
220 sed "s|HUB_DOMAIN_PLACEHOLDER|hub.waow.tech|g" deploy/hub-ingress.yaml | kubectl apply -f -
221 kubectl rollout restart deployment/hub -n prefect
222
223# build, push, and deploy hub
224web: push-web deploy-web