this repo has no description
POC: K8s-based Tangled jobs
+249
-215
+46
.tangled/workflows/notify.yml
+46
.tangled/workflows/notify.yml
···
1
+
when:
2
+
- event: ["push"]
3
+
branch: ["main"]
4
+
5
+
engine: "kubernetes"
6
+
7
+
job:
8
+
apiVersion: batch/v1
9
+
kind: Job
10
+
spec:
11
+
backoffLimit: 0
12
+
template:
13
+
spec:
14
+
restartPolicy: Never
15
+
volumes:
16
+
- name: secrets
17
+
secret:
18
+
secretName: telegram-credentials
19
+
containers:
20
+
- name: notify
21
+
image: busybox:1
22
+
volumeMounts:
23
+
- name: secrets
24
+
mountPath: /secrets
25
+
readOnly: true
26
+
command: ["sh", "-c"]
27
+
args:
28
+
- |
29
+
BOT_TOKEN=$(tr -d '[:space:]' < /secrets/bot-token)
30
+
CHAT_ID=$(tr -d '[:space:]' < /secrets/chat-id)
31
+
32
+
[ "$CI_REF_TYPE" = "tag" ] \
33
+
&& ref_label="Tag: \`$CI_REF_NAME\`" \
34
+
|| ref_label="Branch: \`$CI_REF_NAME\`"
35
+
36
+
text="*Push to $CI_REPO_NAME*
37
+
$ref_label
38
+
39
+
\`$CI_SHORT_SHA\`"
40
+
41
+
json_text=$(printf '%s' "$text" | sed 's/\\/\\\\/g; s/"/\\"/g' | awk 'NR>1{printf "\\n"}{printf "%s",$0}')
42
+
43
+
wget -q -O /dev/null \
44
+
--header='Content-Type: application/json' \
45
+
--post-data="{\"chat_id\":\"$CHAT_ID\",\"text\":\"$json_text\",\"parse_mode\":\"Markdown\"}" \
46
+
"https://api.telegram.org/bot${BOT_TOKEN}/sendMessage"
-20
k8s/ci/ci-hook.sh
-20
k8s/ci/ci-hook.sh
···
1
-
#!/bin/sh
2
-
# Post-receive CI hook — notifies push-webhook service on every push.
3
-
# Installed by the ci-hook-installer sidecar.
4
-
5
-
WEBHOOK_URL="http://push-webhook.ci.svc.cluster.local:3000/cgi-bin/webhook"
6
-
REPO_PATH="${GIT_DIR#/home/git/repositories/}"
7
-
8
-
while read -r oldrev newrev refname; do
9
-
author=$(git log -1 --format='%an' "$newrev" 2>/dev/null || echo '')
10
-
message=$(git log -1 --format='%s' "$newrev" 2>/dev/null || echo '')
11
-
12
-
wget -q -O /dev/null --post-data \
13
-
"repo=${REPO_PATH}
14
-
ref=${refname}
15
-
before=${oldrev}
16
-
after=${newrev}
17
-
author=${author}
18
-
message=${message}" \
19
-
"$WEBHOOK_URL" 2>/dev/null || true
20
-
done
+27
-37
k8s/ci/deployment.yaml
+27
-37
k8s/ci/deployment.yaml
···
1
1
apiVersion: apps/v1
2
2
kind: Deployment
3
3
metadata:
4
-
name: push-webhook
4
+
name: ci-listener
5
5
namespace: ci
6
6
spec:
7
7
replicas: 1
8
8
selector:
9
9
matchLabels:
10
-
app: push-webhook
10
+
app: ci-listener
11
11
template:
12
12
metadata:
13
13
labels:
14
-
app: push-webhook
14
+
app: ci-listener
15
15
spec:
16
+
serviceAccountName: ci-runner
17
+
initContainers:
18
+
- name: install-deps
19
+
image: python:3-alpine
20
+
command: ["pip", "install", "--target=/deps", "websockets", "pyyaml"]
21
+
volumeMounts:
22
+
- name: deps
23
+
mountPath: /deps
16
24
containers:
17
-
- name: webhook
18
-
image: busybox:1
19
-
command: ["httpd", "-f", "-p", "3000", "-h", "/app"]
20
-
ports:
21
-
- name: http
22
-
containerPort: 3000
25
+
- name: listener
26
+
image: python:3-alpine
27
+
command: ["python3", "/app/listener.py"]
23
28
env:
24
-
- name: WATCH_REPOS
25
-
value: "infrastructure"
26
-
- name: WATCH_BRANCH
27
-
value: "main"
29
+
- name: PYTHONPATH
30
+
value: /deps
28
31
volumeMounts:
29
-
- name: cgi-bin
30
-
mountPath: /app/cgi-bin
31
-
- name: secrets
32
-
mountPath: /secrets
32
+
- name: script
33
+
mountPath: /app
33
34
readOnly: true
34
-
livenessProbe:
35
-
httpGet:
36
-
path: /cgi-bin/health
37
-
port: 3000
38
-
initialDelaySeconds: 3
39
-
periodSeconds: 30
40
-
readinessProbe:
41
-
httpGet:
42
-
path: /cgi-bin/health
43
-
port: 3000
44
-
initialDelaySeconds: 2
45
-
periodSeconds: 10
35
+
- name: deps
36
+
mountPath: /deps
37
+
readOnly: true
46
38
securityContext:
47
39
runAsNonRoot: true
48
40
runAsUser: 1000
···
54
46
resources:
55
47
requests:
56
48
cpu: 10m
57
-
memory: 16Mi
49
+
memory: 32Mi
58
50
limits:
59
51
cpu: 100m
60
-
memory: 32Mi
52
+
memory: 64Mi
61
53
volumes:
62
-
- name: cgi-bin
54
+
- name: script
63
55
configMap:
64
-
name: webhook-script
65
-
defaultMode: 0755
66
-
- name: secrets
67
-
secret:
68
-
secretName: telegram-credentials
56
+
name: ci-listener-script
57
+
- name: deps
58
+
emptyDir: {}
+3
-9
k8s/ci/kustomization.yaml
+3
-9
k8s/ci/kustomization.yaml
···
7
7
resources:
8
8
- namespace.yaml
9
9
- deployment.yaml
10
-
- service.yaml
11
-
- network-policy.yaml
10
+
- rbac.yaml
12
11
13
12
configMapGenerator:
14
-
- name: webhook-script
13
+
- name: ci-listener-script
15
14
namespace: ci
16
15
files:
17
-
- webhook=webhook.sh
18
-
- health=health.sh
19
-
- name: ci-hooks
20
-
namespace: knot
21
-
files:
22
-
- ci-hook.sh
16
+
- listener.py
+137
k8s/ci/listener.py
+137
k8s/ci/listener.py
···
1
+
import asyncio
2
+
import json
3
+
import os
4
+
import ssl
5
+
import sys
6
+
import urllib.request
7
+
8
+
import websockets
9
+
import yaml
10
+
11
+
KNOT_WS_URL = os.environ.get("KNOT_WS_URL", "ws://knot.knot.svc.cluster.local:5555/events")
12
+
NAMESPACE = "ci"
13
+
14
+
K8S_API = "https://kubernetes.default.svc"
15
+
TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token"
16
+
CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
17
+
18
+
19
+
def log(msg):
20
+
print(msg, flush=True)
21
+
22
+
23
+
def k8s_ssl():
24
+
return ssl.create_default_context(cafile=CA_PATH)
25
+
26
+
27
+
def k8s_token():
28
+
with open(TOKEN_PATH) as f:
29
+
return f.read().strip()
30
+
31
+
32
+
def create_job(job_spec):
33
+
url = f"{K8S_API}/apis/batch/v1/namespaces/{NAMESPACE}/jobs"
34
+
data = json.dumps(job_spec).encode()
35
+
req = urllib.request.Request(url, data=data, method="POST")
36
+
req.add_header("Authorization", f"Bearer {k8s_token()}")
37
+
req.add_header("Content-Type", "application/json")
38
+
39
+
try:
40
+
resp = urllib.request.urlopen(req, context=k8s_ssl())
41
+
result = json.loads(resp.read())
42
+
log(f" created job: {result['metadata']['name']}")
43
+
except urllib.error.HTTPError as e:
44
+
body = e.read().decode()
45
+
log(f" k8s API error {e.code}: {body}")
46
+
47
+
48
+
def inject_env(job_spec, env_vars):
49
+
env_list = [{"name": k, "value": v} for k, v in env_vars.items()]
50
+
containers = job_spec.get("spec", {}).get("template", {}).get("spec", {}).get("containers", [])
51
+
for container in containers:
52
+
container["env"] = container.get("env", []) + env_list
53
+
54
+
55
+
def handle_pipeline(event):
56
+
trigger = event.get("triggerMetadata", {})
57
+
push = trigger.get("push", {})
58
+
repo = trigger.get("repo", {})
59
+
60
+
ref = push.get("ref", "")
61
+
new_sha = push.get("newSha", "")
62
+
63
+
ref_name = ref.removeprefix("refs/heads/").removeprefix("refs/tags/")
64
+
ref_type = "tag" if ref.startswith("refs/tags/") else "branch"
65
+
66
+
env_vars = {
67
+
"CI_REPO_NAME": repo.get("repo", ""),
68
+
"CI_REPO_DID": repo.get("did", ""),
69
+
"CI_KNOT": repo.get("knot", ""),
70
+
"CI_REF": ref,
71
+
"CI_REF_NAME": ref_name,
72
+
"CI_REF_TYPE": ref_type,
73
+
"CI_NEW_SHA": new_sha,
74
+
"CI_OLD_SHA": push.get("oldSha", ""),
75
+
"CI_SHORT_SHA": new_sha[:7],
76
+
}
77
+
78
+
for wf in event.get("workflows", []):
79
+
name = wf.get("name", "unknown")
80
+
raw = wf.get("raw", "")
81
+
82
+
try:
83
+
parsed = yaml.safe_load(raw)
84
+
except yaml.YAMLError as e:
85
+
log(f" {name}: bad yaml: {e}")
86
+
continue
87
+
88
+
if parsed.get("engine") != "kubernetes":
89
+
continue
90
+
91
+
job_spec = parsed.get("job")
92
+
if not job_spec:
93
+
log(f" {name}: engine=kubernetes but no 'job' field")
94
+
continue
95
+
96
+
repo_name = repo.get("repo", "unknown")
97
+
job_spec.setdefault("metadata", {})
98
+
job_spec["metadata"]["generateName"] = f"ci-{repo_name}-"
99
+
job_spec["metadata"]["namespace"] = NAMESPACE
100
+
job_spec["metadata"].setdefault("labels", {})
101
+
job_spec["metadata"]["labels"]["ci.sans-self.org/workflow"] = name
102
+
job_spec["metadata"]["labels"]["ci.sans-self.org/repo"] = repo_name[:63]
103
+
104
+
inject_env(job_spec, env_vars)
105
+
106
+
log(f" {name}: creating job")
107
+
create_job(job_spec)
108
+
109
+
110
+
async def listen():
111
+
while True:
112
+
try:
113
+
log(f"connecting to {KNOT_WS_URL}")
114
+
async with websockets.connect(KNOT_WS_URL) as ws:
115
+
log("connected, listening for events")
116
+
async for msg in ws:
117
+
data = json.loads(msg)
118
+
nsid = data.get("nsid", "")
119
+
rkey = data.get("rkey", "")
120
+
event = data.get("event", {})
121
+
122
+
log(f"event: {nsid} rkey={rkey}")
123
+
124
+
if nsid == "sh.tangled.pipeline":
125
+
handle_pipeline(event)
126
+
127
+
except websockets.ConnectionClosed:
128
+
log("connection closed, reconnecting in 5s")
129
+
except Exception as e:
130
+
log(f"error: {e}, reconnecting in 5s")
131
+
132
+
await asyncio.sleep(5)
133
+
134
+
135
+
if __name__ == "__main__":
136
+
log("ci listener starting")
137
+
asyncio.run(listen())
-18
k8s/ci/network-policy.yaml
-18
k8s/ci/network-policy.yaml
···
1
-
apiVersion: networking.k8s.io/v1
2
-
kind: NetworkPolicy
3
-
metadata:
4
-
name: push-webhook-ingress
5
-
namespace: ci
6
-
spec:
7
-
podSelector:
8
-
matchLabels:
9
-
app: push-webhook
10
-
policyTypes:
11
-
- Ingress
12
-
ingress:
13
-
- from:
14
-
- namespaceSelector:
15
-
matchLabels:
16
-
kubernetes.io/metadata.name: knot
17
-
ports:
18
-
- port: 3000
+29
k8s/ci/rbac.yaml
+29
k8s/ci/rbac.yaml
···
1
+
apiVersion: v1
2
+
kind: ServiceAccount
3
+
metadata:
4
+
name: ci-runner
5
+
namespace: ci
6
+
---
7
+
apiVersion: rbac.authorization.k8s.io/v1
8
+
kind: Role
9
+
metadata:
10
+
name: ci-runner
11
+
namespace: ci
12
+
rules:
13
+
- apiGroups: ["batch"]
14
+
resources: ["jobs"]
15
+
verbs: ["create", "get", "list", "delete"]
16
+
---
17
+
apiVersion: rbac.authorization.k8s.io/v1
18
+
kind: RoleBinding
19
+
metadata:
20
+
name: ci-runner
21
+
namespace: ci
22
+
subjects:
23
+
- kind: ServiceAccount
24
+
name: ci-runner
25
+
namespace: ci
26
+
roleRef:
27
+
kind: Role
28
+
name: ci-runner
29
+
apiGroup: rbac.authorization.k8s.io
-12
k8s/ci/service.yaml
-12
k8s/ci/service.yaml
-74
k8s/ci/webhook.sh
-74
k8s/ci/webhook.sh
···
1
-
#!/bin/sh
2
-
# CGI handler — receives push notifications from ci-hook.sh
3
-
4
-
body=$(head -c "${CONTENT_LENGTH:-0}")
5
-
printf '%s\n' "$body" >&2
6
-
7
-
field() { printf '%s\n' "$body" | sed -n "s/^$1=//p" | head -1; }
8
-
9
-
repo=$(field repo)
10
-
ref=$(field ref)
11
-
after=$(field after)
12
-
author=$(field author)
13
-
message=$(field message)
14
-
15
-
case "$ref" in
16
-
refs/tags/*) ref_type=tag; ref_name="${ref#refs/tags/}" ;;
17
-
refs/heads/*) ref_type=branch; ref_name="${ref#refs/heads/}" ;;
18
-
*) ref_type=ref; ref_name="$ref" ;;
19
-
esac
20
-
21
-
repo_name="${repo##*/}"
22
-
23
-
printf '%s %s=%s by %s\n' "$repo" "$ref_type" "$ref_name" "${author:-unknown}" >&2
24
-
25
-
# Filtering
26
-
watched_repo=false
27
-
if [ -z "$WATCH_REPOS" ]; then
28
-
watched_repo=true
29
-
else
30
-
for r in $(printf '%s' "$WATCH_REPOS" | tr ',' ' '); do
31
-
[ "$r" = "$repo" ] || [ "$r" = "$repo_name" ] && watched_repo=true
32
-
done
33
-
fi
34
-
35
-
watched_ref=false
36
-
[ "$ref_type" = "tag" ] || [ "$ref_name" = "${WATCH_BRANCH:-main}" ] && watched_ref=true
37
-
38
-
printf 'Content-Type: text/plain\r\n\r\n'
39
-
40
-
if [ "$watched_repo" = false ] || [ "$watched_ref" = false ]; then
41
-
echo "ok (not watched)"
42
-
exit 0
43
-
fi
44
-
45
-
# Build notification
46
-
[ "$ref_type" = "tag" ] \
47
-
&& ref_label="Tag: \`$ref_name\`" \
48
-
|| ref_label="Branch: \`$ref_name\`"
49
-
50
-
short_sha=$(printf '%.7s' "$after")
51
-
[ -n "$message" ] \
52
-
&& commit_line="\`$short_sha\` $message" \
53
-
|| commit_line="\`$short_sha\`"
54
-
55
-
text="*Push to $repo_name*
56
-
$ref_label"
57
-
[ -n "$author" ] && text="$text
58
-
By: $author"
59
-
text="$text
60
-
61
-
$commit_line"
62
-
63
-
# JSON-escape and collapse newlines
64
-
json_text=$(printf '%s' "$text" | sed 's/\\/\\\\/g; s/"/\\"/g' | awk 'NR>1{printf "\\n"}{printf "%s",$0}')
65
-
66
-
BOT_TOKEN=$(tr -d '[:space:]' < /secrets/bot-token)
67
-
CHAT_ID=$(tr -d '[:space:]' < /secrets/chat-id)
68
-
69
-
wget -q -O /dev/null \
70
-
--header='Content-Type: application/json' \
71
-
--post-data="{\"chat_id\":\"$CHAT_ID\",\"text\":\"$json_text\",\"parse_mode\":\"Markdown\"}" \
72
-
"https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" 2>&1 | cat >&2
73
-
74
-
echo "ok (notified)"
-43
k8s/knot/deployment.yaml
-43
k8s/knot/deployment.yaml
···
77
77
limits:
78
78
cpu: 500m
79
79
memory: 512Mi
80
-
- name: ci-hook-installer
81
-
image: busybox:1
82
-
command: ["sh", "-c"]
83
-
args:
84
-
- |
85
-
REPO_ROOT=/repos
86
-
HOOK_SRC=/ci-hooks/ci-hook.sh
87
-
88
-
while true; do
89
-
for hookdir in "$REPO_ROOT"/*/*/hooks; do
90
-
[ -d "$hookdir/post-receive.d" ] || continue
91
-
dest="$hookdir/post-receive.d/50-ci-notify"
92
-
if [ ! -f "$dest" ] || ! cmp -s "$HOOK_SRC" "$dest"; then
93
-
cp "$HOOK_SRC" "$dest" && chmod +x "$dest" && echo "installed: $dest"
94
-
fi
95
-
done
96
-
sleep 30
97
-
done
98
-
volumeMounts:
99
-
- name: data
100
-
mountPath: /repos
101
-
subPath: repositories
102
-
- name: ci-hooks
103
-
mountPath: /ci-hooks
104
-
readOnly: true
105
-
securityContext:
106
-
runAsUser: 1000
107
-
runAsGroup: 1000
108
-
readOnlyRootFilesystem: true
109
-
allowPrivilegeEscalation: false
110
-
capabilities:
111
-
drop:
112
-
- ALL
113
-
resources:
114
-
requests:
115
-
cpu: 10m
116
-
memory: 16Mi
117
-
limits:
118
-
cpu: 50m
119
-
memory: 32Mi
120
80
volumes:
121
81
- name: data
122
82
persistentVolumeClaim:
···
124
84
- name: sshd-hardening
125
85
configMap:
126
86
name: sshd-hardening
127
-
- name: ci-hooks
128
-
configMap:
129
-
name: ci-hooks
+7
k8s/knot/network-policy.yaml
+7
k8s/knot/network-policy.yaml
···
32
32
ports:
33
33
- port: 5555
34
34
- port: 5444
35
+
# CI listener subscribes to the event stream via WebSocket
36
+
- from:
37
+
- namespaceSelector:
38
+
matchLabels:
39
+
kubernetes.io/metadata.name: ci
40
+
ports:
41
+
- port: 5555
35
42
---
36
43
apiVersion: networking.k8s.io/v1
37
44
kind: NetworkPolicy