my prefect server setup
prefect-metrics.waow.tech
python
orchestration
1name: my-prefect-server
2
3pull:
4 - prefect.deployments.steps.run_shell_script:
5 script: |
6 bash -c "rm -rf my-prefect-server && (git clone --depth 1 --branch main https://tangled.sh/zzstoatzz.io/my-prefect-server.git my-prefect-server || git clone --depth 1 --branch main https://github.com/zzstoatzz/my-prefect-server.git my-prefect-server)"
7 - prefect.deployments.steps.set_working_directory:
8 directory: my-prefect-server
9
10definitions:
11 work_pools:
12 k8s: &k8s
13 name: kubernetes-pool
14 job_variables:
15 command: >-
16 uv run
17 --with 'my-prefect-server @ git+https://github.com/zzstoatzz/my-prefect-server.git'
18 prefect flow-run execute
19
20deployments:
21 - name: diagnostics
22 entrypoint: flows/diagnostics.py:diagnostics
23 work_pool: *k8s
24 schedules:
25 - cron: "*/5 * * * *"
26 active: false
27
28 - name: ingest
29 entrypoint: flows/ingest.py:ingest
30 work_pool: *k8s
31 schedules:
32 - cron: "0 * * * *" # hourly
33 parameters:
34 only_unread: true
35
36 - name: transform
37 entrypoint: flows/transform.py:transform
38 work_pool:
39 name: kubernetes-pool
40 job_variables:
41 command: >-
42 uv run --python 3.13
43 --with 'my-prefect-server @ git+https://github.com/zzstoatzz/my-prefect-server.git'
44 prefect flow-run execute
45 concurrency_limit: 1
46 triggers:
47 - type: event
48 expect:
49 - "prefect.flow-run.Completed"
50 match_related:
51 prefect.resource.name: "ingest"
52 prefect.resource.role: "deployment"
53
54 - name: cleanup
55 entrypoint: flows/cleanup.py:cleanup
56 work_pool: *k8s
57 schedules:
58 - cron: "0 2 * * 0"
59 parameters:
60 config:
61 days_to_keep: 30
62 states_to_clean: ["Completed", "Failed", "Cancelled", "Crashed"]
63 batch_size: 100
64 rate_limit_delay: 0.5
65 dry_run: false
66
67 - name: brief
68 entrypoint: flows/brief.py:brief
69 work_pool: *k8s
70 triggers:
71 - type: event
72 expect:
73 - "prefect.flow-run.Completed"
74 match_related:
75 prefect.resource.name: "transform"
76 prefect.resource.role: "deployment"
77
78 - name: compact
79 entrypoint: flows/compact.py:compact
80 work_pool: *k8s
81 triggers:
82 - type: event
83 expect:
84 - "prefect.flow-run.Completed"
85 match_related:
86 prefect.resource.name: "transform"
87 prefect.resource.role: "deployment"
88
89 - name: morning
90 entrypoint: flows/morning.py:morning
91 work_pool: *k8s
92 schedules:
93 - cron: "0 13 * * *" # 8am CT daily
94
95 - name: curate
96 entrypoint: flows/curate.py:curate
97 work_pool: *k8s
98 triggers:
99 - type: event
100 expect:
101 - "prefect.flow-run.Completed"
102 match_related:
103 prefect.resource.name: "morning"
104 prefect.resource.role: "deployment"
105
106 - name: rebuild-atlas
107 entrypoint: flows/atlas.py:rebuild_atlas
108 work_pool: *k8s
109 schedules:
110 - cron: "0 */6 * * *" # every 6 hours
111
112 - name: pds-records
113 entrypoint: flows/pds_records.py:pds_records
114 work_pool: *k8s
115 parameters:
116 config:
117 action: list
118 collection: network.cosmik.connection
119 dry_run: true