my prefect server setup
prefect-metrics.waow.tech
python
orchestration
initial setup: k3s + prefect server + monitoring + kubernetes worker
hetzner VM with k3s, prefect server (2 replicas + background services),
postgresql, redis, kube-prometheus-stack, and a kubernetes work pool
for isolated flow runs.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+7045
+6
.env.example
+6
.env.example
+6
.gitignore
+6
.gitignore
+14
deploy/cluster-issuer.yaml
+14
deploy/cluster-issuer.yaml
···
1
+
apiVersion: cert-manager.io/v1
2
+
kind: ClusterIssuer
3
+
metadata:
4
+
name: letsencrypt-prod
5
+
spec:
6
+
acme:
7
+
server: https://acme-v02.api.letsencrypt.org/directory
8
+
email: LETSENCRYPT_EMAIL_PLACEHOLDER
9
+
privateKeySecretRef:
10
+
name: letsencrypt-prod
11
+
solvers:
12
+
- http01:
13
+
ingress:
14
+
class: traefik
+2241
deploy/dashboards/prefect-flow-run-overview.json
+2241
deploy/dashboards/prefect-flow-run-overview.json
···
1
+
{
2
+
"__inputs": [
3
+
{
4
+
"name": "${DS_PROMETHEUS}",
5
+
"label": "Prometheus",
6
+
"description": "",
7
+
"type": "datasource",
8
+
"pluginId": "prometheus",
9
+
"pluginName": "Prometheus"
10
+
},
11
+
{
12
+
"name": "DS_LOKI",
13
+
"label": "Loki",
14
+
"description": "",
15
+
"type": "datasource",
16
+
"pluginId": "loki",
17
+
"pluginName": "Loki"
18
+
}
19
+
],
20
+
"__elements": {},
21
+
"__requires": [
22
+
{
23
+
"type": "grafana",
24
+
"id": "grafana",
25
+
"name": "Grafana",
26
+
"version": "10.1.5"
27
+
},
28
+
{
29
+
"type": "panel",
30
+
"id": "logs",
31
+
"name": "Logs",
32
+
"version": ""
33
+
},
34
+
{
35
+
"type": "datasource",
36
+
"id": "loki",
37
+
"name": "Loki",
38
+
"version": "1.0.0"
39
+
},
40
+
{
41
+
"type": "panel",
42
+
"id": "piechart",
43
+
"name": "Pie chart",
44
+
"version": ""
45
+
},
46
+
{
47
+
"type": "datasource",
48
+
"id": "prometheus",
49
+
"name": "Prometheus",
50
+
"version": "1.0.0"
51
+
},
52
+
{
53
+
"type": "panel",
54
+
"id": "stat",
55
+
"name": "Stat",
56
+
"version": ""
57
+
},
58
+
{
59
+
"type": "panel",
60
+
"id": "table",
61
+
"name": "Table",
62
+
"version": ""
63
+
},
64
+
{
65
+
"type": "panel",
66
+
"id": "timeseries",
67
+
"name": "Time series",
68
+
"version": ""
69
+
}
70
+
],
71
+
"annotations": {
72
+
"list": [
73
+
{
74
+
"builtIn": 1,
75
+
"datasource": {
76
+
"type": "grafana",
77
+
"uid": "-- Grafana --"
78
+
},
79
+
"enable": true,
80
+
"hide": true,
81
+
"iconColor": "rgba(0, 211, 255, 1)",
82
+
"name": "Annotations & Alerts",
83
+
"type": "dashboard"
84
+
}
85
+
]
86
+
},
87
+
"description": "Dashboard for Prefect Exporter chart",
88
+
"editable": true,
89
+
"fiscalYearStartMonth": 0,
90
+
"graphTooltip": 0,
91
+
"id": null,
92
+
"links": [],
93
+
"liveNow": false,
94
+
"panels": [
95
+
{
96
+
"gridPos": {
97
+
"h": 1,
98
+
"w": 24,
99
+
"x": 0,
100
+
"y": 0
101
+
},
102
+
"id": 45,
103
+
"title": "Overview",
104
+
"type": "row"
105
+
},
106
+
{
107
+
"datasource": {
108
+
"type": "prometheus",
109
+
"uid": "${DS_PROMETHEUS}"
110
+
},
111
+
"fieldConfig": {
112
+
"defaults": {
113
+
"color": {
114
+
"mode": "thresholds"
115
+
},
116
+
"decimals": 0,
117
+
"mappings": [],
118
+
"thresholds": {
119
+
"mode": "absolute",
120
+
"steps": [
121
+
{
122
+
"color": "green",
123
+
"value": null
124
+
}
125
+
]
126
+
},
127
+
"unit": "none"
128
+
},
129
+
"overrides": []
130
+
},
131
+
"gridPos": {
132
+
"h": 5,
133
+
"w": 3,
134
+
"x": 0,
135
+
"y": 1
136
+
},
137
+
"id": 2,
138
+
"options": {
139
+
"colorMode": "value",
140
+
"graphMode": "none",
141
+
"justifyMode": "center",
142
+
"orientation": "auto",
143
+
"reduceOptions": {
144
+
"calcs": [
145
+
"lastNotNull"
146
+
],
147
+
"fields": "",
148
+
"values": false
149
+
},
150
+
"textMode": "auto"
151
+
},
152
+
"pluginVersion": "10.1.5",
153
+
"targets": [
154
+
{
155
+
"datasource": {
156
+
"type": "prometheus",
157
+
"uid": "${DS_PROMETHEUS}"
158
+
},
159
+
"editorMode": "code",
160
+
"expr": "sum(prefect_flows_total)",
161
+
"instant": false,
162
+
"legendFormat": "__auto",
163
+
"range": true,
164
+
"refId": "A"
165
+
}
166
+
],
167
+
"title": "Flows total",
168
+
"type": "stat"
169
+
},
170
+
{
171
+
"datasource": {
172
+
"type": "prometheus",
173
+
"uid": "${DS_PROMETHEUS}"
174
+
},
175
+
"fieldConfig": {
176
+
"defaults": {
177
+
"color": {
178
+
"mode": "thresholds"
179
+
},
180
+
"decimals": 0,
181
+
"mappings": [],
182
+
"thresholds": {
183
+
"mode": "absolute",
184
+
"steps": [
185
+
{
186
+
"color": "green",
187
+
"value": null
188
+
}
189
+
]
190
+
},
191
+
"unit": "none"
192
+
},
193
+
"overrides": []
194
+
},
195
+
"gridPos": {
196
+
"h": 5,
197
+
"w": 3,
198
+
"x": 3,
199
+
"y": 1
200
+
},
201
+
"id": 3,
202
+
"options": {
203
+
"colorMode": "value",
204
+
"graphMode": "none",
205
+
"justifyMode": "center",
206
+
"orientation": "auto",
207
+
"reduceOptions": {
208
+
"calcs": [
209
+
"lastNotNull"
210
+
],
211
+
"fields": "",
212
+
"values": false
213
+
},
214
+
"textMode": "auto"
215
+
},
216
+
"pluginVersion": "10.1.5",
217
+
"targets": [
218
+
{
219
+
"datasource": {
220
+
"type": "prometheus",
221
+
"uid": "${DS_PROMETHEUS}"
222
+
},
223
+
"editorMode": "code",
224
+
"expr": "sum(prefect_deployments_total)",
225
+
"instant": false,
226
+
"legendFormat": "__auto",
227
+
"range": true,
228
+
"refId": "A"
229
+
}
230
+
],
231
+
"title": "Deployments total",
232
+
"type": "stat"
233
+
},
234
+
{
235
+
"datasource": {
236
+
"type": "prometheus",
237
+
"uid": "${DS_PROMETHEUS}"
238
+
},
239
+
"fieldConfig": {
240
+
"defaults": {
241
+
"color": {
242
+
"mode": "thresholds"
243
+
},
244
+
"decimals": 0,
245
+
"mappings": [],
246
+
"noValue": "0",
247
+
"thresholds": {
248
+
"mode": "absolute",
249
+
"steps": [
250
+
{
251
+
"color": "green",
252
+
"value": null
253
+
},
254
+
{
255
+
"color": "yellow",
256
+
"value": 1
257
+
}
258
+
]
259
+
},
260
+
"unit": "none"
261
+
},
262
+
"overrides": []
263
+
},
264
+
"gridPos": {
265
+
"h": 5,
266
+
"w": 3,
267
+
"x": 6,
268
+
"y": 1
269
+
},
270
+
"id": 14,
271
+
"options": {
272
+
"colorMode": "value",
273
+
"graphMode": "none",
274
+
"justifyMode": "center",
275
+
"orientation": "auto",
276
+
"reduceOptions": {
277
+
"calcs": [
278
+
"lastNotNull"
279
+
],
280
+
"fields": "",
281
+
"values": false
282
+
},
283
+
"textMode": "auto"
284
+
},
285
+
"pluginVersion": "10.1.5",
286
+
"targets": [
287
+
{
288
+
"datasource": {
289
+
"type": "prometheus",
290
+
"uid": "${DS_PROMETHEUS}"
291
+
},
292
+
"editorMode": "code",
293
+
"expr": "sum(prefect_info_deployment{is_schedule_active=\"False\"} == 1)",
294
+
"instant": false,
295
+
"legendFormat": "__auto",
296
+
"range": true,
297
+
"refId": "A"
298
+
}
299
+
],
300
+
"title": "Deployments paused",
301
+
"type": "stat"
302
+
},
303
+
{
304
+
"datasource": {
305
+
"type": "prometheus",
306
+
"uid": "${DS_PROMETHEUS}"
307
+
},
308
+
"description": "",
309
+
"fieldConfig": {
310
+
"defaults": {
311
+
"color": {
312
+
"mode": "thresholds"
313
+
},
314
+
"decimals": 0,
315
+
"mappings": [],
316
+
"thresholds": {
317
+
"mode": "absolute",
318
+
"steps": [
319
+
{
320
+
"color": "green",
321
+
"value": null
322
+
}
323
+
]
324
+
},
325
+
"unit": "none"
326
+
},
327
+
"overrides": []
328
+
},
329
+
"gridPos": {
330
+
"h": 5,
331
+
"w": 3,
332
+
"x": 9,
333
+
"y": 1
334
+
},
335
+
"id": 49,
336
+
"options": {
337
+
"colorMode": "value",
338
+
"graphMode": "area",
339
+
"justifyMode": "center",
340
+
"orientation": "auto",
341
+
"reduceOptions": {
342
+
"calcs": [
343
+
"lastNotNull"
344
+
],
345
+
"fields": "",
346
+
"values": false
347
+
},
348
+
"textMode": "auto"
349
+
},
350
+
"pluginVersion": "10.1.5",
351
+
"targets": [
352
+
{
353
+
"datasource": {
354
+
"type": "prometheus",
355
+
"uid": "${DS_PROMETHEUS}"
356
+
},
357
+
"editorMode": "code",
358
+
"exemplar": false,
359
+
"expr": "count(prefect_info_flow_runs == 1)",
360
+
"instant": false,
361
+
"legendFormat": "__auto",
362
+
"range": true,
363
+
"refId": "A"
364
+
}
365
+
],
366
+
"title": "Flow Runs running",
367
+
"type": "stat"
368
+
},
369
+
{
370
+
"datasource": {
371
+
"type": "prometheus",
372
+
"uid": "${DS_PROMETHEUS}"
373
+
},
374
+
"description": "",
375
+
"fieldConfig": {
376
+
"defaults": {
377
+
"color": {
378
+
"mode": "thresholds"
379
+
},
380
+
"decimals": 0,
381
+
"mappings": [],
382
+
"thresholds": {
383
+
"mode": "absolute",
384
+
"steps": [
385
+
{
386
+
"color": "green",
387
+
"value": null
388
+
}
389
+
]
390
+
},
391
+
"unit": "none"
392
+
},
393
+
"overrides": []
394
+
},
395
+
"gridPos": {
396
+
"h": 5,
397
+
"w": 4,
398
+
"x": 12,
399
+
"y": 1
400
+
},
401
+
"id": 48,
402
+
"options": {
403
+
"colorMode": "value",
404
+
"graphMode": "none",
405
+
"justifyMode": "center",
406
+
"orientation": "auto",
407
+
"reduceOptions": {
408
+
"calcs": [
409
+
"lastNotNull"
410
+
],
411
+
"fields": "",
412
+
"values": false
413
+
},
414
+
"textMode": "auto"
415
+
},
416
+
"pluginVersion": "10.1.5",
417
+
"targets": [
418
+
{
419
+
"datasource": {
420
+
"type": "prometheus",
421
+
"uid": "${DS_PROMETHEUS}"
422
+
},
423
+
"editorMode": "code",
424
+
"expr": "count(prefect_info_flow_runs)",
425
+
"instant": false,
426
+
"legendFormat": "__auto",
427
+
"range": true,
428
+
"refId": "A"
429
+
}
430
+
],
431
+
"title": "Flow Runs total",
432
+
"type": "stat"
433
+
},
434
+
{
435
+
"datasource": {
436
+
"type": "prometheus",
437
+
"uid": "${DS_PROMETHEUS}"
438
+
},
439
+
"fieldConfig": {
440
+
"defaults": {
441
+
"color": {
442
+
"mode": "thresholds"
443
+
},
444
+
"decimals": 0,
445
+
"mappings": [],
446
+
"thresholds": {
447
+
"mode": "absolute",
448
+
"steps": [
449
+
{
450
+
"color": "green",
451
+
"value": null
452
+
}
453
+
]
454
+
},
455
+
"unit": "s"
456
+
},
457
+
"overrides": []
458
+
},
459
+
"gridPos": {
460
+
"h": 5,
461
+
"w": 4,
462
+
"x": 16,
463
+
"y": 1
464
+
},
465
+
"id": 60,
466
+
"options": {
467
+
"colorMode": "value",
468
+
"graphMode": "none",
469
+
"justifyMode": "center",
470
+
"orientation": "auto",
471
+
"reduceOptions": {
472
+
"calcs": [
473
+
"lastNotNull"
474
+
],
475
+
"fields": "",
476
+
"values": false
477
+
},
478
+
"textMode": "auto"
479
+
},
480
+
"pluginVersion": "10.1.5",
481
+
"targets": [
482
+
{
483
+
"datasource": {
484
+
"type": "prometheus",
485
+
"uid": "${DS_PROMETHEUS}"
486
+
},
487
+
"editorMode": "code",
488
+
"expr": "avg(prefect_flow_runs_total_run_time)",
489
+
"instant": false,
490
+
"legendFormat": "__auto",
491
+
"range": true,
492
+
"refId": "A"
493
+
}
494
+
],
495
+
"title": "Flow Runs average run time",
496
+
"type": "stat"
497
+
},
498
+
{
499
+
"datasource": {
500
+
"type": "prometheus",
501
+
"uid": "${DS_PROMETHEUS}"
502
+
},
503
+
"description": "",
504
+
"fieldConfig": {
505
+
"defaults": {
506
+
"color": {
507
+
"mode": "palette-classic"
508
+
},
509
+
"custom": {
510
+
"hideFrom": {
511
+
"legend": false,
512
+
"tooltip": false,
513
+
"viz": false
514
+
}
515
+
},
516
+
"decimals": 0,
517
+
"mappings": [],
518
+
"unit": "none"
519
+
},
520
+
"overrides": [
521
+
{
522
+
"matcher": {
523
+
"id": "byName",
524
+
"options": "Failed"
525
+
},
526
+
"properties": [
527
+
{
528
+
"id": "color",
529
+
"value": {
530
+
"fixedColor": "red",
531
+
"mode": "fixed"
532
+
}
533
+
}
534
+
]
535
+
},
536
+
{
537
+
"matcher": {
538
+
"id": "byName",
539
+
"options": "Running"
540
+
},
541
+
"properties": [
542
+
{
543
+
"id": "color",
544
+
"value": {
545
+
"fixedColor": "blue",
546
+
"mode": "fixed"
547
+
}
548
+
}
549
+
]
550
+
},
551
+
{
552
+
"matcher": {
553
+
"id": "byName",
554
+
"options": "Completed"
555
+
},
556
+
"properties": [
557
+
{
558
+
"id": "color",
559
+
"value": {
560
+
"fixedColor": "green",
561
+
"mode": "fixed"
562
+
}
563
+
}
564
+
]
565
+
},
566
+
{
567
+
"matcher": {
568
+
"id": "byName",
569
+
"options": "Cancelled"
570
+
},
571
+
"properties": [
572
+
{
573
+
"id": "color",
574
+
"value": {
575
+
"fixedColor": "yellow",
576
+
"mode": "fixed"
577
+
}
578
+
}
579
+
]
580
+
}
581
+
]
582
+
},
583
+
"gridPos": {
584
+
"h": 5,
585
+
"w": 4,
586
+
"x": 20,
587
+
"y": 1
588
+
},
589
+
"id": 47,
590
+
"options": {
591
+
"legend": {
592
+
"displayMode": "table",
593
+
"placement": "right",
594
+
"showLegend": true,
595
+
"values": [
596
+
"percent"
597
+
]
598
+
},
599
+
"pieType": "pie",
600
+
"reduceOptions": {
601
+
"calcs": [
602
+
"lastNotNull"
603
+
],
604
+
"fields": "",
605
+
"values": false
606
+
},
607
+
"tooltip": {
608
+
"mode": "single",
609
+
"sort": "none"
610
+
}
611
+
},
612
+
"pluginVersion": "10.1.1",
613
+
"targets": [
614
+
{
615
+
"datasource": {
616
+
"type": "prometheus",
617
+
"uid": "${DS_PROMETHEUS}"
618
+
},
619
+
"editorMode": "code",
620
+
"expr": "count by (state_name) (prefect_info_flow_runs)",
621
+
"instant": false,
622
+
"legendFormat": "{{label_name}}",
623
+
"range": true,
624
+
"refId": "A"
625
+
}
626
+
],
627
+
"title": "Flow Runs by state",
628
+
"type": "piechart"
629
+
},
630
+
{
631
+
"collapsed": false,
632
+
"gridPos": {
633
+
"h": 1,
634
+
"w": 24,
635
+
"x": 0,
636
+
"y": 6
637
+
},
638
+
"id": 10,
639
+
"panels": [],
640
+
"title": "Flows Runs: $flow_name (last 24h)",
641
+
"type": "row"
642
+
},
643
+
{
644
+
"datasource": {
645
+
"type": "prometheus",
646
+
"uid": "${DS_PROMETHEUS}"
647
+
},
648
+
"description": "Last 24h",
649
+
"fieldConfig": {
650
+
"defaults": {
651
+
"color": {
652
+
"mode": "thresholds"
653
+
},
654
+
"decimals": 0,
655
+
"mappings": [],
656
+
"thresholds": {
657
+
"mode": "absolute",
658
+
"steps": [
659
+
{
660
+
"color": "green",
661
+
"value": null
662
+
}
663
+
]
664
+
},
665
+
"unit": "none"
666
+
},
667
+
"overrides": []
668
+
},
669
+
"gridPos": {
670
+
"h": 4,
671
+
"w": 3,
672
+
"x": 0,
673
+
"y": 7
674
+
},
675
+
"id": 51,
676
+
"options": {
677
+
"colorMode": "value",
678
+
"graphMode": "none",
679
+
"justifyMode": "center",
680
+
"orientation": "auto",
681
+
"reduceOptions": {
682
+
"calcs": [
683
+
"lastNotNull"
684
+
],
685
+
"fields": "",
686
+
"values": false
687
+
},
688
+
"textMode": "auto"
689
+
},
690
+
"pluginVersion": "10.1.5",
691
+
"targets": [
692
+
{
693
+
"datasource": {
694
+
"type": "prometheus",
695
+
"uid": "${DS_PROMETHEUS}"
696
+
},
697
+
"editorMode": "code",
698
+
"expr": "count(prefect_info_flow_runs{flow_name=~\"$flow_name\"})",
699
+
"instant": false,
700
+
"legendFormat": "__auto",
701
+
"range": true,
702
+
"refId": "A"
703
+
}
704
+
],
705
+
"title": "Flow Runs total",
706
+
"type": "stat"
707
+
},
708
+
{
709
+
"datasource": {
710
+
"type": "prometheus",
711
+
"uid": "${DS_PROMETHEUS}"
712
+
},
713
+
"description": "",
714
+
"fieldConfig": {
715
+
"defaults": {
716
+
"color": {
717
+
"mode": "thresholds"
718
+
},
719
+
"custom": {
720
+
"align": "auto",
721
+
"cellOptions": {
722
+
"type": "auto"
723
+
},
724
+
"filterable": true,
725
+
"inspect": false
726
+
},
727
+
"mappings": [],
728
+
"thresholds": {
729
+
"mode": "absolute",
730
+
"steps": [
731
+
{
732
+
"color": "green",
733
+
"value": null
734
+
}
735
+
]
736
+
}
737
+
},
738
+
"overrides": [
739
+
{
740
+
"matcher": {
741
+
"id": "byName",
742
+
"options": "created"
743
+
},
744
+
"properties": [
745
+
{
746
+
"id": "unit",
747
+
"value": "dateTimeAsIso"
748
+
}
749
+
]
750
+
},
751
+
{
752
+
"matcher": {
753
+
"id": "byName",
754
+
"options": "name"
755
+
},
756
+
"properties": [
757
+
{
758
+
"id": "custom.width",
759
+
"value": 201
760
+
}
761
+
]
762
+
},
763
+
{
764
+
"matcher": {
765
+
"id": "byName",
766
+
"options": "spent time"
767
+
},
768
+
"properties": [
769
+
{
770
+
"id": "unit",
771
+
"value": "s"
772
+
}
773
+
]
774
+
},
775
+
{
776
+
"matcher": {
777
+
"id": "byName",
778
+
"options": "state"
779
+
},
780
+
"properties": [
781
+
{
782
+
"id": "custom.cellOptions",
783
+
"value": {
784
+
"mode": "basic",
785
+
"type": "color-background"
786
+
}
787
+
},
788
+
{
789
+
"id": "mappings",
790
+
"value": [
791
+
{
792
+
"options": {
793
+
"Cancelled": {
794
+
"color": "orange",
795
+
"index": 2
796
+
},
797
+
"Completed": {
798
+
"color": "green",
799
+
"index": 0
800
+
},
801
+
"Crashed": {
802
+
"color": "red",
803
+
"index": 3
804
+
},
805
+
"Failed": {
806
+
"color": "red",
807
+
"index": 1
808
+
},
809
+
"Running": {
810
+
"color": "blue",
811
+
"index": 4
812
+
}
813
+
},
814
+
"type": "value"
815
+
}
816
+
]
817
+
}
818
+
]
819
+
},
820
+
{
821
+
"matcher": {
822
+
"id": "byName",
823
+
"options": "start_time"
824
+
},
825
+
"properties": [
826
+
{
827
+
"id": "unit",
828
+
"value": "dateTimeAsIso"
829
+
}
830
+
]
831
+
},
832
+
{
833
+
"matcher": {
834
+
"id": "byName",
835
+
"options": "end_time"
836
+
},
837
+
"properties": [
838
+
{
839
+
"id": "unit",
840
+
"value": "dateTimeAsIso"
841
+
},
842
+
{
843
+
"id": "mappings",
844
+
"value": [
845
+
{
846
+
"options": {
847
+
"match": "nan",
848
+
"result": {
849
+
"index": 0,
850
+
"text": "-"
851
+
}
852
+
},
853
+
"type": "special"
854
+
}
855
+
]
856
+
}
857
+
]
858
+
},
859
+
{
860
+
"matcher": {
861
+
"id": "byName",
862
+
"options": "runs"
863
+
},
864
+
"properties": [
865
+
{
866
+
"id": "custom.width",
867
+
"value": 85
868
+
}
869
+
]
870
+
},
871
+
{
872
+
"matcher": {
873
+
"id": "byName",
874
+
"options": "flow_run_name"
875
+
},
876
+
"properties": [
877
+
{
878
+
"id": "custom.width",
879
+
"value": 280
880
+
}
881
+
]
882
+
}
883
+
]
884
+
},
885
+
"gridPos": {
886
+
"h": 12,
887
+
"w": 21,
888
+
"x": 3,
889
+
"y": 7
890
+
},
891
+
"hideTimeOverride": true,
892
+
"id": 50,
893
+
"options": {
894
+
"cellHeight": "sm",
895
+
"footer": {
896
+
"countRows": false,
897
+
"enablePagination": true,
898
+
"fields": [],
899
+
"reducer": [
900
+
"sum"
901
+
],
902
+
"show": false
903
+
},
904
+
"frameIndex": 0,
905
+
"showHeader": true,
906
+
"sortBy": []
907
+
},
908
+
"pluginVersion": "10.1.5",
909
+
"targets": [
910
+
{
911
+
"datasource": {
912
+
"type": "prometheus",
913
+
"uid": "${DS_PROMETHEUS}"
914
+
},
915
+
"editorMode": "code",
916
+
"exemplar": false,
917
+
"expr": "prefect_info_flow_runs{state_name=~\"$flow_run_state_name\", flow_name=~\"$flow_name\"}",
918
+
"format": "table",
919
+
"instant": true,
920
+
"legendFormat": "",
921
+
"range": false,
922
+
"refId": "A"
923
+
}
924
+
],
925
+
"transformations": [
926
+
{
927
+
"id": "organize",
928
+
"options": {
929
+
"excludeByName": {
930
+
"Time": true,
931
+
"Value": true,
932
+
"__name__": true,
933
+
"container": true,
934
+
"created": false,
935
+
"deployment_id": true,
936
+
"deployment_name": true,
937
+
"endpoint": true,
938
+
"flow_id": true,
939
+
"flow_name": true,
940
+
"flow_run_id": true,
941
+
"instance": true,
942
+
"job": true,
943
+
"name": false,
944
+
"namespace": true,
945
+
"pod": true,
946
+
"service": true,
947
+
"start_time": false,
948
+
"state_id": true,
949
+
"updated": true,
950
+
"work_queue_name": true
951
+
},
952
+
"indexByName": {
953
+
"Time": 8,
954
+
"Value": 22,
955
+
"__name__": 9,
956
+
"container": 10,
957
+
"created": 18,
958
+
"deployment_id": 6,
959
+
"deployment_name": 7,
960
+
"end_time": 20,
961
+
"endpoint": 11,
962
+
"flow_id": 4,
963
+
"flow_name": 5,
964
+
"flow_run_id": 1,
965
+
"flow_run_name": 0,
966
+
"instance": 12,
967
+
"job": 13,
968
+
"namespace": 14,
969
+
"pod": 15,
970
+
"prefect_info_flow_runs": 23,
971
+
"run_count": 2,
972
+
"service": 16,
973
+
"start_time": 19,
974
+
"state_id": 21,
975
+
"total_run_time": 3,
976
+
"work_queue_name": 17
977
+
},
978
+
"renameByName": {
979
+
"is_schedule_active": "scheduled",
980
+
"job": "",
981
+
"prefect_info_flow_runs": "state",
982
+
"run_count": "runs",
983
+
"start_time": "",
984
+
"state_name": "state",
985
+
"total_run_time": "spent time"
986
+
}
987
+
}
988
+
}
989
+
],
990
+
"type": "table"
991
+
},
992
+
{
993
+
"datasource": {
994
+
"type": "prometheus",
995
+
"uid": "${DS_PROMETHEUS}"
996
+
},
997
+
"description": "Last 24h",
998
+
"fieldConfig": {
999
+
"defaults": {
1000
+
"color": {
1001
+
"mode": "thresholds"
1002
+
},
1003
+
"mappings": [],
1004
+
"thresholds": {
1005
+
"mode": "absolute",
1006
+
"steps": [
1007
+
{
1008
+
"color": "green",
1009
+
"value": null
1010
+
}
1011
+
]
1012
+
},
1013
+
"unit": "short"
1014
+
},
1015
+
"overrides": []
1016
+
},
1017
+
"gridPos": {
1018
+
"h": 4,
1019
+
"w": 3,
1020
+
"x": 0,
1021
+
"y": 11
1022
+
},
1023
+
"id": 56,
1024
+
"options": {
1025
+
"colorMode": "value",
1026
+
"graphMode": "none",
1027
+
"justifyMode": "center",
1028
+
"orientation": "auto",
1029
+
"reduceOptions": {
1030
+
"calcs": [
1031
+
"lastNotNull"
1032
+
],
1033
+
"fields": "",
1034
+
"values": false
1035
+
},
1036
+
"textMode": "auto"
1037
+
},
1038
+
"pluginVersion": "10.1.5",
1039
+
"targets": [
1040
+
{
1041
+
"datasource": {
1042
+
"type": "prometheus",
1043
+
"uid": "${DS_PROMETHEUS}"
1044
+
},
1045
+
"editorMode": "code",
1046
+
"expr": "max(max(max_over_time(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\".*$flow_name.*\"}[$__range])) by (pod))",
1047
+
"instant": false,
1048
+
"legendFormat": "__auto",
1049
+
"range": true,
1050
+
"refId": "A"
1051
+
}
1052
+
],
1053
+
"title": "CPU usage peak",
1054
+
"type": "stat"
1055
+
},
1056
+
{
1057
+
"datasource": {
1058
+
"type": "prometheus",
1059
+
"uid": "${DS_PROMETHEUS}"
1060
+
},
1061
+
"description": "Last 24h",
1062
+
"fieldConfig": {
1063
+
"defaults": {
1064
+
"color": {
1065
+
"mode": "thresholds"
1066
+
},
1067
+
"decimals": 0,
1068
+
"mappings": [],
1069
+
"thresholds": {
1070
+
"mode": "absolute",
1071
+
"steps": [
1072
+
{
1073
+
"color": "green",
1074
+
"value": null
1075
+
}
1076
+
]
1077
+
},
1078
+
"unit": "bytes"
1079
+
},
1080
+
"overrides": []
1081
+
},
1082
+
"gridPos": {
1083
+
"h": 4,
1084
+
"w": 3,
1085
+
"x": 0,
1086
+
"y": 15
1087
+
},
1088
+
"id": 55,
1089
+
"options": {
1090
+
"colorMode": "value",
1091
+
"graphMode": "none",
1092
+
"justifyMode": "center",
1093
+
"orientation": "auto",
1094
+
"reduceOptions": {
1095
+
"calcs": [
1096
+
"lastNotNull"
1097
+
],
1098
+
"fields": "",
1099
+
"values": false
1100
+
},
1101
+
"textMode": "auto"
1102
+
},
1103
+
"pluginVersion": "10.1.5",
1104
+
"targets": [
1105
+
{
1106
+
"datasource": {
1107
+
"type": "prometheus",
1108
+
"uid": "${DS_PROMETHEUS}"
1109
+
},
1110
+
"editorMode": "code",
1111
+
"expr": "max(max(max_over_time(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", container!=\"\", image!=\"\"}[$__range])) by (pod))",
1112
+
"instant": false,
1113
+
"legendFormat": "__auto",
1114
+
"range": true,
1115
+
"refId": "A"
1116
+
}
1117
+
],
1118
+
"title": "MEM usage peak",
1119
+
"type": "stat"
1120
+
},
1121
+
{
1122
+
"collapsed": false,
1123
+
"gridPos": {
1124
+
"h": 1,
1125
+
"w": 24,
1126
+
"x": 0,
1127
+
"y": 19
1128
+
},
1129
+
"id": 9,
1130
+
"panels": [],
1131
+
"title": "Flows",
1132
+
"type": "row"
1133
+
},
1134
+
{
1135
+
"datasource": {
1136
+
"type": "prometheus",
1137
+
"uid": "${DS_PROMETHEUS}"
1138
+
},
1139
+
"description": "Last 24h",
1140
+
"fieldConfig": {
1141
+
"defaults": {
1142
+
"color": {
1143
+
"mode": "thresholds"
1144
+
},
1145
+
"mappings": [],
1146
+
"thresholds": {
1147
+
"mode": "absolute",
1148
+
"steps": [
1149
+
{
1150
+
"color": "green",
1151
+
"value": null
1152
+
}
1153
+
]
1154
+
},
1155
+
"unit": "s"
1156
+
},
1157
+
"overrides": []
1158
+
},
1159
+
"gridPos": {
1160
+
"h": 8,
1161
+
"w": 3,
1162
+
"x": 0,
1163
+
"y": 20
1164
+
},
1165
+
"id": 58,
1166
+
"options": {
1167
+
"colorMode": "value",
1168
+
"graphMode": "none",
1169
+
"justifyMode": "center",
1170
+
"orientation": "auto",
1171
+
"reduceOptions": {
1172
+
"calcs": [
1173
+
"lastNotNull"
1174
+
],
1175
+
"fields": "",
1176
+
"values": false
1177
+
},
1178
+
"textMode": "auto"
1179
+
},
1180
+
"pluginVersion": "10.1.5",
1181
+
"targets": [
1182
+
{
1183
+
"datasource": {
1184
+
"type": "prometheus",
1185
+
"uid": "${DS_PROMETHEUS}"
1186
+
},
1187
+
"editorMode": "code",
1188
+
"expr": "avg by (flow_name) (prefect_flow_runs_total_run_time{flow_name=~\"$flow_name\"})",
1189
+
"instant": false,
1190
+
"legendFormat": "__auto",
1191
+
"range": true,
1192
+
"refId": "A"
1193
+
}
1194
+
],
1195
+
"title": "Average time",
1196
+
"type": "stat"
1197
+
},
1198
+
{
1199
+
"datasource": {
1200
+
"type": "prometheus",
1201
+
"uid": "${DS_PROMETHEUS}"
1202
+
},
1203
+
"fieldConfig": {
1204
+
"defaults": {
1205
+
"color": {
1206
+
"mode": "palette-classic"
1207
+
},
1208
+
"custom": {
1209
+
"axisCenteredZero": false,
1210
+
"axisColorMode": "text",
1211
+
"axisLabel": "",
1212
+
"axisPlacement": "auto",
1213
+
"barAlignment": 0,
1214
+
"drawStyle": "line",
1215
+
"fillOpacity": 0,
1216
+
"gradientMode": "none",
1217
+
"hideFrom": {
1218
+
"legend": false,
1219
+
"tooltip": false,
1220
+
"viz": false
1221
+
},
1222
+
"insertNulls": false,
1223
+
"lineInterpolation": "linear",
1224
+
"lineWidth": 1,
1225
+
"pointSize": 5,
1226
+
"scaleDistribution": {
1227
+
"type": "linear"
1228
+
},
1229
+
"showPoints": "auto",
1230
+
"spanNulls": false,
1231
+
"stacking": {
1232
+
"group": "A",
1233
+
"mode": "none"
1234
+
},
1235
+
"thresholdsStyle": {
1236
+
"mode": "off"
1237
+
}
1238
+
},
1239
+
"mappings": [],
1240
+
"min": 0,
1241
+
"thresholds": {
1242
+
"mode": "absolute",
1243
+
"steps": [
1244
+
{
1245
+
"color": "green",
1246
+
"value": null
1247
+
},
1248
+
{
1249
+
"color": "red",
1250
+
"value": 80
1251
+
}
1252
+
]
1253
+
},
1254
+
"unit": "s"
1255
+
},
1256
+
"overrides": []
1257
+
},
1258
+
"gridPos": {
1259
+
"h": 8,
1260
+
"w": 21,
1261
+
"x": 3,
1262
+
"y": 20
1263
+
},
1264
+
"id": 61,
1265
+
"options": {
1266
+
"legend": {
1267
+
"calcs": [],
1268
+
"displayMode": "list",
1269
+
"placement": "bottom",
1270
+
"showLegend": false
1271
+
},
1272
+
"tooltip": {
1273
+
"mode": "single",
1274
+
"sort": "none"
1275
+
}
1276
+
},
1277
+
"pluginVersion": "10.1.5",
1278
+
"targets": [
1279
+
{
1280
+
"datasource": {
1281
+
"type": "prometheus",
1282
+
"uid": "${DS_PROMETHEUS}"
1283
+
},
1284
+
"editorMode": "code",
1285
+
"expr": "avg by (flow_name) (prefect_flow_runs_total_run_time{flow_name=~\"$flow_name\"})",
1286
+
"instant": false,
1287
+
"legendFormat": "{{flow_run_name}}",
1288
+
"range": true,
1289
+
"refId": "A"
1290
+
}
1291
+
],
1292
+
"title": "Flow Run total run time",
1293
+
"type": "timeseries"
1294
+
},
1295
+
{
1296
+
"datasource": {
1297
+
"type": "prometheus",
1298
+
"uid": "${DS_PROMETHEUS}"
1299
+
},
1300
+
"description": "",
1301
+
"fieldConfig": {
1302
+
"defaults": {
1303
+
"color": {
1304
+
"mode": "thresholds"
1305
+
},
1306
+
"custom": {
1307
+
"align": "auto",
1308
+
"cellOptions": {
1309
+
"type": "auto"
1310
+
},
1311
+
"filterable": true,
1312
+
"inspect": false
1313
+
},
1314
+
"mappings": [],
1315
+
"thresholds": {
1316
+
"mode": "absolute",
1317
+
"steps": [
1318
+
{
1319
+
"color": "green",
1320
+
"value": null
1321
+
}
1322
+
]
1323
+
}
1324
+
},
1325
+
"overrides": [
1326
+
{
1327
+
"matcher": {
1328
+
"id": "byName",
1329
+
"options": "created"
1330
+
},
1331
+
"properties": [
1332
+
{
1333
+
"id": "unit",
1334
+
"value": "dateTimeAsIso"
1335
+
}
1336
+
]
1337
+
}
1338
+
]
1339
+
},
1340
+
"gridPos": {
1341
+
"h": 10,
1342
+
"w": 24,
1343
+
"x": 0,
1344
+
"y": 28
1345
+
},
1346
+
"id": 12,
1347
+
"options": {
1348
+
"cellHeight": "sm",
1349
+
"footer": {
1350
+
"countRows": false,
1351
+
"enablePagination": true,
1352
+
"fields": [],
1353
+
"reducer": [
1354
+
"sum"
1355
+
],
1356
+
"show": false
1357
+
},
1358
+
"frameIndex": 0,
1359
+
"showHeader": true,
1360
+
"sortBy": [
1361
+
{
1362
+
"desc": true,
1363
+
"displayName": "scheduled"
1364
+
}
1365
+
]
1366
+
},
1367
+
"pluginVersion": "10.1.5",
1368
+
"targets": [
1369
+
{
1370
+
"datasource": {
1371
+
"type": "prometheus",
1372
+
"uid": "${DS_PROMETHEUS}"
1373
+
},
1374
+
"editorMode": "code",
1375
+
"exemplar": false,
1376
+
"expr": "prefect_info_flows{flow_name=~\"$flow_name\"} == 1",
1377
+
"format": "table",
1378
+
"instant": true,
1379
+
"legendFormat": "",
1380
+
"range": false,
1381
+
"refId": "A"
1382
+
}
1383
+
],
1384
+
"transformations": [
1385
+
{
1386
+
"id": "organize",
1387
+
"options": {
1388
+
"excludeByName": {
1389
+
"Time": true,
1390
+
"Value": true,
1391
+
"__name__": true,
1392
+
"container": true,
1393
+
"endpoint": true,
1394
+
"instance": true,
1395
+
"job": true,
1396
+
"namespace": true,
1397
+
"pod": true,
1398
+
"service": true
1399
+
},
1400
+
"indexByName": {
1401
+
"Time": 2,
1402
+
"Value": 12,
1403
+
"__name__": 3,
1404
+
"container": 4,
1405
+
"created": 5,
1406
+
"endpoint": 6,
1407
+
"flow_id": 1,
1408
+
"flow_name": 0,
1409
+
"instance": 7,
1410
+
"job": 8,
1411
+
"namespace": 9,
1412
+
"pod": 10,
1413
+
"service": 11
1414
+
},
1415
+
"renameByName": {}
1416
+
}
1417
+
}
1418
+
],
1419
+
"type": "table"
1420
+
},
1421
+
{
1422
+
"datasource": {
1423
+
"type": "prometheus",
1424
+
"uid": "$datasource"
1425
+
},
1426
+
"fieldConfig": {
1427
+
"defaults": {
1428
+
"color": {
1429
+
"mode": "palette-classic"
1430
+
},
1431
+
"custom": {
1432
+
"axisCenteredZero": false,
1433
+
"axisColorMode": "text",
1434
+
"axisLabel": "",
1435
+
"axisPlacement": "auto",
1436
+
"barAlignment": 0,
1437
+
"drawStyle": "line",
1438
+
"fillOpacity": 100,
1439
+
"gradientMode": "none",
1440
+
"hideFrom": {
1441
+
"legend": false,
1442
+
"tooltip": false,
1443
+
"viz": false
1444
+
},
1445
+
"insertNulls": false,
1446
+
"lineInterpolation": "linear",
1447
+
"lineStyle": {
1448
+
"fill": "solid"
1449
+
},
1450
+
"lineWidth": 0,
1451
+
"pointSize": 5,
1452
+
"scaleDistribution": {
1453
+
"type": "linear"
1454
+
},
1455
+
"showPoints": "never",
1456
+
"spanNulls": false,
1457
+
"stacking": {
1458
+
"group": "A",
1459
+
"mode": "normal"
1460
+
},
1461
+
"thresholdsStyle": {
1462
+
"mode": "off"
1463
+
}
1464
+
},
1465
+
"mappings": [],
1466
+
"min": 0,
1467
+
"thresholds": {
1468
+
"mode": "absolute",
1469
+
"steps": [
1470
+
{
1471
+
"color": "green",
1472
+
"value": null
1473
+
},
1474
+
{
1475
+
"color": "red",
1476
+
"value": 80
1477
+
}
1478
+
]
1479
+
},
1480
+
"unit": "short"
1481
+
},
1482
+
"overrides": [
1483
+
{
1484
+
"matcher": {
1485
+
"id": "byName",
1486
+
"options": "requests"
1487
+
},
1488
+
"properties": [
1489
+
{
1490
+
"id": "color",
1491
+
"value": {
1492
+
"fixedColor": "orange",
1493
+
"mode": "fixed"
1494
+
}
1495
+
},
1496
+
{
1497
+
"id": "custom.fillOpacity",
1498
+
"value": 0
1499
+
},
1500
+
{
1501
+
"id": "custom.lineWidth",
1502
+
"value": 2
1503
+
},
1504
+
{
1505
+
"id": "custom.stacking",
1506
+
"value": {
1507
+
"group": "A",
1508
+
"mode": "none"
1509
+
}
1510
+
},
1511
+
{
1512
+
"id": "custom.transform",
1513
+
"value": "constant"
1514
+
}
1515
+
]
1516
+
},
1517
+
{
1518
+
"matcher": {
1519
+
"id": "byName",
1520
+
"options": "limits"
1521
+
},
1522
+
"properties": [
1523
+
{
1524
+
"id": "color",
1525
+
"value": {
1526
+
"fixedColor": "red",
1527
+
"mode": "fixed"
1528
+
}
1529
+
},
1530
+
{
1531
+
"id": "custom.fillOpacity",
1532
+
"value": 0
1533
+
},
1534
+
{
1535
+
"id": "custom.lineWidth",
1536
+
"value": 2
1537
+
},
1538
+
{
1539
+
"id": "custom.stacking",
1540
+
"value": {
1541
+
"group": "A",
1542
+
"mode": "none"
1543
+
}
1544
+
},
1545
+
{
1546
+
"id": "custom.transform",
1547
+
"value": "constant"
1548
+
}
1549
+
]
1550
+
}
1551
+
]
1552
+
},
1553
+
"gridPos": {
1554
+
"h": 7,
1555
+
"w": 13,
1556
+
"x": 0,
1557
+
"y": 38
1558
+
},
1559
+
"id": 53,
1560
+
"interval": "1m",
1561
+
"links": [],
1562
+
"options": {
1563
+
"legend": {
1564
+
"calcs": [],
1565
+
"displayMode": "table",
1566
+
"placement": "bottom",
1567
+
"showLegend": false
1568
+
},
1569
+
"tooltip": {
1570
+
"mode": "single",
1571
+
"sort": "none"
1572
+
}
1573
+
},
1574
+
"pluginVersion": "10.1.1",
1575
+
"targets": [
1576
+
{
1577
+
"datasource": {
1578
+
"uid": "$datasource"
1579
+
},
1580
+
"editorMode": "code",
1581
+
"expr": "avg(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\".*$flow_name.*\"}) by (pod))",
1582
+
"format": "time_series",
1583
+
"intervalFactor": 2,
1584
+
"legendFormat": "usage",
1585
+
"range": true,
1586
+
"refId": "A"
1587
+
},
1588
+
{
1589
+
"datasource": {
1590
+
"uid": "$datasource"
1591
+
},
1592
+
"editorMode": "code",
1593
+
"expr": "kube_pod_container_resource_requests{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", resource=\"cpu\"}\n",
1594
+
"format": "time_series",
1595
+
"intervalFactor": 2,
1596
+
"legendFormat": "requests",
1597
+
"range": true,
1598
+
"refId": "B"
1599
+
},
1600
+
{
1601
+
"datasource": {
1602
+
"uid": "$datasource"
1603
+
},
1604
+
"editorMode": "code",
1605
+
"expr": "kube_pod_container_resource_limits{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", resource=\"cpu\"}\n",
1606
+
"format": "time_series",
1607
+
"intervalFactor": 2,
1608
+
"legendFormat": "limits",
1609
+
"range": true,
1610
+
"refId": "C"
1611
+
}
1612
+
],
1613
+
"title": "CPU Usage $flow_name",
1614
+
"type": "timeseries"
1615
+
},
1616
+
{
1617
+
"datasource": {
1618
+
"type": "prometheus",
1619
+
"uid": "$datasource"
1620
+
},
1621
+
"fieldConfig": {
1622
+
"defaults": {
1623
+
"color": {
1624
+
"mode": "palette-classic"
1625
+
},
1626
+
"custom": {
1627
+
"axisCenteredZero": false,
1628
+
"axisColorMode": "text",
1629
+
"axisLabel": "",
1630
+
"axisPlacement": "auto",
1631
+
"barAlignment": 0,
1632
+
"drawStyle": "line",
1633
+
"fillOpacity": 100,
1634
+
"gradientMode": "none",
1635
+
"hideFrom": {
1636
+
"legend": false,
1637
+
"tooltip": false,
1638
+
"viz": false
1639
+
},
1640
+
"insertNulls": false,
1641
+
"lineInterpolation": "linear",
1642
+
"lineStyle": {
1643
+
"fill": "solid"
1644
+
},
1645
+
"lineWidth": 0,
1646
+
"pointSize": 5,
1647
+
"scaleDistribution": {
1648
+
"type": "linear"
1649
+
},
1650
+
"showPoints": "never",
1651
+
"spanNulls": false,
1652
+
"stacking": {
1653
+
"group": "A",
1654
+
"mode": "normal"
1655
+
},
1656
+
"thresholdsStyle": {
1657
+
"mode": "off"
1658
+
}
1659
+
},
1660
+
"mappings": [],
1661
+
"min": 0,
1662
+
"thresholds": {
1663
+
"mode": "absolute",
1664
+
"steps": [
1665
+
{
1666
+
"color": "green",
1667
+
"value": null
1668
+
},
1669
+
{
1670
+
"color": "red",
1671
+
"value": 80
1672
+
}
1673
+
]
1674
+
},
1675
+
"unit": "bytes"
1676
+
},
1677
+
"overrides": [
1678
+
{
1679
+
"matcher": {
1680
+
"id": "byName",
1681
+
"options": "requests"
1682
+
},
1683
+
"properties": [
1684
+
{
1685
+
"id": "color",
1686
+
"value": {
1687
+
"fixedColor": "orange",
1688
+
"mode": "fixed"
1689
+
}
1690
+
},
1691
+
{
1692
+
"id": "custom.fillOpacity",
1693
+
"value": 0
1694
+
},
1695
+
{
1696
+
"id": "custom.lineWidth",
1697
+
"value": 2
1698
+
},
1699
+
{
1700
+
"id": "custom.stacking",
1701
+
"value": {
1702
+
"group": "A",
1703
+
"mode": "none"
1704
+
}
1705
+
},
1706
+
{
1707
+
"id": "custom.transform",
1708
+
"value": "constant"
1709
+
}
1710
+
]
1711
+
},
1712
+
{
1713
+
"matcher": {
1714
+
"id": "byName",
1715
+
"options": "limits"
1716
+
},
1717
+
"properties": [
1718
+
{
1719
+
"id": "color",
1720
+
"value": {
1721
+
"fixedColor": "red",
1722
+
"mode": "fixed"
1723
+
}
1724
+
},
1725
+
{
1726
+
"id": "custom.fillOpacity",
1727
+
"value": 0
1728
+
},
1729
+
{
1730
+
"id": "custom.lineWidth",
1731
+
"value": 2
1732
+
},
1733
+
{
1734
+
"id": "custom.stacking",
1735
+
"value": {
1736
+
"group": "A",
1737
+
"mode": "none"
1738
+
}
1739
+
},
1740
+
{
1741
+
"id": "custom.transform",
1742
+
"value": "constant"
1743
+
}
1744
+
]
1745
+
}
1746
+
]
1747
+
},
1748
+
"gridPos": {
1749
+
"h": 7,
1750
+
"w": 11,
1751
+
"x": 13,
1752
+
"y": 38
1753
+
},
1754
+
"id": 54,
1755
+
"interval": "1m",
1756
+
"links": [],
1757
+
"options": {
1758
+
"legend": {
1759
+
"calcs": [],
1760
+
"displayMode": "table",
1761
+
"placement": "right",
1762
+
"showLegend": false
1763
+
},
1764
+
"tooltip": {
1765
+
"mode": "single",
1766
+
"sort": "none"
1767
+
}
1768
+
},
1769
+
"pluginVersion": "10.1.1",
1770
+
"targets": [
1771
+
{
1772
+
"datasource": {
1773
+
"uid": "$datasource"
1774
+
},
1775
+
"editorMode": "code",
1776
+
"expr": "avg(sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", container!=\"\", image!=\"\"}) by (pod))",
1777
+
"format": "time_series",
1778
+
"intervalFactor": 2,
1779
+
"legendFormat": "usage",
1780
+
"range": true,
1781
+
"refId": "A"
1782
+
},
1783
+
{
1784
+
"datasource": {
1785
+
"uid": "$datasource"
1786
+
},
1787
+
"editorMode": "code",
1788
+
"expr": "kube_pod_container_resource_requests{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", resource=\"memory\"}",
1789
+
"format": "time_series",
1790
+
"intervalFactor": 2,
1791
+
"legendFormat": "requests",
1792
+
"range": true,
1793
+
"refId": "B"
1794
+
},
1795
+
{
1796
+
"datasource": {
1797
+
"uid": "$datasource"
1798
+
},
1799
+
"editorMode": "code",
1800
+
"expr": "kube_pod_container_resource_limits{job=\"kube-state-metrics\", namespace=\"$namespace\", pod=~\".*$flow_name.*\", resource=\"memory\"}",
1801
+
"format": "time_series",
1802
+
"intervalFactor": 2,
1803
+
"legendFormat": "limits",
1804
+
"range": true,
1805
+
"refId": "C"
1806
+
}
1807
+
],
1808
+
"title": "Memory Usage (WSS)",
1809
+
"type": "timeseries"
1810
+
},
1811
+
{
1812
+
"datasource": {
1813
+
"type": "loki",
1814
+
"uid": "${DS_LOKI}"
1815
+
},
1816
+
"description": "Include all flow_runs",
1817
+
"gridPos": {
1818
+
"h": 18,
1819
+
"w": 24,
1820
+
"x": 0,
1821
+
"y": 45
1822
+
},
1823
+
"id": 46,
1824
+
"options": {
1825
+
"dedupStrategy": "none",
1826
+
"enableLogDetails": true,
1827
+
"prettifyLogMessage": false,
1828
+
"showCommonLabels": false,
1829
+
"showLabels": false,
1830
+
"showTime": false,
1831
+
"sortOrder": "Descending",
1832
+
"wrapLogMessage": false
1833
+
},
1834
+
"targets": [
1835
+
{
1836
+
"datasource": {
1837
+
"type": "loki",
1838
+
"uid": "${DS_LOKI}"
1839
+
},
1840
+
"editorMode": "code",
1841
+
"expr": "{namespace=\"$namespace\", pod=~\".*$flow_name.*\"} |= \"$search\"",
1842
+
"queryType": "range",
1843
+
"refId": "A"
1844
+
}
1845
+
],
1846
+
"title": "Flow logs: flow-$flow_name",
1847
+
"type": "logs"
1848
+
},
1849
+
{
1850
+
"collapsed": true,
1851
+
"gridPos": {
1852
+
"h": 1,
1853
+
"w": 24,
1854
+
"x": 0,
1855
+
"y": 63
1856
+
},
1857
+
"id": 8,
1858
+
"panels": [
1859
+
{
1860
+
"datasource": {
1861
+
"type": "prometheus",
1862
+
"uid": "${DS_PROMETHEUS}"
1863
+
},
1864
+
"description": "",
1865
+
"fieldConfig": {
1866
+
"defaults": {
1867
+
"color": {
1868
+
"mode": "thresholds"
1869
+
},
1870
+
"custom": {
1871
+
"align": "auto",
1872
+
"cellOptions": {
1873
+
"type": "auto"
1874
+
},
1875
+
"filterable": true,
1876
+
"inspect": false
1877
+
},
1878
+
"mappings": [],
1879
+
"thresholds": {
1880
+
"mode": "absolute",
1881
+
"steps": [
1882
+
{
1883
+
"color": "green"
1884
+
}
1885
+
]
1886
+
}
1887
+
},
1888
+
"overrides": [
1889
+
{
1890
+
"matcher": {
1891
+
"id": "byName",
1892
+
"options": "is_schedule_active"
1893
+
},
1894
+
"properties": [
1895
+
{
1896
+
"id": "custom.cellOptions",
1897
+
"value": {
1898
+
"mode": "basic",
1899
+
"type": "color-background"
1900
+
}
1901
+
},
1902
+
{
1903
+
"id": "mappings",
1904
+
"value": [
1905
+
{
1906
+
"options": {
1907
+
"False": {
1908
+
"color": "red",
1909
+
"index": 1,
1910
+
"text": "Disable"
1911
+
},
1912
+
"True": {
1913
+
"color": "green",
1914
+
"index": 0,
1915
+
"text": "Enable"
1916
+
}
1917
+
},
1918
+
"type": "value"
1919
+
}
1920
+
]
1921
+
}
1922
+
]
1923
+
},
1924
+
{
1925
+
"matcher": {
1926
+
"id": "byName",
1927
+
"options": "created"
1928
+
},
1929
+
"properties": [
1930
+
{
1931
+
"id": "unit",
1932
+
"value": "dateTimeAsIso"
1933
+
}
1934
+
]
1935
+
},
1936
+
{
1937
+
"matcher": {
1938
+
"id": "byName",
1939
+
"options": "updated"
1940
+
},
1941
+
"properties": [
1942
+
{
1943
+
"id": "unit",
1944
+
"value": "dateTimeAsIso"
1945
+
}
1946
+
]
1947
+
},
1948
+
{
1949
+
"matcher": {
1950
+
"id": "byName",
1951
+
"options": "name"
1952
+
},
1953
+
"properties": [
1954
+
{
1955
+
"id": "custom.width",
1956
+
"value": 233
1957
+
}
1958
+
]
1959
+
}
1960
+
]
1961
+
},
1962
+
"gridPos": {
1963
+
"h": 10,
1964
+
"w": 24,
1965
+
"x": 0,
1966
+
"y": 29
1967
+
},
1968
+
"id": 11,
1969
+
"options": {
1970
+
"cellHeight": "sm",
1971
+
"footer": {
1972
+
"countRows": false,
1973
+
"enablePagination": true,
1974
+
"fields": [],
1975
+
"reducer": [
1976
+
"sum"
1977
+
],
1978
+
"show": false
1979
+
},
1980
+
"frameIndex": 0,
1981
+
"showHeader": true,
1982
+
"sortBy": [
1983
+
{
1984
+
"desc": true,
1985
+
"displayName": "scheduled"
1986
+
}
1987
+
]
1988
+
},
1989
+
"pluginVersion": "10.1.5",
1990
+
"targets": [
1991
+
{
1992
+
"datasource": {
1993
+
"type": "prometheus",
1994
+
"uid": "${DS_PROMETHEUS}"
1995
+
},
1996
+
"editorMode": "code",
1997
+
"exemplar": false,
1998
+
"expr": "prefect_info_deployment{flow_name=~\"$flow_name\"}",
1999
+
"format": "table",
2000
+
"instant": true,
2001
+
"legendFormat": "",
2002
+
"range": false,
2003
+
"refId": "A"
2004
+
}
2005
+
],
2006
+
"transformations": [
2007
+
{
2008
+
"id": "organize",
2009
+
"options": {
2010
+
"excludeByName": {
2011
+
"Time": true,
2012
+
"Value": true,
2013
+
"__name__": true,
2014
+
"container": true,
2015
+
"created": false,
2016
+
"endpoint": true,
2017
+
"instance": true,
2018
+
"job": true,
2019
+
"name": false,
2020
+
"namespace": true,
2021
+
"pod": true,
2022
+
"service": true,
2023
+
"work_pool_name": true,
2024
+
"work_queue_name": true
2025
+
},
2026
+
"indexByName": {
2027
+
"Time": 7,
2028
+
"Value": 18,
2029
+
"__name__": 8,
2030
+
"container": 9,
2031
+
"created": 5,
2032
+
"deployment_id": 1,
2033
+
"deployment_name": 0,
2034
+
"endpoint": 10,
2035
+
"flow_id": 4,
2036
+
"flow_name": 3,
2037
+
"instance": 11,
2038
+
"is_schedule_active": 2,
2039
+
"job": 12,
2040
+
"namespace": 13,
2041
+
"path": 6,
2042
+
"pod": 14,
2043
+
"service": 15,
2044
+
"work_pool_name": 16,
2045
+
"work_queue_name": 17
2046
+
},
2047
+
"renameByName": {
2048
+
"flow_id": "",
2049
+
"id": "",
2050
+
"is_schedule_active": "scheduled",
2051
+
"job": ""
2052
+
}
2053
+
}
2054
+
}
2055
+
],
2056
+
"type": "table"
2057
+
}
2058
+
],
2059
+
"title": "Deployments",
2060
+
"type": "row"
2061
+
}
2062
+
],
2063
+
"refresh": "30s",
2064
+
"schemaVersion": 38,
2065
+
"style": "dark",
2066
+
"tags": [
2067
+
"prefect",
2068
+
"infrastructure",
2069
+
"chart",
2070
+
"monitoring",
2071
+
"flow-runs"
2072
+
],
2073
+
"templating": {
2074
+
"list": [
2075
+
{
2076
+
"current": {
2077
+
"selected": false,
2078
+
"text": "DS_PROMETHEUS",
2079
+
"value": "DS_PROMETHEUS"
2080
+
},
2081
+
"hide": 2,
2082
+
"includeAll": false,
2083
+
"label": "",
2084
+
"multi": false,
2085
+
"name": "DS_PROMETHEUS",
2086
+
"options": [
2087
+
{
2088
+
"selected": true,
2089
+
"text": "DS_PROMETHEUS",
2090
+
"value": "DS_PROMETHEUS"
2091
+
}
2092
+
],
2093
+
"query": "DS_PROMETHEUS",
2094
+
"skipUrlSync": false,
2095
+
"type": "constant"
2096
+
},
2097
+
{
2098
+
"current": {},
2099
+
"datasource": {
2100
+
"type": "prometheus",
2101
+
"uid": "${DS_PROMETHEUS}"
2102
+
},
2103
+
"definition": "label_values(prefect_info_flows,namespace)",
2104
+
"hide": 0,
2105
+
"includeAll": false,
2106
+
"label": "Namespace",
2107
+
"multi": false,
2108
+
"name": "namespace",
2109
+
"options": [],
2110
+
"query": {
2111
+
"query": "label_values(prefect_info_flows,namespace)",
2112
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
2113
+
},
2114
+
"refresh": 1,
2115
+
"regex": "",
2116
+
"skipUrlSync": false,
2117
+
"sort": 0,
2118
+
"type": "query"
2119
+
},
2120
+
{
2121
+
"current": {},
2122
+
"datasource": {
2123
+
"type": "prometheus",
2124
+
"uid": "${DS_PROMETHEUS}"
2125
+
},
2126
+
"definition": "label_values(prefect_info_flows,flow_name)",
2127
+
"hide": 0,
2128
+
"includeAll": false,
2129
+
"label": "Flow Name",
2130
+
"multi": false,
2131
+
"name": "flow_name",
2132
+
"options": [],
2133
+
"query": {
2134
+
"query": "label_values(prefect_info_flows,flow_name)",
2135
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
2136
+
},
2137
+
"refresh": 1,
2138
+
"regex": "",
2139
+
"skipUrlSync": false,
2140
+
"sort": 0,
2141
+
"type": "query"
2142
+
},
2143
+
{
2144
+
"current": {},
2145
+
"datasource": {
2146
+
"type": "prometheus",
2147
+
"uid": "${DS_PROMETHEUS}"
2148
+
},
2149
+
"definition": "label_values(prefect_info_flow_runs{flow_name=\"$flow_name\"},flow_run_name)",
2150
+
"hide": 0,
2151
+
"includeAll": false,
2152
+
"label": "Flow Run Name",
2153
+
"multi": false,
2154
+
"name": "flow_run_name",
2155
+
"options": [],
2156
+
"query": {
2157
+
"query": "label_values(prefect_info_flow_runs{flow_name=\"$flow_name\"},flow_run_name)",
2158
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
2159
+
},
2160
+
"refresh": 2,
2161
+
"regex": "",
2162
+
"skipUrlSync": false,
2163
+
"sort": 0,
2164
+
"type": "query"
2165
+
},
2166
+
{
2167
+
"current": {},
2168
+
"datasource": {
2169
+
"type": "prometheus",
2170
+
"uid": "${DS_PROMETHEUS}"
2171
+
},
2172
+
"definition": "label_values(prefect_info_flow_runs,state_name)",
2173
+
"hide": 0,
2174
+
"includeAll": true,
2175
+
"label": "Flow Run State",
2176
+
"multi": false,
2177
+
"name": "flow_run_state_name",
2178
+
"options": [],
2179
+
"query": {
2180
+
"query": "label_values(prefect_info_flow_runs,state_name)",
2181
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
2182
+
},
2183
+
"refresh": 1,
2184
+
"regex": "",
2185
+
"skipUrlSync": false,
2186
+
"sort": 0,
2187
+
"type": "query"
2188
+
},
2189
+
{
2190
+
"current": {
2191
+
"selected": false,
2192
+
"text": "",
2193
+
"value": ""
2194
+
},
2195
+
"hide": 0,
2196
+
"label": "Search",
2197
+
"name": "search",
2198
+
"options": [
2199
+
{
2200
+
"selected": true,
2201
+
"text": "",
2202
+
"value": ""
2203
+
}
2204
+
],
2205
+
"query": "",
2206
+
"skipUrlSync": false,
2207
+
"type": "textbox"
2208
+
},
2209
+
{
2210
+
"current": {
2211
+
"selected": false,
2212
+
"text": "Prometheus",
2213
+
"value": "prometheus"
2214
+
},
2215
+
"hide": 2,
2216
+
"includeAll": false,
2217
+
"label": "Data Source",
2218
+
"multi": false,
2219
+
"name": "datasource",
2220
+
"options": [],
2221
+
"query": "prometheus",
2222
+
"refresh": 1,
2223
+
"regex": "",
2224
+
"skipUrlSync": false,
2225
+
"type": "datasource"
2226
+
}
2227
+
]
2228
+
},
2229
+
"time": {
2230
+
"from": "now-24h",
2231
+
"to": "now"
2232
+
},
2233
+
"timepicker": {
2234
+
"hidden": false
2235
+
},
2236
+
"timezone": "browser",
2237
+
"title": "Prefect / Flow Runs Overview",
2238
+
"uid": "prefect-flow-runs-overview",
2239
+
"version": 4,
2240
+
"weekStart": "monday"
2241
+
}
+4189
deploy/dashboards/prefect-platform-overview.json
+4189
deploy/dashboards/prefect-platform-overview.json
···
1
+
{
2
+
"__inputs": [
3
+
{
4
+
"name": "${DS_PROMETHEUS}",
5
+
"label": "Prometheus",
6
+
"description": "",
7
+
"type": "datasource",
8
+
"pluginId": "prometheus",
9
+
"pluginName": "Prometheus"
10
+
},
11
+
{
12
+
"name": "DS_LOKI",
13
+
"label": "Loki",
14
+
"description": "",
15
+
"type": "datasource",
16
+
"pluginId": "loki",
17
+
"pluginName": "Loki"
18
+
}
19
+
],
20
+
"__elements": {},
21
+
"__requires": [
22
+
{
23
+
"type": "grafana",
24
+
"id": "grafana",
25
+
"name": "Grafana",
26
+
"version": "10.2.2"
27
+
},
28
+
{
29
+
"type": "panel",
30
+
"id": "logs",
31
+
"name": "Logs",
32
+
"version": ""
33
+
},
34
+
{
35
+
"type": "datasource",
36
+
"id": "loki",
37
+
"name": "Loki",
38
+
"version": "1.0.0"
39
+
},
40
+
{
41
+
"type": "panel",
42
+
"id": "piechart",
43
+
"name": "Pie chart",
44
+
"version": ""
45
+
},
46
+
{
47
+
"type": "datasource",
48
+
"id": "prometheus",
49
+
"name": "Prometheus",
50
+
"version": "1.0.0"
51
+
},
52
+
{
53
+
"type": "panel",
54
+
"id": "stat",
55
+
"name": "Stat",
56
+
"version": ""
57
+
},
58
+
{
59
+
"type": "panel",
60
+
"id": "table",
61
+
"name": "Table",
62
+
"version": ""
63
+
},
64
+
{
65
+
"type": "panel",
66
+
"id": "timeseries",
67
+
"name": "Time series",
68
+
"version": ""
69
+
}
70
+
],
71
+
"annotations": {
72
+
"list": [
73
+
{
74
+
"builtIn": 1,
75
+
"datasource": {
76
+
"type": "grafana",
77
+
"uid": "-- Grafana --"
78
+
},
79
+
"enable": true,
80
+
"hide": true,
81
+
"iconColor": "rgba(0, 211, 255, 1)",
82
+
"name": "Annotations & Alerts",
83
+
"type": "dashboard"
84
+
}
85
+
]
86
+
},
87
+
"description": "Dashboard for Prefect Exporter chart",
88
+
"editable": true,
89
+
"fiscalYearStartMonth": 0,
90
+
"graphTooltip": 0,
91
+
"id": null,
92
+
"links": [],
93
+
"liveNow": false,
94
+
"panels": [
95
+
{
96
+
"gridPos": {
97
+
"h": 1,
98
+
"w": 24,
99
+
"x": 0,
100
+
"y": 0
101
+
},
102
+
"id": 45,
103
+
"title": "Overview",
104
+
"type": "row"
105
+
},
106
+
{
107
+
"datasource": {
108
+
"type": "prometheus",
109
+
"uid": "${DS_PROMETHEUS}"
110
+
},
111
+
"fieldConfig": {
112
+
"defaults": {
113
+
"color": {
114
+
"mode": "thresholds"
115
+
},
116
+
"decimals": 0,
117
+
"mappings": [],
118
+
"thresholds": {
119
+
"mode": "absolute",
120
+
"steps": [
121
+
{
122
+
"color": "green",
123
+
"value": null
124
+
}
125
+
]
126
+
},
127
+
"unit": "none"
128
+
},
129
+
"overrides": []
130
+
},
131
+
"gridPos": {
132
+
"h": 5,
133
+
"w": 4,
134
+
"x": 0,
135
+
"y": 1
136
+
},
137
+
"id": 2,
138
+
"options": {
139
+
"colorMode": "value",
140
+
"graphMode": "none",
141
+
"justifyMode": "center",
142
+
"orientation": "auto",
143
+
"reduceOptions": {
144
+
"calcs": [
145
+
"lastNotNull"
146
+
],
147
+
"fields": "",
148
+
"values": false
149
+
},
150
+
"textMode": "auto",
151
+
"wideLayout": true
152
+
},
153
+
"pluginVersion": "10.2.2",
154
+
"targets": [
155
+
{
156
+
"datasource": {
157
+
"type": "prometheus",
158
+
"uid": "${DS_PROMETHEUS}"
159
+
},
160
+
"editorMode": "code",
161
+
"expr": "sum(prefect_flows_total)",
162
+
"instant": false,
163
+
"legendFormat": "__auto",
164
+
"range": true,
165
+
"refId": "A"
166
+
}
167
+
],
168
+
"title": "Flows total",
169
+
"type": "stat"
170
+
},
171
+
{
172
+
"datasource": {
173
+
"type": "prometheus",
174
+
"uid": "${DS_PROMETHEUS}"
175
+
},
176
+
"description": "",
177
+
"fieldConfig": {
178
+
"defaults": {
179
+
"color": {
180
+
"mode": "thresholds"
181
+
},
182
+
"decimals": 0,
183
+
"mappings": [],
184
+
"thresholds": {
185
+
"mode": "absolute",
186
+
"steps": [
187
+
{
188
+
"color": "green",
189
+
"value": null
190
+
}
191
+
]
192
+
},
193
+
"unit": "none"
194
+
},
195
+
"overrides": []
196
+
},
197
+
"gridPos": {
198
+
"h": 5,
199
+
"w": 4,
200
+
"x": 4,
201
+
"y": 1
202
+
},
203
+
"id": 52,
204
+
"options": {
205
+
"colorMode": "value",
206
+
"graphMode": "none",
207
+
"justifyMode": "center",
208
+
"orientation": "auto",
209
+
"reduceOptions": {
210
+
"calcs": [
211
+
"lastNotNull"
212
+
],
213
+
"fields": "",
214
+
"values": false
215
+
},
216
+
"textMode": "auto",
217
+
"wideLayout": true
218
+
},
219
+
"pluginVersion": "10.2.2",
220
+
"targets": [
221
+
{
222
+
"datasource": {
223
+
"type": "prometheus",
224
+
"uid": "${DS_PROMETHEUS}"
225
+
},
226
+
"editorMode": "code",
227
+
"expr": "count(prefect_info_flow_runs)",
228
+
"instant": false,
229
+
"legendFormat": "__auto",
230
+
"range": true,
231
+
"refId": "A"
232
+
}
233
+
],
234
+
"title": "Flow Runs total",
235
+
"type": "stat"
236
+
},
237
+
{
238
+
"datasource": {
239
+
"type": "prometheus",
240
+
"uid": "${DS_PROMETHEUS}"
241
+
},
242
+
"fieldConfig": {
243
+
"defaults": {
244
+
"color": {
245
+
"mode": "thresholds"
246
+
},
247
+
"decimals": 0,
248
+
"mappings": [],
249
+
"thresholds": {
250
+
"mode": "absolute",
251
+
"steps": [
252
+
{
253
+
"color": "green",
254
+
"value": null
255
+
}
256
+
]
257
+
},
258
+
"unit": "none"
259
+
},
260
+
"overrides": []
261
+
},
262
+
"gridPos": {
263
+
"h": 5,
264
+
"w": 4,
265
+
"x": 8,
266
+
"y": 1
267
+
},
268
+
"id": 20,
269
+
"options": {
270
+
"colorMode": "value",
271
+
"graphMode": "area",
272
+
"justifyMode": "center",
273
+
"orientation": "auto",
274
+
"reduceOptions": {
275
+
"calcs": [
276
+
"lastNotNull"
277
+
],
278
+
"fields": "",
279
+
"values": false
280
+
},
281
+
"textMode": "auto",
282
+
"wideLayout": true
283
+
},
284
+
"pluginVersion": "10.2.2",
285
+
"targets": [
286
+
{
287
+
"datasource": {
288
+
"type": "prometheus",
289
+
"uid": "${DS_PROMETHEUS}"
290
+
},
291
+
"editorMode": "code",
292
+
"exemplar": false,
293
+
"expr": "count(prefect_info_flow_runs == 1)",
294
+
"instant": false,
295
+
"legendFormat": "__auto",
296
+
"range": true,
297
+
"refId": "A"
298
+
}
299
+
],
300
+
"title": "Flow Runs running",
301
+
"type": "stat"
302
+
},
303
+
{
304
+
"datasource": {
305
+
"type": "prometheus",
306
+
"uid": "${DS_PROMETHEUS}"
307
+
},
308
+
"fieldConfig": {
309
+
"defaults": {
310
+
"color": {
311
+
"mode": "thresholds"
312
+
},
313
+
"decimals": 0,
314
+
"mappings": [],
315
+
"thresholds": {
316
+
"mode": "absolute",
317
+
"steps": [
318
+
{
319
+
"color": "green",
320
+
"value": null
321
+
}
322
+
]
323
+
},
324
+
"unit": "none"
325
+
},
326
+
"overrides": []
327
+
},
328
+
"gridPos": {
329
+
"h": 5,
330
+
"w": 4,
331
+
"x": 12,
332
+
"y": 1
333
+
},
334
+
"id": 50,
335
+
"options": {
336
+
"colorMode": "value",
337
+
"graphMode": "none",
338
+
"justifyMode": "center",
339
+
"orientation": "auto",
340
+
"reduceOptions": {
341
+
"calcs": [
342
+
"lastNotNull"
343
+
],
344
+
"fields": "",
345
+
"values": false
346
+
},
347
+
"textMode": "auto",
348
+
"wideLayout": true
349
+
},
350
+
"pluginVersion": "10.2.2",
351
+
"targets": [
352
+
{
353
+
"datasource": {
354
+
"type": "prometheus",
355
+
"uid": "${DS_PROMETHEUS}"
356
+
},
357
+
"editorMode": "code",
358
+
"expr": "sum(prefect_work_pools_total)",
359
+
"instant": false,
360
+
"legendFormat": "__auto",
361
+
"range": true,
362
+
"refId": "A"
363
+
}
364
+
],
365
+
"title": "Work Pools total",
366
+
"type": "stat"
367
+
},
368
+
{
369
+
"datasource": {
370
+
"type": "prometheus",
371
+
"uid": "${DS_PROMETHEUS}"
372
+
},
373
+
"description": "",
374
+
"fieldConfig": {
375
+
"defaults": {
376
+
"color": {
377
+
"mode": "palette-classic"
378
+
},
379
+
"custom": {
380
+
"hideFrom": {
381
+
"legend": false,
382
+
"tooltip": false,
383
+
"viz": false
384
+
}
385
+
},
386
+
"decimals": 0,
387
+
"mappings": [],
388
+
"unit": "none"
389
+
},
390
+
"overrides": [
391
+
{
392
+
"matcher": {
393
+
"id": "byName",
394
+
"options": "Failed"
395
+
},
396
+
"properties": [
397
+
{
398
+
"id": "color",
399
+
"value": {
400
+
"fixedColor": "red",
401
+
"mode": "fixed"
402
+
}
403
+
}
404
+
]
405
+
},
406
+
{
407
+
"matcher": {
408
+
"id": "byName",
409
+
"options": "Running"
410
+
},
411
+
"properties": [
412
+
{
413
+
"id": "color",
414
+
"value": {
415
+
"fixedColor": "blue",
416
+
"mode": "fixed"
417
+
}
418
+
}
419
+
]
420
+
},
421
+
{
422
+
"matcher": {
423
+
"id": "byName",
424
+
"options": "Cancelled"
425
+
},
426
+
"properties": [
427
+
{
428
+
"id": "color",
429
+
"value": {
430
+
"fixedColor": "yellow",
431
+
"mode": "fixed"
432
+
}
433
+
}
434
+
]
435
+
},
436
+
{
437
+
"matcher": {
438
+
"id": "byName",
439
+
"options": "Completed"
440
+
},
441
+
"properties": [
442
+
{
443
+
"id": "color",
444
+
"value": {
445
+
"fixedColor": "green",
446
+
"mode": "fixed"
447
+
}
448
+
}
449
+
]
450
+
}
451
+
]
452
+
},
453
+
"gridPos": {
454
+
"h": 10,
455
+
"w": 8,
456
+
"x": 16,
457
+
"y": 1
458
+
},
459
+
"id": 21,
460
+
"options": {
461
+
"legend": {
462
+
"displayMode": "table",
463
+
"placement": "right",
464
+
"showLegend": true,
465
+
"values": [
466
+
"percent"
467
+
]
468
+
},
469
+
"pieType": "pie",
470
+
"reduceOptions": {
471
+
"calcs": [
472
+
"lastNotNull"
473
+
],
474
+
"fields": "",
475
+
"values": false
476
+
},
477
+
"tooltip": {
478
+
"mode": "single",
479
+
"sort": "none"
480
+
}
481
+
},
482
+
"pluginVersion": "10.1.1",
483
+
"targets": [
484
+
{
485
+
"datasource": {
486
+
"type": "prometheus",
487
+
"uid": "${DS_PROMETHEUS}"
488
+
},
489
+
"editorMode": "code",
490
+
"expr": "count by (state_name) (prefect_info_flow_runs)",
491
+
"instant": false,
492
+
"legendFormat": "{{label_name}}",
493
+
"range": true,
494
+
"refId": "A"
495
+
}
496
+
],
497
+
"title": "Flow Runs by state",
498
+
"type": "piechart"
499
+
},
500
+
{
501
+
"datasource": {
502
+
"type": "prometheus",
503
+
"uid": "${DS_PROMETHEUS}"
504
+
},
505
+
"fieldConfig": {
506
+
"defaults": {
507
+
"color": {
508
+
"mode": "thresholds"
509
+
},
510
+
"decimals": 0,
511
+
"mappings": [],
512
+
"thresholds": {
513
+
"mode": "absolute",
514
+
"steps": [
515
+
{
516
+
"color": "green",
517
+
"value": null
518
+
}
519
+
]
520
+
},
521
+
"unit": "none"
522
+
},
523
+
"overrides": []
524
+
},
525
+
"gridPos": {
526
+
"h": 5,
527
+
"w": 4,
528
+
"x": 0,
529
+
"y": 6
530
+
},
531
+
"id": 3,
532
+
"options": {
533
+
"colorMode": "value",
534
+
"graphMode": "none",
535
+
"justifyMode": "center",
536
+
"orientation": "auto",
537
+
"reduceOptions": {
538
+
"calcs": [
539
+
"lastNotNull"
540
+
],
541
+
"fields": "",
542
+
"values": false
543
+
},
544
+
"textMode": "auto",
545
+
"wideLayout": true
546
+
},
547
+
"pluginVersion": "10.2.2",
548
+
"targets": [
549
+
{
550
+
"datasource": {
551
+
"type": "prometheus",
552
+
"uid": "${DS_PROMETHEUS}"
553
+
},
554
+
"editorMode": "code",
555
+
"expr": "sum(prefect_deployments_total)",
556
+
"instant": false,
557
+
"legendFormat": "__auto",
558
+
"range": true,
559
+
"refId": "A"
560
+
}
561
+
],
562
+
"title": "Deployments total",
563
+
"type": "stat"
564
+
},
565
+
{
566
+
"datasource": {
567
+
"type": "prometheus",
568
+
"uid": "${DS_PROMETHEUS}"
569
+
},
570
+
"fieldConfig": {
571
+
"defaults": {
572
+
"color": {
573
+
"mode": "thresholds"
574
+
},
575
+
"decimals": 0,
576
+
"mappings": [],
577
+
"noValue": "0",
578
+
"thresholds": {
579
+
"mode": "absolute",
580
+
"steps": [
581
+
{
582
+
"color": "green",
583
+
"value": null
584
+
},
585
+
{
586
+
"color": "yellow",
587
+
"value": 1
588
+
}
589
+
]
590
+
},
591
+
"unit": "none"
592
+
},
593
+
"overrides": []
594
+
},
595
+
"gridPos": {
596
+
"h": 5,
597
+
"w": 4,
598
+
"x": 4,
599
+
"y": 6
600
+
},
601
+
"id": 14,
602
+
"options": {
603
+
"colorMode": "value",
604
+
"graphMode": "none",
605
+
"justifyMode": "center",
606
+
"orientation": "auto",
607
+
"reduceOptions": {
608
+
"calcs": [
609
+
"lastNotNull"
610
+
],
611
+
"fields": "",
612
+
"values": false
613
+
},
614
+
"textMode": "auto",
615
+
"wideLayout": true
616
+
},
617
+
"pluginVersion": "10.2.2",
618
+
"targets": [
619
+
{
620
+
"datasource": {
621
+
"type": "prometheus",
622
+
"uid": "${DS_PROMETHEUS}"
623
+
},
624
+
"editorMode": "code",
625
+
"expr": "count(prefect_info_deployment{is_schedule_active=\"False\"} == 1)",
626
+
"instant": false,
627
+
"legendFormat": "__auto",
628
+
"range": true,
629
+
"refId": "A"
630
+
}
631
+
],
632
+
"title": "Deployments paused",
633
+
"type": "stat"
634
+
},
635
+
{
636
+
"datasource": {
637
+
"type": "prometheus",
638
+
"uid": "${DS_PROMETHEUS}"
639
+
},
640
+
"fieldConfig": {
641
+
"defaults": {
642
+
"color": {
643
+
"mode": "thresholds"
644
+
},
645
+
"decimals": 0,
646
+
"mappings": [],
647
+
"thresholds": {
648
+
"mode": "absolute",
649
+
"steps": [
650
+
{
651
+
"color": "green",
652
+
"value": null
653
+
}
654
+
]
655
+
},
656
+
"unit": "s"
657
+
},
658
+
"overrides": []
659
+
},
660
+
"gridPos": {
661
+
"h": 5,
662
+
"w": 4,
663
+
"x": 8,
664
+
"y": 6
665
+
},
666
+
"id": 49,
667
+
"options": {
668
+
"colorMode": "value",
669
+
"graphMode": "none",
670
+
"justifyMode": "center",
671
+
"orientation": "auto",
672
+
"reduceOptions": {
673
+
"calcs": [
674
+
"lastNotNull"
675
+
],
676
+
"fields": "",
677
+
"values": false
678
+
},
679
+
"textMode": "auto",
680
+
"wideLayout": true
681
+
},
682
+
"pluginVersion": "10.2.2",
683
+
"targets": [
684
+
{
685
+
"datasource": {
686
+
"type": "prometheus",
687
+
"uid": "${DS_PROMETHEUS}"
688
+
},
689
+
"editorMode": "code",
690
+
"expr": "avg(prefect_flow_runs_total_run_time)",
691
+
"instant": false,
692
+
"legendFormat": "__auto",
693
+
"range": true,
694
+
"refId": "A"
695
+
}
696
+
],
697
+
"title": "Flow Runs average run time",
698
+
"type": "stat"
699
+
},
700
+
{
701
+
"datasource": {
702
+
"type": "prometheus",
703
+
"uid": "${DS_PROMETHEUS}"
704
+
},
705
+
"fieldConfig": {
706
+
"defaults": {
707
+
"color": {
708
+
"mode": "thresholds"
709
+
},
710
+
"decimals": 0,
711
+
"mappings": [],
712
+
"thresholds": {
713
+
"mode": "absolute",
714
+
"steps": [
715
+
{
716
+
"color": "green",
717
+
"value": null
718
+
}
719
+
]
720
+
},
721
+
"unit": "none"
722
+
},
723
+
"overrides": []
724
+
},
725
+
"gridPos": {
726
+
"h": 5,
727
+
"w": 4,
728
+
"x": 12,
729
+
"y": 6
730
+
},
731
+
"id": 51,
732
+
"options": {
733
+
"colorMode": "value",
734
+
"graphMode": "none",
735
+
"justifyMode": "center",
736
+
"orientation": "auto",
737
+
"reduceOptions": {
738
+
"calcs": [
739
+
"lastNotNull"
740
+
],
741
+
"fields": "",
742
+
"values": false
743
+
},
744
+
"textMode": "auto",
745
+
"wideLayout": true
746
+
},
747
+
"pluginVersion": "10.2.2",
748
+
"targets": [
749
+
{
750
+
"datasource": {
751
+
"type": "prometheus",
752
+
"uid": "${DS_PROMETHEUS}"
753
+
},
754
+
"editorMode": "code",
755
+
"expr": "sum(prefect_work_queues_total)",
756
+
"instant": false,
757
+
"legendFormat": "__auto",
758
+
"range": true,
759
+
"refId": "A"
760
+
}
761
+
],
762
+
"title": "Work Queues total",
763
+
"type": "stat"
764
+
},
765
+
{
766
+
"collapsed": true,
767
+
"gridPos": {
768
+
"h": 1,
769
+
"w": 24,
770
+
"x": 0,
771
+
"y": 11
772
+
},
773
+
"id": 22,
774
+
"panels": [
775
+
{
776
+
"datasource": {
777
+
"type": "prometheus",
778
+
"uid": "${DS_PROMETHEUS}"
779
+
},
780
+
"fieldConfig": {
781
+
"defaults": {
782
+
"color": {
783
+
"mode": "thresholds"
784
+
},
785
+
"mappings": [
786
+
{
787
+
"options": {
788
+
"0": {
789
+
"index": 0,
790
+
"text": "DOWN"
791
+
},
792
+
"1": {
793
+
"index": 1,
794
+
"text": "UP"
795
+
},
796
+
"null": {
797
+
"index": 2,
798
+
"text": "N/A"
799
+
}
800
+
},
801
+
"type": "value"
802
+
}
803
+
],
804
+
"thresholds": {
805
+
"mode": "absolute",
806
+
"steps": [
807
+
{
808
+
"color": "red",
809
+
"value": null
810
+
},
811
+
{
812
+
"color": "orange",
813
+
"value": 1
814
+
},
815
+
{
816
+
"color": "dark-green",
817
+
"value": 1
818
+
}
819
+
]
820
+
}
821
+
},
822
+
"overrides": []
823
+
},
824
+
"gridPos": {
825
+
"h": 7,
826
+
"w": 2,
827
+
"x": 0,
828
+
"y": 12
829
+
},
830
+
"id": 26,
831
+
"options": {
832
+
"colorMode": "background_solid",
833
+
"graphMode": "none",
834
+
"justifyMode": "auto",
835
+
"orientation": "auto",
836
+
"reduceOptions": {
837
+
"calcs": [
838
+
"lastNotNull"
839
+
],
840
+
"fields": "",
841
+
"values": false
842
+
},
843
+
"textMode": "auto",
844
+
"wideLayout": true
845
+
},
846
+
"pluginVersion": "10.2.2",
847
+
"targets": [
848
+
{
849
+
"datasource": {
850
+
"type": "prometheus",
851
+
"uid": "${DS_PROMETHEUS}"
852
+
},
853
+
"editorMode": "code",
854
+
"expr": "sum(probe_success{instance=~\".*prefect.*health\"})",
855
+
"instant": false,
856
+
"legendFormat": "__auto",
857
+
"range": true,
858
+
"refId": "A"
859
+
}
860
+
],
861
+
"title": "Status",
862
+
"type": "stat"
863
+
},
864
+
{
865
+
"datasource": {
866
+
"type": "prometheus",
867
+
"uid": "${DS_PROMETHEUS}"
868
+
},
869
+
"fieldConfig": {
870
+
"defaults": {
871
+
"color": {
872
+
"mode": "thresholds"
873
+
},
874
+
"decimals": 0,
875
+
"mappings": [],
876
+
"thresholds": {
877
+
"mode": "absolute",
878
+
"steps": [
879
+
{
880
+
"color": "red",
881
+
"value": null
882
+
},
883
+
{
884
+
"color": "green",
885
+
"value": 1
886
+
}
887
+
]
888
+
},
889
+
"unit": "none"
890
+
},
891
+
"overrides": []
892
+
},
893
+
"gridPos": {
894
+
"h": 7,
895
+
"w": 2,
896
+
"x": 2,
897
+
"y": 12
898
+
},
899
+
"id": 15,
900
+
"options": {
901
+
"colorMode": "value",
902
+
"graphMode": "area",
903
+
"justifyMode": "center",
904
+
"orientation": "auto",
905
+
"reduceOptions": {
906
+
"calcs": [
907
+
"lastNotNull"
908
+
],
909
+
"fields": "",
910
+
"values": false
911
+
},
912
+
"textMode": "auto",
913
+
"wideLayout": true
914
+
},
915
+
"pluginVersion": "10.2.2",
916
+
"targets": [
917
+
{
918
+
"datasource": {
919
+
"type": "prometheus",
920
+
"uid": "${DS_PROMETHEUS}"
921
+
},
922
+
"editorMode": "code",
923
+
"expr": "sum(kube_pod_info{namespace=\"$namespace\", pod=~\"prefect-server.*\"})",
924
+
"instant": false,
925
+
"legendFormat": "__auto",
926
+
"range": true,
927
+
"refId": "A"
928
+
}
929
+
],
930
+
"title": "Prefect Server replicas",
931
+
"type": "stat"
932
+
},
933
+
{
934
+
"datasource": {
935
+
"type": "loki",
936
+
"uid": "${DS_LOKI}"
937
+
},
938
+
"gridPos": {
939
+
"h": 7,
940
+
"w": 20,
941
+
"x": 4,
942
+
"y": 12
943
+
},
944
+
"id": 38,
945
+
"options": {
946
+
"dedupStrategy": "none",
947
+
"enableLogDetails": true,
948
+
"prettifyLogMessage": false,
949
+
"showCommonLabels": false,
950
+
"showLabels": false,
951
+
"showTime": false,
952
+
"sortOrder": "Descending",
953
+
"wrapLogMessage": false
954
+
},
955
+
"targets": [
956
+
{
957
+
"datasource": {
958
+
"type": "loki",
959
+
"uid": "${DS_LOKI}"
960
+
},
961
+
"editorMode": "code",
962
+
"expr": "{namespace=\"$namespace\", pod=~\"prefect-server.*\"}",
963
+
"queryType": "range",
964
+
"refId": "A"
965
+
}
966
+
],
967
+
"title": "Logs",
968
+
"type": "logs"
969
+
},
970
+
{
971
+
"datasource": {
972
+
"type": "prometheus",
973
+
"uid": "${DS_PROMETHEUS}"
974
+
},
975
+
"fieldConfig": {
976
+
"defaults": {
977
+
"color": {
978
+
"mode": "thresholds"
979
+
},
980
+
"custom": {
981
+
"align": "auto",
982
+
"cellOptions": {
983
+
"type": "auto"
984
+
},
985
+
"filterable": false,
986
+
"inspect": false
987
+
},
988
+
"decimals": 2,
989
+
"displayName": "",
990
+
"mappings": [],
991
+
"noValue": "-",
992
+
"thresholds": {
993
+
"mode": "absolute",
994
+
"steps": [
995
+
{
996
+
"color": "green",
997
+
"value": null
998
+
},
999
+
{
1000
+
"color": "red",
1001
+
"value": 80
1002
+
}
1003
+
]
1004
+
},
1005
+
"unit": "short"
1006
+
},
1007
+
"overrides": [
1008
+
{
1009
+
"matcher": {
1010
+
"id": "byName",
1011
+
"options": "Time"
1012
+
},
1013
+
"properties": [
1014
+
{
1015
+
"id": "displayName",
1016
+
"value": "Time"
1017
+
},
1018
+
{
1019
+
"id": "custom.align"
1020
+
}
1021
+
]
1022
+
},
1023
+
{
1024
+
"matcher": {
1025
+
"id": "byName",
1026
+
"options": "Value #A"
1027
+
},
1028
+
"properties": [
1029
+
{
1030
+
"id": "displayName",
1031
+
"value": "CPU Usage"
1032
+
},
1033
+
{
1034
+
"id": "unit",
1035
+
"value": "short"
1036
+
},
1037
+
{
1038
+
"id": "decimals",
1039
+
"value": 2
1040
+
},
1041
+
{
1042
+
"id": "custom.align"
1043
+
}
1044
+
]
1045
+
},
1046
+
{
1047
+
"matcher": {
1048
+
"id": "byName",
1049
+
"options": "Value #B"
1050
+
},
1051
+
"properties": [
1052
+
{
1053
+
"id": "displayName",
1054
+
"value": "CPU Requests"
1055
+
},
1056
+
{
1057
+
"id": "unit",
1058
+
"value": "short"
1059
+
},
1060
+
{
1061
+
"id": "decimals",
1062
+
"value": 2
1063
+
},
1064
+
{
1065
+
"id": "custom.align"
1066
+
}
1067
+
]
1068
+
},
1069
+
{
1070
+
"matcher": {
1071
+
"id": "byName",
1072
+
"options": "Value #C"
1073
+
},
1074
+
"properties": [
1075
+
{
1076
+
"id": "displayName",
1077
+
"value": "CPU Requests %"
1078
+
},
1079
+
{
1080
+
"id": "unit",
1081
+
"value": "percentunit"
1082
+
},
1083
+
{
1084
+
"id": "decimals",
1085
+
"value": 2
1086
+
},
1087
+
{
1088
+
"id": "custom.align"
1089
+
}
1090
+
]
1091
+
},
1092
+
{
1093
+
"matcher": {
1094
+
"id": "byName",
1095
+
"options": "Value #D"
1096
+
},
1097
+
"properties": [
1098
+
{
1099
+
"id": "displayName",
1100
+
"value": "CPU Limits"
1101
+
},
1102
+
{
1103
+
"id": "unit",
1104
+
"value": "short"
1105
+
},
1106
+
{
1107
+
"id": "decimals",
1108
+
"value": 2
1109
+
},
1110
+
{
1111
+
"id": "custom.align"
1112
+
}
1113
+
]
1114
+
},
1115
+
{
1116
+
"matcher": {
1117
+
"id": "byName",
1118
+
"options": "Value #E"
1119
+
},
1120
+
"properties": [
1121
+
{
1122
+
"id": "displayName",
1123
+
"value": "CPU Limits %"
1124
+
},
1125
+
{
1126
+
"id": "unit",
1127
+
"value": "percentunit"
1128
+
},
1129
+
{
1130
+
"id": "decimals",
1131
+
"value": 2
1132
+
},
1133
+
{
1134
+
"id": "custom.align"
1135
+
}
1136
+
]
1137
+
},
1138
+
{
1139
+
"matcher": {
1140
+
"id": "byName",
1141
+
"options": "pod"
1142
+
},
1143
+
"properties": [
1144
+
{
1145
+
"id": "displayName",
1146
+
"value": "Pod"
1147
+
},
1148
+
{
1149
+
"id": "unit",
1150
+
"value": "short"
1151
+
},
1152
+
{
1153
+
"id": "decimals",
1154
+
"value": 2
1155
+
},
1156
+
{
1157
+
"id": "links",
1158
+
"value": [
1159
+
{
1160
+
"targetBlank": false,
1161
+
"title": "Drill down",
1162
+
"url": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell"
1163
+
}
1164
+
]
1165
+
},
1166
+
{
1167
+
"id": "custom.align"
1168
+
}
1169
+
]
1170
+
}
1171
+
]
1172
+
},
1173
+
"gridPos": {
1174
+
"h": 6,
1175
+
"w": 12,
1176
+
"x": 0,
1177
+
"y": 19
1178
+
},
1179
+
"id": 25,
1180
+
"interval": "1m",
1181
+
"links": [],
1182
+
"options": {
1183
+
"cellHeight": "sm",
1184
+
"footer": {
1185
+
"countRows": false,
1186
+
"fields": "",
1187
+
"reducer": [
1188
+
"sum"
1189
+
],
1190
+
"show": false
1191
+
},
1192
+
"showHeader": true,
1193
+
"sortBy": [
1194
+
{
1195
+
"desc": true,
1196
+
"displayName": "CPU Usage"
1197
+
}
1198
+
]
1199
+
},
1200
+
"pluginVersion": "10.2.2",
1201
+
"targets": [
1202
+
{
1203
+
"datasource": {
1204
+
"uid": "$datasource"
1205
+
},
1206
+
"editorMode": "code",
1207
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1208
+
"format": "table",
1209
+
"instant": true,
1210
+
"intervalFactor": 2,
1211
+
"legendFormat": "",
1212
+
"refId": "A"
1213
+
},
1214
+
{
1215
+
"datasource": {
1216
+
"uid": "$datasource"
1217
+
},
1218
+
"editorMode": "code",
1219
+
"expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1220
+
"format": "table",
1221
+
"instant": true,
1222
+
"intervalFactor": 2,
1223
+
"legendFormat": "",
1224
+
"refId": "B"
1225
+
},
1226
+
{
1227
+
"datasource": {
1228
+
"uid": "$datasource"
1229
+
},
1230
+
"editorMode": "code",
1231
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1232
+
"format": "table",
1233
+
"instant": true,
1234
+
"intervalFactor": 2,
1235
+
"legendFormat": "",
1236
+
"refId": "C"
1237
+
},
1238
+
{
1239
+
"datasource": {
1240
+
"uid": "$datasource"
1241
+
},
1242
+
"editorMode": "code",
1243
+
"expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1244
+
"format": "table",
1245
+
"instant": true,
1246
+
"intervalFactor": 2,
1247
+
"legendFormat": "",
1248
+
"refId": "D"
1249
+
},
1250
+
{
1251
+
"datasource": {
1252
+
"uid": "$datasource"
1253
+
},
1254
+
"editorMode": "code",
1255
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1256
+
"format": "table",
1257
+
"instant": true,
1258
+
"intervalFactor": 2,
1259
+
"legendFormat": "",
1260
+
"refId": "E"
1261
+
}
1262
+
],
1263
+
"title": "CPU Quota",
1264
+
"transformations": [
1265
+
{
1266
+
"id": "merge",
1267
+
"options": {
1268
+
"reducers": []
1269
+
}
1270
+
},
1271
+
{
1272
+
"id": "organize",
1273
+
"options": {
1274
+
"excludeByName": {
1275
+
"Time": true
1276
+
},
1277
+
"indexByName": {},
1278
+
"renameByName": {}
1279
+
}
1280
+
}
1281
+
],
1282
+
"type": "table"
1283
+
},
1284
+
{
1285
+
"datasource": {
1286
+
"type": "prometheus",
1287
+
"uid": "${DS_PROMETHEUS}"
1288
+
},
1289
+
"fieldConfig": {
1290
+
"defaults": {
1291
+
"color": {
1292
+
"mode": "palette-classic"
1293
+
},
1294
+
"custom": {
1295
+
"axisBorderShow": false,
1296
+
"axisCenteredZero": false,
1297
+
"axisColorMode": "text",
1298
+
"axisLabel": "",
1299
+
"axisPlacement": "auto",
1300
+
"barAlignment": 0,
1301
+
"drawStyle": "line",
1302
+
"fillOpacity": 100,
1303
+
"gradientMode": "none",
1304
+
"hideFrom": {
1305
+
"legend": false,
1306
+
"tooltip": false,
1307
+
"viz": false
1308
+
},
1309
+
"insertNulls": false,
1310
+
"lineInterpolation": "linear",
1311
+
"lineWidth": 0,
1312
+
"pointSize": 5,
1313
+
"scaleDistribution": {
1314
+
"type": "linear"
1315
+
},
1316
+
"showPoints": "never",
1317
+
"spanNulls": false,
1318
+
"stacking": {
1319
+
"group": "A",
1320
+
"mode": "normal"
1321
+
},
1322
+
"thresholdsStyle": {
1323
+
"mode": "off"
1324
+
}
1325
+
},
1326
+
"mappings": [],
1327
+
"min": 0,
1328
+
"thresholds": {
1329
+
"mode": "absolute",
1330
+
"steps": [
1331
+
{
1332
+
"color": "green",
1333
+
"value": null
1334
+
},
1335
+
{
1336
+
"color": "red",
1337
+
"value": 80
1338
+
}
1339
+
]
1340
+
},
1341
+
"unit": "short"
1342
+
},
1343
+
"overrides": [
1344
+
{
1345
+
"matcher": {
1346
+
"id": "byName",
1347
+
"options": "quota - requests"
1348
+
},
1349
+
"properties": [
1350
+
{
1351
+
"id": "color",
1352
+
"value": {
1353
+
"fixedColor": "#F2495C",
1354
+
"mode": "fixed"
1355
+
}
1356
+
},
1357
+
{
1358
+
"id": "custom.fillOpacity",
1359
+
"value": 0
1360
+
},
1361
+
{
1362
+
"id": "custom.lineWidth",
1363
+
"value": 2
1364
+
},
1365
+
{
1366
+
"id": "custom.stacking",
1367
+
"value": {
1368
+
"group": "A",
1369
+
"mode": "none"
1370
+
}
1371
+
},
1372
+
{
1373
+
"id": "custom.lineStyle",
1374
+
"value": {
1375
+
"dash": [
1376
+
10,
1377
+
10
1378
+
],
1379
+
"fill": "dash"
1380
+
}
1381
+
}
1382
+
]
1383
+
},
1384
+
{
1385
+
"matcher": {
1386
+
"id": "byName",
1387
+
"options": "quota - limits"
1388
+
},
1389
+
"properties": [
1390
+
{
1391
+
"id": "color",
1392
+
"value": {
1393
+
"fixedColor": "#FF9830",
1394
+
"mode": "fixed"
1395
+
}
1396
+
},
1397
+
{
1398
+
"id": "custom.fillOpacity",
1399
+
"value": 0
1400
+
},
1401
+
{
1402
+
"id": "custom.lineWidth",
1403
+
"value": 2
1404
+
},
1405
+
{
1406
+
"id": "custom.stacking",
1407
+
"value": {
1408
+
"group": "A",
1409
+
"mode": "none"
1410
+
}
1411
+
},
1412
+
{
1413
+
"id": "custom.lineStyle",
1414
+
"value": {
1415
+
"dash": [
1416
+
10,
1417
+
10
1418
+
],
1419
+
"fill": "dash"
1420
+
}
1421
+
}
1422
+
]
1423
+
}
1424
+
]
1425
+
},
1426
+
"gridPos": {
1427
+
"h": 6,
1428
+
"w": 12,
1429
+
"x": 12,
1430
+
"y": 19
1431
+
},
1432
+
"id": 42,
1433
+
"interval": "1m",
1434
+
"links": [],
1435
+
"options": {
1436
+
"legend": {
1437
+
"calcs": [],
1438
+
"displayMode": "table",
1439
+
"placement": "right",
1440
+
"showLegend": true
1441
+
},
1442
+
"tooltip": {
1443
+
"mode": "single",
1444
+
"sort": "none"
1445
+
}
1446
+
},
1447
+
"pluginVersion": "10.1.1",
1448
+
"targets": [
1449
+
{
1450
+
"datasource": {
1451
+
"uid": "$datasource"
1452
+
},
1453
+
"editorMode": "code",
1454
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1455
+
"format": "time_series",
1456
+
"intervalFactor": 2,
1457
+
"legendFormat": "{{pod}}",
1458
+
"range": true,
1459
+
"refId": "A"
1460
+
}
1461
+
],
1462
+
"title": "CPU Usage",
1463
+
"type": "timeseries"
1464
+
},
1465
+
{
1466
+
"datasource": {
1467
+
"type": "prometheus",
1468
+
"uid": "$datasource"
1469
+
},
1470
+
"fieldConfig": {
1471
+
"defaults": {
1472
+
"color": {
1473
+
"mode": "thresholds"
1474
+
},
1475
+
"custom": {
1476
+
"align": "auto",
1477
+
"cellOptions": {
1478
+
"type": "auto"
1479
+
},
1480
+
"filterable": false,
1481
+
"inspect": false
1482
+
},
1483
+
"decimals": 2,
1484
+
"displayName": "",
1485
+
"mappings": [],
1486
+
"thresholds": {
1487
+
"mode": "absolute",
1488
+
"steps": [
1489
+
{
1490
+
"color": "green",
1491
+
"value": null
1492
+
},
1493
+
{
1494
+
"color": "red",
1495
+
"value": 80
1496
+
}
1497
+
]
1498
+
},
1499
+
"unit": "short"
1500
+
},
1501
+
"overrides": [
1502
+
{
1503
+
"matcher": {
1504
+
"id": "byName",
1505
+
"options": "Time"
1506
+
},
1507
+
"properties": [
1508
+
{
1509
+
"id": "displayName",
1510
+
"value": "Time"
1511
+
},
1512
+
{
1513
+
"id": "custom.align"
1514
+
}
1515
+
]
1516
+
},
1517
+
{
1518
+
"matcher": {
1519
+
"id": "byName",
1520
+
"options": "Value #A"
1521
+
},
1522
+
"properties": [
1523
+
{
1524
+
"id": "displayName",
1525
+
"value": "Mem Usage"
1526
+
},
1527
+
{
1528
+
"id": "unit",
1529
+
"value": "bytes"
1530
+
},
1531
+
{
1532
+
"id": "decimals",
1533
+
"value": 2
1534
+
},
1535
+
{
1536
+
"id": "custom.align"
1537
+
}
1538
+
]
1539
+
},
1540
+
{
1541
+
"matcher": {
1542
+
"id": "byName",
1543
+
"options": "Value #B"
1544
+
},
1545
+
"properties": [
1546
+
{
1547
+
"id": "displayName",
1548
+
"value": "Mem Requests"
1549
+
},
1550
+
{
1551
+
"id": "unit",
1552
+
"value": "bytes"
1553
+
},
1554
+
{
1555
+
"id": "decimals",
1556
+
"value": 2
1557
+
},
1558
+
{
1559
+
"id": "custom.align"
1560
+
}
1561
+
]
1562
+
},
1563
+
{
1564
+
"matcher": {
1565
+
"id": "byName",
1566
+
"options": "Value #C"
1567
+
},
1568
+
"properties": [
1569
+
{
1570
+
"id": "displayName",
1571
+
"value": "Mem Requests %"
1572
+
},
1573
+
{
1574
+
"id": "unit",
1575
+
"value": "percentunit"
1576
+
},
1577
+
{
1578
+
"id": "decimals",
1579
+
"value": 2
1580
+
},
1581
+
{
1582
+
"id": "custom.align"
1583
+
}
1584
+
]
1585
+
},
1586
+
{
1587
+
"matcher": {
1588
+
"id": "byName",
1589
+
"options": "Value #D"
1590
+
},
1591
+
"properties": [
1592
+
{
1593
+
"id": "displayName",
1594
+
"value": "Mem Limits"
1595
+
},
1596
+
{
1597
+
"id": "unit",
1598
+
"value": "bytes"
1599
+
},
1600
+
{
1601
+
"id": "decimals",
1602
+
"value": 2
1603
+
},
1604
+
{
1605
+
"id": "custom.align"
1606
+
}
1607
+
]
1608
+
},
1609
+
{
1610
+
"matcher": {
1611
+
"id": "byName",
1612
+
"options": "Value #E"
1613
+
},
1614
+
"properties": [
1615
+
{
1616
+
"id": "displayName",
1617
+
"value": "Mem Limits %"
1618
+
},
1619
+
{
1620
+
"id": "unit",
1621
+
"value": "percentunit"
1622
+
},
1623
+
{
1624
+
"id": "decimals",
1625
+
"value": 2
1626
+
},
1627
+
{
1628
+
"id": "custom.align"
1629
+
}
1630
+
]
1631
+
},
1632
+
{
1633
+
"matcher": {
1634
+
"id": "byName",
1635
+
"options": "Value #F"
1636
+
},
1637
+
"properties": [
1638
+
{
1639
+
"id": "displayName",
1640
+
"value": "Mem Usage (RSS)"
1641
+
},
1642
+
{
1643
+
"id": "unit",
1644
+
"value": "bytes"
1645
+
},
1646
+
{
1647
+
"id": "decimals",
1648
+
"value": 2
1649
+
},
1650
+
{
1651
+
"id": "custom.align"
1652
+
}
1653
+
]
1654
+
},
1655
+
{
1656
+
"matcher": {
1657
+
"id": "byName",
1658
+
"options": "Value #G"
1659
+
},
1660
+
"properties": [
1661
+
{
1662
+
"id": "displayName",
1663
+
"value": "Mem Usage (Cache)"
1664
+
},
1665
+
{
1666
+
"id": "unit",
1667
+
"value": "bytes"
1668
+
},
1669
+
{
1670
+
"id": "decimals",
1671
+
"value": 2
1672
+
},
1673
+
{
1674
+
"id": "custom.align"
1675
+
}
1676
+
]
1677
+
},
1678
+
{
1679
+
"matcher": {
1680
+
"id": "byName",
1681
+
"options": "Value #H"
1682
+
},
1683
+
"properties": [
1684
+
{
1685
+
"id": "displayName",
1686
+
"value": "Mem Usage (Swap)"
1687
+
},
1688
+
{
1689
+
"id": "unit",
1690
+
"value": "bytes"
1691
+
},
1692
+
{
1693
+
"id": "decimals",
1694
+
"value": 2
1695
+
},
1696
+
{
1697
+
"id": "custom.align"
1698
+
}
1699
+
]
1700
+
},
1701
+
{
1702
+
"matcher": {
1703
+
"id": "byName",
1704
+
"options": "pod"
1705
+
},
1706
+
"properties": [
1707
+
{
1708
+
"id": "displayName",
1709
+
"value": "Pod"
1710
+
},
1711
+
{
1712
+
"id": "unit",
1713
+
"value": "short"
1714
+
},
1715
+
{
1716
+
"id": "decimals",
1717
+
"value": 2
1718
+
},
1719
+
{
1720
+
"id": "links",
1721
+
"value": [
1722
+
{
1723
+
"targetBlank": false,
1724
+
"title": "Drill down",
1725
+
"url": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell"
1726
+
}
1727
+
]
1728
+
},
1729
+
{
1730
+
"id": "custom.align"
1731
+
}
1732
+
]
1733
+
},
1734
+
{
1735
+
"matcher": {
1736
+
"id": "byName",
1737
+
"options": "Pod"
1738
+
},
1739
+
"properties": [
1740
+
{
1741
+
"id": "custom.width",
1742
+
"value": 208
1743
+
}
1744
+
]
1745
+
},
1746
+
{
1747
+
"matcher": {
1748
+
"id": "byName",
1749
+
"options": "Mem Usage"
1750
+
},
1751
+
"properties": [
1752
+
{
1753
+
"id": "custom.width"
1754
+
}
1755
+
]
1756
+
},
1757
+
{
1758
+
"matcher": {
1759
+
"id": "byName",
1760
+
"options": "Mem Requests"
1761
+
},
1762
+
"properties": [
1763
+
{
1764
+
"id": "custom.width"
1765
+
}
1766
+
]
1767
+
},
1768
+
{
1769
+
"matcher": {
1770
+
"id": "byName",
1771
+
"options": "Mem Usage (RSS)"
1772
+
},
1773
+
"properties": [
1774
+
{
1775
+
"id": "custom.width"
1776
+
}
1777
+
]
1778
+
},
1779
+
{
1780
+
"matcher": {
1781
+
"id": "byName",
1782
+
"options": "Mem Usage (Cache)"
1783
+
},
1784
+
"properties": [
1785
+
{
1786
+
"id": "custom.width",
1787
+
"value": 162
1788
+
}
1789
+
]
1790
+
}
1791
+
]
1792
+
},
1793
+
"gridPos": {
1794
+
"h": 6,
1795
+
"w": 12,
1796
+
"x": 0,
1797
+
"y": 25
1798
+
},
1799
+
"id": 46,
1800
+
"interval": "1m",
1801
+
"links": [],
1802
+
"options": {
1803
+
"cellHeight": "sm",
1804
+
"footer": {
1805
+
"countRows": false,
1806
+
"enablePagination": false,
1807
+
"fields": "",
1808
+
"reducer": [
1809
+
"sum"
1810
+
],
1811
+
"show": false
1812
+
},
1813
+
"showHeader": true,
1814
+
"sortBy": [
1815
+
{
1816
+
"desc": false,
1817
+
"displayName": "Mem Requests"
1818
+
}
1819
+
]
1820
+
},
1821
+
"pluginVersion": "10.2.2",
1822
+
"targets": [
1823
+
{
1824
+
"datasource": {
1825
+
"uid": "$datasource"
1826
+
},
1827
+
"editorMode": "code",
1828
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\", image!=\"\"}) by (pod)",
1829
+
"format": "table",
1830
+
"instant": true,
1831
+
"intervalFactor": 2,
1832
+
"legendFormat": "",
1833
+
"refId": "A"
1834
+
},
1835
+
{
1836
+
"datasource": {
1837
+
"uid": "$datasource"
1838
+
},
1839
+
"editorMode": "code",
1840
+
"expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1841
+
"format": "table",
1842
+
"instant": true,
1843
+
"intervalFactor": 2,
1844
+
"legendFormat": "",
1845
+
"refId": "B"
1846
+
},
1847
+
{
1848
+
"datasource": {
1849
+
"uid": "$datasource"
1850
+
},
1851
+
"editorMode": "code",
1852
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1853
+
"format": "table",
1854
+
"instant": true,
1855
+
"intervalFactor": 2,
1856
+
"legendFormat": "",
1857
+
"refId": "C"
1858
+
},
1859
+
{
1860
+
"datasource": {
1861
+
"uid": "$datasource"
1862
+
},
1863
+
"editorMode": "code",
1864
+
"expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1865
+
"format": "table",
1866
+
"instant": true,
1867
+
"intervalFactor": 2,
1868
+
"legendFormat": "",
1869
+
"refId": "D"
1870
+
},
1871
+
{
1872
+
"datasource": {
1873
+
"uid": "$datasource"
1874
+
},
1875
+
"editorMode": "code",
1876
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"prefect-server.*\"}) by (pod)",
1877
+
"format": "table",
1878
+
"instant": true,
1879
+
"intervalFactor": 2,
1880
+
"legendFormat": "",
1881
+
"refId": "E"
1882
+
},
1883
+
{
1884
+
"datasource": {
1885
+
"uid": "$datasource"
1886
+
},
1887
+
"editorMode": "code",
1888
+
"expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\"}) by (pod)",
1889
+
"format": "table",
1890
+
"instant": true,
1891
+
"intervalFactor": 2,
1892
+
"legendFormat": "",
1893
+
"refId": "F"
1894
+
},
1895
+
{
1896
+
"datasource": {
1897
+
"uid": "$datasource"
1898
+
},
1899
+
"editorMode": "code",
1900
+
"expr": "sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\"}) by (pod)",
1901
+
"format": "table",
1902
+
"instant": true,
1903
+
"intervalFactor": 2,
1904
+
"legendFormat": "",
1905
+
"refId": "G"
1906
+
},
1907
+
{
1908
+
"datasource": {
1909
+
"uid": "$datasource"
1910
+
},
1911
+
"editorMode": "code",
1912
+
"expr": "sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\"}) by (pod)",
1913
+
"format": "table",
1914
+
"instant": true,
1915
+
"intervalFactor": 2,
1916
+
"legendFormat": "",
1917
+
"refId": "H"
1918
+
}
1919
+
],
1920
+
"title": "Memory Quota",
1921
+
"transformations": [
1922
+
{
1923
+
"id": "merge",
1924
+
"options": {
1925
+
"reducers": []
1926
+
}
1927
+
},
1928
+
{
1929
+
"id": "organize",
1930
+
"options": {
1931
+
"excludeByName": {
1932
+
"Time": true
1933
+
},
1934
+
"indexByName": {},
1935
+
"renameByName": {}
1936
+
}
1937
+
}
1938
+
],
1939
+
"type": "table"
1940
+
},
1941
+
{
1942
+
"datasource": {
1943
+
"type": "prometheus",
1944
+
"uid": "$datasource"
1945
+
},
1946
+
"fieldConfig": {
1947
+
"defaults": {
1948
+
"color": {
1949
+
"mode": "palette-classic"
1950
+
},
1951
+
"custom": {
1952
+
"axisBorderShow": false,
1953
+
"axisCenteredZero": false,
1954
+
"axisColorMode": "text",
1955
+
"axisLabel": "",
1956
+
"axisPlacement": "auto",
1957
+
"barAlignment": 0,
1958
+
"drawStyle": "line",
1959
+
"fillOpacity": 100,
1960
+
"gradientMode": "none",
1961
+
"hideFrom": {
1962
+
"legend": false,
1963
+
"tooltip": false,
1964
+
"viz": false
1965
+
},
1966
+
"insertNulls": false,
1967
+
"lineInterpolation": "linear",
1968
+
"lineWidth": 0,
1969
+
"pointSize": 5,
1970
+
"scaleDistribution": {
1971
+
"type": "linear"
1972
+
},
1973
+
"showPoints": "never",
1974
+
"spanNulls": false,
1975
+
"stacking": {
1976
+
"group": "A",
1977
+
"mode": "normal"
1978
+
},
1979
+
"thresholdsStyle": {
1980
+
"mode": "off"
1981
+
}
1982
+
},
1983
+
"mappings": [],
1984
+
"min": 0,
1985
+
"thresholds": {
1986
+
"mode": "absolute",
1987
+
"steps": [
1988
+
{
1989
+
"color": "green",
1990
+
"value": null
1991
+
},
1992
+
{
1993
+
"color": "red",
1994
+
"value": 80
1995
+
}
1996
+
]
1997
+
},
1998
+
"unit": "bytes"
1999
+
},
2000
+
"overrides": [
2001
+
{
2002
+
"matcher": {
2003
+
"id": "byName",
2004
+
"options": "quota - requests"
2005
+
},
2006
+
"properties": [
2007
+
{
2008
+
"id": "color",
2009
+
"value": {
2010
+
"fixedColor": "#F2495C",
2011
+
"mode": "fixed"
2012
+
}
2013
+
},
2014
+
{
2015
+
"id": "custom.fillOpacity",
2016
+
"value": 0
2017
+
},
2018
+
{
2019
+
"id": "custom.lineWidth",
2020
+
"value": 2
2021
+
},
2022
+
{
2023
+
"id": "custom.stacking",
2024
+
"value": {
2025
+
"group": "A",
2026
+
"mode": "none"
2027
+
}
2028
+
},
2029
+
{
2030
+
"id": "custom.lineStyle",
2031
+
"value": {
2032
+
"dash": [
2033
+
10,
2034
+
10
2035
+
],
2036
+
"fill": "dash"
2037
+
}
2038
+
}
2039
+
]
2040
+
},
2041
+
{
2042
+
"matcher": {
2043
+
"id": "byName",
2044
+
"options": "quota - limits"
2045
+
},
2046
+
"properties": [
2047
+
{
2048
+
"id": "color",
2049
+
"value": {
2050
+
"fixedColor": "#FF9830",
2051
+
"mode": "fixed"
2052
+
}
2053
+
},
2054
+
{
2055
+
"id": "custom.fillOpacity",
2056
+
"value": 0
2057
+
},
2058
+
{
2059
+
"id": "custom.lineWidth",
2060
+
"value": 2
2061
+
},
2062
+
{
2063
+
"id": "custom.stacking",
2064
+
"value": {
2065
+
"group": "A",
2066
+
"mode": "none"
2067
+
}
2068
+
},
2069
+
{
2070
+
"id": "custom.lineStyle",
2071
+
"value": {
2072
+
"dash": [
2073
+
10,
2074
+
10
2075
+
],
2076
+
"fill": "dash"
2077
+
}
2078
+
}
2079
+
]
2080
+
}
2081
+
]
2082
+
},
2083
+
"gridPos": {
2084
+
"h": 6,
2085
+
"w": 12,
2086
+
"x": 12,
2087
+
"y": 25
2088
+
},
2089
+
"id": 27,
2090
+
"interval": "1m",
2091
+
"links": [],
2092
+
"options": {
2093
+
"legend": {
2094
+
"calcs": [],
2095
+
"displayMode": "table",
2096
+
"placement": "right",
2097
+
"showLegend": true
2098
+
},
2099
+
"tooltip": {
2100
+
"mode": "single",
2101
+
"sort": "none"
2102
+
}
2103
+
},
2104
+
"pluginVersion": "10.1.1",
2105
+
"targets": [
2106
+
{
2107
+
"datasource": {
2108
+
"uid": "$datasource"
2109
+
},
2110
+
"editorMode": "code",
2111
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-server.*\", container!=\"\", image!=\"\"}) by (pod)",
2112
+
"format": "time_series",
2113
+
"intervalFactor": 2,
2114
+
"legendFormat": "{{pod}}",
2115
+
"range": true,
2116
+
"refId": "A"
2117
+
}
2118
+
],
2119
+
"title": "Memory Usage (w/o cache)",
2120
+
"type": "timeseries"
2121
+
}
2122
+
],
2123
+
"title": "Prefect Server overview",
2124
+
"type": "row"
2125
+
},
2126
+
{
2127
+
"collapsed": false,
2128
+
"gridPos": {
2129
+
"h": 1,
2130
+
"w": 24,
2131
+
"x": 0,
2132
+
"y": 12
2133
+
},
2134
+
"id": 34,
2135
+
"panels": [],
2136
+
"title": "Prefect Agent overview",
2137
+
"type": "row"
2138
+
},
2139
+
{
2140
+
"datasource": {
2141
+
"type": "prometheus",
2142
+
"uid": "${DS_PROMETHEUS}"
2143
+
},
2144
+
"fieldConfig": {
2145
+
"defaults": {
2146
+
"color": {
2147
+
"mode": "thresholds"
2148
+
},
2149
+
"decimals": 0,
2150
+
"mappings": [],
2151
+
"thresholds": {
2152
+
"mode": "absolute",
2153
+
"steps": [
2154
+
{
2155
+
"color": "red",
2156
+
"value": null
2157
+
},
2158
+
{
2159
+
"color": "green",
2160
+
"value": 1
2161
+
}
2162
+
]
2163
+
},
2164
+
"unit": "none"
2165
+
},
2166
+
"overrides": []
2167
+
},
2168
+
"gridPos": {
2169
+
"h": 7,
2170
+
"w": 4,
2171
+
"x": 0,
2172
+
"y": 13
2173
+
},
2174
+
"id": 39,
2175
+
"options": {
2176
+
"colorMode": "value",
2177
+
"graphMode": "area",
2178
+
"justifyMode": "center",
2179
+
"orientation": "auto",
2180
+
"reduceOptions": {
2181
+
"calcs": [
2182
+
"lastNotNull"
2183
+
],
2184
+
"fields": "",
2185
+
"values": false
2186
+
},
2187
+
"textMode": "auto",
2188
+
"wideLayout": true
2189
+
},
2190
+
"pluginVersion": "10.2.2",
2191
+
"targets": [
2192
+
{
2193
+
"datasource": {
2194
+
"type": "prometheus",
2195
+
"uid": "${DS_PROMETHEUS}"
2196
+
},
2197
+
"editorMode": "code",
2198
+
"expr": "sum(kube_deployment_status_replicas{deployment=~\".*agent|.*worker\"})",
2199
+
"instant": false,
2200
+
"legendFormat": "__auto",
2201
+
"range": true,
2202
+
"refId": "A"
2203
+
}
2204
+
],
2205
+
"title": "Agents/Worker total",
2206
+
"type": "stat"
2207
+
},
2208
+
{
2209
+
"datasource": {
2210
+
"type": "loki",
2211
+
"uid": "${DS_LOKI}"
2212
+
},
2213
+
"gridPos": {
2214
+
"h": 7,
2215
+
"w": 20,
2216
+
"x": 4,
2217
+
"y": 13
2218
+
},
2219
+
"id": 40,
2220
+
"options": {
2221
+
"dedupStrategy": "none",
2222
+
"enableLogDetails": true,
2223
+
"prettifyLogMessage": false,
2224
+
"showCommonLabels": false,
2225
+
"showLabels": false,
2226
+
"showTime": false,
2227
+
"sortOrder": "Descending",
2228
+
"wrapLogMessage": false
2229
+
},
2230
+
"targets": [
2231
+
{
2232
+
"datasource": {
2233
+
"type": "loki",
2234
+
"uid": "${DS_LOKI}"
2235
+
},
2236
+
"editorMode": "code",
2237
+
"expr": "{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}",
2238
+
"queryType": "range",
2239
+
"refId": "A"
2240
+
}
2241
+
],
2242
+
"title": "Logs",
2243
+
"type": "logs"
2244
+
},
2245
+
{
2246
+
"datasource": {
2247
+
"type": "prometheus",
2248
+
"uid": "${DS_PROMETHEUS}"
2249
+
},
2250
+
"fieldConfig": {
2251
+
"defaults": {
2252
+
"color": {
2253
+
"mode": "thresholds"
2254
+
},
2255
+
"custom": {
2256
+
"align": "auto",
2257
+
"cellOptions": {
2258
+
"type": "auto"
2259
+
},
2260
+
"filterable": false,
2261
+
"inspect": false
2262
+
},
2263
+
"decimals": 2,
2264
+
"displayName": "",
2265
+
"mappings": [],
2266
+
"noValue": "-",
2267
+
"thresholds": {
2268
+
"mode": "absolute",
2269
+
"steps": [
2270
+
{
2271
+
"color": "green",
2272
+
"value": null
2273
+
},
2274
+
{
2275
+
"color": "red",
2276
+
"value": 80
2277
+
}
2278
+
]
2279
+
},
2280
+
"unit": "short"
2281
+
},
2282
+
"overrides": [
2283
+
{
2284
+
"matcher": {
2285
+
"id": "byName",
2286
+
"options": "Time"
2287
+
},
2288
+
"properties": [
2289
+
{
2290
+
"id": "displayName",
2291
+
"value": "Time"
2292
+
},
2293
+
{
2294
+
"id": "custom.align"
2295
+
}
2296
+
]
2297
+
},
2298
+
{
2299
+
"matcher": {
2300
+
"id": "byName",
2301
+
"options": "Value #A"
2302
+
},
2303
+
"properties": [
2304
+
{
2305
+
"id": "displayName",
2306
+
"value": "CPU Usage"
2307
+
},
2308
+
{
2309
+
"id": "unit",
2310
+
"value": "short"
2311
+
},
2312
+
{
2313
+
"id": "decimals",
2314
+
"value": 2
2315
+
},
2316
+
{
2317
+
"id": "custom.align"
2318
+
}
2319
+
]
2320
+
},
2321
+
{
2322
+
"matcher": {
2323
+
"id": "byName",
2324
+
"options": "Value #B"
2325
+
},
2326
+
"properties": [
2327
+
{
2328
+
"id": "displayName",
2329
+
"value": "CPU Requests"
2330
+
},
2331
+
{
2332
+
"id": "unit",
2333
+
"value": "short"
2334
+
},
2335
+
{
2336
+
"id": "decimals",
2337
+
"value": 2
2338
+
},
2339
+
{
2340
+
"id": "custom.align"
2341
+
}
2342
+
]
2343
+
},
2344
+
{
2345
+
"matcher": {
2346
+
"id": "byName",
2347
+
"options": "Value #C"
2348
+
},
2349
+
"properties": [
2350
+
{
2351
+
"id": "displayName",
2352
+
"value": "CPU Requests %"
2353
+
},
2354
+
{
2355
+
"id": "unit",
2356
+
"value": "percentunit"
2357
+
},
2358
+
{
2359
+
"id": "decimals",
2360
+
"value": 2
2361
+
},
2362
+
{
2363
+
"id": "custom.align"
2364
+
}
2365
+
]
2366
+
},
2367
+
{
2368
+
"matcher": {
2369
+
"id": "byName",
2370
+
"options": "Value #D"
2371
+
},
2372
+
"properties": [
2373
+
{
2374
+
"id": "displayName",
2375
+
"value": "CPU Limits"
2376
+
},
2377
+
{
2378
+
"id": "unit",
2379
+
"value": "short"
2380
+
},
2381
+
{
2382
+
"id": "decimals",
2383
+
"value": 2
2384
+
},
2385
+
{
2386
+
"id": "custom.align"
2387
+
}
2388
+
]
2389
+
},
2390
+
{
2391
+
"matcher": {
2392
+
"id": "byName",
2393
+
"options": "Value #E"
2394
+
},
2395
+
"properties": [
2396
+
{
2397
+
"id": "displayName",
2398
+
"value": "CPU Limits %"
2399
+
},
2400
+
{
2401
+
"id": "unit",
2402
+
"value": "percentunit"
2403
+
},
2404
+
{
2405
+
"id": "decimals",
2406
+
"value": 2
2407
+
},
2408
+
{
2409
+
"id": "custom.align"
2410
+
}
2411
+
]
2412
+
},
2413
+
{
2414
+
"matcher": {
2415
+
"id": "byName",
2416
+
"options": "pod"
2417
+
},
2418
+
"properties": [
2419
+
{
2420
+
"id": "displayName",
2421
+
"value": "Pod"
2422
+
},
2423
+
{
2424
+
"id": "unit",
2425
+
"value": "short"
2426
+
},
2427
+
{
2428
+
"id": "decimals",
2429
+
"value": 2
2430
+
},
2431
+
{
2432
+
"id": "links",
2433
+
"value": [
2434
+
{
2435
+
"targetBlank": false,
2436
+
"title": "Drill down",
2437
+
"url": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell"
2438
+
}
2439
+
]
2440
+
},
2441
+
{
2442
+
"id": "custom.align"
2443
+
}
2444
+
]
2445
+
}
2446
+
]
2447
+
},
2448
+
"gridPos": {
2449
+
"h": 6,
2450
+
"w": 12,
2451
+
"x": 0,
2452
+
"y": 20
2453
+
},
2454
+
"id": 41,
2455
+
"interval": "1m",
2456
+
"links": [],
2457
+
"options": {
2458
+
"cellHeight": "sm",
2459
+
"footer": {
2460
+
"countRows": false,
2461
+
"fields": "",
2462
+
"reducer": [
2463
+
"sum"
2464
+
],
2465
+
"show": false
2466
+
},
2467
+
"showHeader": true,
2468
+
"sortBy": [
2469
+
{
2470
+
"desc": true,
2471
+
"displayName": "CPU Usage"
2472
+
}
2473
+
]
2474
+
},
2475
+
"pluginVersion": "10.2.2",
2476
+
"targets": [
2477
+
{
2478
+
"datasource": {
2479
+
"uid": "$datasource"
2480
+
},
2481
+
"editorMode": "code",
2482
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
2483
+
"format": "table",
2484
+
"instant": true,
2485
+
"intervalFactor": 2,
2486
+
"legendFormat": "",
2487
+
"refId": "A"
2488
+
},
2489
+
{
2490
+
"datasource": {
2491
+
"uid": "$datasource"
2492
+
},
2493
+
"editorMode": "code",
2494
+
"expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
2495
+
"format": "table",
2496
+
"instant": true,
2497
+
"intervalFactor": 2,
2498
+
"legendFormat": "",
2499
+
"refId": "B"
2500
+
},
2501
+
{
2502
+
"datasource": {
2503
+
"uid": "$datasource"
2504
+
},
2505
+
"editorMode": "code",
2506
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
2507
+
"format": "table",
2508
+
"instant": true,
2509
+
"intervalFactor": 2,
2510
+
"legendFormat": "",
2511
+
"refId": "C"
2512
+
},
2513
+
{
2514
+
"datasource": {
2515
+
"uid": "$datasource"
2516
+
},
2517
+
"editorMode": "code",
2518
+
"expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
2519
+
"format": "table",
2520
+
"instant": true,
2521
+
"intervalFactor": 2,
2522
+
"legendFormat": "",
2523
+
"refId": "D"
2524
+
},
2525
+
{
2526
+
"datasource": {
2527
+
"uid": "$datasource"
2528
+
},
2529
+
"editorMode": "code",
2530
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
2531
+
"format": "table",
2532
+
"instant": true,
2533
+
"intervalFactor": 2,
2534
+
"legendFormat": "",
2535
+
"refId": "E"
2536
+
}
2537
+
],
2538
+
"title": "CPU Quota",
2539
+
"transformations": [
2540
+
{
2541
+
"id": "merge",
2542
+
"options": {
2543
+
"reducers": []
2544
+
}
2545
+
},
2546
+
{
2547
+
"id": "organize",
2548
+
"options": {
2549
+
"excludeByName": {
2550
+
"Time": true
2551
+
},
2552
+
"indexByName": {},
2553
+
"renameByName": {}
2554
+
}
2555
+
}
2556
+
],
2557
+
"type": "table"
2558
+
},
2559
+
{
2560
+
"datasource": {
2561
+
"type": "prometheus",
2562
+
"uid": "${DS_PROMETHEUS}"
2563
+
},
2564
+
"fieldConfig": {
2565
+
"defaults": {
2566
+
"color": {
2567
+
"mode": "palette-classic"
2568
+
},
2569
+
"custom": {
2570
+
"axisBorderShow": false,
2571
+
"axisCenteredZero": false,
2572
+
"axisColorMode": "text",
2573
+
"axisLabel": "",
2574
+
"axisPlacement": "auto",
2575
+
"barAlignment": 0,
2576
+
"drawStyle": "line",
2577
+
"fillOpacity": 100,
2578
+
"gradientMode": "none",
2579
+
"hideFrom": {
2580
+
"legend": false,
2581
+
"tooltip": false,
2582
+
"viz": false
2583
+
},
2584
+
"insertNulls": false,
2585
+
"lineInterpolation": "linear",
2586
+
"lineWidth": 0,
2587
+
"pointSize": 5,
2588
+
"scaleDistribution": {
2589
+
"type": "linear"
2590
+
},
2591
+
"showPoints": "never",
2592
+
"spanNulls": false,
2593
+
"stacking": {
2594
+
"group": "A",
2595
+
"mode": "normal"
2596
+
},
2597
+
"thresholdsStyle": {
2598
+
"mode": "off"
2599
+
}
2600
+
},
2601
+
"mappings": [],
2602
+
"min": 0,
2603
+
"thresholds": {
2604
+
"mode": "absolute",
2605
+
"steps": [
2606
+
{
2607
+
"color": "green",
2608
+
"value": null
2609
+
},
2610
+
{
2611
+
"color": "red",
2612
+
"value": 80
2613
+
}
2614
+
]
2615
+
},
2616
+
"unit": "short"
2617
+
},
2618
+
"overrides": [
2619
+
{
2620
+
"matcher": {
2621
+
"id": "byName",
2622
+
"options": "quota - requests"
2623
+
},
2624
+
"properties": [
2625
+
{
2626
+
"id": "color",
2627
+
"value": {
2628
+
"fixedColor": "#F2495C",
2629
+
"mode": "fixed"
2630
+
}
2631
+
},
2632
+
{
2633
+
"id": "custom.fillOpacity",
2634
+
"value": 0
2635
+
},
2636
+
{
2637
+
"id": "custom.lineWidth",
2638
+
"value": 2
2639
+
},
2640
+
{
2641
+
"id": "custom.stacking",
2642
+
"value": {
2643
+
"group": "A",
2644
+
"mode": "none"
2645
+
}
2646
+
},
2647
+
{
2648
+
"id": "custom.lineStyle",
2649
+
"value": {
2650
+
"dash": [
2651
+
10,
2652
+
10
2653
+
],
2654
+
"fill": "dash"
2655
+
}
2656
+
}
2657
+
]
2658
+
},
2659
+
{
2660
+
"matcher": {
2661
+
"id": "byName",
2662
+
"options": "quota - limits"
2663
+
},
2664
+
"properties": [
2665
+
{
2666
+
"id": "color",
2667
+
"value": {
2668
+
"fixedColor": "#FF9830",
2669
+
"mode": "fixed"
2670
+
}
2671
+
},
2672
+
{
2673
+
"id": "custom.fillOpacity",
2674
+
"value": 0
2675
+
},
2676
+
{
2677
+
"id": "custom.lineWidth",
2678
+
"value": 2
2679
+
},
2680
+
{
2681
+
"id": "custom.stacking",
2682
+
"value": {
2683
+
"group": "A",
2684
+
"mode": "none"
2685
+
}
2686
+
},
2687
+
{
2688
+
"id": "custom.lineStyle",
2689
+
"value": {
2690
+
"dash": [
2691
+
10,
2692
+
10
2693
+
],
2694
+
"fill": "dash"
2695
+
}
2696
+
}
2697
+
]
2698
+
}
2699
+
]
2700
+
},
2701
+
"gridPos": {
2702
+
"h": 6,
2703
+
"w": 12,
2704
+
"x": 12,
2705
+
"y": 20
2706
+
},
2707
+
"id": 24,
2708
+
"interval": "1m",
2709
+
"links": [],
2710
+
"options": {
2711
+
"legend": {
2712
+
"calcs": [],
2713
+
"displayMode": "table",
2714
+
"placement": "right",
2715
+
"showLegend": true
2716
+
},
2717
+
"tooltip": {
2718
+
"mode": "single",
2719
+
"sort": "none"
2720
+
}
2721
+
},
2722
+
"pluginVersion": "10.1.1",
2723
+
"targets": [
2724
+
{
2725
+
"datasource": {
2726
+
"uid": "$datasource"
2727
+
},
2728
+
"editorMode": "code",
2729
+
"expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=~\"prefect-agent.*\"}) by (pod)",
2730
+
"format": "time_series",
2731
+
"intervalFactor": 2,
2732
+
"legendFormat": "{{pod}}",
2733
+
"range": true,
2734
+
"refId": "A"
2735
+
}
2736
+
],
2737
+
"title": "CPU Usage",
2738
+
"type": "timeseries"
2739
+
},
2740
+
{
2741
+
"datasource": {
2742
+
"type": "prometheus",
2743
+
"uid": "$datasource"
2744
+
},
2745
+
"fieldConfig": {
2746
+
"defaults": {
2747
+
"color": {
2748
+
"mode": "thresholds"
2749
+
},
2750
+
"custom": {
2751
+
"align": "auto",
2752
+
"cellOptions": {
2753
+
"type": "auto"
2754
+
},
2755
+
"filterable": false,
2756
+
"inspect": false
2757
+
},
2758
+
"decimals": 2,
2759
+
"displayName": "",
2760
+
"mappings": [],
2761
+
"thresholds": {
2762
+
"mode": "absolute",
2763
+
"steps": [
2764
+
{
2765
+
"color": "green",
2766
+
"value": null
2767
+
},
2768
+
{
2769
+
"color": "red",
2770
+
"value": 80
2771
+
}
2772
+
]
2773
+
},
2774
+
"unit": "short"
2775
+
},
2776
+
"overrides": [
2777
+
{
2778
+
"matcher": {
2779
+
"id": "byName",
2780
+
"options": "Time"
2781
+
},
2782
+
"properties": [
2783
+
{
2784
+
"id": "displayName",
2785
+
"value": "Time"
2786
+
},
2787
+
{
2788
+
"id": "custom.align"
2789
+
}
2790
+
]
2791
+
},
2792
+
{
2793
+
"matcher": {
2794
+
"id": "byName",
2795
+
"options": "Value #A"
2796
+
},
2797
+
"properties": [
2798
+
{
2799
+
"id": "displayName",
2800
+
"value": "Mem Usage"
2801
+
},
2802
+
{
2803
+
"id": "unit",
2804
+
"value": "bytes"
2805
+
},
2806
+
{
2807
+
"id": "decimals",
2808
+
"value": 2
2809
+
},
2810
+
{
2811
+
"id": "custom.align"
2812
+
}
2813
+
]
2814
+
},
2815
+
{
2816
+
"matcher": {
2817
+
"id": "byName",
2818
+
"options": "Value #B"
2819
+
},
2820
+
"properties": [
2821
+
{
2822
+
"id": "displayName",
2823
+
"value": "Mem Requests"
2824
+
},
2825
+
{
2826
+
"id": "unit",
2827
+
"value": "bytes"
2828
+
},
2829
+
{
2830
+
"id": "decimals",
2831
+
"value": 2
2832
+
},
2833
+
{
2834
+
"id": "custom.align"
2835
+
}
2836
+
]
2837
+
},
2838
+
{
2839
+
"matcher": {
2840
+
"id": "byName",
2841
+
"options": "Value #C"
2842
+
},
2843
+
"properties": [
2844
+
{
2845
+
"id": "displayName",
2846
+
"value": "Mem Requests %"
2847
+
},
2848
+
{
2849
+
"id": "unit",
2850
+
"value": "percentunit"
2851
+
},
2852
+
{
2853
+
"id": "decimals",
2854
+
"value": 2
2855
+
},
2856
+
{
2857
+
"id": "custom.align"
2858
+
}
2859
+
]
2860
+
},
2861
+
{
2862
+
"matcher": {
2863
+
"id": "byName",
2864
+
"options": "Value #D"
2865
+
},
2866
+
"properties": [
2867
+
{
2868
+
"id": "displayName",
2869
+
"value": "Mem Limits"
2870
+
},
2871
+
{
2872
+
"id": "unit",
2873
+
"value": "bytes"
2874
+
},
2875
+
{
2876
+
"id": "decimals",
2877
+
"value": 2
2878
+
},
2879
+
{
2880
+
"id": "custom.align"
2881
+
}
2882
+
]
2883
+
},
2884
+
{
2885
+
"matcher": {
2886
+
"id": "byName",
2887
+
"options": "Value #E"
2888
+
},
2889
+
"properties": [
2890
+
{
2891
+
"id": "displayName",
2892
+
"value": "Mem Limits %"
2893
+
},
2894
+
{
2895
+
"id": "unit",
2896
+
"value": "percentunit"
2897
+
},
2898
+
{
2899
+
"id": "decimals",
2900
+
"value": 2
2901
+
},
2902
+
{
2903
+
"id": "custom.align"
2904
+
}
2905
+
]
2906
+
},
2907
+
{
2908
+
"matcher": {
2909
+
"id": "byName",
2910
+
"options": "Value #F"
2911
+
},
2912
+
"properties": [
2913
+
{
2914
+
"id": "displayName",
2915
+
"value": "Mem Usage (RSS)"
2916
+
},
2917
+
{
2918
+
"id": "unit",
2919
+
"value": "bytes"
2920
+
},
2921
+
{
2922
+
"id": "decimals",
2923
+
"value": 2
2924
+
},
2925
+
{
2926
+
"id": "custom.align"
2927
+
}
2928
+
]
2929
+
},
2930
+
{
2931
+
"matcher": {
2932
+
"id": "byName",
2933
+
"options": "Value #G"
2934
+
},
2935
+
"properties": [
2936
+
{
2937
+
"id": "displayName",
2938
+
"value": "Mem Usage (Cache)"
2939
+
},
2940
+
{
2941
+
"id": "unit",
2942
+
"value": "bytes"
2943
+
},
2944
+
{
2945
+
"id": "decimals",
2946
+
"value": 2
2947
+
},
2948
+
{
2949
+
"id": "custom.align"
2950
+
}
2951
+
]
2952
+
},
2953
+
{
2954
+
"matcher": {
2955
+
"id": "byName",
2956
+
"options": "Value #H"
2957
+
},
2958
+
"properties": [
2959
+
{
2960
+
"id": "displayName",
2961
+
"value": "Mem Usage (Swap)"
2962
+
},
2963
+
{
2964
+
"id": "unit",
2965
+
"value": "bytes"
2966
+
},
2967
+
{
2968
+
"id": "decimals",
2969
+
"value": 2
2970
+
},
2971
+
{
2972
+
"id": "custom.align"
2973
+
}
2974
+
]
2975
+
},
2976
+
{
2977
+
"matcher": {
2978
+
"id": "byName",
2979
+
"options": "pod"
2980
+
},
2981
+
"properties": [
2982
+
{
2983
+
"id": "displayName",
2984
+
"value": "Pod"
2985
+
},
2986
+
{
2987
+
"id": "unit",
2988
+
"value": "short"
2989
+
},
2990
+
{
2991
+
"id": "decimals",
2992
+
"value": 2
2993
+
},
2994
+
{
2995
+
"id": "links",
2996
+
"value": [
2997
+
{
2998
+
"targetBlank": false,
2999
+
"title": "Drill down",
3000
+
"url": "/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell"
3001
+
}
3002
+
]
3003
+
},
3004
+
{
3005
+
"id": "custom.align"
3006
+
}
3007
+
]
3008
+
},
3009
+
{
3010
+
"matcher": {
3011
+
"id": "byName",
3012
+
"options": "Pod"
3013
+
},
3014
+
"properties": [
3015
+
{
3016
+
"id": "custom.width",
3017
+
"value": 208
3018
+
}
3019
+
]
3020
+
},
3021
+
{
3022
+
"matcher": {
3023
+
"id": "byName",
3024
+
"options": "Mem Usage"
3025
+
},
3026
+
"properties": [
3027
+
{
3028
+
"id": "custom.width"
3029
+
}
3030
+
]
3031
+
},
3032
+
{
3033
+
"matcher": {
3034
+
"id": "byName",
3035
+
"options": "Mem Requests"
3036
+
},
3037
+
"properties": [
3038
+
{
3039
+
"id": "custom.width"
3040
+
}
3041
+
]
3042
+
},
3043
+
{
3044
+
"matcher": {
3045
+
"id": "byName",
3046
+
"options": "Mem Usage (RSS)"
3047
+
},
3048
+
"properties": [
3049
+
{
3050
+
"id": "custom.width"
3051
+
}
3052
+
]
3053
+
},
3054
+
{
3055
+
"matcher": {
3056
+
"id": "byName",
3057
+
"options": "Mem Usage (Cache)"
3058
+
},
3059
+
"properties": [
3060
+
{
3061
+
"id": "custom.width",
3062
+
"value": 162
3063
+
}
3064
+
]
3065
+
}
3066
+
]
3067
+
},
3068
+
"gridPos": {
3069
+
"h": 6,
3070
+
"w": 12,
3071
+
"x": 0,
3072
+
"y": 26
3073
+
},
3074
+
"id": 43,
3075
+
"interval": "1m",
3076
+
"links": [],
3077
+
"options": {
3078
+
"cellHeight": "sm",
3079
+
"footer": {
3080
+
"countRows": false,
3081
+
"enablePagination": false,
3082
+
"fields": "",
3083
+
"reducer": [
3084
+
"sum"
3085
+
],
3086
+
"show": false
3087
+
},
3088
+
"showHeader": true,
3089
+
"sortBy": [
3090
+
{
3091
+
"desc": false,
3092
+
"displayName": "Mem Requests"
3093
+
}
3094
+
]
3095
+
},
3096
+
"pluginVersion": "10.2.2",
3097
+
"targets": [
3098
+
{
3099
+
"datasource": {
3100
+
"uid": "$datasource"
3101
+
},
3102
+
"editorMode": "code",
3103
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\", image!=\"\"}) by (pod)",
3104
+
"format": "table",
3105
+
"instant": true,
3106
+
"intervalFactor": 2,
3107
+
"legendFormat": "",
3108
+
"refId": "A"
3109
+
},
3110
+
{
3111
+
"datasource": {
3112
+
"uid": "$datasource"
3113
+
},
3114
+
"editorMode": "code",
3115
+
"expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
3116
+
"format": "table",
3117
+
"instant": true,
3118
+
"intervalFactor": 2,
3119
+
"legendFormat": "",
3120
+
"refId": "B"
3121
+
},
3122
+
{
3123
+
"datasource": {
3124
+
"uid": "$datasource"
3125
+
},
3126
+
"editorMode": "code",
3127
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
3128
+
"format": "table",
3129
+
"instant": true,
3130
+
"intervalFactor": 2,
3131
+
"legendFormat": "",
3132
+
"refId": "C"
3133
+
},
3134
+
{
3135
+
"datasource": {
3136
+
"uid": "$datasource"
3137
+
},
3138
+
"editorMode": "code",
3139
+
"expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
3140
+
"format": "table",
3141
+
"instant": true,
3142
+
"intervalFactor": 2,
3143
+
"legendFormat": "",
3144
+
"refId": "D"
3145
+
},
3146
+
{
3147
+
"datasource": {
3148
+
"uid": "$datasource"
3149
+
},
3150
+
"editorMode": "code",
3151
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\"}) by (pod)",
3152
+
"format": "table",
3153
+
"instant": true,
3154
+
"intervalFactor": 2,
3155
+
"legendFormat": "",
3156
+
"refId": "E"
3157
+
},
3158
+
{
3159
+
"datasource": {
3160
+
"uid": "$datasource"
3161
+
},
3162
+
"editorMode": "code",
3163
+
"expr": "sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\"}) by (pod)",
3164
+
"format": "table",
3165
+
"instant": true,
3166
+
"intervalFactor": 2,
3167
+
"legendFormat": "",
3168
+
"refId": "F"
3169
+
},
3170
+
{
3171
+
"datasource": {
3172
+
"uid": "$datasource"
3173
+
},
3174
+
"editorMode": "code",
3175
+
"expr": "sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\"}) by (pod)",
3176
+
"format": "table",
3177
+
"instant": true,
3178
+
"intervalFactor": 2,
3179
+
"legendFormat": "",
3180
+
"refId": "G"
3181
+
},
3182
+
{
3183
+
"datasource": {
3184
+
"uid": "$datasource"
3185
+
},
3186
+
"editorMode": "code",
3187
+
"expr": "sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"(prefect-agent.*|prefect-worker.*)\", container!=\"\"}) by (pod)",
3188
+
"format": "table",
3189
+
"instant": true,
3190
+
"intervalFactor": 2,
3191
+
"legendFormat": "",
3192
+
"refId": "H"
3193
+
}
3194
+
],
3195
+
"title": "Memory Quota",
3196
+
"transformations": [
3197
+
{
3198
+
"id": "merge",
3199
+
"options": {
3200
+
"reducers": []
3201
+
}
3202
+
},
3203
+
{
3204
+
"id": "organize",
3205
+
"options": {
3206
+
"excludeByName": {
3207
+
"Time": true
3208
+
},
3209
+
"indexByName": {},
3210
+
"renameByName": {}
3211
+
}
3212
+
}
3213
+
],
3214
+
"type": "table"
3215
+
},
3216
+
{
3217
+
"datasource": {
3218
+
"type": "prometheus",
3219
+
"uid": "$datasource"
3220
+
},
3221
+
"fieldConfig": {
3222
+
"defaults": {
3223
+
"color": {
3224
+
"mode": "palette-classic"
3225
+
},
3226
+
"custom": {
3227
+
"axisBorderShow": false,
3228
+
"axisCenteredZero": false,
3229
+
"axisColorMode": "text",
3230
+
"axisLabel": "",
3231
+
"axisPlacement": "auto",
3232
+
"barAlignment": 0,
3233
+
"drawStyle": "line",
3234
+
"fillOpacity": 100,
3235
+
"gradientMode": "none",
3236
+
"hideFrom": {
3237
+
"legend": false,
3238
+
"tooltip": false,
3239
+
"viz": false
3240
+
},
3241
+
"insertNulls": false,
3242
+
"lineInterpolation": "linear",
3243
+
"lineWidth": 0,
3244
+
"pointSize": 5,
3245
+
"scaleDistribution": {
3246
+
"type": "linear"
3247
+
},
3248
+
"showPoints": "never",
3249
+
"spanNulls": false,
3250
+
"stacking": {
3251
+
"group": "A",
3252
+
"mode": "normal"
3253
+
},
3254
+
"thresholdsStyle": {
3255
+
"mode": "off"
3256
+
}
3257
+
},
3258
+
"mappings": [],
3259
+
"min": 0,
3260
+
"thresholds": {
3261
+
"mode": "absolute",
3262
+
"steps": [
3263
+
{
3264
+
"color": "green",
3265
+
"value": null
3266
+
},
3267
+
{
3268
+
"color": "red",
3269
+
"value": 80
3270
+
}
3271
+
]
3272
+
},
3273
+
"unit": "bytes"
3274
+
},
3275
+
"overrides": [
3276
+
{
3277
+
"matcher": {
3278
+
"id": "byName",
3279
+
"options": "quota - requests"
3280
+
},
3281
+
"properties": [
3282
+
{
3283
+
"id": "color",
3284
+
"value": {
3285
+
"fixedColor": "#F2495C",
3286
+
"mode": "fixed"
3287
+
}
3288
+
},
3289
+
{
3290
+
"id": "custom.fillOpacity",
3291
+
"value": 0
3292
+
},
3293
+
{
3294
+
"id": "custom.lineWidth",
3295
+
"value": 2
3296
+
},
3297
+
{
3298
+
"id": "custom.stacking",
3299
+
"value": {
3300
+
"group": "A",
3301
+
"mode": "none"
3302
+
}
3303
+
},
3304
+
{
3305
+
"id": "custom.lineStyle",
3306
+
"value": {
3307
+
"dash": [
3308
+
10,
3309
+
10
3310
+
],
3311
+
"fill": "dash"
3312
+
}
3313
+
}
3314
+
]
3315
+
},
3316
+
{
3317
+
"matcher": {
3318
+
"id": "byName",
3319
+
"options": "quota - limits"
3320
+
},
3321
+
"properties": [
3322
+
{
3323
+
"id": "color",
3324
+
"value": {
3325
+
"fixedColor": "#FF9830",
3326
+
"mode": "fixed"
3327
+
}
3328
+
},
3329
+
{
3330
+
"id": "custom.fillOpacity",
3331
+
"value": 0
3332
+
},
3333
+
{
3334
+
"id": "custom.lineWidth",
3335
+
"value": 2
3336
+
},
3337
+
{
3338
+
"id": "custom.stacking",
3339
+
"value": {
3340
+
"group": "A",
3341
+
"mode": "none"
3342
+
}
3343
+
},
3344
+
{
3345
+
"id": "custom.lineStyle",
3346
+
"value": {
3347
+
"dash": [
3348
+
10,
3349
+
10
3350
+
],
3351
+
"fill": "dash"
3352
+
}
3353
+
}
3354
+
]
3355
+
}
3356
+
]
3357
+
},
3358
+
"gridPos": {
3359
+
"h": 6,
3360
+
"w": 12,
3361
+
"x": 12,
3362
+
"y": 26
3363
+
},
3364
+
"id": 44,
3365
+
"interval": "1m",
3366
+
"links": [],
3367
+
"options": {
3368
+
"legend": {
3369
+
"calcs": [],
3370
+
"displayMode": "table",
3371
+
"placement": "right",
3372
+
"showLegend": true
3373
+
},
3374
+
"tooltip": {
3375
+
"mode": "single",
3376
+
"sort": "none"
3377
+
}
3378
+
},
3379
+
"pluginVersion": "10.1.1",
3380
+
"targets": [
3381
+
{
3382
+
"datasource": {
3383
+
"uid": "$datasource"
3384
+
},
3385
+
"editorMode": "code",
3386
+
"expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=~\"prefect-agent.*\", container!=\"\", image!=\"\"}) by (pod)",
3387
+
"format": "time_series",
3388
+
"intervalFactor": 2,
3389
+
"legendFormat": "{{pod}}",
3390
+
"range": true,
3391
+
"refId": "A"
3392
+
}
3393
+
],
3394
+
"title": "Memory Usage (w/o cache)",
3395
+
"type": "timeseries"
3396
+
},
3397
+
{
3398
+
"collapsed": true,
3399
+
"gridPos": {
3400
+
"h": 1,
3401
+
"w": 24,
3402
+
"x": 0,
3403
+
"y": 32
3404
+
},
3405
+
"id": 10,
3406
+
"panels": [
3407
+
{
3408
+
"datasource": {
3409
+
"type": "prometheus",
3410
+
"uid": "${DS_PROMETHEUS}"
3411
+
},
3412
+
"description": "",
3413
+
"fieldConfig": {
3414
+
"defaults": {
3415
+
"color": {
3416
+
"mode": "thresholds"
3417
+
},
3418
+
"custom": {
3419
+
"align": "auto",
3420
+
"cellOptions": {
3421
+
"type": "auto"
3422
+
},
3423
+
"filterable": true,
3424
+
"inspect": false
3425
+
},
3426
+
"mappings": [],
3427
+
"thresholds": {
3428
+
"mode": "absolute",
3429
+
"steps": [
3430
+
{
3431
+
"color": "green",
3432
+
"value": null
3433
+
}
3434
+
]
3435
+
}
3436
+
},
3437
+
"overrides": [
3438
+
{
3439
+
"matcher": {
3440
+
"id": "byName",
3441
+
"options": "created"
3442
+
},
3443
+
"properties": [
3444
+
{
3445
+
"id": "unit",
3446
+
"value": "dateTimeAsIso"
3447
+
}
3448
+
]
3449
+
},
3450
+
{
3451
+
"matcher": {
3452
+
"id": "byName",
3453
+
"options": "name"
3454
+
},
3455
+
"properties": [
3456
+
{
3457
+
"id": "custom.width",
3458
+
"value": 201
3459
+
}
3460
+
]
3461
+
},
3462
+
{
3463
+
"matcher": {
3464
+
"id": "byName",
3465
+
"options": "spent time"
3466
+
},
3467
+
"properties": [
3468
+
{
3469
+
"id": "unit",
3470
+
"value": "s"
3471
+
},
3472
+
{
3473
+
"id": "custom.width",
3474
+
"value": 316
3475
+
}
3476
+
]
3477
+
},
3478
+
{
3479
+
"matcher": {
3480
+
"id": "byName",
3481
+
"options": "state"
3482
+
},
3483
+
"properties": [
3484
+
{
3485
+
"id": "custom.cellOptions",
3486
+
"value": {
3487
+
"mode": "basic",
3488
+
"type": "color-background"
3489
+
}
3490
+
},
3491
+
{
3492
+
"id": "mappings",
3493
+
"value": [
3494
+
{
3495
+
"options": {
3496
+
"Cancelled": {
3497
+
"color": "orange",
3498
+
"index": 2
3499
+
},
3500
+
"Completed": {
3501
+
"color": "green",
3502
+
"index": 0
3503
+
},
3504
+
"Crashed": {
3505
+
"color": "red",
3506
+
"index": 3
3507
+
},
3508
+
"Failed": {
3509
+
"color": "red",
3510
+
"index": 1
3511
+
},
3512
+
"Running": {
3513
+
"color": "blue",
3514
+
"index": 4
3515
+
}
3516
+
},
3517
+
"type": "value"
3518
+
}
3519
+
]
3520
+
}
3521
+
]
3522
+
},
3523
+
{
3524
+
"matcher": {
3525
+
"id": "byName",
3526
+
"options": "start_time"
3527
+
},
3528
+
"properties": [
3529
+
{
3530
+
"id": "unit",
3531
+
"value": "dateTimeAsIso"
3532
+
}
3533
+
]
3534
+
},
3535
+
{
3536
+
"matcher": {
3537
+
"id": "byName",
3538
+
"options": "end_time"
3539
+
},
3540
+
"properties": [
3541
+
{
3542
+
"id": "unit",
3543
+
"value": "dateTimeAsIso"
3544
+
},
3545
+
{
3546
+
"id": "mappings",
3547
+
"value": [
3548
+
{
3549
+
"options": {
3550
+
"match": "nan",
3551
+
"result": {
3552
+
"index": 0,
3553
+
"text": "-"
3554
+
}
3555
+
},
3556
+
"type": "special"
3557
+
}
3558
+
]
3559
+
}
3560
+
]
3561
+
},
3562
+
{
3563
+
"matcher": {
3564
+
"id": "byName",
3565
+
"options": "runs"
3566
+
},
3567
+
"properties": [
3568
+
{
3569
+
"id": "custom.width",
3570
+
"value": 201
3571
+
}
3572
+
]
3573
+
},
3574
+
{
3575
+
"matcher": {
3576
+
"id": "byName",
3577
+
"options": "flow_run_name"
3578
+
},
3579
+
"properties": [
3580
+
{
3581
+
"id": "custom.width",
3582
+
"value": 447
3583
+
}
3584
+
]
3585
+
}
3586
+
]
3587
+
},
3588
+
"gridPos": {
3589
+
"h": 10,
3590
+
"w": 24,
3591
+
"x": 0,
3592
+
"y": 14
3593
+
},
3594
+
"hideTimeOverride": true,
3595
+
"id": 13,
3596
+
"options": {
3597
+
"cellHeight": "sm",
3598
+
"footer": {
3599
+
"countRows": false,
3600
+
"enablePagination": true,
3601
+
"fields": [],
3602
+
"reducer": [
3603
+
"sum"
3604
+
],
3605
+
"show": false
3606
+
},
3607
+
"frameIndex": 0,
3608
+
"showHeader": true,
3609
+
"sortBy": [
3610
+
{
3611
+
"desc": true,
3612
+
"displayName": "end_time"
3613
+
}
3614
+
]
3615
+
},
3616
+
"pluginVersion": "10.2.2",
3617
+
"targets": [
3618
+
{
3619
+
"datasource": {
3620
+
"type": "prometheus",
3621
+
"uid": "${DS_PROMETHEUS}"
3622
+
},
3623
+
"editorMode": "code",
3624
+
"exemplar": false,
3625
+
"expr": "prefect_info_flow_runs{state_name=~\"$flow_run_state_name\", flow_name=~\"$flow_name\"}",
3626
+
"format": "table",
3627
+
"instant": true,
3628
+
"legendFormat": "",
3629
+
"range": false,
3630
+
"refId": "A"
3631
+
}
3632
+
],
3633
+
"transformations": [
3634
+
{
3635
+
"id": "organize",
3636
+
"options": {
3637
+
"excludeByName": {
3638
+
"Time": true,
3639
+
"Value": true,
3640
+
"__name__": true,
3641
+
"container": true,
3642
+
"created": true,
3643
+
"deployment_name": true,
3644
+
"endpoint": true,
3645
+
"flow_id": true,
3646
+
"flow_name": true,
3647
+
"flow_run_id": true,
3648
+
"instance": true,
3649
+
"job": true,
3650
+
"name": false,
3651
+
"namespace": true,
3652
+
"pod": true,
3653
+
"service": true,
3654
+
"start_time": false,
3655
+
"state_id": true,
3656
+
"updated": true,
3657
+
"work_queue_name": true
3658
+
},
3659
+
"indexByName": {
3660
+
"Time": 8,
3661
+
"Value": 22,
3662
+
"__name__": 9,
3663
+
"container": 10,
3664
+
"created": 18,
3665
+
"deployment_id": 6,
3666
+
"deployment_name": 7,
3667
+
"end_time": 20,
3668
+
"endpoint": 11,
3669
+
"flow_id": 4,
3670
+
"flow_name": 5,
3671
+
"flow_run_id": 1,
3672
+
"flow_run_name": 0,
3673
+
"instance": 12,
3674
+
"job": 13,
3675
+
"namespace": 14,
3676
+
"pod": 15,
3677
+
"prefect_info_flow_runs": 23,
3678
+
"run_count": 2,
3679
+
"service": 16,
3680
+
"start_time": 19,
3681
+
"state_id": 21,
3682
+
"total_run_time": 3,
3683
+
"work_queue_name": 17
3684
+
},
3685
+
"renameByName": {
3686
+
"is_schedule_active": "scheduled",
3687
+
"job": "",
3688
+
"prefect_info_flow_runs": "state",
3689
+
"run_count": "runs",
3690
+
"start_time": "",
3691
+
"state_name": "state",
3692
+
"total_run_time": "spent time"
3693
+
}
3694
+
}
3695
+
}
3696
+
],
3697
+
"type": "table"
3698
+
}
3699
+
],
3700
+
"title": "Flows Runs (last 24h)",
3701
+
"type": "row"
3702
+
},
3703
+
{
3704
+
"collapsed": true,
3705
+
"gridPos": {
3706
+
"h": 1,
3707
+
"w": 24,
3708
+
"x": 0,
3709
+
"y": 33
3710
+
},
3711
+
"id": 9,
3712
+
"panels": [
3713
+
{
3714
+
"datasource": {
3715
+
"type": "prometheus",
3716
+
"uid": "${DS_PROMETHEUS}"
3717
+
},
3718
+
"description": "",
3719
+
"fieldConfig": {
3720
+
"defaults": {
3721
+
"color": {
3722
+
"mode": "thresholds"
3723
+
},
3724
+
"custom": {
3725
+
"align": "auto",
3726
+
"cellOptions": {
3727
+
"type": "auto"
3728
+
},
3729
+
"filterable": true,
3730
+
"inspect": false
3731
+
},
3732
+
"mappings": [],
3733
+
"thresholds": {
3734
+
"mode": "absolute",
3735
+
"steps": [
3736
+
{
3737
+
"color": "green"
3738
+
}
3739
+
]
3740
+
}
3741
+
},
3742
+
"overrides": [
3743
+
{
3744
+
"matcher": {
3745
+
"id": "byName",
3746
+
"options": "created"
3747
+
},
3748
+
"properties": [
3749
+
{
3750
+
"id": "unit",
3751
+
"value": "dateTimeAsIso"
3752
+
}
3753
+
]
3754
+
}
3755
+
]
3756
+
},
3757
+
"gridPos": {
3758
+
"h": 10,
3759
+
"w": 24,
3760
+
"x": 0,
3761
+
"y": 61
3762
+
},
3763
+
"id": 12,
3764
+
"options": {
3765
+
"cellHeight": "sm",
3766
+
"footer": {
3767
+
"countRows": false,
3768
+
"enablePagination": true,
3769
+
"fields": [],
3770
+
"reducer": [
3771
+
"sum"
3772
+
],
3773
+
"show": false
3774
+
},
3775
+
"frameIndex": 0,
3776
+
"showHeader": true,
3777
+
"sortBy": [
3778
+
{
3779
+
"desc": true,
3780
+
"displayName": "scheduled"
3781
+
}
3782
+
]
3783
+
},
3784
+
"pluginVersion": "10.1.5",
3785
+
"targets": [
3786
+
{
3787
+
"datasource": {
3788
+
"type": "prometheus",
3789
+
"uid": "${DS_PROMETHEUS}"
3790
+
},
3791
+
"editorMode": "code",
3792
+
"exemplar": false,
3793
+
"expr": "prefect_info_flows{flow_name=~\"$flow_name\"} == 1",
3794
+
"format": "table",
3795
+
"instant": true,
3796
+
"legendFormat": "",
3797
+
"range": false,
3798
+
"refId": "A"
3799
+
}
3800
+
],
3801
+
"transformations": [
3802
+
{
3803
+
"id": "organize",
3804
+
"options": {
3805
+
"excludeByName": {
3806
+
"Time": true,
3807
+
"Value": true,
3808
+
"__name__": true,
3809
+
"container": true,
3810
+
"endpoint": true,
3811
+
"instance": true,
3812
+
"job": true,
3813
+
"namespace": true,
3814
+
"pod": true,
3815
+
"service": true
3816
+
},
3817
+
"indexByName": {
3818
+
"Time": 2,
3819
+
"Value": 12,
3820
+
"__name__": 3,
3821
+
"container": 4,
3822
+
"created": 5,
3823
+
"endpoint": 6,
3824
+
"flow_id": 1,
3825
+
"flow_name": 0,
3826
+
"instance": 7,
3827
+
"job": 8,
3828
+
"namespace": 9,
3829
+
"pod": 10,
3830
+
"service": 11
3831
+
},
3832
+
"renameByName": {}
3833
+
}
3834
+
}
3835
+
],
3836
+
"type": "table"
3837
+
}
3838
+
],
3839
+
"title": "Flows",
3840
+
"type": "row"
3841
+
},
3842
+
{
3843
+
"collapsed": true,
3844
+
"gridPos": {
3845
+
"h": 1,
3846
+
"w": 24,
3847
+
"x": 0,
3848
+
"y": 34
3849
+
},
3850
+
"id": 8,
3851
+
"panels": [
3852
+
{
3853
+
"datasource": {
3854
+
"type": "prometheus",
3855
+
"uid": "${DS_PROMETHEUS}"
3856
+
},
3857
+
"description": "",
3858
+
"fieldConfig": {
3859
+
"defaults": {
3860
+
"color": {
3861
+
"mode": "thresholds"
3862
+
},
3863
+
"custom": {
3864
+
"align": "auto",
3865
+
"cellOptions": {
3866
+
"type": "auto"
3867
+
},
3868
+
"filterable": true,
3869
+
"inspect": false
3870
+
},
3871
+
"mappings": [],
3872
+
"thresholds": {
3873
+
"mode": "absolute",
3874
+
"steps": [
3875
+
{
3876
+
"color": "green"
3877
+
}
3878
+
]
3879
+
}
3880
+
},
3881
+
"overrides": [
3882
+
{
3883
+
"matcher": {
3884
+
"id": "byName",
3885
+
"options": "is_schedule_active"
3886
+
},
3887
+
"properties": [
3888
+
{
3889
+
"id": "custom.cellOptions",
3890
+
"value": {
3891
+
"mode": "basic",
3892
+
"type": "color-background"
3893
+
}
3894
+
},
3895
+
{
3896
+
"id": "mappings",
3897
+
"value": [
3898
+
{
3899
+
"options": {
3900
+
"False": {
3901
+
"color": "red",
3902
+
"index": 1,
3903
+
"text": "Disable"
3904
+
},
3905
+
"True": {
3906
+
"color": "green",
3907
+
"index": 0,
3908
+
"text": "Enable"
3909
+
}
3910
+
},
3911
+
"type": "value"
3912
+
}
3913
+
]
3914
+
}
3915
+
]
3916
+
},
3917
+
{
3918
+
"matcher": {
3919
+
"id": "byName",
3920
+
"options": "created"
3921
+
},
3922
+
"properties": [
3923
+
{
3924
+
"id": "unit",
3925
+
"value": "dateTimeAsIso"
3926
+
}
3927
+
]
3928
+
},
3929
+
{
3930
+
"matcher": {
3931
+
"id": "byName",
3932
+
"options": "updated"
3933
+
},
3934
+
"properties": [
3935
+
{
3936
+
"id": "unit",
3937
+
"value": "dateTimeAsIso"
3938
+
}
3939
+
]
3940
+
},
3941
+
{
3942
+
"matcher": {
3943
+
"id": "byName",
3944
+
"options": "name"
3945
+
},
3946
+
"properties": [
3947
+
{
3948
+
"id": "custom.width",
3949
+
"value": 233
3950
+
}
3951
+
]
3952
+
}
3953
+
]
3954
+
},
3955
+
"gridPos": {
3956
+
"h": 10,
3957
+
"w": 24,
3958
+
"x": 0,
3959
+
"y": 39
3960
+
},
3961
+
"id": 11,
3962
+
"options": {
3963
+
"cellHeight": "sm",
3964
+
"footer": {
3965
+
"countRows": false,
3966
+
"enablePagination": true,
3967
+
"fields": [],
3968
+
"reducer": [
3969
+
"sum"
3970
+
],
3971
+
"show": false
3972
+
},
3973
+
"frameIndex": 0,
3974
+
"showHeader": true,
3975
+
"sortBy": [
3976
+
{
3977
+
"desc": true,
3978
+
"displayName": "scheduled"
3979
+
}
3980
+
]
3981
+
},
3982
+
"pluginVersion": "10.1.5",
3983
+
"targets": [
3984
+
{
3985
+
"datasource": {
3986
+
"type": "prometheus",
3987
+
"uid": "${DS_PROMETHEUS}"
3988
+
},
3989
+
"editorMode": "code",
3990
+
"exemplar": false,
3991
+
"expr": "prefect_info_deployment{flow_name=~\"$flow_name\"}",
3992
+
"format": "table",
3993
+
"instant": true,
3994
+
"legendFormat": "",
3995
+
"range": false,
3996
+
"refId": "A"
3997
+
}
3998
+
],
3999
+
"transformations": [
4000
+
{
4001
+
"id": "organize",
4002
+
"options": {
4003
+
"excludeByName": {
4004
+
"Time": true,
4005
+
"Value": true,
4006
+
"__name__": true,
4007
+
"container": true,
4008
+
"created": false,
4009
+
"endpoint": true,
4010
+
"instance": true,
4011
+
"job": true,
4012
+
"name": false,
4013
+
"namespace": true,
4014
+
"pod": true,
4015
+
"service": true,
4016
+
"work_pool_name": true,
4017
+
"work_queue_name": true
4018
+
},
4019
+
"indexByName": {
4020
+
"Time": 7,
4021
+
"Value": 18,
4022
+
"__name__": 8,
4023
+
"container": 9,
4024
+
"created": 5,
4025
+
"deployment_id": 1,
4026
+
"deployment_name": 0,
4027
+
"endpoint": 10,
4028
+
"flow_id": 4,
4029
+
"flow_name": 3,
4030
+
"instance": 11,
4031
+
"is_schedule_active": 2,
4032
+
"job": 12,
4033
+
"namespace": 13,
4034
+
"path": 6,
4035
+
"pod": 14,
4036
+
"service": 15,
4037
+
"work_pool_name": 16,
4038
+
"work_queue_name": 17
4039
+
},
4040
+
"renameByName": {
4041
+
"flow_id": "",
4042
+
"id": "",
4043
+
"is_schedule_active": "scheduled",
4044
+
"job": ""
4045
+
}
4046
+
}
4047
+
}
4048
+
],
4049
+
"type": "table"
4050
+
}
4051
+
],
4052
+
"title": "Deployments",
4053
+
"type": "row"
4054
+
}
4055
+
],
4056
+
"refresh": "30s",
4057
+
"schemaVersion": 38,
4058
+
"tags": [
4059
+
"prefect",
4060
+
"infrastructure",
4061
+
"chart",
4062
+
"monitoring"
4063
+
],
4064
+
"templating": {
4065
+
"list": [
4066
+
{
4067
+
"current": {
4068
+
"selected": false,
4069
+
"text": "DS_PROMETHEUS",
4070
+
"value": "DS_PROMETHEUS"
4071
+
},
4072
+
"hide": 2,
4073
+
"includeAll": false,
4074
+
"label": "",
4075
+
"multi": false,
4076
+
"name": "DS_PROMETHEUS",
4077
+
"options": [
4078
+
{
4079
+
"selected": true,
4080
+
"text": "DS_PROMETHEUS",
4081
+
"value": "DS_PROMETHEUS"
4082
+
}
4083
+
],
4084
+
"query": "DS_PROMETHEUS",
4085
+
"skipUrlSync": false,
4086
+
"type": "constant"
4087
+
},
4088
+
{
4089
+
"current": {},
4090
+
"datasource": {
4091
+
"type": "prometheus",
4092
+
"uid": "${DS_PROMETHEUS}"
4093
+
},
4094
+
"definition": "label_values(prefect_info_deployment,namespace)",
4095
+
"hide": 0,
4096
+
"includeAll": false,
4097
+
"label": "Namespace",
4098
+
"multi": false,
4099
+
"name": "namespace",
4100
+
"options": [],
4101
+
"query": {
4102
+
"query": "label_values(prefect_info_deployment,namespace)",
4103
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
4104
+
},
4105
+
"refresh": 1,
4106
+
"regex": "",
4107
+
"skipUrlSync": false,
4108
+
"sort": 0,
4109
+
"type": "query"
4110
+
},
4111
+
{
4112
+
"current": {},
4113
+
"datasource": {
4114
+
"type": "prometheus",
4115
+
"uid": "${DS_PROMETHEUS}"
4116
+
},
4117
+
"definition": "label_values(prefect_info_flows,flow_name)",
4118
+
"hide": 0,
4119
+
"includeAll": true,
4120
+
"label": "Flow Name",
4121
+
"multi": false,
4122
+
"name": "flow_name",
4123
+
"options": [],
4124
+
"query": {
4125
+
"query": "label_values(prefect_info_flows,flow_name)",
4126
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
4127
+
},
4128
+
"refresh": 1,
4129
+
"regex": "",
4130
+
"skipUrlSync": false,
4131
+
"sort": 0,
4132
+
"type": "query"
4133
+
},
4134
+
{
4135
+
"current": {},
4136
+
"datasource": {
4137
+
"type": "prometheus",
4138
+
"uid": "${DS_PROMETHEUS}"
4139
+
},
4140
+
"definition": "label_values(prefect_info_flow_runs,state_name)",
4141
+
"hide": 0,
4142
+
"includeAll": true,
4143
+
"label": "Flow Run State",
4144
+
"multi": false,
4145
+
"name": "flow_run_state_name",
4146
+
"options": [],
4147
+
"query": {
4148
+
"query": "label_values(prefect_info_flow_runs,state_name)",
4149
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
4150
+
},
4151
+
"refresh": 1,
4152
+
"regex": "",
4153
+
"skipUrlSync": false,
4154
+
"sort": 0,
4155
+
"type": "query"
4156
+
},
4157
+
{
4158
+
"current": {
4159
+
"selected": false,
4160
+
"text": "Prometheus",
4161
+
"value": "prometheus"
4162
+
},
4163
+
"hide": 2,
4164
+
"includeAll": false,
4165
+
"label": "Data Source",
4166
+
"multi": false,
4167
+
"name": "datasource",
4168
+
"options": [],
4169
+
"query": "prometheus",
4170
+
"refresh": 1,
4171
+
"regex": "",
4172
+
"skipUrlSync": false,
4173
+
"type": "datasource"
4174
+
}
4175
+
]
4176
+
},
4177
+
"time": {
4178
+
"from": "now-24h",
4179
+
"to": "now"
4180
+
},
4181
+
"timepicker": {
4182
+
"hidden": false
4183
+
},
4184
+
"timezone": "browser",
4185
+
"title": "Prefect / Platform Overview",
4186
+
"uid": "prefect-platform-overview",
4187
+
"version": 5,
4188
+
"weekStart": "monday"
4189
+
}
+12
deploy/exporter-values.yaml
+12
deploy/exporter-values.yaml
···
1
+
# prometheus-prefect-exporter values
2
+
prefectApiUrl: "http://prefect-server.prefect.svc.cluster.local:4200/api"
3
+
4
+
basicAuth:
5
+
enabled: true
6
+
existingSecret: prefect-auth
7
+
existingSecretKey: auth-string
8
+
9
+
serviceMonitor:
10
+
enabled: true
11
+
interval: 30s
12
+
scrapeTimeout: 10s
+24
deploy/grafana-ingress.yaml
+24
deploy/grafana-ingress.yaml
···
1
+
apiVersion: networking.k8s.io/v1
2
+
kind: Ingress
3
+
metadata:
4
+
name: grafana
5
+
namespace: monitoring
6
+
annotations:
7
+
cert-manager.io/cluster-issuer: letsencrypt-prod
8
+
spec:
9
+
ingressClassName: traefik
10
+
tls:
11
+
- hosts:
12
+
- GRAFANA_DOMAIN_PLACEHOLDER
13
+
secretName: grafana-tls
14
+
rules:
15
+
- host: GRAFANA_DOMAIN_PLACEHOLDER
16
+
http:
17
+
paths:
18
+
- path: /
19
+
pathType: Prefix
20
+
backend:
21
+
service:
22
+
name: kube-prometheus-stack-grafana
23
+
port:
24
+
number: 80
+81
deploy/monitoring-values.yaml
+81
deploy/monitoring-values.yaml
···
1
+
# lightweight kube-prometheus-stack — follows relay/indigo pattern
2
+
alertmanager:
3
+
enabled: false
4
+
5
+
nodeExporter:
6
+
enabled: true
7
+
8
+
kubeStateMetrics:
9
+
enabled: true
10
+
11
+
kubeApiServer:
12
+
enabled: false
13
+
14
+
kubeControllerManager:
15
+
enabled: false
16
+
17
+
kubeScheduler:
18
+
enabled: false
19
+
20
+
kubeProxy:
21
+
enabled: false
22
+
23
+
kubeEtcd:
24
+
enabled: false
25
+
26
+
kubeDns:
27
+
enabled: false
28
+
29
+
coreDns:
30
+
enabled: false
31
+
32
+
kubelet:
33
+
enabled: true
34
+
serviceMonitor:
35
+
cAdvisor: true
36
+
37
+
prometheus:
38
+
prometheusSpec:
39
+
scrapeInterval: 30s
40
+
retention: 14d
41
+
serviceMonitorSelectorNilUsesHelmValues: false
42
+
storageSpec:
43
+
volumeClaimTemplate:
44
+
spec:
45
+
accessModes: ["ReadWriteOnce"]
46
+
resources:
47
+
requests:
48
+
storage: 10Gi
49
+
resources:
50
+
requests:
51
+
memory: 256Mi
52
+
limits:
53
+
memory: 1Gi
54
+
55
+
prometheusOperator:
56
+
resources:
57
+
requests:
58
+
memory: 64Mi
59
+
limits:
60
+
memory: 128Mi
61
+
62
+
grafana:
63
+
enabled: true
64
+
adminPassword: bufobufobufo
65
+
grafana.ini:
66
+
auth.anonymous:
67
+
enabled: true
68
+
org_role: Viewer
69
+
server:
70
+
root_url: "https://GRAFANA_DOMAIN_PLACEHOLDER"
71
+
72
+
sidecar:
73
+
dashboards:
74
+
enabled: true
75
+
label: grafana_dashboard
76
+
77
+
resources:
78
+
requests:
79
+
memory: 128Mi
80
+
limits:
81
+
memory: 256Mi
+39
deploy/prefect-values.yaml
+39
deploy/prefect-values.yaml
···
1
+
server:
2
+
replicaCount: 2
3
+
4
+
basicAuth:
5
+
enabled: true
6
+
existingSecret: prefect-auth
7
+
8
+
uiConfig:
9
+
prefectUiApiUrl: "https://DOMAIN_PLACEHOLDER/api"
10
+
11
+
backgroundServices:
12
+
runAsSeparateDeployment: true
13
+
replicaCount: 1
14
+
15
+
postgresql:
16
+
enabled: true
17
+
auth:
18
+
enablePostgresUser: false
19
+
database: prefect
20
+
username: prefect
21
+
primary:
22
+
persistence:
23
+
enabled: true
24
+
size: 10Gi
25
+
26
+
redis:
27
+
enabled: true
28
+
architecture: standalone
29
+
30
+
ingress:
31
+
enabled: true
32
+
className: traefik
33
+
host:
34
+
hostname: DOMAIN_PLACEHOLDER
35
+
path: /
36
+
pathType: Prefix
37
+
annotations:
38
+
cert-manager.io/cluster-issuer: letsencrypt-prod
39
+
tls: true
+66
deploy/worker.yaml
+66
deploy/worker.yaml
···
1
+
apiVersion: apps/v1
2
+
kind: Deployment
3
+
metadata:
4
+
name: prefect-worker
5
+
namespace: prefect
6
+
spec:
7
+
replicas: 1
8
+
selector:
9
+
matchLabels:
10
+
app: prefect-worker
11
+
template:
12
+
metadata:
13
+
labels:
14
+
app: prefect-worker
15
+
spec:
16
+
serviceAccountName: prefect-worker
17
+
containers:
18
+
- name: worker
19
+
image: prefecthq/prefect:3-latest
20
+
command:
21
+
- prefect
22
+
- worker
23
+
- start
24
+
- --pool
25
+
- kubernetes-pool
26
+
env:
27
+
- name: PREFECT_API_URL
28
+
value: http://prefect-server.prefect.svc.cluster.local:4200/api
29
+
- name: PREFECT_API_AUTH_STRING
30
+
valueFrom:
31
+
secretKeyRef:
32
+
name: prefect-auth
33
+
key: auth-string
34
+
---
35
+
apiVersion: v1
36
+
kind: ServiceAccount
37
+
metadata:
38
+
name: prefect-worker
39
+
namespace: prefect
40
+
---
41
+
apiVersion: rbac.authorization.k8s.io/v1
42
+
kind: Role
43
+
metadata:
44
+
name: prefect-worker
45
+
namespace: prefect
46
+
rules:
47
+
- apiGroups: ["batch"]
48
+
resources: ["jobs"]
49
+
verbs: ["create", "get", "list", "watch", "delete"]
50
+
- apiGroups: [""]
51
+
resources: ["pods", "pods/log"]
52
+
verbs: ["get", "list", "watch"]
53
+
---
54
+
apiVersion: rbac.authorization.k8s.io/v1
55
+
kind: RoleBinding
56
+
metadata:
57
+
name: prefect-worker
58
+
namespace: prefect
59
+
subjects:
60
+
- kind: ServiceAccount
61
+
name: prefect-worker
62
+
namespace: prefect
63
+
roleRef:
64
+
kind: Role
65
+
name: prefect-worker
66
+
apiGroup: rbac.authorization.k8s.io
+27
flows/diagnostics.py
+27
flows/diagnostics.py
···
1
+
"""simple diagnostic flow — runs every 5 minutes, prints system info."""
2
+
3
+
import datetime
4
+
import os
5
+
import platform
6
+
7
+
from prefect import flow
8
+
9
+
10
+
@flow(log_prints=True)
11
+
def diagnostics():
12
+
now = datetime.datetime.now(datetime.timezone.utc)
13
+
print(f"time: {now.isoformat()}")
14
+
print(f"hostname: {platform.node()}")
15
+
print(f"python: {platform.python_version()}")
16
+
print(f"platform: {platform.platform()}")
17
+
print(f"pid: {os.getpid()}")
18
+
print(f"cwd: {os.getcwd()}")
19
+
20
+
21
+
if __name__ == "__main__":
22
+
diagnostics.deploy(
23
+
name="diagnostics-every-5m",
24
+
work_pool_name="kubernetes-pool",
25
+
image="prefecthq/prefect:3-latest",
26
+
cron="*/5 * * * *",
27
+
)
+65
infra/main.tf
+65
infra/main.tf
···
1
+
resource "hcloud_ssh_key" "default" {
2
+
name = "${var.server_name}-key"
3
+
public_key = file(pathexpand(var.ssh_public_key_path))
4
+
}
5
+
6
+
resource "hcloud_firewall" "prefect" {
7
+
name = "${var.server_name}-fw"
8
+
9
+
# ssh
10
+
rule {
11
+
direction = "in"
12
+
protocol = "tcp"
13
+
port = "22"
14
+
source_ips = ["0.0.0.0/0", "::/0"]
15
+
}
16
+
17
+
# http
18
+
rule {
19
+
direction = "in"
20
+
protocol = "tcp"
21
+
port = "80"
22
+
source_ips = ["0.0.0.0/0", "::/0"]
23
+
}
24
+
25
+
# https
26
+
rule {
27
+
direction = "in"
28
+
protocol = "tcp"
29
+
port = "443"
30
+
source_ips = ["0.0.0.0/0", "::/0"]
31
+
}
32
+
33
+
# k3s API
34
+
rule {
35
+
direction = "in"
36
+
protocol = "tcp"
37
+
port = "6443"
38
+
source_ips = ["0.0.0.0/0", "::/0"]
39
+
}
40
+
}
41
+
42
+
resource "hcloud_server" "prefect" {
43
+
name = var.server_name
44
+
server_type = var.server_type
45
+
location = var.location
46
+
image = "ubuntu-24.04"
47
+
48
+
ssh_keys = [hcloud_ssh_key.default.id]
49
+
firewall_ids = [hcloud_firewall.prefect.id]
50
+
51
+
user_data = <<-CLOUDINIT
52
+
#cloud-config
53
+
package_update: true
54
+
packages:
55
+
- curl
56
+
- jq
57
+
58
+
runcmd:
59
+
- |
60
+
PUBLIC_IP=$(curl -s http://169.254.169.254/hetzner/v1/metadata/public-ipv4)
61
+
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --tls-san $PUBLIC_IP" sh -
62
+
- while ! kubectl get nodes >/dev/null 2>&1; do sleep 2; done
63
+
- touch /run/k3s-ready
64
+
CLOUDINIT
65
+
}
+9
infra/outputs.tf
+9
infra/outputs.tf
+29
infra/variables.tf
+29
infra/variables.tf
···
1
+
variable "hcloud_token" {
2
+
description = "Hetzner Cloud API token"
3
+
type = string
4
+
sensitive = true
5
+
}
6
+
7
+
variable "ssh_public_key_path" {
8
+
description = "Path to SSH public key"
9
+
type = string
10
+
default = "~/.ssh/id_ed25519.pub"
11
+
}
12
+
13
+
variable "server_type" {
14
+
description = "Hetzner server type (cpx21 = 3 vCPU, 4 GB RAM, 80 GB disk)"
15
+
type = string
16
+
default = "cpx21"
17
+
}
18
+
19
+
variable "location" {
20
+
description = "Hetzner datacenter location"
21
+
type = string
22
+
default = "ash"
23
+
}
24
+
25
+
variable "server_name" {
26
+
description = "Name for the server"
27
+
type = string
28
+
default = "prefect-server"
29
+
}
+14
infra/versions.tf
+14
infra/versions.tf
+187
justfile
+187
justfile
···
1
+
# my-prefect-server deployment (k3s + helm)
2
+
# required env vars: HCLOUD_TOKEN, POSTGRES_PASSWORD, AUTH_STRING, DOMAIN, LETSENCRYPT_EMAIL
3
+
# optional env vars: GRAFANA_DOMAIN (default: prefect-metrics.waow.tech)
4
+
5
+
set dotenv-load
6
+
7
+
export KUBECONFIG := source_directory() / "kubeconfig.yaml"
8
+
9
+
# --- infrastructure ---
10
+
11
+
# initialize terraform
12
+
init:
13
+
terraform -chdir=infra init
14
+
15
+
# create the hetzner server with k3s
16
+
infra:
17
+
terraform -chdir=infra apply -var="hcloud_token=$HCLOUD_TOKEN"
18
+
19
+
# plan infra changes
20
+
infra-plan:
21
+
terraform -chdir=infra plan -var="hcloud_token=$HCLOUD_TOKEN"
22
+
23
+
# destroy all infrastructure
24
+
destroy:
25
+
terraform -chdir=infra destroy -var="hcloud_token=$HCLOUD_TOKEN"
26
+
27
+
# get the server IP from terraform
28
+
server-ip:
29
+
@terraform -chdir=infra output -raw server_ip
30
+
31
+
# ssh into the server
32
+
ssh:
33
+
ssh root@$(just server-ip)
34
+
35
+
# --- cluster access ---
36
+
37
+
# fetch kubeconfig from the server (run after cloud-init finishes)
38
+
kubeconfig:
39
+
#!/usr/bin/env bash
40
+
set -euo pipefail
41
+
IP=$(just server-ip)
42
+
echo "fetching kubeconfig from $IP..."
43
+
until ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new root@$IP test -f /run/k3s-ready 2>/dev/null; do
44
+
echo " waiting for k3s..."
45
+
sleep 5
46
+
done
47
+
scp root@$IP:/etc/rancher/k3s/k3s.yaml kubeconfig.yaml
48
+
if [[ "$(uname)" == "Darwin" ]]; then
49
+
sed -i '' "s|127.0.0.1|$IP|g" kubeconfig.yaml
50
+
else
51
+
sed -i "s|127.0.0.1|$IP|g" kubeconfig.yaml
52
+
fi
53
+
chmod 600 kubeconfig.yaml
54
+
echo "kubeconfig written"
55
+
kubectl get nodes
56
+
57
+
# --- deployment ---
58
+
59
+
# deploy everything to the cluster
60
+
deploy:
61
+
#!/usr/bin/env bash
62
+
set -euo pipefail
63
+
64
+
helm repo add prefect https://prefecthq.github.io/prefect-helm
65
+
helm repo add jetstack https://charts.jetstack.io
66
+
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
67
+
helm repo update
68
+
69
+
: "${DOMAIN:?set DOMAIN}"
70
+
: "${AUTH_STRING:?set AUTH_STRING}"
71
+
: "${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}"
72
+
: "${LETSENCRYPT_EMAIL:?set LETSENCRYPT_EMAIL}"
73
+
GRAFANA_DOMAIN="${GRAFANA_DOMAIN:-prefect-metrics.waow.tech}"
74
+
75
+
echo "==> creating namespaces"
76
+
kubectl create namespace prefect --dry-run=client -o yaml | kubectl apply -f -
77
+
kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
78
+
79
+
echo "==> installing cert-manager"
80
+
helm upgrade --install cert-manager jetstack/cert-manager \
81
+
--namespace cert-manager --create-namespace \
82
+
--set crds.enabled=true \
83
+
--wait
84
+
85
+
echo "==> applying cluster issuer"
86
+
sed "s|LETSENCRYPT_EMAIL_PLACEHOLDER|$LETSENCRYPT_EMAIL|g" deploy/cluster-issuer.yaml \
87
+
| kubectl apply -f -
88
+
89
+
echo "==> creating prefect auth secret"
90
+
kubectl create secret generic prefect-auth \
91
+
--namespace prefect \
92
+
--from-literal=auth-string="$AUTH_STRING" \
93
+
--dry-run=client -o yaml | kubectl apply -f -
94
+
95
+
echo "==> installing prefect server"
96
+
sed "s|DOMAIN_PLACEHOLDER|$DOMAIN|g" deploy/prefect-values.yaml \
97
+
| helm upgrade --install prefect-server prefect/prefect-server \
98
+
--namespace prefect \
99
+
--values - \
100
+
--set postgresql.auth.password="$POSTGRES_PASSWORD" \
101
+
--wait --timeout 5m
102
+
103
+
echo "==> installing monitoring stack"
104
+
sed "s|GRAFANA_DOMAIN_PLACEHOLDER|$GRAFANA_DOMAIN|g" deploy/monitoring-values.yaml \
105
+
| helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
106
+
--namespace monitoring \
107
+
--values - \
108
+
--wait --timeout 5m
109
+
110
+
echo "==> applying grafana ingress"
111
+
sed "s|GRAFANA_DOMAIN_PLACEHOLDER|$GRAFANA_DOMAIN|g" deploy/grafana-ingress.yaml \
112
+
| kubectl apply -f -
113
+
114
+
echo "==> loading prefect dashboards"
115
+
for dashboard in deploy/dashboards/*.json; do
116
+
name=$(basename "$dashboard" .json | tr '.' '-')
117
+
kubectl create configmap "prefect-dashboard-$name" \
118
+
--namespace monitoring \
119
+
--from-file="$dashboard" \
120
+
--dry-run=client -o yaml \
121
+
| kubectl label --local -f - grafana_dashboard=1 -o yaml \
122
+
| kubectl apply -f -
123
+
done
124
+
125
+
echo "==> installing prefect exporter"
126
+
helm upgrade --install prometheus-prefect-exporter prefect/prometheus-prefect-exporter \
127
+
--namespace prefect \
128
+
--values deploy/exporter-values.yaml \
129
+
--wait --timeout 2m
130
+
131
+
echo ""
132
+
echo "done. point DNS:"
133
+
echo " $DOMAIN -> $(just server-ip)"
134
+
echo " $GRAFANA_DOMAIN -> $(just server-ip)"
135
+
echo "then check:"
136
+
echo " curl https://$DOMAIN/api/health"
137
+
echo " curl https://$GRAFANA_DOMAIN"
138
+
139
+
# deploy the kubernetes worker to the cluster
140
+
worker:
141
+
kubectl apply -f deploy/worker.yaml
142
+
143
+
# register flow deployments (run locally with PREFECT_API_URL + PREFECT_API_AUTH_STRING)
144
+
register-flows:
145
+
PREFECT_API_URL="https://$DOMAIN/api" PREFECT_API_AUTH_STRING="$AUTH_STRING" \
146
+
uv run --with prefect flows/diagnostics.py
147
+
148
+
# --- prefect resources (terraform) ---
149
+
150
+
# initialize prefect terraform
151
+
prefect-init:
152
+
terraform -chdir=prefect init
153
+
154
+
# apply prefect resources
155
+
prefect-apply:
156
+
terraform -chdir=prefect apply \
157
+
-var="domain=$DOMAIN" \
158
+
-var="auth_string=$AUTH_STRING"
159
+
160
+
# plan prefect resource changes
161
+
prefect-plan:
162
+
terraform -chdir=prefect plan \
163
+
-var="domain=$DOMAIN" \
164
+
-var="auth_string=$AUTH_STRING"
165
+
166
+
# --- operations ---
167
+
168
+
# check the state of everything
169
+
status:
170
+
@echo "==> nodes"
171
+
@kubectl get nodes
172
+
@echo ""
173
+
@echo "==> pods (prefect)"
174
+
@kubectl get pods -n prefect
175
+
@echo ""
176
+
@echo "==> pods (monitoring)"
177
+
@kubectl get pods -n monitoring
178
+
179
+
# tail prefect server logs
180
+
logs:
181
+
kubectl logs -n prefect -l app.kubernetes.io/name=prefect-server -f
182
+
183
+
# check prefect health via public endpoint
184
+
health:
185
+
#!/usr/bin/env bash
186
+
: "${DOMAIN:?set DOMAIN}"
187
+
curl -sf "https://$DOMAIN/api/health" | jq .
+16
prefect/main.tf
+16
prefect/main.tf
···
1
+
provider "prefect" {
2
+
endpoint = "https://${var.domain}/api"
3
+
basic_auth_key = var.auth_string
4
+
}
5
+
6
+
# kubernetes work pool — each flow run gets its own pod
7
+
resource "prefect_work_pool" "k8s" {
8
+
name = "kubernetes-pool"
9
+
type = "kubernetes"
10
+
}
11
+
12
+
# environment identifier
13
+
resource "prefect_variable" "environment" {
14
+
name = "environment"
15
+
value = "dogfood"
16
+
}
+10
prefect/variables.tf
+10
prefect/variables.tf