declarative relay deployment on hetzner
relay-eval.waow.tech
atproto
relay
fix: update dashboard — remove nonexistent metrics panels
- remove rocksdb memory panel (metrics never implemented)
- update resolver panel: queue length + dedup set (remove recycles)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+84
-284
+84
-284
deploy/zlay-dashboard.json
+84
-284
deploy/zlay-dashboard.json
···
8
8
"links": [],
9
9
"panels": [
10
10
{
11
-
"title": "events/sec",
11
+
"title": "throughput",
12
12
"type": "timeseries",
13
13
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 },
14
14
"datasource": { "type": "prometheus", "uid": "prometheus" },
···
27
27
"targets": [
28
28
{
29
29
"expr": "sum(rate(relay_frames_received_total{job=\"zlay\"}[5m]))",
30
-
"legendFormat": "events received",
30
+
"legendFormat": "received",
31
31
"refId": "A"
32
+
},
33
+
{
34
+
"expr": "sum(rate(relay_frames_broadcast_total{job=\"zlay\"}[5m]))",
35
+
"legendFormat": "broadcast",
36
+
"refId": "B"
32
37
}
33
38
]
34
39
},
···
64
69
]
65
70
},
66
71
{
67
-
"title": "downstream consumers",
68
-
"type": "timeseries",
69
-
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
70
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
71
-
"fieldConfig": {
72
-
"defaults": {
73
-
"unit": "ops",
74
-
"color": { "mode": "palette-classic" },
75
-
"custom": {
76
-
"fillOpacity": 15,
77
-
"lineWidth": 2,
78
-
"spanNulls": false
79
-
}
80
-
},
81
-
"overrides": []
82
-
},
83
-
"targets": [
84
-
{
85
-
"expr": "sum(rate(relay_frames_broadcast_total{job=\"zlay\"}[5m]))",
86
-
"legendFormat": "events sent",
87
-
"refId": "A"
88
-
}
89
-
]
90
-
},
91
-
{
92
72
"title": "validation/sec",
93
73
"type": "timeseries",
94
-
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
74
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 },
95
75
"datasource": { "type": "prometheus", "uid": "prometheus" },
96
76
"fieldConfig": {
97
77
"defaults": {
···
124
104
]
125
105
},
126
106
{
127
-
"title": "memory",
128
-
"type": "timeseries",
129
-
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
130
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
131
-
"fieldConfig": {
132
-
"defaults": {
133
-
"unit": "bytes",
134
-
"color": { "mode": "palette-classic" },
135
-
"custom": {
136
-
"fillOpacity": 15,
137
-
"lineWidth": 2,
138
-
"spanNulls": false
139
-
}
140
-
},
141
-
"overrides": []
142
-
},
143
-
"targets": [
144
-
{
145
-
"expr": "sum(container_memory_working_set_bytes{namespace=\"zlay\",pod=~\"zlay.*\",container=\"main\"})",
146
-
"legendFormat": "working set",
147
-
"refId": "A"
148
-
},
149
-
{
150
-
"expr": "max(kube_pod_container_resource_limits{namespace=\"zlay\",pod=~\"zlay.*\",container=\"main\",resource=\"memory\"})",
151
-
"legendFormat": "limit",
152
-
"refId": "B"
153
-
}
154
-
]
155
-
},
156
-
{
157
-
"title": "threads",
158
-
"type": "timeseries",
159
-
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
160
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
161
-
"fieldConfig": {
162
-
"defaults": {
163
-
"color": { "mode": "palette-classic" },
164
-
"custom": {
165
-
"fillOpacity": 15,
166
-
"lineWidth": 2,
167
-
"spanNulls": false
168
-
}
169
-
},
170
-
"overrides": []
171
-
},
172
-
"targets": [
173
-
{
174
-
"expr": "sum(container_threads{namespace=\"zlay\",pod=~\"zlay-[a-z0-9].*\",container=\"main\"})",
175
-
"legendFormat": "threads",
176
-
"refId": "A"
177
-
}
178
-
]
179
-
},
180
-
{
181
-
"title": "key cache/sec",
182
-
"type": "timeseries",
183
-
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
184
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
185
-
"fieldConfig": {
186
-
"defaults": {
187
-
"unit": "ops",
188
-
"color": { "mode": "palette-classic" },
189
-
"custom": {
190
-
"fillOpacity": 15,
191
-
"lineWidth": 2,
192
-
"spanNulls": false
193
-
}
194
-
},
195
-
"overrides": []
196
-
},
197
-
"targets": [
198
-
{
199
-
"expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"hit\"}[5m]))",
200
-
"legendFormat": "hits",
201
-
"refId": "A"
202
-
},
203
-
{
204
-
"expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"miss\"}[5m]))",
205
-
"legendFormat": "misses",
206
-
"refId": "B"
207
-
}
208
-
]
209
-
},
210
-
{
211
-
"title": "errors/sec",
212
-
"type": "timeseries",
213
-
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
214
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
215
-
"fieldConfig": {
216
-
"defaults": {
217
-
"unit": "ops",
218
-
"color": { "mode": "palette-classic" },
219
-
"custom": {
220
-
"fillOpacity": 15,
221
-
"lineWidth": 2,
222
-
"spanNulls": false
223
-
}
224
-
},
225
-
"overrides": []
226
-
},
227
-
"targets": [
228
-
{
229
-
"expr": "sum(rate(relay_decode_errors_total{job=\"zlay\"}[5m]))",
230
-
"legendFormat": "decode errors",
231
-
"refId": "A"
232
-
},
233
-
{
234
-
"expr": "sum(rate(relay_slow_consumers_total{job=\"zlay\"}[5m]))",
235
-
"legendFormat": "slow consumer drops",
236
-
"refId": "B"
237
-
}
238
-
]
239
-
},
240
-
{
241
-
"title": "validator cache entries",
242
-
"type": "timeseries",
243
-
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
244
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
245
-
"fieldConfig": {
246
-
"defaults": {
247
-
"color": { "mode": "palette-classic" },
248
-
"custom": {
249
-
"fillOpacity": 15,
250
-
"lineWidth": 2,
251
-
"spanNulls": false
252
-
}
253
-
},
254
-
"overrides": []
255
-
},
256
-
"targets": [
257
-
{
258
-
"expr": "sum(relay_validator_cache_entries{job=\"zlay\"})",
259
-
"legendFormat": "validator cache",
260
-
"refId": "A"
261
-
},
262
-
{
263
-
"expr": "sum(relay_did_cache_entries{job=\"zlay\"})",
264
-
"legendFormat": "DID cache",
265
-
"refId": "B"
266
-
}
267
-
]
268
-
},
269
-
{
270
-
"title": "cache evictions/sec",
271
-
"type": "timeseries",
272
-
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
273
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
274
-
"fieldConfig": {
275
-
"defaults": {
276
-
"unit": "ops",
277
-
"color": { "mode": "palette-classic" },
278
-
"custom": {
279
-
"fillOpacity": 15,
280
-
"lineWidth": 2,
281
-
"spanNulls": false
282
-
}
283
-
},
284
-
"overrides": []
285
-
},
286
-
"targets": [
287
-
{
288
-
"expr": "sum(rate(relay_validator_cache_evictions_total{job=\"zlay\"}[5m]))",
289
-
"legendFormat": "evictions",
290
-
"refId": "A"
291
-
}
292
-
]
293
-
},
294
-
{
295
107
"title": "malloc breakdown",
296
108
"type": "timeseries",
297
-
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
109
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
298
110
"datasource": { "type": "prometheus", "uid": "prometheus" },
299
111
"fieldConfig": {
300
112
"defaults": {
···
332
144
]
333
145
},
334
146
{
335
-
"title": "rss breakdown",
147
+
"title": "process memory",
336
148
"type": "timeseries",
337
-
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
149
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
338
150
"datasource": { "type": "prometheus", "uid": "prometheus" },
339
151
"fieldConfig": {
340
152
"defaults": {
···
346
158
"spanNulls": false
347
159
}
348
160
},
349
-
"overrides": []
161
+
"overrides": [
162
+
{
163
+
"matcher": { "id": "byName", "options": "limit" },
164
+
"properties": [
165
+
{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } },
166
+
{ "id": "custom.fillOpacity", "value": 0 },
167
+
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
168
+
]
169
+
}
170
+
]
350
171
},
351
172
"targets": [
352
173
{
···
363
184
"expr": "max(relay_vm_hwm_kb{job=\"zlay\"}) * 1024",
364
185
"legendFormat": "VmHWM (peak RSS)",
365
186
"refId": "C"
366
-
}
367
-
]
368
-
},
369
-
{
370
-
"title": "disk used %",
371
-
"type": "gauge",
372
-
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 40 },
373
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
374
-
"fieldConfig": {
375
-
"defaults": {
376
-
"unit": "percentunit",
377
-
"min": 0,
378
-
"max": 1,
379
-
"color": { "mode": "thresholds" },
380
-
"thresholds": {
381
-
"steps": [
382
-
{ "color": "green", "value": null },
383
-
{ "color": "yellow", "value": 0.7 },
384
-
{ "color": "red", "value": 0.85 }
385
-
]
386
-
}
387
187
},
388
-
"overrides": []
389
-
},
390
-
"options": {
391
-
"reduceOptions": { "calcs": ["lastNotNull"] }
392
-
},
393
-
"targets": [
394
188
{
395
-
"expr": "max(1 - (relay_disk_available_bytes{job=\"zlay\"} / relay_disk_total_bytes{job=\"zlay\"}))",
396
-
"legendFormat": "used",
397
-
"refId": "A"
189
+
"expr": "max(kube_pod_container_resource_limits{namespace=\"zlay\",pod=~\"zlay-[a-z0-9].*\",container=\"main\",resource=\"memory\"})",
190
+
"legendFormat": "limit",
191
+
"refId": "D"
398
192
}
399
193
]
400
194
},
401
195
{
402
-
"title": "disk usage",
196
+
"title": "caches",
403
197
"type": "timeseries",
404
-
"gridPos": { "h": 8, "w": 16, "x": 8, "y": 40 },
198
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
405
199
"datasource": { "type": "prometheus", "uid": "prometheus" },
406
200
"fieldConfig": {
407
201
"defaults": {
408
-
"unit": "bytes",
409
202
"color": { "mode": "palette-classic" },
410
203
"custom": {
411
204
"fillOpacity": 15,
412
205
"lineWidth": 2,
413
-
"spanNulls": false
206
+
"spanNulls": false,
207
+
"axisSoftMin": 0
414
208
}
415
209
},
416
-
"overrides": []
210
+
"overrides": [
211
+
{
212
+
"matcher": { "id": "byRegexp", "options": "/(hits|misses|evictions)/" },
213
+
"properties": [
214
+
{ "id": "custom.axisPlacement", "value": "right" },
215
+
{ "id": "unit", "value": "ops" },
216
+
{ "id": "custom.fillOpacity", "value": 0 },
217
+
{ "id": "custom.lineWidth", "value": 1 }
218
+
]
219
+
}
220
+
]
417
221
},
418
222
"targets": [
419
223
{
420
-
"expr": "max(relay_disk_total_bytes{job=\"zlay\"})",
421
-
"legendFormat": "total",
224
+
"expr": "sum(relay_validator_cache_entries{job=\"zlay\"})",
225
+
"legendFormat": "validator cache",
422
226
"refId": "A"
423
227
},
424
228
{
425
-
"expr": "max(relay_disk_total_bytes{job=\"zlay\"}) - max(relay_disk_available_bytes{job=\"zlay\"})",
426
-
"legendFormat": "used",
229
+
"expr": "sum(relay_did_cache_entries{job=\"zlay\"})",
230
+
"legendFormat": "DID cache",
427
231
"refId": "B"
428
-
}
429
-
]
430
-
},
431
-
{
432
-
"title": "resolve queue",
433
-
"type": "timeseries",
434
-
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 },
435
-
"datasource": { "type": "prometheus", "uid": "prometheus" },
436
-
"fieldConfig": {
437
-
"defaults": {
438
-
"color": { "mode": "palette-classic" },
439
-
"custom": {
440
-
"fillOpacity": 15,
441
-
"lineWidth": 2,
442
-
"spanNulls": false
443
-
}
444
232
},
445
-
"overrides": []
446
-
},
447
-
"targets": [
448
233
{
449
-
"expr": "max(relay_resolve_queue_len{job=\"zlay\"})",
450
-
"legendFormat": "queue length",
451
-
"refId": "A"
234
+
"expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"hit\"}[5m]))",
235
+
"legendFormat": "hits",
236
+
"refId": "C"
237
+
},
238
+
{
239
+
"expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"miss\"}[5m]))",
240
+
"legendFormat": "misses",
241
+
"refId": "D"
452
242
},
453
243
{
454
-
"expr": "max(relay_resolve_queued_set_count{job=\"zlay\"})",
455
-
"legendFormat": "dedup set count",
456
-
"refId": "B"
244
+
"expr": "sum(rate(relay_validator_cache_evictions_total{job=\"zlay\"}[5m]))",
245
+
"legendFormat": "evictions",
246
+
"refId": "E"
457
247
}
458
248
]
459
249
},
460
250
{
461
-
"title": "rocksdb memory",
251
+
"title": "errors",
462
252
"type": "timeseries",
463
-
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 },
253
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
464
254
"datasource": { "type": "prometheus", "uid": "prometheus" },
465
255
"fieldConfig": {
466
256
"defaults": {
467
-
"unit": "bytes",
257
+
"unit": "ops",
468
258
"color": { "mode": "palette-classic" },
469
259
"custom": {
470
260
"fillOpacity": 15,
···
476
266
},
477
267
"targets": [
478
268
{
479
-
"expr": "max(relay_rocksdb_table_readers_bytes{job=\"zlay\"})",
480
-
"legendFormat": "table readers",
269
+
"expr": "sum(rate(relay_decode_errors_total{job=\"zlay\"}[5m]))",
270
+
"legendFormat": "decode errors",
481
271
"refId": "A"
482
272
},
483
273
{
484
-
"expr": "max(relay_rocksdb_memtable_bytes{job=\"zlay\"})",
485
-
"legendFormat": "memtables",
274
+
"expr": "sum(rate(relay_slow_consumers_total{job=\"zlay\"}[5m]))",
275
+
"legendFormat": "slow consumer drops",
486
276
"refId": "B"
277
+
},
278
+
{
279
+
"expr": "sum(rate(relay_rate_limited_total{job=\"zlay\"}[5m]))",
280
+
"legendFormat": "rate limited",
281
+
"refId": "C"
487
282
}
488
283
]
489
284
},
490
285
{
491
-
"title": "resolves/sec",
286
+
"title": "resolver",
492
287
"type": "timeseries",
493
-
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 },
288
+
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
494
289
"datasource": { "type": "prometheus", "uid": "prometheus" },
495
290
"fieldConfig": {
496
291
"defaults": {
497
-
"unit": "ops",
498
292
"color": { "mode": "palette-classic" },
499
293
"custom": {
500
294
"fillOpacity": 15,
501
295
"lineWidth": 2,
502
-
"spanNulls": false
296
+
"spanNulls": false,
297
+
"axisSoftMin": 0
503
298
}
504
299
},
505
300
"overrides": []
506
301
},
507
302
"targets": [
508
303
{
509
-
"expr": "sum(rate(relay_resolves_total{job=\"zlay\"}[5m]))",
510
-
"legendFormat": "resolves",
304
+
"expr": "max(relay_resolve_queue_len{job=\"zlay\"})",
305
+
"legendFormat": "queue length",
511
306
"refId": "A"
512
307
},
513
308
{
514
-
"expr": "sum(rate(relay_resolver_recycles_total{job=\"zlay\"}[5m]))",
515
-
"legendFormat": "recycles",
309
+
"expr": "max(relay_resolve_queued_set_count{job=\"zlay\"})",
310
+
"legendFormat": "dedup set",
516
311
"refId": "B"
517
312
}
518
313
]
519
314
},
520
315
{
521
-
"title": "bytes/resolve (heap slope)",
316
+
"title": "disk usage",
522
317
"type": "timeseries",
523
-
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 },
318
+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
524
319
"datasource": { "type": "prometheus", "uid": "prometheus" },
525
320
"fieldConfig": {
526
321
"defaults": {
···
536
331
},
537
332
"targets": [
538
333
{
539
-
"expr": "deriv(max(relay_malloc_in_use_bytes{job=\"zlay\"})[5m:]) / clamp_min(sum(rate(relay_resolves_total{job=\"zlay\"}[5m])), 0.001)",
540
-
"legendFormat": "bytes/resolve",
334
+
"expr": "max(relay_disk_total_bytes{job=\"zlay\"})",
335
+
"legendFormat": "total",
541
336
"refId": "A"
337
+
},
338
+
{
339
+
"expr": "max(relay_disk_total_bytes{job=\"zlay\"}) - max(relay_disk_available_bytes{job=\"zlay\"})",
340
+
"legendFormat": "used",
341
+
"refId": "B"
542
342
}
543
343
]
544
344
}
···
546
346
"schemaVersion": 39,
547
347
"tags": ["zlay", "atproto"],
548
348
"templating": { "list": [] },
549
-
"time": { "from": "now-1h", "to": "now" },
349
+
"time": { "from": "now-3h", "to": "now" },
550
350
"timepicker": {},
551
351
"timezone": "browser",
552
352
"title": "zlay.waow.tech",