declarative relay deployment on hetzner relay-eval.waow.tech
atproto relay
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

fix: update dashboard — remove nonexistent metrics panels

- remove rocksdb memory panel (metrics never implemented)
- update resolver panel: queue length + dedup set (remove recycles)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

zzstoatzz dc29553a 41603f3c

+84 -284
+84 -284
deploy/zlay-dashboard.json
··· 8 8 "links": [], 9 9 "panels": [ 10 10 { 11 - "title": "events/sec", 11 + "title": "throughput", 12 12 "type": "timeseries", 13 13 "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, 14 14 "datasource": { "type": "prometheus", "uid": "prometheus" }, ··· 27 27 "targets": [ 28 28 { 29 29 "expr": "sum(rate(relay_frames_received_total{job=\"zlay\"}[5m]))", 30 - "legendFormat": "events received", 30 + "legendFormat": "received", 31 31 "refId": "A" 32 + }, 33 + { 34 + "expr": "sum(rate(relay_frames_broadcast_total{job=\"zlay\"}[5m]))", 35 + "legendFormat": "broadcast", 36 + "refId": "B" 32 37 } 33 38 ] 34 39 }, ··· 64 69 ] 65 70 }, 66 71 { 67 - "title": "downstream consumers", 68 - "type": "timeseries", 69 - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, 70 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 71 - "fieldConfig": { 72 - "defaults": { 73 - "unit": "ops", 74 - "color": { "mode": "palette-classic" }, 75 - "custom": { 76 - "fillOpacity": 15, 77 - "lineWidth": 2, 78 - "spanNulls": false 79 - } 80 - }, 81 - "overrides": [] 82 - }, 83 - "targets": [ 84 - { 85 - "expr": "sum(rate(relay_frames_broadcast_total{job=\"zlay\"}[5m]))", 86 - "legendFormat": "events sent", 87 - "refId": "A" 88 - } 89 - ] 90 - }, 91 - { 92 72 "title": "validation/sec", 93 73 "type": "timeseries", 94 - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, 74 + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, 95 75 "datasource": { "type": "prometheus", "uid": "prometheus" }, 96 76 "fieldConfig": { 97 77 "defaults": { ··· 124 104 ] 125 105 }, 126 106 { 127 - "title": "memory", 128 - "type": "timeseries", 129 - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, 130 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 131 - "fieldConfig": { 132 - "defaults": { 133 - "unit": "bytes", 134 - "color": { "mode": "palette-classic" }, 135 - "custom": { 136 - "fillOpacity": 15, 137 - "lineWidth": 2, 138 - "spanNulls": false 139 - } 140 - }, 141 - "overrides": [] 142 - }, 143 - "targets": [ 144 - { 145 - "expr": "sum(container_memory_working_set_bytes{namespace=\"zlay\",pod=~\"zlay.*\",container=\"main\"})", 146 - "legendFormat": "working set", 147 - "refId": "A" 148 - }, 149 - { 150 - "expr": "max(kube_pod_container_resource_limits{namespace=\"zlay\",pod=~\"zlay.*\",container=\"main\",resource=\"memory\"})", 151 - "legendFormat": "limit", 152 - "refId": "B" 153 - } 154 - ] 155 - }, 156 - { 157 - "title": "threads", 158 - "type": "timeseries", 159 - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, 160 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 161 - "fieldConfig": { 162 - "defaults": { 163 - "color": { "mode": "palette-classic" }, 164 - "custom": { 165 - "fillOpacity": 15, 166 - "lineWidth": 2, 167 - "spanNulls": false 168 - } 169 - }, 170 - "overrides": [] 171 - }, 172 - "targets": [ 173 - { 174 - "expr": "sum(container_threads{namespace=\"zlay\",pod=~\"zlay-[a-z0-9].*\",container=\"main\"})", 175 - "legendFormat": "threads", 176 - "refId": "A" 177 - } 178 - ] 179 - }, 180 - { 181 - "title": "key cache/sec", 182 - "type": "timeseries", 183 - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, 184 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 185 - "fieldConfig": { 186 - "defaults": { 187 - "unit": "ops", 188 - "color": { "mode": "palette-classic" }, 189 - "custom": { 190 - "fillOpacity": 15, 191 - "lineWidth": 2, 192 - "spanNulls": false 193 - } 194 - }, 195 - "overrides": [] 196 - }, 197 - "targets": [ 198 - { 199 - "expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"hit\"}[5m]))", 200 - "legendFormat": "hits", 201 - "refId": "A" 202 - }, 203 - { 204 - "expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"miss\"}[5m]))", 205 - "legendFormat": "misses", 206 - "refId": "B" 207 - } 208 - ] 209 - }, 210 - { 211 - "title": "errors/sec", 212 - "type": "timeseries", 213 - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, 214 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 215 - "fieldConfig": { 216 - "defaults": { 217 - "unit": "ops", 218 - "color": { "mode": "palette-classic" }, 219 - "custom": { 220 - "fillOpacity": 15, 221 - "lineWidth": 2, 222 - "spanNulls": false 223 - } 224 - }, 225 - "overrides": [] 226 - }, 227 - "targets": [ 228 - { 229 - "expr": "sum(rate(relay_decode_errors_total{job=\"zlay\"}[5m]))", 230 - "legendFormat": "decode errors", 231 - "refId": "A" 232 - }, 233 - { 234 - "expr": "sum(rate(relay_slow_consumers_total{job=\"zlay\"}[5m]))", 235 - "legendFormat": "slow consumer drops", 236 - "refId": "B" 237 - } 238 - ] 239 - }, 240 - { 241 - "title": "validator cache entries", 242 - "type": "timeseries", 243 - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, 244 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 245 - "fieldConfig": { 246 - "defaults": { 247 - "color": { "mode": "palette-classic" }, 248 - "custom": { 249 - "fillOpacity": 15, 250 - "lineWidth": 2, 251 - "spanNulls": false 252 - } 253 - }, 254 - "overrides": [] 255 - }, 256 - "targets": [ 257 - { 258 - "expr": "sum(relay_validator_cache_entries{job=\"zlay\"})", 259 - "legendFormat": "validator cache", 260 - "refId": "A" 261 - }, 262 - { 263 - "expr": "sum(relay_did_cache_entries{job=\"zlay\"})", 264 - "legendFormat": "DID cache", 265 - "refId": "B" 266 - } 267 - ] 268 - }, 269 - { 270 - "title": "cache evictions/sec", 271 - "type": "timeseries", 272 - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, 273 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 274 - "fieldConfig": { 275 - "defaults": { 276 - "unit": "ops", 277 - "color": { "mode": "palette-classic" }, 278 - "custom": { 279 - "fillOpacity": 15, 280 - "lineWidth": 2, 281 - "spanNulls": false 282 - } 283 - }, 284 - "overrides": [] 285 - }, 286 - "targets": [ 287 - { 288 - "expr": "sum(rate(relay_validator_cache_evictions_total{job=\"zlay\"}[5m]))", 289 - "legendFormat": "evictions", 290 - "refId": "A" 291 - } 292 - ] 293 - }, 294 - { 295 107 "title": "malloc breakdown", 296 108 "type": "timeseries", 297 - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, 109 + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, 298 110 "datasource": { "type": "prometheus", "uid": "prometheus" }, 299 111 "fieldConfig": { 300 112 "defaults": { ··· 332 144 ] 333 145 }, 334 146 { 335 - "title": "rss breakdown", 147 + "title": "process memory", 336 148 "type": "timeseries", 337 - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, 149 + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, 338 150 "datasource": { "type": "prometheus", "uid": "prometheus" }, 339 151 "fieldConfig": { 340 152 "defaults": { ··· 346 158 "spanNulls": false 347 159 } 348 160 }, 349 - "overrides": [] 161 + "overrides": [ 162 + { 163 + "matcher": { "id": "byName", "options": "limit" }, 164 + "properties": [ 165 + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, 166 + { "id": "custom.fillOpacity", "value": 0 }, 167 + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } 168 + ] 169 + } 170 + ] 350 171 }, 351 172 "targets": [ 352 173 { ··· 363 184 "expr": "max(relay_vm_hwm_kb{job=\"zlay\"}) * 1024", 364 185 "legendFormat": "VmHWM (peak RSS)", 365 186 "refId": "C" 366 - } 367 - ] 368 - }, 369 - { 370 - "title": "disk used %", 371 - "type": "gauge", 372 - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 40 }, 373 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 374 - "fieldConfig": { 375 - "defaults": { 376 - "unit": "percentunit", 377 - "min": 0, 378 - "max": 1, 379 - "color": { "mode": "thresholds" }, 380 - "thresholds": { 381 - "steps": [ 382 - { "color": "green", "value": null }, 383 - { "color": "yellow", "value": 0.7 }, 384 - { "color": "red", "value": 0.85 } 385 - ] 386 - } 387 187 }, 388 - "overrides": [] 389 - }, 390 - "options": { 391 - "reduceOptions": { "calcs": ["lastNotNull"] } 392 - }, 393 - "targets": [ 394 188 { 395 - "expr": "max(1 - (relay_disk_available_bytes{job=\"zlay\"} / relay_disk_total_bytes{job=\"zlay\"}))", 396 - "legendFormat": "used", 397 - "refId": "A" 189 + "expr": "max(kube_pod_container_resource_limits{namespace=\"zlay\",pod=~\"zlay-[a-z0-9].*\",container=\"main\",resource=\"memory\"})", 190 + "legendFormat": "limit", 191 + "refId": "D" 398 192 } 399 193 ] 400 194 }, 401 195 { 402 - "title": "disk usage", 196 + "title": "caches", 403 197 "type": "timeseries", 404 - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 40 }, 198 + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, 405 199 "datasource": { "type": "prometheus", "uid": "prometheus" }, 406 200 "fieldConfig": { 407 201 "defaults": { 408 - "unit": "bytes", 409 202 "color": { "mode": "palette-classic" }, 410 203 "custom": { 411 204 "fillOpacity": 15, 412 205 "lineWidth": 2, 413 - "spanNulls": false 206 + "spanNulls": false, 207 + "axisSoftMin": 0 414 208 } 415 209 }, 416 - "overrides": [] 210 + "overrides": [ 211 + { 212 + "matcher": { "id": "byRegexp", "options": "/(hits|misses|evictions)/" }, 213 + "properties": [ 214 + { "id": "custom.axisPlacement", "value": "right" }, 215 + { "id": "unit", "value": "ops" }, 216 + { "id": "custom.fillOpacity", "value": 0 }, 217 + { "id": "custom.lineWidth", "value": 1 } 218 + ] 219 + } 220 + ] 417 221 }, 418 222 "targets": [ 419 223 { 420 - "expr": "max(relay_disk_total_bytes{job=\"zlay\"})", 421 - "legendFormat": "total", 224 + "expr": "sum(relay_validator_cache_entries{job=\"zlay\"})", 225 + "legendFormat": "validator cache", 422 226 "refId": "A" 423 227 }, 424 228 { 425 - "expr": "max(relay_disk_total_bytes{job=\"zlay\"}) - max(relay_disk_available_bytes{job=\"zlay\"})", 426 - "legendFormat": "used", 229 + "expr": "sum(relay_did_cache_entries{job=\"zlay\"})", 230 + "legendFormat": "DID cache", 427 231 "refId": "B" 428 - } 429 - ] 430 - }, 431 - { 432 - "title": "resolve queue", 433 - "type": "timeseries", 434 - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, 435 - "datasource": { "type": "prometheus", "uid": "prometheus" }, 436 - "fieldConfig": { 437 - "defaults": { 438 - "color": { "mode": "palette-classic" }, 439 - "custom": { 440 - "fillOpacity": 15, 441 - "lineWidth": 2, 442 - "spanNulls": false 443 - } 444 232 }, 445 - "overrides": [] 446 - }, 447 - "targets": [ 448 233 { 449 - "expr": "max(relay_resolve_queue_len{job=\"zlay\"})", 450 - "legendFormat": "queue length", 451 - "refId": "A" 234 + "expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"hit\"}[5m]))", 235 + "legendFormat": "hits", 236 + "refId": "C" 237 + }, 238 + { 239 + "expr": "sum(rate(relay_cache_total{job=\"zlay\",result=\"miss\"}[5m]))", 240 + "legendFormat": "misses", 241 + "refId": "D" 452 242 }, 453 243 { 454 - "expr": "max(relay_resolve_queued_set_count{job=\"zlay\"})", 455 - "legendFormat": "dedup set count", 456 - "refId": "B" 244 + "expr": "sum(rate(relay_validator_cache_evictions_total{job=\"zlay\"}[5m]))", 245 + "legendFormat": "evictions", 246 + "refId": "E" 457 247 } 458 248 ] 459 249 }, 460 250 { 461 - "title": "rocksdb memory", 251 + "title": "errors", 462 252 "type": "timeseries", 463 - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, 253 + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, 464 254 "datasource": { "type": "prometheus", "uid": "prometheus" }, 465 255 "fieldConfig": { 466 256 "defaults": { 467 - "unit": "bytes", 257 + "unit": "ops", 468 258 "color": { "mode": "palette-classic" }, 469 259 "custom": { 470 260 "fillOpacity": 15, ··· 476 266 }, 477 267 "targets": [ 478 268 { 479 - "expr": "max(relay_rocksdb_table_readers_bytes{job=\"zlay\"})", 480 - "legendFormat": "table readers", 269 + "expr": "sum(rate(relay_decode_errors_total{job=\"zlay\"}[5m]))", 270 + "legendFormat": "decode errors", 481 271 "refId": "A" 482 272 }, 483 273 { 484 - "expr": "max(relay_rocksdb_memtable_bytes{job=\"zlay\"})", 485 - "legendFormat": "memtables", 274 + "expr": "sum(rate(relay_slow_consumers_total{job=\"zlay\"}[5m]))", 275 + "legendFormat": "slow consumer drops", 486 276 "refId": "B" 277 + }, 278 + { 279 + "expr": "sum(rate(relay_rate_limited_total{job=\"zlay\"}[5m]))", 280 + "legendFormat": "rate limited", 281 + "refId": "C" 487 282 } 488 283 ] 489 284 }, 490 285 { 491 - "title": "resolves/sec", 286 + "title": "resolver", 492 287 "type": "timeseries", 493 - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, 288 + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, 494 289 "datasource": { "type": "prometheus", "uid": "prometheus" }, 495 290 "fieldConfig": { 496 291 "defaults": { 497 - "unit": "ops", 498 292 "color": { "mode": "palette-classic" }, 499 293 "custom": { 500 294 "fillOpacity": 15, 501 295 "lineWidth": 2, 502 - "spanNulls": false 296 + "spanNulls": false, 297 + "axisSoftMin": 0 503 298 } 504 299 }, 505 300 "overrides": [] 506 301 }, 507 302 "targets": [ 508 303 { 509 - "expr": "sum(rate(relay_resolves_total{job=\"zlay\"}[5m]))", 510 - "legendFormat": "resolves", 304 + "expr": "max(relay_resolve_queue_len{job=\"zlay\"})", 305 + "legendFormat": "queue length", 511 306 "refId": "A" 512 307 }, 513 308 { 514 - "expr": "sum(rate(relay_resolver_recycles_total{job=\"zlay\"}[5m]))", 515 - "legendFormat": "recycles", 309 + "expr": "max(relay_resolve_queued_set_count{job=\"zlay\"})", 310 + "legendFormat": "dedup set", 516 311 "refId": "B" 517 312 } 518 313 ] 519 314 }, 520 315 { 521 - "title": "bytes/resolve (heap slope)", 316 + "title": "disk usage", 522 317 "type": "timeseries", 523 - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, 318 + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, 524 319 "datasource": { "type": "prometheus", "uid": "prometheus" }, 525 320 "fieldConfig": { 526 321 "defaults": { ··· 536 331 }, 537 332 "targets": [ 538 333 { 539 - "expr": "deriv(max(relay_malloc_in_use_bytes{job=\"zlay\"})[5m:]) / clamp_min(sum(rate(relay_resolves_total{job=\"zlay\"}[5m])), 0.001)", 540 - "legendFormat": "bytes/resolve", 334 + "expr": "max(relay_disk_total_bytes{job=\"zlay\"})", 335 + "legendFormat": "total", 541 336 "refId": "A" 337 + }, 338 + { 339 + "expr": "max(relay_disk_total_bytes{job=\"zlay\"}) - max(relay_disk_available_bytes{job=\"zlay\"})", 340 + "legendFormat": "used", 341 + "refId": "B" 542 342 } 543 343 ] 544 344 } ··· 546 346 "schemaVersion": 39, 547 347 "tags": ["zlay", "atproto"], 548 348 "templating": { "list": [] }, 549 - "time": { "from": "now-1h", "to": "now" }, 349 + "time": { "from": "now-3h", "to": "now" }, 550 350 "timepicker": {}, 551 351 "timezone": "browser", 552 352 "title": "zlay.waow.tech",