this repo has no description
Document sonar and proivde a dashboard
+1581
-12
+24
cmd/sonar/README.md
+24
cmd/sonar/README.md
···
1
+
# Sonar
2
+
Sonar is an AT Proto Firehose Montioring tool
3
+
4
+
Sonar connects to an AT Proto Firehose (either from a PDS or a Relay) following the semantics of `com.atproto.sync.subscribeRepos`.
5
+
6
+
Sonar monitors the throughput of events, producing prometheus metrics on the frequency of different kinds of events.
7
+
8
+
Sonar additionally walks through repo operations and tracks the frequency of creation/update/deletion of different record collections.
9
+
10
+
Sonar's main use is to provide an operational dashboard of activity on the network, allowing us to view changes in event rate over time and understand what kinds of traffic flow through the firehose over time.
11
+
12
+
## Running Sonar
13
+
14
+
To run sonar in Docker locally, you can run: `make sonar-up` from the root of the `indigo` directory.
15
+
16
+
This will start a sonar instance that connects to the Bluesky-operated Relay firehose at `bsky.network` and will expose metrics at `http://localhost:8345`
17
+
18
+
Feel free to modify the `docker-compose.yml` in this directory to change any settings via environment variables i.e. to change the firehose host `SONAR_WS_URL` or the listen port `SONAR_PORT`.
19
+
20
+
## Dashboard
21
+
22
+
Sonar emits Prometheus metrics which you can scrape and then visualize with the Grafana dashboard (JSON template provided in this directory) shown below:
23
+
24
+
+1541
cmd/sonar/grafana-dashboard.json
+1541
cmd/sonar/grafana-dashboard.json
···
1
+
{
2
+
"__inputs": [
3
+
{
4
+
"name": "DS_MIMIR",
5
+
"label": "Mimir",
6
+
"description": "",
7
+
"type": "datasource",
8
+
"pluginId": "prometheus",
9
+
"pluginName": "Prometheus"
10
+
}
11
+
],
12
+
"__elements": {},
13
+
"__requires": [
14
+
{
15
+
"type": "grafana",
16
+
"id": "grafana",
17
+
"name": "Grafana",
18
+
"version": "10.4.1"
19
+
},
20
+
{
21
+
"type": "panel",
22
+
"id": "piechart",
23
+
"name": "Pie chart",
24
+
"version": ""
25
+
},
26
+
{
27
+
"type": "datasource",
28
+
"id": "prometheus",
29
+
"name": "Prometheus",
30
+
"version": "1.0.0"
31
+
},
32
+
{
33
+
"type": "panel",
34
+
"id": "stat",
35
+
"name": "Stat",
36
+
"version": ""
37
+
},
38
+
{
39
+
"type": "panel",
40
+
"id": "timeseries",
41
+
"name": "Time series",
42
+
"version": ""
43
+
}
44
+
],
45
+
"annotations": {
46
+
"list": [
47
+
{
48
+
"builtIn": 1,
49
+
"datasource": {
50
+
"type": "grafana",
51
+
"uid": "-- Grafana --"
52
+
},
53
+
"enable": true,
54
+
"hide": true,
55
+
"iconColor": "rgba(0, 211, 255, 1)",
56
+
"name": "Annotations & Alerts",
57
+
"target": {
58
+
"limit": 100,
59
+
"matchAny": false,
60
+
"tags": [],
61
+
"type": "dashboard"
62
+
},
63
+
"type": "dashboard"
64
+
}
65
+
]
66
+
},
67
+
"editable": true,
68
+
"fiscalYearStartMonth": 0,
69
+
"graphTooltip": 1,
70
+
"id": null,
71
+
"links": [],
72
+
"liveNow": false,
73
+
"panels": [
74
+
{
75
+
"datasource": {
76
+
"type": "prometheus",
77
+
"uid": "${DS_MIMIR}"
78
+
},
79
+
"fieldConfig": {
80
+
"defaults": {
81
+
"color": {
82
+
"mode": "palette-classic"
83
+
},
84
+
"custom": {
85
+
"axisBorderShow": false,
86
+
"axisCenteredZero": false,
87
+
"axisColorMode": "text",
88
+
"axisLabel": "",
89
+
"axisPlacement": "auto",
90
+
"barAlignment": 0,
91
+
"drawStyle": "line",
92
+
"fillOpacity": 55,
93
+
"gradientMode": "none",
94
+
"hideFrom": {
95
+
"legend": false,
96
+
"tooltip": false,
97
+
"viz": false
98
+
},
99
+
"insertNulls": false,
100
+
"lineInterpolation": "linear",
101
+
"lineWidth": 1,
102
+
"pointSize": 5,
103
+
"scaleDistribution": {
104
+
"type": "linear"
105
+
},
106
+
"showPoints": "auto",
107
+
"spanNulls": false,
108
+
"stacking": {
109
+
"group": "A",
110
+
"mode": "normal"
111
+
},
112
+
"thresholdsStyle": {
113
+
"mode": "line+area"
114
+
}
115
+
},
116
+
"mappings": [],
117
+
"thresholds": {
118
+
"mode": "absolute",
119
+
"steps": [
120
+
{
121
+
"color": "green",
122
+
"value": null
123
+
},
124
+
{
125
+
"color": "yellow",
126
+
"value": 1000
127
+
},
128
+
{
129
+
"color": "orange",
130
+
"value": 1200
131
+
},
132
+
{
133
+
"color": "red",
134
+
"value": 1500
135
+
}
136
+
]
137
+
},
138
+
"unit": "cps"
139
+
},
140
+
"overrides": [
141
+
{
142
+
"matcher": {
143
+
"id": "byName",
144
+
"options": "repo_commit"
145
+
},
146
+
"properties": [
147
+
{
148
+
"id": "color",
149
+
"value": {
150
+
"fixedColor": "green",
151
+
"mode": "fixed"
152
+
}
153
+
}
154
+
]
155
+
},
156
+
{
157
+
"matcher": {
158
+
"id": "byName",
159
+
"options": "identity"
160
+
},
161
+
"properties": [
162
+
{
163
+
"id": "color",
164
+
"value": {
165
+
"fixedColor": "yellow",
166
+
"mode": "fixed"
167
+
}
168
+
}
169
+
]
170
+
},
171
+
{
172
+
"matcher": {
173
+
"id": "byName",
174
+
"options": "repo_tombstone"
175
+
},
176
+
"properties": [
177
+
{
178
+
"id": "color",
179
+
"value": {
180
+
"fixedColor": "red",
181
+
"mode": "fixed"
182
+
}
183
+
}
184
+
]
185
+
}
186
+
]
187
+
},
188
+
"gridPos": {
189
+
"h": 9,
190
+
"w": 12,
191
+
"x": 0,
192
+
"y": 0
193
+
},
194
+
"id": 1,
195
+
"options": {
196
+
"legend": {
197
+
"calcs": [
198
+
"mean"
199
+
],
200
+
"displayMode": "list",
201
+
"placement": "bottom",
202
+
"showLegend": true
203
+
},
204
+
"tooltip": {
205
+
"mode": "multi",
206
+
"sort": "desc"
207
+
}
208
+
},
209
+
"targets": [
210
+
{
211
+
"datasource": {
212
+
"type": "prometheus",
213
+
"uid": "${DS_MIMIR}"
214
+
},
215
+
"editorMode": "code",
216
+
"expr": "avg(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (event_type)",
217
+
"legendFormat": "__auto",
218
+
"range": true,
219
+
"refId": "A"
220
+
}
221
+
],
222
+
"title": "Event Throughput",
223
+
"type": "timeseries"
224
+
},
225
+
{
226
+
"datasource": {
227
+
"type": "prometheus",
228
+
"uid": "${DS_MIMIR}"
229
+
},
230
+
"fieldConfig": {
231
+
"defaults": {
232
+
"color": {
233
+
"mode": "palette-classic"
234
+
},
235
+
"custom": {
236
+
"axisBorderShow": false,
237
+
"axisCenteredZero": false,
238
+
"axisColorMode": "text",
239
+
"axisLabel": "",
240
+
"axisPlacement": "auto",
241
+
"barAlignment": 0,
242
+
"drawStyle": "line",
243
+
"fillOpacity": 20,
244
+
"gradientMode": "none",
245
+
"hideFrom": {
246
+
"legend": false,
247
+
"tooltip": false,
248
+
"viz": false
249
+
},
250
+
"insertNulls": false,
251
+
"lineInterpolation": "linear",
252
+
"lineWidth": 1,
253
+
"pointSize": 5,
254
+
"scaleDistribution": {
255
+
"type": "linear"
256
+
},
257
+
"showPoints": "auto",
258
+
"spanNulls": false,
259
+
"stacking": {
260
+
"group": "A",
261
+
"mode": "none"
262
+
},
263
+
"thresholdsStyle": {
264
+
"mode": "off"
265
+
}
266
+
},
267
+
"mappings": [],
268
+
"thresholds": {
269
+
"mode": "absolute",
270
+
"steps": [
271
+
{
272
+
"color": "green",
273
+
"value": null
274
+
},
275
+
{
276
+
"color": "yellow",
277
+
"value": 1000
278
+
},
279
+
{
280
+
"color": "orange",
281
+
"value": 1200
282
+
},
283
+
{
284
+
"color": "red",
285
+
"value": 1500
286
+
}
287
+
]
288
+
},
289
+
"unit": "cps"
290
+
},
291
+
"overrides": [
292
+
{
293
+
"matcher": {
294
+
"id": "byName",
295
+
"options": "1 Day Ago"
296
+
},
297
+
"properties": [
298
+
{
299
+
"id": "custom.fillOpacity",
300
+
"value": 0
301
+
},
302
+
{
303
+
"id": "custom.lineStyle",
304
+
"value": {
305
+
"fill": "solid"
306
+
}
307
+
},
308
+
{
309
+
"id": "color",
310
+
"value": {
311
+
"fixedColor": "yellow",
312
+
"mode": "fixed"
313
+
}
314
+
}
315
+
]
316
+
},
317
+
{
318
+
"matcher": {
319
+
"id": "byName",
320
+
"options": "1 Week Ago"
321
+
},
322
+
"properties": [
323
+
{
324
+
"id": "custom.fillOpacity",
325
+
"value": 0
326
+
},
327
+
{
328
+
"id": "color",
329
+
"value": {
330
+
"fixedColor": "blue",
331
+
"mode": "fixed"
332
+
}
333
+
}
334
+
]
335
+
},
336
+
{
337
+
"matcher": {
338
+
"id": "byName",
339
+
"options": "4 Weeks Ago"
340
+
},
341
+
"properties": [
342
+
{
343
+
"id": "custom.fillOpacity",
344
+
"value": 0
345
+
}
346
+
]
347
+
},
348
+
{
349
+
"__systemRef": "hideSeriesFrom",
350
+
"matcher": {
351
+
"id": "byNames",
352
+
"options": {
353
+
"mode": "exclude",
354
+
"names": [
355
+
"Current",
356
+
"1 Day Ago"
357
+
],
358
+
"prefix": "All except:",
359
+
"readOnly": true
360
+
}
361
+
},
362
+
"properties": [
363
+
{
364
+
"id": "custom.hideFrom",
365
+
"value": {
366
+
"legend": false,
367
+
"tooltip": false,
368
+
"viz": true
369
+
}
370
+
}
371
+
]
372
+
}
373
+
]
374
+
},
375
+
"gridPos": {
376
+
"h": 9,
377
+
"w": 12,
378
+
"x": 12,
379
+
"y": 0
380
+
},
381
+
"id": 12,
382
+
"options": {
383
+
"legend": {
384
+
"calcs": [
385
+
"mean"
386
+
],
387
+
"displayMode": "list",
388
+
"placement": "bottom",
389
+
"showLegend": true
390
+
},
391
+
"tooltip": {
392
+
"mode": "multi",
393
+
"sort": "desc"
394
+
}
395
+
},
396
+
"targets": [
397
+
{
398
+
"datasource": {
399
+
"type": "prometheus",
400
+
"uid": "${DS_MIMIR}"
401
+
},
402
+
"editorMode": "code",
403
+
"expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval]))",
404
+
"legendFormat": "Current",
405
+
"range": true,
406
+
"refId": "A"
407
+
},
408
+
{
409
+
"datasource": {
410
+
"type": "prometheus",
411
+
"uid": "${DS_MIMIR}"
412
+
},
413
+
"editorMode": "code",
414
+
"expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 1d))",
415
+
"hide": false,
416
+
"instant": false,
417
+
"legendFormat": "1 Day Ago",
418
+
"range": true,
419
+
"refId": "B"
420
+
},
421
+
{
422
+
"datasource": {
423
+
"type": "prometheus",
424
+
"uid": "${DS_MIMIR}"
425
+
},
426
+
"editorMode": "code",
427
+
"expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 1w))",
428
+
"hide": false,
429
+
"instant": false,
430
+
"legendFormat": "1 Week Ago",
431
+
"range": true,
432
+
"refId": "C"
433
+
},
434
+
{
435
+
"datasource": {
436
+
"type": "prometheus",
437
+
"uid": "${DS_MIMIR}"
438
+
},
439
+
"editorMode": "code",
440
+
"expr": "sum(rate(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval] offset 4w))",
441
+
"hide": false,
442
+
"instant": false,
443
+
"legendFormat": "4 Weeks Ago",
444
+
"range": true,
445
+
"refId": "D"
446
+
}
447
+
],
448
+
"title": "Total Event Throughput",
449
+
"type": "timeseries"
450
+
},
451
+
{
452
+
"datasource": {
453
+
"type": "prometheus",
454
+
"uid": "${DS_MIMIR}"
455
+
},
456
+
"fieldConfig": {
457
+
"defaults": {
458
+
"color": {
459
+
"mode": "palette-classic"
460
+
},
461
+
"custom": {
462
+
"axisBorderShow": false,
463
+
"axisCenteredZero": false,
464
+
"axisColorMode": "text",
465
+
"axisLabel": "",
466
+
"axisPlacement": "auto",
467
+
"barAlignment": 0,
468
+
"drawStyle": "line",
469
+
"fillOpacity": 55,
470
+
"gradientMode": "none",
471
+
"hideFrom": {
472
+
"legend": false,
473
+
"tooltip": false,
474
+
"viz": false
475
+
},
476
+
"insertNulls": false,
477
+
"lineInterpolation": "linear",
478
+
"lineWidth": 1,
479
+
"pointSize": 5,
480
+
"scaleDistribution": {
481
+
"type": "linear"
482
+
},
483
+
"showPoints": "auto",
484
+
"spanNulls": false,
485
+
"stacking": {
486
+
"group": "A",
487
+
"mode": "normal"
488
+
},
489
+
"thresholdsStyle": {
490
+
"mode": "line+area"
491
+
}
492
+
},
493
+
"mappings": [],
494
+
"thresholds": {
495
+
"mode": "absolute",
496
+
"steps": [
497
+
{
498
+
"color": "green",
499
+
"value": null
500
+
},
501
+
{
502
+
"color": "yellow",
503
+
"value": 1000
504
+
},
505
+
{
506
+
"color": "orange",
507
+
"value": 1200
508
+
},
509
+
{
510
+
"color": "red",
511
+
"value": 1500
512
+
}
513
+
]
514
+
},
515
+
"unit": "cps"
516
+
},
517
+
"overrides": []
518
+
},
519
+
"gridPos": {
520
+
"h": 9,
521
+
"w": 12,
522
+
"x": 0,
523
+
"y": 9
524
+
},
525
+
"id": 2,
526
+
"options": {
527
+
"legend": {
528
+
"calcs": [
529
+
"mean",
530
+
"max"
531
+
],
532
+
"displayMode": "table",
533
+
"placement": "right",
534
+
"showLegend": true,
535
+
"sortBy": "Mean",
536
+
"sortDesc": true
537
+
},
538
+
"tooltip": {
539
+
"mode": "multi",
540
+
"sort": "desc"
541
+
}
542
+
},
543
+
"targets": [
544
+
{
545
+
"datasource": {
546
+
"type": "prometheus",
547
+
"uid": "${DS_MIMIR}"
548
+
},
549
+
"editorMode": "code",
550
+
"expr": "avg(rate(sonar_records_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (record_type)",
551
+
"legendFormat": "__auto",
552
+
"range": true,
553
+
"refId": "A"
554
+
}
555
+
],
556
+
"title": "Record Throughput",
557
+
"type": "timeseries"
558
+
},
559
+
{
560
+
"datasource": {
561
+
"type": "prometheus",
562
+
"uid": "${DS_MIMIR}"
563
+
},
564
+
"fieldConfig": {
565
+
"defaults": {
566
+
"color": {
567
+
"mode": "palette-classic"
568
+
},
569
+
"custom": {
570
+
"axisBorderShow": false,
571
+
"axisCenteredZero": false,
572
+
"axisColorMode": "text",
573
+
"axisLabel": "",
574
+
"axisPlacement": "auto",
575
+
"barAlignment": 0,
576
+
"drawStyle": "line",
577
+
"fillOpacity": 0,
578
+
"gradientMode": "none",
579
+
"hideFrom": {
580
+
"legend": false,
581
+
"tooltip": false,
582
+
"viz": false
583
+
},
584
+
"insertNulls": false,
585
+
"lineInterpolation": "linear",
586
+
"lineWidth": 1,
587
+
"pointSize": 5,
588
+
"scaleDistribution": {
589
+
"type": "linear"
590
+
},
591
+
"showPoints": "auto",
592
+
"spanNulls": false,
593
+
"stacking": {
594
+
"group": "A",
595
+
"mode": "none"
596
+
},
597
+
"thresholdsStyle": {
598
+
"mode": "line+area"
599
+
}
600
+
},
601
+
"mappings": [],
602
+
"thresholds": {
603
+
"mode": "absolute",
604
+
"steps": [
605
+
{
606
+
"color": "green",
607
+
"value": null
608
+
},
609
+
{
610
+
"color": "yellow",
611
+
"value": 15000
612
+
},
613
+
{
614
+
"color": "orange",
615
+
"value": 60000
616
+
},
617
+
{
618
+
"color": "red",
619
+
"value": 300000
620
+
}
621
+
]
622
+
},
623
+
"unit": "s"
624
+
},
625
+
"overrides": [
626
+
{
627
+
"__systemRef": "hideSeriesFrom",
628
+
"matcher": {
629
+
"id": "byNames",
630
+
"options": {
631
+
"mode": "exclude",
632
+
"names": [
633
+
"Since Event Creation"
634
+
],
635
+
"prefix": "All except:",
636
+
"readOnly": true
637
+
}
638
+
},
639
+
"properties": [
640
+
{
641
+
"id": "custom.hideFrom",
642
+
"value": {
643
+
"legend": false,
644
+
"tooltip": false,
645
+
"viz": true
646
+
}
647
+
}
648
+
]
649
+
}
650
+
]
651
+
},
652
+
"gridPos": {
653
+
"h": 8,
654
+
"w": 8,
655
+
"x": 12,
656
+
"y": 9
657
+
},
658
+
"id": 3,
659
+
"options": {
660
+
"legend": {
661
+
"calcs": [
662
+
"mean"
663
+
],
664
+
"displayMode": "list",
665
+
"placement": "bottom",
666
+
"showLegend": true
667
+
},
668
+
"tooltip": {
669
+
"mode": "multi",
670
+
"sort": "desc"
671
+
}
672
+
},
673
+
"targets": [
674
+
{
675
+
"datasource": {
676
+
"type": "prometheus",
677
+
"uid": "${DS_MIMIR}"
678
+
},
679
+
"editorMode": "code",
680
+
"expr": "sonar_last_evt_created_evt_processed_gap{box=~\"${box}\",socket_url=~\"${socket}\"}",
681
+
"legendFormat": "Since Event Creation",
682
+
"range": true,
683
+
"refId": "A"
684
+
},
685
+
{
686
+
"datasource": {
687
+
"type": "prometheus",
688
+
"uid": "${DS_MIMIR}"
689
+
},
690
+
"editorMode": "code",
691
+
"expr": "sonar_last_record_created_evt_processed_gap{box=~\"${box}\",socket_url=~\"${socket}\"}",
692
+
"hide": false,
693
+
"legendFormat": "Since Record Creation",
694
+
"range": true,
695
+
"refId": "B"
696
+
}
697
+
],
698
+
"title": "Firehose Latency",
699
+
"type": "timeseries"
700
+
},
701
+
{
702
+
"datasource": {
703
+
"type": "prometheus",
704
+
"uid": "${DS_MIMIR}"
705
+
},
706
+
"fieldConfig": {
707
+
"defaults": {
708
+
"color": {
709
+
"mode": "palette-classic"
710
+
},
711
+
"custom": {
712
+
"hideFrom": {
713
+
"legend": false,
714
+
"tooltip": false,
715
+
"viz": false
716
+
}
717
+
},
718
+
"decimals": 0,
719
+
"mappings": [],
720
+
"unit": "locale"
721
+
},
722
+
"overrides": []
723
+
},
724
+
"gridPos": {
725
+
"h": 8,
726
+
"w": 4,
727
+
"x": 20,
728
+
"y": 9
729
+
},
730
+
"id": 9,
731
+
"options": {
732
+
"displayLabels": [
733
+
"name"
734
+
],
735
+
"legend": {
736
+
"displayMode": "table",
737
+
"placement": "right",
738
+
"showLegend": true,
739
+
"values": [
740
+
"percent",
741
+
"value"
742
+
]
743
+
},
744
+
"pieType": "donut",
745
+
"reduceOptions": {
746
+
"calcs": [
747
+
"lastNotNull"
748
+
],
749
+
"fields": "",
750
+
"values": false
751
+
},
752
+
"tooltip": {
753
+
"mode": "single",
754
+
"sort": "none"
755
+
}
756
+
},
757
+
"targets": [
758
+
{
759
+
"datasource": {
760
+
"type": "prometheus",
761
+
"uid": "${DS_MIMIR}"
762
+
},
763
+
"editorMode": "code",
764
+
"expr": "avg(increase(sonar_ops_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"}[$__range])) by (kind)",
765
+
"format": "time_series",
766
+
"instant": false,
767
+
"legendFormat": "{{kind}}",
768
+
"range": true,
769
+
"refId": "A"
770
+
}
771
+
],
772
+
"title": "Operation Kind Distribution",
773
+
"type": "piechart"
774
+
},
775
+
{
776
+
"datasource": {
777
+
"type": "prometheus",
778
+
"uid": "${DS_MIMIR}"
779
+
},
780
+
"fieldConfig": {
781
+
"defaults": {
782
+
"color": {
783
+
"mode": "palette-classic"
784
+
},
785
+
"custom": {
786
+
"axisBorderShow": false,
787
+
"axisCenteredZero": false,
788
+
"axisColorMode": "text",
789
+
"axisLabel": "",
790
+
"axisPlacement": "auto",
791
+
"barAlignment": 0,
792
+
"drawStyle": "line",
793
+
"fillOpacity": 0,
794
+
"gradientMode": "none",
795
+
"hideFrom": {
796
+
"legend": false,
797
+
"tooltip": false,
798
+
"viz": false
799
+
},
800
+
"insertNulls": false,
801
+
"lineInterpolation": "linear",
802
+
"lineWidth": 1,
803
+
"pointSize": 5,
804
+
"scaleDistribution": {
805
+
"type": "linear"
806
+
},
807
+
"showPoints": "auto",
808
+
"spanNulls": false,
809
+
"stacking": {
810
+
"group": "A",
811
+
"mode": "none"
812
+
},
813
+
"thresholdsStyle": {
814
+
"mode": "off"
815
+
}
816
+
},
817
+
"mappings": [],
818
+
"thresholds": {
819
+
"mode": "absolute",
820
+
"steps": [
821
+
{
822
+
"color": "green",
823
+
"value": null
824
+
},
825
+
{
826
+
"color": "red",
827
+
"value": 80
828
+
}
829
+
]
830
+
},
831
+
"unit": "s"
832
+
},
833
+
"overrides": []
834
+
},
835
+
"gridPos": {
836
+
"h": 8,
837
+
"w": 6,
838
+
"x": 12,
839
+
"y": 17
840
+
},
841
+
"id": 8,
842
+
"options": {
843
+
"legend": {
844
+
"calcs": [],
845
+
"displayMode": "list",
846
+
"placement": "bottom",
847
+
"showLegend": true
848
+
},
849
+
"tooltip": {
850
+
"mode": "single",
851
+
"sort": "none"
852
+
}
853
+
},
854
+
"targets": [
855
+
{
856
+
"datasource": {
857
+
"type": "prometheus",
858
+
"uid": "${DS_MIMIR}"
859
+
},
860
+
"editorMode": "code",
861
+
"expr": "rate(process_cpu_seconds_total{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])",
862
+
"instant": false,
863
+
"legendFormat": "{{instance}}",
864
+
"range": true,
865
+
"refId": "A"
866
+
}
867
+
],
868
+
"title": "CPU Usage",
869
+
"type": "timeseries"
870
+
},
871
+
{
872
+
"datasource": {
873
+
"type": "prometheus",
874
+
"uid": "${DS_MIMIR}"
875
+
},
876
+
"fieldConfig": {
877
+
"defaults": {
878
+
"color": {
879
+
"mode": "palette-classic"
880
+
},
881
+
"custom": {
882
+
"hideFrom": {
883
+
"legend": false,
884
+
"tooltip": false,
885
+
"viz": false
886
+
}
887
+
},
888
+
"decimals": 0,
889
+
"mappings": [],
890
+
"unit": "locale"
891
+
},
892
+
"overrides": []
893
+
},
894
+
"gridPos": {
895
+
"h": 9,
896
+
"w": 6,
897
+
"x": 18,
898
+
"y": 17
899
+
},
900
+
"id": 7,
901
+
"options": {
902
+
"displayLabels": [
903
+
"name"
904
+
],
905
+
"legend": {
906
+
"displayMode": "table",
907
+
"placement": "right",
908
+
"showLegend": true,
909
+
"values": [
910
+
"percent",
911
+
"value"
912
+
]
913
+
},
914
+
"pieType": "donut",
915
+
"reduceOptions": {
916
+
"calcs": [
917
+
"lastNotNull"
918
+
],
919
+
"fields": "",
920
+
"values": false
921
+
},
922
+
"tooltip": {
923
+
"mode": "single",
924
+
"sort": "none"
925
+
}
926
+
},
927
+
"targets": [
928
+
{
929
+
"datasource": {
930
+
"type": "prometheus",
931
+
"uid": "${DS_MIMIR}"
932
+
},
933
+
"editorMode": "code",
934
+
"expr": "avg(increase(sonar_records_processed_total{box=~\"${box}\",socket_url=~\"${socket}\", action=\"create\"}[$__range])) by (record_type)",
935
+
"format": "time_series",
936
+
"instant": false,
937
+
"legendFormat": "{{record_type}}",
938
+
"range": true,
939
+
"refId": "A"
940
+
}
941
+
],
942
+
"title": "Record Distribution",
943
+
"type": "piechart"
944
+
},
945
+
{
946
+
"datasource": {
947
+
"type": "prometheus",
948
+
"uid": "${DS_MIMIR}"
949
+
},
950
+
"fieldConfig": {
951
+
"defaults": {
952
+
"color": {
953
+
"mode": "palette-classic"
954
+
},
955
+
"custom": {
956
+
"axisBorderShow": false,
957
+
"axisCenteredZero": false,
958
+
"axisColorMode": "text",
959
+
"axisLabel": "",
960
+
"axisPlacement": "auto",
961
+
"barAlignment": 0,
962
+
"drawStyle": "line",
963
+
"fillOpacity": 0,
964
+
"gradientMode": "none",
965
+
"hideFrom": {
966
+
"legend": false,
967
+
"tooltip": false,
968
+
"viz": false
969
+
},
970
+
"insertNulls": false,
971
+
"lineInterpolation": "linear",
972
+
"lineWidth": 1,
973
+
"pointSize": 5,
974
+
"scaleDistribution": {
975
+
"type": "linear"
976
+
},
977
+
"showPoints": "auto",
978
+
"spanNulls": false,
979
+
"stacking": {
980
+
"group": "A",
981
+
"mode": "none"
982
+
},
983
+
"thresholdsStyle": {
984
+
"mode": "off"
985
+
}
986
+
},
987
+
"mappings": [],
988
+
"thresholds": {
989
+
"mode": "absolute",
990
+
"steps": [
991
+
{
992
+
"color": "green",
993
+
"value": null
994
+
}
995
+
]
996
+
},
997
+
"unit": "binbps"
998
+
},
999
+
"overrides": []
1000
+
},
1001
+
"gridPos": {
1002
+
"h": 9,
1003
+
"w": 12,
1004
+
"x": 0,
1005
+
"y": 18
1006
+
},
1007
+
"id": 10,
1008
+
"options": {
1009
+
"legend": {
1010
+
"calcs": [
1011
+
"mean",
1012
+
"max"
1013
+
],
1014
+
"displayMode": "list",
1015
+
"placement": "bottom",
1016
+
"showLegend": true,
1017
+
"sortBy": "Mean",
1018
+
"sortDesc": true
1019
+
},
1020
+
"tooltip": {
1021
+
"mode": "multi",
1022
+
"sort": "desc"
1023
+
}
1024
+
},
1025
+
"targets": [
1026
+
{
1027
+
"datasource": {
1028
+
"type": "prometheus",
1029
+
"uid": "${DS_MIMIR}"
1030
+
},
1031
+
"editorMode": "code",
1032
+
"expr": "sum(rate(indigo_repo_stream_bytes_total{box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance) * 8",
1033
+
"legendFormat": "{{instance}}",
1034
+
"range": true,
1035
+
"refId": "A"
1036
+
}
1037
+
],
1038
+
"title": "Socket Bandwidth",
1039
+
"type": "timeseries"
1040
+
},
1041
+
{
1042
+
"datasource": {
1043
+
"type": "prometheus",
1044
+
"uid": "${DS_MIMIR}"
1045
+
},
1046
+
"fieldConfig": {
1047
+
"defaults": {
1048
+
"color": {
1049
+
"mode": "palette-classic"
1050
+
},
1051
+
"custom": {
1052
+
"axisBorderShow": false,
1053
+
"axisCenteredZero": false,
1054
+
"axisColorMode": "text",
1055
+
"axisLabel": "",
1056
+
"axisPlacement": "auto",
1057
+
"barAlignment": 0,
1058
+
"drawStyle": "line",
1059
+
"fillOpacity": 0,
1060
+
"gradientMode": "none",
1061
+
"hideFrom": {
1062
+
"legend": false,
1063
+
"tooltip": false,
1064
+
"viz": false
1065
+
},
1066
+
"insertNulls": false,
1067
+
"lineInterpolation": "linear",
1068
+
"lineWidth": 1,
1069
+
"pointSize": 5,
1070
+
"scaleDistribution": {
1071
+
"type": "linear"
1072
+
},
1073
+
"showPoints": "auto",
1074
+
"spanNulls": false,
1075
+
"stacking": {
1076
+
"group": "A",
1077
+
"mode": "none"
1078
+
},
1079
+
"thresholdsStyle": {
1080
+
"mode": "off"
1081
+
}
1082
+
},
1083
+
"mappings": [],
1084
+
"thresholds": {
1085
+
"mode": "absolute",
1086
+
"steps": [
1087
+
{
1088
+
"color": "green",
1089
+
"value": null
1090
+
},
1091
+
{
1092
+
"color": "red",
1093
+
"value": 80
1094
+
}
1095
+
]
1096
+
},
1097
+
"unit": "bytes"
1098
+
},
1099
+
"overrides": []
1100
+
},
1101
+
"gridPos": {
1102
+
"h": 7,
1103
+
"w": 6,
1104
+
"x": 12,
1105
+
"y": 25
1106
+
},
1107
+
"id": 6,
1108
+
"options": {
1109
+
"legend": {
1110
+
"calcs": [],
1111
+
"displayMode": "list",
1112
+
"placement": "bottom",
1113
+
"showLegend": true
1114
+
},
1115
+
"tooltip": {
1116
+
"mode": "single",
1117
+
"sort": "none"
1118
+
}
1119
+
},
1120
+
"targets": [
1121
+
{
1122
+
"datasource": {
1123
+
"type": "prometheus",
1124
+
"uid": "${DS_MIMIR}"
1125
+
},
1126
+
"editorMode": "code",
1127
+
"expr": "process_resident_memory_bytes{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}",
1128
+
"instant": false,
1129
+
"legendFormat": "{{instance}}",
1130
+
"range": true,
1131
+
"refId": "A"
1132
+
}
1133
+
],
1134
+
"title": "Memory Usage",
1135
+
"type": "timeseries"
1136
+
},
1137
+
{
1138
+
"datasource": {
1139
+
"type": "prometheus",
1140
+
"uid": "${DS_MIMIR}"
1141
+
},
1142
+
"fieldConfig": {
1143
+
"defaults": {
1144
+
"color": {
1145
+
"mode": "palette-classic"
1146
+
},
1147
+
"custom": {
1148
+
"axisBorderShow": false,
1149
+
"axisCenteredZero": false,
1150
+
"axisColorMode": "text",
1151
+
"axisLabel": "",
1152
+
"axisPlacement": "auto",
1153
+
"barAlignment": 0,
1154
+
"drawStyle": "line",
1155
+
"fillOpacity": 0,
1156
+
"gradientMode": "none",
1157
+
"hideFrom": {
1158
+
"legend": false,
1159
+
"tooltip": false,
1160
+
"viz": false
1161
+
},
1162
+
"insertNulls": false,
1163
+
"lineInterpolation": "linear",
1164
+
"lineWidth": 1,
1165
+
"pointSize": 5,
1166
+
"scaleDistribution": {
1167
+
"type": "linear"
1168
+
},
1169
+
"showPoints": "auto",
1170
+
"spanNulls": false,
1171
+
"stacking": {
1172
+
"group": "A",
1173
+
"mode": "none"
1174
+
},
1175
+
"thresholdsStyle": {
1176
+
"mode": "off"
1177
+
}
1178
+
},
1179
+
"mappings": [],
1180
+
"thresholds": {
1181
+
"mode": "absolute",
1182
+
"steps": [
1183
+
{
1184
+
"color": "green",
1185
+
"value": null
1186
+
},
1187
+
{
1188
+
"color": "red",
1189
+
"value": 80
1190
+
}
1191
+
]
1192
+
},
1193
+
"unit": "s"
1194
+
},
1195
+
"overrides": []
1196
+
},
1197
+
"gridPos": {
1198
+
"h": 6,
1199
+
"w": 6,
1200
+
"x": 18,
1201
+
"y": 26
1202
+
},
1203
+
"id": 5,
1204
+
"options": {
1205
+
"legend": {
1206
+
"calcs": [],
1207
+
"displayMode": "list",
1208
+
"placement": "bottom",
1209
+
"showLegend": true
1210
+
},
1211
+
"tooltip": {
1212
+
"mode": "single",
1213
+
"sort": "none"
1214
+
}
1215
+
},
1216
+
"targets": [
1217
+
{
1218
+
"datasource": {
1219
+
"type": "prometheus",
1220
+
"uid": "${DS_MIMIR}"
1221
+
},
1222
+
"editorMode": "code",
1223
+
"expr": "sum(rate(process_cpu_seconds_total{job=\"sonar\", box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance) / sum(rate(sonar_events_processed_total{box=~\"${box}\",instance=~\"${instance}\"}[$__rate_interval])) by (instance)",
1224
+
"format": "time_series",
1225
+
"instant": false,
1226
+
"interval": "",
1227
+
"legendFormat": "CPU Seconds per Event",
1228
+
"range": true,
1229
+
"refId": "A"
1230
+
}
1231
+
],
1232
+
"title": "CPU per Event",
1233
+
"type": "timeseries"
1234
+
},
1235
+
{
1236
+
"datasource": {
1237
+
"type": "prometheus",
1238
+
"uid": "${DS_MIMIR}"
1239
+
},
1240
+
"fieldConfig": {
1241
+
"defaults": {
1242
+
"color": {
1243
+
"mode": "thresholds"
1244
+
},
1245
+
"decimals": 0,
1246
+
"mappings": [],
1247
+
"thresholds": {
1248
+
"mode": "absolute",
1249
+
"steps": [
1250
+
{
1251
+
"color": "green",
1252
+
"value": null
1253
+
},
1254
+
{
1255
+
"color": "orange",
1256
+
"value": 500000
1257
+
},
1258
+
{
1259
+
"color": "#EAB839",
1260
+
"value": 850000
1261
+
},
1262
+
{
1263
+
"color": "red",
1264
+
"value": 1000000
1265
+
}
1266
+
]
1267
+
},
1268
+
"unit": "locale"
1269
+
},
1270
+
"overrides": []
1271
+
},
1272
+
"gridPos": {
1273
+
"h": 4,
1274
+
"w": 2,
1275
+
"x": 10,
1276
+
"y": 27
1277
+
},
1278
+
"id": 11,
1279
+
"options": {
1280
+
"colorMode": "value",
1281
+
"graphMode": "area",
1282
+
"justifyMode": "auto",
1283
+
"orientation": "auto",
1284
+
"reduceOptions": {
1285
+
"calcs": [
1286
+
"lastNotNull"
1287
+
],
1288
+
"fields": "",
1289
+
"values": false
1290
+
},
1291
+
"showPercentChange": false,
1292
+
"textMode": "auto",
1293
+
"wideLayout": true
1294
+
},
1295
+
"pluginVersion": "10.4.1",
1296
+
"targets": [
1297
+
{
1298
+
"datasource": {
1299
+
"type": "prometheus",
1300
+
"uid": "${DS_MIMIR}"
1301
+
},
1302
+
"editorMode": "code",
1303
+
"expr": "increase(sonar_ops_processed_total{kind=~\"create\",socket_url=~\"wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos\", op_path=\"app.bsky.actor.profile\"}[$__range])",
1304
+
"instant": false,
1305
+
"legendFormat": "__auto",
1306
+
"range": true,
1307
+
"refId": "A"
1308
+
}
1309
+
],
1310
+
"title": "Profiles Created",
1311
+
"type": "stat"
1312
+
},
1313
+
{
1314
+
"datasource": {
1315
+
"type": "prometheus",
1316
+
"uid": "${DS_MIMIR}"
1317
+
},
1318
+
"fieldConfig": {
1319
+
"defaults": {
1320
+
"color": {
1321
+
"mode": "palette-classic"
1322
+
},
1323
+
"custom": {
1324
+
"axisBorderShow": false,
1325
+
"axisCenteredZero": false,
1326
+
"axisColorMode": "text",
1327
+
"axisLabel": "",
1328
+
"axisPlacement": "auto",
1329
+
"barAlignment": 0,
1330
+
"drawStyle": "line",
1331
+
"fillOpacity": 55,
1332
+
"gradientMode": "none",
1333
+
"hideFrom": {
1334
+
"legend": false,
1335
+
"tooltip": false,
1336
+
"viz": false
1337
+
},
1338
+
"insertNulls": false,
1339
+
"lineInterpolation": "linear",
1340
+
"lineWidth": 1,
1341
+
"pointSize": 5,
1342
+
"scaleDistribution": {
1343
+
"type": "linear"
1344
+
},
1345
+
"showPoints": "auto",
1346
+
"spanNulls": false,
1347
+
"stacking": {
1348
+
"group": "A",
1349
+
"mode": "normal"
1350
+
},
1351
+
"thresholdsStyle": {
1352
+
"mode": "line+area"
1353
+
}
1354
+
},
1355
+
"mappings": [],
1356
+
"thresholds": {
1357
+
"mode": "absolute",
1358
+
"steps": [
1359
+
{
1360
+
"color": "green",
1361
+
"value": null
1362
+
},
1363
+
{
1364
+
"color": "yellow",
1365
+
"value": 1000
1366
+
},
1367
+
{
1368
+
"color": "orange",
1369
+
"value": 1200
1370
+
},
1371
+
{
1372
+
"color": "red",
1373
+
"value": 1500
1374
+
}
1375
+
]
1376
+
},
1377
+
"unit": "cps"
1378
+
},
1379
+
"overrides": []
1380
+
},
1381
+
"gridPos": {
1382
+
"h": 14,
1383
+
"w": 24,
1384
+
"x": 0,
1385
+
"y": 32
1386
+
},
1387
+
"id": 4,
1388
+
"options": {
1389
+
"legend": {
1390
+
"calcs": [
1391
+
"mean",
1392
+
"max"
1393
+
],
1394
+
"displayMode": "table",
1395
+
"placement": "right",
1396
+
"showLegend": true,
1397
+
"sortBy": "Mean",
1398
+
"sortDesc": true
1399
+
},
1400
+
"tooltip": {
1401
+
"mode": "multi",
1402
+
"sort": "desc"
1403
+
}
1404
+
},
1405
+
"targets": [
1406
+
{
1407
+
"datasource": {
1408
+
"type": "prometheus",
1409
+
"uid": "${DS_MIMIR}"
1410
+
},
1411
+
"editorMode": "code",
1412
+
"expr": "avg(rate(sonar_ops_processed_total{kind=~\"${op_kind}\", box=~\"${box}\",socket_url=~\"${socket}\"}[$__rate_interval])) by (op_path, kind) > 0",
1413
+
"legendFormat": "{{kind}} {{op_path}}",
1414
+
"range": true,
1415
+
"refId": "A"
1416
+
}
1417
+
],
1418
+
"title": "Operation Throughput",
1419
+
"transformations": [
1420
+
{
1421
+
"id": "renameByRegex",
1422
+
"options": {
1423
+
"regex": "(.*)(app.bsky.)(.*)",
1424
+
"renamePattern": "$1 $3"
1425
+
}
1426
+
}
1427
+
],
1428
+
"type": "timeseries"
1429
+
}
1430
+
],
1431
+
"refresh": "30s",
1432
+
"schemaVersion": 39,
1433
+
"tags": [],
1434
+
"templating": {
1435
+
"list": [
1436
+
{
1437
+
"current": {},
1438
+
"datasource": {
1439
+
"type": "prometheus",
1440
+
"uid": "${DS_MIMIR}"
1441
+
},
1442
+
"definition": "label_values(sonar_ops_processed_total,kind)",
1443
+
"hide": 0,
1444
+
"includeAll": true,
1445
+
"label": "Operation Kind",
1446
+
"multi": true,
1447
+
"name": "op_kind",
1448
+
"options": [],
1449
+
"query": {
1450
+
"query": "label_values(sonar_ops_processed_total,kind)",
1451
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
1452
+
},
1453
+
"refresh": 1,
1454
+
"regex": "",
1455
+
"skipUrlSync": false,
1456
+
"sort": 0,
1457
+
"type": "query"
1458
+
},
1459
+
{
1460
+
"current": {},
1461
+
"datasource": {
1462
+
"type": "prometheus",
1463
+
"uid": "${DS_MIMIR}"
1464
+
},
1465
+
"definition": "label_values(indigo_repo_stream_bytes_total{job=\"sonar\"},box)",
1466
+
"hide": 0,
1467
+
"includeAll": false,
1468
+
"label": "Sonar Host",
1469
+
"multi": false,
1470
+
"name": "box",
1471
+
"options": [],
1472
+
"query": {
1473
+
"qryType": 1,
1474
+
"query": "label_values(indigo_repo_stream_bytes_total{job=\"sonar\"},box)",
1475
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
1476
+
},
1477
+
"refresh": 1,
1478
+
"regex": "",
1479
+
"skipUrlSync": false,
1480
+
"sort": 0,
1481
+
"type": "query"
1482
+
},
1483
+
{
1484
+
"current": {},
1485
+
"datasource": {
1486
+
"type": "prometheus",
1487
+
"uid": "${DS_MIMIR}"
1488
+
},
1489
+
"definition": "label_values(socket_url)",
1490
+
"hide": 0,
1491
+
"includeAll": false,
1492
+
"label": "Socket",
1493
+
"multi": false,
1494
+
"name": "socket",
1495
+
"options": [],
1496
+
"query": {
1497
+
"query": "label_values(socket_url)",
1498
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
1499
+
},
1500
+
"refresh": 1,
1501
+
"regex": "",
1502
+
"skipUrlSync": false,
1503
+
"sort": 0,
1504
+
"type": "query"
1505
+
},
1506
+
{
1507
+
"current": {},
1508
+
"datasource": {
1509
+
"type": "prometheus",
1510
+
"uid": "${DS_MIMIR}"
1511
+
},
1512
+
"definition": "label_values(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"},instance)",
1513
+
"hide": 2,
1514
+
"includeAll": false,
1515
+
"label": "Instance",
1516
+
"multi": false,
1517
+
"name": "instance",
1518
+
"options": [],
1519
+
"query": {
1520
+
"query": "label_values(sonar_events_processed_total{box=~\"${box}\",socket_url=~\"${socket}\"},instance)",
1521
+
"refId": "PrometheusVariableQueryEditor-VariableQuery"
1522
+
},
1523
+
"refresh": 2,
1524
+
"regex": "",
1525
+
"skipUrlSync": false,
1526
+
"sort": 0,
1527
+
"type": "query"
1528
+
}
1529
+
]
1530
+
},
1531
+
"time": {
1532
+
"from": "now-24h",
1533
+
"to": "now"
1534
+
},
1535
+
"timepicker": {},
1536
+
"timezone": "",
1537
+
"title": "Sonar",
1538
+
"uid": "e5c542d0-07d6-44e9-bc30-a4b973bedd86",
1539
+
"version": 24,
1540
+
"weekStart": ""
1541
+
}
+16
-12
cmd/sonar/main.go
+16
-12
cmd/sonar/main.go
···
33
33
34
34
app.Flags = []cli.Flag{
35
35
&cli.StringFlag{
36
-
Name: "ws-url",
37
-
Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint",
38
-
Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos",
36
+
Name: "ws-url",
37
+
Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint",
38
+
Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos",
39
+
EnvVars: []string{"SONAR_WS_URL"},
39
40
},
40
41
&cli.StringFlag{
41
-
Name: "log-level",
42
-
Usage: "log level",
43
-
Value: "info",
42
+
Name: "log-level",
43
+
Usage: "log level",
44
+
Value: "info",
45
+
EnvVars: []string{"SONAR_LOG_LEVEL"},
44
46
},
45
47
&cli.IntFlag{
46
-
Name: "port",
47
-
Usage: "listen port for metrics server",
48
-
Value: 8345,
48
+
Name: "port",
49
+
Usage: "listen port for metrics server",
50
+
Value: 8345,
51
+
EnvVars: []string{"SONAR_PORT"},
49
52
},
50
53
&cli.IntFlag{
51
54
Name: "max-queue-size",
···
53
56
Value: 10,
54
57
},
55
58
&cli.StringFlag{
56
-
Name: "cursor-file",
57
-
Usage: "path to cursor file",
58
-
Value: "sonar_cursor.json",
59
+
Name: "cursor-file",
60
+
Usage: "path to cursor file",
61
+
Value: "sonar_cursor.json",
62
+
EnvVars: []string{"SONAR_CURSOR_FILE"},
59
63
},
60
64
}
61
65
cmd/sonar/sonar_dash.png
cmd/sonar/sonar_dash.png
This is a binary file and will not be displayed.