feat: promtheus metrics in ingest · mia.omg.lol/flashback@f432805

+178 -3

Cargo.lock

··· 352 352 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 353 353 354 354 [[package]] 355 + name = "aws-lc-rs" 356 + version = "1.15.1" 357 + source = "registry+https://github.com/rust-lang/crates.io-index" 358 + checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f" 359 + dependencies = [ 360 + "aws-lc-sys", 361 + "zeroize", 362 + ] 363 + 364 + [[package]] 365 + name = "aws-lc-sys" 366 + version = "0.34.0" 367 + source = "registry+https://github.com/rust-lang/crates.io-index" 368 + checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6" 369 + dependencies = [ 370 + "cc", 371 + "cmake", 372 + "dunce", 373 + "fs_extra", 374 + ] 375 + 376 + [[package]] 355 377 name = "axum" 356 378 version = "0.8.6" 357 379 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 769 791 checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" 770 792 771 793 [[package]] 794 + name = "cmake" 795 + version = "0.1.54" 796 + source = "registry+https://github.com/rust-lang/crates.io-index" 797 + checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" 798 + dependencies = [ 799 + "cc", 800 + ] 801 + 802 + [[package]] 772 803 name = "colorchoice" 773 804 version = "1.0.4" 774 805 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 903 934 version = "0.5.15" 904 935 source = "registry+https://github.com/rust-lang/crates.io-index" 905 936 checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" 937 + dependencies = [ 938 + "crossbeam-utils", 939 + ] 940 + 941 + [[package]] 942 + name = "crossbeam-epoch" 943 + version = "0.9.18" 944 + source = "registry+https://github.com/rust-lang/crates.io-index" 945 + checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 906 946 dependencies = [ 907 947 "crossbeam-utils", 908 948 ] ··· 1119 1159 "strum 0.27.2", 1120 1160 "uuid", 1121 1161 ] 1162 + 1163 + [[package]] 1164 + name = "dunce" 1165 + version = "1.0.5" 1166 + source = "registry+https://github.com/rust-lang/crates.io-index" 1167 + checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 1122 1168 1123 1169 [[package]] 1124 1170 name = "dyn-clone" ··· 1281 1327 "jacquard-common", 1282 1328 "jacquard-identity", 1283 1329 "jacquard-lexicon", 1330 + "metrics", 1331 + "metrics-exporter-prometheus", 1284 1332 "r2d2", 1285 1333 "reqwest", 1286 1334 "serde", ··· 1316 1364 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" 1317 1365 1318 1366 [[package]] 1367 + name = "foldhash" 1368 + version = "0.2.0" 1369 + source = "registry+https://github.com/rust-lang/crates.io-index" 1370 + checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" 1371 + 1372 + [[package]] 1319 1373 name = "foreign-types" 1320 1374 version = "0.3.2" 1321 1375 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1338 1392 dependencies = [ 1339 1393 "percent-encoding", 1340 1394 ] 1395 + 1396 + [[package]] 1397 + name = "fs_extra" 1398 + version = "1.3.0" 1399 + source = "registry+https://github.com/rust-lang/crates.io-index" 1400 + checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" 1341 1401 1342 1402 [[package]] 1343 1403 name = "funty" ··· 1619 1679 source = "registry+https://github.com/rust-lang/crates.io-index" 1620 1680 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" 1621 1681 dependencies = [ 1622 - "foldhash", 1682 + "foldhash 0.1.5", 1623 1683 ] 1624 1684 1625 1685 [[package]] ··· 1627 1687 version = "0.16.0" 1628 1688 source = "registry+https://github.com/rust-lang/crates.io-index" 1629 1689 checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" 1690 + dependencies = [ 1691 + "foldhash 0.2.0", 1692 + ] 1630 1693 1631 1694 [[package]] 1632 1695 name = "hashlink" ··· 1777 1840 1778 1841 [[package]] 1779 1842 name = "hyper" 1780 - version = "1.7.0" 1843 + version = "1.8.1" 1781 1844 source = "registry+https://github.com/rust-lang/crates.io-index" 1782 - checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" 1845 + checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" 1783 1846 dependencies = [ 1784 1847 "atomic-waker", 1785 1848 "bytes", ··· 1808 1871 "hyper", 1809 1872 "hyper-util", 1810 1873 "rustls", 1874 + "rustls-native-certs", 1811 1875 "rustls-pki-types", 1812 1876 "tokio", 1813 1877 "tokio-rustls", ··· 2615 2679 checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" 2616 2680 2617 2681 [[package]] 2682 + name = "metrics" 2683 + version = "0.24.3" 2684 + source = "registry+https://github.com/rust-lang/crates.io-index" 2685 + checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8" 2686 + dependencies = [ 2687 + "ahash 0.8.12", 2688 + "portable-atomic", 2689 + ] 2690 + 2691 + [[package]] 2692 + name = "metrics-exporter-prometheus" 2693 + version = "0.18.0" 2694 + source = "registry+https://github.com/rust-lang/crates.io-index" 2695 + checksum = "2bac37bd902eaf3f9028708c4fbeed677e738cb2b34c2da8524c4731ebeb301e" 2696 + dependencies = [ 2697 + "base64", 2698 + "http-body-util", 2699 + "hyper", 2700 + "hyper-rustls", 2701 + "hyper-util", 2702 + "indexmap 2.12.0", 2703 + "ipnet", 2704 + "metrics", 2705 + "metrics-util", 2706 + "quanta", 2707 + "rustls", 2708 + "thiserror 2.0.17", 2709 + "tokio", 2710 + "tracing", 2711 + ] 2712 + 2713 + [[package]] 2714 + name = "metrics-util" 2715 + version = "0.20.1" 2716 + source = "registry+https://github.com/rust-lang/crates.io-index" 2717 + checksum = "cdfb1365fea27e6dd9dc1dbc19f570198bc86914533ad639dae939635f096be4" 2718 + dependencies = [ 2719 + "crossbeam-epoch", 2720 + "crossbeam-utils", 2721 + "hashbrown 0.16.0", 2722 + "metrics", 2723 + "quanta", 2724 + "rand 0.9.2", 2725 + "rand_xoshiro", 2726 + "sketches-ddsketch", 2727 + ] 2728 + 2729 + [[package]] 2618 2730 name = "miette" 2619 2731 version = "7.6.0" 2620 2732 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3251 3363 ] 3252 3364 3253 3365 [[package]] 3366 + name = "quanta" 3367 + version = "0.12.6" 3368 + source = "registry+https://github.com/rust-lang/crates.io-index" 3369 + checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" 3370 + dependencies = [ 3371 + "crossbeam-utils", 3372 + "libc", 3373 + "once_cell", 3374 + "raw-cpuid", 3375 + "wasi", 3376 + "web-sys", 3377 + "winapi", 3378 + ] 3379 + 3380 + [[package]] 3254 3381 name = "quinn" 3255 3382 version = "0.11.9" 3256 3383 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3397 3524 ] 3398 3525 3399 3526 [[package]] 3527 + name = "rand_xoshiro" 3528 + version = "0.7.0" 3529 + source = "registry+https://github.com/rust-lang/crates.io-index" 3530 + checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" 3531 + dependencies = [ 3532 + "rand_core 0.9.3", 3533 + ] 3534 + 3535 + [[package]] 3400 3536 name = "range-traits" 3401 3537 version = "0.3.2" 3402 3538 source = "registry+https://github.com/rust-lang/crates.io-index" 3403 3539 checksum = "d20581732dd76fa913c7dff1a2412b714afe3573e94d41c34719de73337cc8ab" 3540 + 3541 + [[package]] 3542 + name = "raw-cpuid" 3543 + version = "11.6.0" 3544 + source = "registry+https://github.com/rust-lang/crates.io-index" 3545 + checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" 3546 + dependencies = [ 3547 + "bitflags", 3548 + ] 3404 3549 3405 3550 [[package]] 3406 3551 name = "redox_syscall" ··· 3643 3788 source = "registry+https://github.com/rust-lang/crates.io-index" 3644 3789 checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" 3645 3790 dependencies = [ 3791 + "aws-lc-rs", 3646 3792 "once_cell", 3647 3793 "ring", 3648 3794 "rustls-pki-types", ··· 3679 3825 source = "registry+https://github.com/rust-lang/crates.io-index" 3680 3826 checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" 3681 3827 dependencies = [ 3828 + "aws-lc-rs", 3682 3829 "ring", 3683 3830 "rustls-pki-types", 3684 3831 "untrusted", ··· 4062 4209 "tempfile", 4063 4210 "walkdir", 4064 4211 ] 4212 + 4213 + [[package]] 4214 + name = "sketches-ddsketch" 4215 + version = "0.3.0" 4216 + source = "registry+https://github.com/rust-lang/crates.io-index" 4217 + checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" 4065 4218 4066 4219 [[package]] 4067 4220 name = "slab" ··· 5022 5175 checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" 5023 5176 5024 5177 [[package]] 5178 + name = "winapi" 5179 + version = "0.3.9" 5180 + source = "registry+https://github.com/rust-lang/crates.io-index" 5181 + checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 5182 + dependencies = [ 5183 + "winapi-i686-pc-windows-gnu", 5184 + "winapi-x86_64-pc-windows-gnu", 5185 + ] 5186 + 5187 + [[package]] 5188 + name = "winapi-i686-pc-windows-gnu" 5189 + version = "0.4.0" 5190 + source = "registry+https://github.com/rust-lang/crates.io-index" 5191 + checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 5192 + 5193 + [[package]] 5025 5194 name = "winapi-util" 5026 5195 version = "0.1.11" 5027 5196 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5029 5198 dependencies = [ 5030 5199 "windows-sys 0.61.2", 5031 5200 ] 5201 + 5202 + [[package]] 5203 + name = "winapi-x86_64-pc-windows-gnu" 5204 + version = "0.4.0" 5205 + source = "registry+https://github.com/rust-lang/crates.io-index" 5206 + checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 5032 5207 5033 5208 [[package]] 5034 5209 name = "windows"

+2

Cargo.toml

··· 19 19 jacquard-common = "0.9" 20 20 jacquard-identity = { version = "0.9.2", features = ["cache"] } 21 21 jacquard-lexicon = "0.9.2" 22 + metrics = "0.24.3" 23 + metrics-exporter-prometheus = "0.18.0" 22 24 r2d2 = "0.8" 23 25 reqwest = "0.12.24" 24 26 serde = { version = "1.0.228", features = ["derive"] }

+1 -1

src/ingest/mod.rs

··· 45 45 while let Some(msg) = messages.next().await { 46 46 match msg { 47 47 Ok(msg) => match tx.send(msg).await { 48 - Ok(_) => {} 48 + Ok(_) => {metrics::counter!("jetstream_events").increment(1)} 49 49 Err(e) => tracing::error!("Error sending jetstream message: {:?}", e), 50 50 }, 51 51 Err(e) if *e.kind() == StreamErrorKind::Closed => break,

+4

src/ingest/scrobbles.rs

··· 18 18 19 19 if let Some(rkey) = check_duplicate_scrobble(&conn, did, &scrobble.track_name, created)? { 20 20 tracing::debug!("skipping duplicate scrobble of {rkey}"); 21 + metrics::counter!("scrobbles", "status"=>"skipped", "source"=>"tealfm").increment(1); 21 22 return Ok(()); 22 23 } 23 24 ··· 36 37 track_discrim: scrobble.track_discriminant.as_deref(), 37 38 }; 38 39 resolve_and_insert_scrobble(&conn, did, rkey, &scrobble.track_name, find, created)?; 40 + metrics::counter!("scrobbles", "status"=>"written", "source"=>"tealfm").increment(1); 39 41 40 42 Ok(()) 41 43 } ··· 51 53 52 54 if let Some(rkey) = check_duplicate_scrobble(&conn, did, &scrobble.title, created)? { 53 55 tracing::debug!("skipping duplicate scrobble of {rkey}"); 56 + metrics::counter!("scrobbles", "status"=>"skipped", "source"=>"rocksky").increment(1); 54 57 return Ok(()); 55 58 } 56 59 ··· 62 65 ..Default::default() 63 66 }; 64 67 resolve_and_insert_scrobble(&conn, did, rkey, &scrobble.title, find, created)?; 68 + metrics::counter!("scrobbles", "status"=>"written", "source"=>"rocksky").increment(1); 65 69 66 70 Ok(()) 67 71 }

+2

src/main.rs

··· 1 1 use clap::Parser; 2 2 use std::sync::Arc; 3 + use metrics_exporter_prometheus::PrometheusBuilder; 3 4 use r2d2::ManageConnection; 4 5 5 6 mod analysis; ··· 13 14 #[tokio::main] 14 15 async fn main() -> eyre::Result<()> { 15 16 tracing_subscriber::fmt::init(); 17 + PrometheusBuilder::new().install()?; 16 18 17 19 let config = config::Config::parse(); 18 20

Configure Feed

Configure Feed