Fast and robust atproto CAR file processing in rust
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

benchmark streaming some cars

phil cf59b7f2 2262fa76

+433
+380
Cargo.lock
··· 27 27 ] 28 28 29 29 [[package]] 30 + name = "anes" 31 + version = "0.1.6" 32 + source = "registry+https://github.com/rust-lang/crates.io-index" 33 + checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" 34 + 35 + [[package]] 30 36 name = "anstream" 31 37 version = "0.6.21" 32 38 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 81 87 version = "1.0.100" 82 88 source = "registry+https://github.com/rust-lang/crates.io-index" 83 89 checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" 90 + 91 + [[package]] 92 + name = "autocfg" 93 + version = "1.5.0" 94 + source = "registry+https://github.com/rust-lang/crates.io-index" 95 + checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 84 96 85 97 [[package]] 86 98 name = "backtrace" ··· 120 132 checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" 121 133 122 134 [[package]] 135 + name = "bumpalo" 136 + version = "3.19.0" 137 + source = "registry+https://github.com/rust-lang/crates.io-index" 138 + checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" 139 + 140 + [[package]] 123 141 name = "bytes" 124 142 version = "1.10.1" 125 143 source = "registry+https://github.com/rust-lang/crates.io-index" 126 144 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 145 + 146 + [[package]] 147 + name = "cast" 148 + version = "0.3.0" 149 + source = "registry+https://github.com/rust-lang/crates.io-index" 150 + checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" 127 151 128 152 [[package]] 129 153 name = "cbor4ii" ··· 141 165 checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" 142 166 143 167 [[package]] 168 + name = "ciborium" 169 + version = "0.2.2" 170 + source = "registry+https://github.com/rust-lang/crates.io-index" 171 + checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" 172 + dependencies = [ 173 + "ciborium-io", 174 + "ciborium-ll", 175 + "serde", 176 + ] 177 + 178 + [[package]] 179 + name = "ciborium-io" 180 + version = "0.2.2" 181 + source = "registry+https://github.com/rust-lang/crates.io-index" 182 + checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" 183 + 184 + [[package]] 185 + name = "ciborium-ll" 186 + version = "0.2.2" 187 + source = "registry+https://github.com/rust-lang/crates.io-index" 188 + checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" 189 + dependencies = [ 190 + "ciborium-io", 191 + "half", 192 + ] 193 + 194 + [[package]] 144 195 name = "cid" 145 196 version = "0.11.1" 146 197 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 216 267 ] 217 268 218 269 [[package]] 270 + name = "criterion" 271 + version = "0.7.0" 272 + source = "registry+https://github.com/rust-lang/crates.io-index" 273 + checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" 274 + dependencies = [ 275 + "anes", 276 + "cast", 277 + "ciborium", 278 + "clap", 279 + "criterion-plot", 280 + "itertools", 281 + "num-traits", 282 + "oorandom", 283 + "plotters", 284 + "rayon", 285 + "regex", 286 + "serde", 287 + "serde_json", 288 + "tinytemplate", 289 + "tokio", 290 + "walkdir", 291 + ] 292 + 293 + [[package]] 294 + name = "criterion-plot" 295 + version = "0.6.0" 296 + source = "registry+https://github.com/rust-lang/crates.io-index" 297 + checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" 298 + dependencies = [ 299 + "cast", 300 + "itertools", 301 + ] 302 + 303 + [[package]] 304 + name = "crossbeam-deque" 305 + version = "0.8.6" 306 + source = "registry+https://github.com/rust-lang/crates.io-index" 307 + checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" 308 + dependencies = [ 309 + "crossbeam-epoch", 310 + "crossbeam-utils", 311 + ] 312 + 313 + [[package]] 314 + name = "crossbeam-epoch" 315 + version = "0.9.18" 316 + source = "registry+https://github.com/rust-lang/crates.io-index" 317 + checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 318 + dependencies = [ 319 + "crossbeam-utils", 320 + ] 321 + 322 + [[package]] 323 + name = "crossbeam-utils" 324 + version = "0.8.21" 325 + source = "registry+https://github.com/rust-lang/crates.io-index" 326 + checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 327 + 328 + [[package]] 329 + name = "crunchy" 330 + version = "0.2.4" 331 + source = "registry+https://github.com/rust-lang/crates.io-index" 332 + checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" 333 + 334 + [[package]] 219 335 name = "data-encoding" 220 336 version = "2.9.0" 221 337 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 242 358 ] 243 359 244 360 [[package]] 361 + name = "either" 362 + version = "1.15.0" 363 + source = "registry+https://github.com/rust-lang/crates.io-index" 364 + checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 365 + 366 + [[package]] 245 367 name = "env_filter" 246 368 version = "0.1.3" 247 369 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 360 482 checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" 361 483 362 484 [[package]] 485 + name = "half" 486 + version = "2.7.0" 487 + source = "registry+https://github.com/rust-lang/crates.io-index" 488 + checksum = "e54c115d4f30f52c67202f079c5f9d8b49db4691f460fdb0b4c2e838261b2ba5" 489 + dependencies = [ 490 + "cfg-if", 491 + "crunchy", 492 + "zerocopy", 493 + ] 494 + 495 + [[package]] 363 496 name = "heck" 364 497 version = "0.5.0" 365 498 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 410 543 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 411 544 412 545 [[package]] 546 + name = "itertools" 547 + version = "0.13.0" 548 + source = "registry+https://github.com/rust-lang/crates.io-index" 549 + checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 550 + dependencies = [ 551 + "either", 552 + ] 553 + 554 + [[package]] 555 + name = "itoa" 556 + version = "1.0.15" 557 + source = "registry+https://github.com/rust-lang/crates.io-index" 558 + checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" 559 + 560 + [[package]] 413 561 name = "jiff" 414 562 version = "0.2.15" 415 563 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 431 579 "proc-macro2", 432 580 "quote", 433 581 "syn 2.0.106", 582 + ] 583 + 584 + [[package]] 585 + name = "js-sys" 586 + version = "0.3.81" 587 + source = "registry+https://github.com/rust-lang/crates.io-index" 588 + checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" 589 + dependencies = [ 590 + "once_cell", 591 + "wasm-bindgen", 434 592 ] 435 593 436 594 [[package]] ··· 512 670 "core2", 513 671 "serde", 514 672 "unsigned-varint 0.8.0", 673 + ] 674 + 675 + [[package]] 676 + name = "num-traits" 677 + version = "0.2.19" 678 + source = "registry+https://github.com/rust-lang/crates.io-index" 679 + checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 680 + dependencies = [ 681 + "autocfg", 515 682 ] 516 683 517 684 [[package]] ··· 524 691 ] 525 692 526 693 [[package]] 694 + name = "once_cell" 695 + version = "1.21.3" 696 + source = "registry+https://github.com/rust-lang/crates.io-index" 697 + checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" 698 + 699 + [[package]] 527 700 name = "once_cell_polyfill" 528 701 version = "1.70.1" 529 702 source = "registry+https://github.com/rust-lang/crates.io-index" 530 703 checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" 704 + 705 + [[package]] 706 + name = "oorandom" 707 + version = "11.1.5" 708 + source = "registry+https://github.com/rust-lang/crates.io-index" 709 + checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" 531 710 532 711 [[package]] 533 712 name = "parking_lot" ··· 565 744 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 566 745 567 746 [[package]] 747 + name = "plotters" 748 + version = "0.3.7" 749 + source = "registry+https://github.com/rust-lang/crates.io-index" 750 + checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" 751 + dependencies = [ 752 + "num-traits", 753 + "plotters-backend", 754 + "plotters-svg", 755 + "wasm-bindgen", 756 + "web-sys", 757 + ] 758 + 759 + [[package]] 760 + name = "plotters-backend" 761 + version = "0.3.7" 762 + source = "registry+https://github.com/rust-lang/crates.io-index" 763 + checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" 764 + 765 + [[package]] 766 + name = "plotters-svg" 767 + version = "0.3.7" 768 + source = "registry+https://github.com/rust-lang/crates.io-index" 769 + checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" 770 + dependencies = [ 771 + "plotters-backend", 772 + ] 773 + 774 + [[package]] 568 775 name = "portable-atomic" 569 776 version = "1.11.1" 570 777 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 598 805 ] 599 806 600 807 [[package]] 808 + name = "rayon" 809 + version = "1.11.0" 810 + source = "registry+https://github.com/rust-lang/crates.io-index" 811 + checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" 812 + dependencies = [ 813 + "either", 814 + "rayon-core", 815 + ] 816 + 817 + [[package]] 818 + name = "rayon-core" 819 + version = "1.13.0" 820 + source = "registry+https://github.com/rust-lang/crates.io-index" 821 + checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" 822 + dependencies = [ 823 + "crossbeam-deque", 824 + "crossbeam-utils", 825 + ] 826 + 827 + [[package]] 601 828 name = "redox_syscall" 602 829 version = "0.5.18" 603 830 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 640 867 version = "0.1.0" 641 868 dependencies = [ 642 869 "clap", 870 + "criterion", 643 871 "env_logger", 644 872 "futures", 645 873 "futures-core", ··· 661 889 checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" 662 890 663 891 [[package]] 892 + name = "rustversion" 893 + version = "1.0.22" 894 + source = "registry+https://github.com/rust-lang/crates.io-index" 895 + checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" 896 + 897 + [[package]] 898 + name = "ryu" 899 + version = "1.0.20" 900 + source = "registry+https://github.com/rust-lang/crates.io-index" 901 + checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" 902 + 903 + [[package]] 904 + name = "same-file" 905 + version = "1.0.6" 906 + source = "registry+https://github.com/rust-lang/crates.io-index" 907 + checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 908 + dependencies = [ 909 + "winapi-util", 910 + ] 911 + 912 + [[package]] 664 913 name = "scopeguard" 665 914 version = "1.2.0" 666 915 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 716 965 "ipld-core", 717 966 "scopeguard", 718 967 "serde", 968 + ] 969 + 970 + [[package]] 971 + name = "serde_json" 972 + version = "1.0.145" 973 + source = "registry+https://github.com/rust-lang/crates.io-index" 974 + checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" 975 + dependencies = [ 976 + "itoa", 977 + "memchr", 978 + "ryu", 979 + "serde", 980 + "serde_core", 719 981 ] 720 982 721 983 [[package]] ··· 818 1080 ] 819 1081 820 1082 [[package]] 1083 + name = "tinytemplate" 1084 + version = "1.2.1" 1085 + source = "registry+https://github.com/rust-lang/crates.io-index" 1086 + checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" 1087 + dependencies = [ 1088 + "serde", 1089 + "serde_json", 1090 + ] 1091 + 1092 + [[package]] 821 1093 name = "tokio" 822 1094 version = "1.47.1" 823 1095 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 873 1145 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 874 1146 875 1147 [[package]] 1148 + name = "walkdir" 1149 + version = "2.5.0" 1150 + source = "registry+https://github.com/rust-lang/crates.io-index" 1151 + checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 1152 + dependencies = [ 1153 + "same-file", 1154 + "winapi-util", 1155 + ] 1156 + 1157 + [[package]] 876 1158 name = "wasi" 877 1159 version = "0.11.1+wasi-snapshot-preview1" 878 1160 source = "registry+https://github.com/rust-lang/crates.io-index" 879 1161 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" 880 1162 881 1163 [[package]] 1164 + name = "wasm-bindgen" 1165 + version = "0.2.104" 1166 + source = "registry+https://github.com/rust-lang/crates.io-index" 1167 + checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" 1168 + dependencies = [ 1169 + "cfg-if", 1170 + "once_cell", 1171 + "rustversion", 1172 + "wasm-bindgen-macro", 1173 + "wasm-bindgen-shared", 1174 + ] 1175 + 1176 + [[package]] 1177 + name = "wasm-bindgen-backend" 1178 + version = "0.2.104" 1179 + source = "registry+https://github.com/rust-lang/crates.io-index" 1180 + checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" 1181 + dependencies = [ 1182 + "bumpalo", 1183 + "log", 1184 + "proc-macro2", 1185 + "quote", 1186 + "syn 2.0.106", 1187 + "wasm-bindgen-shared", 1188 + ] 1189 + 1190 + [[package]] 1191 + name = "wasm-bindgen-macro" 1192 + version = "0.2.104" 1193 + source = "registry+https://github.com/rust-lang/crates.io-index" 1194 + checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" 1195 + dependencies = [ 1196 + "quote", 1197 + "wasm-bindgen-macro-support", 1198 + ] 1199 + 1200 + [[package]] 1201 + name = "wasm-bindgen-macro-support" 1202 + version = "0.2.104" 1203 + source = "registry+https://github.com/rust-lang/crates.io-index" 1204 + checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" 1205 + dependencies = [ 1206 + "proc-macro2", 1207 + "quote", 1208 + "syn 2.0.106", 1209 + "wasm-bindgen-backend", 1210 + "wasm-bindgen-shared", 1211 + ] 1212 + 1213 + [[package]] 1214 + name = "wasm-bindgen-shared" 1215 + version = "0.2.104" 1216 + source = "registry+https://github.com/rust-lang/crates.io-index" 1217 + checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" 1218 + dependencies = [ 1219 + "unicode-ident", 1220 + ] 1221 + 1222 + [[package]] 1223 + name = "web-sys" 1224 + version = "0.3.81" 1225 + source = "registry+https://github.com/rust-lang/crates.io-index" 1226 + checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" 1227 + dependencies = [ 1228 + "js-sys", 1229 + "wasm-bindgen", 1230 + ] 1231 + 1232 + [[package]] 1233 + name = "winapi-util" 1234 + version = "0.1.11" 1235 + source = "registry+https://github.com/rust-lang/crates.io-index" 1236 + checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 1237 + dependencies = [ 1238 + "windows-sys 0.60.2", 1239 + ] 1240 + 1241 + [[package]] 882 1242 name = "windows-link" 883 1243 version = "0.2.1" 884 1244 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1030 1390 version = "0.53.1" 1031 1391 source = "registry+https://github.com/rust-lang/crates.io-index" 1032 1392 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" 1393 + 1394 + [[package]] 1395 + name = "zerocopy" 1396 + version = "0.8.27" 1397 + source = "registry+https://github.com/rust-lang/crates.io-index" 1398 + checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" 1399 + dependencies = [ 1400 + "zerocopy-derive", 1401 + ] 1402 + 1403 + [[package]] 1404 + name = "zerocopy-derive" 1405 + version = "0.8.27" 1406 + source = "registry+https://github.com/rust-lang/crates.io-index" 1407 + checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" 1408 + dependencies = [ 1409 + "proc-macro2", 1410 + "quote", 1411 + "syn 2.0.106", 1412 + ]
+5
Cargo.toml
··· 18 18 19 19 [dev-dependencies] 20 20 clap = { version = "4.5.48", features = ["derive"] } 21 + criterion = { version = "0.7.0", features = ["async_tokio"] } 21 22 env_logger = "0.11.8" 22 23 multibase = "0.9.2" 23 24 tokio = { version = "1.47.1", features = ["full"] } ··· 25 26 [profile.profiling] 26 27 inherits = "release" 27 28 debug = true 29 + 30 + [[bench]] 31 + name = "non-huge-cars" 32 + harness = false
+48
benches/non-huge-cars.rs
··· 1 + extern crate repo_stream; 2 + use futures::TryStreamExt; 3 + use iroh_car::CarReader; 4 + use std::convert::Infallible; 5 + 6 + use criterion::{criterion_group, criterion_main, Criterion}; 7 + 8 + const TINY_CAR: &'static [u8] = include_bytes!("../car-samples/tiny.car"); 9 + const LITTLE_CAR: &'static [u8] = include_bytes!("../car-samples/little.car"); 10 + const MIDSIZE_CAR: &'static [u8] = include_bytes!("../car-samples/midsize.car"); 11 + 12 + pub fn criterion_benchmark(c: &mut Criterion) { 13 + let rt = tokio::runtime::Builder::new_multi_thread() 14 + .enable_all() 15 + .build() 16 + .expect("Creating runtime failed"); 17 + 18 + c.bench_function("tiny-car", |b| b.to_async(&rt).iter(async || drive_car(TINY_CAR).await )); 19 + c.bench_function("little-car", |b| b.to_async(&rt).iter(async || drive_car(LITTLE_CAR).await )); 20 + c.bench_function("midsize-car", |b| b.to_async(&rt).iter(async || drive_car(MIDSIZE_CAR).await )); 21 + } 22 + 23 + async fn drive_car(bytes: &[u8]) { 24 + let reader = CarReader::new(bytes).await.unwrap(); 25 + 26 + let root = reader 27 + .header() 28 + .roots() 29 + .first() 30 + .ok_or("missing root").unwrap() 31 + .clone(); 32 + 33 + let stream = std::pin::pin!(reader.stream()); 34 + 35 + let (_commit, v) = 36 + repo_stream::drive::Vehicle::init(root, stream, |block| Ok::<_, Infallible>(block.len())) 37 + .await 38 + .unwrap(); 39 + let mut record_stream = std::pin::pin!(v.stream()); 40 + 41 + while let Some(_) = record_stream.try_next().await.unwrap() { 42 + // just here for the drive 43 + } 44 + } 45 + 46 + criterion_group!(benches, criterion_benchmark); 47 + criterion_main!(benches); 48 +
car-samples/little.car

This is a binary file and will not be displayed.

car-samples/midsize.car

This is a binary file and will not be displayed.

car-samples/tiny.car

This is a binary file and will not be displayed.