+2

.gitignore

reviewed

··· 1 1 + /target 2 2 + /local

+1465

Cargo.lock

reviewed

··· 1 1 + # This file is automatically @generated by Cargo. 2 2 + # It is not intended for manual editing. 3 3 + version = 4 4 4 + 5 5 + [[package]] 6 6 + name = "aho-corasick" 7 7 + version = "1.1.4" 8 8 + source = "registry+https://github.com/rust-lang/crates.io-index" 9 9 + checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" 10 10 + dependencies = [ 11 11 + "memchr", 12 12 + ] 13 13 + 14 14 + [[package]] 15 15 + name = "anstream" 16 16 + version = "1.0.0" 17 17 + source = "registry+https://github.com/rust-lang/crates.io-index" 18 18 + checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" 19 19 + dependencies = [ 20 20 + "anstyle", 21 21 + "anstyle-parse", 22 22 + "anstyle-query", 23 23 + "anstyle-wincon", 24 24 + "colorchoice", 25 25 + "is_terminal_polyfill", 26 26 + "utf8parse", 27 27 + ] 28 28 + 29 29 + [[package]] 30 30 + name = "anstyle" 31 31 + version = "1.0.14" 32 32 + source = "registry+https://github.com/rust-lang/crates.io-index" 33 33 + checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" 34 34 + 35 35 + [[package]] 36 36 + name = "anstyle-parse" 37 37 + version = "1.0.0" 38 38 + source = "registry+https://github.com/rust-lang/crates.io-index" 39 39 + checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" 40 40 + dependencies = [ 41 41 + "utf8parse", 42 42 + ] 43 43 + 44 44 + [[package]] 45 45 + name = "anstyle-query" 46 46 + version = "1.1.5" 47 47 + source = "registry+https://github.com/rust-lang/crates.io-index" 48 48 + checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" 49 49 + dependencies = [ 50 50 + "windows-sys", 51 51 + ] 52 52 + 53 53 + [[package]] 54 54 + name = "anstyle-wincon" 55 55 + version = "3.0.11" 56 56 + source = "registry+https://github.com/rust-lang/crates.io-index" 57 57 + checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" 58 58 + dependencies = [ 59 59 + "anstyle", 60 60 + "once_cell_polyfill", 61 61 + "windows-sys", 62 62 + ] 63 63 + 64 64 + [[package]] 65 65 + name = "anyhow" 66 66 + version = "1.0.102" 67 67 + source = "registry+https://github.com/rust-lang/crates.io-index" 68 68 + checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" 69 69 + 70 70 + [[package]] 71 71 + name = "async-channel" 72 72 + version = "2.5.0" 73 73 + source = "registry+https://github.com/rust-lang/crates.io-index" 74 74 + checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" 75 75 + dependencies = [ 76 76 + "concurrent-queue", 77 77 + "event-listener-strategy", 78 78 + "futures-core", 79 79 + "pin-project-lite", 80 80 + ] 81 81 + 82 82 + [[package]] 83 83 + name = "base-x" 84 84 + version = "0.2.11" 85 85 + source = "registry+https://github.com/rust-lang/crates.io-index" 86 86 + checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" 87 87 + 88 88 + [[package]] 89 89 + name = "base256emoji" 90 90 + version = "1.0.2" 91 91 + source = "registry+https://github.com/rust-lang/crates.io-index" 92 92 + checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" 93 93 + dependencies = [ 94 94 + "const-str", 95 95 + "match-lookup", 96 96 + ] 97 97 + 98 98 + [[package]] 99 99 + name = "bindgen" 100 100 + version = "0.72.1" 101 101 + source = "registry+https://github.com/rust-lang/crates.io-index" 102 102 + checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" 103 103 + dependencies = [ 104 104 + "bitflags", 105 105 + "cexpr", 106 106 + "clang-sys", 107 107 + "itertools", 108 108 + "proc-macro2", 109 109 + "quote", 110 110 + "regex", 111 111 + "rustc-hash", 112 112 + "shlex", 113 113 + "syn", 114 114 + ] 115 115 + 116 116 + [[package]] 117 117 + name = "bitflags" 118 118 + version = "2.11.0" 119 119 + source = "registry+https://github.com/rust-lang/crates.io-index" 120 120 + checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" 121 121 + 122 122 + [[package]] 123 123 + name = "block-buffer" 124 124 + version = "0.10.4" 125 125 + source = "registry+https://github.com/rust-lang/crates.io-index" 126 126 + checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 127 127 + dependencies = [ 128 128 + "generic-array", 129 129 + ] 130 130 + 131 131 + [[package]] 132 132 + name = "byteorder-lite" 133 133 + version = "0.1.0" 134 134 + source = "registry+https://github.com/rust-lang/crates.io-index" 135 135 + checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" 136 136 + 137 137 + [[package]] 138 138 + name = "bytes" 139 139 + version = "1.11.1" 140 140 + source = "registry+https://github.com/rust-lang/crates.io-index" 141 141 + checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" 142 142 + 143 143 + [[package]] 144 144 + name = "byteview" 145 145 + version = "0.10.1" 146 146 + source = "registry+https://github.com/rust-lang/crates.io-index" 147 147 + checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" 148 148 + 149 149 + [[package]] 150 150 + name = "bzip2-sys" 151 151 + version = "0.1.13+1.0.8" 152 152 + source = "registry+https://github.com/rust-lang/crates.io-index" 153 153 + checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" 154 154 + dependencies = [ 155 155 + "cc", 156 156 + "pkg-config", 157 157 + ] 158 158 + 159 159 + [[package]] 160 160 + name = "cbor4ii" 161 161 + version = "0.2.14" 162 162 + source = "registry+https://github.com/rust-lang/crates.io-index" 163 163 + checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" 164 164 + dependencies = [ 165 165 + "serde", 166 166 + ] 167 167 + 168 168 + [[package]] 169 169 + name = "cc" 170 170 + version = "1.2.59" 171 171 + source = "registry+https://github.com/rust-lang/crates.io-index" 172 172 + checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" 173 173 + dependencies = [ 174 174 + "find-msvc-tools", 175 175 + "jobserver", 176 176 + "libc", 177 177 + "shlex", 178 178 + ] 179 179 + 180 180 + [[package]] 181 181 + name = "cexpr" 182 182 + version = "0.6.0" 183 183 + source = "registry+https://github.com/rust-lang/crates.io-index" 184 184 + checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" 185 185 + dependencies = [ 186 186 + "nom", 187 187 + ] 188 188 + 189 189 + [[package]] 190 190 + name = "cfg-if" 191 191 + version = "1.0.4" 192 192 + source = "registry+https://github.com/rust-lang/crates.io-index" 193 193 + checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" 194 194 + 195 195 + [[package]] 196 196 + name = "cid" 197 197 + version = "0.11.1" 198 198 + source = "registry+https://github.com/rust-lang/crates.io-index" 199 199 + checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" 200 200 + dependencies = [ 201 201 + "core2", 202 202 + "multibase", 203 203 + "multihash", 204 204 + "serde", 205 205 + "serde_bytes", 206 206 + "unsigned-varint 0.8.0", 207 207 + ] 208 208 + 209 209 + [[package]] 210 210 + name = "clang-sys" 211 211 + version = "1.8.1" 212 212 + source = "registry+https://github.com/rust-lang/crates.io-index" 213 213 + checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" 214 214 + dependencies = [ 215 215 + "glob", 216 216 + "libc", 217 217 + "libloading", 218 218 + ] 219 219 + 220 220 + [[package]] 221 221 + name = "clap" 222 222 + version = "4.6.0" 223 223 + source = "registry+https://github.com/rust-lang/crates.io-index" 224 224 + checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" 225 225 + dependencies = [ 226 226 + "clap_builder", 227 227 + "clap_derive", 228 228 + ] 229 229 + 230 230 + [[package]] 231 231 + name = "clap_builder" 232 232 + version = "4.6.0" 233 233 + source = "registry+https://github.com/rust-lang/crates.io-index" 234 234 + checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" 235 235 + dependencies = [ 236 236 + "anstream", 237 237 + "anstyle", 238 238 + "clap_lex", 239 239 + "strsim", 240 240 + ] 241 241 + 242 242 + [[package]] 243 243 + name = "clap_derive" 244 244 + version = "4.6.0" 245 245 + source = "registry+https://github.com/rust-lang/crates.io-index" 246 246 + checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" 247 247 + dependencies = [ 248 248 + "heck", 249 249 + "proc-macro2", 250 250 + "quote", 251 251 + "syn", 252 252 + ] 253 253 + 254 254 + [[package]] 255 255 + name = "clap_lex" 256 256 + version = "1.1.0" 257 257 + source = "registry+https://github.com/rust-lang/crates.io-index" 258 258 + checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" 259 259 + 260 260 + [[package]] 261 261 + name = "colorchoice" 262 262 + version = "1.0.5" 263 263 + source = "registry+https://github.com/rust-lang/crates.io-index" 264 264 + checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" 265 265 + 266 266 + [[package]] 267 267 + name = "compare" 268 268 + version = "0.0.6" 269 269 + source = "registry+https://github.com/rust-lang/crates.io-index" 270 270 + checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" 271 271 + 272 272 + [[package]] 273 273 + name = "concurrent-queue" 274 274 + version = "2.5.0" 275 275 + source = "registry+https://github.com/rust-lang/crates.io-index" 276 276 + checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" 277 277 + dependencies = [ 278 278 + "crossbeam-utils", 279 279 + ] 280 280 + 281 281 + [[package]] 282 282 + name = "const-str" 283 283 + version = "0.4.3" 284 284 + source = "registry+https://github.com/rust-lang/crates.io-index" 285 285 + checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" 286 286 + 287 287 + [[package]] 288 288 + name = "core2" 289 289 + version = "0.4.0" 290 290 + source = "registry+https://github.com/rust-lang/crates.io-index" 291 291 + checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" 292 292 + dependencies = [ 293 293 + "memchr", 294 294 + ] 295 295 + 296 296 + [[package]] 297 297 + name = "cpufeatures" 298 298 + version = "0.2.17" 299 299 + source = "registry+https://github.com/rust-lang/crates.io-index" 300 300 + checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 301 301 + dependencies = [ 302 302 + "libc", 303 303 + ] 304 304 + 305 305 + [[package]] 306 306 + name = "crossbeam-epoch" 307 307 + version = "0.9.18" 308 308 + source = "registry+https://github.com/rust-lang/crates.io-index" 309 309 + checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 310 310 + dependencies = [ 311 311 + "crossbeam-utils", 312 312 + ] 313 313 + 314 314 + [[package]] 315 315 + name = "crossbeam-skiplist" 316 316 + version = "0.1.3" 317 317 + source = "registry+https://github.com/rust-lang/crates.io-index" 318 318 + checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" 319 319 + dependencies = [ 320 320 + "crossbeam-epoch", 321 321 + "crossbeam-utils", 322 322 + ] 323 323 + 324 324 + [[package]] 325 325 + name = "crossbeam-utils" 326 326 + version = "0.8.21" 327 327 + source = "registry+https://github.com/rust-lang/crates.io-index" 328 328 + checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 329 329 + 330 330 + [[package]] 331 331 + name = "crypto-common" 332 332 + version = "0.1.7" 333 333 + source = "registry+https://github.com/rust-lang/crates.io-index" 334 334 + checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" 335 335 + dependencies = [ 336 336 + "generic-array", 337 337 + "typenum", 338 338 + ] 339 339 + 340 340 + [[package]] 341 341 + name = "dashmap" 342 342 + version = "6.1.0" 343 343 + source = "registry+https://github.com/rust-lang/crates.io-index" 344 344 + checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" 345 345 + dependencies = [ 346 346 + "cfg-if", 347 347 + "crossbeam-utils", 348 348 + "hashbrown 0.14.5", 349 349 + "lock_api", 350 350 + "once_cell", 351 351 + "parking_lot_core", 352 352 + ] 353 353 + 354 354 + [[package]] 355 355 + name = "data-encoding" 356 356 + version = "2.10.0" 357 357 + source = "registry+https://github.com/rust-lang/crates.io-index" 358 358 + checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" 359 359 + 360 360 + [[package]] 361 361 + name = "data-encoding-macro" 362 362 + version = "0.1.19" 363 363 + source = "registry+https://github.com/rust-lang/crates.io-index" 364 364 + checksum = "8142a83c17aa9461d637e649271eae18bf2edd00e91f2e105df36c3c16355bdb" 365 365 + dependencies = [ 366 366 + "data-encoding", 367 367 + "data-encoding-macro-internal", 368 368 + ] 369 369 + 370 370 + [[package]] 371 371 + name = "data-encoding-macro-internal" 372 372 + version = "0.1.17" 373 373 + source = "registry+https://github.com/rust-lang/crates.io-index" 374 374 + checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" 375 375 + dependencies = [ 376 376 + "data-encoding", 377 377 + "syn", 378 378 + ] 379 379 + 380 380 + [[package]] 381 381 + name = "digest" 382 382 + version = "0.10.7" 383 383 + source = "registry+https://github.com/rust-lang/crates.io-index" 384 384 + checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 385 385 + dependencies = [ 386 386 + "block-buffer", 387 387 + "crypto-common", 388 388 + ] 389 389 + 390 390 + [[package]] 391 391 + name = "either" 392 392 + version = "1.15.0" 393 393 + source = "registry+https://github.com/rust-lang/crates.io-index" 394 394 + checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 395 395 + 396 396 + [[package]] 397 397 + name = "enum_dispatch" 398 398 + version = "0.3.13" 399 399 + source = "registry+https://github.com/rust-lang/crates.io-index" 400 400 + checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" 401 401 + dependencies = [ 402 402 + "once_cell", 403 403 + "proc-macro2", 404 404 + "quote", 405 405 + "syn", 406 406 + ] 407 407 + 408 408 + [[package]] 409 409 + name = "equivalent" 410 410 + version = "1.0.2" 411 411 + source = "registry+https://github.com/rust-lang/crates.io-index" 412 412 + checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" 413 413 + 414 414 + [[package]] 415 415 + name = "errno" 416 416 + version = "0.3.14" 417 417 + source = "registry+https://github.com/rust-lang/crates.io-index" 418 418 + checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" 419 419 + dependencies = [ 420 420 + "libc", 421 421 + "windows-sys", 422 422 + ] 423 423 + 424 424 + [[package]] 425 425 + name = "event-listener" 426 426 + version = "5.4.1" 427 427 + source = "registry+https://github.com/rust-lang/crates.io-index" 428 428 + checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" 429 429 + dependencies = [ 430 430 + "concurrent-queue", 431 431 + "parking", 432 432 + "pin-project-lite", 433 433 + ] 434 434 + 435 435 + [[package]] 436 436 + name = "event-listener-strategy" 437 437 + version = "0.5.4" 438 438 + source = "registry+https://github.com/rust-lang/crates.io-index" 439 439 + checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" 440 440 + dependencies = [ 441 441 + "event-listener", 442 442 + "pin-project-lite", 443 443 + ] 444 444 + 445 445 + [[package]] 446 446 + name = "fastrand" 447 447 + version = "2.4.1" 448 448 + source = "registry+https://github.com/rust-lang/crates.io-index" 449 449 + checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" 450 450 + 451 451 + [[package]] 452 452 + name = "find-msvc-tools" 453 453 + version = "0.1.9" 454 454 + source = "registry+https://github.com/rust-lang/crates.io-index" 455 455 + checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" 456 456 + 457 457 + [[package]] 458 458 + name = "fjall" 459 459 + version = "3.1.3" 460 460 + source = "registry+https://github.com/rust-lang/crates.io-index" 461 461 + checksum = "fdf46551c9abc5fb0e0d540da36c875197285af2a29833892d7d3434b8617343" 462 462 + dependencies = [ 463 463 + "byteorder-lite", 464 464 + "byteview", 465 465 + "dashmap", 466 466 + "flume", 467 467 + "log", 468 468 + "lsm-tree", 469 469 + "tempfile", 470 470 + "xxhash-rust", 471 471 + ] 472 472 + 473 473 + [[package]] 474 474 + name = "flume" 475 475 + version = "0.12.0" 476 476 + source = "registry+https://github.com/rust-lang/crates.io-index" 477 477 + checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" 478 478 + dependencies = [ 479 479 + "spin", 480 480 + ] 481 481 + 482 482 + [[package]] 483 483 + name = "futures" 484 484 + version = "0.3.32" 485 485 + source = "registry+https://github.com/rust-lang/crates.io-index" 486 486 + checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" 487 487 + dependencies = [ 488 488 + "futures-channel", 489 489 + "futures-core", 490 490 + "futures-executor", 491 491 + "futures-io", 492 492 + "futures-sink", 493 493 + "futures-task", 494 494 + "futures-util", 495 495 + ] 496 496 + 497 497 + [[package]] 498 498 + name = "futures-channel" 499 499 + version = "0.3.32" 500 500 + source = "registry+https://github.com/rust-lang/crates.io-index" 501 501 + checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" 502 502 + dependencies = [ 503 503 + "futures-core", 504 504 + "futures-sink", 505 505 + ] 506 506 + 507 507 + [[package]] 508 508 + name = "futures-core" 509 509 + version = "0.3.32" 510 510 + source = "registry+https://github.com/rust-lang/crates.io-index" 511 511 + checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" 512 512 + 513 513 + [[package]] 514 514 + name = "futures-executor" 515 515 + version = "0.3.32" 516 516 + source = "registry+https://github.com/rust-lang/crates.io-index" 517 517 + checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" 518 518 + dependencies = [ 519 519 + "futures-core", 520 520 + "futures-task", 521 521 + "futures-util", 522 522 + ] 523 523 + 524 524 + [[package]] 525 525 + name = "futures-io" 526 526 + version = "0.3.32" 527 527 + source = "registry+https://github.com/rust-lang/crates.io-index" 528 528 + checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" 529 529 + 530 530 + [[package]] 531 531 + name = "futures-macro" 532 532 + version = "0.3.32" 533 533 + source = "registry+https://github.com/rust-lang/crates.io-index" 534 534 + checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" 535 535 + dependencies = [ 536 536 + "proc-macro2", 537 537 + "quote", 538 538 + "syn", 539 539 + ] 540 540 + 541 541 + [[package]] 542 542 + name = "futures-sink" 543 543 + version = "0.3.32" 544 544 + source = "registry+https://github.com/rust-lang/crates.io-index" 545 545 + checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" 546 546 + 547 547 + [[package]] 548 548 + name = "futures-task" 549 549 + version = "0.3.32" 550 550 + source = "registry+https://github.com/rust-lang/crates.io-index" 551 551 + checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" 552 552 + 553 553 + [[package]] 554 554 + name = "futures-util" 555 555 + version = "0.3.32" 556 556 + source = "registry+https://github.com/rust-lang/crates.io-index" 557 557 + checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" 558 558 + dependencies = [ 559 559 + "futures-channel", 560 560 + "futures-core", 561 561 + "futures-io", 562 562 + "futures-macro", 563 563 + "futures-sink", 564 564 + "futures-task", 565 565 + "memchr", 566 566 + "pin-project-lite", 567 567 + "slab", 568 568 + ] 569 569 + 570 570 + [[package]] 571 571 + name = "generic-array" 572 572 + version = "0.14.7" 573 573 + source = "registry+https://github.com/rust-lang/crates.io-index" 574 574 + checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 575 575 + dependencies = [ 576 576 + "typenum", 577 577 + "version_check", 578 578 + ] 579 579 + 580 580 + [[package]] 581 581 + name = "getrandom" 582 582 + version = "0.3.4" 583 583 + source = "registry+https://github.com/rust-lang/crates.io-index" 584 584 + checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" 585 585 + dependencies = [ 586 586 + "cfg-if", 587 587 + "libc", 588 588 + "r-efi", 589 589 + "wasip2", 590 590 + ] 591 591 + 592 592 + [[package]] 593 593 + name = "glob" 594 594 + version = "0.3.3" 595 595 + source = "registry+https://github.com/rust-lang/crates.io-index" 596 596 + checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" 597 597 + 598 598 + [[package]] 599 599 + name = "hashbrown" 600 600 + version = "0.14.5" 601 601 + source = "registry+https://github.com/rust-lang/crates.io-index" 602 602 + checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" 603 603 + 604 604 + [[package]] 605 605 + name = "hashbrown" 606 606 + version = "0.16.1" 607 607 + source = "registry+https://github.com/rust-lang/crates.io-index" 608 608 + checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" 609 609 + 610 610 + [[package]] 611 611 + name = "heck" 612 612 + version = "0.5.0" 613 613 + source = "registry+https://github.com/rust-lang/crates.io-index" 614 614 + checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 615 615 + 616 616 + [[package]] 617 617 + name = "interval-heap" 618 618 + version = "0.0.5" 619 619 + source = "registry+https://github.com/rust-lang/crates.io-index" 620 620 + checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" 621 621 + dependencies = [ 622 622 + "compare", 623 623 + ] 624 624 + 625 625 + [[package]] 626 626 + name = "ipld-core" 627 627 + version = "0.4.3" 628 628 + source = "registry+https://github.com/rust-lang/crates.io-index" 629 629 + checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" 630 630 + dependencies = [ 631 631 + "cid", 632 632 + "serde", 633 633 + "serde_bytes", 634 634 + ] 635 635 + 636 636 + [[package]] 637 637 + name = "iroh-car" 638 638 + version = "0.5.1" 639 639 + source = "registry+https://github.com/rust-lang/crates.io-index" 640 640 + checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" 641 641 + dependencies = [ 642 642 + "anyhow", 643 643 + "cid", 644 644 + "futures", 645 645 + "serde", 646 646 + "serde_ipld_dagcbor", 647 647 + "thiserror 1.0.69", 648 648 + "tokio", 649 649 + "unsigned-varint 0.7.2", 650 650 + ] 651 651 + 652 652 + [[package]] 653 653 + name = "is_terminal_polyfill" 654 654 + version = "1.70.2" 655 655 + source = "registry+https://github.com/rust-lang/crates.io-index" 656 656 + checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" 657 657 + 658 658 + [[package]] 659 659 + name = "itertools" 660 660 + version = "0.13.0" 661 661 + source = "registry+https://github.com/rust-lang/crates.io-index" 662 662 + checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 663 663 + dependencies = [ 664 664 + "either", 665 665 + ] 666 666 + 667 667 + [[package]] 668 668 + name = "jobserver" 669 669 + version = "0.1.34" 670 670 + source = "registry+https://github.com/rust-lang/crates.io-index" 671 671 + checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" 672 672 + dependencies = [ 673 673 + "getrandom", 674 674 + "libc", 675 675 + ] 676 676 + 677 677 + [[package]] 678 678 + name = "lazy_static" 679 679 + version = "1.5.0" 680 680 + source = "registry+https://github.com/rust-lang/crates.io-index" 681 681 + checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 682 682 + 683 683 + [[package]] 684 684 + name = "libc" 685 685 + version = "0.2.184" 686 686 + source = "registry+https://github.com/rust-lang/crates.io-index" 687 687 + checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" 688 688 + 689 689 + [[package]] 690 690 + name = "libloading" 691 691 + version = "0.8.9" 692 692 + source = "registry+https://github.com/rust-lang/crates.io-index" 693 693 + checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" 694 694 + dependencies = [ 695 695 + "cfg-if", 696 696 + "windows-link", 697 697 + ] 698 698 + 699 699 + [[package]] 700 700 + name = "libz-sys" 701 701 + version = "1.1.28" 702 702 + source = "registry+https://github.com/rust-lang/crates.io-index" 703 703 + checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" 704 704 + dependencies = [ 705 705 + "cc", 706 706 + "pkg-config", 707 707 + "vcpkg", 708 708 + ] 709 709 + 710 710 + [[package]] 711 711 + name = "linux-raw-sys" 712 712 + version = "0.12.1" 713 713 + source = "registry+https://github.com/rust-lang/crates.io-index" 714 714 + checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" 715 715 + 716 716 + [[package]] 717 717 + name = "lock_api" 718 718 + version = "0.4.14" 719 719 + source = "registry+https://github.com/rust-lang/crates.io-index" 720 720 + checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" 721 721 + dependencies = [ 722 722 + "scopeguard", 723 723 + ] 724 724 + 725 725 + [[package]] 726 726 + name = "log" 727 727 + version = "0.4.29" 728 728 + source = "registry+https://github.com/rust-lang/crates.io-index" 729 729 + checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" 730 730 + 731 731 + [[package]] 732 732 + name = "lsm-tree" 733 733 + version = "3.1.3" 734 734 + source = "registry+https://github.com/rust-lang/crates.io-index" 735 735 + checksum = "f38ed727e0965f218f4e419231b9ecacc1db8fb5066cf1df0d2bafeef88459d4" 736 736 + dependencies = [ 737 737 + "byteorder-lite", 738 738 + "byteview", 739 739 + "crossbeam-skiplist", 740 740 + "enum_dispatch", 741 741 + "interval-heap", 742 742 + "log", 743 743 + "quick_cache", 744 744 + "rustc-hash", 745 745 + "self_cell", 746 746 + "sfa", 747 747 + "tempfile", 748 748 + "varint-rs", 749 749 + "xxhash-rust", 750 750 + ] 751 751 + 752 752 + [[package]] 753 753 + name = "lz4-sys" 754 754 + version = "1.11.1+lz4-1.10.0" 755 755 + source = "registry+https://github.com/rust-lang/crates.io-index" 756 756 + checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" 757 757 + dependencies = [ 758 758 + "cc", 759 759 + "libc", 760 760 + ] 761 761 + 762 762 + [[package]] 763 763 + name = "match-lookup" 764 764 + version = "0.1.2" 765 765 + source = "registry+https://github.com/rust-lang/crates.io-index" 766 766 + checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" 767 767 + dependencies = [ 768 768 + "proc-macro2", 769 769 + "quote", 770 770 + "syn", 771 771 + ] 772 772 + 773 773 + [[package]] 774 774 + name = "matchers" 775 775 + version = "0.2.0" 776 776 + source = "registry+https://github.com/rust-lang/crates.io-index" 777 777 + checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" 778 778 + dependencies = [ 779 779 + "regex-automata", 780 780 + ] 781 781 + 782 782 + [[package]] 783 783 + name = "memchr" 784 784 + version = "2.8.0" 785 785 + source = "registry+https://github.com/rust-lang/crates.io-index" 786 786 + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" 787 787 + 788 788 + [[package]] 789 789 + name = "minimal-lexical" 790 790 + version = "0.2.1" 791 791 + source = "registry+https://github.com/rust-lang/crates.io-index" 792 792 + checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 793 793 + 794 794 + [[package]] 795 795 + name = "mio" 796 796 + version = "1.2.0" 797 797 + source = "registry+https://github.com/rust-lang/crates.io-index" 798 798 + checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" 799 799 + dependencies = [ 800 800 + "libc", 801 801 + "wasi", 802 802 + "windows-sys", 803 803 + ] 804 804 + 805 805 + [[package]] 806 806 + name = "multibase" 807 807 + version = "0.9.2" 808 808 + source = "registry+https://github.com/rust-lang/crates.io-index" 809 809 + checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" 810 810 + dependencies = [ 811 811 + "base-x", 812 812 + "base256emoji", 813 813 + "data-encoding", 814 814 + "data-encoding-macro", 815 815 + ] 816 816 + 817 817 + [[package]] 818 818 + name = "multihash" 819 819 + version = "0.19.3" 820 820 + source = "registry+https://github.com/rust-lang/crates.io-index" 821 821 + checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" 822 822 + dependencies = [ 823 823 + "core2", 824 824 + "serde", 825 825 + "unsigned-varint 0.8.0", 826 826 + ] 827 827 + 828 828 + [[package]] 829 829 + name = "nom" 830 830 + version = "7.1.3" 831 831 + source = "registry+https://github.com/rust-lang/crates.io-index" 832 832 + checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 833 833 + dependencies = [ 834 834 + "memchr", 835 835 + "minimal-lexical", 836 836 + ] 837 837 + 838 838 + [[package]] 839 839 + name = "nu-ansi-term" 840 840 + version = "0.50.3" 841 841 + source = "registry+https://github.com/rust-lang/crates.io-index" 842 842 + checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" 843 843 + dependencies = [ 844 844 + "windows-sys", 845 845 + ] 846 846 + 847 847 + [[package]] 848 848 + name = "once_cell" 849 849 + version = "1.21.4" 850 850 + source = "registry+https://github.com/rust-lang/crates.io-index" 851 851 + checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" 852 852 + 853 853 + [[package]] 854 854 + name = "once_cell_polyfill" 855 855 + version = "1.70.2" 856 856 + source = "registry+https://github.com/rust-lang/crates.io-index" 857 857 + checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" 858 858 + 859 859 + [[package]] 860 860 + name = "parking" 861 861 + version = "2.2.1" 862 862 + source = "registry+https://github.com/rust-lang/crates.io-index" 863 863 + checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" 864 864 + 865 865 + [[package]] 866 866 + name = "parking_lot" 867 867 + version = "0.12.5" 868 868 + source = "registry+https://github.com/rust-lang/crates.io-index" 869 869 + checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" 870 870 + dependencies = [ 871 871 + "lock_api", 872 872 + "parking_lot_core", 873 873 + ] 874 874 + 875 875 + [[package]] 876 876 + name = "parking_lot_core" 877 877 + version = "0.9.12" 878 878 + source = "registry+https://github.com/rust-lang/crates.io-index" 879 879 + checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" 880 880 + dependencies = [ 881 881 + "cfg-if", 882 882 + "libc", 883 883 + "redox_syscall", 884 884 + "smallvec", 885 885 + "windows-link", 886 886 + ] 887 887 + 888 888 + [[package]] 889 889 + name = "pin-project-lite" 890 890 + version = "0.2.17" 891 891 + source = "registry+https://github.com/rust-lang/crates.io-index" 892 892 + checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" 893 893 + 894 894 + [[package]] 895 895 + name = "pkg-config" 896 896 + version = "0.3.32" 897 897 + source = "registry+https://github.com/rust-lang/crates.io-index" 898 898 + checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 899 899 + 900 900 + [[package]] 901 901 + name = "proc-macro2" 902 902 + version = "1.0.106" 903 903 + source = "registry+https://github.com/rust-lang/crates.io-index" 904 904 + checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" 905 905 + dependencies = [ 906 906 + "unicode-ident", 907 907 + ] 908 908 + 909 909 + [[package]] 910 910 + name = "quick_cache" 911 911 + version = "0.6.21" 912 912 + source = "registry+https://github.com/rust-lang/crates.io-index" 913 913 + checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3" 914 914 + dependencies = [ 915 915 + "equivalent", 916 916 + "hashbrown 0.16.1", 917 917 + ] 918 918 + 919 919 + [[package]] 920 920 + name = "quote" 921 921 + version = "1.0.45" 922 922 + source = "registry+https://github.com/rust-lang/crates.io-index" 923 923 + checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" 924 924 + dependencies = [ 925 925 + "proc-macro2", 926 926 + ] 927 927 + 928 928 + [[package]] 929 929 + name = "r-efi" 930 930 + version = "5.3.0" 931 931 + source = "registry+https://github.com/rust-lang/crates.io-index" 932 932 + checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" 933 933 + 934 934 + [[package]] 935 935 + name = "redox_syscall" 936 936 + version = "0.5.18" 937 937 + source = "registry+https://github.com/rust-lang/crates.io-index" 938 938 + checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" 939 939 + dependencies = [ 940 940 + "bitflags", 941 941 + ] 942 942 + 943 943 + [[package]] 944 944 + name = "regex" 945 945 + version = "1.12.3" 946 946 + source = "registry+https://github.com/rust-lang/crates.io-index" 947 947 + checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" 948 948 + dependencies = [ 949 949 + "aho-corasick", 950 950 + "memchr", 951 951 + "regex-automata", 952 952 + "regex-syntax", 953 953 + ] 954 954 + 955 955 + [[package]] 956 956 + name = "regex-automata" 957 957 + version = "0.4.14" 958 958 + source = "registry+https://github.com/rust-lang/crates.io-index" 959 959 + checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" 960 960 + dependencies = [ 961 961 + "aho-corasick", 962 962 + "memchr", 963 963 + "regex-syntax", 964 964 + ] 965 965 + 966 966 + [[package]] 967 967 + name = "regex-syntax" 968 968 + version = "0.8.10" 969 969 + source = "registry+https://github.com/rust-lang/crates.io-index" 970 970 + checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" 971 971 + 972 972 + [[package]] 973 973 + name = "repo-stream" 974 974 + version = "0.5.0-alpha.5" 975 975 + source = "registry+https://github.com/rust-lang/crates.io-index" 976 976 + checksum = "c885e481796dcf2662a515dd76719120160c399de7fac74809c2f56e354fee41" 977 977 + dependencies = [ 978 978 + "cid", 979 979 + "fjall", 980 980 + "iroh-car", 981 981 + "log", 982 982 + "serde", 983 983 + "serde_bytes", 984 984 + "serde_ipld_dagcbor", 985 985 + "sha2", 986 986 + "thiserror 2.0.18", 987 987 + "tokio", 988 988 + ] 989 989 + 990 990 + [[package]] 991 991 + name = "rust-librocksdb-sys" 992 992 + version = "0.42.0+10.10.1" 993 993 + source = "registry+https://github.com/rust-lang/crates.io-index" 994 994 + checksum = "6522f8dfc02acd618477cd9aa94234f6a823ce578bba66215f15df8cb09a97d5" 995 995 + dependencies = [ 996 996 + "bindgen", 997 997 + "bzip2-sys", 998 998 + "cc", 999 999 + "glob", 1000 1000 + "libc", 1001 1001 + "libz-sys", 1002 1002 + "lz4-sys", 1003 1003 + "zstd-sys", 1004 1004 + ] 1005 1005 + 1006 1006 + [[package]] 1007 1007 + name = "rust-rocksdb" 1008 1008 + version = "0.46.0" 1009 1009 + source = "registry+https://github.com/rust-lang/crates.io-index" 1010 1010 + checksum = "5573cd9e65281eb049123284fb8e161192f34ff76a7d635945d5fedbb7a59db7" 1011 1011 + dependencies = [ 1012 1012 + "libc", 1013 1013 + "parking_lot", 1014 1014 + "rust-librocksdb-sys", 1015 1015 + ] 1016 1016 + 1017 1017 + [[package]] 1018 1018 + name = "rustc-hash" 1019 1019 + version = "2.1.2" 1020 1020 + source = "registry+https://github.com/rust-lang/crates.io-index" 1021 1021 + checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" 1022 1022 + 1023 1023 + [[package]] 1024 1024 + name = "rustix" 1025 1025 + version = "1.1.4" 1026 1026 + source = "registry+https://github.com/rust-lang/crates.io-index" 1027 1027 + checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" 1028 1028 + dependencies = [ 1029 1029 + "bitflags", 1030 1030 + "errno", 1031 1031 + "libc", 1032 1032 + "linux-raw-sys", 1033 1033 + "windows-sys", 1034 1034 + ] 1035 1035 + 1036 1036 + [[package]] 1037 1037 + name = "scopeguard" 1038 1038 + version = "1.2.0" 1039 1039 + source = "registry+https://github.com/rust-lang/crates.io-index" 1040 1040 + checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 1041 1041 + 1042 1042 + [[package]] 1043 1043 + name = "self_cell" 1044 1044 + version = "1.2.2" 1045 1045 + source = "registry+https://github.com/rust-lang/crates.io-index" 1046 1046 + checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" 1047 1047 + 1048 1048 + [[package]] 1049 1049 + name = "serde" 1050 1050 + version = "1.0.228" 1051 1051 + source = "registry+https://github.com/rust-lang/crates.io-index" 1052 1052 + checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" 1053 1053 + dependencies = [ 1054 1054 + "serde_core", 1055 1055 + "serde_derive", 1056 1056 + ] 1057 1057 + 1058 1058 + [[package]] 1059 1059 + name = "serde_bytes" 1060 1060 + version = "0.11.19" 1061 1061 + source = "registry+https://github.com/rust-lang/crates.io-index" 1062 1062 + checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" 1063 1063 + dependencies = [ 1064 1064 + "serde", 1065 1065 + "serde_core", 1066 1066 + ] 1067 1067 + 1068 1068 + [[package]] 1069 1069 + name = "serde_core" 1070 1070 + version = "1.0.228" 1071 1071 + source = "registry+https://github.com/rust-lang/crates.io-index" 1072 1072 + checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" 1073 1073 + dependencies = [ 1074 1074 + "serde_derive", 1075 1075 + ] 1076 1076 + 1077 1077 + [[package]] 1078 1078 + name = "serde_derive" 1079 1079 + version = "1.0.228" 1080 1080 + source = "registry+https://github.com/rust-lang/crates.io-index" 1081 1081 + checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" 1082 1082 + dependencies = [ 1083 1083 + "proc-macro2", 1084 1084 + "quote", 1085 1085 + "syn", 1086 1086 + ] 1087 1087 + 1088 1088 + [[package]] 1089 1089 + name = "serde_ipld_dagcbor" 1090 1090 + version = "0.6.4" 1091 1091 + source = "registry+https://github.com/rust-lang/crates.io-index" 1092 1092 + checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" 1093 1093 + dependencies = [ 1094 1094 + "cbor4ii", 1095 1095 + "ipld-core", 1096 1096 + "scopeguard", 1097 1097 + "serde", 1098 1098 + ] 1099 1099 + 1100 1100 + [[package]] 1101 1101 + name = "sfa" 1102 1102 + version = "1.0.0" 1103 1103 + source = "registry+https://github.com/rust-lang/crates.io-index" 1104 1104 + checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175" 1105 1105 + dependencies = [ 1106 1106 + "byteorder-lite", 1107 1107 + "log", 1108 1108 + "xxhash-rust", 1109 1109 + ] 1110 1110 + 1111 1111 + [[package]] 1112 1112 + name = "sha2" 1113 1113 + version = "0.10.9" 1114 1114 + source = "registry+https://github.com/rust-lang/crates.io-index" 1115 1115 + checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 1116 1116 + dependencies = [ 1117 1117 + "cfg-if", 1118 1118 + "cpufeatures", 1119 1119 + "digest", 1120 1120 + ] 1121 1121 + 1122 1122 + [[package]] 1123 1123 + name = "sharded-slab" 1124 1124 + version = "0.1.7" 1125 1125 + source = "registry+https://github.com/rust-lang/crates.io-index" 1126 1126 + checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" 1127 1127 + dependencies = [ 1128 1128 + "lazy_static", 1129 1129 + ] 1130 1130 + 1131 1131 + [[package]] 1132 1132 + name = "shlex" 1133 1133 + version = "1.3.0" 1134 1134 + source = "registry+https://github.com/rust-lang/crates.io-index" 1135 1135 + checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 1136 1136 + 1137 1137 + [[package]] 1138 1138 + name = "signal-hook-registry" 1139 1139 + version = "1.4.8" 1140 1140 + source = "registry+https://github.com/rust-lang/crates.io-index" 1141 1141 + checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" 1142 1142 + dependencies = [ 1143 1143 + "errno", 1144 1144 + "libc", 1145 1145 + ] 1146 1146 + 1147 1147 + [[package]] 1148 1148 + name = "slab" 1149 1149 + version = "0.4.12" 1150 1150 + source = "registry+https://github.com/rust-lang/crates.io-index" 1151 1151 + checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" 1152 1152 + 1153 1153 + [[package]] 1154 1154 + name = "smallvec" 1155 1155 + version = "1.15.1" 1156 1156 + source = "registry+https://github.com/rust-lang/crates.io-index" 1157 1157 + checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" 1158 1158 + 1159 1159 + [[package]] 1160 1160 + name = "socket2" 1161 1161 + version = "0.6.3" 1162 1162 + source = "registry+https://github.com/rust-lang/crates.io-index" 1163 1163 + checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" 1164 1164 + dependencies = [ 1165 1165 + "libc", 1166 1166 + "windows-sys", 1167 1167 + ] 1168 1168 + 1169 1169 + [[package]] 1170 1170 + name = "space-efficiency-check" 1171 1171 + version = "0.1.0" 1172 1172 + dependencies = [ 1173 1173 + "async-channel", 1174 1174 + "clap", 1175 1175 + "repo-stream", 1176 1176 + "rust-rocksdb", 1177 1177 + "thiserror 2.0.18", 1178 1178 + "tokio", 1179 1179 + "tracing", 1180 1180 + "tracing-subscriber", 1181 1181 + ] 1182 1182 + 1183 1183 + [[package]] 1184 1184 + name = "spin" 1185 1185 + version = "0.9.8" 1186 1186 + source = "registry+https://github.com/rust-lang/crates.io-index" 1187 1187 + checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" 1188 1188 + dependencies = [ 1189 1189 + "lock_api", 1190 1190 + ] 1191 1191 + 1192 1192 + [[package]] 1193 1193 + name = "strsim" 1194 1194 + version = "0.11.1" 1195 1195 + source = "registry+https://github.com/rust-lang/crates.io-index" 1196 1196 + checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 1197 1197 + 1198 1198 + [[package]] 1199 1199 + name = "syn" 1200 1200 + version = "2.0.117" 1201 1201 + source = "registry+https://github.com/rust-lang/crates.io-index" 1202 1202 + checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" 1203 1203 + dependencies = [ 1204 1204 + "proc-macro2", 1205 1205 + "quote", 1206 1206 + "unicode-ident", 1207 1207 + ] 1208 1208 + 1209 1209 + [[package]] 1210 1210 + name = "tempfile" 1211 1211 + version = "3.27.0" 1212 1212 + source = "registry+https://github.com/rust-lang/crates.io-index" 1213 1213 + checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" 1214 1214 + dependencies = [ 1215 1215 + "fastrand", 1216 1216 + "getrandom", 1217 1217 + "once_cell", 1218 1218 + "rustix", 1219 1219 + "windows-sys", 1220 1220 + ] 1221 1221 + 1222 1222 + [[package]] 1223 1223 + name = "thiserror" 1224 1224 + version = "1.0.69" 1225 1225 + source = "registry+https://github.com/rust-lang/crates.io-index" 1226 1226 + checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" 1227 1227 + dependencies = [ 1228 1228 + "thiserror-impl 1.0.69", 1229 1229 + ] 1230 1230 + 1231 1231 + [[package]] 1232 1232 + name = "thiserror" 1233 1233 + version = "2.0.18" 1234 1234 + source = "registry+https://github.com/rust-lang/crates.io-index" 1235 1235 + checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" 1236 1236 + dependencies = [ 1237 1237 + "thiserror-impl 2.0.18", 1238 1238 + ] 1239 1239 + 1240 1240 + [[package]] 1241 1241 + name = "thiserror-impl" 1242 1242 + version = "1.0.69" 1243 1243 + source = "registry+https://github.com/rust-lang/crates.io-index" 1244 1244 + checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" 1245 1245 + dependencies = [ 1246 1246 + "proc-macro2", 1247 1247 + "quote", 1248 1248 + "syn", 1249 1249 + ] 1250 1250 + 1251 1251 + [[package]] 1252 1252 + name = "thiserror-impl" 1253 1253 + version = "2.0.18" 1254 1254 + source = "registry+https://github.com/rust-lang/crates.io-index" 1255 1255 + checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" 1256 1256 + dependencies = [ 1257 1257 + "proc-macro2", 1258 1258 + "quote", 1259 1259 + "syn", 1260 1260 + ] 1261 1261 + 1262 1262 + [[package]] 1263 1263 + name = "thread_local" 1264 1264 + version = "1.1.9" 1265 1265 + source = "registry+https://github.com/rust-lang/crates.io-index" 1266 1266 + checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" 1267 1267 + dependencies = [ 1268 1268 + "cfg-if", 1269 1269 + ] 1270 1270 + 1271 1271 + [[package]] 1272 1272 + name = "tokio" 1273 1273 + version = "1.51.0" 1274 1274 + source = "registry+https://github.com/rust-lang/crates.io-index" 1275 1275 + checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" 1276 1276 + dependencies = [ 1277 1277 + "bytes", 1278 1278 + "libc", 1279 1279 + "mio", 1280 1280 + "parking_lot", 1281 1281 + "pin-project-lite", 1282 1282 + "signal-hook-registry", 1283 1283 + "socket2", 1284 1284 + "tokio-macros", 1285 1285 + "windows-sys", 1286 1286 + ] 1287 1287 + 1288 1288 + [[package]] 1289 1289 + name = "tokio-macros" 1290 1290 + version = "2.7.0" 1291 1291 + source = "registry+https://github.com/rust-lang/crates.io-index" 1292 1292 + checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" 1293 1293 + dependencies = [ 1294 1294 + "proc-macro2", 1295 1295 + "quote", 1296 1296 + "syn", 1297 1297 + ] 1298 1298 + 1299 1299 + [[package]] 1300 1300 + name = "tracing" 1301 1301 + version = "0.1.44" 1302 1302 + source = "registry+https://github.com/rust-lang/crates.io-index" 1303 1303 + checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" 1304 1304 + dependencies = [ 1305 1305 + "pin-project-lite", 1306 1306 + "tracing-attributes", 1307 1307 + "tracing-core", 1308 1308 + ] 1309 1309 + 1310 1310 + [[package]] 1311 1311 + name = "tracing-attributes" 1312 1312 + version = "0.1.31" 1313 1313 + source = "registry+https://github.com/rust-lang/crates.io-index" 1314 1314 + checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" 1315 1315 + dependencies = [ 1316 1316 + "proc-macro2", 1317 1317 + "quote", 1318 1318 + "syn", 1319 1319 + ] 1320 1320 + 1321 1321 + [[package]] 1322 1322 + name = "tracing-core" 1323 1323 + version = "0.1.36" 1324 1324 + source = "registry+https://github.com/rust-lang/crates.io-index" 1325 1325 + checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" 1326 1326 + dependencies = [ 1327 1327 + "once_cell", 1328 1328 + "valuable", 1329 1329 + ] 1330 1330 + 1331 1331 + [[package]] 1332 1332 + name = "tracing-log" 1333 1333 + version = "0.2.0" 1334 1334 + source = "registry+https://github.com/rust-lang/crates.io-index" 1335 1335 + checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" 1336 1336 + dependencies = [ 1337 1337 + "log", 1338 1338 + "once_cell", 1339 1339 + "tracing-core", 1340 1340 + ] 1341 1341 + 1342 1342 + [[package]] 1343 1343 + name = "tracing-subscriber" 1344 1344 + version = "0.3.23" 1345 1345 + source = "registry+https://github.com/rust-lang/crates.io-index" 1346 1346 + checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" 1347 1347 + dependencies = [ 1348 1348 + "matchers", 1349 1349 + "nu-ansi-term", 1350 1350 + "once_cell", 1351 1351 + "regex-automata", 1352 1352 + "sharded-slab", 1353 1353 + "smallvec", 1354 1354 + "thread_local", 1355 1355 + "tracing", 1356 1356 + "tracing-core", 1357 1357 + "tracing-log", 1358 1358 + ] 1359 1359 + 1360 1360 + [[package]] 1361 1361 + name = "typenum" 1362 1362 + version = "1.19.0" 1363 1363 + source = "registry+https://github.com/rust-lang/crates.io-index" 1364 1364 + checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" 1365 1365 + 1366 1366 + [[package]] 1367 1367 + name = "unicode-ident" 1368 1368 + version = "1.0.24" 1369 1369 + source = "registry+https://github.com/rust-lang/crates.io-index" 1370 1370 + checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" 1371 1371 + 1372 1372 + [[package]] 1373 1373 + name = "unsigned-varint" 1374 1374 + version = "0.7.2" 1375 1375 + source = "registry+https://github.com/rust-lang/crates.io-index" 1376 1376 + checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" 1377 1377 + 1378 1378 + [[package]] 1379 1379 + name = "unsigned-varint" 1380 1380 + version = "0.8.0" 1381 1381 + source = "registry+https://github.com/rust-lang/crates.io-index" 1382 1382 + checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" 1383 1383 + 1384 1384 + [[package]] 1385 1385 + name = "utf8parse" 1386 1386 + version = "0.2.2" 1387 1387 + source = "registry+https://github.com/rust-lang/crates.io-index" 1388 1388 + checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 1389 1389 + 1390 1390 + [[package]] 1391 1391 + name = "valuable" 1392 1392 + version = "0.1.1" 1393 1393 + source = "registry+https://github.com/rust-lang/crates.io-index" 1394 1394 + checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" 1395 1395 + 1396 1396 + [[package]] 1397 1397 + name = "varint-rs" 1398 1398 + version = "2.2.0" 1399 1399 + source = "registry+https://github.com/rust-lang/crates.io-index" 1400 1400 + checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" 1401 1401 + 1402 1402 + [[package]] 1403 1403 + name = "vcpkg" 1404 1404 + version = "0.2.15" 1405 1405 + source = "registry+https://github.com/rust-lang/crates.io-index" 1406 1406 + checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1407 1407 + 1408 1408 + [[package]] 1409 1409 + name = "version_check" 1410 1410 + version = "0.9.5" 1411 1411 + source = "registry+https://github.com/rust-lang/crates.io-index" 1412 1412 + checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" 1413 1413 + 1414 1414 + [[package]] 1415 1415 + name = "wasi" 1416 1416 + version = "0.11.1+wasi-snapshot-preview1" 1417 1417 + source = "registry+https://github.com/rust-lang/crates.io-index" 1418 1418 + checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" 1419 1419 + 1420 1420 + [[package]] 1421 1421 + name = "wasip2" 1422 1422 + version = "1.0.2+wasi-0.2.9" 1423 1423 + source = "registry+https://github.com/rust-lang/crates.io-index" 1424 1424 + checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" 1425 1425 + dependencies = [ 1426 1426 + "wit-bindgen", 1427 1427 + ] 1428 1428 + 1429 1429 + [[package]] 1430 1430 + name = "windows-link" 1431 1431 + version = "0.2.1" 1432 1432 + source = "registry+https://github.com/rust-lang/crates.io-index" 1433 1433 + checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" 1434 1434 + 1435 1435 + [[package]] 1436 1436 + name = "windows-sys" 1437 1437 + version = "0.61.2" 1438 1438 + source = "registry+https://github.com/rust-lang/crates.io-index" 1439 1439 + checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" 1440 1440 + dependencies = [ 1441 1441 + "windows-link", 1442 1442 + ] 1443 1443 + 1444 1444 + [[package]] 1445 1445 + name = "wit-bindgen" 1446 1446 + version = "0.51.0" 1447 1447 + source = "registry+https://github.com/rust-lang/crates.io-index" 1448 1448 + checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" 1449 1449 + 1450 1450 + [[package]] 1451 1451 + name = "xxhash-rust" 1452 1452 + version = "0.8.15" 1453 1453 + source = "registry+https://github.com/rust-lang/crates.io-index" 1454 1454 + checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" 1455 1455 + 1456 1456 + [[package]] 1457 1457 + name = "zstd-sys" 1458 1458 + version = "2.0.16+zstd.1.5.7" 1459 1459 + source = "registry+https://github.com/rust-lang/crates.io-index" 1460 1460 + checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" 1461 1461 + dependencies = [ 1462 1462 + "bindgen", 1463 1463 + "cc", 1464 1464 + "pkg-config", 1465 1465 + ]

+15

Cargo.toml

reviewed

··· 1 1 + [workspace] 2 2 + resolver = "2" 3 3 + members = [ 4 4 + "space-efficiency-check" 5 5 + ] 6 6 + 7 7 + [workspace.dependencies] 8 8 + async-channel = "2.5.0" 9 9 + clap = { version = "4.6.0", features = ["env", "derive"] } 10 10 + repo-stream = "0.5.0-alpha.5" 11 11 + rust-rocksdb = "0.46.0" 12 12 + thiserror = "2.0.18" 13 13 + tokio = "1.51.0" 14 14 + tracing = "0.1.44" 15 15 + tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }

+14

space-efficiency-check/Cargo.toml

reviewed

··· 1 1 + [package] 2 2 + name = "space-efficiency-check" 3 3 + version = "0.1.0" 4 4 + edition = "2024" 5 5 + 6 6 + [dependencies] 7 7 + async-channel = { workspace = true } 8 8 + clap = { workspace = true } 9 9 + repo-stream = { workspace = true } 10 10 + rust-rocksdb = { workspace = true } 11 11 + thiserror = { workspace = true } 12 12 + tokio = { workspace = true, features = ["full"] } 13 13 + tracing = { workspace = true } 14 14 + tracing-subscriber = { workspace = true }

+1

space-efficiency-check/src/lib.rs

reviewed

··· 1 1 + pub mod work;

+101

space-efficiency-check/src/main.rs

reviewed

··· 1 1 + use clap::Parser; 2 2 + use rust_rocksdb::{DB, Options}; 3 3 + 4 4 + use std::path::{Path, PathBuf}; 5 5 + use std::sync::{Arc, atomic::Ordering}; 6 6 + 7 7 + use space_efficiency_check::work::{ProcessError, run_workers}; 8 8 + 9 9 + /// import a directory of car files into rocks to check space efficiency 10 10 + #[derive(Parser)] 11 11 + struct Args { 12 12 + /// directory of car files 13 13 + car_dir: PathBuf, 14 14 + 15 15 + /// where to kick rocks 16 16 + #[arg(long, default_value = "./local/space-efficiency-check-db")] 17 17 + db_dir: PathBuf, 18 18 + 19 19 + /// concurrent car workers 20 20 + #[arg(long, default_value_t = 10)] 21 21 + workers: usize, 22 22 + 23 23 + /// repo-stream memory limit: we don't disk spill; set higher than largest car 24 24 + #[arg(long, default_value_t = 1024)] 25 25 + mem_limit_mb: usize, 26 26 + } 27 27 + 28 28 + fn open_db(path: &Path) -> Result<DB, rust_rocksdb::Error> { 29 29 + let mut opts = Options::default(); 30 30 + opts.create_if_missing(true); 31 31 + 32 32 + // we don't need durability 33 33 + opts.set_manual_wal_flush(true); 34 34 + 35 35 + // best compression option 36 36 + opts.set_compression_type(rust_rocksdb::DBCompressionType::Zstd); 37 37 + 38 38 + // larger write buffer for less frequent flushes 39 39 + opts.set_write_buffer_size(64 * 1024 * 1024); 40 40 + 41 41 + DB::open(&opts, path) 42 42 + } 43 43 + 44 44 + #[tokio::main] 45 45 + async fn main() -> Result<(), ProcessError> { 46 46 + tracing_subscriber::fmt::init(); 47 47 + 48 48 + let args = Args::parse(); 49 49 + 50 50 + let db = Arc::new(open_db(&args.db_dir)?); 51 51 + 52 52 + let stats = run_workers(&args.car_dir, db.clone(), args.workers, args.mem_limit_mb).await?; 53 53 + 54 54 + let repos = stats.repos.load(Ordering::Relaxed); 55 55 + let empty = stats.empty_repos.load(Ordering::Relaxed); 56 56 + let records = stats.records.load(Ordering::Relaxed); 57 57 + let failed = stats.failed_repos.load(Ordering::Relaxed); 58 58 + let car_bytes = stats.car_bytes.load(Ordering::Relaxed); 59 59 + 60 60 + tracing::info!(repos, empty, records, failed, "import complete"); 61 61 + tracing::info!( 62 62 + car_bytes, 63 63 + car_mib = car_bytes / 2_u64.pow(20), 64 64 + "total input size" 65 65 + ); 66 66 + 67 67 + let size_before = db_size(&db); 68 68 + tracing::info!( 69 69 + bytes = size_before, 70 70 + mib = size_before / 2_u64.pow(20), 71 71 + "db size before compaction" 72 72 + ); 73 73 + 74 74 + tracing::info!("running manual compaction..."); 75 75 + let compact_start = std::time::Instant::now(); 76 76 + db.compact_range(None::<&[u8]>, None::<&[u8]>); 77 77 + let compact_elapsed = compact_start.elapsed(); 78 78 + 79 79 + let size_after = db_size(&db); 80 80 + tracing::info!( 81 81 + bytes = size_after, 82 82 + mib = size_after / (1024 * 1024), 83 83 + ?compact_elapsed, 84 84 + "db size after compaction", 85 85 + ); 86 86 + 87 87 + if car_bytes > 0 { 88 88 + let ratio = size_after as f64 / car_bytes as f64; 89 89 + tracing::info!(ratio = format!("{ratio:.3}"), "db size / car bytes"); 90 90 + } 91 91 + 92 92 + Ok(()) 93 93 + } 94 94 + 95 95 + fn db_size(db: &DB) -> u64 { 96 96 + db.property_value("rocksdb.total-sst-files-size") 97 97 + .ok() 98 98 + .flatten() 99 99 + .and_then(|v| v.parse::<u64>().ok()) 100 100 + .unwrap_or(0) 101 101 + }

+155

space-efficiency-check/src/work.rs

reviewed

··· 1 1 + use std::path::{Path, PathBuf}; 2 2 + use std::sync::{ 3 3 + Arc, 4 4 + atomic::{AtomicU64, Ordering}, 5 5 + }; 6 6 + use std::time::Instant; 7 7 + 8 8 + use repo_stream::DriverBuilder; 9 9 + use rust_rocksdb::{DB, WriteBatch}; 10 10 + use tokio::fs; 11 11 + use tokio::io::BufReader; 12 12 + use tokio::task::JoinSet; 13 13 + use tracing::warn; 14 14 + 15 15 + #[derive(Debug, thiserror::Error)] 16 16 + pub enum ProcessError { 17 17 + #[error("io: {0}")] 18 18 + Io(#[from] std::io::Error), 19 19 + 20 20 + #[error("load_car: {0}")] 21 21 + Load(String), 22 22 + 23 23 + #[error("walk: {0}")] 24 24 + Walk(#[from] repo_stream::WalkError), 25 25 + 26 26 + #[error("rocksdb: {0}")] 27 27 + Rocks(#[from] rust_rocksdb::Error), 28 28 + 29 29 + #[error("channel send: {0}")] 30 30 + SendError(String), 31 31 + } 32 32 + 33 33 + /// Stats collected across all workers. 34 34 + #[derive(Debug, Default)] 35 35 + pub struct Stats { 36 36 + pub repos: AtomicU64, 37 37 + pub empty_repos: AtomicU64, 38 38 + pub records: AtomicU64, 39 39 + pub failed_repos: AtomicU64, 40 40 + pub car_bytes: AtomicU64, 41 41 + } 42 42 + 43 43 + pub async fn run_workers( 44 44 + car_dir: &Path, 45 45 + db: Arc<DB>, 46 46 + workers: usize, 47 47 + mem_limit_mb: usize, 48 48 + ) -> Result<Stats, ProcessError> { 49 49 + let stats = Arc::new(Stats::default()); 50 50 + let (tx, rx) = async_channel::bounded(workers * 2); 51 51 + let start = Instant::now(); 52 52 + 53 53 + let mut set = JoinSet::new(); 54 54 + for _ in 0..workers { 55 55 + let rx = rx.clone(); 56 56 + let db = db.clone(); 57 57 + let stats = stats.clone(); 58 58 + set.spawn(worker(rx, db, stats, mem_limit_mb)); 59 59 + } 60 60 + 61 61 + let mut file_count = 0; 62 62 + let mut entries = fs::read_dir(car_dir).await?; 63 63 + while let Some(entry) = entries.next_entry().await? { 64 64 + let meta = entry.metadata().await?; 65 65 + if !meta.is_file() { 66 66 + continue; 67 67 + } 68 68 + stats.car_bytes.fetch_add(meta.len(), Ordering::Relaxed); 69 69 + file_count += 1; 70 70 + tx.send(entry.path()) 71 71 + .await 72 72 + .map_err(|e| ProcessError::SendError(e.to_string()))? 73 73 + } 74 74 + drop(tx); 75 75 + 76 76 + tracing::info!( 77 77 + files = file_count, 78 78 + car_bytes = stats.car_bytes.load(Ordering::Relaxed), 79 79 + car_mib = stats.car_bytes.load(Ordering::Relaxed) / 2_u64.pow(20), 80 80 + "enumerated car directory", 81 81 + ); 82 82 + 83 83 + while let Some(result) = set.join_next().await { 84 84 + result.expect("tasks not to panic (joinerror)"); 85 85 + } 86 86 + 87 87 + let elapsed = start.elapsed(); 88 88 + tracing::info!(?elapsed, "workers finished"); 89 89 + 90 90 + Ok(Arc::try_unwrap(stats).expect("all workers to have finished")) 91 91 + } 92 92 + 93 93 + async fn worker( 94 94 + rx: async_channel::Receiver<PathBuf>, 95 95 + db: Arc<DB>, 96 96 + stats: Arc<Stats>, 97 97 + mem_limit_mb: usize, 98 98 + ) { 99 99 + let driver = DriverBuilder::new().with_mem_limit_mb(mem_limit_mb); 100 100 + 101 101 + while let Ok(path) = rx.recv().await { 102 102 + if let Err(e) = process_car(&path, &driver, &db, &stats).await { 103 103 + warn!(?path, %e, "failed to process CAR"); 104 104 + stats.failed_repos.fetch_add(1, Ordering::Relaxed); 105 105 + } 106 106 + } 107 107 + } 108 108 + 109 109 + async fn process_car( 110 110 + path: &Path, 111 111 + driver: &DriverBuilder, 112 112 + db: &DB, 113 113 + stats: &Stats, 114 114 + ) -> Result<(), ProcessError> { 115 115 + let file = tokio::fs::File::open(path).await?; 116 116 + let reader = BufReader::new(file); 117 117 + 118 118 + let mut car = driver 119 119 + .load_car(reader) 120 120 + .await 121 121 + .map_err(|e| ProcessError::Load(e.to_string()))?; 122 122 + 123 123 + let did = car.commit.did.clone(); 124 124 + let mut record_count: u64 = 0; 125 125 + 126 126 + const CHUNK_SIZE: usize = 8192; 127 127 + 128 128 + while let Some(chunk) = car.next_chunk_strict(CHUNK_SIZE)? { 129 129 + record_count += chunk.len() as u64; 130 130 + 131 131 + let mut batch = WriteBatch::default(); 132 132 + for output in &chunk { 133 133 + let key = format!("{did}/{}", output.key); 134 134 + batch.put(key.as_bytes(), &output.data); 135 135 + } 136 136 + db.write(&batch)?; 137 137 + } 138 138 + 139 139 + if record_count == 0 { 140 140 + stats.empty_repos.fetch_add(1, Ordering::Relaxed); 141 141 + } 142 142 + stats.records.fetch_add(record_count, Ordering::Relaxed); 143 143 + 144 144 + let completed = stats.repos.fetch_add(1, Ordering::Relaxed) + 1; 145 145 + if completed.is_multiple_of(10_000) { 146 146 + tracing::info!( 147 147 + repos = completed, 148 148 + records = stats.records.load(Ordering::Relaxed), 149 149 + failed = stats.failed_repos.load(Ordering::Relaxed), 150 150 + "progress", 151 151 + ); 152 152 + } 153 153 + 154 154 + Ok(()) 155 155 + }