Constellation: restore direct from object storage #6

+1 -1

.github/workflows/checks.yml

··· 28 28 - name: get nightly toolchain for jetstream fmt 29 29 run: rustup toolchain install nightly --allow-downgrade -c rustfmt 30 30 - name: fmt 31 - run: cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot --package pocket -- --check 31 + run: cargo fmt --package constellation --package links --package pocket --package quasar --package slingshot --package spacedust --package ufos -- --check 32 32 - name: fmt jetstream (nightly) 33 33 run: cargo +nightly fmt --package jetstream -- --check 34 34 - name: clippy

+355 -14

Cargo.lock

··· 778 778 source = "registry+https://github.com/rust-lang/crates.io-index" 779 779 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" 780 780 781 + [[package]] 782 + name = "chacha20" 783 + version = "0.10.0" 784 + source = "registry+https://github.com/rust-lang/crates.io-index" 785 + checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" 786 + dependencies = [ 787 + "cfg-if", 788 + "cpufeatures 0.3.0", 789 + "rand_core 0.10.0", 790 + ] 791 + 781 792 [[package]] 782 793 name = "chrono" 783 794 version = "0.4.41" ··· 953 964 "bincode 1.3.3", 954 965 "clap", 955 966 "ctrlc", 967 + "eat-rocks", 956 968 "flume", 957 969 "fs4 0.12.0", 958 970 "headers-accept", ··· 972 984 "tokio", 973 985 "tokio-util", 974 986 "tower-http", 987 + "tracing", 988 + "tracing-subscriber", 975 989 "tungstenite 0.26.2", 976 990 "zstd", 977 991 ] ··· 1031 1045 "libc", 1032 1046 ] 1033 1047 1048 + [[package]] 1049 + name = "cpufeatures" 1050 + version = "0.3.0" 1051 + source = "registry+https://github.com/rust-lang/crates.io-index" 1052 + checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" 1053 + dependencies = [ 1054 + "libc", 1055 + ] 1056 + 1057 + [[package]] 1058 + name = "crc32c" 1059 + version = "0.6.8" 1060 + source = "registry+https://github.com/rust-lang/crates.io-index" 1061 + checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" 1062 + dependencies = [ 1063 + "rustc_version", 1064 + ] 1065 + 1034 1066 [[package]] 1035 1067 name = "crc32fast" 1036 1068 version = "1.4.2" ··· 1381 1413 source = "registry+https://github.com/rust-lang/crates.io-index" 1382 1414 checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" 1383 1415 1416 + [[package]] 1417 + name = "eat-rocks" 1418 + version = "0.1.1" 1419 + source = "registry+https://github.com/rust-lang/crates.io-index" 1420 + checksum = "19a562da0f74abda90eb1cc96f7adf23a33ce1c71a8d714ae93afa27d33a7a31" 1421 + dependencies = [ 1422 + "crc32c", 1423 + "futures", 1424 + "object_store", 1425 + "thiserror 2.0.16", 1426 + "tokio", 1427 + "tracing", 1428 + ] 1429 + 1384 1430 [[package]] 1385 1431 name = "ecdsa" 1386 1432 version = "0.16.9" ··· 1930 1976 "cfg-if", 1931 1977 "js-sys", 1932 1978 "libc", 1933 - "r-efi", 1979 + "r-efi 5.2.0", 1934 1980 "wasi 0.14.2+wasi-0.2.4", 1935 1981 "wasm-bindgen", 1936 1982 ] 1937 1983 1984 + [[package]] 1985 + name = "getrandom" 1986 + version = "0.4.2" 1987 + source = "registry+https://github.com/rust-lang/crates.io-index" 1988 + checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" 1989 + dependencies = [ 1990 + "cfg-if", 1991 + "libc", 1992 + "r-efi 6.0.0", 1993 + "rand_core 0.10.0", 1994 + "wasip2", 1995 + "wasip3", 1996 + ] 1997 + 1938 1998 [[package]] 1939 1999 name = "gimli" 1940 2000 version = "0.31.1" ··· 2239 2299 "libm", 2240 2300 ] 2241 2301 2302 + [[package]] 2303 + name = "humantime" 2304 + version = "2.3.0" 2305 + source = "registry+https://github.com/rust-lang/crates.io-index" 2306 + checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" 2307 + 2242 2308 [[package]] 2243 2309 name = "hyper" 2244 2310 version = "1.6.0" ··· 2461 2527 "syn", 2462 2528 ] 2463 2529 2530 + [[package]] 2531 + name = "id-arena" 2532 + version = "2.3.0" 2533 + source = "registry+https://github.com/rust-lang/crates.io-index" 2534 + checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" 2535 + 2464 2536 [[package]] 2465 2537 name = "ident_case" 2466 2538 version = "1.0.1" ··· 2764 2836 source = "registry+https://github.com/rust-lang/crates.io-index" 2765 2837 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 2766 2838 2839 + [[package]] 2840 + name = "leb128fmt" 2841 + version = "0.1.0" 2842 + source = "registry+https://github.com/rust-lang/crates.io-index" 2843 + checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" 2844 + 2767 2845 [[package]] 2768 2846 name = "libc" 2769 2847 version = "0.2.174" ··· 2787 2865 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" 2788 2866 dependencies = [ 2789 2867 "cfg-if", 2790 - "windows-targets 0.48.5", 2868 + "windows-targets 0.52.6", 2791 2869 ] 2792 2870 2793 2871 [[package]] ··· 3005 3083 source = "registry+https://github.com/rust-lang/crates.io-index" 3006 3084 checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" 3007 3085 3086 + [[package]] 3087 + name = "md-5" 3088 + version = "0.10.6" 3089 + source = "registry+https://github.com/rust-lang/crates.io-index" 3090 + checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" 3091 + dependencies = [ 3092 + "cfg-if", 3093 + "digest", 3094 + ] 3095 + 3008 3096 [[package]] 3009 3097 name = "mea" 3010 3098 version = "0.6.3" ··· 3368 3456 "memchr", 3369 3457 ] 3370 3458 3459 + [[package]] 3460 + name = "object_store" 3461 + version = "0.13.2" 3462 + source = "registry+https://github.com/rust-lang/crates.io-index" 3463 + checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" 3464 + dependencies = [ 3465 + "async-trait", 3466 + "base64 0.22.1", 3467 + "bytes", 3468 + "chrono", 3469 + "form_urlencoded", 3470 + "futures-channel", 3471 + "futures-core", 3472 + "futures-util", 3473 + "http", 3474 + "http-body-util", 3475 + "humantime", 3476 + "hyper", 3477 + "itertools 0.14.0", 3478 + "md-5", 3479 + "parking_lot", 3480 + "percent-encoding", 3481 + "quick-xml 0.39.2", 3482 + "rand 0.10.1", 3483 + "reqwest", 3484 + "ring", 3485 + "serde", 3486 + "serde_json", 3487 + "serde_urlencoded", 3488 + "thiserror 2.0.16", 3489 + "tokio", 3490 + "tracing", 3491 + "url", 3492 + "walkdir", 3493 + "wasm-bindgen-futures", 3494 + "web-time", 3495 + ] 3496 + 3371 3497 [[package]] 3372 3498 name = "oid-registry" 3373 3499 version = "0.8.1" ··· 3624 3750 "percent-encoding", 3625 3751 "pin-project-lite", 3626 3752 "poem-derive", 3627 - "quick-xml", 3753 + "quick-xml 0.36.2", 3628 3754 "rcgen", 3629 3755 "regex", 3630 3756 "reqwest", ··· 3676 3802 "num-traits", 3677 3803 "poem", 3678 3804 "poem-openapi-derive", 3679 - "quick-xml", 3805 + "quick-xml 0.36.2", 3680 3806 "regex", 3681 3807 "serde", 3682 3808 "serde_json", ··· 3826 3952 "serde", 3827 3953 ] 3828 3954 3955 + [[package]] 3956 + name = "quick-xml" 3957 + version = "0.39.2" 3958 + source = "registry+https://github.com/rust-lang/crates.io-index" 3959 + checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" 3960 + dependencies = [ 3961 + "memchr", 3962 + "serde", 3963 + ] 3964 + 3829 3965 [[package]] 3830 3966 name = "quick_cache" 3831 3967 version = "0.6.12" ··· 3906 4042 source = "registry+https://github.com/rust-lang/crates.io-index" 3907 4043 checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" 3908 4044 4045 + [[package]] 4046 + name = "r-efi" 4047 + version = "6.0.0" 4048 + source = "registry+https://github.com/rust-lang/crates.io-index" 4049 + checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" 4050 + 3909 4051 [[package]] 3910 4052 name = "rand" 3911 4053 version = "0.8.5" ··· 3927 4069 "rand_core 0.9.3", 3928 4070 ] 3929 4071 4072 + [[package]] 4073 + name = "rand" 4074 + version = "0.10.1" 4075 + source = "registry+https://github.com/rust-lang/crates.io-index" 4076 + checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" 4077 + dependencies = [ 4078 + "chacha20", 4079 + "getrandom 0.4.2", 4080 + "rand_core 0.10.0", 4081 + ] 4082 + 3930 4083 [[package]] 3931 4084 name = "rand_chacha" 3932 4085 version = "0.3.1" ··· 3965 4118 "getrandom 0.3.3", 3966 4119 ] 3967 4120 4121 + [[package]] 4122 + name = "rand_core" 4123 + version = "0.10.0" 4124 + source = "registry+https://github.com/rust-lang/crates.io-index" 4125 + checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" 4126 + 3968 4127 [[package]] 3969 4128 name = "rand_xoshiro" 3970 4129 version = "0.6.0" ··· 4128 4287 "url", 4129 4288 "wasm-bindgen", 4130 4289 "wasm-bindgen-futures", 4290 + "wasm-streams", 4131 4291 "web-sys", 4132 4292 ] 4133 4293 ··· 4361 4521 source = "registry+https://github.com/rust-lang/crates.io-index" 4362 4522 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" 4363 4523 4524 + [[package]] 4525 + name = "same-file" 4526 + version = "1.0.6" 4527 + source = "registry+https://github.com/rust-lang/crates.io-index" 4528 + checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 4529 + dependencies = [ 4530 + "winapi-util", 4531 + ] 4532 + 4364 4533 [[package]] 4365 4534 name = "schannel" 4366 4535 version = "0.1.27" ··· 4672 4841 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" 4673 4842 dependencies = [ 4674 4843 "cfg-if", 4675 - "cpufeatures", 4844 + "cpufeatures 0.2.17", 4676 4845 "digest", 4677 4846 ] 4678 4847 ··· 4683 4852 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 4684 4853 dependencies = [ 4685 4854 "cfg-if", 4686 - "cpufeatures", 4855 + "cpufeatures 0.2.17", 4687 4856 "digest", 4688 4857 ] 4689 4858 ··· 5361 5530 5362 5531 [[package]] 5363 5532 name = "tracing" 5364 - version = "0.1.41" 5533 + version = "0.1.44" 5365 5534 source = "registry+https://github.com/rust-lang/crates.io-index" 5366 - checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" 5535 + checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" 5367 5536 dependencies = [ 5368 5537 "log", 5369 5538 "pin-project-lite", ··· 5373 5542 5374 5543 [[package]] 5375 5544 name = "tracing-attributes" 5376 - version = "0.1.30" 5545 + version = "0.1.31" 5377 5546 source = "registry+https://github.com/rust-lang/crates.io-index" 5378 - checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" 5547 + checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" 5379 5548 dependencies = [ 5380 5549 "proc-macro2", 5381 5550 "quote", ··· 5384 5553 5385 5554 [[package]] 5386 5555 name = "tracing-core" 5387 - version = "0.1.33" 5556 + version = "0.1.36" 5388 5557 source = "registry+https://github.com/rust-lang/crates.io-index" 5389 - checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" 5558 + checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" 5390 5559 dependencies = [ 5391 5560 "once_cell", 5392 5561 "valuable", ··· 5405 5574 5406 5575 [[package]] 5407 5576 name = "tracing-subscriber" 5408 - version = "0.3.20" 5577 + version = "0.3.23" 5409 5578 source = "registry+https://github.com/rust-lang/crates.io-index" 5410 - checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" 5579 + checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" 5411 5580 dependencies = [ 5412 5581 "matchers", 5413 5582 "nu-ansi-term", ··· 5689 5858 "atomic-waker", 5690 5859 ] 5691 5860 5861 + [[package]] 5862 + name = "walkdir" 5863 + version = "2.5.0" 5864 + source = "registry+https://github.com/rust-lang/crates.io-index" 5865 + checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 5866 + dependencies = [ 5867 + "same-file", 5868 + "winapi-util", 5869 + ] 5870 + 5692 5871 [[package]] 5693 5872 name = "want" 5694 5873 version = "0.3.1" ··· 5713 5892 "wit-bindgen-rt", 5714 5893 ] 5715 5894 5895 + [[package]] 5896 + name = "wasip2" 5897 + version = "1.0.2+wasi-0.2.9" 5898 + source = "registry+https://github.com/rust-lang/crates.io-index" 5899 + checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" 5900 + dependencies = [ 5901 + "wit-bindgen", 5902 + ] 5903 + 5904 + [[package]] 5905 + name = "wasip3" 5906 + version = "0.4.0+wasi-0.3.0-rc-2026-01-06" 5907 + source = "registry+https://github.com/rust-lang/crates.io-index" 5908 + checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" 5909 + dependencies = [ 5910 + "wit-bindgen", 5911 + ] 5912 + 5716 5913 [[package]] 5717 5914 name = "wasm-bindgen" 5718 5915 version = "0.2.100" ··· 5784 5981 "unicode-ident", 5785 5982 ] 5786 5983 5984 + [[package]] 5985 + name = "wasm-encoder" 5986 + version = "0.244.0" 5987 + source = "registry+https://github.com/rust-lang/crates.io-index" 5988 + checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" 5989 + dependencies = [ 5990 + "leb128fmt", 5991 + "wasmparser", 5992 + ] 5993 + 5994 + [[package]] 5995 + name = "wasm-metadata" 5996 + version = "0.244.0" 5997 + source = "registry+https://github.com/rust-lang/crates.io-index" 5998 + checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" 5999 + dependencies = [ 6000 + "anyhow", 6001 + "indexmap 2.11.4", 6002 + "wasm-encoder", 6003 + "wasmparser", 6004 + ] 6005 + 6006 + [[package]] 6007 + name = "wasm-streams" 6008 + version = "0.4.2" 6009 + source = "registry+https://github.com/rust-lang/crates.io-index" 6010 + checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" 6011 + dependencies = [ 6012 + "futures-util", 6013 + "js-sys", 6014 + "wasm-bindgen", 6015 + "wasm-bindgen-futures", 6016 + "web-sys", 6017 + ] 6018 + 6019 + [[package]] 6020 + name = "wasmparser" 6021 + version = "0.244.0" 6022 + source = "registry+https://github.com/rust-lang/crates.io-index" 6023 + checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" 6024 + dependencies = [ 6025 + "bitflags", 6026 + "hashbrown 0.15.2", 6027 + "indexmap 2.11.4", 6028 + "semver", 6029 + ] 6030 + 5787 6031 [[package]] 5788 6032 name = "web-sys" 5789 6033 version = "0.3.77" ··· 5844 6088 source = "registry+https://github.com/rust-lang/crates.io-index" 5845 6089 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 5846 6090 6091 + [[package]] 6092 + name = "winapi-util" 6093 + version = "0.1.11" 6094 + source = "registry+https://github.com/rust-lang/crates.io-index" 6095 + checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 6096 + dependencies = [ 6097 + "windows-sys 0.59.0", 6098 + ] 6099 + 5847 6100 [[package]] 5848 6101 name = "winapi-x86_64-pc-windows-gnu" 5849 6102 version = "0.4.0" ··· 6116 6369 "windows-sys 0.48.0", 6117 6370 ] 6118 6371 6372 + [[package]] 6373 + name = "wit-bindgen" 6374 + version = "0.51.0" 6375 + source = "registry+https://github.com/rust-lang/crates.io-index" 6376 + checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" 6377 + dependencies = [ 6378 + "wit-bindgen-rust-macro", 6379 + ] 6380 + 6381 + [[package]] 6382 + name = "wit-bindgen-core" 6383 + version = "0.51.0" 6384 + source = "registry+https://github.com/rust-lang/crates.io-index" 6385 + checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" 6386 + dependencies = [ 6387 + "anyhow", 6388 + "heck", 6389 + "wit-parser", 6390 + ] 6391 + 6119 6392 [[package]] 6120 6393 name = "wit-bindgen-rt" 6121 6394 version = "0.39.0" ··· 6125 6398 "bitflags", 6126 6399 ] 6127 6400 6401 + [[package]] 6402 + name = "wit-bindgen-rust" 6403 + version = "0.51.0" 6404 + source = "registry+https://github.com/rust-lang/crates.io-index" 6405 + checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" 6406 + dependencies = [ 6407 + "anyhow", 6408 + "heck", 6409 + "indexmap 2.11.4", 6410 + "prettyplease", 6411 + "syn", 6412 + "wasm-metadata", 6413 + "wit-bindgen-core", 6414 + "wit-component", 6415 + ] 6416 + 6417 + [[package]] 6418 + name = "wit-bindgen-rust-macro" 6419 + version = "0.51.0" 6420 + source = "registry+https://github.com/rust-lang/crates.io-index" 6421 + checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" 6422 + dependencies = [ 6423 + "anyhow", 6424 + "prettyplease", 6425 + "proc-macro2", 6426 + "quote", 6427 + "syn", 6428 + "wit-bindgen-core", 6429 + "wit-bindgen-rust", 6430 + ] 6431 + 6432 + [[package]] 6433 + name = "wit-component" 6434 + version = "0.244.0" 6435 + source = "registry+https://github.com/rust-lang/crates.io-index" 6436 + checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" 6437 + dependencies = [ 6438 + "anyhow", 6439 + "bitflags", 6440 + "indexmap 2.11.4", 6441 + "log", 6442 + "serde", 6443 + "serde_derive", 6444 + "serde_json", 6445 + "wasm-encoder", 6446 + "wasm-metadata", 6447 + "wasmparser", 6448 + "wit-parser", 6449 + ] 6450 + 6451 + [[package]] 6452 + name = "wit-parser" 6453 + version = "0.244.0" 6454 + source = "registry+https://github.com/rust-lang/crates.io-index" 6455 + checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" 6456 + dependencies = [ 6457 + "anyhow", 6458 + "id-arena", 6459 + "indexmap 2.11.4", 6460 + "log", 6461 + "semver", 6462 + "serde", 6463 + "serde_derive", 6464 + "serde_json", 6465 + "unicode-xid", 6466 + "wasmparser", 6467 + ] 6468 + 6128 6469 [[package]] 6129 6470 name = "write16" 6130 6471 version = "1.0.0"

+2

Cargo.toml

··· 14 14 15 15 [workspace.dependencies] 16 16 clap = { version = "4.5.56", features = ["derive", "env"] } 17 + tracing = "0.1.44" 18 + tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }

+4 -1

constellation/Cargo.toml

··· 13 13 bincode = "1.3.3" 14 14 clap = { workspace = true } 15 15 ctrlc = "3.4.5" 16 + eat-rocks = { version = "0.1.1", optional = true, features = ["easy"] } 16 17 flume = { version = "0.11.1", default-features = false } 17 18 fs4 = { version = "0.12.0", features = ["sync"] } 18 19 headers-accept = "0.1.4" ··· 30 31 tinyjson = "2.5.1" 31 32 tokio-util = "0.7.13" 32 33 tower-http = { version = "0.6.2", features = ["cors"] } 34 + tracing = { version = "0.1.44", workspace = true } 35 + tracing-subscriber = { version = "0.3.23", workspace = true } 33 36 zstd = "0.13.2" 34 37 35 38 [dependencies.tokio] ··· 45 48 46 49 [features] 47 50 default = ["rocks"] 48 - rocks = ["dep:rocksdb"] 51 + rocks = ["dep:rocksdb", "dep:eat-rocks"]

+113 -30

constellation/src/bin/rocks-restore-from-backup.rs

··· 2 2 use anyhow::Result; 3 3 use clap::Parser; 4 4 use std::path::PathBuf; 5 + use std::time; 6 + use tracing::{error, info}; 5 7 6 8 #[cfg(feature = "rocks")] 7 9 use rocksdb::backup::{BackupEngine, BackupEngineOptions, RestoreOptions}; 8 10 9 - use std::time; 10 - 11 - /// Aggregate links in the at-mosphere 11 + /// Restore a rocksdb database 12 12 #[derive(Parser, Debug)] 13 13 #[command(version, about, long_about = None)] 14 14 struct Args { 15 - /// the backup directory to restore *from* 15 + /// the directory to restore the database into 16 + dest: PathBuf, 17 + /// restore from a rocksdb backup folder 18 + #[arg(long, conflicts_with_all = ["endpoint", "prefix", "concurrency"])] 19 + fs_dir: Option<PathBuf>, 20 + /// restore from public object storage 21 + #[arg(long, default_value = "https://constellation.t3.storage.dev")] 22 + endpoint: String, 23 + /// specific backup to restore (default: latest) 16 24 #[arg(long)] 17 - from_backup_dir: PathBuf, 18 - /// the db dir to restore *to* 25 + backup: Option<u32>, 26 + /// object-store key prefix 27 + #[arg(long, default_value = "")] 28 + prefix: String, 29 + /// concurrency limit for object storage operations 30 + /// 31 + /// TODO: wire this through for the filesystem mode as well. 19 32 #[arg(long)] 20 - to_data_dir: PathBuf, 33 + concurrency: Option<usize>, 21 34 } 22 35 23 36 #[cfg(feature = "rocks")] 24 37 fn main() -> Result<()> { 25 - let args = Args::parse(); 26 - 27 - eprintln!( 28 - "restoring latest rocksdb backup from {:?} to {:?}...", 29 - args.from_backup_dir, args.to_data_dir 30 - ); 31 - 32 - let mut engine = BackupEngine::open( 33 - &BackupEngineOptions::new(args.from_backup_dir)?, 34 - &rocksdb::Env::new()?, 35 - )?; 38 + use tracing_subscriber::EnvFilter; 39 + tracing_subscriber::fmt() 40 + .with_env_filter( 41 + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), 42 + ) 43 + .init(); 36 44 45 + let args = Args::parse(); 37 46 let t0 = Instant::now(); 38 - if let Err(e) = engine.restore_from_latest_backup( 39 - &args.to_data_dir, 40 - &args.to_data_dir, 41 - &RestoreOptions::default(), 42 - ) { 43 - eprintln!( 44 - "restoring from backup failed after {:?}: {e:?}", 45 - t0.elapsed() 47 + 48 + let result = if let Some(fs_dir) = args.fs_dir { 49 + info!( 50 + ?fs_dir, 51 + ?args.dest, 52 + backup = describe_backup(args.backup), 53 + mode = "rocksdb fs", 54 + "restoring rocksdb backup..." 46 55 ); 56 + restore_from_dir(&fs_dir, &args.dest, args.backup) 47 57 } else { 48 - eprintln!( 49 - "success, restored latest from backup after {:?}", 50 - t0.elapsed() 58 + info!( 59 + args.endpoint, 60 + args.prefix, 61 + ?args.dest, 62 + ?args.concurrency, 63 + backup = describe_backup(args.backup), 64 + mode = "eat-rocks object storage", 65 + "restoring rocksdb backup..." 51 66 ); 67 + restore_from_object_store( 68 + &args.endpoint, 69 + &args.prefix, 70 + &args.dest, 71 + args.backup, 72 + args.concurrency, 73 + ) 74 + }; 75 + 76 + match result { 77 + Ok(()) => info!( 78 + elapsed = ?t0.elapsed(), 79 + "backup restored." 80 + ), 81 + Err(err) => error!( 82 + ?err, 83 + elapsed = ?t0.elapsed(), 84 + "failed to restore backup." 85 + ), 86 + } 87 + Ok(()) 88 + } 89 + 90 + fn describe_backup(backup: Option<u32>) -> String { 91 + match backup { 92 + Some(id) => format!("#{id}"), 93 + None => "(latest)".to_string(), 52 94 } 95 + } 53 96 54 - eprintln!("bye."); 97 + #[cfg(feature = "rocks")] 98 + fn restore_from_dir( 99 + fs_dir: &std::path::Path, 100 + dest: &std::path::Path, 101 + backup_id: Option<u32>, 102 + ) -> Result<()> { 103 + let mut engine = BackupEngine::open(&BackupEngineOptions::new(fs_dir)?, &rocksdb::Env::new()?)?; 104 + let opts = RestoreOptions::default(); 105 + match backup_id { 106 + Some(id) => engine.restore_from_backup(dest, dest, &opts, id)?, 107 + None => engine.restore_from_latest_backup(dest, dest, &opts)?, 108 + } 109 + Ok(()) 110 + } 111 + 112 + #[cfg(feature = "rocks")] 113 + fn restore_from_object_store( 114 + endpoint: &str, 115 + prefix: &str, 116 + dest: &std::path::Path, 117 + backup_id: Option<u32>, 118 + concurrency: Option<usize>, 119 + ) -> Result<()> { 120 + use eat_rocks::{public_bucket, restore, RestoreOptions}; 121 + use tokio::runtime::Runtime; 122 + 123 + let rt = Runtime::new()?; 124 + rt.block_on(async { 125 + let store = public_bucket(endpoint)?; 126 + restore( 127 + store, 128 + prefix, 129 + dest, 130 + RestoreOptions { 131 + backup_id: backup_id.map(u64::from), 132 + concurrency: concurrency.unwrap_or(RestoreOptions::default().concurrency), 133 + ..Default::default() 134 + }, 135 + ) 136 + .await 137 + })?; 55 138 Ok(()) 56 139 }

Configure Feed

Configure Feed

History