microcosm.blue / hubble
A public mirror for the whole atmosphere hubble.microcosm.blue
27
fork

Configure Feed

Select the types of activity you want to include in your feed.

bench checkpoint

phil 85d8ef3c e3cf014b

+1109 -55
+20 -23
Cargo.lock
··· 697 697 ] 698 698 699 699 [[package]] 700 + name = "librocksdb-sys" 701 + version = "0.17.3+10.4.2" 702 + source = "registry+https://github.com/rust-lang/crates.io-index" 703 + checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" 704 + dependencies = [ 705 + "bindgen", 706 + "bzip2-sys", 707 + "cc", 708 + "libc", 709 + "libz-sys", 710 + "lz4-sys", 711 + "zstd-sys", 712 + ] 713 + 714 + [[package]] 700 715 name = "libz-sys" 701 716 version = "1.1.28" 702 717 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 988 1003 ] 989 1004 990 1005 [[package]] 991 - name = "rust-librocksdb-sys" 992 - version = "0.42.0+10.10.1" 1006 + name = "rocksdb" 1007 + version = "0.24.0" 993 1008 source = "registry+https://github.com/rust-lang/crates.io-index" 994 - checksum = "6522f8dfc02acd618477cd9aa94234f6a823ce578bba66215f15df8cb09a97d5" 1009 + checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" 995 1010 dependencies = [ 996 - "bindgen", 997 - "bzip2-sys", 998 - "cc", 999 - "glob", 1000 1011 "libc", 1001 - "libz-sys", 1002 - "lz4-sys", 1003 - "zstd-sys", 1004 - ] 1005 - 1006 - [[package]] 1007 - name = "rust-rocksdb" 1008 - version = "0.46.0" 1009 - source = "registry+https://github.com/rust-lang/crates.io-index" 1010 - checksum = "5573cd9e65281eb049123284fb8e161192f34ff76a7d635945d5fedbb7a59db7" 1011 - dependencies = [ 1012 - "libc", 1013 - "parking_lot", 1014 - "rust-librocksdb-sys", 1012 + "librocksdb-sys", 1015 1013 ] 1016 1014 1017 1015 [[package]] ··· 1173 1171 "async-channel", 1174 1172 "clap", 1175 1173 "repo-stream", 1176 - "rust-rocksdb", 1174 + "rocksdb", 1177 1175 "thiserror 2.0.18", 1178 1176 "tokio", 1179 1177 "tracing", ··· 1459 1457 source = "registry+https://github.com/rust-lang/crates.io-index" 1460 1458 checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" 1461 1459 dependencies = [ 1462 - "bindgen", 1463 1460 "cc", 1464 1461 "pkg-config", 1465 1462 ]
+1 -1
Cargo.toml
··· 8 8 async-channel = "2.5.0" 9 9 clap = { version = "4.6.0", features = ["env", "derive"] } 10 10 repo-stream = "0.5.0-alpha.5" 11 - rust-rocksdb = "0.46.0" 11 + rocksdb = "0.24.0" 12 12 thiserror = "2.0.18" 13 13 tokio = "1.51.0" 14 14 tracing = "0.1.44"
+1 -1
space-efficiency-check/Cargo.toml
··· 7 7 async-channel = { workspace = true } 8 8 clap = { workspace = true } 9 9 repo-stream = { workspace = true } 10 - rust-rocksdb = { workspace = true } 10 + rocksdb = { workspace = true } 11 11 thiserror = { workspace = true } 12 12 tokio = { workspace = true, features = ["full"] } 13 13 tracing = { workspace = true }
+95
space-efficiency-check/examples/inspect.rs
··· 1 + use rocksdb::{Options, DB}; 2 + use std::path::PathBuf; 3 + 4 + fn main() { 5 + let db_path = std::env::args() 6 + .nth(1) 7 + .map(PathBuf::from) 8 + .unwrap_or_else(|| PathBuf::from("./local/space-efficiency-check-db")); 9 + 10 + let opts = Options::default(); 11 + let db = DB::open_for_read_only(&opts, &db_path, false).expect("failed to open db"); 12 + 13 + // per-level file counts and sizes 14 + if let Ok(Some(v)) = db.property_value("rocksdb.levelstats") { 15 + println!("=== Level Stats ==="); 16 + println!("{v}"); 17 + } 18 + 19 + // aggregated table properties (includes compression, block counts, sizes) 20 + if let Ok(Some(v)) = db.property_value("rocksdb.aggregated-table-properties") { 21 + println!("=== Aggregated Table Properties ==="); 22 + println!("{v}"); 23 + } 24 + 25 + // per-level aggregated table properties (for levels that have files) 26 + for level in 0..7 { 27 + let prop = format!("rocksdb.aggregated-table-properties-at-level{level}"); 28 + if let Ok(Some(v)) = db.property_value(&*prop) { 29 + let trimmed = v.trim(); 30 + if !trimmed.is_empty() { 31 + println!("=== Table Properties Level {level} ==="); 32 + println!("{v}"); 33 + } 34 + } 35 + } 36 + 37 + // sstables summary (per-file compression details) 38 + if let Ok(Some(v)) = db.property_value("rocksdb.sstables") { 39 + println!("=== SST Tables Summary ==="); 40 + // this can be very long, just print the first ~100 lines 41 + for line in v.lines().take(100) { 42 + println!("{line}"); 43 + } 44 + let total_lines = v.lines().count(); 45 + if total_lines > 100 { 46 + println!("... ({} more lines)", total_lines - 100); 47 + } 48 + println!(); 49 + } 50 + 51 + // per-level compression ratios 52 + println!("=== Compression Ratios ==="); 53 + for level in 0..7 { 54 + let prop = format!("rocksdb.compression-ratio-at-level{level}"); 55 + if let Ok(Some(v)) = db.property_value(&*prop) { 56 + if v != "-1.0" { 57 + println!(" L{level}: {v}"); 58 + } 59 + } 60 + } 61 + println!(); 62 + 63 + // live files summary 64 + match db.live_files() { 65 + Ok(mut files) => { 66 + files.sort_by_key(|f| (f.level, f.name.clone())); 67 + println!("=== Live SST Files ({} total) ===", files.len()); 68 + println!( 69 + "{:<6} {:<30} {:>14} {:>14} {:>10}", 70 + "Level", "Name", "Size (MiB)", "Size (bytes)", "Entries" 71 + ); 72 + for f in &files { 73 + println!( 74 + "L{:<5} {:<30} {:>14.1} {:>14} {:>10}", 75 + f.level, 76 + f.name, 77 + f.size as f64 / (1 << 20) as f64, 78 + f.size, 79 + f.num_entries, 80 + ); 81 + } 82 + let total_size: usize = files.iter().map(|f| f.size).sum(); 83 + let total_entries: u64 = files.iter().map(|f| f.num_entries).sum(); 84 + println!( 85 + "{:<6} {:<30} {:>14.1} {:>14} {:>10}", 86 + "", 87 + "TOTAL", 88 + total_size as f64 / (1 << 20) as f64, 89 + total_size, 90 + total_entries, 91 + ); 92 + } 93 + Err(e) => eprintln!("failed to get live files: {e}"), 94 + } 95 + }
+484
space-efficiency-check/inspect-output.txt
··· 1 + === Level Stats === 2 + Level Files Size(MB) 3 + -------------------- 4 + 0 0 0 5 + 1 0 0 6 + 2 0 0 7 + 3 0 0 8 + 4 0 0 9 + 5 0 0 10 + 6 342 52214 11 + 12 + === Aggregated Table Properties === 13 + # data blocks=5502269; # entries=665928371; # deletions=0; # merge operands=0; # range deletions=0; raw key size=48950837002; raw average key size=73.507661; raw value size=160502129948; raw average value size=241.020111; data block size=54659623307; index block size (user-key? 342, delta-value? 342)=385489305; filter block size=0; # entries for filter=0; (estimated) table size=55045112612; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 14 + === Table Properties Level 0 === 15 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 16 + === Table Properties Level 1 === 17 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 18 + === Table Properties Level 2 === 19 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 20 + === Table Properties Level 3 === 21 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 22 + === Table Properties Level 4 === 23 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 24 + === Table Properties Level 5 === 25 + # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 26 + === Table Properties Level 6 === 27 + # data blocks=5502269; # entries=665928371; # deletions=0; # merge operands=0; # range deletions=0; raw key size=48950837002; raw average key size=73.507661; raw value size=160502129948; raw average value size=241.020111; data block size=54659623307; index block size (user-key? 342, delta-value? 342)=385489305; filter block size=0; # entries for filter=0; (estimated) table size=55045112612; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 28 + === SST Tables Summary === 29 + --- level 0 --- version# 1 --- 30 + --- level 1 --- version# 1 --- 31 + --- level 2 --- version# 1 --- 32 + --- level 3 --- version# 1 --- 33 + --- level 4 --- version# 1 --- 34 + --- level 5 --- version# 1 --- 35 + --- level 6 --- version# 1 --- 36 + 2985:268861442[0 .. 0]['6469643A706C633A3232323263786E7464346333717474796437736D6C6473702F6170702E62736B792E6163746F722E70726F66696C652F73656C66' seq:0, type:1 .. '6469643A706C633A32326E6F7262796A626C6C35336877676B637776706A37622F6170702E62736B792E666565642E6C696B652F336B687A656F6662346B35326E' seq:0, type:1](0) 37 + 2987:58754567[0 .. 0]['6469643A706C633A32326E6F7262796A626C6C35336877676B637776706A37622F6170702E62736B792E666565642E6C696B652F336B687A6732666D72376C3274' seq:0, type:1 .. '6469643A706C633A3232726273346A6F32796A7A6833366A686D7877613364322F6170702E62736B792E666565642E6C696B652F336C63626C6B67373765643270' seq:0, type:1](0) 38 + 2979:268861560[0 .. 0]['6469643A706C633A3232726273346A6F32796A7A6833366A686D7877613364322F6170702E62736B792E666565642E6C696B652F336C63626D626F757079793264' seq:0, type:1 .. '6469643A706C633A32336536716D7567327677636E73667632667977757934622F6170702E62736B792E666565642E6C696B652F336C6332796F737964656A326E' seq:0, type:1](0) 39 + 2996:58726230[0 .. 0]['6469643A706C633A32336536716D7567327677636E73667632667977757934622F6170702E62736B792E666565642E6C696B652F336C6332797032346870623270' seq:0, type:1 .. '6469643A706C633A323367756A646176637578646A34326568376A66733661642F6170702E62736B792E666565642E6C696B652F336D34773333376978736B326A' seq:0, type:1](0) 40 + 2980:268873207[0 .. 0]['6469643A706C633A323367756A646176637578646A34326568376A66733661642F6170702E62736B792E666565642E6C696B652F336D3477333378626535613275' seq:0, type:1 .. '6469643A706C633A323436786F613662706267616C33326579366F70337A6E612F6170702E62736B792E666565642E7265706F73742F336B6966346D70746561333261' seq:0, type:1](0) 41 + 2995:58143950[0 .. 0]['6469643A706C633A323436786F613662706267616C33326579366F70337A6E612F6170702E62736B792E666565642E7265706F73742F336B6966376C37637661663234' seq:0, type:1 .. '6469643A706C633A3234647A727971656E6A7670667A337863686875327474732F6170702E62736B792E666565642E6C696B652F336C6461646569767374663271' seq:0, type:1](0) 42 + 2981:268868054[0 .. 0]['6469643A706C633A3234647A727971656E6A7670667A337863686875327474732F6170702E62736B792E666565642E6C696B652F336C646164656A643562623268' seq:0, type:1 .. '6469643A706C633A3234777A73666A326B696532737177347263706D6E7774632F6170702E62736B792E666565642E706F73742F336C6E6A3534686279336B3234' seq:0, type:1](0) 43 + 2990:58243804[0 .. 0]['6469643A706C633A3234777A73666A326B696532737177347263706D6E7774632F6170702E62736B792E666565642E706F73742F336C6E6A617569367036733234' seq:0, type:1 .. '6469643A706C633A3235327978727462327A7978737A7979696E7032757A34692F6170702E62736B792E666565642E706F73742F336C36766C706C766635713272' seq:0, type:1](0) 44 + 2978:268863540[0 .. 0]['6469643A706C633A3235327978727462327A7978737A7979696E7032757A34692F6170702E62736B792E666565642E706F73742F336C36766E3562786962633263' seq:0, type:1 .. '6469643A706C633A32356E756F686A63326F32326D777A756C696464737479792F6170702E62736B792E666565642E6C696B652F336B366A6E3365373562663276' seq:0, type:1](0) 45 + 2988:56596617[0 .. 0]['6469643A706C633A32356E756F686A63326F32326D777A756C696464737479792F6170702E62736B792E666565642E6C696B652F336B366A6E7432767A77673271' seq:0, type:1 .. '6469643A706C633A3235727464776E7037797A3573707A7976617A673275786B2F6170702E62736B792E666565642E706F73742F336B69666A357176616A783277' seq:0, type:1](0) 46 + 2984:268868826[0 .. 0]['6469643A706C633A3235727464776E7037797A3573707A7976617A673275786B2F6170702E62736B792E666565642E706F73742F336B69666A70697A716B753265' seq:0, type:1 .. '6469643A706C633A32366A646C71356F6B71376C77373365366278326D7836792F6170702E62736B792E666565642E6C696B652F336B33646F72686D643733326D' seq:0, type:1](0) 47 + 2991:57796974[0 .. 0]['6469643A706C633A32366A646C71356F6B71376C77373365366278326D7836792F6170702E62736B792E666565642E6C696B652F336B33647234657A78336C327A' seq:0, type:1 .. '6469643A706C633A323670336B33746B7271676D787273326D65766E673235792F6170702E62736B792E67726170682E626C6F636B2F336C696574646D6F6236373272' seq:0, type:1](0) 48 + 2986:268866712[0 .. 0]['6469643A706C633A323670336B33746B7271676D787273326D65766E673235792F6170702E62736B792E67726170682E626C6F636B2F336C696574646F37777768326C' seq:0, type:1 .. '6469643A706C633A32376535786C74796A64776A777766746C676863646672652F6170702E62736B792E666565642E6C696B652F336C646E706F6D376936773277' seq:0, type:1](0) 49 + 2992:58269042[0 .. 0]['6469643A706C633A32376535786C74796A64776A777766746C676863646672652F6170702E62736B792E666565642E6C696B652F336C646E707262353478673235' seq:0, type:1 .. '6469643A706C633A32376864346A6D6935793678646D68726E32677A6A3277322F6170702E62736B792E67726170682E666F6C6C6F772F336D3233706772663265673271' seq:0, type:1](0) 50 + 2977:268862614[0 .. 0]['6469643A706C633A32376864346A6D6935793678646D68726E32677A6A3277322F6170702E62736B792E67726170682E666F6C6C6F772F336D323370677279747A343266' seq:0, type:1 .. '6469643A706C633A32377A70726461786D7A6A6466666F6235367A326A6278332F6170702E62736B792E666565642E6C696B652F336C3462326B67613465373274' seq:0, type:1](0) 51 + 2994:58826933[0 .. 0]['6469643A706C633A32377A70726461786D7A6A6466666F6235367A326A6278332F6170702E62736B792E666565642E6C696B652F336C3462326B6F67366F7A326A' seq:0, type:1 .. '6469643A706C633A3261346E3762347A746A65746F796D7A7536716E78666D322F6170702E62736B792E666565642E7265706F73742F336C6B6F647374636A68373264' seq:0, type:1](0) 52 + 2982:268858158[0 .. 0]['6469643A706C633A3261346E3762347A746A65746F796D7A7536716E78666D322F6170702E62736B792E666565642E7265706F73742F336C6B6F7A7865756D7833326E' seq:0, type:1 .. '6469643A706C633A3261716F6F796968676C737861696137706E373472616B692F6170702E62736B792E666565642E6C696B652F336C7A74683735363671613261' seq:0, type:1](0) 53 + 2989:57240801[0 .. 0]['6469643A706C633A3261716F6F796968676C737861696137706E373472616B692F6170702E62736B792E666565642E6C696B652F336C7A746C6E77717671643237' seq:0, type:1 .. '6469643A706C633A32617475366E79367A726D70626778613273673237616F712F6170702E62736B792E666565642E6C696B652F336C6968736E7973777463326F' seq:0, type:1](0) 54 + 2983:268861238[0 .. 0]['6469643A706C633A32617475366E79367A726D70626778613273673237616F712F6170702E62736B792E666565642E6C696B652F336C6968736F36763377653270' seq:0, type:1 .. '6469643A706C633A3262666478727074696C6F663671687735376B6F626C63632F6170702E62736B792E666565642E706F73742F336C716E716E6D6C636173326E' seq:0, type:1](0) 55 + 2993:57408071[0 .. 0]['6469643A706C633A3262666478727074696C6F663671687735376B6F626C63632F6170702E62736B792E666565642E706F73742F336C716E717A617A70766B3276' seq:0, type:1 .. '6469643A706C633A32626A6666353435746D327A753361666E6468747A6475652F6170702E62736B792E666565642E6C696B652F336C6373797178747778673237' seq:0, type:1](0) 56 + 2999:268859763[0 .. 0]['6469643A706C633A32626A6666353435746D327A753361666E6468747A6475652F6170702E62736B792E666565642E6C696B652F336C63743266796C7671663237' seq:0, type:1 .. '6469643A706C633A32633737347477626778657679357378646C63756379636E2F6170702E62736B792E666565642E6C696B652F336B786A726A77646664783279' seq:0, type:1](0) 57 + 3016:49005391[0 .. 0]['6469643A706C633A32633737347477626778657679357378646C63756379636E2F6170702E62736B792E666565642E6C696B652F336B786C6B756F696D37773271' seq:0, type:1 .. '6469643A706C633A326364357A6933796F6C68366E746A696E6273323370357A2F6170702E62736B792E666565642E6C696B652F336D3374796E357174336A3267' seq:0, type:1](0) 58 + 2998:268865193[0 .. 0]['6469643A706C633A326364357A6933796F6C68366E746A696E6273323370357A2F6170702E62736B792E666565642E6C696B652F336D33747A336C6B6E6B723264' seq:0, type:1 .. '6469643A706C633A32637769627A366F62357979346269326D6F616F626762632F6170702E62736B792E666565642E6C696B652F336C676C6171326F67796E3265' seq:0, type:1](0) 59 + 3008:49941547[0 .. 0]['6469643A706C633A32637769627A366F62357979346269326D6F616F626762632F6170702E62736B792E666565642E6C696B652F336C676C623778326B75373262' seq:0, type:1 .. '6469643A706C633A3263793564356279706C6973757234793770706B7A356E682F6170702E62736B792E666565642E706F73742F336C6E616B6E78736968323272' seq:0, type:1](0) 60 + 2997:268860983[0 .. 0]['6469643A706C633A3263793564356279706C6973757234793770706B7A356E682F6170702E62736B792E666565642E706F73742F336C6E616B77767763726B326D' seq:0, type:1 .. '6469643A706C633A326469626B6232686161676E786C6C6E79636763643561642F6170702E62736B792E666565642E706F73742F336C73336C757062696132326C' seq:0, type:1](0) 61 + 3013:49013595[0 .. 0]['6469643A706C633A326469626B6232686161676E786C6C6E79636763643561642F6170702E62736B792E666565642E706F73742F336C73336D363474747173326C' seq:0, type:1 .. '6469643A706C633A32646C796536363368696376676F706964777862623269722F6170702E62736B792E666565642E706F73742F336C346A737279766F6774326A' seq:0, type:1](0) 62 + 3000:268869110[0 .. 0]['6469643A706C633A32646C796536363368696376676F706964777862623269722F6170702E62736B792E666565642E706F73742F336C346A737676656B766C327A' seq:0, type:1 .. '6469643A706C633A326536756B7A796C6E6D66703766786B357937696C6674672F6170702E62736B792E666565642E6C696B652F336C726A767665623775663271' seq:0, type:1](0) 63 + 3012:48660516[0 .. 0]['6469643A706C633A326536756B7A796C6E6D66703766786B357937696C6674672F6170702E62736B792E666565642E6C696B652F336C726A776274636A7A733271' seq:0, type:1 .. '6469643A706C633A3265626C776D7961736B756D37697A35346C32726374736A2F6170702E62736B792E666565642E6C696B652F336C756A666C6679366272326E' seq:0, type:1](0) 64 + 3001:268851211[0 .. 0]['6469643A706C633A3265626C776D7961736B756D37697A35346C32726374736A2F6170702E62736B792E666565642E6C696B652F336C756A666C6D793234653271' seq:0, type:1 .. '6469643A706C633A326573756E656A786E7875717778667576636E3673716C702F6170702E62736B792E666565642E6C696B652F336C6367687436616E76743274' seq:0, type:1](0) 65 + 3015:49418209[0 .. 0]['6469643A706C633A326573756E656A786E7875717778667576636E3673716C702F6170702E62736B792E666565642E6C696B652F336C6367687476783478773279' seq:0, type:1 .. '6469643A706C633A3265767A71766A75713472326A7274377479737269676A742F6170702E62736B792E666565642E6C696B652F336B716E767477647464733277' seq:0, type:1](0) 66 + 3006:268867983[0 .. 0]['6469643A706C633A3265767A71766A75713472326A7274377479737269676A742F6170702E62736B792E666565642E6C696B652F336B716E783773686477733233' seq:0, type:1 .. '6469643A706C633A32666C7833666E6F7036716F707632736273733567347A652F6170702E62736B792E666565642E6C696B652F336C6736706171643779613236' seq:0, type:1](0) 67 + 3011:48111091[0 .. 0]['6469643A706C633A32666C7833666E6F7036716F707632736273733567347A652F6170702E62736B792E666565642E6C696B652F336C6736706263367232613236' seq:0, type:1 .. '6469643A706C633A32666E79366F7733686A6F32636476716A6767616B656F682F6170702E62736B792E666565642E6C696B652F336C36716F357A6F3737763272' seq:0, type:1](0) 68 + 3002:268858725[0 .. 0]['6469643A706C633A32666E79366F7733686A6F32636476716A6767616B656F682F6170702E62736B792E666565642E6C696B652F336C36716F373532746C753266' seq:0, type:1 .. '6469643A706C633A32673736666F6E75756A6D377765366E7A656636746F666C2F6170702E62736B792E666565642E7265706F73742F336C6E6F33627374783435326B' seq:0, type:1](0) 69 + 3009:47761296[0 .. 0]['6469643A706C633A32673736666F6E75756A6D377765366E7A656636746F666C2F6170702E62736B792E666565642E7265706F73742F336C6E6F336363696966663237' seq:0, type:1 .. '6469643A706C633A326762686D76696E617379687A7977646E6963346A7369332F6170702E62736B792E666565642E6C696B652F336C356575767235696D373275' seq:0, type:1](0) 70 + 3004:268878428[0 .. 0]['6469643A706C633A326762686D76696E617379687A7977646E6963346A7369332F6170702E62736B792E666565642E6C696B652F336C3565757763766F77783275' seq:0, type:1 .. '6469643A706C633A3267776B346771346F627A676D7A79617A76366F71377A342F6170702E62736B792E666565642E6C696B652F336C696E326C61736C72733261' seq:0, type:1](0) 71 + 3010:47692215[0 .. 0]['6469643A706C633A3267776B346771346F627A676D7A79617A76366F71377A342F6170702E62736B792E666565642E6C696B652F336C696E326C666B61666D3271' seq:0, type:1 .. '6469643A706C633A326779626A62336734646F686F79366C34377335346175722F6170702E62736B792E666565642E6C696B652F336B6B773233747167656B327A' seq:0, type:1](0) 72 + 3003:268859183[0 .. 0]['6469643A706C633A326779626A62336734646F686F79366C34377335346175722F6170702E62736B792E666565642E6C696B652F336B6B77326F33733370333266' seq:0, type:1 .. '6469643A706C633A32686D376466617069326B67726B6A626B6F6D366C6579632F6170702E62736B792E666565642E706F73742F336C72717271666A32796B3262' seq:0, type:1](0) 73 + 3007:50456087[0 .. 0]['6469643A706C633A32686D376466617069326B67726B6A626B6F6D366C6579632F6170702E62736B792E666565642E706F73742F336C7335637A71683375633234' seq:0, type:1 .. '6469643A706C633A32687067656B3670356964797465716F757A37636F7577642F6170702E62736B792E666565642E6C696B652F336C62687A3677346B64663278' seq:0, type:1](0) 74 + 3005:268875961[0 .. 0]['6469643A706C633A32687067656B3670356964797465716F757A37636F7577642F6170702E62736B792E666565642E6C696B652F336C62687A3734346F69693270' seq:0, type:1 .. '6469643A706C633A326967736D73676D6C737732336170373567746C3636756E2F6170702E62736B792E666565642E6C696B652F336C747976326768676F373263' seq:0, type:1](0) 75 + 3014:47460314[0 .. 0]['6469643A706C633A326967736D73676D6C737732336170373567746C3636756E2F6170702E62736B792E666565642E6C696B652F336C747976333275366C6B3269' seq:0, type:1 .. '6469643A706C633A32696C6777366D6A6365703468673471636A6E6F727032332F6170702E62736B792E666565642E6C696B652F336B6A62656E6C6F6C696A3236' seq:0, type:1](0) 76 + 3026:268866516[0 .. 0]['6469643A706C633A32696C6777366D6A6365703468673471636A6E6F727032332F6170702E62736B792E666565642E6C696B652F336B6A62716876706F6A62326E' seq:0, type:1 .. '6469643A706C633A326A37787A7A767A6A6D776D73647334717661786E7263722F6170702E62736B792E666565642E6C696B652F336C69646F627179716478326D' seq:0, type:1](0) 77 + 3028:44965751[0 .. 0]['6469643A706C633A326A37787A7A767A6A6D776D73647334717661786E7263722F6170702E62736B792E666565642E6C696B652F336C69646F637A77677970326E' seq:0, type:1 .. '6469643A706C633A326A6532617237657964747A76327075376C7577616333722F6170702E62736B792E666565642E6C696B652F336C377464786A636F34323273' seq:0, type:1](0) 78 + 3025:268853156[0 .. 0]['6469643A706C633A326A6532617237657964747A76327075376C7577616333722F6170702E62736B792E666565642E6C696B652F336C3774647973656E796E3277' seq:0, type:1 .. '6469643A706C633A326A736F6A65776D6D7A72746B6163766A627933697936332F6170702E62736B792E666565642E706F73742F336C7166717932366472323237' seq:0, type:1](0) 79 + 3036:46122242[0 .. 0]['6469643A706C633A326A736F6A65776D6D7A72746B6163766A627933697936332F6170702E62736B792E666565642E706F73742F336C7166717A7862326B323237' seq:0, type:1 .. '6469643A706C633A326A7479613273766C7763706F733564727567346C7376782F6170702E62736B792E666565642E6C696B652F336C686A787A696B346A663274' seq:0, type:1](0) 80 + 3024:268857116[0 .. 0]['6469643A706C633A326A7479613273766C7763706F733564727567346C7376782F6170702E62736B792E666565642E6C696B652F336C686A793233796A6A65326B' seq:0, type:1 .. '6469643A706C633A326B67786537707873656734787268796A647A646B326A662F6170702E62736B792E666565642E6C696B652F336C7A7275356A6A6F6E353271' seq:0, type:1](0) 81 + 3035:46421856[0 .. 0]['6469643A706C633A326B67786537707873656734787268796A647A646B326A662F6170702E62736B792E666565642E6C696B652F336D32647962786B336C613265' seq:0, type:1 .. '6469643A706C633A326B6B6976766772636C62776B336774646D626E6B7576792F6170702E62736B792E666565642E6C696B652F336B71626E7662723566773234' seq:0, type:1](0) 82 + 3017:268865566[0 .. 0]['6469643A706C633A326B6B6976766772636C62776B336774646D626E6B7576792F6170702E62736B792E666565642E6C696B652F336B71626E766F676A62683232' seq:0, type:1 .. '6469643A706C633A326C36376A7676617667377262336368727A61616F3773322F6170702E62736B792E666565642E6C696B652F336D336264726C7A78706D3261' seq:0, type:1](0) 83 + 3034:43720209[0 .. 0]['6469643A706C633A326C36376A7676617667377262336368727A61616F3773322F6170702E62736B792E666565642E6C696B652F336D33626472706A617A643237' seq:0, type:1 .. '6469643A706C633A326C636E64697032677A7966666E67636274333264716E352F6170702E62736B792E666565642E6C696B652F336C6878696E6179626B353261' seq:0, type:1](0) 84 + 3019:268860135[0 .. 0]['6469643A706C633A326C636E64697032677A7966666E67636274333264716E352F6170702E62736B792E666565642E6C696B652F336C6878697367353571763261' seq:0, type:1 .. '6469643A706C633A326C74787265747433786D347176797477626E626B6636732F6170702E62736B792E666565642E706F73742F336C676A36367A696869633276' seq:0, type:1](0) 85 + 3031:43662350[0 .. 0]['6469643A706C633A326C74787265747433786D347176797477626E626B6636732F6170702E62736B792E666565642E706F73742F336C676A6164326A3678733236' seq:0, type:1 .. '6469643A706C633A326C78616A633771717A78366E346B6E776D7363736C64332F6170702E62736B792E666565642E6C696B652F336C786E69366C62796668326C' seq:0, type:1](0) 86 + 3021:268870507[0 .. 0]['6469643A706C633A326C78616A633771717A78366E346B6E776D7363736C64332F6170702E62736B792E666565642E6C696B652F336C786E69366D76757778326A' seq:0, type:1 .. '6469643A706C633A326D6963357A7277367A69776169686E3773767933687A6E2F6170702E62736B792E666565642E6C696B652F336C3568626A6D733674373235' seq:0, type:1](0) 87 + 3029:45036982[0 .. 0]['6469643A706C633A326D6963357A7277367A69776169686E3773767933687A6E2F6170702E62736B792E666565642E6C696B652F336C3568626B35326D35683235' seq:0, type:1 .. '6469643A706C633A326D6A69767234736B6E71696232737A79376B6D6F7669682F6170702E62736B792E666565642E706F73742F336B35723333667A3536683265' seq:0, type:1](0) 88 + 3023:268865620[0 .. 0]['6469643A706C633A326D6A69767234736B6E71696232737A79376B6D6F7669682F6170702E62736B792E666565642E706F73742F336B3572336267777A6D6F326F' seq:0, type:1 .. '6469643A706C633A326E64356C6362796571793464743272733479336B7965362F6170702E62736B792E666565642E706F73742F336C7473333476343262323273' seq:0, type:1](0) 89 + 3032:45881657[0 .. 0]['6469643A706C633A326E64356C6362796571793464743272733479336B7965362F6170702E62736B792E666565642E706F73742F336C7473333577636C61633273' seq:0, type:1 .. '6469643A706C633A326E686E6F33757068773570367671327972366C64627A6A2F6170702E62736B792E666565642E6C696B652F336C687A746A6173656564326F' seq:0, type:1](0) 90 + 3018:268866581[0 .. 0]['6469643A706C633A326E686E6F33757068773570367671327972366C64627A6A2F6170702E62736B792E666565642E6C696B652F336C687A746B70636A66773270' seq:0, type:1 .. '6469643A706C633A326F616D663234357879706573616A6E6E3263616A7867742F6170702E62736B792E666565642E7265706F73742F336C6C79696167737A6F633275' seq:0, type:1](0) 91 + 3027:46042194[0 .. 0]['6469643A706C633A326F616D663234357879706573616A6E6E3263616A7867742F6170702E62736B792E666565642E7265706F73742F336C6C796A61706F327733326E' seq:0, type:1 .. '6469643A706C633A326F62746C6472663668346A7967347474656136336E62642F6170702E62736B792E666565642E706F73742F336C716F676E76616C36633265' seq:0, type:1](0) 92 + 3022:268864661[0 .. 0]['6469643A706C633A326F62746C6472663668346A7967347474656136336E62642F6170702E62736B792E666565642E706F73742F336C716F677062353667733265' seq:0, type:1 .. '6469643A706C633A326F78347537717765357977757962693777667368326C682F6170702E62736B792E666565642E6C696B652F336C6669616D61797A346B3237' seq:0, type:1](0) 93 + 3033:44306441[0 .. 0]['6469643A706C633A326F78347537717765357977757962693777667368326C682F6170702E62736B792E666565642E6C696B652F336C6669616D677764666A3265' seq:0, type:1 .. '6469643A706C633A3270336274613435376B69787A6D323679656574323763652F6170702E62736B792E666565642E6C696B652F336C6E64766576717768753273' seq:0, type:1](0) 94 + 3020:268855547[0 .. 0]['6469643A706C633A3270336274613435376B69787A6D323679656574323763652F6170702E62736B792E666565642E6C696B652F336C6E6476666B746E6A713272' seq:0, type:1 .. '6469643A706C633A32706D6A786B7467783635616362766F67343567357435692F6170702E62736B792E666565642E6C696B652F336D34373268676E7568323266' seq:0, type:1](0) 95 + 3030:43303643[0 .. 0]['6469643A706C633A32706D6A786B7467783635616362766F67343567357435692F6170702E62736B792E666565642E6C696B652F336D3437326B61797934783232' seq:0, type:1 .. '6469643A706C633A3270717269336169367372326364796F7673357A696735662F6170702E62736B792E666565642E706F73742F336C68626E6474676163633270' seq:0, type:1](0) 96 + 3039:268859073[0 .. 0]['6469643A706C633A3270717269336169367372326364796F7673357A696735662F6170702E62736B792E666565642E706F73742F336C68673668376732716B3233' seq:0, type:1 .. '6469643A706C633A3271623473356775686F626F696E64616C337635346A61622F6170702E62736B792E666565642E6C696B652F336C6D7264746C32723778326F' seq:0, type:1](0) 97 + 3054:54499248[0 .. 0]['6469643A706C633A3271623473356775686F626F696E64616C337635346A61622F6170702E62736B792E666565642E6C696B652F336C6D726475327436776D3265' seq:0, type:1 .. '6469643A706C633A32716536756E7A336870693277627436793272327A3662722F6170702E62736B792E666565642E6C696B652F336C6261776E7236356B373233' seq:0, type:1](0) 98 + 3045:268863107[0 .. 0]['6469643A706C633A32716536756E7A336870693277627436793272327A3662722F6170702E62736B792E666565642E6C696B652F336C6261776F6B6F72367A3261' seq:0, type:1 .. '6469643A706C633A3271777065697378687A6E77646A66796C696875726470342F6170702E62736B792E666565642E6C696B652F336C3232703279636563793261' seq:0, type:1](0) 99 + 3050:54582384[0 .. 0]['6469643A706C633A3271777065697378687A6E77646A66796C696875726470342F6170702E62736B792E666565642E6C696B652F336C323271673373647132326E' seq:0, type:1 .. '6469643A706C633A3272326D786E6B34637176697A7676346F32666E797437742F6170702E62736B792E666565642E6C696B652F336C6B6B7935323473716C3272' seq:0, type:1](0) 100 + 3040:268861807[0 .. 0]['6469643A706C633A3272326D786E6B34637176697A7676346F32666E797437742F6170702E62736B792E666565642E6C696B652F336C6B6B793768657969673273' seq:0, type:1 .. '6469643A706C633A32726D6A35747636777A75716E32657532763332766274662F6170702E62736B792E666565642E6C696B652F336C746C6964626A67376D3272' seq:0, type:1](0) 101 + 3056:54348105[0 .. 0]['6469643A706C633A32726D6A35747636777A75716E32657532763332766274662F6170702E62736B792E666565642E6C696B652F336C746C6964736F32796F3276' seq:0, type:1 .. '6469643A706C633A3272727037616D626C686733786F6C79356E3568756971762F6170702E62736B792E666565642E6C696B652F336C71766B7A656534796B3233' seq:0, type:1](0) 102 + 3041:268862386[0 .. 0]['6469643A706C633A3272727037616D626C686733786F6C79356E3568756971762F6170702E62736B792E666565642E6C696B652F336C71766C336232646A623276' seq:0, type:1 .. '6469643A706C633A327364356B33743633333364356E64676F7A70716D6932652F6170702E62736B792E666565642E706F73742F336B616837626332726F743232' seq:0, type:1](0) 103 + 3047:53169771[0 .. 0]['6469643A706C633A327364356B33743633333364356E64676F7A70716D6932652F6170702E62736B792E666565642E706F73742F336B616A6E756470647963326F' seq:0, type:1 .. '6469643A706C633A32736762786D737266737867686772736A6E726F753735782F6170702E62736B792E666565642E6C696B652F336C3672696A6C7533356D3233' seq:0, type:1](0) 104 + 3042:268865393[0 .. 0]['6469643A706C633A32736762786D737266737867686772736A6E726F753735782F6170702E62736B792E666565642E6C696B652F336C3672696B6C713774723233' seq:0, type:1 .. '6469643A706C633A3274336E73666836777A796B6F796E7867347A73647732332F6170702E62736B792E666565642E6C696B652F336D3437637672706A64643261' seq:0, type:1](0) 105 + 3049:55289645[0 .. 0]['6469643A706C633A3274336E73666836777A796B6F796E7867347A73647732332F6170702E62736B792E666565642E6C696B652F336D3437657A326D656C6B326E' seq:0, type:1 .. '6469643A706C633A327437666E756A347079677A353561667A79636E32746F652F6170702E62736B792E666565642E6C696B652F336C7173656F75327A33713261' seq:0, type:1](0) 106 + 3038:268857948[0 .. 0]['6469643A706C633A327437666E756A347079677A353561667A79636E32746F652F6170702E62736B792E666565642E6C696B652F336C71736570376E3264713234' seq:0, type:1 .. '6469643A706C633A327471336873613663756B7A77696D643665666C61626F362F6170702E62736B792E67726170682E666F6C6C6F772F336C626E626A623263676C3271' seq:0, type:1](0) 107 + 3055:52683502[0 .. 0]['6469643A706C633A327471336873613663756B7A77696D643665666C61626F362F6170702E62736B792E67726170682E666F6C6C6F772F336C626E626A623263676D3271' seq:0, type:1 .. '6469643A706C633A3274756E6176766D66697136627974726C793334726B63322F6170702E62736B792E666565642E706F73742F336C616F6D336A717276323264' seq:0, type:1](0) 108 + 3043:268872932[0 .. 0]['6469643A706C633A3274756E6176766D66697136627974726C793334726B63322F6170702E62736B792E666565642E706F73742F336C616F736334666834633263' seq:0, type:1 .. '6469643A706C633A32756A74756B736369686D70776E77767A6A6D6B356E35682F6170702E62736B792E666565642E7265706F73742F336C70366F616B647073343263' seq:0, type:1](0) 109 + 3051:53198609[0 .. 0]['6469643A706C633A32756A74756B736369686D70776E77767A6A6D6B356E35682F6170702E62736B792E666565642E7265706F73742F336C706D6468656E32336D3279' seq:0, type:1 .. '6469643A706C633A32756F776434793466787A7470346A3379676F72336F757A2F6170702E62736B792E666565642E706F73742F336A797365746D796E766F3237' seq:0, type:1](0) 110 + 3037:268866317[0 .. 0]['6469643A706C633A32756F776434793466787A7470346A3379676F72336F757A2F6170702E62736B792E666565642E706F73742F336A7A697A6F79796D76713270' seq:0, type:1 .. '6469643A706C633A3276636161793279367161786D65646F366164746765677A2F6170702E62736B792E666565642E6C696B652F336B786835686A67626F733264' seq:0, type:1](0) 111 + 3048:53124111[0 .. 0]['6469643A706C633A3276636161793279367161786D65646F366164746765677A2F6170702E62736B792E666565642E6C696B652F336B786836716F6C7474323232' seq:0, type:1 .. '6469643A706C633A327665776871777676376271363764346A676E69777736612F6170702E62736B792E666565642E6C696B652F336D326C6A34336F6E6E783237' seq:0, type:1](0) 112 + 3046:268864680[0 .. 0]['6469643A706C633A327665776871777676376271363764346A676E69777736612F6170702E62736B792E666565642E6C696B652F336D326C6A36786F62786F3262' seq:0, type:1 .. '6469643A706C633A3276756B71626434623573686937727034723662657172642F6170702E62736B792E666565642E706F73742F336C6F723566633261636B3276' seq:0, type:1](0) 113 + 3053:56261157[0 .. 0]['6469643A706C633A3276756B71626434623573686937727034723662657172642F6170702E62736B792E666565642E706F73742F336C6F747174763768766B326A' seq:0, type:1 .. '6469643A706C633A32767A6C63726E656E7A6B79686B7674766E75376D676E342F6170702E62736B792E666565642E6C696B652F336C6C68676434716366733262' seq:0, type:1](0) 114 + 3044:268861905[0 .. 0]['6469643A706C633A32767A6C63726E656E7A6B79686B7674766E75376D676E342F6170702E62736B792E666565642E6C696B652F336C6C686764666B6A36663271' seq:0, type:1 .. '6469643A706C633A32776F336B757876617A7A366C7865346366667079626E692F6170702E62736B792E666565642E6C696B652F336C37627A35326E666A353237' seq:0, type:1](0) 115 + 3052:53666253[0 .. 0]['6469643A706C633A32776F336B757876617A7A366C7865346366667079626E692F6170702E62736B792E666565642E6C696B652F336C37627A35777A3370663278' seq:0, type:1 .. '6469643A706C633A32777437756C666161686162323372796E716C616D7933332F6170702E62736B792E666565642E6C696B652F336D337136786D6A7367753274' seq:0, type:1](0) 116 + 3060:268857163[0 .. 0]['6469643A706C633A32777437756C666161686162323372796E716C616D7933332F6170702E62736B792E666565642E6C696B652F336D3371367A6F36696D743261' seq:0, type:1 .. '6469643A706C633A32786536756A6D767078737262733662746D64756432696F2F6170702E62736B792E666565642E6C696B652F336C637933666F6E617A683232' seq:0, type:1](0) 117 + 3069:60238326[0 .. 0]['6469643A706C633A32786536756A6D767078737262733662746D64756432696F2F6170702E62736B792E666565642E6C696B652F336C6379336736727562763237' seq:0, type:1 .. '6469643A706C633A3278696568766E797136376F656E75716137376B377336782F6170702E62736B792E666565642E706F73742F336C6A326D6D3669676F32326A' seq:0, type:1](0) 118 + 3057:268858500[0 .. 0]['6469643A706C633A3278696568766E797136376F656E75716137376B377336782F6170702E62736B792E666565642E706F73742F336C6A32707A6761366373327A' seq:0, type:1 .. '6469643A706C633A327877686369736B7A63346E686C723475736F71763775762F6170702E62736B792E666565642E7265706F73742F336C796B7A6372747A736D3233' seq:0, type:1](0) 119 + 3067:60756781[0 .. 0]['6469643A706C633A327877686369736B7A63346E686C723475736F71763775762F6170702E62736B792E666565642E7265706F73742F336C796B7A6433643778353276' seq:0, type:1 .. '6469643A706C633A32793537656C377A7679616F68366E74766762766865666A2F6170702E62736B792E666565642E6C696B652F336B7A7565667377786D653232' seq:0, type:1](0) 120 + 3058:268860209[0 .. 0]['6469643A706C633A32793537656C377A7679616F68366E74766762766865666A2F6170702E62736B792E666565642E6C696B652F336B7A7565677877626777326B' seq:0, type:1 .. '6469643A706C633A3279706171646F7375657532616B6363766336617177787A2F6170702E62736B792E666565642E706F73742F336D3562617375657A32623275' seq:0, type:1](0) 121 + 3073:58976748[0 .. 0]['6469643A706C633A3279706171646F7375657532616B6363766336617177787A2F6170702E62736B792E666565642E706F73742F336D356261737570717832327A' seq:0, type:1 .. '6469643A706C633A32797468706A34707777706B61326C6A6B61626F7575626D2F6170702E62736B792E666565642E706F73742F336C6F733268776F78346B326B' seq:0, type:1](0) 122 + 3059:268867287[0 .. 0]['6469643A706C633A32797468706A34707777706B61326C6A6B61626F7575626D2F6170702E62736B792E666565642E706F73742F336C6F737370786B337A733274' seq:0, type:1 .. '6469643A706C633A327A676C7573753333346A3474366B6173776475356C73652F6170702E62736B792E666565642E6C696B652F336C76656F64703473646B3266' seq:0, type:1](0) 123 + 3076:58758534[0 .. 0]['6469643A706C633A327A676C7573753333346A3474366B6173776475356C73652F6170702E62736B792E666565642E6C696B652F336C76656F647773676D713273' seq:0, type:1 .. '6469643A706C633A327A6C68706F6F6D6F7572746B356C726B3335726F6565642F6170702E62736B792E666565642E6C696B652F336C646B6E796562667671326B' seq:0, type:1](0) 124 + 3062:268865371[0 .. 0]['6469643A706C633A327A6C68706F6F6D6F7572746B356C726B3335726F6565642F6170702E62736B792E666565642E6C696B652F336C646B6E79657A7634643278' seq:0, type:1 .. '6469643A706C633A333237706474616173723767687A367064796568787A34722F6170702E62736B792E666565642E6C696B652F336C79697A6D77346B6F6D326A' seq:0, type:1](0) 125 + 3070:60458923[0 .. 0]['6469643A706C633A333237706474616173723767687A367064796568787A34722F6170702E62736B792E666565642E6C696B652F336C79697A6E343333646E326E' seq:0, type:1 .. '6469643A706C633A33326373787678703335706B746F676B6A667370706575782F6170702E62736B792E666565642E6C696B652F336C746F7934766B6C35743232' seq:0, type:1](0) 126 + 3063:268854904[0 .. 0]['6469643A706C633A33326373787678703335706B746F676B6A667370706575782F6170702E62736B792E666565642E6C696B652F336C746F793563656366683269' seq:0, type:1 .. '6469643A706C633A333272613765336F6D79637A65343376736E7968727433652F6170702E62736B792E666565642E6C696B652F336B783279366F366262763263' seq:0, type:1](0) 127 + 3068:59185510[0 .. 0]['6469643A706C633A333272613765336F6D79637A65343376736E7968727433652F6170702E62736B792E666565642E6C696B652F336B7832793737626437743265' seq:0, type:1 .. '6469643A706C633A333277663274646263686763693233357963676B627869332F6170702E62736B792E666565642E6C696B652F336C726A6C7370747632753234' seq:0, type:1](0) 128 + 3061:268870898[0 .. 0]['6469643A706C633A333277663274646263686763693233357963676B627869332F6170702E62736B792E666565642E6C696B652F336C726A6C7377726833733274' seq:0, type:1 .. '6469643A706C633A333369753675327937663275637865666172376C6A7433762F6170702E62736B792E666565642E6C696B652F336B3472733279666464673267' seq:0, type:1](0) 129 + ... (249 more lines) 130 + 131 + === Compression Ratios === 132 + L0: -1.000000 133 + L1: -1.000000 134 + L2: -1.000000 135 + L3: -1.000000 136 + L4: -1.000000 137 + L5: -1.000000 138 + L6: 3.825617 139 + 140 + === Live SST Files (342 total) === 141 + Level Name Size (MiB) Size (bytes) Entries 142 + L6 /002977.sst 256.4 268862614 3276383 143 + L6 /002978.sst 256.4 268863540 3205414 144 + L6 /002979.sst 256.4 268861560 3274927 145 + L6 /002980.sst 256.4 268873207 3271956 146 + L6 /002981.sst 256.4 268868054 3242270 147 + L6 /002982.sst 256.4 268858158 3249309 148 + L6 /002983.sst 256.4 268861238 3224275 149 + L6 /002984.sst 256.4 268868826 3161092 150 + L6 /002985.sst 256.4 268861442 3216341 151 + L6 /002986.sst 256.4 268866712 3230393 152 + L6 /002987.sst 56.0 58754567 699554 153 + L6 /002988.sst 54.0 56596617 667028 154 + L6 /002989.sst 54.6 57240801 661240 155 + L6 /002990.sst 55.5 58243804 729394 156 + L6 /002991.sst 55.1 57796974 709922 157 + L6 /002992.sst 55.6 58269042 708449 158 + L6 /002993.sst 54.7 57408071 690180 159 + L6 /002994.sst 56.1 58826933 706139 160 + L6 /002995.sst 55.5 58143950 692731 161 + L6 /002996.sst 56.0 58726230 740125 162 + L6 /002997.sst 256.4 268860983 3172277 163 + L6 /002998.sst 256.4 268865193 3165917 164 + L6 /002999.sst 256.4 268859763 3240811 165 + L6 /003000.sst 256.4 268869110 3182763 166 + L6 /003001.sst 256.4 268851211 3122652 167 + L6 /003002.sst 256.4 268858725 3237410 168 + L6 /003003.sst 256.4 268859183 3146922 169 + L6 /003004.sst 256.4 268878428 3293679 170 + L6 /003005.sst 256.4 268875961 3334403 171 + L6 /003006.sst 256.4 268867983 3215218 172 + L6 /003007.sst 48.1 50456087 610756 173 + L6 /003008.sst 47.6 49941547 589929 174 + L6 /003009.sst 45.5 47761296 629798 175 + L6 /003010.sst 45.5 47692215 586233 176 + L6 /003011.sst 45.9 48111091 661140 177 + L6 /003012.sst 46.4 48660516 594008 178 + L6 /003013.sst 46.7 49013595 611531 179 + L6 /003014.sst 45.3 47460314 573788 180 + L6 /003015.sst 47.1 49418209 610541 181 + L6 /003016.sst 46.7 49005391 552302 182 + L6 /003017.sst 256.4 268865566 3214633 183 + L6 /003018.sst 256.4 268866581 3386094 184 + L6 /003019.sst 256.4 268860135 3236831 185 + L6 /003020.sst 256.4 268855547 3030425 186 + L6 /003021.sst 256.4 268870507 3276826 187 + L6 /003022.sst 256.4 268864661 3121457 188 + L6 /003023.sst 256.4 268865620 3129425 189 + L6 /003024.sst 256.4 268857116 3101500 190 + L6 /003025.sst 256.4 268853156 3202160 191 + L6 /003026.sst 256.4 268866516 3168814 192 + L6 /003027.sst 43.9 46042194 548655 193 + L6 /003028.sst 42.9 44965751 490843 194 + L6 /003029.sst 43.0 45036982 556370 195 + L6 /003030.sst 41.3 43303643 532114 196 + L6 /003031.sst 41.6 43662350 543547 197 + L6 /003032.sst 43.8 45881657 527788 198 + L6 /003033.sst 42.3 44306441 523605 199 + L6 /003034.sst 41.7 43720209 514003 200 + L6 /003035.sst 44.3 46421856 569326 201 + L6 /003036.sst 44.0 46122242 597276 202 + L6 /003037.sst 256.4 268866317 3190186 203 + L6 /003038.sst 256.4 268857948 3263342 204 + L6 /003039.sst 256.4 268859073 3293917 205 + L6 /003040.sst 256.4 268861807 3259109 206 + L6 /003041.sst 256.4 268862386 3200493 207 + L6 /003042.sst 256.4 268865393 3117844 208 + L6 /003043.sst 256.4 268872932 3348477 209 + L6 /003044.sst 256.4 268861905 3207536 210 + L6 /003045.sst 256.4 268863107 3216575 211 + L6 /003046.sst 256.4 268864680 3207041 212 + L6 /003047.sst 50.7 53169771 638878 213 + L6 /003048.sst 50.7 53124111 654481 214 + L6 /003049.sst 52.7 55289645 668810 215 + L6 /003050.sst 52.1 54582384 616540 216 + L6 /003051.sst 50.7 53198609 630495 217 + L6 /003052.sst 51.2 53666253 665800 218 + L6 /003053.sst 53.7 56261157 670967 219 + L6 /003054.sst 52.0 54499248 647974 220 + L6 /003055.sst 50.2 52683502 696254 221 + L6 /003056.sst 51.8 54348105 664675 222 + L6 /003057.sst 256.4 268858500 3208643 223 + L6 /003058.sst 256.4 268860209 3188353 224 + L6 /003059.sst 256.4 268867287 3240825 225 + L6 /003060.sst 256.4 268857163 3246988 226 + L6 /003061.sst 256.4 268870898 3200174 227 + L6 /003062.sst 256.4 268865371 3147083 228 + L6 /003063.sst 256.4 268854904 3263976 229 + L6 /003064.sst 256.4 268864822 3146851 230 + L6 /003065.sst 256.4 268865182 3286606 231 + L6 /003066.sst 256.4 268856043 3161095 232 + L6 /003067.sst 57.9 60756781 723462 233 + L6 /003068.sst 56.4 59185510 725641 234 + L6 /003069.sst 57.4 60238326 702111 235 + L6 /003070.sst 57.7 60458923 702165 236 + L6 /003071.sst 56.0 58743595 662586 237 + L6 /003072.sst 56.4 59117632 709524 238 + L6 /003073.sst 56.2 58976748 800752 239 + L6 /003074.sst 55.8 58464758 710214 240 + L6 /003075.sst 54.1 56738797 657114 241 + L6 /003076.sst 56.0 58758534 688227 242 + L6 /003077.sst 256.4 268858476 3157217 243 + L6 /003078.sst 256.4 268866422 3250229 244 + L6 /003079.sst 256.4 268858514 3193715 245 + L6 /003080.sst 256.4 268884543 3578473 246 + L6 /003081.sst 256.4 268855902 3202340 247 + L6 /003082.sst 256.4 268858119 3157036 248 + L6 /003083.sst 256.4 268867157 3211522 249 + L6 /003084.sst 256.4 268863876 3239316 250 + L6 /003085.sst 256.4 268855862 3193150 251 + L6 /003086.sst 256.4 268862526 3115013 252 + L6 /003087.sst 62.4 65436067 876006 253 + L6 /003088.sst 62.5 65492722 773555 254 + L6 /003089.sst 63.7 66791661 824052 255 + L6 /003090.sst 61.0 63951634 757055 256 + L6 /003091.sst 61.4 64398887 695459 257 + L6 /003092.sst 59.9 62813746 784690 258 + L6 /003093.sst 64.5 67649396 792956 259 + L6 /003094.sst 62.9 65915250 771150 260 + L6 /003095.sst 63.5 66544239 859051 261 + L6 /003096.sst 65.1 68285246 799075 262 + L6 /003097.sst 256.4 268871964 3236503 263 + L6 /003098.sst 256.4 268861038 3175946 264 + L6 /003099.sst 256.4 268860480 3172882 265 + L6 /003100.sst 256.4 268853986 3179349 266 + L6 /003101.sst 256.4 268863689 3319562 267 + L6 /003102.sst 256.4 268854744 3228953 268 + L6 /003103.sst 256.4 268858448 3170355 269 + L6 /003104.sst 256.4 268860635 3080658 270 + L6 /003105.sst 256.4 268862019 3099253 271 + L6 /003106.sst 256.4 268867301 3249732 272 + L6 /003107.sst 59.2 62045964 749435 273 + L6 /003108.sst 58.4 61200642 703378 274 + L6 /003109.sst 58.6 61438491 710299 275 + L6 /003110.sst 57.6 60362675 746173 276 + L6 /003111.sst 56.9 59650909 725793 277 + L6 /003112.sst 58.6 61482712 718285 278 + L6 /003113.sst 57.4 60160017 718722 279 + L6 /003114.sst 59.3 62214995 779507 280 + L6 /003115.sst 58.7 61501498 725176 281 + L6 /003116.sst 58.6 61462917 730847 282 + L6 /003117.sst 256.4 268861405 3133537 283 + L6 /003118.sst 256.4 268864479 3107500 284 + L6 /003119.sst 256.4 268875937 3343080 285 + L6 /003120.sst 256.4 268862517 3165597 286 + L6 /003121.sst 256.4 268864544 3272407 287 + L6 /003122.sst 256.4 268859636 3137050 288 + L6 /003123.sst 256.4 268872081 3301664 289 + L6 /003124.sst 256.4 268865987 3176258 290 + L6 /003125.sst 256.4 268864638 3222350 291 + L6 /003126.sst 256.4 268870166 3165072 292 + L6 /003127.sst 50.6 53063838 622272 293 + L6 /003128.sst 49.8 52251897 626433 294 + L6 /003129.sst 51.7 54222857 684449 295 + L6 /003130.sst 51.1 53618671 628301 296 + L6 /003131.sst 50.1 52563871 684837 297 + L6 /003132.sst 48.9 51327189 563544 298 + L6 /003133.sst 52.3 54828571 663558 299 + L6 /003134.sst 51.3 53837215 687009 300 + L6 /003135.sst 51.0 53489582 642071 301 + L6 /003136.sst 51.1 53633885 632123 302 + L6 /003137.sst 256.4 268858369 3221625 303 + L6 /003138.sst 256.4 268862675 3246344 304 + L6 /003139.sst 256.4 268856225 3230236 305 + L6 /003140.sst 256.4 268863140 3180595 306 + L6 /003141.sst 256.4 268865880 3175348 307 + L6 /003142.sst 256.4 268866128 3212405 308 + L6 /003143.sst 256.4 268866020 3126200 309 + L6 /003144.sst 256.4 268861169 3269356 310 + L6 /003145.sst 256.4 268863083 3182215 311 + L6 /003146.sst 256.4 268861222 3199114 312 + L6 /003147.sst 58.5 61325241 775069 313 + L6 /003148.sst 58.5 61352571 760372 314 + L6 /003149.sst 60.0 62887437 688991 315 + L6 /003150.sst 61.5 64477536 763386 316 + L6 /003151.sst 59.8 62729456 724418 317 + L6 /003152.sst 56.7 59454702 697947 318 + L6 /003153.sst 58.4 61270256 733206 319 + L6 /003154.sst 59.8 62677541 767872 320 + L6 /003155.sst 57.6 60422099 733541 321 + L6 /003156.sst 60.3 63269003 748565 322 + L6 /003157.sst 256.4 268856968 3005972 323 + L6 /003158.sst 256.4 268862341 3187355 324 + L6 /003159.sst 256.4 268863460 3212984 325 + L6 /003160.sst 256.4 268869217 3435330 326 + L6 /003161.sst 256.4 268871188 3288067 327 + L6 /003162.sst 256.4 268862053 3198586 328 + L6 /003163.sst 256.4 268854731 3150601 329 + L6 /003164.sst 256.4 268872799 3280981 330 + L6 /003165.sst 256.4 268864842 3198084 331 + L6 /003166.sst 256.4 268864613 3196915 332 + L6 /003167.sst 39.5 41391480 462030 333 + L6 /003168.sst 41.9 43895962 520435 334 + L6 /003169.sst 41.3 43326836 542909 335 + L6 /003170.sst 42.0 43989612 526032 336 + L6 /003171.sst 41.6 43642571 511527 337 + L6 /003172.sst 41.9 43921683 543282 338 + L6 /003173.sst 38.3 40198677 496308 339 + L6 /003174.sst 41.0 43018499 500878 340 + L6 /003175.sst 43.8 45901033 554405 341 + L6 /003176.sst 41.5 43565630 509942 342 + L6 /003177.sst 256.4 268860179 3220701 343 + L6 /003178.sst 256.4 268867739 3195633 344 + L6 /003179.sst 256.4 268865409 3185029 345 + L6 /003180.sst 256.4 268868613 3136484 346 + L6 /003181.sst 256.4 268863095 3199370 347 + L6 /003182.sst 256.4 268865885 3183188 348 + L6 /003183.sst 256.4 268865968 3281885 349 + L6 /003184.sst 256.4 268847298 2552178 350 + L6 /003185.sst 256.4 268867204 3227566 351 + L6 /003186.sst 256.4 268859917 3257118 352 + L6 /003187.sst 42.0 43993578 546307 353 + L6 /003188.sst 43.1 45145734 524024 354 + L6 /003189.sst 42.3 44383195 527793 355 + L6 /003190.sst 42.2 44200532 513463 356 + L6 /003191.sst 40.1 42027341 502349 357 + L6 /003192.sst 40.9 42886471 563526 358 + L6 /003193.sst 42.5 44521889 534889 359 + L6 /003194.sst 42.4 44408635 718293 360 + L6 /003195.sst 39.3 41234508 513861 361 + L6 /003196.sst 42.4 44499863 572143 362 + L6 /003197.sst 256.4 268866913 3193767 363 + L6 /003198.sst 256.4 268864501 3142515 364 + L6 /003199.sst 256.4 268854026 3169061 365 + L6 /003200.sst 256.4 268867177 3276247 366 + L6 /003201.sst 256.4 268863298 3252091 367 + L6 /003202.sst 256.4 268867723 3223326 368 + L6 /003203.sst 256.4 268862295 3180425 369 + L6 /003204.sst 256.4 268863849 3131459 370 + L6 /003205.sst 256.4 268864104 3199325 371 + L6 /003206.sst 256.4 268864314 3259808 372 + L6 /003207.sst 42.6 44677378 545414 373 + L6 /003208.sst 40.8 42820691 501173 374 + L6 /003209.sst 42.5 44569396 543574 375 + L6 /003210.sst 42.3 44369122 512980 376 + L6 /003211.sst 41.5 43516885 532859 377 + L6 /003212.sst 41.7 43690697 528902 378 + L6 /003213.sst 40.4 42346831 508398 379 + L6 /003214.sst 42.0 44014410 530676 380 + L6 /003215.sst 40.8 42808388 551741 381 + L6 /003216.sst 42.5 44549734 522489 382 + L6 /003217.sst 256.4 268862641 3207204 383 + L6 /003218.sst 256.4 268863668 3082556 384 + L6 /003219.sst 256.4 268868964 3293881 385 + L6 /003220.sst 256.4 268866876 3296490 386 + L6 /003221.sst 256.4 268871694 3336165 387 + L6 /003222.sst 256.4 268863736 3210560 388 + L6 /003223.sst 256.4 268866602 3128406 389 + L6 /003224.sst 256.4 268858377 3246305 390 + L6 /003225.sst 256.4 268866849 3293066 391 + L6 /003226.sst 256.4 268857998 3230417 392 + L6 /003227.sst 40.8 42801140 583170 393 + L6 /003228.sst 40.2 42162156 521559 394 + L6 /003229.sst 40.7 42682876 492196 395 + L6 /003230.sst 38.9 40784147 514353 396 + L6 /003231.sst 38.2 40099737 495375 397 + L6 /003232.sst 39.0 40900395 518788 398 + L6 /003233.sst 36.9 38735645 489202 399 + L6 /003234.sst 39.4 41270032 467314 400 + L6 /003235.sst 40.3 42210365 491418 401 + L6 /003236.sst 40.9 42859293 523434 402 + L6 /003237.sst 256.4 268872024 3221528 403 + L6 /003238.sst 256.4 268871391 3152584 404 + L6 /003239.sst 256.4 268873934 3223183 405 + L6 /003240.sst 256.4 268896298 3461847 406 + L6 /003241.sst 256.4 268871677 3271598 407 + L6 /003242.sst 256.4 268897511 5015832 408 + L6 /003243.sst 256.4 268899179 3351433 409 + L6 /003244.sst 256.5 268911361 3408847 410 + L6 /003245.sst 256.5 268919905 3472088 411 + L6 /003246.sst 256.4 268900265 3464565 412 + L6 /003247.sst 38.0 39806264 496882 413 + L6 /003248.sst 41.4 43435386 517387 414 + L6 /003249.sst 39.4 41262438 524043 415 + L6 /003250.sst 42.4 44412123 550949 416 + L6 /003251.sst 41.2 43233815 508729 417 + L6 /003252.sst 40.4 42410623 502580 418 + L6 /003253.sst 39.5 41409168 533025 419 + L6 /003254.sst 38.1 39911925 505291 420 + L6 /003255.sst 39.6 41520873 490207 421 + L6 /003256.sst 41.5 43486711 585633 422 + L6 /003257.sst 256.5 268912316 3472178 423 + L6 /003258.sst 256.5 268912117 3312977 424 + L6 /003259.sst 256.4 268905922 3330836 425 + L6 /003260.sst 256.4 268901023 3490335 426 + L6 /003261.sst 256.5 268912424 3402032 427 + L6 /003262.sst 256.4 268900819 3453537 428 + L6 /003263.sst 256.4 268902592 3378925 429 + L6 /003264.sst 256.4 268897802 3237374 430 + L6 /003265.sst 256.5 268908390 3442247 431 + L6 /003266.sst 256.4 268903779 3465077 432 + L6 /003267.sst 43.1 45245683 602088 433 + L6 /003268.sst 42.9 44946336 633779 434 + L6 /003269.sst 42.0 44051626 602150 435 + L6 /003270.sst 43.4 45516123 580797 436 + L6 /003271.sst 41.1 43065426 500121 437 + L6 /003272.sst 42.0 44067669 571477 438 + L6 /003273.sst 41.9 43972387 570966 439 + L6 /003274.sst 40.0 41984262 530728 440 + L6 /003275.sst 43.3 45452602 580964 441 + L6 /003276.sst 43.0 45045736 619937 442 + L6 /003277.sst 256.4 268902254 3453584 443 + L6 /003278.sst 256.4 268901773 3495945 444 + L6 /003279.sst 256.5 268911172 3454322 445 + L6 /003280.sst 256.5 268913116 3444020 446 + L6 /003281.sst 256.5 268915939 3475868 447 + L6 /003282.sst 256.4 268896334 3396266 448 + L6 /003283.sst 256.4 268905118 3427140 449 + L6 /003284.sst 256.5 268907822 3387635 450 + L6 /003285.sst 256.5 268916113 3612414 451 + L6 /003286.sst 256.4 268897249 3468662 452 + L6 /003287.sst 53.4 56020775 767446 453 + L6 /003288.sst 53.4 56019269 712525 454 + L6 /003289.sst 54.6 57244373 758709 455 + L6 /003290.sst 55.2 57863501 642612 456 + L6 /003291.sst 55.1 57810294 749960 457 + L6 /003292.sst 55.5 58239100 762226 458 + L6 /003293.sst 55.8 58534092 735171 459 + L6 /003294.sst 53.9 56542603 741244 460 + L6 /003295.sst 53.5 56128633 739028 461 + L6 /003296.sst 54.5 57109049 738403 462 + L6 /003297.sst 256.4 268898782 3381552 463 + L6 /003298.sst 256.5 268909804 3538990 464 + L6 /003299.sst 256.5 268910476 3365519 465 + L6 /003300.sst 256.4 268907014 3431448 466 + L6 /003301.sst 256.5 268918208 3565244 467 + L6 /003302.sst 256.5 268914078 3451541 468 + L6 /003303.sst 256.5 268935459 3186169 469 + L6 /003304.sst 256.4 268905433 3448609 470 + L6 /003305.sst 256.5 268908458 3439849 471 + L6 /003306.sst 256.5 268916977 3459214 472 + L6 /003307.sst 45.8 47999980 600957 473 + L6 /003308.sst 49.0 51383035 696103 474 + L6 /003309.sst 50.6 53062575 727276 475 + L6 /003310.sst 48.7 51071404 644918 476 + L6 /003311.sst 50.2 52614219 680154 477 + L6 /003312.sst 48.3 50617420 621601 478 + L6 /003313.sst 49.9 52294191 673927 479 + L6 /003314.sst 47.2 49515619 605872 480 + L6 /003315.sst 49.0 51350586 666331 481 + L6 /003316.sst 47.5 49756657 619661 482 + L6 /003317.sst 33.0 34569387 442654 483 + L6 /003318.sst 256.5 268919693 3533873 484 + TOTAL 52213.8 54750108385 665928371
+383
space-efficiency-check/readme.md
··· 56 56 just 1mb better, eeh 57 57 494s 58 58 59 + #### more attempts at dict settings 60 + 50236 61 + 497.587948917s 59 62 63 + #### 256KB blocks 64 + mb after: 49151 (51539349907) (4.8x) 65 + 66 + 67 + #### rerunning from cars with new config 68 + 69 + - no compression of L0 70 + - lz4 for L1 71 + - zstd below 72 + 73 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 74 + workers finished elapsed=1131.757322333s 75 + import complete repos=490328 empty=19172 records=665928371 failed=366 76 + total input size car_bytes=246501100813 car_mib=235081 77 + db size before compaction bytes=76787876531 mib=73230 78 + running manual compaction... 79 + db size after compaction bytes=51539292437 mib=49151 compact_elapsed=470.095283125s 80 + db size / car bytes ratio="0.209" 81 + 82 + 83 + - 22% faster load 84 + - 11% bigger before compaction 85 + - 38% faster compaction 86 + - same size post-compaction 87 + 88 + to try: 89 + 90 + - lz4 for L0 91 + - snappy for L1 92 + - zstd below 93 + 94 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 95 + workers finished elapsed=769.148841542s 96 + import complete repos=490328 empty=19172 records=665928371 failed=366 97 + total input size car_bytes=246501100813 car_mib=235081 98 + db size before compaction bytes=72627779170 mib=69263 99 + running manual compaction... 100 + db size after compaction bytes=51539427006 mib=49151 compact_elapsed=440.631236042s 101 + db size / car bytes ratio="0.209" 102 + 103 + - 48% faster! 104 + - 5% bigger before compaction 105 + - 41% faster compaction 106 + - same size post 107 + 108 + try: 109 + - snappy L0 110 + - zstd 3 below 111 + - zstd 6 bottom 112 + 113 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 114 + workers finished elapsed=706.999255833s 115 + import complete repos=490328 empty=19172 records=665928371 failed=366 116 + total input size car_bytes=246501100813 car_mib=235081 117 + db size before compaction bytes=74153258877 mib=70718 118 + running manual compaction... 119 + db size after compaction bytes=49509919781 mib=47216 compact_elapsed=736.388699625s 120 + db size / car bytes ratio="0.201" 121 + 122 + - 51% faster 123 + - 7% bigger before compaction 124 + - 2% faster compaction 125 + - 4% smaller post! (allllmost 5x) 126 + 127 + 128 + zstd 4 above bottom (snappy L0): 129 + 130 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 131 + workers finished elapsed=796.088198709s 132 + import complete repos=490328 empty=19172 records=665928371 failed=366 133 + total input size car_bytes=246501100813 car_mib=235081 134 + db size before compaction bytes=63662316943 mib=60713 135 + running manual compaction... 136 + db size after compaction bytes=49511141686 mib=47217 compact_elapsed=771.272409958s 137 + db size / car bytes ratio="0.201" 138 + 139 + - 45% faster 140 + - 8% bigger before compaction 141 + - 2% slower compaction 142 + - 4% smaller post (allllllmot 5x still) 143 + 144 + 145 + try: 146 + no zstd dictionaries at all 147 + 148 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 149 + workers finished elapsed=831.602000625s 150 + import complete repos=490328 empty=19172 records=665928371 failed=366 151 + total input size car_bytes=246501100813 car_mib=235081 152 + db size before compaction bytes=77854990237 mib=74248 153 + running manual compaction... 154 + db size after compaction bytes=49510481839 mib=47216 compact_elapsed=628.108967125s 155 + db size / car bytes ratio="0.201" 156 + 157 + - 43% faster 158 + - 13% bigger 159 + - 16% faster manual compaction 160 + - 4% smaller post 161 + 162 + 163 + ==== so dictionaries are actually not worth it (at least for records) ==== 164 + 165 + try: 166 + - lz4 L0 then zstd 3 (default options, no bottommost setting) 167 + - optimize_level_style_compaction(2GB) replaces write_buffer_size, max_write_buffer_number 168 + 169 + -- 170 + 171 + turn off unordered write: 172 + 173 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 174 + workers finished elapsed=1461.684272667s 175 + import complete repos=490327 empty=19172 records=663121972 failed=367 176 + total input size car_bytes=246501100813 car_mib=235081 177 + db size before compaction bytes=85902581906 mib=81923 178 + running manual compaction... 179 + db size after compaction bytes=78751866099 mib=75103 compact_elapsed=340.036456667s 180 + db size / car bytes ratio="0.319" 181 + 182 + - same speed 183 + - 25% bigger 184 + - 55% faster compact 185 + - 19% bigger post 186 + wtf 187 + 188 + same but dictionary again: 189 + 190 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 191 + progress repos=490000 records=665287800 failed=366 192 + workers finished elapsed=1510.41171175s 193 + import complete repos=490328 empty=19172 records=665928371 failed=366 194 + total input size car_bytes=246501100813 car_mib=235081 195 + db size before compaction bytes=96642919507 mib=92165 196 + running manual compaction... 197 + db size after compaction bytes=78766420371 mib=75117 compact_elapsed=328.5108135s 198 + db size / car bytes ratio="0.320" 199 + 200 + what. 201 + 202 + zstd 4 again (no dict): 203 + 204 + 2026-04-08T23:39:58.226569Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 205 + 2026-04-08T23:40:01.813198Z INFO space_efficiency_check::work: progress repos=490000 records=665341081 failed=366 206 + 2026-04-08T23:40:02.553634Z INFO space_efficiency_check::work: workers finished elapsed=1444.968804209s 207 + 2026-04-08T23:40:02.553657Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 208 + 2026-04-08T23:40:02.553660Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 209 + 2026-04-08T23:40:02.553700Z INFO space_efficiency_check: db size before compaction bytes=87279007693 mib=83235 210 + 2026-04-08T23:40:02.553702Z INFO space_efficiency_check: running manual compaction... 211 + 2026-04-08T23:45:32.431196Z INFO space_efficiency_check: db size after compaction bytes=78823738786 mib=75172 compact_elapsed=329.875497584s 212 + 2026-04-08T23:45:32.442654Z INFO space_efficiency_check: db size / car bytes ratio="0.320" 213 + 214 + zstd default includinb bottom (lz4 l0): 215 + 216 + 2026-04-09T01:16:41.254396Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 217 + 2026-04-09T01:16:41.460409Z INFO space_efficiency_check::work: workers finished elapsed=1345.288276167s 218 + 2026-04-09T01:16:41.462956Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 219 + 2026-04-09T01:16:41.462967Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 220 + 2026-04-09T01:16:41.463007Z INFO space_efficiency_check: db size before compaction bytes=65930950758 mib=62876 221 + 2026-04-09T01:16:41.463010Z INFO space_efficiency_check: running manual compaction... 222 + 2026-04-09T01:22:07.400826Z INFO space_efficiency_check: db size after compaction bytes=51537661824 mib=49150 compact_elapsed=325.940944625s 223 + 2026-04-09T01:22:07.408148Z INFO space_efficiency_check: db size / car bytes ratio="0.209" 224 + 225 + 226 + ok now, 256k => 64k block size: 227 + 228 + 2026-04-09T01:46:26.893956Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 229 + 2026-04-09T01:46:31.098925Z INFO space_efficiency_check::work: progress repos=490000 records=665025090 failed=367 230 + 2026-04-09T01:46:33.994992Z INFO space_efficiency_check::work: workers finished elapsed=1355.049800208s 231 + 2026-04-09T01:46:33.997804Z INFO space_efficiency_check: import complete repos=490327 empty=19172 records=665612139 failed=367 232 + 2026-04-09T01:46:33.997811Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 233 + 2026-04-09T01:46:33.997858Z INFO space_efficiency_check: db size before compaction bytes=75196160153 mib=71712 234 + 2026-04-09T01:46:33.997860Z INFO space_efficiency_check: running manual compaction... 235 + 2026-04-09T01:52:06.451996Z INFO space_efficiency_check: db size after compaction bytes=52672889189 mib=50232 compact_elapsed=332.454359958s 236 + 2026-04-09T01:52:06.466039Z INFO space_efficiency_check: db size / car bytes ratio="0.214" 237 + 238 + 128K block 239 + 240 + 2026-04-09T02:45:21.213202Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 241 + 2026-04-09T02:45:22.317758Z INFO space_efficiency_check::work: progress repos=490000 records=665288131 failed=366 242 + 2026-04-09T02:45:22.885135Z INFO space_efficiency_check::work: workers finished elapsed=1323.177367625s 243 + 2026-04-09T02:45:22.885177Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 244 + 2026-04-09T02:45:22.885186Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 245 + 2026-04-09T02:45:22.885234Z INFO space_efficiency_check: db size before compaction bytes=80563021019 mib=76830 246 + 2026-04-09T02:45:22.885242Z INFO space_efficiency_check: running manual compaction... 247 + 2026-04-09T02:50:39.873626Z INFO space_efficiency_check: db size after compaction bytes=51708170815 mib=49312 compact_elapsed=316.988863208s 248 + 2026-04-09T02:50:39.885508Z INFO space_efficiency_check: db size / car bytes ratio="0.210" 249 + 250 + 16K block 251 + 252 + 2026-04-09T03:15:58.493554Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 253 + 2026-04-09T03:15:58.540900Z INFO space_efficiency_check::work: workers finished elapsed=1398.781832625s 254 + 2026-04-09T03:15:58.545808Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 255 + 2026-04-09T03:15:58.545820Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 256 + 2026-04-09T03:15:58.545865Z INFO space_efficiency_check: db size before compaction bytes=86378449887 mib=82376 257 + 2026-04-09T03:15:58.545866Z INFO space_efficiency_check: running manual compaction... 258 + 2026-04-09T03:21:26.082768Z INFO space_efficiency_check: db size after compaction bytes=56222871034 mib=53618 compact_elapsed=327.471128792s 259 + 2026-04-09T03:21:26.136856Z INFO space_efficiency_check: db size / car bytes ratio="0.228" 260 + 261 + 8K block 262 + 263 + 2026-04-09T03:46:15.381506Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 264 + 2026-04-09T03:46:15.520773Z INFO space_efficiency_check::work: workers finished elapsed=1367.07904175s 265 + 2026-04-09T03:46:15.527295Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 266 + 2026-04-09T03:46:15.527307Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 267 + 2026-04-09T03:46:15.527348Z INFO space_efficiency_check: db size before compaction bytes=78343679632 mib=74714 268 + 2026-04-09T03:46:15.527350Z INFO space_efficiency_check: running manual compaction... 269 + 2026-04-09T03:53:05.373778Z INFO space_efficiency_check: db size after compaction bytes=59702615786 mib=56936 compact_elapsed=409.843443917s 270 + 2026-04-09T03:53:05.457495Z INFO space_efficiency_check: db size / car bytes ratio="0.242" 271 + 272 + 8k block + 16k dictionary (256k training): 273 + 274 + 2026-04-09T04:29:27.446190Z INFO space_efficiency_check::work: enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 275 + 2026-04-09T04:29:27.613743Z INFO space_efficiency_check::work: workers finished elapsed=1496.465470458s 276 + 2026-04-09T04:29:27.616045Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 277 + 2026-04-09T04:29:27.616057Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 278 + 2026-04-09T04:29:27.616111Z INFO space_efficiency_check: db size before compaction bytes=88278600013 mib=84189 279 + 2026-04-09T04:29:27.616113Z INFO space_efficiency_check: running manual compaction... 280 + 2026-04-09T04:38:28.940272Z INFO space_efficiency_check: db size after compaction bytes=57737984223 mib=55063 compact_elapsed=541.320268792s 281 + 2026-04-09T04:38:29.004843Z INFO space_efficiency_check: db size / car bytes ratio="0.234" 282 + 283 + 16k block + 16k dict: 284 + 285 + db size after compaction bytes=55663485496 mib=53084 compact_elapsed=204.063369459s 286 + 287 + 32k block + 16k dict: 288 + 289 + db size after compaction bytes=53999703450 mib=51498 compact_elapsed=143.929752417s 290 + 291 + 4k block + 16k dict: 292 + 293 + db size after compaction bytes=61098626429 mib=58268 compact_elapsed=136.135160458s 294 + 295 + 4k block + 0 dict: 296 + 297 + db size after compaction bytes=65956690117 mib=62901 compact_elapsed=136.727676042s 298 + 299 + 32k block + 0 dict: 300 + 301 + db size after compaction bytes=54041334794 mib=51537 compact_elapsed=98.191982917s 302 + 303 + 32k block + 0 dict, z6: 304 + 305 + db size after compaction bytes=53352328748 mib=50880 compact_elapsed=162.159684083s 306 + 307 + 32k block + 16k dict, z6: 308 + 309 + db size after compaction bytes=53021812545 mib=50565 compact_elapsed=203.065514958s 310 + 311 + 8k block + 16k dict, z6: 312 + 313 + db size after compaction bytes=57114197996 mib=54468 compact_elapsed=220.894172333s 314 + 315 + 8k block + 0 dict, z6: 316 + 317 + db size after compaction bytes=58704811352 mib=55985 compact_elapsed=177.26946475s 318 + 319 + 4k block + 0 dict, z6: 320 + 321 + db size after compaction bytes=64943379652 mib=61934 compact_elapsed=170.830192667s 322 + 323 + 4k block + 16k dict, z6: 324 + 325 + db size after compaction bytes=60537637714 mib=57733 compact_elapsed=242.577487375s 326 + 327 + ---- 328 + 329 + 4k block + 16k dict, z3, restart 8: 330 + 331 + db size after compaction bytes=61942405450 mib=59072 compact_elapsed=184.055436916s 332 + 333 + 4k block + 16k dict, z3, restart 16: 334 + 335 + db size after compaction bytes=61527257614 mib=58676 compact_elapsed=181.6921695s 336 + 337 + 4k block + 16k dict, z3, restart 32: 338 + 339 + db size after compaction bytes=61070969206 mib=58241 compact_elapsed=179.042228292s 340 + 341 + // 64 we already have 342 + 343 + 4k block + 16k dict, z3, restart 128: 344 + 345 + db size after compaction bytes=61079509127 mib=58249 compact_elapsed=180.653354292s 346 + 347 + 4k block + 0 dict, z3, restart 8: 348 + 349 + db size after compaction bytes=66798905955 mib=63704 compact_elapsed=140.909784041s 350 + 351 + 32k block + 0 dict, z3, restart 8: 352 + 353 + db size after compaction bytes=54771771953 mib=52234 compact_elapsed=102.226233584s 354 + 355 + db size after compaction bytes=54765640022 mib=52228 compact_elapsed=130.654591209s 356 + 357 + --= 358 + 359 + 4k block + 0 dict z3 restart 8: 360 + 361 + db size after compaction bytes=54771844999 mib=52234 compact_elapsed=77.579734459s 362 + 363 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 364 + workers finished elapsed=1366.711822333s 365 + import complete repos=490328 empty=19172 records=665928371 failed=366 366 + total input size car_bytes=246501100813 car_mib=235081 367 + db size before compaction bytes=69391838577 mib=66177 368 + running manual compaction... 369 + db size after compaction bytes=54771946443 mib=52234 compact_elapsed=292.145749542s 370 + db size / car bytes ratio="0.222" 371 + 372 + 373 + 4k block + 8 dict: 374 + 375 + 376 + 4k block + 16 dict: 377 + 378 + db size after compaction bytes=54771869920 mib=52234 compact_elapsed=83.646350833s 379 + 380 + 4k block + 32 dict: 381 + 382 + db size after compaction bytes=54771873968 mib=52234 compact_elapsed=82.5698695s 383 + 384 + 4k block + 64 dict: 385 + 386 + db size after compaction bytes=54771906488 mib=52234 compact_elapsed=83.8019415s 387 + 388 + 4k block + 128 dict: 389 + 390 + db size after compaction bytes=54771932651 mib=52234 compact_elapsed=80.826865292s 391 + 392 + 4k block + 256 dict: 393 + 394 + db size after compaction bytes=54771899715 mib=52234 compact_elapsed=81.088142667s 395 + 396 + 4k block + 512 dict: 397 + 398 + db size after compaction bytes=54771905067 mib=52234 compact_elapsed=82.468925208s 399 + 400 + 4k block + 1k dict: 401 + 402 + db size after compaction bytes=54771892224 mib=52234 compact_elapsed=85.311880917s 403 + 404 + 4k block + 2k dict: 405 + 406 + db size after compaction bytes=54771897772 mib=52234 compact_elapsed=81.808258125s 407 + 408 + 4k block + 4k dict: 409 + 410 + db size after compaction bytes=54771899652 mib=52234 compact_elapsed=81.196579125s 411 + 412 + 4k block + 8k dict: 413 + 414 + db size after compaction bytes=54771894044 mib=52234 compact_elapsed=81.155235041s 415 + 416 + 4k block + 16k dict: 417 + 418 + 026-04-09T14:40:02.228661Z INFO space_efficiency_check::work: workers finished elapsed=1516.877558667s 419 + 2026-04-09T14:40:02.260445Z INFO space_efficiency_check: import complete repos=490328 empty=19172 records=665928371 failed=366 420 + 2026-04-09T14:40:02.260455Z INFO space_efficiency_check: total input size car_bytes=246501100813 car_mib=235081 421 + 2026-04-09T14:40:02.260503Z INFO space_efficiency_check: db size before compaction bytes=83032996404 mib=79186 422 + 2026-04-09T14:40:02.260506Z INFO space_efficiency_check: running manual compaction... 423 + 2026-04-09T14:46:33.171340Z INFO space_efficiency_check: db size after compaction bytes=54739156417 mib=52203 compact_elapsed=390.910325s 424 + 2026-04-09T14:46:33.233832Z INFO space_efficiency_check: db size / car bytes ratio="0.222" 425 + 426 + 4k block + 16K dict (again, fresh run again): 427 + 428 + db size after compaction bytes=54750108385 mib=52213 compact_elapsed=397.067736125s 429 + 430 + 431 + 432 + 433 + 434 + and then maybe: 435 + 436 + - snappy L0 L1 437 + - zstd below 438 + 439 + and: 440 + 441 + - snappy L0 442 + - zstd below
+22
space-efficiency-check/src/lib.rs
··· 1 1 pub mod work; 2 + 3 + pub const PREFIX_ACCOUNT: u8 = b'a'; 4 + pub const PREFIX_RECORD: u8 = b'r'; 5 + 6 + /// account status values into bytes 7 + /// 8 + /// [active 0 or 1] [rev str] [0x00] [cid bytes] 9 + /// 10 + /// (rev can't contain null) 11 + pub fn encode_account_status(active: bool, rev: &str, cid: &[u8]) -> Vec<u8> { 12 + let mut buf = Vec::with_capacity( 13 + 1 + // active bool 14 + rev.len() + 15 + 1 + // null sep 16 + cid.len() 17 + ); 18 + buf.push(if active { 0x01 } else { 0x00 }); 19 + buf.extend_from_slice(rev.as_bytes()); 20 + buf.push(0x00); 21 + buf.extend_from_slice(cid); 22 + buf 23 + }
+69 -22
space-efficiency-check/src/main.rs
··· 1 1 use clap::Parser; 2 - use rust_rocksdb::{DB, CompactOptions, DBCompressionType, Options}; 2 + use rocksdb::{DB, CompactOptions, DBCompressionType, Options}; 3 3 4 4 use std::path::{Path, PathBuf}; 5 - use std::sync::{Arc, atomic::Ordering}; 5 + use std::sync::{Arc, atomic::{AtomicBool, Ordering}}; 6 6 7 7 use space_efficiency_check::work::{ProcessError, run_workers}; 8 8 ··· 29 29 just_compact: bool, 30 30 } 31 31 32 - fn open_db(path: &Path) -> Result<DB, rust_rocksdb::Error> { 32 + fn open_db(path: &Path) -> Result<DB, rocksdb::Error> { 33 33 let mut opts = Options::default(); 34 34 opts.create_if_missing(true); 35 35 36 36 // we don't need durability 37 - opts.set_manual_wal_flush(true); 37 + // redundant: we do `disable_wal(true)` on all writes 38 + // opts.set_manual_wal_flush(true); 38 39 39 40 // compress lower levels more 40 41 opts.set_compression_per_level(&[ 41 - DBCompressionType::None, // L0 42 - DBCompressionType::Lz4, // L1 42 + DBCompressionType::Lz4, // L0 43 + DBCompressionType::Zstd, // L1 43 44 DBCompressionType::Zstd, // L2 44 45 DBCompressionType::Zstd, // L3 45 46 DBCompressionType::Zstd, // L4 46 47 DBCompressionType::Zstd, // L5 47 48 DBCompressionType::Zstd, // L6 48 49 ]); 50 + 51 + const D: i32 = 16_384; 49 52 50 53 opts.set_compression_options( 51 54 -14, // window_bits (default, -14 means use zstd default) 52 55 3, // level (default 3) 53 56 0, // strategy (0 = default) 54 - 16_384, // max_dict_bytes (0 = no dictionary) (tried 16k) 57 + D, // max_dict_bytes (0 = no dictionary) (tried 16k) 55 58 ); 56 - opts.set_zstd_max_train_bytes(16 * 16_384); 59 + opts.set_zstd_max_train_bytes(16 * D); 57 60 58 - opts.set_bottommost_compression_type(rust_rocksdb::DBCompressionType::Zstd); 61 + opts.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); 59 62 60 63 opts.set_bottommost_compression_options( 61 64 -14, // window_bits (default, -14 means use zstd default) 62 65 3, // level (default 3) 63 66 0, // strategy (0 = default) 64 - 16_384, // max_dict_bytes (0 = no dictionary) (tried 16k) 67 + D, // max_dict_bytes (0 = no dictionary) (tried 16k) 65 68 true, // enabled 66 69 ); 67 70 68 71 // probably investigate this one: use more training data for dictionary 69 - opts.set_bottommost_zstd_max_train_bytes(16 * 16_384, true); // 256KB of samples for a 16KB dict 72 + opts.set_bottommost_zstd_max_train_bytes(16 * D, true); // 256KB of samples for a 16KB dict 70 73 71 74 // skip filters on bottom-most layer 72 75 opts.set_optimize_filters_for_hits(true); 73 76 74 - // write into new buffer while flushing? 75 - opts.set_max_write_buffer_number(4); 77 + // // write into new buffer while flushing? 78 + // opts.set_max_write_buffer_number(4); 79 + 80 + opts.set_max_subcompactions(10); // default 1 76 81 77 82 // ugh this probably wasn't exactly the right move 78 83 // there's increase_parallelism() ··· 80 85 // opts.set_max_background_jobs(8); 81 86 opts.increase_parallelism(10); 82 87 83 - // larger write buffer for less frequent flushes 84 - opts.set_write_buffer_size(256 * 2_usize.pow(20)); // default 64MB 85 - 86 88 // apparently can help with space amp, less partially-filled levels 87 89 opts.set_level_compaction_dynamic_level_bytes(true); 88 90 91 + opts.optimize_level_style_compaction(2 * 2_usize.pow(30)); 92 + 93 + // larger write buffer for less frequent flushes 94 + // opts.set_write_buffer_size(256 * 2_usize.pow(20)); // default 64MB 95 + 89 96 // larger sst files (ehhh) 90 97 // don't help 91 98 // opts.set_target_file_size_base(128 * 2_u64.pow(20)); // default 64MB 92 99 93 100 // larger blocks for better compression (good scan, worse point read) 94 - let mut block_opts = rust_rocksdb::BlockBasedOptions::default(); 95 - block_opts.set_block_size(256 * 2_usize.pow(10)); // N * KB blocks 96 - block_opts.set_block_restart_interval(64); 101 + let mut block_opts = rocksdb::BlockBasedOptions::default(); 102 + block_opts.set_block_size(32 * 2_usize.pow(10)); // N * KB blocks 103 + block_opts.set_block_restart_interval(8); 97 104 opts.set_block_based_table_factory(&block_opts); 98 105 99 106 // enable ONLY during bulk-import 100 - opts.set_unordered_write(true); 107 + // opts.set_unordered_write(true); 108 + // opts.set_level_zero_slowdown_writes_trigger(40); 109 + // opts.set_level_zero_stop_writes_trigger(80); 101 110 102 - opts.set_max_subcompactions(4); // default 1 103 111 104 112 DB::open(&opts, path) 105 113 } ··· 111 119 let args = Args::parse(); 112 120 113 121 let db = Arc::new(open_db(&args.db_dir)?); 122 + 123 + // let _guard = Monitor::start(db.clone()); 114 124 115 125 if args.just_compact { 116 126 manually_compact(db)?; ··· 151 161 152 162 fn manually_compact(db: Arc<DB>) -> Result<u64, ProcessError> { 153 163 let mut compact_opts = CompactOptions::default(); 154 - compact_opts.set_bottommost_level_compaction(rust_rocksdb::BottommostLevelCompaction::Force); 164 + compact_opts.set_bottommost_level_compaction(rocksdb::BottommostLevelCompaction::Force); 155 165 156 166 tracing::info!("running manual compaction..."); 157 167 let compact_start = std::time::Instant::now(); ··· 175 185 .and_then(|v| v.parse::<u64>().ok()) 176 186 .unwrap_or(0) 177 187 } 188 + 189 + struct Monitor { 190 + stop: Arc<AtomicBool>, 191 + } 192 + 193 + impl Drop for Monitor { 194 + fn drop(&mut self) { 195 + self.stop.store(true, Ordering::Relaxed) 196 + } 197 + } 198 + 199 + impl Monitor { 200 + fn start(db: Arc<DB>) -> Self { 201 + let stop: Arc<AtomicBool> = Arc::new(false.into()); 202 + tokio::task::spawn(Monitor::run(db, stop.clone())); 203 + Self { stop } 204 + } 205 + 206 + async fn run(db: Arc<DB>, stop: Arc<AtomicBool>) { 207 + while !stop.load(Ordering::Relaxed) { 208 + tracing::info!( 209 + is_stopped = db.property_value("rocksdb.is-write-stopped").unwrap(), 210 + actual_delayed_wirte_rate = db.property_value("rocksdb.actual-delayed-write-rate").unwrap(), 211 + l0_files = db.property_value("rocksdb.num-files-at-level0").unwrap(), 212 + "db status", 213 + ); 214 + 215 + // delay 5s but also check if we need to bail 216 + for _ in 0..10 { 217 + tokio::time::sleep(std::time::Duration::from_secs_f64(0.5)).await; 218 + if stop.load(Ordering::Relaxed) { 219 + return; 220 + } 221 + } 222 + } 223 + } 224 + }
+34 -8
space-efficiency-check/src/work.rs
··· 1 + use crate::{PREFIX_ACCOUNT, encode_account_status}; 1 2 use std::path::{Path, PathBuf}; 2 3 use std::sync::{ 3 4 Arc, 4 5 atomic::{AtomicU64, Ordering}, 5 6 }; 6 - use std::time::Instant; 7 + use std::time::{Duration, Instant}; 7 8 8 9 use repo_stream::DriverBuilder; 9 - use rust_rocksdb::{DB, WriteBatch}; 10 + use rocksdb::{DB, WriteBatch}; 10 11 use tokio::fs; 11 12 use tokio::io::BufReader; 12 13 use tokio::task::JoinSet; ··· 24 25 Walk(#[from] repo_stream::WalkError), 25 26 26 27 #[error("rocksdb: {0}")] 27 - Rocks(#[from] rust_rocksdb::Error), 28 + Rocks(#[from] rocksdb::Error), 28 29 29 30 #[error("channel send: {0}")] 30 31 SendError(String), ··· 97 98 mem_limit_mb: usize, 98 99 ) { 99 100 while let Ok(path) = rx.recv().await { 100 - if let Err(e) = process_car(&path, mem_limit_mb, db.clone(), &stats).await { 101 - warn!(?path, %e, "failed to process CAR"); 102 - stats.failed_repos.fetch_add(1, Ordering::Relaxed); 101 + let p = process_car(&path, mem_limit_mb, db.clone(), &stats); 102 + match tokio::time::timeout(Duration::from_secs(30), p).await { 103 + Ok(Ok(_)) => {} 104 + Ok(Err(e)) => { 105 + warn!(?path, %e, "failed to process CAR"); 106 + stats.failed_repos.fetch_add(1, Ordering::Relaxed); 107 + } 108 + Err(_) => { 109 + warn!(?path, "timed out on CAR processing"); 110 + stats.failed_repos.fetch_add(1, Ordering::Relaxed); 111 + } 103 112 } 104 113 } 105 114 } ··· 121 130 .map_err(|e| ProcessError::Load(e.to_string()))?; 122 131 123 132 let did = car.commit.did.clone(); 133 + let cid = car.commit.data.to_bytes(); 134 + 135 + if false { 136 + let mut account_key = Vec::with_capacity(1 + did.len()); 137 + account_key.push(PREFIX_ACCOUNT); 138 + account_key.extend_from_slice(&did.as_bytes()); 139 + let account_val = encode_account_status(true, &car.commit.rev, &cid); 140 + let db = db.clone(); 141 + tokio::task::spawn_blocking(move || { 142 + let mut write_opts = rocksdb::WriteOptions::default(); 143 + write_opts.disable_wal(true); 144 + db.put_opt(&account_key, &account_val, &write_opts) 145 + }) 146 + .await 147 + .expect("account status write not to joinerror")?; 148 + } 149 + 124 150 let mut record_count: u64 = 0; 125 151 126 152 const CHUNK_SIZE: usize = 32768; ··· 135 161 } 136 162 let db = db.clone(); 137 163 tokio::task::spawn_blocking(move || { 138 - let mut write_opts = rust_rocksdb::WriteOptions::default(); 164 + let mut write_opts = rocksdb::WriteOptions::default(); 139 165 write_opts.disable_wal(true); 140 - db.write_opt(&batch, &write_opts) 166 + db.write_opt(batch, &write_opts) 141 167 }) 142 168 .await 143 169 .expect("write batch not to joinerror")?;