+35

Cargo.lock

reviewed

··· 338 338 ] 339 339 340 340 [[package]] 341 341 + name = "csv" 342 342 + version = "1.4.0" 343 343 + source = "registry+https://github.com/rust-lang/crates.io-index" 344 344 + checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" 345 345 + dependencies = [ 346 346 + "csv-core", 347 347 + "itoa", 348 348 + "ryu", 349 349 + "serde_core", 350 350 + ] 351 351 + 352 352 + [[package]] 353 353 + name = "csv-core" 354 354 + version = "0.1.13" 355 355 + source = "registry+https://github.com/rust-lang/crates.io-index" 356 356 + checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" 357 357 + dependencies = [ 358 358 + "memchr", 359 359 + ] 360 360 + 361 361 + [[package]] 341 362 name = "dashmap" 342 363 version = "6.1.0" 343 364 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 663 684 dependencies = [ 664 685 "either", 665 686 ] 687 687 + 688 688 + [[package]] 689 689 + name = "itoa" 690 690 + version = "1.0.18" 691 691 + source = "registry+https://github.com/rust-lang/crates.io-index" 692 692 + checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" 666 693 667 694 [[package]] 668 695 name = "jobserver" ··· 1032 1059 ] 1033 1060 1034 1061 [[package]] 1062 1062 + name = "ryu" 1063 1063 + version = "1.0.23" 1064 1064 + source = "registry+https://github.com/rust-lang/crates.io-index" 1065 1065 + checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" 1066 1066 + 1067 1067 + [[package]] 1035 1068 name = "scopeguard" 1036 1069 version = "1.2.0" 1037 1070 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1170 1203 dependencies = [ 1171 1204 "async-channel", 1172 1205 "clap", 1206 1206 + "csv", 1173 1207 "repo-stream", 1174 1208 "rocksdb", 1209 1209 + "serde", 1175 1210 "thiserror 2.0.18", 1176 1211 "tokio", 1177 1212 "tracing",

+2

space-efficiency-check/Cargo.toml

reviewed

··· 12 12 tokio = { workspace = true, features = ["full"] } 13 13 tracing = { workspace = true } 14 14 tracing-subscriber = { workspace = true } 15 15 + csv = "1" 16 16 + serde = { version = "1", features = ["derive"] } 15 17 16 18 [profile.release] 17 19 debug = true

+147

space-efficiency-check/examples/sample.rs

reviewed

··· 1 1 + use clap::Parser; 2 2 + use rocksdb::{Options, WriteBatch, WriteOptions, DB}; 3 3 + use std::path::PathBuf; 4 4 + use std::time::Instant; 5 5 + 6 6 + /// Create a smaller test DB by copying contiguous key ranges from a compacted source DB. 7 7 + /// Picks every Nth SST file's key range to preserve DID-level key locality. 8 8 + #[derive(Parser)] 9 9 + struct Args { 10 10 + /// source DB directory (should be fully compacted to L6) 11 11 + source: PathBuf, 12 12 + 13 13 + /// destination DB directory (will be created) 14 14 + dest: PathBuf, 15 15 + 16 16 + /// fraction of SST files to sample (0.0–1.0) 17 17 + #[arg(long, default_value_t = 0.1)] 18 18 + fraction: f64, 19 19 + } 20 20 + 21 21 + fn main() { 22 22 + let args = Args::parse(); 23 23 + assert!( 24 24 + args.fraction > 0.0 && args.fraction <= 1.0, 25 25 + "fraction must be in (0.0, 1.0]" 26 26 + ); 27 27 + 28 28 + // open source read-only to get file list 29 29 + let source_opts = Options::default(); 30 30 + let source = 31 31 + DB::open_for_read_only(&source_opts, &args.source, false).expect("failed to open source db"); 32 32 + 33 33 + let mut files = source.live_files().expect("failed to get live files"); 34 34 + files.sort_by(|a, b| a.start_key.cmp(&b.start_key)); 35 35 + 36 36 + let total_size: usize = files.iter().map(|f| f.size).sum(); 37 37 + let total_entries: u64 = files.iter().map(|f| f.num_entries).sum(); 38 38 + eprintln!( 39 39 + "source: {} SST files, {} MiB, {} entries", 40 40 + files.len(), 41 41 + total_size / (1 << 20), 42 42 + total_entries, 43 43 + ); 44 44 + 45 45 + // pick every Nth file's key range 46 46 + let step = (1.0 / args.fraction).round().max(1.0) as usize; 47 47 + let selected: Vec<_> = files.iter().step_by(step).collect(); 48 48 + 49 49 + let sel_entries: u64 = selected.iter().map(|f| f.num_entries).sum(); 50 50 + eprintln!( 51 51 + "sampling {} file ranges (every {}th), ~{} entries", 52 52 + selected.len(), 53 53 + step, 54 54 + sel_entries, 55 55 + ); 56 56 + 57 57 + // collect key ranges to copy 58 58 + let ranges: Vec<_> = selected 59 59 + .iter() 60 60 + .filter_map(|f| { 61 61 + let start = f.start_key.as_ref()?; 62 62 + let end = f.end_key.as_ref()?; 63 63 + Some((start.clone(), end.clone())) 64 64 + }) 65 65 + .collect(); 66 66 + 67 67 + // create dest DB 68 68 + let mut dest_opts = Options::default(); 69 69 + dest_opts.create_if_missing(true); 70 70 + let dest = DB::open(&dest_opts, &args.dest).expect("failed to create dest db"); 71 71 + 72 72 + let mut write_opts = WriteOptions::default(); 73 73 + write_opts.disable_wal(true); 74 74 + 75 75 + // iterate source key ranges and write to dest 76 76 + let start = Instant::now(); 77 77 + let mut total_written: u64 = 0; 78 78 + 79 79 + for (i, (range_start, range_end)) in ranges.iter().enumerate() { 80 80 + let mut iter = source.raw_iterator(); 81 81 + iter.seek(range_start); 82 82 + 83 83 + let mut batch = WriteBatch::default(); 84 84 + let mut batch_count = 0; 85 85 + 86 86 + while iter.valid() { 87 87 + let key = iter.key().unwrap(); 88 88 + if key > range_end.as_slice() { 89 89 + break; 90 90 + } 91 91 + batch.put(key, iter.value().unwrap()); 92 92 + batch_count += 1; 93 93 + total_written += 1; 94 94 + 95 95 + if batch_count >= 32_768 { 96 96 + dest.write_opt(std::mem::replace(&mut batch, WriteBatch::default()), &write_opts) 97 97 + .expect("failed to write batch"); 98 98 + batch_count = 0; 99 99 + } 100 100 + 101 101 + iter.next(); 102 102 + } 103 103 + 104 104 + if batch_count > 0 { 105 105 + dest.write_opt(batch, &write_opts) 106 106 + .expect("failed to write batch"); 107 107 + } 108 108 + 109 109 + if (i + 1) % 10 == 0 || i + 1 == ranges.len() { 110 110 + eprintln!( 111 111 + " copied {}/{} ranges, {} entries so far ({:.1}s)", 112 112 + i + 1, 113 113 + ranges.len(), 114 114 + total_written, 115 115 + start.elapsed().as_secs_f64(), 116 116 + ); 117 117 + } 118 118 + } 119 119 + 120 120 + eprintln!("flushing..."); 121 121 + dest.flush().expect("failed to flush"); 122 122 + 123 123 + // compact so the dest DB is in a clean state 124 124 + eprintln!("compacting..."); 125 125 + let compact_start = Instant::now(); 126 126 + let mut compact_opts = rocksdb::CompactOptions::default(); 127 127 + compact_opts.set_bottommost_level_compaction(rocksdb::BottommostLevelCompaction::Force); 128 128 + dest.compact_range_opt(None::<&[u8]>, None::<&[u8]>, &compact_opts); 129 129 + eprintln!("compacted in {:.1}s", compact_start.elapsed().as_secs_f64()); 130 130 + 131 131 + // report 132 132 + let dest_size: u64 = dest 133 133 + .property_value("rocksdb.total-sst-files-size") 134 134 + .ok() 135 135 + .flatten() 136 136 + .and_then(|v| v.parse().ok()) 137 137 + .unwrap_or(0); 138 138 + let dest_files = dest.live_files().map(|f| f.len()).unwrap_or(0); 139 139 + 140 140 + eprintln!( 141 141 + "done: {} entries → {} SST files, {} MiB ({:.1}s total)", 142 142 + total_written, 143 143 + dest_files, 144 144 + dest_size / (1 << 20), 145 145 + start.elapsed().as_secs_f64(), 146 146 + ); 147 147 + }

+99

space-efficiency-check/examples/stats.rs

reviewed

··· 1 1 + use clap::Parser; 2 2 + use rocksdb::{Options, DB}; 3 3 + use std::collections::HashMap; 4 4 + use std::path::PathBuf; 5 5 + use std::time::Instant; 6 6 + 7 7 + /// Scan a DB and output data profile stats as JSON (for embedding in the dashboard). 8 8 + #[derive(Parser)] 9 9 + struct Args { 10 10 + /// DB directory to scan 11 11 + db_dir: PathBuf, 12 12 + } 13 13 + 14 14 + fn main() { 15 15 + let args = Args::parse(); 16 16 + 17 17 + let opts = Options::default(); 18 18 + let db = DB::open_for_read_only(&opts, &args.db_dir, false).expect("failed to open db"); 19 19 + 20 20 + let start = Instant::now(); 21 21 + let mut total_keys: u64 = 0; 22 22 + let mut value_size_buckets: HashMap<u32, u64> = HashMap::new(); // log2 bucket → count 23 23 + let mut account_keys: HashMap<Vec<u8>, u64> = HashMap::new(); // did → key count 24 24 + 25 25 + let mut iter = db.raw_iterator(); 26 26 + iter.seek_to_first(); 27 27 + 28 28 + while iter.valid() { 29 29 + let key = iter.key().unwrap(); 30 30 + let value = iter.value().unwrap(); 31 31 + total_keys += 1; 32 32 + 33 33 + // value size histogram (log2 buckets) 34 34 + let bucket = if value.is_empty() { 0 } else { (value.len() as f64).log2().floor() as u32 }; 35 35 + *value_size_buckets.entry(bucket).or_default() += 1; 36 36 + 37 37 + // extract DID prefix (everything before first '/') 38 38 + let did_end = key.iter().position(|&b| b == b'/').unwrap_or(key.len()); 39 39 + let did = &key[..did_end]; 40 40 + *account_keys.entry(did.to_vec()).or_default() += 1; 41 41 + 42 42 + if total_keys % 10_000_000 == 0 { 43 43 + eprintln!( 44 44 + " scanned {}M keys, {}s", 45 45 + total_keys / 1_000_000, 46 46 + start.elapsed().as_secs(), 47 47 + ); 48 48 + } 49 49 + 50 50 + iter.next(); 51 51 + } 52 52 + 53 53 + let total_accounts = account_keys.len() as u64; 54 54 + eprintln!( 55 55 + "done: {} keys, {} accounts in {:.1}s", 56 56 + total_keys, 57 57 + total_accounts, 58 58 + start.elapsed().as_secs_f64(), 59 59 + ); 60 60 + 61 61 + // build keys-per-account histogram (log2 buckets) 62 62 + let mut acct_size_buckets: HashMap<u32, u64> = HashMap::new(); 63 63 + for &count in account_keys.values() { 64 64 + let bucket = if count == 0 { 0 } else { (count as f64).log2().floor() as u32 }; 65 65 + *acct_size_buckets.entry(bucket).or_default() += 1; 66 66 + } 67 67 + 68 68 + // output JSON 69 69 + println!("{{"); 70 70 + println!(" \"total_keys\": {},", total_keys); 71 71 + println!(" \"total_accounts\": {},", total_accounts); 72 72 + 73 73 + // value size histogram: [[bucket_low, bucket_high, count], ...] 74 74 + let mut vs_buckets: Vec<u32> = value_size_buckets.keys().copied().collect(); 75 75 + vs_buckets.sort(); 76 76 + println!(" \"value_size_hist\": ["); 77 77 + for (i, &b) in vs_buckets.iter().enumerate() { 78 78 + let lo = 1u64 << b; 79 79 + let hi = 1u64 << (b + 1); 80 80 + let count = value_size_buckets[&b]; 81 81 + let comma = if i + 1 < vs_buckets.len() { "," } else { "" }; 82 82 + println!(" [{}, {}, {}]{}", lo, hi, count, comma); 83 83 + } 84 84 + println!(" ],"); 85 85 + 86 86 + // keys per account histogram 87 87 + let mut ka_buckets: Vec<u32> = acct_size_buckets.keys().copied().collect(); 88 88 + ka_buckets.sort(); 89 89 + println!(" \"keys_per_account_hist\": ["); 90 90 + for (i, &b) in ka_buckets.iter().enumerate() { 91 91 + let lo = 1u64 << b; 92 92 + let hi = 1u64 << (b + 1); 93 93 + let count = acct_size_buckets[&b]; 94 94 + let comma = if i + 1 < ka_buckets.len() { "," } else { "" }; 95 95 + println!(" [{}, {}, {}]{}", lo, hi, count, comma); 96 96 + } 97 97 + println!(" ]"); 98 98 + println!("}}"); 99 99 + }

+340

space-efficiency-check/examples/sweep.rs

reviewed

··· 1 1 + use clap::Parser; 2 2 + use csv::{ReaderBuilder, WriterBuilder}; 3 3 + use rocksdb::{BlockBasedOptions, CompactOptions, DBCompressionType, Options, DB}; 4 4 + use serde::{Deserialize, Serialize}; 5 5 + use std::collections::HashSet; 6 6 + use std::path::PathBuf; 7 7 + use std::time::Instant; 8 8 + 9 9 + /// Sweep RocksDB compression configs, measuring size after forced compaction. 10 10 + #[derive(Parser)] 11 11 + struct Args { 12 12 + /// DB directory to sweep (will be modified in-place each run) 13 13 + db_dir: PathBuf, 14 14 + 15 15 + /// output CSV file (appends to existing results) 16 16 + #[arg(long, default_value = "sweep-results.csv")] 17 17 + output: PathBuf, 18 18 + } 19 19 + 20 20 + // ── config + result row ─────────────────────────────────────────── 21 21 + 22 22 + #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] 23 23 + struct SweepConfig { 24 24 + block_size: usize, 25 25 + zstd_level: i32, 26 26 + restart_interval: i32, 27 27 + dict_bytes: i32, 28 28 + /// None = don't call set_zstd_max_train_bytes (RocksDB default). 29 29 + /// Some(N) = set train bytes to dict_bytes * N. 30 30 + #[serde(deserialize_with = "csv::invalid_option")] 31 31 + train_mult: Option<i32>, 32 32 + opt_filters: bool, 33 33 + subcompactions: u32, 34 34 + } 35 35 + 36 36 + #[derive(Debug, Serialize, Deserialize)] 37 37 + struct ResultRow { 38 38 + block_size: usize, 39 39 + zstd_level: i32, 40 40 + restart_interval: i32, 41 41 + dict_bytes: i32, 42 42 + #[serde(deserialize_with = "csv::invalid_option")] 43 43 + train_mult: Option<i32>, 44 44 + opt_filters: bool, 45 45 + subcompactions: u32, 46 46 + size_bytes: u64, 47 47 + size_mib: u64, 48 48 + compact_secs: f64, 49 49 + } 50 50 + 51 51 + impl ResultRow { 52 52 + fn config(&self) -> SweepConfig { 53 53 + SweepConfig { 54 54 + block_size: self.block_size, 55 55 + zstd_level: self.zstd_level, 56 56 + restart_interval: self.restart_interval, 57 57 + dict_bytes: self.dict_bytes, 58 58 + train_mult: self.train_mult, 59 59 + opt_filters: self.opt_filters, 60 60 + subcompactions: self.subcompactions, 61 61 + } 62 62 + } 63 63 + 64 64 + fn new(config: &SweepConfig, size: u64, compact_secs: f64) -> Self { 65 65 + Self { 66 66 + block_size: config.block_size, 67 67 + zstd_level: config.zstd_level, 68 68 + restart_interval: config.restart_interval, 69 69 + dict_bytes: config.dict_bytes, 70 70 + train_mult: config.train_mult, 71 71 + opt_filters: config.opt_filters, 72 72 + subcompactions: config.subcompactions, 73 73 + size_bytes: size, 74 74 + size_mib: size / (1 << 20), 75 75 + compact_secs, 76 76 + } 77 77 + } 78 78 + } 79 79 + 80 80 + impl std::fmt::Display for SweepConfig { 81 81 + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 82 82 + let train = match self.train_mult { 83 83 + None => "default".into(), 84 84 + Some(n) => format!("{n}x"), 85 85 + }; 86 86 + let algo = if self.zstd_level == 0 { "lz4".into() } else { format!("z{}", self.zstd_level) }; 87 87 + write!( 88 88 + f, 89 89 + "blk={}k {} ri={} dict={} train={} filt={} sub={}", 90 90 + self.block_size / 1024, 91 91 + algo, 92 92 + self.restart_interval, 93 93 + self.dict_bytes, 94 94 + train, 95 95 + if self.opt_filters { "on" } else { "off" }, 96 96 + self.subcompactions, 97 97 + ) 98 98 + } 99 99 + } 100 100 + 101 101 + // ── options builder ──────────────────────────────────────────────── 102 102 + 103 103 + fn build_options(config: &SweepConfig) -> Options { 104 104 + let mut opts = Options::default(); 105 105 + 106 106 + if config.zstd_level == 0 { 107 107 + // level 0 = LZ4 (no dictionary support) 108 108 + opts.set_compression_type(DBCompressionType::Lz4); 109 109 + } else { 110 110 + opts.set_compression_type(DBCompressionType::Zstd); 111 111 + opts.set_compression_options(-14, config.zstd_level, 0, config.dict_bytes); 112 112 + 113 113 + if config.dict_bytes > 0 { 114 114 + if let Some(mult) = config.train_mult { 115 115 + opts.set_zstd_max_train_bytes(config.dict_bytes * mult); 116 116 + } 117 117 + } 118 118 + } 119 119 + 120 120 + if config.opt_filters { 121 121 + opts.set_optimize_filters_for_hits(true); 122 122 + } 123 123 + 124 124 + opts.set_max_subcompactions(config.subcompactions); 125 125 + 126 126 + let mut block_opts = BlockBasedOptions::default(); 127 127 + block_opts.set_block_size(config.block_size); 128 128 + block_opts.set_block_restart_interval(config.restart_interval); 129 129 + opts.set_block_based_table_factory(&block_opts); 130 130 + 131 131 + opts 132 132 + } 133 133 + 134 134 + // ── config matrix ────────────────────────────────────────────────── 135 135 + // Edit this function to change what gets tested. 136 136 + 137 137 + fn sweep_configs() -> Vec<SweepConfig> { 138 138 + let base = SweepConfig { 139 139 + block_size: 32 * 1024, 140 140 + zstd_level: 3, 141 141 + restart_interval: 8, 142 142 + dict_bytes: 0, 143 143 + train_mult: None, 144 144 + opt_filters: true, 145 145 + subcompactions: 10, 146 146 + }; 147 147 + 148 148 + let mut configs = Vec::new(); 149 149 + 150 150 + // ── Block size × zstd level × dictionary ─────────────────── 151 151 + for &block_kb in &[4, 8, 16, 32, 64, 128] { 152 152 + for &level in &[0, 1, 3, 6, 9] { 153 153 + for &dict in &[0, 16_384] { 154 154 + if level == 0 && dict > 0 { continue; } // lz4 has no dictionary 155 155 + configs.push(SweepConfig { 156 156 + block_size: block_kb * 1024, 157 157 + zstd_level: level, 158 158 + dict_bytes: dict, 159 159 + train_mult: if dict > 0 { Some(16) } else { None }, 160 160 + ..base 161 161 + }); 162 162 + } 163 163 + } 164 164 + } 165 165 + 166 166 + // ── Zstd level deep (adds level 9, focused block sizes) ─── 167 167 + for &level in &[1, 3, 6, 9] { 168 168 + for &block_kb in &[4, 16, 64] { 169 169 + for &dict in &[0, 16_384] { 170 170 + configs.push(SweepConfig { 171 171 + block_size: block_kb * 1024, 172 172 + zstd_level: level, 173 173 + dict_bytes: dict, 174 174 + train_mult: if dict > 0 { Some(16) } else { None }, 175 175 + ..base 176 176 + }); 177 177 + } 178 178 + } 179 179 + } 180 180 + 181 181 + // ── Restart interval ─────────────────────────────────────── 182 182 + for &ri in &[4, 8, 16, 32, 64] { 183 183 + for &block_kb in &[4, 32] { 184 184 + configs.push(SweepConfig { 185 185 + block_size: block_kb * 1024, 186 186 + restart_interval: ri, 187 187 + ..base 188 188 + }); 189 189 + } 190 190 + } 191 191 + 192 192 + // ── Dictionary training sweep ────────────────────────────── 193 193 + for &block_kb in &[4, 8] { 194 194 + for &dict in &[1024, 2048, 4096, 8192, 16_384, 65_536] { 195 195 + // without explicit training config (RocksDB default) 196 196 + configs.push(SweepConfig { 197 197 + block_size: block_kb * 1024, 198 198 + dict_bytes: dict, 199 199 + train_mult: None, 200 200 + ..base 201 201 + }); 202 202 + // with training multipliers 203 203 + for &mult in &[4, 8, 16, 32, 64] { 204 204 + configs.push(SweepConfig { 205 205 + block_size: block_kb * 1024, 206 206 + dict_bytes: dict, 207 207 + train_mult: Some(mult), 208 208 + ..base 209 209 + }); 210 210 + } 211 211 + } 212 212 + } 213 213 + 214 214 + // ── Filters on/off ───────────────────────────────────────── 215 215 + for &block_kb in &[8, 32] { 216 216 + configs.push(SweepConfig { 217 217 + block_size: block_kb * 1024, 218 218 + opt_filters: false, 219 219 + ..base 220 220 + }); 221 221 + } 222 222 + 223 223 + // ── Subcompactions ───────────────────────────────────────── 224 224 + for &sub in &[1, 4, 10] { 225 225 + configs.push(SweepConfig { 226 226 + block_size: 16 * 1024, 227 227 + subcompactions: sub, 228 228 + ..base 229 229 + }); 230 230 + } 231 231 + 232 232 + // deduplicate (sweeps overlap) 233 233 + configs.sort_by(|a, b| { 234 234 + a.block_size 235 235 + .cmp(&b.block_size) 236 236 + .then(a.zstd_level.cmp(&b.zstd_level)) 237 237 + .then(a.restart_interval.cmp(&b.restart_interval)) 238 238 + .then(a.dict_bytes.cmp(&b.dict_bytes)) 239 239 + .then(a.train_mult.cmp(&b.train_mult)) 240 240 + .then(a.opt_filters.cmp(&b.opt_filters)) 241 241 + .then(a.subcompactions.cmp(&b.subcompactions)) 242 242 + }); 243 243 + configs.dedup(); 244 244 + 245 245 + configs 246 246 + } 247 247 + 248 248 + // ── load existing results ───────────────────────────────────────── 249 249 + 250 250 + fn load_existing(path: &std::path::Path) -> Vec<ResultRow> { 251 251 + let Ok(rdr) = ReaderBuilder::new().from_path(path) else { 252 252 + return Vec::new(); 253 253 + }; 254 254 + rdr.into_deserialize::<ResultRow>() 255 255 + .filter_map(|r| r.ok()) 256 256 + .collect() 257 257 + } 258 258 + 259 259 + // ── main ─────────────────────────────────────────────────────────── 260 260 + 261 261 + fn main() { 262 262 + let args = Args::parse(); 263 263 + let configs = sweep_configs(); 264 264 + 265 265 + let existing = load_existing(&args.output); 266 266 + let done: HashSet<_> = existing.iter().map(|r| r.config()).collect(); 267 267 + let remaining: Vec<_> = configs.iter().filter(|c| !done.contains(c)).collect(); 268 268 + 269 269 + eprintln!( 270 270 + "sweep: {} configs total, {} already done, {} to run", 271 271 + configs.len(), 272 272 + done.len(), 273 273 + remaining.len(), 274 274 + ); 275 275 + 276 276 + if remaining.is_empty() { 277 277 + eprintln!("nothing to do!"); 278 278 + return; 279 279 + } 280 280 + 281 281 + // open CSV for appending (write header only if new file) 282 282 + let write_header = existing.is_empty(); 283 283 + let file = std::fs::OpenOptions::new() 284 284 + .create(true) 285 285 + .append(true) 286 286 + .open(&args.output) 287 287 + .expect("failed to open CSV for writing"); 288 288 + let mut wtr = WriterBuilder::new() 289 289 + .has_headers(write_header) 290 290 + .from_writer(file); 291 291 + 292 292 + for (i, config) in remaining.iter().enumerate() { 293 293 + let opts = build_options(config); 294 294 + let db = match DB::open(&opts, &args.db_dir) { 295 295 + Ok(db) => db, 296 296 + Err(e) => { 297 297 + eprintln!( 298 298 + "[{}/{}] FAILED to open: {} — {}", 299 299 + i + 1, 300 300 + remaining.len(), 301 301 + config, 302 302 + e 303 303 + ); 304 304 + continue; 305 305 + } 306 306 + }; 307 307 + 308 308 + let start = Instant::now(); 309 309 + let mut compact_opts = CompactOptions::default(); 310 310 + compact_opts.set_bottommost_level_compaction(rocksdb::BottommostLevelCompaction::Force); 311 311 + db.compact_range_opt(None::<&[u8]>, None::<&[u8]>, &compact_opts); 312 312 + let elapsed = start.elapsed(); 313 313 + 314 314 + let size: u64 = db 315 315 + .property_value("rocksdb.total-sst-files-size") 316 316 + .ok() 317 317 + .flatten() 318 318 + .and_then(|v| v.parse().ok()) 319 319 + .unwrap_or(0); 320 320 + 321 321 + drop(db); 322 322 + 323 323 + let compact_secs = (elapsed.as_secs_f64() * 10.0).round() / 10.0; 324 324 + let row = ResultRow::new(config, size, compact_secs); 325 325 + 326 326 + wtr.serialize(&row).expect("failed to write CSV row"); 327 327 + wtr.flush().expect("failed to flush CSV"); 328 328 + 329 329 + eprintln!( 330 330 + "[{}/{}] {} → {} MiB ({:.1}s)", 331 331 + i + 1, 332 332 + remaining.len(), 333 333 + config, 334 334 + row.size_mib, 335 335 + compact_secs, 336 336 + ); 337 337 + } 338 338 + 339 339 + eprintln!("done! results in {}", args.output.display()); 340 340 + }

-484

space-efficiency-check/inspect-output.txt

reviewed

··· 1 1 - === Level Stats === 2 2 - Level Files Size(MB) 3 3 - -------------------- 4 4 - 0 0 0 5 5 - 1 0 0 6 6 - 2 0 0 7 7 - 3 0 0 8 8 - 4 0 0 9 9 - 5 0 0 10 10 - 6 342 52214 11 11 - 12 12 - === Aggregated Table Properties === 13 13 - # data blocks=5502269; # entries=665928371; # deletions=0; # merge operands=0; # range deletions=0; raw key size=48950837002; raw average key size=73.507661; raw value size=160502129948; raw average value size=241.020111; data block size=54659623307; index block size (user-key? 342, delta-value? 342)=385489305; filter block size=0; # entries for filter=0; (estimated) table size=55045112612; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 14 14 - === Table Properties Level 0 === 15 15 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 16 16 - === Table Properties Level 1 === 17 17 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 18 18 - === Table Properties Level 2 === 19 19 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 20 20 - === Table Properties Level 3 === 21 21 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 22 22 - === Table Properties Level 4 === 23 23 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 24 24 - === Table Properties Level 5 === 25 25 - # data blocks=0; # entries=0; # deletions=0; # merge operands=0; # range deletions=0; raw key size=0; raw average key size=0.000000; raw value size=0; raw average value size=0.000000; data block size=0; index block size (user-key? 0, delta-value? 0)=0; filter block size=0; # entries for filter=0; (estimated) table size=0; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 26 26 - === Table Properties Level 6 === 27 27 - # data blocks=5502269; # entries=665928371; # deletions=0; # merge operands=0; # range deletions=0; raw key size=48950837002; raw average key size=73.507661; raw value size=160502129948; raw average value size=241.020111; data block size=54659623307; index block size (user-key? 342, delta-value? 342)=385489305; filter block size=0; # entries for filter=0; (estimated) table size=55045112612; filter policy name=N/A; prefix extractor name=N/A; column family ID=N/A; column family name=N/A; comparator name=N/A; user defined timestamps persisted=true; largest sequence number in file=18446744073709551615; merge operator name=N/A; property collectors names=N/A; SST file compression algo=N/A; SST file compression options=N/A; creation time=0; time stamp of earliest key=0; time stamp of newest key=0; file creation time=0; slow compression estimated data size=0; fast compression estimated data size=0; DB identity=; DB session identity=; DB host id=; original file number=0; unique ID=N/A; Sequence number to time mapping=; 28 28 - === SST Tables Summary === 29 29 - --- level 0 --- version# 1 --- 30 30 - --- level 1 --- version# 1 --- 31 31 - --- level 2 --- version# 1 --- 32 32 - --- level 3 --- version# 1 --- 33 33 - --- level 4 --- version# 1 --- 34 34 - --- level 5 --- version# 1 --- 35 35 - --- level 6 --- version# 1 --- 36 36 - 2985:268861442[0 .. 0]['6469643A706C633A3232323263786E7464346333717474796437736D6C6473702F6170702E62736B792E6163746F722E70726F66696C652F73656C66' seq:0, type:1 .. '6469643A706C633A32326E6F7262796A626C6C35336877676B637776706A37622F6170702E62736B792E666565642E6C696B652F336B687A656F6662346B35326E' seq:0, type:1](0) 37 37 - 2987:58754567[0 .. 0]['6469643A706C633A32326E6F7262796A626C6C35336877676B637776706A37622F6170702E62736B792E666565642E6C696B652F336B687A6732666D72376C3274' seq:0, type:1 .. '6469643A706C633A3232726273346A6F32796A7A6833366A686D7877613364322F6170702E62736B792E666565642E6C696B652F336C63626C6B67373765643270' seq:0, type:1](0) 38 38 - 2979:268861560[0 .. 0]['6469643A706C633A3232726273346A6F32796A7A6833366A686D7877613364322F6170702E62736B792E666565642E6C696B652F336C63626D626F757079793264' seq:0, type:1 .. '6469643A706C633A32336536716D7567327677636E73667632667977757934622F6170702E62736B792E666565642E6C696B652F336C6332796F737964656A326E' seq:0, type:1](0) 39 39 - 2996:58726230[0 .. 0]['6469643A706C633A32336536716D7567327677636E73667632667977757934622F6170702E62736B792E666565642E6C696B652F336C6332797032346870623270' seq:0, type:1 .. '6469643A706C633A323367756A646176637578646A34326568376A66733661642F6170702E62736B792E666565642E6C696B652F336D34773333376978736B326A' seq:0, type:1](0) 40 40 - 2980:268873207[0 .. 0]['6469643A706C633A323367756A646176637578646A34326568376A66733661642F6170702E62736B792E666565642E6C696B652F336D3477333378626535613275' seq:0, type:1 .. '6469643A706C633A323436786F613662706267616C33326579366F70337A6E612F6170702E62736B792E666565642E7265706F73742F336B6966346D70746561333261' seq:0, type:1](0) 41 41 - 2995:58143950[0 .. 0]['6469643A706C633A323436786F613662706267616C33326579366F70337A6E612F6170702E62736B792E666565642E7265706F73742F336B6966376C37637661663234' seq:0, type:1 .. '6469643A706C633A3234647A727971656E6A7670667A337863686875327474732F6170702E62736B792E666565642E6C696B652F336C6461646569767374663271' seq:0, type:1](0) 42 42 - 2981:268868054[0 .. 0]['6469643A706C633A3234647A727971656E6A7670667A337863686875327474732F6170702E62736B792E666565642E6C696B652F336C646164656A643562623268' seq:0, type:1 .. '6469643A706C633A3234777A73666A326B696532737177347263706D6E7774632F6170702E62736B792E666565642E706F73742F336C6E6A3534686279336B3234' seq:0, type:1](0) 43 43 - 2990:58243804[0 .. 0]['6469643A706C633A3234777A73666A326B696532737177347263706D6E7774632F6170702E62736B792E666565642E706F73742F336C6E6A617569367036733234' seq:0, type:1 .. '6469643A706C633A3235327978727462327A7978737A7979696E7032757A34692F6170702E62736B792E666565642E706F73742F336C36766C706C766635713272' seq:0, type:1](0) 44 44 - 2978:268863540[0 .. 0]['6469643A706C633A3235327978727462327A7978737A7979696E7032757A34692F6170702E62736B792E666565642E706F73742F336C36766E3562786962633263' seq:0, type:1 .. '6469643A706C633A32356E756F686A63326F32326D777A756C696464737479792F6170702E62736B792E666565642E6C696B652F336B366A6E3365373562663276' seq:0, type:1](0) 45 45 - 2988:56596617[0 .. 0]['6469643A706C633A32356E756F686A63326F32326D777A756C696464737479792F6170702E62736B792E666565642E6C696B652F336B366A6E7432767A77673271' seq:0, type:1 .. '6469643A706C633A3235727464776E7037797A3573707A7976617A673275786B2F6170702E62736B792E666565642E706F73742F336B69666A357176616A783277' seq:0, type:1](0) 46 46 - 2984:268868826[0 .. 0]['6469643A706C633A3235727464776E7037797A3573707A7976617A673275786B2F6170702E62736B792E666565642E706F73742F336B69666A70697A716B753265' seq:0, type:1 .. '6469643A706C633A32366A646C71356F6B71376C77373365366278326D7836792F6170702E62736B792E666565642E6C696B652F336B33646F72686D643733326D' seq:0, type:1](0) 47 47 - 2991:57796974[0 .. 0]['6469643A706C633A32366A646C71356F6B71376C77373365366278326D7836792F6170702E62736B792E666565642E6C696B652F336B33647234657A78336C327A' seq:0, type:1 .. '6469643A706C633A323670336B33746B7271676D787273326D65766E673235792F6170702E62736B792E67726170682E626C6F636B2F336C696574646D6F6236373272' seq:0, type:1](0) 48 48 - 2986:268866712[0 .. 0]['6469643A706C633A323670336B33746B7271676D787273326D65766E673235792F6170702E62736B792E67726170682E626C6F636B2F336C696574646F37777768326C' seq:0, type:1 .. '6469643A706C633A32376535786C74796A64776A777766746C676863646672652F6170702E62736B792E666565642E6C696B652F336C646E706F6D376936773277' seq:0, type:1](0) 49 49 - 2992:58269042[0 .. 0]['6469643A706C633A32376535786C74796A64776A777766746C676863646672652F6170702E62736B792E666565642E6C696B652F336C646E707262353478673235' seq:0, type:1 .. '6469643A706C633A32376864346A6D6935793678646D68726E32677A6A3277322F6170702E62736B792E67726170682E666F6C6C6F772F336D3233706772663265673271' seq:0, type:1](0) 50 50 - 2977:268862614[0 .. 0]['6469643A706C633A32376864346A6D6935793678646D68726E32677A6A3277322F6170702E62736B792E67726170682E666F6C6C6F772F336D323370677279747A343266' seq:0, type:1 .. '6469643A706C633A32377A70726461786D7A6A6466666F6235367A326A6278332F6170702E62736B792E666565642E6C696B652F336C3462326B67613465373274' seq:0, type:1](0) 51 51 - 2994:58826933[0 .. 0]['6469643A706C633A32377A70726461786D7A6A6466666F6235367A326A6278332F6170702E62736B792E666565642E6C696B652F336C3462326B6F67366F7A326A' seq:0, type:1 .. '6469643A706C633A3261346E3762347A746A65746F796D7A7536716E78666D322F6170702E62736B792E666565642E7265706F73742F336C6B6F647374636A68373264' seq:0, type:1](0) 52 52 - 2982:268858158[0 .. 0]['6469643A706C633A3261346E3762347A746A65746F796D7A7536716E78666D322F6170702E62736B792E666565642E7265706F73742F336C6B6F7A7865756D7833326E' seq:0, type:1 .. '6469643A706C633A3261716F6F796968676C737861696137706E373472616B692F6170702E62736B792E666565642E6C696B652F336C7A74683735363671613261' seq:0, type:1](0) 53 53 - 2989:57240801[0 .. 0]['6469643A706C633A3261716F6F796968676C737861696137706E373472616B692F6170702E62736B792E666565642E6C696B652F336C7A746C6E77717671643237' seq:0, type:1 .. '6469643A706C633A32617475366E79367A726D70626778613273673237616F712F6170702E62736B792E666565642E6C696B652F336C6968736E7973777463326F' seq:0, type:1](0) 54 54 - 2983:268861238[0 .. 0]['6469643A706C633A32617475366E79367A726D70626778613273673237616F712F6170702E62736B792E666565642E6C696B652F336C6968736F36763377653270' seq:0, type:1 .. '6469643A706C633A3262666478727074696C6F663671687735376B6F626C63632F6170702E62736B792E666565642E706F73742F336C716E716E6D6C636173326E' seq:0, type:1](0) 55 55 - 2993:57408071[0 .. 0]['6469643A706C633A3262666478727074696C6F663671687735376B6F626C63632F6170702E62736B792E666565642E706F73742F336C716E717A617A70766B3276' seq:0, type:1 .. '6469643A706C633A32626A6666353435746D327A753361666E6468747A6475652F6170702E62736B792E666565642E6C696B652F336C6373797178747778673237' seq:0, type:1](0) 56 56 - 2999:268859763[0 .. 0]['6469643A706C633A32626A6666353435746D327A753361666E6468747A6475652F6170702E62736B792E666565642E6C696B652F336C63743266796C7671663237' seq:0, type:1 .. '6469643A706C633A32633737347477626778657679357378646C63756379636E2F6170702E62736B792E666565642E6C696B652F336B786A726A77646664783279' seq:0, type:1](0) 57 57 - 3016:49005391[0 .. 0]['6469643A706C633A32633737347477626778657679357378646C63756379636E2F6170702E62736B792E666565642E6C696B652F336B786C6B756F696D37773271' seq:0, type:1 .. '6469643A706C633A326364357A6933796F6C68366E746A696E6273323370357A2F6170702E62736B792E666565642E6C696B652F336D3374796E357174336A3267' seq:0, type:1](0) 58 58 - 2998:268865193[0 .. 0]['6469643A706C633A326364357A6933796F6C68366E746A696E6273323370357A2F6170702E62736B792E666565642E6C696B652F336D33747A336C6B6E6B723264' seq:0, type:1 .. '6469643A706C633A32637769627A366F62357979346269326D6F616F626762632F6170702E62736B792E666565642E6C696B652F336C676C6171326F67796E3265' seq:0, type:1](0) 59 59 - 3008:49941547[0 .. 0]['6469643A706C633A32637769627A366F62357979346269326D6F616F626762632F6170702E62736B792E666565642E6C696B652F336C676C623778326B75373262' seq:0, type:1 .. '6469643A706C633A3263793564356279706C6973757234793770706B7A356E682F6170702E62736B792E666565642E706F73742F336C6E616B6E78736968323272' seq:0, type:1](0) 60 60 - 2997:268860983[0 .. 0]['6469643A706C633A3263793564356279706C6973757234793770706B7A356E682F6170702E62736B792E666565642E706F73742F336C6E616B77767763726B326D' seq:0, type:1 .. '6469643A706C633A326469626B6232686161676E786C6C6E79636763643561642F6170702E62736B792E666565642E706F73742F336C73336C757062696132326C' seq:0, type:1](0) 61 61 - 3013:49013595[0 .. 0]['6469643A706C633A326469626B6232686161676E786C6C6E79636763643561642F6170702E62736B792E666565642E706F73742F336C73336D363474747173326C' seq:0, type:1 .. '6469643A706C633A32646C796536363368696376676F706964777862623269722F6170702E62736B792E666565642E706F73742F336C346A737279766F6774326A' seq:0, type:1](0) 62 62 - 3000:268869110[0 .. 0]['6469643A706C633A32646C796536363368696376676F706964777862623269722F6170702E62736B792E666565642E706F73742F336C346A737676656B766C327A' seq:0, type:1 .. '6469643A706C633A326536756B7A796C6E6D66703766786B357937696C6674672F6170702E62736B792E666565642E6C696B652F336C726A767665623775663271' seq:0, type:1](0) 63 63 - 3012:48660516[0 .. 0]['6469643A706C633A326536756B7A796C6E6D66703766786B357937696C6674672F6170702E62736B792E666565642E6C696B652F336C726A776274636A7A733271' seq:0, type:1 .. '6469643A706C633A3265626C776D7961736B756D37697A35346C32726374736A2F6170702E62736B792E666565642E6C696B652F336C756A666C6679366272326E' seq:0, type:1](0) 64 64 - 3001:268851211[0 .. 0]['6469643A706C633A3265626C776D7961736B756D37697A35346C32726374736A2F6170702E62736B792E666565642E6C696B652F336C756A666C6D793234653271' seq:0, type:1 .. '6469643A706C633A326573756E656A786E7875717778667576636E3673716C702F6170702E62736B792E666565642E6C696B652F336C6367687436616E76743274' seq:0, type:1](0) 65 65 - 3015:49418209[0 .. 0]['6469643A706C633A326573756E656A786E7875717778667576636E3673716C702F6170702E62736B792E666565642E6C696B652F336C6367687476783478773279' seq:0, type:1 .. '6469643A706C633A3265767A71766A75713472326A7274377479737269676A742F6170702E62736B792E666565642E6C696B652F336B716E767477647464733277' seq:0, type:1](0) 66 66 - 3006:268867983[0 .. 0]['6469643A706C633A3265767A71766A75713472326A7274377479737269676A742F6170702E62736B792E666565642E6C696B652F336B716E783773686477733233' seq:0, type:1 .. '6469643A706C633A32666C7833666E6F7036716F707632736273733567347A652F6170702E62736B792E666565642E6C696B652F336C6736706171643779613236' seq:0, type:1](0) 67 67 - 3011:48111091[0 .. 0]['6469643A706C633A32666C7833666E6F7036716F707632736273733567347A652F6170702E62736B792E666565642E6C696B652F336C6736706263367232613236' seq:0, type:1 .. '6469643A706C633A32666E79366F7733686A6F32636476716A6767616B656F682F6170702E62736B792E666565642E6C696B652F336C36716F357A6F3737763272' seq:0, type:1](0) 68 68 - 3002:268858725[0 .. 0]['6469643A706C633A32666E79366F7733686A6F32636476716A6767616B656F682F6170702E62736B792E666565642E6C696B652F336C36716F373532746C753266' seq:0, type:1 .. '6469643A706C633A32673736666F6E75756A6D377765366E7A656636746F666C2F6170702E62736B792E666565642E7265706F73742F336C6E6F33627374783435326B' seq:0, type:1](0) 69 69 - 3009:47761296[0 .. 0]['6469643A706C633A32673736666F6E75756A6D377765366E7A656636746F666C2F6170702E62736B792E666565642E7265706F73742F336C6E6F336363696966663237' seq:0, type:1 .. '6469643A706C633A326762686D76696E617379687A7977646E6963346A7369332F6170702E62736B792E666565642E6C696B652F336C356575767235696D373275' seq:0, type:1](0) 70 70 - 3004:268878428[0 .. 0]['6469643A706C633A326762686D76696E617379687A7977646E6963346A7369332F6170702E62736B792E666565642E6C696B652F336C3565757763766F77783275' seq:0, type:1 .. '6469643A706C633A3267776B346771346F627A676D7A79617A76366F71377A342F6170702E62736B792E666565642E6C696B652F336C696E326C61736C72733261' seq:0, type:1](0) 71 71 - 3010:47692215[0 .. 0]['6469643A706C633A3267776B346771346F627A676D7A79617A76366F71377A342F6170702E62736B792E666565642E6C696B652F336C696E326C666B61666D3271' seq:0, type:1 .. '6469643A706C633A326779626A62336734646F686F79366C34377335346175722F6170702E62736B792E666565642E6C696B652F336B6B773233747167656B327A' seq:0, type:1](0) 72 72 - 3003:268859183[0 .. 0]['6469643A706C633A326779626A62336734646F686F79366C34377335346175722F6170702E62736B792E666565642E6C696B652F336B6B77326F33733370333266' seq:0, type:1 .. '6469643A706C633A32686D376466617069326B67726B6A626B6F6D366C6579632F6170702E62736B792E666565642E706F73742F336C72717271666A32796B3262' seq:0, type:1](0) 73 73 - 3007:50456087[0 .. 0]['6469643A706C633A32686D376466617069326B67726B6A626B6F6D366C6579632F6170702E62736B792E666565642E706F73742F336C7335637A71683375633234' seq:0, type:1 .. '6469643A706C633A32687067656B3670356964797465716F757A37636F7577642F6170702E62736B792E666565642E6C696B652F336C62687A3677346B64663278' seq:0, type:1](0) 74 74 - 3005:268875961[0 .. 0]['6469643A706C633A32687067656B3670356964797465716F757A37636F7577642F6170702E62736B792E666565642E6C696B652F336C62687A3734346F69693270' seq:0, type:1 .. '6469643A706C633A326967736D73676D6C737732336170373567746C3636756E2F6170702E62736B792E666565642E6C696B652F336C747976326768676F373263' seq:0, type:1](0) 75 75 - 3014:47460314[0 .. 0]['6469643A706C633A326967736D73676D6C737732336170373567746C3636756E2F6170702E62736B792E666565642E6C696B652F336C747976333275366C6B3269' seq:0, type:1 .. '6469643A706C633A32696C6777366D6A6365703468673471636A6E6F727032332F6170702E62736B792E666565642E6C696B652F336B6A62656E6C6F6C696A3236' seq:0, type:1](0) 76 76 - 3026:268866516[0 .. 0]['6469643A706C633A32696C6777366D6A6365703468673471636A6E6F727032332F6170702E62736B792E666565642E6C696B652F336B6A62716876706F6A62326E' seq:0, type:1 .. '6469643A706C633A326A37787A7A767A6A6D776D73647334717661786E7263722F6170702E62736B792E666565642E6C696B652F336C69646F627179716478326D' seq:0, type:1](0) 77 77 - 3028:44965751[0 .. 0]['6469643A706C633A326A37787A7A767A6A6D776D73647334717661786E7263722F6170702E62736B792E666565642E6C696B652F336C69646F637A77677970326E' seq:0, type:1 .. '6469643A706C633A326A6532617237657964747A76327075376C7577616333722F6170702E62736B792E666565642E6C696B652F336C377464786A636F34323273' seq:0, type:1](0) 78 78 - 3025:268853156[0 .. 0]['6469643A706C633A326A6532617237657964747A76327075376C7577616333722F6170702E62736B792E666565642E6C696B652F336C3774647973656E796E3277' seq:0, type:1 .. '6469643A706C633A326A736F6A65776D6D7A72746B6163766A627933697936332F6170702E62736B792E666565642E706F73742F336C7166717932366472323237' seq:0, type:1](0) 79 79 - 3036:46122242[0 .. 0]['6469643A706C633A326A736F6A65776D6D7A72746B6163766A627933697936332F6170702E62736B792E666565642E706F73742F336C7166717A7862326B323237' seq:0, type:1 .. '6469643A706C633A326A7479613273766C7763706F733564727567346C7376782F6170702E62736B792E666565642E6C696B652F336C686A787A696B346A663274' seq:0, type:1](0) 80 80 - 3024:268857116[0 .. 0]['6469643A706C633A326A7479613273766C7763706F733564727567346C7376782F6170702E62736B792E666565642E6C696B652F336C686A793233796A6A65326B' seq:0, type:1 .. '6469643A706C633A326B67786537707873656734787268796A647A646B326A662F6170702E62736B792E666565642E6C696B652F336C7A7275356A6A6F6E353271' seq:0, type:1](0) 81 81 - 3035:46421856[0 .. 0]['6469643A706C633A326B67786537707873656734787268796A647A646B326A662F6170702E62736B792E666565642E6C696B652F336D32647962786B336C613265' seq:0, type:1 .. '6469643A706C633A326B6B6976766772636C62776B336774646D626E6B7576792F6170702E62736B792E666565642E6C696B652F336B71626E7662723566773234' seq:0, type:1](0) 82 82 - 3017:268865566[0 .. 0]['6469643A706C633A326B6B6976766772636C62776B336774646D626E6B7576792F6170702E62736B792E666565642E6C696B652F336B71626E766F676A62683232' seq:0, type:1 .. '6469643A706C633A326C36376A7676617667377262336368727A61616F3773322F6170702E62736B792E666565642E6C696B652F336D336264726C7A78706D3261' seq:0, type:1](0) 83 83 - 3034:43720209[0 .. 0]['6469643A706C633A326C36376A7676617667377262336368727A61616F3773322F6170702E62736B792E666565642E6C696B652F336D33626472706A617A643237' seq:0, type:1 .. '6469643A706C633A326C636E64697032677A7966666E67636274333264716E352F6170702E62736B792E666565642E6C696B652F336C6878696E6179626B353261' seq:0, type:1](0) 84 84 - 3019:268860135[0 .. 0]['6469643A706C633A326C636E64697032677A7966666E67636274333264716E352F6170702E62736B792E666565642E6C696B652F336C6878697367353571763261' seq:0, type:1 .. '6469643A706C633A326C74787265747433786D347176797477626E626B6636732F6170702E62736B792E666565642E706F73742F336C676A36367A696869633276' seq:0, type:1](0) 85 85 - 3031:43662350[0 .. 0]['6469643A706C633A326C74787265747433786D347176797477626E626B6636732F6170702E62736B792E666565642E706F73742F336C676A6164326A3678733236' seq:0, type:1 .. '6469643A706C633A326C78616A633771717A78366E346B6E776D7363736C64332F6170702E62736B792E666565642E6C696B652F336C786E69366C62796668326C' seq:0, type:1](0) 86 86 - 3021:268870507[0 .. 0]['6469643A706C633A326C78616A633771717A78366E346B6E776D7363736C64332F6170702E62736B792E666565642E6C696B652F336C786E69366D76757778326A' seq:0, type:1 .. '6469643A706C633A326D6963357A7277367A69776169686E3773767933687A6E2F6170702E62736B792E666565642E6C696B652F336C3568626A6D733674373235' seq:0, type:1](0) 87 87 - 3029:45036982[0 .. 0]['6469643A706C633A326D6963357A7277367A69776169686E3773767933687A6E2F6170702E62736B792E666565642E6C696B652F336C3568626B35326D35683235' seq:0, type:1 .. '6469643A706C633A326D6A69767234736B6E71696232737A79376B6D6F7669682F6170702E62736B792E666565642E706F73742F336B35723333667A3536683265' seq:0, type:1](0) 88 88 - 3023:268865620[0 .. 0]['6469643A706C633A326D6A69767234736B6E71696232737A79376B6D6F7669682F6170702E62736B792E666565642E706F73742F336B3572336267777A6D6F326F' seq:0, type:1 .. '6469643A706C633A326E64356C6362796571793464743272733479336B7965362F6170702E62736B792E666565642E706F73742F336C7473333476343262323273' seq:0, type:1](0) 89 89 - 3032:45881657[0 .. 0]['6469643A706C633A326E64356C6362796571793464743272733479336B7965362F6170702E62736B792E666565642E706F73742F336C7473333577636C61633273' seq:0, type:1 .. '6469643A706C633A326E686E6F33757068773570367671327972366C64627A6A2F6170702E62736B792E666565642E6C696B652F336C687A746A6173656564326F' seq:0, type:1](0) 90 90 - 3018:268866581[0 .. 0]['6469643A706C633A326E686E6F33757068773570367671327972366C64627A6A2F6170702E62736B792E666565642E6C696B652F336C687A746B70636A66773270' seq:0, type:1 .. '6469643A706C633A326F616D663234357879706573616A6E6E3263616A7867742F6170702E62736B792E666565642E7265706F73742F336C6C79696167737A6F633275' seq:0, type:1](0) 91 91 - 3027:46042194[0 .. 0]['6469643A706C633A326F616D663234357879706573616A6E6E3263616A7867742F6170702E62736B792E666565642E7265706F73742F336C6C796A61706F327733326E' seq:0, type:1 .. '6469643A706C633A326F62746C6472663668346A7967347474656136336E62642F6170702E62736B792E666565642E706F73742F336C716F676E76616C36633265' seq:0, type:1](0) 92 92 - 3022:268864661[0 .. 0]['6469643A706C633A326F62746C6472663668346A7967347474656136336E62642F6170702E62736B792E666565642E706F73742F336C716F677062353667733265' seq:0, type:1 .. '6469643A706C633A326F78347537717765357977757962693777667368326C682F6170702E62736B792E666565642E6C696B652F336C6669616D61797A346B3237' seq:0, type:1](0) 93 93 - 3033:44306441[0 .. 0]['6469643A706C633A326F78347537717765357977757962693777667368326C682F6170702E62736B792E666565642E6C696B652F336C6669616D677764666A3265' seq:0, type:1 .. '6469643A706C633A3270336274613435376B69787A6D323679656574323763652F6170702E62736B792E666565642E6C696B652F336C6E64766576717768753273' seq:0, type:1](0) 94 94 - 3020:268855547[0 .. 0]['6469643A706C633A3270336274613435376B69787A6D323679656574323763652F6170702E62736B792E666565642E6C696B652F336C6E6476666B746E6A713272' seq:0, type:1 .. '6469643A706C633A32706D6A786B7467783635616362766F67343567357435692F6170702E62736B792E666565642E6C696B652F336D34373268676E7568323266' seq:0, type:1](0) 95 95 - 3030:43303643[0 .. 0]['6469643A706C633A32706D6A786B7467783635616362766F67343567357435692F6170702E62736B792E666565642E6C696B652F336D3437326B61797934783232' seq:0, type:1 .. '6469643A706C633A3270717269336169367372326364796F7673357A696735662F6170702E62736B792E666565642E706F73742F336C68626E6474676163633270' seq:0, type:1](0) 96 96 - 3039:268859073[0 .. 0]['6469643A706C633A3270717269336169367372326364796F7673357A696735662F6170702E62736B792E666565642E706F73742F336C68673668376732716B3233' seq:0, type:1 .. '6469643A706C633A3271623473356775686F626F696E64616C337635346A61622F6170702E62736B792E666565642E6C696B652F336C6D7264746C32723778326F' seq:0, type:1](0) 97 97 - 3054:54499248[0 .. 0]['6469643A706C633A3271623473356775686F626F696E64616C337635346A61622F6170702E62736B792E666565642E6C696B652F336C6D726475327436776D3265' seq:0, type:1 .. '6469643A706C633A32716536756E7A336870693277627436793272327A3662722F6170702E62736B792E666565642E6C696B652F336C6261776E7236356B373233' seq:0, type:1](0) 98 98 - 3045:268863107[0 .. 0]['6469643A706C633A32716536756E7A336870693277627436793272327A3662722F6170702E62736B792E666565642E6C696B652F336C6261776F6B6F72367A3261' seq:0, type:1 .. '6469643A706C633A3271777065697378687A6E77646A66796C696875726470342F6170702E62736B792E666565642E6C696B652F336C3232703279636563793261' seq:0, type:1](0) 99 99 - 3050:54582384[0 .. 0]['6469643A706C633A3271777065697378687A6E77646A66796C696875726470342F6170702E62736B792E666565642E6C696B652F336C323271673373647132326E' seq:0, type:1 .. '6469643A706C633A3272326D786E6B34637176697A7676346F32666E797437742F6170702E62736B792E666565642E6C696B652F336C6B6B7935323473716C3272' seq:0, type:1](0) 100 100 - 3040:268861807[0 .. 0]['6469643A706C633A3272326D786E6B34637176697A7676346F32666E797437742F6170702E62736B792E666565642E6C696B652F336C6B6B793768657969673273' seq:0, type:1 .. '6469643A706C633A32726D6A35747636777A75716E32657532763332766274662F6170702E62736B792E666565642E6C696B652F336C746C6964626A67376D3272' seq:0, type:1](0) 101 101 - 3056:54348105[0 .. 0]['6469643A706C633A32726D6A35747636777A75716E32657532763332766274662F6170702E62736B792E666565642E6C696B652F336C746C6964736F32796F3276' seq:0, type:1 .. '6469643A706C633A3272727037616D626C686733786F6C79356E3568756971762F6170702E62736B792E666565642E6C696B652F336C71766B7A656534796B3233' seq:0, type:1](0) 102 102 - 3041:268862386[0 .. 0]['6469643A706C633A3272727037616D626C686733786F6C79356E3568756971762F6170702E62736B792E666565642E6C696B652F336C71766C336232646A623276' seq:0, type:1 .. '6469643A706C633A327364356B33743633333364356E64676F7A70716D6932652F6170702E62736B792E666565642E706F73742F336B616837626332726F743232' seq:0, type:1](0) 103 103 - 3047:53169771[0 .. 0]['6469643A706C633A327364356B33743633333364356E64676F7A70716D6932652F6170702E62736B792E666565642E706F73742F336B616A6E756470647963326F' seq:0, type:1 .. '6469643A706C633A32736762786D737266737867686772736A6E726F753735782F6170702E62736B792E666565642E6C696B652F336C3672696A6C7533356D3233' seq:0, type:1](0) 104 104 - 3042:268865393[0 .. 0]['6469643A706C633A32736762786D737266737867686772736A6E726F753735782F6170702E62736B792E666565642E6C696B652F336C3672696B6C713774723233' seq:0, type:1 .. '6469643A706C633A3274336E73666836777A796B6F796E7867347A73647732332F6170702E62736B792E666565642E6C696B652F336D3437637672706A64643261' seq:0, type:1](0) 105 105 - 3049:55289645[0 .. 0]['6469643A706C633A3274336E73666836777A796B6F796E7867347A73647732332F6170702E62736B792E666565642E6C696B652F336D3437657A326D656C6B326E' seq:0, type:1 .. '6469643A706C633A327437666E756A347079677A353561667A79636E32746F652F6170702E62736B792E666565642E6C696B652F336C7173656F75327A33713261' seq:0, type:1](0) 106 106 - 3038:268857948[0 .. 0]['6469643A706C633A327437666E756A347079677A353561667A79636E32746F652F6170702E62736B792E666565642E6C696B652F336C71736570376E3264713234' seq:0, type:1 .. '6469643A706C633A327471336873613663756B7A77696D643665666C61626F362F6170702E62736B792E67726170682E666F6C6C6F772F336C626E626A623263676C3271' seq:0, type:1](0) 107 107 - 3055:52683502[0 .. 0]['6469643A706C633A327471336873613663756B7A77696D643665666C61626F362F6170702E62736B792E67726170682E666F6C6C6F772F336C626E626A623263676D3271' seq:0, type:1 .. '6469643A706C633A3274756E6176766D66697136627974726C793334726B63322F6170702E62736B792E666565642E706F73742F336C616F6D336A717276323264' seq:0, type:1](0) 108 108 - 3043:268872932[0 .. 0]['6469643A706C633A3274756E6176766D66697136627974726C793334726B63322F6170702E62736B792E666565642E706F73742F336C616F736334666834633263' seq:0, type:1 .. '6469643A706C633A32756A74756B736369686D70776E77767A6A6D6B356E35682F6170702E62736B792E666565642E7265706F73742F336C70366F616B647073343263' seq:0, type:1](0) 109 109 - 3051:53198609[0 .. 0]['6469643A706C633A32756A74756B736369686D70776E77767A6A6D6B356E35682F6170702E62736B792E666565642E7265706F73742F336C706D6468656E32336D3279' seq:0, type:1 .. '6469643A706C633A32756F776434793466787A7470346A3379676F72336F757A2F6170702E62736B792E666565642E706F73742F336A797365746D796E766F3237' seq:0, type:1](0) 110 110 - 3037:268866317[0 .. 0]['6469643A706C633A32756F776434793466787A7470346A3379676F72336F757A2F6170702E62736B792E666565642E706F73742F336A7A697A6F79796D76713270' seq:0, type:1 .. '6469643A706C633A3276636161793279367161786D65646F366164746765677A2F6170702E62736B792E666565642E6C696B652F336B786835686A67626F733264' seq:0, type:1](0) 111 111 - 3048:53124111[0 .. 0]['6469643A706C633A3276636161793279367161786D65646F366164746765677A2F6170702E62736B792E666565642E6C696B652F336B786836716F6C7474323232' seq:0, type:1 .. '6469643A706C633A327665776871777676376271363764346A676E69777736612F6170702E62736B792E666565642E6C696B652F336D326C6A34336F6E6E783237' seq:0, type:1](0) 112 112 - 3046:268864680[0 .. 0]['6469643A706C633A327665776871777676376271363764346A676E69777736612F6170702E62736B792E666565642E6C696B652F336D326C6A36786F62786F3262' seq:0, type:1 .. '6469643A706C633A3276756B71626434623573686937727034723662657172642F6170702E62736B792E666565642E706F73742F336C6F723566633261636B3276' seq:0, type:1](0) 113 113 - 3053:56261157[0 .. 0]['6469643A706C633A3276756B71626434623573686937727034723662657172642F6170702E62736B792E666565642E706F73742F336C6F747174763768766B326A' seq:0, type:1 .. '6469643A706C633A32767A6C63726E656E7A6B79686B7674766E75376D676E342F6170702E62736B792E666565642E6C696B652F336C6C68676434716366733262' seq:0, type:1](0) 114 114 - 3044:268861905[0 .. 0]['6469643A706C633A32767A6C63726E656E7A6B79686B7674766E75376D676E342F6170702E62736B792E666565642E6C696B652F336C6C686764666B6A36663271' seq:0, type:1 .. '6469643A706C633A32776F336B757876617A7A366C7865346366667079626E692F6170702E62736B792E666565642E6C696B652F336C37627A35326E666A353237' seq:0, type:1](0) 115 115 - 3052:53666253[0 .. 0]['6469643A706C633A32776F336B757876617A7A366C7865346366667079626E692F6170702E62736B792E666565642E6C696B652F336C37627A35777A3370663278' seq:0, type:1 .. '6469643A706C633A32777437756C666161686162323372796E716C616D7933332F6170702E62736B792E666565642E6C696B652F336D337136786D6A7367753274' seq:0, type:1](0) 116 116 - 3060:268857163[0 .. 0]['6469643A706C633A32777437756C666161686162323372796E716C616D7933332F6170702E62736B792E666565642E6C696B652F336D3371367A6F36696D743261' seq:0, type:1 .. '6469643A706C633A32786536756A6D767078737262733662746D64756432696F2F6170702E62736B792E666565642E6C696B652F336C637933666F6E617A683232' seq:0, type:1](0) 117 117 - 3069:60238326[0 .. 0]['6469643A706C633A32786536756A6D767078737262733662746D64756432696F2F6170702E62736B792E666565642E6C696B652F336C6379336736727562763237' seq:0, type:1 .. '6469643A706C633A3278696568766E797136376F656E75716137376B377336782F6170702E62736B792E666565642E706F73742F336C6A326D6D3669676F32326A' seq:0, type:1](0) 118 118 - 3057:268858500[0 .. 0]['6469643A706C633A3278696568766E797136376F656E75716137376B377336782F6170702E62736B792E666565642E706F73742F336C6A32707A6761366373327A' seq:0, type:1 .. '6469643A706C633A327877686369736B7A63346E686C723475736F71763775762F6170702E62736B792E666565642E7265706F73742F336C796B7A6372747A736D3233' seq:0, type:1](0) 119 119 - 3067:60756781[0 .. 0]['6469643A706C633A327877686369736B7A63346E686C723475736F71763775762F6170702E62736B792E666565642E7265706F73742F336C796B7A6433643778353276' seq:0, type:1 .. '6469643A706C633A32793537656C377A7679616F68366E74766762766865666A2F6170702E62736B792E666565642E6C696B652F336B7A7565667377786D653232' seq:0, type:1](0) 120 120 - 3058:268860209[0 .. 0]['6469643A706C633A32793537656C377A7679616F68366E74766762766865666A2F6170702E62736B792E666565642E6C696B652F336B7A7565677877626777326B' seq:0, type:1 .. '6469643A706C633A3279706171646F7375657532616B6363766336617177787A2F6170702E62736B792E666565642E706F73742F336D3562617375657A32623275' seq:0, type:1](0) 121 121 - 3073:58976748[0 .. 0]['6469643A706C633A3279706171646F7375657532616B6363766336617177787A2F6170702E62736B792E666565642E706F73742F336D356261737570717832327A' seq:0, type:1 .. '6469643A706C633A32797468706A34707777706B61326C6A6B61626F7575626D2F6170702E62736B792E666565642E706F73742F336C6F733268776F78346B326B' seq:0, type:1](0) 122 122 - 3059:268867287[0 .. 0]['6469643A706C633A32797468706A34707777706B61326C6A6B61626F7575626D2F6170702E62736B792E666565642E706F73742F336C6F737370786B337A733274' seq:0, type:1 .. '6469643A706C633A327A676C7573753333346A3474366B6173776475356C73652F6170702E62736B792E666565642E6C696B652F336C76656F64703473646B3266' seq:0, type:1](0) 123 123 - 3076:58758534[0 .. 0]['6469643A706C633A327A676C7573753333346A3474366B6173776475356C73652F6170702E62736B792E666565642E6C696B652F336C76656F647773676D713273' seq:0, type:1 .. '6469643A706C633A327A6C68706F6F6D6F7572746B356C726B3335726F6565642F6170702E62736B792E666565642E6C696B652F336C646B6E796562667671326B' seq:0, type:1](0) 124 124 - 3062:268865371[0 .. 0]['6469643A706C633A327A6C68706F6F6D6F7572746B356C726B3335726F6565642F6170702E62736B792E666565642E6C696B652F336C646B6E79657A7634643278' seq:0, type:1 .. '6469643A706C633A333237706474616173723767687A367064796568787A34722F6170702E62736B792E666565642E6C696B652F336C79697A6D77346B6F6D326A' seq:0, type:1](0) 125 125 - 3070:60458923[0 .. 0]['6469643A706C633A333237706474616173723767687A367064796568787A34722F6170702E62736B792E666565642E6C696B652F336C79697A6E343333646E326E' seq:0, type:1 .. '6469643A706C633A33326373787678703335706B746F676B6A667370706575782F6170702E62736B792E666565642E6C696B652F336C746F7934766B6C35743232' seq:0, type:1](0) 126 126 - 3063:268854904[0 .. 0]['6469643A706C633A33326373787678703335706B746F676B6A667370706575782F6170702E62736B792E666565642E6C696B652F336C746F793563656366683269' seq:0, type:1 .. '6469643A706C633A333272613765336F6D79637A65343376736E7968727433652F6170702E62736B792E666565642E6C696B652F336B783279366F366262763263' seq:0, type:1](0) 127 127 - 3068:59185510[0 .. 0]['6469643A706C633A333272613765336F6D79637A65343376736E7968727433652F6170702E62736B792E666565642E6C696B652F336B7832793737626437743265' seq:0, type:1 .. '6469643A706C633A333277663274646263686763693233357963676B627869332F6170702E62736B792E666565642E6C696B652F336C726A6C7370747632753234' seq:0, type:1](0) 128 128 - 3061:268870898[0 .. 0]['6469643A706C633A333277663274646263686763693233357963676B627869332F6170702E62736B792E666565642E6C696B652F336C726A6C7377726833733274' seq:0, type:1 .. '6469643A706C633A333369753675327937663275637865666172376C6A7433762F6170702E62736B792E666565642E6C696B652F336B3472733279666464673267' seq:0, type:1](0) 129 129 - ... (249 more lines) 130 130 - 131 131 - === Compression Ratios === 132 132 - L0: -1.000000 133 133 - L1: -1.000000 134 134 - L2: -1.000000 135 135 - L3: -1.000000 136 136 - L4: -1.000000 137 137 - L5: -1.000000 138 138 - L6: 3.825617 139 139 - 140 140 - === Live SST Files (342 total) === 141 141 - Level Name Size (MiB) Size (bytes) Entries 142 142 - L6 /002977.sst 256.4 268862614 3276383 143 143 - L6 /002978.sst 256.4 268863540 3205414 144 144 - L6 /002979.sst 256.4 268861560 3274927 145 145 - L6 /002980.sst 256.4 268873207 3271956 146 146 - L6 /002981.sst 256.4 268868054 3242270 147 147 - L6 /002982.sst 256.4 268858158 3249309 148 148 - L6 /002983.sst 256.4 268861238 3224275 149 149 - L6 /002984.sst 256.4 268868826 3161092 150 150 - L6 /002985.sst 256.4 268861442 3216341 151 151 - L6 /002986.sst 256.4 268866712 3230393 152 152 - L6 /002987.sst 56.0 58754567 699554 153 153 - L6 /002988.sst 54.0 56596617 667028 154 154 - L6 /002989.sst 54.6 57240801 661240 155 155 - L6 /002990.sst 55.5 58243804 729394 156 156 - L6 /002991.sst 55.1 57796974 709922 157 157 - L6 /002992.sst 55.6 58269042 708449 158 158 - L6 /002993.sst 54.7 57408071 690180 159 159 - L6 /002994.sst 56.1 58826933 706139 160 160 - L6 /002995.sst 55.5 58143950 692731 161 161 - L6 /002996.sst 56.0 58726230 740125 162 162 - L6 /002997.sst 256.4 268860983 3172277 163 163 - L6 /002998.sst 256.4 268865193 3165917 164 164 - L6 /002999.sst 256.4 268859763 3240811 165 165 - L6 /003000.sst 256.4 268869110 3182763 166 166 - L6 /003001.sst 256.4 268851211 3122652 167 167 - L6 /003002.sst 256.4 268858725 3237410 168 168 - L6 /003003.sst 256.4 268859183 3146922 169 169 - L6 /003004.sst 256.4 268878428 3293679 170 170 - L6 /003005.sst 256.4 268875961 3334403 171 171 - L6 /003006.sst 256.4 268867983 3215218 172 172 - L6 /003007.sst 48.1 50456087 610756 173 173 - L6 /003008.sst 47.6 49941547 589929 174 174 - L6 /003009.sst 45.5 47761296 629798 175 175 - L6 /003010.sst 45.5 47692215 586233 176 176 - L6 /003011.sst 45.9 48111091 661140 177 177 - L6 /003012.sst 46.4 48660516 594008 178 178 - L6 /003013.sst 46.7 49013595 611531 179 179 - L6 /003014.sst 45.3 47460314 573788 180 180 - L6 /003015.sst 47.1 49418209 610541 181 181 - L6 /003016.sst 46.7 49005391 552302 182 182 - L6 /003017.sst 256.4 268865566 3214633 183 183 - L6 /003018.sst 256.4 268866581 3386094 184 184 - L6 /003019.sst 256.4 268860135 3236831 185 185 - L6 /003020.sst 256.4 268855547 3030425 186 186 - L6 /003021.sst 256.4 268870507 3276826 187 187 - L6 /003022.sst 256.4 268864661 3121457 188 188 - L6 /003023.sst 256.4 268865620 3129425 189 189 - L6 /003024.sst 256.4 268857116 3101500 190 190 - L6 /003025.sst 256.4 268853156 3202160 191 191 - L6 /003026.sst 256.4 268866516 3168814 192 192 - L6 /003027.sst 43.9 46042194 548655 193 193 - L6 /003028.sst 42.9 44965751 490843 194 194 - L6 /003029.sst 43.0 45036982 556370 195 195 - L6 /003030.sst 41.3 43303643 532114 196 196 - L6 /003031.sst 41.6 43662350 543547 197 197 - L6 /003032.sst 43.8 45881657 527788 198 198 - L6 /003033.sst 42.3 44306441 523605 199 199 - L6 /003034.sst 41.7 43720209 514003 200 200 - L6 /003035.sst 44.3 46421856 569326 201 201 - L6 /003036.sst 44.0 46122242 597276 202 202 - L6 /003037.sst 256.4 268866317 3190186 203 203 - L6 /003038.sst 256.4 268857948 3263342 204 204 - L6 /003039.sst 256.4 268859073 3293917 205 205 - L6 /003040.sst 256.4 268861807 3259109 206 206 - L6 /003041.sst 256.4 268862386 3200493 207 207 - L6 /003042.sst 256.4 268865393 3117844 208 208 - L6 /003043.sst 256.4 268872932 3348477 209 209 - L6 /003044.sst 256.4 268861905 3207536 210 210 - L6 /003045.sst 256.4 268863107 3216575 211 211 - L6 /003046.sst 256.4 268864680 3207041 212 212 - L6 /003047.sst 50.7 53169771 638878 213 213 - L6 /003048.sst 50.7 53124111 654481 214 214 - L6 /003049.sst 52.7 55289645 668810 215 215 - L6 /003050.sst 52.1 54582384 616540 216 216 - L6 /003051.sst 50.7 53198609 630495 217 217 - L6 /003052.sst 51.2 53666253 665800 218 218 - L6 /003053.sst 53.7 56261157 670967 219 219 - L6 /003054.sst 52.0 54499248 647974 220 220 - L6 /003055.sst 50.2 52683502 696254 221 221 - L6 /003056.sst 51.8 54348105 664675 222 222 - L6 /003057.sst 256.4 268858500 3208643 223 223 - L6 /003058.sst 256.4 268860209 3188353 224 224 - L6 /003059.sst 256.4 268867287 3240825 225 225 - L6 /003060.sst 256.4 268857163 3246988 226 226 - L6 /003061.sst 256.4 268870898 3200174 227 227 - L6 /003062.sst 256.4 268865371 3147083 228 228 - L6 /003063.sst 256.4 268854904 3263976 229 229 - L6 /003064.sst 256.4 268864822 3146851 230 230 - L6 /003065.sst 256.4 268865182 3286606 231 231 - L6 /003066.sst 256.4 268856043 3161095 232 232 - L6 /003067.sst 57.9 60756781 723462 233 233 - L6 /003068.sst 56.4 59185510 725641 234 234 - L6 /003069.sst 57.4 60238326 702111 235 235 - L6 /003070.sst 57.7 60458923 702165 236 236 - L6 /003071.sst 56.0 58743595 662586 237 237 - L6 /003072.sst 56.4 59117632 709524 238 238 - L6 /003073.sst 56.2 58976748 800752 239 239 - L6 /003074.sst 55.8 58464758 710214 240 240 - L6 /003075.sst 54.1 56738797 657114 241 241 - L6 /003076.sst 56.0 58758534 688227 242 242 - L6 /003077.sst 256.4 268858476 3157217 243 243 - L6 /003078.sst 256.4 268866422 3250229 244 244 - L6 /003079.sst 256.4 268858514 3193715 245 245 - L6 /003080.sst 256.4 268884543 3578473 246 246 - L6 /003081.sst 256.4 268855902 3202340 247 247 - L6 /003082.sst 256.4 268858119 3157036 248 248 - L6 /003083.sst 256.4 268867157 3211522 249 249 - L6 /003084.sst 256.4 268863876 3239316 250 250 - L6 /003085.sst 256.4 268855862 3193150 251 251 - L6 /003086.sst 256.4 268862526 3115013 252 252 - L6 /003087.sst 62.4 65436067 876006 253 253 - L6 /003088.sst 62.5 65492722 773555 254 254 - L6 /003089.sst 63.7 66791661 824052 255 255 - L6 /003090.sst 61.0 63951634 757055 256 256 - L6 /003091.sst 61.4 64398887 695459 257 257 - L6 /003092.sst 59.9 62813746 784690 258 258 - L6 /003093.sst 64.5 67649396 792956 259 259 - L6 /003094.sst 62.9 65915250 771150 260 260 - L6 /003095.sst 63.5 66544239 859051 261 261 - L6 /003096.sst 65.1 68285246 799075 262 262 - L6 /003097.sst 256.4 268871964 3236503 263 263 - L6 /003098.sst 256.4 268861038 3175946 264 264 - L6 /003099.sst 256.4 268860480 3172882 265 265 - L6 /003100.sst 256.4 268853986 3179349 266 266 - L6 /003101.sst 256.4 268863689 3319562 267 267 - L6 /003102.sst 256.4 268854744 3228953 268 268 - L6 /003103.sst 256.4 268858448 3170355 269 269 - L6 /003104.sst 256.4 268860635 3080658 270 270 - L6 /003105.sst 256.4 268862019 3099253 271 271 - L6 /003106.sst 256.4 268867301 3249732 272 272 - L6 /003107.sst 59.2 62045964 749435 273 273 - L6 /003108.sst 58.4 61200642 703378 274 274 - L6 /003109.sst 58.6 61438491 710299 275 275 - L6 /003110.sst 57.6 60362675 746173 276 276 - L6 /003111.sst 56.9 59650909 725793 277 277 - L6 /003112.sst 58.6 61482712 718285 278 278 - L6 /003113.sst 57.4 60160017 718722 279 279 - L6 /003114.sst 59.3 62214995 779507 280 280 - L6 /003115.sst 58.7 61501498 725176 281 281 - L6 /003116.sst 58.6 61462917 730847 282 282 - L6 /003117.sst 256.4 268861405 3133537 283 283 - L6 /003118.sst 256.4 268864479 3107500 284 284 - L6 /003119.sst 256.4 268875937 3343080 285 285 - L6 /003120.sst 256.4 268862517 3165597 286 286 - L6 /003121.sst 256.4 268864544 3272407 287 287 - L6 /003122.sst 256.4 268859636 3137050 288 288 - L6 /003123.sst 256.4 268872081 3301664 289 289 - L6 /003124.sst 256.4 268865987 3176258 290 290 - L6 /003125.sst 256.4 268864638 3222350 291 291 - L6 /003126.sst 256.4 268870166 3165072 292 292 - L6 /003127.sst 50.6 53063838 622272 293 293 - L6 /003128.sst 49.8 52251897 626433 294 294 - L6 /003129.sst 51.7 54222857 684449 295 295 - L6 /003130.sst 51.1 53618671 628301 296 296 - L6 /003131.sst 50.1 52563871 684837 297 297 - L6 /003132.sst 48.9 51327189 563544 298 298 - L6 /003133.sst 52.3 54828571 663558 299 299 - L6 /003134.sst 51.3 53837215 687009 300 300 - L6 /003135.sst 51.0 53489582 642071 301 301 - L6 /003136.sst 51.1 53633885 632123 302 302 - L6 /003137.sst 256.4 268858369 3221625 303 303 - L6 /003138.sst 256.4 268862675 3246344 304 304 - L6 /003139.sst 256.4 268856225 3230236 305 305 - L6 /003140.sst 256.4 268863140 3180595 306 306 - L6 /003141.sst 256.4 268865880 3175348 307 307 - L6 /003142.sst 256.4 268866128 3212405 308 308 - L6 /003143.sst 256.4 268866020 3126200 309 309 - L6 /003144.sst 256.4 268861169 3269356 310 310 - L6 /003145.sst 256.4 268863083 3182215 311 311 - L6 /003146.sst 256.4 268861222 3199114 312 312 - L6 /003147.sst 58.5 61325241 775069 313 313 - L6 /003148.sst 58.5 61352571 760372 314 314 - L6 /003149.sst 60.0 62887437 688991 315 315 - L6 /003150.sst 61.5 64477536 763386 316 316 - L6 /003151.sst 59.8 62729456 724418 317 317 - L6 /003152.sst 56.7 59454702 697947 318 318 - L6 /003153.sst 58.4 61270256 733206 319 319 - L6 /003154.sst 59.8 62677541 767872 320 320 - L6 /003155.sst 57.6 60422099 733541 321 321 - L6 /003156.sst 60.3 63269003 748565 322 322 - L6 /003157.sst 256.4 268856968 3005972 323 323 - L6 /003158.sst 256.4 268862341 3187355 324 324 - L6 /003159.sst 256.4 268863460 3212984 325 325 - L6 /003160.sst 256.4 268869217 3435330 326 326 - L6 /003161.sst 256.4 268871188 3288067 327 327 - L6 /003162.sst 256.4 268862053 3198586 328 328 - L6 /003163.sst 256.4 268854731 3150601 329 329 - L6 /003164.sst 256.4 268872799 3280981 330 330 - L6 /003165.sst 256.4 268864842 3198084 331 331 - L6 /003166.sst 256.4 268864613 3196915 332 332 - L6 /003167.sst 39.5 41391480 462030 333 333 - L6 /003168.sst 41.9 43895962 520435 334 334 - L6 /003169.sst 41.3 43326836 542909 335 335 - L6 /003170.sst 42.0 43989612 526032 336 336 - L6 /003171.sst 41.6 43642571 511527 337 337 - L6 /003172.sst 41.9 43921683 543282 338 338 - L6 /003173.sst 38.3 40198677 496308 339 339 - L6 /003174.sst 41.0 43018499 500878 340 340 - L6 /003175.sst 43.8 45901033 554405 341 341 - L6 /003176.sst 41.5 43565630 509942 342 342 - L6 /003177.sst 256.4 268860179 3220701 343 343 - L6 /003178.sst 256.4 268867739 3195633 344 344 - L6 /003179.sst 256.4 268865409 3185029 345 345 - L6 /003180.sst 256.4 268868613 3136484 346 346 - L6 /003181.sst 256.4 268863095 3199370 347 347 - L6 /003182.sst 256.4 268865885 3183188 348 348 - L6 /003183.sst 256.4 268865968 3281885 349 349 - L6 /003184.sst 256.4 268847298 2552178 350 350 - L6 /003185.sst 256.4 268867204 3227566 351 351 - L6 /003186.sst 256.4 268859917 3257118 352 352 - L6 /003187.sst 42.0 43993578 546307 353 353 - L6 /003188.sst 43.1 45145734 524024 354 354 - L6 /003189.sst 42.3 44383195 527793 355 355 - L6 /003190.sst 42.2 44200532 513463 356 356 - L6 /003191.sst 40.1 42027341 502349 357 357 - L6 /003192.sst 40.9 42886471 563526 358 358 - L6 /003193.sst 42.5 44521889 534889 359 359 - L6 /003194.sst 42.4 44408635 718293 360 360 - L6 /003195.sst 39.3 41234508 513861 361 361 - L6 /003196.sst 42.4 44499863 572143 362 362 - L6 /003197.sst 256.4 268866913 3193767 363 363 - L6 /003198.sst 256.4 268864501 3142515 364 364 - L6 /003199.sst 256.4 268854026 3169061 365 365 - L6 /003200.sst 256.4 268867177 3276247 366 366 - L6 /003201.sst 256.4 268863298 3252091 367 367 - L6 /003202.sst 256.4 268867723 3223326 368 368 - L6 /003203.sst 256.4 268862295 3180425 369 369 - L6 /003204.sst 256.4 268863849 3131459 370 370 - L6 /003205.sst 256.4 268864104 3199325 371 371 - L6 /003206.sst 256.4 268864314 3259808 372 372 - L6 /003207.sst 42.6 44677378 545414 373 373 - L6 /003208.sst 40.8 42820691 501173 374 374 - L6 /003209.sst 42.5 44569396 543574 375 375 - L6 /003210.sst 42.3 44369122 512980 376 376 - L6 /003211.sst 41.5 43516885 532859 377 377 - L6 /003212.sst 41.7 43690697 528902 378 378 - L6 /003213.sst 40.4 42346831 508398 379 379 - L6 /003214.sst 42.0 44014410 530676 380 380 - L6 /003215.sst 40.8 42808388 551741 381 381 - L6 /003216.sst 42.5 44549734 522489 382 382 - L6 /003217.sst 256.4 268862641 3207204 383 383 - L6 /003218.sst 256.4 268863668 3082556 384 384 - L6 /003219.sst 256.4 268868964 3293881 385 385 - L6 /003220.sst 256.4 268866876 3296490 386 386 - L6 /003221.sst 256.4 268871694 3336165 387 387 - L6 /003222.sst 256.4 268863736 3210560 388 388 - L6 /003223.sst 256.4 268866602 3128406 389 389 - L6 /003224.sst 256.4 268858377 3246305 390 390 - L6 /003225.sst 256.4 268866849 3293066 391 391 - L6 /003226.sst 256.4 268857998 3230417 392 392 - L6 /003227.sst 40.8 42801140 583170 393 393 - L6 /003228.sst 40.2 42162156 521559 394 394 - L6 /003229.sst 40.7 42682876 492196 395 395 - L6 /003230.sst 38.9 40784147 514353 396 396 - L6 /003231.sst 38.2 40099737 495375 397 397 - L6 /003232.sst 39.0 40900395 518788 398 398 - L6 /003233.sst 36.9 38735645 489202 399 399 - L6 /003234.sst 39.4 41270032 467314 400 400 - L6 /003235.sst 40.3 42210365 491418 401 401 - L6 /003236.sst 40.9 42859293 523434 402 402 - L6 /003237.sst 256.4 268872024 3221528 403 403 - L6 /003238.sst 256.4 268871391 3152584 404 404 - L6 /003239.sst 256.4 268873934 3223183 405 405 - L6 /003240.sst 256.4 268896298 3461847 406 406 - L6 /003241.sst 256.4 268871677 3271598 407 407 - L6 /003242.sst 256.4 268897511 5015832 408 408 - L6 /003243.sst 256.4 268899179 3351433 409 409 - L6 /003244.sst 256.5 268911361 3408847 410 410 - L6 /003245.sst 256.5 268919905 3472088 411 411 - L6 /003246.sst 256.4 268900265 3464565 412 412 - L6 /003247.sst 38.0 39806264 496882 413 413 - L6 /003248.sst 41.4 43435386 517387 414 414 - L6 /003249.sst 39.4 41262438 524043 415 415 - L6 /003250.sst 42.4 44412123 550949 416 416 - L6 /003251.sst 41.2 43233815 508729 417 417 - L6 /003252.sst 40.4 42410623 502580 418 418 - L6 /003253.sst 39.5 41409168 533025 419 419 - L6 /003254.sst 38.1 39911925 505291 420 420 - L6 /003255.sst 39.6 41520873 490207 421 421 - L6 /003256.sst 41.5 43486711 585633 422 422 - L6 /003257.sst 256.5 268912316 3472178 423 423 - L6 /003258.sst 256.5 268912117 3312977 424 424 - L6 /003259.sst 256.4 268905922 3330836 425 425 - L6 /003260.sst 256.4 268901023 3490335 426 426 - L6 /003261.sst 256.5 268912424 3402032 427 427 - L6 /003262.sst 256.4 268900819 3453537 428 428 - L6 /003263.sst 256.4 268902592 3378925 429 429 - L6 /003264.sst 256.4 268897802 3237374 430 430 - L6 /003265.sst 256.5 268908390 3442247 431 431 - L6 /003266.sst 256.4 268903779 3465077 432 432 - L6 /003267.sst 43.1 45245683 602088 433 433 - L6 /003268.sst 42.9 44946336 633779 434 434 - L6 /003269.sst 42.0 44051626 602150 435 435 - L6 /003270.sst 43.4 45516123 580797 436 436 - L6 /003271.sst 41.1 43065426 500121 437 437 - L6 /003272.sst 42.0 44067669 571477 438 438 - L6 /003273.sst 41.9 43972387 570966 439 439 - L6 /003274.sst 40.0 41984262 530728 440 440 - L6 /003275.sst 43.3 45452602 580964 441 441 - L6 /003276.sst 43.0 45045736 619937 442 442 - L6 /003277.sst 256.4 268902254 3453584 443 443 - L6 /003278.sst 256.4 268901773 3495945 444 444 - L6 /003279.sst 256.5 268911172 3454322 445 445 - L6 /003280.sst 256.5 268913116 3444020 446 446 - L6 /003281.sst 256.5 268915939 3475868 447 447 - L6 /003282.sst 256.4 268896334 3396266 448 448 - L6 /003283.sst 256.4 268905118 3427140 449 449 - L6 /003284.sst 256.5 268907822 3387635 450 450 - L6 /003285.sst 256.5 268916113 3612414 451 451 - L6 /003286.sst 256.4 268897249 3468662 452 452 - L6 /003287.sst 53.4 56020775 767446 453 453 - L6 /003288.sst 53.4 56019269 712525 454 454 - L6 /003289.sst 54.6 57244373 758709 455 455 - L6 /003290.sst 55.2 57863501 642612 456 456 - L6 /003291.sst 55.1 57810294 749960 457 457 - L6 /003292.sst 55.5 58239100 762226 458 458 - L6 /003293.sst 55.8 58534092 735171 459 459 - L6 /003294.sst 53.9 56542603 741244 460 460 - L6 /003295.sst 53.5 56128633 739028 461 461 - L6 /003296.sst 54.5 57109049 738403 462 462 - L6 /003297.sst 256.4 268898782 3381552 463 463 - L6 /003298.sst 256.5 268909804 3538990 464 464 - L6 /003299.sst 256.5 268910476 3365519 465 465 - L6 /003300.sst 256.4 268907014 3431448 466 466 - L6 /003301.sst 256.5 268918208 3565244 467 467 - L6 /003302.sst 256.5 268914078 3451541 468 468 - L6 /003303.sst 256.5 268935459 3186169 469 469 - L6 /003304.sst 256.4 268905433 3448609 470 470 - L6 /003305.sst 256.5 268908458 3439849 471 471 - L6 /003306.sst 256.5 268916977 3459214 472 472 - L6 /003307.sst 45.8 47999980 600957 473 473 - L6 /003308.sst 49.0 51383035 696103 474 474 - L6 /003309.sst 50.6 53062575 727276 475 475 - L6 /003310.sst 48.7 51071404 644918 476 476 - L6 /003311.sst 50.2 52614219 680154 477 477 - L6 /003312.sst 48.3 50617420 621601 478 478 - L6 /003313.sst 49.9 52294191 673927 479 479 - L6 /003314.sst 47.2 49515619 605872 480 480 - L6 /003315.sst 49.0 51350586 666331 481 481 - L6 /003316.sst 47.5 49756657 619661 482 482 - L6 /003317.sst 33.0 34569387 442654 483 483 - L6 /003318.sst 256.5 268919693 3533873 484 484 - TOTAL 52213.8 54750108385 665928371

+5

space-efficiency-check/readme.md

reviewed

··· 427 427 428 428 db size after compaction bytes=54750108385 mib=52213 compact_elapsed=397.067736125s 429 429 430 430 + after fixing compression: 430 431 432 432 + db size after compaction bytes=54699871870 mib=52165 compact_elapsed=128.5052185s 431 433 434 434 + and with no dictionary: 435 435 + 436 436 + db size after compaction bytes=54772637044 mib=52235 compact_elapsed=85.942000375s 432 437 433 438 434 439 and then maybe:

+9 -96

space-efficiency-check/src/main.rs

reviewed

··· 2 2 use rocksdb::{DB, CompactOptions, DBCompressionType, Options}; 3 3 4 4 use std::path::{Path, PathBuf}; 5 5 - use std::sync::{Arc, atomic::{AtomicBool, Ordering}}; 5 5 + use std::sync::{Arc, atomic::Ordering}; 6 6 7 7 use space_efficiency_check::work::{ProcessError, run_workers}; 8 8 ··· 33 33 let mut opts = Options::default(); 34 34 opts.create_if_missing(true); 35 35 36 36 - // we don't need durability 37 37 - // redundant: we do `disable_wal(true)` on all writes 38 38 - // opts.set_manual_wal_flush(true); 39 39 - 40 36 // compress lower levels more 41 41 - opts.set_compression_per_level(&[ 42 42 - DBCompressionType::Lz4, // L0 43 43 - DBCompressionType::Zstd, // L1 44 44 - DBCompressionType::Zstd, // L2 45 45 - DBCompressionType::Zstd, // L3 46 46 - DBCompressionType::Zstd, // L4 47 47 - DBCompressionType::Zstd, // L5 48 48 - DBCompressionType::Zstd, // L6 49 49 - ]); 37 37 + opts.set_compression_type(DBCompressionType::Zstd); 50 38 51 51 - const D: i32 = 16_384; 39 39 + const D: i32 = 0; 52 40 53 41 opts.set_compression_options( 54 42 -14, // window_bits (default, -14 means use zstd default) ··· 56 44 0, // strategy (0 = default) 57 45 D, // max_dict_bytes (0 = no dictionary) (tried 16k) 58 46 ); 59 59 - opts.set_zstd_max_train_bytes(16 * D); 60 60 - 61 61 - opts.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); 62 62 - 63 63 - opts.set_bottommost_compression_options( 64 64 - -14, // window_bits (default, -14 means use zstd default) 65 65 - 3, // level (default 3) 66 66 - 0, // strategy (0 = default) 67 67 - D, // max_dict_bytes (0 = no dictionary) (tried 16k) 68 68 - true, // enabled 69 69 - ); 70 70 - 71 71 - // probably investigate this one: use more training data for dictionary 72 72 - opts.set_bottommost_zstd_max_train_bytes(16 * D, true); // 256KB of samples for a 16KB dict 47 47 + // opts.set_zstd_max_train_bytes(16 * D); 73 48 74 49 // skip filters on bottom-most layer 50 50 + // TODO: verify how much this does 75 51 opts.set_optimize_filters_for_hits(true); 76 52 77 77 - // // write into new buffer while flushing? 78 78 - // opts.set_max_write_buffer_number(4); 79 79 - 80 80 - opts.set_max_subcompactions(10); // default 1 81 81 - 82 82 - // ugh this probably wasn't exactly the right move 83 83 - // there's increase_parallelism() 84 84 - // // try to speed up compaction? 85 85 - // opts.set_max_background_jobs(8); 86 86 - opts.increase_parallelism(10); 87 87 - 88 88 - // apparently can help with space amp, less partially-filled levels 89 89 - opts.set_level_compaction_dynamic_level_bytes(true); 90 90 - 91 91 - opts.optimize_level_style_compaction(2 * 2_usize.pow(30)); 92 92 - 93 93 - // larger write buffer for less frequent flushes 94 94 - // opts.set_write_buffer_size(256 * 2_usize.pow(20)); // default 64MB 95 95 - 96 96 - // larger sst files (ehhh) 97 97 - // don't help 98 98 - // opts.set_target_file_size_base(128 * 2_u64.pow(20)); // default 64MB 53 53 + // keep this for now while running on the full set since it speeds things up a lot 54 54 + // but probably drop it if we sample SSTs and don't need that? 55 55 + // (or maybe actually measure how much it affects total compression ratio? expecting minimal) 56 56 + opts.set_max_subcompactions(10); // default 1, probably do 3–4 in prod 99 57 100 58 // larger blocks for better compression (good scan, worse point read) 101 59 let mut block_opts = rocksdb::BlockBasedOptions::default(); ··· 103 61 block_opts.set_block_restart_interval(8); 104 62 opts.set_block_based_table_factory(&block_opts); 105 63 106 106 - // enable ONLY during bulk-import 107 107 - // opts.set_unordered_write(true); 108 108 - // opts.set_level_zero_slowdown_writes_trigger(40); 109 109 - // opts.set_level_zero_stop_writes_trigger(80); 110 110 - 111 111 - 112 64 DB::open(&opts, path) 113 65 } 114 66 ··· 119 71 let args = Args::parse(); 120 72 121 73 let db = Arc::new(open_db(&args.db_dir)?); 122 122 - 123 123 - // let _guard = Monitor::start(db.clone()); 124 74 125 75 if args.just_compact { 126 76 manually_compact(db)?; ··· 185 135 .and_then(|v| v.parse::<u64>().ok()) 186 136 .unwrap_or(0) 187 137 } 188 188 - 189 189 - struct Monitor { 190 190 - stop: Arc<AtomicBool>, 191 191 - } 192 192 - 193 193 - impl Drop for Monitor { 194 194 - fn drop(&mut self) { 195 195 - self.stop.store(true, Ordering::Relaxed) 196 196 - } 197 197 - } 198 198 - 199 199 - impl Monitor { 200 200 - fn start(db: Arc<DB>) -> Self { 201 201 - let stop: Arc<AtomicBool> = Arc::new(false.into()); 202 202 - tokio::task::spawn(Monitor::run(db, stop.clone())); 203 203 - Self { stop } 204 204 - } 205 205 - 206 206 - async fn run(db: Arc<DB>, stop: Arc<AtomicBool>) { 207 207 - while !stop.load(Ordering::Relaxed) { 208 208 - tracing::info!( 209 209 - is_stopped = db.property_value("rocksdb.is-write-stopped").unwrap(), 210 210 - actual_delayed_wirte_rate = db.property_value("rocksdb.actual-delayed-write-rate").unwrap(), 211 211 - l0_files = db.property_value("rocksdb.num-files-at-level0").unwrap(), 212 212 - "db status", 213 213 - ); 214 214 - 215 215 - // delay 5s but also check if we need to bail 216 216 - for _ in 0..10 { 217 217 - tokio::time::sleep(std::time::Duration::from_secs_f64(0.5)).await; 218 218 - if stop.load(Ordering::Relaxed) { 219 219 - return; 220 220 - } 221 221 - } 222 222 - } 223 223 - } 224 224 - }

+43

space-efficiency-check/stats.json

reviewed

··· 1 1 + { 2 2 + "total_keys": 110471379, 3 3 + "total_accounts": 78568, 4 4 + "value_size_hist": [ 5 5 + [16, 32, 32], 6 6 + [32, 64, 30165], 7 7 + [64, 128, 13339156], 8 8 + [128, 256, 79513656], 9 9 + [256, 512, 13262139], 10 10 + [512, 1024, 3916643], 11 11 + [1024, 2048, 384505], 12 12 + [2048, 4096, 20803], 13 13 + [4096, 8192, 3652], 14 14 + [8192, 16384, 472], 15 15 + [16384, 32768, 68], 16 16 + [32768, 65536, 44], 17 17 + [65536, 131072, 37], 18 18 + [131072, 262144, 7] 19 19 + ], 20 20 + "keys_per_account_hist": [ 21 21 + [1, 2, 2630], 22 22 + [2, 4, 8451], 23 23 + [4, 8, 6955], 24 24 + [8, 16, 7082], 25 25 + [16, 32, 7424], 26 26 + [32, 64, 7898], 27 27 + [64, 128, 7817], 28 28 + [128, 256, 7196], 29 29 + [256, 512, 6323], 30 30 + [512, 1024, 5230], 31 31 + [1024, 2048, 4087], 32 32 + [2048, 4096, 2869], 33 33 + [4096, 8192, 2038], 34 34 + [8192, 16384, 1223], 35 35 + [16384, 32768, 747], 36 36 + [32768, 65536, 379], 37 37 + [65536, 131072, 165], 38 38 + [131072, 262144, 45], 39 39 + [262144, 524288, 6], 40 40 + [524288, 1048576, 2], 41 41 + [1048576, 2097152, 1] 42 42 + ] 43 43 + }

+137

space-efficiency-check/sweep-results.csv

reviewed

··· 1 1 + block_size,zstd_level,restart_interval,dict_bytes,train_mult,opt_filters,subcompactions,size_bytes,size_mib,compact_secs 2 2 + 4096,1,8,0,,true,10,11079791854,10566,16.2 3 3 + 4096,1,8,16384,16,true,10,10365069306,9884,20.8 4 4 + 4096,3,4,0,,true,10,11208454885,10689,24.0 5 5 + 4096,3,8,0,,true,10,11070990214,10558,20.2 6 6 + 4096,3,8,1024,,true,10,10617592768,10125,27.7 7 7 + 4096,3,8,1024,4,true,10,11071036189,10558,20.5 8 8 + 4096,3,8,1024,16,true,10,11071022120,10558,21.0 9 9 + 4096,3,8,1024,64,true,10,10782814873,10283,27.3 10 10 + 4096,3,8,4096,,true,10,10598956252,10107,30.5 11 11 + 4096,3,8,4096,4,true,10,11071016375,10558,20.8 12 12 + 4096,3,8,4096,16,true,10,10271883281,9796,25.6 13 13 + 4096,3,8,4096,64,true,10,10213224021,9740,29.1 14 14 + 4096,3,8,16384,,true,10,10556340899,10067,30.9 15 15 + 4096,3,8,16384,4,true,10,10301070056,9823,27.1 16 16 + 4096,3,8,16384,16,true,10,10227379189,9753,30.4 17 17 + 4096,3,8,16384,64,true,10,10213306710,9740,33.9 18 18 + 4096,3,8,65536,,true,10,10465976570,9981,33.0 19 19 + 4096,3,8,65536,4,true,10,10217242670,9743,30.0 20 20 + 4096,3,8,65536,16,true,10,10164362837,9693,34.1 21 21 + 4096,3,8,65536,64,true,10,10139725001,9669,38.2 22 22 + 4096,3,16,0,,true,10,10974836115,10466,25.6 23 23 + 4096,3,32,0,,true,10,10931177138,10424,21.3 24 24 + 4096,3,64,0,,true,10,10931160646,10424,20.8 25 25 + 4096,6,8,0,,true,10,10895448640,10390,33.9 26 26 + 4096,6,8,16384,16,true,10,10093706553,9626,43.9 27 27 + 4096,9,8,0,,true,10,10893183784,10388,58.0 28 28 + 4096,9,8,16384,16,true,10,9946861812,9486,58.1 29 29 + 8192,1,8,0,,true,10,10028074705,9563,20.8 30 30 + 8192,1,8,16384,16,true,10,9811325586,9356,17.4 31 31 + 8192,3,8,0,,false,10,10040289523,9575,18.0 32 32 + 8192,3,8,0,,true,10,10040284381,9575,16.2 33 33 + 8192,3,8,1024,,true,10,9835665629,9380,25.4 34 34 + 8192,3,8,1024,4,true,10,10040279221,9575,16.8 35 35 + 8192,3,8,1024,16,true,10,10040292654,9575,16.8 36 36 + 8192,3,8,1024,64,true,10,10119180082,9650,25.0 37 37 + 8192,3,8,4096,,true,10,9848434052,9392,24.6 38 38 + 8192,3,8,4096,4,true,10,10040294086,9575,16.8 39 39 + 8192,3,8,4096,16,true,10,9717871364,9267,22.5 40 40 + 8192,3,8,4096,64,true,10,9665939593,9218,25.4 41 41 + 8192,3,8,16384,,true,10,9859705853,9402,25.6 42 42 + 8192,3,8,16384,4,true,10,9782101780,9328,24.5 43 43 + 8192,3,8,16384,16,true,10,9713733091,9263,26.0 44 44 + 8192,3,8,16384,64,true,10,9694061307,9244,27.5 45 45 + 8192,3,8,65536,,true,10,9849121697,9392,26.2 46 46 + 8192,3,8,65536,4,true,10,9745808012,9294,24.9 47 47 + 8192,3,8,65536,16,true,10,9696039387,9246,27.5 48 48 + 8192,3,8,65536,64,true,10,9672733565,9224,31.5 49 49 + 8192,6,8,0,,true,10,9874369448,9416,31.7 50 50 + 8192,6,8,16384,16,true,10,9555000218,9112,38.5 51 51 + 16384,1,8,0,,true,10,9401904407,8966,15.6 52 52 + 16384,1,8,16384,16,true,10,9384259110,8949,15.2 53 53 + 16384,3,8,0,,true,4,9439746342,9002,22.6 54 54 + 16384,3,8,0,,true,10,9439798265,9002,14.1 55 55 + 16384,3,8,16384,16,true,10,9336363403,8903,21.6 56 56 + 16384,6,8,0,,true,10,9238560723,8810,28.8 57 57 + 16384,6,8,16384,16,true,10,9157064709,8732,35.4 58 58 + 16384,9,8,0,,true,10,9227102912,8799,55.0 59 59 + 16384,9,8,16384,16,true,10,9002507344,8585,48.0 60 60 + 32768,1,8,0,,true,10,9161174067,8736,12.6 61 61 + 32768,1,8,16384,16,true,10,9129717484,8706,13.8 62 62 + 32768,3,4,0,,true,10,9186445992,8760,13.7 63 63 + 32768,3,8,0,,false,10,9074744264,8654,13.3 64 64 + 32768,3,8,0,,true,10,9074587560,8654,13.3 65 65 + 32768,3,8,16384,16,true,10,9060352672,8640,22.0 66 66 + 32768,3,16,0,,true,10,9009532082,8592,13.8 67 67 + 32768,3,32,0,,true,10,8974169294,8558,13.3 68 68 + 32768,3,64,0,,true,10,8953489124,8538,13.3 69 69 + 32768,6,8,0,,true,10,8959226119,8544,27.5 70 70 + 32768,6,8,16384,16,true,10,8891715689,8479,34.9 71 71 + 65536,1,8,0,,true,10,8940392508,8526,11.5 72 72 + 65536,1,8,16384,16,true,10,8940404881,8526,10.8 73 73 + 65536,3,8,0,,true,10,8861096723,8450,12.9 74 74 + 65536,3,8,16384,16,true,10,8861073512,8450,13.4 75 75 + 65536,6,8,0,,true,10,8742392375,8337,27.9 76 76 + 65536,6,8,16384,16,true,10,8742335065,8337,29.1 77 77 + 65536,9,8,0,,true,10,8633639539,8233,36.6 78 78 + 65536,9,8,16384,16,true,10,8633630034,8233,37.0 79 79 + 8192,9,8,0,,true,10,9869739461,9412,50.2 80 80 + 8192,9,8,16384,16,true,10,9399964893,8964,54.3 81 81 + 32768,9,8,0,,true,10,8857646963,8447,42.4 82 82 + 32768,9,8,16384,16,true,10,8747823404,8342,51.4 83 83 + 16384,3,8,0,,true,1,9439775263,9002,81.6 84 84 + 4096,0,8,0,,true,10,15722034231,14993,13.5 85 85 + 8192,0,8,0,,true,10,14715967944,14034,12.8 86 86 + 16384,0,8,0,,true,10,14045987598,13395,11.7 87 87 + 32768,0,8,0,,true,10,13597519906,12967,12.9 88 88 + 65536,0,8,0,,true,10,13332609861,12714,16.7 89 89 + 4096,3,8,1024,8,true,10,11071194210,10558,16.7 90 90 + 4096,3,8,1024,32,true,10,10803699447,10303,27.1 91 91 + 4096,3,8,2048,,true,10,10618567648,10126,30.1 92 92 + 4096,3,8,2048,4,true,10,11070980059,10558,21.3 93 93 + 4096,3,8,2048,8,true,10,11070998773,10558,21.2 94 94 + 4096,3,8,2048,16,true,10,10344211112,9865,25.7 95 95 + 4096,3,8,2048,32,true,10,10310192553,9832,29.4 96 96 + 4096,3,8,2048,64,true,10,10275154282,9799,29.1 97 97 + 4096,3,8,4096,8,true,10,10323537489,9845,30.1 98 98 + 4096,3,8,4096,32,true,10,10235055391,9760,29.2 99 99 + 4096,3,8,8192,,true,10,10554955403,10065,30.6 100 100 + 4096,3,8,8192,4,true,10,10343018691,9863,26.8 101 101 + 4096,3,8,8192,8,true,10,10259861034,9784,29.0 102 102 + 4096,3,8,8192,16,true,10,10226087656,9752,29.2 103 103 + 4096,3,8,8192,32,true,10,10208952277,9736,29.4 104 104 + 4096,3,8,8192,64,true,10,10201640423,9729,29.8 105 105 + 4096,3,8,16384,8,true,10,10249137059,9774,30.1 106 106 + 4096,3,8,16384,32,true,10,10224396141,9750,30.7 107 107 + 4096,3,8,65536,8,true,10,10187080812,9715,30.9 108 108 + 4096,3,8,65536,32,true,10,10154462199,9684,33.1 109 109 + 8192,3,8,1024,8,true,10,10040350281,9575,22.2 110 110 + 8192,3,8,1024,32,true,10,10040330045,9575,16.4 111 111 + 8192,3,8,2048,,true,10,9829590607,9374,23.4 112 112 + 8192,3,8,2048,4,true,10,10040326898,9575,16.5 113 113 + 8192,3,8,2048,8,true,10,10040331586,9575,16.5 114 114 + 8192,3,8,2048,16,true,10,10040326454,9575,16.5 115 115 + 8192,3,8,2048,32,true,10,10107002899,9638,23.1 116 116 + 8192,3,8,2048,64,true,10,10089763846,9622,24.7 117 117 + 8192,3,8,4096,8,true,10,10040335592,9575,18.1 118 118 + 8192,3,8,4096,32,true,10,9694641840,9245,22.3 119 119 + 8192,3,8,8192,,true,10,9844667804,9388,24.5 120 120 + 8192,3,8,8192,4,true,10,10040337106,9575,16.8 121 121 + 8192,3,8,8192,8,true,10,9746958438,9295,23.3 122 122 + 8192,3,8,8192,16,true,10,9708134370,9258,25.7 123 123 + 8192,3,8,8192,32,true,10,9682607740,9234,25.7 124 124 + 8192,3,8,8192,64,true,10,9674036954,9225,26.2 125 125 + 8192,3,8,16384,8,true,10,9737625952,9286,26.8 126 126 + 8192,3,8,16384,32,true,10,9699236736,9249,26.5 127 127 + 8192,3,8,65536,8,true,10,9710583278,9260,26.5 128 128 + 8192,3,8,65536,32,true,10,9683941181,9235,28.6 129 129 + 131072,0,8,0,,true,10,13201586596,12590,13.5 130 130 + 131072,1,8,0,,true,10,8823092338,8414,10.4 131 131 + 131072,1,8,16384,16,true,10,8823145617,8414,10.1 132 132 + 131072,3,8,0,,true,10,8712125430,8308,12.5 133 133 + 131072,3,8,16384,16,true,10,8712317883,8308,12.9 134 134 + 131072,6,8,0,,true,10,8600177778,8201,27.0 135 135 + 131072,6,8,16384,16,true,10,8600107633,8201,27.6 136 136 + 131072,9,8,0,,true,10,8487562590,8094,38.2 137 137 + 131072,9,8,16384,16,true,10,8487490042,8094,38.8

+746

space-efficiency-check/sweep-results.html

reviewed

··· 1 1 + <!DOCTYPE html> 2 2 + <html lang="en"> 3 3 + <head> 4 4 + <meta charset="utf-8"> 5 5 + <title>rocks space efficiency</title> 6 6 + <script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script> 7 7 + <style> 8 8 + * { box-sizing: border-box; margin: 0; padding: 0; } 9 9 + body { font-family: system-ui, -apple-system, sans-serif; background: #f5f5f5; color: #222; padding: 20px; } 10 10 + h1 { margin-bottom: 10px; } 11 11 + .subtitle { color: #666; margin-bottom: 20px; } 12 12 + .chart-container { background: #fff; border-radius: 8px; padding: 16px; margin-bottom: 20px; box-shadow: 0 1px 3px rgba(0,0,0,0.1); } 13 13 + .chart-container h2 { font-size: 16px; margin-bottom: 8px; } 14 14 + .chart-row { display: flex; gap: 20px; flex-wrap: wrap; } 15 15 + .chart-row > .chart-container { flex: 1; min-width: 500px; } 16 16 + table { border-collapse: collapse; width: 100%; font-size: 13px; } 17 17 + th, td { text-align: right; padding: 4px 10px; border-bottom: 1px solid #e0e0e0; } 18 18 + th { background: #f0f0f0; cursor: pointer; user-select: none; position: sticky; top: 0; } 19 19 + th:hover { background: #e0e0e0; } 20 20 + td:first-child, th:first-child { text-align: left; } 21 21 + tr:hover td { background: #f8f8ff; } 22 22 + .table-wrap { max-height: 500px; overflow-y: auto; } 23 23 + .filters { display: flex; gap: 16px; flex-wrap: wrap; margin-bottom: 16px; align-items: center; } 24 24 + .filters label { font-size: 13px; font-weight: 600; } 25 25 + .filters select { font-size: 13px; padding: 2px 6px; } 26 26 + .zero-toggle { float: right; font-size: 12px; font-weight: normal; color: #666; cursor: pointer; user-select: none; } 27 27 + .zero-toggle input { vertical-align: middle; cursor: pointer; } 28 28 + .dl-btn { float: right; font-size: 12px; padding: 2px 10px; cursor: pointer; border: 1px solid #ccc; border-radius: 4px; background: #f8f8f8; color: #444; } 29 29 + .dl-btn:hover { background: #e8e8e8; } 30 30 + </style> 31 31 + </head> 32 32 + <body> 33 33 + 34 34 + <h1>rocks records space efficiency</h1> 35 35 + <p class="subtitle">atproto records: <span id="total-keys"></span> keys across <span id="total-accounts"></span> accounts. <span id="config-count"></span> rocksdb configs measured in <span id="total-time"></span>.</p> 36 36 + 37 37 + <div class="chart-row"> 38 38 + <div class="chart-container"> 39 39 + <h2>value size distribution</h2> 40 40 + <div id="chart-value-hist"></div> 41 41 + </div> 42 42 + <div class="chart-container"> 43 43 + <h2>keys per account distribution</h2> 44 44 + <div id="chart-account-hist"></div> 45 45 + </div> 46 46 + </div> 47 47 + 48 48 + <div class="chart-row"> 49 49 + <div class="chart-container"> 50 50 + <h2>total size vs block size by zstd level &amp; dictionary <label class="zero-toggle"><input type="checkbox" data-chart="chart-block-size"> y from 0</label></h2> 51 51 + <div id="chart-block-size"></div> 52 52 + </div> 53 53 + <div class="chart-container"> 54 54 + <h2>total size vs compaction time <label class="zero-toggle"><input type="checkbox" data-chart="chart-pareto"> y from 0</label></h2> 55 55 + <div id="chart-pareto"></div> 56 56 + </div> 57 57 + </div> 58 58 + 59 59 + <div class="chart-row"> 60 60 + <div class="chart-container"> 61 61 + <h2>dictionary training vs 4k, 8k blocks <label class="zero-toggle"><input type="checkbox" data-chart="chart-dict"> y from 0</label></h2> 62 62 + <div id="chart-dict"></div> 63 63 + </div> 64 64 + <div class="chart-container"> 65 65 + <h2>key restart interval <label class="zero-toggle"><input type="checkbox" data-chart="chart-restart"> y from 0</label></h2> 66 66 + <div id="chart-restart"></div> 67 67 + </div> 68 68 + </div> 69 69 + 70 70 + <div class="chart-row"> 71 71 + <div class="chart-container"> 72 72 + <h2>subcompactions: size <label class="zero-toggle"><input type="checkbox" data-chart="chart-subcomp-size"> y from 0</label></h2> 73 73 + <div id="chart-subcomp-size"></div> 74 74 + </div> 75 75 + <div class="chart-container"> 76 76 + <h2>subcompactions: time <label class="zero-toggle"><input type="checkbox" data-chart="chart-subcomp-time"> y from 0</label></h2> 77 77 + <div id="chart-subcomp-time"></div> 78 78 + </div> 79 79 + </div> 80 80 + 81 81 + <div class="chart-container"> 82 82 + <h2>all data <button class="dl-btn" id="download-csv">download csv</button></h2> 83 83 + <div class="filters"> 84 84 + <label>block size: <select id="f-block"><option value="">all</option></select></label> 85 85 + <label>zstd level: <select id="f-level"><option value="">all</option></select></label> 86 86 + <label>dict bytes: <select id="f-dict"><option value="">all</option></select></label> 87 87 + </div> 88 88 + <div class="table-wrap"> 89 89 + <table id="results-table"> 90 90 + <thead><tr> 91 91 + <th data-col="block_size">block_size</th> 92 92 + <th data-col="zstd_level">zstd_level</th> 93 93 + <th data-col="restart_interval">restart_int</th> 94 94 + <th data-col="dict_bytes">dict_bytes</th> 95 95 + <th data-col="train_mult">train_mult</th> 96 96 + <th data-col="opt_filters">filters</th> 97 97 + <th data-col="subcompactions">subcomp</th> 98 98 + <th data-col="size_mib">size_gib</th> 99 99 + <th data-col="compact_secs">compact_s</th> 100 100 + </tr></thead> 101 101 + <tbody id="results-body"></tbody> 102 102 + </table> 103 103 + </div> 104 104 + </div> 105 105 + 106 106 + <script> 107 107 + // ── embed CSV data ──────────────────────────────────────────────── 108 108 + const csvText = `block_size,zstd_level,restart_interval,dict_bytes,train_mult,opt_filters,subcompactions,size_bytes,size_mib,compact_secs 109 109 + 4096,1,8,0,,true,10,11079791854,10566,16.2 110 110 + 4096,1,8,16384,16,true,10,10365069306,9884,20.8 111 111 + 4096,3,4,0,,true,10,11208454885,10689,24.0 112 112 + 4096,3,8,0,,true,10,11070990214,10558,20.2 113 113 + 4096,3,8,1024,,true,10,10617592768,10125,27.7 114 114 + 4096,3,8,1024,4,true,10,11071036189,10558,20.5 115 115 + 4096,3,8,1024,16,true,10,11071022120,10558,21.0 116 116 + 4096,3,8,1024,64,true,10,10782814873,10283,27.3 117 117 + 4096,3,8,4096,,true,10,10598956252,10107,30.5 118 118 + 4096,3,8,4096,4,true,10,11071016375,10558,20.8 119 119 + 4096,3,8,4096,16,true,10,10271883281,9796,25.6 120 120 + 4096,3,8,4096,64,true,10,10213224021,9740,29.1 121 121 + 4096,3,8,16384,,true,10,10556340899,10067,30.9 122 122 + 4096,3,8,16384,4,true,10,10301070056,9823,27.1 123 123 + 4096,3,8,16384,16,true,10,10227379189,9753,30.4 124 124 + 4096,3,8,16384,64,true,10,10213306710,9740,33.9 125 125 + 4096,3,8,65536,,true,10,10465976570,9981,33.0 126 126 + 4096,3,8,65536,4,true,10,10217242670,9743,30.0 127 127 + 4096,3,8,65536,16,true,10,10164362837,9693,34.1 128 128 + 4096,3,8,65536,64,true,10,10139725001,9669,38.2 129 129 + 4096,3,16,0,,true,10,10974836115,10466,25.6 130 130 + 4096,3,32,0,,true,10,10931177138,10424,21.3 131 131 + 4096,3,64,0,,true,10,10931160646,10424,20.8 132 132 + 4096,6,8,0,,true,10,10895448640,10390,33.9 133 133 + 4096,6,8,16384,16,true,10,10093706553,9626,43.9 134 134 + 4096,9,8,0,,true,10,10893183784,10388,58.0 135 135 + 4096,9,8,16384,16,true,10,9946861812,9486,58.1 136 136 + 8192,1,8,0,,true,10,10028074705,9563,20.8 137 137 + 8192,1,8,16384,16,true,10,9811325586,9356,17.4 138 138 + 8192,3,8,0,,false,10,10040289523,9575,18.0 139 139 + 8192,3,8,0,,true,10,10040284381,9575,16.2 140 140 + 8192,3,8,1024,,true,10,9835665629,9380,25.4 141 141 + 8192,3,8,1024,4,true,10,10040279221,9575,16.8 142 142 + 8192,3,8,1024,16,true,10,10040292654,9575,16.8 143 143 + 8192,3,8,1024,64,true,10,10119180082,9650,25.0 144 144 + 8192,3,8,4096,,true,10,9848434052,9392,24.6 145 145 + 8192,3,8,4096,4,true,10,10040294086,9575,16.8 146 146 + 8192,3,8,4096,16,true,10,9717871364,9267,22.5 147 147 + 8192,3,8,4096,64,true,10,9665939593,9218,25.4 148 148 + 8192,3,8,16384,,true,10,9859705853,9402,25.6 149 149 + 8192,3,8,16384,4,true,10,9782101780,9328,24.5 150 150 + 8192,3,8,16384,16,true,10,9713733091,9263,26.0 151 151 + 8192,3,8,16384,64,true,10,9694061307,9244,27.5 152 152 + 8192,3,8,65536,,true,10,9849121697,9392,26.2 153 153 + 8192,3,8,65536,4,true,10,9745808012,9294,24.9 154 154 + 8192,3,8,65536,16,true,10,9696039387,9246,27.5 155 155 + 8192,3,8,65536,64,true,10,9672733565,9224,31.5 156 156 + 8192,6,8,0,,true,10,9874369448,9416,31.7 157 157 + 8192,6,8,16384,16,true,10,9555000218,9112,38.5 158 158 + 16384,1,8,0,,true,10,9401904407,8966,15.6 159 159 + 16384,1,8,16384,16,true,10,9384259110,8949,15.2 160 160 + 16384,3,8,0,,true,4,9439746342,9002,22.6 161 161 + 16384,3,8,0,,true,10,9439798265,9002,14.1 162 162 + 16384,3,8,16384,16,true,10,9336363403,8903,21.6 163 163 + 16384,6,8,0,,true,10,9238560723,8810,28.8 164 164 + 16384,6,8,16384,16,true,10,9157064709,8732,35.4 165 165 + 16384,9,8,0,,true,10,9227102912,8799,55.0 166 166 + 16384,9,8,16384,16,true,10,9002507344,8585,48.0 167 167 + 32768,1,8,0,,true,10,9161174067,8736,12.6 168 168 + 32768,1,8,16384,16,true,10,9129717484,8706,13.8 169 169 + 32768,3,4,0,,true,10,9186445992,8760,13.7 170 170 + 32768,3,8,0,,false,10,9074744264,8654,13.3 171 171 + 32768,3,8,0,,true,10,9074587560,8654,13.3 172 172 + 32768,3,8,16384,16,true,10,9060352672,8640,22.0 173 173 + 32768,3,16,0,,true,10,9009532082,8592,13.8 174 174 + 32768,3,32,0,,true,10,8974169294,8558,13.3 175 175 + 32768,3,64,0,,true,10,8953489124,8538,13.3 176 176 + 32768,6,8,0,,true,10,8959226119,8544,27.5 177 177 + 32768,6,8,16384,16,true,10,8891715689,8479,34.9 178 178 + 65536,1,8,0,,true,10,8940392508,8526,11.5 179 179 + 65536,1,8,16384,16,true,10,8940404881,8526,10.8 180 180 + 65536,3,8,0,,true,10,8861096723,8450,12.9 181 181 + 65536,3,8,16384,16,true,10,8861073512,8450,13.4 182 182 + 65536,6,8,0,,true,10,8742392375,8337,27.9 183 183 + 65536,6,8,16384,16,true,10,8742335065,8337,29.1 184 184 + 65536,9,8,0,,true,10,8633639539,8233,36.6 185 185 + 65536,9,8,16384,16,true,10,8633630034,8233,37.0 186 186 + 8192,9,8,0,,true,10,9869739461,9412,50.2 187 187 + 8192,9,8,16384,16,true,10,9399964893,8964,54.3 188 188 + 32768,9,8,0,,true,10,8857646963,8447,42.4 189 189 + 32768,9,8,16384,16,true,10,8747823404,8342,51.4 190 190 + 16384,3,8,0,,true,1,9439775263,9002,81.6 191 191 + 4096,0,8,0,,true,10,15722034231,14993,13.5 192 192 + 8192,0,8,0,,true,10,14715967944,14034,12.8 193 193 + 16384,0,8,0,,true,10,14045987598,13395,11.7 194 194 + 32768,0,8,0,,true,10,13597519906,12967,12.9 195 195 + 65536,0,8,0,,true,10,13332609861,12714,16.7 196 196 + 4096,3,8,1024,8,true,10,11071194210,10558,16.7 197 197 + 4096,3,8,1024,32,true,10,10803699447,10303,27.1 198 198 + 4096,3,8,2048,,true,10,10618567648,10126,30.1 199 199 + 4096,3,8,2048,4,true,10,11070980059,10558,21.3 200 200 + 4096,3,8,2048,8,true,10,11070998773,10558,21.2 201 201 + 4096,3,8,2048,16,true,10,10344211112,9865,25.7 202 202 + 4096,3,8,2048,32,true,10,10310192553,9832,29.4 203 203 + 4096,3,8,2048,64,true,10,10275154282,9799,29.1 204 204 + 4096,3,8,4096,8,true,10,10323537489,9845,30.1 205 205 + 4096,3,8,4096,32,true,10,10235055391,9760,29.2 206 206 + 4096,3,8,8192,,true,10,10554955403,10065,30.6 207 207 + 4096,3,8,8192,4,true,10,10343018691,9863,26.8 208 208 + 4096,3,8,8192,8,true,10,10259861034,9784,29.0 209 209 + 4096,3,8,8192,16,true,10,10226087656,9752,29.2 210 210 + 4096,3,8,8192,32,true,10,10208952277,9736,29.4 211 211 + 4096,3,8,8192,64,true,10,10201640423,9729,29.8 212 212 + 4096,3,8,16384,8,true,10,10249137059,9774,30.1 213 213 + 4096,3,8,16384,32,true,10,10224396141,9750,30.7 214 214 + 4096,3,8,65536,8,true,10,10187080812,9715,30.9 215 215 + 4096,3,8,65536,32,true,10,10154462199,9684,33.1 216 216 + 8192,3,8,1024,8,true,10,10040350281,9575,22.2 217 217 + 8192,3,8,1024,32,true,10,10040330045,9575,16.4 218 218 + 8192,3,8,2048,,true,10,9829590607,9374,23.4 219 219 + 8192,3,8,2048,4,true,10,10040326898,9575,16.5 220 220 + 8192,3,8,2048,8,true,10,10040331586,9575,16.5 221 221 + 8192,3,8,2048,16,true,10,10040326454,9575,16.5 222 222 + 8192,3,8,2048,32,true,10,10107002899,9638,23.1 223 223 + 8192,3,8,2048,64,true,10,10089763846,9622,24.7 224 224 + 8192,3,8,4096,8,true,10,10040335592,9575,18.1 225 225 + 8192,3,8,4096,32,true,10,9694641840,9245,22.3 226 226 + 8192,3,8,8192,,true,10,9844667804,9388,24.5 227 227 + 8192,3,8,8192,4,true,10,10040337106,9575,16.8 228 228 + 8192,3,8,8192,8,true,10,9746958438,9295,23.3 229 229 + 8192,3,8,8192,16,true,10,9708134370,9258,25.7 230 230 + 8192,3,8,8192,32,true,10,9682607740,9234,25.7 231 231 + 8192,3,8,8192,64,true,10,9674036954,9225,26.2 232 232 + 8192,3,8,16384,8,true,10,9737625952,9286,26.8 233 233 + 8192,3,8,16384,32,true,10,9699236736,9249,26.5 234 234 + 8192,3,8,65536,8,true,10,9710583278,9260,26.5 235 235 + 8192,3,8,65536,32,true,10,9683941181,9235,28.6 236 236 + 131072,0,8,0,,true,10,13201586596,12590,13.5 237 237 + 131072,1,8,0,,true,10,8823092338,8414,10.4 238 238 + 131072,1,8,16384,16,true,10,8823145617,8414,10.1 239 239 + 131072,3,8,0,,true,10,8712125430,8308,12.5 240 240 + 131072,3,8,16384,16,true,10,8712317883,8308,12.9 241 241 + 131072,6,8,0,,true,10,8600177778,8201,27.0 242 242 + 131072,6,8,16384,16,true,10,8600107633,8201,27.6 243 243 + 131072,9,8,0,,true,10,8487562590,8094,38.2 244 244 + 131072,9,8,16384,16,true,10,8487490042,8094,38.8 245 245 + `; 246 246 + 247 247 + // ── parse ───────────────────────────────────────────────────────── 248 248 + function parseCSV(text) { 249 249 + const lines = text.trim().split('\n'); 250 250 + const headers = lines[0].split(','); 251 251 + return lines.slice(1).map(line => { 252 252 + const vals = line.split(','); 253 253 + const row = {}; 254 254 + headers.forEach((h, i) => { 255 255 + const v = vals[i]; 256 256 + if (h === 'opt_filters') row[h] = v === 'true'; 257 257 + else if (h === 'train_mult') row[h] = v === '' ? null : +v; 258 258 + else row[h] = +v; 259 259 + }); 260 260 + return row; 261 261 + }); 262 262 + } 263 263 + 264 264 + const data = parseCSV(csvText); 265 265 + document.getElementById('config-count').textContent = data.length; 266 266 + const totalSecs = data.reduce((s, d) => s + d.compact_secs, 0); 267 267 + const totalMins = (totalSecs / 60).toFixed(1); 268 268 + document.getElementById('total-time').textContent = `${totalMins}min`; 269 269 + 270 270 + // ── data profile stats ─────────────────────────────────────────── 271 271 + const stats = { 272 272 + total_keys: 110471379, 273 273 + total_accounts: 78568, 274 274 + value_size_hist: [ 275 275 + [16, 32, 32], [32, 64, 30165], [64, 128, 13339156], [128, 256, 79513656], 276 276 + [256, 512, 13262139], [512, 1024, 3916643], [1024, 2048, 384505], 277 277 + [2048, 4096, 20803], [4096, 8192, 3652], [8192, 16384, 472], 278 278 + [16384, 32768, 68], [32768, 65536, 44], [65536, 131072, 37], [131072, 262144, 7], 279 279 + ], 280 280 + keys_per_account_hist: [ 281 281 + [1, 2, 2630], [2, 4, 8451], [4, 8, 6955], [8, 16, 7082], [16, 32, 7424], 282 282 + [32, 64, 7898], [64, 128, 7817], [128, 256, 7196], [256, 512, 6323], 283 283 + [512, 1024, 5230], [1024, 2048, 4087], [2048, 4096, 2869], [4096, 8192, 2038], 284 284 + [8192, 16384, 1223], [16384, 32768, 747], [32768, 65536, 379], 285 285 + [65536, 131072, 165], [131072, 262144, 45], [262144, 524288, 6], 286 286 + [524288, 1048576, 2], [1048576, 2097152, 1], 287 287 + ], 288 288 + }; 289 289 + 290 290 + const fmtNum = n => n.toLocaleString(); 291 291 + document.getElementById('total-keys').textContent = fmtNum(stats.total_keys); 292 292 + document.getElementById('total-accounts').textContent = fmtNum(stats.total_accounts); 293 293 + 294 294 + // ── helpers ─────────────────────────────────────────────────────── 295 295 + const gib = d => +(d.size_mib / 1024).toFixed(2); 296 296 + const levelName = l => l === 0 ? 'lz4' : `z${l}`; 297 297 + const levelColors = { 0: '#888', 1: '#4ecdc4', 3: '#45b7d1', 6: '#f7b731', 9: '#fc5c65' }; 298 298 + 299 299 + const plotLayout = (opts = {}) => ({ 300 300 + margin: { t: 30, b: 50, l: 60, r: 20 }, 301 301 + hovermode: 'closest', 302 302 + font: { family: 'system-ui', size: 12 }, 303 303 + ...opts, 304 304 + }); 305 305 + 306 306 + function groupBy(arr, keyFn) { 307 307 + const groups = {}; 308 308 + arr.forEach(d => { 309 309 + const k = keyFn(d); 310 310 + (groups[k] = groups[k] || []).push(d); 311 311 + }); 312 312 + return groups; 313 313 + } 314 314 + 315 315 + // ── chart 1: size vs block size (with level slider) ────────────── 316 316 + function chartBlockSize() { 317 317 + // filter to standard configs: ri=8, filters=on, subcomp=10 318 318 + const base = data.filter(d => 319 319 + d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10 && 320 320 + ((d.dict_bytes === 0 && d.train_mult === null) || 321 321 + (d.dict_bytes === 16384 && d.train_mult === 16)) 322 322 + ); 323 323 + 324 324 + const levels = [...new Set(base.map(d => d.zstd_level))].sort((a, b) => a - b); 325 325 + 326 326 + // build all traces (one per level × dict combo) 327 327 + const traces = []; 328 328 + const traceLevel = []; // which level each trace belongs to 329 329 + for (const level of levels) { 330 330 + for (const dict of [0, 16384]) { 331 331 + const subset = base.filter(d => d.zstd_level === level && d.dict_bytes === dict); 332 332 + if (subset.length === 0) continue; 333 333 + subset.sort((a, b) => a.block_size - b.block_size); 334 334 + traces.push({ 335 335 + x: subset.map(d => d.block_size / 1024 + 'k'), 336 336 + y: subset.map(gib), 337 337 + name: `${levelName(level)} ${dict > 0 ? 'dict=16k' : 'no dict'}`, 338 338 + mode: 'lines+markers', 339 339 + line: { dash: dict > 0 ? 'dash' : 'solid', color: levelColors[level] || '#888', width: 2.5 }, 340 340 + marker: { symbol: dict > 0 ? 'diamond' : 'circle', size: 8 }, 341 341 + text: subset.map(d => `${d.compact_secs}s`), 342 342 + hovertemplate: '%{x} block, ' + levelName(level) + (dict > 0 ? ' +dict' : '') + 343 343 + '<br>%{y:.2f} GiB<br>%{text}<extra></extra>', 344 344 + }); 345 345 + traceLevel.push(level); 346 346 + } 347 347 + } 348 348 + 349 349 + // per-level steps: highlight one, fade the rest 350 350 + const steps = [{ 351 351 + label: 'all', 352 352 + method: 'restyle', 353 353 + args: [{ 354 354 + 'line.color': traceLevel.map(l => levelColors[l] || '#888'), 355 355 + 'line.width': traceLevel.map(() => 2.5), 356 356 + 'marker.opacity': traceLevel.map(() => 1), 357 357 + 'marker.size': traceLevel.map(() => 8), 358 358 + }], 359 359 + }]; 360 360 + 361 361 + for (const hl of levels) { 362 362 + steps.push({ 363 363 + label: levelName(hl), 364 364 + method: 'restyle', 365 365 + args: [{ 366 366 + 'line.color': traceLevel.map(l => l === hl ? levelColors[l] : (levelColors[l] || '#888') + '30'), 367 367 + 'line.width': traceLevel.map(l => l === hl ? 3.5 : 1), 368 368 + 'marker.opacity': traceLevel.map(l => l === hl ? 1 : 0.2), 369 369 + 'marker.size': traceLevel.map(l => l === hl ? 10 : 5), 370 370 + }], 371 371 + }); 372 372 + } 373 373 + 374 374 + Plotly.newPlot('chart-block-size', traces, plotLayout({ 375 375 + xaxis: { title: 'Block Size', type: 'category' }, 376 376 + yaxis: { title: 'Size (GiB)' }, 377 377 + legend: { x: 1, xanchor: 'right', y: 1 }, 378 378 + sliders: [{ 379 379 + active: 0, 380 380 + pad: { t: 30 }, 381 381 + currentvalue: { prefix: 'zstd level: ', font: { size: 13 } }, 382 382 + steps: steps, 383 383 + }], 384 384 + })); 385 385 + } 386 386 + 387 387 + // ── chart 2: pareto (size vs time, with zstd level slider) ─────── 388 388 + function chartPareto() { 389 389 + const blockColors = { 390 390 + 4096: '#fc5c65', 8192: '#f7b731', 16384: '#45b7d1', 391 391 + 32768: '#4ecdc4', 65536: '#a55eea', 392 392 + }; 393 393 + const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b); 394 394 + const levels = [...new Set(data.map(d => d.zstd_level))].sort((a, b) => a - b); 395 395 + 396 396 + const traces = []; 397 397 + const traceLevel = []; 398 398 + 399 399 + for (const bs of blocks) { 400 400 + const subset = data.filter(d => d.block_size === bs && d.subcompactions === 10); 401 401 + traces.push({ 402 402 + x: subset.map(d => d.compact_secs), 403 403 + y: subset.map(gib), 404 404 + name: (bs / 1024) + 'k', 405 405 + mode: 'markers', 406 406 + marker: { size: 7, color: subset.map(() => blockColors[bs] || '#888'), opacity: 1 }, 407 407 + customdata: subset.map(d => d.zstd_level), 408 408 + text: subset.map(d => { 409 409 + const parts = [levelName(d.zstd_level), `ri=${d.restart_interval}`]; 410 410 + if (d.dict_bytes > 0) parts.push(`dict=${d.dict_bytes}`); 411 411 + if (d.train_mult !== null) parts.push(`train=${d.train_mult}x`); 412 412 + return parts.join(', '); 413 413 + }), 414 414 + hovertemplate: '%{text}<br>%{y:.2f} GiB, %{x:.1f}s<extra>%{fullData.name} blocks</extra>', 415 415 + }); 416 416 + traceLevel.push(subset.map(d => d.zstd_level)); 417 417 + } 418 418 + 419 419 + // slider: highlight by zstd level 420 420 + const steps = [{ 421 421 + label: 'all', 422 422 + method: 'restyle', 423 423 + args: [{ 424 424 + 'marker.opacity': traceLevel.map(levels => levels.map(() => 1)), 425 425 + 'marker.size': traceLevel.map(levels => levels.map(() => 7)), 426 426 + }], 427 427 + }]; 428 428 + 429 429 + for (const hl of levels) { 430 430 + steps.push({ 431 431 + label: levelName(hl), 432 432 + method: 'restyle', 433 433 + args: [{ 434 434 + 'marker.opacity': traceLevel.map(levels => levels.map(l => l === hl ? 1 : 0.12)), 435 435 + 'marker.size': traceLevel.map(levels => levels.map(l => l === hl ? 10 : 5)), 436 436 + }], 437 437 + }); 438 438 + } 439 439 + 440 440 + Plotly.newPlot('chart-pareto', traces, plotLayout({ 441 441 + xaxis: { title: 'Compaction Time (s)' }, 442 442 + yaxis: { title: 'Size (GiB)' }, 443 443 + legend: { title: { text: 'Block Size' } }, 444 444 + sliders: [{ 445 445 + active: 0, 446 446 + pad: { t: 30 }, 447 447 + currentvalue: { prefix: 'zstd level: ', font: { size: 13 } }, 448 448 + steps: steps, 449 449 + }], 450 450 + })); 451 451 + } 452 452 + 453 453 + // ── chart 3: dictionary training sweep (with block size slider) ─── 454 454 + function chartDict() { 455 455 + const traces = []; 456 456 + const traceBlock = []; // which block size each trace belongs to 457 457 + const colors = { 'None': '#45b7d1', '4': '#4ecdc4', '16': '#f7b731', '64': '#fc5c65' }; 458 458 + const blockSizes = [4, 8]; 459 459 + 460 460 + for (const blockKb of blockSizes) { 461 461 + const bs = blockKb * 1024; 462 462 + const subset = data.filter(d => 463 463 + d.block_size === bs && d.dict_bytes > 0 && 464 464 + d.zstd_level === 3 && d.restart_interval === 8 465 465 + ); 466 466 + if (subset.length === 0) continue; 467 467 + 468 468 + const byTrain = groupBy(subset, d => d.train_mult === null ? 'None' : String(d.train_mult)); 469 469 + 470 470 + for (const [tm, rows] of Object.entries(byTrain)) { 471 471 + rows.sort((a, b) => a.dict_bytes - b.dict_bytes); 472 472 + const label = tm === 'None' ? 'default' : tm + 'x'; 473 473 + traces.push({ 474 474 + x: rows.map(d => d.dict_bytes / 1024 + 'k'), 475 475 + y: rows.map(gib), 476 476 + name: `${blockKb}k block, train=${label}`, 477 477 + mode: 'lines+markers', 478 478 + line: { color: colors[tm] || '#888', dash: blockKb === 4 ? 'solid' : 'dot', width: 2.5 }, 479 479 + marker: { size: 7 }, 480 480 + hovertemplate: `${blockKb}k block, dict=%{x}, train=${label}<br>%{y:.2f} GiB<extra></extra>`, 481 481 + }); 482 482 + traceBlock.push(blockKb); 483 483 + } 484 484 + 485 485 + // add no-dict baseline 486 486 + const baseline = data.find(d => 487 487 + d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 && 488 488 + d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10 489 489 + ); 490 490 + if (baseline) { 491 491 + traces.push({ 492 492 + x: ['1k', '4k', '16k', '64k'], 493 493 + y: Array(4).fill(gib(baseline)), 494 494 + name: `${blockKb}k block, no dict`, 495 495 + mode: 'lines', 496 496 + line: { color: '#ccc', dash: blockKb === 4 ? 'solid' : 'dot', width: 1 }, 497 497 + hoverinfo: 'skip', 498 498 + }); 499 499 + traceBlock.push(blockKb); 500 500 + } 501 501 + } 502 502 + 503 503 + // slider steps 504 504 + const fadedColors = { 'None': '#45b7d130', '4': '#4ecdc430', '16': '#f7b73130', '64': '#fc5c6530' }; 505 505 + const steps = [{ 506 506 + label: 'both', 507 507 + method: 'restyle', 508 508 + args: [{ 509 509 + 'line.width': traceBlock.map(() => 2.5), 510 510 + 'marker.opacity': traceBlock.map(() => 1), 511 511 + 'marker.size': traceBlock.map(() => 7), 512 512 + }], 513 513 + }]; 514 514 + 515 515 + for (const hl of blockSizes) { 516 516 + steps.push({ 517 517 + label: `${hl}k`, 518 518 + method: 'restyle', 519 519 + args: [{ 520 520 + 'line.width': traceBlock.map(b => b === hl ? 3.5 : 1), 521 521 + 'marker.opacity': traceBlock.map(b => b === hl ? 1 : 0.15), 522 522 + 'marker.size': traceBlock.map(b => b === hl ? 9 : 4), 523 523 + }], 524 524 + }); 525 525 + } 526 526 + 527 527 + Plotly.newPlot('chart-dict', traces, plotLayout({ 528 528 + xaxis: { title: 'Dictionary Size', type: 'category' }, 529 529 + yaxis: { title: 'Size (GiB)' }, 530 530 + margin: { t: 30, b: 50, l: 60, r: 160 }, 531 531 + legend: { x: 1.02, xanchor: 'left', y: 1, font: { size: 10 } }, 532 532 + sliders: [{ 533 533 + active: 0, 534 534 + pad: { t: 30 }, 535 535 + currentvalue: { prefix: 'block size: ', font: { size: 13 } }, 536 536 + steps: steps, 537 537 + }], 538 538 + })); 539 539 + } 540 540 + 541 541 + // ── chart 4: restart interval ───────────────────────────────────── 542 542 + function chartRestart() { 543 543 + const traces = []; 544 544 + const colors = { 4: '#fc5c65', 32: '#45b7d1' }; 545 545 + 546 546 + for (const blockKb of [4, 32]) { 547 547 + const bs = blockKb * 1024; 548 548 + const subset = data.filter(d => 549 549 + d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 && 550 550 + d.opt_filters && d.subcompactions === 10 && 551 551 + [4, 8, 16, 32, 64].includes(d.restart_interval) 552 552 + ).sort((a, b) => a.restart_interval - b.restart_interval); 553 553 + 554 554 + if (subset.length > 0) { 555 555 + traces.push({ 556 556 + x: subset.map(d => d.restart_interval), 557 557 + y: subset.map(gib), 558 558 + name: `${blockKb}k blocks`, 559 559 + mode: 'lines+markers', 560 560 + marker: { size: 8 }, 561 561 + line: { color: colors[blockKb] }, 562 562 + hovertemplate: `${blockKb}k blocks, restart interval %{x}<br>%{y:.2f} GiB<extra></extra>`, 563 563 + }); 564 564 + } 565 565 + } 566 566 + 567 567 + Plotly.newPlot('chart-restart', traces, plotLayout({ 568 568 + xaxis: { title: 'Key Restart Interval', type: 'category' }, 569 569 + yaxis: { title: 'Size (GiB)' }, 570 570 + legend: { x: 1, xanchor: 'right', y: 1 }, 571 571 + })); 572 572 + } 573 573 + 574 574 + // ── chart 5: subcompactions ─────────────────────────────────────── 575 575 + function chartSubcomp() { 576 576 + const subData = data.filter(d => 577 577 + d.block_size === 16384 && d.dict_bytes === 0 && d.zstd_level === 3 && 578 578 + d.restart_interval === 8 && d.opt_filters 579 579 + ).sort((a, b) => a.subcompactions - b.subcompactions); 580 580 + 581 581 + if (subData.length > 0) { 582 582 + Plotly.newPlot('chart-subcomp-size', [{ 583 583 + x: subData.map(d => d.subcompactions), 584 584 + y: subData.map(gib), 585 585 + mode: 'lines+markers', 586 586 + marker: { size: 8 }, 587 587 + line: { color: '#45b7d1' }, 588 588 + hovertemplate: 'subcompactions=%{x}<br>%{y:.2f} GiB<extra></extra>', 589 589 + }], plotLayout({ 590 590 + xaxis: { title: 'Max Subcompactions', type: 'category' }, 591 591 + yaxis: { title: 'Size (GiB)' }, 592 592 + showlegend: false, 593 593 + })); 594 594 + 595 595 + Plotly.newPlot('chart-subcomp-time', [{ 596 596 + x: subData.map(d => d.subcompactions), 597 597 + y: subData.map(d => d.compact_secs), 598 598 + mode: 'lines+markers', 599 599 + marker: { size: 8 }, 600 600 + line: { color: '#f7b731' }, 601 601 + hovertemplate: 'subcompactions=%{x}<br>%{y:.1f}s<extra></extra>', 602 602 + }], plotLayout({ 603 603 + xaxis: { title: 'Max Subcompactions', type: 'category' }, 604 604 + yaxis: { title: 'Time (s)' }, 605 605 + showlegend: false, 606 606 + })); 607 607 + } 608 608 + } 609 609 + 610 610 + // ── data table ──────────────────────────────────────────────────── 611 611 + function buildTable(filteredData) { 612 612 + const tbody = document.getElementById('results-body'); 613 613 + tbody.innerHTML = filteredData.map(d => `<tr> 614 614 + <td>${d.block_size / 1024}k</td> 615 615 + <td>${levelName(d.zstd_level)}</td> 616 616 + <td>${d.restart_interval}</td> 617 617 + <td>${d.dict_bytes}</td> 618 618 + <td>${d.train_mult === null ? '' : d.train_mult + 'x'}</td> 619 619 + <td>${d.opt_filters ? 'on' : 'off'}</td> 620 620 + <td>${d.subcompactions}</td> 621 621 + <td><strong>${(d.size_mib / 1024).toFixed(2)}</strong></td> 622 622 + <td>${d.compact_secs.toFixed(1)}</td> 623 623 + </tr>`).join(''); 624 624 + } 625 625 + 626 626 + function setupFilters() { 627 627 + const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b); 628 628 + const levels = [...new Set(data.map(d => d.zstd_level))].sort((a, b) => a - b); 629 629 + const dicts = [...new Set(data.map(d => d.dict_bytes))].sort((a, b) => a - b); 630 630 + 631 631 + const fBlock = document.getElementById('f-block'); 632 632 + const fLevel = document.getElementById('f-level'); 633 633 + const fDict = document.getElementById('f-dict'); 634 634 + 635 635 + blocks.forEach(b => { const o = document.createElement('option'); o.value = b; o.textContent = (b/1024) + 'k'; fBlock.appendChild(o); }); 636 636 + levels.forEach(l => { const o = document.createElement('option'); o.value = l; o.textContent = levelName(l); fLevel.appendChild(o); }); 637 637 + dicts.forEach(d => { const o = document.createElement('option'); o.value = d; o.textContent = d; fDict.appendChild(o); }); 638 638 + 639 639 + const applyFilters = () => { 640 640 + let filtered = data; 641 641 + if (fBlock.value) filtered = filtered.filter(d => d.block_size === +fBlock.value); 642 642 + if (fLevel.value) filtered = filtered.filter(d => d.zstd_level === +fLevel.value); 643 643 + if (fDict.value !== '') filtered = filtered.filter(d => d.dict_bytes === +fDict.value); 644 644 + buildTable(filtered); 645 645 + }; 646 646 + 647 647 + [fBlock, fLevel, fDict].forEach(el => el.addEventListener('change', applyFilters)); 648 648 + } 649 649 + 650 650 + // sortable headers 651 651 + let sortCol = 'size_mib', sortAsc = true; 652 652 + document.querySelectorAll('#results-table th').forEach(th => { 653 653 + th.addEventListener('click', () => { 654 654 + const col = th.dataset.col; 655 655 + if (sortCol === col) sortAsc = !sortAsc; 656 656 + else { sortCol = col; sortAsc = true; } 657 657 + const filtered = getFilteredData(); 658 658 + filtered.sort((a, b) => { 659 659 + const av = a[col] ?? -1, bv = b[col] ?? -1; 660 660 + return sortAsc ? av - bv : bv - av; 661 661 + }); 662 662 + buildTable(filtered); 663 663 + }); 664 664 + }); 665 665 + 666 666 + function getFilteredData() { 667 667 + let filtered = [...data]; 668 668 + const fBlock = document.getElementById('f-block').value; 669 669 + const fLevel = document.getElementById('f-level').value; 670 670 + const fDict = document.getElementById('f-dict').value; 671 671 + if (fBlock) filtered = filtered.filter(d => d.block_size === +fBlock); 672 672 + if (fLevel) filtered = filtered.filter(d => d.zstd_level === +fLevel); 673 673 + if (fDict !== '') filtered = filtered.filter(d => d.dict_bytes === +fDict); 674 674 + return filtered; 675 675 + } 676 676 + 677 677 + // ── CSV download ────────────────────────────────────────────────── 678 678 + document.getElementById('download-csv').addEventListener('click', () => { 679 679 + const blob = new Blob([csvText], { type: 'text/csv' }); 680 680 + const a = document.createElement('a'); 681 681 + a.href = URL.createObjectURL(blob); 682 682 + a.download = 'sweep-results.csv'; 683 683 + a.click(); 684 684 + URL.revokeObjectURL(a.href); 685 685 + }); 686 686 + 687 687 + // ── zero-Y toggles ──────────────────────────────────────────────── 688 688 + document.querySelectorAll('.zero-toggle input').forEach(cb => { 689 689 + cb.checked = false; // override browser autofill on refresh 690 690 + cb.addEventListener('change', () => { 691 691 + const chartId = cb.dataset.chart; 692 692 + const mode = cb.checked ? 'tozero' : 'normal'; 693 693 + Plotly.relayout(chartId, { 'yaxis.rangemode': mode }); 694 694 + }); 695 695 + }); 696 696 + 697 697 + // ── data profile histograms ──────────────────────────────────────── 698 698 + function fmtBytes(n) { 699 699 + if (n >= 1048576) return (n / 1048576).toFixed(0) + 'M'; 700 700 + if (n >= 1024) return (n / 1024).toFixed(0) + 'k'; 701 701 + return n + ''; 702 702 + } 703 703 + 704 704 + function chartValueHist() { 705 705 + const h = stats.value_size_hist; 706 706 + Plotly.newPlot('chart-value-hist', [{ 707 707 + x: h.map(([lo, hi]) => `${fmtBytes(lo)}-${fmtBytes(hi)}`), 708 708 + y: h.map(([,, c]) => c), 709 709 + type: 'bar', 710 710 + marker: { color: '#45b7d1' }, 711 711 + hovertemplate: '%{x} bytes<br>%{y:,} keys<extra></extra>', 712 712 + }], plotLayout({ 713 713 + xaxis: { title: 'Value Size (bytes)', type: 'category' }, 714 714 + yaxis: { title: 'Keys', type: 'log' }, 715 715 + showlegend: false, 716 716 + })); 717 717 + } 718 718 + 719 719 + function chartAccountHist() { 720 720 + const h = stats.keys_per_account_hist; 721 721 + Plotly.newPlot('chart-account-hist', [{ 722 722 + x: h.map(([lo, hi]) => `${fmtNum(lo)}-${fmtNum(hi)}`), 723 723 + y: h.map(([,, c]) => c), 724 724 + type: 'bar', 725 725 + marker: { color: '#4ecdc4' }, 726 726 + hovertemplate: '%{x} keys<br>%{y:,} accounts<extra></extra>', 727 727 + }], plotLayout({ 728 728 + xaxis: { title: 'Keys per Account', type: 'category' }, 729 729 + yaxis: { title: 'Accounts', type: 'log' }, 730 730 + showlegend: false, 731 731 + })); 732 732 + } 733 733 + 734 734 + // ── init ────────────────────────────────────────────────────────── 735 735 + chartValueHist(); 736 736 + chartAccountHist(); 737 737 + chartBlockSize(); 738 738 + chartPareto(); 739 739 + chartDict(); 740 740 + chartRestart(); 741 741 + chartSubcomp(); 742 742 + setupFilters(); 743 743 + buildTable(data); 744 744 + </script> 745 745 + </body> 746 746 + </html>