A public mirror for the whole atmosphere hubble.microcosm.blue
27
fork

Configure Feed

Select the types of activity you want to include in your feed.

try a lot of rocksdb options

phil e3cf014b 7e9d71e2

+134 -14
+59
space-efficiency-check/readme.md
··· 1 + # space efficiency check 2 + 3 + assessing rocksdb's space efficiency 4 + 5 + 6 + ### first full run: zstd for layers 2+ 7 + 8 + format `<did>/<repo path> => <cbor>` 9 + 10 + enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081 11 + progress repos=490000 records=665231031 failed=366 12 + workers finished elapsed=1457.296831042s 13 + import complete repos=490328 empty=19172 records=665928371 failed=366 14 + total input size car_bytes=246501100813 car_mib=235081 15 + db size before compaction bytes=69084296913 mib=65883 16 + running manual compaction... 17 + db size after compaction bytes=66225638581 mib=63157 compact_elapsed=751.790672875s 18 + db size / car bytes ratio="0.269" 19 + 20 + 21 + #### manually compact with bottommost zstd 6 22 + 23 + mb after: 62186 (65207279363) or another 1.5% only 24 + 25 + #### manually compact with bottommost zstd 9 26 + 27 + mb after: 62173 (65193700507) or barely any better (30m to manually compact) 28 + 29 + #### larger block size 4 -> 64KB (zstd 3) 30 + 31 + mb after: 50576 (53033071353) or 20ish% hey! (4.6x vs raw car) 32 + 521s 33 + 34 + #### zstd dictionary (16k) (64KB block zstd 3) 35 + 36 + mb after: 50647 (53108254067) or slightly worse actually (4.6x vs raw car) 37 + 860s 38 + 39 + #### zstd optimize for hits (64KB block zstd 3 no dict) 40 + 41 + mb after: 50576 (53033105629) or basically nothin more 42 + 429s 43 + 44 + #### zstd key restart interval 64 (optimize hits, 64KB block, zstd3 no dict) 45 + 46 + mb after: 50237 (52677638685) (4.7x) 47 + 425s 48 + 49 + #### zstd 32k dict w 50 + 51 + 50237 52677524253 52 + 455s 53 + 54 + #### 128m sst 55 + 56 + just 1mb better, eeh 57 + 494s 58 + 59 +
+75 -14
space-efficiency-check/src/main.rs
··· 1 1 use clap::Parser; 2 - use rust_rocksdb::{DB, DBCompressionType, Options}; 2 + use rust_rocksdb::{DB, CompactOptions, DBCompressionType, Options}; 3 3 4 4 use std::path::{Path, PathBuf}; 5 5 use std::sync::{Arc, atomic::Ordering}; ··· 23 23 /// repo-stream memory limit: we don't disk spill; set higher than largest car 24 24 #[arg(long, default_value_t = 1024)] 25 25 mem_limit_mb: usize, 26 + 27 + /// skip import and just do manual compact 28 + #[arg(long, action)] 29 + just_compact: bool, 26 30 } 27 31 28 32 fn open_db(path: &Path) -> Result<DB, rust_rocksdb::Error> { ··· 34 38 35 39 // compress lower levels more 36 40 opts.set_compression_per_level(&[ 37 - DBCompressionType::Lz4, // L0 41 + DBCompressionType::None, // L0 38 42 DBCompressionType::Lz4, // L1 39 43 DBCompressionType::Zstd, // L2 40 44 DBCompressionType::Zstd, // L3 ··· 42 46 DBCompressionType::Zstd, // L5 43 47 DBCompressionType::Zstd, // L6 44 48 ]); 49 + 50 + opts.set_compression_options( 51 + -14, // window_bits (default, -14 means use zstd default) 52 + 3, // level (default 3) 53 + 0, // strategy (0 = default) 54 + 16_384, // max_dict_bytes (0 = no dictionary) (tried 16k) 55 + ); 56 + opts.set_zstd_max_train_bytes(16 * 16_384); 57 + 45 58 opts.set_bottommost_compression_type(rust_rocksdb::DBCompressionType::Zstd); 46 59 60 + opts.set_bottommost_compression_options( 61 + -14, // window_bits (default, -14 means use zstd default) 62 + 3, // level (default 3) 63 + 0, // strategy (0 = default) 64 + 16_384, // max_dict_bytes (0 = no dictionary) (tried 16k) 65 + true, // enabled 66 + ); 67 + 68 + // probably investigate this one: use more training data for dictionary 69 + opts.set_bottommost_zstd_max_train_bytes(16 * 16_384, true); // 256KB of samples for a 16KB dict 70 + 71 + // skip filters on bottom-most layer 72 + opts.set_optimize_filters_for_hits(true); 73 + 47 74 // write into new buffer while flushing? 48 75 opts.set_max_write_buffer_number(4); 49 76 50 - // try to speed up compaction? 51 - opts.set_max_background_jobs(8); 77 + // ugh this probably wasn't exactly the right move 78 + // there's increase_parallelism() 79 + // // try to speed up compaction? 80 + // opts.set_max_background_jobs(8); 81 + opts.increase_parallelism(10); 52 82 53 83 // larger write buffer for less frequent flushes 54 - opts.set_write_buffer_size(64 * 1024 * 1024); 84 + opts.set_write_buffer_size(256 * 2_usize.pow(20)); // default 64MB 85 + 86 + // apparently can help with space amp, less partially-filled levels 87 + opts.set_level_compaction_dynamic_level_bytes(true); 88 + 89 + // larger sst files (ehhh) 90 + // don't help 91 + // opts.set_target_file_size_base(128 * 2_u64.pow(20)); // default 64MB 92 + 93 + // larger blocks for better compression (good scan, worse point read) 94 + let mut block_opts = rust_rocksdb::BlockBasedOptions::default(); 95 + block_opts.set_block_size(256 * 2_usize.pow(10)); // N * KB blocks 96 + block_opts.set_block_restart_interval(64); 97 + opts.set_block_based_table_factory(&block_opts); 98 + 99 + // enable ONLY during bulk-import 100 + opts.set_unordered_write(true); 101 + 102 + opts.set_max_subcompactions(4); // default 1 55 103 56 104 DB::open(&opts, path) 57 105 } ··· 63 111 let args = Args::parse(); 64 112 65 113 let db = Arc::new(open_db(&args.db_dir)?); 114 + 115 + if args.just_compact { 116 + manually_compact(db)?; 117 + return Ok(()) 118 + } 66 119 67 120 let stats = run_workers(&args.car_dir, db.clone(), args.workers, args.mem_limit_mb).await?; 68 121 ··· 86 139 "db size before compaction" 87 140 ); 88 141 142 + let size_after = manually_compact(db)?; 143 + 144 + if car_bytes > 0 { 145 + let ratio = size_after as f64 / car_bytes as f64; 146 + tracing::info!(ratio = format!("{ratio:.3}"), "db size / car bytes"); 147 + } 148 + 149 + Ok(()) 150 + } 151 + 152 + fn manually_compact(db: Arc<DB>) -> Result<u64, ProcessError> { 153 + let mut compact_opts = CompactOptions::default(); 154 + compact_opts.set_bottommost_level_compaction(rust_rocksdb::BottommostLevelCompaction::Force); 155 + 89 156 tracing::info!("running manual compaction..."); 90 157 let compact_start = std::time::Instant::now(); 91 - db.compact_range(None::<&[u8]>, None::<&[u8]>); 158 + db.compact_range_opt(None::<&[u8]>, None::<&[u8]>, &compact_opts); 92 159 let compact_elapsed = compact_start.elapsed(); 93 160 94 161 let size_after = db_size(&db); 95 162 tracing::info!( 96 163 bytes = size_after, 97 - mib = size_after / (1024 * 1024), 164 + mib = size_after / 2_u64.pow(20), 98 165 ?compact_elapsed, 99 166 "db size after compaction", 100 167 ); 101 - 102 - if car_bytes > 0 { 103 - let ratio = size_after as f64 / car_bytes as f64; 104 - tracing::info!(ratio = format!("{ratio:.3}"), "db size / car bytes"); 105 - } 106 - 107 - Ok(()) 168 + Ok(size_after) 108 169 } 109 170 110 171 fn db_size(db: &DB) -> u64 {