A public mirror for the whole atmosphere
hubble.microcosm.blue
try a lot of rocksdb options
+134
-14
+59
space-efficiency-check/readme.md
+59
space-efficiency-check/readme.md
···
1
+
# space efficiency check
2
+
3
+
assessing rocksdb's space efficiency
4
+
5
+
6
+
### first full run: zstd for layers 2+
7
+
8
+
format `<did>/<repo path> => <cbor>`
9
+
10
+
enumerated car directory files=490694 car_bytes=246501100813 car_mib=235081
11
+
progress repos=490000 records=665231031 failed=366
12
+
workers finished elapsed=1457.296831042s
13
+
import complete repos=490328 empty=19172 records=665928371 failed=366
14
+
total input size car_bytes=246501100813 car_mib=235081
15
+
db size before compaction bytes=69084296913 mib=65883
16
+
running manual compaction...
17
+
db size after compaction bytes=66225638581 mib=63157 compact_elapsed=751.790672875s
18
+
db size / car bytes ratio="0.269"
19
+
20
+
21
+
#### manually compact with bottommost zstd 6
22
+
23
+
mb after: 62186 (65207279363) or another 1.5% only
24
+
25
+
#### manually compact with bottommost zstd 9
26
+
27
+
mb after: 62173 (65193700507) or barely any better (30m to manually compact)
28
+
29
+
#### larger block size 4 -> 64KB (zstd 3)
30
+
31
+
mb after: 50576 (53033071353) or 20ish% hey! (4.6x vs raw car)
32
+
521s
33
+
34
+
#### zstd dictionary (16k) (64KB block zstd 3)
35
+
36
+
mb after: 50647 (53108254067) or slightly worse actually (4.6x vs raw car)
37
+
860s
38
+
39
+
#### zstd optimize for hits (64KB block zstd 3 no dict)
40
+
41
+
mb after: 50576 (53033105629) or basically nothin more
42
+
429s
43
+
44
+
#### zstd key restart interval 64 (optimize hits, 64KB block, zstd3 no dict)
45
+
46
+
mb after: 50237 (52677638685) (4.7x)
47
+
425s
48
+
49
+
#### zstd 32k dict w
50
+
51
+
50237 52677524253
52
+
455s
53
+
54
+
#### 128m sst
55
+
56
+
just 1mb better, eeh
57
+
494s
58
+
59
+
+75
-14
space-efficiency-check/src/main.rs
+75
-14
space-efficiency-check/src/main.rs
···
1
1
use clap::Parser;
2
-
use rust_rocksdb::{DB, DBCompressionType, Options};
2
+
use rust_rocksdb::{DB, CompactOptions, DBCompressionType, Options};
3
3
4
4
use std::path::{Path, PathBuf};
5
5
use std::sync::{Arc, atomic::Ordering};
···
23
23
/// repo-stream memory limit: we don't disk spill; set higher than largest car
24
24
#[arg(long, default_value_t = 1024)]
25
25
mem_limit_mb: usize,
26
+
27
+
/// skip import and just do manual compact
28
+
#[arg(long, action)]
29
+
just_compact: bool,
26
30
}
27
31
28
32
fn open_db(path: &Path) -> Result<DB, rust_rocksdb::Error> {
···
34
38
35
39
// compress lower levels more
36
40
opts.set_compression_per_level(&[
37
-
DBCompressionType::Lz4, // L0
41
+
DBCompressionType::None, // L0
38
42
DBCompressionType::Lz4, // L1
39
43
DBCompressionType::Zstd, // L2
40
44
DBCompressionType::Zstd, // L3
···
42
46
DBCompressionType::Zstd, // L5
43
47
DBCompressionType::Zstd, // L6
44
48
]);
49
+
50
+
opts.set_compression_options(
51
+
-14, // window_bits (default, -14 means use zstd default)
52
+
3, // level (default 3)
53
+
0, // strategy (0 = default)
54
+
16_384, // max_dict_bytes (0 = no dictionary) (tried 16k)
55
+
);
56
+
opts.set_zstd_max_train_bytes(16 * 16_384);
57
+
45
58
opts.set_bottommost_compression_type(rust_rocksdb::DBCompressionType::Zstd);
46
59
60
+
opts.set_bottommost_compression_options(
61
+
-14, // window_bits (default, -14 means use zstd default)
62
+
3, // level (default 3)
63
+
0, // strategy (0 = default)
64
+
16_384, // max_dict_bytes (0 = no dictionary) (tried 16k)
65
+
true, // enabled
66
+
);
67
+
68
+
// probably investigate this one: use more training data for dictionary
69
+
opts.set_bottommost_zstd_max_train_bytes(16 * 16_384, true); // 256KB of samples for a 16KB dict
70
+
71
+
// skip filters on bottom-most layer
72
+
opts.set_optimize_filters_for_hits(true);
73
+
47
74
// write into new buffer while flushing?
48
75
opts.set_max_write_buffer_number(4);
49
76
50
-
// try to speed up compaction?
51
-
opts.set_max_background_jobs(8);
77
+
// ugh this probably wasn't exactly the right move
78
+
// there's increase_parallelism()
79
+
// // try to speed up compaction?
80
+
// opts.set_max_background_jobs(8);
81
+
opts.increase_parallelism(10);
52
82
53
83
// larger write buffer for less frequent flushes
54
-
opts.set_write_buffer_size(64 * 1024 * 1024);
84
+
opts.set_write_buffer_size(256 * 2_usize.pow(20)); // default 64MB
85
+
86
+
// apparently can help with space amp, less partially-filled levels
87
+
opts.set_level_compaction_dynamic_level_bytes(true);
88
+
89
+
// larger sst files (ehhh)
90
+
// don't help
91
+
// opts.set_target_file_size_base(128 * 2_u64.pow(20)); // default 64MB
92
+
93
+
// larger blocks for better compression (good scan, worse point read)
94
+
let mut block_opts = rust_rocksdb::BlockBasedOptions::default();
95
+
block_opts.set_block_size(256 * 2_usize.pow(10)); // N * KB blocks
96
+
block_opts.set_block_restart_interval(64);
97
+
opts.set_block_based_table_factory(&block_opts);
98
+
99
+
// enable ONLY during bulk-import
100
+
opts.set_unordered_write(true);
101
+
102
+
opts.set_max_subcompactions(4); // default 1
55
103
56
104
DB::open(&opts, path)
57
105
}
···
63
111
let args = Args::parse();
64
112
65
113
let db = Arc::new(open_db(&args.db_dir)?);
114
+
115
+
if args.just_compact {
116
+
manually_compact(db)?;
117
+
return Ok(())
118
+
}
66
119
67
120
let stats = run_workers(&args.car_dir, db.clone(), args.workers, args.mem_limit_mb).await?;
68
121
···
86
139
"db size before compaction"
87
140
);
88
141
142
+
let size_after = manually_compact(db)?;
143
+
144
+
if car_bytes > 0 {
145
+
let ratio = size_after as f64 / car_bytes as f64;
146
+
tracing::info!(ratio = format!("{ratio:.3}"), "db size / car bytes");
147
+
}
148
+
149
+
Ok(())
150
+
}
151
+
152
+
fn manually_compact(db: Arc<DB>) -> Result<u64, ProcessError> {
153
+
let mut compact_opts = CompactOptions::default();
154
+
compact_opts.set_bottommost_level_compaction(rust_rocksdb::BottommostLevelCompaction::Force);
155
+
89
156
tracing::info!("running manual compaction...");
90
157
let compact_start = std::time::Instant::now();
91
-
db.compact_range(None::<&[u8]>, None::<&[u8]>);
158
+
db.compact_range_opt(None::<&[u8]>, None::<&[u8]>, &compact_opts);
92
159
let compact_elapsed = compact_start.elapsed();
93
160
94
161
let size_after = db_size(&db);
95
162
tracing::info!(
96
163
bytes = size_after,
97
-
mib = size_after / (1024 * 1024),
164
+
mib = size_after / 2_u64.pow(20),
98
165
?compact_elapsed,
99
166
"db size after compaction",
100
167
);
101
-
102
-
if car_bytes > 0 {
103
-
let ratio = size_after as f64 / car_bytes as f64;
104
-
tracing::info!(ratio = format!("{ratio:.3}"), "db size / car bytes");
105
-
}
106
-
107
-
Ok(())
168
+
Ok(size_after)
108
169
}
109
170
110
171
fn db_size(db: &DB) -> u64 {