+1

server/Cargo.lock

reviewed

··· 1580 1580 "tower-http", 1581 1581 "tracing", 1582 1582 "tracing-subscriber", 1583 1583 + "zstd", 1583 1584 ] 1584 1585 1585 1586 [[package]]

+1

server/Cargo.toml

reviewed

··· 30 30 rayon = "1.10.0" 31 31 parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] } 32 32 rclite = "0.2.7" 33 33 + zstd = "0.13.3" 33 34 34 35 [target.'cfg(target_env = "msvc")'.dependencies] 35 36 snmalloc-rs = "0.3.8"

+19

server/src/db/mod.rs

reviewed

··· 380 380 }) 381 381 } 382 382 383 383 + // train zstd dict with 100 blocks from every lexicon 384 384 + pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> { 385 385 + let samples = self 386 386 + .get_nsids() 387 387 + .filter_map(|nsid| self.get_handle(&nsid)) 388 388 + .map(|handle| { 389 389 + handle 390 390 + .iter() 391 391 + .rev() 392 392 + .map(|res| { 393 393 + res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) 394 394 + .map(|(_, value)| Cursor::new(value)) 395 395 + }) 396 396 + .take(1000) 397 397 + }) 398 398 + .flatten(); 399 399 + zstd::dict::from_sample_iterator(samples, 1024 * 128).map_err(AppError::from) 400 400 + } 401 401 + 383 402 pub fn get_hits( 384 403 &self, 385 404 nsid: &str,

+10

server/src/main.rs

reviewed

··· 53 53 debug(); 54 54 return; 55 55 } 56 56 + Some("traindict") => { 57 57 + train_zstd_dict(); 58 58 + return; 59 59 + } 56 60 Some(x) => { 57 61 tracing::error!("unknown command: {}", x); 58 62 return; ··· 205 209 ingest_events.join().expect("failed to join ingest events"); 206 210 db_task.await.expect("cant join db task"); 207 211 db.sync(true).expect("cant sync db"); 212 212 + } 213 213 + 214 214 + fn train_zstd_dict() { 215 215 + let db = Db::new(DbConfig::default(), CancellationToken::new()).expect("couldnt create db"); 216 216 + let dict_data = db.train_zstd_dict().expect("cant train zstd dict"); 217 217 + std::fs::write("zstd_dict", dict_data).expect("cant save zstd dict") 208 218 } 209 219 210 220 fn debug() {