-1

server/Cargo.lock

reviewed

··· 1580 1580 "tower-http", 1581 1581 "tracing", 1582 1582 "tracing-subscriber", 1583 1583 - "zstd", 1584 1583 ] 1585 1584 1586 1585 [[package]]

-1

server/Cargo.toml

reviewed

··· 30 30 rayon = "1.10.0" 31 31 parking_lot = { version = "0.12", features = ["send_guard", "hardware-lock-elision"] } 32 32 rclite = "0.2.7" 33 33 - zstd = "0.13.3" 34 33 35 34 [target.'cfg(target_env = "msvc")'.dependencies] 36 35 snmalloc-rs = "0.3.8"

-19

server/src/db/mod.rs

reviewed

··· 380 380 }) 381 381 } 382 382 383 383 - // train zstd dict with 100 blocks from every lexicon 384 384 - pub fn train_zstd_dict(&self) -> AppResult<Vec<u8>> { 385 385 - let samples = self 386 386 - .get_nsids() 387 387 - .filter_map(|nsid| self.get_handle(&nsid)) 388 388 - .map(|handle| { 389 389 - handle 390 390 - .iter() 391 391 - .rev() 392 392 - .map(|res| { 393 393 - res.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) 394 394 - .map(|(_, value)| Cursor::new(value)) 395 395 - }) 396 396 - .take(1000) 397 397 - }) 398 398 - .flatten(); 399 399 - zstd::dict::from_sample_iterator(samples, 1024 * 128).map_err(AppError::from) 400 400 - } 401 401 - 402 383 pub fn get_hits( 403 384 &self, 404 385 nsid: &str,

-10

server/src/main.rs

reviewed

··· 53 53 debug(); 54 54 return; 55 55 } 56 56 - Some("traindict") => { 57 57 - train_zstd_dict(); 58 58 - return; 59 59 - } 60 56 Some(x) => { 61 57 tracing::error!("unknown command: {}", x); 62 58 return; ··· 209 205 ingest_events.join().expect("failed to join ingest events"); 210 206 db_task.await.expect("cant join db task"); 211 207 db.sync(true).expect("cant sync db"); 212 212 - } 213 213 - 214 214 - fn train_zstd_dict() { 215 215 - let db = Db::new(DbConfig::default(), CancellationToken::new()).expect("couldnt create db"); 216 216 - let dict_data = db.train_zstd_dict().expect("cant train zstd dict"); 217 217 - std::fs::write("zstd_dict", dict_data).expect("cant save zstd dict") 218 208 } 219 209 220 210 fn debug() {