very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[db] fix train dict broken when backlinks not enabled, also increase dict sizes
+9
-6
+1
-1
Cargo.toml
+1
-1
Cargo.toml
+8
-5
src/db/mod.rs
+8
-5
src/db/mod.rs
···
443
443
"blocks" => &self.blocks,
444
444
"events" => &self.events,
445
445
"repos" => &self.repos,
446
+
#[cfg(feature = "backlinks")]
446
447
"backlinks" => &self.backlinks,
447
448
_ => miette::bail!("unknown keyspace for training: {ks_name}"),
448
449
};
449
450
450
451
let dict_size = match ks_name {
451
-
"blocks" => 32_768,
452
-
"events" => 16_384,
453
-
"repos" => 16_384,
454
-
_ => 16_384,
452
+
"blocks" => kb(128),
453
+
"events" => kb(64),
454
+
"repos" => kb(64),
455
+
"backlinks" => kb(64),
456
+
_ => kb(32),
455
457
};
456
458
457
459
let samples: Vec<Vec<u8>> = if ks_name == "blocks" {
···
505
507
"training zstd dictionary for keyspace {ks_name} ({} samples, {dict_size} bytes limit)...",
506
508
samples.len(),
507
509
);
508
-
let dict_bytes = lsm_tree::train_zstd_dict(&samples, dict_size).into_diagnostic()?;
510
+
let dict_bytes =
511
+
lsm_tree::train_zstd_dict(&samples, dict_size as usize).into_diagnostic()?;
509
512
let path = self.path.join(format!("dict_{ks_name}.bin"));
510
513
std::fs::write(&path, &dict_bytes).into_diagnostic()?;
511
514
tracing::info!(