very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[db] tune repo_metadata and relay_events keyspaces
+19
-12
+1
-1
src/config.rs
+1
-1
src/config.rs
···
488
488
db_worker_threads: 8,
489
489
db_max_journaling_size_mb: 1024,
490
490
db_blocks_memtable_size_mb: BASE_MEMTABLE_MB,
491
-
db_repos_memtable_size_mb: BASE_MEMTABLE_MB / 2,
491
+
db_repos_memtable_size_mb: BASE_MEMTABLE_MB / 4,
492
492
db_events_memtable_size_mb: BASE_MEMTABLE_MB,
493
493
db_records_memtable_size_mb: BASE_MEMTABLE_MB / 3 * 2,
494
494
..Self::default()
+18
-11
src/db/mod.rs
+18
-11
src/db/mod.rs
···
211
211
// did plc are random so the interval wont rlly matter
212
212
.data_block_restart_interval_policy(RestartIntervalPolicy::new([2, 4])),
213
213
)?;
214
+
let repo_metadata = open_ks(
215
+
"repo_metadata",
216
+
opts()
217
+
.expect_point_read_hits(true)
218
+
.max_memtable_size(mb(cfg.db_repos_memtable_size_mb / 2))
219
+
// its did -> random u64 id + bool, not much to compress, very small
220
+
.data_block_size_policy(BlockSizePolicy::new([kb(2), kb(4), kb(8)]))
221
+
.data_block_compression_policy(CompressionPolicy::new([
222
+
CompressionType::None,
223
+
CompressionType::None,
224
+
get_compression("repos", 3),
225
+
]))
226
+
// did plc are random so the interval wont rlly matter
227
+
.data_block_restart_interval_policy(RestartIntervalPolicy::new([2, 4])),
228
+
)?;
214
229
let pending = open_ks(
215
230
"pending",
216
231
opts()
···
298
313
.data_block_compression_policy(CompressionPolicy::disabled())
299
314
.data_block_restart_interval_policy(RestartIntervalPolicy::all(16)),
300
315
)?;
301
-
let repo_metadata = open_ks(
302
-
"repo_metadata",
303
-
opts()
304
-
// point reads for tracking check
305
-
.expect_point_read_hits(true)
306
-
.max_memtable_size(mb(8))
307
-
.data_block_size_policy(BlockSizePolicy::all(kb(4)))
308
-
.data_block_compression_policy(CompressionPolicy::disabled())
309
-
.data_block_restart_interval_policy(RestartIntervalPolicy::all(4)),
310
-
)?;
311
316
let events = open_ks(
312
317
"events",
313
318
opts()
···
387
392
// only iterated for cursor replay
388
393
.expect_point_read_hits(true)
389
394
.max_memtable_size(mb(cfg.db_events_memtable_size_mb))
390
-
.data_block_size_policy(BlockSizePolicy::new([kb(64), kb(128)]))
395
+
.data_block_size_policy(BlockSizePolicy::new([kb(64), kb(128), kb(256)]))
391
396
.data_block_compression_policy(CompressionPolicy::new([
392
397
CompressionType::None,
393
398
get_compression("events", 3),
399
+
get_compression("events", 3),
400
+
get_compression("events", 5),
394
401
]))
395
402
.data_block_restart_interval_policy(RestartIntervalPolicy::new([64, 128])),
396
403
)?;