Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
further reduce ufos batch limits, avoid oom?
....
+7
-5
+5
-5
ufos/src/consumer.rs
+5
-5
ufos/src/consumer.rs
···
14
14
15
15
use crate::{CreateRecord, DeleteAccount, DeleteRecord, EventBatch, ModifyRecord, UpdateRecord};
16
16
17
-
const MAX_BATCHED_RECORDS: usize = 128; // *non-blocking* limit. drops oldest batched record per collection once reached.
18
-
const MAX_BATCHED_MODIFIES: usize = 512; // hard limit, total updates and deletes across all collections.
19
-
const MAX_ACCOUNT_REMOVES: usize = 512; // hard limit, total account deletions. actually the least frequent event, but tiny.
20
-
const MAX_BATCHED_COLLECTIONS: usize = 64; // hard limit, MAX_BATCHED_RECORDS applies per collection
17
+
const MAX_BATCHED_RECORDS: usize = 64; // *non-blocking* limit. drops oldest batched record per collection once reached.
18
+
const MAX_BATCHED_MODIFIES: usize = 32; // hard limit, total updates and deletes across all collections.
19
+
const MAX_ACCOUNT_REMOVES: usize = 128; // hard limit, total account deletions. actually the least frequent event, but tiny.
20
+
const MAX_BATCHED_COLLECTIONS: usize = 32; // hard limit, MAX_BATCHED_RECORDS applies per collection
21
21
const MIN_BATCH_SPAN_SECS: f64 = 2.; // try to get a bit of rest a bit.
22
22
const MAX_BATCH_SPAN_SECS: f64 = 60.; // hard limit of duration from oldest to latest event cursor within a batch, in seconds.
23
23
24
24
const SEND_TIMEOUT_S: f64 = 60.;
25
-
const BATCH_QUEUE_SIZE: usize = 512; // 4096 got OOM'd. update: 1024 also got OOM'd during L0 compaction blocking
25
+
const BATCH_QUEUE_SIZE: usize = 64; // 4096 got OOM'd. update: 1024 also got OOM'd during L0 compaction blocking
26
26
27
27
#[derive(Debug)]
28
28
struct Batcher {
+2
ufos/src/store.rs
+2
ufos/src/store.rs
···
271
271
// 4. reverse and try to walk back MAX_RETAINED steps
272
272
// 5. if we didn't end iteration yet, start deleting records (and their forward links) until we get to the end
273
273
274
+
// oh we might be able to walk *forward* instead of reverse from the cursor, which might help avoid iterating over a lot of deletion tombstones
275
+
274
276
// ... we can probably do even better with cursor ranges too, since we'll have a cursor range from rollup and it's in the by_collection key
275
277
276
278
Ok(())