very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[backfill] dont hold store / mst tree over db commit, drop early when we can
+61
-3
+44
AGENTS.md
+44
AGENTS.md
···
139
139
- `[ingest] implement backfill buffer replay`
140
140
- `[api] add accurate count parameter to stats`
141
141
- `[db] migrate block storage to msgpack`
142
+
143
+
144
+
<!-- headroom:rtk-instructions -->
145
+
# RTK (Rust Token Killer) - Token-Optimized Commands
146
+
147
+
When running shell commands, **always prefix with `rtk`**. This reduces context
148
+
usage by 60-90% with zero behavior change. If rtk has no filter for a command,
149
+
it passes through unchanged — so it is always safe to use.
150
+
151
+
## Key Commands
152
+
```bash
153
+
# Git (59-80% savings)
154
+
rtk git status rtk git diff rtk git log
155
+
156
+
# Files & Search (60-75% savings)
157
+
rtk ls <path> rtk read <file> rtk grep <pattern>
158
+
rtk find <pattern> rtk diff <file>
159
+
160
+
# Test (90-99% savings) — shows failures only
161
+
rtk pytest tests/ rtk cargo test rtk test <cmd>
162
+
163
+
# Build & Lint (80-90% savings) — shows errors only
164
+
rtk tsc rtk lint rtk cargo build
165
+
rtk prettier --check rtk mypy rtk ruff check
166
+
167
+
# Analysis (70-90% savings)
168
+
rtk err <cmd> rtk log <file> rtk json <file>
169
+
rtk summary <cmd> rtk deps rtk env
170
+
171
+
# GitHub (26-87% savings)
172
+
rtk gh pr view <n> rtk gh run list rtk gh issue list
173
+
174
+
# Infrastructure (85% savings)
175
+
rtk docker ps rtk kubectl get rtk docker logs <c>
176
+
177
+
# Package managers (70-90% savings)
178
+
rtk pip list rtk pnpm install rtk npm run <script>
179
+
```
180
+
181
+
## Rules
182
+
- In command chains, prefix each segment: `rtk git add . && rtk git commit -m "msg"`
183
+
- For debugging, use raw command without rtk prefix
184
+
- `rtk proxy <cmd>` runs command without filtering but tracks usage
185
+
<!-- /headroom:rtk-instructions -->
+1
flake.nix
+1
flake.nix
+15
-2
src/backfill/mod.rs
+15
-2
src/backfill/mod.rs
···
520
520
trace!(elapsed = %start.elapsed().as_secs_f32(), "parsed car");
521
521
522
522
let start = Instant::now();
523
+
let root_cid = parsed.root;
523
524
let store = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks));
525
+
// parsed.blocks was moved into the store; parsed.root is now root_cid. drop the raw
526
+
// CAR bytes here so we don't hold two copies of the block data (raw + parsed) for
527
+
// the entire duration of the spawn_blocking call below.
528
+
drop(car_bytes);
524
529
trace!(
525
530
blocks = store.len(),
526
531
elapsed = ?start.elapsed(),
···
529
534
530
535
// 4. parse root commit to get mst root
531
536
let root_bytes = store
532
-
.get(&parsed.root)
537
+
.get(&root_cid)
533
538
.await
534
539
.into_diagnostic()?
535
540
.ok_or_else(|| miette::miette!("root block missing from CAR"))?;
···
575
580
let mut added_blocks = 0;
576
581
let mut collection_counts: HashMap<SmolStr, u64> = HashMap::new();
577
582
let mut batch = app_state.db.inner.batch();
578
-
let store = mst.storage();
583
+
// clone the Arc so we hold an independent reference to the block store,
584
+
// allowing mst (and its entire loaded node tree) to be freed immediately
585
+
// rather than surviving until the end of spawn_blocking.
586
+
let store = mst.storage().clone();
587
+
drop(mst);
579
588
580
589
let prefix = keys::record_prefix_did(&did);
581
590
let mut existing_cids: HashMap<(SmolStr, DbRkey), SmolStr> = HashMap::new();
···
698
707
count += 1;
699
708
}
700
709
}
710
+
711
+
// all blocks have been read; free the MemoryBlockStore now so it does not
712
+
// survive through the final batch commit or any fjall backpressure stall.
713
+
drop(store);
701
714
702
715
// remove any remaining existing records (they weren't in the new MST)
703
716
for ((collection, rkey), cid) in existing_cids {
+1
-1
src/control/mod.rs
+1
-1
src/control/mod.rs