Fast and robust atproto CAR file processing in rust
type alias for rkey finally
+10
-7
+2
-2
readme.md
+2
-2
readme.md
···
62
62
63
63
more recent todo
64
64
- [ ] add a zero-copy rkyv process function example
65
-
- [ ] repo car slices
66
-
- [ ] lazy-value stream (rkey -> CID diffing for tap-like `#sync` handling)
65
+
- [ ] car slices
66
+
- [ ] lazy-value stream (for rkey -> CID diffing; tap-like `#sync` handling; save a fjall record `.get` when not needed)
67
67
- [x] get an *emtpy* car for the test suite
68
68
- [x] implement a max size on disk limit
69
69
+2
src/lib.rs
+2
src/lib.rs
+2
-1
src/mst.rs
+2
-1
src/mst.rs
···
3
3
//! The primary aim is to work through the **tree** structure. Non-node blocks
4
4
//! are left as raw bytes, for upper levels to parse into DAG-CBOR or whatever.
5
5
6
+
use crate::Rkey;
6
7
use cid::Cid;
7
8
use serde::Deserialize;
8
9
use sha2::{Digest, Sha256};
···
63
64
#[derive(Debug)]
64
65
pub(crate) enum ThingKind {
65
66
Tree,
66
-
Value { rkey: String },
67
+
Value { rkey: Rkey },
67
68
}
68
69
69
70
impl<'de> Deserialize<'de> for MstNode {
+4
-4
src/walk.rs
+4
-4
src/walk.rs
···
1
1
//! Depth-first MST traversal
2
2
3
3
use crate::mst::{Depth, MstNode, NodeThing, ThingKind};
4
-
use crate::{Bytes, HashMap, disk::DiskStore, drive::MaybeProcessedBlock};
4
+
use crate::{Bytes, HashMap, Rkey, disk::DiskStore, drive::MaybeProcessedBlock};
5
5
use cid::Cid;
6
6
use std::convert::Infallible;
7
7
···
30
30
#[error("MST depth underflow: depth-0 node with child trees")]
31
31
DepthUnderflow,
32
32
#[error("Encountered rkey {rkey:?} which cannot follow the previous: {prev:?}")]
33
-
RkeyOutOfOrder { prev: String, rkey: String },
33
+
RkeyOutOfOrder { prev: Rkey, rkey: Rkey },
34
34
}
35
35
36
36
/// Walker outputs
37
37
#[derive(Debug, PartialEq)]
38
38
pub struct Output {
39
-
pub rkey: String,
39
+
pub rkey: Rkey,
40
40
pub cid: Cid,
41
41
pub data: Bytes,
42
42
}
···
46
46
/// Walks the tree from left-to-right in depth-first order
47
47
#[derive(Debug)]
48
48
pub struct Walker {
49
-
prev_rkey: String,
49
+
prev_rkey: Rkey,
50
50
root_depth: Depth,
51
51
todo: Vec<Vec<NodeThing>>,
52
52
}