Fast and robust atproto CAR file processing in rust
add failing test for record slice
+54
-2
car-samples/slice-one.car
car-samples/slice-one.car
This is a binary file and will not be displayed.
-1
src/walk.rs
-1
src/walk.rs
+53
tests/car-slices.rs
+53
tests/car-slices.rs
···
1
+
extern crate repo_stream;
2
+
use repo_stream::{Driver, Output, Step};
3
+
4
+
const RECORD_SLICE: &'static [u8] = include_bytes!("../car-samples/slice-one.car");
5
+
6
+
async fn test_car_slice(
7
+
bytes: &[u8],
8
+
expected_records: usize,
9
+
expected_sum: usize,
10
+
expect_rkey: &str,
11
+
) {
12
+
let mut driver = match Driver::load_car(
13
+
bytes,
14
+
|block| block.len().to_ne_bytes().to_vec(),
15
+
10, /* MiB */
16
+
)
17
+
.await
18
+
.unwrap()
19
+
{
20
+
Driver::Memory(_commit, _, mem_driver) => mem_driver,
21
+
Driver::Disk(_) => panic!("too big"),
22
+
};
23
+
24
+
let mut records = 0;
25
+
let mut sum = 0;
26
+
let mut found_expected_rkey = false;
27
+
let mut prev_rkey = "".to_string();
28
+
29
+
while let Step::Value(pairs) = driver.next_chunk(256).await.unwrap() {
30
+
for Output { rkey, cid: _, data } in pairs {
31
+
records += 1;
32
+
33
+
let (int_bytes, _) = data.split_at(size_of::<usize>());
34
+
let size = usize::from_ne_bytes(int_bytes.try_into().unwrap());
35
+
36
+
sum += size;
37
+
if rkey == expect_rkey {
38
+
found_expected_rkey = true;
39
+
}
40
+
assert!(rkey > prev_rkey, "rkeys are streamed in order");
41
+
prev_rkey = rkey;
42
+
}
43
+
}
44
+
45
+
assert_eq!(records, expected_records);
46
+
assert_eq!(sum, expected_sum);
47
+
assert!(found_expected_rkey);
48
+
}
49
+
50
+
#[tokio::test]
51
+
async fn test_record_slice_car() {
52
+
test_car_slice(RECORD_SLICE, 1, 0, "app.bsky.feed.like/3mcg72x6bi32z").await
53
+
}