Fast and robust atproto CAR file processing in rust
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

add failing test for record slice

phil d062193e 469ba2d8

+54 -2
car-samples/slice-one.car

This is a binary file and will not be displayed.

-1
src/walk.rs
··· 156 156 Ok(Step::End(None)) 157 157 } 158 158 159 - #[allow(dead_code)] 160 159 pub fn step_to_edge( 161 160 &mut self, 162 161 blocks: &HashMap<ObjectLink, MaybeProcessedBlock>,
+53
tests/car-slices.rs
··· 1 + extern crate repo_stream; 2 + use repo_stream::{Driver, Output, Step}; 3 + 4 + const RECORD_SLICE: &'static [u8] = include_bytes!("../car-samples/slice-one.car"); 5 + 6 + async fn test_car_slice( 7 + bytes: &[u8], 8 + expected_records: usize, 9 + expected_sum: usize, 10 + expect_rkey: &str, 11 + ) { 12 + let mut driver = match Driver::load_car( 13 + bytes, 14 + |block| block.len().to_ne_bytes().to_vec(), 15 + 10, /* MiB */ 16 + ) 17 + .await 18 + .unwrap() 19 + { 20 + Driver::Memory(_commit, _, mem_driver) => mem_driver, 21 + Driver::Disk(_) => panic!("too big"), 22 + }; 23 + 24 + let mut records = 0; 25 + let mut sum = 0; 26 + let mut found_expected_rkey = false; 27 + let mut prev_rkey = "".to_string(); 28 + 29 + while let Step::Value(pairs) = driver.next_chunk(256).await.unwrap() { 30 + for Output { rkey, cid: _, data } in pairs { 31 + records += 1; 32 + 33 + let (int_bytes, _) = data.split_at(size_of::<usize>()); 34 + let size = usize::from_ne_bytes(int_bytes.try_into().unwrap()); 35 + 36 + sum += size; 37 + if rkey == expect_rkey { 38 + found_expected_rkey = true; 39 + } 40 + assert!(rkey > prev_rkey, "rkeys are streamed in order"); 41 + prev_rkey = rkey; 42 + } 43 + } 44 + 45 + assert_eq!(records, expected_records); 46 + assert_eq!(sum, expected_sum); 47 + assert!(found_expected_rkey); 48 + } 49 + 50 + #[tokio::test] 51 + async fn test_record_slice_car() { 52 + test_car_slice(RECORD_SLICE, 1, 0, "app.bsky.feed.like/3mcg72x6bi32z").await 53 + }
+1 -1
tests/non-huge-cars.rs
··· 14 14 ) { 15 15 let mut driver = match Driver::load_car( 16 16 bytes, 17 - |block| block.len().to_ne_bytes().to_vec().into(), 17 + |block| block.len().to_ne_bytes().to_vec(), 18 18 10, /* MiB */ 19 19 ) 20 20 .await