Fast and robust atproto CAR file processing in rust
clean up builder
authored by
phil
and committed by
tangled.org
35c37793
8c2ff0aa
+18
-41
+18
-41
src/drive.rs
+18
-41
src/drive.rs
···
115
115
Disk(NeedDisk<R>),
116
116
}
117
117
118
+
/// Processor that just returns the raw blocks
119
+
#[inline]
120
+
pub fn noop(block: Bytes) -> Bytes {
121
+
block
122
+
}
123
+
118
124
/// Builder-style driver setup
119
125
#[derive(Debug, Clone)]
120
126
pub struct DriverBuilder {
121
127
pub mem_limit_mb: usize,
128
+
pub block_processor: fn(Bytes) -> Bytes,
122
129
}
123
130
124
131
impl Default for DriverBuilder {
125
132
fn default() -> Self {
126
-
Self { mem_limit_mb: 16 }
133
+
Self {
134
+
mem_limit_mb: 16,
135
+
block_processor: noop,
136
+
}
127
137
}
128
138
}
129
139
130
-
/// Processor that just returns the raw blocks
131
-
#[inline]
132
-
pub fn noop(block: Bytes) -> Bytes {
133
-
block
134
-
}
135
-
136
140
impl DriverBuilder {
137
141
/// Begin configuring the driver with defaults
138
142
pub fn new() -> Self {
···
141
145
/// Set the in-memory size limit, in MiB
142
146
///
143
147
/// Default: 16 MiB
144
-
pub fn with_mem_limit_mb(self, new_limit: usize) -> Self {
145
-
Self {
146
-
mem_limit_mb: new_limit,
147
-
}
148
+
pub fn with_mem_limit_mb(mut self, new_limit: usize) -> Self {
149
+
self.mem_limit_mb = new_limit;
150
+
self
148
151
}
152
+
149
153
/// Set the block processor
150
154
///
151
155
/// Default: noop, raw blocks will be emitted
152
-
pub fn with_block_processor(
153
-
self,
154
-
block_processor: fn(Bytes) -> Bytes,
155
-
) -> DriverBuilderWithProcessor {
156
-
DriverBuilderWithProcessor {
157
-
mem_limit_mb: self.mem_limit_mb,
158
-
block_processor,
159
-
}
156
+
pub fn with_block_processor(mut self, new_processor: fn(Bytes) -> Bytes) -> DriverBuilder {
157
+
self.block_processor = new_processor;
158
+
self
160
159
}
161
-
/// Begin processing an atproto MST from a CAR file
162
-
pub async fn load_car<R: AsyncRead + Unpin>(&self, reader: R) -> Result<Driver<R>, DriveError> {
163
-
Driver::load_car(reader, noop, self.mem_limit_mb).await
164
-
}
165
-
}
166
160
167
-
/// Builder-style driver intermediate step
168
-
///
169
-
/// start from `DriverBuilder`
170
-
#[derive(Debug, Clone)]
171
-
pub struct DriverBuilderWithProcessor {
172
-
pub mem_limit_mb: usize,
173
-
pub block_processor: fn(Bytes) -> Bytes,
174
-
}
175
-
176
-
impl DriverBuilderWithProcessor {
177
-
/// Set the in-memory size limit, in MiB
178
-
///
179
-
/// Default: 16 MiB
180
-
pub fn with_mem_limit_mb(mut self, new_limit: usize) -> Self {
181
-
self.mem_limit_mb = new_limit;
182
-
self
183
-
}
184
161
/// Begin processing an atproto MST from a CAR file
185
162
pub async fn load_car<R: AsyncRead + Unpin>(&self, reader: R) -> Result<Driver<R>, DriveError> {
186
163
Driver::load_car(reader, self.block_processor, self.mem_limit_mb).await