lightweight
com.atproto.sync.listReposByCollection
wip slice tricks
+57
-52
+2
-2
Cargo.lock
+2
-2
Cargo.lock
···
3330
3330
3331
3331
[[package]]
3332
3332
name = "repo-stream"
3333
-
version = "0.5.0-alpha.3"
3333
+
version = "0.5.0-alpha.5"
3334
3334
source = "registry+https://github.com/rust-lang/crates.io-index"
3335
-
checksum = "7df620415aa6a02c68a83a699f48f23472724367ae2301e6498b8f5957866340"
3335
+
checksum = "c885e481796dcf2662a515dd76719120160c399de7fac74809c2f56e354fee41"
3336
3336
dependencies = [
3337
3337
"cid",
3338
3338
"fjall",
+1
-1
Cargo.toml
+1
-1
Cargo.toml
···
25
25
metrics-exporter-prometheus = { version = "0.18.1", features = ["http-listener"] }
26
26
mini-moka = "0.10"
27
27
n0-future = "0.1"
28
-
repo-stream = { version = "0.5.0-alpha.3", features = ["jacquard"] }
28
+
repo-stream = { version = "0.5.0-alpha.5", features = ["jacquard"] }
29
29
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
30
30
rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs"] }
31
31
rustversion = "1"
+52
-47
src/mst/slice_tricks.rs
+52
-47
src/mst/slice_tricks.rs
···
2
2
3
3
use super::Span;
4
4
use jacquard_common::types::string::Nsid;
5
-
use repo_stream::{MemCar, Output as WalkOutput, WalkItem};
5
+
use repo_stream::{MemCar, WalkItem};
6
6
use std::cmp::Ordering;
7
7
use std::collections::BTreeSet;
8
8
···
51
51
type CollectionSpan = Span<TerminatedNsid>;
52
52
53
53
impl CollectionSpan {
54
-
// fn contains_nsid(&self, collection: &Nsid<'_>) -> Option<bool> {
55
-
// self.contains((&collection).into())
56
-
// }
57
-
// fn may_contain_nsid(&self, collection: &Nsid<'_>) -> bool {
58
-
// self.may_contain(&collection.clone().into())
59
-
// }
60
54
/// get a list of NSIDs if the span has no gaps
55
+
///
56
+
/// a CollectionSpan can be complete even if it contains gaps, as long as
57
+
/// those gaps are *within* collection-bounding keys
61
58
fn complete(&self) -> Option<Vec<Nsid<'static>>> {
62
59
self.is_complete()
63
60
.then(|| self.things.keys().map(Into::into).collect())
···
109
106
}
110
107
}
111
108
}
109
+
}
110
+
111
+
/// extract a span of collection NSIDs (with possible gaps) from a CAR slice
112
+
fn span_from_slice(car: &mut MemCar) -> Result<CollectionSpan> {
113
+
let mut prev_gap = false;
114
+
let mut prev_collection = None;
115
+
116
+
let mut span = CollectionSpan::empty();
117
+
118
+
while let Some(item) = car.next()? {
119
+
assert!(
120
+
!matches!(item, WalkItem::Node { .. }),
121
+
"car.next() does not return nodes"
122
+
);
123
+
let Some(key) = item.key() else {
124
+
prev_gap = true;
125
+
continue;
126
+
};
127
+
let collection: Nsid<'_> = key
128
+
.parse::<Nsid<'_>>()
129
+
.map_err(|e| MstSliceTricksError::BadPath(format!("nsid parse: {e}")))?;
130
+
131
+
if let Some(prev) = prev_collection {
132
+
// last-from-collection wins setting gap_after
133
+
span.things.insert(prev, prev_gap);
134
+
} else {
135
+
span.gap_before = prev_gap;
136
+
}
137
+
138
+
prev_collection = Some((&collection).into());
139
+
}
140
+
141
+
if let Some(prev) = prev_collection {
142
+
// last-from-collection wins setting gap_after
143
+
span.things.insert(prev, prev_gap);
144
+
} else {
145
+
span.gap_before = prev_gap;
146
+
}
147
+
148
+
Ok(span)
149
+
}
150
+
151
+
/// Estimate whether or not the full CAR is small, from a slice
152
+
///
153
+
/// pretty basic rn, just checks if its root node is layer 0 or 1 (hard max 200,
154
+
/// 800 entries, respectively)
155
+
fn likely_from_small_repo(car: &mut MemCar) -> bool {
156
+
car.height() <= 1
112
157
}
113
158
114
159
#[cfg(test)]
···
269
314
assert!(s.could_cover(&nsids(&["a.b.c.d"])));
270
315
}
271
316
}
272
-
273
-
fn span_from_slice(car: &mut MemCar) -> Result<CollectionSpan> {
274
-
let mut prev_gap = false;
275
-
let mut prev_collection = None;
276
-
277
-
let mut span = CollectionSpan::empty();
278
-
279
-
while let Some(item) = car.next()? {
280
-
match item {
281
-
WalkItem::MissingSubtree { .. } => {
282
-
prev_gap = true;
283
-
}
284
-
WalkItem::Record(WalkOutput { key, .. }) | WalkItem::MissingRecord { key, .. } => {
285
-
let collection: Nsid<'_> = key
286
-
.parse()
287
-
.map_err(|e| MstSliceTricksError::BadPath(format!("nsid parse: {e}")))?;
288
-
let terminated = (&collection).into();
289
-
290
-
if let Some(prev) = prev_collection {
291
-
// last-from-collection wins setting gap_after
292
-
span.things.insert(prev, prev_gap);
293
-
} else {
294
-
span.gap_before = prev_gap;
295
-
}
296
-
297
-
prev_collection = Some(terminated);
298
-
}
299
-
WalkItem::Node { .. } => unreachable!("repostream mem::next doesn't output nodes"),
300
-
}
301
-
}
302
-
303
-
if let Some(prev) = prev_collection {
304
-
// last-from-collection wins setting gap_after
305
-
span.things.insert(prev, prev_gap);
306
-
} else {
307
-
span.gap_before = prev_gap;
308
-
}
309
-
310
-
Ok(span)
311
-
}
+1
-1
src/sync/resync/describe_repo.rs
+1
-1
src/sync/resync/describe_repo.rs
···
104
104
.map_err(|e| GetCollectionsError::InvalidData(format!("bad rev in commit: {e}")))?;
105
105
106
106
// shuffle repo-stream raw cid => jacquard cid wrapper
107
-
let data = mem_car.commit.data.clone().into();
107
+
let data = mem_car.commit.data.into();
108
108
109
109
let proven_collections =
110
110
crate::mst::collections::extract_from_slice(&mut mem_car).map_err(|e| {
+1
-1
src/sync/resync/get_repo.rs
+1
-1
src/sync/resync/get_repo.rs
···
70
70
let rev = Tid::new(&mem_car.commit.rev)
71
71
.map_err(|e| GetCollectionsError::InvalidData(format!("bad rev in commit: {e}")))?;
72
72
73
-
let data = mem_car.commit.data.clone().into();
73
+
let data = mem_car.commit.data.into();
74
74
75
75
let collections = crate::mst::collections::extract_from_full(mem_car)
76
76
.await