forked from
tranquil.farm/tranquil-pds
Our Personal Data Server from scratch!
feat(tranquil-store): beginnings of the gauntlet test suite
Lewis: May this revision serve well! <lu5a@proton.me>
+1290
-106
+2
-3
crates/tranquil-pds/src/repo_ops.rs
+2
-3
crates/tranquil-pds/src/repo_ops.rs
···
246
246
247
247
let obsolete_cids = match original_settled.diff(&new_settled).await {
248
248
Ok(diff) => {
249
-
let mut obsolete: Vec<Cid> = Vec::with_capacity(
250
-
1 + diff.removed_mst_blocks.len() + diff.removed_cids.len(),
251
-
);
249
+
let mut obsolete: Vec<Cid> =
250
+
Vec::with_capacity(1 + diff.removed_mst_blocks.len() + diff.removed_cids.len());
252
251
obsolete.push(ctx.current_root_cid);
253
252
obsolete.extend(diff.removed_mst_blocks);
254
253
obsolete.extend(diff.removed_cids);
+10
-16
crates/tranquil-pds/tests/gc_compaction_restart.rs
+10
-16
crates/tranquil-pds/tests/gc_compaction_restart.rs
···
13
13
.map(|(&fid, _)| fid)
14
14
.collect::<Vec<_>>()
15
15
.into_iter()
16
-
.for_each(|fid| {
17
-
match store.compact_file(fid, 0) {
18
-
Ok(_) => {}
19
-
Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
20
-
Err(e) => eprintln!("compaction: {e}"),
21
-
}
16
+
.for_each(|fid| match store.compact_file(fid, 0) {
17
+
Ok(_) => {}
18
+
Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
19
+
Err(e) => eprintln!("compaction: {e}"),
22
20
});
23
21
}
24
22
···
84
82
}
85
83
86
84
let data_dir = store.data_dir().to_path_buf();
87
-
let index_dir = data_dir
88
-
.parent()
89
-
.unwrap()
90
-
.join("index");
85
+
let index_dir = data_dir.parent().unwrap().join("index");
91
86
92
87
let store_clone = store.clone();
93
88
tokio::task::spawn_blocking(move || {
···
107
102
108
103
let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid");
109
104
110
-
let car_blocks =
111
-
tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
112
-
.await
113
-
.expect("collect blocks");
105
+
let car_blocks = tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
106
+
.await
107
+
.expect("collect blocks");
114
108
115
109
let block_count_before = car_blocks.len();
116
110
···
131
125
group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
132
126
shard_count: 1,
133
127
};
134
-
let fresh = tranquil_store::blockstore::TranquilBlockStore::open(config)
135
-
.expect("reopen failed");
128
+
let fresh =
129
+
tranquil_store::blockstore::TranquilBlockStore::open(config).expect("reopen failed");
136
130
137
131
let missing: Vec<String> = car_blocks
138
132
.iter()
+7
-5
crates/tranquil-pds/tests/lifecycle_session.rs
+7
-5
crates/tranquil-pds/tests/lifecycle_session.rs
···
626
626
.send()
627
627
.await
628
628
.expect("Failed to login with app password");
629
-
assert_eq!(login_res.status(), StatusCode::OK, "App password login for '{}' failed", name);
629
+
assert_eq!(
630
+
login_res.status(),
631
+
StatusCode::OK,
632
+
"App password login for '{}' failed",
633
+
name
634
+
);
630
635
let session: Value = login_res.json().await.unwrap();
631
636
let jwt = session["accessJwt"].as_str().unwrap().to_string();
632
637
(jwt, scopes_response)
···
635
640
async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
636
641
let base = base_url().await;
637
642
let res = client
638
-
.get(format!(
639
-
"{}/xrpc/com.atproto.server.getServiceAuth",
640
-
base
641
-
))
643
+
.get(format!("{}/xrpc/com.atproto.server.getServiceAuth", base))
642
644
.bearer_auth(jwt)
643
645
.query(&[
644
646
("aud", "did:web:api.bsky.app"),
+62
-22
crates/tranquil-pds/tests/mst_diff_equivalence.rs
+62
-22
crates/tranquil-pds/tests/mst_diff_equivalence.rs
···
15
15
Cid::new_v1(0x71, mh)
16
16
}
17
17
18
-
async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync + Send + 'static>(
18
+
async fn compute_obsolete_full_walk<
19
+
S: jacquard_repo::storage::BlockStore + Sync + Send + 'static,
20
+
>(
19
21
old: &Mst<S>,
20
22
new: &Mst<S>,
21
23
) -> BTreeSet<Cid> {
···
34
36
.collect()
35
37
}
36
38
37
-
fn compute_obsolete_from_diff(
38
-
diff: &jacquard_repo::mst::diff::MstDiff,
39
-
) -> BTreeSet<Cid> {
39
+
fn compute_obsolete_from_diff(diff: &jacquard_repo::mst::diff::MstDiff) -> BTreeSet<Cid> {
40
40
diff.removed_mst_blocks
41
41
.iter()
42
42
.copied()
···
74
74
let diff_obsolete = compute_obsolete_from_diff(&diff);
75
75
76
76
assert_eq!(
77
-
full_walk_obsolete, diff_obsolete,
77
+
full_walk_obsolete,
78
+
diff_obsolete,
78
79
"MISMATCH in scenario: {scenario}\n full_walk count: {}\n diff count: {}\n in full_walk but not diff: {:?}\n in diff but not full_walk: {:?}",
79
80
full_walk_obsolete.len(),
80
81
diff_obsolete.len(),
81
-
full_walk_obsolete.difference(&diff_obsolete).collect::<Vec<_>>(),
82
-
diff_obsolete.difference(&full_walk_obsolete).collect::<Vec<_>>(),
82
+
full_walk_obsolete
83
+
.difference(&diff_obsolete)
84
+
.collect::<Vec<_>>(),
85
+
diff_obsolete
86
+
.difference(&full_walk_obsolete)
87
+
.collect::<Vec<_>>(),
83
88
);
84
89
}
85
90
···
256
261
async fn massive_complete_replacement() {
257
262
let old = generate_records("app.bsky.feed.post", 0..1000);
258
263
let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
259
-
assert_equivalence(&old, &new_rec, "1000 records fully replaced with 1000 different").await;
264
+
assert_equivalence(
265
+
&old,
266
+
&new_rec,
267
+
"1000 records fully replaced with 1000 different",
268
+
)
269
+
.await;
260
270
}
261
271
262
272
#[tokio::test]
···
276
286
];
277
287
let old = generate_multi_collection_records(&collections, 500);
278
288
let new_rec = apply_scattered_updates(&old, 4, 30000);
279
-
assert_equivalence(&old, &new_rec, "5 collections x 500 records - update every 4th").await;
289
+
assert_equivalence(
290
+
&old,
291
+
&new_rec,
292
+
"5 collections x 500 records - update every 4th",
293
+
)
294
+
.await;
280
295
}
281
296
282
297
#[tokio::test]
···
294
309
.filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
295
310
.cloned()
296
311
.collect();
297
-
assert_equivalence(&old, &new_rec, "4 collections x 400 - wipe repost collection").await;
312
+
assert_equivalence(
313
+
&old,
314
+
&new_rec,
315
+
"4 collections x 400 - wipe repost collection",
316
+
)
317
+
.await;
298
318
}
299
319
300
320
#[tokio::test]
···
313
333
314
334
#[tokio::test]
315
335
async fn multi_collection_add_new_collection() {
316
-
let old_collections = [
317
-
"app.bsky.feed.like",
318
-
"app.bsky.feed.post",
319
-
];
336
+
let old_collections = ["app.bsky.feed.like", "app.bsky.feed.post"];
320
337
let old = generate_multi_collection_records(&old_collections, 500);
321
338
let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
322
339
assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
···
378
395
let new_rec: Vec<_> = (0..1000u32)
379
396
.map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
380
397
.collect();
381
-
assert_equivalence(&old, &new_rec, "1000 even-keyed records replaced by 1000 odd-keyed").await;
398
+
assert_equivalence(
399
+
&old,
400
+
&new_rec,
401
+
"1000 even-keyed records replaced by 1000 odd-keyed",
402
+
)
403
+
.await;
382
404
}
383
405
384
406
#[tokio::test]
···
426
448
})
427
449
.collect();
428
450
429
-
assert_equivalence(&old, &new_rec, "50 collections x 20 records - delete every 15th, update every 7th").await;
451
+
assert_equivalence(
452
+
&old,
453
+
&new_rec,
454
+
"50 collections x 20 records - delete every 15th, update every 7th",
455
+
)
456
+
.await;
430
457
}
431
458
432
459
#[tokio::test]
···
457
484
async fn delete_head_and_tail() {
458
485
let old = generate_records("app.bsky.feed.post", 0..2000);
459
486
let new_rec: Vec<_> = old[200..1800].to_vec();
460
-
assert_equivalence(&old, &new_rec, "2000 records - delete first 200 and last 200").await;
487
+
assert_equivalence(
488
+
&old,
489
+
&new_rec,
490
+
"2000 records - delete first 200 and last 200",
491
+
)
492
+
.await;
461
493
}
462
494
463
495
#[tokio::test]
···
465
497
let old = generate_records("app.bsky.feed.post", 0..2000);
466
498
let mut new_rec: Vec<_> = old[..100].to_vec();
467
499
new_rec.extend_from_slice(&old[1900..]);
468
-
assert_equivalence(&old, &new_rec, "2000 records - keep only first 100 and last 100").await;
500
+
assert_equivalence(
501
+
&old,
502
+
&new_rec,
503
+
"2000 records - keep only first 100 and last 100",
504
+
)
505
+
.await;
469
506
}
470
507
471
508
#[tokio::test]
···
515
552
})
516
553
.map(|(_, r)| r.clone())
517
554
.collect();
518
-
assert_equivalence(&old, &new_rec, "1500 records - delete every 3rd chunk of 50").await;
555
+
assert_equivalence(
556
+
&old,
557
+
&new_rec,
558
+
"1500 records - delete every 3rd chunk of 50",
559
+
)
560
+
.await;
519
561
}
520
562
521
563
#[tokio::test]
···
529
571
.filter(|(_, val)| val % 4 != 0)
530
572
.cloned()
531
573
.collect();
532
-
new_rec.extend((0..500u32).map(|i| {
533
-
(make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)
534
-
}));
574
+
new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)));
535
575
new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
536
576
537
577
assert_equivalence(
+4
-1
crates/tranquil-scopes/src/permissions.rs
+4
-1
crates/tranquil-scopes/src/permissions.rs
···
164
164
if self.has_transition_generic && !self.has_transition_chat {
165
165
return Err(ScopeError::InsufficientScope {
166
166
required: "transition:chat.bsky".to_string(),
167
-
message: format!("Chat access requires transition:chat.bsky scope to call {}", lxm),
167
+
message: format!(
168
+
"Chat access requires transition:chat.bsky scope to call {}",
169
+
lxm
170
+
),
168
171
});
169
172
}
170
173
}
+3
-2
crates/tranquil-store/Cargo.toml
+3
-2
crates/tranquil-store/Cargo.toml
···
14
14
fjall = "3"
15
15
lsm-tree = "3"
16
16
flume = "0.11"
17
-
tokio = { workspace = true, features = ["sync", "rt"] }
17
+
tokio = { workspace = true, features = ["sync", "rt", "time"] }
18
18
bytes = "1"
19
19
memmap2 = "0.9"
20
20
tracing = { workspace = true }
···
34
34
rayon = "1"
35
35
smallvec = "1"
36
36
uuid = { workspace = true }
37
+
tempfile = { version = "3", optional = true }
37
38
38
39
[features]
39
-
test-harness = []
40
+
test-harness = ["dep:tempfile"]
40
41
41
42
[dev-dependencies]
42
43
tranquil-store = { path = ".", features = ["test-harness"] }
+17
-1
crates/tranquil-store/src/blockstore/hash_index.rs
+17
-1
crates/tranquil-store/src/blockstore/hash_index.rs
···
1202
1202
self.table.read().contains_live(cid)
1203
1203
}
1204
1204
1205
+
pub fn live_entries_snapshot(&self) -> Vec<([u8; CID_SIZE], RefCount)> {
1206
+
self.table
1207
+
.read()
1208
+
.iter()
1209
+
.filter(|s| !s.refcount.is_zero())
1210
+
.map(|s| (s.cid, s.refcount))
1211
+
.collect()
1212
+
}
1213
+
1205
1214
pub fn batch_put(
1206
1215
&self,
1207
1216
entries: &[([u8; CID_SIZE], BlockLocation)],
···
1222
1231
now: WallClockMs,
1223
1232
position_update: PositionUpdate<'_>,
1224
1233
) -> Result<(), BlockIndexError> {
1225
-
self.batch_put_inner(entries, decrements, cursor, epoch, now, Some(position_update))
1234
+
self.batch_put_inner(
1235
+
entries,
1236
+
decrements,
1237
+
cursor,
1238
+
epoch,
1239
+
now,
1240
+
Some(position_update),
1241
+
)
1226
1242
}
1227
1243
1228
1244
fn batch_put_inner(
+7
-5
crates/tranquil-store/src/blockstore/hint.rs
+7
-5
crates/tranquil-store/src/blockstore/hint.rs
···
57
57
58
58
fn encode_location_fields(record: &mut [u8; HINT_RECORD_SIZE], loc: &BlockLocation) {
59
59
record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&loc.file_id.raw().to_le_bytes());
60
-
record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8]
61
-
.copy_from_slice(&loc.length.raw().to_le_bytes());
62
-
record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8]
63
-
.copy_from_slice(&loc.offset.raw().to_le_bytes());
60
+
record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&loc.length.raw().to_le_bytes());
61
+
record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&loc.offset.raw().to_le_bytes());
64
62
}
65
63
66
64
pub(crate) fn encode_hint_record<S: StorageIO>(
···
841
839
let offset = BlockOffset::new(1024);
842
840
let length = BlockLength::new(256);
843
841
844
-
let loc = BlockLocation { file_id, offset, length };
842
+
let loc = BlockLocation {
843
+
file_id,
844
+
offset,
845
+
length,
846
+
};
845
847
encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();
846
848
847
849
let file_size = sim.file_size(fd).unwrap();
+1
-1
crates/tranquil-store/src/eventlog/reader.rs
+1
-1
crates/tranquil-store/src/eventlog/reader.rs
+1
-1
crates/tranquil-store/src/eventlog/writer.rs
+1
-1
crates/tranquil-store/src/eventlog/writer.rs
···
305
305
306
306
match self.build_sidecar_for_segment(old_id) {
307
307
Ok(()) => {}
308
-
Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
308
+
Err(e) => warn!(segment = %old_id, error = %e, "non-fatal sidecar build failure"),
309
309
}
310
310
311
311
let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
+41
crates/tranquil-store/src/gauntlet/farm.rs
+41
crates/tranquil-store/src/gauntlet/farm.rs
···
1
+
use std::cell::RefCell;
2
+
3
+
use rayon::prelude::*;
4
+
use tokio::runtime::Runtime;
5
+
6
+
use super::op::Seed;
7
+
use super::runner::{Gauntlet, GauntletConfig, GauntletReport};
8
+
9
+
thread_local! {
10
+
static RUNTIME: RefCell<Option<Runtime>> = const { RefCell::new(None) };
11
+
}
12
+
13
+
fn with_runtime<R>(f: impl FnOnce(&Runtime) -> R) -> R {
14
+
RUNTIME.with(|cell| {
15
+
let mut slot = cell.borrow_mut();
16
+
if slot.is_none() {
17
+
*slot = Some(
18
+
tokio::runtime::Builder::new_current_thread()
19
+
.enable_all()
20
+
.build()
21
+
.expect("build rt"),
22
+
);
23
+
}
24
+
f(slot.as_ref().expect("runtime present"))
25
+
})
26
+
}
27
+
28
+
pub fn run_many<F>(make_config: F, seeds: impl IntoIterator<Item = Seed>) -> Vec<GauntletReport>
29
+
where
30
+
F: Fn(Seed) -> GauntletConfig + Sync + Send,
31
+
{
32
+
let seeds: Vec<Seed> = seeds.into_iter().collect();
33
+
seeds
34
+
.into_par_iter()
35
+
.map(|s| {
36
+
let cfg = make_config(s);
37
+
let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
38
+
with_runtime(|rt| rt.block_on(gauntlet.run()))
39
+
})
40
+
.collect()
41
+
}
+141
crates/tranquil-store/src/gauntlet/invariants.rs
+141
crates/tranquil-store/src/gauntlet/invariants.rs
···
1
+
use std::collections::{HashMap, HashSet};
2
+
3
+
use super::oracle::Oracle;
4
+
use crate::blockstore::{CidBytes, TranquilBlockStore};
5
+
6
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7
+
pub struct InvariantSet(u32);
8
+
9
+
impl InvariantSet {
10
+
pub const EMPTY: Self = Self(0);
11
+
pub const REFCOUNT_CONSERVATION: Self = Self(1 << 0);
12
+
pub const REACHABILITY: Self = Self(1 << 1);
13
+
14
+
const ALL_KNOWN: u32 = Self::REFCOUNT_CONSERVATION.0 | Self::REACHABILITY.0;
15
+
16
+
pub const fn contains(self, other: Self) -> bool {
17
+
(self.0 & other.0) == other.0
18
+
}
19
+
20
+
pub const fn union(self, other: Self) -> Self {
21
+
Self(self.0 | other.0)
22
+
}
23
+
24
+
pub const fn unknown_bits(self) -> u32 {
25
+
self.0 & !Self::ALL_KNOWN
26
+
}
27
+
}
28
+
29
+
impl std::ops::BitOr for InvariantSet {
30
+
type Output = Self;
31
+
fn bitor(self, rhs: Self) -> Self {
32
+
self.union(rhs)
33
+
}
34
+
}
35
+
36
+
#[derive(Debug)]
37
+
pub struct InvariantViolation {
38
+
pub invariant: &'static str,
39
+
pub detail: String,
40
+
}
41
+
42
+
pub trait Invariant {
43
+
fn name(&self) -> &'static str;
44
+
fn check(&self, store: &TranquilBlockStore, oracle: &Oracle) -> Result<(), InvariantViolation>;
45
+
}
46
+
47
+
pub struct RefcountConservation;
48
+
49
+
impl Invariant for RefcountConservation {
50
+
fn name(&self) -> &'static str {
51
+
"RefcountConservation"
52
+
}
53
+
54
+
fn check(&self, store: &TranquilBlockStore, oracle: &Oracle) -> Result<(), InvariantViolation> {
55
+
let live: Vec<(String, CidBytes)> = oracle.live_cids_labeled();
56
+
let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect();
57
+
let index: HashMap<CidBytes, u32> = store
58
+
.block_index()
59
+
.live_entries_snapshot()
60
+
.into_iter()
61
+
.map(|(c, r)| (c, r.raw()))
62
+
.collect();
63
+
64
+
let forward: Vec<String> = live
65
+
.iter()
66
+
.filter_map(|(label, cid)| match index.get(cid) {
67
+
Some(&r) if r >= 1 => None,
68
+
Some(&r) => Some(format!("{label}: refcount {r}")),
69
+
None => Some(format!("{label}: missing from index")),
70
+
})
71
+
.collect();
72
+
73
+
let inverse: Vec<String> = index
74
+
.iter()
75
+
.filter(|(cid, _)| !live_set.contains(*cid))
76
+
.map(|(cid, r)| format!("orphan cid {} refcount {}", hex_short(cid), r))
77
+
.collect();
78
+
79
+
let violations: Vec<String> = forward.into_iter().chain(inverse).collect();
80
+
if violations.is_empty() {
81
+
Ok(())
82
+
} else {
83
+
Err(InvariantViolation {
84
+
invariant: "RefcountConservation",
85
+
detail: violations.join("; "),
86
+
})
87
+
}
88
+
}
89
+
}
90
+
91
+
pub struct Reachability;
92
+
93
+
impl Invariant for Reachability {
94
+
fn name(&self) -> &'static str {
95
+
"Reachability"
96
+
}
97
+
98
+
fn check(&self, store: &TranquilBlockStore, oracle: &Oracle) -> Result<(), InvariantViolation> {
99
+
let violations: Vec<String> = oracle
100
+
.live_cids_labeled()
101
+
.into_iter()
102
+
.filter_map(|(label, fixed)| match store.get_block_sync(&fixed) {
103
+
Ok(Some(_)) => None,
104
+
Ok(None) => Some(format!("{label}: missing")),
105
+
Err(e) => Some(format!("{label}: read error {e}")),
106
+
})
107
+
.collect();
108
+
109
+
if violations.is_empty() {
110
+
Ok(())
111
+
} else {
112
+
Err(InvariantViolation {
113
+
invariant: "Reachability",
114
+
detail: violations.join("; "),
115
+
})
116
+
}
117
+
}
118
+
}
119
+
120
+
fn hex_short(cid: &CidBytes) -> String {
121
+
let tail = &cid[cid.len() - 6..];
122
+
tail.iter().map(|b| format!("{b:02x}")).collect()
123
+
}
124
+
125
+
pub fn invariants_for(set: InvariantSet) -> Vec<Box<dyn Invariant>> {
126
+
let unknown = set.unknown_bits();
127
+
assert!(
128
+
unknown == 0,
129
+
"invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl"
130
+
);
131
+
[
132
+
(
133
+
InvariantSet::REFCOUNT_CONSERVATION,
134
+
Box::new(RefcountConservation) as Box<dyn Invariant>,
135
+
),
136
+
(InvariantSet::REACHABILITY, Box::new(Reachability)),
137
+
]
138
+
.into_iter()
139
+
.filter_map(|(flag, inv)| set.contains(flag).then_some(inv))
140
+
.collect()
141
+
}
+20
crates/tranquil-store/src/gauntlet/mod.rs
+20
crates/tranquil-store/src/gauntlet/mod.rs
···
1
+
pub mod farm;
2
+
pub mod invariants;
3
+
pub mod op;
4
+
pub mod oracle;
5
+
pub mod runner;
6
+
pub mod scenarios;
7
+
pub mod workload;
8
+
9
+
pub use invariants::{Invariant, InvariantSet, InvariantViolation, invariants_for};
10
+
pub use op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
11
+
pub use oracle::Oracle;
12
+
pub use runner::{
13
+
CompactInterval, Gauntlet, GauntletBuildError, GauntletConfig, GauntletReport, IoBackend,
14
+
MaxFileSize, OpIndex, OpInterval, OpsExecuted, RestartCount, RestartPolicy, RunLimits,
15
+
ShardCount, StoreConfig, WallMs,
16
+
};
17
+
pub use scenarios::{Scenario, config_for};
18
+
pub use workload::{
19
+
ByteRange, KeySpaceSize, OpCount, OpWeights, SizeDistribution, ValueBytes, WorkloadModel,
20
+
};
+60
crates/tranquil-store/src/gauntlet/op.rs
+60
crates/tranquil-store/src/gauntlet/op.rs
···
1
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2
+
pub struct Seed(pub u64);
3
+
4
+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
5
+
pub struct CollectionName(pub String);
6
+
7
+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
8
+
pub struct RecordKey(pub String);
9
+
10
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
11
+
pub struct ValueSeed(pub u32);
12
+
13
+
#[derive(Debug, Clone)]
14
+
pub enum Op {
15
+
AddRecord {
16
+
collection: CollectionName,
17
+
rkey: RecordKey,
18
+
value_seed: ValueSeed,
19
+
},
20
+
DeleteRecord {
21
+
collection: CollectionName,
22
+
rkey: RecordKey,
23
+
},
24
+
Compact,
25
+
Checkpoint,
26
+
}
27
+
28
+
#[derive(Debug, Clone)]
29
+
pub struct OpStream {
30
+
ops: Vec<Op>,
31
+
}
32
+
33
+
impl OpStream {
34
+
pub fn from_vec(ops: Vec<Op>) -> Self {
35
+
Self { ops }
36
+
}
37
+
38
+
pub fn into_vec(self) -> Vec<Op> {
39
+
self.ops
40
+
}
41
+
42
+
pub fn iter(&self) -> impl Iterator<Item = &Op> {
43
+
self.ops.iter()
44
+
}
45
+
46
+
pub fn len(&self) -> usize {
47
+
self.ops.len()
48
+
}
49
+
50
+
pub fn is_empty(&self) -> bool {
51
+
self.ops.is_empty()
52
+
}
53
+
54
+
pub fn shrink(&self) -> Option<OpStream> {
55
+
(self.ops.len() >= 2).then(|| {
56
+
let half = self.ops.len() / 2;
57
+
OpStream::from_vec(self.ops[..half].to_vec())
58
+
})
59
+
}
60
+
}
+75
crates/tranquil-store/src/gauntlet/oracle.rs
+75
crates/tranquil-store/src/gauntlet/oracle.rs
···
1
+
use std::collections::HashMap;
2
+
3
+
use cid::Cid;
4
+
5
+
use super::op::{CollectionName, RecordKey};
6
+
use crate::blockstore::CidBytes;
7
+
8
+
#[derive(Debug, Default)]
9
+
pub struct Oracle {
10
+
live: HashMap<(CollectionName, RecordKey), CidBytes>,
11
+
current_root: Option<Cid>,
12
+
mst_node_cids: Vec<Cid>,
13
+
}
14
+
15
+
impl Oracle {
16
+
pub fn new() -> Self {
17
+
Self::default()
18
+
}
19
+
20
+
pub fn add(
21
+
&mut self,
22
+
coll: CollectionName,
23
+
rkey: RecordKey,
24
+
record_cid: CidBytes,
25
+
) -> Option<CidBytes> {
26
+
self.live.insert((coll, rkey), record_cid)
27
+
}
28
+
29
+
pub fn delete(&mut self, coll: &CollectionName, rkey: &RecordKey) -> Option<CidBytes> {
30
+
self.live.remove(&(coll.clone(), rkey.clone()))
31
+
}
32
+
33
+
pub fn set_root(&mut self, root: Cid) {
34
+
self.current_root = Some(root);
35
+
}
36
+
37
+
pub fn root(&self) -> Option<Cid> {
38
+
self.current_root
39
+
}
40
+
41
+
pub fn set_node_cids(&mut self, cids: Vec<Cid>) {
42
+
self.mst_node_cids = cids;
43
+
}
44
+
45
+
pub fn mst_node_cids(&self) -> &[Cid] {
46
+
&self.mst_node_cids
47
+
}
48
+
49
+
pub fn live_records(&self) -> impl Iterator<Item = (&CollectionName, &RecordKey, &CidBytes)> {
50
+
self.live.iter().map(|((c, r), v)| (c, r, v))
51
+
}
52
+
53
+
pub fn live_count(&self) -> usize {
54
+
self.live.len()
55
+
}
56
+
57
+
pub fn live_cids_labeled(&self) -> Vec<(String, CidBytes)> {
58
+
let nodes = self
59
+
.mst_node_cids
60
+
.iter()
61
+
.map(|cid| (format!("mst {cid}"), cid_to_fixed(cid)));
62
+
let records = self
63
+
.live_records()
64
+
.map(|(c, r, v)| (format!("record {}/{}", c.0, r.0), *v));
65
+
nodes.chain(records).collect()
66
+
}
67
+
}
68
+
69
+
pub(super) fn cid_to_fixed(cid: &Cid) -> CidBytes {
70
+
let bytes = cid.to_bytes();
71
+
debug_assert_eq!(bytes.len(), 36, "expected 36 byte CIDv1+sha256");
72
+
let mut arr = [0u8; 36];
73
+
arr.copy_from_slice(&bytes[..36]);
74
+
arr
75
+
}
+438
crates/tranquil-store/src/gauntlet/runner.rs
+438
crates/tranquil-store/src/gauntlet/runner.rs
···
1
+
use std::sync::Arc;
2
+
use std::sync::atomic::{AtomicUsize, Ordering};
3
+
use std::time::Duration;
4
+
5
+
use cid::Cid;
6
+
use jacquard_repo::mst::Mst;
7
+
use jacquard_repo::storage::BlockStore;
8
+
9
+
use super::invariants::{InvariantSet, InvariantViolation, invariants_for};
10
+
use super::op::{Op, OpStream, Seed, ValueSeed};
11
+
use super::oracle::{Oracle, cid_to_fixed};
12
+
use super::workload::{Lcg, OpCount, SizeDistribution, ValueBytes, WorkloadModel};
13
+
use crate::blockstore::{
14
+
BlockStoreConfig, CidBytes, CompactionError, GroupCommitConfig, TranquilBlockStore,
15
+
};
16
+
17
+
#[derive(Debug, Clone, Copy)]
18
+
pub enum IoBackend {
19
+
Real,
20
+
Simulated,
21
+
}
22
+
23
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
24
+
pub struct OpInterval(pub usize);
25
+
26
+
#[derive(Debug, Clone, Copy)]
27
+
pub enum RestartPolicy {
28
+
Never,
29
+
EveryNOps(OpInterval),
30
+
PoissonByOps(OpInterval),
31
+
}
32
+
33
+
#[derive(Debug, Clone, Copy)]
34
+
pub struct WallMs(pub u64);
35
+
36
+
#[derive(Debug, Clone, Copy)]
37
+
pub struct RunLimits {
38
+
pub max_wall_ms: Option<WallMs>,
39
+
}
40
+
41
+
#[derive(Debug, Clone, Copy)]
42
+
pub struct MaxFileSize(pub u64);
43
+
44
+
#[derive(Debug, Clone, Copy)]
45
+
pub struct ShardCount(pub u8);
46
+
47
+
#[derive(Debug, Clone, Copy)]
48
+
pub struct CompactInterval(pub u32);
49
+
50
+
#[derive(Debug, Clone)]
51
+
pub struct StoreConfig {
52
+
pub max_file_size: MaxFileSize,
53
+
pub group_commit: GroupCommitConfig,
54
+
pub shard_count: ShardCount,
55
+
pub compact_every: CompactInterval,
56
+
}
57
+
58
+
#[derive(Debug, Clone)]
59
+
pub struct GauntletConfig {
60
+
pub seed: Seed,
61
+
pub io: IoBackend,
62
+
pub workload: WorkloadModel,
63
+
pub op_count: OpCount,
64
+
pub invariants: InvariantSet,
65
+
pub limits: RunLimits,
66
+
pub restart_policy: RestartPolicy,
67
+
pub store: StoreConfig,
68
+
}
69
+
70
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
71
+
pub struct OpsExecuted(pub usize);
72
+
73
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
74
+
pub struct RestartCount(pub usize);
75
+
76
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
77
+
pub struct OpIndex(pub usize);
78
+
79
+
#[derive(Debug)]
80
+
pub struct GauntletReport {
81
+
pub seed: Seed,
82
+
pub ops_executed: OpsExecuted,
83
+
pub restarts: RestartCount,
84
+
pub violations: Vec<InvariantViolation>,
85
+
}
86
+
87
+
impl GauntletReport {
88
+
pub fn is_clean(&self) -> bool {
89
+
self.violations.is_empty()
90
+
}
91
+
}
92
+
93
+
#[derive(Debug, thiserror::Error)]
94
+
enum OpError {
95
+
#[error("put record: {0}")]
96
+
PutRecord(String),
97
+
#[error("mst add: {0}")]
98
+
MstAdd(String),
99
+
#[error("mst delete: {0}")]
100
+
MstDelete(String),
101
+
#[error("mst persist: {0}")]
102
+
MstPersist(String),
103
+
#[error("mst diff: {0}")]
104
+
MstDiff(String),
105
+
#[error("apply commit: {0}")]
106
+
ApplyCommit(String),
107
+
#[error("compact_file: {0}")]
108
+
CompactFile(String),
109
+
#[error("join: {0}")]
110
+
Join(String),
111
+
}
112
+
113
+
pub struct Gauntlet {
114
+
config: GauntletConfig,
115
+
}
116
+
117
+
#[derive(Debug, thiserror::Error)]
118
+
pub enum GauntletBuildError {
119
+
#[error("IoBackend::Simulated not wired yet")]
120
+
UnsupportedIoBackend,
121
+
}
122
+
123
+
impl Gauntlet {
124
+
pub fn new(config: GauntletConfig) -> Result<Self, GauntletBuildError> {
125
+
match config.io {
126
+
IoBackend::Real => Ok(Self { config }),
127
+
IoBackend::Simulated => Err(GauntletBuildError::UnsupportedIoBackend),
128
+
}
129
+
}
130
+
131
+
pub async fn run(self) -> GauntletReport {
132
+
let deadline = self
133
+
.config
134
+
.limits
135
+
.max_wall_ms
136
+
.map(|WallMs(ms)| Duration::from_millis(ms));
137
+
138
+
let seed = self.config.seed;
139
+
let ops_counter = Arc::new(AtomicUsize::new(0));
140
+
let restarts_counter = Arc::new(AtomicUsize::new(0));
141
+
let fut = run_real_inner(self.config, ops_counter.clone(), restarts_counter.clone());
142
+
match deadline {
143
+
Some(d) => match tokio::time::timeout(d, fut).await {
144
+
Ok(r) => r,
145
+
Err(_) => GauntletReport {
146
+
seed,
147
+
ops_executed: OpsExecuted(ops_counter.load(Ordering::Relaxed)),
148
+
restarts: RestartCount(restarts_counter.load(Ordering::Relaxed)),
149
+
violations: vec![InvariantViolation {
150
+
invariant: "WallClockBudget",
151
+
detail: format!("exceeded max_wall_ms ({} ms)", d.as_millis()),
152
+
}],
153
+
},
154
+
},
155
+
None => fut.await,
156
+
}
157
+
}
158
+
}
159
+
160
+
async fn run_real_inner(
161
+
config: GauntletConfig,
162
+
ops_counter: Arc<AtomicUsize>,
163
+
restarts_counter: Arc<AtomicUsize>,
164
+
) -> GauntletReport {
165
+
let dir = tempfile::TempDir::new().expect("tempdir");
166
+
let op_stream: OpStream = config.workload.generate(config.seed, config.op_count);
167
+
168
+
let mut oracle = Oracle::new();
169
+
let mut violations: Vec<InvariantViolation> = Vec::new();
170
+
171
+
let mut store = Arc::new(
172
+
TranquilBlockStore::open(blockstore_config(dir.path(), &config.store)).expect("open store"),
173
+
);
174
+
let mut root: Option<Cid> = None;
175
+
let mut restart_rng = Lcg::new(Seed(config.seed.0 ^ 0xA5A5_A5A5_A5A5_A5A5));
176
+
let mut halt_ops = false;
177
+
178
+
for (idx, op) in op_stream.iter().enumerate() {
179
+
if halt_ops {
180
+
break;
181
+
}
182
+
match apply_op(&store, &mut root, &mut oracle, op, &config.workload).await {
183
+
Ok(()) => {}
184
+
Err(e) => {
185
+
violations.push(InvariantViolation {
186
+
invariant: "OpExecution",
187
+
detail: format!("op {idx}: {e}"),
188
+
});
189
+
ops_counter.store(idx + 1, Ordering::Relaxed);
190
+
halt_ops = true;
191
+
continue;
192
+
}
193
+
}
194
+
ops_counter.store(idx + 1, Ordering::Relaxed);
195
+
196
+
if should_restart(config.restart_policy, OpIndex(idx), &mut restart_rng) {
197
+
drop(store);
198
+
store = Arc::new(
199
+
TranquilBlockStore::open(blockstore_config(dir.path(), &config.store))
200
+
.expect("reopen store"),
201
+
);
202
+
let n = restarts_counter.fetch_add(1, Ordering::Relaxed) + 1;
203
+
204
+
if let Err(e) = refresh_oracle_graph(&store, &mut oracle, root).await {
205
+
violations.push(InvariantViolation {
206
+
invariant: "OpExecution",
207
+
detail: format!("refresh after restart {n}: {e}"),
208
+
});
209
+
halt_ops = true;
210
+
continue;
211
+
}
212
+
violations.extend(check_all(&store, &oracle, config.invariants));
213
+
if !violations.is_empty() {
214
+
halt_ops = true;
215
+
}
216
+
}
217
+
}
218
+
219
+
match refresh_oracle_graph(&store, &mut oracle, root).await {
220
+
Ok(()) => violations.extend(check_all(&store, &oracle, config.invariants)),
221
+
Err(e) => violations.push(InvariantViolation {
222
+
invariant: "OpExecution",
223
+
detail: format!("refresh at end: {e}"),
224
+
}),
225
+
}
226
+
227
+
GauntletReport {
228
+
seed: config.seed,
229
+
ops_executed: OpsExecuted(ops_counter.load(Ordering::Relaxed)),
230
+
restarts: RestartCount(restarts_counter.load(Ordering::Relaxed)),
231
+
violations,
232
+
}
233
+
}
234
+
235
+
fn check_all(
236
+
store: &TranquilBlockStore,
237
+
oracle: &Oracle,
238
+
set: InvariantSet,
239
+
) -> Vec<InvariantViolation> {
240
+
invariants_for(set)
241
+
.into_iter()
242
+
.filter_map(|inv| inv.check(store, oracle).err())
243
+
.collect()
244
+
}
245
+
246
+
async fn refresh_oracle_graph(
247
+
store: &Arc<TranquilBlockStore>,
248
+
oracle: &mut Oracle,
249
+
root: Option<Cid>,
250
+
) -> Result<(), String> {
251
+
match root {
252
+
None => {
253
+
oracle.set_node_cids(Vec::new());
254
+
Ok(())
255
+
}
256
+
Some(r) => {
257
+
let settled = Mst::load(store.clone(), r, None);
258
+
let cids = settled
259
+
.collect_node_cids()
260
+
.await
261
+
.map_err(|e| format!("collect_node_cids: {e}"))?;
262
+
oracle.set_root(r);
263
+
oracle.set_node_cids(cids);
264
+
Ok(())
265
+
}
266
+
}
267
+
}
268
+
269
+
fn should_restart(policy: RestartPolicy, idx: OpIndex, rng: &mut Lcg) -> bool {
270
+
match policy {
271
+
RestartPolicy::Never => false,
272
+
RestartPolicy::EveryNOps(OpInterval(n)) => n > 0 && (idx.0 + 1).is_multiple_of(n),
273
+
RestartPolicy::PoissonByOps(OpInterval(n)) => {
274
+
if n == 0 {
275
+
false
276
+
} else {
277
+
rng.next_u64().is_multiple_of(n as u64)
278
+
}
279
+
}
280
+
}
281
+
}
282
+
283
+
fn blockstore_config(dir: &std::path::Path, s: &StoreConfig) -> BlockStoreConfig {
284
+
BlockStoreConfig {
285
+
data_dir: dir.join("data"),
286
+
index_dir: dir.join("index"),
287
+
max_file_size: s.max_file_size.0,
288
+
group_commit: s.group_commit.clone(),
289
+
shard_count: s.shard_count.0,
290
+
}
291
+
}
292
+
293
+
fn make_record_bytes(value_seed: ValueSeed, dist: SizeDistribution) -> Vec<u8> {
294
+
let raw = value_seed.0;
295
+
let target_len: usize = match dist {
296
+
SizeDistribution::Fixed(ValueBytes(n)) => n as usize,
297
+
SizeDistribution::Uniform(range) => {
298
+
let ValueBytes(lo) = range.min();
299
+
let ValueBytes(hi) = range.max();
300
+
let span = u64::from(hi.saturating_sub(lo)).max(1);
301
+
let rng_state = u64::from(raw);
302
+
(lo as usize) + (rng_state % span) as usize
303
+
}
304
+
};
305
+
serde_ipld_dagcbor::to_vec(&serde_json::json!({
306
+
"$type": "app.bsky.feed.post",
307
+
"text": format!("record-{raw}"),
308
+
"createdAt": "2026-01-01T00:00:00Z",
309
+
"pad": "x".repeat(target_len.saturating_sub(64)),
310
+
}))
311
+
.expect("encode record")
312
+
}
313
+
314
+
async fn apply_op(
315
+
store: &Arc<TranquilBlockStore>,
316
+
root: &mut Option<Cid>,
317
+
oracle: &mut Oracle,
318
+
op: &Op,
319
+
workload: &WorkloadModel,
320
+
) -> Result<(), OpError> {
321
+
match op {
322
+
Op::AddRecord {
323
+
collection,
324
+
rkey,
325
+
value_seed,
326
+
} => {
327
+
let record_bytes = make_record_bytes(*value_seed, workload.size_distribution);
328
+
let record_cid = store
329
+
.put(&record_bytes)
330
+
.await
331
+
.map_err(|e| OpError::PutRecord(e.to_string()))?;
332
+
let key = format!("{}/{}", collection.0, rkey.0);
333
+
let loaded = match *root {
334
+
None => Mst::new(store.clone()),
335
+
Some(r) => Mst::load(store.clone(), r, None),
336
+
};
337
+
let updated = loaded
338
+
.add(&key, record_cid)
339
+
.await
340
+
.map_err(|e| OpError::MstAdd(e.to_string()))?;
341
+
let new_root = updated
342
+
.persist()
343
+
.await
344
+
.map_err(|e| OpError::MstPersist(e.to_string()))?;
345
+
346
+
if let Some(old_root) = *root {
347
+
apply_mst_diff(store, old_root, new_root).await?;
348
+
}
349
+
350
+
*root = Some(new_root);
351
+
oracle.add(collection.clone(), rkey.clone(), cid_to_fixed(&record_cid));
352
+
Ok(())
353
+
}
354
+
Op::DeleteRecord { collection, rkey } => {
355
+
let Some(old_root) = *root else { return Ok(()) };
356
+
if oracle.delete(collection, rkey).is_none() {
357
+
return Ok(());
358
+
}
359
+
let key = format!("{}/{}", collection.0, rkey.0);
360
+
let loaded = Mst::load(store.clone(), old_root, None);
361
+
let updated = loaded
362
+
.delete(&key)
363
+
.await
364
+
.map_err(|e| OpError::MstDelete(e.to_string()))?;
365
+
let new_root = updated
366
+
.persist()
367
+
.await
368
+
.map_err(|e| OpError::MstPersist(e.to_string()))?;
369
+
apply_mst_diff(store, old_root, new_root).await?;
370
+
*root = Some(new_root);
371
+
Ok(())
372
+
}
373
+
Op::Compact => {
374
+
let s = store.clone();
375
+
tokio::task::spawn_blocking(move || compact_by_liveness(&s))
376
+
.await
377
+
.map_err(|e| OpError::Join(e.to_string()))?
378
+
}
379
+
Op::Checkpoint => {
380
+
let s = store.clone();
381
+
tokio::task::spawn_blocking(move || {
382
+
s.apply_commit_blocking(vec![], vec![])
383
+
.map_err(|e| e.to_string())
384
+
})
385
+
.await
386
+
.map_err(|e| OpError::Join(e.to_string()))?
387
+
.map_err(OpError::ApplyCommit)
388
+
}
389
+
}
390
+
}
391
+
392
+
async fn apply_mst_diff(
393
+
store: &Arc<TranquilBlockStore>,
394
+
old_root: Cid,
395
+
new_root: Cid,
396
+
) -> Result<(), OpError> {
397
+
let old_m = Mst::load(store.clone(), old_root, None);
398
+
let new_m = Mst::load(store.clone(), new_root, None);
399
+
let diff = old_m
400
+
.diff(&new_m)
401
+
.await
402
+
.map_err(|e| OpError::MstDiff(e.to_string()))?;
403
+
let obsolete: Vec<CidBytes> = diff
404
+
.removed_mst_blocks
405
+
.into_iter()
406
+
.chain(diff.removed_cids.into_iter())
407
+
.map(|c| cid_to_fixed(&c))
408
+
.collect();
409
+
let s = store.clone();
410
+
tokio::task::spawn_blocking(move || {
411
+
s.apply_commit_blocking(vec![], obsolete)
412
+
.map_err(|e| e.to_string())
413
+
})
414
+
.await
415
+
.map_err(|e| OpError::Join(e.to_string()))?
416
+
.map_err(OpError::ApplyCommit)
417
+
}
418
+
419
+
const COMPACT_LIVENESS_CEILING: f64 = 0.99;
420
+
421
+
fn compact_by_liveness(store: &TranquilBlockStore) -> Result<(), OpError> {
422
+
let liveness = match store.compaction_liveness(0) {
423
+
Ok(l) => l,
424
+
Err(_) => return Ok(()),
425
+
};
426
+
let targets: Vec<_> = liveness
427
+
.iter()
428
+
.filter(|(_, info)| info.total_blocks > 0 && info.ratio() < COMPACT_LIVENESS_CEILING)
429
+
.map(|(&fid, _)| fid)
430
+
.collect();
431
+
targets
432
+
.into_iter()
433
+
.try_for_each(|fid| match store.compact_file(fid, 0) {
434
+
Ok(_) => Ok(()),
435
+
Err(CompactionError::ActiveFileCannotBeCompacted) => Ok(()),
436
+
Err(e) => Err(OpError::CompactFile(format!("{fid}: {e}"))),
437
+
})
438
+
}
+120
crates/tranquil-store/src/gauntlet/scenarios.rs
+120
crates/tranquil-store/src/gauntlet/scenarios.rs
···
1
+
use super::invariants::InvariantSet;
2
+
use super::op::{CollectionName, Seed};
3
+
use super::runner::{
4
+
CompactInterval, GauntletConfig, IoBackend, MaxFileSize, OpInterval, RestartPolicy, RunLimits,
5
+
ShardCount, StoreConfig, WallMs,
6
+
};
7
+
use super::workload::{
8
+
KeySpaceSize, OpCount, OpWeights, SizeDistribution, ValueBytes, WorkloadModel,
9
+
};
10
+
use crate::blockstore::GroupCommitConfig;
11
+
12
+
#[derive(Debug, Clone, Copy)]
13
+
pub enum Scenario {
14
+
SmokePR,
15
+
MstChurn,
16
+
MstRestartChurn,
17
+
}
18
+
19
+
pub fn config_for(scenario: Scenario, seed: Seed) -> GauntletConfig {
20
+
match scenario {
21
+
Scenario::SmokePR => smoke_pr(seed),
22
+
Scenario::MstChurn => mst_churn(seed),
23
+
Scenario::MstRestartChurn => mst_restart_churn(seed),
24
+
}
25
+
}
26
+
27
+
fn default_collections() -> Vec<CollectionName> {
28
+
vec![
29
+
CollectionName("app.bsky.feed.post".to_string()),
30
+
CollectionName("app.bsky.feed.like".to_string()),
31
+
]
32
+
}
33
+
34
+
fn tiny_store() -> StoreConfig {
35
+
StoreConfig {
36
+
max_file_size: MaxFileSize(300),
37
+
group_commit: GroupCommitConfig {
38
+
checkpoint_interval_ms: 100,
39
+
checkpoint_write_threshold: 10,
40
+
..GroupCommitConfig::default()
41
+
},
42
+
shard_count: ShardCount(1),
43
+
compact_every: CompactInterval(5),
44
+
}
45
+
}
46
+
47
+
fn smoke_pr(seed: Seed) -> GauntletConfig {
48
+
GauntletConfig {
49
+
seed,
50
+
io: IoBackend::Real,
51
+
workload: WorkloadModel {
52
+
weights: OpWeights {
53
+
add: 80,
54
+
delete: 0,
55
+
compact: 10,
56
+
checkpoint: 10,
57
+
},
58
+
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
59
+
collections: default_collections(),
60
+
key_space: KeySpaceSize(200),
61
+
},
62
+
op_count: OpCount(10_000),
63
+
invariants: InvariantSet::REFCOUNT_CONSERVATION | InvariantSet::REACHABILITY,
64
+
limits: RunLimits {
65
+
max_wall_ms: Some(WallMs(60_000)),
66
+
},
67
+
restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
68
+
store: tiny_store(),
69
+
}
70
+
}
71
+
72
+
fn mst_churn(seed: Seed) -> GauntletConfig {
73
+
GauntletConfig {
74
+
seed,
75
+
io: IoBackend::Real,
76
+
workload: WorkloadModel {
77
+
weights: OpWeights {
78
+
add: 85,
79
+
delete: 0,
80
+
compact: 10,
81
+
checkpoint: 5,
82
+
},
83
+
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
84
+
collections: default_collections(),
85
+
key_space: KeySpaceSize(2_000),
86
+
},
87
+
op_count: OpCount(100_000),
88
+
invariants: InvariantSet::REFCOUNT_CONSERVATION | InvariantSet::REACHABILITY,
89
+
limits: RunLimits {
90
+
max_wall_ms: Some(WallMs(600_000)),
91
+
},
92
+
restart_policy: RestartPolicy::Never,
93
+
store: tiny_store(),
94
+
}
95
+
}
96
+
97
+
fn mst_restart_churn(seed: Seed) -> GauntletConfig {
98
+
GauntletConfig {
99
+
seed,
100
+
io: IoBackend::Real,
101
+
workload: WorkloadModel {
102
+
weights: OpWeights {
103
+
add: 85,
104
+
delete: 0,
105
+
compact: 10,
106
+
checkpoint: 5,
107
+
},
108
+
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
109
+
collections: default_collections(),
110
+
key_space: KeySpaceSize(2_000),
111
+
},
112
+
op_count: OpCount(100_000),
113
+
invariants: InvariantSet::REFCOUNT_CONSERVATION | InvariantSet::REACHABILITY,
114
+
limits: RunLimits {
115
+
max_wall_ms: Some(WallMs(600_000)),
116
+
},
117
+
restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
118
+
store: tiny_store(),
119
+
}
120
+
}
+133
crates/tranquil-store/src/gauntlet/workload.rs
+133
crates/tranquil-store/src/gauntlet/workload.rs
···
1
+
use super::op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
2
+
3
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
4
+
pub struct ValueBytes(pub u32);
5
+
6
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7
+
pub struct KeySpaceSize(pub u32);
8
+
9
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
10
+
pub struct OpCount(pub usize);
11
+
12
+
#[derive(Debug, Clone, Copy)]
13
+
pub struct OpWeights {
14
+
pub add: u32,
15
+
pub delete: u32,
16
+
pub compact: u32,
17
+
pub checkpoint: u32,
18
+
}
19
+
20
+
impl OpWeights {
21
+
pub const fn total(&self) -> u32 {
22
+
self.add + self.delete + self.compact + self.checkpoint
23
+
}
24
+
}
25
+
26
+
#[derive(Debug, Clone, Copy)]
27
+
pub struct ByteRange {
28
+
min: ValueBytes,
29
+
max: ValueBytes,
30
+
}
31
+
32
+
impl ByteRange {
33
+
pub fn new(min: ValueBytes, max: ValueBytes) -> Result<Self, String> {
34
+
if max.0 < min.0 {
35
+
Err(format!("ByteRange: max {} < min {}", max.0, min.0))
36
+
} else {
37
+
Ok(Self { min, max })
38
+
}
39
+
}
40
+
41
+
pub fn min(&self) -> ValueBytes {
42
+
self.min
43
+
}
44
+
45
+
pub fn max(&self) -> ValueBytes {
46
+
self.max
47
+
}
48
+
}
49
+
50
+
#[derive(Debug, Clone, Copy)]
51
+
pub enum SizeDistribution {
52
+
Fixed(ValueBytes),
53
+
Uniform(ByteRange),
54
+
}
55
+
56
+
#[derive(Debug, Clone)]
57
+
pub struct WorkloadModel {
58
+
pub weights: OpWeights,
59
+
pub size_distribution: SizeDistribution,
60
+
pub collections: Vec<CollectionName>,
61
+
pub key_space: KeySpaceSize,
62
+
}
63
+
64
+
impl WorkloadModel {
65
+
pub fn generate(&self, seed: Seed, op_count: OpCount) -> OpStream {
66
+
let mut rng = Lcg::new(seed);
67
+
let total = self.weights.total();
68
+
assert!(total > 0, "workload weights must sum to > 0");
69
+
assert!(
70
+
!self.collections.is_empty(),
71
+
"workload needs at least 1 collection"
72
+
);
73
+
74
+
let ops: Vec<Op> = (0..op_count.0)
75
+
.map(|_| {
76
+
let bucket = rng.next_u32() % total;
77
+
let coll = self.collections[rng.next_usize() % self.collections.len()].clone();
78
+
let rkey = RecordKey(format!("{:06}", rng.next_u32() % self.key_space.0.max(1)));
79
+
80
+
let (a, d, c) = (
81
+
self.weights.add,
82
+
self.weights.add + self.weights.delete,
83
+
self.weights.add + self.weights.delete + self.weights.compact,
84
+
);
85
+
match bucket {
86
+
b if b < a => Op::AddRecord {
87
+
collection: coll,
88
+
rkey,
89
+
value_seed: ValueSeed(rng.next_u32()),
90
+
},
91
+
b if b < d => Op::DeleteRecord {
92
+
collection: coll,
93
+
rkey,
94
+
},
95
+
b if b < c => Op::Compact,
96
+
_ => Op::Checkpoint,
97
+
}
98
+
})
99
+
.collect();
100
+
OpStream::from_vec(ops)
101
+
}
102
+
}
103
+
104
+
pub struct Lcg {
105
+
state: u64,
106
+
}
107
+
108
+
impl Lcg {
109
+
pub fn new(seed: Seed) -> Self {
110
+
Self {
111
+
state: seed
112
+
.0
113
+
.wrapping_mul(6364136223846793005)
114
+
.wrapping_add(1442695040888963407),
115
+
}
116
+
}
117
+
118
+
pub fn next_u64(&mut self) -> u64 {
119
+
self.state = self
120
+
.state
121
+
.wrapping_mul(6364136223846793005)
122
+
.wrapping_add(1442695040888963407);
123
+
self.state
124
+
}
125
+
126
+
pub fn next_u32(&mut self) -> u32 {
127
+
(self.next_u64() >> 16) as u32
128
+
}
129
+
130
+
pub fn next_usize(&mut self) -> usize {
131
+
self.next_u32() as usize
132
+
}
133
+
}
+2
crates/tranquil-store/src/lib.rs
+2
crates/tranquil-store/src/lib.rs
+1
-1
crates/tranquil-store/src/metastore/handler.rs
+1
-1
crates/tranquil-store/src/metastore/handler.rs
+5
-5
crates/tranquil-store/tests/checkpoint_race.rs
+5
-5
crates/tranquil-store/tests/checkpoint_race.rs
···
4
4
use std::sync::Arc;
5
5
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6
6
7
+
use tranquil_store::PostBlockstoreHook;
7
8
use tranquil_store::blockstore::{
8
9
BlockStoreConfig, BlocksSynced, CidBytes, GroupCommitConfig, TranquilBlockStore,
9
10
};
10
-
use tranquil_store::PostBlockstoreHook;
11
11
12
12
struct SlowHook;
13
13
···
72
72
store
73
73
.put_blocks_blocking(vec![(cid, vec![shard; 60])])
74
74
.unwrap();
75
-
store
76
-
.apply_commit_blocking(vec![], vec![cid])
77
-
.unwrap();
75
+
store.apply_commit_blocking(vec![], vec![cid]).unwrap();
78
76
targets.push(cid);
79
77
seq += 1;
80
78
total_cycles.fetch_add(1, Ordering::Relaxed);
···
208
206
Ok(d) => d,
209
207
Err(_) => return,
210
208
};
211
-
let use_hook = std::env::var("CRASH_TEST_HOOK").map(|v| v == "1").unwrap_or(false);
209
+
let use_hook = std::env::var("CRASH_TEST_HOOK")
210
+
.map(|v| v == "1")
211
+
.unwrap_or(false);
212
212
let base = std::path::Path::new(&dir);
213
213
214
214
let rt = tokio::runtime::Runtime::new().unwrap();
+1
-1
crates/tranquil-store/tests/compaction_liveness.rs
+1
-1
crates/tranquil-store/tests/compaction_liveness.rs
+11
-11
crates/tranquil-store/tests/compaction_restart.rs
+11
-11
crates/tranquil-store/tests/compaction_restart.rs
···
26
26
let blockstore_data = base_dir.join("blockstore").join("data");
27
27
let blockstore_index = base_dir.join("blockstore").join("index");
28
28
29
-
[&metastore_dir, &segments_dir, &blockstore_data, &blockstore_index]
30
-
.iter()
31
-
.for_each(|d| std::fs::create_dir_all(d).unwrap());
29
+
[
30
+
&metastore_dir,
31
+
&segments_dir,
32
+
&blockstore_data,
33
+
&blockstore_index,
34
+
]
35
+
.iter()
36
+
.for_each(|d| std::fs::create_dir_all(d).unwrap());
32
37
33
38
let metastore = Metastore::open(&metastore_dir, MetastoreConfig::default()).unwrap();
34
39
···
59
64
60
65
let indexes = metastore.partition(Partition::Indexes).clone();
61
66
let event_ops = metastore.event_ops(Arc::clone(&bridge));
62
-
let recovered = event_ops
63
-
.recover_metastore_mutations(&indexes)
64
-
.unwrap();
67
+
let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap();
65
68
if recovered > 0 {
66
69
eprintln!("replayed {recovered} metastore mutations from eventlog");
67
70
}
···
240
243
241
244
if round > 0 {
242
245
let prev_mst = test_cid(6000 + round - 1);
243
-
store
244
-
.apply_commit_blocking(vec![], vec![prev_mst])
245
-
.unwrap();
246
+
store.apply_commit_blocking(vec![], vec![prev_mst]).unwrap();
246
247
}
247
248
248
249
prev_commit = new_commit;
···
408
409
409
410
(0..10u32).for_each(|cycle| {
410
411
{
411
-
let store =
412
-
TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
412
+
let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
413
413
414
414
(0..50u32).for_each(|round| {
415
415
let churn = test_cid(9000 + cycle * 100 + round);
+2
-2
crates/tranquil-store/tests/eventlog_properties.rs
+2
-2
crates/tranquil-store/tests/eventlog_properties.rs
···
579
579
580
580
assert!(
581
581
reads_with_index < reads_without_index,
582
-
"read with index ({reads_with_index} reads) should require fewer reads than without ({reads_without_index} reads)"
582
+
"indexed read took {reads_with_index} reads but unindexed took only {reads_without_index}"
583
583
);
584
584
}
585
585
···
654
654
assert_eq!(
655
655
event_writer.synced_seq(),
656
656
EventSequence::BEFORE_ALL,
657
-
"crash between blockstore sync and eventlog sync must not persist the event (blocks exist, event does not = orphan, not inconsistency)"
657
+
"crash between blockstore sync and eventlog sync must leave blocks orphaned rather than persist the event"
658
658
);
659
659
660
660
drop(event_writer);
+1
-1
crates/tranquil-store/tests/eventlog_retention.rs
+1
-1
crates/tranquil-store/tests/eventlog_retention.rs
+111
crates/tranquil-store/tests/gauntlet_smoke.rs
+111
crates/tranquil-store/tests/gauntlet_smoke.rs
···
1
+
use tranquil_store::blockstore::GroupCommitConfig;
2
+
use tranquil_store::gauntlet::{
3
+
CollectionName, CompactInterval, Gauntlet, GauntletConfig, InvariantSet, IoBackend,
4
+
KeySpaceSize, MaxFileSize, OpCount, OpInterval, OpWeights, RestartPolicy, RunLimits, Scenario,
5
+
Seed, ShardCount, SizeDistribution, StoreConfig, ValueBytes, WallMs, WorkloadModel, config_for,
6
+
farm,
7
+
};
8
+
9
+
#[test]
10
+
#[ignore = "long running, 30 seeds of 10k ops each"]
11
+
fn smoke_pr_30_seeds() {
12
+
let reports = farm::run_many(
13
+
|seed| config_for(Scenario::SmokePR, seed),
14
+
(0..30).map(Seed),
15
+
);
16
+
let failures: Vec<String> = reports
17
+
.iter()
18
+
.filter(|r| !r.is_clean())
19
+
.map(|r| {
20
+
format!(
21
+
"seed {}: {} violations\n {}",
22
+
r.seed.0,
23
+
r.violations.len(),
24
+
r.violations
25
+
.iter()
26
+
.map(|v| format!("{}: {}", v.invariant, v.detail))
27
+
.collect::<Vec<_>>()
28
+
.join("\n ")
29
+
)
30
+
})
31
+
.collect();
32
+
assert!(failures.is_empty(), "{}", failures.join("\n---\n"));
33
+
}
34
+
35
+
fn fast_sanity_config(seed: Seed) -> GauntletConfig {
36
+
GauntletConfig {
37
+
seed,
38
+
io: IoBackend::Real,
39
+
workload: WorkloadModel {
40
+
weights: OpWeights {
41
+
add: 80,
42
+
delete: 0,
43
+
compact: 10,
44
+
checkpoint: 10,
45
+
},
46
+
size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
47
+
collections: vec![CollectionName("app.bsky.feed.post".to_string())],
48
+
key_space: KeySpaceSize(100),
49
+
},
50
+
op_count: OpCount(200),
51
+
invariants: InvariantSet::REFCOUNT_CONSERVATION | InvariantSet::REACHABILITY,
52
+
limits: RunLimits {
53
+
max_wall_ms: Some(WallMs(30_000)),
54
+
},
55
+
restart_policy: RestartPolicy::EveryNOps(OpInterval(80)),
56
+
store: StoreConfig {
57
+
max_file_size: MaxFileSize(512),
58
+
group_commit: GroupCommitConfig {
59
+
checkpoint_interval_ms: 50,
60
+
checkpoint_write_threshold: 8,
61
+
..GroupCommitConfig::default()
62
+
},
63
+
shard_count: ShardCount(1),
64
+
compact_every: CompactInterval(5),
65
+
},
66
+
}
67
+
}
68
+
69
+
#[tokio::test]
70
+
async fn gauntlet_fast_sanity() {
71
+
let report = Gauntlet::new(fast_sanity_config(Seed(7)))
72
+
.expect("build gauntlet")
73
+
.run()
74
+
.await;
75
+
assert!(
76
+
report.is_clean(),
77
+
"violations: {:?}",
78
+
report
79
+
.violations
80
+
.iter()
81
+
.map(|v| format!("{}: {}", v.invariant, v.detail))
82
+
.collect::<Vec<_>>()
83
+
);
84
+
assert!(
85
+
report.restarts.0 >= 2,
86
+
"expected at least 2 restarts, got {}",
87
+
report.restarts.0
88
+
);
89
+
assert_eq!(report.ops_executed.0, 200);
90
+
}
91
+
92
+
#[tokio::test]
93
+
#[ignore = "long running, 100k ops with around 20 restarts"]
94
+
async fn mst_restart_churn_single_seed() {
95
+
let cfg = config_for(Scenario::MstRestartChurn, Seed(42));
96
+
let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
97
+
assert!(
98
+
report.is_clean(),
99
+
"violations: {:?}",
100
+
report
101
+
.violations
102
+
.iter()
103
+
.map(|v| format!("{}: {}", v.invariant, v.detail))
104
+
.collect::<Vec<_>>()
105
+
);
106
+
assert!(
107
+
report.restarts.0 >= 1,
108
+
"PoissonByOps(5000) over 100k ops should fire at least 1 restart, got {}",
109
+
report.restarts.0
110
+
);
111
+
}
+1
-1
crates/tranquil-store/tests/gc.rs
+1
-1
crates/tranquil-store/tests/gc.rs
+3
-7
crates/tranquil-store/tests/mst_refcount_integrity.rs
+3
-7
crates/tranquil-store/tests/mst_refcount_integrity.rs
···
80
80
81
81
let obsolete =
82
82
compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await;
83
-
let obsolete_fixed: Vec<[u8; 36]> =
84
-
obsolete.iter().map(|c| cid_to_fixed(c)).collect();
83
+
let obsolete_fixed: Vec<[u8; 36]> = obsolete.iter().map(cid_to_fixed).collect();
85
84
let s = store.clone();
86
85
tokio::task::spawn_blocking(move || {
87
86
s.apply_commit_blocking(vec![], obsolete_fixed).unwrap();
···
115
114
116
115
let obsolete =
117
116
compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await;
118
-
let obsolete_fixed: Vec<[u8; 36]> =
119
-
obsolete.iter().map(|c| cid_to_fixed(c)).collect();
117
+
let obsolete_fixed: Vec<[u8; 36]> = obsolete.iter().map(cid_to_fixed).collect();
120
118
let s = store.clone();
121
119
tokio::task::spawn_blocking(move || {
122
120
s.apply_commit_blocking(vec![], obsolete_fixed).unwrap();
···
168
166
}
169
167
170
168
{
171
-
let store = Arc::new(
172
-
TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(),
173
-
);
169
+
let store = Arc::new(TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap());
174
170
175
171
let missing: Vec<String> = final_node_cids
176
172
.iter()
+2
-6
crates/tranquil-store/tests/sim_blockstore.rs
+2
-6
crates/tranquil-store/tests/sim_blockstore.rs
···
74
74
let cid = test_cid(seed as u32);
75
75
let data = vec![seed as u8; data_size];
76
76
let loc = writer.append_block(&cid, &data).unwrap();
77
-
hint_writer
78
-
.append_hint(&cid, &loc)
79
-
.unwrap();
77
+
hint_writer.append_hint(&cid, &loc).unwrap();
80
78
(cid, loc)
81
79
})
82
80
.collect();
···
538
536
let cid = test_cid(i as u32);
539
537
let data = vec![i as u8; 64];
540
538
let loc = writer.append_block(&cid, &data).ok()?;
541
-
hint_writer
542
-
.append_hint(&cid, &loc)
543
-
.ok()?;
539
+
hint_writer.append_hint(&cid, &loc).ok()?;
544
540
Some(())
545
541
})?;
546
542