+4

.config/nextest.toml

reviewed

··· 72 72 filter = "test(/test_scale_/) | test(/full_backup_and_restore/)" 73 73 slow-timeout = { period = "120s", terminate-after = 4 } 74 74 75 75 + [[profile.default.overrides]] 76 76 + filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)" 77 77 + slow-timeout = { period = "120s", terminate-after = 4 } 78 78 + 75 79 [[profile.ci.overrides]] 76 80 filter = "test(/import_with_verification/) | test(/plc_migration/)" 77 81 test-group = "serial-env-tests"

+22 -22

Cargo.lock

reviewed

··· 7405 7405 7406 7406 [[package]] 7407 7407 name = "tranquil-api" 7408 7408 - version = "0.5.3" 7408 7408 + version = "0.5.4" 7409 7409 dependencies = [ 7410 7410 "anyhow", 7411 7411 "axum", ··· 7456 7456 7457 7457 [[package]] 7458 7458 name = "tranquil-auth" 7459 7459 - version = "0.5.3" 7459 7459 + version = "0.5.4" 7460 7460 dependencies = [ 7461 7461 "anyhow", 7462 7462 "base32", ··· 7479 7479 7480 7480 [[package]] 7481 7481 name = "tranquil-cache" 7482 7482 - version = "0.5.3" 7482 7482 + version = "0.5.4" 7483 7483 dependencies = [ 7484 7484 "async-trait", 7485 7485 "base64 0.22.1", ··· 7493 7493 7494 7494 [[package]] 7495 7495 name = "tranquil-comms" 7496 7496 - version = "0.5.3" 7496 7496 + version = "0.5.4" 7497 7497 dependencies = [ 7498 7498 "async-trait", 7499 7499 "base64 0.22.1", ··· 7511 7511 7512 7512 [[package]] 7513 7513 name = "tranquil-config" 7514 7514 - version = "0.5.3" 7514 7514 + version = "0.5.4" 7515 7515 dependencies = [ 7516 7516 "confique", 7517 7517 "serde", ··· 7519 7519 7520 7520 [[package]] 7521 7521 name = "tranquil-crypto" 7522 7522 - version = "0.5.3" 7522 7522 + version = "0.5.4" 7523 7523 dependencies = [ 7524 7524 "aes-gcm", 7525 7525 "base64 0.22.1", ··· 7535 7535 7536 7536 [[package]] 7537 7537 name = "tranquil-db" 7538 7538 - version = "0.5.3" 7538 7538 + version = "0.5.4" 7539 7539 dependencies = [ 7540 7540 "async-trait", 7541 7541 "chrono", ··· 7552 7552 7553 7553 [[package]] 7554 7554 name = "tranquil-db-traits" 7555 7555 - version = "0.5.3" 7555 7555 + version = "0.5.4" 7556 7556 dependencies = [ 7557 7557 "async-trait", 7558 7558 "base64 0.22.1", ··· 7568 7568 7569 7569 [[package]] 7570 7570 name = "tranquil-infra" 7571 7571 - version = "0.5.3" 7571 7571 + version = "0.5.4" 7572 7572 dependencies = [ 7573 7573 "async-trait", 7574 7574 "bytes", ··· 7579 7579 7580 7580 [[package]] 7581 7581 name = "tranquil-lexicon" 7582 7582 - version = "0.5.3" 7582 7582 + version = "0.5.4" 7583 7583 dependencies = [ 7584 7584 "chrono", 7585 7585 "hickory-resolver", ··· 7597 7597 7598 7598 [[package]] 7599 7599 name = "tranquil-oauth" 7600 7600 - version = "0.5.3" 7600 7600 + version = "0.5.4" 7601 7601 dependencies = [ 7602 7602 "anyhow", 7603 7603 "axum", ··· 7620 7620 7621 7621 [[package]] 7622 7622 name = "tranquil-oauth-server" 7623 7623 - version = "0.5.3" 7623 7623 + version = "0.5.4" 7624 7624 dependencies = [ 7625 7625 "axum", 7626 7626 "base64 0.22.1", ··· 7653 7653 7654 7654 [[package]] 7655 7655 name = "tranquil-pds" 7656 7656 - version = "0.5.3" 7656 7656 + version = "0.5.4" 7657 7657 dependencies = [ 7658 7658 "aes-gcm", 7659 7659 "anyhow", ··· 7745 7745 7746 7746 [[package]] 7747 7747 name = "tranquil-repo" 7748 7748 - version = "0.5.3" 7748 7748 + version = "0.5.4" 7749 7749 dependencies = [ 7750 7750 "bytes", 7751 7751 "cid", ··· 7757 7757 7758 7758 [[package]] 7759 7759 name = "tranquil-ripple" 7760 7760 - version = "0.5.3" 7760 7760 + version = "0.5.4" 7761 7761 dependencies = [ 7762 7762 "async-trait", 7763 7763 "backon", ··· 7782 7782 7783 7783 [[package]] 7784 7784 name = "tranquil-scopes" 7785 7785 - version = "0.5.3" 7785 7785 + version = "0.5.4" 7786 7786 dependencies = [ 7787 7787 "axum", 7788 7788 "futures", ··· 7798 7798 7799 7799 [[package]] 7800 7800 name = "tranquil-server" 7801 7801 - version = "0.5.3" 7801 7801 + version = "0.5.4" 7802 7802 dependencies = [ 7803 7803 "axum", 7804 7804 "clap", ··· 7819 7819 7820 7820 [[package]] 7821 7821 name = "tranquil-signal" 7822 7822 - version = "0.5.3" 7822 7822 + version = "0.5.4" 7823 7823 dependencies = [ 7824 7824 "async-trait", 7825 7825 "chrono", ··· 7842 7842 7843 7843 [[package]] 7844 7844 name = "tranquil-storage" 7845 7845 - version = "0.5.3" 7845 7845 + version = "0.5.4" 7846 7846 dependencies = [ 7847 7847 "async-trait", 7848 7848 "aws-config", ··· 7859 7859 7860 7860 [[package]] 7861 7861 name = "tranquil-store" 7862 7862 - version = "0.5.3" 7862 7862 + version = "0.5.4" 7863 7863 dependencies = [ 7864 7864 "async-trait", 7865 7865 "bytes", ··· 7906 7906 7907 7907 [[package]] 7908 7908 name = "tranquil-sync" 7909 7909 - version = "0.5.3" 7909 7909 + version = "0.5.4" 7910 7910 dependencies = [ 7911 7911 "anyhow", 7912 7912 "axum", ··· 7928 7928 7929 7929 [[package]] 7930 7930 name = "tranquil-types" 7931 7931 - version = "0.5.3" 7931 7931 + version = "0.5.4" 7932 7932 dependencies = [ 7933 7933 "chrono", 7934 7934 "cid",

+1 -1

Cargo.toml

reviewed

··· 26 26 ] 27 27 28 28 [workspace.package] 29 29 - version = "0.5.3" 29 29 + version = "0.5.4" 30 30 edition = "2024" 31 31 license = "AGPL-3.0-or-later" 32 32

+175

crates/tranquil-pds/tests/gc_compaction_restart.rs

reviewed

··· 1 1 + mod common; 2 2 + 3 3 + use chrono::Utc; 4 4 + use common::*; 5 5 + use reqwest::StatusCode; 6 6 + use serde_json::{Value, json}; 7 7 + 8 8 + fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) { 9 9 + let liveness = store.compaction_liveness(0).unwrap(); 10 10 + liveness 11 11 + .iter() 12 12 + .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.95) 13 13 + .map(|(&fid, _)| fid) 14 14 + .collect::<Vec<_>>() 15 15 + .into_iter() 16 16 + .for_each(|fid| { 17 17 + match store.compact_file(fid, 0) { 18 18 + Ok(_) => {} 19 19 + Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {} 20 20 + Err(e) => eprintln!("compaction: {e}"), 21 21 + } 22 22 + }); 23 23 + } 24 24 + 25 25 + #[tokio::test] 26 26 + async fn mst_blocks_survive_full_store_reopen() { 27 27 + if !is_store_backend() { 28 28 + eprintln!("skipping: only meaningful with tranquil-store backend"); 29 29 + return; 30 30 + } 31 31 + 32 32 + let client = client(); 33 33 + let base = base_url().await; 34 34 + let block_store = get_test_block_store().await; 35 35 + 36 36 + let store = block_store 37 37 + .as_tranquil_store() 38 38 + .expect("expected tranquil-store backend"); 39 39 + 40 40 + let (jwt, did) = create_account_and_login(&client).await; 41 41 + 42 42 + let mut posts = Vec::new(); 43 43 + for i in 0..30 { 44 44 + let res = client 45 45 + .post(format!("{base}/xrpc/com.atproto.repo.createRecord")) 46 46 + .bearer_auth(&jwt) 47 47 + .json(&json!({ 48 48 + "repo": did, 49 49 + "collection": "app.bsky.feed.post", 50 50 + "record": { 51 51 + "$type": "app.bsky.feed.post", 52 52 + "text": format!("compaction test post {i}"), 53 53 + "createdAt": Utc::now().to_rfc3339() 54 54 + } 55 55 + })) 56 56 + .send() 57 57 + .await 58 58 + .unwrap(); 59 59 + assert_eq!(res.status(), StatusCode::OK); 60 60 + let body: Value = res.json().await.unwrap(); 61 61 + posts.push(( 62 62 + body["uri"].as_str().unwrap().to_string(), 63 63 + body["cid"].as_str().unwrap().to_string(), 64 64 + )); 65 65 + } 66 66 + 67 67 + for (uri, cid) in &posts[..20] { 68 68 + let res = client 69 69 + .post(format!("{base}/xrpc/com.atproto.repo.createRecord")) 70 70 + .bearer_auth(&jwt) 71 71 + .json(&json!({ 72 72 + "repo": did, 73 73 + "collection": "app.bsky.feed.like", 74 74 + "record": { 75 75 + "$type": "app.bsky.feed.like", 76 76 + "subject": { "uri": uri, "cid": cid }, 77 77 + "createdAt": Utc::now().to_rfc3339() 78 78 + } 79 79 + })) 80 80 + .send() 81 81 + .await 82 82 + .unwrap(); 83 83 + assert_eq!(res.status(), StatusCode::OK, "like failed for {uri}"); 84 84 + } 85 85 + 86 86 + let data_dir = store.data_dir().to_path_buf(); 87 87 + let index_dir = data_dir 88 88 + .parent() 89 89 + .unwrap() 90 90 + .join("index"); 91 91 + 92 92 + let store_clone = store.clone(); 93 93 + tokio::task::spawn_blocking(move || { 94 94 + (0..40).for_each(|_| run_compaction(&store_clone)); 95 95 + }) 96 96 + .await 97 97 + .unwrap(); 98 98 + 99 99 + let repo_root_str: String = get_test_repos() 100 100 + .await 101 101 + .repo 102 102 + .get_repo_root_by_did(&tranquil_types::Did::new(did.clone()).unwrap()) 103 103 + .await 104 104 + .expect("db error") 105 105 + .expect("no repo root") 106 106 + .to_string(); 107 107 + 108 108 + let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid"); 109 109 + 110 110 + let car_blocks = 111 111 + tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid) 112 112 + .await 113 113 + .expect("collect blocks"); 114 114 + 115 115 + let block_count_before = car_blocks.len(); 116 116 + 117 117 + let max_file_size = store 118 118 + .list_data_files() 119 119 + .ok() 120 120 + .and_then(|_| Some(4 * 1024 * 1024u64)) 121 121 + .unwrap_or(4 * 1024 * 1024); 122 122 + 123 123 + let reopened_missing = tokio::task::spawn_blocking(move || { 124 124 + let rt = tokio::runtime::Handle::current(); 125 125 + let _guard = rt.enter(); 126 126 + 127 127 + let config = tranquil_store::blockstore::BlockStoreConfig { 128 128 + data_dir: data_dir.clone(), 129 129 + index_dir, 130 130 + max_file_size, 131 131 + group_commit: tranquil_store::blockstore::GroupCommitConfig::default(), 132 132 + shard_count: 1, 133 133 + }; 134 134 + let fresh = tranquil_store::blockstore::TranquilBlockStore::open(config) 135 135 + .expect("reopen failed"); 136 136 + 137 137 + let missing: Vec<String> = car_blocks 138 138 + .iter() 139 139 + .filter_map(|cid_bytes| { 140 140 + if cid_bytes.len() < 36 { 141 141 + return None; 142 142 + } 143 143 + let mut arr = [0u8; 36]; 144 144 + arr.copy_from_slice(&cid_bytes[..36]); 145 145 + match fresh.get_block_sync(&arr) { 146 146 + Ok(Some(_)) => None, 147 147 + Ok(None) => Some(format!( 148 148 + "missing {}", 149 149 + cid::Cid::try_from(cid_bytes.as_slice()) 150 150 + .map(|c| c.to_string()) 151 151 + .unwrap_or_else(|_| hex::encode(cid_bytes)) 152 152 + )), 153 153 + Err(e) => Some(format!("error: {e}")), 154 154 + } 155 155 + }) 156 156 + .collect(); 157 157 + 158 158 + drop(fresh); 159 159 + missing 160 160 + }) 161 161 + .await 162 162 + .unwrap(); 163 163 + 164 164 + assert!( 165 165 + reopened_missing.is_empty(), 166 166 + "{} of {block_count_before} blocks missing after blockstore reopen:\n{}", 167 167 + reopened_missing.len(), 168 168 + reopened_missing 169 169 + .iter() 170 170 + .take(20) 171 171 + .map(|s| s.as_str()) 172 172 + .collect::<Vec<_>>() 173 173 + .join("\n"), 174 174 + ); 175 175 + }

+5 -1

crates/tranquil-store/src/blockstore/hash_index.rs

reviewed

··· 215 215 self.get(cid).is_some() 216 216 } 217 217 218 218 + pub fn contains_live(&self, cid: &[u8; CID_SIZE]) -> bool { 219 219 + self.get(cid).is_some_and(|s| !s.refcount.is_zero()) 220 220 + } 221 221 + 218 222 pub fn insert(&mut self, new_slot: Slot) -> Result<Option<Slot>, CapacityExhausted> { 219 223 if is_empty(&new_slot.cid) { 220 224 tracing::error!("attempted to insert all-zero CID into hash table"); ··· 1195 1199 } 1196 1200 1197 1201 pub fn has(&self, cid: &[u8; CID_SIZE]) -> bool { 1198 1198 - self.table.read().contains(cid) 1202 1202 + self.table.read().contains_live(cid) 1199 1203 } 1200 1204 1201 1205 pub fn batch_put(

+64

crates/tranquil-store/tests/common/mod.rs

reviewed

··· 104 104 }); 105 105 } 106 106 107 107 + pub fn tiny_blockstore_config(dir: &std::path::Path) -> BlockStoreConfig { 108 108 + BlockStoreConfig { 109 109 + data_dir: dir.join("data"), 110 110 + index_dir: dir.join("index"), 111 111 + max_file_size: 300, 112 112 + group_commit: GroupCommitConfig { 113 113 + checkpoint_interval_ms: 100, 114 114 + checkpoint_write_threshold: 10, 115 115 + ..GroupCommitConfig::default() 116 116 + }, 117 117 + shard_count: 1, 118 118 + } 119 119 + } 120 120 + 121 121 + pub fn compact_by_liveness(store: &TranquilBlockStore) { 122 122 + let liveness = store.compaction_liveness(0).unwrap(); 123 123 + liveness 124 124 + .iter() 125 125 + .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.99) 126 126 + .map(|(&fid, _)| fid) 127 127 + .collect::<Vec<_>>() 128 128 + .into_iter() 129 129 + .for_each(|fid| match store.compact_file(fid, 0) { 130 130 + Ok(_) => {} 131 131 + Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {} 132 132 + Err(e) => eprintln!("compaction: {e}"), 133 133 + }); 134 134 + } 135 135 + 136 136 + pub fn compact_lowest_liveness(store: &TranquilBlockStore) { 137 137 + let liveness = store.compaction_liveness(0).unwrap(); 138 138 + let candidate = liveness 139 139 + .iter() 140 140 + .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.99) 141 141 + .min_by(|(_, a), (_, b)| { 142 142 + a.ratio() 143 143 + .partial_cmp(&b.ratio()) 144 144 + .unwrap_or(std::cmp::Ordering::Equal) 145 145 + }) 146 146 + .map(|(&fid, _)| fid); 147 147 + 148 148 + if let Some(fid) = candidate { 149 149 + match store.compact_file(fid, 0) { 150 150 + Ok(_) => {} 151 151 + Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {} 152 152 + Err(e) => eprintln!("compaction: {e}"), 153 153 + } 154 154 + } 155 155 + } 156 156 + 157 157 + pub fn collect_refcounts(store: &TranquilBlockStore, cids: &[CidBytes]) -> Vec<(u32, u32)> { 158 158 + cids.iter() 159 159 + .map(|cid| { 160 160 + let seed = u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]]); 161 161 + let rc = store 162 162 + .block_index() 163 163 + .get(cid) 164 164 + .map(|e| e.refcount.raw()) 165 165 + .unwrap_or(0); 166 166 + (seed, rc) 167 167 + }) 168 168 + .collect() 169 169 + } 170 170 + 107 171 pub struct TestStores { 108 172 pub blockstore: TranquilBlockStore, 109 173 pub eventlog: Arc<EventLog<RealIO>>,

+567

crates/tranquil-store/tests/compaction_restart.rs

reviewed

··· 1 1 + mod common; 2 2 + 3 3 + use std::path::Path; 4 4 + use std::sync::Arc; 5 5 + 6 6 + use common::{ 7 7 + collect_refcounts, compact_by_liveness, compact_lowest_liveness, test_cid, 8 8 + tiny_blockstore_config, with_runtime, 9 9 + }; 10 10 + use tranquil_store::RealIO; 11 11 + use tranquil_store::blockstore::{CidBytes, TranquilBlockStore}; 12 12 + use tranquil_store::eventlog::{EventLog, EventLogBridge, EventLogConfig}; 13 13 + use tranquil_store::metastore::handler::HandlerPool; 14 14 + use tranquil_store::metastore::partitions::Partition; 15 15 + use tranquil_store::metastore::{Metastore, MetastoreConfig}; 16 16 + 17 17 + struct FullStack { 18 18 + blockstore: TranquilBlockStore, 19 19 + _pool: Arc<HandlerPool>, 20 20 + _event_log: Arc<EventLog<RealIO>>, 21 21 + } 22 22 + 23 23 + fn open_full_stack(base_dir: &Path) -> FullStack { 24 24 + let metastore_dir = base_dir.join("metastore"); 25 25 + let segments_dir = base_dir.join("eventlog").join("segments"); 26 26 + let blockstore_data = base_dir.join("blockstore").join("data"); 27 27 + let blockstore_index = base_dir.join("blockstore").join("index"); 28 28 + 29 29 + [&metastore_dir, &segments_dir, &blockstore_data, &blockstore_index] 30 30 + .iter() 31 31 + .for_each(|d| std::fs::create_dir_all(d).unwrap()); 32 32 + 33 33 + let metastore = Metastore::open(&metastore_dir, MetastoreConfig::default()).unwrap(); 34 34 + 35 35 + let blockstore = TranquilBlockStore::open(tranquil_store::blockstore::BlockStoreConfig { 36 36 + data_dir: blockstore_data, 37 37 + index_dir: blockstore_index, 38 38 + max_file_size: 512, 39 39 + group_commit: tranquil_store::blockstore::GroupCommitConfig::default(), 40 40 + shard_count: 1, 41 41 + }) 42 42 + .unwrap(); 43 43 + 44 44 + let event_log = Arc::new( 45 45 + EventLog::open( 46 46 + EventLogConfig { 47 47 + segments_dir, 48 48 + ..EventLogConfig::default() 49 49 + }, 50 50 + RealIO::new(), 51 51 + ) 52 52 + .unwrap(), 53 53 + ); 54 54 + 55 55 + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); 56 56 + 57 57 + let was_clean = tranquil_store::consistency::had_clean_shutdown(base_dir); 58 58 + tranquil_store::consistency::remove_clean_shutdown_marker(base_dir).ok(); 59 59 + 60 60 + let indexes = metastore.partition(Partition::Indexes).clone(); 61 61 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 62 62 + let recovered = event_ops 63 63 + .recover_metastore_mutations(&indexes) 64 64 + .unwrap(); 65 65 + if recovered > 0 { 66 66 + eprintln!("replayed {recovered} metastore mutations from eventlog"); 67 67 + } 68 68 + 69 69 + if !was_clean || recovered > 0 { 70 70 + let report = tranquil_store::consistency::verify_store_consistency( 71 71 + &blockstore, 72 72 + &metastore, 73 73 + &event_log, 74 74 + ); 75 75 + report.log_findings(); 76 76 + if report.has_repairable_issues() { 77 77 + let repair = tranquil_store::consistency::repair_known_issues(&blockstore, &report); 78 78 + if repair.orphan_files_removed > 0 { 79 79 + eprintln!("removed {} orphan files", repair.orphan_files_removed); 80 80 + } 81 81 + } 82 82 + } 83 83 + 84 84 + let pool = Arc::new(HandlerPool::spawn::<RealIO>( 85 85 + metastore, 86 86 + bridge, 87 87 + Some(blockstore.clone()), 88 88 + None, 89 89 + )); 90 90 + 91 91 + FullStack { 92 92 + blockstore, 93 93 + _pool: pool, 94 94 + _event_log: event_log, 95 95 + } 96 96 + } 97 97 + 98 98 + fn close_full_stack(stack: FullStack, base_dir: &Path) { 99 99 + let rt = tokio::runtime::Handle::current(); 100 100 + rt.block_on(stack._pool.close()); 101 101 + if let Err(e) = stack._event_log.shutdown() { 102 102 + eprintln!("eventlog shutdown: {e}"); 103 103 + } 104 104 + tranquil_store::consistency::write_clean_shutdown_marker(base_dir).ok(); 105 105 + drop(stack.blockstore); 106 106 + } 107 107 + 108 108 + fn verify_blocks_and_refcounts( 109 109 + store: &TranquilBlockStore, 110 110 + live_cids: &[CidBytes], 111 111 + expected_refcounts: Option<&[(u32, u32)]>, 112 112 + label: &str, 113 113 + ) { 114 114 + let missing: Vec<u32> = live_cids 115 115 + .iter() 116 116 + .filter(|cid| store.get_block_sync(cid).unwrap().is_none()) 117 117 + .map(|cid| u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]])) 118 118 + .collect(); 119 119 + 120 120 + assert!( 121 121 + missing.is_empty(), 122 122 + "{label}: live blocks missing after reopen: {missing:?}" 123 123 + ); 124 124 + 125 125 + match expected_refcounts { 126 126 + Some(expected) => { 127 127 + let actual = collect_refcounts(store, live_cids); 128 128 + let mismatches: Vec<_> = expected 129 129 + .iter() 130 130 + .zip(actual.iter()) 131 131 + .filter(|((_, exp_rc), (_, act_rc))| exp_rc != act_rc) 132 132 + .map(|((seed, exp), (_, act))| format!("seed {seed}: before={exp} after={act}")) 133 133 + .collect(); 134 134 + 135 135 + assert!( 136 136 + mismatches.is_empty(), 137 137 + "{label}: refcounts changed across reopen:\n{}", 138 138 + mismatches.join("\n"), 139 139 + ); 140 140 + } 141 141 + None => { 142 142 + live_cids.iter().for_each(|cid| { 143 143 + let rc = store 144 144 + .block_index() 145 145 + .get(cid) 146 146 + .map(|e| e.refcount.raw()) 147 147 + .unwrap_or(0); 148 148 + assert!( 149 149 + rc > 0, 150 150 + "{label}: refcount dropped to 0 for seed {}", 151 151 + u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]]) 152 152 + ); 153 153 + }); 154 154 + } 155 155 + } 156 156 + } 157 157 + 158 158 + #[test] 159 159 + fn hundreds_of_compaction_cycles() { 160 160 + with_runtime(|| { 161 161 + let dir = tempfile::TempDir::new().unwrap(); 162 162 + 163 163 + let live_cids: Vec<CidBytes> = (0..15u32).map(test_cid).collect(); 164 164 + 165 165 + { 166 166 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 167 167 + 168 168 + live_cids.iter().for_each(|cid| { 169 169 + store 170 170 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 171 171 + .unwrap(); 172 172 + }); 173 173 + 174 174 + (0..500u32).for_each(|round| { 175 175 + let churn = test_cid(2000 + round); 176 176 + store 177 177 + .put_blocks_blocking(vec![(churn, vec![0xDD; 80])]) 178 178 + .unwrap(); 179 179 + store.apply_commit_blocking(vec![], vec![churn]).unwrap(); 180 180 + 181 181 + if round % 3 == 0 { 182 182 + compact_lowest_liveness(&store); 183 183 + } 184 184 + }); 185 185 + 186 186 + (0..200).for_each(|_| compact_by_liveness(&store)); 187 187 + 188 188 + live_cids.iter().for_each(|cid| { 189 189 + assert!( 190 190 + store.get_block_sync(cid).unwrap().is_some(), 191 191 + "sanity: block present before drop" 192 192 + ); 193 193 + }); 194 194 + 195 195 + drop(store); 196 196 + } 197 197 + 198 198 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 199 199 + verify_blocks_and_refcounts(&store, &live_cids, None, "500 churn + 200 compact rounds"); 200 200 + }); 201 201 + } 202 202 + 203 203 + #[test] 204 204 + fn commit_style_decrements() { 205 205 + with_runtime(|| { 206 206 + let dir = tempfile::TempDir::new().unwrap(); 207 207 + 208 208 + let shared_nodes: Vec<CidBytes> = (0..10u32).map(test_cid).collect(); 209 209 + 210 210 + let refcounts_before = { 211 211 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 212 212 + 213 213 + shared_nodes.iter().for_each(|cid| { 214 214 + store 215 215 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 216 216 + .unwrap(); 217 217 + }); 218 218 + 219 219 + let mut prev_commit = test_cid(5000); 220 220 + store 221 221 + .put_blocks_blocking(vec![(prev_commit, vec![0xCC; 80])]) 222 222 + .unwrap(); 223 223 + 224 224 + (0..500u32).for_each(|round| { 225 225 + let new_commit = test_cid(5001 + round); 226 226 + let new_mst_node = test_cid(6000 + round); 227 227 + let old_mst_node = test_cid(7000 + round); 228 228 + 229 229 + store 230 230 + .put_blocks_blocking(vec![ 231 231 + (new_commit, vec![0xBB; 80]), 232 232 + (new_mst_node, vec![0xCC; 60]), 233 233 + (old_mst_node, vec![0xDD; 60]), 234 234 + ]) 235 235 + .unwrap(); 236 236 + 237 237 + store 238 238 + .apply_commit_blocking(vec![], vec![prev_commit, old_mst_node]) 239 239 + .unwrap(); 240 240 + 241 241 + if round > 0 { 242 242 + let prev_mst = test_cid(6000 + round - 1); 243 243 + store 244 244 + .apply_commit_blocking(vec![], vec![prev_mst]) 245 245 + .unwrap(); 246 246 + } 247 247 + 248 248 + prev_commit = new_commit; 249 249 + 250 250 + if round % 2 == 0 { 251 251 + compact_lowest_liveness(&store); 252 252 + } 253 253 + }); 254 254 + 255 255 + (0..300).for_each(|_| { 256 256 + compact_by_liveness(&store); 257 257 + std::thread::sleep(std::time::Duration::from_millis(1)); 258 258 + }); 259 259 + 260 260 + shared_nodes.iter().for_each(|cid| { 261 261 + assert!( 262 262 + store.get_block_sync(cid).unwrap().is_some(), 263 263 + "sanity: shared node present before drop" 264 264 + ); 265 265 + }); 266 266 + 267 267 + let rc = collect_refcounts(&store, &shared_nodes); 268 268 + drop(store); 269 269 + rc 270 270 + }; 271 271 + 272 272 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 273 273 + verify_blocks_and_refcounts( 274 274 + &store, 275 275 + &shared_nodes, 276 276 + Some(&refcounts_before), 277 277 + "500 commits + 300 compact rounds", 278 278 + ); 279 279 + }); 280 280 + } 281 281 + 282 282 + #[test] 283 283 + fn extreme_file_churn_with_dedup_hits() { 284 284 + with_runtime(|| { 285 285 + let dir = tempfile::TempDir::new().unwrap(); 286 286 + 287 287 + let live_cids: Vec<CidBytes> = (0..8u32).map(test_cid).collect(); 288 288 + let live_data: Vec<u8> = vec![0xAA; 80]; 289 289 + 290 290 + let refcounts_before = { 291 291 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 292 292 + 293 293 + live_cids.iter().for_each(|cid| { 294 294 + store 295 295 + .put_blocks_blocking(vec![(*cid, live_data.clone())]) 296 296 + .unwrap(); 297 297 + }); 298 298 + 299 299 + (0..300u32).for_each(|round| { 300 300 + let churn = test_cid(3000 + round); 301 301 + store 302 302 + .put_blocks_blocking(vec![(churn, vec![0xEE; 80])]) 303 303 + .unwrap(); 304 304 + store.apply_commit_blocking(vec![], vec![churn]).unwrap(); 305 305 + 306 306 + if round % 50 == 0 { 307 307 + live_cids.iter().for_each(|cid| { 308 308 + store 309 309 + .put_blocks_blocking(vec![(*cid, live_data.clone())]) 310 310 + .unwrap(); 311 311 + }); 312 312 + } 313 313 + 314 314 + if round % 2 == 0 { 315 315 + compact_lowest_liveness(&store); 316 316 + } 317 317 + }); 318 318 + 319 319 + (0..200).for_each(|_| compact_by_liveness(&store)); 320 320 + 321 321 + let rc = collect_refcounts(&store, &live_cids); 322 322 + drop(store); 323 323 + rc 324 324 + }; 325 325 + 326 326 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 327 327 + verify_blocks_and_refcounts( 328 328 + &store, 329 329 + &live_cids, 330 330 + Some(&refcounts_before), 331 331 + "300 churn + dedup re-puts + 200 compacts", 332 332 + ); 333 333 + }); 334 334 + } 335 335 + 336 336 + #[test] 337 337 + fn long_idle_compaction_only_phase() { 338 338 + with_runtime(|| { 339 339 + let dir = tempfile::TempDir::new().unwrap(); 340 340 + 341 341 + let live_cids: Vec<CidBytes> = (0..20u32).map(test_cid).collect(); 342 342 + 343 343 + { 344 344 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 345 345 + 346 346 + live_cids.iter().for_each(|cid| { 347 347 + store 348 348 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 349 349 + .unwrap(); 350 350 + }); 351 351 + 352 352 + (0..100u32).for_each(|round| { 353 353 + let churn = test_cid(4000 + round); 354 354 + store 355 355 + .put_blocks_blocking(vec![(churn, vec![0xFF; 80])]) 356 356 + .unwrap(); 357 357 + store.apply_commit_blocking(vec![], vec![churn]).unwrap(); 358 358 + }); 359 359 + 360 360 + (0..500).for_each(|_| { 361 361 + compact_by_liveness(&store); 362 362 + std::thread::sleep(std::time::Duration::from_millis(1)); 363 363 + }); 364 364 + 365 365 + live_cids.iter().for_each(|cid| { 366 366 + assert!( 367 367 + store.get_block_sync(cid).unwrap().is_some(), 368 368 + "sanity: block present before drop" 369 369 + ); 370 370 + }); 371 371 + 372 372 + drop(store); 373 373 + } 374 374 + 375 375 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 376 376 + verify_blocks_and_refcounts( 377 377 + &store, 378 378 + &live_cids, 379 379 + None, 380 380 + "idle with 500 compaction-only rounds", 381 381 + ); 382 382 + }); 383 383 + } 384 384 + 385 385 + #[test] 386 386 + fn multiple_restart_cycles_blockstore() { 387 387 + with_runtime(|| { 388 388 + let dir = tempfile::TempDir::new().unwrap(); 389 389 + 390 390 + let live_cids: Vec<CidBytes> = (0..10u32).map(test_cid).collect(); 391 391 + 392 392 + { 393 393 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 394 394 + live_cids.iter().for_each(|cid| { 395 395 + store 396 396 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 397 397 + .unwrap(); 398 398 + }); 399 399 + (0..50u32).for_each(|round| { 400 400 + let churn = test_cid(8000 + round); 401 401 + store 402 402 + .put_blocks_blocking(vec![(churn, vec![0xBB; 80])]) 403 403 + .unwrap(); 404 404 + store.apply_commit_blocking(vec![], vec![churn]).unwrap(); 405 405 + }); 406 406 + drop(store); 407 407 + } 408 408 + 409 409 + (0..10u32).for_each(|cycle| { 410 410 + { 411 411 + let store = 412 412 + TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 413 413 + 414 414 + (0..50u32).for_each(|round| { 415 415 + let churn = test_cid(9000 + cycle * 100 + round); 416 416 + store 417 417 + .put_blocks_blocking(vec![(churn, vec![0xCC; 80])]) 418 418 + .unwrap(); 419 419 + store.apply_commit_blocking(vec![], vec![churn]).unwrap(); 420 420 + compact_lowest_liveness(&store); 421 421 + }); 422 422 + 423 423 + (0..50).for_each(|_| compact_by_liveness(&store)); 424 424 + 425 425 + live_cids.iter().for_each(|cid| { 426 426 + assert!( 427 427 + store.get_block_sync(cid).unwrap().is_some(), 428 428 + "cycle {cycle}: block missing before drop" 429 429 + ); 430 430 + }); 431 431 + 432 432 + drop(store); 433 433 + } 434 434 + 435 435 + let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(); 436 436 + verify_blocks_and_refcounts( 437 437 + &store, 438 438 + &live_cids, 439 439 + None, 440 440 + &format!("blockstore restart cycle {cycle}"), 441 441 + ); 442 442 + }); 443 443 + }); 444 444 + } 445 445 + 446 446 + #[test] 447 447 + fn full_stack_compaction_restart_preserves_refcounts() { 448 448 + with_runtime(|| { 449 449 + let base = tempfile::TempDir::new().unwrap(); 450 450 + let base_dir = base.path().to_path_buf(); 451 451 + 452 452 + let live_cids: Vec<CidBytes> = (0..15u32).map(test_cid).collect(); 453 453 + 454 454 + let refcounts_before = { 455 455 + let stack = open_full_stack(&base_dir); 456 456 + 457 457 + live_cids.iter().for_each(|cid| { 458 458 + stack 459 459 + .blockstore 460 460 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 461 461 + .unwrap(); 462 462 + }); 463 463 + 464 464 + (0..500u32).for_each(|round| { 465 465 + let churn = test_cid(2000 + round); 466 466 + stack 467 467 + .blockstore 468 468 + .put_blocks_blocking(vec![(churn, vec![0xDD; 80])]) 469 469 + .unwrap(); 470 470 + stack 471 471 + .blockstore 472 472 + .apply_commit_blocking(vec![], vec![churn]) 473 473 + .unwrap(); 474 474 + 475 475 + if round % 3 == 0 { 476 476 + compact_lowest_liveness(&stack.blockstore); 477 477 + } 478 478 + }); 479 479 + 480 480 + (0..200).for_each(|_| compact_by_liveness(&stack.blockstore)); 481 481 + 482 482 + let rc = collect_refcounts(&stack.blockstore, &live_cids); 483 483 + 484 484 + live_cids.iter().for_each(|cid| { 485 485 + assert!( 486 486 + stack.blockstore.get_block_sync(cid).unwrap().is_some(), 487 487 + "sanity: block present before shutdown" 488 488 + ); 489 489 + }); 490 490 + 491 491 + close_full_stack(stack, &base_dir); 492 492 + rc 493 493 + }; 494 494 + 495 495 + let stack = open_full_stack(&base_dir); 496 496 + verify_blocks_and_refcounts( 497 497 + &stack.blockstore, 498 498 + &live_cids, 499 499 + Some(&refcounts_before), 500 500 + "full stack restart", 501 501 + ); 502 502 + close_full_stack(stack, &base_dir); 503 503 + }); 504 504 + } 505 505 + 506 506 + #[test] 507 507 + fn full_stack_multiple_restart_cycles() { 508 508 + with_runtime(|| { 509 509 + let base = tempfile::TempDir::new().unwrap(); 510 510 + let base_dir = base.path().to_path_buf(); 511 511 + 512 512 + let live_cids: Vec<CidBytes> = (0..10u32).map(test_cid).collect(); 513 513 + 514 514 + { 515 515 + let stack = open_full_stack(&base_dir); 516 516 + live_cids.iter().for_each(|cid| { 517 517 + stack 518 518 + .blockstore 519 519 + .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])]) 520 520 + .unwrap(); 521 521 + }); 522 522 + close_full_stack(stack, &base_dir); 523 523 + } 524 524 + 525 525 + (0..10u32).for_each(|cycle| { 526 526 + let refcounts_before = { 527 527 + let stack = open_full_stack(&base_dir); 528 528 + 529 529 + (0..50u32).for_each(|round| { 530 530 + let churn = test_cid(5000 + cycle * 100 + round); 531 531 + stack 532 532 + .blockstore 533 533 + .put_blocks_blocking(vec![(churn, vec![0xBB; 80])]) 534 534 + .unwrap(); 535 535 + stack 536 536 + .blockstore 537 537 + .apply_commit_blocking(vec![], vec![churn]) 538 538 + .unwrap(); 539 539 + compact_lowest_liveness(&stack.blockstore); 540 540 + }); 541 541 + 542 542 + (0..30).for_each(|_| compact_by_liveness(&stack.blockstore)); 543 543 + 544 544 + let rc = collect_refcounts(&stack.blockstore, &live_cids); 545 545 + 546 546 + live_cids.iter().for_each(|cid| { 547 547 + assert!( 548 548 + stack.blockstore.get_block_sync(cid).unwrap().is_some(), 549 549 + "cycle {cycle}: block missing before shutdown" 550 550 + ); 551 551 + }); 552 552 + 553 553 + close_full_stack(stack, &base_dir); 554 554 + rc 555 555 + }; 556 556 + 557 557 + let stack = open_full_stack(&base_dir); 558 558 + verify_blocks_and_refcounts( 559 559 + &stack.blockstore, 560 560 + &live_cids, 561 561 + Some(&refcounts_before), 562 562 + &format!("full stack cycle {cycle}"), 563 563 + ); 564 564 + close_full_stack(stack, &base_dir); 565 565 + }); 566 566 + }); 567 567 + }

+6 -6

crates/tranquil-store/tests/mst_integration.rs

reviewed

··· 268 268 assert_eq!(&retrieved_a_v2.unwrap()[..], &record_a_v2); 269 269 270 270 assert!( 271 271 - store.has(&cid_a_v1).await.unwrap(), 272 272 - "cid_a_v1 should still exist, tombstoned but not GC'd" 271 271 + store.get(&cid_a_v1).await.unwrap().is_some(), 272 272 + "cid_a_v1 data should still exist, tombstoned but not GC'd" 273 273 ); 274 274 assert!( 275 275 - store.has(&cid_b).await.unwrap(), 276 276 - "cid_b should still exist, tombstoned but not GC'd" 275 275 + store.get(&cid_b).await.unwrap().is_some(), 276 276 + "cid_b data should still exist, tombstoned but not GC'd" 277 277 ); 278 278 279 279 assert!( ··· 284 284 assert_eq!(&retrieved_c[..], &record_c); 285 285 286 286 assert!( 287 287 - store.has(&cid_shared).await.unwrap(), 288 288 - "shared-content block should still exist, tombstoned but not GC'd" 287 287 + store.get(&cid_shared).await.unwrap().is_some(), 288 288 + "shared-content block data should still exist, tombstoned but not GC'd" 289 289 ); 290 290 291 291 let loaded_mst = Mst::load(storage.clone(), mst_root_v2, None);

+217

crates/tranquil-store/tests/mst_refcount_integrity.rs

reviewed

··· 1 1 + mod common; 2 2 + 3 3 + use std::sync::Arc; 4 4 + 5 5 + use cid::Cid; 6 6 + use common::{compact_by_liveness, tiny_blockstore_config}; 7 7 + use jacquard_repo::mst::Mst; 8 8 + use jacquard_repo::storage::BlockStore; 9 9 + use tranquil_store::blockstore::TranquilBlockStore; 10 10 + 11 11 + fn cid_to_fixed(cid: &Cid) -> [u8; 36] { 12 12 + let bytes = cid.to_bytes(); 13 13 + let mut arr = [0u8; 36]; 14 14 + arr.copy_from_slice(&bytes[..36]); 15 15 + arr 16 16 + } 17 17 + 18 18 + fn make_record_bytes(seed: u32) -> Vec<u8> { 19 19 + serde_ipld_dagcbor::to_vec(&serde_json::json!({ 20 20 + "$type": "app.bsky.feed.post", 21 21 + "text": format!("record {seed}"), 22 22 + "createdAt": "2026-01-01T00:00:00Z" 23 23 + })) 24 24 + .unwrap() 25 25 + } 26 26 + 27 27 + fn make_fake_commit_cid(counter: u32) -> Cid { 28 28 + let data = format!("commit-{counter}"); 29 29 + let mh = multihash::Multihash::wrap(0x12, &{ 30 30 + use sha2::Digest; 31 31 + sha2::Sha256::digest(data.as_bytes()) 32 32 + }) 33 33 + .unwrap(); 34 34 + Cid::new_v1(0x71, mh) 35 35 + } 36 36 + 37 37 + async fn compute_obsolete_from_diff<S: BlockStore + Sync + Send + 'static>( 38 38 + old_mst: &Mst<S>, 39 39 + new_mst: &Mst<S>, 40 40 + old_commit_cid: Cid, 41 41 + ) -> Vec<Cid> { 42 42 + let diff = old_mst.diff(new_mst).await.unwrap(); 43 43 + std::iter::once(old_commit_cid) 44 44 + .chain(diff.removed_mst_blocks.into_iter()) 45 45 + .chain(diff.removed_cids.into_iter()) 46 46 + .collect() 47 47 + } 48 48 + 49 49 + #[tokio::test] 50 50 + async fn mst_shared_subtrees_survive_incremental_writes_compaction_restart() { 51 51 + let dir = tempfile::TempDir::new().unwrap(); 52 52 + 53 53 + let mut commit_counter = 0u32; 54 54 + let final_node_cids: Vec<Cid>; 55 55 + 56 56 + { 57 57 + let store = Arc::new(TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap()); 58 58 + 59 59 + let mut mst = Mst::new(store.clone()); 60 60 + let mut root: Option<Cid> = None; 61 61 + let mut prev_commit = make_fake_commit_cid(commit_counter); 62 62 + commit_counter += 1; 63 63 + 64 64 + for i in 0..30u32 { 65 65 + let record_bytes = make_record_bytes(i); 66 66 + let record_cid = store.put(&record_bytes).await.unwrap(); 67 67 + let key = format!("app.bsky.feed.post/{i:06}"); 68 68 + mst = match root { 69 69 + None => mst.add(&key, record_cid).await.unwrap(), 70 70 + Some(r) => { 71 71 + let loaded = Mst::load(store.clone(), r, None); 72 72 + loaded.add(&key, record_cid).await.unwrap() 73 73 + } 74 74 + }; 75 75 + let new_root = mst.persist().await.unwrap(); 76 76 + 77 77 + if let Some(old_root) = root { 78 78 + let old_settled = Mst::load(store.clone(), old_root, None); 79 79 + let new_settled = Mst::load(store.clone(), new_root, None); 80 80 + 81 81 + let obsolete = 82 82 + compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await; 83 83 + let obsolete_fixed: Vec<[u8; 36]> = 84 84 + obsolete.iter().map(|c| cid_to_fixed(c)).collect(); 85 85 + let s = store.clone(); 86 86 + tokio::task::spawn_blocking(move || { 87 87 + s.apply_commit_blocking(vec![], obsolete_fixed).unwrap(); 88 88 + }) 89 89 + .await 90 90 + .unwrap(); 91 91 + } 92 92 + 93 93 + root = Some(new_root); 94 94 + prev_commit = make_fake_commit_cid(commit_counter); 95 95 + commit_counter += 1; 96 96 + 97 97 + if i % 5 == 0 { 98 98 + let s = store.clone(); 99 99 + tokio::task::spawn_blocking(move || compact_by_liveness(&s)) 100 100 + .await 101 101 + .unwrap(); 102 102 + } 103 103 + } 104 104 + 105 105 + for i in 0..15u32 { 106 106 + let record_bytes = make_record_bytes(1000 + i); 107 107 + let record_cid = store.put(&record_bytes).await.unwrap(); 108 108 + let key = format!("app.bsky.feed.like/{i:06}"); 109 109 + let loaded = Mst::load(store.clone(), root.unwrap(), None); 110 110 + mst = loaded.add(&key, record_cid).await.unwrap(); 111 111 + let new_root = mst.persist().await.unwrap(); 112 112 + 113 113 + let old_settled = Mst::load(store.clone(), root.unwrap(), None); 114 114 + let new_settled = Mst::load(store.clone(), new_root, None); 115 115 + 116 116 + let obsolete = 117 117 + compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await; 118 118 + let obsolete_fixed: Vec<[u8; 36]> = 119 119 + obsolete.iter().map(|c| cid_to_fixed(c)).collect(); 120 120 + let s = store.clone(); 121 121 + tokio::task::spawn_blocking(move || { 122 122 + s.apply_commit_blocking(vec![], obsolete_fixed).unwrap(); 123 123 + }) 124 124 + .await 125 125 + .unwrap(); 126 126 + 127 127 + root = Some(new_root); 128 128 + prev_commit = make_fake_commit_cid(commit_counter); 129 129 + commit_counter += 1; 130 130 + 131 131 + if i % 3 == 0 { 132 132 + let s = store.clone(); 133 133 + tokio::task::spawn_blocking(move || compact_by_liveness(&s)) 134 134 + .await 135 135 + .unwrap(); 136 136 + } 137 137 + } 138 138 + 139 139 + let final_settled = Mst::load(store.clone(), root.unwrap(), None); 140 140 + final_node_cids = final_settled.collect_node_cids().await.unwrap(); 141 141 + 142 142 + final_node_cids.iter().for_each(|cid| { 143 143 + let fixed = cid_to_fixed(cid); 144 144 + let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw()); 145 145 + assert!( 146 146 + rc.is_some_and(|r| r > 0), 147 147 + "MST node {cid} has refcount {rc:?} before shutdown" 148 148 + ); 149 149 + }); 150 150 + 151 151 + let s = store.clone(); 152 152 + tokio::task::spawn_blocking(move || { 153 153 + (0..100).for_each(|_| compact_by_liveness(&s)); 154 154 + }) 155 155 + .await 156 156 + .unwrap(); 157 157 + 158 158 + final_node_cids.iter().for_each(|cid| { 159 159 + let fixed = cid_to_fixed(cid); 160 160 + let block = store.get_block_sync(&fixed).unwrap(); 161 161 + assert!( 162 162 + block.is_some(), 163 163 + "MST node {cid} missing after compaction before shutdown" 164 164 + ); 165 165 + }); 166 166 + 167 167 + drop(store); 168 168 + } 169 169 + 170 170 + { 171 171 + let store = Arc::new( 172 172 + TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap(), 173 173 + ); 174 174 + 175 175 + let missing: Vec<String> = final_node_cids 176 176 + .iter() 177 177 + .filter_map(|cid| { 178 178 + let fixed = cid_to_fixed(cid); 179 179 + match store.get_block_sync(&fixed) { 180 180 + Ok(Some(_)) => None, 181 181 + Ok(None) => { 182 182 + let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw()); 183 183 + Some(format!("{cid} missing, index refcount {rc:?}")) 184 184 + } 185 185 + Err(e) => Some(format!("{cid} error: {e}")), 186 186 + } 187 187 + }) 188 188 + .collect(); 189 189 + 190 190 + assert!( 191 191 + missing.is_empty(), 192 192 + "{} of {} MST nodes missing after reopen:\n{}", 193 193 + missing.len(), 194 194 + final_node_cids.len(), 195 195 + missing.join("\n"), 196 196 + ); 197 197 + 198 198 + let refcount_issues: Vec<String> = final_node_cids 199 199 + .iter() 200 200 + .filter_map(|cid| { 201 201 + let fixed = cid_to_fixed(cid); 202 202 + let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw()); 203 203 + match rc { 204 204 + Some(0) => Some(format!("{cid} refcount dropped to 0")), 205 205 + None => Some(format!("{cid} not in index")), 206 206 + _ => None, 207 207 + } 208 208 + }) 209 209 + .collect(); 210 210 + 211 211 + assert!( 212 212 + refcount_issues.is_empty(), 213 213 + "MST nodes with bad refcounts after reopen:\n{}", 214 214 + refcount_issues.join("\n"), 215 215 + ); 216 216 + } 217 217 + }