bas.sh / tranquil-pds
forked from tranquil.farm/tranquil-pds
Our Personal Data Server from scratch!
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(tranquil-store): metastore

Lewis: May this revision serve well! <lu5a@proton.me>

lu5a.myatproto.social a5c68a35 9ea33def

+16813 -92
+22
.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json
··· 1 + { 2 + "db_name": "PostgreSQL", 3 + "query": "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1", 4 + "describe": { 5 + "columns": [ 6 + { 7 + "ordinal": 0, 8 + "name": "cid", 9 + "type_info": "Bytea" 10 + } 11 + ], 12 + "parameters": { 13 + "Left": [ 14 + "Int8" 15 + ] 16 + }, 17 + "nullable": [ 18 + false 19 + ] 20 + }, 21 + "hash": "12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b" 22 + }
+22
.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json
··· 1 + { 2 + "db_name": "PostgreSQL", 3 + "query": "\n SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)\n WHERE NOT EXISTS (\n SELECT 1 FROM user_blocks WHERE block_cid = t.cid\n )\n ", 4 + "describe": { 5 + "columns": [ 6 + { 7 + "ordinal": 0, 8 + "name": "cid", 9 + "type_info": "Bytea" 10 + } 11 + ], 12 + "parameters": { 13 + "Left": [ 14 + "ByteaArray" 15 + ] 16 + }, 17 + "nullable": [ 18 + null 19 + ] 20 + }, 21 + "hash": "18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8" 22 + }
+17
.sqlx/query-47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f.json
··· 1 + { 2 + "db_name": "PostgreSQL", 3 + "query": "\n INSERT INTO backlinks (uri, path, link_to, repo_id)\n SELECT unnest($1::text[]), unnest($2::text[]), unnest($3::text[]), $4\n ON CONFLICT (uri, path) DO NOTHING\n ", 4 + "describe": { 5 + "columns": [], 6 + "parameters": { 7 + "Left": [ 8 + "TextArray", 9 + "TextArray", 10 + "TextArray", 11 + "Uuid" 12 + ] 13 + }, 14 + "nullable": [] 15 + }, 16 + "hash": "47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f" 17 + }
+14
.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json
··· 1 + { 2 + "db_name": "PostgreSQL", 3 + "query": "DELETE FROM blocks WHERE cid = ANY($1)", 4 + "describe": { 5 + "columns": [], 6 + "parameters": { 7 + "Left": [ 8 + "ByteaArray" 9 + ] 10 + }, 11 + "nullable": [] 12 + }, 13 + "hash": "8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366" 14 + }
+14
.sqlx/query-cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49.json
··· 1 + { 2 + "db_name": "PostgreSQL", 3 + "query": "DELETE FROM backlinks WHERE uri = ANY($1::text[])", 4 + "describe": { 5 + "columns": [], 6 + "parameters": { 7 + "Left": [ 8 + "TextArray" 9 + ] 10 + }, 11 + "nullable": [] 12 + }, 13 + "hash": "cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49" 14 + }
+13
Cargo.lock
··· 6390 6390 ] 6391 6391 6392 6392 [[package]] 6393 + name = "siphasher" 6394 + version = "1.0.2" 6395 + source = "registry+https://github.com/rust-lang/crates.io-index" 6396 + checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" 6397 + 6398 + [[package]] 6393 6399 name = "sketches-ddsketch" 6394 6400 version = "0.3.1" 6395 6401 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 7705 7711 "tranquil-scopes", 7706 7712 "tranquil-signal", 7707 7713 "tranquil-storage", 7714 + "tranquil-store", 7708 7715 "tranquil-sync", 7709 7716 "tranquil-types", 7710 7717 "urlencoding", ··· 7833 7840 "bytes", 7834 7841 "chrono", 7835 7842 "cid", 7843 + "dashmap", 7836 7844 "fjall", 7837 7845 "flume 0.11.1", 7838 7846 "futures", 7839 7847 "jacquard-common", 7840 7848 "jacquard-repo", 7841 7849 "k256", 7850 + "lsm-tree", 7842 7851 "memmap2", 7843 7852 "multihash", 7844 7853 "parking_lot", ··· 7849 7858 "serde_ipld_dagcbor", 7850 7859 "serde_json", 7851 7860 "sha2", 7861 + "siphasher", 7862 + "smallvec", 7852 7863 "sqlx", 7853 7864 "tempfile", 7854 7865 "thiserror 2.0.18", 7855 7866 "tokio", 7856 7867 "tracing", 7868 + "tranquil-db", 7857 7869 "tranquil-db-traits", 7858 7870 "tranquil-repo", 7859 7871 "tranquil-types", 7872 + "uuid", 7860 7873 "xxhash-rust", 7861 7874 ] 7862 7875
+4
Cargo.toml
··· 144 144 strip = true 145 145 codegen-units = 1 146 146 panic = "abort" 147 + 148 + [profile.bench] 149 + debug = 1 150 + strip = false
+18 -8
crates/tranquil-api/src/admin/status.rs
··· 195 195 ApiError::InternalError(Some("Failed to update deactivation status".into())) 196 196 })?; 197 197 } 198 + let takedown_update = input.takedown.as_ref().map(|t| t.applied); 199 + let takedown_ref = input.takedown.as_ref().and_then(|t| t.r#ref.as_deref()); 200 + let deactivated_update = input.deactivated.as_ref().map(|d| d.applied); 201 + if (takedown_update.is_some() || deactivated_update.is_some()) 202 + && let Err(e) = state 203 + .repos 204 + .repo 205 + .update_repo_status(&did, takedown_update, takedown_ref, deactivated_update) 206 + .await 207 + { 208 + warn!("failed to sync status to repo backend: {e:?}"); 209 + } 198 210 if let Some(takedown) = &input.takedown { 199 - let status = if takedown.applied { 200 - tranquil_db_traits::AccountStatus::Takendown 201 - } else { 202 - tranquil_db_traits::AccountStatus::Active 211 + let status = match takedown.applied { 212 + true => tranquil_db_traits::AccountStatus::Takendown, 213 + false => tranquil_db_traits::AccountStatus::Active, 203 214 }; 204 215 if let Err(e) = 205 216 tranquil_pds::repo_ops::sequence_account_event(&state, &did, status).await ··· 208 219 } 209 220 } 210 221 if let Some(deactivated) = &input.deactivated { 211 - let status = if deactivated.applied { 212 - tranquil_db_traits::AccountStatus::Deactivated 213 - } else { 214 - tranquil_db_traits::AccountStatus::Active 222 + let status = match deactivated.applied { 223 + true => tranquil_db_traits::AccountStatus::Deactivated, 224 + false => tranquil_db_traits::AccountStatus::Active, 215 225 }; 216 226 if let Err(e) = 217 227 tranquil_pds::repo_ops::sequence_account_event(&state, &did, status).await
+17
crates/tranquil-api/src/delegation.rs
··· 18 18 use tranquil_pds::rate_limit::{AccountCreationLimit, RateLimited}; 19 19 use tranquil_pds::state::AppState; 20 20 use tranquil_pds::types::{Did, Handle}; 21 + use tranquil_types::CidLink; 21 22 22 23 pub async fn list_controllers( 23 24 State(state): State<AppState>, ··· 418 419 return Err(ApiError::InternalError(None)); 419 420 } 420 421 }; 422 + 423 + state 424 + .repos 425 + .repo 426 + .create_repo( 427 + user_id, 428 + &did, 429 + &handle, 430 + &CidLink::from(&repo.commit_cid), 431 + &repo.repo_rev, 432 + ) 433 + .await 434 + .map_err(|e| { 435 + error!("failed to register repo in backend: {e:?}"); 436 + ApiError::InternalError(None) 437 + })?; 421 438 422 439 if let Some(validated) = validated_invite_code 423 440 && let Err(e) = state
+18 -2
crates/tranquil-api/src/identity/account.rs
··· 15 15 use tranquil_pds::state::AppState; 16 16 use tranquil_pds::types::{Did, Handle, PlainPassword}; 17 17 use tranquil_pds::validation::validate_password; 18 + use tranquil_types::CidLink; 18 19 19 20 #[derive(Deserialize)] 20 21 #[serde(rename_all = "camelCase")] ··· 160 161 .fetch_did_document(did) 161 162 .await 162 163 .ok() 163 - .and_then(|f| Some((*f).clone())), 164 + .map(|f| (*f).clone()), 164 165 access_jwt: access_meta.token, 165 166 refresh_jwt: refresh_meta.token, 166 167 verification_required, ··· 547 548 } 548 549 }; 549 550 let user_id = create_result.user_id; 551 + if let Err(e) = state 552 + .repos 553 + .repo 554 + .create_repo( 555 + user_id, 556 + &did_for_commit, 557 + &handle_typed, 558 + &CidLink::from(&repo.commit_cid), 559 + &repo.repo_rev, 560 + ) 561 + .await 562 + { 563 + error!("failed to register repo in backend: {e:?}"); 564 + return ApiError::InternalError(None).into_response(); 565 + } 550 566 if !is_migration && !is_did_web_byod { 551 567 super::provision::sequence_new_account( 552 568 &state, ··· 607 623 Json(CreateAccountOutput { 608 624 handle: handle.clone().into(), 609 625 did: did_for_commit, 610 - did_doc: did_doc.and_then(|f| Some((*f).clone())), 626 + did_doc: did_doc.map(|f| (*f).clone()), 611 627 access_jwt: session.access_jwt, 612 628 refresh_jwt: session.refresh_jwt, 613 629 verification_required: !is_migration,
+10
crates/tranquil-api/src/repo/import.rs
··· 195 195 } 196 196 let max_blocks = tranquil_config::get().import.max_blocks as usize; 197 197 let _write_lock = state.repo_write_locks.lock(user_id).await; 198 + 199 + state 200 + .block_store 201 + .put_many(blocks.clone()) 202 + .await 203 + .map_err(|e| { 204 + error!("Failed to store import blocks: {:?}", e); 205 + ApiError::InternalError(None) 206 + })?; 207 + 198 208 match apply_import( 199 209 &state.repos.repo, 200 210 user_id,
+23 -1
crates/tranquil-api/src/repo/record/batch.rs
··· 6 6 use serde::{Deserialize, Serialize}; 7 7 use serde_json::json; 8 8 use tracing::info; 9 + use tranquil_db_traits::Backlink; 9 10 use tranquil_pds::api::error::{ApiError, DbResultExt}; 10 11 use tranquil_pds::auth::{ 11 12 Active, Auth, WriteOpKind, require_not_migrated, require_verified_or_delegated, ··· 13 14 }; 14 15 use tranquil_pds::repo::TrackingBlockStore; 15 16 use tranquil_pds::repo_ops::{ 16 - FinalizeParams, RecordOp, begin_repo_write, extract_blob_cids, finalize_repo_write, 17 + FinalizeParams, RecordOp, begin_repo_write, extract_backlinks, extract_blob_cids, 18 + finalize_repo_write, 17 19 }; 18 20 use tranquil_pds::state::AppState; 19 21 use tranquil_pds::types::{AtIdentifier, AtUri, Did, Nsid, Rkey}; ··· 27 29 ops: Vec<RecordOp>, 28 30 modified_keys: Vec<String>, 29 31 all_blob_cids: Vec<String>, 32 + backlinks_to_add: Vec<Backlink>, 33 + backlinks_to_remove: Vec<AtUri>, 30 34 } 31 35 32 36 async fn process_single_write( ··· 42 46 mut ops, 43 47 mut modified_keys, 44 48 mut all_blob_cids, 49 + mut backlinks_to_add, 50 + mut backlinks_to_remove, 45 51 } = acc; 46 52 47 53 match write { ··· 79 85 .await 80 86 .map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?; 81 87 let uri = AtUri::from_parts(did, collection, &rkey); 88 + backlinks_to_add.extend(extract_backlinks(&uri, value)); 82 89 results.push(WriteResult::CreateResult { 83 90 uri, 84 91 cid: record_cid.to_string(), ··· 95 102 ops, 96 103 modified_keys, 97 104 all_blob_cids, 105 + backlinks_to_add, 106 + backlinks_to_remove, 98 107 }) 99 108 } 100 109 WriteOp::Update { ··· 131 140 .await 132 141 .map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?; 133 142 let uri = AtUri::from_parts(did, collection, rkey); 143 + backlinks_to_remove.push(uri.clone()); 144 + backlinks_to_add.extend(extract_backlinks(&uri, value)); 134 145 results.push(WriteResult::UpdateResult { 135 146 uri, 136 147 cid: record_cid.to_string(), ··· 148 159 ops, 149 160 modified_keys, 150 161 all_blob_cids, 162 + backlinks_to_add, 163 + backlinks_to_remove, 151 164 }) 152 165 } 153 166 WriteOp::Delete { collection, rkey } => { ··· 158 171 .delete(&key) 159 172 .await 160 173 .map_err(|_| ApiError::InternalError(Some("Failed to delete from MST".into())))?; 174 + backlinks_to_remove.push(AtUri::from_parts(did, collection, rkey)); 161 175 results.push(WriteResult::DeleteResult {}); 162 176 ops.push(RecordOp::Delete { 163 177 collection: collection.clone(), ··· 170 184 ops, 171 185 modified_keys, 172 186 all_blob_cids, 187 + backlinks_to_add, 188 + backlinks_to_remove, 173 189 }) 174 190 } 175 191 } ··· 189 205 ops: Vec::new(), 190 206 modified_keys: Vec::new(), 191 207 all_blob_cids: Vec::new(), 208 + backlinks_to_add: Vec::new(), 209 + backlinks_to_remove: Vec::new(), 192 210 }; 193 211 stream::iter(writes.iter().map(Ok::<_, ApiError>)) 194 212 .try_fold(initial_acc, |acc, write| async move { ··· 319 337 ops, 320 338 modified_keys, 321 339 all_blob_cids, 340 + backlinks_to_add, 341 + backlinks_to_remove, 322 342 } = process_writes( 323 343 &input.writes, 324 344 mst, ··· 373 393 ops, 374 394 modified_keys: &modified_keys, 375 395 blob_cids: &all_blob_cids, 396 + backlinks_to_add, 397 + backlinks_to_remove, 376 398 }, 377 399 ) 378 400 .await?;
+6 -10
crates/tranquil-api/src/repo/record/delete.rs
··· 76 76 }; 77 77 78 78 let modified_keys = [key]; 79 + let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); 79 80 80 81 let commit_result = finalize_repo_write( 81 82 &state, ··· 95 96 ops: vec![op], 96 97 modified_keys: &modified_keys, 97 98 blob_cids: &[], 99 + backlinks_to_add: vec![], 100 + backlinks_to_remove: vec![deleted_uri], 98 101 }, 99 102 ) 100 103 .await?; 101 - 102 - let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); 103 - if let Err(e) = state 104 - .repos 105 - .backlink 106 - .remove_backlinks_by_uri(&deleted_uri) 107 - .await 108 - { 109 - error!("Failed to remove backlinks for {}: {}", deleted_uri, e); 110 - } 111 104 112 105 Ok(Json(DeleteRecordOutput { 113 106 commit: Some(CommitInfo { ··· 216 209 217 210 let written_cids_str: Vec<String> = written_cids.iter().map(ToString::to_string).collect(); 218 211 212 + let deleted_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); 219 213 commit_and_log( 220 214 state, 221 215 CommitParams { ··· 228 222 blocks_cids: &written_cids_str, 229 223 blobs: &[], 230 224 obsolete_cids, 225 + backlinks_to_add: vec![], 226 + backlinks_to_remove: vec![deleted_uri], 231 227 }, 232 228 ) 233 229 .await?;
+1 -1
crates/tranquil-api/src/repo/record/read.rs
··· 178 178 }; 179 179 let records: Vec<Value> = parsed_rows 180 180 .iter() 181 - .zip(blocks.into_iter()) 181 + .zip(blocks) 182 182 .filter_map(|((_, rkey, cid_str), block_opt)| { 183 183 block_opt.and_then(|block| { 184 184 serde_ipld_dagcbor::from_slice::<Ipld>(&block)
+30 -33
crates/tranquil-api/src/repo/record/write.rs
··· 147 147 148 148 let prev_cid = match mst.get(&conflict_key).await { 149 149 Ok(Some(cid)) => cid, 150 - _ => continue, 151 - }; 152 - 153 - mst = match mst.delete(&conflict_key).await { 154 - Ok(m) => m, 150 + Ok(None) => continue, 155 151 Err(e) => { 156 152 error!( 157 - "Failed to delete conflict from MST {}: {:?}", 153 + "Failed to read conflict record from MST {}: {:?}", 158 154 conflict_uri, e 159 155 ); 160 - continue; 156 + return Err(ApiError::InternalError(Some( 157 + "Failed to read conflicting record from MST".into(), 158 + ))); 161 159 } 162 160 }; 161 + 162 + mst = mst.delete(&conflict_key).await.map_err(|e| { 163 + error!( 164 + "Failed to delete conflict from MST {}: {:?}", 165 + conflict_uri, e 166 + ); 167 + ApiError::InternalError(Some( 168 + "Failed to delete conflicting record from MST".into(), 169 + )) 170 + })?; 163 171 164 172 ops.push(RecordOp::Delete { 165 173 collection: conflict_collection, ··· 208 216 .collect(); 209 217 let blob_cids = extract_blob_cids(&input.record); 210 218 219 + let created_uri = AtUri::from_parts(&did, &input.collection, &rkey); 220 + let backlinks_to_add = extract_backlinks(&created_uri, &input.record); 221 + 211 222 let commit_result = finalize_repo_write( 212 223 &state, 213 224 ctx, ··· 226 237 ops, 227 238 modified_keys: &modified_keys, 228 239 blob_cids: &blob_cids, 240 + backlinks_to_add, 241 + backlinks_to_remove: conflict_uris_to_cleanup, 229 242 }, 230 243 ) 231 244 .await?; 232 245 233 - { 234 - let backlink_repo = state.repos.backlink.clone(); 235 - futures::future::join_all(conflict_uris_to_cleanup.iter().map(|uri| { 236 - let backlink_repo = backlink_repo.clone(); 237 - async move { 238 - if let Err(e) = backlink_repo.remove_backlinks_by_uri(uri).await { 239 - error!("Failed to remove backlinks for {}: {}", uri, e); 240 - } 241 - } 242 - })) 243 - .await; 244 - } 245 - 246 - let created_uri = AtUri::from_parts(&did, &input.collection, &rkey); 247 - let backlinks = extract_backlinks(&created_uri, &input.record); 248 - if !backlinks.is_empty() 249 - && let Err(e) = state 250 - .repos 251 - .backlink 252 - .add_backlinks(user_id, &backlinks) 253 - .await 254 - { 255 - error!("Failed to add backlinks for {}: {}", created_uri, e); 256 - } 257 - 258 246 Ok(Json(CreateRecordOutput { 259 247 uri: created_uri, 260 248 cid: record_cid.to_string(), ··· 379 367 let modified_keys = [key]; 380 368 let blob_cids = extract_blob_cids(&input.record); 381 369 370 + let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); 371 + let backlinks_to_add = extract_backlinks(&record_uri, &input.record); 372 + let backlinks_to_remove = match is_update { 373 + true => vec![record_uri.clone()], 374 + false => vec![], 375 + }; 376 + 382 377 let commit_result = finalize_repo_write( 383 378 &state, 384 379 ctx, ··· 397 392 ops: vec![op], 398 393 modified_keys: &modified_keys, 399 394 blob_cids: &blob_cids, 395 + backlinks_to_add, 396 + backlinks_to_remove, 400 397 }, 401 398 ) 402 399 .await?; 403 400 404 401 Ok(Json(PutRecordOutput { 405 - uri: AtUri::from_parts(&did, &input.collection, &input.rkey), 402 + uri: record_uri, 406 403 cid: record_cid.to_string(), 407 404 commit: Some(CommitInfo { 408 405 cid: commit_result.commit_cid.to_string(),
+16
crates/tranquil-api/src/server/account_status.rs
··· 382 382 did 383 383 ); 384 384 } 385 + if let Err(e) = state 386 + .repos 387 + .repo 388 + .update_repo_status(&did, None, None, Some(false)) 389 + .await 390 + { 391 + warn!("failed to sync activation to repo backend: {e:?}"); 392 + } 385 393 info!( 386 394 "[MIGRATION] activateAccount: Sequencing account event (active=true) for did={}", 387 395 did ··· 513 521 .cache 514 522 .delete(&tranquil_pds::cache_keys::handle_key(h)) 515 523 .await; 524 + } 525 + if let Err(e) = state 526 + .repos 527 + .repo 528 + .update_repo_status(&did, None, None, Some(true)) 529 + .await 530 + { 531 + warn!("failed to sync deactivation to repo backend: {e:?}"); 516 532 } 517 533 if let Err(e) = tranquil_pds::repo_ops::sequence_account_event( 518 534 &state,
+17
crates/tranquil-api/src/server/passkey_account.rs
··· 15 15 use tranquil_pds::state::AppState; 16 16 use tranquil_pds::types::{Did, Handle, PlainPassword}; 17 17 use tranquil_pds::validation::validate_password; 18 + use tranquil_types::CidLink; 18 19 19 20 fn generate_setup_token() -> String { 20 21 let mut rng = rand::thread_rng(); ··· 365 366 } 366 367 }; 367 368 let user_id = create_result.user_id; 369 + 370 + state 371 + .repos 372 + .repo 373 + .create_repo( 374 + user_id, 375 + &did_typed, 376 + &handle_typed, 377 + &CidLink::from(&repo.commit_cid), 378 + &repo.repo_rev, 379 + ) 380 + .await 381 + .map_err(|e| { 382 + error!("failed to register repo in backend: {e:?}"); 383 + ApiError::InternalError(None) 384 + })?; 368 385 369 386 if !is_byod_did_web { 370 387 crate::identity::provision::sequence_new_account(
+2 -2
crates/tranquil-api/src/server/password.rs
··· 153 153 } 154 154 return Err(ApiError::ExpiredToken(None)); 155 155 } 156 - let password_hash = crate::common::hash_password_async(&password).await?; 156 + let password_hash = crate::common::hash_password_async(password).await?; 157 157 let result = match state 158 158 .repos 159 159 .user ··· 345 345 )); 346 346 } 347 347 348 - let new_hash = crate::common::hash_password_async(&new_password).await?; 348 + let new_hash = crate::common::hash_password_async(new_password).await?; 349 349 350 350 state 351 351 .repos
+3 -3
crates/tranquil-api/src/server/session.rs
··· 313 313 refresh_jwt: refresh_meta.token, 314 314 handle, 315 315 did: row.did, 316 - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), 316 + did_doc: did_doc.ok().map(|f| (*f).clone()), 317 317 email: row.email, 318 318 email_confirmed: Some(row.channel_verification.email), 319 319 email_auth_factor: email_auth_factor_out, ··· 406 406 status: account_state.status_for_session().map(String::from), 407 407 migrated_to_pds, 408 408 migrated_at, 409 - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), 409 + did_doc: did_doc.ok().map(|f| (*f).clone()), 410 410 })) 411 411 } 412 412 Ok(None) => Err(ApiError::AuthenticationFailed(None)), ··· 604 604 preferred_locale: u.preferred_locale, 605 605 is_admin: u.is_admin, 606 606 active: account_state.is_active(), 607 - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), 607 + did_doc: did_doc.ok().map(|f| (*f).clone()), 608 608 status: account_state.status_for_session().map(String::from), 609 609 })) 610 610 }
+83
crates/tranquil-config/src/lib.rs
··· 143 143 144 144 #[config(nested)] 145 145 pub scheduled: ScheduledConfig, 146 + 147 + #[config(nested)] 148 + pub tranquil_store: TranquilStoreConfig, 146 149 } 147 150 148 151 impl TranquilConfig { ··· 248 251 must both be set or both be unset" 249 252 .to_string(), 250 253 ); 254 + } 255 + 256 + // -- repo backend ----------------------------------------------------- 257 + if let Err(e) = self.storage.repo_backend.parse::<RepoBackend>() { 258 + errors.push(e); 259 + } 260 + 261 + // -- tranquil-store --------------------------------------------------- 262 + if let Some(mb) = self.tranquil_store.memory_budget_mb 263 + && mb == 0 264 + { 265 + errors.push("tranquil_store.memory_budget_mb must be at least 1".to_string()); 266 + } 267 + if let Some(threads) = self.tranquil_store.handler_threads 268 + && threads == 0 269 + { 270 + errors.push("tranquil_store.handler_threads must be at least 1".to_string()); 251 271 } 252 272 253 273 // -- cache ------------------------------------------------------------ ··· 561 581 } 562 582 } 563 583 584 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 585 + pub enum RepoBackend { 586 + Postgres, 587 + TranquilStore, 588 + } 589 + 590 + impl std::str::FromStr for RepoBackend { 591 + type Err = String; 592 + 593 + fn from_str(s: &str) -> Result<Self, Self::Err> { 594 + match s { 595 + "postgres" => Ok(Self::Postgres), 596 + "tranquil-store" => Ok(Self::TranquilStore), 597 + other => Err(format!( 598 + "unknown repo backend \"{other}\", expected \"postgres\" or \"tranquil-store\"" 599 + )), 600 + } 601 + } 602 + } 603 + 604 + impl fmt::Display for RepoBackend { 605 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 606 + match self { 607 + Self::Postgres => f.write_str("postgres"), 608 + Self::TranquilStore => f.write_str("tranquil-store"), 609 + } 610 + } 611 + } 612 + 564 613 #[derive(Debug, Config)] 565 614 pub struct StorageConfig { 566 615 /// Storage backend: `filesystem` or `s3`. ··· 578 627 /// Custom S3 endpoint URL (for MinIO, R2, etc.). 579 628 #[config(env = "S3_ENDPOINT")] 580 629 pub s3_endpoint: Option<String>, 630 + 631 + #[config(env = "REPO_BACKEND", default = "postgres")] 632 + pub repo_backend: String, 633 + } 634 + 635 + impl StorageConfig { 636 + pub fn repo_backend(&self) -> RepoBackend { 637 + self.repo_backend 638 + .parse() 639 + .expect("repo_backend must be validated before use") 640 + } 581 641 } 582 642 583 643 #[derive(Debug, Config)] ··· 996 1056 /// Interval in seconds between scheduled delete checks. 997 1057 #[config(env = "SCHEDULED_DELETE_CHECK_INTERVAL_SECS", default = 3600)] 998 1058 pub delete_check_interval_secs: u64, 1059 + 1060 + /// Interval in seconds between block garbage collection cycles. 1061 + #[config(env = "BLOCK_GC_INTERVAL_SECS", default = 21600)] 1062 + pub block_gc_interval_secs: u64, 1063 + } 1064 + 1065 + #[derive(Debug, Config)] 1066 + pub struct TranquilStoreConfig { 1067 + /// Directory for tranquil-store data (metastore, eventlog). 1068 + #[config( 1069 + env = "TRANQUIL_STORE_DATA_DIR", 1070 + default = "/var/lib/tranquil-pds/store" 1071 + )] 1072 + pub data_dir: String, 1073 + 1074 + /// Fjall block cache size in megabytes. Defaults to 20% of system RAM 1075 + /// when unset. 1076 + #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")] 1077 + pub memory_budget_mb: Option<u64>, 1078 + 1079 + /// Number of handler threads. Defaults to available_parallelism / 2. 1080 + #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")] 1081 + pub handler_threads: Option<usize>, 999 1082 } 1000 1083 1001 1084 /// Generate a TOML configuration template with all available options,
+18
crates/tranquil-db-traits/src/repo.rs
··· 5 5 use uuid::Uuid; 6 6 7 7 use crate::DbError; 8 + use crate::backlink::Backlink; 8 9 use crate::sequence::SequenceNumber; 9 10 10 11 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, sqlx::Type)] ··· 279 280 pub obsolete_block_cids: Vec<Vec<u8>>, 280 281 pub record_upserts: Vec<RecordUpsert>, 281 282 pub record_deletes: Vec<RecordDelete>, 283 + pub backlinks_to_add: Vec<Backlink>, 284 + pub backlinks_to_remove: Vec<AtUri>, 282 285 pub commit_event: CommitEventData, 283 286 } 284 287 ··· 300 303 async fn create_repo( 301 304 &self, 302 305 user_id: Uuid, 306 + did: &Did, 307 + handle: &Handle, 303 308 repo_root_cid: &CidLink, 304 309 repo_rev: &str, 305 310 ) -> Result<(), DbError>; ··· 312 317 ) -> Result<(), DbError>; 313 318 314 319 async fn update_repo_rev(&self, user_id: Uuid, repo_rev: &str) -> Result<(), DbError>; 320 + 321 + async fn update_repo_status( 322 + &self, 323 + did: &Did, 324 + takedown: Option<bool>, 325 + takedown_ref: Option<&str>, 326 + deactivated: Option<bool>, 327 + ) -> Result<(), DbError>; 315 328 316 329 async fn delete_repo(&self, user_id: Uuid) -> Result<(), DbError>; 317 330 ··· 399 412 ) -> Result<Vec<Vec<u8>>, DbError>; 400 413 401 414 async fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, DbError>; 415 + 416 + async fn find_unreferenced_blocks( 417 + &self, 418 + candidate_cids: &[Vec<u8>], 419 + ) -> Result<Vec<Vec<u8>>, DbError>; 402 420 403 421 async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError>; 404 422
+4
crates/tranquil-db-traits/src/sequence.rs
··· 23 23 pub fn is_valid(&self) -> bool { 24 24 self.0 >= 0 25 25 } 26 + 27 + pub fn as_u64(&self) -> Option<u64> { 28 + u64::try_from(self.0).ok() 29 + } 26 30 } 27 31 28 32 impl fmt::Display for SequenceNumber {
+83
crates/tranquil-db/src/postgres/repo.rs
··· 46 46 47 47 #[async_trait] 48 48 impl RepoRepository for PostgresRepoRepository { 49 + async fn update_repo_status( 50 + &self, 51 + _did: &Did, 52 + _takedown: Option<bool>, 53 + _takedown_ref: Option<&str>, 54 + _deactivated: Option<bool>, 55 + ) -> Result<(), DbError> { 56 + Ok(()) 57 + } 58 + 49 59 async fn create_repo( 50 60 &self, 51 61 user_id: Uuid, 62 + _did: &Did, 63 + _handle: &Handle, 52 64 repo_root_cid: &CidLink, 53 65 repo_rev: &str, 54 66 ) -> Result<(), DbError> { ··· 604 616 .map_err(map_sqlx_error)?; 605 617 606 618 Ok(count) 619 + } 620 + 621 + async fn find_unreferenced_blocks( 622 + &self, 623 + candidate_cids: &[Vec<u8>], 624 + ) -> Result<Vec<Vec<u8>>, DbError> { 625 + match candidate_cids.is_empty() { 626 + true => Ok(Vec::new()), 627 + false => { 628 + let rows = sqlx::query!( 629 + r#" 630 + SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid) 631 + WHERE NOT EXISTS ( 632 + SELECT 1 FROM user_blocks WHERE block_cid = t.cid 633 + ) 634 + "#, 635 + candidate_cids, 636 + ) 637 + .fetch_all(&self.pool) 638 + .await 639 + .map_err(map_sqlx_error)?; 640 + Ok(rows.into_iter().filter_map(|r| r.cid).collect()) 641 + } 642 + } 607 643 } 608 644 609 645 async fn get_user_block_cids_since_rev( ··· 1362 1398 .bind(input.user_id) 1363 1399 .bind(&collections) 1364 1400 .bind(&rkeys) 1401 + .execute(&mut *tx) 1402 + .await 1403 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 1404 + } 1405 + 1406 + if !input.backlinks_to_remove.is_empty() { 1407 + let remove_uris: Vec<&str> = input 1408 + .backlinks_to_remove 1409 + .iter() 1410 + .map(|u| u.as_str()) 1411 + .collect(); 1412 + sqlx::query!( 1413 + "DELETE FROM backlinks WHERE uri = ANY($1::text[])", 1414 + &remove_uris as &[&str], 1415 + ) 1416 + .execute(&mut *tx) 1417 + .await 1418 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 1419 + } 1420 + 1421 + if !input.backlinks_to_add.is_empty() { 1422 + let uris: Vec<&str> = input 1423 + .backlinks_to_add 1424 + .iter() 1425 + .map(|b| b.uri.as_str()) 1426 + .collect(); 1427 + let paths: Vec<&str> = input 1428 + .backlinks_to_add 1429 + .iter() 1430 + .map(|b| b.path.as_str()) 1431 + .collect(); 1432 + let link_tos: Vec<&str> = input 1433 + .backlinks_to_add 1434 + .iter() 1435 + .map(|b| b.link_to.as_str()) 1436 + .collect(); 1437 + sqlx::query!( 1438 + r#" 1439 + INSERT INTO backlinks (uri, path, link_to, repo_id) 1440 + SELECT unnest($1::text[]), unnest($2::text[]), unnest($3::text[]), $4 1441 + ON CONFLICT (uri, path) DO NOTHING 1442 + "#, 1443 + &uris as &[&str], 1444 + &paths as &[&str], 1445 + &link_tos as &[&str], 1446 + input.user_id, 1447 + ) 1365 1448 .execute(&mut *tx) 1366 1449 .await 1367 1450 .map_err(|e| ApplyCommitError::Database(e.to_string()))?;
+1
crates/tranquil-pds/Cargo.toml
··· 18 18 tranquil-signal = { workspace = true } 19 19 tranquil-db = { workspace = true } 20 20 tranquil-db-traits = { workspace = true } 21 + tranquil-store = { workspace = true } 21 22 tranquil-lexicon = { workspace = true, features = ["resolve"] } 22 23 23 24 aes-gcm = { workspace = true }
+4 -5
crates/tranquil-pds/src/delegation/mod.rs
··· 49 49 None 50 50 } 51 51 }); 52 - let handle = did_doc.also_known_as.iter().find_map(|alias| { 53 - alias 54 - .strip_prefix("at://") 55 - .and_then(|s| Some(s.to_string())) 56 - }); 52 + let handle = did_doc 53 + .also_known_as 54 + .iter() 55 + .find_map(|alias| alias.strip_prefix("at://").map(|s| s.to_string())); 57 56 58 57 Ok(ResolvedIdentity { 59 58 did: did.clone(),
+6 -4
crates/tranquil-pds/src/did.rs
··· 4 4 use std::sync::Arc; 5 5 use std::time::{Duration, Instant}; 6 6 use tokio::sync::RwLock; 7 - use tracing::{debug, error, info, warn}; 7 + use tracing::{debug, info, warn}; 8 8 9 9 #[derive(Debug, thiserror::Error)] 10 10 pub enum DidResolutionError { ··· 55 55 pub service_id: String, 56 56 } 57 57 58 + type TimedCache<T> = RwLock<HashMap<Box<str>, (Instant, Arc<T>)>>; 59 + 58 60 pub struct DidResolver { 59 - did_doc_cache: RwLock<HashMap<Box<str>, (Instant, Arc<serde_json::Value>)>>, 60 - parsed_did_doc_cache: RwLock<HashMap<Box<str>, (Instant, Arc<DidDocument>)>>, 61 - service_cache: RwLock<HashMap<Box<str>, (Instant, Arc<ResolvedService>)>>, 61 + did_doc_cache: TimedCache<serde_json::Value>, 62 + parsed_did_doc_cache: TimedCache<DidDocument>, 63 + service_cache: TimedCache<ResolvedService>, 62 64 client: Client, 63 65 cache_ttl: Duration, 64 66 plc_directory_url: String,
+15
crates/tranquil-pds/src/repo_ops.rs
··· 159 159 pub ops: Vec<RecordOp>, 160 160 pub modified_keys: &'a [String], 161 161 pub blob_cids: &'a [String], 162 + pub backlinks_to_add: Vec<Backlink>, 163 + pub backlinks_to_remove: Vec<AtUri>, 162 164 } 163 165 164 166 pub async fn begin_repo_write( ··· 249 251 blocks_cids: &written_cids_str, 250 252 blobs: params.blob_cids, 251 253 obsolete_cids: vec![ctx.current_root_cid], 254 + backlinks_to_add: params.backlinks_to_add, 255 + backlinks_to_remove: params.backlinks_to_remove, 252 256 }, 253 257 ) 254 258 .await?; ··· 331 335 pub blocks_cids: &'a [String], 332 336 pub blobs: &'a [String], 333 337 pub obsolete_cids: Vec<Cid>, 338 + pub backlinks_to_add: Vec<Backlink>, 339 + pub backlinks_to_remove: Vec<AtUri>, 334 340 } 335 341 336 342 pub async fn commit_and_log( ··· 342 348 RepoEventType, 343 349 }; 344 350 351 + let backlinks_to_add = params.backlinks_to_add; 352 + let backlinks_to_remove = params.backlinks_to_remove; 345 353 let CommitParams { 346 354 did, 347 355 user_id, ··· 352 360 blocks_cids, 353 361 blobs, 354 362 obsolete_cids, 363 + .. 355 364 } = params; 356 365 let key_row = state 357 366 .repos ··· 485 494 obsolete_block_cids: obsolete_bytes, 486 495 record_upserts, 487 496 record_deletes, 497 + backlinks_to_add, 498 + backlinks_to_remove, 488 499 commit_event, 489 500 }; 490 501 ··· 592 603 .collect(); 593 604 let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 594 605 let blob_cids = extract_blob_cids(record); 606 + let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); 607 + let backlinks = extract_backlinks(&record_uri, record); 595 608 let result = commit_and_log( 596 609 state, 597 610 CommitParams { ··· 604 617 blocks_cids: &written_cids_str, 605 618 blobs: &blob_cids, 606 619 obsolete_cids, 620 + backlinks_to_add: backlinks, 621 + backlinks_to_remove: vec![], 607 622 }, 608 623 ) 609 624 .await?;
+62 -2
crates/tranquil-pds/src/scheduled.rs
··· 436 436 blob_repo: Arc<dyn BlobRepository>, 437 437 blob_store: Arc<dyn BlobStorage>, 438 438 sso_repo: Arc<dyn SsoRepository>, 439 + repo_repo: Arc<dyn RepoRepository>, 440 + block_store: PostgresBlockStore, 439 441 shutdown: CancellationToken, 440 442 ) { 441 - let check_interval = 442 - Duration::from_secs(tranquil_config::get().scheduled.delete_check_interval_secs); 443 + let cfg = tranquil_config::get(); 444 + let check_interval = Duration::from_secs(cfg.scheduled.delete_check_interval_secs); 445 + let gc_interval = Duration::from_secs(cfg.scheduled.block_gc_interval_secs); 443 446 444 447 info!( 445 448 check_interval_secs = check_interval.as_secs(), 449 + gc_interval_secs = gc_interval.as_secs(), 446 450 "Starting scheduled tasks service" 447 451 ); 448 452 449 453 let mut ticker = interval(check_interval); 450 454 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); 455 + 456 + let mut gc_ticker = interval(gc_interval); 457 + gc_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); 451 458 452 459 loop { 453 460 tokio::select! { ··· 494 501 } 495 502 } 496 503 } 504 + _ = gc_ticker.tick() => { 505 + if let Err(e) = run_block_gc(repo_repo.as_ref(), &block_store).await { 506 + error!("Block GC error: {e}"); 507 + } 508 + } 497 509 } 498 510 } 511 + } 512 + 513 + const BLOCK_GC_BATCH_SIZE: i64 = 1000; 514 + 515 + async fn run_block_gc( 516 + repo_repo: &dyn RepoRepository, 517 + block_store: &PostgresBlockStore, 518 + ) -> anyhow::Result<()> { 519 + let mut total_deleted: u64 = 0; 520 + 521 + loop { 522 + let candidates = block_store 523 + .get_oldest_block_cids(BLOCK_GC_BATCH_SIZE) 524 + .await 525 + .context("failed to fetch candidate blocks")?; 526 + 527 + match candidates.is_empty() { 528 + true => break, 529 + false => { 530 + let batch_len = candidates.len(); 531 + let unreferenced = repo_repo 532 + .find_unreferenced_blocks(&candidates) 533 + .await 534 + .context("failed to check block references")?; 535 + 536 + let deleted = match unreferenced.is_empty() { 537 + true => 0, 538 + false => block_store 539 + .delete_blocks(&unreferenced) 540 + .await 541 + .context("failed to delete unreferenced blocks")?, 542 + }; 543 + 544 + total_deleted = total_deleted.saturating_add(deleted); 545 + 546 + match unreferenced.len() == batch_len { 547 + true => continue, 548 + false => break, 549 + } 550 + } 551 + } 552 + } 553 + 554 + match total_deleted > 0 { 555 + true => info!(total_deleted, "Block GC cycle complete"), 556 + false => debug!("Block GC cycle: no orphaned blocks found"), 557 + } 558 + Ok(()) 499 559 } 500 560 501 561 async fn process_scheduled_deletions(
+75 -1
crates/tranquil-pds/src/state.rs
··· 12 12 use crate::storage::{BlobStorage, create_blob_storage}; 13 13 use sqlx::PgPool; 14 14 use std::error::Error; 15 + use std::path::PathBuf; 15 16 use std::sync::Arc; 16 17 use std::sync::atomic::{AtomicBool, Ordering}; 17 18 use tokio::sync::broadcast; ··· 262 263 AuthConfig::init(); 263 264 init_rate_limit_override(); 264 265 265 - let repos = Arc::new(PostgresRepositories::new(db.clone())); 266 + let mut repos = PostgresRepositories::new(db.clone()); 267 + 268 + let cfg = tranquil_config::get(); 269 + if cfg.storage.repo_backend() == tranquil_config::RepoBackend::TranquilStore { 270 + wire_tranquil_store(&mut repos, &cfg.tranquil_store); 271 + } 272 + 273 + let repos = Arc::new(repos); 266 274 let block_store = PostgresBlockStore::new(db); 267 275 let blob_store = create_blob_storage().await; 268 276 ··· 377 385 true 378 386 } 379 387 } 388 + 389 + fn wire_tranquil_store( 390 + repos: &mut PostgresRepositories, 391 + store_cfg: &tranquil_config::TranquilStoreConfig, 392 + ) { 393 + use tranquil_store::RealIO; 394 + use tranquil_store::eventlog::{EventLog, EventLogBridge, EventLogConfig}; 395 + use tranquil_store::metastore::client::MetastoreClient; 396 + use tranquil_store::metastore::handler::HandlerPool; 397 + use tranquil_store::metastore::partitions::Partition; 398 + use tranquil_store::metastore::{Metastore, MetastoreConfig}; 399 + 400 + let base_dir = PathBuf::from(&store_cfg.data_dir); 401 + let data_dir = match std::env::var("TRANQUIL_PDS_TEST_INFRA_READY").as_deref() { 402 + Ok("1") => base_dir.join(format!("pid-{}", std::process::id())), 403 + _ => base_dir, 404 + }; 405 + let metastore_dir = data_dir.join("metastore"); 406 + let segments_dir = data_dir.join("eventlog").join("segments"); 407 + 408 + std::fs::create_dir_all(&metastore_dir).expect("failed to create metastore directory"); 409 + std::fs::create_dir_all(&segments_dir).expect("failed to create eventlog segments directory"); 410 + 411 + let metastore_config = store_cfg 412 + .memory_budget_mb 413 + .map(|mb| MetastoreConfig { 414 + cache_size_bytes: mb.saturating_mul(1024 * 1024), 415 + }) 416 + .unwrap_or_default(); 417 + 418 + let metastore = 419 + Metastore::open(&metastore_dir, metastore_config).expect("failed to open metastore"); 420 + 421 + let event_log = EventLog::open( 422 + EventLogConfig { 423 + segments_dir, 424 + ..EventLogConfig::default() 425 + }, 426 + RealIO::new(), 427 + ) 428 + .expect("failed to open eventlog"); 429 + let event_log = Arc::new(event_log); 430 + 431 + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); 432 + 433 + let indexes = metastore.partition(Partition::Indexes).clone(); 434 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 435 + let recovered = event_ops 436 + .recover_metastore_mutations(&indexes) 437 + .expect("metastore crash recovery failed"); 438 + if recovered > 0 { 439 + tracing::info!(recovered, "replayed metastore mutations from eventlog"); 440 + } 441 + 442 + let notifier = bridge.notifier(); 443 + 444 + let pool = HandlerPool::spawn::<RealIO>(metastore, bridge, None, store_cfg.handler_threads); 445 + 446 + let client = MetastoreClient::<RealIO>::new(Arc::new(pool)); 447 + 448 + tracing::info!(data_dir = %store_cfg.data_dir, "tranquil-store data directory"); 449 + 450 + repos.repo = Arc::new(client.clone()); 451 + repos.backlink = Arc::new(client); 452 + repos.event_notifier = Arc::new(notifier); 453 + }
+44 -20
crates/tranquil-pds/tests/helpers/mod.rs
··· 302 302 303 303 #[allow(dead_code)] 304 304 pub async fn set_account_takedown(did: &str, takedown_ref: Option<&str>) { 305 - let pool = get_test_db_pool().await; 306 - sqlx::query!( 307 - "UPDATE users SET takedown_ref = $1 WHERE did = $2", 308 - takedown_ref, 309 - did 310 - ) 311 - .execute(pool) 312 - .await 313 - .expect("Failed to update takedown_ref"); 305 + let client = client(); 306 + let (admin_jwt, _) = create_admin_account_and_login(&client).await; 307 + let applied = takedown_ref.is_some(); 308 + let res = client 309 + .post(format!( 310 + "{}/xrpc/com.atproto.admin.updateSubjectStatus", 311 + base_url().await, 312 + )) 313 + .bearer_auth(&admin_jwt) 314 + .json(&json!({ 315 + "subject": { 316 + "$type": "com.atproto.admin.defs#repoRef", 317 + "did": did 318 + }, 319 + "takedown": { 320 + "applied": applied, 321 + "ref": takedown_ref 322 + } 323 + })) 324 + .send() 325 + .await 326 + .expect("Failed to send takedown request"); 327 + assert_eq!(res.status(), StatusCode::OK, "Failed to set takedown"); 314 328 } 315 329 316 330 #[allow(dead_code)] 317 331 pub async fn set_account_deactivated(did: &str, deactivated: bool) { 318 - let pool = get_test_db_pool().await; 319 - let deactivated_at: Option<chrono::DateTime<Utc>> = 320 - if deactivated { Some(Utc::now()) } else { None }; 321 - sqlx::query!( 322 - "UPDATE users SET deactivated_at = $1 WHERE did = $2", 323 - deactivated_at, 324 - did 325 - ) 326 - .execute(pool) 327 - .await 328 - .expect("Failed to update deactivated_at"); 332 + let client = client(); 333 + let (admin_jwt, _) = create_admin_account_and_login(&client).await; 334 + let res = client 335 + .post(format!( 336 + "{}/xrpc/com.atproto.admin.updateSubjectStatus", 337 + base_url().await, 338 + )) 339 + .bearer_auth(&admin_jwt) 340 + .json(&json!({ 341 + "subject": { 342 + "$type": "com.atproto.admin.defs#repoRef", 343 + "did": did 344 + }, 345 + "deactivated": { 346 + "applied": deactivated 347 + } 348 + })) 349 + .send() 350 + .await 351 + .expect("Failed to send deactivation request"); 352 + assert_eq!(res.status(), StatusCode::OK, "Failed to set deactivation"); 329 353 } 330 354 331 355 #[allow(dead_code)]
+26
crates/tranquil-repo/src/lib.rs
··· 24 24 } 25 25 } 26 26 27 + impl PostgresBlockStore { 28 + pub async fn get_oldest_block_cids(&self, limit: i64) -> Result<Vec<Vec<u8>>, RepoError> { 29 + let rows = sqlx::query!( 30 + "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1", 31 + limit, 32 + ) 33 + .fetch_all(&self.pool) 34 + .await 35 + .map_err(RepoError::storage)?; 36 + Ok(rows.into_iter().map(|r| r.cid).collect()) 37 + } 38 + 39 + pub async fn delete_blocks(&self, cids: &[Vec<u8>]) -> Result<u64, RepoError> { 40 + match cids.is_empty() { 41 + true => Ok(0), 42 + false => { 43 + let result = sqlx::query!("DELETE FROM blocks WHERE cid = ANY($1)", cids,) 44 + .execute(&self.pool) 45 + .await 46 + .map_err(RepoError::storage)?; 47 + Ok(result.rows_affected()) 48 + } 49 + } 50 + } 51 + } 52 + 27 53 impl BlockStore for PostgresBlockStore { 28 54 async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> { 29 55 let cid_bytes = cid.to_bytes();
+2
crates/tranquil-server/src/main.rs
··· 253 253 state.repos.blob.clone(), 254 254 state.blob_store.clone(), 255 255 state.repos.sso.clone(), 256 + state.repos.repo.clone(), 257 + state.block_store.clone(), 256 258 shutdown.clone(), 257 259 )); 258 260
+5
crates/tranquil-store/Cargo.toml
··· 12 12 postcard = { version = "1", features = ["alloc"] } 13 13 parking_lot = { workspace = true } 14 14 fjall = "3" 15 + lsm-tree = "3" 15 16 flume = "0.11" 16 17 tokio = { workspace = true, features = ["sync", "rt"] } 17 18 bytes = "1" ··· 26 27 cid = { workspace = true } 27 28 multihash = { workspace = true } 28 29 sha2 = { workspace = true } 30 + siphasher = "1" 31 + dashmap = "6" 32 + smallvec = "1" 33 + uuid = { workspace = true } 29 34 30 35 [features] 31 36 test-harness = []
+812
crates/tranquil-store/benches/metastore.rs
··· 1 + use std::sync::Arc; 2 + use std::time::{Duration, Instant}; 3 + 4 + use futures::StreamExt; 5 + use tokio::sync::oneshot; 6 + use tranquil_db_traits::{ 7 + ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType, RepoRepository, 8 + }; 9 + use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; 10 + use uuid::Uuid; 11 + 12 + use tranquil_store::RealIO; 13 + use tranquil_store::eventlog::{EventLog, EventLogConfig}; 14 + use tranquil_store::metastore::handler::{ 15 + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, 16 + }; 17 + use tranquil_store::metastore::{Metastore, MetastoreConfig}; 18 + 19 + struct LatencyStats { 20 + p50: Duration, 21 + p95: Duration, 22 + p99: Duration, 23 + max: Duration, 24 + mean: Duration, 25 + } 26 + 27 + fn compute_stats(durations: &mut [Duration]) -> Option<LatencyStats> { 28 + match durations.is_empty() { 29 + true => None, 30 + false => { 31 + durations.sort(); 32 + let len = durations.len(); 33 + let sum: Duration = durations.iter().sum(); 34 + let divisor = u32::try_from(len).unwrap_or(u32::MAX); 35 + let last = len - 1; 36 + Some(LatencyStats { 37 + p50: durations[last * 50 / 100], 38 + p95: durations[last * 95 / 100], 39 + p99: durations[last * 99 / 100], 40 + max: durations[last], 41 + mean: sum / divisor, 42 + }) 43 + } 44 + } 45 + } 46 + 47 + fn print_result(ops: usize, elapsed: Duration, stats: Option<&LatencyStats>) { 48 + let throughput = ops as f64 / elapsed.as_secs_f64(); 49 + match stats { 50 + Some(s) => println!( 51 + "{throughput:.0} ops/sec, {:.1}ms | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}", 52 + elapsed.as_secs_f64() * 1000.0, 53 + s.p50, 54 + s.p95, 55 + s.p99, 56 + s.max, 57 + s.mean 58 + ), 59 + None => println!( 60 + "{throughput:.0} ops/sec, {:.1}ms", 61 + elapsed.as_secs_f64() * 1000.0, 62 + ), 63 + } 64 + } 65 + 66 + async fn collect_latencies(handles: Vec<tokio::task::JoinHandle<Vec<Duration>>>) -> Vec<Duration> { 67 + futures::stream::iter(handles) 68 + .fold(Vec::new(), |mut acc, h| async move { 69 + acc.extend(h.await.unwrap()); 70 + acc 71 + }) 72 + .await 73 + } 74 + 75 + fn test_cid(seed: u8) -> CidLink { 76 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 77 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 78 + let c = cid::Cid::new_v1(0x71, mh); 79 + CidLink::from_cid(&c) 80 + } 81 + 82 + fn test_cid_bytes(seed: u8) -> Vec<u8> { 83 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 84 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 85 + cid::Cid::new_v1(0x71, mh).to_bytes() 86 + } 87 + 88 + fn make_rev(n: u64) -> String { 89 + format!("rev{n:010}") 90 + } 91 + 92 + struct BenchHarness { 93 + pool: Arc<HandlerPool>, 94 + _metastore_dir: tempfile::TempDir, 95 + _eventlog_dir: tempfile::TempDir, 96 + } 97 + 98 + fn setup(thread_count: usize) -> BenchHarness { 99 + let metastore_dir = tempfile::TempDir::new().unwrap(); 100 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 101 + let segments_dir = eventlog_dir.path().join("segments"); 102 + std::fs::create_dir_all(&segments_dir).unwrap(); 103 + 104 + let metastore = Metastore::open( 105 + metastore_dir.path(), 106 + MetastoreConfig { 107 + cache_size_bytes: 256 * 1024 * 1024, 108 + }, 109 + ) 110 + .unwrap(); 111 + 112 + let event_log = EventLog::open( 113 + EventLogConfig { 114 + segments_dir, 115 + ..EventLogConfig::default() 116 + }, 117 + RealIO::new(), 118 + ) 119 + .unwrap(); 120 + 121 + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( 122 + event_log, 123 + ))); 124 + 125 + let pool = Arc::new(HandlerPool::spawn::<RealIO>( 126 + metastore, 127 + bridge, 128 + None, 129 + Some(thread_count), 130 + )); 131 + 132 + BenchHarness { 133 + pool, 134 + _metastore_dir: metastore_dir, 135 + _eventlog_dir: eventlog_dir, 136 + } 137 + } 138 + 139 + async fn create_user(pool: &HandlerPool, user_id: Uuid, did: &Did, cid: &CidLink) { 140 + let (tx, rx) = oneshot::channel(); 141 + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 142 + user_id, 143 + did: did.clone(), 144 + handle: Handle::from(format!("bench.{}.invalid", user_id.as_simple())), 145 + repo_root_cid: cid.clone(), 146 + repo_rev: "rev0000000000".to_string(), 147 + tx, 148 + })) 149 + .unwrap(); 150 + rx.await.unwrap().unwrap(); 151 + } 152 + 153 + fn make_commit_input( 154 + user_id: Uuid, 155 + did: &Did, 156 + collection: &Nsid, 157 + rev_n: u64, 158 + cid_seed: u8, 159 + ) -> ApplyCommitInput { 160 + ApplyCommitInput { 161 + user_id, 162 + did: did.clone(), 163 + expected_root_cid: None, 164 + new_root_cid: test_cid(cid_seed), 165 + new_rev: make_rev(rev_n), 166 + new_block_cids: vec![test_cid_bytes(cid_seed)], 167 + obsolete_block_cids: vec![], 168 + record_upserts: vec![RecordUpsert { 169 + collection: collection.clone(), 170 + rkey: Rkey::from(format!("r{rev_n:010}")), 171 + cid: test_cid(cid_seed), 172 + }], 173 + record_deletes: vec![], 174 + backlinks_to_add: vec![], 175 + backlinks_to_remove: vec![], 176 + commit_event: CommitEventData { 177 + did: did.clone(), 178 + event_type: RepoEventType::Commit, 179 + commit_cid: Some(test_cid(cid_seed)), 180 + prev_cid: None, 181 + ops: None, 182 + blobs: None, 183 + blocks_cids: None, 184 + prev_data_cid: None, 185 + rev: Some(make_rev(rev_n)), 186 + }, 187 + } 188 + } 189 + 190 + async fn seed_records( 191 + pool: &HandlerPool, 192 + user_id: Uuid, 193 + did: &Did, 194 + collection: &Nsid, 195 + count: usize, 196 + ) { 197 + let batches: Vec<(usize, usize, u64, u8)> = (0..) 198 + .map(|i| { 199 + let start = i * 50; 200 + let end = (start + 50).min(count); 201 + let rev_n = (i as u64) + 1; 202 + let cid_seed = ((i + 10) & 0xFF) as u8; 203 + (start, end, rev_n, cid_seed) 204 + }) 205 + .take_while(|(start, _, _, _)| *start < count) 206 + .collect(); 207 + 208 + futures::stream::iter(batches) 209 + .fold((), |(), (batch_start, batch_end, rev_n, cid_seed)| { 210 + let did = did.clone(); 211 + let collection = collection.clone(); 212 + async move { 213 + let record_upserts: Vec<RecordUpsert> = (batch_start..batch_end) 214 + .map(|i| RecordUpsert { 215 + collection: collection.clone(), 216 + rkey: Rkey::from(format!("rec{i:08}")), 217 + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), 218 + }) 219 + .collect(); 220 + 221 + let new_block_cids: Vec<Vec<u8>> = (batch_start..batch_end) 222 + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) 223 + .collect(); 224 + 225 + let input = ApplyCommitInput { 226 + user_id, 227 + did: did.clone(), 228 + expected_root_cid: None, 229 + new_root_cid: test_cid(cid_seed), 230 + new_rev: make_rev(rev_n), 231 + new_block_cids, 232 + obsolete_block_cids: vec![], 233 + record_upserts, 234 + record_deletes: vec![], 235 + backlinks_to_add: vec![], 236 + backlinks_to_remove: vec![], 237 + commit_event: CommitEventData { 238 + did, 239 + event_type: RepoEventType::Commit, 240 + commit_cid: Some(test_cid(cid_seed)), 241 + prev_cid: None, 242 + ops: None, 243 + blobs: None, 244 + blocks_cids: None, 245 + prev_data_cid: None, 246 + rev: Some(make_rev(rev_n)), 247 + }, 248 + }; 249 + 250 + let (tx, rx) = oneshot::channel(); 251 + pool.send(MetastoreRequest::Commit(Box::new( 252 + CommitRequest::ApplyCommit { 253 + input: Box::new(input), 254 + tx, 255 + }, 256 + ))) 257 + .unwrap(); 258 + rx.await.unwrap().unwrap(); 259 + } 260 + }) 261 + .await; 262 + } 263 + 264 + async fn bench_apply_commit(pool: &Arc<HandlerPool>, concurrency: usize, ops_per_task: usize) { 265 + let user_ids: Vec<Uuid> = (0..concurrency).map(|_| Uuid::new_v4()).collect(); 266 + let dids: Vec<Did> = user_ids 267 + .iter() 268 + .map(|u| Did::from(format!("did:plc:bench{}", u.as_simple()))) 269 + .collect(); 270 + 271 + futures::stream::iter(user_ids.iter().zip(dids.iter())) 272 + .fold((), |(), (uid, did)| async { 273 + create_user(pool, *uid, did, &test_cid(1)).await; 274 + }) 275 + .await; 276 + 277 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 278 + let start = Instant::now(); 279 + let handles: Vec<_> = (0..concurrency) 280 + .map(|task_id| { 281 + let pool = Arc::clone(pool); 282 + let user_id = user_ids[task_id]; 283 + let did = dids[task_id].clone(); 284 + let collection = collection.clone(); 285 + tokio::spawn(async move { 286 + futures::stream::iter(0..ops_per_task) 287 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 288 + let pool = &pool; 289 + let did = &did; 290 + let collection = &collection; 291 + async move { 292 + let rev_n = (task_id * ops_per_task + i + 1) as u64; 293 + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; 294 + let input = 295 + make_commit_input(user_id, did, collection, rev_n, cid_seed); 296 + let t = Instant::now(); 297 + let (tx, rx) = oneshot::channel(); 298 + pool.send(MetastoreRequest::Commit(Box::new( 299 + CommitRequest::ApplyCommit { 300 + input: Box::new(input), 301 + tx, 302 + }, 303 + ))) 304 + .unwrap(); 305 + rx.await.unwrap().unwrap(); 306 + latencies.push(t.elapsed()); 307 + latencies 308 + } 309 + }) 310 + .await 311 + }) 312 + }) 313 + .collect(); 314 + 315 + let mut all_latencies = collect_latencies(handles).await; 316 + let elapsed = start.elapsed(); 317 + let total_ops = concurrency * ops_per_task; 318 + let stats = compute_stats(&mut all_latencies); 319 + print_result(total_ops, elapsed, stats.as_ref()); 320 + } 321 + 322 + async fn bench_get_record_cid(pool: &Arc<HandlerPool>, concurrency: usize, ops_per_task: usize) { 323 + let user_id = Uuid::new_v4(); 324 + let did = Did::from(format!("did:plc:getrecord{}", user_id.as_simple())); 325 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 326 + create_user(pool, user_id, &did, &test_cid(1)).await; 327 + seed_records(pool, user_id, &did, &collection, 1000).await; 328 + 329 + let total_records = 1000usize; 330 + let start = Instant::now(); 331 + let handles: Vec<_> = (0..concurrency) 332 + .map(|task_id| { 333 + let pool = Arc::clone(pool); 334 + let collection = collection.clone(); 335 + tokio::spawn(async move { 336 + futures::stream::iter(0..ops_per_task) 337 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 338 + let pool = &pool; 339 + let collection = &collection; 340 + async move { 341 + let rec_idx = (task_id * 7 + i * 13) % total_records; 342 + let rkey = Rkey::from(format!("rec{rec_idx:08}")); 343 + let t = Instant::now(); 344 + let (tx, rx) = oneshot::channel(); 345 + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { 346 + repo_id: user_id, 347 + collection: collection.clone(), 348 + rkey, 349 + tx, 350 + })) 351 + .unwrap(); 352 + let result = rx.await.unwrap().unwrap(); 353 + assert!(result.is_some()); 354 + latencies.push(t.elapsed()); 355 + latencies 356 + } 357 + }) 358 + .await 359 + }) 360 + }) 361 + .collect(); 362 + 363 + let mut all_latencies = collect_latencies(handles).await; 364 + let elapsed = start.elapsed(); 365 + let total_ops = concurrency * ops_per_task; 366 + let stats = compute_stats(&mut all_latencies); 367 + print_result(total_ops, elapsed, stats.as_ref()); 368 + } 369 + 370 + async fn bench_list_records(pool: &Arc<HandlerPool>, concurrency: usize, ops_per_task: usize) { 371 + let user_id = Uuid::new_v4(); 372 + let did = Did::from(format!("did:plc:listrecords{}", user_id.as_simple())); 373 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 374 + create_user(pool, user_id, &did, &test_cid(1)).await; 375 + seed_records(pool, user_id, &did, &collection, 1000).await; 376 + 377 + let start = Instant::now(); 378 + let handles: Vec<_> = (0..concurrency) 379 + .map(|_| { 380 + let pool = Arc::clone(pool); 381 + let collection = collection.clone(); 382 + tokio::spawn(async move { 383 + futures::stream::iter(0..ops_per_task) 384 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, _| { 385 + let pool = &pool; 386 + let collection = &collection; 387 + async move { 388 + let t = Instant::now(); 389 + let (tx, rx) = oneshot::channel(); 390 + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { 391 + repo_id: user_id, 392 + collection: collection.clone(), 393 + cursor: None, 394 + limit: 50, 395 + reverse: false, 396 + rkey_start: None, 397 + rkey_end: None, 398 + tx, 399 + })) 400 + .unwrap(); 401 + let result = rx.await.unwrap().unwrap(); 402 + assert!(!result.is_empty()); 403 + latencies.push(t.elapsed()); 404 + latencies 405 + } 406 + }) 407 + .await 408 + }) 409 + }) 410 + .collect(); 411 + 412 + let mut all_latencies = collect_latencies(handles).await; 413 + let elapsed = start.elapsed(); 414 + let total_ops = concurrency * ops_per_task; 415 + let stats = compute_stats(&mut all_latencies); 416 + print_result(total_ops, elapsed, stats.as_ref()); 417 + } 418 + 419 + async fn setup_pg_bench_schema(pool: &sqlx::PgPool) { 420 + sqlx::query( 421 + "CREATE TABLE IF NOT EXISTS users ( 422 + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), 423 + handle TEXT NOT NULL UNIQUE, 424 + email TEXT, 425 + did TEXT NOT NULL UNIQUE, 426 + password_hash TEXT NOT NULL DEFAULT '', 427 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 428 + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 429 + deactivated_at TIMESTAMPTZ, 430 + invites_disabled BOOLEAN DEFAULT FALSE, 431 + takedown_ref TEXT, 432 + preferred_comms_channel TEXT NOT NULL DEFAULT 'email', 433 + password_reset_code TEXT, 434 + password_reset_code_expires_at TIMESTAMPTZ, 435 + email_verified BOOLEAN NOT NULL DEFAULT FALSE, 436 + two_factor_enabled BOOLEAN NOT NULL DEFAULT FALSE, 437 + discord_id TEXT, 438 + discord_verified BOOLEAN NOT NULL DEFAULT FALSE, 439 + telegram_username TEXT, 440 + telegram_verified BOOLEAN NOT NULL DEFAULT FALSE, 441 + signal_number TEXT, 442 + signal_verified BOOLEAN NOT NULL DEFAULT FALSE, 443 + is_admin BOOLEAN NOT NULL DEFAULT FALSE, 444 + migrated_to_pds TEXT, 445 + migrated_at TIMESTAMPTZ, 446 + preferred_locale TEXT, 447 + signal_uuid TEXT 448 + )", 449 + ) 450 + .execute(pool) 451 + .await 452 + .unwrap(); 453 + 454 + sqlx::query( 455 + "CREATE TABLE IF NOT EXISTS repos ( 456 + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, 457 + repo_root_cid TEXT NOT NULL, 458 + repo_rev TEXT, 459 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 460 + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() 461 + )", 462 + ) 463 + .execute(pool) 464 + .await 465 + .unwrap(); 466 + 467 + sqlx::query( 468 + "CREATE TABLE IF NOT EXISTS records ( 469 + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), 470 + repo_id UUID NOT NULL REFERENCES repos(user_id) ON DELETE CASCADE, 471 + collection TEXT NOT NULL, 472 + rkey TEXT NOT NULL, 473 + record_cid TEXT NOT NULL, 474 + takedown_ref TEXT, 475 + repo_rev TEXT, 476 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 477 + UNIQUE(repo_id, collection, rkey) 478 + )", 479 + ) 480 + .execute(pool) 481 + .await 482 + .unwrap(); 483 + } 484 + 485 + async fn pg_create_user(pg: &sqlx::PgPool, user_id: Uuid, did: &str) { 486 + sqlx::query("INSERT INTO users (id, handle, did) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING") 487 + .bind(user_id) 488 + .bind(format!("bench.{}.invalid", Uuid::new_v4().as_simple())) 489 + .bind(did) 490 + .execute(pg) 491 + .await 492 + .unwrap(); 493 + } 494 + 495 + async fn bench_pg_upsert_records( 496 + repo: &dyn RepoRepository, 497 + pg: &sqlx::PgPool, 498 + concurrency: usize, 499 + ops_per_task: usize, 500 + ) { 501 + let user_ids: Vec<Uuid> = (0..concurrency).map(|_| Uuid::new_v4()).collect(); 502 + let dids: Vec<String> = user_ids 503 + .iter() 504 + .map(|u| format!("did:plc:pgbench{}", u.as_simple())) 505 + .collect(); 506 + 507 + futures::stream::iter(user_ids.iter().zip(dids.iter())) 508 + .fold((), |(), (uid, did)| async { 509 + pg_create_user(pg, *uid, did).await; 510 + let did_typed = Did::from(did.clone()); 511 + let handle = Handle::from(format!("bench.{}.invalid", uid.as_simple())); 512 + repo.create_repo(*uid, &did_typed, &handle, &test_cid(1), "rev0000000000") 513 + .await 514 + .unwrap(); 515 + }) 516 + .await; 517 + 518 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 519 + let start = Instant::now(); 520 + let handles: Vec<_> = (0..concurrency) 521 + .map(|task_id| { 522 + let user_id = user_ids[task_id]; 523 + let collection = collection.clone(); 524 + let pg = pg.clone(); 525 + tokio::spawn(async move { 526 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 527 + futures::stream::iter(0..ops_per_task) 528 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 529 + let repo = &repo; 530 + let collection = &collection; 531 + async move { 532 + let rev_n = (task_id * ops_per_task + i + 1) as u64; 533 + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; 534 + let rkey = Rkey::from(format!("r{rev_n:010}")); 535 + let t = Instant::now(); 536 + repo.upsert_records( 537 + user_id, 538 + std::slice::from_ref(collection), 539 + &[rkey], 540 + &[test_cid(cid_seed)], 541 + &make_rev(rev_n), 542 + ) 543 + .await 544 + .unwrap(); 545 + latencies.push(t.elapsed()); 546 + latencies 547 + } 548 + }) 549 + .await 550 + }) 551 + }) 552 + .collect(); 553 + 554 + let mut all_latencies = collect_latencies(handles).await; 555 + let elapsed = start.elapsed(); 556 + let total_ops = concurrency * ops_per_task; 557 + let stats = compute_stats(&mut all_latencies); 558 + print_result(total_ops, elapsed, stats.as_ref()); 559 + } 560 + 561 + async fn pg_seed_records( 562 + repo: &dyn RepoRepository, 563 + user_id: Uuid, 564 + collection: &Nsid, 565 + count: usize, 566 + ) { 567 + let batches: Vec<(usize, usize)> = (0..) 568 + .map(|i| { 569 + let start = i * 50; 570 + let end = (start + 50).min(count); 571 + (start, end) 572 + }) 573 + .take_while(|(start, _)| *start < count) 574 + .collect(); 575 + 576 + futures::stream::iter(batches) 577 + .fold((), |(), (batch_start, batch_end)| { 578 + let collection = collection.clone(); 579 + async move { 580 + let collections: Vec<Nsid> = (batch_start..batch_end) 581 + .map(|_| collection.clone()) 582 + .collect(); 583 + let rkeys: Vec<Rkey> = (batch_start..batch_end) 584 + .map(|i| Rkey::from(format!("rec{i:08}"))) 585 + .collect(); 586 + let cids: Vec<CidLink> = (batch_start..batch_end) 587 + .map(|i| test_cid(((i * 7 + 3) & 0xFF) as u8)) 588 + .collect(); 589 + repo.upsert_records( 590 + user_id, 591 + &collections, 592 + &rkeys, 593 + &cids, 594 + &make_rev(batch_start as u64), 595 + ) 596 + .await 597 + .unwrap(); 598 + } 599 + }) 600 + .await; 601 + } 602 + 603 + async fn bench_pg_get_record_cid(pg: &sqlx::PgPool, concurrency: usize, ops_per_task: usize) { 604 + let user_id = Uuid::new_v4(); 605 + let did = format!("did:plc:pgget{}", user_id.as_simple()); 606 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 607 + pg_create_user(pg, user_id, &did).await; 608 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); 609 + let did_typed = Did::from(did.clone()); 610 + let handle = Handle::from(format!("bench.{}.invalid", user_id.as_simple())); 611 + repo.create_repo(user_id, &did_typed, &handle, &test_cid(1), "rev0000000000") 612 + .await 613 + .unwrap(); 614 + pg_seed_records(&repo, user_id, &collection, 1000).await; 615 + 616 + let total_records = 1000usize; 617 + let start = Instant::now(); 618 + let handles: Vec<_> = (0..concurrency) 619 + .map(|task_id| { 620 + let pg = pg.clone(); 621 + let collection = collection.clone(); 622 + tokio::spawn(async move { 623 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 624 + futures::stream::iter(0..ops_per_task) 625 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 626 + let repo = &repo; 627 + let collection = &collection; 628 + async move { 629 + let rec_idx = (task_id * 7 + i * 13) % total_records; 630 + let rkey = Rkey::from(format!("rec{rec_idx:08}")); 631 + let t = Instant::now(); 632 + let result = repo 633 + .get_record_cid(user_id, collection, &rkey) 634 + .await 635 + .unwrap(); 636 + assert!(result.is_some()); 637 + latencies.push(t.elapsed()); 638 + latencies 639 + } 640 + }) 641 + .await 642 + }) 643 + }) 644 + .collect(); 645 + 646 + let mut all_latencies = collect_latencies(handles).await; 647 + let elapsed = start.elapsed(); 648 + let total_ops = concurrency * ops_per_task; 649 + let stats = compute_stats(&mut all_latencies); 650 + print_result(total_ops, elapsed, stats.as_ref()); 651 + } 652 + 653 + async fn bench_pg_list_records(pg: &sqlx::PgPool, concurrency: usize, ops_per_task: usize) { 654 + let user_id = Uuid::new_v4(); 655 + let did = format!("did:plc:pglist{}", user_id.as_simple()); 656 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 657 + pg_create_user(pg, user_id, &did).await; 658 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); 659 + let did_typed = Did::from(did.clone()); 660 + let handle = Handle::from(format!("bench.{}.invalid", user_id.as_simple())); 661 + repo.create_repo(user_id, &did_typed, &handle, &test_cid(1), "rev0000000000") 662 + .await 663 + .unwrap(); 664 + pg_seed_records(&repo, user_id, &collection, 1000).await; 665 + 666 + let start = Instant::now(); 667 + let handles: Vec<_> = (0..concurrency) 668 + .map(|_| { 669 + let pg = pg.clone(); 670 + let collection = collection.clone(); 671 + tokio::spawn(async move { 672 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 673 + futures::stream::iter(0..ops_per_task) 674 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, _| { 675 + let repo = &repo; 676 + let collection = &collection; 677 + async move { 678 + let t = Instant::now(); 679 + let result = repo 680 + .list_records(user_id, collection, None, 50, false, None, None) 681 + .await 682 + .unwrap(); 683 + assert!(!result.is_empty()); 684 + latencies.push(t.elapsed()); 685 + latencies 686 + } 687 + }) 688 + .await 689 + }) 690 + }) 691 + .collect(); 692 + 693 + let mut all_latencies = collect_latencies(handles).await; 694 + let elapsed = start.elapsed(); 695 + let total_ops = concurrency * ops_per_task; 696 + let stats = compute_stats(&mut all_latencies); 697 + print_result(total_ops, elapsed, stats.as_ref()); 698 + } 699 + 700 + #[tokio::main] 701 + async fn main() { 702 + let handler_threads = std::thread::available_parallelism() 703 + .map(|n| n.get().max(2) / 2) 704 + .unwrap_or(2); 705 + println!("handler threads: {handler_threads}"); 706 + 707 + let concurrency_levels = [1, 10, 100, 1000]; 708 + let ops_per_concurrency = |c: usize| match c { 709 + 1 => 5000, 710 + 10 => 1000, 711 + 100 => 200, 712 + 1000 => 50, 713 + _ => 100, 714 + }; 715 + 716 + futures::stream::iter(concurrency_levels.iter()) 717 + .fold((), |(), &c| async move { 718 + let ops = ops_per_concurrency(c); 719 + 720 + println!("-- apply_commit: {} ops, {} callers --", ops * c, c); 721 + let h = setup(handler_threads); 722 + bench_apply_commit(&h.pool, c, ops).await; 723 + 724 + println!("-- get_record_cid: {} ops, {} callers --", ops * c, c); 725 + let h = setup(handler_threads); 726 + bench_get_record_cid(&h.pool, c, ops).await; 727 + 728 + println!("-- list_records: {} ops, {} callers --", ops * c, c); 729 + let h = setup(handler_threads); 730 + bench_list_records(&h.pool, c, ops).await; 731 + }) 732 + .await; 733 + 734 + let pg_url = match std::env::var("DATABASE_URL") { 735 + Ok(url) => url, 736 + Err(_) => { 737 + println!("set DATABASE_URL for postgres comparison"); 738 + return; 739 + } 740 + }; 741 + 742 + let pg_concurrency_levels: &[usize] = &[1, 10, 50]; 743 + 744 + let setup_pg = |max_conns: u32| { 745 + let url = pg_url.clone(); 746 + async move { 747 + sqlx::postgres::PgPoolOptions::new() 748 + .max_connections(max_conns) 749 + .connect(&url) 750 + .await 751 + .unwrap() 752 + } 753 + }; 754 + 755 + let pg = setup_pg(60).await; 756 + setup_pg_bench_schema(&pg).await; 757 + pg.close().await; 758 + 759 + futures::stream::iter(pg_concurrency_levels.iter()) 760 + .fold((), |(), &c| { 761 + let setup_pg = &setup_pg; 762 + async move { 763 + let ops = ops_per_concurrency(c); 764 + let max_conns = u32::try_from(c).unwrap_or(50) + 10; 765 + let pg = setup_pg(max_conns).await; 766 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); 767 + 768 + println!( 769 + "-- postgres upsert_records: {} ops, {} callers --", 770 + ops * c, 771 + c 772 + ); 773 + bench_pg_upsert_records(&repo, &pg, c, ops).await; 774 + 775 + println!( 776 + "-- postgres get_record_cid: {} ops, {} callers --", 777 + ops * c, 778 + c 779 + ); 780 + bench_pg_get_record_cid(&pg, c, ops).await; 781 + 782 + println!( 783 + "-- postgres list_records: {} ops, {} callers --", 784 + ops * c, 785 + c 786 + ); 787 + bench_pg_list_records(&pg, c, ops).await; 788 + 789 + sqlx::query("TRUNCATE records, repos, users CASCADE") 790 + .execute(&pg) 791 + .await 792 + .unwrap(); 793 + pg.close().await; 794 + } 795 + }) 796 + .await; 797 + 798 + let pg = setup_pg(5).await; 799 + sqlx::query("DROP TABLE IF EXISTS records CASCADE") 800 + .execute(&pg) 801 + .await 802 + .unwrap(); 803 + sqlx::query("DROP TABLE IF EXISTS repos CASCADE") 804 + .execute(&pg) 805 + .await 806 + .unwrap(); 807 + sqlx::query("DROP TABLE IF EXISTS users CASCADE") 808 + .execute(&pg) 809 + .await 810 + .unwrap(); 811 + pg.close().await; 812 + }
+1083
crates/tranquil-store/benches/metastore_scale.rs
··· 1 + use std::sync::Arc; 2 + use std::time::{Duration, Instant}; 3 + 4 + use futures::StreamExt; 5 + use tokio::sync::oneshot; 6 + use tranquil_db_traits::{ 7 + ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType, RepoRepository, 8 + }; 9 + use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; 10 + use uuid::Uuid; 11 + 12 + use tranquil_store::RealIO; 13 + use tranquil_store::eventlog::{EventLog, EventLogConfig}; 14 + use tranquil_store::metastore::handler::{ 15 + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, 16 + }; 17 + use tranquil_store::metastore::{Metastore, MetastoreConfig}; 18 + 19 + struct LatencyStats { 20 + p50: Duration, 21 + p95: Duration, 22 + p99: Duration, 23 + max: Duration, 24 + mean: Duration, 25 + } 26 + 27 + fn compute_stats(durations: &mut [Duration]) -> Option<LatencyStats> { 28 + match durations.is_empty() { 29 + true => None, 30 + false => { 31 + durations.sort(); 32 + let len = durations.len(); 33 + let sum: Duration = durations.iter().sum(); 34 + let divisor = u32::try_from(len).unwrap_or(u32::MAX); 35 + let last = len - 1; 36 + Some(LatencyStats { 37 + p50: durations[last * 50 / 100], 38 + p95: durations[last * 95 / 100], 39 + p99: durations[last * 99 / 100], 40 + max: durations[last], 41 + mean: sum / divisor, 42 + }) 43 + } 44 + } 45 + } 46 + 47 + fn print_result(label: &str, ops: usize, elapsed: Duration, stats: Option<&LatencyStats>) { 48 + let throughput = ops as f64 / elapsed.as_secs_f64(); 49 + match stats { 50 + Some(s) => println!( 51 + "{label}: {throughput:.0} ops/sec, {:.1}ms | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}", 52 + elapsed.as_secs_f64() * 1000.0, 53 + s.p50, 54 + s.p95, 55 + s.p99, 56 + s.max, 57 + s.mean 58 + ), 59 + None => println!( 60 + "{label}: {throughput:.0} ops/sec, {:.1}ms", 61 + elapsed.as_secs_f64() * 1000.0, 62 + ), 63 + } 64 + } 65 + 66 + async fn collect_latencies(handles: Vec<tokio::task::JoinHandle<Vec<Duration>>>) -> Vec<Duration> { 67 + futures::stream::iter(handles) 68 + .fold(Vec::new(), |mut acc, h| async move { 69 + acc.extend(h.await.unwrap()); 70 + acc 71 + }) 72 + .await 73 + } 74 + 75 + fn test_cid(seed: u8) -> CidLink { 76 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 77 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 78 + let c = cid::Cid::new_v1(0x71, mh); 79 + CidLink::from_cid(&c) 80 + } 81 + 82 + fn test_cid_bytes(seed: u8) -> Vec<u8> { 83 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 84 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 85 + cid::Cid::new_v1(0x71, mh).to_bytes() 86 + } 87 + 88 + fn make_rev(n: u64) -> String { 89 + format!("rev{n:010}") 90 + } 91 + 92 + struct BenchHarness { 93 + pool: Arc<HandlerPool>, 94 + metastore: Metastore, 95 + _metastore_dir: tempfile::TempDir, 96 + _eventlog_dir: tempfile::TempDir, 97 + } 98 + 99 + fn cache_size_for_users(user_count: usize) -> u64 { 100 + let estimated_dataset_bytes = user_count as u64 * 10 * 300; 101 + let cache_bytes = estimated_dataset_bytes 102 + .saturating_mul(2) 103 + .max(512 * 1024 * 1024); 104 + let cap = 8u64 * 1024 * 1024 * 1024; 105 + cache_bytes.min(cap) 106 + } 107 + 108 + fn setup(thread_count: usize, user_count: usize) -> BenchHarness { 109 + let cache_bytes = cache_size_for_users(user_count); 110 + println!( 111 + "cache size: {} MB (for {user_count} users)", 112 + cache_bytes / (1024 * 1024) 113 + ); 114 + 115 + let metastore_dir = tempfile::TempDir::new().unwrap(); 116 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 117 + let segments_dir = eventlog_dir.path().join("segments"); 118 + std::fs::create_dir_all(&segments_dir).unwrap(); 119 + 120 + let metastore = Metastore::open( 121 + metastore_dir.path(), 122 + MetastoreConfig { 123 + cache_size_bytes: cache_bytes, 124 + }, 125 + ) 126 + .unwrap(); 127 + 128 + let event_log = EventLog::open( 129 + EventLogConfig { 130 + segments_dir, 131 + ..EventLogConfig::default() 132 + }, 133 + RealIO::new(), 134 + ) 135 + .unwrap(); 136 + 137 + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( 138 + event_log, 139 + ))); 140 + 141 + let pool = Arc::new(HandlerPool::spawn::<RealIO>( 142 + metastore.clone(), 143 + bridge, 144 + None, 145 + Some(thread_count), 146 + )); 147 + 148 + BenchHarness { 149 + pool, 150 + metastore, 151 + _metastore_dir: metastore_dir, 152 + _eventlog_dir: eventlog_dir, 153 + } 154 + } 155 + 156 + fn compact_and_report(metastore: &Metastore) { 157 + println!("running major compaction..."); 158 + let start = Instant::now(); 159 + metastore.major_compact().unwrap(); 160 + println!( 161 + "major compaction complete in {:.1}s", 162 + start.elapsed().as_secs_f64() 163 + ); 164 + } 165 + 166 + struct UserInfo { 167 + user_id: Uuid, 168 + did: Did, 169 + } 170 + 171 + async fn seed_users(pool: &HandlerPool, count: usize) -> Vec<UserInfo> { 172 + let users: Vec<UserInfo> = (0..count) 173 + .map(|i| { 174 + let user_id = Uuid::new_v4(); 175 + UserInfo { 176 + did: Did::from(format!("did:plc:scale{i:06x}{}", user_id.as_simple())), 177 + user_id, 178 + } 179 + }) 180 + .collect(); 181 + 182 + let batch_size = 500; 183 + let batches: Vec<&[UserInfo]> = users.chunks(batch_size).collect(); 184 + let total_batches = batches.len(); 185 + 186 + let start = Instant::now(); 187 + futures::stream::iter(batches.into_iter().enumerate()) 188 + .fold((), |(), (batch_idx, batch)| async move { 189 + futures::stream::iter(batch.iter()) 190 + .fold((), |(), user| async { 191 + let (tx, rx) = oneshot::channel(); 192 + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 193 + user_id: user.user_id, 194 + did: user.did.clone(), 195 + handle: Handle::from(format!( 196 + "u{}.scale.invalid", 197 + user.user_id.as_simple() 198 + )), 199 + repo_root_cid: test_cid(1), 200 + repo_rev: "rev0000000000".to_string(), 201 + tx, 202 + })) 203 + .unwrap(); 204 + rx.await.unwrap().unwrap(); 205 + }) 206 + .await; 207 + if (batch_idx + 1) % 20 == 0 || batch_idx + 1 == total_batches { 208 + println!( 209 + "seeded {}/{} users, {:.1}s", 210 + (batch_idx + 1) * batch_size, 211 + count, 212 + start.elapsed().as_secs_f64() 213 + ); 214 + } 215 + }) 216 + .await; 217 + println!( 218 + "seeded {} users in {:.1}s, {:.0} users/sec", 219 + count, 220 + start.elapsed().as_secs_f64(), 221 + count as f64 / start.elapsed().as_secs_f64() 222 + ); 223 + 224 + users 225 + } 226 + 227 + async fn seed_records_for_user(pool: &HandlerPool, user: &UserInfo, record_count: usize) { 228 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 229 + let record_upserts: Vec<RecordUpsert> = (0..record_count) 230 + .map(|i| RecordUpsert { 231 + collection: collection.clone(), 232 + rkey: Rkey::from(format!("rec{i:08}")), 233 + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), 234 + }) 235 + .collect(); 236 + 237 + let new_block_cids: Vec<Vec<u8>> = (0..record_count) 238 + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) 239 + .collect(); 240 + 241 + let input = ApplyCommitInput { 242 + user_id: user.user_id, 243 + did: user.did.clone(), 244 + expected_root_cid: None, 245 + new_root_cid: test_cid(2), 246 + new_rev: make_rev(1), 247 + new_block_cids, 248 + obsolete_block_cids: vec![], 249 + record_upserts, 250 + record_deletes: vec![], 251 + backlinks_to_add: vec![], 252 + backlinks_to_remove: vec![], 253 + commit_event: CommitEventData { 254 + did: user.did.clone(), 255 + event_type: RepoEventType::Commit, 256 + commit_cid: Some(test_cid(2)), 257 + prev_cid: None, 258 + ops: None, 259 + blobs: None, 260 + blocks_cids: None, 261 + prev_data_cid: None, 262 + rev: Some(make_rev(1)), 263 + }, 264 + }; 265 + 266 + let (tx, rx) = oneshot::channel(); 267 + pool.send(MetastoreRequest::Commit(Box::new( 268 + CommitRequest::ApplyCommit { 269 + input: Box::new(input), 270 + tx, 271 + }, 272 + ))) 273 + .unwrap(); 274 + rx.await.unwrap().unwrap(); 275 + } 276 + 277 + async fn seed_all_records(pool: &Arc<HandlerPool>, users: &[UserInfo], records_per_user: usize) { 278 + let start = Instant::now(); 279 + let total = users.len(); 280 + let chunk_size = 500; 281 + let chunks: Vec<&[UserInfo]> = users.chunks(chunk_size).collect(); 282 + let total_chunks = chunks.len(); 283 + 284 + futures::stream::iter(chunks.into_iter().enumerate()) 285 + .fold((), |(), (chunk_idx, chunk)| { 286 + let pool = Arc::clone(pool); 287 + async move { 288 + futures::stream::iter(chunk.iter()) 289 + .fold((), |(), user| { 290 + let pool = &pool; 291 + async move { 292 + seed_records_for_user(pool, user, records_per_user).await; 293 + } 294 + }) 295 + .await; 296 + if (chunk_idx + 1) % 20 == 0 || chunk_idx + 1 == total_chunks { 297 + println!( 298 + "seeded records for {}/{} users, {:.1}s", 299 + (chunk_idx + 1) * chunk_size, 300 + total, 301 + start.elapsed().as_secs_f64() 302 + ); 303 + } 304 + } 305 + }) 306 + .await; 307 + println!( 308 + "seeded {} records across {} users in {:.1}s", 309 + total * records_per_user, 310 + total, 311 + start.elapsed().as_secs_f64() 312 + ); 313 + } 314 + 315 + async fn bench_single_user_commit(pool: &Arc<HandlerPool>, user: &UserInfo, ops: usize) { 316 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 317 + let start = Instant::now(); 318 + let mut latencies: Vec<Duration> = Vec::with_capacity(ops); 319 + 320 + futures::stream::iter(0..ops) 321 + .fold(&mut latencies, |latencies, i| { 322 + let pool = &pool; 323 + let user = &user; 324 + let collection = &collection; 325 + async move { 326 + let rev_n = (i + 100) as u64; 327 + let cid_seed = ((i * 7 + 42) & 0xFF) as u8; 328 + let input = ApplyCommitInput { 329 + user_id: user.user_id, 330 + did: user.did.clone(), 331 + expected_root_cid: None, 332 + new_root_cid: test_cid(cid_seed), 333 + new_rev: make_rev(rev_n), 334 + new_block_cids: vec![test_cid_bytes(cid_seed)], 335 + obsolete_block_cids: vec![], 336 + record_upserts: vec![RecordUpsert { 337 + collection: collection.clone(), 338 + rkey: Rkey::from(format!("new{rev_n:010}")), 339 + cid: test_cid(cid_seed), 340 + }], 341 + record_deletes: vec![], 342 + backlinks_to_add: vec![], 343 + backlinks_to_remove: vec![], 344 + commit_event: CommitEventData { 345 + did: user.did.clone(), 346 + event_type: RepoEventType::Commit, 347 + commit_cid: Some(test_cid(cid_seed)), 348 + prev_cid: None, 349 + ops: None, 350 + blobs: None, 351 + blocks_cids: None, 352 + prev_data_cid: None, 353 + rev: Some(make_rev(rev_n)), 354 + }, 355 + }; 356 + let t = Instant::now(); 357 + let (tx, rx) = oneshot::channel(); 358 + pool.send(MetastoreRequest::Commit(Box::new( 359 + CommitRequest::ApplyCommit { 360 + input: Box::new(input), 361 + tx, 362 + }, 363 + ))) 364 + .unwrap(); 365 + rx.await.unwrap().unwrap(); 366 + latencies.push(t.elapsed()); 367 + latencies 368 + } 369 + }) 370 + .await; 371 + 372 + let elapsed = start.elapsed(); 373 + let stats = compute_stats(&mut latencies); 374 + print_result("single-user commit", ops, elapsed, stats.as_ref()); 375 + } 376 + 377 + async fn bench_multi_user_commit( 378 + pool: &Arc<HandlerPool>, 379 + users: &[UserInfo], 380 + concurrency: usize, 381 + ops_per_task: usize, 382 + ) { 383 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 384 + let active_users: Vec<&UserInfo> = users.iter().take(concurrency).collect(); 385 + 386 + let start = Instant::now(); 387 + let handles: Vec<_> = active_users 388 + .iter() 389 + .enumerate() 390 + .map(|(task_id, user)| { 391 + let pool = Arc::clone(pool); 392 + let user_id = user.user_id; 393 + let did = user.did.clone(); 394 + let collection = collection.clone(); 395 + tokio::spawn(async move { 396 + futures::stream::iter(0..ops_per_task) 397 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 398 + let pool = &pool; 399 + let did = &did; 400 + let collection = &collection; 401 + async move { 402 + let rev_n = (task_id * ops_per_task + i + 200) as u64; 403 + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; 404 + let input = ApplyCommitInput { 405 + user_id, 406 + did: did.clone(), 407 + expected_root_cid: None, 408 + new_root_cid: test_cid(cid_seed), 409 + new_rev: make_rev(rev_n), 410 + new_block_cids: vec![test_cid_bytes(cid_seed)], 411 + obsolete_block_cids: vec![], 412 + record_upserts: vec![RecordUpsert { 413 + collection: collection.clone(), 414 + rkey: Rkey::from(format!("mu{rev_n:010}")), 415 + cid: test_cid(cid_seed), 416 + }], 417 + record_deletes: vec![], 418 + backlinks_to_add: vec![], 419 + backlinks_to_remove: vec![], 420 + commit_event: CommitEventData { 421 + did: did.clone(), 422 + event_type: RepoEventType::Commit, 423 + commit_cid: Some(test_cid(cid_seed)), 424 + prev_cid: None, 425 + ops: None, 426 + blobs: None, 427 + blocks_cids: None, 428 + prev_data_cid: None, 429 + rev: Some(make_rev(rev_n)), 430 + }, 431 + }; 432 + let t = Instant::now(); 433 + let (tx, rx) = oneshot::channel(); 434 + pool.send(MetastoreRequest::Commit(Box::new( 435 + CommitRequest::ApplyCommit { 436 + input: Box::new(input), 437 + tx, 438 + }, 439 + ))) 440 + .unwrap(); 441 + rx.await.unwrap().unwrap(); 442 + latencies.push(t.elapsed()); 443 + latencies 444 + } 445 + }) 446 + .await 447 + }) 448 + }) 449 + .collect(); 450 + 451 + let mut all_latencies = collect_latencies(handles).await; 452 + let elapsed = start.elapsed(); 453 + let total_ops = concurrency * ops_per_task; 454 + let stats = compute_stats(&mut all_latencies); 455 + print_result( 456 + &format!("multi-user commit ({concurrency} writers)"), 457 + total_ops, 458 + elapsed, 459 + stats.as_ref(), 460 + ); 461 + } 462 + 463 + async fn bench_list_records_at_scale( 464 + pool: &Arc<HandlerPool>, 465 + users: &[UserInfo], 466 + concurrency: usize, 467 + ops_per_task: usize, 468 + ) { 469 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 470 + let user_count = users.len(); 471 + 472 + let start = Instant::now(); 473 + let handles: Vec<_> = (0..concurrency) 474 + .map(|task_id| { 475 + let pool = Arc::clone(pool); 476 + let collection = collection.clone(); 477 + let users: Vec<(Uuid, Did)> = 478 + users.iter().map(|u| (u.user_id, u.did.clone())).collect(); 479 + tokio::spawn(async move { 480 + futures::stream::iter(0..ops_per_task) 481 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 482 + let pool = &pool; 483 + let collection = &collection; 484 + let users = &users; 485 + async move { 486 + let idx = (task_id * 997 + i * 31) % user_count; 487 + let (user_id, _) = &users[idx]; 488 + let t = Instant::now(); 489 + let (tx, rx) = oneshot::channel(); 490 + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { 491 + repo_id: *user_id, 492 + collection: collection.clone(), 493 + cursor: None, 494 + limit: 50, 495 + reverse: false, 496 + rkey_start: None, 497 + rkey_end: None, 498 + tx, 499 + })) 500 + .unwrap(); 501 + let _result = rx.await.unwrap().unwrap(); 502 + latencies.push(t.elapsed()); 503 + latencies 504 + } 505 + }) 506 + .await 507 + }) 508 + }) 509 + .collect(); 510 + 511 + let mut all_latencies = collect_latencies(handles).await; 512 + let elapsed = start.elapsed(); 513 + let total_ops = concurrency * ops_per_task; 514 + let stats = compute_stats(&mut all_latencies); 515 + print_result( 516 + &format!("listRecords ({concurrency} readers)"), 517 + total_ops, 518 + elapsed, 519 + stats.as_ref(), 520 + ); 521 + } 522 + 523 + async fn bench_get_record_at_scale( 524 + pool: &Arc<HandlerPool>, 525 + users: &[UserInfo], 526 + concurrency: usize, 527 + ops_per_task: usize, 528 + ) { 529 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 530 + let user_count = users.len(); 531 + let records_per_user = 10usize; 532 + 533 + let start = Instant::now(); 534 + let handles: Vec<_> = (0..concurrency) 535 + .map(|task_id| { 536 + let pool = Arc::clone(pool); 537 + let collection = collection.clone(); 538 + let users: Vec<Uuid> = users.iter().map(|u| u.user_id).collect(); 539 + tokio::spawn(async move { 540 + futures::stream::iter(0..ops_per_task) 541 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 542 + let pool = &pool; 543 + let collection = &collection; 544 + let users = &users; 545 + async move { 546 + let user_idx = (task_id * 997 + i * 31) % user_count; 547 + let rec_idx = (task_id * 13 + i * 7) % records_per_user; 548 + let rkey = Rkey::from(format!("rec{rec_idx:08}")); 549 + let t = Instant::now(); 550 + let (tx, rx) = oneshot::channel(); 551 + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { 552 + repo_id: users[user_idx], 553 + collection: collection.clone(), 554 + rkey, 555 + tx, 556 + })) 557 + .unwrap(); 558 + let _result = rx.await.unwrap().unwrap(); 559 + latencies.push(t.elapsed()); 560 + latencies 561 + } 562 + }) 563 + .await 564 + }) 565 + }) 566 + .collect(); 567 + 568 + let mut all_latencies = collect_latencies(handles).await; 569 + let elapsed = start.elapsed(); 570 + let total_ops = concurrency * ops_per_task; 571 + let stats = compute_stats(&mut all_latencies); 572 + print_result( 573 + &format!("getRecordCid ({concurrency} readers)"), 574 + total_ops, 575 + elapsed, 576 + stats.as_ref(), 577 + ); 578 + } 579 + 580 + async fn pg_seed_users(pg: &sqlx::PgPool, count: usize) -> Vec<UserInfo> { 581 + let users: Vec<UserInfo> = (0..count) 582 + .map(|i| { 583 + let user_id = Uuid::new_v4(); 584 + UserInfo { 585 + did: Did::from(format!("did:plc:pgscale{i:06x}{}", user_id.as_simple())), 586 + user_id, 587 + } 588 + }) 589 + .collect(); 590 + 591 + let start = Instant::now(); 592 + let batch_size = 500; 593 + let batches: Vec<&[UserInfo]> = users.chunks(batch_size).collect(); 594 + let total_batches = batches.len(); 595 + 596 + futures::stream::iter(batches.into_iter().enumerate()) 597 + .fold((), |(), (batch_idx, batch)| async move { 598 + futures::stream::iter(batch.iter()) 599 + .fold((), |(), user| async { 600 + sqlx::query( 601 + "INSERT INTO users (id, handle, did) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", 602 + ) 603 + .bind(user.user_id) 604 + .bind(format!("u{}.pgscale.invalid", user.user_id.as_simple())) 605 + .bind(user.did.as_str()) 606 + .execute(pg) 607 + .await 608 + .unwrap(); 609 + 610 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); 611 + let handle = Handle::from(format!( 612 + "u{}.pgscale.invalid", 613 + user.user_id.as_simple() 614 + )); 615 + repo.create_repo(user.user_id, &user.did, &handle, &test_cid(1), "rev0000000000") 616 + .await 617 + .unwrap(); 618 + }) 619 + .await; 620 + if (batch_idx + 1) % 20 == 0 || batch_idx + 1 == total_batches { 621 + println!( 622 + "seeded {}/{} postgres users, {:.1}s", 623 + (batch_idx + 1) * batch_size, 624 + count, 625 + start.elapsed().as_secs_f64() 626 + ); 627 + } 628 + }) 629 + .await; 630 + println!( 631 + "seeded {} postgres users in {:.1}s, {:.0} users/sec", 632 + count, 633 + start.elapsed().as_secs_f64(), 634 + count as f64 / start.elapsed().as_secs_f64() 635 + ); 636 + 637 + users 638 + } 639 + 640 + async fn pg_seed_all_records(pg: &sqlx::PgPool, users: &[UserInfo], records_per_user: usize) { 641 + let start = Instant::now(); 642 + let total = users.len(); 643 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 644 + let chunk_size = 500; 645 + let chunks: Vec<&[UserInfo]> = users.chunks(chunk_size).collect(); 646 + let total_chunks = chunks.len(); 647 + 648 + futures::stream::iter(chunks.into_iter().enumerate()) 649 + .fold((), |(), (chunk_idx, chunk)| { 650 + let collection = collection.clone(); 651 + let pg = pg.clone(); 652 + async move { 653 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 654 + futures::stream::iter(chunk.iter()) 655 + .fold((), |(), user| { 656 + let repo = &repo; 657 + let collection = &collection; 658 + async move { 659 + let collections: Vec<Nsid> = 660 + (0..records_per_user).map(|_| collection.clone()).collect(); 661 + let rkeys: Vec<Rkey> = (0..records_per_user) 662 + .map(|i| Rkey::from(format!("rec{i:08}"))) 663 + .collect(); 664 + let cids: Vec<CidLink> = (0..records_per_user) 665 + .map(|i| test_cid(((i * 7 + 3) & 0xFF) as u8)) 666 + .collect(); 667 + repo.upsert_records( 668 + user.user_id, 669 + &collections, 670 + &rkeys, 671 + &cids, 672 + "rev0000000001", 673 + ) 674 + .await 675 + .unwrap(); 676 + } 677 + }) 678 + .await; 679 + if (chunk_idx + 1) % 20 == 0 || chunk_idx + 1 == total_chunks { 680 + println!( 681 + "seeded records for {}/{} postgres users, {:.1}s", 682 + (chunk_idx + 1) * chunk_size, 683 + total, 684 + start.elapsed().as_secs_f64() 685 + ); 686 + } 687 + } 688 + }) 689 + .await; 690 + println!( 691 + "seeded {} postgres records in {:.1}s", 692 + total * records_per_user, 693 + start.elapsed().as_secs_f64() 694 + ); 695 + } 696 + 697 + async fn bench_pg_single_user_commit(pg: &sqlx::PgPool, user: &UserInfo, ops: usize) { 698 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); 699 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 700 + let start = Instant::now(); 701 + let mut latencies: Vec<Duration> = Vec::with_capacity(ops); 702 + 703 + futures::stream::iter(0..ops) 704 + .fold(&mut latencies, |latencies, i| { 705 + let repo = &repo; 706 + let user = &user; 707 + let collection = &collection; 708 + async move { 709 + let rev_n = (i + 100) as u64; 710 + let cid_seed = ((i * 7 + 42) & 0xFF) as u8; 711 + let rkey = Rkey::from(format!("new{rev_n:010}")); 712 + let t = Instant::now(); 713 + repo.upsert_records( 714 + user.user_id, 715 + std::slice::from_ref(collection), 716 + &[rkey], 717 + &[test_cid(cid_seed)], 718 + &make_rev(rev_n), 719 + ) 720 + .await 721 + .unwrap(); 722 + latencies.push(t.elapsed()); 723 + latencies 724 + } 725 + }) 726 + .await; 727 + 728 + let elapsed = start.elapsed(); 729 + let stats = compute_stats(&mut latencies); 730 + print_result("single-user commit", ops, elapsed, stats.as_ref()); 731 + } 732 + 733 + async fn bench_pg_multi_user_commit( 734 + pg: &sqlx::PgPool, 735 + users: &[UserInfo], 736 + concurrency: usize, 737 + ops_per_task: usize, 738 + ) { 739 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 740 + let active_users: Vec<&UserInfo> = users.iter().take(concurrency).collect(); 741 + 742 + let start = Instant::now(); 743 + let handles: Vec<_> = active_users 744 + .iter() 745 + .enumerate() 746 + .map(|(task_id, user)| { 747 + let pg = pg.clone(); 748 + let user_id = user.user_id; 749 + let collection = collection.clone(); 750 + tokio::spawn(async move { 751 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 752 + futures::stream::iter(0..ops_per_task) 753 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 754 + let repo = &repo; 755 + let collection = &collection; 756 + async move { 757 + let rev_n = (task_id * ops_per_task + i + 200) as u64; 758 + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; 759 + let rkey = Rkey::from(format!("mu{rev_n:010}")); 760 + let t = Instant::now(); 761 + repo.upsert_records( 762 + user_id, 763 + std::slice::from_ref(collection), 764 + &[rkey], 765 + &[test_cid(cid_seed)], 766 + &make_rev(rev_n), 767 + ) 768 + .await 769 + .unwrap(); 770 + latencies.push(t.elapsed()); 771 + latencies 772 + } 773 + }) 774 + .await 775 + }) 776 + }) 777 + .collect(); 778 + 779 + let mut all_latencies = collect_latencies(handles).await; 780 + let elapsed = start.elapsed(); 781 + let total_ops = concurrency * ops_per_task; 782 + let stats = compute_stats(&mut all_latencies); 783 + print_result( 784 + &format!("multi-user commit ({concurrency} writers)"), 785 + total_ops, 786 + elapsed, 787 + stats.as_ref(), 788 + ); 789 + } 790 + 791 + async fn bench_pg_list_records( 792 + pg: &sqlx::PgPool, 793 + users: &[UserInfo], 794 + concurrency: usize, 795 + ops_per_task: usize, 796 + ) { 797 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 798 + let user_count = users.len(); 799 + 800 + let start = Instant::now(); 801 + let handles: Vec<_> = (0..concurrency) 802 + .map(|task_id| { 803 + let pg = pg.clone(); 804 + let collection = collection.clone(); 805 + let user_ids: Vec<Uuid> = users.iter().map(|u| u.user_id).collect(); 806 + tokio::spawn(async move { 807 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 808 + futures::stream::iter(0..ops_per_task) 809 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 810 + let repo = &repo; 811 + let collection = &collection; 812 + let user_ids = &user_ids; 813 + async move { 814 + let idx = (task_id * 997 + i * 31) % user_count; 815 + let t = Instant::now(); 816 + let _result = repo 817 + .list_records( 818 + user_ids[idx], 819 + collection, 820 + None, 821 + 50, 822 + false, 823 + None, 824 + None, 825 + ) 826 + .await 827 + .unwrap(); 828 + latencies.push(t.elapsed()); 829 + latencies 830 + } 831 + }) 832 + .await 833 + }) 834 + }) 835 + .collect(); 836 + 837 + let mut all_latencies = collect_latencies(handles).await; 838 + let elapsed = start.elapsed(); 839 + let total_ops = concurrency * ops_per_task; 840 + let stats = compute_stats(&mut all_latencies); 841 + print_result( 842 + &format!("listRecords ({concurrency} readers)"), 843 + total_ops, 844 + elapsed, 845 + stats.as_ref(), 846 + ); 847 + } 848 + 849 + async fn bench_pg_get_record( 850 + pg: &sqlx::PgPool, 851 + users: &[UserInfo], 852 + concurrency: usize, 853 + ops_per_task: usize, 854 + ) { 855 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 856 + let user_count = users.len(); 857 + let records_per_user = 10usize; 858 + 859 + let start = Instant::now(); 860 + let handles: Vec<_> = (0..concurrency) 861 + .map(|task_id| { 862 + let pg = pg.clone(); 863 + let collection = collection.clone(); 864 + let user_ids: Vec<Uuid> = users.iter().map(|u| u.user_id).collect(); 865 + tokio::spawn(async move { 866 + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); 867 + futures::stream::iter(0..ops_per_task) 868 + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { 869 + let repo = &repo; 870 + let collection = &collection; 871 + let user_ids = &user_ids; 872 + async move { 873 + let user_idx = (task_id * 997 + i * 31) % user_count; 874 + let rec_idx = (task_id * 13 + i * 7) % records_per_user; 875 + let rkey = Rkey::from(format!("rec{rec_idx:08}")); 876 + let t = Instant::now(); 877 + let _result = repo 878 + .get_record_cid(user_ids[user_idx], collection, &rkey) 879 + .await 880 + .unwrap(); 881 + latencies.push(t.elapsed()); 882 + latencies 883 + } 884 + }) 885 + .await 886 + }) 887 + }) 888 + .collect(); 889 + 890 + let mut all_latencies = collect_latencies(handles).await; 891 + let elapsed = start.elapsed(); 892 + let total_ops = concurrency * ops_per_task; 893 + let stats = compute_stats(&mut all_latencies); 894 + print_result( 895 + &format!("getRecordCid ({concurrency} readers)"), 896 + total_ops, 897 + elapsed, 898 + stats.as_ref(), 899 + ); 900 + } 901 + 902 + async fn setup_pg_bench_schema(pool: &sqlx::PgPool) { 903 + sqlx::query( 904 + "CREATE TABLE IF NOT EXISTS users ( 905 + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), 906 + handle TEXT NOT NULL UNIQUE, 907 + email TEXT, 908 + did TEXT NOT NULL UNIQUE, 909 + password_hash TEXT NOT NULL DEFAULT '', 910 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 911 + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 912 + deactivated_at TIMESTAMPTZ, 913 + invites_disabled BOOLEAN DEFAULT FALSE, 914 + takedown_ref TEXT, 915 + preferred_comms_channel TEXT NOT NULL DEFAULT 'email', 916 + password_reset_code TEXT, 917 + password_reset_code_expires_at TIMESTAMPTZ, 918 + email_verified BOOLEAN NOT NULL DEFAULT FALSE, 919 + two_factor_enabled BOOLEAN NOT NULL DEFAULT FALSE, 920 + discord_id TEXT, 921 + discord_verified BOOLEAN NOT NULL DEFAULT FALSE, 922 + telegram_username TEXT, 923 + telegram_verified BOOLEAN NOT NULL DEFAULT FALSE, 924 + signal_number TEXT, 925 + signal_verified BOOLEAN NOT NULL DEFAULT FALSE, 926 + is_admin BOOLEAN NOT NULL DEFAULT FALSE, 927 + migrated_to_pds TEXT, 928 + migrated_at TIMESTAMPTZ, 929 + preferred_locale TEXT, 930 + signal_uuid TEXT 931 + )", 932 + ) 933 + .execute(pool) 934 + .await 935 + .unwrap(); 936 + 937 + sqlx::query( 938 + "CREATE TABLE IF NOT EXISTS repos ( 939 + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, 940 + repo_root_cid TEXT NOT NULL, 941 + repo_rev TEXT, 942 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 943 + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() 944 + )", 945 + ) 946 + .execute(pool) 947 + .await 948 + .unwrap(); 949 + 950 + sqlx::query( 951 + "CREATE TABLE IF NOT EXISTS records ( 952 + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), 953 + repo_id UUID NOT NULL REFERENCES repos(user_id) ON DELETE CASCADE, 954 + collection TEXT NOT NULL, 955 + rkey TEXT NOT NULL, 956 + record_cid TEXT NOT NULL, 957 + takedown_ref TEXT, 958 + repo_rev TEXT, 959 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 960 + UNIQUE(repo_id, collection, rkey) 961 + )", 962 + ) 963 + .execute(pool) 964 + .await 965 + .unwrap(); 966 + 967 + sqlx::query( 968 + "CREATE TABLE IF NOT EXISTS user_blocks ( 969 + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), 970 + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, 971 + block_cid BYTEA NOT NULL, 972 + repo_rev TEXT, 973 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 974 + UNIQUE(user_id, block_cid) 975 + )", 976 + ) 977 + .execute(pool) 978 + .await 979 + .unwrap(); 980 + } 981 + 982 + #[tokio::main] 983 + async fn main() { 984 + let handler_threads = std::thread::available_parallelism() 985 + .map(|n| n.get().max(2) / 2) 986 + .unwrap_or(2); 987 + println!("-- metastore scale --"); 988 + println!("handler threads: {handler_threads}"); 989 + 990 + let scale_levels: &[usize] = &[1_000, 10_000, 100_000, 300_000]; 991 + let records_per_user = 10; 992 + let bench_ops = 2000; 993 + let concurrency = 100; 994 + let ops_per_writer = 20; 995 + 996 + futures::stream::iter(scale_levels.iter()) 997 + .fold((), |(), &user_count| async move { 998 + println!("-- tranquil-store, {user_count} users, {records_per_user} records each --"); 999 + 1000 + let h = setup(handler_threads, user_count); 1001 + let users = seed_users(&h.pool, user_count).await; 1002 + seed_all_records(&h.pool, &users, records_per_user).await; 1003 + 1004 + compact_and_report(&h.metastore); 1005 + 1006 + bench_single_user_commit(&h.pool, &users[0], bench_ops).await; 1007 + 1008 + let writers = concurrency.min(user_count); 1009 + bench_multi_user_commit(&h.pool, &users, writers, ops_per_writer).await; 1010 + 1011 + let warmup_ops = 50; 1012 + println!("warming read cache, {} ops per task...", warmup_ops); 1013 + bench_list_records_at_scale(&h.pool, &users, concurrency, warmup_ops).await; 1014 + bench_get_record_at_scale(&h.pool, &users, concurrency, warmup_ops).await; 1015 + 1016 + let read_ops = 500; 1017 + println!("measuring reads, {} ops per task...", read_ops); 1018 + bench_list_records_at_scale(&h.pool, &users, concurrency, read_ops).await; 1019 + bench_get_record_at_scale(&h.pool, &users, concurrency, read_ops).await; 1020 + }) 1021 + .await; 1022 + 1023 + let pg_url = match std::env::var("DATABASE_URL") { 1024 + Ok(url) => url, 1025 + Err(_) => { 1026 + println!("set DATABASE_URL for postgres comparison"); 1027 + return; 1028 + } 1029 + }; 1030 + 1031 + let pg_scale_levels: &[usize] = &[1_000, 10_000, 100_000]; 1032 + 1033 + let setup_pg = |max_conns: u32| { 1034 + let url = pg_url.clone(); 1035 + async move { 1036 + sqlx::postgres::PgPoolOptions::new() 1037 + .max_connections(max_conns) 1038 + .acquire_timeout(Duration::from_secs(30)) 1039 + .connect(&url) 1040 + .await 1041 + .unwrap() 1042 + } 1043 + }; 1044 + 1045 + let pg = setup_pg(60).await; 1046 + setup_pg_bench_schema(&pg).await; 1047 + pg.close().await; 1048 + 1049 + futures::stream::iter(pg_scale_levels.iter()) 1050 + .fold((), |(), &user_count| { 1051 + let setup_pg = &setup_pg; 1052 + async move { 1053 + println!("-- postgres, {user_count} users, {records_per_user} records each --"); 1054 + 1055 + let pg = setup_pg(120).await; 1056 + 1057 + let users = pg_seed_users(&pg, user_count).await; 1058 + pg_seed_all_records(&pg, &users, records_per_user).await; 1059 + 1060 + bench_pg_single_user_commit(&pg, &users[0], bench_ops).await; 1061 + 1062 + let writers = concurrency.min(user_count); 1063 + bench_pg_multi_user_commit(&pg, &users, writers, ops_per_writer).await; 1064 + 1065 + bench_pg_list_records(&pg, &users, concurrency, ops_per_writer).await; 1066 + bench_pg_get_record(&pg, &users, concurrency, ops_per_writer).await; 1067 + 1068 + sqlx::query("TRUNCATE user_blocks, records, repos, users CASCADE") 1069 + .execute(&pg) 1070 + .await 1071 + .unwrap(); 1072 + pg.close().await; 1073 + } 1074 + }) 1075 + .await; 1076 + 1077 + let pg = setup_pg(5).await; 1078 + sqlx::query("DROP TABLE IF EXISTS user_blocks, records, repos, users CASCADE") 1079 + .execute(&pg) 1080 + .await 1081 + .unwrap(); 1082 + pg.close().await; 1083 + }
+358
crates/tranquil-store/benches/profile_reads.rs
··· 1 + use std::sync::Arc; 2 + use std::time::{Duration, Instant}; 3 + 4 + use futures::StreamExt; 5 + use tokio::sync::oneshot; 6 + use tranquil_db_traits::{ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType}; 7 + use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; 8 + use uuid::Uuid; 9 + 10 + use tranquil_store::RealIO; 11 + use tranquil_store::eventlog::{EventLog, EventLogConfig}; 12 + use tranquil_store::metastore::handler::{ 13 + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, 14 + }; 15 + use tranquil_store::metastore::{Metastore, MetastoreConfig}; 16 + 17 + fn test_cid(seed: u8) -> CidLink { 18 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 19 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 20 + let c = cid::Cid::new_v1(0x71, mh); 21 + CidLink::from_cid(&c) 22 + } 23 + 24 + fn test_cid_bytes(seed: u8) -> Vec<u8> { 25 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 26 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 27 + cid::Cid::new_v1(0x71, mh).to_bytes() 28 + } 29 + 30 + struct UserInfo { 31 + user_id: Uuid, 32 + did: Did, 33 + } 34 + 35 + async fn seed_users(pool: &HandlerPool, count: usize) -> Vec<UserInfo> { 36 + let users: Vec<UserInfo> = (0..count) 37 + .map(|i| { 38 + let user_id = Uuid::new_v4(); 39 + UserInfo { 40 + did: Did::from(format!("did:plc:prof{i:06x}{}", user_id.as_simple())), 41 + user_id, 42 + } 43 + }) 44 + .collect(); 45 + 46 + let batch_size = 500; 47 + let start = Instant::now(); 48 + let total_batches = count.div_ceil(batch_size); 49 + futures::stream::iter(users.chunks(batch_size).enumerate()) 50 + .fold((), |(), (batch_idx, batch)| async move { 51 + futures::stream::iter(batch.iter()) 52 + .fold((), |(), user| async { 53 + let (tx, rx) = oneshot::channel(); 54 + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 55 + user_id: user.user_id, 56 + did: user.did.clone(), 57 + handle: Handle::from(format!("u{}.prof.invalid", user.user_id.as_simple())), 58 + repo_root_cid: test_cid(1), 59 + repo_rev: "rev0000000000".to_string(), 60 + tx, 61 + })) 62 + .unwrap(); 63 + rx.await.unwrap().unwrap(); 64 + }) 65 + .await; 66 + if (batch_idx + 1) % 100 == 0 || batch_idx + 1 == total_batches { 67 + println!( 68 + "seeded {}/{} users, {:.1}s", 69 + ((batch_idx + 1) * batch_size).min(count), 70 + count, 71 + start.elapsed().as_secs_f64() 72 + ); 73 + } 74 + }) 75 + .await; 76 + println!( 77 + "seeded {} users in {:.1}s, {:.0} users/sec", 78 + count, 79 + start.elapsed().as_secs_f64(), 80 + count as f64 / start.elapsed().as_secs_f64() 81 + ); 82 + users 83 + } 84 + 85 + async fn seed_records(pool: &Arc<HandlerPool>, users: &[UserInfo], records_per_user: usize) { 86 + let start = Instant::now(); 87 + let total = users.len(); 88 + let batch_size = 500; 89 + let total_batches = total.div_ceil(batch_size); 90 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 91 + 92 + futures::stream::iter(users.chunks(batch_size).enumerate()) 93 + .fold((), |(), (chunk_idx, chunk)| { 94 + let pool = Arc::clone(pool); 95 + let collection = collection.clone(); 96 + async move { 97 + futures::stream::iter(chunk.iter()) 98 + .fold((), |(), user| { 99 + let pool = &pool; 100 + let collection = &collection; 101 + async move { 102 + let record_upserts: Vec<RecordUpsert> = (0..records_per_user) 103 + .map(|i| RecordUpsert { 104 + collection: collection.clone(), 105 + rkey: Rkey::from(format!("rec{i:08}")), 106 + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), 107 + }) 108 + .collect(); 109 + let new_block_cids: Vec<Vec<u8>> = (0..records_per_user) 110 + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) 111 + .collect(); 112 + let input = ApplyCommitInput { 113 + user_id: user.user_id, 114 + did: user.did.clone(), 115 + expected_root_cid: None, 116 + new_root_cid: test_cid(2), 117 + new_rev: "rev0000000001".to_string(), 118 + new_block_cids, 119 + obsolete_block_cids: vec![], 120 + record_upserts, 121 + record_deletes: vec![], 122 + backlinks_to_add: vec![], 123 + backlinks_to_remove: vec![], 124 + commit_event: CommitEventData { 125 + did: user.did.clone(), 126 + event_type: RepoEventType::Commit, 127 + commit_cid: Some(test_cid(2)), 128 + prev_cid: None, 129 + ops: None, 130 + blobs: None, 131 + blocks_cids: None, 132 + prev_data_cid: None, 133 + rev: Some("rev0000000001".to_string()), 134 + }, 135 + }; 136 + let (tx, rx) = oneshot::channel(); 137 + pool.send(MetastoreRequest::Commit(Box::new( 138 + CommitRequest::ApplyCommit { 139 + input: Box::new(input), 140 + tx, 141 + }, 142 + ))) 143 + .unwrap(); 144 + rx.await.unwrap().unwrap(); 145 + } 146 + }) 147 + .await; 148 + if (chunk_idx + 1) % 100 == 0 || chunk_idx + 1 == total_batches { 149 + println!( 150 + "seeded records for {}/{} users, {:.1}s", 151 + ((chunk_idx + 1) * batch_size).min(total), 152 + total, 153 + start.elapsed().as_secs_f64() 154 + ); 155 + } 156 + } 157 + }) 158 + .await; 159 + println!( 160 + "seeded {} records across {} users in {:.1}s", 161 + total * records_per_user, 162 + total, 163 + start.elapsed().as_secs_f64() 164 + ); 165 + } 166 + 167 + async fn profile_list_records( 168 + pool: &Arc<HandlerPool>, 169 + user_ids: &Arc<Vec<Uuid>>, 170 + concurrency: usize, 171 + seconds: u64, 172 + ) -> u64 { 173 + let deadline = Instant::now() + Duration::from_secs(seconds); 174 + let user_count = user_ids.len(); 175 + 176 + let handles: Vec<_> = (0..concurrency) 177 + .map(|task_id| { 178 + let pool = Arc::clone(pool); 179 + let user_ids = Arc::clone(user_ids); 180 + tokio::spawn(async move { 181 + futures::stream::unfold(0usize, |i| { 182 + let cont = Instant::now() < deadline; 183 + async move { cont.then_some((i, i + 1)) } 184 + }) 185 + .fold(0u64, |ops, i| { 186 + let pool = &pool; 187 + let user_ids = &user_ids; 188 + async move { 189 + let idx = (task_id * 997 + i * 31) % user_count; 190 + let (tx, rx) = oneshot::channel(); 191 + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { 192 + repo_id: user_ids[idx], 193 + collection: Nsid::from("app.bsky.feed.post".to_string()), 194 + cursor: None, 195 + limit: 50, 196 + reverse: false, 197 + rkey_start: None, 198 + rkey_end: None, 199 + tx, 200 + })) 201 + .unwrap(); 202 + let _ = rx.await.unwrap().unwrap(); 203 + ops + 1 204 + } 205 + }) 206 + .await 207 + }) 208 + }) 209 + .collect(); 210 + 211 + futures::stream::iter(handles) 212 + .fold(0u64, |acc, h| async move { acc + h.await.unwrap() }) 213 + .await 214 + } 215 + 216 + async fn profile_get_record_cid( 217 + pool: &Arc<HandlerPool>, 218 + user_ids: &Arc<Vec<Uuid>>, 219 + concurrency: usize, 220 + seconds: u64, 221 + records_per_user: usize, 222 + ) -> u64 { 223 + let deadline = Instant::now() + Duration::from_secs(seconds); 224 + let user_count = user_ids.len(); 225 + 226 + let handles: Vec<_> = (0..concurrency) 227 + .map(|task_id| { 228 + let pool = Arc::clone(pool); 229 + let user_ids = Arc::clone(user_ids); 230 + tokio::spawn(async move { 231 + futures::stream::unfold(0usize, |i| { 232 + let cont = Instant::now() < deadline; 233 + async move { cont.then_some((i, i + 1)) } 234 + }) 235 + .fold(0u64, |ops, i| { 236 + let pool = &pool; 237 + let user_ids = &user_ids; 238 + async move { 239 + let user_idx = (task_id * 997 + i * 31) % user_count; 240 + let rec_idx = (task_id * 13 + i * 7) % records_per_user; 241 + let rkey = Rkey::from(format!("rec{rec_idx:08}")); 242 + let (tx, rx) = oneshot::channel(); 243 + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { 244 + repo_id: user_ids[user_idx], 245 + collection: Nsid::from("app.bsky.feed.post".to_string()), 246 + rkey, 247 + tx, 248 + })) 249 + .unwrap(); 250 + let _ = rx.await.unwrap().unwrap(); 251 + ops + 1 252 + } 253 + }) 254 + .await 255 + }) 256 + }) 257 + .collect(); 258 + 259 + futures::stream::iter(handles) 260 + .fold(0u64, |acc, h| async move { acc + h.await.unwrap() }) 261 + .await 262 + } 263 + 264 + #[tokio::main] 265 + async fn main() { 266 + let user_count = std::env::var("PROFILE_USERS") 267 + .ok() 268 + .and_then(|s| s.parse().ok()) 269 + .unwrap_or(300_000usize); 270 + let records_per_user = 10; 271 + let profile_seconds = std::env::var("PROFILE_SECONDS") 272 + .ok() 273 + .and_then(|s| s.parse().ok()) 274 + .unwrap_or(30u64); 275 + let concurrency = 100usize; 276 + 277 + let handler_threads = std::thread::available_parallelism() 278 + .map(|n| n.get().max(2) / 2) 279 + .unwrap_or(2); 280 + 281 + println!("-- profile reads --"); 282 + println!("handler threads: {handler_threads}"); 283 + println!("{user_count} users, {records_per_user} records each, {profile_seconds}s per phase"); 284 + 285 + let metastore_dir = tempfile::TempDir::new().unwrap(); 286 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 287 + let segments_dir = eventlog_dir.path().join("segments"); 288 + std::fs::create_dir_all(&segments_dir).unwrap(); 289 + 290 + let cache_bytes = (user_count as u64 * 10 * 300) 291 + .saturating_mul(2) 292 + .clamp(512 * 1024 * 1024, 8 * 1024 * 1024 * 1024); 293 + println!("cache size: {} MB", cache_bytes / (1024 * 1024)); 294 + 295 + let metastore = Metastore::open( 296 + metastore_dir.path(), 297 + MetastoreConfig { 298 + cache_size_bytes: cache_bytes, 299 + }, 300 + ) 301 + .unwrap(); 302 + 303 + let event_log = EventLog::open( 304 + EventLogConfig { 305 + segments_dir, 306 + ..EventLogConfig::default() 307 + }, 308 + RealIO::new(), 309 + ) 310 + .unwrap(); 311 + 312 + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( 313 + event_log, 314 + ))); 315 + 316 + let pool = Arc::new(HandlerPool::spawn::<RealIO>( 317 + metastore.clone(), 318 + bridge, 319 + None, 320 + Some(handler_threads), 321 + )); 322 + 323 + let users = seed_users(&pool, user_count).await; 324 + seed_records(&pool, &users, records_per_user).await; 325 + 326 + println!("running major compaction..."); 327 + let t = Instant::now(); 328 + metastore.major_compact().unwrap(); 329 + println!( 330 + "major compaction complete in {:.1}s", 331 + t.elapsed().as_secs_f64() 332 + ); 333 + 334 + let user_ids: Arc<Vec<Uuid>> = Arc::new(users.iter().map(|u| u.user_id).collect()); 335 + 336 + println!("-- listRecords, {concurrency} readers, {profile_seconds}s --"); 337 + let list_ops = profile_list_records(&pool, &user_ids, concurrency, profile_seconds).await; 338 + println!( 339 + "listRecords: {list_ops} ops, {:.0} ops/sec", 340 + list_ops as f64 / profile_seconds as f64 341 + ); 342 + 343 + println!("-- getRecordCid, {concurrency} readers, {profile_seconds}s --"); 344 + let get_ops = profile_get_record_cid( 345 + &pool, 346 + &user_ids, 347 + concurrency, 348 + profile_seconds, 349 + records_per_user, 350 + ) 351 + .await; 352 + println!( 353 + "getRecordCid: {get_ops} ops, {:.0} ops/sec", 354 + get_ops as f64 / profile_seconds as f64 355 + ); 356 + 357 + println!("-- profile reads complete :D --"); 358 + }
+1
crates/tranquil-store/src/lib.rs
··· 3 3 pub mod fsync_order; 4 4 mod harness; 5 5 mod io; 6 + pub mod metastore; 6 7 mod record; 7 8 #[cfg(any(test, feature = "test-harness"))] 8 9 mod sim;
+257
crates/tranquil-store/src/metastore/backlinks.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use smallvec::SmallVec; 3 + 4 + use super::encoding::KeyBuilder; 5 + use super::keys::{KeyTag, UserHash}; 6 + 7 + use tranquil_db_traits::BacklinkPath; 8 + 9 + const SCHEMA_VERSION: u8 = 1; 10 + 11 + pub fn path_to_discriminant(path: BacklinkPath) -> u8 { 12 + match path { 13 + BacklinkPath::Subject => 0, 14 + BacklinkPath::SubjectUri => 1, 15 + } 16 + } 17 + 18 + pub fn discriminant_to_path(d: u8) -> Option<BacklinkPath> { 19 + match d { 20 + 0 => Some(BacklinkPath::Subject), 21 + 1 => Some(BacklinkPath::SubjectUri), 22 + _ => None, 23 + } 24 + } 25 + 26 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 27 + pub struct BacklinkValue { 28 + pub source_uri: String, 29 + pub path: u8, 30 + } 31 + 32 + impl BacklinkValue { 33 + pub fn serialize(&self) -> Vec<u8> { 34 + let payload = postcard::to_allocvec(self).expect("BacklinkValue serialization cannot fail"); 35 + let mut buf = Vec::with_capacity(1 + payload.len()); 36 + buf.push(SCHEMA_VERSION); 37 + buf.extend_from_slice(&payload); 38 + buf 39 + } 40 + 41 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 42 + let (&version, payload) = bytes.split_first()?; 43 + match version { 44 + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 45 + _ => None, 46 + } 47 + } 48 + } 49 + 50 + pub fn backlink_key( 51 + link_target: &str, 52 + user_hash: UserHash, 53 + collection: &str, 54 + rkey: &str, 55 + ) -> SmallVec<[u8; 128]> { 56 + KeyBuilder::new() 57 + .tag(KeyTag::BACKLINKS) 58 + .string(link_target) 59 + .u64(user_hash.raw()) 60 + .string(collection) 61 + .string(rkey) 62 + .build() 63 + } 64 + 65 + pub fn backlink_target_prefix(link_target: &str) -> SmallVec<[u8; 128]> { 66 + KeyBuilder::new() 67 + .tag(KeyTag::BACKLINKS) 68 + .string(link_target) 69 + .build() 70 + } 71 + 72 + pub fn backlink_target_user_prefix(link_target: &str, user_hash: UserHash) -> SmallVec<[u8; 128]> { 73 + KeyBuilder::new() 74 + .tag(KeyTag::BACKLINKS) 75 + .string(link_target) 76 + .u64(user_hash.raw()) 77 + .build() 78 + } 79 + 80 + pub fn backlink_by_user_key( 81 + user_hash: UserHash, 82 + collection: &str, 83 + rkey: &str, 84 + link_target: &str, 85 + ) -> SmallVec<[u8; 128]> { 86 + KeyBuilder::new() 87 + .tag(KeyTag::BACKLINK_BY_USER) 88 + .u64(user_hash.raw()) 89 + .string(collection) 90 + .string(rkey) 91 + .string(link_target) 92 + .build() 93 + } 94 + 95 + pub fn backlink_by_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 96 + KeyBuilder::new() 97 + .tag(KeyTag::BACKLINK_BY_USER) 98 + .u64(user_hash.raw()) 99 + .build() 100 + } 101 + 102 + pub fn backlink_by_user_record_prefix( 103 + user_hash: UserHash, 104 + collection: &str, 105 + rkey: &str, 106 + ) -> SmallVec<[u8; 128]> { 107 + KeyBuilder::new() 108 + .tag(KeyTag::BACKLINK_BY_USER) 109 + .u64(user_hash.raw()) 110 + .string(collection) 111 + .string(rkey) 112 + .build() 113 + } 114 + 115 + #[cfg(test)] 116 + mod tests { 117 + use super::*; 118 + use crate::metastore::encoding::KeyReader; 119 + 120 + #[test] 121 + fn discriminant_roundtrip() { 122 + assert_eq!( 123 + discriminant_to_path(path_to_discriminant(BacklinkPath::Subject)), 124 + Some(BacklinkPath::Subject) 125 + ); 126 + assert_eq!( 127 + discriminant_to_path(path_to_discriminant(BacklinkPath::SubjectUri)), 128 + Some(BacklinkPath::SubjectUri) 129 + ); 130 + assert_eq!(discriminant_to_path(255), None); 131 + assert_eq!(discriminant_to_path(2), None); 132 + } 133 + 134 + #[test] 135 + fn backlink_value_roundtrip() { 136 + let value = BacklinkValue { 137 + source_uri: "at://did:plc:abc/app.bsky.feed.like/3k2xyz".to_string(), 138 + path: path_to_discriminant(BacklinkPath::SubjectUri), 139 + }; 140 + let bytes = value.serialize(); 141 + let decoded = BacklinkValue::deserialize(&bytes).unwrap(); 142 + assert_eq!(decoded, value); 143 + } 144 + 145 + #[test] 146 + fn schema_version_is_first_byte() { 147 + let value = BacklinkValue { 148 + source_uri: "at://x".to_string(), 149 + path: path_to_discriminant(BacklinkPath::Subject), 150 + }; 151 + let bytes = value.serialize(); 152 + assert_eq!(bytes[0], SCHEMA_VERSION); 153 + } 154 + 155 + #[test] 156 + fn deserialize_rejects_unknown_version() { 157 + let value = BacklinkValue { 158 + source_uri: "at://x".to_string(), 159 + path: path_to_discriminant(BacklinkPath::Subject), 160 + }; 161 + let mut bytes = value.serialize(); 162 + bytes[0] = 99; 163 + assert!(BacklinkValue::deserialize(&bytes).is_none()); 164 + } 165 + 166 + #[test] 167 + fn deserialize_rejects_empty() { 168 + assert!(BacklinkValue::deserialize(&[]).is_none()); 169 + } 170 + 171 + #[test] 172 + fn backlink_key_roundtrip() { 173 + let hash = UserHash::from_raw(0xCAFE_BABE_DEAD_BEEF); 174 + let key = backlink_key( 175 + "at://did:plc:target/app.bsky.feed.post/3k2abc", 176 + hash, 177 + "app.bsky.feed.like", 178 + "3k2xyz", 179 + ); 180 + let mut reader = KeyReader::new(&key); 181 + assert_eq!(reader.tag(), Some(KeyTag::BACKLINKS.raw())); 182 + assert_eq!( 183 + reader.string(), 184 + Some("at://did:plc:target/app.bsky.feed.post/3k2abc".to_string()) 185 + ); 186 + assert_eq!(reader.u64(), Some(0xCAFE_BABE_DEAD_BEEF)); 187 + assert_eq!(reader.string(), Some("app.bsky.feed.like".to_string())); 188 + assert_eq!(reader.string(), Some("3k2xyz".to_string())); 189 + assert!(reader.is_empty()); 190 + } 191 + 192 + #[test] 193 + fn backlink_keys_sort_by_target_then_user_then_collection_then_rkey() { 194 + let h1 = UserHash::from_raw(1); 195 + let h2 = UserHash::from_raw(2); 196 + 197 + let k1 = backlink_key("aaa", h1, "col_a", "r1"); 198 + let k2 = backlink_key("aaa", h1, "col_a", "r2"); 199 + let k3 = backlink_key("aaa", h1, "col_b", "r1"); 200 + let k4 = backlink_key("aaa", h2, "col_a", "r1"); 201 + let k5 = backlink_key("bbb", h1, "col_a", "r1"); 202 + 203 + assert!(k1.as_slice() < k2.as_slice()); 204 + assert!(k2.as_slice() < k3.as_slice()); 205 + assert!(k3.as_slice() < k4.as_slice()); 206 + assert!(k4.as_slice() < k5.as_slice()); 207 + } 208 + 209 + #[test] 210 + fn target_prefix_is_prefix_of_full_key() { 211 + let hash = UserHash::from_raw(42); 212 + let prefix = backlink_target_prefix("did:plc:target"); 213 + let full = backlink_key("did:plc:target", hash, "col", "rk"); 214 + assert!(full.as_slice().starts_with(prefix.as_slice())); 215 + } 216 + 217 + #[test] 218 + fn by_user_key_roundtrip() { 219 + let hash = UserHash::from_raw(0xDEAD_BEEF_1234_5678); 220 + let key = backlink_by_user_key(hash, "app.bsky.feed.like", "3k2abc", "did:plc:target"); 221 + let mut reader = KeyReader::new(&key); 222 + assert_eq!(reader.tag(), Some(KeyTag::BACKLINK_BY_USER.raw())); 223 + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_1234_5678)); 224 + assert_eq!(reader.string(), Some("app.bsky.feed.like".to_string())); 225 + assert_eq!(reader.string(), Some("3k2abc".to_string())); 226 + assert_eq!(reader.string(), Some("did:plc:target".to_string())); 227 + assert!(reader.is_empty()); 228 + } 229 + 230 + #[test] 231 + fn by_user_prefix_is_prefix_of_full_key() { 232 + let hash = UserHash::from_raw(42); 233 + let prefix = backlink_by_user_prefix(hash); 234 + let full = backlink_by_user_key(hash, "col", "rk", "target"); 235 + assert!(full.as_slice().starts_with(prefix.as_slice())); 236 + } 237 + 238 + #[test] 239 + fn by_user_record_prefix_is_prefix_of_full_key() { 240 + let hash = UserHash::from_raw(42); 241 + let prefix = backlink_by_user_record_prefix(hash, "col", "rk"); 242 + let full = backlink_by_user_key(hash, "col", "rk", "target"); 243 + assert!(full.as_slice().starts_with(prefix.as_slice())); 244 + } 245 + 246 + #[test] 247 + fn same_rkey_different_collection_produces_distinct_keys() { 248 + let hash = UserHash::from_raw(42); 249 + let k1 = backlink_key("target", hash, "app.bsky.feed.like", "self"); 250 + let k2 = backlink_key("target", hash, "app.bsky.graph.follow", "self"); 251 + assert_ne!(k1.as_slice(), k2.as_slice()); 252 + 253 + let r1 = backlink_by_user_key(hash, "app.bsky.feed.like", "self", "target"); 254 + let r2 = backlink_by_user_key(hash, "app.bsky.graph.follow", "self", "target"); 255 + assert_ne!(r1.as_slice(), r2.as_slice()); 256 + } 257 + }
+785
crates/tranquil-store/src/metastore/blob_ops.rs
··· 1 + use std::collections::{BTreeMap, BTreeSet}; 2 + use std::ops::Bound; 3 + use std::sync::Arc; 4 + 5 + use fjall::{Database, Keyspace}; 6 + use smallvec::SmallVec; 7 + use uuid::Uuid; 8 + 9 + use super::MetastoreError; 10 + use super::blobs::{BlobMetaValue, blob_by_cid_key, blob_meta_key, blob_user_prefix, blobs_prefix}; 11 + use super::commit_ops::{RecordBlobsValue, record_blobs_user_prefix}; 12 + use super::encoding::{KeyReader, exclusive_upper_bound}; 13 + use super::keys::{KeyTag, UserHash}; 14 + use super::repo_ops::bytes_to_cid_link; 15 + use super::scan::{count_prefix, point_lookup}; 16 + use super::user_hash::UserHashMap; 17 + use tranquil_types::CidLink; 18 + 19 + const DELETE_BATCH_SIZE: usize = 1024; 20 + 21 + pub struct BlobOps { 22 + db: Database, 23 + repo_data: Keyspace, 24 + user_hashes: Arc<UserHashMap>, 25 + } 26 + 27 + impl BlobOps { 28 + pub fn new(db: Database, repo_data: Keyspace, user_hashes: Arc<UserHashMap>) -> Self { 29 + Self { 30 + db, 31 + repo_data, 32 + user_hashes, 33 + } 34 + } 35 + 36 + fn resolve_user_hash(&self, user_id: Uuid) -> Result<UserHash, MetastoreError> { 37 + self.user_hashes 38 + .get(&user_id) 39 + .ok_or(MetastoreError::InvalidInput("unknown user_id")) 40 + } 41 + 42 + pub fn insert_blob( 43 + &self, 44 + cid: &CidLink, 45 + mime_type: &str, 46 + size_bytes: i64, 47 + created_by_user: Uuid, 48 + storage_key: &str, 49 + ) -> Result<Option<CidLink>, MetastoreError> { 50 + if size_bytes < 0 { 51 + return Err(MetastoreError::InvalidInput( 52 + "size_bytes must be non-negative", 53 + )); 54 + } 55 + 56 + let user_hash = self.resolve_user_hash(created_by_user)?; 57 + let cid_str = cid.as_str(); 58 + 59 + let cid_index_key = blob_by_cid_key(cid_str); 60 + let existing = self 61 + .repo_data 62 + .get(cid_index_key.as_slice()) 63 + .map_err(MetastoreError::Fjall)?; 64 + if existing.is_some() { 65 + return Ok(None); 66 + } 67 + 68 + let value = BlobMetaValue { 69 + size_bytes, 70 + mime_type: mime_type.to_owned(), 71 + storage_key: storage_key.to_owned(), 72 + takedown_ref: None, 73 + created_at_ms: chrono::Utc::now().timestamp_millis(), 74 + }; 75 + 76 + let primary_key = blob_meta_key(user_hash, cid_str); 77 + 78 + let mut batch = self.db.batch(); 79 + batch.insert(&self.repo_data, primary_key.as_slice(), value.serialize()); 80 + batch.insert( 81 + &self.repo_data, 82 + cid_index_key.as_slice(), 83 + user_hash.raw().to_be_bytes(), 84 + ); 85 + batch.commit().map_err(MetastoreError::Fjall)?; 86 + 87 + Ok(Some(cid.clone())) 88 + } 89 + 90 + fn lookup_user_hash_by_cid(&self, cid_str: &str) -> Result<Option<UserHash>, MetastoreError> { 91 + let key = blob_by_cid_key(cid_str); 92 + match self 93 + .repo_data 94 + .get(key.as_slice()) 95 + .map_err(MetastoreError::Fjall)? 96 + { 97 + Some(raw) => { 98 + let arr: [u8; 8] = raw 99 + .as_ref() 100 + .try_into() 101 + .map_err(|_| MetastoreError::CorruptData("blob_by_cid value not 8 bytes"))?; 102 + Ok(Some(UserHash::from_raw(u64::from_be_bytes(arr)))) 103 + } 104 + None => Ok(None), 105 + } 106 + } 107 + 108 + fn get_blob_value(&self, cid: &CidLink) -> Result<Option<BlobMetaValue>, MetastoreError> { 109 + let cid_str = cid.as_str(); 110 + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { 111 + Some(h) => h, 112 + None => return Ok(None), 113 + }; 114 + let key = blob_meta_key(user_hash, cid_str); 115 + point_lookup( 116 + &self.repo_data, 117 + key.as_slice(), 118 + BlobMetaValue::deserialize, 119 + "corrupt blob_meta value", 120 + ) 121 + } 122 + 123 + pub fn get_blob_metadata( 124 + &self, 125 + cid: &CidLink, 126 + ) -> Result<Option<tranquil_db_traits::BlobMetadata>, MetastoreError> { 127 + Ok(self 128 + .get_blob_value(cid)? 129 + .map(|v| tranquil_db_traits::BlobMetadata { 130 + storage_key: v.storage_key, 131 + mime_type: v.mime_type, 132 + size_bytes: v.size_bytes, 133 + })) 134 + } 135 + 136 + pub fn get_blob_with_takedown( 137 + &self, 138 + cid: &CidLink, 139 + ) -> Result<Option<tranquil_db_traits::BlobWithTakedown>, MetastoreError> { 140 + Ok(self 141 + .get_blob_value(cid)? 142 + .map(|v| tranquil_db_traits::BlobWithTakedown { 143 + cid: cid.clone(), 144 + takedown_ref: v.takedown_ref, 145 + })) 146 + } 147 + 148 + pub fn get_blob_storage_key(&self, cid: &CidLink) -> Result<Option<String>, MetastoreError> { 149 + Ok(self.get_blob_value(cid)?.map(|v| v.storage_key)) 150 + } 151 + 152 + pub fn list_blobs_by_user( 153 + &self, 154 + user_id: Uuid, 155 + cursor: Option<&str>, 156 + limit: usize, 157 + ) -> Result<Vec<CidLink>, MetastoreError> { 158 + let user_hash = self.resolve_user_hash(user_id)?; 159 + let prefix = blob_user_prefix(user_hash); 160 + let upper = exclusive_upper_bound(prefix.as_slice()) 161 + .expect("blob user prefix always contains non-0xFF bytes"); 162 + 163 + let range_start: SmallVec<[u8; 128]> = match cursor { 164 + Some(c) => { 165 + let mut cursor_key = blob_meta_key(user_hash, c); 166 + cursor_key.push(0x00); 167 + cursor_key 168 + } 169 + None => prefix, 170 + }; 171 + 172 + self.repo_data 173 + .range(range_start.as_slice()..upper.as_slice()) 174 + .map(|guard| { 175 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 176 + parse_blob_cid_from_key(key_bytes.as_ref()) 177 + }) 178 + .take(limit) 179 + .collect() 180 + } 181 + 182 + pub fn count_blobs_by_user(&self, user_id: Uuid) -> Result<i64, MetastoreError> { 183 + let user_hash = self.resolve_user_hash(user_id)?; 184 + let prefix = blob_user_prefix(user_hash); 185 + count_prefix(&self.repo_data, prefix.as_slice()) 186 + } 187 + 188 + pub fn sum_blob_storage(&self) -> Result<i64, MetastoreError> { 189 + let prefix = blobs_prefix(); 190 + self.repo_data 191 + .prefix(prefix.as_slice()) 192 + .try_fold(0i64, |acc, guard| { 193 + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 194 + let value = BlobMetaValue::deserialize(&val_bytes) 195 + .ok_or(MetastoreError::CorruptData("corrupt blob_meta in sum"))?; 196 + Ok::<_, MetastoreError>(acc.saturating_add(value.size_bytes)) 197 + }) 198 + } 199 + 200 + pub fn update_blob_takedown( 201 + &self, 202 + cid: &CidLink, 203 + takedown_ref: Option<&str>, 204 + ) -> Result<bool, MetastoreError> { 205 + let cid_str = cid.as_str(); 206 + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { 207 + Some(h) => h, 208 + None => return Ok(false), 209 + }; 210 + let key = blob_meta_key(user_hash, cid_str); 211 + let mut value = match point_lookup( 212 + &self.repo_data, 213 + key.as_slice(), 214 + BlobMetaValue::deserialize, 215 + "corrupt blob_meta value", 216 + )? { 217 + Some(v) => v, 218 + None => return Ok(false), 219 + }; 220 + 221 + value.takedown_ref = takedown_ref.map(str::to_owned); 222 + let mut batch = self.db.batch(); 223 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 224 + batch.commit().map_err(MetastoreError::Fjall)?; 225 + Ok(true) 226 + } 227 + 228 + pub fn delete_blob_by_cid(&self, cid: &CidLink) -> Result<bool, MetastoreError> { 229 + let cid_str = cid.as_str(); 230 + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { 231 + Some(h) => h, 232 + None => return Ok(false), 233 + }; 234 + 235 + let primary_key = blob_meta_key(user_hash, cid_str); 236 + let exists = self 237 + .repo_data 238 + .get(primary_key.as_slice()) 239 + .map_err(MetastoreError::Fjall)? 240 + .is_some(); 241 + if !exists { 242 + return Ok(false); 243 + } 244 + 245 + let cid_index_key = blob_by_cid_key(cid_str); 246 + 247 + let mut batch = self.db.batch(); 248 + batch.remove(&self.repo_data, primary_key.as_slice()); 249 + batch.remove(&self.repo_data, cid_index_key.as_slice()); 250 + batch.commit().map_err(MetastoreError::Fjall)?; 251 + 252 + Ok(true) 253 + } 254 + 255 + pub fn delete_blobs_by_user(&self, user_id: Uuid) -> Result<u64, MetastoreError> { 256 + let user_hash = self.resolve_user_hash(user_id)?; 257 + let prefix = blob_user_prefix(user_hash); 258 + let user_hash_bytes = user_hash.raw().to_be_bytes(); 259 + 260 + let (final_batch, remaining, total) = self 261 + .repo_data 262 + .prefix(prefix.as_slice()) 263 + .map(|guard| { 264 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 265 + parse_blob_cid_from_key(key_bytes.as_ref()).map(|c| c.as_str().to_owned()) 266 + }) 267 + .try_fold( 268 + (self.db.batch(), 0usize, 0u64), 269 + |(mut batch, count, total), entry: Result<_, MetastoreError>| { 270 + let cid_str = entry?; 271 + batch.remove( 272 + &self.repo_data, 273 + blob_meta_key(user_hash, &cid_str).as_slice(), 274 + ); 275 + let cid_index_key = blob_by_cid_key(&cid_str); 276 + let owns_cid = self 277 + .repo_data 278 + .get(cid_index_key.as_slice()) 279 + .map_err(MetastoreError::Fjall)? 280 + .is_some_and(|raw| raw.as_ref() == user_hash_bytes); 281 + if owns_cid { 282 + batch.remove(&self.repo_data, cid_index_key.as_slice()); 283 + } 284 + let new_count = count + 1; 285 + if new_count >= DELETE_BATCH_SIZE { 286 + batch.commit().map_err(MetastoreError::Fjall)?; 287 + let flushed = u64::try_from(new_count).unwrap_or(u64::MAX); 288 + Ok::<_, MetastoreError>((self.db.batch(), 0, total.saturating_add(flushed))) 289 + } else { 290 + Ok((batch, new_count, total)) 291 + } 292 + }, 293 + )?; 294 + 295 + if remaining > 0 { 296 + final_batch.commit().map_err(MetastoreError::Fjall)?; 297 + let flushed = u64::try_from(remaining).unwrap_or(u64::MAX); 298 + Ok(total.saturating_add(flushed)) 299 + } else { 300 + Ok(total) 301 + } 302 + } 303 + 304 + pub fn get_blob_storage_keys_by_user( 305 + &self, 306 + user_id: Uuid, 307 + ) -> Result<Vec<String>, MetastoreError> { 308 + let user_hash = self.resolve_user_hash(user_id)?; 309 + let prefix = blob_user_prefix(user_hash); 310 + 311 + self.repo_data 312 + .prefix(prefix.as_slice()) 313 + .map(|guard| { 314 + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 315 + let value = BlobMetaValue::deserialize(&val_bytes) 316 + .ok_or(MetastoreError::CorruptData("corrupt blob_meta value"))?; 317 + Ok(value.storage_key) 318 + }) 319 + .collect() 320 + } 321 + 322 + pub fn list_missing_blobs( 323 + &self, 324 + repo_id: Uuid, 325 + cursor: Option<&str>, 326 + limit: usize, 327 + ) -> Result<Vec<tranquil_db_traits::MissingBlobInfo>, MetastoreError> { 328 + let user_hash = self.resolve_user_hash(repo_id)?; 329 + let rb_prefix = record_blobs_user_prefix(user_hash); 330 + 331 + let missing: BTreeMap<String, String> = self 332 + .repo_data 333 + .prefix(rb_prefix.as_slice()) 334 + .try_fold(BTreeMap::new(), |mut acc, guard| { 335 + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 336 + let record_uri = parse_record_blobs_uri(&key_bytes) 337 + .ok_or(MetastoreError::CorruptData("corrupt record_blobs key"))?; 338 + let blob_cid_bytes = RecordBlobsValue::deserialize(&val_bytes) 339 + .map(|v| v.blob_cid_bytes) 340 + .ok_or(MetastoreError::CorruptData("corrupt record_blobs value"))?; 341 + 342 + blob_cid_bytes.into_iter().try_for_each( 343 + |cid_bytes| -> Result<(), MetastoreError> { 344 + let cid_link = bytes_to_cid_link(&cid_bytes)?; 345 + let cid_str = cid_link.as_str().to_owned(); 346 + if acc.contains_key(&cid_str) { 347 + return Ok(()); 348 + } 349 + let key = blob_meta_key(user_hash, &cid_str); 350 + let exists = self 351 + .repo_data 352 + .get(key.as_slice()) 353 + .map_err(MetastoreError::Fjall)? 354 + .is_some(); 355 + if !exists { 356 + acc.insert(cid_str, record_uri.clone()); 357 + } 358 + Ok(()) 359 + }, 360 + )?; 361 + 362 + Ok::<_, MetastoreError>(acc) 363 + })?; 364 + 365 + let start = cursor.map_or(Bound::Unbounded, Bound::Excluded); 366 + Ok(missing 367 + .range::<str, _>((start, Bound::Unbounded)) 368 + .take(limit) 369 + .map(|(cid_str, uri)| tranquil_db_traits::MissingBlobInfo { 370 + blob_cid: CidLink::from(cid_str.clone()), 371 + record_uri: tranquil_types::AtUri::from(uri.clone()), 372 + }) 373 + .collect()) 374 + } 375 + 376 + fn collect_referenced_cid_bytes( 377 + &self, 378 + user_hash: UserHash, 379 + ) -> Result<BTreeSet<Vec<u8>>, MetastoreError> { 380 + let rb_prefix = record_blobs_user_prefix(user_hash); 381 + self.repo_data 382 + .prefix(rb_prefix.as_slice()) 383 + .try_fold(BTreeSet::new(), |mut acc, guard| { 384 + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 385 + let blob_cids = RecordBlobsValue::deserialize(&val_bytes) 386 + .map(|v| v.blob_cid_bytes) 387 + .ok_or(MetastoreError::CorruptData("corrupt record_blobs value"))?; 388 + acc.extend(blob_cids); 389 + Ok::<_, MetastoreError>(acc) 390 + }) 391 + } 392 + 393 + pub fn count_distinct_record_blobs(&self, repo_id: Uuid) -> Result<i64, MetastoreError> { 394 + let user_hash = self.resolve_user_hash(repo_id)?; 395 + let distinct = self.collect_referenced_cid_bytes(user_hash)?; 396 + Ok(i64::try_from(distinct.len()).unwrap_or(i64::MAX)) 397 + } 398 + 399 + pub fn get_blobs_for_export( 400 + &self, 401 + repo_id: Uuid, 402 + ) -> Result<Vec<tranquil_db_traits::BlobForExport>, MetastoreError> { 403 + let user_hash = self.resolve_user_hash(repo_id)?; 404 + let referenced_cids = self.collect_referenced_cid_bytes(user_hash)?; 405 + 406 + referenced_cids 407 + .into_iter() 408 + .filter_map(|cid_bytes| { 409 + let cid_link = match bytes_to_cid_link(&cid_bytes) { 410 + Ok(c) => c, 411 + Err(e) => return Some(Err(e)), 412 + }; 413 + let key = blob_meta_key(user_hash, cid_link.as_str()); 414 + match point_lookup( 415 + &self.repo_data, 416 + key.as_slice(), 417 + BlobMetaValue::deserialize, 418 + "corrupt blob_meta value", 419 + ) { 420 + Ok(Some(v)) => Some(Ok(tranquil_db_traits::BlobForExport { 421 + cid: cid_link, 422 + storage_key: v.storage_key, 423 + mime_type: v.mime_type, 424 + })), 425 + Ok(None) => None, 426 + Err(e) => Some(Err(e)), 427 + } 428 + }) 429 + .collect() 430 + } 431 + } 432 + 433 + fn parse_blob_cid_from_key(key: &[u8]) -> Result<CidLink, MetastoreError> { 434 + let mut reader = KeyReader::new(key); 435 + let tag = reader 436 + .tag() 437 + .ok_or(MetastoreError::CorruptData("corrupt blob key: missing tag"))?; 438 + if tag != KeyTag::BLOBS.raw() { 439 + return Err(MetastoreError::CorruptData( 440 + "corrupt blob key: unexpected tag", 441 + )); 442 + } 443 + reader.u64().ok_or(MetastoreError::CorruptData( 444 + "corrupt blob key: missing user_hash", 445 + ))?; 446 + reader 447 + .string() 448 + .map(CidLink::from) 449 + .ok_or(MetastoreError::CorruptData("corrupt blob key: missing cid")) 450 + } 451 + 452 + fn parse_record_blobs_uri(key: &[u8]) -> Option<String> { 453 + let mut reader = KeyReader::new(key); 454 + let tag = reader.tag()?; 455 + if tag != KeyTag::RECORD_BLOBS.raw() { 456 + return None; 457 + } 458 + reader.u64()?; 459 + reader.string() 460 + } 461 + 462 + #[cfg(test)] 463 + mod tests { 464 + use super::*; 465 + use crate::metastore::{Metastore, MetastoreConfig}; 466 + 467 + fn open_fresh() -> (tempfile::TempDir, Metastore) { 468 + let dir = tempfile::TempDir::new().unwrap(); 469 + let ms = Metastore::open( 470 + dir.path(), 471 + MetastoreConfig { 472 + cache_size_bytes: 64 * 1024 * 1024, 473 + }, 474 + ) 475 + .unwrap(); 476 + (dir, ms) 477 + } 478 + 479 + fn setup_user(ms: &Metastore) -> (Uuid, UserHash) { 480 + let user_id = Uuid::new_v4(); 481 + let did = format!("did:plc:blob_test_{}", user_id); 482 + let user_hash = UserHash::from_did(&did); 483 + let mut batch = ms.database().batch(); 484 + ms.user_hashes() 485 + .stage_insert(&mut batch, user_id, user_hash) 486 + .unwrap(); 487 + batch.commit().unwrap(); 488 + (user_id, user_hash) 489 + } 490 + 491 + fn test_cid_link(seed: u8) -> CidLink { 492 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 493 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 494 + let c = cid::Cid::new_v1(0x71, mh); 495 + CidLink::from_cid(&c) 496 + } 497 + 498 + #[test] 499 + fn insert_and_get_metadata_roundtrip() { 500 + let (_dir, ms) = open_fresh(); 501 + let (user_id, _) = setup_user(&ms); 502 + let ops = ms.blob_ops(); 503 + 504 + let cid = test_cid_link(1); 505 + let result = ops 506 + .insert_blob(&cid, "image/png", 1024, user_id, "blobs/a/b") 507 + .unwrap(); 508 + assert_eq!(result, Some(cid.clone())); 509 + 510 + let meta = ops.get_blob_metadata(&cid).unwrap().unwrap(); 511 + assert_eq!(meta.storage_key, "blobs/a/b"); 512 + assert_eq!(meta.mime_type, "image/png"); 513 + assert_eq!(meta.size_bytes, 1024); 514 + } 515 + 516 + #[test] 517 + fn insert_duplicate_returns_none() { 518 + let (_dir, ms) = open_fresh(); 519 + let (user_id, _) = setup_user(&ms); 520 + let ops = ms.blob_ops(); 521 + 522 + let cid = test_cid_link(2); 523 + assert!( 524 + ops.insert_blob(&cid, "image/png", 100, user_id, "k1") 525 + .unwrap() 526 + .is_some() 527 + ); 528 + assert!( 529 + ops.insert_blob(&cid, "image/png", 100, user_id, "k1") 530 + .unwrap() 531 + .is_none() 532 + ); 533 + } 534 + 535 + #[test] 536 + fn insert_same_cid_different_user_returns_none() { 537 + let (_dir, ms) = open_fresh(); 538 + let (user_a, _) = setup_user(&ms); 539 + let (user_b, _) = setup_user(&ms); 540 + let ops = ms.blob_ops(); 541 + 542 + let cid = test_cid_link(80); 543 + assert!( 544 + ops.insert_blob(&cid, "image/png", 100, user_a, "ka") 545 + .unwrap() 546 + .is_some() 547 + ); 548 + assert!( 549 + ops.insert_blob(&cid, "image/png", 100, user_b, "kb") 550 + .unwrap() 551 + .is_none() 552 + ); 553 + } 554 + 555 + #[test] 556 + fn get_blob_with_takedown_no_takedown() { 557 + let (_dir, ms) = open_fresh(); 558 + let (user_id, _) = setup_user(&ms); 559 + let ops = ms.blob_ops(); 560 + 561 + let cid = test_cid_link(3); 562 + ops.insert_blob(&cid, "text/plain", 10, user_id, "k") 563 + .unwrap(); 564 + 565 + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); 566 + assert_eq!(result.cid, cid); 567 + assert!(result.takedown_ref.is_none()); 568 + } 569 + 570 + #[test] 571 + fn update_takedown_and_read_back() { 572 + let (_dir, ms) = open_fresh(); 573 + let (user_id, _) = setup_user(&ms); 574 + let ops = ms.blob_ops(); 575 + 576 + let cid = test_cid_link(4); 577 + ops.insert_blob(&cid, "text/plain", 10, user_id, "k") 578 + .unwrap(); 579 + 580 + assert!(ops.update_blob_takedown(&cid, Some("mod-42")).unwrap()); 581 + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); 582 + assert_eq!(result.takedown_ref.as_deref(), Some("mod-42")); 583 + 584 + assert!(ops.update_blob_takedown(&cid, None).unwrap()); 585 + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); 586 + assert!(result.takedown_ref.is_none()); 587 + } 588 + 589 + #[test] 590 + fn update_takedown_nonexistent_returns_false() { 591 + let (_dir, ms) = open_fresh(); 592 + let ops = ms.blob_ops(); 593 + let cid = test_cid_link(99); 594 + assert!(!ops.update_blob_takedown(&cid, Some("x")).unwrap()); 595 + } 596 + 597 + #[test] 598 + fn get_blob_storage_key() { 599 + let (_dir, ms) = open_fresh(); 600 + let (user_id, _) = setup_user(&ms); 601 + let ops = ms.blob_ops(); 602 + 603 + let cid = test_cid_link(5); 604 + ops.insert_blob(&cid, "image/jpeg", 500, user_id, "blobs/x/y") 605 + .unwrap(); 606 + 607 + assert_eq!( 608 + ops.get_blob_storage_key(&cid).unwrap().as_deref(), 609 + Some("blobs/x/y") 610 + ); 611 + } 612 + 613 + #[test] 614 + fn list_blobs_by_user_with_pagination() { 615 + let (_dir, ms) = open_fresh(); 616 + let (user_id, _) = setup_user(&ms); 617 + let ops = ms.blob_ops(); 618 + 619 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(10 + i)).collect(); 620 + cids.iter().enumerate().for_each(|(i, cid)| { 621 + ops.insert_blob(cid, "image/png", i as i64, user_id, &format!("k{i}")) 622 + .unwrap(); 623 + }); 624 + 625 + let page1 = ops.list_blobs_by_user(user_id, None, 3).unwrap(); 626 + assert_eq!(page1.len(), 3); 627 + 628 + let cursor = page1.last().unwrap().as_str(); 629 + let page2 = ops.list_blobs_by_user(user_id, Some(cursor), 3).unwrap(); 630 + assert_eq!(page2.len(), 2); 631 + 632 + let all = ops.list_blobs_by_user(user_id, None, 100).unwrap(); 633 + assert_eq!(all.len(), 5); 634 + } 635 + 636 + #[test] 637 + fn count_blobs_by_user() { 638 + let (_dir, ms) = open_fresh(); 639 + let (user_id, _) = setup_user(&ms); 640 + let ops = ms.blob_ops(); 641 + 642 + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 0); 643 + 644 + (0..3).for_each(|i| { 645 + ops.insert_blob( 646 + &test_cid_link(20 + i), 647 + "image/png", 648 + 100, 649 + user_id, 650 + &format!("k{i}"), 651 + ) 652 + .unwrap(); 653 + }); 654 + 655 + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 3); 656 + } 657 + 658 + #[test] 659 + fn sum_blob_storage() { 660 + let (_dir, ms) = open_fresh(); 661 + let (user_id, _) = setup_user(&ms); 662 + let ops = ms.blob_ops(); 663 + 664 + assert_eq!(ops.sum_blob_storage().unwrap(), 0); 665 + 666 + ops.insert_blob(&test_cid_link(30), "a/b", 100, user_id, "k0") 667 + .unwrap(); 668 + ops.insert_blob(&test_cid_link(31), "a/b", 250, user_id, "k1") 669 + .unwrap(); 670 + 671 + assert_eq!(ops.sum_blob_storage().unwrap(), 350); 672 + } 673 + 674 + #[test] 675 + fn delete_blob_by_cid() { 676 + let (_dir, ms) = open_fresh(); 677 + let (user_id, _) = setup_user(&ms); 678 + let ops = ms.blob_ops(); 679 + 680 + let cid = test_cid_link(40); 681 + ops.insert_blob(&cid, "image/png", 100, user_id, "k") 682 + .unwrap(); 683 + 684 + assert!(ops.delete_blob_by_cid(&cid).unwrap()); 685 + assert!(ops.get_blob_metadata(&cid).unwrap().is_none()); 686 + assert!(!ops.delete_blob_by_cid(&cid).unwrap()); 687 + } 688 + 689 + #[test] 690 + fn delete_blob_cleans_up_indexes() { 691 + let (_dir, ms) = open_fresh(); 692 + let (user_id, _) = setup_user(&ms); 693 + let ops = ms.blob_ops(); 694 + 695 + let cid = test_cid_link(41); 696 + ops.insert_blob(&cid, "image/png", 100, user_id, "storage/abc") 697 + .unwrap(); 698 + assert!(ops.get_blob_storage_key(&cid).unwrap().is_some()); 699 + 700 + ops.delete_blob_by_cid(&cid).unwrap(); 701 + 702 + assert!(ops.lookup_user_hash_by_cid(cid.as_str()).unwrap().is_none()); 703 + } 704 + 705 + #[test] 706 + fn delete_blobs_by_user() { 707 + let (_dir, ms) = open_fresh(); 708 + let (user_id, _) = setup_user(&ms); 709 + let ops = ms.blob_ops(); 710 + 711 + (0..4).for_each(|i| { 712 + ops.insert_blob(&test_cid_link(50 + i), "a/b", 10, user_id, &format!("k{i}")) 713 + .unwrap(); 714 + }); 715 + 716 + let deleted = ops.delete_blobs_by_user(user_id).unwrap(); 717 + assert_eq!(deleted, 4); 718 + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 0); 719 + } 720 + 721 + #[test] 722 + fn delete_blobs_by_user_cleans_all_indexes() { 723 + let (_dir, ms) = open_fresh(); 724 + let (user_id, _) = setup_user(&ms); 725 + let ops = ms.blob_ops(); 726 + 727 + let cid = test_cid_link(81); 728 + ops.insert_blob(&cid, "a/b", 10, user_id, "storage/del_test") 729 + .unwrap(); 730 + 731 + ops.delete_blobs_by_user(user_id).unwrap(); 732 + 733 + assert!(ops.lookup_user_hash_by_cid(cid.as_str()).unwrap().is_none()); 734 + } 735 + 736 + #[test] 737 + fn insert_blob_rejects_negative_size() { 738 + let (_dir, ms) = open_fresh(); 739 + let (user_id, _) = setup_user(&ms); 740 + let ops = ms.blob_ops(); 741 + 742 + let result = ops.insert_blob(&test_cid_link(90), "a/b", -1, user_id, "k"); 743 + assert!(result.is_err()); 744 + } 745 + 746 + #[test] 747 + fn get_blob_storage_keys_by_user() { 748 + let (_dir, ms) = open_fresh(); 749 + let (user_id, _) = setup_user(&ms); 750 + let ops = ms.blob_ops(); 751 + 752 + ops.insert_blob(&test_cid_link(60), "a/b", 10, user_id, "alpha") 753 + .unwrap(); 754 + ops.insert_blob(&test_cid_link(61), "a/b", 10, user_id, "beta") 755 + .unwrap(); 756 + 757 + let mut keys = ops.get_blob_storage_keys_by_user(user_id).unwrap(); 758 + keys.sort(); 759 + assert_eq!(keys, vec!["alpha", "beta"]); 760 + } 761 + 762 + #[test] 763 + fn get_metadata_for_nonexistent_returns_none() { 764 + let (_dir, ms) = open_fresh(); 765 + let ops = ms.blob_ops(); 766 + assert!(ops.get_blob_metadata(&test_cid_link(99)).unwrap().is_none()); 767 + } 768 + 769 + #[test] 770 + fn blobs_isolated_between_users() { 771 + let (_dir, ms) = open_fresh(); 772 + let (user_a, _) = setup_user(&ms); 773 + let (user_b, _) = setup_user(&ms); 774 + let ops = ms.blob_ops(); 775 + 776 + ops.insert_blob(&test_cid_link(70), "a/b", 10, user_a, "ka") 777 + .unwrap(); 778 + ops.insert_blob(&test_cid_link(71), "a/b", 20, user_b, "kb") 779 + .unwrap(); 780 + 781 + assert_eq!(ops.count_blobs_by_user(user_a).unwrap(), 1); 782 + assert_eq!(ops.count_blobs_by_user(user_b).unwrap(), 1); 783 + assert_eq!(ops.sum_blob_storage().unwrap(), 30); 784 + } 785 + }
+144
crates/tranquil-store/src/metastore/blobs.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use smallvec::SmallVec; 3 + 4 + use super::encoding::KeyBuilder; 5 + use super::keys::{KeyTag, UserHash}; 6 + 7 + const BLOB_META_SCHEMA_VERSION: u8 = 1; 8 + 9 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 10 + pub struct BlobMetaValue { 11 + pub size_bytes: i64, 12 + pub mime_type: String, 13 + pub storage_key: String, 14 + pub takedown_ref: Option<String>, 15 + pub created_at_ms: i64, 16 + } 17 + 18 + impl BlobMetaValue { 19 + pub fn serialize(&self) -> Vec<u8> { 20 + let payload = postcard::to_allocvec(self).expect("BlobMetaValue serialization cannot fail"); 21 + let mut buf = Vec::with_capacity(1 + payload.len()); 22 + buf.push(BLOB_META_SCHEMA_VERSION); 23 + buf.extend_from_slice(&payload); 24 + buf 25 + } 26 + 27 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 28 + let (&version, payload) = bytes.split_first()?; 29 + match version { 30 + BLOB_META_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 31 + _ => None, 32 + } 33 + } 34 + } 35 + 36 + pub fn blob_meta_key(user_hash: UserHash, cid_str: &str) -> SmallVec<[u8; 128]> { 37 + KeyBuilder::new() 38 + .tag(KeyTag::BLOBS) 39 + .u64(user_hash.raw()) 40 + .string(cid_str) 41 + .build() 42 + } 43 + 44 + pub fn blob_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 45 + KeyBuilder::new() 46 + .tag(KeyTag::BLOBS) 47 + .u64(user_hash.raw()) 48 + .build() 49 + } 50 + 51 + pub fn blobs_prefix() -> SmallVec<[u8; 128]> { 52 + KeyBuilder::new().tag(KeyTag::BLOBS).build() 53 + } 54 + 55 + pub fn blob_by_cid_key(cid_str: &str) -> SmallVec<[u8; 128]> { 56 + KeyBuilder::new() 57 + .tag(KeyTag::BLOB_BY_CID) 58 + .string(cid_str) 59 + .build() 60 + } 61 + 62 + #[cfg(test)] 63 + mod tests { 64 + use super::*; 65 + use crate::metastore::encoding::KeyReader; 66 + 67 + #[test] 68 + fn blob_meta_value_roundtrip() { 69 + let val = BlobMetaValue { 70 + size_bytes: 1024, 71 + mime_type: "image/png".to_owned(), 72 + storage_key: "blobs/abc/def".to_owned(), 73 + takedown_ref: None, 74 + created_at_ms: 1700000000000, 75 + }; 76 + let bytes = val.serialize(); 77 + let decoded = BlobMetaValue::deserialize(&bytes).unwrap(); 78 + assert_eq!(val, decoded); 79 + } 80 + 81 + #[test] 82 + fn blob_meta_value_with_takedown_roundtrip() { 83 + let val = BlobMetaValue { 84 + size_bytes: 42, 85 + mime_type: "text/plain".to_owned(), 86 + storage_key: "k".to_owned(), 87 + takedown_ref: Some("mod-123".to_owned()), 88 + created_at_ms: 0, 89 + }; 90 + let bytes = val.serialize(); 91 + let decoded = BlobMetaValue::deserialize(&bytes).unwrap(); 92 + assert_eq!(val, decoded); 93 + } 94 + 95 + #[test] 96 + fn blob_meta_key_roundtrip() { 97 + let uh = UserHash::from_did("did:plc:test"); 98 + let key = blob_meta_key(uh, "bafyreiabc"); 99 + let mut reader = KeyReader::new(&key); 100 + assert_eq!(reader.tag(), Some(KeyTag::BLOBS.raw())); 101 + assert_eq!(reader.u64(), Some(uh.raw())); 102 + assert_eq!(reader.string(), Some("bafyreiabc".to_owned())); 103 + assert!(reader.is_empty()); 104 + } 105 + 106 + #[test] 107 + fn blob_user_prefix_is_prefix_of_key() { 108 + let uh = UserHash::from_did("did:plc:test"); 109 + let prefix = blob_user_prefix(uh); 110 + let key = blob_meta_key(uh, "bafyreiabc"); 111 + assert!(key.starts_with(prefix.as_slice())); 112 + } 113 + 114 + #[test] 115 + fn blob_by_cid_key_roundtrip() { 116 + let key = blob_by_cid_key("bafyreiabc"); 117 + let mut reader = KeyReader::new(&key); 118 + assert_eq!(reader.tag(), Some(KeyTag::BLOB_BY_CID.raw())); 119 + assert_eq!(reader.string(), Some("bafyreiabc".to_owned())); 120 + assert!(reader.is_empty()); 121 + } 122 + 123 + #[test] 124 + fn blob_meta_key_ordering_by_cid() { 125 + let uh = UserHash::from_did("did:plc:test"); 126 + let key_a = blob_meta_key(uh, "aaa"); 127 + let key_b = blob_meta_key(uh, "bbb"); 128 + assert!(key_a.as_slice() < key_b.as_slice()); 129 + } 130 + 131 + #[test] 132 + fn deserialize_unknown_version_returns_none() { 133 + let val = BlobMetaValue { 134 + size_bytes: 0, 135 + mime_type: String::new(), 136 + storage_key: String::new(), 137 + takedown_ref: None, 138 + created_at_ms: 0, 139 + }; 140 + let mut bytes = val.serialize(); 141 + bytes[0] = 99; 142 + assert!(BlobMetaValue::deserialize(&bytes).is_none()); 143 + } 144 + }
+1023
crates/tranquil-store/src/metastore/client.rs
··· 1 + use std::marker::PhantomData; 2 + use std::sync::Arc; 3 + 4 + use async_trait::async_trait; 5 + use chrono::{DateTime, Utc}; 6 + use tokio::sync::oneshot; 7 + use tranquil_db_traits::{ 8 + AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, Backlink, 9 + BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, ImportBlock, ImportRecord, 10 + ImportRepoError, RepoAccountInfo, RepoInfo, RepoListItem, RepoWithoutRev, SequenceNumber, 11 + SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks, 12 + }; 13 + use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey}; 14 + use uuid::Uuid; 15 + 16 + use super::handler::{ 17 + BacklinkRequest, BlobRequest, CommitRequest, EventRequest, HandlerPool, MetastoreRequest, 18 + RecordRequest, RepoRequest, UserBlockRequest, 19 + }; 20 + use super::keys::UserHash; 21 + use crate::io::StorageIO; 22 + 23 + async fn recv<T>(rx: oneshot::Receiver<Result<T, DbError>>) -> Result<T, DbError> { 24 + rx.await 25 + .map_err(|_| DbError::Connection("metastore handler thread closed".to_string()))? 26 + } 27 + 28 + async fn recv_commit( 29 + rx: oneshot::Receiver<Result<ApplyCommitResult, ApplyCommitError>>, 30 + ) -> Result<ApplyCommitResult, ApplyCommitError> { 31 + rx.await 32 + .map_err(|_| ApplyCommitError::Database("metastore handler thread closed".to_string()))? 33 + } 34 + 35 + async fn recv_import( 36 + rx: oneshot::Receiver<Result<(), ImportRepoError>>, 37 + ) -> Result<(), ImportRepoError> { 38 + rx.await 39 + .map_err(|_| ImportRepoError::Database("metastore handler thread closed".to_string()))? 40 + } 41 + 42 + pub struct MetastoreClient<S: StorageIO> { 43 + pool: Arc<HandlerPool>, 44 + _phantom: PhantomData<S>, 45 + } 46 + 47 + impl<S: StorageIO> Clone for MetastoreClient<S> { 48 + fn clone(&self) -> Self { 49 + Self { 50 + pool: Arc::clone(&self.pool), 51 + _phantom: PhantomData, 52 + } 53 + } 54 + } 55 + 56 + impl<S: StorageIO> MetastoreClient<S> { 57 + pub fn new(pool: Arc<HandlerPool>) -> Self { 58 + Self { 59 + pool, 60 + _phantom: PhantomData, 61 + } 62 + } 63 + 64 + pub fn pool(&self) -> &Arc<HandlerPool> { 65 + &self.pool 66 + } 67 + 68 + pub async fn create_repo_full( 69 + &self, 70 + user_id: Uuid, 71 + did: &Did, 72 + handle: &Handle, 73 + repo_root_cid: &CidLink, 74 + repo_rev: &str, 75 + ) -> Result<(), DbError> { 76 + let (tx, rx) = oneshot::channel(); 77 + self.pool 78 + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 79 + user_id, 80 + did: did.clone(), 81 + handle: handle.clone(), 82 + repo_root_cid: repo_root_cid.clone(), 83 + repo_rev: repo_rev.to_string(), 84 + tx, 85 + }))?; 86 + recv(rx).await 87 + } 88 + } 89 + 90 + #[async_trait] 91 + impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreClient<S> { 92 + async fn create_repo( 93 + &self, 94 + user_id: Uuid, 95 + did: &Did, 96 + handle: &Handle, 97 + repo_root_cid: &CidLink, 98 + repo_rev: &str, 99 + ) -> Result<(), DbError> { 100 + self.create_repo_full(user_id, did, handle, repo_root_cid, repo_rev) 101 + .await 102 + } 103 + 104 + async fn update_repo_status( 105 + &self, 106 + did: &Did, 107 + takedown: Option<bool>, 108 + takedown_ref: Option<&str>, 109 + deactivated: Option<bool>, 110 + ) -> Result<(), DbError> { 111 + let (tx, rx) = oneshot::channel(); 112 + self.pool 113 + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoStatus { 114 + did: did.clone(), 115 + takedown, 116 + takedown_ref: takedown_ref.map(str::to_owned), 117 + deactivated, 118 + tx, 119 + }))?; 120 + recv(rx).await 121 + } 122 + 123 + async fn update_repo_root( 124 + &self, 125 + user_id: Uuid, 126 + repo_root_cid: &CidLink, 127 + repo_rev: &str, 128 + ) -> Result<(), DbError> { 129 + let (tx, rx) = oneshot::channel(); 130 + self.pool 131 + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoRoot { 132 + user_id, 133 + repo_root_cid: repo_root_cid.clone(), 134 + repo_rev: repo_rev.to_string(), 135 + tx, 136 + }))?; 137 + recv(rx).await 138 + } 139 + 140 + async fn update_repo_rev(&self, user_id: Uuid, repo_rev: &str) -> Result<(), DbError> { 141 + let (tx, rx) = oneshot::channel(); 142 + self.pool 143 + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoRev { 144 + user_id, 145 + repo_rev: repo_rev.to_string(), 146 + tx, 147 + }))?; 148 + recv(rx).await 149 + } 150 + 151 + async fn delete_repo(&self, user_id: Uuid) -> Result<(), DbError> { 152 + let (tx, rx) = oneshot::channel(); 153 + self.pool 154 + .send(MetastoreRequest::Repo(RepoRequest::DeleteRepo { 155 + user_id, 156 + tx, 157 + }))?; 158 + recv(rx).await 159 + } 160 + 161 + async fn get_repo_root_for_update(&self, user_id: Uuid) -> Result<Option<CidLink>, DbError> { 162 + let (tx, rx) = oneshot::channel(); 163 + self.pool 164 + .send(MetastoreRequest::Repo(RepoRequest::GetRepoRootForUpdate { 165 + user_id, 166 + tx, 167 + }))?; 168 + recv(rx).await 169 + } 170 + 171 + async fn get_repo(&self, user_id: Uuid) -> Result<Option<RepoInfo>, DbError> { 172 + let (tx, rx) = oneshot::channel(); 173 + self.pool 174 + .send(MetastoreRequest::Repo(RepoRequest::GetRepo { user_id, tx }))?; 175 + recv(rx).await 176 + } 177 + 178 + async fn get_repo_root_by_did(&self, did: &Did) -> Result<Option<CidLink>, DbError> { 179 + let (tx, rx) = oneshot::channel(); 180 + self.pool 181 + .send(MetastoreRequest::Repo(RepoRequest::GetRepoRootByDid { 182 + did: did.clone(), 183 + tx, 184 + }))?; 185 + recv(rx).await 186 + } 187 + 188 + async fn count_repos(&self) -> Result<i64, DbError> { 189 + let (tx, rx) = oneshot::channel(); 190 + self.pool 191 + .send(MetastoreRequest::Repo(RepoRequest::CountRepos { tx }))?; 192 + recv(rx).await 193 + } 194 + 195 + async fn get_repos_without_rev(&self) -> Result<Vec<RepoWithoutRev>, DbError> { 196 + let (tx, rx) = oneshot::channel(); 197 + self.pool 198 + .send(MetastoreRequest::Repo(RepoRequest::GetReposWithoutRev { 199 + tx, 200 + }))?; 201 + recv(rx).await 202 + } 203 + 204 + async fn upsert_records( 205 + &self, 206 + repo_id: Uuid, 207 + collections: &[Nsid], 208 + rkeys: &[Rkey], 209 + record_cids: &[CidLink], 210 + repo_rev: &str, 211 + ) -> Result<(), DbError> { 212 + let (tx, rx) = oneshot::channel(); 213 + self.pool 214 + .send(MetastoreRequest::Record(RecordRequest::UpsertRecords { 215 + repo_id, 216 + collections: collections.to_vec(), 217 + rkeys: rkeys.to_vec(), 218 + record_cids: record_cids.to_vec(), 219 + repo_rev: repo_rev.to_string(), 220 + tx, 221 + }))?; 222 + recv(rx).await 223 + } 224 + 225 + async fn delete_records( 226 + &self, 227 + repo_id: Uuid, 228 + collections: &[Nsid], 229 + rkeys: &[Rkey], 230 + ) -> Result<(), DbError> { 231 + let (tx, rx) = oneshot::channel(); 232 + self.pool 233 + .send(MetastoreRequest::Record(RecordRequest::DeleteRecords { 234 + repo_id, 235 + collections: collections.to_vec(), 236 + rkeys: rkeys.to_vec(), 237 + tx, 238 + }))?; 239 + recv(rx).await 240 + } 241 + 242 + async fn delete_all_records(&self, repo_id: Uuid) -> Result<(), DbError> { 243 + let (tx, rx) = oneshot::channel(); 244 + self.pool 245 + .send(MetastoreRequest::Record(RecordRequest::DeleteAllRecords { 246 + repo_id, 247 + tx, 248 + }))?; 249 + recv(rx).await 250 + } 251 + 252 + async fn get_record_cid( 253 + &self, 254 + repo_id: Uuid, 255 + collection: &Nsid, 256 + rkey: &Rkey, 257 + ) -> Result<Option<CidLink>, DbError> { 258 + let (tx, rx) = oneshot::channel(); 259 + self.pool 260 + .send(MetastoreRequest::Record(RecordRequest::GetRecordCid { 261 + repo_id, 262 + collection: collection.clone(), 263 + rkey: rkey.clone(), 264 + tx, 265 + }))?; 266 + recv(rx).await 267 + } 268 + 269 + async fn list_records( 270 + &self, 271 + repo_id: Uuid, 272 + collection: &Nsid, 273 + cursor: Option<&Rkey>, 274 + limit: i64, 275 + reverse: bool, 276 + rkey_start: Option<&Rkey>, 277 + rkey_end: Option<&Rkey>, 278 + ) -> Result<Vec<tranquil_db_traits::RecordInfo>, DbError> { 279 + let (tx, rx) = oneshot::channel(); 280 + self.pool 281 + .send(MetastoreRequest::Record(RecordRequest::ListRecords { 282 + repo_id, 283 + collection: collection.clone(), 284 + cursor: cursor.cloned(), 285 + limit, 286 + reverse, 287 + rkey_start: rkey_start.cloned(), 288 + rkey_end: rkey_end.cloned(), 289 + tx, 290 + }))?; 291 + recv(rx).await 292 + } 293 + 294 + async fn get_all_records( 295 + &self, 296 + repo_id: Uuid, 297 + ) -> Result<Vec<tranquil_db_traits::FullRecordInfo>, DbError> { 298 + let (tx, rx) = oneshot::channel(); 299 + self.pool 300 + .send(MetastoreRequest::Record(RecordRequest::GetAllRecords { 301 + repo_id, 302 + tx, 303 + }))?; 304 + recv(rx).await 305 + } 306 + 307 + async fn list_collections(&self, repo_id: Uuid) -> Result<Vec<Nsid>, DbError> { 308 + let (tx, rx) = oneshot::channel(); 309 + self.pool 310 + .send(MetastoreRequest::Record(RecordRequest::ListCollections { 311 + repo_id, 312 + tx, 313 + }))?; 314 + recv(rx).await 315 + } 316 + 317 + async fn count_records(&self, repo_id: Uuid) -> Result<i64, DbError> { 318 + let (tx, rx) = oneshot::channel(); 319 + self.pool 320 + .send(MetastoreRequest::Record(RecordRequest::CountRecords { 321 + repo_id, 322 + tx, 323 + }))?; 324 + recv(rx).await 325 + } 326 + 327 + async fn count_all_records(&self) -> Result<i64, DbError> { 328 + let (tx, rx) = oneshot::channel(); 329 + self.pool 330 + .send(MetastoreRequest::Record(RecordRequest::CountAllRecords { 331 + tx, 332 + }))?; 333 + recv(rx).await 334 + } 335 + 336 + async fn get_record_by_cid( 337 + &self, 338 + cid: &CidLink, 339 + ) -> Result<Option<tranquil_db_traits::RecordWithTakedown>, DbError> { 340 + let (tx, rx) = oneshot::channel(); 341 + self.pool 342 + .send(MetastoreRequest::Record(RecordRequest::GetRecordByCid { 343 + cid: cid.clone(), 344 + tx, 345 + }))?; 346 + recv(rx).await 347 + } 348 + 349 + async fn set_record_takedown( 350 + &self, 351 + cid: &CidLink, 352 + takedown_ref: Option<&str>, 353 + ) -> Result<(), DbError> { 354 + let (tx, rx) = oneshot::channel(); 355 + self.pool 356 + .send(MetastoreRequest::Record(RecordRequest::SetRecordTakedown { 357 + cid: cid.clone(), 358 + takedown_ref: takedown_ref.map(str::to_owned), 359 + scope_user: None, 360 + tx, 361 + }))?; 362 + recv(rx).await 363 + } 364 + 365 + async fn insert_user_blocks( 366 + &self, 367 + user_id: Uuid, 368 + block_cids: &[Vec<u8>], 369 + repo_rev: &str, 370 + ) -> Result<(), DbError> { 371 + let (tx, rx) = oneshot::channel(); 372 + self.pool.send(MetastoreRequest::UserBlock( 373 + UserBlockRequest::InsertUserBlocks { 374 + user_id, 375 + block_cids: block_cids.to_vec(), 376 + repo_rev: repo_rev.to_string(), 377 + tx, 378 + }, 379 + ))?; 380 + recv(rx).await 381 + } 382 + 383 + async fn delete_user_blocks( 384 + &self, 385 + user_id: Uuid, 386 + block_cids: &[Vec<u8>], 387 + ) -> Result<(), DbError> { 388 + let (tx, rx) = oneshot::channel(); 389 + self.pool.send(MetastoreRequest::UserBlock( 390 + UserBlockRequest::DeleteUserBlocks { 391 + user_id, 392 + block_cids: block_cids.to_vec(), 393 + tx, 394 + }, 395 + ))?; 396 + recv(rx).await 397 + } 398 + 399 + async fn get_user_block_cids_since_rev( 400 + &self, 401 + user_id: Uuid, 402 + since_rev: &str, 403 + ) -> Result<Vec<Vec<u8>>, DbError> { 404 + let (tx, rx) = oneshot::channel(); 405 + self.pool.send(MetastoreRequest::UserBlock( 406 + UserBlockRequest::GetUserBlockCidsSinceRev { 407 + user_id, 408 + since_rev: since_rev.to_string(), 409 + tx, 410 + }, 411 + ))?; 412 + recv(rx).await 413 + } 414 + 415 + async fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, DbError> { 416 + let (tx, rx) = oneshot::channel(); 417 + self.pool.send(MetastoreRequest::UserBlock( 418 + UserBlockRequest::CountUserBlocks { user_id, tx }, 419 + ))?; 420 + recv(rx).await 421 + } 422 + 423 + async fn find_unreferenced_blocks( 424 + &self, 425 + candidate_cids: &[Vec<u8>], 426 + ) -> Result<Vec<Vec<u8>>, DbError> { 427 + let (tx, rx) = oneshot::channel(); 428 + self.pool.send(MetastoreRequest::UserBlock( 429 + UserBlockRequest::FindUnreferencedBlocks { 430 + candidate_cids: candidate_cids.to_vec(), 431 + tx, 432 + }, 433 + ))?; 434 + recv(rx).await 435 + } 436 + 437 + async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> { 438 + let (tx, rx) = oneshot::channel(); 439 + self.pool 440 + .send(MetastoreRequest::Event(EventRequest::InsertCommitEvent { 441 + data: data.clone(), 442 + tx, 443 + }))?; 444 + recv(rx).await 445 + } 446 + 447 + async fn insert_identity_event( 448 + &self, 449 + did: &Did, 450 + handle: Option<&Handle>, 451 + ) -> Result<SequenceNumber, DbError> { 452 + let (tx, rx) = oneshot::channel(); 453 + self.pool 454 + .send(MetastoreRequest::Event(EventRequest::InsertIdentityEvent { 455 + did: did.clone(), 456 + handle: handle.cloned(), 457 + tx, 458 + }))?; 459 + recv(rx).await 460 + } 461 + 462 + async fn insert_account_event( 463 + &self, 464 + did: &Did, 465 + status: AccountStatus, 466 + ) -> Result<SequenceNumber, DbError> { 467 + let (tx, rx) = oneshot::channel(); 468 + self.pool 469 + .send(MetastoreRequest::Event(EventRequest::InsertAccountEvent { 470 + did: did.clone(), 471 + status, 472 + tx, 473 + }))?; 474 + recv(rx).await 475 + } 476 + 477 + async fn insert_sync_event( 478 + &self, 479 + did: &Did, 480 + commit_cid: &CidLink, 481 + rev: Option<&str>, 482 + ) -> Result<SequenceNumber, DbError> { 483 + let (tx, rx) = oneshot::channel(); 484 + self.pool 485 + .send(MetastoreRequest::Event(EventRequest::InsertSyncEvent { 486 + did: did.clone(), 487 + commit_cid: commit_cid.clone(), 488 + rev: rev.map(str::to_owned), 489 + tx, 490 + }))?; 491 + recv(rx).await 492 + } 493 + 494 + async fn insert_genesis_commit_event( 495 + &self, 496 + did: &Did, 497 + commit_cid: &CidLink, 498 + mst_root_cid: &CidLink, 499 + rev: &str, 500 + ) -> Result<SequenceNumber, DbError> { 501 + let (tx, rx) = oneshot::channel(); 502 + self.pool.send(MetastoreRequest::Event( 503 + EventRequest::InsertGenesisCommitEvent { 504 + did: did.clone(), 505 + commit_cid: commit_cid.clone(), 506 + mst_root_cid: mst_root_cid.clone(), 507 + rev: rev.to_string(), 508 + tx, 509 + }, 510 + ))?; 511 + recv(rx).await 512 + } 513 + 514 + async fn update_seq_blocks_cids( 515 + &self, 516 + seq: SequenceNumber, 517 + blocks_cids: &[String], 518 + ) -> Result<(), DbError> { 519 + let (tx, rx) = oneshot::channel(); 520 + self.pool 521 + .send(MetastoreRequest::Event(EventRequest::UpdateSeqBlocksCids { 522 + seq, 523 + blocks_cids: blocks_cids.to_vec(), 524 + tx, 525 + }))?; 526 + recv(rx).await 527 + } 528 + 529 + async fn delete_sequences_except( 530 + &self, 531 + did: &Did, 532 + keep_seq: SequenceNumber, 533 + ) -> Result<(), DbError> { 534 + let (tx, rx) = oneshot::channel(); 535 + self.pool.send(MetastoreRequest::Event( 536 + EventRequest::DeleteSequencesExcept { 537 + did: did.clone(), 538 + keep_seq, 539 + tx, 540 + }, 541 + ))?; 542 + recv(rx).await 543 + } 544 + 545 + async fn get_max_seq(&self) -> Result<SequenceNumber, DbError> { 546 + let (tx, rx) = oneshot::channel(); 547 + self.pool 548 + .send(MetastoreRequest::Event(EventRequest::GetMaxSeq { tx }))?; 549 + recv(rx).await 550 + } 551 + 552 + async fn get_min_seq_since( 553 + &self, 554 + since: DateTime<Utc>, 555 + ) -> Result<Option<SequenceNumber>, DbError> { 556 + let (tx, rx) = oneshot::channel(); 557 + self.pool 558 + .send(MetastoreRequest::Event(EventRequest::GetMinSeqSince { 559 + since, 560 + tx, 561 + }))?; 562 + recv(rx).await 563 + } 564 + 565 + async fn get_account_with_repo(&self, did: &Did) -> Result<Option<RepoAccountInfo>, DbError> { 566 + let (tx, rx) = oneshot::channel(); 567 + self.pool 568 + .send(MetastoreRequest::Repo(RepoRequest::GetAccountWithRepo { 569 + did: did.clone(), 570 + tx, 571 + }))?; 572 + recv(rx).await 573 + } 574 + 575 + async fn get_events_since_seq( 576 + &self, 577 + since_seq: SequenceNumber, 578 + limit: Option<i64>, 579 + ) -> Result<Vec<SequencedEvent>, DbError> { 580 + let (tx, rx) = oneshot::channel(); 581 + self.pool 582 + .send(MetastoreRequest::Event(EventRequest::GetEventsSinceSeq { 583 + since_seq, 584 + limit, 585 + tx, 586 + }))?; 587 + recv(rx).await 588 + } 589 + 590 + async fn get_events_in_seq_range( 591 + &self, 592 + start_seq: SequenceNumber, 593 + end_seq: SequenceNumber, 594 + ) -> Result<Vec<SequencedEvent>, DbError> { 595 + let (tx, rx) = oneshot::channel(); 596 + self.pool 597 + .send(MetastoreRequest::Event(EventRequest::GetEventsInSeqRange { 598 + start_seq, 599 + end_seq, 600 + tx, 601 + }))?; 602 + recv(rx).await 603 + } 604 + 605 + async fn get_event_by_seq( 606 + &self, 607 + seq: SequenceNumber, 608 + ) -> Result<Option<SequencedEvent>, DbError> { 609 + let (tx, rx) = oneshot::channel(); 610 + self.pool 611 + .send(MetastoreRequest::Event(EventRequest::GetEventBySeq { 612 + seq, 613 + tx, 614 + }))?; 615 + recv(rx).await 616 + } 617 + 618 + async fn get_events_since_cursor( 619 + &self, 620 + cursor: SequenceNumber, 621 + limit: i64, 622 + ) -> Result<Vec<SequencedEvent>, DbError> { 623 + let (tx, rx) = oneshot::channel(); 624 + self.pool.send(MetastoreRequest::Event( 625 + EventRequest::GetEventsSinceCursor { cursor, limit, tx }, 626 + ))?; 627 + recv(rx).await 628 + } 629 + 630 + async fn get_events_since_rev( 631 + &self, 632 + did: &Did, 633 + since_rev: &str, 634 + ) -> Result<Vec<EventBlocksCids>, DbError> { 635 + let (tx, rx) = oneshot::channel(); 636 + self.pool 637 + .send(MetastoreRequest::Event(EventRequest::GetEventsSinceRev { 638 + did: did.clone(), 639 + since_rev: since_rev.to_string(), 640 + tx, 641 + }))?; 642 + recv(rx).await 643 + } 644 + 645 + async fn list_repos_paginated( 646 + &self, 647 + cursor_did: Option<&Did>, 648 + limit: i64, 649 + ) -> Result<Vec<RepoListItem>, DbError> { 650 + let cursor_hash = cursor_did.map(|d| UserHash::from_did(d.as_str()).raw()); 651 + let (tx, rx) = oneshot::channel(); 652 + self.pool 653 + .send(MetastoreRequest::Repo(RepoRequest::ListReposPaginated { 654 + cursor_user_hash: cursor_hash, 655 + limit: usize::try_from(limit).unwrap_or(usize::MAX), 656 + tx, 657 + }))?; 658 + recv(rx).await 659 + } 660 + 661 + async fn get_repo_root_cid_by_user_id( 662 + &self, 663 + user_id: Uuid, 664 + ) -> Result<Option<CidLink>, DbError> { 665 + let (tx, rx) = oneshot::channel(); 666 + self.pool.send(MetastoreRequest::Repo( 667 + RepoRequest::GetRepoRootCidByUserId { user_id, tx }, 668 + ))?; 669 + recv(rx).await 670 + } 671 + 672 + async fn notify_update(&self, seq: SequenceNumber) -> Result<(), DbError> { 673 + let (tx, rx) = oneshot::channel(); 674 + self.pool 675 + .send(MetastoreRequest::Event(EventRequest::NotifyUpdate { 676 + seq, 677 + tx, 678 + }))?; 679 + recv(rx).await 680 + } 681 + 682 + async fn import_repo_data( 683 + &self, 684 + user_id: Uuid, 685 + blocks: &[ImportBlock], 686 + records: &[ImportRecord], 687 + expected_root_cid: Option<&CidLink>, 688 + ) -> Result<(), ImportRepoError> { 689 + let (tx, rx) = oneshot::channel(); 690 + self.pool 691 + .send(MetastoreRequest::Commit(Box::new( 692 + CommitRequest::ImportRepoData { 693 + user_id, 694 + blocks: blocks.to_vec(), 695 + records: records.to_vec(), 696 + expected_root_cid: expected_root_cid.cloned(), 697 + tx, 698 + }, 699 + ))) 700 + .map_err(|e| ImportRepoError::Database(e.to_string()))?; 701 + recv_import(rx).await 702 + } 703 + 704 + async fn apply_commit( 705 + &self, 706 + input: ApplyCommitInput, 707 + ) -> Result<ApplyCommitResult, ApplyCommitError> { 708 + let (tx, rx) = oneshot::channel(); 709 + self.pool 710 + .send(MetastoreRequest::Commit(Box::new( 711 + CommitRequest::ApplyCommit { 712 + input: Box::new(input), 713 + tx, 714 + }, 715 + ))) 716 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 717 + recv_commit(rx).await 718 + } 719 + 720 + async fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, DbError> { 721 + let (tx, rx) = oneshot::channel(); 722 + self.pool.send(MetastoreRequest::Commit(Box::new( 723 + CommitRequest::GetBrokenGenesisCommits { tx }, 724 + )))?; 725 + recv(rx).await 726 + } 727 + 728 + async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError> { 729 + let (tx, rx) = oneshot::channel(); 730 + self.pool.send(MetastoreRequest::Commit(Box::new( 731 + CommitRequest::GetUsersWithoutBlocks { tx }, 732 + )))?; 733 + recv(rx).await 734 + } 735 + 736 + async fn get_users_needing_record_blobs_backfill( 737 + &self, 738 + limit: i64, 739 + ) -> Result<Vec<UserNeedingRecordBlobsBackfill>, DbError> { 740 + let (tx, rx) = oneshot::channel(); 741 + self.pool.send(MetastoreRequest::Commit(Box::new( 742 + CommitRequest::GetUsersNeedingRecordBlobsBackfill { limit, tx }, 743 + )))?; 744 + recv(rx).await 745 + } 746 + 747 + async fn insert_record_blobs( 748 + &self, 749 + repo_id: Uuid, 750 + record_uris: &[AtUri], 751 + blob_cids: &[CidLink], 752 + ) -> Result<(), DbError> { 753 + let (tx, rx) = oneshot::channel(); 754 + self.pool.send(MetastoreRequest::Commit(Box::new( 755 + CommitRequest::InsertRecordBlobs { 756 + repo_id, 757 + record_uris: record_uris.to_vec(), 758 + blob_cids: blob_cids.to_vec(), 759 + tx, 760 + }, 761 + )))?; 762 + recv(rx).await 763 + } 764 + } 765 + 766 + #[async_trait] 767 + impl<S: StorageIO + 'static> tranquil_db_traits::BacklinkRepository for MetastoreClient<S> { 768 + async fn get_backlink_conflicts( 769 + &self, 770 + repo_id: Uuid, 771 + collection: &Nsid, 772 + backlinks: &[Backlink], 773 + ) -> Result<Vec<AtUri>, DbError> { 774 + let (tx, rx) = oneshot::channel(); 775 + self.pool.send(MetastoreRequest::Backlink( 776 + BacklinkRequest::GetBacklinkConflicts { 777 + repo_id, 778 + collection: collection.clone(), 779 + backlinks: backlinks.to_vec(), 780 + tx, 781 + }, 782 + ))?; 783 + recv(rx).await 784 + } 785 + 786 + async fn add_backlinks(&self, repo_id: Uuid, backlinks: &[Backlink]) -> Result<(), DbError> { 787 + let (tx, rx) = oneshot::channel(); 788 + self.pool 789 + .send(MetastoreRequest::Backlink(BacklinkRequest::AddBacklinks { 790 + repo_id, 791 + backlinks: backlinks.to_vec(), 792 + tx, 793 + }))?; 794 + recv(rx).await 795 + } 796 + 797 + async fn remove_backlinks_by_uri(&self, uri: &AtUri) -> Result<(), DbError> { 798 + let (tx, rx) = oneshot::channel(); 799 + self.pool.send(MetastoreRequest::Backlink( 800 + BacklinkRequest::RemoveBacklinksByUri { 801 + uri: uri.clone(), 802 + tx, 803 + }, 804 + ))?; 805 + recv(rx).await 806 + } 807 + 808 + async fn remove_backlinks_by_repo(&self, repo_id: Uuid) -> Result<(), DbError> { 809 + let (tx, rx) = oneshot::channel(); 810 + self.pool.send(MetastoreRequest::Backlink( 811 + BacklinkRequest::RemoveBacklinksByRepo { repo_id, tx }, 812 + ))?; 813 + recv(rx).await 814 + } 815 + } 816 + 817 + #[async_trait] 818 + impl<S: StorageIO + 'static> tranquil_db_traits::BlobRepository for MetastoreClient<S> { 819 + async fn insert_blob( 820 + &self, 821 + cid: &CidLink, 822 + mime_type: &str, 823 + size_bytes: i64, 824 + created_by_user: Uuid, 825 + storage_key: &str, 826 + ) -> Result<Option<CidLink>, DbError> { 827 + let (tx, rx) = oneshot::channel(); 828 + self.pool 829 + .send(MetastoreRequest::Blob(BlobRequest::InsertBlob { 830 + cid: cid.clone(), 831 + mime_type: mime_type.to_owned(), 832 + size_bytes, 833 + created_by_user, 834 + storage_key: storage_key.to_owned(), 835 + tx, 836 + }))?; 837 + recv(rx).await 838 + } 839 + 840 + async fn get_blob_metadata( 841 + &self, 842 + cid: &CidLink, 843 + ) -> Result<Option<tranquil_db_traits::BlobMetadata>, DbError> { 844 + let (tx, rx) = oneshot::channel(); 845 + self.pool 846 + .send(MetastoreRequest::Blob(BlobRequest::GetBlobMetadata { 847 + cid: cid.clone(), 848 + tx, 849 + }))?; 850 + recv(rx).await 851 + } 852 + 853 + async fn get_blob_with_takedown( 854 + &self, 855 + cid: &CidLink, 856 + ) -> Result<Option<tranquil_db_traits::BlobWithTakedown>, DbError> { 857 + let (tx, rx) = oneshot::channel(); 858 + self.pool 859 + .send(MetastoreRequest::Blob(BlobRequest::GetBlobWithTakedown { 860 + cid: cid.clone(), 861 + tx, 862 + }))?; 863 + recv(rx).await 864 + } 865 + 866 + async fn get_blob_storage_key(&self, cid: &CidLink) -> Result<Option<String>, DbError> { 867 + let (tx, rx) = oneshot::channel(); 868 + self.pool 869 + .send(MetastoreRequest::Blob(BlobRequest::GetBlobStorageKey { 870 + cid: cid.clone(), 871 + tx, 872 + }))?; 873 + recv(rx).await 874 + } 875 + 876 + async fn list_blobs_by_user( 877 + &self, 878 + user_id: Uuid, 879 + cursor: Option<&str>, 880 + limit: i64, 881 + ) -> Result<Vec<CidLink>, DbError> { 882 + let (tx, rx) = oneshot::channel(); 883 + self.pool 884 + .send(MetastoreRequest::Blob(BlobRequest::ListBlobsByUser { 885 + user_id, 886 + cursor: cursor.map(str::to_owned), 887 + limit, 888 + tx, 889 + }))?; 890 + recv(rx).await 891 + } 892 + 893 + async fn list_blobs_since_rev( 894 + &self, 895 + did: &tranquil_types::Did, 896 + since: &str, 897 + ) -> Result<Vec<CidLink>, DbError> { 898 + let (tx, rx) = oneshot::channel(); 899 + self.pool 900 + .send(MetastoreRequest::Blob(BlobRequest::ListBlobsSinceRev { 901 + did: did.clone(), 902 + since: since.to_owned(), 903 + tx, 904 + }))?; 905 + recv(rx).await 906 + } 907 + 908 + async fn count_blobs_by_user(&self, user_id: Uuid) -> Result<i64, DbError> { 909 + let (tx, rx) = oneshot::channel(); 910 + self.pool 911 + .send(MetastoreRequest::Blob(BlobRequest::CountBlobsByUser { 912 + user_id, 913 + tx, 914 + }))?; 915 + recv(rx).await 916 + } 917 + 918 + async fn sum_blob_storage(&self) -> Result<i64, DbError> { 919 + let (tx, rx) = oneshot::channel(); 920 + self.pool 921 + .send(MetastoreRequest::Blob(BlobRequest::SumBlobStorage { tx }))?; 922 + recv(rx).await 923 + } 924 + 925 + async fn update_blob_takedown( 926 + &self, 927 + cid: &CidLink, 928 + takedown_ref: Option<&str>, 929 + ) -> Result<bool, DbError> { 930 + let (tx, rx) = oneshot::channel(); 931 + self.pool 932 + .send(MetastoreRequest::Blob(BlobRequest::UpdateBlobTakedown { 933 + cid: cid.clone(), 934 + takedown_ref: takedown_ref.map(str::to_owned), 935 + tx, 936 + }))?; 937 + recv(rx).await 938 + } 939 + 940 + async fn delete_blob_by_cid(&self, cid: &CidLink) -> Result<bool, DbError> { 941 + let (tx, rx) = oneshot::channel(); 942 + self.pool 943 + .send(MetastoreRequest::Blob(BlobRequest::DeleteBlobByCid { 944 + cid: cid.clone(), 945 + tx, 946 + }))?; 947 + recv(rx).await 948 + } 949 + 950 + async fn delete_blobs_by_user(&self, user_id: Uuid) -> Result<u64, DbError> { 951 + let (tx, rx) = oneshot::channel(); 952 + self.pool 953 + .send(MetastoreRequest::Blob(BlobRequest::DeleteBlobsByUser { 954 + user_id, 955 + tx, 956 + }))?; 957 + recv(rx).await 958 + } 959 + 960 + async fn get_blob_storage_keys_by_user(&self, user_id: Uuid) -> Result<Vec<String>, DbError> { 961 + let (tx, rx) = oneshot::channel(); 962 + self.pool.send(MetastoreRequest::Blob( 963 + BlobRequest::GetBlobStorageKeysByUser { user_id, tx }, 964 + ))?; 965 + recv(rx).await 966 + } 967 + 968 + async fn insert_record_blobs( 969 + &self, 970 + repo_id: Uuid, 971 + record_uris: &[AtUri], 972 + blob_cids: &[CidLink], 973 + ) -> Result<(), DbError> { 974 + let (tx, rx) = oneshot::channel(); 975 + self.pool.send(MetastoreRequest::Commit(Box::new( 976 + CommitRequest::InsertRecordBlobs { 977 + repo_id, 978 + record_uris: record_uris.to_vec(), 979 + blob_cids: blob_cids.to_vec(), 980 + tx, 981 + }, 982 + )))?; 983 + recv(rx).await 984 + } 985 + 986 + async fn list_missing_blobs( 987 + &self, 988 + repo_id: Uuid, 989 + cursor: Option<&str>, 990 + limit: i64, 991 + ) -> Result<Vec<tranquil_db_traits::MissingBlobInfo>, DbError> { 992 + let (tx, rx) = oneshot::channel(); 993 + self.pool 994 + .send(MetastoreRequest::Blob(BlobRequest::ListMissingBlobs { 995 + repo_id, 996 + cursor: cursor.map(str::to_owned), 997 + limit, 998 + tx, 999 + }))?; 1000 + recv(rx).await 1001 + } 1002 + 1003 + async fn count_distinct_record_blobs(&self, repo_id: Uuid) -> Result<i64, DbError> { 1004 + let (tx, rx) = oneshot::channel(); 1005 + self.pool.send(MetastoreRequest::Blob( 1006 + BlobRequest::CountDistinctRecordBlobs { repo_id, tx }, 1007 + ))?; 1008 + recv(rx).await 1009 + } 1010 + 1011 + async fn get_blobs_for_export( 1012 + &self, 1013 + repo_id: Uuid, 1014 + ) -> Result<Vec<tranquil_db_traits::BlobForExport>, DbError> { 1015 + let (tx, rx) = oneshot::channel(); 1016 + self.pool 1017 + .send(MetastoreRequest::Blob(BlobRequest::GetBlobsForExport { 1018 + repo_id, 1019 + tx, 1020 + }))?; 1021 + recv(rx).await 1022 + } 1023 + }
+1504
crates/tranquil-store/src/metastore/commit_ops.rs
··· 1 + use std::sync::Arc; 2 + 3 + use fjall::{Database, Keyspace}; 4 + use smallvec::SmallVec; 5 + use uuid::Uuid; 6 + 7 + use super::MetastoreError; 8 + use super::backlink_ops::BacklinkOps; 9 + use super::backlinks::path_to_discriminant; 10 + use super::encoding::KeyBuilder; 11 + use super::event_ops::EventOps; 12 + use super::keys::{KeyTag, UserHash}; 13 + use super::record_ops::{RecordDelete, RecordOps, RecordWrite}; 14 + use super::recovery::{ 15 + BacklinkMutation, CommitMutationSet, RecordMutationDelete, RecordMutationUpsert, 16 + }; 17 + use super::repo_meta::{RepoMetaValue, repo_meta_key, repo_meta_prefix}; 18 + use super::repo_ops::{RepoOps, bytes_to_cid_link, cid_link_to_bytes}; 19 + use super::user_block_ops::UserBlockOps; 20 + use super::user_blocks::user_block_user_prefix; 21 + use super::user_hash::UserHashMap; 22 + use crate::blockstore::TranquilBlockStore; 23 + use crate::eventlog::EventLogBridge; 24 + use crate::io::StorageIO; 25 + 26 + use tranquil_db_traits::{ 27 + ApplyCommitError, ApplyCommitInput, ApplyCommitResult, BrokenGenesisCommit, ImportBlock, 28 + ImportRecord, ImportRepoError, RepoEventType, SequenceNumber, UserNeedingRecordBlobsBackfill, 29 + UserWithoutBlocks, 30 + }; 31 + use tranquil_types::{AtUri, CidLink, Did}; 32 + 33 + use serde::{Deserialize, Serialize}; 34 + 35 + pub(crate) const RECORD_BLOBS_SCHEMA_VERSION: u8 = 1; 36 + 37 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 38 + pub(crate) struct RecordBlobsValue { 39 + pub(crate) blob_cid_bytes: Vec<Vec<u8>>, 40 + } 41 + 42 + impl RecordBlobsValue { 43 + pub(crate) fn serialize(&self) -> Vec<u8> { 44 + let payload = 45 + postcard::to_allocvec(self).expect("RecordBlobsValue serialization cannot fail"); 46 + let mut buf = Vec::with_capacity(1 + payload.len()); 47 + buf.push(RECORD_BLOBS_SCHEMA_VERSION); 48 + buf.extend_from_slice(&payload); 49 + buf 50 + } 51 + 52 + pub(crate) fn deserialize(bytes: &[u8]) -> Option<Self> { 53 + let (&version, payload) = bytes.split_first()?; 54 + match version { 55 + RECORD_BLOBS_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 56 + _ => None, 57 + } 58 + } 59 + } 60 + 61 + fn record_blobs_key(user_hash: UserHash, uri: &AtUri) -> SmallVec<[u8; 128]> { 62 + KeyBuilder::new() 63 + .tag(KeyTag::RECORD_BLOBS) 64 + .u64(user_hash.raw()) 65 + .string(uri.as_str()) 66 + .build() 67 + } 68 + 69 + pub(crate) fn record_blobs_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 70 + KeyBuilder::new() 71 + .tag(KeyTag::RECORD_BLOBS) 72 + .u64(user_hash.raw()) 73 + .build() 74 + } 75 + 76 + pub struct CommitOps<S: StorageIO> { 77 + db: Database, 78 + repo_data: Keyspace, 79 + user_hashes: Arc<UserHashMap>, 80 + repo_ops: RepoOps, 81 + record_ops: RecordOps, 82 + user_block_ops: UserBlockOps, 83 + backlink_ops: BacklinkOps, 84 + event_ops: EventOps<S>, 85 + blockstore: Option<TranquilBlockStore>, 86 + } 87 + 88 + impl<S: StorageIO> CommitOps<S> { 89 + pub fn new( 90 + db: Database, 91 + repo_data: Keyspace, 92 + indexes: Keyspace, 93 + user_hashes: Arc<UserHashMap>, 94 + bridge: Arc<EventLogBridge<S>>, 95 + ) -> Self { 96 + let repo_ops = RepoOps::new(repo_data.clone(), Arc::clone(&user_hashes)); 97 + let record_ops = RecordOps::new(repo_data.clone(), Arc::clone(&user_hashes)); 98 + let user_block_ops = UserBlockOps::new(repo_data.clone(), Arc::clone(&user_hashes)); 99 + let backlink_ops = BacklinkOps::new(indexes, Arc::clone(&user_hashes)); 100 + let event_ops = EventOps::new(db.clone(), repo_data.clone(), bridge); 101 + Self { 102 + db, 103 + repo_data, 104 + user_hashes, 105 + repo_ops, 106 + record_ops, 107 + user_block_ops, 108 + backlink_ops, 109 + event_ops, 110 + blockstore: None, 111 + } 112 + } 113 + 114 + pub fn with_blockstore(mut self, blockstore: TranquilBlockStore) -> Self { 115 + self.blockstore = Some(blockstore); 116 + self 117 + } 118 + 119 + pub fn apply_commit( 120 + &self, 121 + input: ApplyCommitInput, 122 + ) -> Result<ApplyCommitResult, ApplyCommitError> { 123 + let user_hash = self 124 + .user_hashes 125 + .get(&input.user_id) 126 + .ok_or(ApplyCommitError::RepoNotFound)?; 127 + 128 + let key = repo_meta_key(user_hash); 129 + let meta = self 130 + .repo_data 131 + .get(key.as_slice()) 132 + .map_err(|e| ApplyCommitError::Database(e.to_string()))? 133 + .and_then(|raw| RepoMetaValue::deserialize(&raw)) 134 + .ok_or(ApplyCommitError::RepoNotFound)?; 135 + 136 + if let Some(expected) = &input.expected_root_cid { 137 + let current = bytes_to_cid_link(&meta.repo_root_cid) 138 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 139 + if current != *expected { 140 + return Err(ApplyCommitError::ConcurrentModification); 141 + } 142 + } 143 + 144 + let new_cid_bytes = cid_link_to_bytes(&input.new_root_cid) 145 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 146 + 147 + let is_active = meta.status.is_active(); 148 + 149 + let updated_meta = RepoMetaValue { 150 + repo_root_cid: new_cid_bytes.clone(), 151 + repo_rev: input.new_rev.clone(), 152 + ..meta 153 + }; 154 + 155 + let mut batch = self.db.batch(); 156 + 157 + self.repo_ops 158 + .write_repo_meta(&mut batch, user_hash, &updated_meta); 159 + 160 + let upserts: Vec<RecordWrite<'_>> = input 161 + .record_upserts 162 + .iter() 163 + .map(|u| RecordWrite { 164 + collection: &u.collection, 165 + rkey: &u.rkey, 166 + cid: &u.cid, 167 + }) 168 + .collect(); 169 + 170 + self.record_ops 171 + .upsert_records(&mut batch, user_hash, &upserts) 172 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 173 + 174 + let deletes: Vec<RecordDelete<'_>> = input 175 + .record_deletes 176 + .iter() 177 + .map(|d| RecordDelete { 178 + collection: &d.collection, 179 + rkey: &d.rkey, 180 + }) 181 + .collect(); 182 + 183 + self.record_ops 184 + .delete_records(&mut batch, user_hash, &deletes); 185 + 186 + self.user_block_ops 187 + .insert_user_blocks(&mut batch, user_hash, &input.new_block_cids, &input.new_rev) 188 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 189 + 190 + self.user_block_ops 191 + .delete_user_blocks_by_cid(&mut batch, user_hash, &input.obsolete_block_cids) 192 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 193 + 194 + input.backlinks_to_remove.iter().try_for_each(|uri| { 195 + self.backlink_ops 196 + .remove_backlinks_by_uri(&mut batch, user_hash, uri) 197 + .map_err(|e| ApplyCommitError::Database(e.to_string())) 198 + })?; 199 + 200 + self.backlink_ops 201 + .add_backlinks(&mut batch, user_hash, &input.backlinks_to_add) 202 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 203 + 204 + let mutation_set = CommitMutationSet { 205 + new_root_cid: new_cid_bytes.clone(), 206 + new_rev: input.new_rev.clone(), 207 + record_upserts: input 208 + .record_upserts 209 + .iter() 210 + .map(|u| { 211 + let cid_bytes = cid_link_to_bytes(&u.cid) 212 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 213 + Ok(RecordMutationUpsert { 214 + collection: u.collection.as_str().to_owned(), 215 + rkey: u.rkey.as_str().to_owned(), 216 + cid_bytes, 217 + }) 218 + }) 219 + .collect::<Result<Vec<_>, ApplyCommitError>>()?, 220 + record_deletes: input 221 + .record_deletes 222 + .iter() 223 + .map(|d| RecordMutationDelete { 224 + collection: d.collection.as_str().to_owned(), 225 + rkey: d.rkey.as_str().to_owned(), 226 + }) 227 + .collect(), 228 + block_inserts: input.new_block_cids.clone(), 229 + block_deletes: input.obsolete_block_cids.clone(), 230 + backlink_adds: input 231 + .backlinks_to_add 232 + .iter() 233 + .map(|bl| BacklinkMutation { 234 + uri: bl.uri.as_str().to_owned(), 235 + path: path_to_discriminant(bl.path), 236 + link_to: bl.link_to.clone(), 237 + }) 238 + .collect(), 239 + backlink_remove_uris: input 240 + .backlinks_to_remove 241 + .iter() 242 + .map(|uri| uri.as_str().to_owned()) 243 + .collect(), 244 + }; 245 + let mutation_set_bytes = mutation_set 246 + .serialize() 247 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 248 + 249 + let (seq, deferred) = self 250 + .event_ops 251 + .append_commit_event_into_batch( 252 + &mut batch, 253 + &input.commit_event, 254 + Some(&mutation_set_bytes), 255 + ) 256 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 257 + 258 + batch 259 + .commit() 260 + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; 261 + 262 + self.event_ops.complete_broadcast(deferred); 263 + 264 + Ok(ApplyCommitResult { 265 + seq: seq.as_i64(), 266 + is_account_active: is_active, 267 + }) 268 + } 269 + 270 + pub fn import_repo_data( 271 + &self, 272 + user_id: Uuid, 273 + blocks: &[ImportBlock], 274 + records: &[ImportRecord], 275 + expected_root_cid: Option<&CidLink>, 276 + ) -> Result<(), ImportRepoError> { 277 + let user_hash = self 278 + .user_hashes 279 + .get(&user_id) 280 + .ok_or(ImportRepoError::RepoNotFound)?; 281 + 282 + let key = repo_meta_key(user_hash); 283 + let meta = self 284 + .repo_data 285 + .get(key.as_slice()) 286 + .map_err(|e| ImportRepoError::Database(e.to_string()))? 287 + .and_then(|raw| RepoMetaValue::deserialize(&raw)) 288 + .ok_or(ImportRepoError::RepoNotFound)?; 289 + 290 + if let Some(expected) = expected_root_cid { 291 + let current = bytes_to_cid_link(&meta.repo_root_cid) 292 + .map_err(|e| ImportRepoError::Database(e.to_string()))?; 293 + if current != *expected { 294 + return Err(ImportRepoError::ConcurrentModification); 295 + } 296 + } 297 + 298 + if let Some(bs) = &self.blockstore 299 + && !blocks.is_empty() 300 + { 301 + let block_pairs: Vec<([u8; 36], Vec<u8>)> = blocks 302 + .iter() 303 + .map(|b| { 304 + let cid: [u8; 36] = b.cid_bytes.as_slice().try_into().map_err(|_| { 305 + ImportRepoError::Database(format!( 306 + "block CID has invalid length: {} (expected 36)", 307 + b.cid_bytes.len() 308 + )) 309 + })?; 310 + Ok((cid, b.data.clone())) 311 + }) 312 + .collect::<Result<Vec<_>, ImportRepoError>>()?; 313 + bs.put_blocks_blocking(block_pairs) 314 + .map_err(|e| ImportRepoError::Database(e.to_string()))?; 315 + } 316 + 317 + let mut batch = self.db.batch(); 318 + 319 + let upserts: Vec<RecordWrite<'_>> = records 320 + .iter() 321 + .map(|r| RecordWrite { 322 + collection: &r.collection, 323 + rkey: &r.rkey, 324 + cid: &r.record_cid, 325 + }) 326 + .collect(); 327 + 328 + self.record_ops 329 + .upsert_records(&mut batch, user_hash, &upserts) 330 + .map_err(|e| ImportRepoError::Database(e.to_string()))?; 331 + 332 + batch 333 + .commit() 334 + .map_err(|e| ImportRepoError::Database(e.to_string())) 335 + } 336 + 337 + pub fn insert_record_blobs( 338 + &self, 339 + repo_id: Uuid, 340 + record_uris: &[AtUri], 341 + blob_cids: &[CidLink], 342 + ) -> Result<(), MetastoreError> { 343 + let user_hash = self 344 + .user_hashes 345 + .get(&repo_id) 346 + .ok_or(MetastoreError::InvalidInput("unknown user_id"))?; 347 + 348 + let blob_bytes: Vec<Vec<u8>> = blob_cids 349 + .iter() 350 + .map(cid_link_to_bytes) 351 + .collect::<Result<_, _>>()?; 352 + 353 + let serialized = RecordBlobsValue { 354 + blob_cid_bytes: blob_bytes, 355 + } 356 + .serialize(); 357 + 358 + let mut batch = self.db.batch(); 359 + record_uris.iter().for_each(|uri| { 360 + let key = record_blobs_key(user_hash, uri); 361 + batch.insert(&self.repo_data, key.as_slice(), serialized.as_slice()); 362 + }); 363 + batch.commit().map_err(MetastoreError::Fjall) 364 + } 365 + 366 + pub fn get_users_needing_record_blobs_backfill( 367 + &self, 368 + limit: i64, 369 + ) -> Result<Vec<UserNeedingRecordBlobsBackfill>, MetastoreError> { 370 + let limit_usize = usize::try_from(limit).unwrap_or(usize::MAX); 371 + 372 + self.scan_users_missing_prefix( 373 + record_blobs_user_prefix, 374 + |meta, user_id| { 375 + let did = meta 376 + .did 377 + .map(Did::from) 378 + .ok_or(MetastoreError::CorruptData("repo_meta missing did field"))?; 379 + Ok(UserNeedingRecordBlobsBackfill { user_id, did }) 380 + }, 381 + limit_usize, 382 + ) 383 + } 384 + 385 + pub fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, MetastoreError> { 386 + const PAGE_SIZE: usize = 4096; 387 + self.collect_broken_genesis_page(SequenceNumber::ZERO, Vec::new(), PAGE_SIZE) 388 + } 389 + 390 + fn collect_broken_genesis_page( 391 + &self, 392 + cursor: SequenceNumber, 393 + acc: Vec<BrokenGenesisCommit>, 394 + page_size: usize, 395 + ) -> Result<Vec<BrokenGenesisCommit>, MetastoreError> { 396 + let limit = i64::try_from(page_size).unwrap_or(i64::MAX); 397 + let events = self 398 + .event_ops 399 + .get_events_since_seq(cursor, Some(limit)) 400 + .map_err(|_| MetastoreError::CorruptData("failed to read events"))?; 401 + 402 + let page_len = events.len(); 403 + let page_high_seq = events.last().map(|e| e.seq).unwrap_or(cursor); 404 + 405 + let results = events.into_iter().fold(acc, |mut results, e| { 406 + if e.event_type == RepoEventType::Commit 407 + && e.prev_cid.is_none() 408 + && e.commit_cid.is_none() 409 + { 410 + results.push(BrokenGenesisCommit { 411 + seq: e.seq, 412 + did: e.did, 413 + commit_cid: e.commit_cid, 414 + }); 415 + } 416 + results 417 + }); 418 + 419 + match page_len < page_size { 420 + true => Ok(results), 421 + false => self.collect_broken_genesis_page(page_high_seq, results, page_size), 422 + } 423 + } 424 + 425 + pub fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, MetastoreError> { 426 + const MAX_RESULTS: usize = 10_000; 427 + 428 + self.scan_users_missing_prefix( 429 + user_block_user_prefix, 430 + |meta, user_id| { 431 + let root_cid = bytes_to_cid_link(&meta.repo_root_cid)?; 432 + Ok(UserWithoutBlocks { 433 + user_id, 434 + repo_root_cid: root_cid, 435 + repo_rev: match meta.repo_rev.is_empty() { 436 + true => None, 437 + false => Some(meta.repo_rev), 438 + }, 439 + }) 440 + }, 441 + MAX_RESULTS, 442 + ) 443 + } 444 + 445 + fn scan_users_missing_prefix<T, F, P>( 446 + &self, 447 + make_prefix: P, 448 + build_result: F, 449 + limit: usize, 450 + ) -> Result<Vec<T>, MetastoreError> 451 + where 452 + F: Fn(RepoMetaValue, Uuid) -> Result<T, MetastoreError>, 453 + P: Fn(UserHash) -> SmallVec<[u8; 128]>, 454 + { 455 + let prefix = repo_meta_prefix(); 456 + 457 + self.repo_data 458 + .prefix(prefix.as_slice()) 459 + .filter_map(|guard| { 460 + let (key_bytes, val_bytes) = match guard.into_inner() { 461 + Ok(pair) => pair, 462 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 463 + }; 464 + 465 + let user_hash = match parse_user_hash_from_key(&key_bytes) { 466 + Some(h) => h, 467 + None => return Some(Err(MetastoreError::CorruptData("invalid repo_meta key"))), 468 + }; 469 + 470 + let check_prefix = make_prefix(user_hash); 471 + let has_entries = match self.repo_data.prefix(check_prefix.as_slice()).next() { 472 + Some(guard) => match guard.into_inner() { 473 + Ok(_) => true, 474 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 475 + }, 476 + None => false, 477 + }; 478 + 479 + match has_entries { 480 + true => None, 481 + false => { 482 + let meta = match RepoMetaValue::deserialize(&val_bytes) { 483 + Some(v) => v, 484 + None => { 485 + return Some(Err(MetastoreError::CorruptData( 486 + "invalid repo_meta value", 487 + ))); 488 + } 489 + }; 490 + let user_id = match self.user_hashes.get_uuid(&user_hash) { 491 + Some(id) => id, 492 + None => { 493 + return Some(Err(MetastoreError::CorruptData( 494 + "user_hash has no reverse mapping", 495 + ))); 496 + } 497 + }; 498 + Some(build_result(meta, user_id)) 499 + } 500 + } 501 + }) 502 + .take(limit) 503 + .collect() 504 + } 505 + } 506 + 507 + fn parse_user_hash_from_key(key_bytes: &[u8]) -> Option<UserHash> { 508 + use super::encoding::KeyReader; 509 + let mut reader = KeyReader::new(key_bytes); 510 + let _tag = reader.tag()?; 511 + let hash = reader.u64()?; 512 + Some(UserHash::from_raw(hash)) 513 + } 514 + 515 + #[cfg(test)] 516 + mod tests { 517 + use super::*; 518 + use crate::eventlog::{EventLog, EventLogConfig}; 519 + use crate::io::RealIO; 520 + use crate::metastore::{Metastore, MetastoreConfig}; 521 + use tranquil_db_traits::CommitEventData; 522 + use tranquil_types::{Handle, Nsid, Rkey}; 523 + 524 + struct TestHarness { 525 + _metastore_dir: tempfile::TempDir, 526 + _eventlog_dir: tempfile::TempDir, 527 + metastore: Metastore, 528 + bridge: Arc<EventLogBridge<RealIO>>, 529 + } 530 + 531 + fn setup() -> TestHarness { 532 + let metastore_dir = tempfile::TempDir::new().unwrap(); 533 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 534 + let segments_dir = eventlog_dir.path().join("segments"); 535 + std::fs::create_dir_all(&segments_dir).unwrap(); 536 + 537 + let metastore = Metastore::open( 538 + metastore_dir.path(), 539 + MetastoreConfig { 540 + cache_size_bytes: 64 * 1024 * 1024, 541 + }, 542 + ) 543 + .unwrap(); 544 + 545 + let event_log = EventLog::open( 546 + EventLogConfig { 547 + segments_dir, 548 + ..EventLogConfig::default() 549 + }, 550 + RealIO::new(), 551 + ) 552 + .unwrap(); 553 + 554 + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); 555 + 556 + TestHarness { 557 + _metastore_dir: metastore_dir, 558 + _eventlog_dir: eventlog_dir, 559 + metastore, 560 + bridge, 561 + } 562 + } 563 + 564 + fn test_cid_link(seed: u8) -> CidLink { 565 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 566 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 567 + let c = cid::Cid::new_v1(0x71, mh); 568 + CidLink::from_cid(&c) 569 + } 570 + 571 + fn test_did(name: &str) -> Did { 572 + Did::from(format!("did:plc:{name}")) 573 + } 574 + 575 + fn test_handle(name: &str) -> Handle { 576 + Handle::from(format!("{name}.test.invalid")) 577 + } 578 + 579 + fn make_commit_ops(h: &TestHarness) -> CommitOps<RealIO> { 580 + use crate::metastore::partitions::Partition; 581 + CommitOps::new( 582 + h.metastore.database().clone(), 583 + h.metastore.partition(Partition::RepoData).clone(), 584 + h.metastore.partition(Partition::Indexes).clone(), 585 + Arc::clone(h.metastore.user_hashes()), 586 + Arc::clone(&h.bridge), 587 + ) 588 + } 589 + 590 + fn create_test_repo(h: &TestHarness, name: &str, seed: u8) -> (Uuid, Did, CidLink) { 591 + let user_id = Uuid::new_v4(); 592 + let did = test_did(name); 593 + let handle = test_handle(name); 594 + let cid = test_cid_link(seed); 595 + h.metastore 596 + .repo_ops() 597 + .create_repo(h.metastore.database(), user_id, &did, &handle, &cid, "rev0") 598 + .unwrap(); 599 + (user_id, did, cid) 600 + } 601 + 602 + #[test] 603 + fn apply_commit_updates_records_and_meta() { 604 + let h = setup(); 605 + let ops = make_commit_ops(&h); 606 + let (user_id, did, root_cid) = create_test_repo(&h, "alice", 1); 607 + 608 + let new_root = test_cid_link(2); 609 + let record_cid = test_cid_link(3); 610 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 611 + let rkey = Rkey::from("3k2abc".to_string()); 612 + 613 + let input = ApplyCommitInput { 614 + user_id, 615 + did: did.clone(), 616 + expected_root_cid: Some(root_cid.clone()), 617 + new_root_cid: new_root.clone(), 618 + new_rev: "rev1".to_string(), 619 + new_block_cids: vec![vec![0x01, 0x02]], 620 + obsolete_block_cids: vec![], 621 + record_upserts: vec![tranquil_db_traits::RecordUpsert { 622 + collection: collection.clone(), 623 + rkey: rkey.clone(), 624 + cid: record_cid.clone(), 625 + }], 626 + record_deletes: vec![], 627 + backlinks_to_add: vec![], 628 + backlinks_to_remove: vec![], 629 + commit_event: CommitEventData { 630 + did: did.clone(), 631 + event_type: RepoEventType::Commit, 632 + commit_cid: Some(new_root.clone()), 633 + prev_cid: Some(root_cid.clone()), 634 + ops: None, 635 + blobs: None, 636 + blocks_cids: None, 637 + prev_data_cid: None, 638 + rev: Some("rev1".to_string()), 639 + }, 640 + }; 641 + 642 + let result = ops.apply_commit(input).unwrap(); 643 + assert!(result.seq > 0); 644 + assert!(result.is_account_active); 645 + 646 + let repo = h.metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); 647 + assert_eq!(repo.repo_root_cid, new_root); 648 + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); 649 + 650 + let found_cid = h 651 + .metastore 652 + .record_ops() 653 + .get_record_cid(user_id, &collection, &rkey) 654 + .unwrap() 655 + .unwrap(); 656 + assert_eq!(found_cid, record_cid); 657 + } 658 + 659 + #[test] 660 + fn apply_commit_cas_rejects_stale_root() { 661 + let h = setup(); 662 + let ops = make_commit_ops(&h); 663 + let (user_id, did, _root_cid) = create_test_repo(&h, "bob", 10); 664 + 665 + let stale_root = test_cid_link(99); 666 + let new_root = test_cid_link(11); 667 + 668 + let input = ApplyCommitInput { 669 + user_id, 670 + did, 671 + expected_root_cid: Some(stale_root), 672 + new_root_cid: new_root, 673 + new_rev: "rev1".to_string(), 674 + new_block_cids: vec![], 675 + obsolete_block_cids: vec![], 676 + record_upserts: vec![], 677 + record_deletes: vec![], 678 + backlinks_to_add: vec![], 679 + backlinks_to_remove: vec![], 680 + commit_event: CommitEventData { 681 + did: test_did("bob"), 682 + event_type: RepoEventType::Commit, 683 + commit_cid: None, 684 + prev_cid: None, 685 + ops: None, 686 + blobs: None, 687 + blocks_cids: None, 688 + prev_data_cid: None, 689 + rev: Some("rev1".to_string()), 690 + }, 691 + }; 692 + 693 + let result = ops.apply_commit(input); 694 + assert_eq!( 695 + result.unwrap_err(), 696 + ApplyCommitError::ConcurrentModification 697 + ); 698 + } 699 + 700 + #[test] 701 + fn apply_commit_returns_repo_not_found_for_unknown_user() { 702 + let h = setup(); 703 + let ops = make_commit_ops(&h); 704 + 705 + let input = ApplyCommitInput { 706 + user_id: Uuid::new_v4(), 707 + did: test_did("nobody"), 708 + expected_root_cid: None, 709 + new_root_cid: test_cid_link(1), 710 + new_rev: "rev1".to_string(), 711 + new_block_cids: vec![], 712 + obsolete_block_cids: vec![], 713 + record_upserts: vec![], 714 + record_deletes: vec![], 715 + backlinks_to_add: vec![], 716 + backlinks_to_remove: vec![], 717 + commit_event: CommitEventData { 718 + did: test_did("nobody"), 719 + event_type: RepoEventType::Commit, 720 + commit_cid: None, 721 + prev_cid: None, 722 + ops: None, 723 + blobs: None, 724 + blocks_cids: None, 725 + prev_data_cid: None, 726 + rev: None, 727 + }, 728 + }; 729 + 730 + assert_eq!( 731 + ops.apply_commit(input).unwrap_err(), 732 + ApplyCommitError::RepoNotFound 733 + ); 734 + } 735 + 736 + #[test] 737 + fn apply_commit_record_deletes() { 738 + let h = setup(); 739 + let ops = make_commit_ops(&h); 740 + let (user_id, did, root_cid) = create_test_repo(&h, "carol", 20); 741 + 742 + let mid_root = test_cid_link(21); 743 + let record_cid = test_cid_link(22); 744 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 745 + let rkey = Rkey::from("3k2del".to_string()); 746 + 747 + let insert_input = ApplyCommitInput { 748 + user_id, 749 + did: did.clone(), 750 + expected_root_cid: Some(root_cid.clone()), 751 + new_root_cid: mid_root.clone(), 752 + new_rev: "rev1".to_string(), 753 + new_block_cids: vec![], 754 + obsolete_block_cids: vec![], 755 + record_upserts: vec![tranquil_db_traits::RecordUpsert { 756 + collection: collection.clone(), 757 + rkey: rkey.clone(), 758 + cid: record_cid, 759 + }], 760 + record_deletes: vec![], 761 + backlinks_to_add: vec![], 762 + backlinks_to_remove: vec![], 763 + commit_event: CommitEventData { 764 + did: did.clone(), 765 + event_type: RepoEventType::Commit, 766 + commit_cid: Some(mid_root.clone()), 767 + prev_cid: None, 768 + ops: None, 769 + blobs: None, 770 + blocks_cids: None, 771 + prev_data_cid: None, 772 + rev: Some("rev1".to_string()), 773 + }, 774 + }; 775 + ops.apply_commit(insert_input).unwrap(); 776 + 777 + assert!( 778 + h.metastore 779 + .record_ops() 780 + .get_record_cid(user_id, &collection, &rkey) 781 + .unwrap() 782 + .is_some() 783 + ); 784 + 785 + let final_root = test_cid_link(23); 786 + let delete_input = ApplyCommitInput { 787 + user_id, 788 + did: did.clone(), 789 + expected_root_cid: Some(mid_root.clone()), 790 + new_root_cid: final_root.clone(), 791 + new_rev: "rev2".to_string(), 792 + new_block_cids: vec![], 793 + obsolete_block_cids: vec![], 794 + record_upserts: vec![], 795 + record_deletes: vec![tranquil_db_traits::RecordDelete { 796 + collection: collection.clone(), 797 + rkey: rkey.clone(), 798 + }], 799 + backlinks_to_add: vec![], 800 + backlinks_to_remove: vec![], 801 + commit_event: CommitEventData { 802 + did: did.clone(), 803 + event_type: RepoEventType::Commit, 804 + commit_cid: Some(final_root.clone()), 805 + prev_cid: Some(mid_root.clone()), 806 + ops: None, 807 + blobs: None, 808 + blocks_cids: None, 809 + prev_data_cid: None, 810 + rev: Some("rev2".to_string()), 811 + }, 812 + }; 813 + ops.apply_commit(delete_input).unwrap(); 814 + 815 + assert!( 816 + h.metastore 817 + .record_ops() 818 + .get_record_cid(user_id, &collection, &rkey) 819 + .unwrap() 820 + .is_none() 821 + ); 822 + } 823 + 824 + #[test] 825 + fn apply_commit_event_visible_after_commit() { 826 + let h = setup(); 827 + let ops = make_commit_ops(&h); 828 + let (user_id, did, root_cid) = create_test_repo(&h, "dave", 30); 829 + 830 + let new_root = test_cid_link(31); 831 + let input = ApplyCommitInput { 832 + user_id, 833 + did: did.clone(), 834 + expected_root_cid: Some(root_cid.clone()), 835 + new_root_cid: new_root.clone(), 836 + new_rev: "rev1".to_string(), 837 + new_block_cids: vec![], 838 + obsolete_block_cids: vec![], 839 + record_upserts: vec![], 840 + record_deletes: vec![], 841 + backlinks_to_add: vec![], 842 + backlinks_to_remove: vec![], 843 + commit_event: CommitEventData { 844 + did: did.clone(), 845 + event_type: RepoEventType::Commit, 846 + commit_cid: Some(new_root.clone()), 847 + prev_cid: Some(root_cid.clone()), 848 + ops: None, 849 + blobs: None, 850 + blocks_cids: None, 851 + prev_data_cid: None, 852 + rev: Some("rev1".to_string()), 853 + }, 854 + }; 855 + 856 + let result = ops.apply_commit(input).unwrap(); 857 + let seq = SequenceNumber::from_raw(result.seq); 858 + 859 + let event = ops.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 860 + assert_eq!(event.did, did); 861 + assert_eq!(event.event_type, RepoEventType::Commit); 862 + assert_eq!(event.rev.as_deref(), Some("rev1")); 863 + } 864 + 865 + #[test] 866 + fn import_repo_data_inserts_records() { 867 + let h = setup(); 868 + let ops = make_commit_ops(&h); 869 + let (user_id, _did, root_cid) = create_test_repo(&h, "eve", 40); 870 + 871 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 872 + let rkey = Rkey::from("3k2import".to_string()); 873 + let record_cid = test_cid_link(41); 874 + 875 + ops.import_repo_data( 876 + user_id, 877 + &[], 878 + &[ImportRecord { 879 + collection: collection.clone(), 880 + rkey: rkey.clone(), 881 + record_cid: record_cid.clone(), 882 + }], 883 + Some(&root_cid), 884 + ) 885 + .unwrap(); 886 + 887 + let found = h 888 + .metastore 889 + .record_ops() 890 + .get_record_cid(user_id, &collection, &rkey) 891 + .unwrap() 892 + .unwrap(); 893 + assert_eq!(found, record_cid); 894 + } 895 + 896 + #[test] 897 + fn import_repo_data_cas_rejects_stale_root() { 898 + let h = setup(); 899 + let ops = make_commit_ops(&h); 900 + let (user_id, _did, _root_cid) = create_test_repo(&h, "frank", 50); 901 + 902 + let stale = test_cid_link(99); 903 + let result = ops.import_repo_data(user_id, &[], &[], Some(&stale)); 904 + assert_eq!(result.unwrap_err(), ImportRepoError::ConcurrentModification); 905 + } 906 + 907 + #[test] 908 + fn insert_record_blobs_and_backfill_query() { 909 + let h = setup(); 910 + let ops = make_commit_ops(&h); 911 + let (user_id_a, did_a, _) = create_test_repo(&h, "grace", 60); 912 + let (user_id_b, _did_b, _) = create_test_repo(&h, "henry", 61); 913 + 914 + let needing = ops.get_users_needing_record_blobs_backfill(100).unwrap(); 915 + assert_eq!(needing.len(), 2); 916 + 917 + let uri = AtUri::from_parts(did_a.as_str(), "app.bsky.feed.post", "3k2abc"); 918 + let blob_cid = test_cid_link(62); 919 + ops.insert_record_blobs(user_id_a, &[uri], &[blob_cid]) 920 + .unwrap(); 921 + 922 + let needing_after = ops.get_users_needing_record_blobs_backfill(100).unwrap(); 923 + assert_eq!(needing_after.len(), 1); 924 + assert_eq!(needing_after[0].user_id, user_id_b); 925 + } 926 + 927 + #[test] 928 + fn get_users_without_blocks_returns_users_with_no_blocks() { 929 + let h = setup(); 930 + let ops = make_commit_ops(&h); 931 + let (user_id_a, did_a, root_a) = create_test_repo(&h, "ivan", 70); 932 + let (user_id_b, _did_b, _root_b) = create_test_repo(&h, "julia", 71); 933 + 934 + let new_root = test_cid_link(72); 935 + let input = ApplyCommitInput { 936 + user_id: user_id_a, 937 + did: did_a.clone(), 938 + expected_root_cid: Some(root_a), 939 + new_root_cid: new_root.clone(), 940 + new_rev: "rev1".to_string(), 941 + new_block_cids: vec![vec![0x01, 0x02, 0x03]], 942 + obsolete_block_cids: vec![], 943 + record_upserts: vec![], 944 + record_deletes: vec![], 945 + backlinks_to_add: vec![], 946 + backlinks_to_remove: vec![], 947 + commit_event: CommitEventData { 948 + did: did_a.clone(), 949 + event_type: RepoEventType::Commit, 950 + commit_cid: Some(new_root), 951 + prev_cid: None, 952 + ops: None, 953 + blobs: None, 954 + blocks_cids: None, 955 + prev_data_cid: None, 956 + rev: Some("rev1".to_string()), 957 + }, 958 + }; 959 + ops.apply_commit(input).unwrap(); 960 + 961 + let without = ops.get_users_without_blocks().unwrap(); 962 + assert_eq!(without.len(), 1); 963 + assert_eq!(without[0].user_id, user_id_b); 964 + } 965 + 966 + #[test] 967 + fn apply_commit_without_expected_root_skips_cas() { 968 + let h = setup(); 969 + let ops = make_commit_ops(&h); 970 + let (user_id, did, _root_cid) = create_test_repo(&h, "kate", 80); 971 + 972 + let new_root = test_cid_link(81); 973 + let input = ApplyCommitInput { 974 + user_id, 975 + did: did.clone(), 976 + expected_root_cid: None, 977 + new_root_cid: new_root.clone(), 978 + new_rev: "rev_force".to_string(), 979 + new_block_cids: vec![], 980 + obsolete_block_cids: vec![], 981 + record_upserts: vec![], 982 + record_deletes: vec![], 983 + backlinks_to_add: vec![], 984 + backlinks_to_remove: vec![], 985 + commit_event: CommitEventData { 986 + did, 987 + event_type: RepoEventType::Commit, 988 + commit_cid: None, 989 + prev_cid: None, 990 + ops: None, 991 + blobs: None, 992 + blocks_cids: None, 993 + prev_data_cid: None, 994 + rev: Some("rev_force".to_string()), 995 + }, 996 + }; 997 + 998 + let result = ops.apply_commit(input).unwrap(); 999 + assert!(result.seq > 0); 1000 + } 1001 + 1002 + #[test] 1003 + fn apply_commit_update_preserves_new_backlinks() { 1004 + use crate::metastore::backlinks::backlink_target_prefix; 1005 + use crate::metastore::partitions::Partition; 1006 + 1007 + let h = setup(); 1008 + let ops = make_commit_ops(&h); 1009 + let (user_id, did, root_cid) = create_test_repo(&h, "backlink_upd", 90); 1010 + 1011 + let collection = Nsid::from("app.bsky.feed.like".to_string()); 1012 + let rkey = Rkey::from("3k2like1".to_string()); 1013 + let record_cid = test_cid_link(91); 1014 + let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); 1015 + 1016 + let mid_root = test_cid_link(92); 1017 + let create_input = ApplyCommitInput { 1018 + user_id, 1019 + did: did.clone(), 1020 + expected_root_cid: Some(root_cid.clone()), 1021 + new_root_cid: mid_root.clone(), 1022 + new_rev: "rev1".to_string(), 1023 + new_block_cids: vec![], 1024 + obsolete_block_cids: vec![], 1025 + record_upserts: vec![tranquil_db_traits::RecordUpsert { 1026 + collection: collection.clone(), 1027 + rkey: rkey.clone(), 1028 + cid: record_cid.clone(), 1029 + }], 1030 + record_deletes: vec![], 1031 + backlinks_to_add: vec![tranquil_db_traits::Backlink { 1032 + uri: record_uri.clone(), 1033 + path: tranquil_db_traits::BacklinkPath::SubjectUri, 1034 + link_to: "at://did:plc:target_a/app.bsky.feed.post/p1".to_string(), 1035 + }], 1036 + backlinks_to_remove: vec![], 1037 + commit_event: CommitEventData { 1038 + did: did.clone(), 1039 + event_type: RepoEventType::Commit, 1040 + commit_cid: Some(mid_root.clone()), 1041 + prev_cid: Some(root_cid.clone()), 1042 + ops: None, 1043 + blobs: None, 1044 + blocks_cids: None, 1045 + prev_data_cid: None, 1046 + rev: Some("rev1".to_string()), 1047 + }, 1048 + }; 1049 + ops.apply_commit(create_input).unwrap(); 1050 + 1051 + let indexes = h.metastore.partition(Partition::Indexes); 1052 + let target_a_prefix = backlink_target_prefix("at://did:plc:target_a/app.bsky.feed.post/p1"); 1053 + let count_a_before = indexes 1054 + .prefix(target_a_prefix.as_slice()) 1055 + .map(|g| g.into_inner().expect("scan must not fail")) 1056 + .fold(0, |acc, _| acc + 1); 1057 + assert_eq!(count_a_before, 1); 1058 + 1059 + let final_root = test_cid_link(93); 1060 + let new_record_cid = test_cid_link(94); 1061 + let update_input = ApplyCommitInput { 1062 + user_id, 1063 + did: did.clone(), 1064 + expected_root_cid: Some(mid_root.clone()), 1065 + new_root_cid: final_root.clone(), 1066 + new_rev: "rev2".to_string(), 1067 + new_block_cids: vec![], 1068 + obsolete_block_cids: vec![], 1069 + record_upserts: vec![tranquil_db_traits::RecordUpsert { 1070 + collection: collection.clone(), 1071 + rkey: rkey.clone(), 1072 + cid: new_record_cid.clone(), 1073 + }], 1074 + record_deletes: vec![], 1075 + backlinks_to_add: vec![tranquil_db_traits::Backlink { 1076 + uri: record_uri.clone(), 1077 + path: tranquil_db_traits::BacklinkPath::SubjectUri, 1078 + link_to: "at://did:plc:target_b/app.bsky.feed.post/p2".to_string(), 1079 + }], 1080 + backlinks_to_remove: vec![record_uri], 1081 + commit_event: CommitEventData { 1082 + did: did.clone(), 1083 + event_type: RepoEventType::Commit, 1084 + commit_cid: Some(final_root.clone()), 1085 + prev_cid: Some(mid_root.clone()), 1086 + ops: None, 1087 + blobs: None, 1088 + blocks_cids: None, 1089 + prev_data_cid: None, 1090 + rev: Some("rev2".to_string()), 1091 + }, 1092 + }; 1093 + ops.apply_commit(update_input).unwrap(); 1094 + 1095 + let count_a_after = indexes 1096 + .prefix(target_a_prefix.as_slice()) 1097 + .map(|g| g.into_inner().expect("scan must not fail")) 1098 + .fold(0, |acc, _| acc + 1); 1099 + assert_eq!(count_a_after, 0); 1100 + 1101 + let target_b_prefix = backlink_target_prefix("at://did:plc:target_b/app.bsky.feed.post/p2"); 1102 + let count_b = indexes 1103 + .prefix(target_b_prefix.as_slice()) 1104 + .map(|g| g.into_inner().expect("scan must not fail")) 1105 + .fold(0, |acc, _| acc + 1); 1106 + assert_eq!(count_b, 1); 1107 + } 1108 + 1109 + #[test] 1110 + fn crash_recovery_replays_mutation_set() { 1111 + let metastore_dir = tempfile::TempDir::new().unwrap(); 1112 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 1113 + let segments_dir = eventlog_dir.path().join("segments"); 1114 + std::fs::create_dir_all(&segments_dir).unwrap(); 1115 + 1116 + let user_id = Uuid::new_v4(); 1117 + let did = test_did("crash_alice"); 1118 + let handle = test_handle("crash_alice"); 1119 + let initial_root = test_cid_link(200); 1120 + let new_root = test_cid_link(201); 1121 + let record_cid = test_cid_link(202); 1122 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1123 + let rkey = Rkey::from("3k2crash".to_string()); 1124 + 1125 + let event_log = EventLog::open( 1126 + EventLogConfig { 1127 + segments_dir: segments_dir.clone(), 1128 + ..EventLogConfig::default() 1129 + }, 1130 + RealIO::new(), 1131 + ) 1132 + .unwrap(); 1133 + let event_log = Arc::new(event_log); 1134 + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); 1135 + 1136 + { 1137 + let metastore = Metastore::open( 1138 + metastore_dir.path(), 1139 + MetastoreConfig { 1140 + cache_size_bytes: 64 * 1024 * 1024, 1141 + }, 1142 + ) 1143 + .unwrap(); 1144 + 1145 + metastore 1146 + .repo_ops() 1147 + .create_repo( 1148 + metastore.database(), 1149 + user_id, 1150 + &did, 1151 + &handle, 1152 + &initial_root, 1153 + "rev0", 1154 + ) 1155 + .unwrap(); 1156 + metastore.persist().unwrap(); 1157 + 1158 + let ops = make_commit_ops_from(&metastore, &bridge); 1159 + let input = ApplyCommitInput { 1160 + user_id, 1161 + did: did.clone(), 1162 + expected_root_cid: Some(initial_root.clone()), 1163 + new_root_cid: new_root.clone(), 1164 + new_rev: "rev1".to_string(), 1165 + new_block_cids: vec![vec![0xAA, 0xBB]], 1166 + obsolete_block_cids: vec![], 1167 + record_upserts: vec![tranquil_db_traits::RecordUpsert { 1168 + collection: collection.clone(), 1169 + rkey: rkey.clone(), 1170 + cid: record_cid.clone(), 1171 + }], 1172 + record_deletes: vec![], 1173 + backlinks_to_add: vec![], 1174 + backlinks_to_remove: vec![], 1175 + commit_event: CommitEventData { 1176 + did: did.clone(), 1177 + event_type: RepoEventType::Commit, 1178 + commit_cid: Some(new_root.clone()), 1179 + prev_cid: Some(initial_root.clone()), 1180 + ops: None, 1181 + blobs: None, 1182 + blocks_cids: None, 1183 + prev_data_cid: None, 1184 + rev: Some("rev1".to_string()), 1185 + }, 1186 + }; 1187 + 1188 + let result = ops.apply_commit(input).unwrap(); 1189 + assert!(result.seq > 0); 1190 + metastore.persist().unwrap(); 1191 + } 1192 + 1193 + { 1194 + let metastore = Metastore::open( 1195 + metastore_dir.path(), 1196 + MetastoreConfig { 1197 + cache_size_bytes: 64 * 1024 * 1024, 1198 + }, 1199 + ) 1200 + .unwrap(); 1201 + 1202 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 1203 + 1204 + event_ops.write_last_applied_cursor_direct(0).unwrap(); 1205 + metastore.persist().unwrap(); 1206 + } 1207 + 1208 + { 1209 + let metastore = Metastore::open( 1210 + metastore_dir.path(), 1211 + MetastoreConfig { 1212 + cache_size_bytes: 64 * 1024 * 1024, 1213 + }, 1214 + ) 1215 + .unwrap(); 1216 + 1217 + let repo_before = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); 1218 + assert_eq!(repo_before.repo_root_cid, new_root); 1219 + 1220 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 1221 + let cursor_before = event_ops.read_last_applied_cursor().unwrap(); 1222 + assert_eq!(cursor_before, Some(0)); 1223 + 1224 + let indexes = metastore 1225 + .partition(crate::metastore::partitions::Partition::Indexes) 1226 + .clone(); 1227 + let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap(); 1228 + assert!(recovered > 0, "should replay at least one event"); 1229 + 1230 + let cursor_after = event_ops.read_last_applied_cursor().unwrap(); 1231 + assert!(cursor_after.unwrap_or(0) > 0); 1232 + } 1233 + } 1234 + 1235 + #[test] 1236 + fn crash_recovery_with_uncommitted_batch() { 1237 + let metastore_dir = tempfile::TempDir::new().unwrap(); 1238 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 1239 + let segments_dir = eventlog_dir.path().join("segments"); 1240 + std::fs::create_dir_all(&segments_dir).unwrap(); 1241 + 1242 + let user_id = Uuid::new_v4(); 1243 + let did = test_did("crash_bob"); 1244 + let handle = test_handle("crash_bob"); 1245 + let initial_root = test_cid_link(210); 1246 + let new_root = test_cid_link(211); 1247 + let record_cid = test_cid_link(212); 1248 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1249 + let rkey = Rkey::from("3k2bob".to_string()); 1250 + 1251 + let event_log = EventLog::open( 1252 + EventLogConfig { 1253 + segments_dir: segments_dir.clone(), 1254 + ..EventLogConfig::default() 1255 + }, 1256 + RealIO::new(), 1257 + ) 1258 + .unwrap(); 1259 + let event_log = Arc::new(event_log); 1260 + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); 1261 + 1262 + { 1263 + let metastore = Metastore::open( 1264 + metastore_dir.path(), 1265 + MetastoreConfig { 1266 + cache_size_bytes: 64 * 1024 * 1024, 1267 + }, 1268 + ) 1269 + .unwrap(); 1270 + 1271 + metastore 1272 + .repo_ops() 1273 + .create_repo( 1274 + metastore.database(), 1275 + user_id, 1276 + &did, 1277 + &handle, 1278 + &initial_root, 1279 + "rev0", 1280 + ) 1281 + .unwrap(); 1282 + metastore.persist().unwrap(); 1283 + 1284 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 1285 + let mutation_set = super::CommitMutationSet { 1286 + new_root_cid: super::cid_link_to_bytes(&new_root).unwrap(), 1287 + new_rev: "rev1".to_string(), 1288 + record_upserts: vec![super::RecordMutationUpsert { 1289 + collection: collection.as_str().to_owned(), 1290 + rkey: rkey.as_str().to_owned(), 1291 + cid_bytes: super::cid_link_to_bytes(&record_cid).unwrap(), 1292 + }], 1293 + record_deletes: vec![], 1294 + block_inserts: vec![vec![0xCC, 0xDD]], 1295 + block_deletes: vec![], 1296 + backlink_adds: vec![], 1297 + backlink_remove_uris: vec![], 1298 + }; 1299 + let ms_bytes = mutation_set.serialize().unwrap(); 1300 + 1301 + let commit_data = CommitEventData { 1302 + did: did.clone(), 1303 + event_type: RepoEventType::Commit, 1304 + commit_cid: Some(new_root.clone()), 1305 + prev_cid: Some(initial_root.clone()), 1306 + ops: None, 1307 + blobs: None, 1308 + blocks_cids: None, 1309 + prev_data_cid: None, 1310 + rev: Some("rev1".to_string()), 1311 + }; 1312 + 1313 + let mut batch = metastore.database().batch(); 1314 + let (_seq, deferred) = event_ops 1315 + .append_commit_event_into_batch(&mut batch, &commit_data, Some(&ms_bytes)) 1316 + .unwrap(); 1317 + 1318 + event_ops.complete_broadcast(deferred); 1319 + 1320 + drop(batch); 1321 + 1322 + metastore.persist().unwrap(); 1323 + } 1324 + 1325 + { 1326 + let metastore = Metastore::open( 1327 + metastore_dir.path(), 1328 + MetastoreConfig { 1329 + cache_size_bytes: 64 * 1024 * 1024, 1330 + }, 1331 + ) 1332 + .unwrap(); 1333 + 1334 + let repo = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); 1335 + assert_eq!(repo.repo_root_cid, initial_root); 1336 + 1337 + let record = metastore 1338 + .record_ops() 1339 + .get_record_cid(user_id, &collection, &rkey) 1340 + .unwrap(); 1341 + assert!(record.is_none()); 1342 + 1343 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 1344 + let indexes = metastore 1345 + .partition(crate::metastore::partitions::Partition::Indexes) 1346 + .clone(); 1347 + let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap(); 1348 + assert_eq!(recovered, 1); 1349 + 1350 + let repo_after = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); 1351 + assert_eq!(repo_after.repo_root_cid, new_root); 1352 + assert_eq!(repo_after.repo_rev.as_deref(), Some("rev1")); 1353 + 1354 + let record_after = metastore 1355 + .record_ops() 1356 + .get_record_cid(user_id, &collection, &rkey) 1357 + .unwrap(); 1358 + assert_eq!(record_after, Some(record_cid)); 1359 + } 1360 + } 1361 + 1362 + fn make_commit_ops_from( 1363 + metastore: &Metastore, 1364 + bridge: &Arc<EventLogBridge<RealIO>>, 1365 + ) -> CommitOps<RealIO> { 1366 + use crate::metastore::partitions::Partition; 1367 + CommitOps::new( 1368 + metastore.database().clone(), 1369 + metastore.partition(Partition::RepoData).clone(), 1370 + metastore.partition(Partition::Indexes).clone(), 1371 + Arc::clone(metastore.user_hashes()), 1372 + Arc::clone(bridge), 1373 + ) 1374 + } 1375 + 1376 + #[test] 1377 + fn apply_commit_backlinks_isolated_by_collection() { 1378 + use crate::metastore::backlinks::{backlink_by_user_prefix, backlink_target_prefix}; 1379 + use crate::metastore::partitions::Partition; 1380 + 1381 + let h = setup(); 1382 + let ops = make_commit_ops(&h); 1383 + let (user_id, did, root_cid) = create_test_repo(&h, "col_iso", 95); 1384 + 1385 + let col_like = Nsid::from("app.bsky.feed.like".to_string()); 1386 + let col_repost = Nsid::from("app.bsky.feed.repost".to_string()); 1387 + let rkey = Rkey::from("same_rkey".to_string()); 1388 + let target = "at://did:plc:someone/app.bsky.feed.post/p1"; 1389 + 1390 + let mid_root = test_cid_link(96); 1391 + let uri_like = AtUri::from_parts(did.as_str(), col_like.as_str(), rkey.as_str()); 1392 + let uri_repost = AtUri::from_parts(did.as_str(), col_repost.as_str(), rkey.as_str()); 1393 + 1394 + let input = ApplyCommitInput { 1395 + user_id, 1396 + did: did.clone(), 1397 + expected_root_cid: Some(root_cid.clone()), 1398 + new_root_cid: mid_root.clone(), 1399 + new_rev: "rev1".to_string(), 1400 + new_block_cids: vec![], 1401 + obsolete_block_cids: vec![], 1402 + record_upserts: vec![ 1403 + tranquil_db_traits::RecordUpsert { 1404 + collection: col_like.clone(), 1405 + rkey: rkey.clone(), 1406 + cid: test_cid_link(97), 1407 + }, 1408 + tranquil_db_traits::RecordUpsert { 1409 + collection: col_repost.clone(), 1410 + rkey: rkey.clone(), 1411 + cid: test_cid_link(98), 1412 + }, 1413 + ], 1414 + record_deletes: vec![], 1415 + backlinks_to_add: vec![ 1416 + tranquil_db_traits::Backlink { 1417 + uri: uri_like.clone(), 1418 + path: tranquil_db_traits::BacklinkPath::SubjectUri, 1419 + link_to: target.to_string(), 1420 + }, 1421 + tranquil_db_traits::Backlink { 1422 + uri: uri_repost.clone(), 1423 + path: tranquil_db_traits::BacklinkPath::SubjectUri, 1424 + link_to: target.to_string(), 1425 + }, 1426 + ], 1427 + backlinks_to_remove: vec![], 1428 + commit_event: CommitEventData { 1429 + did: did.clone(), 1430 + event_type: RepoEventType::Commit, 1431 + commit_cid: Some(mid_root.clone()), 1432 + prev_cid: Some(root_cid.clone()), 1433 + ops: None, 1434 + blobs: None, 1435 + blocks_cids: None, 1436 + prev_data_cid: None, 1437 + rev: Some("rev1".to_string()), 1438 + }, 1439 + }; 1440 + ops.apply_commit(input).unwrap(); 1441 + 1442 + let indexes = h.metastore.partition(Partition::Indexes); 1443 + let user_hash = h.metastore.user_hashes().get(&user_id).unwrap(); 1444 + let target_prefix = backlink_target_prefix(target); 1445 + let user_prefix = backlink_by_user_prefix(user_hash); 1446 + 1447 + assert_eq!( 1448 + indexes 1449 + .prefix(target_prefix.as_slice()) 1450 + .map(|g| g.into_inner().expect("scan must not fail")) 1451 + .fold(0, |acc, _| acc + 1), 1452 + 2 1453 + ); 1454 + assert_eq!( 1455 + indexes 1456 + .prefix(user_prefix.as_slice()) 1457 + .map(|g| g.into_inner().expect("scan must not fail")) 1458 + .fold(0, |acc, _| acc + 1), 1459 + 2 1460 + ); 1461 + 1462 + let final_root = test_cid_link(99); 1463 + let remove_like = ApplyCommitInput { 1464 + user_id, 1465 + did: did.clone(), 1466 + expected_root_cid: Some(mid_root.clone()), 1467 + new_root_cid: final_root.clone(), 1468 + new_rev: "rev2".to_string(), 1469 + new_block_cids: vec![], 1470 + obsolete_block_cids: vec![], 1471 + record_upserts: vec![], 1472 + record_deletes: vec![], 1473 + backlinks_to_add: vec![], 1474 + backlinks_to_remove: vec![uri_like], 1475 + commit_event: CommitEventData { 1476 + did: did.clone(), 1477 + event_type: RepoEventType::Commit, 1478 + commit_cid: Some(final_root.clone()), 1479 + prev_cid: Some(mid_root.clone()), 1480 + ops: None, 1481 + blobs: None, 1482 + blocks_cids: None, 1483 + prev_data_cid: None, 1484 + rev: Some("rev2".to_string()), 1485 + }, 1486 + }; 1487 + ops.apply_commit(remove_like).unwrap(); 1488 + 1489 + assert_eq!( 1490 + indexes 1491 + .prefix(target_prefix.as_slice()) 1492 + .map(|g| g.into_inner().expect("scan must not fail")) 1493 + .fold(0, |acc, _| acc + 1), 1494 + 1 1495 + ); 1496 + assert_eq!( 1497 + indexes 1498 + .prefix(user_prefix.as_slice()) 1499 + .map(|g| g.into_inner().expect("scan must not fail")) 1500 + .fold(0, |acc, _| acc + 1), 1501 + 1 1502 + ); 1503 + } 1504 + }
+599
crates/tranquil-store/src/metastore/encoding.rs
··· 1 + use smallvec::SmallVec; 2 + 3 + const NULL_ESCAPE: u8 = 0x01; 4 + const NULL_TERMINATOR: [u8; 2] = [0x00, 0x00]; 5 + 6 + pub fn encode_u64(buf: &mut SmallVec<[u8; 128]>, value: u64) { 7 + buf.extend_from_slice(&value.to_be_bytes()); 8 + } 9 + 10 + pub fn decode_u64(src: &[u8]) -> Option<(u64, &[u8])> { 11 + let (bytes, rest) = src.split_first_chunk::<8>()?; 12 + Some((u64::from_be_bytes(*bytes), rest)) 13 + } 14 + 15 + pub fn encode_i64(buf: &mut SmallVec<[u8; 128]>, value: i64) { 16 + let encoded = (value as u64) ^ (1u64 << 63); 17 + buf.extend_from_slice(&encoded.to_be_bytes()); 18 + } 19 + 20 + pub fn decode_i64(src: &[u8]) -> Option<(i64, &[u8])> { 21 + let (bytes, rest) = src.split_first_chunk::<8>()?; 22 + let raw = u64::from_be_bytes(*bytes) ^ (1u64 << 63); 23 + Some((raw as i64, rest)) 24 + } 25 + 26 + pub fn encode_u32(buf: &mut SmallVec<[u8; 128]>, value: u32) { 27 + buf.extend_from_slice(&value.to_be_bytes()); 28 + } 29 + 30 + pub fn decode_u32(src: &[u8]) -> Option<(u32, &[u8])> { 31 + let (bytes, rest) = src.split_first_chunk::<4>()?; 32 + Some((u32::from_be_bytes(*bytes), rest)) 33 + } 34 + 35 + pub fn encode_u16(buf: &mut SmallVec<[u8; 128]>, value: u16) { 36 + buf.extend_from_slice(&value.to_be_bytes()); 37 + } 38 + 39 + pub fn decode_u16(src: &[u8]) -> Option<(u16, &[u8])> { 40 + let (bytes, rest) = src.split_first_chunk::<2>()?; 41 + Some((u16::from_be_bytes(*bytes), rest)) 42 + } 43 + 44 + pub fn encode_bool(buf: &mut SmallVec<[u8; 128]>, value: bool) { 45 + buf.push(u8::from(value)); 46 + } 47 + 48 + pub fn decode_bool(src: &[u8]) -> Option<(bool, &[u8])> { 49 + let (&byte, rest) = src.split_first()?; 50 + match byte { 51 + 0 => Some((false, rest)), 52 + 1 => Some((true, rest)), 53 + _ => None, 54 + } 55 + } 56 + 57 + pub fn encode_bytes(buf: &mut SmallVec<[u8; 128]>, value: &[u8]) { 58 + value.iter().for_each(|&b| match b { 59 + 0x00 => { 60 + buf.push(0x00); 61 + buf.push(NULL_ESCAPE); 62 + } 63 + other => buf.push(other), 64 + }); 65 + buf.extend_from_slice(&NULL_TERMINATOR); 66 + } 67 + 68 + pub fn decode_bytes(src: &[u8]) -> Option<(Vec<u8>, &[u8])> { 69 + let mut result = Vec::new(); 70 + let mut i = 0; 71 + loop { 72 + match src.get(i)? { 73 + 0x00 => match src.get(i + 1)? { 74 + 0x00 => return Some((result, &src[i + 2..])), 75 + &NULL_ESCAPE => { 76 + result.push(0x00); 77 + i += 2; 78 + } 79 + _ => return None, 80 + }, 81 + &b => { 82 + result.push(b); 83 + i += 1; 84 + } 85 + } 86 + } 87 + } 88 + 89 + pub fn encode_string(buf: &mut SmallVec<[u8; 128]>, value: &str) { 90 + encode_bytes(buf, value.as_bytes()); 91 + } 92 + 93 + pub fn decode_string(src: &[u8]) -> Option<(String, &[u8])> { 94 + let (bytes, rest) = decode_bytes(src)?; 95 + String::from_utf8(bytes).ok().map(|s| (s, rest)) 96 + } 97 + 98 + pub struct KeyBuilder(SmallVec<[u8; 128]>); 99 + 100 + impl KeyBuilder { 101 + pub fn new() -> Self { 102 + Self(SmallVec::new()) 103 + } 104 + 105 + pub fn with_capacity(cap: usize) -> Self { 106 + Self(SmallVec::with_capacity(cap)) 107 + } 108 + 109 + pub fn u64(mut self, value: u64) -> Self { 110 + encode_u64(&mut self.0, value); 111 + self 112 + } 113 + 114 + pub fn i64(mut self, value: i64) -> Self { 115 + encode_i64(&mut self.0, value); 116 + self 117 + } 118 + 119 + pub fn u32(mut self, value: u32) -> Self { 120 + encode_u32(&mut self.0, value); 121 + self 122 + } 123 + 124 + pub fn u16(mut self, value: u16) -> Self { 125 + encode_u16(&mut self.0, value); 126 + self 127 + } 128 + 129 + pub fn bool(mut self, value: bool) -> Self { 130 + encode_bool(&mut self.0, value); 131 + self 132 + } 133 + 134 + pub fn bytes(mut self, value: &[u8]) -> Self { 135 + encode_bytes(&mut self.0, value); 136 + self 137 + } 138 + 139 + pub fn string(mut self, value: &str) -> Self { 140 + encode_string(&mut self.0, value); 141 + self 142 + } 143 + 144 + pub fn tag(mut self, tag: super::keys::KeyTag) -> Self { 145 + self.0.push(tag.raw()); 146 + self 147 + } 148 + 149 + pub fn fixed<const N: usize>(mut self, bytes: &[u8; N]) -> Self { 150 + self.0.extend_from_slice(bytes); 151 + self 152 + } 153 + 154 + pub fn raw(mut self, bytes: &[u8]) -> Self { 155 + self.0.extend_from_slice(bytes); 156 + self 157 + } 158 + 159 + pub fn build(self) -> SmallVec<[u8; 128]> { 160 + self.0 161 + } 162 + 163 + pub fn as_bytes(&self) -> &[u8] { 164 + &self.0 165 + } 166 + } 167 + 168 + impl Default for KeyBuilder { 169 + fn default() -> Self { 170 + Self::new() 171 + } 172 + } 173 + 174 + pub struct KeyReader<'a>(&'a [u8]); 175 + 176 + impl<'a> KeyReader<'a> { 177 + pub fn new(src: &'a [u8]) -> Self { 178 + Self(src) 179 + } 180 + 181 + pub fn u64(&mut self) -> Option<u64> { 182 + let (val, rest) = decode_u64(self.0)?; 183 + self.0 = rest; 184 + Some(val) 185 + } 186 + 187 + pub fn i64(&mut self) -> Option<i64> { 188 + let (val, rest) = decode_i64(self.0)?; 189 + self.0 = rest; 190 + Some(val) 191 + } 192 + 193 + pub fn u32(&mut self) -> Option<u32> { 194 + let (val, rest) = decode_u32(self.0)?; 195 + self.0 = rest; 196 + Some(val) 197 + } 198 + 199 + pub fn u16(&mut self) -> Option<u16> { 200 + let (val, rest) = decode_u16(self.0)?; 201 + self.0 = rest; 202 + Some(val) 203 + } 204 + 205 + pub fn bool(&mut self) -> Option<bool> { 206 + let (val, rest) = decode_bool(self.0)?; 207 + self.0 = rest; 208 + Some(val) 209 + } 210 + 211 + pub fn bytes(&mut self) -> Option<Vec<u8>> { 212 + let (val, rest) = decode_bytes(self.0)?; 213 + self.0 = rest; 214 + Some(val) 215 + } 216 + 217 + pub fn string(&mut self) -> Option<String> { 218 + let (val, rest) = decode_string(self.0)?; 219 + self.0 = rest; 220 + Some(val) 221 + } 222 + 223 + pub fn tag(&mut self) -> Option<u8> { 224 + let (&tag, rest) = self.0.split_first()?; 225 + self.0 = rest; 226 + Some(tag) 227 + } 228 + 229 + pub fn remaining(&self) -> &'a [u8] { 230 + self.0 231 + } 232 + 233 + pub fn is_empty(&self) -> bool { 234 + self.0.is_empty() 235 + } 236 + } 237 + 238 + pub fn exclusive_upper_bound(prefix: &[u8]) -> Option<SmallVec<[u8; 128]>> { 239 + prefix.iter().rposition(|&b| b != 0xFF).map(|pos| { 240 + let mut result = SmallVec::from_slice(&prefix[..=pos]); 241 + result[pos] = prefix[pos].wrapping_add(1); 242 + result 243 + }) 244 + } 245 + 246 + #[cfg(test)] 247 + mod tests { 248 + use super::*; 249 + use proptest::prelude::*; 250 + 251 + #[test] 252 + fn u64_roundtrip_boundaries() { 253 + [0u64, 1, u64::MAX / 2, u64::MAX - 1, u64::MAX] 254 + .iter() 255 + .for_each(|&v| { 256 + let mut buf = SmallVec::new(); 257 + encode_u64(&mut buf, v); 258 + let (decoded, rest) = decode_u64(&buf).unwrap(); 259 + assert_eq!(decoded, v); 260 + assert!(rest.is_empty()); 261 + }); 262 + } 263 + 264 + #[test] 265 + fn i64_roundtrip_boundaries() { 266 + [i64::MIN, -1, 0, 1, i64::MAX].iter().for_each(|&v| { 267 + let mut buf = SmallVec::new(); 268 + encode_i64(&mut buf, v); 269 + let (decoded, rest) = decode_i64(&buf).unwrap(); 270 + assert_eq!(decoded, v); 271 + assert!(rest.is_empty()); 272 + }); 273 + } 274 + 275 + #[test] 276 + fn bool_roundtrip() { 277 + [false, true].iter().for_each(|&v| { 278 + let mut buf = SmallVec::new(); 279 + encode_bool(&mut buf, v); 280 + let (decoded, rest) = decode_bool(&buf).unwrap(); 281 + assert_eq!(decoded, v); 282 + assert!(rest.is_empty()); 283 + }); 284 + } 285 + 286 + #[test] 287 + fn bytes_with_nulls() { 288 + let input = &[0x00, 0x01, 0x00, 0xFF, 0x00]; 289 + let mut buf = SmallVec::new(); 290 + encode_bytes(&mut buf, input); 291 + let (decoded, rest) = decode_bytes(&buf).unwrap(); 292 + assert_eq!(decoded, input); 293 + assert!(rest.is_empty()); 294 + } 295 + 296 + #[test] 297 + fn empty_bytes_roundtrip() { 298 + let mut buf = SmallVec::new(); 299 + encode_bytes(&mut buf, &[]); 300 + let (decoded, rest) = decode_bytes(&buf).unwrap(); 301 + assert!(decoded.is_empty()); 302 + assert!(rest.is_empty()); 303 + } 304 + 305 + #[test] 306 + fn empty_string_roundtrip() { 307 + let mut buf = SmallVec::new(); 308 + encode_string(&mut buf, ""); 309 + let (decoded, rest) = decode_string(&buf).unwrap(); 310 + assert_eq!(decoded, ""); 311 + assert!(rest.is_empty()); 312 + } 313 + 314 + #[test] 315 + fn string_with_null_bytes() { 316 + let input = "hello\x00world"; 317 + let mut buf = SmallVec::new(); 318 + encode_string(&mut buf, input); 319 + let (decoded, rest) = decode_string(&buf).unwrap(); 320 + assert_eq!(decoded, input); 321 + assert!(rest.is_empty()); 322 + } 323 + 324 + #[test] 325 + fn key_builder_composite_roundtrip() { 326 + let key = KeyBuilder::new() 327 + .tag(super::super::keys::KeyTag::RECORDS) 328 + .u64(42) 329 + .string("app.bsky.feed.post") 330 + .string("3k2a") 331 + .build(); 332 + 333 + let mut reader = KeyReader::new(&key); 334 + assert_eq!( 335 + reader.tag(), 336 + Some(super::super::keys::KeyTag::RECORDS.raw()) 337 + ); 338 + assert_eq!(reader.u64(), Some(42)); 339 + assert_eq!(reader.string(), Some("app.bsky.feed.post".to_string())); 340 + assert_eq!(reader.string(), Some("3k2a".to_string())); 341 + assert!(reader.is_empty()); 342 + } 343 + 344 + #[test] 345 + fn key_builder_ordering_preserves_field_order() { 346 + let key_a = KeyBuilder::new().u64(1).string("aaa").build(); 347 + let key_b = KeyBuilder::new().u64(1).string("bbb").build(); 348 + let key_c = KeyBuilder::new().u64(2).string("aaa").build(); 349 + 350 + assert!(key_a.as_slice() < key_b.as_slice()); 351 + assert!(key_b.as_slice() < key_c.as_slice()); 352 + } 353 + 354 + #[test] 355 + fn decode_bytes_rejects_invalid_escape() { 356 + assert!(decode_bytes(&[0x00, 0x02]).is_none()); 357 + assert!(decode_bytes(&[0x00, 0xFF]).is_none()); 358 + assert!(decode_bytes(&[0x41, 0x00, 0x03]).is_none()); 359 + } 360 + 361 + #[test] 362 + fn decode_bytes_rejects_truncated_input() { 363 + assert!(decode_bytes(&[]).is_none()); 364 + assert!(decode_bytes(&[0x00]).is_none()); 365 + assert!(decode_bytes(&[0x41]).is_none()); 366 + assert!(decode_bytes(&[0x41, 0x00]).is_none()); 367 + assert!(decode_bytes(&[0x00, 0x01]).is_none()); 368 + } 369 + 370 + #[test] 371 + fn decode_bool_rejects_invalid_byte() { 372 + assert!(decode_bool(&[0x02]).is_none()); 373 + assert!(decode_bool(&[0xFF]).is_none()); 374 + assert!(decode_bool(&[]).is_none()); 375 + } 376 + 377 + #[test] 378 + fn decode_string_rejects_invalid_utf8() { 379 + let mut buf = SmallVec::new(); 380 + encode_bytes(&mut buf, &[0xFF, 0xFE]); 381 + assert!(decode_string(&buf).is_none()); 382 + } 383 + 384 + #[test] 385 + fn decode_u64_rejects_short_input() { 386 + assert!(decode_u64(&[]).is_none()); 387 + assert!(decode_u64(&[0x00; 7]).is_none()); 388 + } 389 + 390 + #[test] 391 + fn decode_u32_rejects_short_input() { 392 + assert!(decode_u32(&[]).is_none()); 393 + assert!(decode_u32(&[0x00; 3]).is_none()); 394 + } 395 + 396 + #[test] 397 + fn decode_u16_rejects_short_input() { 398 + assert!(decode_u16(&[]).is_none()); 399 + assert!(decode_u16(&[0x00]).is_none()); 400 + } 401 + 402 + #[test] 403 + fn decode_i64_rejects_short_input() { 404 + assert!(decode_i64(&[]).is_none()); 405 + assert!(decode_i64(&[0x00; 7]).is_none()); 406 + } 407 + 408 + #[test] 409 + fn fixed_key_roundtrip() { 410 + let data: [u8; 4] = [0xDE, 0xAD, 0xBE, 0xEF]; 411 + let key = KeyBuilder::new() 412 + .tag(super::super::keys::KeyTag::RECORDS) 413 + .fixed(&data) 414 + .build(); 415 + 416 + let mut reader = KeyReader::new(&key); 417 + assert_eq!( 418 + reader.tag(), 419 + Some(super::super::keys::KeyTag::RECORDS.raw()) 420 + ); 421 + assert_eq!(reader.remaining(), &data); 422 + } 423 + 424 + proptest! { 425 + #[test] 426 + fn prop_u64_roundtrip(v: u64) { 427 + let mut buf = SmallVec::new(); 428 + encode_u64(&mut buf, v); 429 + let (decoded, rest) = decode_u64(&buf).unwrap(); 430 + prop_assert_eq!(decoded, v); 431 + prop_assert!(rest.is_empty()); 432 + } 433 + 434 + #[test] 435 + fn prop_u64_ordering(a: u64, b: u64) { 436 + let mut buf_a = SmallVec::new(); 437 + let mut buf_b = SmallVec::new(); 438 + encode_u64(&mut buf_a, a); 439 + encode_u64(&mut buf_b, b); 440 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 441 + } 442 + 443 + #[test] 444 + fn prop_i64_roundtrip(v: i64) { 445 + let mut buf = SmallVec::new(); 446 + encode_i64(&mut buf, v); 447 + let (decoded, rest) = decode_i64(&buf).unwrap(); 448 + prop_assert_eq!(decoded, v); 449 + prop_assert!(rest.is_empty()); 450 + } 451 + 452 + #[test] 453 + fn prop_i64_ordering(a: i64, b: i64) { 454 + let mut buf_a = SmallVec::new(); 455 + let mut buf_b = SmallVec::new(); 456 + encode_i64(&mut buf_a, a); 457 + encode_i64(&mut buf_b, b); 458 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 459 + } 460 + 461 + #[test] 462 + fn prop_u32_roundtrip(v: u32) { 463 + let mut buf = SmallVec::new(); 464 + encode_u32(&mut buf, v); 465 + let (decoded, rest) = decode_u32(&buf).unwrap(); 466 + prop_assert_eq!(decoded, v); 467 + prop_assert!(rest.is_empty()); 468 + } 469 + 470 + #[test] 471 + fn prop_u32_ordering(a: u32, b: u32) { 472 + let mut buf_a = SmallVec::new(); 473 + let mut buf_b = SmallVec::new(); 474 + encode_u32(&mut buf_a, a); 475 + encode_u32(&mut buf_b, b); 476 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 477 + } 478 + 479 + #[test] 480 + fn prop_u16_roundtrip(v: u16) { 481 + let mut buf = SmallVec::new(); 482 + encode_u16(&mut buf, v); 483 + let (decoded, rest) = decode_u16(&buf).unwrap(); 484 + prop_assert_eq!(decoded, v); 485 + prop_assert!(rest.is_empty()); 486 + } 487 + 488 + #[test] 489 + fn prop_u16_ordering(a: u16, b: u16) { 490 + let mut buf_a = SmallVec::new(); 491 + let mut buf_b = SmallVec::new(); 492 + encode_u16(&mut buf_a, a); 493 + encode_u16(&mut buf_b, b); 494 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 495 + } 496 + 497 + #[test] 498 + fn prop_bool_roundtrip(v: bool) { 499 + let mut buf = SmallVec::new(); 500 + encode_bool(&mut buf, v); 501 + let (decoded, rest) = decode_bool(&buf).unwrap(); 502 + prop_assert_eq!(decoded, v); 503 + prop_assert!(rest.is_empty()); 504 + } 505 + 506 + #[test] 507 + fn prop_bool_ordering(a: bool, b: bool) { 508 + let mut buf_a = SmallVec::new(); 509 + let mut buf_b = SmallVec::new(); 510 + encode_bool(&mut buf_a, a); 511 + encode_bool(&mut buf_b, b); 512 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 513 + } 514 + 515 + #[test] 516 + fn prop_bytes_roundtrip(v in proptest::collection::vec(any::<u8>(), 0..256)) { 517 + let mut buf = SmallVec::new(); 518 + encode_bytes(&mut buf, &v); 519 + let (decoded, rest) = decode_bytes(&buf).unwrap(); 520 + prop_assert_eq!(decoded, v); 521 + prop_assert!(rest.is_empty()); 522 + } 523 + 524 + #[test] 525 + fn prop_bytes_ordering( 526 + a in proptest::collection::vec(any::<u8>(), 0..64), 527 + b in proptest::collection::vec(any::<u8>(), 0..64), 528 + ) { 529 + let mut buf_a = SmallVec::new(); 530 + let mut buf_b = SmallVec::new(); 531 + encode_bytes(&mut buf_a, &a); 532 + encode_bytes(&mut buf_b, &b); 533 + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); 534 + } 535 + 536 + #[test] 537 + fn prop_string_roundtrip(v in "\\PC{0,128}") { 538 + let mut buf = SmallVec::new(); 539 + encode_string(&mut buf, &v); 540 + let (decoded, rest) = decode_string(&buf).unwrap(); 541 + prop_assert_eq!(decoded, v); 542 + prop_assert!(rest.is_empty()); 543 + } 544 + 545 + #[test] 546 + fn prop_string_ordering( 547 + a in "[\\x00-\\xff]{0,32}", 548 + b in "[\\x00-\\xff]{0,32}", 549 + ) { 550 + let mut buf_a = SmallVec::new(); 551 + let mut buf_b = SmallVec::new(); 552 + encode_string(&mut buf_a, &a); 553 + encode_string(&mut buf_b, &b); 554 + prop_assert_eq!( 555 + buf_a.as_slice().cmp(buf_b.as_slice()), 556 + a.as_bytes().cmp(b.as_bytes()) 557 + ); 558 + } 559 + 560 + #[test] 561 + fn prop_composite_roundtrip( 562 + tag_raw in 0u8..=255, 563 + num in any::<u64>(), 564 + s1 in "\\PC{0,32}", 565 + s2 in "\\PC{0,32}", 566 + ) { 567 + let tag = super::super::keys::KeyTag::from_raw_unchecked(tag_raw); 568 + let key = KeyBuilder::new() 569 + .tag(tag) 570 + .u64(num) 571 + .string(&s1) 572 + .string(&s2) 573 + .build(); 574 + 575 + let mut reader = KeyReader::new(&key); 576 + prop_assert_eq!(reader.tag(), Some(tag_raw)); 577 + prop_assert_eq!(reader.u64(), Some(num)); 578 + prop_assert_eq!(reader.string(), Some(s1)); 579 + prop_assert_eq!(reader.string(), Some(s2)); 580 + prop_assert!(reader.is_empty()); 581 + } 582 + 583 + #[test] 584 + fn prop_composite_ordering( 585 + tag_raw in 0u8..=10, 586 + a_num in any::<u64>(), 587 + b_num in any::<u64>(), 588 + a_str in "[a-z]{0,8}", 589 + b_str in "[a-z]{0,8}", 590 + ) { 591 + let tag = super::super::keys::KeyTag::from_raw_unchecked(tag_raw); 592 + let key_a = KeyBuilder::new().tag(tag).u64(a_num).string(&a_str).build(); 593 + let key_b = KeyBuilder::new().tag(tag).u64(b_num).string(&b_str).build(); 594 + 595 + let expected = a_num.cmp(&b_num).then_with(|| a_str.as_bytes().cmp(b_str.as_bytes())); 596 + prop_assert_eq!(key_a.as_slice().cmp(key_b.as_slice()), expected); 597 + } 598 + } 599 + }
+230
crates/tranquil-store/src/metastore/event_keys.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use smallvec::SmallVec; 3 + 4 + use super::encoding::KeyBuilder; 5 + use super::keys::{KeyTag, UserHash}; 6 + 7 + const SEQ_META_SCHEMA_VERSION: u8 = 1; 8 + 9 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 10 + pub struct SeqMetaValue { 11 + pub blocks_cids: Vec<String>, 12 + } 13 + 14 + impl SeqMetaValue { 15 + pub fn serialize(&self) -> Vec<u8> { 16 + let payload = postcard::to_allocvec(self).expect("SeqMetaValue serialization cannot fail"); 17 + let mut buf = Vec::with_capacity(1 + payload.len()); 18 + buf.push(SEQ_META_SCHEMA_VERSION); 19 + buf.extend_from_slice(&payload); 20 + buf 21 + } 22 + 23 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 24 + let (&version, payload) = bytes.split_first()?; 25 + match version { 26 + SEQ_META_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 27 + _ => None, 28 + } 29 + } 30 + } 31 + 32 + pub fn rev_to_seq_key(user_hash: UserHash, rev: &str) -> SmallVec<[u8; 128]> { 33 + KeyBuilder::new() 34 + .tag(KeyTag::REV_TO_SEQ) 35 + .u64(user_hash.raw()) 36 + .string(rev) 37 + .build() 38 + } 39 + 40 + pub fn rev_to_seq_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 41 + KeyBuilder::new() 42 + .tag(KeyTag::REV_TO_SEQ) 43 + .u64(user_hash.raw()) 44 + .build() 45 + } 46 + 47 + pub fn seq_meta_key(seq: u64) -> SmallVec<[u8; 128]> { 48 + KeyBuilder::new().tag(KeyTag::SEQ_META).u64(seq).build() 49 + } 50 + 51 + pub fn seq_tombstone_key(seq: u64) -> SmallVec<[u8; 128]> { 52 + KeyBuilder::new() 53 + .tag(KeyTag::SEQ_TOMBSTONE) 54 + .u64(seq) 55 + .build() 56 + } 57 + 58 + pub fn did_events_key(user_hash: UserHash, seq: u64) -> SmallVec<[u8; 128]> { 59 + KeyBuilder::new() 60 + .tag(KeyTag::DID_EVENTS) 61 + .u64(user_hash.raw()) 62 + .u64(seq) 63 + .build() 64 + } 65 + 66 + pub fn did_events_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 67 + KeyBuilder::new() 68 + .tag(KeyTag::DID_EVENTS) 69 + .u64(user_hash.raw()) 70 + .build() 71 + } 72 + 73 + pub fn metastore_cursor_key() -> SmallVec<[u8; 128]> { 74 + KeyBuilder::new() 75 + .tag(KeyTag::METASTORE_CURSOR) 76 + .raw(&[0x00]) 77 + .build() 78 + } 79 + 80 + #[cfg(test)] 81 + mod tests { 82 + use super::*; 83 + use crate::metastore::encoding::KeyReader; 84 + 85 + #[test] 86 + fn seq_meta_value_roundtrip() { 87 + let value = SeqMetaValue { 88 + blocks_cids: vec!["bafyreiblock1".to_owned(), "bafyreiblock2".to_owned()], 89 + }; 90 + let bytes = value.serialize(); 91 + let decoded = SeqMetaValue::deserialize(&bytes).unwrap(); 92 + assert_eq!(decoded, value); 93 + } 94 + 95 + #[test] 96 + fn seq_meta_value_empty_blocks() { 97 + let value = SeqMetaValue { 98 + blocks_cids: vec![], 99 + }; 100 + let bytes = value.serialize(); 101 + let decoded = SeqMetaValue::deserialize(&bytes).unwrap(); 102 + assert_eq!(decoded, value); 103 + } 104 + 105 + #[test] 106 + fn seq_meta_schema_version_first_byte() { 107 + let value = SeqMetaValue { 108 + blocks_cids: vec![], 109 + }; 110 + let bytes = value.serialize(); 111 + assert_eq!(bytes[0], SEQ_META_SCHEMA_VERSION); 112 + } 113 + 114 + #[test] 115 + fn seq_meta_rejects_unknown_version() { 116 + let value = SeqMetaValue { 117 + blocks_cids: vec![], 118 + }; 119 + let mut bytes = value.serialize(); 120 + bytes[0] = 99; 121 + assert!(SeqMetaValue::deserialize(&bytes).is_none()); 122 + } 123 + 124 + #[test] 125 + fn seq_meta_rejects_empty_input() { 126 + assert!(SeqMetaValue::deserialize(&[]).is_none()); 127 + } 128 + 129 + #[test] 130 + fn rev_to_seq_key_roundtrip() { 131 + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); 132 + let key = rev_to_seq_key(hash, "3k2abcde"); 133 + let mut reader = KeyReader::new(&key); 134 + assert_eq!(reader.tag(), Some(KeyTag::REV_TO_SEQ.raw())); 135 + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); 136 + assert_eq!(reader.string(), Some("3k2abcde".to_owned())); 137 + assert!(reader.is_empty()); 138 + } 139 + 140 + #[test] 141 + fn rev_to_seq_keys_sort_by_user_then_rev() { 142 + let h1 = UserHash::from_raw(1); 143 + let h2 = UserHash::from_raw(2); 144 + let k1 = rev_to_seq_key(h1, "abc"); 145 + let k2 = rev_to_seq_key(h1, "def"); 146 + let k3 = rev_to_seq_key(h2, "abc"); 147 + assert!(k1.as_slice() < k2.as_slice()); 148 + assert!(k2.as_slice() < k3.as_slice()); 149 + } 150 + 151 + #[test] 152 + fn rev_to_seq_user_prefix_is_prefix_of_full_key() { 153 + let hash = UserHash::from_raw(42); 154 + let prefix = rev_to_seq_user_prefix(hash); 155 + let full = rev_to_seq_key(hash, "some_rev"); 156 + assert!(full.as_slice().starts_with(prefix.as_slice())); 157 + } 158 + 159 + #[test] 160 + fn seq_meta_key_roundtrip() { 161 + let key = seq_meta_key(12345); 162 + let mut reader = KeyReader::new(&key); 163 + assert_eq!(reader.tag(), Some(KeyTag::SEQ_META.raw())); 164 + assert_eq!(reader.u64(), Some(12345)); 165 + assert!(reader.is_empty()); 166 + } 167 + 168 + #[test] 169 + fn seq_meta_keys_sort_by_seq() { 170 + let k1 = seq_meta_key(1); 171 + let k2 = seq_meta_key(2); 172 + let k3 = seq_meta_key(100); 173 + assert!(k1.as_slice() < k2.as_slice()); 174 + assert!(k2.as_slice() < k3.as_slice()); 175 + } 176 + 177 + #[test] 178 + fn seq_tombstone_key_roundtrip() { 179 + let key = seq_tombstone_key(999); 180 + let mut reader = KeyReader::new(&key); 181 + assert_eq!(reader.tag(), Some(KeyTag::SEQ_TOMBSTONE.raw())); 182 + assert_eq!(reader.u64(), Some(999)); 183 + assert!(reader.is_empty()); 184 + } 185 + 186 + #[test] 187 + fn did_events_key_roundtrip() { 188 + let hash = UserHash::from_raw(0xCAFE_BABE_DEAD_BEEF); 189 + let key = did_events_key(hash, 42); 190 + let mut reader = KeyReader::new(&key); 191 + assert_eq!(reader.tag(), Some(KeyTag::DID_EVENTS.raw())); 192 + assert_eq!(reader.u64(), Some(0xCAFE_BABE_DEAD_BEEF)); 193 + assert_eq!(reader.u64(), Some(42)); 194 + assert!(reader.is_empty()); 195 + } 196 + 197 + #[test] 198 + fn did_events_keys_sort_by_user_then_seq() { 199 + let h1 = UserHash::from_raw(1); 200 + let h2 = UserHash::from_raw(2); 201 + let k1 = did_events_key(h1, 10); 202 + let k2 = did_events_key(h1, 20); 203 + let k3 = did_events_key(h2, 5); 204 + assert!(k1.as_slice() < k2.as_slice()); 205 + assert!(k2.as_slice() < k3.as_slice()); 206 + } 207 + 208 + #[test] 209 + fn did_events_prefix_is_prefix_of_full_key() { 210 + let hash = UserHash::from_raw(99); 211 + let prefix = did_events_prefix(hash); 212 + let full = did_events_key(hash, 1); 213 + assert!(full.as_slice().starts_with(prefix.as_slice())); 214 + } 215 + 216 + #[test] 217 + fn metastore_cursor_key_roundtrip() { 218 + let key = metastore_cursor_key(); 219 + let mut reader = KeyReader::new(&key); 220 + assert_eq!(reader.tag(), Some(KeyTag::METASTORE_CURSOR.raw())); 221 + assert_eq!(reader.remaining(), &[0x00]); 222 + } 223 + 224 + #[test] 225 + fn metastore_cursor_key_is_stable() { 226 + let k1 = metastore_cursor_key(); 227 + let k2 = metastore_cursor_key(); 228 + assert_eq!(k1.as_slice(), k2.as_slice()); 229 + } 230 + }
+1734
crates/tranquil-store/src/metastore/event_ops.rs
··· 1 + use std::collections::HashSet; 2 + use std::sync::Arc; 3 + 4 + use chrono::{DateTime, Utc}; 5 + use fjall::{Database, Keyspace}; 6 + use tracing::warn; 7 + use tranquil_db_traits::{ 8 + AccountStatus, CommitEventData, DbError, EventBlocksCids, RepoEventType, SequenceNumber, 9 + SequencedEvent, 10 + }; 11 + use tranquil_types::{CidLink, Did, Handle}; 12 + 13 + use super::encoding::{KeyReader, exclusive_upper_bound}; 14 + use super::event_keys::{ 15 + SeqMetaValue, did_events_key, did_events_prefix, metastore_cursor_key, rev_to_seq_key, 16 + rev_to_seq_user_prefix, seq_meta_key, seq_tombstone_key, 17 + }; 18 + use super::keys::UserHash; 19 + use super::recovery::CommitMutationSet; 20 + use super::repo_meta::RepoMetaValue; 21 + use crate::eventlog::{DeferredBroadcast, EventLogBridge, EventLogNotifier}; 22 + use crate::io::StorageIO; 23 + 24 + const RECOVERY_BATCH_SIZE: usize = 4096; 25 + 26 + pub struct EventOps<S: StorageIO> { 27 + db: Database, 28 + repo_data: Keyspace, 29 + bridge: Arc<EventLogBridge<S>>, 30 + } 31 + 32 + impl<S: StorageIO> EventOps<S> { 33 + pub fn new(db: Database, repo_data: Keyspace, bridge: Arc<EventLogBridge<S>>) -> Self { 34 + Self { 35 + db, 36 + repo_data, 37 + bridge, 38 + } 39 + } 40 + 41 + pub fn notifier(&self) -> EventLogNotifier<S> { 42 + self.bridge.notifier() 43 + } 44 + 45 + pub fn notify_update(&self, _seq: SequenceNumber) -> Result<(), DbError> { 46 + Ok(()) 47 + } 48 + 49 + pub fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> { 50 + let event = Self::build_commit_event(data); 51 + self.append_and_index(&event, &data.did, data.rev.as_deref()) 52 + } 53 + 54 + pub fn append_commit_event_into_batch( 55 + &self, 56 + batch: &mut fjall::OwnedWriteBatch, 57 + data: &CommitEventData, 58 + mutation_set_bytes: Option<&[u8]>, 59 + ) -> Result<(SequenceNumber, DeferredBroadcast), DbError> { 60 + let event = Self::build_commit_event(data); 61 + let payload = crate::eventlog::encode_payload_with_mutations(&event, mutation_set_bytes); 62 + let (seq, deferred) = self 63 + .bridge 64 + .insert_event_deferred_raw(&data.did, data.event_type, payload) 65 + .map_err(|e| DbError::Query(e.to_string()))?; 66 + 67 + let seq_u64 = seq_to_u64(seq)?; 68 + let user_hash = UserHash::from_did(data.did.as_str()); 69 + self.stage_did_event(batch, user_hash, seq_u64); 70 + if let Some(rev) = &data.rev { 71 + self.stage_rev_to_seq(batch, user_hash, rev, seq_u64); 72 + } 73 + self.write_last_applied_cursor(batch, seq_u64); 74 + 75 + Ok((seq, deferred)) 76 + } 77 + 78 + pub fn complete_broadcast(&self, deferred: DeferredBroadcast) { 79 + self.bridge.complete_broadcast(deferred); 80 + } 81 + 82 + fn build_commit_event(data: &CommitEventData) -> SequencedEvent { 83 + SequencedEvent { 84 + seq: SequenceNumber::ZERO, 85 + did: data.did.clone(), 86 + created_at: Utc::now(), 87 + event_type: data.event_type, 88 + commit_cid: data.commit_cid.clone(), 89 + prev_cid: data.prev_cid.clone(), 90 + prev_data_cid: data.prev_data_cid.clone(), 91 + ops: data.ops.clone(), 92 + blobs: data.blobs.clone(), 93 + blocks_cids: data.blocks_cids.clone(), 94 + handle: None, 95 + active: None, 96 + status: None, 97 + rev: data.rev.clone(), 98 + } 99 + } 100 + 101 + pub fn insert_identity_event( 102 + &self, 103 + did: &Did, 104 + handle: Option<&Handle>, 105 + ) -> Result<SequenceNumber, DbError> { 106 + let event = SequencedEvent { 107 + seq: SequenceNumber::ZERO, 108 + did: did.clone(), 109 + created_at: Utc::now(), 110 + event_type: RepoEventType::Identity, 111 + commit_cid: None, 112 + prev_cid: None, 113 + prev_data_cid: None, 114 + ops: None, 115 + blobs: None, 116 + blocks_cids: None, 117 + handle: handle.cloned(), 118 + active: None, 119 + status: None, 120 + rev: None, 121 + }; 122 + 123 + self.append_and_index(&event, did, None) 124 + } 125 + 126 + pub fn insert_account_event( 127 + &self, 128 + did: &Did, 129 + status: AccountStatus, 130 + ) -> Result<SequenceNumber, DbError> { 131 + let active = Some(status.is_active()); 132 + let event = SequencedEvent { 133 + seq: SequenceNumber::ZERO, 134 + did: did.clone(), 135 + created_at: Utc::now(), 136 + event_type: RepoEventType::Account, 137 + commit_cid: None, 138 + prev_cid: None, 139 + prev_data_cid: None, 140 + ops: None, 141 + blobs: None, 142 + blocks_cids: None, 143 + handle: None, 144 + active, 145 + status: Some(status), 146 + rev: None, 147 + }; 148 + 149 + self.append_and_index(&event, did, None) 150 + } 151 + 152 + pub fn insert_sync_event( 153 + &self, 154 + did: &Did, 155 + commit_cid: &CidLink, 156 + rev: Option<&str>, 157 + ) -> Result<SequenceNumber, DbError> { 158 + let event = SequencedEvent { 159 + seq: SequenceNumber::ZERO, 160 + did: did.clone(), 161 + created_at: Utc::now(), 162 + event_type: RepoEventType::Sync, 163 + commit_cid: Some(commit_cid.clone()), 164 + prev_cid: None, 165 + prev_data_cid: None, 166 + ops: None, 167 + blobs: None, 168 + blocks_cids: None, 169 + handle: None, 170 + active: None, 171 + status: None, 172 + rev: rev.map(str::to_owned), 173 + }; 174 + 175 + self.append_and_index(&event, did, rev) 176 + } 177 + 178 + pub fn insert_genesis_commit_event( 179 + &self, 180 + did: &Did, 181 + commit_cid: &CidLink, 182 + mst_root_cid: &CidLink, 183 + rev: &str, 184 + ) -> Result<SequenceNumber, DbError> { 185 + let event = SequencedEvent { 186 + seq: SequenceNumber::ZERO, 187 + did: did.clone(), 188 + created_at: Utc::now(), 189 + event_type: RepoEventType::Commit, 190 + commit_cid: Some(commit_cid.clone()), 191 + prev_cid: None, 192 + prev_data_cid: Some(mst_root_cid.clone()), 193 + ops: None, 194 + blobs: None, 195 + blocks_cids: None, 196 + handle: None, 197 + active: None, 198 + status: None, 199 + rev: Some(rev.to_owned()), 200 + }; 201 + 202 + self.append_and_index(&event, did, Some(rev)) 203 + } 204 + 205 + pub fn get_events_since_seq( 206 + &self, 207 + since: SequenceNumber, 208 + limit: Option<i64>, 209 + ) -> Result<Vec<SequencedEvent>, DbError> { 210 + let events = self.bridge.get_events_since_seq(since, limit)?; 211 + self.apply_sidecars_and_filter(events) 212 + } 213 + 214 + pub fn get_events_in_seq_range( 215 + &self, 216 + start: SequenceNumber, 217 + end: SequenceNumber, 218 + ) -> Result<Vec<SequencedEvent>, DbError> { 219 + let events = self.bridge.get_events_in_seq_range(start, end)?; 220 + self.apply_sidecars_and_filter(events) 221 + } 222 + 223 + pub fn get_event_by_seq(&self, seq: SequenceNumber) -> Result<Option<SequencedEvent>, DbError> { 224 + let seq_u64 = match seq.as_u64() { 225 + Some(v) => v, 226 + None => return Ok(None), 227 + }; 228 + 229 + if self.is_tombstoned(seq_u64)? { 230 + return Ok(None); 231 + } 232 + 233 + self.bridge 234 + .get_event_by_seq(seq) 235 + .map(|opt| opt.map(|e| self.merge_sidecar(e))) 236 + } 237 + 238 + pub fn get_events_since_cursor( 239 + &self, 240 + cursor: SequenceNumber, 241 + limit: i64, 242 + ) -> Result<Vec<SequencedEvent>, DbError> { 243 + let events = self.bridge.get_events_since_cursor(cursor, limit)?; 244 + self.apply_sidecars_and_filter(events) 245 + } 246 + 247 + pub fn get_max_seq(&self) -> SequenceNumber { 248 + self.bridge.get_max_seq() 249 + } 250 + 251 + pub fn get_min_seq_since( 252 + &self, 253 + since: DateTime<Utc>, 254 + ) -> Result<Option<SequenceNumber>, DbError> { 255 + self.bridge.get_min_seq_since(since) 256 + } 257 + 258 + pub fn get_events_since_rev( 259 + &self, 260 + did: &Did, 261 + since_rev: &str, 262 + ) -> Result<Vec<EventBlocksCids>, DbError> { 263 + let user_hash = UserHash::from_did(did.as_str()); 264 + 265 + let key = rev_to_seq_key(user_hash, since_rev); 266 + let since_seq_u64 = match self.repo_data.get(key).map_err(fjall_to_db)? { 267 + Some(bytes) => { 268 + let arr: [u8; 8] = bytes 269 + .as_ref() 270 + .try_into() 271 + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; 272 + u64::from_be_bytes(arr) 273 + } 274 + None => return Ok(Vec::new()), 275 + }; 276 + 277 + let start_seq = match since_seq_u64.checked_add(1) { 278 + Some(s) => s, 279 + None => return Ok(Vec::new()), 280 + }; 281 + 282 + let user_seqs = self.scan_did_events(user_hash, start_seq)?; 283 + 284 + user_seqs 285 + .into_iter() 286 + .try_fold(Vec::new(), |mut acc, seq_u64| { 287 + if self.is_tombstoned(seq_u64)? { 288 + return Ok(acc); 289 + } 290 + let seq_sn = SequenceNumber::from_raw( 291 + i64::try_from(seq_u64) 292 + .map_err(|_| DbError::Query("seq exceeds i64::MAX".to_owned()))?, 293 + ); 294 + match self.bridge.get_event_by_seq(seq_sn)? { 295 + Some(event) if event.rev.is_some() => { 296 + let merged = self.merge_sidecar(event); 297 + acc.push(EventBlocksCids { 298 + blocks_cids: merged.blocks_cids, 299 + commit_cid: merged.commit_cid, 300 + }); 301 + Ok(acc) 302 + } 303 + _ => Ok(acc), 304 + } 305 + }) 306 + } 307 + 308 + pub fn get_blob_cids_since_rev( 309 + &self, 310 + did: &Did, 311 + since_rev: &str, 312 + ) -> Result<Vec<CidLink>, DbError> { 313 + let user_hash = UserHash::from_did(did.as_str()); 314 + 315 + let key = rev_to_seq_key(user_hash, since_rev); 316 + let since_seq_u64 = match self.repo_data.get(key).map_err(fjall_to_db)? { 317 + Some(bytes) => { 318 + let arr: [u8; 8] = bytes 319 + .as_ref() 320 + .try_into() 321 + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; 322 + u64::from_be_bytes(arr) 323 + } 324 + None => return Ok(Vec::new()), 325 + }; 326 + 327 + let start_seq = match since_seq_u64.checked_add(1) { 328 + Some(s) => s, 329 + None => return Ok(Vec::new()), 330 + }; 331 + 332 + let user_seqs = self.scan_did_events(user_hash, start_seq)?; 333 + 334 + let mut seen = std::collections::BTreeSet::new(); 335 + user_seqs 336 + .into_iter() 337 + .try_fold(Vec::new(), |mut acc, seq_u64| { 338 + if self.is_tombstoned(seq_u64)? { 339 + return Ok(acc); 340 + } 341 + let seq_sn = SequenceNumber::from_raw( 342 + i64::try_from(seq_u64) 343 + .map_err(|_| DbError::Query("seq exceeds i64::MAX".to_owned()))?, 344 + ); 345 + match self.bridge.get_event_by_seq(seq_sn)? { 346 + Some(event) if event.rev.is_some() => { 347 + if let Some(blobs) = event.blobs { 348 + acc.extend( 349 + blobs 350 + .into_iter() 351 + .filter(|b| seen.insert(b.clone())) 352 + .map(CidLink::from), 353 + ); 354 + } 355 + Ok(acc) 356 + } 357 + _ => Ok(acc), 358 + } 359 + }) 360 + } 361 + 362 + pub fn update_seq_blocks_cids( 363 + &self, 364 + seq: SequenceNumber, 365 + blocks_cids: &[String], 366 + ) -> Result<(), DbError> { 367 + let seq_u64 = seq 368 + .as_u64() 369 + .ok_or_else(|| DbError::Query("invalid sequence number".to_owned()))?; 370 + let key = seq_meta_key(seq_u64); 371 + let value = SeqMetaValue { 372 + blocks_cids: blocks_cids.to_vec(), 373 + }; 374 + self.repo_data 375 + .insert(key.as_slice(), value.serialize()) 376 + .map_err(fjall_to_db) 377 + } 378 + 379 + pub fn delete_sequences_except( 380 + &self, 381 + did: &Did, 382 + keep_seq: SequenceNumber, 383 + ) -> Result<(), DbError> { 384 + let keep_raw = keep_seq.as_u64().ok_or_else(|| { 385 + DbError::Query("invalid keep_seq: negative sequence number".to_owned()) 386 + })?; 387 + 388 + let user_hash = UserHash::from_did(did.as_str()); 389 + 390 + let prefix = did_events_prefix(user_hash); 391 + let upper = exclusive_upper_bound(prefix.as_slice()) 392 + .expect("did_events prefix can never be all-0xFF"); 393 + 394 + let seqs_to_tombstone: Result<Vec<u64>, DbError> = self 395 + .repo_data 396 + .range(prefix.as_slice()..upper.as_slice()) 397 + .map(|guard| { 398 + let (key, _) = guard.into_inner().map_err(fjall_to_db)?; 399 + decode_did_events_seq(key.as_ref()) 400 + }) 401 + .filter(|result| match result { 402 + Ok(seq) => *seq != keep_raw, 403 + Err(_) => true, 404 + }) 405 + .collect(); 406 + 407 + let seqs = seqs_to_tombstone?; 408 + let tombstone_set: HashSet<u64> = seqs.iter().copied().collect(); 409 + 410 + let stale_rev_keys = self.collect_stale_rev_keys(user_hash, &tombstone_set)?; 411 + 412 + let mut batch = self.db.batch(); 413 + seqs.iter().for_each(|&seq| { 414 + batch.insert(&self.repo_data, seq_tombstone_key(seq).as_slice(), []); 415 + batch.remove(&self.repo_data, did_events_key(user_hash, seq).as_slice()); 416 + batch.remove(&self.repo_data, seq_meta_key(seq).as_slice()); 417 + }); 418 + stale_rev_keys.iter().for_each(|key| { 419 + batch.remove(&self.repo_data, key.as_slice()); 420 + }); 421 + batch.commit().map_err(fjall_to_db)?; 422 + 423 + Ok(()) 424 + } 425 + 426 + pub fn read_last_applied_cursor(&self) -> Result<Option<u64>, DbError> { 427 + let key = metastore_cursor_key(); 428 + match self.repo_data.get(key.as_slice()).map_err(fjall_to_db)? { 429 + Some(bytes) => { 430 + let arr: [u8; 8] = bytes 431 + .as_ref() 432 + .try_into() 433 + .map_err(|_| DbError::Query("corrupt metastore cursor".to_owned()))?; 434 + Ok(Some(u64::from_be_bytes(arr))) 435 + } 436 + None => Ok(None), 437 + } 438 + } 439 + 440 + pub fn write_last_applied_cursor(&self, batch: &mut fjall::OwnedWriteBatch, seq: u64) { 441 + let key = metastore_cursor_key(); 442 + batch.insert(&self.repo_data, key.as_slice(), seq.to_be_bytes()); 443 + } 444 + 445 + pub fn write_last_applied_cursor_direct(&self, seq: u64) -> Result<(), DbError> { 446 + let key = metastore_cursor_key(); 447 + self.repo_data 448 + .insert(key.as_slice(), seq.to_be_bytes()) 449 + .map_err(fjall_to_db) 450 + } 451 + 452 + pub fn recover_sidecar_indexes(&self) -> Result<u64, DbError> { 453 + let cursor_seq = self.read_last_applied_cursor()?.unwrap_or(0); 454 + let max_raw = self.bridge.get_max_seq().as_u64().unwrap_or(0); 455 + 456 + match max_raw <= cursor_seq { 457 + true => Ok(0), 458 + false => self.recover_page(cursor_seq, 0), 459 + } 460 + } 461 + 462 + pub fn recover_metastore_mutations(&self, indexes: &fjall::Keyspace) -> Result<u64, DbError> { 463 + let cursor_seq = self.read_last_applied_cursor()?.unwrap_or(0); 464 + let max_raw = self.bridge.get_max_seq().as_u64().unwrap_or(0); 465 + 466 + match max_raw <= cursor_seq { 467 + true => Ok(0), 468 + false => { 469 + tracing::info!( 470 + cursor = cursor_seq, 471 + eventlog_max = max_raw, 472 + gap = max_raw.saturating_sub(cursor_seq), 473 + "replaying metastore mutations from eventlog" 474 + ); 475 + self.recover_mutations_page(indexes, cursor_seq, 0) 476 + } 477 + } 478 + } 479 + 480 + fn recover_mutations_page( 481 + &self, 482 + indexes: &fjall::Keyspace, 483 + cursor: u64, 484 + total: u64, 485 + ) -> Result<u64, DbError> { 486 + let cursor_sn = SequenceNumber::from_raw( 487 + i64::try_from(cursor) 488 + .map_err(|_| DbError::Query("recovery cursor exceeds i64::MAX".to_owned()))?, 489 + ); 490 + let events_with_mutations = self 491 + .bridge 492 + .get_events_with_mutations_since(cursor_sn, RECOVERY_BATCH_SIZE)?; 493 + 494 + match events_with_mutations.is_empty() { 495 + true => Ok(total), 496 + false => { 497 + let page_len = events_with_mutations.len(); 498 + let mut page_high = cursor; 499 + let mut count = 0u64; 500 + 501 + events_with_mutations.iter().try_for_each(|ewm| { 502 + let seq_u64 = match ewm.event.seq.as_u64() { 503 + Some(v) => v, 504 + None => return Ok(()), 505 + }; 506 + let user_hash = UserHash::from_did(ewm.event.did.as_str()); 507 + 508 + let mut batch = self.db.batch(); 509 + 510 + self.stage_did_event(&mut batch, user_hash, seq_u64); 511 + if let Some(rev) = &ewm.event.rev { 512 + self.stage_rev_to_seq(&mut batch, user_hash, rev, seq_u64); 513 + } 514 + 515 + if let Some(ms_bytes) = &ewm.mutation_set { 516 + let ms = CommitMutationSet::deserialize(ms_bytes).ok_or_else(|| { 517 + DbError::Query(format!("corrupt CommitMutationSet at seq {seq_u64}")) 518 + })?; 519 + 520 + let meta_key = super::repo_meta::repo_meta_key(user_hash); 521 + let current_meta = self 522 + .repo_data 523 + .get(meta_key.as_slice()) 524 + .map_err(fjall_to_db)? 525 + .and_then(|raw| RepoMetaValue::deserialize(&raw)) 526 + .unwrap_or_else(|| RepoMetaValue { 527 + repo_root_cid: vec![], 528 + repo_rev: String::new(), 529 + handle: String::new(), 530 + status: super::repo_meta::RepoStatus::Active, 531 + deactivated_at_ms: None, 532 + takedown_ref: None, 533 + did: Some(ewm.event.did.as_str().to_owned()), 534 + }); 535 + 536 + super::recovery::replay_mutation_set( 537 + &mut batch, 538 + &self.repo_data, 539 + indexes, 540 + user_hash, 541 + &current_meta, 542 + &ms, 543 + ) 544 + .map_err(|e| DbError::Query(e.to_string()))?; 545 + } 546 + 547 + self.write_last_applied_cursor(&mut batch, seq_u64); 548 + batch.commit().map_err(fjall_to_db)?; 549 + 550 + page_high = seq_u64.max(page_high); 551 + count = count.saturating_add(1); 552 + Ok::<_, DbError>(()) 553 + })?; 554 + 555 + let new_total = total.saturating_add(count); 556 + match page_len < RECOVERY_BATCH_SIZE { 557 + true => Ok(new_total), 558 + false => self.recover_mutations_page(indexes, page_high, new_total), 559 + } 560 + } 561 + } 562 + } 563 + 564 + fn recover_page(&self, cursor: u64, total: u64) -> Result<u64, DbError> { 565 + let cursor_sn = SequenceNumber::from_raw( 566 + i64::try_from(cursor) 567 + .map_err(|_| DbError::Query("recovery cursor exceeds i64::MAX".to_owned()))?, 568 + ); 569 + let events = self 570 + .bridge 571 + .get_events_since_seq(cursor_sn, Some(RECOVERY_BATCH_SIZE as i64))?; 572 + 573 + match events.is_empty() { 574 + true => Ok(total), 575 + false => { 576 + let page_len = events.len(); 577 + let mut batch = self.db.batch(); 578 + 579 + let (batch_high, count) = 580 + events.iter().fold((cursor, 0u64), |(high, count), event| { 581 + let seq_u64 = match event.seq.as_u64() { 582 + Some(v) => v, 583 + None => return (high, count), 584 + }; 585 + let user_hash = UserHash::from_did(event.did.as_str()); 586 + 587 + self.stage_did_event(&mut batch, user_hash, seq_u64); 588 + if let Some(rev) = &event.rev { 589 + self.stage_rev_to_seq(&mut batch, user_hash, rev, seq_u64); 590 + } 591 + 592 + (seq_u64.max(high), count.saturating_add(1)) 593 + }); 594 + 595 + self.write_last_applied_cursor(&mut batch, batch_high); 596 + batch.commit().map_err(fjall_to_db)?; 597 + 598 + let new_total = total.saturating_add(count); 599 + match page_len < RECOVERY_BATCH_SIZE { 600 + true => Ok(new_total), 601 + false => self.recover_page(batch_high, new_total), 602 + } 603 + } 604 + } 605 + } 606 + 607 + pub fn append_and_stage_indexes( 608 + &self, 609 + batch: &mut fjall::OwnedWriteBatch, 610 + event: &SequencedEvent, 611 + did: &Did, 612 + rev: Option<&str>, 613 + ) -> Result<SequenceNumber, DbError> { 614 + let seq = self 615 + .bridge 616 + .insert_event(event) 617 + .map_err(|e| DbError::Query(e.to_string()))?; 618 + 619 + let seq_u64 = seq_to_u64(seq)?; 620 + let user_hash = UserHash::from_did(did.as_str()); 621 + self.stage_did_event(batch, user_hash, seq_u64); 622 + if let Some(rev) = rev { 623 + self.stage_rev_to_seq(batch, user_hash, rev, seq_u64); 624 + } 625 + self.write_last_applied_cursor(batch, seq_u64); 626 + 627 + Ok(seq) 628 + } 629 + 630 + fn append_and_index( 631 + &self, 632 + event: &SequencedEvent, 633 + did: &Did, 634 + rev: Option<&str>, 635 + ) -> Result<SequenceNumber, DbError> { 636 + let mut batch = self.db.batch(); 637 + let seq = self.append_and_stage_indexes(&mut batch, event, did, rev)?; 638 + batch.commit().map_err(fjall_to_db)?; 639 + Ok(seq) 640 + } 641 + 642 + fn stage_did_event(&self, batch: &mut fjall::OwnedWriteBatch, user_hash: UserHash, seq: u64) { 643 + let key = did_events_key(user_hash, seq); 644 + batch.insert(&self.repo_data, key.as_slice(), []); 645 + } 646 + 647 + fn stage_rev_to_seq( 648 + &self, 649 + batch: &mut fjall::OwnedWriteBatch, 650 + user_hash: UserHash, 651 + rev: &str, 652 + seq: u64, 653 + ) { 654 + let key = rev_to_seq_key(user_hash, rev); 655 + batch.insert(&self.repo_data, key.as_slice(), seq.to_be_bytes()); 656 + } 657 + 658 + fn scan_did_events(&self, user_hash: UserHash, start_seq: u64) -> Result<Vec<u64>, DbError> { 659 + let range_start = did_events_key(user_hash, start_seq); 660 + let range_end = exclusive_upper_bound(did_events_prefix(user_hash).as_slice()) 661 + .expect("did_events prefix can never be all-0xFF"); 662 + 663 + self.repo_data 664 + .range(range_start.as_slice()..range_end.as_slice()) 665 + .try_fold(Vec::new(), |mut acc, guard| { 666 + let (key, _) = guard.into_inner().map_err(fjall_to_db)?; 667 + let seq = decode_did_events_seq(key.as_ref())?; 668 + acc.push(seq); 669 + Ok(acc) 670 + }) 671 + } 672 + 673 + fn collect_stale_rev_keys( 674 + &self, 675 + user_hash: UserHash, 676 + tombstone_set: &HashSet<u64>, 677 + ) -> Result<Vec<Vec<u8>>, DbError> { 678 + let rev_prefix = rev_to_seq_user_prefix(user_hash); 679 + let rev_upper = exclusive_upper_bound(rev_prefix.as_slice()) 680 + .expect("rev_to_seq prefix can never be all-0xFF"); 681 + 682 + self.repo_data 683 + .range(rev_prefix.as_slice()..rev_upper.as_slice()) 684 + .try_fold(Vec::new(), |mut acc, guard| { 685 + let (key, val) = guard.into_inner().map_err(fjall_to_db)?; 686 + let val_arr: [u8; 8] = val 687 + .as_ref() 688 + .try_into() 689 + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; 690 + let stored_seq = u64::from_be_bytes(val_arr); 691 + if tombstone_set.contains(&stored_seq) { 692 + acc.push(key.as_ref().to_vec()); 693 + } 694 + Ok(acc) 695 + }) 696 + } 697 + 698 + fn is_tombstoned(&self, seq: u64) -> Result<bool, DbError> { 699 + let key = seq_tombstone_key(seq); 700 + match self.repo_data.get(key.as_slice()) { 701 + Ok(Some(_)) => Ok(true), 702 + Ok(None) => Ok(false), 703 + Err(e) => { 704 + warn!(seq, error = %e, "tombstone check failed, propagating error"); 705 + Err(fjall_to_db(e)) 706 + } 707 + } 708 + } 709 + 710 + fn merge_sidecar(&self, mut event: SequencedEvent) -> SequencedEvent { 711 + let seq_u64 = match event.seq.as_u64() { 712 + Some(v) => v, 713 + None => return event, 714 + }; 715 + 716 + let key = seq_meta_key(seq_u64); 717 + match self.repo_data.get(key.as_slice()) { 718 + Ok(Some(sidecar_bytes)) => { 719 + if let Some(sidecar) = SeqMetaValue::deserialize(sidecar_bytes.as_ref()) { 720 + event.blocks_cids = Some(sidecar.blocks_cids); 721 + } 722 + } 723 + Ok(None) => {} 724 + Err(e) => { 725 + warn!(seq = seq_u64, error = %e, "failed to read seq sidecar, returning event without sidecar merge"); 726 + } 727 + } 728 + 729 + event 730 + } 731 + 732 + fn apply_sidecars_and_filter( 733 + &self, 734 + events: Vec<SequencedEvent>, 735 + ) -> Result<Vec<SequencedEvent>, DbError> { 736 + events.into_iter().try_fold(Vec::new(), |mut acc, e| { 737 + let tombstoned = match e.seq.as_u64() { 738 + Some(seq_u64) => self.is_tombstoned(seq_u64)?, 739 + None => false, 740 + }; 741 + if !tombstoned { 742 + acc.push(self.merge_sidecar(e)); 743 + } 744 + Ok(acc) 745 + }) 746 + } 747 + } 748 + 749 + fn fjall_to_db(e: fjall::Error) -> DbError { 750 + DbError::Query(e.to_string()) 751 + } 752 + 753 + fn seq_to_u64(seq: SequenceNumber) -> Result<u64, DbError> { 754 + seq.as_u64() 755 + .ok_or_else(|| DbError::Query("sequence number is negative".to_owned())) 756 + } 757 + 758 + fn decode_did_events_seq(key_bytes: &[u8]) -> Result<u64, DbError> { 759 + let mut reader = KeyReader::new(key_bytes); 760 + let _tag = reader.tag(); 761 + let _user_hash = reader.u64(); 762 + reader 763 + .u64() 764 + .ok_or_else(|| DbError::Query("corrupt did_events key: missing seq field".to_owned())) 765 + } 766 + 767 + #[cfg(test)] 768 + mod tests { 769 + use super::*; 770 + use crate::eventlog::{EventLog, EventLogConfig}; 771 + use crate::io::RealIO; 772 + use sha2::Digest; 773 + use tranquil_db_traits::RepoEventType; 774 + 775 + struct TestHarness { 776 + _metastore_dir: tempfile::TempDir, 777 + _eventlog_dir: tempfile::TempDir, 778 + event_ops: EventOps<RealIO>, 779 + } 780 + 781 + fn setup() -> TestHarness { 782 + let metastore_dir = tempfile::TempDir::new().unwrap(); 783 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 784 + let segments_dir = eventlog_dir.path().join("segments"); 785 + std::fs::create_dir_all(&segments_dir).unwrap(); 786 + 787 + let db = fjall::Database::builder(metastore_dir.path()) 788 + .open() 789 + .unwrap(); 790 + let repo_data = db 791 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 792 + .unwrap(); 793 + 794 + let event_log = EventLog::open( 795 + EventLogConfig { 796 + segments_dir, 797 + ..EventLogConfig::default() 798 + }, 799 + RealIO::new(), 800 + ) 801 + .unwrap(); 802 + 803 + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); 804 + let event_ops = EventOps::new(db, repo_data, bridge); 805 + 806 + TestHarness { 807 + _metastore_dir: metastore_dir, 808 + _eventlog_dir: eventlog_dir, 809 + event_ops, 810 + } 811 + } 812 + 813 + fn test_did() -> Did { 814 + Did::new("did:plc:testuser1234567890abcdef").unwrap() 815 + } 816 + 817 + fn test_cid_link() -> CidLink { 818 + let hash = sha2::Digest::finalize(sha2::Sha256::new()); 819 + let mh = multihash::Multihash::<64>::wrap(0x12, &hash).unwrap(); 820 + let c = cid::Cid::new_v1(0x71, mh); 821 + CidLink::from_cid(&c) 822 + } 823 + 824 + #[test] 825 + fn insert_and_query_commit_event() { 826 + let h = setup(); 827 + let cid = test_cid_link(); 828 + let data = CommitEventData { 829 + did: test_did(), 830 + event_type: RepoEventType::Commit, 831 + commit_cid: Some(cid.clone()), 832 + prev_cid: None, 833 + ops: Some(serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}])), 834 + blobs: None, 835 + blocks_cids: None, 836 + prev_data_cid: None, 837 + rev: Some("3k2abcde".to_owned()), 838 + }; 839 + 840 + let seq = h.event_ops.insert_commit_event(&data).unwrap(); 841 + assert!(seq.as_i64() > 0); 842 + 843 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 844 + assert_eq!(event.did.as_str(), test_did().as_str()); 845 + assert_eq!(event.event_type, RepoEventType::Commit); 846 + assert_eq!(event.commit_cid, Some(cid)); 847 + assert_eq!(event.rev, Some("3k2abcde".to_owned())); 848 + } 849 + 850 + #[test] 851 + fn insert_and_query_identity_event() { 852 + let h = setup(); 853 + let handle = Handle::new("alice.test").unwrap(); 854 + 855 + let seq = h 856 + .event_ops 857 + .insert_identity_event(&test_did(), Some(&handle)) 858 + .unwrap(); 859 + assert!(seq.as_i64() > 0); 860 + 861 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 862 + assert_eq!(event.event_type, RepoEventType::Identity); 863 + assert_eq!( 864 + event.handle.as_ref().map(|h| h.as_str()), 865 + Some("alice.test") 866 + ); 867 + } 868 + 869 + #[test] 870 + fn insert_and_query_account_event() { 871 + let h = setup(); 872 + 873 + let seq = h 874 + .event_ops 875 + .insert_account_event(&test_did(), AccountStatus::Deactivated) 876 + .unwrap(); 877 + assert!(seq.as_i64() > 0); 878 + 879 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 880 + assert_eq!(event.event_type, RepoEventType::Account); 881 + assert_eq!(event.status, Some(AccountStatus::Deactivated)); 882 + } 883 + 884 + #[test] 885 + fn insert_and_query_sync_event() { 886 + let h = setup(); 887 + let cid = test_cid_link(); 888 + 889 + let seq = h 890 + .event_ops 891 + .insert_sync_event(&test_did(), &cid, Some("rev1")) 892 + .unwrap(); 893 + assert!(seq.as_i64() > 0); 894 + 895 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 896 + assert_eq!(event.event_type, RepoEventType::Sync); 897 + assert_eq!(event.commit_cid, Some(cid)); 898 + assert_eq!(event.rev, Some("rev1".to_owned())); 899 + } 900 + 901 + #[test] 902 + fn insert_genesis_commit_event() { 903 + let h = setup(); 904 + let commit_cid = test_cid_link(); 905 + let mst_cid = test_cid_link(); 906 + 907 + let seq = h 908 + .event_ops 909 + .insert_genesis_commit_event(&test_did(), &commit_cid, &mst_cid, "genesis_rev") 910 + .unwrap(); 911 + assert!(seq.as_i64() > 0); 912 + 913 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 914 + assert_eq!(event.event_type, RepoEventType::Commit); 915 + assert_eq!(event.commit_cid, Some(commit_cid)); 916 + assert_eq!(event.prev_data_cid, Some(mst_cid)); 917 + assert_eq!(event.rev, Some("genesis_rev".to_owned())); 918 + } 919 + 920 + #[test] 921 + fn get_events_since_seq_returns_ordered() { 922 + let h = setup(); 923 + let did = test_did(); 924 + 925 + let seq1 = h 926 + .event_ops 927 + .insert_account_event(&did, AccountStatus::Active) 928 + .unwrap(); 929 + let seq2 = h.event_ops.insert_identity_event(&did, None).unwrap(); 930 + 931 + let events = h 932 + .event_ops 933 + .get_events_since_seq(SequenceNumber::ZERO, None) 934 + .unwrap(); 935 + assert_eq!(events.len(), 2); 936 + assert_eq!(events[0].seq, seq1); 937 + assert_eq!(events[1].seq, seq2); 938 + } 939 + 940 + #[test] 941 + fn get_events_since_seq_with_limit() { 942 + let h = setup(); 943 + let did = test_did(); 944 + 945 + h.event_ops 946 + .insert_account_event(&did, AccountStatus::Active) 947 + .unwrap(); 948 + h.event_ops.insert_identity_event(&did, None).unwrap(); 949 + 950 + let events = h 951 + .event_ops 952 + .get_events_since_seq(SequenceNumber::ZERO, Some(1)) 953 + .unwrap(); 954 + assert_eq!(events.len(), 1); 955 + } 956 + 957 + #[test] 958 + fn get_events_in_seq_range() { 959 + let h = setup(); 960 + let did = test_did(); 961 + 962 + let seq1 = h 963 + .event_ops 964 + .insert_account_event(&did, AccountStatus::Active) 965 + .unwrap(); 966 + let seq2 = h.event_ops.insert_identity_event(&did, None).unwrap(); 967 + let _seq3 = h.event_ops.insert_identity_event(&did, None).unwrap(); 968 + 969 + let events = h 970 + .event_ops 971 + .get_events_in_seq_range(SequenceNumber::ZERO, seq2) 972 + .unwrap(); 973 + assert_eq!(events.len(), 1); 974 + assert_eq!(events[0].seq, seq1); 975 + } 976 + 977 + #[test] 978 + fn cursor_pagination() { 979 + let h = setup(); 980 + let did = test_did(); 981 + 982 + let seqs: Vec<SequenceNumber> = (0..5) 983 + .map(|_| h.event_ops.insert_identity_event(&did, None).unwrap()) 984 + .collect(); 985 + 986 + let page1 = h 987 + .event_ops 988 + .get_events_since_cursor(SequenceNumber::ZERO, 2) 989 + .unwrap(); 990 + assert_eq!(page1.len(), 2); 991 + assert_eq!(page1[0].seq, seqs[0]); 992 + assert_eq!(page1[1].seq, seqs[1]); 993 + 994 + let page2 = h 995 + .event_ops 996 + .get_events_since_cursor(page1[1].seq, 2) 997 + .unwrap(); 998 + assert_eq!(page2.len(), 2); 999 + assert_eq!(page2[0].seq, seqs[2]); 1000 + assert_eq!(page2[1].seq, seqs[3]); 1001 + 1002 + let page3 = h 1003 + .event_ops 1004 + .get_events_since_cursor(page2[1].seq, 2) 1005 + .unwrap(); 1006 + assert_eq!(page3.len(), 1); 1007 + assert_eq!(page3[0].seq, seqs[4]); 1008 + } 1009 + 1010 + #[test] 1011 + fn get_max_seq() { 1012 + let h = setup(); 1013 + assert_eq!(h.event_ops.get_max_seq(), SequenceNumber::ZERO); 1014 + 1015 + let seq = h 1016 + .event_ops 1017 + .insert_identity_event(&test_did(), None) 1018 + .unwrap(); 1019 + assert_eq!(h.event_ops.get_max_seq(), seq); 1020 + } 1021 + 1022 + #[test] 1023 + fn update_seq_blocks_cids_merges_on_query() { 1024 + let h = setup(); 1025 + let data = CommitEventData { 1026 + did: test_did(), 1027 + event_type: RepoEventType::Commit, 1028 + commit_cid: Some(test_cid_link()), 1029 + prev_cid: None, 1030 + ops: None, 1031 + blobs: None, 1032 + blocks_cids: None, 1033 + prev_data_cid: None, 1034 + rev: Some("rev1".to_owned()), 1035 + }; 1036 + 1037 + let seq = h.event_ops.insert_commit_event(&data).unwrap(); 1038 + 1039 + let blocks = vec!["bafyblock1".to_owned(), "bafyblock2".to_owned()]; 1040 + h.event_ops.update_seq_blocks_cids(seq, &blocks).unwrap(); 1041 + 1042 + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); 1043 + assert_eq!(event.blocks_cids, Some(blocks)); 1044 + } 1045 + 1046 + #[test] 1047 + fn delete_sequences_except_tombstones_others() { 1048 + let h = setup(); 1049 + let did = test_did(); 1050 + let cid = test_cid_link(); 1051 + 1052 + let seq1 = h 1053 + .event_ops 1054 + .insert_commit_event(&CommitEventData { 1055 + did: did.clone(), 1056 + event_type: RepoEventType::Commit, 1057 + commit_cid: Some(cid.clone()), 1058 + prev_cid: None, 1059 + ops: None, 1060 + blobs: None, 1061 + blocks_cids: None, 1062 + prev_data_cid: None, 1063 + rev: Some("rev_a".to_owned()), 1064 + }) 1065 + .unwrap(); 1066 + 1067 + let seq2 = h 1068 + .event_ops 1069 + .insert_commit_event(&CommitEventData { 1070 + did: did.clone(), 1071 + event_type: RepoEventType::Commit, 1072 + commit_cid: Some(cid.clone()), 1073 + prev_cid: None, 1074 + ops: None, 1075 + blobs: None, 1076 + blocks_cids: None, 1077 + prev_data_cid: None, 1078 + rev: Some("rev_b".to_owned()), 1079 + }) 1080 + .unwrap(); 1081 + 1082 + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); 1083 + 1084 + assert!(h.event_ops.get_event_by_seq(seq1).unwrap().is_none()); 1085 + assert!(h.event_ops.get_event_by_seq(seq2).unwrap().is_some()); 1086 + } 1087 + 1088 + #[test] 1089 + fn tombstoned_events_filtered_from_range_queries() { 1090 + let h = setup(); 1091 + let did = test_did(); 1092 + let cid = test_cid_link(); 1093 + 1094 + let _seq1 = h 1095 + .event_ops 1096 + .insert_commit_event(&CommitEventData { 1097 + did: did.clone(), 1098 + event_type: RepoEventType::Commit, 1099 + commit_cid: Some(cid.clone()), 1100 + prev_cid: None, 1101 + ops: None, 1102 + blobs: None, 1103 + blocks_cids: None, 1104 + prev_data_cid: None, 1105 + rev: Some("rev_x".to_owned()), 1106 + }) 1107 + .unwrap(); 1108 + 1109 + let seq2 = h 1110 + .event_ops 1111 + .insert_commit_event(&CommitEventData { 1112 + did: did.clone(), 1113 + event_type: RepoEventType::Commit, 1114 + commit_cid: Some(cid.clone()), 1115 + prev_cid: None, 1116 + ops: None, 1117 + blobs: None, 1118 + blocks_cids: None, 1119 + prev_data_cid: None, 1120 + rev: Some("rev_y".to_owned()), 1121 + }) 1122 + .unwrap(); 1123 + 1124 + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); 1125 + 1126 + let events = h 1127 + .event_ops 1128 + .get_events_since_seq(SequenceNumber::ZERO, None) 1129 + .unwrap(); 1130 + assert_eq!(events.len(), 1); 1131 + assert_eq!(events[0].seq, seq2); 1132 + } 1133 + 1134 + #[test] 1135 + fn get_events_since_rev() { 1136 + let h = setup(); 1137 + let did = test_did(); 1138 + let cid = test_cid_link(); 1139 + 1140 + h.event_ops 1141 + .insert_commit_event(&CommitEventData { 1142 + did: did.clone(), 1143 + event_type: RepoEventType::Commit, 1144 + commit_cid: Some(cid.clone()), 1145 + prev_cid: None, 1146 + ops: None, 1147 + blobs: None, 1148 + blocks_cids: Some(vec!["block_a".to_owned()]), 1149 + prev_data_cid: None, 1150 + rev: Some("rev_1".to_owned()), 1151 + }) 1152 + .unwrap(); 1153 + 1154 + h.event_ops 1155 + .insert_commit_event(&CommitEventData { 1156 + did: did.clone(), 1157 + event_type: RepoEventType::Commit, 1158 + commit_cid: Some(cid.clone()), 1159 + prev_cid: None, 1160 + ops: None, 1161 + blobs: None, 1162 + blocks_cids: Some(vec!["block_b".to_owned()]), 1163 + prev_data_cid: None, 1164 + rev: Some("rev_2".to_owned()), 1165 + }) 1166 + .unwrap(); 1167 + 1168 + let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap(); 1169 + 1170 + assert_eq!(events.len(), 1); 1171 + assert_eq!(events[0].blocks_cids, Some(vec!["block_b".to_owned()])); 1172 + } 1173 + 1174 + #[test] 1175 + fn get_events_since_rev_unknown_rev_returns_empty() { 1176 + let h = setup(); 1177 + let events = h 1178 + .event_ops 1179 + .get_events_since_rev(&test_did(), "nonexistent_rev") 1180 + .unwrap(); 1181 + assert!(events.is_empty()); 1182 + } 1183 + 1184 + #[test] 1185 + fn metastore_cursor_read_write() { 1186 + let h = setup(); 1187 + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), None); 1188 + 1189 + h.event_ops.write_last_applied_cursor_direct(42).unwrap(); 1190 + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), Some(42)); 1191 + 1192 + h.event_ops.write_last_applied_cursor_direct(100).unwrap(); 1193 + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), Some(100)); 1194 + } 1195 + 1196 + #[test] 1197 + fn inserts_advance_cursor() { 1198 + let h = setup(); 1199 + let did = test_did(); 1200 + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), None); 1201 + 1202 + let seq1 = h.event_ops.insert_identity_event(&did, None).unwrap(); 1203 + assert_eq!( 1204 + h.event_ops.read_last_applied_cursor().unwrap(), 1205 + seq1.as_u64() 1206 + ); 1207 + 1208 + let seq2 = h 1209 + .event_ops 1210 + .insert_account_event(&did, AccountStatus::Active) 1211 + .unwrap(); 1212 + assert_eq!( 1213 + h.event_ops.read_last_applied_cursor().unwrap(), 1214 + seq2.as_u64() 1215 + ); 1216 + assert!(seq2 > seq1); 1217 + } 1218 + 1219 + #[test] 1220 + fn get_event_by_seq_none_for_missing() { 1221 + let h = setup(); 1222 + let result = h 1223 + .event_ops 1224 + .get_event_by_seq(SequenceNumber::from_raw(9999)) 1225 + .unwrap(); 1226 + assert!(result.is_none()); 1227 + } 1228 + 1229 + #[test] 1230 + fn get_event_by_seq_none_for_negative() { 1231 + let h = setup(); 1232 + let result = h 1233 + .event_ops 1234 + .get_event_by_seq(SequenceNumber::from_raw(-1)) 1235 + .unwrap(); 1236 + assert!(result.is_none()); 1237 + } 1238 + 1239 + #[test] 1240 + fn multiple_event_types_interleaved() { 1241 + let h = setup(); 1242 + let did = test_did(); 1243 + let cid = test_cid_link(); 1244 + 1245 + let s1 = h 1246 + .event_ops 1247 + .insert_commit_event(&CommitEventData { 1248 + did: did.clone(), 1249 + event_type: RepoEventType::Commit, 1250 + commit_cid: Some(cid.clone()), 1251 + prev_cid: None, 1252 + ops: None, 1253 + blobs: None, 1254 + blocks_cids: None, 1255 + prev_data_cid: None, 1256 + rev: Some("r1".to_owned()), 1257 + }) 1258 + .unwrap(); 1259 + let s2 = h.event_ops.insert_identity_event(&did, None).unwrap(); 1260 + let s3 = h 1261 + .event_ops 1262 + .insert_account_event(&did, AccountStatus::Active) 1263 + .unwrap(); 1264 + let s4 = h 1265 + .event_ops 1266 + .insert_sync_event(&did, &cid, Some("r2")) 1267 + .unwrap(); 1268 + 1269 + let events = h 1270 + .event_ops 1271 + .get_events_since_seq(SequenceNumber::ZERO, None) 1272 + .unwrap(); 1273 + assert_eq!(events.len(), 4); 1274 + assert_eq!(events[0].event_type, RepoEventType::Commit); 1275 + assert_eq!(events[1].event_type, RepoEventType::Identity); 1276 + assert_eq!(events[2].event_type, RepoEventType::Account); 1277 + assert_eq!(events[3].event_type, RepoEventType::Sync); 1278 + 1279 + assert!(s1 < s2); 1280 + assert!(s2 < s3); 1281 + assert!(s3 < s4); 1282 + } 1283 + 1284 + #[test] 1285 + fn delete_sequences_except_tombstones_all_event_types() { 1286 + let h = setup(); 1287 + let did = test_did(); 1288 + let cid = test_cid_link(); 1289 + 1290 + let _commit_seq = h 1291 + .event_ops 1292 + .insert_commit_event(&CommitEventData { 1293 + did: did.clone(), 1294 + event_type: RepoEventType::Commit, 1295 + commit_cid: Some(cid.clone()), 1296 + prev_cid: None, 1297 + ops: None, 1298 + blobs: None, 1299 + blocks_cids: None, 1300 + prev_data_cid: None, 1301 + rev: Some("rev_keep".to_owned()), 1302 + }) 1303 + .unwrap(); 1304 + 1305 + let identity_seq = h.event_ops.insert_identity_event(&did, None).unwrap(); 1306 + 1307 + let account_seq = h 1308 + .event_ops 1309 + .insert_account_event(&did, AccountStatus::Active) 1310 + .unwrap(); 1311 + 1312 + let keep_seq = h 1313 + .event_ops 1314 + .insert_sync_event(&did, &cid, Some("rev_sync")) 1315 + .unwrap(); 1316 + 1317 + h.event_ops.delete_sequences_except(&did, keep_seq).unwrap(); 1318 + 1319 + assert!( 1320 + h.event_ops 1321 + .get_event_by_seq(identity_seq) 1322 + .unwrap() 1323 + .is_none() 1324 + ); 1325 + assert!(h.event_ops.get_event_by_seq(account_seq).unwrap().is_none()); 1326 + assert!(h.event_ops.get_event_by_seq(keep_seq).unwrap().is_some()); 1327 + } 1328 + 1329 + #[test] 1330 + fn delete_sequences_except_rejects_negative_keep_seq() { 1331 + let h = setup(); 1332 + let result = h 1333 + .event_ops 1334 + .delete_sequences_except(&test_did(), SequenceNumber::from_raw(-1)); 1335 + assert!(result.is_err()); 1336 + } 1337 + 1338 + #[test] 1339 + fn delete_sequences_except_cleans_rev_to_seq_entries() { 1340 + let h = setup(); 1341 + let did = test_did(); 1342 + let cid = test_cid_link(); 1343 + let user_hash = super::UserHash::from_did(did.as_str()); 1344 + 1345 + let _seq1 = h 1346 + .event_ops 1347 + .insert_commit_event(&CommitEventData { 1348 + did: did.clone(), 1349 + event_type: RepoEventType::Commit, 1350 + commit_cid: Some(cid.clone()), 1351 + prev_cid: None, 1352 + ops: None, 1353 + blobs: None, 1354 + blocks_cids: None, 1355 + prev_data_cid: None, 1356 + rev: Some("rev_old".to_owned()), 1357 + }) 1358 + .unwrap(); 1359 + 1360 + let seq2 = h 1361 + .event_ops 1362 + .insert_commit_event(&CommitEventData { 1363 + did: did.clone(), 1364 + event_type: RepoEventType::Commit, 1365 + commit_cid: Some(cid.clone()), 1366 + prev_cid: None, 1367 + ops: None, 1368 + blobs: None, 1369 + blocks_cids: None, 1370 + prev_data_cid: None, 1371 + rev: Some("rev_keep".to_owned()), 1372 + }) 1373 + .unwrap(); 1374 + 1375 + let old_key = super::super::event_keys::rev_to_seq_key(user_hash, "rev_old"); 1376 + assert!( 1377 + h.event_ops 1378 + .repo_data 1379 + .get(old_key.as_slice()) 1380 + .unwrap() 1381 + .is_some() 1382 + ); 1383 + 1384 + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); 1385 + 1386 + assert!( 1387 + h.event_ops 1388 + .repo_data 1389 + .get(old_key.as_slice()) 1390 + .unwrap() 1391 + .is_none() 1392 + ); 1393 + 1394 + let keep_key = super::super::event_keys::rev_to_seq_key(user_hash, "rev_keep"); 1395 + assert!( 1396 + h.event_ops 1397 + .repo_data 1398 + .get(keep_key.as_slice()) 1399 + .unwrap() 1400 + .is_some() 1401 + ); 1402 + } 1403 + 1404 + #[test] 1405 + fn delete_sequences_except_cleans_did_events_entries() { 1406 + let h = setup(); 1407 + let did = test_did(); 1408 + let cid = test_cid_link(); 1409 + let user_hash = super::UserHash::from_did(did.as_str()); 1410 + 1411 + let seq1 = h 1412 + .event_ops 1413 + .insert_commit_event(&CommitEventData { 1414 + did: did.clone(), 1415 + event_type: RepoEventType::Commit, 1416 + commit_cid: Some(cid.clone()), 1417 + prev_cid: None, 1418 + ops: None, 1419 + blobs: None, 1420 + blocks_cids: None, 1421 + prev_data_cid: None, 1422 + rev: Some("rev_a".to_owned()), 1423 + }) 1424 + .unwrap(); 1425 + 1426 + let seq2 = h 1427 + .event_ops 1428 + .insert_commit_event(&CommitEventData { 1429 + did: did.clone(), 1430 + event_type: RepoEventType::Commit, 1431 + commit_cid: Some(cid.clone()), 1432 + prev_cid: None, 1433 + ops: None, 1434 + blobs: None, 1435 + blocks_cids: None, 1436 + prev_data_cid: None, 1437 + rev: Some("rev_b".to_owned()), 1438 + }) 1439 + .unwrap(); 1440 + 1441 + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); 1442 + 1443 + let removed_key = 1444 + super::super::event_keys::did_events_key(user_hash, seq1.as_u64().unwrap()); 1445 + assert!( 1446 + h.event_ops 1447 + .repo_data 1448 + .get(removed_key.as_slice()) 1449 + .unwrap() 1450 + .is_none() 1451 + ); 1452 + 1453 + let kept_key = super::super::event_keys::did_events_key(user_hash, seq2.as_u64().unwrap()); 1454 + assert!( 1455 + h.event_ops 1456 + .repo_data 1457 + .get(kept_key.as_slice()) 1458 + .unwrap() 1459 + .is_some() 1460 + ); 1461 + } 1462 + 1463 + #[test] 1464 + fn delete_sequences_except_cleans_seq_meta_entries() { 1465 + let h = setup(); 1466 + let did = test_did(); 1467 + let cid = test_cid_link(); 1468 + 1469 + let seq1 = h 1470 + .event_ops 1471 + .insert_commit_event(&CommitEventData { 1472 + did: did.clone(), 1473 + event_type: RepoEventType::Commit, 1474 + commit_cid: Some(cid.clone()), 1475 + prev_cid: None, 1476 + ops: None, 1477 + blobs: None, 1478 + blocks_cids: None, 1479 + prev_data_cid: None, 1480 + rev: Some("rev_a".to_owned()), 1481 + }) 1482 + .unwrap(); 1483 + 1484 + let seq2 = h 1485 + .event_ops 1486 + .insert_commit_event(&CommitEventData { 1487 + did: did.clone(), 1488 + event_type: RepoEventType::Commit, 1489 + commit_cid: Some(cid.clone()), 1490 + prev_cid: None, 1491 + ops: None, 1492 + blobs: None, 1493 + blocks_cids: None, 1494 + prev_data_cid: None, 1495 + rev: Some("rev_b".to_owned()), 1496 + }) 1497 + .unwrap(); 1498 + 1499 + h.event_ops 1500 + .update_seq_blocks_cids(seq1, &["block1".to_owned()]) 1501 + .unwrap(); 1502 + h.event_ops 1503 + .update_seq_blocks_cids(seq2, &["block2".to_owned()]) 1504 + .unwrap(); 1505 + 1506 + let stale_key = super::super::event_keys::seq_meta_key(seq1.as_u64().unwrap()); 1507 + assert!( 1508 + h.event_ops 1509 + .repo_data 1510 + .get(stale_key.as_slice()) 1511 + .unwrap() 1512 + .is_some() 1513 + ); 1514 + 1515 + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); 1516 + 1517 + assert!( 1518 + h.event_ops 1519 + .repo_data 1520 + .get(stale_key.as_slice()) 1521 + .unwrap() 1522 + .is_none() 1523 + ); 1524 + 1525 + let kept_key = super::super::event_keys::seq_meta_key(seq2.as_u64().unwrap()); 1526 + assert!( 1527 + h.event_ops 1528 + .repo_data 1529 + .get(kept_key.as_slice()) 1530 + .unwrap() 1531 + .is_some() 1532 + ); 1533 + } 1534 + 1535 + #[test] 1536 + fn get_events_since_rev_excludes_events_without_rev() { 1537 + let h = setup(); 1538 + let did = test_did(); 1539 + let cid = test_cid_link(); 1540 + 1541 + h.event_ops 1542 + .insert_commit_event(&CommitEventData { 1543 + did: did.clone(), 1544 + event_type: RepoEventType::Commit, 1545 + commit_cid: Some(cid.clone()), 1546 + prev_cid: None, 1547 + ops: None, 1548 + blobs: None, 1549 + blocks_cids: Some(vec!["block_1".to_owned()]), 1550 + prev_data_cid: None, 1551 + rev: Some("rev_1".to_owned()), 1552 + }) 1553 + .unwrap(); 1554 + 1555 + h.event_ops.insert_identity_event(&did, None).unwrap(); 1556 + 1557 + h.event_ops 1558 + .insert_account_event(&did, AccountStatus::Active) 1559 + .unwrap(); 1560 + 1561 + h.event_ops 1562 + .insert_commit_event(&CommitEventData { 1563 + did: did.clone(), 1564 + event_type: RepoEventType::Commit, 1565 + commit_cid: Some(cid.clone()), 1566 + prev_cid: None, 1567 + ops: None, 1568 + blobs: None, 1569 + blocks_cids: Some(vec!["block_2".to_owned()]), 1570 + prev_data_cid: None, 1571 + rev: Some("rev_2".to_owned()), 1572 + }) 1573 + .unwrap(); 1574 + 1575 + let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap(); 1576 + 1577 + assert_eq!(events.len(), 1); 1578 + assert_eq!(events[0].blocks_cids, Some(vec!["block_2".to_owned()])); 1579 + } 1580 + 1581 + #[test] 1582 + fn sync_event_with_rev_appears_in_get_events_since_rev() { 1583 + let h = setup(); 1584 + let did = test_did(); 1585 + let cid = test_cid_link(); 1586 + 1587 + h.event_ops 1588 + .insert_commit_event(&CommitEventData { 1589 + did: did.clone(), 1590 + event_type: RepoEventType::Commit, 1591 + commit_cid: Some(cid.clone()), 1592 + prev_cid: None, 1593 + ops: None, 1594 + blobs: None, 1595 + blocks_cids: None, 1596 + prev_data_cid: None, 1597 + rev: Some("rev_a".to_owned()), 1598 + }) 1599 + .unwrap(); 1600 + 1601 + h.event_ops 1602 + .insert_sync_event(&did, &cid, Some("rev_b")) 1603 + .unwrap(); 1604 + 1605 + let events = h.event_ops.get_events_since_rev(&did, "rev_a").unwrap(); 1606 + 1607 + assert_eq!(events.len(), 1); 1608 + assert_eq!(events[0].commit_cid, Some(cid)); 1609 + } 1610 + 1611 + #[test] 1612 + fn recover_sidecar_indexes_no_gap() { 1613 + let h = setup(); 1614 + let did = test_did(); 1615 + 1616 + let seq = h 1617 + .event_ops 1618 + .insert_commit_event(&CommitEventData { 1619 + did: did.clone(), 1620 + event_type: RepoEventType::Commit, 1621 + commit_cid: Some(test_cid_link()), 1622 + prev_cid: None, 1623 + ops: None, 1624 + blobs: None, 1625 + blocks_cids: None, 1626 + prev_data_cid: None, 1627 + rev: Some("rev_1".to_owned()), 1628 + }) 1629 + .unwrap(); 1630 + 1631 + h.event_ops 1632 + .write_last_applied_cursor_direct(seq.as_u64().unwrap()) 1633 + .unwrap(); 1634 + 1635 + let recovered = h.event_ops.recover_sidecar_indexes().unwrap(); 1636 + assert_eq!(recovered, 0); 1637 + } 1638 + 1639 + #[test] 1640 + fn recover_sidecar_indexes_rebuilds_after_gap() { 1641 + let h = setup(); 1642 + let did = test_did(); 1643 + let cid = test_cid_link(); 1644 + 1645 + let seq1 = h 1646 + .event_ops 1647 + .insert_commit_event(&CommitEventData { 1648 + did: did.clone(), 1649 + event_type: RepoEventType::Commit, 1650 + commit_cid: Some(cid.clone()), 1651 + prev_cid: None, 1652 + ops: None, 1653 + blobs: None, 1654 + blocks_cids: None, 1655 + prev_data_cid: None, 1656 + rev: Some("rev_1".to_owned()), 1657 + }) 1658 + .unwrap(); 1659 + 1660 + h.event_ops 1661 + .write_last_applied_cursor_direct(seq1.as_u64().unwrap()) 1662 + .unwrap(); 1663 + 1664 + let crash_event = SequencedEvent { 1665 + seq: SequenceNumber::ZERO, 1666 + did: did.clone(), 1667 + created_at: Utc::now(), 1668 + event_type: RepoEventType::Commit, 1669 + commit_cid: Some(cid.clone()), 1670 + prev_cid: None, 1671 + prev_data_cid: None, 1672 + ops: None, 1673 + blobs: None, 1674 + blocks_cids: None, 1675 + handle: None, 1676 + active: None, 1677 + status: None, 1678 + rev: Some("rev_2".to_owned()), 1679 + }; 1680 + h.event_ops.bridge.insert_event(&crash_event).unwrap(); 1681 + 1682 + let identity_event = SequencedEvent { 1683 + seq: SequenceNumber::ZERO, 1684 + did: did.clone(), 1685 + created_at: Utc::now(), 1686 + event_type: RepoEventType::Identity, 1687 + commit_cid: None, 1688 + prev_cid: None, 1689 + prev_data_cid: None, 1690 + ops: None, 1691 + blobs: None, 1692 + blocks_cids: None, 1693 + handle: None, 1694 + active: None, 1695 + status: None, 1696 + rev: None, 1697 + }; 1698 + h.event_ops.bridge.insert_event(&identity_event).unwrap(); 1699 + 1700 + let crash_event_3 = SequencedEvent { 1701 + seq: SequenceNumber::ZERO, 1702 + did: did.clone(), 1703 + created_at: Utc::now(), 1704 + event_type: RepoEventType::Commit, 1705 + commit_cid: Some(cid.clone()), 1706 + prev_cid: None, 1707 + prev_data_cid: None, 1708 + ops: None, 1709 + blobs: None, 1710 + blocks_cids: None, 1711 + handle: None, 1712 + active: None, 1713 + status: None, 1714 + rev: Some("rev_3".to_owned()), 1715 + }; 1716 + h.event_ops.bridge.insert_event(&crash_event_3).unwrap(); 1717 + 1718 + assert!( 1719 + h.event_ops 1720 + .get_events_since_rev(&did, "rev_2") 1721 + .unwrap() 1722 + .is_empty() 1723 + ); 1724 + 1725 + let recovered = h.event_ops.recover_sidecar_indexes().unwrap(); 1726 + assert_eq!(recovered, 3); 1727 + 1728 + let events = h.event_ops.get_events_since_rev(&did, "rev_2").unwrap(); 1729 + assert_eq!(events.len(), 1); 1730 + 1731 + let cursor = h.event_ops.read_last_applied_cursor().unwrap(); 1732 + assert!(cursor.is_some()); 1733 + } 1734 + }
+1729
crates/tranquil-store/src/metastore/handler.rs
··· 1 + use std::sync::Arc; 2 + use std::sync::atomic::{AtomicUsize, Ordering}; 3 + use std::thread::JoinHandle; 4 + 5 + use chrono::{DateTime, Utc}; 6 + use tokio::sync::oneshot; 7 + use tranquil_db_traits::{ 8 + AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, Backlink, 9 + BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, ImportBlock, ImportRecord, 10 + ImportRepoError, SequenceNumber, SequencedEvent, UserNeedingRecordBlobsBackfill, 11 + UserWithoutBlocks, 12 + }; 13 + use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey}; 14 + use uuid::Uuid; 15 + 16 + use super::MetastoreError; 17 + use super::commit_ops::CommitOps; 18 + use super::event_ops::EventOps; 19 + use super::keys::UserHash; 20 + use super::record_ops::ListRecordsQuery; 21 + use super::user_hash::UserHashMap; 22 + use crate::blockstore::TranquilBlockStore; 23 + use crate::eventlog::EventLogBridge; 24 + use crate::io::StorageIO; 25 + use crate::metastore::Metastore; 26 + 27 + type Tx<T> = oneshot::Sender<Result<T, DbError>>; 28 + 29 + fn metastore_to_db(e: MetastoreError) -> DbError { 30 + match e { 31 + MetastoreError::Fjall(e) => DbError::Query(e.to_string()), 32 + MetastoreError::Lsm(e) => DbError::Query(e.to_string()), 33 + MetastoreError::VersionMismatch { expected, found } => DbError::Query(format!( 34 + "format version mismatch: expected {expected}, found {found}" 35 + )), 36 + MetastoreError::CorruptData(msg) => DbError::CorruptData(msg), 37 + MetastoreError::InvalidInput(msg) => DbError::Query(msg.to_string()), 38 + MetastoreError::UserHashCollision { 39 + hash, 40 + existing_uuid, 41 + new_uuid, 42 + } => DbError::Constraint(format!( 43 + "user hash collision: {hash} maps to both {existing_uuid} and {new_uuid}" 44 + )), 45 + } 46 + } 47 + 48 + enum Routing { 49 + Sharded(u64), 50 + Global, 51 + } 52 + 53 + fn uuid_to_routing(user_hashes: &UserHashMap, user_id: &Uuid) -> Routing { 54 + match user_hashes.get(user_id) { 55 + Some(h) => Routing::Sharded(h.raw()), 56 + None => Routing::Sharded(user_id.as_u128() as u64), 57 + } 58 + } 59 + 60 + fn did_to_routing(did: &str) -> Routing { 61 + Routing::Sharded(UserHash::from_did(did).raw()) 62 + } 63 + 64 + fn cid_to_routing(cid: &CidLink) -> Routing { 65 + use siphasher::sip::SipHasher24; 66 + use std::hash::{Hash, Hasher}; 67 + let mut hasher = SipHasher24::new(); 68 + cid.as_str().hash(&mut hasher); 69 + Routing::Sharded(hasher.finish()) 70 + } 71 + 72 + pub enum MetastoreRequest { 73 + Repo(RepoRequest), 74 + Record(RecordRequest), 75 + UserBlock(UserBlockRequest), 76 + Event(EventRequest), 77 + Commit(Box<CommitRequest>), 78 + Backlink(BacklinkRequest), 79 + Blob(BlobRequest), 80 + } 81 + 82 + impl MetastoreRequest { 83 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 84 + match self { 85 + Self::Repo(r) => r.routing(user_hashes), 86 + Self::Record(r) => r.routing(user_hashes), 87 + Self::UserBlock(r) => r.routing(user_hashes), 88 + Self::Event(r) => r.routing(), 89 + Self::Commit(r) => r.routing(user_hashes), 90 + Self::Backlink(r) => r.routing(user_hashes), 91 + Self::Blob(r) => r.routing(user_hashes), 92 + } 93 + } 94 + } 95 + 96 + pub enum RepoRequest { 97 + CreateRepoFull { 98 + user_id: Uuid, 99 + did: Did, 100 + handle: Handle, 101 + repo_root_cid: CidLink, 102 + repo_rev: String, 103 + tx: Tx<()>, 104 + }, 105 + UpdateRepoRoot { 106 + user_id: Uuid, 107 + repo_root_cid: CidLink, 108 + repo_rev: String, 109 + tx: Tx<()>, 110 + }, 111 + UpdateRepoRev { 112 + user_id: Uuid, 113 + repo_rev: String, 114 + tx: Tx<()>, 115 + }, 116 + DeleteRepo { 117 + user_id: Uuid, 118 + tx: Tx<()>, 119 + }, 120 + GetRepoRootForUpdate { 121 + user_id: Uuid, 122 + tx: Tx<Option<CidLink>>, 123 + }, 124 + GetRepo { 125 + user_id: Uuid, 126 + tx: Tx<Option<tranquil_db_traits::RepoInfo>>, 127 + }, 128 + GetRepoRootByDid { 129 + did: Did, 130 + tx: Tx<Option<CidLink>>, 131 + }, 132 + CountRepos { 133 + tx: Tx<i64>, 134 + }, 135 + GetReposWithoutRev { 136 + tx: Tx<Vec<tranquil_db_traits::RepoWithoutRev>>, 137 + }, 138 + GetRepoRootCidByUserId { 139 + user_id: Uuid, 140 + tx: Tx<Option<CidLink>>, 141 + }, 142 + GetAccountWithRepo { 143 + did: Did, 144 + tx: Tx<Option<tranquil_db_traits::RepoAccountInfo>>, 145 + }, 146 + ListReposPaginated { 147 + cursor_user_hash: Option<u64>, 148 + limit: usize, 149 + tx: Tx<Vec<tranquil_db_traits::RepoListItem>>, 150 + }, 151 + UpdateRepoStatus { 152 + did: Did, 153 + takedown: Option<bool>, 154 + takedown_ref: Option<String>, 155 + deactivated: Option<bool>, 156 + tx: Tx<()>, 157 + }, 158 + } 159 + 160 + impl RepoRequest { 161 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 162 + match self { 163 + Self::CreateRepoFull { did, .. } => did_to_routing(did.as_str()), 164 + Self::UpdateRepoRoot { user_id, .. } 165 + | Self::UpdateRepoRev { user_id, .. } 166 + | Self::DeleteRepo { user_id, .. } 167 + | Self::GetRepoRootForUpdate { user_id, .. } 168 + | Self::GetRepo { user_id, .. } 169 + | Self::GetRepoRootCidByUserId { user_id, .. } => uuid_to_routing(user_hashes, user_id), 170 + Self::GetRepoRootByDid { did, .. } 171 + | Self::GetAccountWithRepo { did, .. } 172 + | Self::UpdateRepoStatus { did, .. } => did_to_routing(did.as_str()), 173 + Self::CountRepos { .. } 174 + | Self::GetReposWithoutRev { .. } 175 + | Self::ListReposPaginated { .. } => Routing::Global, 176 + } 177 + } 178 + } 179 + 180 + pub enum RecordRequest { 181 + UpsertRecords { 182 + repo_id: Uuid, 183 + collections: Vec<Nsid>, 184 + rkeys: Vec<Rkey>, 185 + record_cids: Vec<CidLink>, 186 + repo_rev: String, 187 + tx: Tx<()>, 188 + }, 189 + DeleteRecords { 190 + repo_id: Uuid, 191 + collections: Vec<Nsid>, 192 + rkeys: Vec<Rkey>, 193 + tx: Tx<()>, 194 + }, 195 + DeleteAllRecords { 196 + repo_id: Uuid, 197 + tx: Tx<()>, 198 + }, 199 + GetRecordCid { 200 + repo_id: Uuid, 201 + collection: Nsid, 202 + rkey: Rkey, 203 + tx: Tx<Option<CidLink>>, 204 + }, 205 + ListRecords { 206 + repo_id: Uuid, 207 + collection: Nsid, 208 + cursor: Option<Rkey>, 209 + limit: i64, 210 + reverse: bool, 211 + rkey_start: Option<Rkey>, 212 + rkey_end: Option<Rkey>, 213 + tx: Tx<Vec<tranquil_db_traits::RecordInfo>>, 214 + }, 215 + GetAllRecords { 216 + repo_id: Uuid, 217 + tx: Tx<Vec<tranquil_db_traits::FullRecordInfo>>, 218 + }, 219 + ListCollections { 220 + repo_id: Uuid, 221 + tx: Tx<Vec<Nsid>>, 222 + }, 223 + CountRecords { 224 + repo_id: Uuid, 225 + tx: Tx<i64>, 226 + }, 227 + CountAllRecords { 228 + tx: Tx<i64>, 229 + }, 230 + GetRecordByCid { 231 + cid: CidLink, 232 + tx: Tx<Option<tranquil_db_traits::RecordWithTakedown>>, 233 + }, 234 + SetRecordTakedown { 235 + cid: CidLink, 236 + takedown_ref: Option<String>, 237 + scope_user: Option<Uuid>, 238 + tx: Tx<()>, 239 + }, 240 + } 241 + 242 + impl RecordRequest { 243 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 244 + match self { 245 + Self::UpsertRecords { repo_id, .. } 246 + | Self::DeleteRecords { repo_id, .. } 247 + | Self::DeleteAllRecords { repo_id, .. } 248 + | Self::GetRecordCid { repo_id, .. } 249 + | Self::ListRecords { repo_id, .. } 250 + | Self::GetAllRecords { repo_id, .. } 251 + | Self::ListCollections { repo_id, .. } 252 + | Self::CountRecords { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), 253 + Self::CountAllRecords { .. } | Self::GetRecordByCid { .. } => Routing::Global, 254 + Self::SetRecordTakedown { 255 + scope_user: Some(user_id), 256 + .. 257 + } => uuid_to_routing(user_hashes, user_id), 258 + Self::SetRecordTakedown { .. } => Routing::Global, 259 + } 260 + } 261 + } 262 + 263 + pub enum UserBlockRequest { 264 + InsertUserBlocks { 265 + user_id: Uuid, 266 + block_cids: Vec<Vec<u8>>, 267 + repo_rev: String, 268 + tx: Tx<()>, 269 + }, 270 + DeleteUserBlocks { 271 + user_id: Uuid, 272 + block_cids: Vec<Vec<u8>>, 273 + tx: Tx<()>, 274 + }, 275 + GetUserBlockCidsSinceRev { 276 + user_id: Uuid, 277 + since_rev: String, 278 + tx: Tx<Vec<Vec<u8>>>, 279 + }, 280 + CountUserBlocks { 281 + user_id: Uuid, 282 + tx: Tx<i64>, 283 + }, 284 + FindUnreferencedBlocks { 285 + candidate_cids: Vec<Vec<u8>>, 286 + tx: Tx<Vec<Vec<u8>>>, 287 + }, 288 + } 289 + 290 + impl UserBlockRequest { 291 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 292 + match self { 293 + Self::InsertUserBlocks { user_id, .. } 294 + | Self::DeleteUserBlocks { user_id, .. } 295 + | Self::GetUserBlockCidsSinceRev { user_id, .. } 296 + | Self::CountUserBlocks { user_id, .. } => uuid_to_routing(user_hashes, user_id), 297 + Self::FindUnreferencedBlocks { .. } => Routing::Global, 298 + } 299 + } 300 + } 301 + 302 + pub enum EventRequest { 303 + InsertCommitEvent { 304 + data: CommitEventData, 305 + tx: Tx<SequenceNumber>, 306 + }, 307 + InsertIdentityEvent { 308 + did: Did, 309 + handle: Option<Handle>, 310 + tx: Tx<SequenceNumber>, 311 + }, 312 + InsertAccountEvent { 313 + did: Did, 314 + status: AccountStatus, 315 + tx: Tx<SequenceNumber>, 316 + }, 317 + InsertSyncEvent { 318 + did: Did, 319 + commit_cid: CidLink, 320 + rev: Option<String>, 321 + tx: Tx<SequenceNumber>, 322 + }, 323 + InsertGenesisCommitEvent { 324 + did: Did, 325 + commit_cid: CidLink, 326 + mst_root_cid: CidLink, 327 + rev: String, 328 + tx: Tx<SequenceNumber>, 329 + }, 330 + UpdateSeqBlocksCids { 331 + seq: SequenceNumber, 332 + blocks_cids: Vec<String>, 333 + tx: Tx<()>, 334 + }, 335 + DeleteSequencesExcept { 336 + did: Did, 337 + keep_seq: SequenceNumber, 338 + tx: Tx<()>, 339 + }, 340 + GetMaxSeq { 341 + tx: Tx<SequenceNumber>, 342 + }, 343 + GetMinSeqSince { 344 + since: DateTime<Utc>, 345 + tx: Tx<Option<SequenceNumber>>, 346 + }, 347 + GetEventsSinceSeq { 348 + since_seq: SequenceNumber, 349 + limit: Option<i64>, 350 + tx: Tx<Vec<SequencedEvent>>, 351 + }, 352 + GetEventsInSeqRange { 353 + start_seq: SequenceNumber, 354 + end_seq: SequenceNumber, 355 + tx: Tx<Vec<SequencedEvent>>, 356 + }, 357 + GetEventBySeq { 358 + seq: SequenceNumber, 359 + tx: Tx<Option<SequencedEvent>>, 360 + }, 361 + GetEventsSinceCursor { 362 + cursor: SequenceNumber, 363 + limit: i64, 364 + tx: Tx<Vec<SequencedEvent>>, 365 + }, 366 + GetEventsSinceRev { 367 + did: Did, 368 + since_rev: String, 369 + tx: Tx<Vec<EventBlocksCids>>, 370 + }, 371 + NotifyUpdate { 372 + seq: SequenceNumber, 373 + tx: Tx<()>, 374 + }, 375 + } 376 + 377 + impl EventRequest { 378 + fn routing(&self) -> Routing { 379 + match self { 380 + Self::InsertCommitEvent { data, .. } => { 381 + Routing::Sharded(UserHash::from_did(data.did.as_str()).raw()) 382 + } 383 + Self::InsertIdentityEvent { did, .. } 384 + | Self::InsertAccountEvent { did, .. } 385 + | Self::InsertSyncEvent { did, .. } 386 + | Self::InsertGenesisCommitEvent { did, .. } 387 + | Self::DeleteSequencesExcept { did, .. } 388 + | Self::GetEventsSinceRev { did, .. } => { 389 + Routing::Sharded(UserHash::from_did(did.as_str()).raw()) 390 + } 391 + Self::UpdateSeqBlocksCids { .. } 392 + | Self::GetMaxSeq { .. } 393 + | Self::GetMinSeqSince { .. } 394 + | Self::GetEventsSinceSeq { .. } 395 + | Self::GetEventsInSeqRange { .. } 396 + | Self::GetEventBySeq { .. } 397 + | Self::GetEventsSinceCursor { .. } 398 + | Self::NotifyUpdate { .. } => Routing::Global, 399 + } 400 + } 401 + } 402 + 403 + pub enum CommitRequest { 404 + ApplyCommit { 405 + input: Box<ApplyCommitInput>, 406 + tx: oneshot::Sender<Result<ApplyCommitResult, ApplyCommitError>>, 407 + }, 408 + ImportRepoData { 409 + user_id: Uuid, 410 + blocks: Vec<ImportBlock>, 411 + records: Vec<ImportRecord>, 412 + expected_root_cid: Option<CidLink>, 413 + tx: oneshot::Sender<Result<(), ImportRepoError>>, 414 + }, 415 + GetBrokenGenesisCommits { 416 + tx: Tx<Vec<BrokenGenesisCommit>>, 417 + }, 418 + GetUsersWithoutBlocks { 419 + tx: Tx<Vec<UserWithoutBlocks>>, 420 + }, 421 + GetUsersNeedingRecordBlobsBackfill { 422 + limit: i64, 423 + tx: Tx<Vec<UserNeedingRecordBlobsBackfill>>, 424 + }, 425 + InsertRecordBlobs { 426 + repo_id: Uuid, 427 + record_uris: Vec<AtUri>, 428 + blob_cids: Vec<CidLink>, 429 + tx: Tx<()>, 430 + }, 431 + } 432 + 433 + impl CommitRequest { 434 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 435 + match self { 436 + Self::ApplyCommit { input, .. } => did_to_routing(input.did.as_str()), 437 + Self::ImportRepoData { user_id, .. } 438 + | Self::InsertRecordBlobs { 439 + repo_id: user_id, .. 440 + } => uuid_to_routing(user_hashes, user_id), 441 + Self::GetBrokenGenesisCommits { .. } 442 + | Self::GetUsersWithoutBlocks { .. } 443 + | Self::GetUsersNeedingRecordBlobsBackfill { .. } => Routing::Global, 444 + } 445 + } 446 + } 447 + 448 + pub enum BacklinkRequest { 449 + GetBacklinkConflicts { 450 + repo_id: Uuid, 451 + collection: Nsid, 452 + backlinks: Vec<Backlink>, 453 + tx: Tx<Vec<AtUri>>, 454 + }, 455 + AddBacklinks { 456 + repo_id: Uuid, 457 + backlinks: Vec<Backlink>, 458 + tx: Tx<()>, 459 + }, 460 + RemoveBacklinksByUri { 461 + uri: AtUri, 462 + tx: Tx<()>, 463 + }, 464 + RemoveBacklinksByRepo { 465 + repo_id: Uuid, 466 + tx: Tx<()>, 467 + }, 468 + } 469 + 470 + impl BacklinkRequest { 471 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 472 + match self { 473 + Self::GetBacklinkConflicts { repo_id, .. } 474 + | Self::AddBacklinks { repo_id, .. } 475 + | Self::RemoveBacklinksByRepo { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), 476 + Self::RemoveBacklinksByUri { uri, .. } => match uri.did() { 477 + Some(did) => did_to_routing(did), 478 + None => Routing::Global, 479 + }, 480 + } 481 + } 482 + } 483 + 484 + pub enum BlobRequest { 485 + InsertBlob { 486 + cid: CidLink, 487 + mime_type: String, 488 + size_bytes: i64, 489 + created_by_user: Uuid, 490 + storage_key: String, 491 + tx: Tx<Option<CidLink>>, 492 + }, 493 + GetBlobMetadata { 494 + cid: CidLink, 495 + tx: Tx<Option<tranquil_db_traits::BlobMetadata>>, 496 + }, 497 + GetBlobWithTakedown { 498 + cid: CidLink, 499 + tx: Tx<Option<tranquil_db_traits::BlobWithTakedown>>, 500 + }, 501 + GetBlobStorageKey { 502 + cid: CidLink, 503 + tx: Tx<Option<String>>, 504 + }, 505 + ListBlobsByUser { 506 + user_id: Uuid, 507 + cursor: Option<String>, 508 + limit: i64, 509 + tx: Tx<Vec<CidLink>>, 510 + }, 511 + ListBlobsSinceRev { 512 + did: Did, 513 + since: String, 514 + tx: Tx<Vec<CidLink>>, 515 + }, 516 + CountBlobsByUser { 517 + user_id: Uuid, 518 + tx: Tx<i64>, 519 + }, 520 + SumBlobStorage { 521 + tx: Tx<i64>, 522 + }, 523 + UpdateBlobTakedown { 524 + cid: CidLink, 525 + takedown_ref: Option<String>, 526 + tx: Tx<bool>, 527 + }, 528 + DeleteBlobByCid { 529 + cid: CidLink, 530 + tx: Tx<bool>, 531 + }, 532 + DeleteBlobsByUser { 533 + user_id: Uuid, 534 + tx: Tx<u64>, 535 + }, 536 + GetBlobStorageKeysByUser { 537 + user_id: Uuid, 538 + tx: Tx<Vec<String>>, 539 + }, 540 + ListMissingBlobs { 541 + repo_id: Uuid, 542 + cursor: Option<String>, 543 + limit: i64, 544 + tx: Tx<Vec<tranquil_db_traits::MissingBlobInfo>>, 545 + }, 546 + CountDistinctRecordBlobs { 547 + repo_id: Uuid, 548 + tx: Tx<i64>, 549 + }, 550 + GetBlobsForExport { 551 + repo_id: Uuid, 552 + tx: Tx<Vec<tranquil_db_traits::BlobForExport>>, 553 + }, 554 + } 555 + 556 + impl BlobRequest { 557 + fn routing(&self, user_hashes: &UserHashMap) -> Routing { 558 + match self { 559 + Self::InsertBlob { cid, .. } 560 + | Self::UpdateBlobTakedown { cid, .. } 561 + | Self::DeleteBlobByCid { cid, .. } => cid_to_routing(cid), 562 + 563 + Self::DeleteBlobsByUser { user_id, .. } => uuid_to_routing(user_hashes, user_id), 564 + 565 + Self::GetBlobMetadata { .. } 566 + | Self::GetBlobWithTakedown { .. } 567 + | Self::GetBlobStorageKey { .. } 568 + | Self::SumBlobStorage { .. } => Routing::Global, 569 + 570 + Self::ListBlobsByUser { user_id, .. } 571 + | Self::CountBlobsByUser { user_id, .. } 572 + | Self::GetBlobStorageKeysByUser { user_id, .. } => { 573 + uuid_to_routing(user_hashes, user_id) 574 + } 575 + Self::ListMissingBlobs { repo_id, .. } 576 + | Self::CountDistinctRecordBlobs { repo_id, .. } 577 + | Self::GetBlobsForExport { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), 578 + Self::ListBlobsSinceRev { did, .. } => did_to_routing(did.as_str()), 579 + } 580 + } 581 + } 582 + 583 + fn convert_repo_info(r: super::repo_ops::RepoInfo) -> tranquil_db_traits::RepoInfo { 584 + tranquil_db_traits::RepoInfo { 585 + user_id: r.user_id, 586 + repo_root_cid: r.repo_root_cid, 587 + repo_rev: r.repo_rev, 588 + } 589 + } 590 + 591 + fn convert_repo_account( 592 + r: super::repo_ops::RepoAccountEntry, 593 + ) -> tranquil_db_traits::RepoAccountInfo { 594 + tranquil_db_traits::RepoAccountInfo { 595 + user_id: r.user_id, 596 + did: r.did, 597 + deactivated_at: r.deactivated_at, 598 + takedown_ref: r.takedown_ref, 599 + repo_root_cid: r.repo_root_cid, 600 + } 601 + } 602 + 603 + fn convert_repo_list_entry( 604 + r: super::repo_ops::RepoListEntry, 605 + ) -> Result<tranquil_db_traits::RepoListItem, DbError> { 606 + let did = r 607 + .did 608 + .ok_or(DbError::CorruptData("repo_meta missing DID field"))?; 609 + Ok(tranquil_db_traits::RepoListItem { 610 + did: Did::from(did), 611 + deactivated_at: r.deactivated_at, 612 + takedown_ref: r.takedown_ref, 613 + repo_root_cid: r.repo_root_cid, 614 + repo_rev: r.repo_rev, 615 + }) 616 + } 617 + 618 + fn convert_record_info(r: super::record_ops::RecordInfo) -> tranquil_db_traits::RecordInfo { 619 + tranquil_db_traits::RecordInfo { 620 + rkey: r.rkey, 621 + record_cid: r.record_cid, 622 + } 623 + } 624 + 625 + fn convert_full_record_info( 626 + r: super::record_ops::FullRecordInfo, 627 + ) -> tranquil_db_traits::FullRecordInfo { 628 + tranquil_db_traits::FullRecordInfo { 629 + collection: r.collection, 630 + rkey: r.rkey, 631 + record_cid: r.record_cid, 632 + } 633 + } 634 + 635 + fn convert_record_with_takedown( 636 + r: super::record_ops::RecordWithTakedown, 637 + ) -> tranquil_db_traits::RecordWithTakedown { 638 + tranquil_db_traits::RecordWithTakedown { 639 + id: r.id, 640 + takedown_ref: r.takedown_ref, 641 + } 642 + } 643 + 644 + fn convert_without_rev( 645 + r: super::repo_ops::RepoWithoutRevEntry, 646 + ) -> tranquil_db_traits::RepoWithoutRev { 647 + tranquil_db_traits::RepoWithoutRev { 648 + user_id: r.user_id, 649 + repo_root_cid: r.repo_root_cid, 650 + } 651 + } 652 + 653 + struct HandlerState<S: StorageIO> { 654 + metastore: Metastore, 655 + event_ops: EventOps<S>, 656 + commit_ops: CommitOps<S>, 657 + } 658 + 659 + fn dispatch_repo<S: StorageIO>(state: &HandlerState<S>, req: RepoRequest) { 660 + match req { 661 + RepoRequest::CreateRepoFull { 662 + user_id, 663 + did, 664 + handle, 665 + repo_root_cid, 666 + repo_rev, 667 + tx, 668 + } => { 669 + let result = state 670 + .metastore 671 + .repo_ops() 672 + .create_repo( 673 + state.metastore.database(), 674 + user_id, 675 + &did, 676 + &handle, 677 + &repo_root_cid, 678 + &repo_rev, 679 + ) 680 + .map_err(metastore_to_db); 681 + let _ = tx.send(result); 682 + } 683 + RepoRequest::UpdateRepoRoot { 684 + user_id, 685 + repo_root_cid, 686 + repo_rev, 687 + tx, 688 + } => { 689 + let result = state 690 + .metastore 691 + .repo_ops() 692 + .update_repo_root( 693 + state.metastore.database(), 694 + user_id, 695 + &repo_root_cid, 696 + &repo_rev, 697 + ) 698 + .map_err(metastore_to_db); 699 + let _ = tx.send(result); 700 + } 701 + RepoRequest::UpdateRepoRev { 702 + user_id, 703 + repo_rev, 704 + tx, 705 + } => { 706 + let result = state 707 + .metastore 708 + .repo_ops() 709 + .update_repo_rev(state.metastore.database(), user_id, &repo_rev) 710 + .map_err(metastore_to_db); 711 + let _ = tx.send(result); 712 + } 713 + RepoRequest::DeleteRepo { user_id, tx } => { 714 + let result = state 715 + .metastore 716 + .repo_ops() 717 + .delete_repo(state.metastore.database(), user_id) 718 + .map_err(metastore_to_db); 719 + let _ = tx.send(result); 720 + } 721 + RepoRequest::GetRepoRootForUpdate { user_id, tx } => { 722 + let result = state 723 + .metastore 724 + .repo_ops() 725 + .get_repo_root_for_update(user_id) 726 + .map_err(metastore_to_db); 727 + let _ = tx.send(result); 728 + } 729 + RepoRequest::GetRepo { user_id, tx } => { 730 + let result = state 731 + .metastore 732 + .repo_ops() 733 + .get_repo(user_id) 734 + .map(|opt| opt.map(convert_repo_info)) 735 + .map_err(metastore_to_db); 736 + let _ = tx.send(result); 737 + } 738 + RepoRequest::GetRepoRootByDid { did, tx } => { 739 + let result = state 740 + .metastore 741 + .repo_ops() 742 + .get_repo_root_by_did(&did) 743 + .map_err(metastore_to_db); 744 + let _ = tx.send(result); 745 + } 746 + RepoRequest::CountRepos { tx } => { 747 + let result = state 748 + .metastore 749 + .repo_ops() 750 + .count_repos() 751 + .map_err(metastore_to_db); 752 + let _ = tx.send(result); 753 + } 754 + RepoRequest::GetReposWithoutRev { tx } => { 755 + let result = state 756 + .metastore 757 + .repo_ops() 758 + .get_repos_without_rev(MAX_REPOS_WITHOUT_REV) 759 + .map(|v| v.into_iter().map(convert_without_rev).collect()) 760 + .map_err(metastore_to_db); 761 + let _ = tx.send(result); 762 + } 763 + RepoRequest::GetRepoRootCidByUserId { user_id, tx } => { 764 + let result = state 765 + .metastore 766 + .repo_ops() 767 + .get_repo_root_cid_by_user_id(user_id) 768 + .map_err(metastore_to_db); 769 + let _ = tx.send(result); 770 + } 771 + RepoRequest::GetAccountWithRepo { did, tx } => { 772 + let result = state 773 + .metastore 774 + .repo_ops() 775 + .get_account_with_repo(&did) 776 + .map(|opt| opt.map(convert_repo_account)) 777 + .map_err(metastore_to_db); 778 + let _ = tx.send(result); 779 + } 780 + RepoRequest::ListReposPaginated { 781 + cursor_user_hash, 782 + limit, 783 + tx, 784 + } => { 785 + let result = state 786 + .metastore 787 + .repo_ops() 788 + .list_repos_paginated(cursor_user_hash, limit) 789 + .map_err(metastore_to_db) 790 + .and_then(|entries| entries.into_iter().map(convert_repo_list_entry).collect()); 791 + let _ = tx.send(result); 792 + } 793 + RepoRequest::UpdateRepoStatus { 794 + did, 795 + takedown, 796 + takedown_ref, 797 + deactivated, 798 + tx, 799 + } => { 800 + let result = state 801 + .metastore 802 + .repo_ops() 803 + .update_repo_status( 804 + state.metastore.database(), 805 + &did, 806 + takedown, 807 + takedown_ref.as_deref(), 808 + deactivated, 809 + ) 810 + .map_err(metastore_to_db); 811 + let _ = tx.send(result); 812 + } 813 + } 814 + } 815 + 816 + fn dispatch_record<S: StorageIO>(state: &HandlerState<S>, req: RecordRequest) { 817 + match req { 818 + RecordRequest::UpsertRecords { 819 + repo_id, 820 + collections, 821 + rkeys, 822 + record_cids, 823 + repo_rev, 824 + tx, 825 + } => { 826 + let result = (|| { 827 + let (user_hash, mut meta) = state 828 + .metastore 829 + .repo_ops() 830 + .get_repo_meta(repo_id) 831 + .map_err(metastore_to_db)? 832 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 833 + let writes: Vec<super::record_ops::RecordWrite<'_>> = collections 834 + .iter() 835 + .zip(rkeys.iter()) 836 + .zip(record_cids.iter()) 837 + .map(|((c, r), cid)| super::record_ops::RecordWrite { 838 + collection: c, 839 + rkey: r, 840 + cid, 841 + }) 842 + .collect(); 843 + let mut batch = state.metastore.database().batch(); 844 + state 845 + .metastore 846 + .record_ops() 847 + .upsert_records(&mut batch, user_hash, &writes) 848 + .map_err(metastore_to_db)?; 849 + meta.repo_rev = repo_rev; 850 + state 851 + .metastore 852 + .repo_ops() 853 + .write_repo_meta(&mut batch, user_hash, &meta); 854 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 855 + })(); 856 + let _ = tx.send(result); 857 + } 858 + RecordRequest::DeleteRecords { 859 + repo_id, 860 + collections, 861 + rkeys, 862 + tx, 863 + } => { 864 + let result = (|| { 865 + let user_hash = state 866 + .metastore 867 + .user_hashes() 868 + .get(&repo_id) 869 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 870 + let deletes: Vec<super::record_ops::RecordDelete<'_>> = collections 871 + .iter() 872 + .zip(rkeys.iter()) 873 + .map(|(c, r)| super::record_ops::RecordDelete { 874 + collection: c, 875 + rkey: r, 876 + }) 877 + .collect(); 878 + let mut batch = state.metastore.database().batch(); 879 + state 880 + .metastore 881 + .record_ops() 882 + .delete_records(&mut batch, user_hash, &deletes); 883 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 884 + })(); 885 + let _ = tx.send(result); 886 + } 887 + RecordRequest::DeleteAllRecords { repo_id, tx } => { 888 + let result = (|| { 889 + let user_hash = state 890 + .metastore 891 + .user_hashes() 892 + .get(&repo_id) 893 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 894 + let mut batch = state.metastore.database().batch(); 895 + state 896 + .metastore 897 + .record_ops() 898 + .delete_all_records(&mut batch, user_hash) 899 + .map_err(metastore_to_db)?; 900 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 901 + })(); 902 + let _ = tx.send(result); 903 + } 904 + RecordRequest::GetRecordCid { 905 + repo_id, 906 + collection, 907 + rkey, 908 + tx, 909 + } => { 910 + let result = state 911 + .metastore 912 + .record_ops() 913 + .get_record_cid(repo_id, &collection, &rkey) 914 + .map_err(metastore_to_db); 915 + let _ = tx.send(result); 916 + } 917 + RecordRequest::ListRecords { 918 + repo_id, 919 + collection, 920 + cursor, 921 + limit, 922 + reverse, 923 + rkey_start, 924 + rkey_end, 925 + tx, 926 + } => { 927 + let query = ListRecordsQuery { 928 + user_id: repo_id, 929 + collection: &collection, 930 + cursor: cursor.as_ref(), 931 + limit: usize::try_from(limit).unwrap_or(usize::MAX), 932 + reverse, 933 + rkey_start: rkey_start.as_ref(), 934 + rkey_end: rkey_end.as_ref(), 935 + }; 936 + let result = state 937 + .metastore 938 + .record_ops() 939 + .list_records(&query) 940 + .map(|v| v.into_iter().map(convert_record_info).collect()) 941 + .map_err(metastore_to_db); 942 + let _ = tx.send(result); 943 + } 944 + RecordRequest::GetAllRecords { repo_id, tx } => { 945 + let result = state 946 + .metastore 947 + .record_ops() 948 + .get_all_records(repo_id) 949 + .map(|v| v.into_iter().map(convert_full_record_info).collect()) 950 + .map_err(metastore_to_db); 951 + let _ = tx.send(result); 952 + } 953 + RecordRequest::ListCollections { repo_id, tx } => { 954 + let result = state 955 + .metastore 956 + .record_ops() 957 + .list_collections(repo_id) 958 + .map_err(metastore_to_db); 959 + let _ = tx.send(result); 960 + } 961 + RecordRequest::CountRecords { repo_id, tx } => { 962 + let result = state 963 + .metastore 964 + .record_ops() 965 + .count_records(repo_id) 966 + .map_err(metastore_to_db); 967 + let _ = tx.send(result); 968 + } 969 + RecordRequest::CountAllRecords { tx } => { 970 + let result = state 971 + .metastore 972 + .record_ops() 973 + .count_all_records() 974 + .map_err(metastore_to_db); 975 + let _ = tx.send(result); 976 + } 977 + RecordRequest::GetRecordByCid { cid, tx } => { 978 + let result = state 979 + .metastore 980 + .record_ops() 981 + .get_record_by_cid(&cid, None) 982 + .map(|opt| opt.map(convert_record_with_takedown)) 983 + .map_err(metastore_to_db); 984 + let _ = tx.send(result); 985 + } 986 + RecordRequest::SetRecordTakedown { 987 + cid, 988 + takedown_ref, 989 + scope_user, 990 + tx, 991 + } => { 992 + let result = state 993 + .metastore 994 + .record_ops() 995 + .set_record_takedown( 996 + state.metastore.database(), 997 + &cid, 998 + takedown_ref.as_deref(), 999 + scope_user, 1000 + ) 1001 + .map_err(metastore_to_db); 1002 + let _ = tx.send(result); 1003 + } 1004 + } 1005 + } 1006 + 1007 + fn dispatch_user_block<S: StorageIO>(state: &HandlerState<S>, req: UserBlockRequest) { 1008 + match req { 1009 + UserBlockRequest::InsertUserBlocks { 1010 + user_id, 1011 + block_cids, 1012 + repo_rev, 1013 + tx, 1014 + } => { 1015 + let result = (|| { 1016 + let user_hash = state 1017 + .metastore 1018 + .user_hashes() 1019 + .get(&user_id) 1020 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 1021 + let mut batch = state.metastore.database().batch(); 1022 + state 1023 + .metastore 1024 + .user_block_ops() 1025 + .insert_user_blocks(&mut batch, user_hash, &block_cids, &repo_rev) 1026 + .map_err(metastore_to_db)?; 1027 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 1028 + })(); 1029 + let _ = tx.send(result); 1030 + } 1031 + UserBlockRequest::DeleteUserBlocks { 1032 + user_id, 1033 + block_cids, 1034 + tx, 1035 + } => { 1036 + let result = (|| { 1037 + let user_hash = state 1038 + .metastore 1039 + .user_hashes() 1040 + .get(&user_id) 1041 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 1042 + let mut batch = state.metastore.database().batch(); 1043 + state 1044 + .metastore 1045 + .user_block_ops() 1046 + .delete_user_blocks_by_cid(&mut batch, user_hash, &block_cids) 1047 + .map_err(metastore_to_db)?; 1048 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 1049 + })(); 1050 + let _ = tx.send(result); 1051 + } 1052 + UserBlockRequest::GetUserBlockCidsSinceRev { 1053 + user_id, 1054 + since_rev, 1055 + tx, 1056 + } => { 1057 + let result = state 1058 + .metastore 1059 + .user_block_ops() 1060 + .get_user_block_cids_since_rev(user_id, &since_rev) 1061 + .map_err(metastore_to_db); 1062 + let _ = tx.send(result); 1063 + } 1064 + UserBlockRequest::CountUserBlocks { user_id, tx } => { 1065 + let result = state 1066 + .metastore 1067 + .user_block_ops() 1068 + .count_user_blocks(user_id) 1069 + .map_err(metastore_to_db); 1070 + let _ = tx.send(result); 1071 + } 1072 + UserBlockRequest::FindUnreferencedBlocks { candidate_cids, tx } => { 1073 + let result = state 1074 + .metastore 1075 + .user_block_ops() 1076 + .find_unreferenced(&candidate_cids); 1077 + let _ = tx.send(Ok(result)); 1078 + } 1079 + } 1080 + } 1081 + 1082 + fn dispatch_event<S: StorageIO>(state: &HandlerState<S>, req: EventRequest) { 1083 + match req { 1084 + EventRequest::InsertCommitEvent { data, tx } => { 1085 + let result = state.event_ops.insert_commit_event(&data); 1086 + let _ = tx.send(result); 1087 + } 1088 + EventRequest::InsertIdentityEvent { did, handle, tx } => { 1089 + let result = state.event_ops.insert_identity_event(&did, handle.as_ref()); 1090 + let _ = tx.send(result); 1091 + } 1092 + EventRequest::InsertAccountEvent { did, status, tx } => { 1093 + let result = state.event_ops.insert_account_event(&did, status); 1094 + let _ = tx.send(result); 1095 + } 1096 + EventRequest::InsertSyncEvent { 1097 + did, 1098 + commit_cid, 1099 + rev, 1100 + tx, 1101 + } => { 1102 + let result = state 1103 + .event_ops 1104 + .insert_sync_event(&did, &commit_cid, rev.as_deref()); 1105 + let _ = tx.send(result); 1106 + } 1107 + EventRequest::InsertGenesisCommitEvent { 1108 + did, 1109 + commit_cid, 1110 + mst_root_cid, 1111 + rev, 1112 + tx, 1113 + } => { 1114 + let result = 1115 + state 1116 + .event_ops 1117 + .insert_genesis_commit_event(&did, &commit_cid, &mst_root_cid, &rev); 1118 + let _ = tx.send(result); 1119 + } 1120 + EventRequest::UpdateSeqBlocksCids { 1121 + seq, 1122 + blocks_cids, 1123 + tx, 1124 + } => { 1125 + let result = state.event_ops.update_seq_blocks_cids(seq, &blocks_cids); 1126 + let _ = tx.send(result); 1127 + } 1128 + EventRequest::DeleteSequencesExcept { did, keep_seq, tx } => { 1129 + let result = state.event_ops.delete_sequences_except(&did, keep_seq); 1130 + let _ = tx.send(result); 1131 + } 1132 + EventRequest::GetMaxSeq { tx } => { 1133 + let _ = tx.send(Ok(state.event_ops.get_max_seq())); 1134 + } 1135 + EventRequest::GetMinSeqSince { since, tx } => { 1136 + let _ = tx.send(state.event_ops.get_min_seq_since(since)); 1137 + } 1138 + EventRequest::GetEventsSinceSeq { 1139 + since_seq, 1140 + limit, 1141 + tx, 1142 + } => { 1143 + let _ = tx.send(state.event_ops.get_events_since_seq(since_seq, limit)); 1144 + } 1145 + EventRequest::GetEventsInSeqRange { 1146 + start_seq, 1147 + end_seq, 1148 + tx, 1149 + } => { 1150 + let _ = tx.send(state.event_ops.get_events_in_seq_range(start_seq, end_seq)); 1151 + } 1152 + EventRequest::GetEventBySeq { seq, tx } => { 1153 + let _ = tx.send(state.event_ops.get_event_by_seq(seq)); 1154 + } 1155 + EventRequest::GetEventsSinceCursor { cursor, limit, tx } => { 1156 + let _ = tx.send(state.event_ops.get_events_since_cursor(cursor, limit)); 1157 + } 1158 + EventRequest::GetEventsSinceRev { did, since_rev, tx } => { 1159 + let _ = tx.send(state.event_ops.get_events_since_rev(&did, &since_rev)); 1160 + } 1161 + EventRequest::NotifyUpdate { seq, tx } => { 1162 + let _ = tx.send(state.event_ops.notify_update(seq)); 1163 + } 1164 + } 1165 + } 1166 + 1167 + fn dispatch_commit<S: StorageIO>(state: &HandlerState<S>, req: CommitRequest) { 1168 + match req { 1169 + CommitRequest::ApplyCommit { input, tx } => { 1170 + let _ = tx.send(state.commit_ops.apply_commit(*input)); 1171 + } 1172 + CommitRequest::ImportRepoData { 1173 + user_id, 1174 + blocks, 1175 + records, 1176 + expected_root_cid, 1177 + tx, 1178 + } => { 1179 + let _ = tx.send(state.commit_ops.import_repo_data( 1180 + user_id, 1181 + &blocks, 1182 + &records, 1183 + expected_root_cid.as_ref(), 1184 + )); 1185 + } 1186 + CommitRequest::GetBrokenGenesisCommits { tx } => { 1187 + let _ = tx.send( 1188 + state 1189 + .commit_ops 1190 + .get_broken_genesis_commits() 1191 + .map_err(metastore_to_db), 1192 + ); 1193 + } 1194 + CommitRequest::GetUsersWithoutBlocks { tx } => { 1195 + let _ = tx.send( 1196 + state 1197 + .commit_ops 1198 + .get_users_without_blocks() 1199 + .map_err(metastore_to_db), 1200 + ); 1201 + } 1202 + CommitRequest::GetUsersNeedingRecordBlobsBackfill { limit, tx } => { 1203 + let _ = tx.send( 1204 + state 1205 + .commit_ops 1206 + .get_users_needing_record_blobs_backfill(limit) 1207 + .map_err(metastore_to_db), 1208 + ); 1209 + } 1210 + CommitRequest::InsertRecordBlobs { 1211 + repo_id, 1212 + record_uris, 1213 + blob_cids, 1214 + tx, 1215 + } => { 1216 + let _ = tx.send( 1217 + state 1218 + .commit_ops 1219 + .insert_record_blobs(repo_id, &record_uris, &blob_cids) 1220 + .map_err(metastore_to_db), 1221 + ); 1222 + } 1223 + } 1224 + } 1225 + 1226 + fn dispatch_backlink<S: StorageIO>(state: &HandlerState<S>, req: BacklinkRequest) { 1227 + match req { 1228 + BacklinkRequest::GetBacklinkConflicts { 1229 + repo_id, 1230 + collection, 1231 + backlinks, 1232 + tx, 1233 + } => { 1234 + let result = state 1235 + .metastore 1236 + .backlink_ops() 1237 + .get_backlink_conflicts(repo_id, &collection, &backlinks) 1238 + .map_err(metastore_to_db); 1239 + let _ = tx.send(result); 1240 + } 1241 + BacklinkRequest::AddBacklinks { 1242 + repo_id, 1243 + backlinks, 1244 + tx, 1245 + } => { 1246 + let result = (|| { 1247 + let user_hash = state 1248 + .metastore 1249 + .user_hashes() 1250 + .get(&repo_id) 1251 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 1252 + let mut batch = state.metastore.database().batch(); 1253 + state 1254 + .metastore 1255 + .backlink_ops() 1256 + .add_backlinks(&mut batch, user_hash, &backlinks) 1257 + .map_err(metastore_to_db)?; 1258 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 1259 + })(); 1260 + let _ = tx.send(result); 1261 + } 1262 + BacklinkRequest::RemoveBacklinksByUri { uri, tx } => { 1263 + let result = (|| { 1264 + let did_str = uri 1265 + .did() 1266 + .ok_or(DbError::Query("backlink uri missing did".to_string()))?; 1267 + let user_hash = UserHash::from_did(did_str); 1268 + let mut batch = state.metastore.database().batch(); 1269 + state 1270 + .metastore 1271 + .backlink_ops() 1272 + .remove_backlinks_by_uri(&mut batch, user_hash, &uri) 1273 + .map_err(metastore_to_db)?; 1274 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 1275 + })(); 1276 + let _ = tx.send(result); 1277 + } 1278 + BacklinkRequest::RemoveBacklinksByRepo { repo_id, tx } => { 1279 + let result = (|| { 1280 + let user_hash = state 1281 + .metastore 1282 + .user_hashes() 1283 + .get(&repo_id) 1284 + .ok_or(DbError::Query("unknown user_id".to_string()))?; 1285 + let mut batch = state.metastore.database().batch(); 1286 + state 1287 + .metastore 1288 + .backlink_ops() 1289 + .remove_backlinks_by_repo(&mut batch, user_hash) 1290 + .map_err(metastore_to_db)?; 1291 + batch.commit().map_err(|e| DbError::Query(e.to_string())) 1292 + })(); 1293 + let _ = tx.send(result); 1294 + } 1295 + } 1296 + } 1297 + 1298 + fn dispatch_blob<S: StorageIO>(state: &HandlerState<S>, req: BlobRequest) { 1299 + match req { 1300 + BlobRequest::InsertBlob { 1301 + cid, 1302 + mime_type, 1303 + size_bytes, 1304 + created_by_user, 1305 + storage_key, 1306 + tx, 1307 + } => { 1308 + let result = state 1309 + .metastore 1310 + .blob_ops() 1311 + .insert_blob(&cid, &mime_type, size_bytes, created_by_user, &storage_key) 1312 + .map_err(metastore_to_db); 1313 + let _ = tx.send(result); 1314 + } 1315 + BlobRequest::GetBlobMetadata { cid, tx } => { 1316 + let result = state 1317 + .metastore 1318 + .blob_ops() 1319 + .get_blob_metadata(&cid) 1320 + .map_err(metastore_to_db); 1321 + let _ = tx.send(result); 1322 + } 1323 + BlobRequest::GetBlobWithTakedown { cid, tx } => { 1324 + let result = state 1325 + .metastore 1326 + .blob_ops() 1327 + .get_blob_with_takedown(&cid) 1328 + .map_err(metastore_to_db); 1329 + let _ = tx.send(result); 1330 + } 1331 + BlobRequest::GetBlobStorageKey { cid, tx } => { 1332 + let result = state 1333 + .metastore 1334 + .blob_ops() 1335 + .get_blob_storage_key(&cid) 1336 + .map_err(metastore_to_db); 1337 + let _ = tx.send(result); 1338 + } 1339 + BlobRequest::ListBlobsByUser { 1340 + user_id, 1341 + cursor, 1342 + limit, 1343 + tx, 1344 + } => { 1345 + let result = state 1346 + .metastore 1347 + .blob_ops() 1348 + .list_blobs_by_user( 1349 + user_id, 1350 + cursor.as_deref(), 1351 + usize::try_from(limit).unwrap_or(usize::MAX), 1352 + ) 1353 + .map_err(metastore_to_db); 1354 + let _ = tx.send(result); 1355 + } 1356 + BlobRequest::ListBlobsSinceRev { did, since, tx } => { 1357 + let result = state.event_ops.get_blob_cids_since_rev(&did, &since); 1358 + let _ = tx.send(result); 1359 + } 1360 + BlobRequest::CountBlobsByUser { user_id, tx } => { 1361 + let result = state 1362 + .metastore 1363 + .blob_ops() 1364 + .count_blobs_by_user(user_id) 1365 + .map_err(metastore_to_db); 1366 + let _ = tx.send(result); 1367 + } 1368 + BlobRequest::SumBlobStorage { tx } => { 1369 + let result = state 1370 + .metastore 1371 + .blob_ops() 1372 + .sum_blob_storage() 1373 + .map_err(metastore_to_db); 1374 + let _ = tx.send(result); 1375 + } 1376 + BlobRequest::UpdateBlobTakedown { 1377 + cid, 1378 + takedown_ref, 1379 + tx, 1380 + } => { 1381 + let result = state 1382 + .metastore 1383 + .blob_ops() 1384 + .update_blob_takedown(&cid, takedown_ref.as_deref()) 1385 + .map_err(metastore_to_db); 1386 + let _ = tx.send(result); 1387 + } 1388 + BlobRequest::DeleteBlobByCid { cid, tx } => { 1389 + let result = state 1390 + .metastore 1391 + .blob_ops() 1392 + .delete_blob_by_cid(&cid) 1393 + .map_err(metastore_to_db); 1394 + let _ = tx.send(result); 1395 + } 1396 + BlobRequest::DeleteBlobsByUser { user_id, tx } => { 1397 + let result = state 1398 + .metastore 1399 + .blob_ops() 1400 + .delete_blobs_by_user(user_id) 1401 + .map_err(metastore_to_db); 1402 + let _ = tx.send(result); 1403 + } 1404 + BlobRequest::GetBlobStorageKeysByUser { user_id, tx } => { 1405 + let result = state 1406 + .metastore 1407 + .blob_ops() 1408 + .get_blob_storage_keys_by_user(user_id) 1409 + .map_err(metastore_to_db); 1410 + let _ = tx.send(result); 1411 + } 1412 + BlobRequest::ListMissingBlobs { 1413 + repo_id, 1414 + cursor, 1415 + limit, 1416 + tx, 1417 + } => { 1418 + let result = state 1419 + .metastore 1420 + .blob_ops() 1421 + .list_missing_blobs( 1422 + repo_id, 1423 + cursor.as_deref(), 1424 + usize::try_from(limit).unwrap_or(usize::MAX), 1425 + ) 1426 + .map_err(metastore_to_db); 1427 + let _ = tx.send(result); 1428 + } 1429 + BlobRequest::CountDistinctRecordBlobs { repo_id, tx } => { 1430 + let result = state 1431 + .metastore 1432 + .blob_ops() 1433 + .count_distinct_record_blobs(repo_id) 1434 + .map_err(metastore_to_db); 1435 + let _ = tx.send(result); 1436 + } 1437 + BlobRequest::GetBlobsForExport { repo_id, tx } => { 1438 + let result = state 1439 + .metastore 1440 + .blob_ops() 1441 + .get_blobs_for_export(repo_id) 1442 + .map_err(metastore_to_db); 1443 + let _ = tx.send(result); 1444 + } 1445 + } 1446 + } 1447 + 1448 + fn dispatch<S: StorageIO>(state: &HandlerState<S>, request: MetastoreRequest) { 1449 + match request { 1450 + MetastoreRequest::Repo(r) => dispatch_repo(state, r), 1451 + MetastoreRequest::Record(r) => dispatch_record(state, r), 1452 + MetastoreRequest::UserBlock(r) => dispatch_user_block(state, r), 1453 + MetastoreRequest::Event(r) => dispatch_event(state, r), 1454 + MetastoreRequest::Commit(r) => dispatch_commit(state, *r), 1455 + MetastoreRequest::Backlink(r) => dispatch_backlink(state, r), 1456 + MetastoreRequest::Blob(r) => dispatch_blob(state, r), 1457 + } 1458 + } 1459 + 1460 + fn handler_loop<S: StorageIO>( 1461 + metastore: Metastore, 1462 + bridge: Arc<EventLogBridge<S>>, 1463 + blockstore: Option<TranquilBlockStore>, 1464 + rx: flume::Receiver<MetastoreRequest>, 1465 + thread_index: usize, 1466 + ) { 1467 + let event_ops = metastore.event_ops(Arc::clone(&bridge)); 1468 + let mut commit_ops = metastore.commit_ops(bridge); 1469 + if let Some(bs) = blockstore { 1470 + commit_ops = commit_ops.with_blockstore(bs); 1471 + } 1472 + let state = HandlerState { 1473 + metastore, 1474 + event_ops, 1475 + commit_ops, 1476 + }; 1477 + tracing::info!(thread_index, "metastore handler thread started"); 1478 + rx.iter().for_each(|req| { 1479 + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dispatch(&state, req))) { 1480 + Ok(()) => {} 1481 + Err(e) => { 1482 + let msg = match e.downcast_ref::<&str>() { 1483 + Some(s) => (*s).to_owned(), 1484 + None => match e.downcast_ref::<String>() { 1485 + Some(s) => s.clone(), 1486 + None => "unknown panic payload".to_owned(), 1487 + }, 1488 + }; 1489 + tracing::error!(thread_index, msg, "metastore handler panic (recovered)"); 1490 + } 1491 + } 1492 + }); 1493 + tracing::info!(thread_index, "metastore handler thread exiting"); 1494 + } 1495 + 1496 + const DEFAULT_CHANNEL_BOUND: usize = 256; 1497 + const MAX_REPOS_WITHOUT_REV: usize = 10_000; 1498 + 1499 + pub struct HandlerPool { 1500 + senders: Vec<flume::Sender<MetastoreRequest>>, 1501 + handles: Option<Vec<JoinHandle<()>>>, 1502 + user_hashes: Arc<UserHashMap>, 1503 + round_robin: AtomicUsize, 1504 + } 1505 + 1506 + impl HandlerPool { 1507 + pub fn spawn<S: StorageIO + 'static>( 1508 + metastore: Metastore, 1509 + bridge: Arc<EventLogBridge<S>>, 1510 + blockstore: Option<TranquilBlockStore>, 1511 + thread_count: Option<usize>, 1512 + ) -> Self { 1513 + let count = thread_count 1514 + .unwrap_or_else(|| { 1515 + std::thread::available_parallelism() 1516 + .map(|n| n.get().max(2) / 2) 1517 + .unwrap_or(1) 1518 + }) 1519 + .max(1); 1520 + 1521 + let user_hashes = Arc::clone(metastore.user_hashes()); 1522 + 1523 + let (senders, handles): (Vec<_>, Vec<_>) = (0..count) 1524 + .map(|i| { 1525 + let (tx, rx) = flume::bounded(DEFAULT_CHANNEL_BOUND); 1526 + let ms = metastore.clone(); 1527 + let br = Arc::clone(&bridge); 1528 + let bs = blockstore.clone(); 1529 + let handle = std::thread::Builder::new() 1530 + .name(format!("metastore-{i}")) 1531 + .spawn(move || handler_loop(ms, br, bs, rx, i)) 1532 + .expect("failed to spawn metastore handler thread"); 1533 + (tx, handle) 1534 + }) 1535 + .unzip(); 1536 + 1537 + Self { 1538 + senders, 1539 + handles: Some(handles), 1540 + user_hashes, 1541 + round_robin: AtomicUsize::new(0), 1542 + } 1543 + } 1544 + 1545 + pub fn send(&self, request: MetastoreRequest) -> Result<(), DbError> { 1546 + let index = match request.routing(&self.user_hashes) { 1547 + Routing::Sharded(bits) => (bits as usize) % self.senders.len(), 1548 + Routing::Global => { 1549 + self.round_robin.fetch_add(1, Ordering::Relaxed) % self.senders.len() 1550 + } 1551 + }; 1552 + self.senders[index].try_send(request).map_err(|e| match e { 1553 + flume::TrySendError::Full(_) => { 1554 + DbError::Query("metastore handler backpressure".to_string()) 1555 + } 1556 + flume::TrySendError::Disconnected(_) => { 1557 + DbError::Connection("metastore handler pool shut down".to_string()) 1558 + } 1559 + }) 1560 + } 1561 + 1562 + pub fn thread_count(&self) -> usize { 1563 + self.senders.len() 1564 + } 1565 + 1566 + pub async fn shutdown(&mut self) { 1567 + self.senders.clear(); 1568 + if let Some(handles) = self.handles.take() { 1569 + let join_fut = tokio::task::spawn_blocking(move || { 1570 + handles.into_iter().for_each(|h| { 1571 + if let Err(e) = h.join() { 1572 + tracing::error!("metastore handler thread panicked: {e:?}"); 1573 + } 1574 + }); 1575 + }); 1576 + match tokio::time::timeout(std::time::Duration::from_secs(30), join_fut).await { 1577 + Ok(_) => tracing::info!("metastore handler threads shut down cleanly"), 1578 + Err(_) => tracing::error!("metastore handler thread shutdown timed out after 30s"), 1579 + } 1580 + } 1581 + } 1582 + } 1583 + 1584 + impl Drop for HandlerPool { 1585 + fn drop(&mut self) { 1586 + self.senders.clear(); 1587 + if let Some(handles) = self.handles.take() { 1588 + tracing::warn!( 1589 + "HandlerPool dropped without calling shutdown(); blocking on thread join" 1590 + ); 1591 + handles.into_iter().for_each(|h| { 1592 + if let Err(e) = h.join() { 1593 + tracing::error!("metastore handler thread panicked: {e:?}"); 1594 + } 1595 + }); 1596 + } 1597 + } 1598 + } 1599 + 1600 + #[cfg(test)] 1601 + mod tests { 1602 + use super::*; 1603 + use crate::eventlog::{EventLog, EventLogConfig}; 1604 + use crate::io::RealIO; 1605 + use crate::metastore::MetastoreConfig; 1606 + use tranquil_types::{Did, Handle}; 1607 + 1608 + struct TestHarness { 1609 + _metastore_dir: tempfile::TempDir, 1610 + _eventlog_dir: tempfile::TempDir, 1611 + pool: HandlerPool, 1612 + } 1613 + 1614 + fn setup() -> TestHarness { 1615 + let metastore_dir = tempfile::TempDir::new().unwrap(); 1616 + let eventlog_dir = tempfile::TempDir::new().unwrap(); 1617 + let segments_dir = eventlog_dir.path().join("segments"); 1618 + std::fs::create_dir_all(&segments_dir).unwrap(); 1619 + 1620 + let metastore = Metastore::open( 1621 + metastore_dir.path(), 1622 + MetastoreConfig { 1623 + cache_size_bytes: 64 * 1024 * 1024, 1624 + }, 1625 + ) 1626 + .unwrap(); 1627 + 1628 + let event_log = EventLog::open( 1629 + EventLogConfig { 1630 + segments_dir, 1631 + ..EventLogConfig::default() 1632 + }, 1633 + RealIO::new(), 1634 + ) 1635 + .unwrap(); 1636 + 1637 + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); 1638 + 1639 + let pool = HandlerPool::spawn::<RealIO>(metastore, bridge, None, Some(2)); 1640 + 1641 + TestHarness { 1642 + _metastore_dir: metastore_dir, 1643 + _eventlog_dir: eventlog_dir, 1644 + pool, 1645 + } 1646 + } 1647 + 1648 + fn test_cid_link(seed: u8) -> CidLink { 1649 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 1650 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 1651 + let c = cid::Cid::new_v1(0x71, mh); 1652 + CidLink::from_cid(&c) 1653 + } 1654 + 1655 + #[tokio::test] 1656 + async fn create_and_get_roundtrip() { 1657 + let h = setup(); 1658 + let user_id = Uuid::new_v4(); 1659 + let did = Did::from("did:plc:handler_test".to_string()); 1660 + let handle = Handle::from("handler.test.invalid".to_string()); 1661 + let cid = test_cid_link(1); 1662 + 1663 + let (tx, rx) = oneshot::channel(); 1664 + h.pool 1665 + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 1666 + user_id, 1667 + did, 1668 + handle, 1669 + repo_root_cid: cid.clone(), 1670 + repo_rev: "rev1".to_string(), 1671 + tx, 1672 + })) 1673 + .unwrap(); 1674 + rx.await.unwrap().unwrap(); 1675 + 1676 + let (tx, rx) = oneshot::channel(); 1677 + h.pool 1678 + .send(MetastoreRequest::Repo(RepoRequest::GetRepo { user_id, tx })) 1679 + .unwrap(); 1680 + let repo = rx.await.unwrap().unwrap().unwrap(); 1681 + assert_eq!(repo.repo_root_cid, cid); 1682 + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); 1683 + } 1684 + 1685 + #[test] 1686 + fn routing_determinism() { 1687 + let user_id = Uuid::from_u128(0x12345678); 1688 + let bits = user_id.as_u128() as u64; 1689 + let thread_count = 4usize; 1690 + let expected = (bits as usize) % thread_count; 1691 + (0..100).for_each(|_| { 1692 + assert_eq!((bits as usize) % thread_count, expected); 1693 + }); 1694 + } 1695 + 1696 + #[test] 1697 + fn global_round_robin_distributes() { 1698 + let counter = AtomicUsize::new(0); 1699 + let thread_count = 4usize; 1700 + let indices: Vec<usize> = (0..8) 1701 + .map(|_| counter.fetch_add(1, Ordering::Relaxed) % thread_count) 1702 + .collect(); 1703 + assert_eq!(indices, vec![0, 1, 2, 3, 0, 1, 2, 3]); 1704 + } 1705 + 1706 + #[tokio::test] 1707 + async fn shutdown_completes_inflight() { 1708 + let mut h = setup(); 1709 + let user_id = Uuid::new_v4(); 1710 + let did = Did::from("did:plc:shutdown_test".to_string()); 1711 + let handle = Handle::from("shutdown.test.invalid".to_string()); 1712 + let cid = test_cid_link(2); 1713 + 1714 + let (tx, rx) = oneshot::channel(); 1715 + h.pool 1716 + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { 1717 + user_id, 1718 + did, 1719 + handle, 1720 + repo_root_cid: cid, 1721 + repo_rev: "rev1".to_string(), 1722 + tx, 1723 + })) 1724 + .unwrap(); 1725 + rx.await.unwrap().unwrap(); 1726 + 1727 + h.pool.shutdown().await; 1728 + } 1729 + }
+149
crates/tranquil-store/src/metastore/keys.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use siphasher::sip::SipHasher24; 3 + use std::hash::Hasher; 4 + 5 + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] 6 + #[serde(transparent)] 7 + pub struct UserHash(u64); 8 + 9 + const SIPHASH_KEY0: u64 = 0x7472_616e_7175_696c; 10 + const SIPHASH_KEY1: u64 = 0x7064_735f_7573_6572; 11 + 12 + impl UserHash { 13 + pub fn from_did(did: &str) -> Self { 14 + let mut hasher = SipHasher24::new_with_keys(SIPHASH_KEY0, SIPHASH_KEY1); 15 + hasher.write(did.as_bytes()); 16 + Self(hasher.finish()) 17 + } 18 + 19 + pub fn from_raw(raw: u64) -> Self { 20 + Self(raw) 21 + } 22 + 23 + pub fn raw(self) -> u64 { 24 + self.0 25 + } 26 + } 27 + 28 + impl std::fmt::Display for UserHash { 29 + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 30 + write!(f, "{:016x}", self.0) 31 + } 32 + } 33 + 34 + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 35 + pub struct KeyTag(u8); 36 + 37 + impl KeyTag { 38 + pub const REPO_META: Self = Self(0x01); 39 + pub const RECORDS: Self = Self(0x02); 40 + pub const USER_BLOCKS: Self = Self(0x03); 41 + pub const HANDLES: Self = Self(0x04); 42 + pub const BLOBS: Self = Self(0x05); 43 + pub const BACKLINKS: Self = Self(0x06); 44 + pub const BLOB_BY_CID: Self = Self(0x07); 45 + 46 + pub const USER_MAP: Self = Self(0x10); 47 + pub const USER_MAP_REVERSE: Self = Self(0x11); 48 + 49 + pub const REV_TO_SEQ: Self = Self(0x20); 50 + pub const SEQ_META: Self = Self(0x21); 51 + pub const SEQ_TOMBSTONE: Self = Self(0x22); 52 + pub const METASTORE_CURSOR: Self = Self(0x23); 53 + pub const DID_EVENTS: Self = Self(0x24); 54 + 55 + pub const RECORD_BLOBS: Self = Self(0x30); 56 + pub const BACKLINK_BY_USER: Self = Self(0x31); 57 + 58 + pub const FORMAT_VERSION: Self = Self(0xFF); 59 + 60 + pub const fn raw(self) -> u8 { 61 + self.0 62 + } 63 + 64 + pub fn exclusive_prefix_bound(self) -> [u8; 1] { 65 + match self.0.checked_add(1) { 66 + Some(next) => [next], 67 + None => panic!("cannot compute exclusive upper bound for tag 0xFF"), 68 + } 69 + } 70 + 71 + #[cfg(test)] 72 + pub fn from_raw_unchecked(raw: u8) -> Self { 73 + Self(raw) 74 + } 75 + } 76 + 77 + #[cfg(test)] 78 + mod tests { 79 + use super::*; 80 + 81 + #[test] 82 + fn user_hash_deterministic() { 83 + let a = UserHash::from_did("did:plc:abc123"); 84 + let b = UserHash::from_did("did:plc:abc123"); 85 + assert_eq!(a, b); 86 + } 87 + 88 + #[test] 89 + fn user_hash_different_dids_differ() { 90 + let a = UserHash::from_did("did:plc:abc123"); 91 + let b = UserHash::from_did("did:plc:xyz789"); 92 + assert_ne!(a, b); 93 + } 94 + 95 + #[test] 96 + fn user_hash_display_is_hex() { 97 + let h = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); 98 + assert_eq!(h.to_string(), "deadbeefcafebabe"); 99 + } 100 + 101 + #[test] 102 + fn key_tags_are_distinct() { 103 + let tags = [ 104 + KeyTag::REPO_META, 105 + KeyTag::RECORDS, 106 + KeyTag::USER_BLOCKS, 107 + KeyTag::HANDLES, 108 + KeyTag::BLOBS, 109 + KeyTag::BACKLINKS, 110 + KeyTag::BLOB_BY_CID, 111 + KeyTag::USER_MAP, 112 + KeyTag::USER_MAP_REVERSE, 113 + KeyTag::REV_TO_SEQ, 114 + KeyTag::SEQ_META, 115 + KeyTag::SEQ_TOMBSTONE, 116 + KeyTag::METASTORE_CURSOR, 117 + KeyTag::DID_EVENTS, 118 + KeyTag::RECORD_BLOBS, 119 + KeyTag::BACKLINK_BY_USER, 120 + KeyTag::FORMAT_VERSION, 121 + ]; 122 + let mut raw: Vec<u8> = tags.iter().map(|t| t.raw()).collect(); 123 + let original_len = raw.len(); 124 + raw.sort(); 125 + raw.dedup(); 126 + assert_eq!(raw.len(), original_len); 127 + } 128 + 129 + #[test] 130 + fn key_tag_ordering() { 131 + assert!(KeyTag::REPO_META < KeyTag::RECORDS); 132 + assert!(KeyTag::RECORDS < KeyTag::USER_BLOCKS); 133 + } 134 + 135 + #[test] 136 + fn exclusive_prefix_bound_is_tag_plus_one() { 137 + assert_eq!( 138 + KeyTag::REPO_META.exclusive_prefix_bound(), 139 + [KeyTag::REPO_META.raw() + 1] 140 + ); 141 + assert_eq!(KeyTag::HANDLES.exclusive_prefix_bound(), [0x05]); 142 + } 143 + 144 + #[test] 145 + #[should_panic(expected = "cannot compute exclusive upper bound for tag 0xFF")] 146 + fn exclusive_prefix_bound_panics_for_0xff() { 147 + KeyTag::FORMAT_VERSION.exclusive_prefix_bound(); 148 + } 149 + }
+417
crates/tranquil-store/src/metastore/mod.rs
··· 1 + pub mod backlink_ops; 2 + pub mod backlinks; 3 + pub mod blob_ops; 4 + pub mod blobs; 5 + pub mod commit_ops; 6 + pub mod encoding; 7 + pub mod event_keys; 8 + pub mod event_ops; 9 + pub mod keys; 10 + pub mod partitions; 11 + pub mod record_ops; 12 + pub mod records; 13 + pub mod recovery; 14 + pub mod repo_meta; 15 + pub mod repo_ops; 16 + pub mod scan; 17 + pub mod user_block_ops; 18 + pub mod user_blocks; 19 + pub mod user_hash; 20 + 21 + use std::path::Path; 22 + use std::sync::Arc; 23 + 24 + use fjall::{Database, Keyspace}; 25 + 26 + use self::keys::KeyTag; 27 + use self::partitions::Partition; 28 + use self::user_hash::UserHashMap; 29 + 30 + const CURRENT_FORMAT_VERSION: u64 = 1; 31 + 32 + #[derive(Debug, Clone)] 33 + pub struct MetastoreConfig { 34 + pub cache_size_bytes: u64, 35 + } 36 + 37 + impl Default for MetastoreConfig { 38 + fn default() -> Self { 39 + let total_ram = total_system_ram_bytes(); 40 + let twenty_percent = total_ram / 5; 41 + 42 + Self { 43 + cache_size_bytes: twenty_percent, 44 + } 45 + } 46 + } 47 + 48 + fn total_system_ram_bytes() -> u64 { 49 + #[cfg(target_os = "linux")] 50 + { 51 + std::fs::read_to_string("/proc/meminfo") 52 + .ok() 53 + .and_then(|contents| { 54 + contents 55 + .lines() 56 + .find(|line| line.starts_with("MemTotal:")) 57 + .and_then(|line| { 58 + line.split_whitespace() 59 + .nth(1) 60 + .and_then(|kb| kb.parse::<u64>().ok()) 61 + .map(|kb| kb.saturating_mul(1024)) 62 + }) 63 + }) 64 + .unwrap_or(4 * 1024 * 1024 * 1024) 65 + } 66 + #[cfg(not(target_os = "linux"))] 67 + { 68 + tracing::warn!("cannot detect system RAM on this platform, defaulting to 4GB"); 69 + 4 * 1024 * 1024 * 1024 70 + } 71 + } 72 + 73 + #[derive(Debug)] 74 + pub enum MetastoreError { 75 + Fjall(fjall::Error), 76 + Lsm(lsm_tree::Error), 77 + VersionMismatch { 78 + expected: u64, 79 + found: u64, 80 + }, 81 + CorruptData(&'static str), 82 + InvalidInput(&'static str), 83 + UserHashCollision { 84 + hash: keys::UserHash, 85 + existing_uuid: uuid::Uuid, 86 + new_uuid: uuid::Uuid, 87 + }, 88 + } 89 + 90 + impl std::fmt::Display for MetastoreError { 91 + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 92 + match self { 93 + Self::Fjall(e) => write!(f, "fjall: {e}"), 94 + Self::Lsm(e) => write!(f, "lsm: {e}"), 95 + Self::VersionMismatch { expected, found } => { 96 + write!( 97 + f, 98 + "format version mismatch: expected {expected}, found {found}" 99 + ) 100 + } 101 + Self::CorruptData(msg) => write!(f, "corrupt data: {msg}"), 102 + Self::InvalidInput(msg) => write!(f, "invalid input: {msg}"), 103 + Self::UserHashCollision { 104 + hash, 105 + existing_uuid, 106 + new_uuid, 107 + } => write!( 108 + f, 109 + "user hash collision: hash {hash} maps to both {existing_uuid} and {new_uuid}" 110 + ), 111 + } 112 + } 113 + } 114 + 115 + impl std::error::Error for MetastoreError { 116 + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 117 + match self { 118 + Self::Fjall(e) => Some(e), 119 + Self::Lsm(e) => Some(e), 120 + _ => None, 121 + } 122 + } 123 + } 124 + 125 + impl From<fjall::Error> for MetastoreError { 126 + fn from(e: fjall::Error) -> Self { 127 + Self::Fjall(e) 128 + } 129 + } 130 + 131 + impl From<lsm_tree::Error> for MetastoreError { 132 + fn from(e: lsm_tree::Error) -> Self { 133 + Self::Lsm(e) 134 + } 135 + } 136 + 137 + type CompactionFilterFn = 138 + Arc<dyn Fn(&str) -> Option<Arc<dyn fjall::compaction::filter::Factory>> + Send + Sync>; 139 + 140 + pub mod client; 141 + pub mod handler; 142 + 143 + #[derive(Clone)] 144 + pub struct Metastore { 145 + db: Database, 146 + partitions: [Keyspace; Partition::ALL.len()], 147 + user_hashes: Arc<UserHashMap>, 148 + } 149 + 150 + impl Metastore { 151 + pub fn open(path: &Path, config: MetastoreConfig) -> Result<Self, MetastoreError> { 152 + let auth_name = Partition::Auth.name(); 153 + let filter_factory: CompactionFilterFn = 154 + Arc::new(move |name: &str| match name == auth_name { 155 + true => Some(Arc::new(partitions::TtlFilterFactory)), 156 + false => None, 157 + }); 158 + 159 + let db = Database::builder(path) 160 + .cache_size(config.cache_size_bytes) 161 + .with_compaction_filter_factories(filter_factory) 162 + .open()?; 163 + 164 + let opened: Vec<Keyspace> = Partition::ALL 165 + .iter() 166 + .map(|&p| { 167 + let opts = p.create_options(); 168 + db.keyspace(p.name(), || opts) 169 + }) 170 + .collect::<Result<_, fjall::Error>>()?; 171 + 172 + let partitions: [Keyspace; Partition::ALL.len()] = opened 173 + .try_into() 174 + .ok() 175 + .expect("opened exactly Partition::ALL.len() keyspaces"); 176 + 177 + let repo_data = partitions[Partition::RepoData.index()].clone(); 178 + Self::check_or_write_version(&db, &repo_data)?; 179 + 180 + let user_hashes = Arc::new(UserHashMap::new(repo_data)); 181 + let loaded = user_hashes.load_all()?; 182 + tracing::info!(count = loaded, "loaded user hash mappings"); 183 + 184 + Ok(Self { 185 + db, 186 + partitions, 187 + user_hashes, 188 + }) 189 + } 190 + 191 + fn check_or_write_version(db: &Database, repo_data: &Keyspace) -> Result<(), MetastoreError> { 192 + let version_key = [KeyTag::FORMAT_VERSION.raw()]; 193 + let version_bytes = CURRENT_FORMAT_VERSION.to_be_bytes(); 194 + 195 + match repo_data.get(version_key)? { 196 + Some(existing) => { 197 + let found_bytes: [u8; 8] = existing 198 + .as_ref() 199 + .try_into() 200 + .map_err(|_| MetastoreError::CorruptData("format version not 8 bytes"))?; 201 + let found = u64::from_be_bytes(found_bytes); 202 + match found == CURRENT_FORMAT_VERSION { 203 + true => Ok(()), 204 + false => Err(MetastoreError::VersionMismatch { 205 + expected: CURRENT_FORMAT_VERSION, 206 + found, 207 + }), 208 + } 209 + } 210 + None => { 211 + repo_data.insert(version_key, version_bytes)?; 212 + db.persist(fjall::PersistMode::SyncData)?; 213 + Ok(()) 214 + } 215 + } 216 + } 217 + 218 + pub fn partition(&self, p: Partition) -> &Keyspace { 219 + &self.partitions[p.index()] 220 + } 221 + 222 + pub fn user_hashes(&self) -> &Arc<UserHashMap> { 223 + &self.user_hashes 224 + } 225 + 226 + pub fn database(&self) -> &Database { 227 + &self.db 228 + } 229 + 230 + pub fn repo_ops(&self) -> repo_ops::RepoOps { 231 + repo_ops::RepoOps::new( 232 + self.partitions[Partition::RepoData.index()].clone(), 233 + Arc::clone(&self.user_hashes), 234 + ) 235 + } 236 + 237 + pub fn record_ops(&self) -> record_ops::RecordOps { 238 + record_ops::RecordOps::new( 239 + self.partitions[Partition::RepoData.index()].clone(), 240 + Arc::clone(&self.user_hashes), 241 + ) 242 + } 243 + 244 + pub fn user_block_ops(&self) -> user_block_ops::UserBlockOps { 245 + user_block_ops::UserBlockOps::new( 246 + self.partitions[Partition::RepoData.index()].clone(), 247 + Arc::clone(&self.user_hashes), 248 + ) 249 + } 250 + 251 + pub fn event_ops<S: crate::io::StorageIO>( 252 + &self, 253 + bridge: Arc<crate::eventlog::EventLogBridge<S>>, 254 + ) -> event_ops::EventOps<S> { 255 + event_ops::EventOps::new( 256 + self.db.clone(), 257 + self.partitions[Partition::RepoData.index()].clone(), 258 + bridge, 259 + ) 260 + } 261 + 262 + pub fn blob_ops(&self) -> blob_ops::BlobOps { 263 + blob_ops::BlobOps::new( 264 + self.db.clone(), 265 + self.partitions[Partition::RepoData.index()].clone(), 266 + Arc::clone(&self.user_hashes), 267 + ) 268 + } 269 + 270 + pub fn backlink_ops(&self) -> backlink_ops::BacklinkOps { 271 + backlink_ops::BacklinkOps::new( 272 + self.partitions[Partition::Indexes.index()].clone(), 273 + Arc::clone(&self.user_hashes), 274 + ) 275 + } 276 + 277 + pub fn commit_ops<S: crate::io::StorageIO>( 278 + &self, 279 + bridge: Arc<crate::eventlog::EventLogBridge<S>>, 280 + ) -> commit_ops::CommitOps<S> { 281 + commit_ops::CommitOps::new( 282 + self.db.clone(), 283 + self.partitions[Partition::RepoData.index()].clone(), 284 + self.partitions[Partition::Indexes.index()].clone(), 285 + Arc::clone(&self.user_hashes), 286 + bridge, 287 + ) 288 + } 289 + 290 + pub fn persist(&self) -> Result<(), MetastoreError> { 291 + self.db 292 + .persist(fjall::PersistMode::SyncData) 293 + .map_err(MetastoreError::Fjall) 294 + } 295 + 296 + pub fn major_compact(&self) -> Result<(), MetastoreError> { 297 + Partition::ALL.iter().try_for_each(|&p| { 298 + tracing::info!(partition = p.name(), "starting major compaction"); 299 + self.partitions[p.index()] 300 + .major_compact() 301 + .map_err(MetastoreError::Fjall)?; 302 + tracing::info!(partition = p.name(), "major compaction complete"); 303 + Ok::<(), MetastoreError>(()) 304 + }) 305 + } 306 + } 307 + 308 + #[cfg(test)] 309 + mod tests { 310 + use super::*; 311 + 312 + fn open_fresh() -> (tempfile::TempDir, Metastore) { 313 + let dir = tempfile::TempDir::new().unwrap(); 314 + let ms = Metastore::open( 315 + dir.path(), 316 + MetastoreConfig { 317 + cache_size_bytes: 64 * 1024 * 1024, 318 + }, 319 + ) 320 + .unwrap(); 321 + (dir, ms) 322 + } 323 + 324 + fn test_config() -> MetastoreConfig { 325 + MetastoreConfig { 326 + cache_size_bytes: 64 * 1024 * 1024, 327 + } 328 + } 329 + 330 + #[test] 331 + fn open_fresh_directory_succeeds() { 332 + let (_dir, ms) = open_fresh(); 333 + assert_eq!(ms.user_hashes().len(), 0); 334 + } 335 + 336 + #[test] 337 + fn all_partitions_accessible() { 338 + let (_dir, ms) = open_fresh(); 339 + Partition::ALL.iter().for_each(|&p| { 340 + let _ = ms.partition(p); 341 + }); 342 + } 343 + 344 + #[test] 345 + fn reopen_preserves_partitions() { 346 + let dir = tempfile::TempDir::new().unwrap(); 347 + 348 + { 349 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 350 + let repo_data = ms.partition(Partition::RepoData); 351 + repo_data.insert(b"test_key", b"test_value").unwrap(); 352 + ms.persist().unwrap(); 353 + } 354 + 355 + { 356 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 357 + let repo_data = ms.partition(Partition::RepoData); 358 + let val = repo_data.get(b"test_key").unwrap().unwrap(); 359 + assert_eq!(val.as_ref(), b"test_value"); 360 + } 361 + } 362 + 363 + #[test] 364 + fn version_mismatch_returns_error() { 365 + let dir = tempfile::TempDir::new().unwrap(); 366 + 367 + { 368 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 369 + let repo_data = ms.partition(Partition::RepoData); 370 + let version_key = [KeyTag::FORMAT_VERSION.raw()]; 371 + repo_data.insert(version_key, 999u64.to_be_bytes()).unwrap(); 372 + ms.persist().unwrap(); 373 + } 374 + 375 + { 376 + let result = Metastore::open(dir.path(), test_config()); 377 + assert!(matches!( 378 + result, 379 + Err(MetastoreError::VersionMismatch { 380 + expected: 1, 381 + found: 999 382 + }) 383 + )); 384 + } 385 + } 386 + 387 + #[test] 388 + fn user_hash_mappings_survive_reopen() { 389 + let dir = tempfile::TempDir::new().unwrap(); 390 + let uuid = uuid::Uuid::new_v4(); 391 + let hash = keys::UserHash::from_did("did:plc:survivor"); 392 + 393 + { 394 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 395 + let mut batch = ms.database().batch(); 396 + ms.user_hashes() 397 + .stage_insert(&mut batch, uuid, hash) 398 + .unwrap(); 399 + batch.commit().unwrap(); 400 + ms.persist().unwrap(); 401 + } 402 + 403 + { 404 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 405 + assert_eq!(ms.user_hashes().len(), 1); 406 + assert_eq!(ms.user_hashes().get(&uuid), Some(hash)); 407 + assert_eq!(ms.user_hashes().get_uuid(&hash), Some(uuid)); 408 + } 409 + } 410 + 411 + #[test] 412 + fn default_config_has_reasonable_cache_size() { 413 + let config = MetastoreConfig::default(); 414 + assert!(config.cache_size_bytes > 0); 415 + assert!(config.cache_size_bytes <= 4 * 1024 * 1024 * 1024); 416 + } 417 + }
+129
crates/tranquil-store/src/metastore/partitions.rs
··· 1 + use std::time::{SystemTime, UNIX_EPOCH}; 2 + 3 + use fjall::KeyspaceCreateOptions; 4 + use fjall::compaction::filter::{CompactionFilter, Context, Factory, ItemAccessor, Verdict}; 5 + use fjall::config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry}; 6 + 7 + pub const EXPIRES_AT_MS_SIZE: usize = 8; 8 + 9 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 10 + pub enum Partition { 11 + RepoData, 12 + Auth, 13 + Users, 14 + Infra, 15 + Indexes, 16 + } 17 + 18 + impl Partition { 19 + pub const ALL: [Partition; 5] = [ 20 + Partition::RepoData, 21 + Partition::Auth, 22 + Partition::Users, 23 + Partition::Infra, 24 + Partition::Indexes, 25 + ]; 26 + 27 + pub const fn index(self) -> usize { 28 + match self { 29 + Self::RepoData => 0, 30 + Self::Auth => 1, 31 + Self::Users => 2, 32 + Self::Infra => 3, 33 + Self::Indexes => 4, 34 + } 35 + } 36 + 37 + pub fn name(self) -> &'static str { 38 + match self { 39 + Self::RepoData => "repo_data", 40 + Self::Auth => "auth", 41 + Self::Users => "users", 42 + Self::Infra => "infra", 43 + Self::Indexes => "indexes", 44 + } 45 + } 46 + 47 + pub fn create_options(self) -> KeyspaceCreateOptions { 48 + match self { 49 + Self::RepoData | Self::Indexes => { 50 + KeyspaceCreateOptions::default().filter_policy(FilterPolicy::new([ 51 + FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)), 52 + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)), 53 + ])) 54 + } 55 + Self::Auth | Self::Users | Self::Infra => KeyspaceCreateOptions::default(), 56 + } 57 + } 58 + } 59 + 60 + pub(crate) struct TtlFilterFactory; 61 + 62 + impl Factory for TtlFilterFactory { 63 + fn name(&self) -> &str { 64 + "ttl_expiry" 65 + } 66 + 67 + fn make_filter(&self, _ctx: &Context) -> Box<dyn CompactionFilter> { 68 + let now_ms = u64::try_from( 69 + SystemTime::now() 70 + .duration_since(UNIX_EPOCH) 71 + .expect("system clock before unix epoch") 72 + .as_millis(), 73 + ) 74 + .unwrap_or(u64::MAX); 75 + Box::new(TtlFilter { now_ms }) 76 + } 77 + } 78 + 79 + struct TtlFilter { 80 + now_ms: u64, 81 + } 82 + 83 + impl CompactionFilter for TtlFilter { 84 + fn filter_item(&mut self, item: ItemAccessor<'_>, _ctx: &Context) -> lsm_tree::Result<Verdict> { 85 + let value = item.value()?; 86 + match value.get(..EXPIRES_AT_MS_SIZE) { 87 + Some(bytes) => { 88 + let expires_at_ms = 89 + u64::from_be_bytes(bytes.try_into().expect("slice is exactly 8 bytes")); 90 + match expires_at_ms > 0 && expires_at_ms < self.now_ms { 91 + true => Ok(Verdict::Remove), 92 + false => Ok(Verdict::Keep), 93 + } 94 + } 95 + None => Ok(Verdict::Keep), 96 + } 97 + } 98 + } 99 + 100 + #[cfg(test)] 101 + mod tests { 102 + use super::*; 103 + 104 + #[test] 105 + fn partition_names_are_distinct() { 106 + let names: Vec<_> = Partition::ALL.iter().map(|p| p.name()).collect(); 107 + let mut deduped = names.clone(); 108 + deduped.sort(); 109 + deduped.dedup(); 110 + assert_eq!(names.len(), deduped.len()); 111 + } 112 + 113 + #[test] 114 + fn all_partitions_covered() { 115 + assert_eq!(Partition::ALL.len(), 5); 116 + } 117 + 118 + #[test] 119 + fn auth_partition_has_filter() { 120 + assert_eq!(Partition::Auth.name(), "auth"); 121 + } 122 + 123 + #[test] 124 + fn index_matches_all_array_position() { 125 + Partition::ALL.iter().enumerate().for_each(|(i, &p)| { 126 + assert_eq!(p.index(), i, "Partition::{:?} index mismatch", p); 127 + }); 128 + } 129 + }
+1506
crates/tranquil-store/src/metastore/record_ops.rs
··· 1 + use std::sync::Arc; 2 + 3 + use fjall::Keyspace; 4 + use smallvec::SmallVec; 5 + use uuid::Uuid; 6 + 7 + use super::MetastoreError; 8 + use super::encoding::{KeyReader, exclusive_upper_bound}; 9 + use super::keys::UserHash; 10 + use super::records::{ 11 + RecordValue, record_collection_prefix, record_key, record_user_prefix, records_prefix, 12 + }; 13 + use super::repo_ops::{bytes_to_cid_link, cid_link_to_bytes}; 14 + use super::scan::{count_prefix, delete_all_by_prefix, point_lookup}; 15 + use super::user_hash::UserHashMap; 16 + 17 + use tranquil_types::{CidLink, Nsid, Rkey}; 18 + 19 + pub struct RecordWrite<'a> { 20 + pub collection: &'a Nsid, 21 + pub rkey: &'a Rkey, 22 + pub cid: &'a CidLink, 23 + } 24 + 25 + pub struct RecordDelete<'a> { 26 + pub collection: &'a Nsid, 27 + pub rkey: &'a Rkey, 28 + } 29 + 30 + pub struct ListRecordsQuery<'a> { 31 + pub user_id: Uuid, 32 + pub collection: &'a Nsid, 33 + pub cursor: Option<&'a Rkey>, 34 + pub limit: usize, 35 + pub reverse: bool, 36 + pub rkey_start: Option<&'a Rkey>, 37 + pub rkey_end: Option<&'a Rkey>, 38 + } 39 + 40 + #[derive(Debug, Clone)] 41 + pub struct RecordInfo { 42 + pub rkey: Rkey, 43 + pub record_cid: CidLink, 44 + } 45 + 46 + #[derive(Debug, Clone)] 47 + pub struct FullRecordInfo { 48 + pub collection: Nsid, 49 + pub rkey: Rkey, 50 + pub record_cid: CidLink, 51 + } 52 + 53 + #[derive(Debug, Clone)] 54 + pub struct RecordWithTakedown { 55 + pub id: Uuid, 56 + pub collection: Nsid, 57 + pub rkey: Rkey, 58 + pub takedown_ref: Option<String>, 59 + } 60 + 61 + pub struct RecordOps { 62 + repo_data: Keyspace, 63 + user_hashes: Arc<UserHashMap>, 64 + } 65 + 66 + impl RecordOps { 67 + pub fn new(repo_data: Keyspace, user_hashes: Arc<UserHashMap>) -> Self { 68 + Self { 69 + repo_data, 70 + user_hashes, 71 + } 72 + } 73 + 74 + pub fn upsert_records( 75 + &self, 76 + batch: &mut fjall::OwnedWriteBatch, 77 + user_hash: UserHash, 78 + records: &[RecordWrite<'_>], 79 + ) -> Result<(), MetastoreError> { 80 + records.iter().try_for_each(|rec| { 81 + let key = record_key(user_hash, rec.collection.as_str(), rec.rkey.as_str()); 82 + let cid_bytes = cid_link_to_bytes(rec.cid)?; 83 + let existing_takedown = self 84 + .repo_data 85 + .get(key.as_slice()) 86 + .map_err(MetastoreError::Fjall)? 87 + .and_then(|raw| RecordValue::deserialize(&raw)) 88 + .and_then(|v| v.takedown_ref); 89 + let value = RecordValue { 90 + record_cid: cid_bytes, 91 + takedown_ref: existing_takedown, 92 + }; 93 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 94 + Ok::<(), MetastoreError>(()) 95 + }) 96 + } 97 + 98 + pub fn delete_records( 99 + &self, 100 + batch: &mut fjall::OwnedWriteBatch, 101 + user_hash: UserHash, 102 + records: &[RecordDelete<'_>], 103 + ) { 104 + records.iter().for_each(|rec| { 105 + let key = record_key(user_hash, rec.collection.as_str(), rec.rkey.as_str()); 106 + batch.remove(&self.repo_data, key.as_slice()); 107 + }); 108 + } 109 + 110 + pub fn delete_all_records( 111 + &self, 112 + batch: &mut fjall::OwnedWriteBatch, 113 + user_hash: UserHash, 114 + ) -> Result<(), MetastoreError> { 115 + let prefix = record_user_prefix(user_hash); 116 + delete_all_by_prefix(&self.repo_data, batch, prefix.as_slice()) 117 + } 118 + 119 + pub fn get_record_cid( 120 + &self, 121 + user_id: Uuid, 122 + collection: &Nsid, 123 + rkey: &Rkey, 124 + ) -> Result<Option<CidLink>, MetastoreError> { 125 + let user_hash = match self.user_hashes.get(&user_id) { 126 + Some(h) => h, 127 + None => return Ok(None), 128 + }; 129 + let key = record_key(user_hash, collection.as_str(), rkey.as_str()); 130 + 131 + point_lookup( 132 + &self.repo_data, 133 + key.as_slice(), 134 + RecordValue::deserialize, 135 + "invalid record value", 136 + )? 137 + .map(|v| bytes_to_cid_link(&v.record_cid)) 138 + .transpose() 139 + } 140 + 141 + pub fn list_records( 142 + &self, 143 + query: &ListRecordsQuery<'_>, 144 + ) -> Result<Vec<RecordInfo>, MetastoreError> { 145 + let user_hash = match self.user_hashes.get(&query.user_id) { 146 + Some(h) => h, 147 + None => return Ok(Vec::new()), 148 + }; 149 + 150 + let coll_str = query.collection.as_str(); 151 + let coll_prefix = record_collection_prefix(user_hash, coll_str); 152 + let coll_upper = exclusive_upper_bound(coll_prefix.as_slice()) 153 + .expect("collection prefix always contains non-0xFF bytes"); 154 + 155 + let start_key = query 156 + .rkey_start 157 + .map(|rs| record_key(user_hash, coll_str, rs.as_str())); 158 + let end_key_upper = query 159 + .rkey_end 160 + .map(|re| record_key(user_hash, coll_str, re.as_str())) 161 + .map(|ek| { 162 + exclusive_upper_bound(ek.as_slice()) 163 + .expect("record key always contains non-0xFF bytes") 164 + }); 165 + let cursor_key = query 166 + .cursor 167 + .map(|c| record_key(user_hash, coll_str, c.as_str())); 168 + 169 + let mut range_lo: &[u8] = coll_prefix.as_slice(); 170 + let mut range_hi: &[u8] = coll_upper.as_slice(); 171 + 172 + if let Some(sk) = start_key.as_ref().filter(|sk| sk.as_slice() > range_lo) { 173 + range_lo = sk.as_slice(); 174 + } 175 + 176 + if let Some(eu) = end_key_upper.as_ref().filter(|eu| eu.as_slice() < range_hi) { 177 + range_hi = eu.as_slice(); 178 + } 179 + 180 + let effective_cursor = match query.reverse { 181 + false => { 182 + let narrowed = cursor_key.as_ref().filter(|ck| ck.as_slice() > range_lo); 183 + match narrowed { 184 + Some(ck) => { 185 + range_lo = ck.as_slice(); 186 + Some(ck.as_slice()) 187 + } 188 + None => None, 189 + } 190 + } 191 + true => { 192 + if let Some(ck) = cursor_key.as_ref().filter(|ck| ck.as_slice() < range_hi) { 193 + range_hi = ck.as_slice(); 194 + } 195 + None 196 + } 197 + }; 198 + 199 + match range_lo >= range_hi { 200 + true => Ok(Vec::new()), 201 + false => match query.reverse { 202 + false => { 203 + self.list_records_forward(range_lo, range_hi, effective_cursor, query.limit) 204 + } 205 + true => self.list_records_reverse(range_lo, range_hi, query.limit), 206 + }, 207 + } 208 + } 209 + 210 + fn list_records_forward( 211 + &self, 212 + range_start: &[u8], 213 + range_end: &[u8], 214 + cursor_key: Option<&[u8]>, 215 + limit: usize, 216 + ) -> Result<Vec<RecordInfo>, MetastoreError> { 217 + self.repo_data 218 + .range(range_start..range_end) 219 + .filter_map(|guard| { 220 + let (key_bytes, val_bytes) = match guard.into_inner() { 221 + Ok(pair) => pair, 222 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 223 + }; 224 + 225 + match cursor_key { 226 + Some(ck) if key_bytes.as_ref() <= ck => None, 227 + _ => Some(decode_record_info(&key_bytes, &val_bytes)), 228 + } 229 + }) 230 + .take(limit) 231 + .collect() 232 + } 233 + 234 + fn list_records_reverse( 235 + &self, 236 + range_start: &[u8], 237 + range_end: &[u8], 238 + limit: usize, 239 + ) -> Result<Vec<RecordInfo>, MetastoreError> { 240 + self.repo_data 241 + .range(range_start..range_end) 242 + .rev() 243 + .map(|guard| { 244 + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 245 + decode_record_info(&key_bytes, &val_bytes) 246 + }) 247 + .take(limit) 248 + .collect() 249 + } 250 + 251 + pub fn get_all_records(&self, user_id: Uuid) -> Result<Vec<FullRecordInfo>, MetastoreError> { 252 + let user_hash = match self.user_hashes.get(&user_id) { 253 + Some(h) => h, 254 + None => return Ok(Vec::new()), 255 + }; 256 + let prefix = record_user_prefix(user_hash); 257 + 258 + self.repo_data 259 + .prefix(prefix.as_slice()) 260 + .map(|guard| { 261 + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 262 + decode_full_record_info(&key_bytes, &val_bytes) 263 + }) 264 + .collect() 265 + } 266 + 267 + pub fn list_collections(&self, user_id: Uuid) -> Result<Vec<Nsid>, MetastoreError> { 268 + let user_hash = match self.user_hashes.get(&user_id) { 269 + Some(h) => h, 270 + None => return Ok(Vec::new()), 271 + }; 272 + let user_pfx = record_user_prefix(user_hash); 273 + let user_upper = exclusive_upper_bound(user_pfx.as_slice()) 274 + .expect("user prefix always contains non-0xFF bytes"); 275 + 276 + let mut collections: Vec<Nsid> = Vec::new(); 277 + let mut seek_from: SmallVec<[u8; 128]> = user_pfx.clone(); 278 + 279 + loop { 280 + let entry = self 281 + .repo_data 282 + .range(seek_from.as_slice()..user_upper.as_slice()) 283 + .next(); 284 + let guard = match entry { 285 + Some(g) => g, 286 + None => break, 287 + }; 288 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 289 + let collection = parse_record_key_collection(&key_bytes) 290 + .ok_or(MetastoreError::CorruptData("invalid record key"))?; 291 + let coll_prefix = record_collection_prefix(user_hash, &collection); 292 + seek_from = exclusive_upper_bound(coll_prefix.as_slice()) 293 + .expect("collection prefix always contains non-0xFF bytes"); 294 + collections.push(Nsid::from(collection)); 295 + } 296 + 297 + Ok(collections) 298 + } 299 + 300 + pub fn count_records(&self, user_id: Uuid) -> Result<i64, MetastoreError> { 301 + let user_hash = match self.user_hashes.get(&user_id) { 302 + Some(h) => h, 303 + None => return Ok(0), 304 + }; 305 + let prefix = record_user_prefix(user_hash); 306 + count_prefix(&self.repo_data, prefix.as_slice()) 307 + } 308 + 309 + pub fn count_all_records(&self) -> Result<i64, MetastoreError> { 310 + let prefix = records_prefix(); 311 + count_prefix(&self.repo_data, prefix.as_slice()) 312 + } 313 + 314 + pub fn get_record_by_cid( 315 + &self, 316 + cid: &CidLink, 317 + scope_user: Option<Uuid>, 318 + ) -> Result<Option<RecordWithTakedown>, MetastoreError> { 319 + let target_bytes = cid_link_to_bytes(cid)?; 320 + let prefix = match scope_user { 321 + Some(uid) => match self.user_hashes.get(&uid) { 322 + Some(hash) => record_user_prefix(hash), 323 + None => return Ok(None), 324 + }, 325 + None => records_prefix(), 326 + }; 327 + 328 + self.repo_data 329 + .prefix(prefix.as_slice()) 330 + .find_map(|guard| { 331 + let (key_bytes, val_bytes) = match guard.into_inner() { 332 + Ok(pair) => pair, 333 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 334 + }; 335 + let value = match RecordValue::deserialize(&val_bytes) { 336 + Some(v) => v, 337 + None => return Some(Err(MetastoreError::CorruptData("invalid record value"))), 338 + }; 339 + 340 + match value.record_cid == target_bytes { 341 + true => { 342 + let (coll_str, rkey_str) = match parse_record_key_fields(&key_bytes) { 343 + Some(pair) => pair, 344 + None => { 345 + return Some(Err(MetastoreError::CorruptData( 346 + "invalid record key", 347 + ))); 348 + } 349 + }; 350 + let user_hash = match parse_record_key_user_hash(&key_bytes) { 351 + Some(h) => h, 352 + None => { 353 + return Some(Err(MetastoreError::CorruptData( 354 + "invalid record key", 355 + ))); 356 + } 357 + }; 358 + let user_id = match self.user_hashes.get_uuid(&user_hash) { 359 + Some(id) => id, 360 + None => { 361 + return Some(Err(MetastoreError::CorruptData( 362 + "record user_hash has no reverse mapping", 363 + ))); 364 + } 365 + }; 366 + Some(Ok(RecordWithTakedown { 367 + id: user_id, 368 + collection: Nsid::from(coll_str), 369 + rkey: Rkey::from(rkey_str), 370 + takedown_ref: value.takedown_ref, 371 + })) 372 + } 373 + false => None, 374 + } 375 + }) 376 + .transpose() 377 + } 378 + 379 + pub fn set_record_takedown( 380 + &self, 381 + db: &fjall::Database, 382 + cid: &CidLink, 383 + takedown_ref: Option<&str>, 384 + scope_user: Option<Uuid>, 385 + ) -> Result<(), MetastoreError> { 386 + let target_bytes = cid_link_to_bytes(cid)?; 387 + let prefix = match scope_user { 388 + Some(uid) => match self.user_hashes.get(&uid) { 389 + Some(hash) => record_user_prefix(hash), 390 + None => return Ok(()), 391 + }, 392 + None => records_prefix(), 393 + }; 394 + 395 + let found = self 396 + .repo_data 397 + .prefix(prefix.as_slice()) 398 + .find_map(|guard| { 399 + let (key_bytes, val_bytes) = match guard.into_inner() { 400 + Ok(pair) => pair, 401 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 402 + }; 403 + let value = match RecordValue::deserialize(&val_bytes) { 404 + Some(v) => v, 405 + None => return Some(Err(MetastoreError::CorruptData("invalid record value"))), 406 + }; 407 + match value.record_cid == target_bytes { 408 + true => Some(Ok((key_bytes.to_vec(), value))), 409 + false => None, 410 + } 411 + }) 412 + .transpose()?; 413 + 414 + match found { 415 + Some((key, mut value)) => { 416 + value.takedown_ref = takedown_ref.map(str::to_string); 417 + let mut batch = db.batch(); 418 + batch.insert(&self.repo_data, &key, value.serialize()); 419 + batch.commit().map_err(MetastoreError::Fjall) 420 + } 421 + None => Ok(()), 422 + } 423 + } 424 + } 425 + 426 + fn decode_record_info(key_bytes: &[u8], val_bytes: &[u8]) -> Result<RecordInfo, MetastoreError> { 427 + let value = RecordValue::deserialize(val_bytes) 428 + .ok_or(MetastoreError::CorruptData("invalid record value"))?; 429 + let (_collection, rkey) = parse_record_key_fields(key_bytes) 430 + .ok_or(MetastoreError::CorruptData("invalid record key"))?; 431 + let cid = bytes_to_cid_link(&value.record_cid)?; 432 + Ok(RecordInfo { 433 + rkey: Rkey::from(rkey), 434 + record_cid: cid, 435 + }) 436 + } 437 + 438 + fn decode_full_record_info( 439 + key_bytes: &[u8], 440 + val_bytes: &[u8], 441 + ) -> Result<FullRecordInfo, MetastoreError> { 442 + let value = RecordValue::deserialize(val_bytes) 443 + .ok_or(MetastoreError::CorruptData("invalid record value"))?; 444 + let (collection, rkey) = parse_record_key_fields(key_bytes) 445 + .ok_or(MetastoreError::CorruptData("invalid record key"))?; 446 + let cid = bytes_to_cid_link(&value.record_cid)?; 447 + Ok(FullRecordInfo { 448 + collection: Nsid::from(collection), 449 + rkey: Rkey::from(rkey), 450 + record_cid: cid, 451 + }) 452 + } 453 + 454 + fn parse_record_key_fields(key_bytes: &[u8]) -> Option<(String, String)> { 455 + let mut reader = KeyReader::new(key_bytes); 456 + let _tag = reader.tag()?; 457 + let _user_hash = reader.u64()?; 458 + let collection = reader.string()?; 459 + let rkey = reader.string()?; 460 + Some((collection, rkey)) 461 + } 462 + 463 + fn parse_record_key_collection(key_bytes: &[u8]) -> Option<String> { 464 + let mut reader = KeyReader::new(key_bytes); 465 + let _tag = reader.tag()?; 466 + let _user_hash = reader.u64()?; 467 + reader.string() 468 + } 469 + 470 + fn parse_record_key_user_hash(key_bytes: &[u8]) -> Option<UserHash> { 471 + let mut reader = KeyReader::new(key_bytes); 472 + let _tag = reader.tag()?; 473 + let hash = reader.u64()?; 474 + Some(UserHash::from_raw(hash)) 475 + } 476 + 477 + #[cfg(test)] 478 + mod tests { 479 + use super::*; 480 + use crate::metastore::{Metastore, MetastoreConfig}; 481 + 482 + fn test_config() -> MetastoreConfig { 483 + MetastoreConfig { 484 + cache_size_bytes: 64 * 1024 * 1024, 485 + } 486 + } 487 + 488 + fn test_cid_link(seed: u8) -> CidLink { 489 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 490 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 491 + let c = cid::Cid::new_v1(0x71, mh); 492 + CidLink::from_cid(&c) 493 + } 494 + 495 + fn open_fresh() -> (tempfile::TempDir, Metastore) { 496 + let dir = tempfile::TempDir::new().unwrap(); 497 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 498 + (dir, ms) 499 + } 500 + 501 + fn test_did(name: &str) -> tranquil_types::Did { 502 + tranquil_types::Did::from(format!("did:plc:{name}")) 503 + } 504 + 505 + fn test_handle(name: &str) -> tranquil_types::Handle { 506 + tranquil_types::Handle::from(format!("{name}.test.invalid")) 507 + } 508 + 509 + fn setup_user(ms: &Metastore) -> (Uuid, super::super::keys::UserHash) { 510 + let user_id = Uuid::new_v4(); 511 + let did = test_did("testuser"); 512 + let handle = test_handle("testuser"); 513 + let cid = test_cid_link(0); 514 + ms.repo_ops() 515 + .create_repo(ms.database(), user_id, &did, &handle, &cid, "rev0") 516 + .unwrap(); 517 + let user_hash = ms.user_hashes().get(&user_id).unwrap(); 518 + (user_id, user_hash) 519 + } 520 + 521 + fn rw<'a>(collection: &'a Nsid, rkey: &'a Rkey, cid: &'a CidLink) -> RecordWrite<'a> { 522 + RecordWrite { 523 + collection, 524 + rkey, 525 + cid, 526 + } 527 + } 528 + 529 + fn rd<'a>(collection: &'a Nsid, rkey: &'a Rkey) -> RecordDelete<'a> { 530 + RecordDelete { collection, rkey } 531 + } 532 + 533 + fn lrq<'a>( 534 + user_id: Uuid, 535 + collection: &'a Nsid, 536 + cursor: Option<&'a Rkey>, 537 + limit: usize, 538 + reverse: bool, 539 + rkey_start: Option<&'a Rkey>, 540 + rkey_end: Option<&'a Rkey>, 541 + ) -> ListRecordsQuery<'a> { 542 + ListRecordsQuery { 543 + user_id, 544 + collection, 545 + cursor, 546 + limit, 547 + reverse, 548 + rkey_start, 549 + rkey_end, 550 + } 551 + } 552 + 553 + #[test] 554 + fn upsert_and_get_record() { 555 + let (_dir, ms) = open_fresh(); 556 + let (user_id, user_hash) = setup_user(&ms); 557 + let rec_ops = ms.record_ops(); 558 + 559 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 560 + let rkey = Rkey::from("3k2abcd".to_string()); 561 + let cid = test_cid_link(1); 562 + 563 + let mut batch = ms.database().batch(); 564 + rec_ops 565 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 566 + .unwrap(); 567 + batch.commit().unwrap(); 568 + 569 + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); 570 + assert_eq!(found, Some(cid)); 571 + } 572 + 573 + #[test] 574 + fn get_record_returns_none_for_missing() { 575 + let (_dir, ms) = open_fresh(); 576 + let (user_id, _) = setup_user(&ms); 577 + let rec_ops = ms.record_ops(); 578 + 579 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 580 + let rkey = Rkey::from("nonexistent".to_string()); 581 + assert!( 582 + rec_ops 583 + .get_record_cid(user_id, &collection, &rkey) 584 + .unwrap() 585 + .is_none() 586 + ); 587 + } 588 + 589 + #[test] 590 + fn upsert_overwrites_existing() { 591 + let (_dir, ms) = open_fresh(); 592 + let (user_id, user_hash) = setup_user(&ms); 593 + let rec_ops = ms.record_ops(); 594 + 595 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 596 + let rkey = Rkey::from("3k2abcd".to_string()); 597 + let cid1 = test_cid_link(1); 598 + let cid2 = test_cid_link(2); 599 + 600 + let mut batch = ms.database().batch(); 601 + rec_ops 602 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid1)]) 603 + .unwrap(); 604 + batch.commit().unwrap(); 605 + 606 + let mut batch = ms.database().batch(); 607 + rec_ops 608 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid2)]) 609 + .unwrap(); 610 + batch.commit().unwrap(); 611 + 612 + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); 613 + assert_eq!(found, Some(cid2)); 614 + } 615 + 616 + #[test] 617 + fn delete_records_removes_entries() { 618 + let (_dir, ms) = open_fresh(); 619 + let (user_id, user_hash) = setup_user(&ms); 620 + let rec_ops = ms.record_ops(); 621 + 622 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 623 + let rkey = Rkey::from("3k2abcd".to_string()); 624 + let cid = test_cid_link(1); 625 + 626 + let mut batch = ms.database().batch(); 627 + rec_ops 628 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 629 + .unwrap(); 630 + batch.commit().unwrap(); 631 + 632 + let mut batch = ms.database().batch(); 633 + rec_ops.delete_records(&mut batch, user_hash, &[rd(&collection, &rkey)]); 634 + batch.commit().unwrap(); 635 + 636 + assert!( 637 + rec_ops 638 + .get_record_cid(user_id, &collection, &rkey) 639 + .unwrap() 640 + .is_none() 641 + ); 642 + } 643 + 644 + #[test] 645 + fn delete_all_records_clears_user() { 646 + let (_dir, ms) = open_fresh(); 647 + let (user_id, user_hash) = setup_user(&ms); 648 + let rec_ops = ms.record_ops(); 649 + 650 + let coll1 = Nsid::from("app.bsky.feed.post".to_string()); 651 + let coll2 = Nsid::from("app.bsky.feed.like".to_string()); 652 + let rkey_a = Rkey::from("a".to_string()); 653 + let rkey_b = Rkey::from("b".to_string()); 654 + let cid1 = test_cid_link(1); 655 + let cid2 = test_cid_link(2); 656 + 657 + let mut batch = ms.database().batch(); 658 + rec_ops 659 + .upsert_records( 660 + &mut batch, 661 + user_hash, 662 + &[rw(&coll1, &rkey_a, &cid1), rw(&coll2, &rkey_b, &cid2)], 663 + ) 664 + .unwrap(); 665 + batch.commit().unwrap(); 666 + 667 + assert_eq!(rec_ops.count_records(user_id).unwrap(), 2); 668 + 669 + let mut batch = ms.database().batch(); 670 + rec_ops.delete_all_records(&mut batch, user_hash).unwrap(); 671 + batch.commit().unwrap(); 672 + 673 + assert_eq!(rec_ops.count_records(user_id).unwrap(), 0); 674 + } 675 + 676 + #[test] 677 + fn list_records_forward_with_limit() { 678 + let (_dir, ms) = open_fresh(); 679 + let (user_id, user_hash) = setup_user(&ms); 680 + let rec_ops = ms.record_ops(); 681 + 682 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 683 + let rkeys: Vec<Rkey> = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 684 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(i + 1)).collect(); 685 + 686 + let writes: Vec<RecordWrite<'_>> = rkeys 687 + .iter() 688 + .zip(cids.iter()) 689 + .map(|(rk, c)| rw(&collection, rk, c)) 690 + .collect(); 691 + 692 + let mut batch = ms.database().batch(); 693 + rec_ops 694 + .upsert_records(&mut batch, user_hash, &writes) 695 + .unwrap(); 696 + batch.commit().unwrap(); 697 + 698 + let results = rec_ops 699 + .list_records(&lrq(user_id, &collection, None, 3, false, None, None)) 700 + .unwrap(); 701 + assert_eq!(results.len(), 3); 702 + assert_eq!(results[0].rkey.as_str(), "rkey000"); 703 + assert_eq!(results[1].rkey.as_str(), "rkey001"); 704 + assert_eq!(results[2].rkey.as_str(), "rkey002"); 705 + } 706 + 707 + #[test] 708 + fn list_records_with_cursor() { 709 + let (_dir, ms) = open_fresh(); 710 + let (user_id, user_hash) = setup_user(&ms); 711 + let rec_ops = ms.record_ops(); 712 + 713 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 714 + let rkeys: Vec<Rkey> = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 715 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(i + 1)).collect(); 716 + 717 + let writes: Vec<RecordWrite<'_>> = rkeys 718 + .iter() 719 + .zip(cids.iter()) 720 + .map(|(rk, c)| rw(&collection, rk, c)) 721 + .collect(); 722 + 723 + let mut batch = ms.database().batch(); 724 + rec_ops 725 + .upsert_records(&mut batch, user_hash, &writes) 726 + .unwrap(); 727 + batch.commit().unwrap(); 728 + 729 + let cursor = Rkey::from("rkey001".to_string()); 730 + let results = rec_ops 731 + .list_records(&lrq( 732 + user_id, 733 + &collection, 734 + Some(&cursor), 735 + 10, 736 + false, 737 + None, 738 + None, 739 + )) 740 + .unwrap(); 741 + assert_eq!(results.len(), 3); 742 + assert_eq!(results[0].rkey.as_str(), "rkey002"); 743 + assert_eq!(results[1].rkey.as_str(), "rkey003"); 744 + assert_eq!(results[2].rkey.as_str(), "rkey004"); 745 + } 746 + 747 + #[test] 748 + fn list_records_reverse() { 749 + let (_dir, ms) = open_fresh(); 750 + let (user_id, user_hash) = setup_user(&ms); 751 + let rec_ops = ms.record_ops(); 752 + 753 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 754 + let rkeys: Vec<Rkey> = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 755 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(i + 1)).collect(); 756 + 757 + let writes: Vec<RecordWrite<'_>> = rkeys 758 + .iter() 759 + .zip(cids.iter()) 760 + .map(|(rk, c)| rw(&collection, rk, c)) 761 + .collect(); 762 + 763 + let mut batch = ms.database().batch(); 764 + rec_ops 765 + .upsert_records(&mut batch, user_hash, &writes) 766 + .unwrap(); 767 + batch.commit().unwrap(); 768 + 769 + let results = rec_ops 770 + .list_records(&lrq(user_id, &collection, None, 3, true, None, None)) 771 + .unwrap(); 772 + assert_eq!(results.len(), 3); 773 + assert_eq!(results[0].rkey.as_str(), "rkey004"); 774 + assert_eq!(results[1].rkey.as_str(), "rkey003"); 775 + assert_eq!(results[2].rkey.as_str(), "rkey002"); 776 + } 777 + 778 + #[test] 779 + fn list_records_reverse_with_cursor() { 780 + let (_dir, ms) = open_fresh(); 781 + let (user_id, user_hash) = setup_user(&ms); 782 + let rec_ops = ms.record_ops(); 783 + 784 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 785 + let rkeys: Vec<Rkey> = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 786 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(i + 1)).collect(); 787 + 788 + let writes: Vec<RecordWrite<'_>> = rkeys 789 + .iter() 790 + .zip(cids.iter()) 791 + .map(|(rk, c)| rw(&collection, rk, c)) 792 + .collect(); 793 + 794 + let mut batch = ms.database().batch(); 795 + rec_ops 796 + .upsert_records(&mut batch, user_hash, &writes) 797 + .unwrap(); 798 + batch.commit().unwrap(); 799 + 800 + let cursor = Rkey::from("rkey003".to_string()); 801 + let results = rec_ops 802 + .list_records(&lrq( 803 + user_id, 804 + &collection, 805 + Some(&cursor), 806 + 10, 807 + true, 808 + None, 809 + None, 810 + )) 811 + .unwrap(); 812 + assert_eq!(results.len(), 3); 813 + assert_eq!(results[0].rkey.as_str(), "rkey002"); 814 + assert_eq!(results[1].rkey.as_str(), "rkey001"); 815 + assert_eq!(results[2].rkey.as_str(), "rkey000"); 816 + } 817 + 818 + #[test] 819 + fn list_records_rkey_range_bounds() { 820 + let (_dir, ms) = open_fresh(); 821 + let (user_id, user_hash) = setup_user(&ms); 822 + let rec_ops = ms.record_ops(); 823 + 824 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 825 + let rkeys: Vec<Rkey> = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 826 + let cids: Vec<CidLink> = (0..10).map(|i| test_cid_link(i + 1)).collect(); 827 + 828 + let writes: Vec<RecordWrite<'_>> = rkeys 829 + .iter() 830 + .zip(cids.iter()) 831 + .map(|(rk, c)| rw(&collection, rk, c)) 832 + .collect(); 833 + 834 + let mut batch = ms.database().batch(); 835 + rec_ops 836 + .upsert_records(&mut batch, user_hash, &writes) 837 + .unwrap(); 838 + batch.commit().unwrap(); 839 + 840 + let rkey_start = Rkey::from("rkey003".to_string()); 841 + let rkey_end = Rkey::from("rkey006".to_string()); 842 + let results = rec_ops 843 + .list_records(&lrq( 844 + user_id, 845 + &collection, 846 + None, 847 + 100, 848 + false, 849 + Some(&rkey_start), 850 + Some(&rkey_end), 851 + )) 852 + .unwrap(); 853 + assert_eq!(results.len(), 4); 854 + assert_eq!(results[0].rkey.as_str(), "rkey003"); 855 + assert_eq!(results[3].rkey.as_str(), "rkey006"); 856 + } 857 + 858 + #[test] 859 + fn get_all_records_across_collections() { 860 + let (_dir, ms) = open_fresh(); 861 + let (user_id, user_hash) = setup_user(&ms); 862 + let rec_ops = ms.record_ops(); 863 + 864 + let coll1 = Nsid::from("app.bsky.feed.like".to_string()); 865 + let coll2 = Nsid::from("app.bsky.feed.post".to_string()); 866 + let rkey_a = Rkey::from("a".to_string()); 867 + let rkey_b = Rkey::from("b".to_string()); 868 + let rkey_c = Rkey::from("c".to_string()); 869 + let cid1 = test_cid_link(1); 870 + let cid2 = test_cid_link(2); 871 + let cid3 = test_cid_link(3); 872 + 873 + let mut batch = ms.database().batch(); 874 + rec_ops 875 + .upsert_records( 876 + &mut batch, 877 + user_hash, 878 + &[ 879 + rw(&coll1, &rkey_a, &cid1), 880 + rw(&coll2, &rkey_b, &cid2), 881 + rw(&coll1, &rkey_c, &cid3), 882 + ], 883 + ) 884 + .unwrap(); 885 + batch.commit().unwrap(); 886 + 887 + let all = rec_ops.get_all_records(user_id).unwrap(); 888 + assert_eq!(all.len(), 3); 889 + assert_eq!(all[0].collection.as_str(), "app.bsky.feed.like"); 890 + assert_eq!(all[1].collection.as_str(), "app.bsky.feed.like"); 891 + assert_eq!(all[2].collection.as_str(), "app.bsky.feed.post"); 892 + } 893 + 894 + #[test] 895 + fn list_collections_returns_distinct_sorted() { 896 + let (_dir, ms) = open_fresh(); 897 + let (user_id, user_hash) = setup_user(&ms); 898 + let rec_ops = ms.record_ops(); 899 + 900 + let coll1 = Nsid::from("app.bsky.feed.like".to_string()); 901 + let coll2 = Nsid::from("app.bsky.feed.post".to_string()); 902 + let coll3 = Nsid::from("app.bsky.graph.follow".to_string()); 903 + let rkeys: Vec<Rkey> = ["a", "b", "c", "d", "e"] 904 + .iter() 905 + .map(|s| Rkey::from(s.to_string())) 906 + .collect(); 907 + let cids: Vec<CidLink> = (1..=5).map(test_cid_link).collect(); 908 + 909 + let mut batch = ms.database().batch(); 910 + rec_ops 911 + .upsert_records( 912 + &mut batch, 913 + user_hash, 914 + &[ 915 + rw(&coll1, &rkeys[0], &cids[0]), 916 + rw(&coll2, &rkeys[1], &cids[1]), 917 + rw(&coll1, &rkeys[2], &cids[2]), 918 + rw(&coll3, &rkeys[3], &cids[3]), 919 + rw(&coll2, &rkeys[4], &cids[4]), 920 + ], 921 + ) 922 + .unwrap(); 923 + batch.commit().unwrap(); 924 + 925 + let collections = rec_ops.list_collections(user_id).unwrap(); 926 + assert_eq!(collections.len(), 3); 927 + assert_eq!(collections[0].as_str(), "app.bsky.feed.like"); 928 + assert_eq!(collections[1].as_str(), "app.bsky.feed.post"); 929 + assert_eq!(collections[2].as_str(), "app.bsky.graph.follow"); 930 + } 931 + 932 + #[test] 933 + fn count_records_and_count_all() { 934 + let (_dir, ms) = open_fresh(); 935 + let (user_id, user_hash) = setup_user(&ms); 936 + let rec_ops = ms.record_ops(); 937 + 938 + assert_eq!(rec_ops.count_records(user_id).unwrap(), 0); 939 + assert_eq!(rec_ops.count_all_records().unwrap(), 0); 940 + 941 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 942 + let rkey_a = Rkey::from("a".to_string()); 943 + let rkey_b = Rkey::from("b".to_string()); 944 + let cid1 = test_cid_link(1); 945 + let cid2 = test_cid_link(2); 946 + 947 + let mut batch = ms.database().batch(); 948 + rec_ops 949 + .upsert_records( 950 + &mut batch, 951 + user_hash, 952 + &[ 953 + rw(&collection, &rkey_a, &cid1), 954 + rw(&collection, &rkey_b, &cid2), 955 + ], 956 + ) 957 + .unwrap(); 958 + batch.commit().unwrap(); 959 + 960 + assert_eq!(rec_ops.count_records(user_id).unwrap(), 2); 961 + assert_eq!(rec_ops.count_all_records().unwrap(), 2); 962 + } 963 + 964 + #[test] 965 + fn records_isolated_between_users() { 966 + let (_dir, ms) = open_fresh(); 967 + let rec_ops = ms.record_ops(); 968 + 969 + let user1 = Uuid::new_v4(); 970 + let did1 = test_did("user1"); 971 + let handle1 = test_handle("user1"); 972 + ms.repo_ops() 973 + .create_repo( 974 + ms.database(), 975 + user1, 976 + &did1, 977 + &handle1, 978 + &test_cid_link(0), 979 + "r", 980 + ) 981 + .unwrap(); 982 + let hash1 = ms.user_hashes().get(&user1).unwrap(); 983 + 984 + let user2 = Uuid::new_v4(); 985 + let did2 = test_did("user2"); 986 + let handle2 = test_handle("user2"); 987 + ms.repo_ops() 988 + .create_repo( 989 + ms.database(), 990 + user2, 991 + &did2, 992 + &handle2, 993 + &test_cid_link(0), 994 + "r", 995 + ) 996 + .unwrap(); 997 + let hash2 = ms.user_hashes().get(&user2).unwrap(); 998 + 999 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1000 + let rkey_a = Rkey::from("a".to_string()); 1001 + let rkey_b = Rkey::from("b".to_string()); 1002 + let cid1 = test_cid_link(1); 1003 + let cid2 = test_cid_link(2); 1004 + 1005 + let mut batch = ms.database().batch(); 1006 + rec_ops 1007 + .upsert_records(&mut batch, hash1, &[rw(&collection, &rkey_a, &cid1)]) 1008 + .unwrap(); 1009 + rec_ops 1010 + .upsert_records(&mut batch, hash2, &[rw(&collection, &rkey_b, &cid2)]) 1011 + .unwrap(); 1012 + batch.commit().unwrap(); 1013 + 1014 + assert_eq!(rec_ops.count_records(user1).unwrap(), 1); 1015 + assert_eq!(rec_ops.count_records(user2).unwrap(), 1); 1016 + assert_eq!(rec_ops.count_all_records().unwrap(), 2); 1017 + } 1018 + 1019 + #[test] 1020 + fn get_record_by_cid_finds_match() { 1021 + let (_dir, ms) = open_fresh(); 1022 + let (user_id, user_hash) = setup_user(&ms); 1023 + let rec_ops = ms.record_ops(); 1024 + 1025 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1026 + let rkey = Rkey::from("r1".to_string()); 1027 + let cid = test_cid_link(42); 1028 + 1029 + let mut batch = ms.database().batch(); 1030 + rec_ops 1031 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 1032 + .unwrap(); 1033 + batch.commit().unwrap(); 1034 + 1035 + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); 1036 + assert_eq!(found.id, user_id); 1037 + assert_eq!(found.collection.as_str(), "app.bsky.feed.post"); 1038 + assert_eq!(found.rkey.as_str(), "r1"); 1039 + assert!(found.takedown_ref.is_none()); 1040 + } 1041 + 1042 + #[test] 1043 + fn get_record_by_cid_returns_none_for_missing() { 1044 + let (_dir, ms) = open_fresh(); 1045 + let rec_ops = ms.record_ops(); 1046 + assert!( 1047 + rec_ops 1048 + .get_record_by_cid(&test_cid_link(99), None) 1049 + .unwrap() 1050 + .is_none() 1051 + ); 1052 + } 1053 + 1054 + #[test] 1055 + fn get_record_by_cid_scoped_to_user() { 1056 + let (_dir, ms) = open_fresh(); 1057 + let (user_id, user_hash) = setup_user(&ms); 1058 + let rec_ops = ms.record_ops(); 1059 + 1060 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1061 + let rkey = Rkey::from("r1".to_string()); 1062 + let cid = test_cid_link(42); 1063 + 1064 + let mut batch = ms.database().batch(); 1065 + rec_ops 1066 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 1067 + .unwrap(); 1068 + batch.commit().unwrap(); 1069 + 1070 + let found = rec_ops 1071 + .get_record_by_cid(&cid, Some(user_id)) 1072 + .unwrap() 1073 + .unwrap(); 1074 + assert_eq!(found.id, user_id); 1075 + 1076 + let other_user = Uuid::new_v4(); 1077 + assert!( 1078 + rec_ops 1079 + .get_record_by_cid(&cid, Some(other_user)) 1080 + .unwrap() 1081 + .is_none() 1082 + ); 1083 + } 1084 + 1085 + #[test] 1086 + fn set_and_get_takedown() { 1087 + let (_dir, ms) = open_fresh(); 1088 + let (user_id, user_hash) = setup_user(&ms); 1089 + let rec_ops = ms.record_ops(); 1090 + 1091 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1092 + let rkey = Rkey::from("r1".to_string()); 1093 + let cid = test_cid_link(42); 1094 + 1095 + let mut batch = ms.database().batch(); 1096 + rec_ops 1097 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 1098 + .unwrap(); 1099 + batch.commit().unwrap(); 1100 + 1101 + rec_ops 1102 + .set_record_takedown(ms.database(), &cid, Some("DMCA-789"), None) 1103 + .unwrap(); 1104 + 1105 + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); 1106 + assert_eq!(found.id, user_id); 1107 + assert_eq!(found.takedown_ref.as_deref(), Some("DMCA-789")); 1108 + 1109 + rec_ops 1110 + .set_record_takedown(ms.database(), &cid, None, None) 1111 + .unwrap(); 1112 + 1113 + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); 1114 + assert!(found.takedown_ref.is_none()); 1115 + } 1116 + 1117 + #[test] 1118 + fn upsert_preserves_existing_takedown() { 1119 + let (_dir, ms) = open_fresh(); 1120 + let (_user_id, user_hash) = setup_user(&ms); 1121 + let rec_ops = ms.record_ops(); 1122 + 1123 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1124 + let rkey = Rkey::from("r1".to_string()); 1125 + let cid1 = test_cid_link(1); 1126 + let cid2 = test_cid_link(2); 1127 + 1128 + let mut batch = ms.database().batch(); 1129 + rec_ops 1130 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid1)]) 1131 + .unwrap(); 1132 + batch.commit().unwrap(); 1133 + 1134 + rec_ops 1135 + .set_record_takedown(ms.database(), &cid1, Some("DMCA-999"), None) 1136 + .unwrap(); 1137 + 1138 + let mut batch = ms.database().batch(); 1139 + rec_ops 1140 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid2)]) 1141 + .unwrap(); 1142 + batch.commit().unwrap(); 1143 + 1144 + let found = rec_ops.get_record_by_cid(&cid2, None).unwrap().unwrap(); 1145 + assert_eq!(found.takedown_ref.as_deref(), Some("DMCA-999")); 1146 + } 1147 + 1148 + #[test] 1149 + fn records_survive_reopen() { 1150 + let dir = tempfile::TempDir::new().unwrap(); 1151 + let user_id; 1152 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1153 + let rkey = Rkey::from("durable".to_string()); 1154 + let cid = test_cid_link(77); 1155 + 1156 + { 1157 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 1158 + user_id = Uuid::new_v4(); 1159 + let did = test_did("persist"); 1160 + let handle = test_handle("persist"); 1161 + ms.repo_ops() 1162 + .create_repo( 1163 + ms.database(), 1164 + user_id, 1165 + &did, 1166 + &handle, 1167 + &test_cid_link(0), 1168 + "r", 1169 + ) 1170 + .unwrap(); 1171 + let user_hash = ms.user_hashes().get(&user_id).unwrap(); 1172 + 1173 + let rec_ops = ms.record_ops(); 1174 + let mut batch = ms.database().batch(); 1175 + rec_ops 1176 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 1177 + .unwrap(); 1178 + batch.commit().unwrap(); 1179 + ms.persist().unwrap(); 1180 + } 1181 + 1182 + { 1183 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 1184 + let rec_ops = ms.record_ops(); 1185 + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); 1186 + assert_eq!(found, Some(cid)); 1187 + } 1188 + } 1189 + 1190 + #[test] 1191 + fn rkey_ordering_matches_lexicographic() { 1192 + let (_dir, ms) = open_fresh(); 1193 + let (user_id, user_hash) = setup_user(&ms); 1194 + let rec_ops = ms.record_ops(); 1195 + 1196 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1197 + let rkeys_unordered = ["zebra", "apple", "mango", "banana"]; 1198 + 1199 + let rkeys: Vec<Rkey> = rkeys_unordered 1200 + .iter() 1201 + .map(|rk| Rkey::from(rk.to_string())) 1202 + .collect(); 1203 + let cids: Vec<CidLink> = (0..rkeys.len()) 1204 + .map(|i| test_cid_link(i as u8 + 1)) 1205 + .collect(); 1206 + 1207 + let writes: Vec<RecordWrite<'_>> = rkeys 1208 + .iter() 1209 + .zip(cids.iter()) 1210 + .map(|(rk, c)| rw(&collection, rk, c)) 1211 + .collect(); 1212 + 1213 + let mut batch = ms.database().batch(); 1214 + rec_ops 1215 + .upsert_records(&mut batch, user_hash, &writes) 1216 + .unwrap(); 1217 + batch.commit().unwrap(); 1218 + 1219 + let results = rec_ops 1220 + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) 1221 + .unwrap(); 1222 + let result_rkeys: Vec<&str> = results.iter().map(|r| r.rkey.as_str()).collect(); 1223 + assert_eq!(result_rkeys, ["apple", "banana", "mango", "zebra"]); 1224 + } 1225 + 1226 + #[test] 1227 + fn record_with_empty_rkey() { 1228 + let (_dir, ms) = open_fresh(); 1229 + let (user_id, user_hash) = setup_user(&ms); 1230 + let rec_ops = ms.record_ops(); 1231 + 1232 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1233 + let rkey = Rkey::from(String::new()); 1234 + let cid = test_cid_link(1); 1235 + 1236 + let mut batch = ms.database().batch(); 1237 + rec_ops 1238 + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) 1239 + .unwrap(); 1240 + batch.commit().unwrap(); 1241 + 1242 + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); 1243 + assert_eq!(found, Some(cid)); 1244 + 1245 + let results = rec_ops 1246 + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) 1247 + .unwrap(); 1248 + assert_eq!(results.len(), 1); 1249 + } 1250 + 1251 + #[test] 1252 + fn record_with_null_bytes_in_rkey() { 1253 + let (_dir, ms) = open_fresh(); 1254 + let (user_id, user_hash) = setup_user(&ms); 1255 + let rec_ops = ms.record_ops(); 1256 + 1257 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1258 + let rkey_with_null = Rkey::from("abc\x00def".to_string()); 1259 + let rkey_plain = Rkey::from("abc".to_string()); 1260 + let cid1 = test_cid_link(1); 1261 + let cid2 = test_cid_link(2); 1262 + 1263 + let mut batch = ms.database().batch(); 1264 + rec_ops 1265 + .upsert_records( 1266 + &mut batch, 1267 + user_hash, 1268 + &[ 1269 + rw(&collection, &rkey_with_null, &cid1), 1270 + rw(&collection, &rkey_plain, &cid2), 1271 + ], 1272 + ) 1273 + .unwrap(); 1274 + batch.commit().unwrap(); 1275 + 1276 + let found1 = rec_ops 1277 + .get_record_cid(user_id, &collection, &rkey_with_null) 1278 + .unwrap(); 1279 + assert_eq!(found1, Some(cid1)); 1280 + 1281 + let found2 = rec_ops 1282 + .get_record_cid(user_id, &collection, &rkey_plain) 1283 + .unwrap(); 1284 + assert_eq!(found2, Some(cid2)); 1285 + 1286 + let results = rec_ops 1287 + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) 1288 + .unwrap(); 1289 + assert_eq!(results.len(), 2); 1290 + assert_eq!(results[0].rkey.as_str(), "abc"); 1291 + assert_eq!(results[1].rkey.as_str(), "abc\x00def"); 1292 + } 1293 + 1294 + #[test] 1295 + fn record_with_null_bytes_in_collection() { 1296 + let (_dir, ms) = open_fresh(); 1297 + let (user_id, user_hash) = setup_user(&ms); 1298 + let rec_ops = ms.record_ops(); 1299 + 1300 + let coll_normal = Nsid::from("app.bsky.feed.post".to_string()); 1301 + let coll_with_null = Nsid::from("app.bsky.feed.post\x00extra".to_string()); 1302 + let rkey = Rkey::from("r1".to_string()); 1303 + let cid1 = test_cid_link(1); 1304 + let cid2 = test_cid_link(2); 1305 + 1306 + let mut batch = ms.database().batch(); 1307 + rec_ops 1308 + .upsert_records( 1309 + &mut batch, 1310 + user_hash, 1311 + &[ 1312 + rw(&coll_normal, &rkey, &cid1), 1313 + rw(&coll_with_null, &rkey, &cid2), 1314 + ], 1315 + ) 1316 + .unwrap(); 1317 + batch.commit().unwrap(); 1318 + 1319 + let found1 = rec_ops 1320 + .get_record_cid(user_id, &coll_normal, &rkey) 1321 + .unwrap(); 1322 + assert_eq!(found1, Some(cid1)); 1323 + 1324 + let found2 = rec_ops 1325 + .get_record_cid(user_id, &coll_with_null, &rkey) 1326 + .unwrap(); 1327 + assert_eq!(found2, Some(cid2)); 1328 + 1329 + let results_normal = rec_ops 1330 + .list_records(&lrq(user_id, &coll_normal, None, 100, false, None, None)) 1331 + .unwrap(); 1332 + assert_eq!(results_normal.len(), 1); 1333 + 1334 + let results_null = rec_ops 1335 + .list_records(&lrq(user_id, &coll_with_null, None, 100, false, None, None)) 1336 + .unwrap(); 1337 + assert_eq!(results_null.len(), 1); 1338 + 1339 + let collections = rec_ops.list_collections(user_id).unwrap(); 1340 + assert_eq!(collections.len(), 2); 1341 + } 1342 + 1343 + #[test] 1344 + fn list_records_cursor_past_rkey_end_returns_empty() { 1345 + let (_dir, ms) = open_fresh(); 1346 + let (user_id, user_hash) = setup_user(&ms); 1347 + let rec_ops = ms.record_ops(); 1348 + 1349 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1350 + let rkeys: Vec<Rkey> = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 1351 + let cids: Vec<CidLink> = (0..5).map(|i| test_cid_link(i + 1)).collect(); 1352 + 1353 + let writes: Vec<RecordWrite<'_>> = rkeys 1354 + .iter() 1355 + .zip(cids.iter()) 1356 + .map(|(rk, c)| rw(&collection, rk, c)) 1357 + .collect(); 1358 + 1359 + let mut batch = ms.database().batch(); 1360 + rec_ops 1361 + .upsert_records(&mut batch, user_hash, &writes) 1362 + .unwrap(); 1363 + batch.commit().unwrap(); 1364 + 1365 + let cursor = Rkey::from("rkey010".to_string()); 1366 + let rkey_end = Rkey::from("rkey003".to_string()); 1367 + let results = rec_ops 1368 + .list_records(&lrq( 1369 + user_id, 1370 + &collection, 1371 + Some(&cursor), 1372 + 100, 1373 + false, 1374 + None, 1375 + Some(&rkey_end), 1376 + )) 1377 + .unwrap(); 1378 + assert!(results.is_empty()); 1379 + } 1380 + 1381 + #[test] 1382 + fn list_records_reverse_with_rkey_bounds() { 1383 + let (_dir, ms) = open_fresh(); 1384 + let (user_id, user_hash) = setup_user(&ms); 1385 + let rec_ops = ms.record_ops(); 1386 + 1387 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1388 + let rkeys: Vec<Rkey> = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 1389 + let cids: Vec<CidLink> = (0..10).map(|i| test_cid_link(i + 1)).collect(); 1390 + 1391 + let writes: Vec<RecordWrite<'_>> = rkeys 1392 + .iter() 1393 + .zip(cids.iter()) 1394 + .map(|(rk, c)| rw(&collection, rk, c)) 1395 + .collect(); 1396 + 1397 + let mut batch = ms.database().batch(); 1398 + rec_ops 1399 + .upsert_records(&mut batch, user_hash, &writes) 1400 + .unwrap(); 1401 + batch.commit().unwrap(); 1402 + 1403 + let rkey_start = Rkey::from("rkey002".to_string()); 1404 + let rkey_end = Rkey::from("rkey007".to_string()); 1405 + let results = rec_ops 1406 + .list_records(&lrq( 1407 + user_id, 1408 + &collection, 1409 + None, 1410 + 100, 1411 + true, 1412 + Some(&rkey_start), 1413 + Some(&rkey_end), 1414 + )) 1415 + .unwrap(); 1416 + assert_eq!(results.len(), 6); 1417 + assert_eq!(results[0].rkey.as_str(), "rkey007"); 1418 + assert_eq!(results[5].rkey.as_str(), "rkey002"); 1419 + } 1420 + 1421 + #[test] 1422 + fn list_records_reverse_cursor_narrows_range() { 1423 + let (_dir, ms) = open_fresh(); 1424 + let (user_id, user_hash) = setup_user(&ms); 1425 + let rec_ops = ms.record_ops(); 1426 + 1427 + let collection = Nsid::from("app.bsky.feed.post".to_string()); 1428 + let rkeys: Vec<Rkey> = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); 1429 + let cids: Vec<CidLink> = (0..10).map(|i| test_cid_link(i + 1)).collect(); 1430 + 1431 + let writes: Vec<RecordWrite<'_>> = rkeys 1432 + .iter() 1433 + .zip(cids.iter()) 1434 + .map(|(rk, c)| rw(&collection, rk, c)) 1435 + .collect(); 1436 + 1437 + let mut batch = ms.database().batch(); 1438 + rec_ops 1439 + .upsert_records(&mut batch, user_hash, &writes) 1440 + .unwrap(); 1441 + batch.commit().unwrap(); 1442 + 1443 + let cursor = Rkey::from("rkey005".to_string()); 1444 + let results = rec_ops 1445 + .list_records(&lrq( 1446 + user_id, 1447 + &collection, 1448 + Some(&cursor), 1449 + 3, 1450 + true, 1451 + None, 1452 + None, 1453 + )) 1454 + .unwrap(); 1455 + assert_eq!(results.len(), 3); 1456 + assert_eq!(results[0].rkey.as_str(), "rkey004"); 1457 + assert_eq!(results[1].rkey.as_str(), "rkey003"); 1458 + assert_eq!(results[2].rkey.as_str(), "rkey002"); 1459 + } 1460 + 1461 + #[test] 1462 + fn exclusive_upper_bound_basic() { 1463 + let prefix = &[0x01, 0x02, 0x03]; 1464 + let upper = exclusive_upper_bound(prefix).unwrap(); 1465 + assert_eq!(upper.as_slice(), &[0x01, 0x02, 0x04]); 1466 + } 1467 + 1468 + #[test] 1469 + fn exclusive_upper_bound_with_trailing_ff() { 1470 + let prefix = &[0x01, 0xFF, 0xFF]; 1471 + let upper = exclusive_upper_bound(prefix).unwrap(); 1472 + assert_eq!(upper.as_slice(), &[0x02]); 1473 + } 1474 + 1475 + #[test] 1476 + fn exclusive_upper_bound_all_ff_returns_none() { 1477 + assert!(exclusive_upper_bound(&[0xFF, 0xFF, 0xFF]).is_none()); 1478 + } 1479 + 1480 + #[test] 1481 + fn exclusive_upper_bound_empty_returns_none() { 1482 + assert!(exclusive_upper_bound(&[]).is_none()); 1483 + } 1484 + 1485 + #[test] 1486 + fn exclusive_upper_bound_preserves_prefix_ordering() { 1487 + let prefix = &[ 1488 + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, 0x61, 0x00, 0x00, 1489 + ]; 1490 + let upper = exclusive_upper_bound(prefix).unwrap(); 1491 + 1492 + let key_inside = { 1493 + let mut k = prefix.to_vec(); 1494 + k.extend_from_slice(&[0x62, 0x00, 0x00]); 1495 + k 1496 + }; 1497 + assert!(key_inside.as_slice() < upper.as_slice()); 1498 + 1499 + let key_outside = { 1500 + let mut k = Vec::from(&prefix[..prefix.len() - 2]); 1501 + k.extend_from_slice(&[0x00, 0x01, 0x00, 0x00]); 1502 + k 1503 + }; 1504 + assert!(key_outside.as_slice() >= upper.as_slice()); 1505 + } 1506 + }
+162
crates/tranquil-store/src/metastore/records.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use smallvec::SmallVec; 3 + 4 + use super::encoding::KeyBuilder; 5 + use super::keys::{KeyTag, UserHash}; 6 + 7 + const SCHEMA_VERSION: u8 = 1; 8 + 9 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 10 + pub struct RecordValue { 11 + pub record_cid: Vec<u8>, 12 + pub takedown_ref: Option<String>, 13 + } 14 + 15 + impl RecordValue { 16 + pub fn serialize(&self) -> Vec<u8> { 17 + let payload = postcard::to_allocvec(self).expect("RecordValue serialization cannot fail"); 18 + let mut buf = Vec::with_capacity(1 + payload.len()); 19 + buf.push(SCHEMA_VERSION); 20 + buf.extend_from_slice(&payload); 21 + buf 22 + } 23 + 24 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 25 + let (&version, payload) = bytes.split_first()?; 26 + match version { 27 + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 28 + _ => None, 29 + } 30 + } 31 + } 32 + 33 + pub fn record_key(user_hash: UserHash, collection: &str, rkey: &str) -> SmallVec<[u8; 128]> { 34 + KeyBuilder::new() 35 + .tag(KeyTag::RECORDS) 36 + .u64(user_hash.raw()) 37 + .string(collection) 38 + .string(rkey) 39 + .build() 40 + } 41 + 42 + pub fn record_collection_prefix(user_hash: UserHash, collection: &str) -> SmallVec<[u8; 128]> { 43 + KeyBuilder::new() 44 + .tag(KeyTag::RECORDS) 45 + .u64(user_hash.raw()) 46 + .string(collection) 47 + .build() 48 + } 49 + 50 + pub fn record_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 51 + KeyBuilder::new() 52 + .tag(KeyTag::RECORDS) 53 + .u64(user_hash.raw()) 54 + .build() 55 + } 56 + 57 + pub fn records_prefix() -> SmallVec<[u8; 128]> { 58 + KeyBuilder::new().tag(KeyTag::RECORDS).build() 59 + } 60 + 61 + #[cfg(test)] 62 + mod tests { 63 + use super::*; 64 + use crate::metastore::encoding::KeyReader; 65 + 66 + #[test] 67 + fn record_value_roundtrip() { 68 + let value = RecordValue { 69 + record_cid: vec![0x01, 0x71, 0x12, 0x20, 0xAB], 70 + takedown_ref: None, 71 + }; 72 + let bytes = value.serialize(); 73 + let decoded = RecordValue::deserialize(&bytes).unwrap(); 74 + assert_eq!(decoded, value); 75 + } 76 + 77 + #[test] 78 + fn record_value_with_takedown() { 79 + let value = RecordValue { 80 + record_cid: vec![0x01], 81 + takedown_ref: Some("DMCA-456".to_string()), 82 + }; 83 + let bytes = value.serialize(); 84 + let decoded = RecordValue::deserialize(&bytes).unwrap(); 85 + assert_eq!(decoded, value); 86 + } 87 + 88 + #[test] 89 + fn schema_version_is_first_byte() { 90 + let value = RecordValue { 91 + record_cid: vec![0x01], 92 + takedown_ref: None, 93 + }; 94 + let bytes = value.serialize(); 95 + assert_eq!(bytes[0], SCHEMA_VERSION); 96 + } 97 + 98 + #[test] 99 + fn deserialize_rejects_unknown_schema_version() { 100 + let value = RecordValue { 101 + record_cid: vec![0x01], 102 + takedown_ref: None, 103 + }; 104 + let mut bytes = value.serialize(); 105 + bytes[0] = 99; 106 + assert!(RecordValue::deserialize(&bytes).is_none()); 107 + } 108 + 109 + #[test] 110 + fn deserialize_rejects_empty_input() { 111 + assert!(RecordValue::deserialize(&[]).is_none()); 112 + } 113 + 114 + #[test] 115 + fn record_key_roundtrip() { 116 + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); 117 + let key = record_key(hash, "app.bsky.feed.post", "3k2abcd"); 118 + let mut reader = KeyReader::new(&key); 119 + assert_eq!(reader.tag(), Some(KeyTag::RECORDS.raw())); 120 + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); 121 + assert_eq!(reader.string(), Some("app.bsky.feed.post".to_string())); 122 + assert_eq!(reader.string(), Some("3k2abcd".to_string())); 123 + assert!(reader.is_empty()); 124 + } 125 + 126 + #[test] 127 + fn record_keys_sort_by_user_then_collection_then_rkey() { 128 + let h1 = UserHash::from_raw(1); 129 + let h2 = UserHash::from_raw(2); 130 + 131 + let k1 = record_key(h1, "app.bsky.feed.like", "aaa"); 132 + let k2 = record_key(h1, "app.bsky.feed.post", "aaa"); 133 + let k3 = record_key(h1, "app.bsky.feed.post", "bbb"); 134 + let k4 = record_key(h2, "app.bsky.feed.like", "aaa"); 135 + 136 + assert!(k1.as_slice() < k2.as_slice()); 137 + assert!(k2.as_slice() < k3.as_slice()); 138 + assert!(k3.as_slice() < k4.as_slice()); 139 + } 140 + 141 + #[test] 142 + fn collection_prefix_is_prefix_of_full_key() { 143 + let hash = UserHash::from_raw(42); 144 + let prefix = record_collection_prefix(hash, "app.bsky.feed.post"); 145 + let full = record_key(hash, "app.bsky.feed.post", "some_rkey"); 146 + assert!(full.as_slice().starts_with(prefix.as_slice())); 147 + } 148 + 149 + #[test] 150 + fn user_prefix_is_prefix_of_collection_prefix() { 151 + let hash = UserHash::from_raw(42); 152 + let user_pfx = record_user_prefix(hash); 153 + let coll_pfx = record_collection_prefix(hash, "app.bsky.feed.post"); 154 + assert!(coll_pfx.as_slice().starts_with(user_pfx.as_slice())); 155 + } 156 + 157 + #[test] 158 + fn records_prefix_is_just_tag() { 159 + let pfx = records_prefix(); 160 + assert_eq!(pfx.as_slice(), &[KeyTag::RECORDS.raw()]); 161 + } 162 + }
+307
crates/tranquil-store/src/metastore/recovery.rs
··· 1 + use std::collections::HashSet; 2 + 3 + use serde::{Deserialize, Serialize}; 4 + 5 + use super::backlink_ops::remove_backlinks_for_record; 6 + use super::backlinks::{BacklinkValue, backlink_by_user_key, backlink_key, discriminant_to_path}; 7 + use super::encoding::KeyReader; 8 + use super::keys::{KeyTag, UserHash}; 9 + use super::records::{RecordValue, record_key}; 10 + use super::repo_meta::{RepoMetaValue, repo_meta_key}; 11 + use super::user_blocks::{user_block_key, user_block_user_prefix}; 12 + use crate::metastore::MetastoreError; 13 + 14 + const MUTATION_SET_VERSION: u8 = 1; 15 + 16 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 17 + pub struct CommitMutationSet { 18 + pub new_root_cid: Vec<u8>, 19 + pub new_rev: String, 20 + pub record_upserts: Vec<RecordMutationUpsert>, 21 + pub record_deletes: Vec<RecordMutationDelete>, 22 + pub block_inserts: Vec<Vec<u8>>, 23 + pub block_deletes: Vec<Vec<u8>>, 24 + pub backlink_adds: Vec<BacklinkMutation>, 25 + pub backlink_remove_uris: Vec<String>, 26 + } 27 + 28 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 29 + pub struct RecordMutationUpsert { 30 + pub collection: String, 31 + pub rkey: String, 32 + pub cid_bytes: Vec<u8>, 33 + } 34 + 35 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 36 + pub struct RecordMutationDelete { 37 + pub collection: String, 38 + pub rkey: String, 39 + } 40 + 41 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 42 + pub struct BacklinkMutation { 43 + pub uri: String, 44 + pub path: u8, 45 + pub link_to: String, 46 + } 47 + 48 + const MAX_MUTATION_SET_ENTRIES: usize = 50_000; 49 + 50 + impl CommitMutationSet { 51 + pub fn serialize(&self) -> Result<Vec<u8>, MetastoreError> { 52 + self.validate_size()?; 53 + let payload = postcard::to_allocvec(self) 54 + .map_err(|_| MetastoreError::CorruptData("CommitMutationSet serialization failed"))?; 55 + let mut buf = Vec::with_capacity(1 + payload.len()); 56 + buf.push(MUTATION_SET_VERSION); 57 + buf.extend_from_slice(&payload); 58 + Ok(buf) 59 + } 60 + 61 + fn validate_size(&self) -> Result<(), MetastoreError> { 62 + let total = self.record_upserts.len() 63 + + self.record_deletes.len() 64 + + self.block_inserts.len() 65 + + self.block_deletes.len() 66 + + self.backlink_adds.len() 67 + + self.backlink_remove_uris.len(); 68 + match total <= MAX_MUTATION_SET_ENTRIES { 69 + true => Ok(()), 70 + false => { 71 + tracing::warn!( 72 + total_entries = total, 73 + max = MAX_MUTATION_SET_ENTRIES, 74 + "CommitMutationSet exceeds entry limit" 75 + ); 76 + Err(MetastoreError::InvalidInput( 77 + "CommitMutationSet exceeds maximum entry count", 78 + )) 79 + } 80 + } 81 + } 82 + 83 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 84 + let (&version, payload) = bytes.split_first()?; 85 + match version { 86 + MUTATION_SET_VERSION => match postcard::from_bytes(payload) { 87 + Ok(v) => Some(v), 88 + Err(e) => { 89 + tracing::warn!(%e, "failed to deserialize CommitMutationSet payload"); 90 + None 91 + } 92 + }, 93 + _ => { 94 + tracing::warn!(version, "unknown CommitMutationSet version"); 95 + None 96 + } 97 + } 98 + } 99 + } 100 + 101 + pub fn replay_mutation_set( 102 + batch: &mut fjall::OwnedWriteBatch, 103 + repo_data: &fjall::Keyspace, 104 + indexes: &fjall::Keyspace, 105 + user_hash: UserHash, 106 + current_meta: &RepoMetaValue, 107 + mutation_set: &CommitMutationSet, 108 + ) -> Result<(), MetastoreError> { 109 + mutation_set.validate_size()?; 110 + 111 + let updated_meta = RepoMetaValue { 112 + repo_root_cid: mutation_set.new_root_cid.clone(), 113 + repo_rev: mutation_set.new_rev.clone(), 114 + ..current_meta.clone() 115 + }; 116 + let meta_key = repo_meta_key(user_hash); 117 + batch.insert(repo_data, meta_key.as_slice(), updated_meta.serialize()); 118 + 119 + mutation_set.record_upserts.iter().for_each(|u| { 120 + let key = record_key(user_hash, &u.collection, &u.rkey); 121 + let value = RecordValue { 122 + record_cid: u.cid_bytes.clone(), 123 + takedown_ref: None, 124 + }; 125 + batch.insert(repo_data, key.as_slice(), value.serialize()); 126 + }); 127 + 128 + mutation_set.record_deletes.iter().for_each(|d| { 129 + let key = record_key(user_hash, &d.collection, &d.rkey); 130 + batch.remove(repo_data, key.as_slice()); 131 + }); 132 + 133 + mutation_set.block_inserts.iter().for_each(|cid_bytes| { 134 + let key = user_block_key(user_hash, &mutation_set.new_rev, cid_bytes); 135 + batch.insert(repo_data, key.as_slice(), []); 136 + }); 137 + 138 + delete_user_blocks_by_cid_scan(batch, repo_data, user_hash, &mutation_set.block_deletes)?; 139 + 140 + mutation_set 141 + .backlink_remove_uris 142 + .iter() 143 + .try_for_each(|uri_str| { 144 + let uri = tranquil_types::AtUri::from(uri_str.clone()); 145 + let collection = uri.collection().ok_or(MetastoreError::CorruptData( 146 + "backlink URI missing collection", 147 + ))?; 148 + let rkey = uri 149 + .rkey() 150 + .ok_or(MetastoreError::CorruptData("backlink URI missing rkey"))?; 151 + 152 + remove_backlinks_for_record(batch, indexes, user_hash, collection, rkey) 153 + })?; 154 + 155 + mutation_set.backlink_adds.iter().try_for_each(|bl| { 156 + let uri = tranquil_types::AtUri::from(bl.uri.clone()); 157 + let collection = uri.collection().ok_or(MetastoreError::CorruptData( 158 + "backlink URI missing collection", 159 + ))?; 160 + let rkey = uri 161 + .rkey() 162 + .ok_or(MetastoreError::CorruptData("backlink URI missing rkey"))?; 163 + 164 + match discriminant_to_path(bl.path) { 165 + None => { 166 + tracing::warn!( 167 + path = bl.path, 168 + uri = %bl.uri, 169 + "skipping backlink with unknown path discriminant during recovery" 170 + ); 171 + } 172 + Some(_) => { 173 + let primary = backlink_key(&bl.link_to, user_hash, collection, rkey); 174 + let value = BacklinkValue { 175 + source_uri: bl.uri.clone(), 176 + path: bl.path, 177 + }; 178 + batch.insert(indexes, primary.as_slice(), value.serialize()); 179 + 180 + let reverse = backlink_by_user_key(user_hash, collection, rkey, &bl.link_to); 181 + batch.insert(indexes, reverse.as_slice(), []); 182 + } 183 + } 184 + Ok::<_, MetastoreError>(()) 185 + }) 186 + } 187 + 188 + fn delete_user_blocks_by_cid_scan( 189 + batch: &mut fjall::OwnedWriteBatch, 190 + repo_data: &fjall::Keyspace, 191 + user_hash: UserHash, 192 + block_cids: &[Vec<u8>], 193 + ) -> Result<(), MetastoreError> { 194 + match block_cids.is_empty() { 195 + true => Ok(()), 196 + false => { 197 + let cid_set: HashSet<&[u8]> = block_cids.iter().map(|c| c.as_slice()).collect(); 198 + let prefix = user_block_user_prefix(user_hash); 199 + repo_data.prefix(prefix.as_slice()).try_for_each(|guard| { 200 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 201 + match extract_cid_from_user_block_key(&key_bytes) { 202 + Some(cid) if cid_set.contains(cid) => { 203 + batch.remove(repo_data, key_bytes.as_ref()); 204 + Ok(()) 205 + } 206 + _ => Ok(()), 207 + } 208 + }) 209 + } 210 + } 211 + } 212 + 213 + fn extract_cid_from_user_block_key(key_bytes: &[u8]) -> Option<&[u8]> { 214 + let mut reader = KeyReader::new(key_bytes); 215 + let tag = reader.tag()?; 216 + 217 + if tag != KeyTag::USER_BLOCKS.raw() { 218 + tracing::warn!( 219 + tag, 220 + "unexpected key tag in user_block prefix scan during recovery" 221 + ); 222 + return None; 223 + } 224 + 225 + if reader.u64().and_then(|_| reader.string()).is_none() { 226 + tracing::warn!("user_block key has corrupt user_hash or rev during recovery"); 227 + return None; 228 + } 229 + 230 + let remaining = reader.remaining(); 231 + match remaining.is_empty() { 232 + true => { 233 + tracing::warn!("user_block key has no CID suffix during recovery"); 234 + None 235 + } 236 + false => Some(remaining), 237 + } 238 + } 239 + 240 + #[cfg(test)] 241 + mod tests { 242 + use super::*; 243 + 244 + #[test] 245 + fn mutation_set_roundtrip() { 246 + let ms = CommitMutationSet { 247 + new_root_cid: vec![0x01, 0x71, 0x12, 0x20], 248 + new_rev: "rev1".to_owned(), 249 + record_upserts: vec![RecordMutationUpsert { 250 + collection: "app.bsky.feed.post".to_owned(), 251 + rkey: "3k2abc".to_owned(), 252 + cid_bytes: vec![0xDE, 0xAD], 253 + }], 254 + record_deletes: vec![RecordMutationDelete { 255 + collection: "app.bsky.feed.like".to_owned(), 256 + rkey: "3k2del".to_owned(), 257 + }], 258 + block_inserts: vec![vec![0x01, 0x02]], 259 + block_deletes: vec![vec![0x03, 0x04]], 260 + backlink_adds: vec![BacklinkMutation { 261 + uri: "at://did:plc:alice/app.bsky.feed.like/3k2abc".to_owned(), 262 + path: 1, 263 + link_to: "at://did:plc:bob/app.bsky.feed.post/3k2xyz".to_owned(), 264 + }], 265 + backlink_remove_uris: vec!["at://did:plc:alice/app.bsky.feed.like/3k2old".to_owned()], 266 + }; 267 + 268 + let bytes = ms.serialize().unwrap(); 269 + assert_eq!(bytes[0], MUTATION_SET_VERSION); 270 + let recovered = CommitMutationSet::deserialize(&bytes).unwrap(); 271 + assert_eq!(recovered, ms); 272 + } 273 + 274 + #[test] 275 + fn mutation_set_empty_roundtrip() { 276 + let ms = CommitMutationSet { 277 + new_root_cid: vec![], 278 + new_rev: String::new(), 279 + record_upserts: vec![], 280 + record_deletes: vec![], 281 + block_inserts: vec![], 282 + block_deletes: vec![], 283 + backlink_adds: vec![], 284 + backlink_remove_uris: vec![], 285 + }; 286 + 287 + let recovered = CommitMutationSet::deserialize(&ms.serialize().unwrap()).unwrap(); 288 + assert_eq!(recovered, ms); 289 + } 290 + 291 + #[test] 292 + fn unknown_version_returns_none() { 293 + let ms = CommitMutationSet { 294 + new_root_cid: vec![], 295 + new_rev: String::new(), 296 + record_upserts: vec![], 297 + record_deletes: vec![], 298 + block_inserts: vec![], 299 + block_deletes: vec![], 300 + backlink_adds: vec![], 301 + backlink_remove_uris: vec![], 302 + }; 303 + let mut bytes = ms.serialize().unwrap(); 304 + bytes[0] = 99; 305 + assert!(CommitMutationSet::deserialize(&bytes).is_none()); 306 + } 307 + }
+226
crates/tranquil-store/src/metastore/repo_meta.rs
··· 1 + use serde::{Deserialize, Serialize}; 2 + use smallvec::SmallVec; 3 + 4 + use super::encoding::KeyBuilder; 5 + use super::keys::{KeyTag, UserHash}; 6 + 7 + const SCHEMA_VERSION: u8 = 1; 8 + 9 + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] 10 + #[repr(u8)] 11 + pub enum RepoStatus { 12 + Active = 0, 13 + Takendown = 1, 14 + Suspended = 2, 15 + Deactivated = 3, 16 + Deleted = 4, 17 + } 18 + 19 + impl RepoStatus { 20 + pub fn is_active(self) -> bool { 21 + matches!(self, Self::Active) 22 + } 23 + } 24 + 25 + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 26 + pub struct RepoMetaValue { 27 + pub repo_root_cid: Vec<u8>, 28 + pub repo_rev: String, 29 + pub handle: String, 30 + pub status: RepoStatus, 31 + pub deactivated_at_ms: Option<u64>, 32 + pub takedown_ref: Option<String>, 33 + #[serde(default)] 34 + pub did: Option<String>, 35 + } 36 + 37 + impl RepoMetaValue { 38 + pub fn serialize(&self) -> Vec<u8> { 39 + let payload = postcard::to_allocvec(self).expect("RepoMetaValue serialization cannot fail"); 40 + let mut buf = Vec::with_capacity(1 + payload.len()); 41 + buf.push(SCHEMA_VERSION); 42 + buf.extend_from_slice(&payload); 43 + buf 44 + } 45 + 46 + pub fn deserialize(bytes: &[u8]) -> Option<Self> { 47 + let (&version, payload) = bytes.split_first()?; 48 + match version { 49 + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), 50 + _ => None, 51 + } 52 + } 53 + } 54 + 55 + pub fn repo_meta_key(user_hash: UserHash) -> SmallVec<[u8; 128]> { 56 + KeyBuilder::new() 57 + .tag(KeyTag::REPO_META) 58 + .u64(user_hash.raw()) 59 + .build() 60 + } 61 + 62 + pub fn repo_meta_prefix() -> SmallVec<[u8; 128]> { 63 + KeyBuilder::new().tag(KeyTag::REPO_META).build() 64 + } 65 + 66 + pub fn handle_key(handle_lower: &str) -> SmallVec<[u8; 128]> { 67 + KeyBuilder::new() 68 + .tag(KeyTag::HANDLES) 69 + .string(handle_lower) 70 + .build() 71 + } 72 + 73 + #[cfg(test)] 74 + mod tests { 75 + use super::*; 76 + use crate::metastore::encoding::KeyReader; 77 + 78 + #[test] 79 + fn repo_meta_value_roundtrip() { 80 + let value = RepoMetaValue { 81 + repo_root_cid: vec![0x01, 0x71, 0x12, 0x20, 0xAB], 82 + repo_rev: "3k2a7bcd".to_string(), 83 + handle: "alice.bsky.social".to_string(), 84 + status: RepoStatus::Active, 85 + deactivated_at_ms: None, 86 + takedown_ref: None, 87 + did: None, 88 + }; 89 + let bytes = value.serialize(); 90 + let decoded = RepoMetaValue::deserialize(&bytes).unwrap(); 91 + assert_eq!(decoded, value); 92 + } 93 + 94 + #[test] 95 + fn repo_meta_value_with_optional_fields() { 96 + let value = RepoMetaValue { 97 + repo_root_cid: vec![0x01], 98 + repo_rev: "rev1".to_string(), 99 + handle: "bob.example.com".to_string(), 100 + status: RepoStatus::Deactivated, 101 + deactivated_at_ms: Some(1700000000000), 102 + takedown_ref: Some("DMCA-123".to_string()), 103 + did: Some("did:plc:bob".to_string()), 104 + }; 105 + let bytes = value.serialize(); 106 + let decoded = RepoMetaValue::deserialize(&bytes).unwrap(); 107 + assert_eq!(decoded, value); 108 + } 109 + 110 + #[test] 111 + fn repo_meta_key_roundtrip() { 112 + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); 113 + let key = repo_meta_key(hash); 114 + let mut reader = KeyReader::new(&key); 115 + assert_eq!(reader.tag(), Some(KeyTag::REPO_META.raw())); 116 + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); 117 + assert!(reader.is_empty()); 118 + } 119 + 120 + #[test] 121 + fn repo_meta_keys_sort_by_user_hash() { 122 + let k1 = repo_meta_key(UserHash::from_raw(1)); 123 + let k2 = repo_meta_key(UserHash::from_raw(2)); 124 + let k3 = repo_meta_key(UserHash::from_raw(u64::MAX)); 125 + assert!(k1.as_slice() < k2.as_slice()); 126 + assert!(k2.as_slice() < k3.as_slice()); 127 + } 128 + 129 + #[test] 130 + fn handle_key_roundtrip() { 131 + let key = handle_key("alice.bsky.social"); 132 + let mut reader = KeyReader::new(&key); 133 + assert_eq!(reader.tag(), Some(KeyTag::HANDLES.raw())); 134 + assert_eq!(reader.string(), Some("alice.bsky.social".to_string())); 135 + assert!(reader.is_empty()); 136 + } 137 + 138 + #[test] 139 + fn handle_keys_sort_lexicographically() { 140 + let k1 = handle_key("alice.example.com"); 141 + let k2 = handle_key("bob.example.com"); 142 + assert!(k1.as_slice() < k2.as_slice()); 143 + } 144 + 145 + #[test] 146 + fn all_repo_statuses_roundtrip() { 147 + [ 148 + RepoStatus::Active, 149 + RepoStatus::Takendown, 150 + RepoStatus::Suspended, 151 + RepoStatus::Deactivated, 152 + RepoStatus::Deleted, 153 + ] 154 + .iter() 155 + .for_each(|&status| { 156 + let value = RepoMetaValue { 157 + repo_root_cid: vec![0x01], 158 + repo_rev: "r".to_string(), 159 + handle: "h.test".to_string(), 160 + status, 161 + deactivated_at_ms: None, 162 + takedown_ref: None, 163 + did: None, 164 + }; 165 + let decoded = RepoMetaValue::deserialize(&value.serialize()).unwrap(); 166 + assert_eq!(decoded.status, status); 167 + }); 168 + } 169 + 170 + #[test] 171 + fn repo_status_serialization_stability() { 172 + [ 173 + (RepoStatus::Active, 0u8), 174 + (RepoStatus::Takendown, 1), 175 + (RepoStatus::Suspended, 2), 176 + (RepoStatus::Deactivated, 3), 177 + (RepoStatus::Deleted, 4), 178 + ] 179 + .iter() 180 + .for_each(|&(status, expected_byte)| { 181 + let bytes = postcard::to_allocvec(&status).unwrap(); 182 + assert_eq!( 183 + bytes, 184 + [expected_byte], 185 + "{status:?} serialized to {bytes:?}, expected [{expected_byte}]" 186 + ); 187 + }); 188 + } 189 + 190 + #[test] 191 + fn schema_version_is_first_byte() { 192 + let value = RepoMetaValue { 193 + repo_root_cid: vec![0x01], 194 + repo_rev: "r".to_string(), 195 + handle: "h.test".to_string(), 196 + status: RepoStatus::Active, 197 + deactivated_at_ms: None, 198 + takedown_ref: None, 199 + did: None, 200 + }; 201 + let bytes = value.serialize(); 202 + assert_eq!(bytes[0], SCHEMA_VERSION); 203 + assert_eq!(bytes[0], 1, "schema version must remain 1 for this format"); 204 + } 205 + 206 + #[test] 207 + fn deserialize_rejects_unknown_schema_version() { 208 + let value = RepoMetaValue { 209 + repo_root_cid: vec![0x01], 210 + repo_rev: "r".to_string(), 211 + handle: "h.test".to_string(), 212 + status: RepoStatus::Active, 213 + deactivated_at_ms: None, 214 + takedown_ref: None, 215 + did: None, 216 + }; 217 + let mut bytes = value.serialize(); 218 + bytes[0] = 99; 219 + assert!(RepoMetaValue::deserialize(&bytes).is_none()); 220 + } 221 + 222 + #[test] 223 + fn deserialize_rejects_empty_input() { 224 + assert!(RepoMetaValue::deserialize(&[]).is_none()); 225 + } 226 + }
+1146
crates/tranquil-store/src/metastore/repo_ops.rs
··· 1 + use chrono::{DateTime, TimeZone, Utc}; 2 + use fjall::Keyspace; 3 + use std::sync::Arc; 4 + use uuid::Uuid; 5 + 6 + use super::MetastoreError; 7 + use super::encoding::KeyReader; 8 + use super::keys::{KeyTag, UserHash}; 9 + use super::repo_meta::{RepoMetaValue, RepoStatus, handle_key, repo_meta_key, repo_meta_prefix}; 10 + use super::scan::{count_prefix, point_lookup}; 11 + use super::user_hash::UserHashMap; 12 + 13 + use tranquil_types::{CidLink, Did, Handle}; 14 + 15 + pub struct RepoOps { 16 + repo_data: Keyspace, 17 + user_hashes: Arc<UserHashMap>, 18 + } 19 + 20 + impl RepoOps { 21 + pub fn new(repo_data: Keyspace, user_hashes: Arc<UserHashMap>) -> Self { 22 + Self { 23 + repo_data, 24 + user_hashes, 25 + } 26 + } 27 + 28 + pub fn create_repo( 29 + &self, 30 + db: &fjall::Database, 31 + user_id: Uuid, 32 + did: &Did, 33 + handle: &Handle, 34 + repo_root_cid: &CidLink, 35 + repo_rev: &str, 36 + ) -> Result<(), MetastoreError> { 37 + let user_hash = UserHash::from_did(did.as_str()); 38 + let mut batch = db.batch(); 39 + 40 + self.user_hashes 41 + .stage_insert(&mut batch, user_id, user_hash)?; 42 + 43 + let cid_bytes = cid_link_to_bytes(repo_root_cid)?; 44 + let handle_lower = handle.as_str().to_ascii_lowercase(); 45 + 46 + let value = RepoMetaValue { 47 + repo_root_cid: cid_bytes, 48 + repo_rev: repo_rev.to_string(), 49 + handle: handle_lower.clone(), 50 + status: RepoStatus::Active, 51 + deactivated_at_ms: None, 52 + takedown_ref: None, 53 + did: Some(did.as_str().to_string()), 54 + }; 55 + 56 + batch.insert( 57 + &self.repo_data, 58 + repo_meta_key(user_hash).as_slice(), 59 + value.serialize(), 60 + ); 61 + 62 + batch.insert( 63 + &self.repo_data, 64 + handle_key(&handle_lower).as_slice(), 65 + user_hash.raw().to_be_bytes(), 66 + ); 67 + 68 + match batch.commit() { 69 + Ok(()) => Ok(()), 70 + Err(e) => { 71 + self.user_hashes.rollback_insert(&user_id, &user_hash); 72 + Err(MetastoreError::Fjall(e)) 73 + } 74 + } 75 + } 76 + 77 + pub fn get_repo_meta( 78 + &self, 79 + user_id: Uuid, 80 + ) -> Result<Option<(UserHash, RepoMetaValue)>, MetastoreError> { 81 + let user_hash = match self.user_hashes.get(&user_id) { 82 + Some(h) => h, 83 + None => return Ok(None), 84 + }; 85 + let key = repo_meta_key(user_hash); 86 + Ok(point_lookup( 87 + &self.repo_data, 88 + key.as_slice(), 89 + RepoMetaValue::deserialize, 90 + "invalid repo_meta value", 91 + )? 92 + .map(|v| (user_hash, v))) 93 + } 94 + 95 + pub fn write_repo_meta( 96 + &self, 97 + batch: &mut fjall::OwnedWriteBatch, 98 + user_hash: UserHash, 99 + value: &RepoMetaValue, 100 + ) { 101 + batch.insert( 102 + &self.repo_data, 103 + repo_meta_key(user_hash).as_slice(), 104 + value.serialize(), 105 + ); 106 + } 107 + 108 + pub(crate) fn update_repo_root( 109 + &self, 110 + db: &fjall::Database, 111 + user_id: Uuid, 112 + repo_root_cid: &CidLink, 113 + repo_rev: &str, 114 + ) -> Result<(), MetastoreError> { 115 + let user_hash = self.resolve_user_hash(user_id)?; 116 + let key = repo_meta_key(user_hash); 117 + 118 + let mut value = self.get_meta_value(key.as_slice())?; 119 + let cid_bytes = cid_link_to_bytes(repo_root_cid)?; 120 + value.repo_root_cid = cid_bytes; 121 + value.repo_rev = repo_rev.to_string(); 122 + 123 + let mut batch = db.batch(); 124 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 125 + batch.commit().map_err(MetastoreError::Fjall) 126 + } 127 + 128 + pub(crate) fn update_repo_rev( 129 + &self, 130 + db: &fjall::Database, 131 + user_id: Uuid, 132 + repo_rev: &str, 133 + ) -> Result<(), MetastoreError> { 134 + let user_hash = self.resolve_user_hash(user_id)?; 135 + let key = repo_meta_key(user_hash); 136 + 137 + let mut value = self.get_meta_value(key.as_slice())?; 138 + value.repo_rev = repo_rev.to_string(); 139 + 140 + let mut batch = db.batch(); 141 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 142 + batch.commit().map_err(MetastoreError::Fjall) 143 + } 144 + 145 + pub fn update_repo_status( 146 + &self, 147 + db: &fjall::Database, 148 + did: &Did, 149 + takedown: Option<bool>, 150 + takedown_ref: Option<&str>, 151 + deactivated: Option<bool>, 152 + ) -> Result<(), MetastoreError> { 153 + let user_hash = UserHash::from_did(did.as_str()); 154 + let key = repo_meta_key(user_hash); 155 + let existing = point_lookup( 156 + &self.repo_data, 157 + key.as_slice(), 158 + RepoMetaValue::deserialize, 159 + "invalid repo_meta value", 160 + )?; 161 + let mut value = match existing { 162 + Some(v) => v, 163 + None => { 164 + tracing::warn!( 165 + did = did.as_str(), 166 + "update_repo_status: repo not found in metastore" 167 + ); 168 + return Ok(()); 169 + } 170 + }; 171 + 172 + match value.status { 173 + RepoStatus::Suspended | RepoStatus::Deleted => return Ok(()), 174 + _ => {} 175 + } 176 + 177 + if let Some(taken_down) = takedown { 178 + value.takedown_ref = match taken_down { 179 + true => Some(takedown_ref.unwrap_or("").to_owned()), 180 + false => None, 181 + }; 182 + } 183 + if let Some(now_deactivated) = deactivated { 184 + value.deactivated_at_ms = match now_deactivated { 185 + true => value.deactivated_at_ms.or_else(|| { 186 + Some(u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0)) 187 + }), 188 + false => None, 189 + }; 190 + } 191 + 192 + let is_taken_down = match takedown { 193 + Some(v) => v, 194 + None => value.takedown_ref.is_some(), 195 + }; 196 + value.status = match (is_taken_down, value.deactivated_at_ms.is_some()) { 197 + (true, _) => RepoStatus::Takendown, 198 + (false, true) => RepoStatus::Deactivated, 199 + (false, false) => RepoStatus::Active, 200 + }; 201 + 202 + let mut batch = db.batch(); 203 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 204 + batch.commit().map_err(MetastoreError::Fjall) 205 + } 206 + 207 + pub fn update_handle( 208 + &self, 209 + db: &fjall::Database, 210 + user_id: Uuid, 211 + new_handle: &Handle, 212 + ) -> Result<(), MetastoreError> { 213 + let user_hash = self.resolve_user_hash(user_id)?; 214 + let key = repo_meta_key(user_hash); 215 + let mut value = self.get_meta_value(key.as_slice())?; 216 + let new_lower = new_handle.as_str().to_ascii_lowercase(); 217 + 218 + let mut batch = db.batch(); 219 + 220 + match value.handle.is_empty() { 221 + true => {} 222 + false => batch.remove(&self.repo_data, handle_key(&value.handle).as_slice()), 223 + } 224 + 225 + batch.insert( 226 + &self.repo_data, 227 + handle_key(&new_lower).as_slice(), 228 + user_hash.raw().to_be_bytes(), 229 + ); 230 + 231 + value.handle = new_lower; 232 + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); 233 + 234 + batch.commit().map_err(MetastoreError::Fjall) 235 + } 236 + 237 + pub fn delete_repo(&self, db: &fjall::Database, user_id: Uuid) -> Result<(), MetastoreError> { 238 + let user_hash = self.resolve_user_hash(user_id)?; 239 + let key = repo_meta_key(user_hash); 240 + 241 + let meta = self.get_meta_value(key.as_slice())?; 242 + 243 + let mut batch = db.batch(); 244 + batch.remove(&self.repo_data, key.as_slice()); 245 + 246 + match meta.handle.is_empty() { 247 + true => {} 248 + false => batch.remove(&self.repo_data, handle_key(&meta.handle).as_slice()), 249 + } 250 + 251 + self.user_hashes.stage_remove(&mut batch, &user_id); 252 + 253 + match batch.commit() { 254 + Ok(()) => Ok(()), 255 + Err(e) => { 256 + self.user_hashes.rollback_remove(user_id, user_hash); 257 + Err(MetastoreError::Fjall(e)) 258 + } 259 + } 260 + } 261 + 262 + pub fn get_repo(&self, user_id: Uuid) -> Result<Option<RepoInfo>, MetastoreError> { 263 + let user_hash = match self.user_hashes.get(&user_id) { 264 + Some(h) => h, 265 + None => return Ok(None), 266 + }; 267 + let key = repo_meta_key(user_hash); 268 + point_lookup( 269 + &self.repo_data, 270 + key.as_slice(), 271 + RepoMetaValue::deserialize, 272 + "invalid repo_meta value", 273 + )? 274 + .map(|value| { 275 + let cid = bytes_to_cid_link(&value.repo_root_cid)?; 276 + Ok(RepoInfo { 277 + user_id, 278 + repo_root_cid: cid, 279 + repo_rev: Some(value.repo_rev), 280 + }) 281 + }) 282 + .transpose() 283 + } 284 + 285 + pub fn get_repo_root_for_update( 286 + &self, 287 + user_id: Uuid, 288 + ) -> Result<Option<CidLink>, MetastoreError> { 289 + let user_hash = match self.user_hashes.get(&user_id) { 290 + Some(h) => h, 291 + None => return Ok(None), 292 + }; 293 + let key = repo_meta_key(user_hash); 294 + point_lookup( 295 + &self.repo_data, 296 + key.as_slice(), 297 + RepoMetaValue::deserialize, 298 + "invalid repo_meta value", 299 + )? 300 + .map(|v| bytes_to_cid_link(&v.repo_root_cid)) 301 + .transpose() 302 + } 303 + 304 + pub fn get_repo_root_by_did(&self, did: &Did) -> Result<Option<CidLink>, MetastoreError> { 305 + let user_hash = UserHash::from_did(did.as_str()); 306 + let key = repo_meta_key(user_hash); 307 + point_lookup( 308 + &self.repo_data, 309 + key.as_slice(), 310 + RepoMetaValue::deserialize, 311 + "invalid repo_meta value", 312 + )? 313 + .map(|v| bytes_to_cid_link(&v.repo_root_cid)) 314 + .transpose() 315 + } 316 + 317 + pub fn get_repo_root_cid_by_user_id( 318 + &self, 319 + user_id: Uuid, 320 + ) -> Result<Option<CidLink>, MetastoreError> { 321 + self.get_repo_root_for_update(user_id) 322 + } 323 + 324 + pub fn count_repos(&self) -> Result<i64, MetastoreError> { 325 + let prefix = repo_meta_prefix(); 326 + count_prefix(&self.repo_data, prefix.as_slice()) 327 + } 328 + 329 + pub fn get_repos_without_rev( 330 + &self, 331 + limit: usize, 332 + ) -> Result<Vec<RepoWithoutRevEntry>, MetastoreError> { 333 + let prefix = repo_meta_prefix(); 334 + self.repo_data 335 + .prefix(prefix.as_slice()) 336 + .filter_map(|guard| { 337 + let (key_bytes, val_bytes) = match guard.into_inner() { 338 + Ok(pair) => pair, 339 + Err(e) => return Some(Err(MetastoreError::Fjall(e))), 340 + }; 341 + let value = match RepoMetaValue::deserialize(&val_bytes) { 342 + Some(v) => v, 343 + None => { 344 + return Some(Err(MetastoreError::CorruptData("invalid repo_meta value"))); 345 + } 346 + }; 347 + match value.repo_rev.is_empty() { 348 + true => Some(decode_without_rev_entry( 349 + &key_bytes, 350 + &value, 351 + &self.user_hashes, 352 + )), 353 + false => None, 354 + } 355 + }) 356 + .take(limit) 357 + .collect() 358 + } 359 + 360 + pub fn get_account_with_repo( 361 + &self, 362 + did: &Did, 363 + ) -> Result<Option<RepoAccountEntry>, MetastoreError> { 364 + let user_hash = UserHash::from_did(did.as_str()); 365 + let key = repo_meta_key(user_hash); 366 + point_lookup( 367 + &self.repo_data, 368 + key.as_slice(), 369 + RepoMetaValue::deserialize, 370 + "invalid repo_meta value", 371 + )? 372 + .map(|value| { 373 + let user_id = 374 + self.user_hashes 375 + .get_uuid(&user_hash) 376 + .ok_or(MetastoreError::CorruptData( 377 + "user_hash has no reverse mapping", 378 + ))?; 379 + let cid = Some(bytes_to_cid_link(&value.repo_root_cid)?); 380 + let deactivated_at = value 381 + .deactivated_at_ms 382 + .and_then(|ms| i64::try_from(ms).ok()) 383 + .and_then(|ms| Utc.timestamp_millis_opt(ms).single()); 384 + Ok(RepoAccountEntry { 385 + user_id, 386 + did: did.clone(), 387 + deactivated_at, 388 + takedown_ref: value.takedown_ref, 389 + repo_root_cid: cid, 390 + }) 391 + }) 392 + .transpose() 393 + } 394 + 395 + pub fn list_repos_paginated( 396 + &self, 397 + cursor_user_hash: Option<u64>, 398 + limit: usize, 399 + ) -> Result<Vec<RepoListEntry>, MetastoreError> { 400 + const _: () = assert!(KeyTag::REPO_META.raw() < 0xFF); 401 + let upper = KeyTag::REPO_META.exclusive_prefix_bound(); 402 + 403 + let start = match cursor_user_hash { 404 + Some(cursor) => match cursor.checked_add(1) { 405 + Some(next) => repo_meta_key(UserHash::from_raw(next)), 406 + None => return Ok(Vec::new()), 407 + }, 408 + None => repo_meta_prefix(), 409 + }; 410 + 411 + self.repo_data 412 + .range(start.as_slice()..upper.as_slice()) 413 + .take(limit) 414 + .map(|guard| { 415 + let (k, v) = guard.into_inner().map_err(MetastoreError::Fjall)?; 416 + decode_list_entry_from_kv(&k, &v, &self.user_hashes) 417 + }) 418 + .collect() 419 + } 420 + 421 + fn resolve_user_hash(&self, user_id: Uuid) -> Result<UserHash, MetastoreError> { 422 + self.user_hashes 423 + .get(&user_id) 424 + .ok_or(MetastoreError::InvalidInput("unknown user_id")) 425 + } 426 + 427 + fn get_meta_value(&self, key: &[u8]) -> Result<RepoMetaValue, MetastoreError> { 428 + point_lookup( 429 + &self.repo_data, 430 + key, 431 + RepoMetaValue::deserialize, 432 + "invalid repo_meta value", 433 + )? 434 + .ok_or(MetastoreError::CorruptData("repo_meta not found")) 435 + } 436 + 437 + pub fn lookup_handle(&self, handle: &Handle) -> Result<Option<Uuid>, MetastoreError> { 438 + let handle_lower = handle.as_str().to_ascii_lowercase(); 439 + let key = handle_key(&handle_lower); 440 + 441 + match self 442 + .repo_data 443 + .get(key.as_slice()) 444 + .map_err(MetastoreError::Fjall)? 445 + { 446 + Some(raw) => { 447 + let user_hash = parse_handle_value(&raw)?; 448 + self.user_hashes 449 + .get_uuid(&user_hash) 450 + .ok_or(MetastoreError::CorruptData( 451 + "handle maps to unknown user_hash", 452 + )) 453 + .map(Some) 454 + } 455 + None => Ok(None), 456 + } 457 + } 458 + } 459 + 460 + #[derive(Debug, Clone)] 461 + pub struct RepoInfo { 462 + pub user_id: Uuid, 463 + pub repo_root_cid: CidLink, 464 + pub repo_rev: Option<String>, 465 + } 466 + 467 + #[derive(Debug, Clone)] 468 + pub struct RepoWithoutRevEntry { 469 + pub user_id: Uuid, 470 + pub repo_root_cid: CidLink, 471 + } 472 + 473 + #[derive(Debug, Clone)] 474 + pub struct RepoAccountEntry { 475 + pub user_id: Uuid, 476 + pub did: Did, 477 + pub deactivated_at: Option<DateTime<Utc>>, 478 + pub takedown_ref: Option<String>, 479 + pub repo_root_cid: Option<CidLink>, 480 + } 481 + 482 + #[derive(Debug, Clone)] 483 + pub struct RepoListEntry { 484 + pub user_id: Uuid, 485 + pub user_hash: UserHash, 486 + pub did: Option<String>, 487 + pub deactivated_at: Option<DateTime<Utc>>, 488 + pub takedown_ref: Option<String>, 489 + pub repo_root_cid: CidLink, 490 + pub repo_rev: Option<String>, 491 + } 492 + 493 + fn decode_list_entry_from_kv( 494 + key_bytes: &[u8], 495 + val_bytes: &[u8], 496 + user_hashes: &UserHashMap, 497 + ) -> Result<RepoListEntry, MetastoreError> { 498 + let value = RepoMetaValue::deserialize(val_bytes) 499 + .ok_or(MetastoreError::CorruptData("invalid repo_meta value"))?; 500 + let user_hash = parse_repo_meta_key_hash(key_bytes) 501 + .ok_or(MetastoreError::CorruptData("invalid repo_meta key"))?; 502 + let user_id = user_hashes 503 + .get_uuid(&user_hash) 504 + .ok_or(MetastoreError::CorruptData( 505 + "user_hash has no reverse mapping", 506 + ))?; 507 + let cid = bytes_to_cid_link(&value.repo_root_cid)?; 508 + let deactivated_at = value 509 + .deactivated_at_ms 510 + .and_then(|ms| i64::try_from(ms).ok()) 511 + .and_then(|ms| Utc.timestamp_millis_opt(ms).single()); 512 + Ok(RepoListEntry { 513 + user_id, 514 + user_hash, 515 + did: value.did, 516 + deactivated_at, 517 + takedown_ref: value.takedown_ref, 518 + repo_root_cid: cid, 519 + repo_rev: match value.repo_rev.is_empty() { 520 + true => None, 521 + false => Some(value.repo_rev), 522 + }, 523 + }) 524 + } 525 + 526 + fn decode_without_rev_entry( 527 + key_bytes: &[u8], 528 + value: &RepoMetaValue, 529 + user_hashes: &UserHashMap, 530 + ) -> Result<RepoWithoutRevEntry, MetastoreError> { 531 + let user_hash = parse_repo_meta_key_hash(key_bytes) 532 + .ok_or(MetastoreError::CorruptData("invalid repo_meta key"))?; 533 + let user_id = user_hashes 534 + .get_uuid(&user_hash) 535 + .ok_or(MetastoreError::CorruptData( 536 + "user_hash has no reverse mapping", 537 + ))?; 538 + let cid = bytes_to_cid_link(&value.repo_root_cid)?; 539 + Ok(RepoWithoutRevEntry { 540 + user_id, 541 + repo_root_cid: cid, 542 + }) 543 + } 544 + 545 + pub(crate) fn cid_link_to_bytes(cid_link: &CidLink) -> Result<Vec<u8>, MetastoreError> { 546 + let cid = cid_link.to_cid().ok_or(MetastoreError::InvalidInput( 547 + "CidLink does not contain a valid CID", 548 + ))?; 549 + Ok(cid.to_bytes()) 550 + } 551 + 552 + pub(crate) fn bytes_to_cid_link(bytes: &[u8]) -> Result<CidLink, MetastoreError> { 553 + let cid = cid::Cid::read_bytes(std::io::Cursor::new(bytes)) 554 + .map_err(|_| MetastoreError::CorruptData("invalid CID bytes in repo_meta"))?; 555 + Ok(CidLink::from_cid(&cid)) 556 + } 557 + 558 + fn parse_handle_value(raw: &[u8]) -> Result<UserHash, MetastoreError> { 559 + let bytes: [u8; 8] = raw 560 + .try_into() 561 + .map_err(|_| MetastoreError::CorruptData("handle value not 8 bytes"))?; 562 + Ok(UserHash::from_raw(u64::from_be_bytes(bytes))) 563 + } 564 + 565 + fn parse_repo_meta_key_hash(key_bytes: &[u8]) -> Option<UserHash> { 566 + let mut reader = KeyReader::new(key_bytes); 567 + let _tag = reader.tag()?; 568 + let hash = reader.u64()?; 569 + Some(UserHash::from_raw(hash)) 570 + } 571 + 572 + #[cfg(test)] 573 + mod tests { 574 + use super::*; 575 + use crate::metastore::{Metastore, MetastoreConfig}; 576 + 577 + fn test_config() -> MetastoreConfig { 578 + MetastoreConfig { 579 + cache_size_bytes: 64 * 1024 * 1024, 580 + } 581 + } 582 + 583 + fn test_cid_link(seed: u8) -> CidLink { 584 + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); 585 + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); 586 + let c = cid::Cid::new_v1(0x71, mh); 587 + CidLink::from_cid(&c) 588 + } 589 + 590 + fn open_fresh() -> (tempfile::TempDir, Metastore) { 591 + let dir = tempfile::TempDir::new().unwrap(); 592 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 593 + (dir, ms) 594 + } 595 + 596 + fn test_did(name: &str) -> Did { 597 + Did::from(format!("did:plc:{name}")) 598 + } 599 + 600 + fn test_handle(name: &str) -> Handle { 601 + Handle::from(format!("{name}.test.invalid")) 602 + } 603 + 604 + #[test] 605 + fn create_and_get_repo() { 606 + let (_dir, ms) = open_fresh(); 607 + let ops = ms.repo_ops(); 608 + let user_id = uuid::Uuid::new_v4(); 609 + let did = test_did("alice"); 610 + let handle = test_handle("alice"); 611 + let cid = test_cid_link(1); 612 + 613 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 614 + .unwrap(); 615 + 616 + let repo = ops.get_repo(user_id).unwrap().unwrap(); 617 + assert_eq!(repo.user_id, user_id); 618 + assert_eq!(repo.repo_root_cid, cid); 619 + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); 620 + } 621 + 622 + #[test] 623 + fn get_repo_returns_none_for_unknown() { 624 + let (_dir, ms) = open_fresh(); 625 + let ops = ms.repo_ops(); 626 + assert!(ops.get_repo(uuid::Uuid::new_v4()).unwrap().is_none()); 627 + } 628 + 629 + #[test] 630 + fn update_repo_root() { 631 + let (_dir, ms) = open_fresh(); 632 + let ops = ms.repo_ops(); 633 + let user_id = uuid::Uuid::new_v4(); 634 + let did = test_did("bob"); 635 + let handle = test_handle("bob"); 636 + let cid1 = test_cid_link(1); 637 + let cid2 = test_cid_link(2); 638 + 639 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid1, "rev1") 640 + .unwrap(); 641 + ops.update_repo_root(ms.database(), user_id, &cid2, "rev2") 642 + .unwrap(); 643 + 644 + let repo = ops.get_repo(user_id).unwrap().unwrap(); 645 + assert_eq!(repo.repo_root_cid, cid2); 646 + assert_eq!(repo.repo_rev.as_deref(), Some("rev2")); 647 + } 648 + 649 + #[test] 650 + fn update_repo_rev() { 651 + let (_dir, ms) = open_fresh(); 652 + let ops = ms.repo_ops(); 653 + let user_id = uuid::Uuid::new_v4(); 654 + let did = test_did("carol"); 655 + let handle = test_handle("carol"); 656 + let cid = test_cid_link(3); 657 + 658 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 659 + .unwrap(); 660 + ops.update_repo_rev(ms.database(), user_id, "rev_updated") 661 + .unwrap(); 662 + 663 + let repo = ops.get_repo(user_id).unwrap().unwrap(); 664 + assert_eq!(repo.repo_root_cid, cid); 665 + assert_eq!(repo.repo_rev.as_deref(), Some("rev_updated")); 666 + } 667 + 668 + #[test] 669 + fn delete_repo_removes_meta_and_handle() { 670 + let (_dir, ms) = open_fresh(); 671 + let ops = ms.repo_ops(); 672 + let user_id = uuid::Uuid::new_v4(); 673 + let did = test_did("dave"); 674 + let handle = test_handle("dave"); 675 + let cid = test_cid_link(4); 676 + 677 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 678 + .unwrap(); 679 + assert!(ops.get_repo(user_id).unwrap().is_some()); 680 + assert!(ops.lookup_handle(&handle).unwrap().is_some()); 681 + 682 + ops.delete_repo(ms.database(), user_id).unwrap(); 683 + assert!(ops.get_repo(user_id).unwrap().is_none()); 684 + assert!(ops.lookup_handle(&handle).unwrap().is_none()); 685 + } 686 + 687 + #[test] 688 + fn delete_repo_clears_user_hash_mapping() { 689 + let (_dir, ms) = open_fresh(); 690 + let ops = ms.repo_ops(); 691 + let user_id = uuid::Uuid::new_v4(); 692 + let did = test_did("mapped"); 693 + let handle = test_handle("mapped"); 694 + let cid = test_cid_link(4); 695 + 696 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 697 + .unwrap(); 698 + assert!(ms.user_hashes().get(&user_id).is_some()); 699 + 700 + ops.delete_repo(ms.database(), user_id).unwrap(); 701 + assert!(ms.user_hashes().get(&user_id).is_none()); 702 + } 703 + 704 + #[test] 705 + fn delete_repo_allows_recreate_with_same_did() { 706 + let (_dir, ms) = open_fresh(); 707 + let ops = ms.repo_ops(); 708 + let did = test_did("recreate"); 709 + let handle_a = test_handle("recreate_a"); 710 + let handle_b = test_handle("recreate_b"); 711 + let uid_a = uuid::Uuid::new_v4(); 712 + let uid_b = uuid::Uuid::new_v4(); 713 + let cid = test_cid_link(70); 714 + 715 + ops.create_repo(ms.database(), uid_a, &did, &handle_a, &cid, "r1") 716 + .unwrap(); 717 + ops.delete_repo(ms.database(), uid_a).unwrap(); 718 + 719 + ops.create_repo(ms.database(), uid_b, &did, &handle_b, &cid, "r2") 720 + .unwrap(); 721 + 722 + let repo = ops.get_repo(uid_b).unwrap().unwrap(); 723 + assert_eq!(repo.user_id, uid_b); 724 + assert_eq!(repo.repo_rev.as_deref(), Some("r2")); 725 + assert!(ops.get_repo(uid_a).unwrap().is_none()); 726 + } 727 + 728 + #[test] 729 + fn delete_nonexistent_user_returns_error() { 730 + let (_dir, ms) = open_fresh(); 731 + let ops = ms.repo_ops(); 732 + let result = ops.delete_repo(ms.database(), uuid::Uuid::new_v4()); 733 + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); 734 + } 735 + 736 + #[test] 737 + fn handle_lookup_case_insensitive() { 738 + let (_dir, ms) = open_fresh(); 739 + let ops = ms.repo_ops(); 740 + let user_id = uuid::Uuid::new_v4(); 741 + let did = test_did("eve"); 742 + let handle = test_handle("eve"); 743 + let cid = test_cid_link(5); 744 + 745 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 746 + .unwrap(); 747 + 748 + let upper_handle = Handle::from("EVE.TEST.INVALID".to_string()); 749 + let found = ops.lookup_handle(&upper_handle).unwrap(); 750 + assert_eq!(found, Some(user_id)); 751 + } 752 + 753 + #[test] 754 + fn get_repo_root_by_did() { 755 + let (_dir, ms) = open_fresh(); 756 + let ops = ms.repo_ops(); 757 + let user_id = uuid::Uuid::new_v4(); 758 + let did = test_did("frank"); 759 + let handle = test_handle("frank"); 760 + let cid = test_cid_link(6); 761 + 762 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 763 + .unwrap(); 764 + 765 + let root = ops.get_repo_root_by_did(&did).unwrap().unwrap(); 766 + assert_eq!(root, cid); 767 + 768 + let unknown = test_did("nobody"); 769 + assert!(ops.get_repo_root_by_did(&unknown).unwrap().is_none()); 770 + } 771 + 772 + #[test] 773 + fn get_repo_root_for_update() { 774 + let (_dir, ms) = open_fresh(); 775 + let ops = ms.repo_ops(); 776 + let user_id = uuid::Uuid::new_v4(); 777 + let did = test_did("grace"); 778 + let handle = test_handle("grace"); 779 + let cid = test_cid_link(7); 780 + 781 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 782 + .unwrap(); 783 + 784 + let root = ops.get_repo_root_for_update(user_id).unwrap().unwrap(); 785 + assert_eq!(root, cid); 786 + } 787 + 788 + #[test] 789 + fn count_repos() { 790 + let (_dir, ms) = open_fresh(); 791 + let ops = ms.repo_ops(); 792 + 793 + assert_eq!(ops.count_repos().unwrap(), 0); 794 + 795 + (0..5u8).for_each(|i| { 796 + let user_id = uuid::Uuid::new_v4(); 797 + let did = test_did(&format!("user{i}")); 798 + let handle = test_handle(&format!("user{i}")); 799 + let cid = test_cid_link(i); 800 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 801 + .unwrap(); 802 + }); 803 + 804 + assert_eq!(ops.count_repos().unwrap(), 5); 805 + } 806 + 807 + #[test] 808 + fn get_account_with_repo() { 809 + let (_dir, ms) = open_fresh(); 810 + let ops = ms.repo_ops(); 811 + let user_id = uuid::Uuid::new_v4(); 812 + let did = test_did("henry"); 813 + let handle = test_handle("henry"); 814 + let cid = test_cid_link(8); 815 + 816 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 817 + .unwrap(); 818 + 819 + let account = ops.get_account_with_repo(&did).unwrap().unwrap(); 820 + assert_eq!(account.user_id, user_id); 821 + assert_eq!(account.did, did); 822 + assert_eq!(account.repo_root_cid, Some(cid)); 823 + assert!(account.deactivated_at.is_none()); 824 + assert!(account.takedown_ref.is_none()); 825 + } 826 + 827 + #[test] 828 + fn list_repos_paginated_all() { 829 + let (_dir, ms) = open_fresh(); 830 + let ops = ms.repo_ops(); 831 + 832 + (0..5u8).for_each(|i| { 833 + let user_id = uuid::Uuid::new_v4(); 834 + let did = test_did(&format!("page{i}")); 835 + let handle = test_handle(&format!("page{i}")); 836 + let cid = test_cid_link(10 + i); 837 + ops.create_repo( 838 + ms.database(), 839 + user_id, 840 + &did, 841 + &handle, 842 + &cid, 843 + &format!("rev{i}"), 844 + ) 845 + .unwrap(); 846 + }); 847 + 848 + let all = ops.list_repos_paginated(None, 100).unwrap(); 849 + assert_eq!(all.len(), 5); 850 + 851 + all.iter() 852 + .zip(all.iter().skip(1)) 853 + .for_each(|(a, b)| assert!(a.user_hash.raw() < b.user_hash.raw())); 854 + } 855 + 856 + #[test] 857 + fn list_repos_paginated_with_cursor() { 858 + let (_dir, ms) = open_fresh(); 859 + let ops = ms.repo_ops(); 860 + 861 + (0..10u8).for_each(|i| { 862 + let user_id = uuid::Uuid::new_v4(); 863 + let did = test_did(&format!("cursor{i}")); 864 + let handle = test_handle(&format!("cursor{i}")); 865 + let cid = test_cid_link(20 + i); 866 + ops.create_repo( 867 + ms.database(), 868 + user_id, 869 + &did, 870 + &handle, 871 + &cid, 872 + &format!("rev{i}"), 873 + ) 874 + .unwrap(); 875 + }); 876 + 877 + let page1 = ops.list_repos_paginated(None, 3).unwrap(); 878 + assert_eq!(page1.len(), 3); 879 + 880 + let cursor = page1.last().unwrap().user_hash.raw(); 881 + let page2 = ops.list_repos_paginated(Some(cursor), 3).unwrap(); 882 + assert_eq!(page2.len(), 3); 883 + 884 + assert!(page2.first().unwrap().user_hash.raw() > cursor); 885 + 886 + let page2_cursor = page2.last().unwrap().user_hash.raw(); 887 + let page3 = ops.list_repos_paginated(Some(page2_cursor), 100).unwrap(); 888 + assert_eq!(page3.len(), 4); 889 + 890 + let total = page1.len() + page2.len() + page3.len(); 891 + assert_eq!(total, 10); 892 + } 893 + 894 + #[test] 895 + fn data_survives_reopen() { 896 + let dir = tempfile::TempDir::new().unwrap(); 897 + let user_id = uuid::Uuid::new_v4(); 898 + let did = test_did("persist"); 899 + let handle = test_handle("persist"); 900 + let cid = test_cid_link(99); 901 + 902 + { 903 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 904 + let ops = ms.repo_ops(); 905 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev_persist") 906 + .unwrap(); 907 + ms.persist().unwrap(); 908 + } 909 + 910 + { 911 + let ms = Metastore::open(dir.path(), test_config()).unwrap(); 912 + let ops = ms.repo_ops(); 913 + let repo = ops.get_repo(user_id).unwrap().unwrap(); 914 + assert_eq!(repo.repo_root_cid, cid); 915 + assert_eq!(repo.repo_rev.as_deref(), Some("rev_persist")); 916 + 917 + let found = ops.lookup_handle(&handle).unwrap(); 918 + assert_eq!(found, Some(user_id)); 919 + } 920 + } 921 + 922 + #[test] 923 + fn get_repos_without_rev() { 924 + let (_dir, ms) = open_fresh(); 925 + let ops = ms.repo_ops(); 926 + 927 + let uid_with = uuid::Uuid::new_v4(); 928 + let did_with = test_did("with_rev"); 929 + let handle_with = test_handle("with_rev"); 930 + ops.create_repo( 931 + ms.database(), 932 + uid_with, 933 + &did_with, 934 + &handle_with, 935 + &test_cid_link(40), 936 + "some_rev", 937 + ) 938 + .unwrap(); 939 + 940 + let uid_without = uuid::Uuid::new_v4(); 941 + let did_without = test_did("without_rev"); 942 + let handle_without = test_handle("without_rev"); 943 + ops.create_repo( 944 + ms.database(), 945 + uid_without, 946 + &did_without, 947 + &handle_without, 948 + &test_cid_link(41), 949 + "", 950 + ) 951 + .unwrap(); 952 + 953 + let result = ops.get_repos_without_rev(100).unwrap(); 954 + assert_eq!(result.len(), 1); 955 + assert_eq!(result[0].user_id, uid_without); 956 + } 957 + 958 + #[test] 959 + fn get_repo_root_cid_by_user_id() { 960 + let (_dir, ms) = open_fresh(); 961 + let ops = ms.repo_ops(); 962 + let user_id = uuid::Uuid::new_v4(); 963 + let did = test_did("root_cid"); 964 + let handle = test_handle("root_cid"); 965 + let cid = test_cid_link(50); 966 + 967 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") 968 + .unwrap(); 969 + 970 + let root = ops.get_repo_root_cid_by_user_id(user_id).unwrap().unwrap(); 971 + assert_eq!(root, cid); 972 + 973 + assert!( 974 + ops.get_repo_root_cid_by_user_id(uuid::Uuid::new_v4()) 975 + .unwrap() 976 + .is_none() 977 + ); 978 + } 979 + 980 + #[test] 981 + fn delete_only_removes_target_handle() { 982 + let (_dir, ms) = open_fresh(); 983 + let ops = ms.repo_ops(); 984 + 985 + let uid_a = uuid::Uuid::new_v4(); 986 + let did_a = test_did("keep_a"); 987 + let handle_a = test_handle("keep_a"); 988 + ops.create_repo( 989 + ms.database(), 990 + uid_a, 991 + &did_a, 992 + &handle_a, 993 + &test_cid_link(60), 994 + "r", 995 + ) 996 + .unwrap(); 997 + 998 + let uid_b = uuid::Uuid::new_v4(); 999 + let did_b = test_did("delete_b"); 1000 + let handle_b = test_handle("delete_b"); 1001 + ops.create_repo( 1002 + ms.database(), 1003 + uid_b, 1004 + &did_b, 1005 + &handle_b, 1006 + &test_cid_link(61), 1007 + "r", 1008 + ) 1009 + .unwrap(); 1010 + 1011 + ops.delete_repo(ms.database(), uid_b).unwrap(); 1012 + 1013 + assert!(ops.lookup_handle(&handle_a).unwrap().is_some()); 1014 + assert!(ops.lookup_handle(&handle_b).unwrap().is_none()); 1015 + assert!(ops.get_repo(uid_a).unwrap().is_some()); 1016 + } 1017 + 1018 + #[test] 1019 + fn create_repo_rejects_hash_collision() { 1020 + let (_dir, ms) = open_fresh(); 1021 + let ops = ms.repo_ops(); 1022 + 1023 + let uid_a = uuid::Uuid::new_v4(); 1024 + let did_a = test_did("collision_a"); 1025 + let handle_a = test_handle("collision_a"); 1026 + let cid = test_cid_link(80); 1027 + 1028 + ops.create_repo(ms.database(), uid_a, &did_a, &handle_a, &cid, "r1") 1029 + .unwrap(); 1030 + 1031 + let uid_b = uuid::Uuid::new_v4(); 1032 + let handle_b = test_handle("collision_b"); 1033 + 1034 + let result = ops.create_repo(ms.database(), uid_b, &did_a, &handle_b, &cid, "r2"); 1035 + 1036 + match result { 1037 + Ok(()) => { 1038 + let repo = ops.get_repo(uid_a).unwrap().unwrap(); 1039 + assert_eq!(repo.repo_root_cid, cid); 1040 + } 1041 + Err(MetastoreError::UserHashCollision { .. }) => { 1042 + let repo = ops.get_repo(uid_a).unwrap().unwrap(); 1043 + assert_eq!(repo.repo_root_cid, cid); 1044 + assert_eq!(repo.repo_rev.as_deref(), Some("r1")); 1045 + } 1046 + Err(e) => panic!("unexpected error: {e}"), 1047 + } 1048 + } 1049 + 1050 + #[test] 1051 + fn get_repo_meta_returns_raw_value() { 1052 + let (_dir, ms) = open_fresh(); 1053 + let ops = ms.repo_ops(); 1054 + let user_id = uuid::Uuid::new_v4(); 1055 + let did = test_did("meta_raw"); 1056 + let handle = test_handle("meta_raw"); 1057 + let cid = test_cid_link(81); 1058 + 1059 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev_meta") 1060 + .unwrap(); 1061 + 1062 + let (user_hash, value) = ops.get_repo_meta(user_id).unwrap().unwrap(); 1063 + assert_eq!(user_hash, UserHash::from_did(did.as_str())); 1064 + assert_eq!(value.repo_rev, "rev_meta"); 1065 + assert_eq!(value.handle, "meta_raw.test.invalid"); 1066 + assert_eq!(value.status, RepoStatus::Active); 1067 + } 1068 + 1069 + #[test] 1070 + fn get_repo_meta_returns_none_for_unknown() { 1071 + let (_dir, ms) = open_fresh(); 1072 + let ops = ms.repo_ops(); 1073 + assert!(ops.get_repo_meta(uuid::Uuid::new_v4()).unwrap().is_none()); 1074 + } 1075 + 1076 + #[test] 1077 + fn write_repo_meta_via_batch() { 1078 + let (_dir, ms) = open_fresh(); 1079 + let ops = ms.repo_ops(); 1080 + let user_id = uuid::Uuid::new_v4(); 1081 + let did = test_did("batch_write"); 1082 + let handle = test_handle("batch_write"); 1083 + let cid1 = test_cid_link(82); 1084 + let cid2 = test_cid_link(83); 1085 + 1086 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid1, "rev1") 1087 + .unwrap(); 1088 + 1089 + let (user_hash, mut value) = ops.get_repo_meta(user_id).unwrap().unwrap(); 1090 + value.repo_root_cid = cid_link_to_bytes(&cid2).unwrap(); 1091 + value.repo_rev = "rev2".to_string(); 1092 + 1093 + let mut batch = ms.database().batch(); 1094 + ops.write_repo_meta(&mut batch, user_hash, &value); 1095 + batch.commit().unwrap(); 1096 + 1097 + let repo = ops.get_repo(user_id).unwrap().unwrap(); 1098 + assert_eq!(repo.repo_root_cid, cid2); 1099 + assert_eq!(repo.repo_rev.as_deref(), Some("rev2")); 1100 + } 1101 + 1102 + #[test] 1103 + fn update_handle_swaps_lookup() { 1104 + let (_dir, ms) = open_fresh(); 1105 + let ops = ms.repo_ops(); 1106 + let user_id = uuid::Uuid::new_v4(); 1107 + let did = test_did("handle_swap"); 1108 + let old_handle = test_handle("old_name"); 1109 + let new_handle = test_handle("new_name"); 1110 + let cid = test_cid_link(84); 1111 + 1112 + ops.create_repo(ms.database(), user_id, &did, &old_handle, &cid, "r1") 1113 + .unwrap(); 1114 + assert!(ops.lookup_handle(&old_handle).unwrap().is_some()); 1115 + 1116 + ops.update_handle(ms.database(), user_id, &new_handle) 1117 + .unwrap(); 1118 + 1119 + assert!(ops.lookup_handle(&old_handle).unwrap().is_none()); 1120 + assert_eq!(ops.lookup_handle(&new_handle).unwrap(), Some(user_id)); 1121 + 1122 + let (_, meta) = ops.get_repo_meta(user_id).unwrap().unwrap(); 1123 + assert_eq!(meta.handle, "new_name.test.invalid"); 1124 + } 1125 + 1126 + #[test] 1127 + fn update_handle_case_insensitive() { 1128 + let (_dir, ms) = open_fresh(); 1129 + let ops = ms.repo_ops(); 1130 + let user_id = uuid::Uuid::new_v4(); 1131 + let did = test_did("handle_case"); 1132 + let handle = test_handle("original"); 1133 + let cid = test_cid_link(85); 1134 + 1135 + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "r1") 1136 + .unwrap(); 1137 + 1138 + let mixed_case = Handle::from("UPPER.TEST.INVALID".to_string()); 1139 + ops.update_handle(ms.database(), user_id, &mixed_case) 1140 + .unwrap(); 1141 + 1142 + let lower_lookup = Handle::from("upper.test.invalid".to_string()); 1143 + assert_eq!(ops.lookup_handle(&lower_lookup).unwrap(), Some(user_id)); 1144 + assert!(ops.lookup_handle(&handle).unwrap().is_none()); 1145 + } 1146 + }
+36
crates/tranquil-store/src/metastore/scan.rs
··· 1 + use fjall::Keyspace; 2 + 3 + use super::MetastoreError; 4 + 5 + pub fn count_prefix(keyspace: &Keyspace, prefix: &[u8]) -> Result<i64, MetastoreError> { 6 + keyspace.prefix(prefix).try_fold(0i64, |acc, guard| { 7 + guard.into_inner().map_err(MetastoreError::Fjall)?; 8 + Ok::<_, MetastoreError>(acc.saturating_add(1)) 9 + }) 10 + } 11 + 12 + pub fn delete_all_by_prefix( 13 + keyspace: &Keyspace, 14 + batch: &mut fjall::OwnedWriteBatch, 15 + prefix: &[u8], 16 + ) -> Result<(), MetastoreError> { 17 + keyspace.prefix(prefix).try_for_each(|guard| { 18 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 19 + batch.remove(keyspace, key_bytes.as_ref()); 20 + Ok::<(), MetastoreError>(()) 21 + }) 22 + } 23 + 24 + pub fn point_lookup<T>( 25 + keyspace: &Keyspace, 26 + key: &[u8], 27 + deserialize: impl FnOnce(&[u8]) -> Option<T>, 28 + corrupt_msg: &'static str, 29 + ) -> Result<Option<T>, MetastoreError> { 30 + match keyspace.get(key).map_err(MetastoreError::Fjall)? { 31 + Some(raw) => deserialize(&raw) 32 + .ok_or(MetastoreError::CorruptData(corrupt_msg)) 33 + .map(Some), 34 + None => Ok(None), 35 + } 36 + }
+560
crates/tranquil-store/src/metastore/user_block_ops.rs
··· 1 + use std::collections::HashSet; 2 + use std::sync::Arc; 3 + 4 + use fjall::Keyspace; 5 + use uuid::Uuid; 6 + 7 + use super::MetastoreError; 8 + use super::encoding::{KeyReader, exclusive_upper_bound}; 9 + use super::keys::UserHash; 10 + use super::scan::{count_prefix, delete_all_by_prefix}; 11 + use super::user_blocks::{user_block_key, user_block_rev_prefix, user_block_user_prefix}; 12 + use super::user_hash::UserHashMap; 13 + 14 + pub struct UserBlockOps { 15 + repo_data: Keyspace, 16 + user_hashes: Arc<UserHashMap>, 17 + } 18 + 19 + impl UserBlockOps { 20 + pub fn new(repo_data: Keyspace, user_hashes: Arc<UserHashMap>) -> Self { 21 + Self { 22 + repo_data, 23 + user_hashes, 24 + } 25 + } 26 + 27 + pub fn insert_user_blocks<C: AsRef<[u8]>>( 28 + &self, 29 + batch: &mut fjall::OwnedWriteBatch, 30 + user_hash: UserHash, 31 + block_cids: &[C], 32 + repo_rev: &str, 33 + ) -> Result<(), MetastoreError> { 34 + let existing: HashSet<Vec<u8>> = match block_cids.is_empty() { 35 + true => HashSet::new(), 36 + false => { 37 + let prefix = user_block_user_prefix(user_hash); 38 + self.repo_data 39 + .prefix(prefix.as_slice()) 40 + .filter_map(|guard| { 41 + let (key_bytes, _) = guard.into_inner().ok()?; 42 + extract_cid_from_key(&key_bytes).map(|c| c.to_vec()) 43 + }) 44 + .collect() 45 + } 46 + }; 47 + 48 + block_cids.iter().try_for_each(|cid| { 49 + let cid = cid.as_ref(); 50 + match cid.is_empty() { 51 + true => Err(MetastoreError::InvalidInput("block CID must not be empty")), 52 + false => { 53 + if !existing.contains(cid) { 54 + let key = user_block_key(user_hash, repo_rev, cid); 55 + batch.insert(&self.repo_data, key.as_slice(), []); 56 + } 57 + Ok(()) 58 + } 59 + } 60 + }) 61 + } 62 + 63 + pub fn delete_user_blocks<C: AsRef<[u8]>>( 64 + &self, 65 + batch: &mut fjall::OwnedWriteBatch, 66 + user_hash: UserHash, 67 + block_cids: &[C], 68 + rev: &str, 69 + ) -> Result<(), MetastoreError> { 70 + block_cids.iter().try_for_each(|cid| { 71 + let cid = cid.as_ref(); 72 + match cid.is_empty() { 73 + true => Err(MetastoreError::InvalidInput("block CID must not be empty")), 74 + false => { 75 + let key = user_block_key(user_hash, rev, cid); 76 + batch.remove(&self.repo_data, key.as_slice()); 77 + Ok(()) 78 + } 79 + } 80 + }) 81 + } 82 + 83 + pub fn delete_user_blocks_by_cid<C: AsRef<[u8]>>( 84 + &self, 85 + batch: &mut fjall::OwnedWriteBatch, 86 + user_hash: UserHash, 87 + block_cids: &[C], 88 + ) -> Result<(), MetastoreError> { 89 + match block_cids.is_empty() { 90 + true => Ok(()), 91 + false => { 92 + let cid_set: HashSet<&[u8]> = block_cids.iter().map(|c| c.as_ref()).collect(); 93 + let prefix = user_block_user_prefix(user_hash); 94 + self.repo_data 95 + .prefix(prefix.as_slice()) 96 + .try_for_each(|guard| { 97 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 98 + match extract_cid_from_key(&key_bytes) { 99 + Some(cid) if cid_set.contains(cid.as_slice()) => { 100 + batch.remove(&self.repo_data, key_bytes.as_ref()); 101 + Ok(()) 102 + } 103 + _ => Ok(()), 104 + } 105 + }) 106 + } 107 + } 108 + } 109 + 110 + pub fn delete_all_user_blocks( 111 + &self, 112 + batch: &mut fjall::OwnedWriteBatch, 113 + user_hash: UserHash, 114 + ) -> Result<(), MetastoreError> { 115 + let prefix = user_block_user_prefix(user_hash); 116 + delete_all_by_prefix(&self.repo_data, batch, prefix.as_slice()) 117 + } 118 + 119 + pub fn get_user_block_cids_since_rev( 120 + &self, 121 + user_id: Uuid, 122 + since_rev: &str, 123 + ) -> Result<Vec<Vec<u8>>, MetastoreError> { 124 + let user_hash = match self.user_hashes.get(&user_id) { 125 + Some(h) => h, 126 + None => return Ok(Vec::new()), 127 + }; 128 + 129 + let since_prefix = user_block_rev_prefix(user_hash, since_rev); 130 + let since_upper = exclusive_upper_bound(since_prefix.as_slice()) 131 + .expect("user block rev prefix always contains non-0xFF bytes"); 132 + 133 + let user_prefix = user_block_user_prefix(user_hash); 134 + let user_upper = exclusive_upper_bound(user_prefix.as_slice()) 135 + .expect("user block user prefix always contains non-0xFF bytes"); 136 + 137 + self.repo_data 138 + .range(since_upper.as_slice()..user_upper.as_slice()) 139 + .map(|guard| { 140 + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; 141 + extract_cid_from_key(&key_bytes) 142 + .ok_or(MetastoreError::CorruptData("invalid user_blocks key")) 143 + }) 144 + .collect() 145 + } 146 + 147 + pub fn find_unreferenced(&self, candidate_cids: &[Vec<u8>]) -> Vec<Vec<u8>> { 148 + match candidate_cids.is_empty() { 149 + true => Vec::new(), 150 + false => { 151 + let mut remaining: HashSet<Vec<u8>> = candidate_cids.iter().cloned().collect(); 152 + let tag_prefix = super::keys::KeyTag::USER_BLOCKS.raw(); 153 + let mut iter = self.repo_data.prefix([tag_prefix]); 154 + loop { 155 + match remaining.is_empty() { 156 + true => break, 157 + false => match iter.next() { 158 + None => break, 159 + Some(guard) => { 160 + if let Ok((key_bytes, _)) = guard.into_inner() 161 + && let Some(cid) = extract_cid_from_key(&key_bytes) 162 + { 163 + remaining.remove(&cid); 164 + } 165 + } 166 + }, 167 + } 168 + } 169 + remaining.into_iter().collect() 170 + } 171 + } 172 + } 173 + 174 + pub fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, MetastoreError> { 175 + let user_hash = match self.user_hashes.get(&user_id) { 176 + Some(h) => h, 177 + None => return Ok(0), 178 + }; 179 + let prefix = user_block_user_prefix(user_hash); 180 + count_prefix(&self.repo_data, prefix.as_slice()) 181 + } 182 + } 183 + 184 + fn extract_cid_from_key(key_bytes: &[u8]) -> Option<Vec<u8>> { 185 + let mut reader = KeyReader::new(key_bytes); 186 + reader.tag()?; 187 + reader.u64()?; 188 + reader.string()?; 189 + match reader.remaining().is_empty() { 190 + true => None, 191 + false => Some(reader.remaining().to_vec()), 192 + } 193 + } 194 + 195 + #[cfg(test)] 196 + mod tests { 197 + use super::*; 198 + use crate::metastore::{Metastore, MetastoreConfig}; 199 + 200 + fn open_fresh() -> (tempfile::TempDir, Metastore) { 201 + let dir = tempfile::TempDir::new().unwrap(); 202 + let ms = Metastore::open( 203 + dir.path(), 204 + MetastoreConfig { 205 + cache_size_bytes: 64 * 1024 * 1024, 206 + }, 207 + ) 208 + .unwrap(); 209 + (dir, ms) 210 + } 211 + 212 + fn setup_user(ms: &Metastore) -> (Uuid, UserHash) { 213 + let uuid = Uuid::new_v4(); 214 + let hash = UserHash::from_did("did:plc:testuser1"); 215 + let mut batch = ms.database().batch(); 216 + ms.user_hashes() 217 + .stage_insert(&mut batch, uuid, hash) 218 + .unwrap(); 219 + batch.commit().unwrap(); 220 + (uuid, hash) 221 + } 222 + 223 + #[test] 224 + fn insert_and_count() { 225 + let (_dir, ms) = open_fresh(); 226 + let (uuid, hash) = setup_user(&ms); 227 + let ops = ms.user_block_ops(); 228 + 229 + let cids = vec![vec![0x01, 0x71], vec![0x02, 0x72], vec![0x03, 0x73]]; 230 + 231 + let mut batch = ms.database().batch(); 232 + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") 233 + .unwrap(); 234 + batch.commit().unwrap(); 235 + 236 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); 237 + } 238 + 239 + #[test] 240 + fn get_since_rev_returns_later_revisions() { 241 + let (_dir, ms) = open_fresh(); 242 + let (uuid, hash) = setup_user(&ms); 243 + let ops = ms.user_block_ops(); 244 + 245 + let cids_abc = vec![vec![0x01], vec![0x02]]; 246 + let cids_def = vec![vec![0x03]]; 247 + 248 + let mut batch = ms.database().batch(); 249 + ops.insert_user_blocks(&mut batch, hash, &cids_abc, "abc") 250 + .unwrap(); 251 + ops.insert_user_blocks(&mut batch, hash, &cids_def, "def") 252 + .unwrap(); 253 + batch.commit().unwrap(); 254 + 255 + let since_abc = ops.get_user_block_cids_since_rev(uuid, "abc").unwrap(); 256 + assert_eq!(since_abc.len(), 1); 257 + assert_eq!(since_abc[0], vec![0x03]); 258 + 259 + let since_def = ops.get_user_block_cids_since_rev(uuid, "def").unwrap(); 260 + assert!(since_def.is_empty()); 261 + } 262 + 263 + #[test] 264 + fn get_since_rev_with_both_revisions() { 265 + let (_dir, ms) = open_fresh(); 266 + let (uuid, hash) = setup_user(&ms); 267 + let ops = ms.user_block_ops(); 268 + 269 + let cids_r1 = vec![vec![0x01]]; 270 + let cids_r2 = vec![vec![0x02]]; 271 + 272 + let mut batch = ms.database().batch(); 273 + ops.insert_user_blocks(&mut batch, hash, &cids_r1, "aaa") 274 + .unwrap(); 275 + ops.insert_user_blocks(&mut batch, hash, &cids_r2, "bbb") 276 + .unwrap(); 277 + batch.commit().unwrap(); 278 + 279 + let since_before = ops.get_user_block_cids_since_rev(uuid, "aaa").unwrap(); 280 + assert_eq!(since_before.len(), 1); 281 + assert_eq!(since_before[0], vec![0x02]); 282 + 283 + let all = ops.get_user_block_cids_since_rev(uuid, "").unwrap(); 284 + assert_eq!(all.len(), 2); 285 + } 286 + 287 + #[test] 288 + fn delete_blocks_at_rev() { 289 + let (_dir, ms) = open_fresh(); 290 + let (uuid, hash) = setup_user(&ms); 291 + let ops = ms.user_block_ops(); 292 + 293 + let cids = vec![vec![0x01], vec![0x02], vec![0x03]]; 294 + 295 + let mut batch = ms.database().batch(); 296 + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") 297 + .unwrap(); 298 + batch.commit().unwrap(); 299 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); 300 + 301 + let to_delete = vec![vec![0x01], vec![0x03]]; 302 + let mut batch = ms.database().batch(); 303 + ops.delete_user_blocks(&mut batch, hash, &to_delete, "rev1") 304 + .unwrap(); 305 + batch.commit().unwrap(); 306 + 307 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 1); 308 + } 309 + 310 + #[test] 311 + fn delete_wrong_rev_is_noop() { 312 + let (_dir, ms) = open_fresh(); 313 + let (uuid, hash) = setup_user(&ms); 314 + let ops = ms.user_block_ops(); 315 + 316 + let cids = vec![vec![0x01], vec![0x02]]; 317 + 318 + let mut batch = ms.database().batch(); 319 + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") 320 + .unwrap(); 321 + batch.commit().unwrap(); 322 + 323 + let mut batch = ms.database().batch(); 324 + ops.delete_user_blocks(&mut batch, hash, &cids, "wrong_rev") 325 + .unwrap(); 326 + batch.commit().unwrap(); 327 + 328 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 2); 329 + } 330 + 331 + #[test] 332 + fn delete_all_user_blocks_clears_all_revisions() { 333 + let (_dir, ms) = open_fresh(); 334 + let (uuid, hash) = setup_user(&ms); 335 + let ops = ms.user_block_ops(); 336 + 337 + let mut batch = ms.database().batch(); 338 + ops.insert_user_blocks(&mut batch, hash, &[vec![0x01], vec![0x02]], "rev1") 339 + .unwrap(); 340 + ops.insert_user_blocks(&mut batch, hash, &[vec![0x03]], "rev2") 341 + .unwrap(); 342 + batch.commit().unwrap(); 343 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); 344 + 345 + let mut batch = ms.database().batch(); 346 + ops.delete_all_user_blocks(&mut batch, hash).unwrap(); 347 + batch.commit().unwrap(); 348 + 349 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 0); 350 + } 351 + 352 + #[test] 353 + fn empty_rev_scan_returns_empty() { 354 + let (_dir, ms) = open_fresh(); 355 + let (uuid, _hash) = setup_user(&ms); 356 + let ops = ms.user_block_ops(); 357 + 358 + let result = ops.get_user_block_cids_since_rev(uuid, "anything").unwrap(); 359 + assert!(result.is_empty()); 360 + } 361 + 362 + #[test] 363 + fn unknown_user_returns_zero_count() { 364 + let (_dir, ms) = open_fresh(); 365 + let ops = ms.user_block_ops(); 366 + let unknown = Uuid::new_v4(); 367 + assert_eq!(ops.count_user_blocks(unknown).unwrap(), 0); 368 + } 369 + 370 + #[test] 371 + fn blocks_survive_reopen() { 372 + let dir = tempfile::TempDir::new().unwrap(); 373 + let uuid = Uuid::new_v4(); 374 + let hash = UserHash::from_did("did:plc:persist"); 375 + 376 + { 377 + let ms = Metastore::open( 378 + dir.path(), 379 + MetastoreConfig { 380 + cache_size_bytes: 64 * 1024 * 1024, 381 + }, 382 + ) 383 + .unwrap(); 384 + let mut batch = ms.database().batch(); 385 + ms.user_hashes() 386 + .stage_insert(&mut batch, uuid, hash) 387 + .unwrap(); 388 + batch.commit().unwrap(); 389 + 390 + let ops = ms.user_block_ops(); 391 + let cids = vec![vec![0xAA, 0xBB], vec![0xCC, 0xDD]]; 392 + let mut batch = ms.database().batch(); 393 + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") 394 + .unwrap(); 395 + batch.commit().unwrap(); 396 + ms.persist().unwrap(); 397 + } 398 + 399 + { 400 + let ms = Metastore::open( 401 + dir.path(), 402 + MetastoreConfig { 403 + cache_size_bytes: 64 * 1024 * 1024, 404 + }, 405 + ) 406 + .unwrap(); 407 + let ops = ms.user_block_ops(); 408 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 2); 409 + } 410 + } 411 + 412 + #[test] 413 + fn multiple_users_isolated() { 414 + let (_dir, ms) = open_fresh(); 415 + 416 + let uuid1 = Uuid::new_v4(); 417 + let hash1 = UserHash::from_did("did:plc:user1"); 418 + let uuid2 = Uuid::new_v4(); 419 + let hash2 = UserHash::from_did("did:plc:user2"); 420 + 421 + let mut batch = ms.database().batch(); 422 + ms.user_hashes() 423 + .stage_insert(&mut batch, uuid1, hash1) 424 + .unwrap(); 425 + ms.user_hashes() 426 + .stage_insert(&mut batch, uuid2, hash2) 427 + .unwrap(); 428 + batch.commit().unwrap(); 429 + 430 + let ops = ms.user_block_ops(); 431 + 432 + let mut batch = ms.database().batch(); 433 + ops.insert_user_blocks(&mut batch, hash1, &[vec![0x01], vec![0x02]], "rev1") 434 + .unwrap(); 435 + ops.insert_user_blocks(&mut batch, hash2, &[vec![0x03]], "rev1") 436 + .unwrap(); 437 + batch.commit().unwrap(); 438 + 439 + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 2); 440 + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); 441 + 442 + let mut batch = ms.database().batch(); 443 + ops.delete_user_blocks(&mut batch, hash1, &[vec![0x01]], "rev1") 444 + .unwrap(); 445 + batch.commit().unwrap(); 446 + 447 + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 1); 448 + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); 449 + } 450 + 451 + #[test] 452 + fn delete_all_does_not_affect_other_users() { 453 + let (_dir, ms) = open_fresh(); 454 + 455 + let uuid1 = Uuid::new_v4(); 456 + let hash1 = UserHash::from_did("did:plc:user1"); 457 + let uuid2 = Uuid::new_v4(); 458 + let hash2 = UserHash::from_did("did:plc:user2"); 459 + 460 + let mut batch = ms.database().batch(); 461 + ms.user_hashes() 462 + .stage_insert(&mut batch, uuid1, hash1) 463 + .unwrap(); 464 + ms.user_hashes() 465 + .stage_insert(&mut batch, uuid2, hash2) 466 + .unwrap(); 467 + batch.commit().unwrap(); 468 + 469 + let ops = ms.user_block_ops(); 470 + 471 + let mut batch = ms.database().batch(); 472 + ops.insert_user_blocks(&mut batch, hash1, &[vec![0x01]], "rev1") 473 + .unwrap(); 474 + ops.insert_user_blocks(&mut batch, hash2, &[vec![0x02]], "rev1") 475 + .unwrap(); 476 + batch.commit().unwrap(); 477 + 478 + let mut batch = ms.database().batch(); 479 + ops.delete_all_user_blocks(&mut batch, hash1).unwrap(); 480 + batch.commit().unwrap(); 481 + 482 + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 0); 483 + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); 484 + } 485 + 486 + #[test] 487 + fn cids_with_null_bytes_roundtrip_through_storage() { 488 + let (_dir, ms) = open_fresh(); 489 + let (uuid, hash) = setup_user(&ms); 490 + let ops = ms.user_block_ops(); 491 + 492 + let cids = vec![ 493 + vec![0x00, 0x00, 0x01], 494 + vec![0x00, 0x01, 0x00, 0x00], 495 + vec![0x00, 0x00], 496 + ]; 497 + 498 + let mut batch = ms.database().batch(); 499 + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") 500 + .unwrap(); 501 + batch.commit().unwrap(); 502 + 503 + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); 504 + 505 + let retrieved = ops.get_user_block_cids_since_rev(uuid, "").unwrap(); 506 + assert_eq!(retrieved.len(), 3); 507 + let mut expected = cids.clone(); 508 + expected.sort(); 509 + assert_eq!(retrieved, expected); 510 + } 511 + 512 + #[test] 513 + fn insert_empty_cid_is_rejected() { 514 + let (_dir, ms) = open_fresh(); 515 + let (_uuid, hash) = setup_user(&ms); 516 + let ops = ms.user_block_ops(); 517 + 518 + let cids = vec![vec![]]; 519 + let mut batch = ms.database().batch(); 520 + let result = ops.insert_user_blocks(&mut batch, hash, &cids, "rev1"); 521 + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); 522 + } 523 + 524 + #[test] 525 + fn delete_empty_cid_is_rejected() { 526 + let (_dir, ms) = open_fresh(); 527 + let (_uuid, hash) = setup_user(&ms); 528 + let ops = ms.user_block_ops(); 529 + 530 + let cids = vec![vec![]]; 531 + let mut batch = ms.database().batch(); 532 + let result = ops.delete_user_blocks(&mut batch, hash, &cids, "rev1"); 533 + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); 534 + } 535 + 536 + #[test] 537 + fn since_rev_nonexistent_returns_later_revisions() { 538 + let (_dir, ms) = open_fresh(); 539 + let (uuid, hash) = setup_user(&ms); 540 + let ops = ms.user_block_ops(); 541 + 542 + let mut batch = ms.database().batch(); 543 + ops.insert_user_blocks(&mut batch, hash, &[vec![0x01]], "aaa") 544 + .unwrap(); 545 + ops.insert_user_blocks(&mut batch, hash, &[vec![0x02]], "bbb") 546 + .unwrap(); 547 + ops.insert_user_blocks(&mut batch, hash, &[vec![0x03]], "ddd") 548 + .unwrap(); 549 + batch.commit().unwrap(); 550 + 551 + let result = ops.get_user_block_cids_since_rev(uuid, "aab").unwrap(); 552 + assert_eq!(result.len(), 2); 553 + assert_eq!(result[0], vec![0x02]); 554 + assert_eq!(result[1], vec![0x03]); 555 + 556 + let result = ops.get_user_block_cids_since_rev(uuid, "ccc").unwrap(); 557 + assert_eq!(result.len(), 1); 558 + assert_eq!(result[0], vec![0x03]); 559 + } 560 + }
+112
crates/tranquil-store/src/metastore/user_blocks.rs
··· 1 + use smallvec::SmallVec; 2 + 3 + use super::encoding::KeyBuilder; 4 + use super::keys::{KeyTag, UserHash}; 5 + 6 + pub fn user_block_key(user_hash: UserHash, rev: &str, cid_bytes: &[u8]) -> SmallVec<[u8; 128]> { 7 + KeyBuilder::new() 8 + .tag(KeyTag::USER_BLOCKS) 9 + .u64(user_hash.raw()) 10 + .string(rev) 11 + .raw(cid_bytes) 12 + .build() 13 + } 14 + 15 + pub fn user_block_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { 16 + KeyBuilder::new() 17 + .tag(KeyTag::USER_BLOCKS) 18 + .u64(user_hash.raw()) 19 + .build() 20 + } 21 + 22 + pub fn user_block_rev_prefix(user_hash: UserHash, rev: &str) -> SmallVec<[u8; 128]> { 23 + KeyBuilder::new() 24 + .tag(KeyTag::USER_BLOCKS) 25 + .u64(user_hash.raw()) 26 + .string(rev) 27 + .build() 28 + } 29 + 30 + #[cfg(test)] 31 + mod tests { 32 + use super::*; 33 + use crate::metastore::encoding::KeyReader; 34 + 35 + #[test] 36 + fn user_block_key_roundtrip() { 37 + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); 38 + let cid = [0x01, 0x71, 0x12, 0x20, 0xAB]; 39 + let key = user_block_key(hash, "3k2abcde", &cid); 40 + 41 + let mut reader = KeyReader::new(&key); 42 + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); 43 + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); 44 + assert_eq!(reader.string(), Some("3k2abcde".to_string())); 45 + assert_eq!(reader.remaining(), &cid); 46 + } 47 + 48 + #[test] 49 + fn keys_sort_by_user_then_rev_then_cid() { 50 + let h1 = UserHash::from_raw(1); 51 + let h2 = UserHash::from_raw(2); 52 + 53 + let k1 = user_block_key(h1, "abc", &[0x01]); 54 + let k2 = user_block_key(h1, "abc", &[0x02]); 55 + let k3 = user_block_key(h1, "def", &[0x01]); 56 + let k4 = user_block_key(h2, "abc", &[0x01]); 57 + 58 + assert!(k1.as_slice() < k2.as_slice()); 59 + assert!(k2.as_slice() < k3.as_slice()); 60 + assert!(k3.as_slice() < k4.as_slice()); 61 + } 62 + 63 + #[test] 64 + fn user_prefix_is_prefix_of_rev_prefix() { 65 + let hash = UserHash::from_raw(42); 66 + let user_pfx = user_block_user_prefix(hash); 67 + let rev_pfx = user_block_rev_prefix(hash, "some_rev"); 68 + assert!(rev_pfx.as_slice().starts_with(user_pfx.as_slice())); 69 + } 70 + 71 + #[test] 72 + fn rev_prefix_is_prefix_of_full_key() { 73 + let hash = UserHash::from_raw(42); 74 + let rev_pfx = user_block_rev_prefix(hash, "some_rev"); 75 + let full = user_block_key(hash, "some_rev", &[0x01, 0x02]); 76 + assert!(full.as_slice().starts_with(rev_pfx.as_slice())); 77 + } 78 + 79 + #[test] 80 + fn cid_with_null_bytes_roundtrips() { 81 + let hash = UserHash::from_raw(99); 82 + let cid = [0x00, 0x01, 0x00, 0xFF]; 83 + let key = user_block_key(hash, "rev1", &cid); 84 + 85 + let mut reader = KeyReader::new(&key); 86 + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); 87 + assert_eq!(reader.u64(), Some(99)); 88 + assert_eq!(reader.string(), Some("rev1".to_string())); 89 + assert_eq!(reader.remaining(), &cid); 90 + } 91 + 92 + #[test] 93 + fn cid_with_double_null_bytes_roundtrips() { 94 + let hash = UserHash::from_raw(99); 95 + let cid = [0x00, 0x00, 0x01, 0x00, 0x00]; 96 + let key = user_block_key(hash, "rev1", &cid); 97 + 98 + let mut reader = KeyReader::new(&key); 99 + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); 100 + assert_eq!(reader.u64(), Some(99)); 101 + assert_eq!(reader.string(), Some("rev1".to_string())); 102 + assert_eq!(reader.remaining(), &cid); 103 + } 104 + 105 + #[test] 106 + fn empty_cid_produces_key_equal_to_rev_prefix() { 107 + let hash = UserHash::from_raw(42); 108 + let rev_pfx = user_block_rev_prefix(hash, "rev1"); 109 + let full = user_block_key(hash, "rev1", &[]); 110 + assert_eq!(full.as_slice(), rev_pfx.as_slice()); 111 + } 112 + }
+437
crates/tranquil-store/src/metastore/user_hash.rs
··· 1 + use dashmap::DashMap; 2 + use fjall::Keyspace; 3 + use parking_lot::Mutex; 4 + use uuid::Uuid; 5 + 6 + use super::MetastoreError; 7 + use super::encoding::KeyBuilder; 8 + use super::keys::{KeyTag, UserHash}; 9 + 10 + pub struct UserHashMap { 11 + cache: DashMap<Uuid, UserHash>, 12 + reverse: DashMap<UserHash, Uuid>, 13 + repo_data: Keyspace, 14 + write_guard: Mutex<()>, 15 + } 16 + 17 + impl UserHashMap { 18 + pub fn new(repo_data: Keyspace) -> Self { 19 + Self { 20 + cache: DashMap::new(), 21 + reverse: DashMap::new(), 22 + repo_data, 23 + write_guard: Mutex::new(()), 24 + } 25 + } 26 + 27 + pub fn load_all(&self) -> Result<usize, MetastoreError> { 28 + let prefix = [KeyTag::USER_MAP.raw()]; 29 + let mut count = 0usize; 30 + 31 + self.repo_data.prefix(prefix).try_for_each(|guard| { 32 + let (key_bytes, value_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; 33 + 34 + let uuid_bytes: [u8; 16] = key_bytes 35 + .get(1..17) 36 + .and_then(|s| <[u8; 16]>::try_from(s).ok()) 37 + .ok_or(MetastoreError::CorruptData( 38 + "user_map key too short for UUID", 39 + ))?; 40 + 41 + let hash_bytes: [u8; 8] = <[u8; 8]>::try_from(value_bytes.as_ref()) 42 + .map_err(|_| MetastoreError::CorruptData("user_map value not 8 bytes"))?; 43 + 44 + let uuid = Uuid::from_bytes(uuid_bytes); 45 + let user_hash = UserHash::from_raw(u64::from_be_bytes(hash_bytes)); 46 + 47 + let existing = self.reverse.get(&user_hash).map(|r| *r); 48 + if let Some(existing_uuid) = existing 49 + && existing_uuid != uuid 50 + { 51 + tracing::error!( 52 + existing_uuid = %existing_uuid, 53 + new_uuid = %uuid, 54 + user_hash = %user_hash, 55 + "user hash collision in persisted data" 56 + ); 57 + return Err(MetastoreError::UserHashCollision { 58 + hash: user_hash, 59 + existing_uuid, 60 + new_uuid: uuid, 61 + }); 62 + } 63 + 64 + self.cache.insert(uuid, user_hash); 65 + self.reverse.insert(user_hash, uuid); 66 + count = count.saturating_add(1); 67 + 68 + if count.is_multiple_of(100_000) { 69 + tracing::info!(count, "loading user hash mappings"); 70 + } 71 + 72 + Ok::<_, MetastoreError>(()) 73 + })?; 74 + 75 + Ok(count) 76 + } 77 + 78 + pub fn stage_insert( 79 + &self, 80 + batch: &mut fjall::OwnedWriteBatch, 81 + uuid: Uuid, 82 + user_hash: UserHash, 83 + ) -> Result<(), MetastoreError> { 84 + let _guard = self.write_guard.lock(); 85 + 86 + let existing = self.reverse.get(&user_hash).map(|r| *r); 87 + if let Some(existing_uuid) = existing 88 + && existing_uuid != uuid 89 + { 90 + tracing::error!( 91 + existing_uuid = %existing_uuid, 92 + new_uuid = %uuid, 93 + user_hash = %user_hash, 94 + "user hash collision detected" 95 + ); 96 + return Err(MetastoreError::UserHashCollision { 97 + hash: user_hash, 98 + existing_uuid, 99 + new_uuid: uuid, 100 + }); 101 + } 102 + 103 + let forward_key = KeyBuilder::new() 104 + .tag(KeyTag::USER_MAP) 105 + .fixed(uuid.as_bytes()) 106 + .build(); 107 + 108 + let reverse_key = KeyBuilder::new() 109 + .tag(KeyTag::USER_MAP_REVERSE) 110 + .u64(user_hash.raw()) 111 + .build(); 112 + 113 + batch.insert( 114 + &self.repo_data, 115 + forward_key.as_slice(), 116 + user_hash.raw().to_be_bytes(), 117 + ); 118 + batch.insert( 119 + &self.repo_data, 120 + reverse_key.as_slice(), 121 + uuid.as_bytes().as_slice(), 122 + ); 123 + 124 + self.cache.insert(uuid, user_hash); 125 + self.reverse.insert(user_hash, uuid); 126 + 127 + Ok(()) 128 + } 129 + 130 + pub fn rollback_insert(&self, uuid: &Uuid, user_hash: &UserHash) { 131 + self.cache.remove(uuid); 132 + self.reverse.remove(user_hash); 133 + } 134 + 135 + pub fn stage_remove( 136 + &self, 137 + batch: &mut fjall::OwnedWriteBatch, 138 + uuid: &Uuid, 139 + ) -> Option<UserHash> { 140 + let _guard = self.write_guard.lock(); 141 + 142 + let (_, user_hash) = self.cache.remove(uuid)?; 143 + self.reverse.remove(&user_hash); 144 + 145 + let forward_key = KeyBuilder::new() 146 + .tag(KeyTag::USER_MAP) 147 + .fixed(uuid.as_bytes()) 148 + .build(); 149 + let reverse_key = KeyBuilder::new() 150 + .tag(KeyTag::USER_MAP_REVERSE) 151 + .u64(user_hash.raw()) 152 + .build(); 153 + 154 + batch.remove(&self.repo_data, forward_key.as_slice()); 155 + batch.remove(&self.repo_data, reverse_key.as_slice()); 156 + 157 + Some(user_hash) 158 + } 159 + 160 + pub fn rollback_remove(&self, uuid: Uuid, user_hash: UserHash) { 161 + self.cache.insert(uuid, user_hash); 162 + self.reverse.insert(user_hash, uuid); 163 + } 164 + 165 + pub fn get(&self, uuid: &Uuid) -> Option<UserHash> { 166 + self.cache.get(uuid).map(|r| *r) 167 + } 168 + 169 + pub fn get_uuid(&self, user_hash: &UserHash) -> Option<Uuid> { 170 + self.reverse.get(user_hash).map(|r| *r) 171 + } 172 + 173 + pub fn len(&self) -> usize { 174 + self.cache.len() 175 + } 176 + 177 + pub fn is_empty(&self) -> bool { 178 + self.cache.is_empty() 179 + } 180 + } 181 + 182 + #[cfg(test)] 183 + mod tests { 184 + use super::*; 185 + 186 + fn open_temp() -> (tempfile::TempDir, fjall::Database, Keyspace) { 187 + let dir = tempfile::TempDir::new().unwrap(); 188 + let db = fjall::Database::builder(dir.path()).open().unwrap(); 189 + let ks = db 190 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 191 + .unwrap(); 192 + (dir, db, ks) 193 + } 194 + 195 + #[test] 196 + fn insert_and_lookup() { 197 + let (_dir, db, ks) = open_temp(); 198 + let map = UserHashMap::new(ks); 199 + 200 + let uuid = Uuid::new_v4(); 201 + let hash = UserHash::from_did("did:plc:test123"); 202 + 203 + let mut batch = db.batch(); 204 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 205 + batch.commit().unwrap(); 206 + 207 + assert_eq!(map.get(&uuid), Some(hash)); 208 + assert_eq!(map.get_uuid(&hash), Some(uuid)); 209 + assert_eq!(map.len(), 1); 210 + } 211 + 212 + #[test] 213 + fn cache_populated_at_stage_time() { 214 + let (_dir, db, ks) = open_temp(); 215 + let map = UserHashMap::new(ks); 216 + 217 + let uuid = Uuid::new_v4(); 218 + let hash = UserHash::from_did("did:plc:staged_only"); 219 + 220 + let mut batch = db.batch(); 221 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 222 + 223 + assert_eq!(map.get(&uuid), Some(hash)); 224 + assert_eq!(map.get_uuid(&hash), Some(uuid)); 225 + assert_eq!(map.len(), 1); 226 + } 227 + 228 + #[test] 229 + fn rollback_removes_from_cache() { 230 + let (_dir, db, ks) = open_temp(); 231 + let map = UserHashMap::new(ks); 232 + 233 + let uuid = Uuid::new_v4(); 234 + let hash = UserHash::from_did("did:plc:rollback"); 235 + 236 + let mut batch = db.batch(); 237 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 238 + assert_eq!(map.len(), 1); 239 + 240 + map.rollback_insert(&uuid, &hash); 241 + assert!(map.is_empty()); 242 + assert_eq!(map.get(&uuid), None); 243 + assert_eq!(map.get_uuid(&hash), None); 244 + 245 + drop(batch); 246 + } 247 + 248 + #[test] 249 + fn load_all_after_reopen() { 250 + let dir = tempfile::TempDir::new().unwrap(); 251 + let uuid = Uuid::new_v4(); 252 + let hash = UserHash::from_did("did:plc:persist"); 253 + 254 + { 255 + let db = fjall::Database::builder(dir.path()).open().unwrap(); 256 + let ks = db 257 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 258 + .unwrap(); 259 + let map = UserHashMap::new(ks); 260 + let mut batch = db.batch(); 261 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 262 + batch.commit().unwrap(); 263 + db.persist(fjall::PersistMode::SyncData).unwrap(); 264 + } 265 + 266 + { 267 + let db = fjall::Database::builder(dir.path()).open().unwrap(); 268 + let ks = db 269 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 270 + .unwrap(); 271 + let map = UserHashMap::new(ks); 272 + let count = map.load_all().unwrap(); 273 + assert_eq!(count, 1); 274 + assert_eq!(map.get(&uuid), Some(hash)); 275 + assert_eq!(map.get_uuid(&hash), Some(uuid)); 276 + } 277 + } 278 + 279 + #[test] 280 + fn multiple_users() { 281 + let (_dir, db, ks) = open_temp(); 282 + let map = UserHashMap::new(ks); 283 + 284 + let pairs: Vec<_> = (0..10) 285 + .map(|i| { 286 + let uuid = Uuid::new_v4(); 287 + let hash = UserHash::from_did(&format!("did:plc:user{i}")); 288 + (uuid, hash) 289 + }) 290 + .collect(); 291 + 292 + let mut batch = db.batch(); 293 + pairs.iter().for_each(|(uuid, hash)| { 294 + map.stage_insert(&mut batch, *uuid, *hash).unwrap(); 295 + }); 296 + batch.commit().unwrap(); 297 + 298 + assert_eq!(map.len(), 10); 299 + pairs.iter().for_each(|(uuid, hash)| { 300 + assert_eq!(map.get(uuid), Some(*hash)); 301 + assert_eq!(map.get_uuid(hash), Some(*uuid)); 302 + }); 303 + } 304 + 305 + #[test] 306 + fn stage_insert_idempotent_for_same_uuid() { 307 + let (_dir, db, ks) = open_temp(); 308 + let map = UserHashMap::new(ks); 309 + 310 + let uuid = Uuid::new_v4(); 311 + let hash = UserHash::from_did("did:plc:same"); 312 + 313 + let mut batch = db.batch(); 314 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 315 + batch.commit().unwrap(); 316 + 317 + let mut batch2 = db.batch(); 318 + map.stage_insert(&mut batch2, uuid, hash).unwrap(); 319 + } 320 + 321 + #[test] 322 + fn stage_insert_rejects_collision() { 323 + let (_dir, db, ks) = open_temp(); 324 + let map = UserHashMap::new(ks); 325 + 326 + let uuid_a = Uuid::new_v4(); 327 + let uuid_b = Uuid::new_v4(); 328 + let same_hash = UserHash::from_raw(0xDEAD_BEEF); 329 + 330 + let mut batch = db.batch(); 331 + map.stage_insert(&mut batch, uuid_a, same_hash).unwrap(); 332 + batch.commit().unwrap(); 333 + 334 + let mut batch2 = db.batch(); 335 + let result = map.stage_insert(&mut batch2, uuid_b, same_hash); 336 + assert!(matches!( 337 + result, 338 + Err(MetastoreError::UserHashCollision { .. }) 339 + )); 340 + } 341 + 342 + #[test] 343 + fn stage_remove_clears_cache_and_persists() { 344 + let dir = tempfile::TempDir::new().unwrap(); 345 + let uuid = Uuid::new_v4(); 346 + let hash = UserHash::from_did("did:plc:removable"); 347 + 348 + let db = fjall::Database::builder(dir.path()).open().unwrap(); 349 + let ks = db 350 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 351 + .unwrap(); 352 + let map = UserHashMap::new(ks); 353 + 354 + let mut batch = db.batch(); 355 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 356 + batch.commit().unwrap(); 357 + assert_eq!(map.len(), 1); 358 + 359 + let mut remove_batch = db.batch(); 360 + let removed = map.stage_remove(&mut remove_batch, &uuid); 361 + assert_eq!(removed, Some(hash)); 362 + remove_batch.commit().unwrap(); 363 + 364 + assert!(map.is_empty()); 365 + assert_eq!(map.get(&uuid), None); 366 + assert_eq!(map.get_uuid(&hash), None); 367 + 368 + db.persist(fjall::PersistMode::SyncData).unwrap(); 369 + drop(map); 370 + drop(db); 371 + 372 + let db2 = fjall::Database::builder(dir.path()).open().unwrap(); 373 + let ks2 = db2 374 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) 375 + .unwrap(); 376 + let map2 = UserHashMap::new(ks2); 377 + assert_eq!(map2.load_all().unwrap(), 0); 378 + } 379 + 380 + #[test] 381 + fn stage_remove_returns_none_for_unknown() { 382 + let (_dir, db, ks) = open_temp(); 383 + let map = UserHashMap::new(ks); 384 + let mut batch = db.batch(); 385 + assert_eq!(map.stage_remove(&mut batch, &Uuid::new_v4()), None); 386 + } 387 + 388 + #[test] 389 + fn rollback_remove_restores_cache() { 390 + let (_dir, db, ks) = open_temp(); 391 + let map = UserHashMap::new(ks); 392 + 393 + let uuid = Uuid::new_v4(); 394 + let hash = UserHash::from_did("did:plc:rollback_remove"); 395 + 396 + let mut batch = db.batch(); 397 + map.stage_insert(&mut batch, uuid, hash).unwrap(); 398 + batch.commit().unwrap(); 399 + 400 + let mut remove_batch = db.batch(); 401 + map.stage_remove(&mut remove_batch, &uuid); 402 + assert!(map.is_empty()); 403 + 404 + map.rollback_remove(uuid, hash); 405 + assert_eq!(map.get(&uuid), Some(hash)); 406 + assert_eq!(map.get_uuid(&hash), Some(uuid)); 407 + assert_eq!(map.len(), 1); 408 + 409 + drop(remove_batch); 410 + } 411 + 412 + #[test] 413 + fn stage_remove_then_reinsert_same_did() { 414 + let (_dir, db, ks) = open_temp(); 415 + let map = UserHashMap::new(ks); 416 + 417 + let uuid_a = Uuid::new_v4(); 418 + let hash = UserHash::from_did("did:plc:reinsert"); 419 + 420 + let mut batch = db.batch(); 421 + map.stage_insert(&mut batch, uuid_a, hash).unwrap(); 422 + batch.commit().unwrap(); 423 + 424 + let mut remove_batch = db.batch(); 425 + map.stage_remove(&mut remove_batch, &uuid_a); 426 + remove_batch.commit().unwrap(); 427 + 428 + let uuid_b = Uuid::new_v4(); 429 + let mut batch2 = db.batch(); 430 + map.stage_insert(&mut batch2, uuid_b, hash).unwrap(); 431 + batch2.commit().unwrap(); 432 + 433 + assert_eq!(map.get(&uuid_b), Some(hash)); 434 + assert_eq!(map.get_uuid(&hash), Some(uuid_b)); 435 + assert_eq!(map.get(&uuid_a), None); 436 + } 437 + }
+34
example.toml
··· 180 180 # Can also be specified via environment variable `S3_ENDPOINT`. 181 181 #s3_endpoint = 182 182 183 + # Repository backend: `postgres` by default, or `tranquil-store`, our embedded db. 184 + # tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS. 185 + # 186 + # Can also be specified via environment variable `REPO_BACKEND`. 187 + # 188 + # Default value: "postgres" 189 + #repo_backend = "postgres" 190 + 191 + [tranquil_store] 192 + # Directory for tranquil-store data: the metastore, eventlog, and blockstore. 193 + # 194 + # Can also be specified via environment variable `TRANQUIL_STORE_DATA_DIR`. 195 + # 196 + # Default value: "/var/lib/tranquil-pds/store" 197 + #data_dir = "/var/lib/tranquil-pds/store" 198 + 199 + # Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset. 200 + # 201 + # Can also be specified via environment variable `TRANQUIL_STORE_MEMORY_BUDGET_MB`. 202 + #memory_budget_mb = 203 + 204 + # Number of handler threads. Defaults to available_parallelism / 2. 205 + # 206 + # Can also be specified via environment variable `TRANQUIL_STORE_HANDLER_THREADS`. 207 + #handler_threads = 208 + 183 209 [cache] 184 210 # Cache backend: `ripple` (default, built-in gossip) or `valkey`. 185 211 # ··· 482 508 # 483 509 # Default value: 3600 484 510 #delete_check_interval_secs = 3600 511 + 512 + # Interval in seconds between block garbage collection cycles. 513 + # Reclaims orphaned ipld blocks that were stored but never committed. 514 + # 515 + # Can also be specified via environment variable `BLOCK_GC_INTERVAL_SECS`. 516 + # 517 + # Default value: 21600 518 + #block_gc_interval_secs = 21600