+57 -1

src/replication/mst-proof.test.ts

reviewed

··· 7 7 import { RepoManager } from "../repo-manager.js"; 8 8 import type { Config } from "../config.js"; 9 9 import { readCarWithRoot } from "@atproto/repo"; 10 10 - import { generateMstProof, verifyMstProof } from "./mst-proof.js"; 10 10 + import { generateMstProof, verifyMstProof, extractAllRecordPaths } from "./mst-proof.js"; 11 11 12 12 function testConfig(dataDir: string): Config { 13 13 return { ··· 583 583 ); 584 584 expect(missingVerify.valid).toBe(true); 585 585 expect(missingVerify.found).toBe(false); 586 586 + }); 587 587 + 588 588 + // ============================================ 589 589 + // extractAllRecordPaths 590 590 + // ============================================ 591 591 + 592 592 + it("extractAllRecordPaths returns all record paths", async () => { 593 593 + await repoManager.createRecord("app.bsky.feed.post", undefined, { 594 594 + $type: "app.bsky.feed.post", 595 595 + text: "Post one", 596 596 + createdAt: new Date().toISOString(), 597 597 + }); 598 598 + await repoManager.createRecord("app.bsky.feed.post", undefined, { 599 599 + $type: "app.bsky.feed.post", 600 600 + text: "Post two", 601 601 + createdAt: new Date().toISOString(), 602 602 + }); 603 603 + await repoManager.putRecord("app.bsky.actor.profile", "self", { 604 604 + $type: "app.bsky.actor.profile", 605 605 + displayName: "Test User", 606 606 + }); 607 607 + 608 608 + const rootCid = await getRepoRootCid(); 609 609 + 610 610 + const paths = await extractAllRecordPaths(ipfsService, rootCid); 611 611 + 612 612 + // Should find all three records 613 613 + expect(paths.length).toBe(3); 614 614 + expect(paths.filter((p) => p.startsWith("app.bsky.feed.post/")).length).toBe(2); 615 615 + expect(paths).toContain("app.bsky.actor.profile/self"); 616 616 + }); 617 617 + 618 618 + it("extractAllRecordPaths returns sorted paths for a large repo", async () => { 619 619 + for (let i = 0; i < 30; i++) { 620 620 + await repoManager.createRecord("app.bsky.feed.post", undefined, { 621 621 + $type: "app.bsky.feed.post", 622 622 + text: `Post ${i}`, 623 623 + createdAt: new Date().toISOString(), 624 624 + }); 625 625 + } 626 626 + 627 627 + const rootCid = await getRepoRootCid(); 628 628 + const paths = await extractAllRecordPaths(ipfsService, rootCid); 629 629 + 630 630 + expect(paths.length).toBe(30); 631 631 + // MST in-order walk produces sorted keys 632 632 + const sorted = [...paths].sort(); 633 633 + expect(paths).toEqual(sorted); 634 634 + }); 635 635 + 636 636 + it("extractAllRecordPaths returns empty for missing commit", async () => { 637 637 + const paths = await extractAllRecordPaths( 638 638 + ipfsService, 639 639 + "bafyreig6mxqmjlb7yjbhhhz6vqmtiw4kgipvhqoowdkggjlpzpd5tcm4", 640 640 + ); 641 641 + expect(paths).toEqual([]); 586 642 }); 587 643 });

+53

src/replication/mst-proof.ts

reviewed

··· 120 120 return result; 121 121 } 122 122 123 123 + // ---------- Full MST walk ---------- 124 124 + 125 125 + /** 126 126 + * Walk an MST node recursively, collecting all record paths (keys). 127 127 + */ 128 128 + async function walkMstNode( 129 129 + blockStore: BlockStore, 130 130 + nodeCid: CID, 131 131 + paths: string[], 132 132 + ): Promise<void> { 133 133 + const bytes = await blockStore.getBlock(nodeCid.toString()); 134 134 + if (!bytes) return; 135 135 + 136 136 + const nodeData = decodeNodeData(bytes); 137 137 + const entries = expandEntries(nodeData); 138 138 + 139 139 + // Visit left subtree first 140 140 + if (nodeData.l) { 141 141 + await walkMstNode(blockStore, nodeData.l, paths); 142 142 + } 143 143 + 144 144 + // Visit each entry: collect key, then recurse into right subtree 145 145 + for (const entry of entries) { 146 146 + paths.push(entry.key); 147 147 + if (entry.subtree) { 148 148 + await walkMstNode(blockStore, entry.subtree, paths); 149 149 + } 150 150 + } 151 151 + } 152 152 + 153 153 + /** 154 154 + * Extract all record paths from a repo by walking the full MST. 155 155 + * 156 156 + * @param blockStore - Block storage containing the repo blocks 157 157 + * @param commitCid - CID of the commit block (repo head) 158 158 + * @returns All record paths in the repo (e.g. "app.bsky.feed.post/abc123") 159 159 + */ 160 160 + export async function extractAllRecordPaths( 161 161 + blockStore: BlockStore, 162 162 + commitCid: string, 163 163 + ): Promise<string[]> { 164 164 + const commitBytes = await blockStore.getBlock(commitCid); 165 165 + if (!commitBytes) return []; 166 166 + 167 167 + const commitObj = cborDecode(commitBytes) as { data: CID }; 168 168 + const mstRootCid = commitObj.data; 169 169 + if (!mstRootCid) return []; 170 170 + 171 171 + const paths: string[] = []; 172 172 + await walkMstNode(blockStore, mstRootCid, paths); 173 173 + return paths; 174 174 + } 175 175 + 123 176 // ---------- Generation ---------- 124 177 125 178 /**

+31

src/replication/replication-manager.ts

reviewed

··· 29 29 import { RepoFetcher } from "./repo-fetcher.js"; 30 30 import { PeerDiscovery } from "./peer-discovery.js"; 31 31 import { BlockVerifier, RemoteVerifier } from "./verification.js"; 32 32 + import { extractAllRecordPaths } from "./mst-proof.js"; 32 33 import { 33 34 FirehoseSubscription, 34 35 type FirehoseCommitEvent, ··· 388 389 // 9b. Invalidate cached ReadableRepo so it reloads with new root 389 390 this.replicatedRepoReader?.invalidateCache(did); 390 391 392 392 + // 9c. Track record paths for challenge generation (full MST walk) 393 393 + try { 394 394 + const recordPaths = await extractAllRecordPaths( 395 395 + this.blockStore, 396 396 + rootCidStr, 397 397 + ); 398 398 + this.syncStorage.clearRecordPaths(did); 399 399 + this.syncStorage.trackRecordPaths(did, recordPaths); 400 400 + } catch { 401 401 + // Non-fatal: path extraction is best-effort 402 402 + } 403 403 + 391 404 // 10. Update manifest record 392 405 const rkey = didToRkey(did); 393 406 const existingManifest = await this.repoManager.getRecord( ··· 659 672 660 673 // 8. Invalidate cached ReadableRepo so it reloads with new root 661 674 this.replicatedRepoReader?.invalidateCache(did); 675 675 + 676 676 + // 8b. Track record paths incrementally from firehose ops 677 677 + try { 678 678 + const createdPaths = event.ops 679 679 + .filter((op) => op.action === "create" || op.action === "update") 680 680 + .map((op) => op.path); 681 681 + const deletedPaths = event.ops 682 682 + .filter((op) => op.action === "delete") 683 683 + .map((op) => op.path); 684 684 + if (createdPaths.length > 0) { 685 685 + this.syncStorage.trackRecordPaths(did, createdPaths); 686 686 + } 687 687 + if (deletedPaths.length > 0) { 688 688 + this.syncStorage.removeRecordPaths(did, deletedPaths); 689 689 + } 690 690 + } catch { 691 691 + // Non-fatal: path tracking is best-effort 692 692 + } 662 693 663 694 // 9. Update manifest record 664 695 const rkey = didToRkey(did);

+16

src/replication/sync-storage.ts

reviewed

··· 300 300 } 301 301 302 302 /** 303 303 + * Remove specific record paths for a DID (batch delete). 304 304 + */ 305 305 + removeRecordPaths(did: string, paths: string[]): void { 306 306 + if (paths.length === 0) return; 307 307 + const remove = this.db.prepare( 308 308 + "DELETE FROM replication_record_paths WHERE did = ? AND record_path = ?", 309 309 + ); 310 310 + const batch = this.db.transaction((items: string[]) => { 311 311 + for (const path of items) { 312 312 + remove.run(did, path); 313 313 + } 314 314 + }); 315 315 + batch(paths); 316 316 + } 317 317 + 318 318 + /** 303 319 * Clear all tracked record paths for a DID. 304 320 */ 305 321 clearRecordPaths(did: string): void {