a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
feat(mst): flesh it out
+1950
-40
+107
packages/utilities/mst/lib/car-writer.test.ts
+107
packages/utilities/mst/lib/car-writer.test.ts
···
1
+
import { describe, expect, it } from 'vitest';
2
+
3
+
import * as CID from '@atcute/cid';
4
+
import { concat, encodeUtf8 } from '@atcute/uint8array';
5
+
6
+
import { type CarBlock, createCarStream, serializeCarEntry, serializeCarHeader } from './car-writer.js';
7
+
8
+
describe('serializeCarHeader', () => {
9
+
it('should serialize a header with one root', async () => {
10
+
const cid = CID.toCidLink(await CID.create(0x55, encodeUtf8('test')));
11
+
const header = serializeCarHeader([cid]);
12
+
13
+
expect(header.length).toBeGreaterThan(0);
14
+
// Should start with a varint length
15
+
expect(header[0]).toBeGreaterThan(0);
16
+
});
17
+
18
+
it('should serialize a header with multiple roots', async () => {
19
+
const cid1 = CID.toCidLink(await CID.create(0x55, encodeUtf8('test1')));
20
+
const cid2 = CID.toCidLink(await CID.create(0x55, encodeUtf8('test2')));
21
+
const header = serializeCarHeader([cid1, cid2]);
22
+
23
+
expect(header.length).toBeGreaterThan(0);
24
+
});
25
+
26
+
it('should serialize a header with no roots', () => {
27
+
const header = serializeCarHeader([]);
28
+
expect(header.length).toBeGreaterThan(0);
29
+
});
30
+
});
31
+
32
+
describe('serializeCarEntry', () => {
33
+
it('should serialize a CAR entry', async () => {
34
+
const cid = await CID.create(0x55, encodeUtf8('test'));
35
+
const data = encodeUtf8('hello world');
36
+
37
+
const entry = serializeCarEntry(cid.bytes, data);
38
+
39
+
expect(entry.length).toBe(1 + cid.bytes.length + data.length); // varint(1 byte) + cid + data
40
+
// Check that the entry starts with the correct length varint
41
+
expect(entry[0]).toBe(cid.bytes.length + data.length);
42
+
});
43
+
});
44
+
45
+
describe('createCarStream', () => {
46
+
it('should stream a CAR with a single block', async () => {
47
+
const rootCid = CID.toCidLink(await CID.create(0x55, encodeUtf8('root')));
48
+
const blockCid = await CID.create(0x55, encodeUtf8('block1'));
49
+
const blockData = encodeUtf8('data1');
50
+
51
+
const blocks = async function* (): AsyncGenerator<CarBlock> {
52
+
yield { cid: blockCid.bytes, data: blockData };
53
+
};
54
+
55
+
const chunks: Uint8Array[] = [];
56
+
for await (const chunk of createCarStream(rootCid, blocks())) {
57
+
chunks.push(chunk);
58
+
}
59
+
60
+
expect(chunks.length).toBe(2); // header + 1 block
61
+
62
+
const carBytes = concat(chunks);
63
+
expect(carBytes.length).toBeGreaterThan(0);
64
+
expect(carBytes[0]).toBeGreaterThan(0); // header starts with varint
65
+
});
66
+
67
+
it('should stream a CAR with multiple blocks', async () => {
68
+
const rootCid = CID.toCidLink(await CID.create(0x55, encodeUtf8('root')));
69
+
70
+
const blocks = async function* (): AsyncGenerator<CarBlock> {
71
+
for (let i = 0; i < 5; i++) {
72
+
const blockCid = await CID.create(0x55, encodeUtf8(`block${i}`));
73
+
const blockData = encodeUtf8(`data${i}`);
74
+
yield { cid: blockCid.bytes, data: blockData };
75
+
}
76
+
};
77
+
78
+
const chunks = await Array.fromAsync(createCarStream(rootCid, blocks()));
79
+
const car = concat(chunks);
80
+
81
+
expect(chunks).toHaveLength(6); // header + 5 blocks
82
+
expect(car.length).toBeGreaterThan(0);
83
+
});
84
+
85
+
it('should produce consistent output', async () => {
86
+
const rootCid = CID.toCidLink(await CID.create(0x55, encodeUtf8('root')));
87
+
const blockCid = await CID.create(0x55, encodeUtf8('block'));
88
+
const blockData = encodeUtf8('data');
89
+
90
+
const blocks: CarBlock[] = [{ cid: blockCid.bytes, data: blockData }];
91
+
92
+
const chunks1: Uint8Array[] = [];
93
+
for await (const chunk of createCarStream(rootCid, blocks)) {
94
+
chunks1.push(chunk);
95
+
}
96
+
97
+
const chunks2: Uint8Array[] = [];
98
+
for await (const chunk of createCarStream(rootCid, blocks)) {
99
+
chunks2.push(chunk);
100
+
}
101
+
102
+
const bytes1 = concat(chunks1);
103
+
const bytes2 = concat(chunks2);
104
+
105
+
expect(bytes1).toEqual(bytes2);
106
+
});
107
+
});
+98
packages/utilities/mst/lib/car-writer.ts
+98
packages/utilities/mst/lib/car-writer.ts
···
1
+
import * as CBOR from '@atcute/cbor';
2
+
import type { CidLink } from '@atcute/cid';
3
+
import { allocUnsafe } from '@atcute/uint8array';
4
+
import * as varint from '@atcute/varint';
5
+
6
+
/**
7
+
* Encodes a number as an unsigned varint (variable-length integer)
8
+
* @param n the number to encode
9
+
* @returns the varint-encoded bytes
10
+
*/
11
+
const encodeVarint = (n: number): Uint8Array<ArrayBuffer> => {
12
+
const length = varint.encodingLength(n);
13
+
const buf = allocUnsafe(length);
14
+
varint.encode(n, buf, 0);
15
+
return buf;
16
+
};
17
+
18
+
/**
19
+
* Serializes a CAR v1 header
20
+
* @param roots array of root CIDs (typically just one)
21
+
* @returns the serialized header bytes
22
+
*/
23
+
export const serializeCarHeader = (roots: readonly CidLink[]): Uint8Array<ArrayBuffer> => {
24
+
const headerData = CBOR.encode({
25
+
version: 1,
26
+
roots: roots,
27
+
});
28
+
29
+
const headerSize = encodeVarint(headerData.length);
30
+
const result = allocUnsafe(headerSize.length + headerData.length);
31
+
32
+
result.set(headerSize, 0);
33
+
result.set(headerData, headerSize.length);
34
+
35
+
return result;
36
+
};
37
+
38
+
/**
39
+
* Serializes a single CAR entry (block)
40
+
* @param cid the CID of the block (as bytes)
41
+
* @param data the block data
42
+
* @returns the serialized entry bytes
43
+
*/
44
+
export const serializeCarEntry = (cid: Uint8Array, data: Uint8Array): Uint8Array<ArrayBuffer> => {
45
+
const entrySize = encodeVarint(cid.length + data.length);
46
+
const result = allocUnsafe(entrySize.length + cid.length + data.length);
47
+
48
+
result.set(entrySize, 0);
49
+
result.set(cid, entrySize.length);
50
+
result.set(data, entrySize.length + cid.length);
51
+
52
+
return result;
53
+
};
54
+
55
+
/**
56
+
* Represents a block to be written to a CAR file
57
+
*/
58
+
export interface CarBlock {
59
+
/** the CID of the block (as bytes) */
60
+
cid: Uint8Array;
61
+
/** the block data */
62
+
data: Uint8Array;
63
+
}
64
+
65
+
/**
66
+
* Creates an async generator that yields CAR file chunks
67
+
* @param root the root CID for the CAR file
68
+
* @param blocks async iterable of blocks to write
69
+
* @yields Uint8Array chunks of the CAR file (header, then entries)
70
+
*
71
+
* @example
72
+
* ```typescript
73
+
* const blocks = async function* () {
74
+
* yield { cid: commitCid.bytes, data: commitBytes };
75
+
* yield { cid: nodeCid.bytes, data: nodeBytes };
76
+
* };
77
+
*
78
+
* // Stream chunks
79
+
* for await (const chunk of createCarStream(rootCid, blocks())) {
80
+
* stream.write(chunk);
81
+
* }
82
+
*
83
+
* // Or collect into array (requires Array.fromAsync or polyfill)
84
+
* const chunks = await Array.fromAsync(createCarStream(rootCid, blocks()));
85
+
* ```
86
+
*/
87
+
export async function* createCarStream(
88
+
root: CidLink,
89
+
blocks: AsyncIterable<CarBlock> | Iterable<CarBlock>,
90
+
): AsyncGenerator<Uint8Array<ArrayBuffer>> {
91
+
// Emit header first
92
+
yield serializeCarHeader([root]);
93
+
94
+
// Then emit each block entry
95
+
for await (const block of blocks) {
96
+
yield serializeCarEntry(block.cid, block.data);
97
+
}
98
+
}
+259
packages/utilities/mst/lib/diff.test.ts
+259
packages/utilities/mst/lib/diff.test.ts
···
1
+
import { describe, expect, it } from 'vitest';
2
+
3
+
import * as CID from '@atcute/cid';
4
+
import { encodeUtf8 } from '@atcute/uint8array';
5
+
6
+
import { DeltaType, mstDiff, recordDiff, verySlowMstDiff } from './diff.js';
7
+
import { NodeStore } from './node-store.js';
8
+
import { NodeWrangler } from './node-wrangler.js';
9
+
import { MemoryBlockStore } from './stores.js';
10
+
11
+
const createCid = async (data: string) => {
12
+
const bytes = encodeUtf8(data);
13
+
return CID.toCidLink(await CID.create(0x55, bytes));
14
+
};
15
+
16
+
describe('mstDiff', () => {
17
+
it('should detect created records', async () => {
18
+
const store = new NodeStore(new MemoryBlockStore());
19
+
const wrangler = new NodeWrangler(store);
20
+
21
+
// Build tree A with 2 records
22
+
let rootA: string | null = null;
23
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
24
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
25
+
26
+
// Build tree B with 3 records (added c/3)
27
+
let rootB: string | null = null;
28
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
29
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2'));
30
+
rootB = await wrangler.putRecord(rootB, 'c/3', await createCid('value-c3'));
31
+
32
+
const [created, deleted] = await mstDiff(store, rootA, rootB);
33
+
34
+
expect(created.size).toBeGreaterThan(0);
35
+
// Note: deleted might contain the old root node that was replaced
36
+
// expect(deleted.size).toBe(0);
37
+
38
+
// Verify record diff
39
+
const deltas = [];
40
+
for await (const delta of recordDiff(store, created, deleted)) {
41
+
deltas.push(delta);
42
+
}
43
+
44
+
expect(deltas.length).toBe(1);
45
+
expect(deltas[0].deltaType).toBe(DeltaType.CREATED);
46
+
expect(deltas[0].path).toBe('c/3');
47
+
expect(deltas[0].priorValue).toBe(null);
48
+
expect(deltas[0].laterValue).not.toBe(null);
49
+
});
50
+
51
+
it('should detect deleted records', async () => {
52
+
const store = new NodeStore(new MemoryBlockStore());
53
+
const wrangler = new NodeWrangler(store);
54
+
55
+
// Build tree A with 3 records
56
+
let rootA: string | null = null;
57
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
58
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
59
+
rootA = await wrangler.putRecord(rootA, 'c/3', await createCid('value-c3'));
60
+
61
+
// Build tree B with 2 records (deleted c/3)
62
+
let rootB: string | null = null;
63
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
64
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2'));
65
+
66
+
const [created, deleted] = await mstDiff(store, rootA, rootB);
67
+
68
+
// Note: created might contain the new root node that was created
69
+
// expect(created.size).toBe(0);
70
+
expect(deleted.size).toBeGreaterThan(0);
71
+
72
+
// Verify record diff
73
+
const deltas = [];
74
+
for await (const delta of recordDiff(store, created, deleted)) {
75
+
deltas.push(delta);
76
+
}
77
+
78
+
expect(deltas.length).toBe(1);
79
+
expect(deltas[0].deltaType).toBe(DeltaType.DELETED);
80
+
expect(deltas[0].path).toBe('c/3');
81
+
expect(deltas[0].priorValue).not.toBe(null);
82
+
expect(deltas[0].laterValue).toBe(null);
83
+
});
84
+
85
+
it('should detect updated records', async () => {
86
+
const store = new NodeStore(new MemoryBlockStore());
87
+
const wrangler = new NodeWrangler(store);
88
+
89
+
// Build tree A
90
+
let rootA: string | null = null;
91
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
92
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
93
+
94
+
// Build tree B with updated value for b/2
95
+
let rootB: string | null = null;
96
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
97
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2-updated'));
98
+
99
+
const [created, deleted] = await mstDiff(store, rootA, rootB);
100
+
101
+
expect(created.size).toBeGreaterThan(0);
102
+
expect(deleted.size).toBeGreaterThan(0);
103
+
104
+
// Verify record diff
105
+
const deltas = [];
106
+
for await (const delta of recordDiff(store, created, deleted)) {
107
+
deltas.push(delta);
108
+
}
109
+
110
+
expect(deltas.length).toBe(1);
111
+
expect(deltas[0].deltaType).toBe(DeltaType.UPDATED);
112
+
expect(deltas[0].path).toBe('b/2');
113
+
expect(deltas[0].priorValue).not.toBe(null);
114
+
expect(deltas[0].laterValue).not.toBe(null);
115
+
expect(deltas[0].priorValue?.$link).not.toBe(deltas[0].laterValue?.$link);
116
+
});
117
+
118
+
it('should handle identical trees', async () => {
119
+
const store = new NodeStore(new MemoryBlockStore());
120
+
const wrangler = new NodeWrangler(store);
121
+
122
+
let root: string | null = null;
123
+
root = await wrangler.putRecord(root, 'a/1', await createCid('value-a1'));
124
+
root = await wrangler.putRecord(root, 'b/2', await createCid('value-b2'));
125
+
126
+
const [created, deleted] = await mstDiff(store, root, root);
127
+
128
+
expect(created.size).toBe(0);
129
+
expect(deleted.size).toBe(0);
130
+
131
+
const deltas = [];
132
+
for await (const delta of recordDiff(store, created, deleted)) {
133
+
deltas.push(delta);
134
+
}
135
+
136
+
expect(deltas.length).toBe(0);
137
+
});
138
+
139
+
it('should handle empty to non-empty tree', async () => {
140
+
const store = new NodeStore(new MemoryBlockStore());
141
+
const wrangler = new NodeWrangler(store);
142
+
143
+
const emptyRoot = (await store.get(null).then((n) => n.cid())).$link;
144
+
145
+
let rootB: string | null = null;
146
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
147
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2'));
148
+
149
+
const [created, deleted] = await mstDiff(store, emptyRoot, rootB);
150
+
151
+
expect(created.size).toBeGreaterThan(0);
152
+
153
+
const deltas = [];
154
+
for await (const delta of recordDiff(store, created, deleted)) {
155
+
deltas.push(delta);
156
+
}
157
+
158
+
expect(deltas.length).toBe(2);
159
+
expect(deltas.every((d) => d.deltaType === DeltaType.CREATED)).toBe(true);
160
+
});
161
+
162
+
it('should handle non-empty to empty tree', async () => {
163
+
const store = new NodeStore(new MemoryBlockStore());
164
+
const wrangler = new NodeWrangler(store);
165
+
166
+
let rootA: string | null = null;
167
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
168
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
169
+
170
+
const emptyRoot = (await store.get(null).then((n) => n.cid())).$link;
171
+
172
+
const [created, deleted] = await mstDiff(store, rootA, emptyRoot);
173
+
174
+
expect(deleted.size).toBeGreaterThan(0);
175
+
176
+
const deltas = [];
177
+
for await (const delta of recordDiff(store, created, deleted)) {
178
+
deltas.push(delta);
179
+
}
180
+
181
+
expect(deltas.length).toBe(2);
182
+
expect(deltas.every((d) => d.deltaType === DeltaType.DELETED)).toBe(true);
183
+
});
184
+
185
+
it('should handle multiple operations', async () => {
186
+
const store = new NodeStore(new MemoryBlockStore());
187
+
const wrangler = new NodeWrangler(store);
188
+
189
+
// Tree A: a/1, b/2, c/3
190
+
let rootA: string | null = null;
191
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
192
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
193
+
rootA = await wrangler.putRecord(rootA, 'c/3', await createCid('value-c3'));
194
+
195
+
// Tree B: a/1 (same), b/2 (updated), d/4 (new), c/3 deleted
196
+
let rootB: string | null = null;
197
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
198
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2-updated'));
199
+
rootB = await wrangler.putRecord(rootB, 'd/4', await createCid('value-d4'));
200
+
201
+
const [created, deleted] = await mstDiff(store, rootA, rootB);
202
+
203
+
const deltas = [];
204
+
for await (const delta of recordDiff(store, created, deleted)) {
205
+
deltas.push(delta);
206
+
}
207
+
208
+
// Should have: 1 created (d/4), 1 updated (b/2), 1 deleted (c/3)
209
+
expect(deltas.length).toBe(3);
210
+
211
+
const deltasByType = {
212
+
[DeltaType.CREATED]: deltas.filter((d) => d.deltaType === DeltaType.CREATED),
213
+
[DeltaType.UPDATED]: deltas.filter((d) => d.deltaType === DeltaType.UPDATED),
214
+
[DeltaType.DELETED]: deltas.filter((d) => d.deltaType === DeltaType.DELETED),
215
+
};
216
+
217
+
expect(deltasByType[DeltaType.CREATED].length).toBe(1);
218
+
expect(deltasByType[DeltaType.CREATED][0].path).toBe('d/4');
219
+
220
+
expect(deltasByType[DeltaType.UPDATED].length).toBe(1);
221
+
expect(deltasByType[DeltaType.UPDATED][0].path).toBe('b/2');
222
+
223
+
expect(deltasByType[DeltaType.DELETED].length).toBe(1);
224
+
expect(deltasByType[DeltaType.DELETED][0].path).toBe('c/3');
225
+
});
226
+
});
227
+
228
+
describe('verySlowMstDiff', () => {
229
+
it('should match mstDiff results', async () => {
230
+
const store = new NodeStore(new MemoryBlockStore());
231
+
const wrangler = new NodeWrangler(store);
232
+
233
+
// Build two different trees
234
+
let rootA: string | null = null;
235
+
rootA = await wrangler.putRecord(rootA, 'a/1', await createCid('value-a1'));
236
+
rootA = await wrangler.putRecord(rootA, 'b/2', await createCid('value-b2'));
237
+
rootA = await wrangler.putRecord(rootA, 'c/3', await createCid('value-c3'));
238
+
239
+
let rootB: string | null = null;
240
+
rootB = await wrangler.putRecord(rootB, 'a/1', await createCid('value-a1'));
241
+
rootB = await wrangler.putRecord(rootB, 'b/2', await createCid('value-b2-updated'));
242
+
rootB = await wrangler.putRecord(rootB, 'd/4', await createCid('value-d4'));
243
+
244
+
const [createdFast, deletedFast] = await mstDiff(store, rootA, rootB);
245
+
const [createdSlow, deletedSlow] = await verySlowMstDiff(store, rootA, rootB);
246
+
247
+
// Both should produce the same sets
248
+
expect(createdFast.size).toBe(createdSlow.size);
249
+
expect(deletedFast.size).toBe(deletedSlow.size);
250
+
251
+
for (const cid of createdFast) {
252
+
expect(createdSlow.has(cid)).toBe(true);
253
+
}
254
+
255
+
for (const cid of deletedFast) {
256
+
expect(deletedSlow.has(cid)).toBe(true);
257
+
}
258
+
});
259
+
});
+292
packages/utilities/mst/lib/diff.ts
+292
packages/utilities/mst/lib/diff.ts
···
1
+
import type { CidLink } from '@atcute/cid';
2
+
3
+
import { MSTNode } from './node.js';
4
+
import type { NodeStore } from './node-store.js';
5
+
import { NodeWalker } from './node-walker.js';
6
+
7
+
/**
8
+
* Type of change to a record
9
+
*/
10
+
export enum DeltaType {
11
+
CREATED = 1,
12
+
UPDATED = 2,
13
+
DELETED = 3,
14
+
}
15
+
16
+
/**
17
+
* Represents a change to a single record
18
+
*/
19
+
export interface RecordDelta {
20
+
/** type of change */
21
+
deltaType: DeltaType;
22
+
/** record path (collection/rkey) */
23
+
path: string;
24
+
/** CID before the change (null for creates) */
25
+
priorValue: CidLink | null;
26
+
/** CID after the change (null for deletes) */
27
+
laterValue: CidLink | null;
28
+
}
29
+
30
+
/**
31
+
* Given two sets of MST nodes, returns an iterator of record-level changes
32
+
* @param ns the node store
33
+
* @param created set of node CIDs that were created
34
+
* @param deleted set of node CIDs that were deleted
35
+
* @yields record deltas describing the changes
36
+
*/
37
+
export async function* recordDiff(
38
+
ns: NodeStore,
39
+
created: Set<string>,
40
+
deleted: Set<string>,
41
+
): AsyncGenerator<RecordDelta> {
42
+
// Build maps of all keys and values in created/deleted nodes
43
+
const createdKv = new Map<string, CidLink>();
44
+
for (const cid of created) {
45
+
const node = await ns.get(cid);
46
+
for (let i = 0; i < node.keys.length; i++) {
47
+
createdKv.set(node.keys[i], node.values[i]);
48
+
}
49
+
}
50
+
51
+
const deletedKv = new Map<string, CidLink>();
52
+
for (const cid of deleted) {
53
+
const node = await ns.get(cid);
54
+
for (let i = 0; i < node.keys.length; i++) {
55
+
deletedKv.set(node.keys[i], node.values[i]);
56
+
}
57
+
}
58
+
59
+
// Find keys that were created (in created but not deleted)
60
+
for (const [key, value] of createdKv) {
61
+
if (!deletedKv.has(key)) {
62
+
yield {
63
+
deltaType: DeltaType.CREATED,
64
+
path: key,
65
+
priorValue: null,
66
+
laterValue: value,
67
+
};
68
+
}
69
+
}
70
+
71
+
// Find keys that were updated (in both, but with different values)
72
+
for (const [key, newValue] of createdKv) {
73
+
const oldValue = deletedKv.get(key);
74
+
if (oldValue !== undefined && oldValue.$link !== newValue.$link) {
75
+
yield {
76
+
deltaType: DeltaType.UPDATED,
77
+
path: key,
78
+
priorValue: oldValue,
79
+
laterValue: newValue,
80
+
};
81
+
}
82
+
}
83
+
84
+
// Find keys that were deleted (in deleted but not created)
85
+
for (const [key, value] of deletedKv) {
86
+
if (!createdKv.has(key)) {
87
+
yield {
88
+
deltaType: DeltaType.DELETED,
89
+
path: key,
90
+
priorValue: value,
91
+
laterValue: null,
92
+
};
93
+
}
94
+
}
95
+
}
96
+
97
+
/**
98
+
* Slow but obvious MST diff implementation for testing
99
+
* Enumerates all nodes in both trees and compares them
100
+
* @param ns the node store
101
+
* @param rootA CID of first MST root
102
+
* @param rootB CID of second MST root
103
+
* @returns tuple of [created nodes, deleted nodes]
104
+
*/
105
+
export const verySlowMstDiff = async (
106
+
ns: NodeStore,
107
+
rootA: string,
108
+
rootB: string,
109
+
): Promise<[Set<string>, Set<string>]> => {
110
+
const walkerA = await NodeWalker.create(ns, rootA);
111
+
const walkerB = await NodeWalker.create(ns, rootB);
112
+
113
+
const nodesA = new Set<string>();
114
+
for await (const cid of iterNodeCids(walkerA)) {
115
+
nodesA.add(cid);
116
+
}
117
+
118
+
const nodesB = new Set<string>();
119
+
for await (const cid of iterNodeCids(walkerB)) {
120
+
nodesB.add(cid);
121
+
}
122
+
123
+
const created = new Set<string>();
124
+
for (const cid of nodesB) {
125
+
if (!nodesA.has(cid)) {
126
+
created.add(cid);
127
+
}
128
+
}
129
+
130
+
const deleted = new Set<string>();
131
+
for (const cid of nodesA) {
132
+
if (!nodesB.has(cid)) {
133
+
deleted.add(cid);
134
+
}
135
+
}
136
+
137
+
return [created, deleted];
138
+
};
139
+
140
+
/**
141
+
* Helper to iterate over all node CIDs in a tree
142
+
*/
143
+
async function* iterNodeCids(walker: NodeWalker): AsyncGenerator<string> {
144
+
// Always yield the current node
145
+
yield (await walker.frame.node.cid()).$link;
146
+
147
+
// Recursively iterate through the tree
148
+
while (!walker.done) {
149
+
if (walker.subtree !== null) {
150
+
await walker.down();
151
+
yield (await walker.frame.node.cid()).$link;
152
+
} else {
153
+
walker.rightOrUp();
154
+
}
155
+
}
156
+
}
157
+
158
+
const EMPTY_NODE_CID = (await MSTNode.empty().cid()).$link;
159
+
160
+
/**
161
+
* Efficiently computes the difference between two MSTs
162
+
* @param ns the node store
163
+
* @param rootA CID of first MST root
164
+
* @param rootB CID of second MST root
165
+
* @returns tuple of [created nodes, deleted nodes]
166
+
*/
167
+
export const mstDiff = async (ns: NodeStore, rootA: string, rootB: string): Promise<[Set<string>, Set<string>]> => {
168
+
const created = new Set<string>(); // nodes in B but not in A
169
+
const deleted = new Set<string>(); // nodes in A but not in B
170
+
171
+
const walkerA = await NodeWalker.create(ns, rootA);
172
+
const walkerB = await NodeWalker.create(ns, rootB);
173
+
174
+
await mstDiffRecursive(created, deleted, walkerA, walkerB);
175
+
176
+
// Remove false positives (nodes that appeared in both sets)
177
+
const middle = new Set<string>();
178
+
for (const cid of created) {
179
+
if (deleted.has(cid)) {
180
+
middle.add(cid);
181
+
}
182
+
}
183
+
184
+
for (const cid of middle) {
185
+
created.delete(cid);
186
+
deleted.delete(cid);
187
+
}
188
+
189
+
// Special case: if one of the root nodes was empty
190
+
if (rootA === EMPTY_NODE_CID && rootB !== EMPTY_NODE_CID) {
191
+
deleted.add(EMPTY_NODE_CID);
192
+
}
193
+
if (rootB === EMPTY_NODE_CID && rootA !== EMPTY_NODE_CID) {
194
+
created.add(EMPTY_NODE_CID);
195
+
}
196
+
197
+
return [created, deleted];
198
+
};
199
+
200
+
/**
201
+
* Recursive helper for mstDiff
202
+
* Theory: most trees that get compared will have lots of shared blocks (which we can skip over)
203
+
* Completely different trees will inevitably have to visit every node.
204
+
*/
205
+
const mstDiffRecursive = async (
206
+
created: Set<string>,
207
+
deleted: Set<string>,
208
+
a: NodeWalker,
209
+
b: NodeWalker,
210
+
): Promise<void> => {
211
+
// Easiest case: nodes are identical
212
+
const aNodeCid = (await a.frame.node.cid()).$link;
213
+
const bNodeCid = (await b.frame.node.cid()).$link;
214
+
215
+
if (aNodeCid === bNodeCid) {
216
+
return; // no difference
217
+
}
218
+
219
+
// Trivial case: a is empty, all of b is new
220
+
if (a.frame.node.isEmpty) {
221
+
for await (const cid of iterNodeCids(b)) {
222
+
created.add(cid);
223
+
}
224
+
return;
225
+
}
226
+
227
+
// Likewise: b is empty, all of a is deleted
228
+
if (b.frame.node.isEmpty) {
229
+
for await (const cid of iterNodeCids(a)) {
230
+
deleted.add(cid);
231
+
}
232
+
return;
233
+
}
234
+
235
+
// Now we're onto the hard part
236
+
// NB: these will end up as false-positives if one tree is a subtree of the other
237
+
created.add(bNodeCid);
238
+
deleted.add(aNodeCid);
239
+
240
+
// General idea:
241
+
// 1. If one cursor is "behind" the other, catch it up
242
+
// 2. When we're matched up, skip over identical subtrees (and recursively diff non-identical subtrees)
243
+
while (true) {
244
+
// Catch up whichever cursor is behind
245
+
while (a.rpath !== b.rpath) {
246
+
// Catch up cursor a if it's behind
247
+
while (a.rpath < b.rpath && !a.done) {
248
+
if (a.subtree !== null) {
249
+
await a.down();
250
+
deleted.add((await a.frame.node.cid()).$link);
251
+
} else {
252
+
a.rightOrUp();
253
+
}
254
+
}
255
+
256
+
// Catch up cursor b likewise
257
+
while (b.rpath < a.rpath && !b.done) {
258
+
if (b.subtree !== null) {
259
+
await b.down();
260
+
created.add((await b.frame.node.cid()).$link);
261
+
} else {
262
+
b.rightOrUp();
263
+
}
264
+
}
265
+
}
266
+
267
+
// The rpaths now match, but the subtrees below us might not
268
+
const aSubtree = a.subtree;
269
+
const bSubtree = b.subtree;
270
+
271
+
// Recursively diff the subtrees
272
+
const aSubWalker = await a.createSubtreeWalker();
273
+
const bSubWalker = await b.createSubtreeWalker();
274
+
await mstDiffRecursive(created, deleted, aSubWalker, bSubWalker);
275
+
276
+
// Check if we can still go right
277
+
const aBottom = a.stack.peekBottom();
278
+
const bBottom = b.stack.peekBottom();
279
+
280
+
if (
281
+
aBottom !== undefined &&
282
+
bBottom !== undefined &&
283
+
a.rpath === aBottom.rpath &&
284
+
b.rpath === bBottom.rpath
285
+
) {
286
+
break;
287
+
}
288
+
289
+
a.rightOrUp();
290
+
b.rightOrUp();
291
+
}
292
+
};
+23
-35
packages/utilities/mst/lib/node-walker.ts
+23
-35
packages/utilities/mst/lib/node-walker.ts
···
40
40
static readonly PATH_MIN = ''; // string that compares less than all legal path strings
41
41
static readonly PATH_MAX = '\xff'; // string that compares greater than all legal path strings
42
42
43
-
/**
44
-
* node store for fetching nodes
45
-
* @internal
46
-
*/
47
-
_store: NodeStore;
48
-
/**
49
-
* stack of frames representing the traversal path
50
-
* @internal
51
-
*/
52
-
_stack: Stack<StackFrame>;
53
-
/**
54
-
* height of the root node
55
-
* @internal
56
-
*/
57
-
_rootHeight: number;
58
-
/**
59
-
* whether to skip height validation (for trusted trees)
60
-
* @internal
61
-
*/
62
-
_trusted: boolean;
43
+
/** node store for fetching nodes */
44
+
readonly store: NodeStore;
45
+
/** stack of frames representing the traversal path */
46
+
readonly stack: Stack<StackFrame>;
47
+
/** height of the root node */
48
+
readonly rootHeight: number;
49
+
/** whether to skip height validation (for trusted trees) */
50
+
readonly trusted: boolean;
63
51
64
52
private constructor(store: NodeStore, stack: Stack<StackFrame>, rootHeight: number, trusted: boolean) {
65
-
this._store = store;
66
-
this._stack = stack;
67
-
this._rootHeight = rootHeight;
68
-
this._trusted = trusted;
53
+
this.store = store;
54
+
this.stack = stack;
55
+
this.rootHeight = rootHeight;
56
+
this.trusted = trusted;
69
57
}
70
58
71
59
/**
···
111
99
*/
112
100
async createSubtreeWalker(): Promise<NodeWalker> {
113
101
return await NodeWalker.create(
114
-
this._store,
102
+
this.store,
115
103
this.subtree?.$link ?? null,
116
104
this.lpath,
117
105
this.rpath,
118
-
this._trusted,
106
+
this.trusted,
119
107
this.height - 1,
120
108
);
121
109
}
122
110
123
111
/** current stack frame */
124
112
get frame(): StackFrame {
125
-
const frame = this._stack.peek();
113
+
const frame = this.stack.peek();
126
114
if (frame === undefined) {
127
115
throw new Error(`stack is empty`);
128
116
}
···
132
120
133
121
/** current height in the tree (decreases as you descend) */
134
122
get height(): number {
135
-
return this._rootHeight - (this._stack.size - 1);
123
+
return this.rootHeight - (this.stack.size - 1);
136
124
}
137
125
138
126
/** key/path to the left of current cursor position */
···
166
154
/** whether the walker has reached the end of the tree */
167
155
get done(): boolean {
168
156
// is (not this.stack) really necessary here? is that a reachable state?
169
-
const bottom = this._stack.peekBottom();
157
+
const bottom = this.stack.peekBottom();
170
158
return (
171
-
this._stack.size === 0 || (this.subtree === null && bottom !== undefined && this.rpath === bottom.rpath)
159
+
this.stack.size === 0 || (this.subtree === null && bottom !== undefined && this.rpath === bottom.rpath)
172
160
);
173
161
}
174
162
···
185
173
rightOrUp(): void {
186
174
if (!this.canGoRight) {
187
175
// we reached the end of this node, go up a level
188
-
this._stack.pop();
189
-
if (this._stack.size === 0) {
176
+
this.stack.pop();
177
+
if (this.stack.size === 0) {
190
178
throw new Error(`cannot navigate beyond root; check .done before calling`);
191
179
}
192
180
return this.rightOrUp(); // we need to recurse, to skip over empty intermediates on the way back up
···
215
203
throw new Error(`cannot descend; no subtree at current position`);
216
204
}
217
205
218
-
const subtreeNode = await this._store.get(subtree.$link);
206
+
const subtreeNode = await this.store.get(subtree.$link);
219
207
220
-
if (!this._trusted) {
208
+
if (!this.trusted) {
221
209
// if we "trust" the source we can elide this check
222
210
// the "null" case occurs for empty intermediate nodes
223
211
const subtreeHeight = await subtreeNode.height();
···
226
214
}
227
215
}
228
216
229
-
this._stack.push({
217
+
this.stack.push({
230
218
node: subtreeNode,
231
219
lpath: this.lpath,
232
220
rpath: this.rpath,
+299
packages/utilities/mst/lib/node-wrangler.test.ts
+299
packages/utilities/mst/lib/node-wrangler.test.ts
···
1
+
import { describe, expect, it } from 'vitest';
2
+
3
+
import * as CID from '@atcute/cid';
4
+
import { encodeUtf8 } from '@atcute/uint8array';
5
+
6
+
import { NodeStore } from './node-store.js';
7
+
import { NodeWalker } from './node-walker.js';
8
+
import { NodeWrangler } from './node-wrangler.js';
9
+
import { MSTNode } from './node.js';
10
+
import { MemoryBlockStore } from './stores.js';
11
+
12
+
const createCid = async (data: string) => {
13
+
const bytes = encodeUtf8(data);
14
+
return CID.toCidLink(await CID.create(0x55, bytes));
15
+
};
16
+
17
+
describe('NodeWrangler', () => {
18
+
it('should put a record into an empty tree', async () => {
19
+
const store = new NodeStore(new MemoryBlockStore());
20
+
const wrangler = new NodeWrangler(store);
21
+
22
+
const emptyNode = MSTNode.empty();
23
+
await store.put(emptyNode);
24
+
const emptyCid = (await emptyNode.cid()).$link;
25
+
26
+
const key = 'test/key';
27
+
const value = await createCid('test-value');
28
+
29
+
const newRootCid = await wrangler.putRecord(emptyCid, key, value);
30
+
31
+
// verify the tree contains the new entry
32
+
const walker = await NodeWalker.create(store, newRootCid);
33
+
const entries: Array<[string, any]> = [];
34
+
for await (const entry of walker.entries()) {
35
+
entries.push(entry);
36
+
}
37
+
38
+
expect(entries.length).toBe(1);
39
+
expect(entries[0][0]).toBe(key);
40
+
expect(entries[0][1].$link).toBe(value.$link);
41
+
});
42
+
43
+
it('should put a record into null (empty tree)', async () => {
44
+
const store = new NodeStore(new MemoryBlockStore());
45
+
const wrangler = new NodeWrangler(store);
46
+
47
+
const key = 'test/key';
48
+
const value = await createCid('test-value');
49
+
50
+
const newRootCid = await wrangler.putRecord(null, key, value);
51
+
52
+
// verify the tree contains the new entry
53
+
const walker = await NodeWalker.create(store, newRootCid);
54
+
const entries: Array<[string, any]> = [];
55
+
for await (const entry of walker.entries()) {
56
+
entries.push(entry);
57
+
}
58
+
59
+
expect(entries.length).toBe(1);
60
+
expect(entries[0][0]).toBe(key);
61
+
expect(entries[0][1].$link).toBe(value.$link);
62
+
});
63
+
64
+
it('should put multiple records', async () => {
65
+
const store = new NodeStore(new MemoryBlockStore());
66
+
const wrangler = new NodeWrangler(store);
67
+
68
+
let rootCid: string | null = null;
69
+
70
+
const keys = ['a', 'b', 'c', 'd'];
71
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
72
+
73
+
for (let i = 0; i < keys.length; i++) {
74
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
75
+
}
76
+
77
+
// verify all entries are present
78
+
const walker = await NodeWalker.create(store, rootCid);
79
+
const entries: Array<[string, any]> = [];
80
+
for await (const entry of walker.entries()) {
81
+
entries.push(entry);
82
+
}
83
+
84
+
expect(entries.length).toBe(keys.length);
85
+
for (let i = 0; i < keys.length; i++) {
86
+
expect(entries[i][0]).toBe(keys[i]);
87
+
expect(entries[i][1].$link).toBe(values[i].$link);
88
+
}
89
+
});
90
+
91
+
it('should be a no-op when putting the same value twice', async () => {
92
+
const store = new NodeStore(new MemoryBlockStore());
93
+
const wrangler = new NodeWrangler(store);
94
+
95
+
const key = 'test/key';
96
+
const value = await createCid('test-value');
97
+
98
+
const rootCid1 = await wrangler.putRecord(null, key, value);
99
+
const rootCid2 = await wrangler.putRecord(rootCid1, key, value);
100
+
101
+
// CIDs should be identical since nothing changed
102
+
expect(rootCid1).toBe(rootCid2);
103
+
});
104
+
105
+
it('should update an existing key with a new value', async () => {
106
+
const store = new NodeStore(new MemoryBlockStore());
107
+
const wrangler = new NodeWrangler(store);
108
+
109
+
const key = 'test/key';
110
+
const value1 = await createCid('value-1');
111
+
const value2 = await createCid('value-2');
112
+
113
+
const rootCid1 = await wrangler.putRecord(null, key, value1);
114
+
const rootCid2 = await wrangler.putRecord(rootCid1, key, value2);
115
+
116
+
// CIDs should be different
117
+
expect(rootCid1).not.toBe(rootCid2);
118
+
119
+
// verify the new value is present
120
+
const walker = await NodeWalker.create(store, rootCid2);
121
+
const entries: Array<[string, any]> = [];
122
+
for await (const entry of walker.entries()) {
123
+
entries.push(entry);
124
+
}
125
+
126
+
expect(entries.length).toBe(1);
127
+
expect(entries[0][0]).toBe(key);
128
+
expect(entries[0][1].$link).toBe(value2.$link);
129
+
});
130
+
131
+
it('should delete a record', async () => {
132
+
const store = new NodeStore(new MemoryBlockStore());
133
+
const wrangler = new NodeWrangler(store);
134
+
135
+
const key = 'test/key';
136
+
const value = await createCid('test-value');
137
+
138
+
const rootCid1 = await wrangler.putRecord(null, key, value);
139
+
const rootCid2 = await wrangler.deleteRecord(rootCid1, key);
140
+
141
+
// verify the tree is empty
142
+
const walker = await NodeWalker.create(store, rootCid2);
143
+
const entries: Array<[string, any]> = [];
144
+
for await (const entry of walker.entries()) {
145
+
entries.push(entry);
146
+
}
147
+
148
+
expect(entries.length).toBe(0);
149
+
});
150
+
151
+
it('should be a no-op when deleting a non-existent key', async () => {
152
+
const store = new NodeStore(new MemoryBlockStore());
153
+
const wrangler = new NodeWrangler(store);
154
+
155
+
const key1 = 'test/key1';
156
+
const key2 = 'test/key2';
157
+
const value = await createCid('test-value');
158
+
159
+
const rootCid1 = await wrangler.putRecord(null, key1, value);
160
+
const rootCid2 = await wrangler.deleteRecord(rootCid1, key2);
161
+
162
+
// CIDs should be identical since nothing changed
163
+
expect(rootCid1).toBe(rootCid2);
164
+
});
165
+
166
+
it('should handle multiple puts and deletes', async () => {
167
+
const store = new NodeStore(new MemoryBlockStore());
168
+
const wrangler = new NodeWrangler(store);
169
+
170
+
let rootCid: string | null = null;
171
+
172
+
// add several keys
173
+
const keys = ['a', 'b', 'c', 'd', 'e'];
174
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
175
+
176
+
for (let i = 0; i < keys.length; i++) {
177
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
178
+
}
179
+
180
+
// delete some keys
181
+
rootCid = await wrangler.deleteRecord(rootCid, 'b');
182
+
rootCid = await wrangler.deleteRecord(rootCid, 'd');
183
+
184
+
// verify remaining entries
185
+
const walker = await NodeWalker.create(store, rootCid);
186
+
const entries: Array<[string, any]> = [];
187
+
for await (const entry of walker.entries()) {
188
+
entries.push(entry);
189
+
}
190
+
191
+
expect(entries.length).toBe(3);
192
+
expect(entries[0][0]).toBe('a');
193
+
expect(entries[1][0]).toBe('c');
194
+
expect(entries[2][0]).toBe('e');
195
+
});
196
+
197
+
it('should maintain sort order across operations', async () => {
198
+
const store = new NodeStore(new MemoryBlockStore());
199
+
const wrangler = new NodeWrangler(store);
200
+
201
+
let rootCid: string | null = null;
202
+
203
+
// add keys in random order
204
+
const keys = ['e', 'b', 'd', 'a', 'c'];
205
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
206
+
207
+
for (let i = 0; i < keys.length; i++) {
208
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
209
+
}
210
+
211
+
// verify entries are in sorted order
212
+
const walker = await NodeWalker.create(store, rootCid);
213
+
const entries: Array<[string, any]> = [];
214
+
for await (const entry of walker.entries()) {
215
+
entries.push(entry);
216
+
}
217
+
218
+
expect(entries.length).toBe(5);
219
+
expect(entries[0][0]).toBe('a');
220
+
expect(entries[1][0]).toBe('b');
221
+
expect(entries[2][0]).toBe('c');
222
+
expect(entries[3][0]).toBe('d');
223
+
expect(entries[4][0]).toBe('e');
224
+
});
225
+
226
+
it('should handle deleting from a tree with multiple levels', async () => {
227
+
const store = new NodeStore(new MemoryBlockStore());
228
+
const wrangler = new NodeWrangler(store);
229
+
230
+
let rootCid: string | null = null;
231
+
232
+
// add many keys to create a multi-level tree
233
+
const keys = Array.from({ length: 20 }, (_, i) => `key-${i.toString().padStart(3, '0')}`);
234
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
235
+
236
+
for (let i = 0; i < keys.length; i++) {
237
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
238
+
}
239
+
240
+
// delete half the keys
241
+
for (let i = 0; i < keys.length; i += 2) {
242
+
rootCid = await wrangler.deleteRecord(rootCid, keys[i]);
243
+
}
244
+
245
+
// verify the remaining keys
246
+
const walker = await NodeWalker.create(store, rootCid);
247
+
const entries: Array<[string, any]> = [];
248
+
for await (const entry of walker.entries()) {
249
+
entries.push(entry);
250
+
}
251
+
252
+
expect(entries.length).toBe(10);
253
+
for (let i = 0; i < 10; i++) {
254
+
expect(entries[i][0]).toBe(keys[i * 2 + 1]);
255
+
}
256
+
});
257
+
258
+
it('should handle putting and deleting the same key multiple times', async () => {
259
+
const store = new NodeStore(new MemoryBlockStore());
260
+
const wrangler = new NodeWrangler(store);
261
+
262
+
let rootCid: string | null = null;
263
+
264
+
const key = 'test/key';
265
+
const value1 = await createCid('value-1');
266
+
const value2 = await createCid('value-2');
267
+
268
+
// put, delete, put again
269
+
rootCid = await wrangler.putRecord(rootCid, key, value1);
270
+
rootCid = await wrangler.deleteRecord(rootCid, key);
271
+
rootCid = await wrangler.putRecord(rootCid, key, value2);
272
+
273
+
// verify the final value
274
+
const walker = await NodeWalker.create(store, rootCid);
275
+
const entries: Array<[string, any]> = [];
276
+
for await (const entry of walker.entries()) {
277
+
entries.push(entry);
278
+
}
279
+
280
+
expect(entries.length).toBe(1);
281
+
expect(entries[0][0]).toBe(key);
282
+
expect(entries[0][1].$link).toBe(value2.$link);
283
+
});
284
+
285
+
it('should handle empty tree deletion', async () => {
286
+
const store = new NodeStore(new MemoryBlockStore());
287
+
const wrangler = new NodeWrangler(store);
288
+
289
+
const emptyNode = MSTNode.empty();
290
+
await store.put(emptyNode);
291
+
const emptyCid = (await emptyNode.cid()).$link;
292
+
293
+
const key = 'test/key';
294
+
const newRootCid = await wrangler.deleteRecord(emptyCid, key);
295
+
296
+
// should be no-op
297
+
expect(newRootCid).toBe(emptyCid);
298
+
});
299
+
});
+330
packages/utilities/mst/lib/node-wrangler.ts
+330
packages/utilities/mst/lib/node-wrangler.ts
···
1
+
import type { CidLink } from '@atcute/cid';
2
+
3
+
import { MSTNode, getKeyHeight } from './node.js';
4
+
import { NodeStore } from './node-store.js';
5
+
6
+
/**
7
+
* array helper: replaces element at index with a new value
8
+
*/
9
+
const replaceAt = <T>(arr: readonly T[], index: number, value: T): readonly T[] => {
10
+
return [...arr.slice(0, index), value, ...arr.slice(index + 1)];
11
+
};
12
+
13
+
/**
14
+
* array helper: inserts element at index
15
+
*/
16
+
const insertAt = <T>(arr: readonly T[], index: number, value: T): readonly T[] => {
17
+
return [...arr.slice(0, index), value, ...arr.slice(index)];
18
+
};
19
+
20
+
/**
21
+
* array helper: removes element at index
22
+
*/
23
+
const removeAt = <T>(arr: readonly T[], index: number): readonly T[] => {
24
+
return [...arr.slice(0, index), ...arr.slice(index + 1)];
25
+
};
26
+
27
+
/**
28
+
* NodeWrangler is where core MST transformation ops are implemented, backed
29
+
* by a NodeStore
30
+
*
31
+
* the external APIs take a CID (the MST root) and return a CID (the new root),
32
+
* while storing any newly created nodes in the NodeStore.
33
+
*
34
+
* neither method should ever fail - deleting a node that doesn't exist is a nop,
35
+
* and adding the same node twice with the same value is also a nop. Callers
36
+
* can detect these cases by seeing if the initial and final CIDs changed.
37
+
*/
38
+
export class NodeWrangler {
39
+
/** underlying node store */
40
+
ns: NodeStore;
41
+
42
+
constructor(ns: NodeStore) {
43
+
this.ns = ns;
44
+
}
45
+
46
+
/**
47
+
* inserts or updates a record in the MST
48
+
* @param rootCid CID of the root node (or null for empty tree)
49
+
* @param key the key to insert/update
50
+
* @param val the value CID to associate with the key
51
+
* @returns the new root CID
52
+
*/
53
+
async putRecord(rootCid: string | null, key: string, val: CidLink): Promise<string> {
54
+
const root = await this.ns.get(rootCid);
55
+
56
+
if (root.isEmpty) {
57
+
// special case for empty tree
58
+
const newNode = await this._putHere(root, key, val);
59
+
return (await newNode.cid()).$link;
60
+
}
61
+
62
+
const newNode = await this._putRecursive(
63
+
root,
64
+
key,
65
+
val,
66
+
await getKeyHeight(key),
67
+
await root.requireHeight(),
68
+
);
69
+
return (await newNode.cid()).$link;
70
+
}
71
+
72
+
/**
73
+
* deletes a record from the MST
74
+
* @param rootCid CID of the root node (or null for empty tree)
75
+
* @param key the key to delete
76
+
* @returns the new root CID
77
+
*/
78
+
async deleteRecord(rootCid: string | null, key: string): Promise<string> {
79
+
const root = await this.ns.get(rootCid);
80
+
81
+
// Note: the seemingly redundant outer .get().cid is required to transform
82
+
// a null cid into the cid representing an empty node
83
+
const resultCid = await this._deleteRecursive(
84
+
root,
85
+
key,
86
+
await getKeyHeight(key),
87
+
await root.requireHeight(),
88
+
);
89
+
const squashed = await this._squashTop(resultCid?.$link ?? null);
90
+
const finalNode = await this.ns.get(squashed);
91
+
92
+
return (await finalNode.cid()).$link;
93
+
}
94
+
95
+
/**
96
+
* inserts a key-value pair into the current node
97
+
* @param node the node to insert into
98
+
* @param key the key to insert
99
+
* @param val the value to insert
100
+
* @returns the updated node
101
+
*/
102
+
private async _putHere(node: MSTNode, key: string, val: CidLink): Promise<MSTNode> {
103
+
const idx = node.lowerBound(key);
104
+
105
+
// the key is already present!
106
+
if (idx < node.keys.length && node.keys[idx] === key) {
107
+
if (node.values[idx].$link === val.$link) {
108
+
return node; // we can return our old self if there is no change
109
+
}
110
+
111
+
return await this.ns.put(
112
+
await MSTNode.create(node.keys, replaceAt(node.values, idx, val), node.subtrees),
113
+
);
114
+
}
115
+
116
+
// split the subtree at the insertion point
117
+
const [lsub, rsub] = await this._splitOnKey(node.subtrees[idx], key);
118
+
119
+
return await this.ns.put(
120
+
await MSTNode.create(insertAt(node.keys, idx, key), insertAt(node.values, idx, val), [
121
+
...node.subtrees.slice(0, idx),
122
+
lsub,
123
+
rsub,
124
+
...node.subtrees.slice(idx + 1),
125
+
]),
126
+
);
127
+
}
128
+
129
+
/**
130
+
* recursively inserts a key-value pair, growing the tree if necessary
131
+
* @param node the current node
132
+
* @param key the key to insert
133
+
* @param val the value to insert
134
+
* @param keyHeight the height of the key (based on hash)
135
+
* @param treeHeight the current tree height
136
+
* @returns the updated node
137
+
*/
138
+
private async _putRecursive(
139
+
node: MSTNode,
140
+
key: string,
141
+
val: CidLink,
142
+
keyHeight: number,
143
+
treeHeight: number,
144
+
): Promise<MSTNode> {
145
+
if (keyHeight > treeHeight) {
146
+
// we need to grow the tree
147
+
return await this._putRecursive(
148
+
await this.ns.put(await MSTNode.create([], [], [await node.cid()])),
149
+
key,
150
+
val,
151
+
keyHeight,
152
+
treeHeight + 1,
153
+
);
154
+
}
155
+
156
+
if (keyHeight < treeHeight) {
157
+
// we need to look below
158
+
const idx = node.lowerBound(key);
159
+
return await this.ns.put(
160
+
await MSTNode.create(
161
+
node.keys,
162
+
node.values,
163
+
replaceAt(
164
+
node.subtrees,
165
+
idx,
166
+
await (
167
+
await this._putRecursive(
168
+
await this.ns.get(node.subtrees[idx]?.$link ?? null),
169
+
key,
170
+
val,
171
+
keyHeight,
172
+
treeHeight - 1,
173
+
)
174
+
).cid(),
175
+
),
176
+
),
177
+
);
178
+
}
179
+
180
+
// we can insert here
181
+
return await this._putHere(node, key, val);
182
+
}
183
+
184
+
/**
185
+
* splits a subtree around a key, producing left and right subtrees
186
+
* @param nodeCid the CID of the subtree to split (or null)
187
+
* @param key the key to split around
188
+
* @returns tuple of [left subtree CID, right subtree CID]
189
+
*/
190
+
private async _splitOnKey(nodeCid: CidLink | null, key: string): Promise<[CidLink | null, CidLink | null]> {
191
+
if (nodeCid === null) {
192
+
return [null, null];
193
+
}
194
+
195
+
const node = await this.ns.get(nodeCid.$link);
196
+
const idx = node.lowerBound(key);
197
+
const [lsub, rsub] = await this._splitOnKey(node.subtrees[idx], key);
198
+
199
+
const leftNode = await this.ns.put(
200
+
await MSTNode.create(node.keys.slice(0, idx), node.values.slice(0, idx), [
201
+
...node.subtrees.slice(0, idx),
202
+
lsub,
203
+
]),
204
+
);
205
+
206
+
const rightNode = await this.ns.put(
207
+
await MSTNode.create(node.keys.slice(idx), node.values.slice(idx), [
208
+
rsub,
209
+
...node.subtrees.slice(idx + 1),
210
+
]),
211
+
);
212
+
213
+
return [await leftNode._toNullable(), await rightNode._toNullable()];
214
+
}
215
+
216
+
/**
217
+
* strips empty nodes from the top of the tree
218
+
* @param nodeCid the CID of the node to check
219
+
* @returns the CID after removing empty top nodes
220
+
*/
221
+
private async _squashTop(nodeCid: string | null): Promise<string | null> {
222
+
const node = await this.ns.get(nodeCid);
223
+
224
+
if (node.keys.length > 0) {
225
+
return nodeCid;
226
+
}
227
+
228
+
if (node.subtrees[0] === null) {
229
+
return nodeCid;
230
+
}
231
+
232
+
return await this._squashTop(node.subtrees[0].$link);
233
+
}
234
+
235
+
/**
236
+
* recursively deletes a key from the tree
237
+
* @param node the current node
238
+
* @param key the key to delete
239
+
* @param keyHeight the height of the key
240
+
* @param treeHeight the current tree height
241
+
* @returns the CID of the updated node, or null if it becomes empty
242
+
*/
243
+
private async _deleteRecursive(
244
+
node: MSTNode,
245
+
key: string,
246
+
keyHeight: number,
247
+
treeHeight: number,
248
+
): Promise<CidLink | null> {
249
+
if (keyHeight > treeHeight) {
250
+
// the key cannot possibly be in this tree, no change needed
251
+
return await node._toNullable();
252
+
}
253
+
254
+
const idx = node.lowerBound(key);
255
+
256
+
if (keyHeight < treeHeight) {
257
+
// the key must be deleted from a subtree
258
+
if (node.subtrees[idx] === null) {
259
+
return await node._toNullable(); // the key cannot be in this subtree, no change needed
260
+
}
261
+
262
+
const updated = await this.ns.put(
263
+
await MSTNode.create(
264
+
node.keys,
265
+
node.values,
266
+
replaceAt(
267
+
node.subtrees,
268
+
idx,
269
+
await this._deleteRecursive(
270
+
await this.ns.get(node.subtrees[idx]!.$link),
271
+
key,
272
+
keyHeight,
273
+
treeHeight - 1,
274
+
),
275
+
),
276
+
),
277
+
);
278
+
279
+
return await updated._toNullable();
280
+
}
281
+
282
+
if (idx === node.keys.length || node.keys[idx] !== key) {
283
+
return await node._toNullable(); // key already not present
284
+
}
285
+
286
+
// merge the subtrees on either side of the deleted key
287
+
const merged = await this._merge(node.subtrees[idx], node.subtrees[idx + 1]);
288
+
289
+
const updated = await this.ns.put(
290
+
await MSTNode.create(removeAt(node.keys, idx), removeAt(node.values, idx), [
291
+
...node.subtrees.slice(0, idx),
292
+
merged,
293
+
...node.subtrees.slice(idx + 2),
294
+
]),
295
+
);
296
+
297
+
return await updated._toNullable();
298
+
}
299
+
300
+
/**
301
+
* merges two adjacent subtrees
302
+
* @param leftCid CID of the left subtree (or null)
303
+
* @param rightCid CID of the right subtree (or null)
304
+
* @returns the CID of the merged subtree (or null if both are null)
305
+
*/
306
+
private async _merge(leftCid: CidLink | null, rightCid: CidLink | null): Promise<CidLink | null> {
307
+
if (leftCid === null) {
308
+
return rightCid; // includes the case where left == right == null
309
+
}
310
+
if (rightCid === null) {
311
+
return leftCid;
312
+
}
313
+
314
+
const left = await this.ns.get(leftCid.$link);
315
+
const right = await this.ns.get(rightCid.$link);
316
+
317
+
// recursively merge the adjacent subtrees at the boundary
318
+
const mergedBoundary = await this._merge(left.subtrees[left.subtrees.length - 1], right.subtrees[0]);
319
+
320
+
const merged = await this.ns.put(
321
+
await MSTNode.create(
322
+
[...left.keys, ...right.keys],
323
+
[...left.values, ...right.values],
324
+
[...left.subtrees.slice(0, -1), mergedBoundary, ...right.subtrees.slice(1)],
325
+
),
326
+
);
327
+
328
+
return await merged._toNullable();
329
+
}
330
+
}
+12
packages/utilities/mst/lib/node.ts
+12
packages/utilities/mst/lib/node.ts
···
234
234
235
235
return len;
236
236
}
237
+
238
+
/**
239
+
* returns the node's CID if it's not empty, null otherwise
240
+
* @returns the CID or null
241
+
* @internal
242
+
*/
243
+
async _toNullable(): Promise<CidLink | null> {
244
+
if (this.isEmpty) {
245
+
return null;
246
+
}
247
+
return await this.cid();
248
+
}
237
249
}
238
250
239
251
/**
+198
packages/utilities/mst/lib/proof.test.ts
+198
packages/utilities/mst/lib/proof.test.ts
···
1
+
import { describe, expect, it } from 'vitest';
2
+
3
+
import * as CID from '@atcute/cid';
4
+
import { encodeUtf8 } from '@atcute/uint8array';
5
+
6
+
import { NodeStore } from './node-store.js';
7
+
import { NodeWrangler } from './node-wrangler.js';
8
+
import {
9
+
buildExclusionProof,
10
+
buildInclusionProof,
11
+
InvalidProofError,
12
+
ProofError,
13
+
verifyExclusion,
14
+
verifyInclusion,
15
+
} from './proof.js';
16
+
import { MemoryBlockStore } from './stores.js';
17
+
18
+
const createCid = async (data: string) => {
19
+
const bytes = encodeUtf8(data);
20
+
return CID.toCidLink(await CID.create(0x55, bytes));
21
+
};
22
+
23
+
describe('Proof', () => {
24
+
it('should build and verify inclusion proof', async () => {
25
+
const store = new NodeStore(new MemoryBlockStore());
26
+
const wrangler = new NodeWrangler(store);
27
+
28
+
// build a tree with some records
29
+
let rootCid: string | null = null;
30
+
const keys = ['a/1', 'b/2', 'c/3'];
31
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
32
+
33
+
for (let i = 0; i < keys.length; i++) {
34
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
35
+
}
36
+
37
+
// build inclusion proof for 'b/2'
38
+
const proof = await buildInclusionProof(store, rootCid!, 'b/2');
39
+
40
+
expect(proof.size).toBeGreaterThan(0);
41
+
42
+
// create a new store with only the proof blocks
43
+
const proofStore = new NodeStore(new MemoryBlockStore());
44
+
for (const cid of proof) {
45
+
const node = await store.get(cid);
46
+
await proofStore.put(node);
47
+
}
48
+
49
+
// verify the inclusion proof
50
+
await expect(verifyInclusion(proofStore, rootCid!, 'b/2')).resolves.toBeUndefined();
51
+
});
52
+
53
+
it('should build and verify exclusion proof', async () => {
54
+
const store = new NodeStore(new MemoryBlockStore());
55
+
const wrangler = new NodeWrangler(store);
56
+
57
+
// build a tree with some records
58
+
let rootCid: string | null = null;
59
+
const keys = ['a/1', 'b/2', 'c/3'];
60
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
61
+
62
+
for (let i = 0; i < keys.length; i++) {
63
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
64
+
}
65
+
66
+
// build exclusion proof for 'd/4' (doesn't exist)
67
+
const proof = await buildExclusionProof(store, rootCid!, 'd/4');
68
+
69
+
expect(proof.size).toBeGreaterThan(0);
70
+
71
+
// create a new store with only the proof blocks
72
+
const proofStore = new NodeStore(new MemoryBlockStore());
73
+
for (const cid of proof) {
74
+
const node = await store.get(cid);
75
+
await proofStore.put(node);
76
+
}
77
+
78
+
// verify the exclusion proof
79
+
await expect(verifyExclusion(proofStore, rootCid!, 'd/4')).resolves.toBeUndefined();
80
+
});
81
+
82
+
it('should throw ProofError when building inclusion proof for non-existent record', async () => {
83
+
const store = new NodeStore(new MemoryBlockStore());
84
+
const wrangler = new NodeWrangler(store);
85
+
86
+
let rootCid: string | null = null;
87
+
rootCid = await wrangler.putRecord(rootCid, 'a/1', await createCid('value-a'));
88
+
89
+
await expect(buildInclusionProof(store, rootCid, 'b/2')).rejects.toThrow(ProofError);
90
+
await expect(buildInclusionProof(store, rootCid, 'b/2')).rejects.toThrow("doesn't exist");
91
+
});
92
+
93
+
it('should throw ProofError when building exclusion proof for existing record', async () => {
94
+
const store = new NodeStore(new MemoryBlockStore());
95
+
const wrangler = new NodeWrangler(store);
96
+
97
+
let rootCid: string | null = null;
98
+
rootCid = await wrangler.putRecord(rootCid, 'a/1', await createCid('value-a'));
99
+
100
+
await expect(buildExclusionProof(store, rootCid, 'a/1')).rejects.toThrow(ProofError);
101
+
await expect(buildExclusionProof(store, rootCid, 'a/1')).rejects.toThrow('that exists');
102
+
});
103
+
104
+
it('should throw InvalidProofError when verifying inclusion with missing blocks', async () => {
105
+
const store = new NodeStore(new MemoryBlockStore());
106
+
const wrangler = new NodeWrangler(store);
107
+
108
+
let rootCid: string | null = null;
109
+
const keys = ['a/1', 'b/2', 'c/3'];
110
+
const values = await Promise.all(keys.map((k) => createCid(`value-${k}`)));
111
+
112
+
for (let i = 0; i < keys.length; i++) {
113
+
rootCid = await wrangler.putRecord(rootCid, keys[i], values[i]);
114
+
}
115
+
116
+
// create a store with no blocks
117
+
const emptyStore = new NodeStore(new MemoryBlockStore());
118
+
119
+
await expect(verifyInclusion(emptyStore, rootCid!, 'b/2')).rejects.toThrow(InvalidProofError);
120
+
await expect(verifyInclusion(emptyStore, rootCid!, 'b/2')).rejects.toThrow('missing MST blocks');
121
+
});
122
+
123
+
it('should throw InvalidProofError when verifying exclusion with missing blocks', async () => {
124
+
const store = new NodeStore(new MemoryBlockStore());
125
+
const wrangler = new NodeWrangler(store);
126
+
127
+
let rootCid: string | null = null;
128
+
rootCid = await wrangler.putRecord(rootCid, 'a/1', await createCid('value-a'));
129
+
130
+
// create a store with no blocks
131
+
const emptyStore = new NodeStore(new MemoryBlockStore());
132
+
133
+
await expect(verifyExclusion(emptyStore, rootCid, 'd/4')).rejects.toThrow(InvalidProofError);
134
+
await expect(verifyExclusion(emptyStore, rootCid, 'd/4')).rejects.toThrow('missing MST blocks');
135
+
});
136
+
137
+
it('should throw InvalidProofError when verifying inclusion proof for non-existent record', async () => {
138
+
const store = new NodeStore(new MemoryBlockStore());
139
+
const wrangler = new NodeWrangler(store);
140
+
141
+
let rootCid: string | null = null;
142
+
rootCid = await wrangler.putRecord(rootCid, 'a/1', await createCid('value-a'));
143
+
144
+
// build proof for existing record
145
+
const proof = await buildInclusionProof(store, rootCid, 'a/1');
146
+
const proofStore = new NodeStore(new MemoryBlockStore());
147
+
for (const cid of proof) {
148
+
const node = await store.get(cid);
149
+
await proofStore.put(node);
150
+
}
151
+
152
+
// try to verify for a different record
153
+
await expect(verifyInclusion(proofStore, rootCid, 'b/2')).rejects.toThrow(InvalidProofError);
154
+
await expect(verifyInclusion(proofStore, rootCid, 'b/2')).rejects.toThrow('not present in MST');
155
+
});
156
+
157
+
it('should throw InvalidProofError when verifying exclusion proof for existing record', async () => {
158
+
const store = new NodeStore(new MemoryBlockStore());
159
+
const wrangler = new NodeWrangler(store);
160
+
161
+
let rootCid: string | null = null;
162
+
rootCid = await wrangler.putRecord(rootCid, 'a/1', await createCid('value-a'));
163
+
164
+
// build proof for non-existing record
165
+
const proof = await buildExclusionProof(store, rootCid, 'b/2');
166
+
const proofStore = new NodeStore(new MemoryBlockStore());
167
+
for (const cid of proof) {
168
+
const node = await store.get(cid);
169
+
await proofStore.put(node);
170
+
}
171
+
172
+
// try to verify exclusion for the existing record
173
+
await expect(verifyExclusion(proofStore, rootCid, 'a/1')).rejects.toThrow(InvalidProofError);
174
+
await expect(verifyExclusion(proofStore, rootCid, 'a/1')).rejects.toThrow('*is* present in MST');
175
+
});
176
+
177
+
it('should handle proofs on empty tree', async () => {
178
+
const store = new NodeStore(new MemoryBlockStore());
179
+
180
+
const emptyNode = await store.get(null);
181
+
const emptyCid = (await emptyNode.cid()).$link;
182
+
183
+
// exclusion proof should work on empty tree
184
+
const proof = await buildExclusionProof(store, emptyCid, 'a/1');
185
+
expect(proof.size).toBeGreaterThan(0);
186
+
187
+
const proofStore = new NodeStore(new MemoryBlockStore());
188
+
for (const cid of proof) {
189
+
const node = await store.get(cid);
190
+
await proofStore.put(node);
191
+
}
192
+
193
+
await expect(verifyExclusion(proofStore, emptyCid, 'a/1')).resolves.toBeUndefined();
194
+
195
+
// inclusion proof should fail on empty tree
196
+
await expect(buildInclusionProof(store, emptyCid, 'a/1')).rejects.toThrow(ProofError);
197
+
});
198
+
});
+132
packages/utilities/mst/lib/proof.ts
+132
packages/utilities/mst/lib/proof.ts
···
1
+
import type { CidLink } from '@atcute/cid';
2
+
3
+
import { MissingBlockError } from './errors.js';
4
+
import type { NodeStore } from './node-store.js';
5
+
import { NodeWalker } from './node-walker.js';
6
+
7
+
/**
8
+
* Error thrown when validating a proof fails
9
+
*/
10
+
export class InvalidProofError extends Error {
11
+
constructor(message: string) {
12
+
super(message);
13
+
this.name = 'InvalidProofError';
14
+
}
15
+
}
16
+
17
+
/**
18
+
* Error thrown when constructing a proof fails
19
+
*/
20
+
export class ProofError extends Error {
21
+
constructor(message: string) {
22
+
super(message);
23
+
this.name = 'ProofError';
24
+
}
25
+
}
26
+
27
+
/**
28
+
* Finds a record path and builds a proof (works for both inclusion and exclusion proofs)
29
+
* @param ns the node store
30
+
* @param rootCid the MST root CID
31
+
* @param rpath the record path to find
32
+
* @returns tuple of [value CID or null, set of proof node CIDs]
33
+
*/
34
+
export const findRpathAndBuildProof = async (
35
+
ns: NodeStore,
36
+
rootCid: string,
37
+
rpath: string,
38
+
): Promise<[CidLink | null, Set<string>]> => {
39
+
const walker = await NodeWalker.create(ns, rootCid);
40
+
const value = await walker.findRpath(rpath);
41
+
42
+
const proof = new Set<string>();
43
+
for (const frame of walker.stack) {
44
+
proof.add((await frame.node.cid()).$link);
45
+
}
46
+
47
+
return [value, proof];
48
+
};
49
+
50
+
/**
51
+
* Builds an exclusion proof for a record that should not exist
52
+
* @param ns the node store
53
+
* @param rootCid the MST root CID
54
+
* @param rpath the record path
55
+
* @returns set of MST node CIDs needed for the exclusion proof
56
+
* @throws {ProofError} if the record exists
57
+
*/
58
+
export const buildExclusionProof = async (
59
+
ns: NodeStore,
60
+
rootCid: string,
61
+
rpath: string,
62
+
): Promise<Set<string>> => {
63
+
const [value, proof] = await findRpathAndBuildProof(ns, rootCid, rpath);
64
+
if (value !== null) {
65
+
throw new ProofError("can't build exclusion proof for a record that exists!");
66
+
}
67
+
return proof;
68
+
};
69
+
70
+
/**
71
+
* Builds an inclusion proof for a record that should exist
72
+
* @param ns the node store
73
+
* @param rootCid the MST root CID
74
+
* @param rpath the record path
75
+
* @returns set of MST node CIDs needed for the inclusion proof
76
+
* @throws {ProofError} if the record doesn't exist
77
+
*/
78
+
export const buildInclusionProof = async (
79
+
ns: NodeStore,
80
+
rootCid: string,
81
+
rpath: string,
82
+
): Promise<Set<string>> => {
83
+
const [value, proof] = await findRpathAndBuildProof(ns, rootCid, rpath);
84
+
if (value === null) {
85
+
throw new ProofError("can't build inclusion proof for a record that doesn't exist!");
86
+
}
87
+
return proof;
88
+
};
89
+
90
+
/**
91
+
* Verifies an inclusion proof - that a record exists in the MST
92
+
* @param ns the node store (should only contain blocks from the proof)
93
+
* @param rootCid the MST root CID
94
+
* @param rpath the record path
95
+
* @throws {InvalidProofError} if the proof is invalid or the record doesn't exist
96
+
*/
97
+
export const verifyInclusion = async (ns: NodeStore, rootCid: string, rpath: string): Promise<void> => {
98
+
try {
99
+
const walker = await NodeWalker.create(ns, rootCid);
100
+
const value = await walker.findRpath(rpath);
101
+
if (value === null) {
102
+
throw new InvalidProofError('rpath not present in MST');
103
+
}
104
+
} catch (err) {
105
+
if (err instanceof MissingBlockError) {
106
+
throw new InvalidProofError('missing MST blocks');
107
+
}
108
+
throw err;
109
+
}
110
+
};
111
+
112
+
/**
113
+
* Verifies an exclusion proof - that a record does not exist in the MST
114
+
* @param ns the node store (should only contain blocks from the proof)
115
+
* @param rootCid the MST root CID
116
+
* @param rpath the record path
117
+
* @throws {InvalidProofError} if the proof is invalid or the record exists
118
+
*/
119
+
export const verifyExclusion = async (ns: NodeStore, rootCid: string, rpath: string): Promise<void> => {
120
+
try {
121
+
const walker = await NodeWalker.create(ns, rootCid);
122
+
const value = await walker.findRpath(rpath);
123
+
if (value !== null) {
124
+
throw new InvalidProofError('rpath *is* present in MST');
125
+
}
126
+
} catch (err) {
127
+
if (err instanceof MissingBlockError) {
128
+
throw new InvalidProofError('missing MST blocks');
129
+
}
130
+
throw err;
131
+
}
132
+
};
+191
packages/utilities/mst/lib/test-suite.test.ts
+191
packages/utilities/mst/lib/test-suite.test.ts
···
1
+
import { readFileSync, readdirSync, statSync } from 'node:fs';
2
+
import { join } from 'node:path';
3
+
import { describe, expect, it } from 'vitest';
4
+
5
+
import { fromUint8Array } from '@atcute/car/v4/car-reader';
6
+
import * as CID from '@atcute/cid';
7
+
8
+
import { DeltaType, mstDiff, recordDiff } from './diff.js';
9
+
import { NodeStore } from './node-store.js';
10
+
import { MemoryBlockStore } from './stores.js';
11
+
12
+
interface MstDiffTestCase {
13
+
$type: 'mst-diff';
14
+
description: string;
15
+
inputs: {
16
+
mst_a: string;
17
+
mst_b: string;
18
+
};
19
+
results: {
20
+
created_nodes: string[];
21
+
deleted_nodes: string[];
22
+
record_ops: Array<{
23
+
rpath: string;
24
+
old_value: string | null;
25
+
new_value: string | null;
26
+
}>;
27
+
proof_nodes: string[];
28
+
inductive_proof_nodes: string[];
29
+
firehose_cids: string | string[];
30
+
};
31
+
}
32
+
33
+
/**
34
+
* Load a CAR file into a MemoryBlockStore and extract the root CID
35
+
*/
36
+
const loadCar = (carPath: string): { store: MemoryBlockStore; root: string } => {
37
+
const testSuiteRoot = join(__dirname, '..', '.research', 'mst-test-suite');
38
+
const fullPath = join(testSuiteRoot, carPath);
39
+
const carBytes = readFileSync(fullPath);
40
+
41
+
const car = fromUint8Array(carBytes);
42
+
const store = new MemoryBlockStore();
43
+
44
+
// Load all blocks from CAR into the store
45
+
for (const entry of car) {
46
+
const cidStr = CID.toCidLink(entry.cid).$link;
47
+
store.blocks.set(cidStr, entry.bytes);
48
+
}
49
+
50
+
// Extract root CID from CAR header
51
+
if (car.roots.length !== 1) {
52
+
throw new Error(`Expected exactly 1 root in CAR, got ${car.roots.length}`);
53
+
}
54
+
55
+
const root = car.roots[0].$link;
56
+
return { store, root };
57
+
};
58
+
59
+
/**
60
+
* Recursively find all .json test files in a directory
61
+
*/
62
+
const findTestFiles = (dir: string): string[] => {
63
+
const results: string[] = [];
64
+
const entries = readdirSync(dir);
65
+
66
+
for (const entry of entries) {
67
+
const fullPath = join(dir, entry);
68
+
const stat = statSync(fullPath);
69
+
70
+
if (stat.isDirectory()) {
71
+
results.push(...findTestFiles(fullPath));
72
+
} else if (entry.endsWith('.json')) {
73
+
results.push(fullPath);
74
+
}
75
+
}
76
+
77
+
return results;
78
+
};
79
+
80
+
/**
81
+
* Load all test cases from the test suite
82
+
*/
83
+
const loadTestCases = (): Array<{ path: string; testCase: MstDiffTestCase }> => {
84
+
const testSuiteRoot = join(__dirname, '..', '.research', 'mst-test-suite');
85
+
const testsDir = join(testSuiteRoot, 'tests');
86
+
const testFiles = findTestFiles(testsDir);
87
+
88
+
const testCases: Array<{ path: string; testCase: MstDiffTestCase }> = [];
89
+
90
+
for (const filePath of testFiles) {
91
+
const content = readFileSync(filePath, 'utf-8');
92
+
const testCase = JSON.parse(content) as MstDiffTestCase;
93
+
94
+
if (testCase.$type === 'mst-diff') {
95
+
testCases.push({ path: filePath, testCase });
96
+
}
97
+
}
98
+
99
+
return testCases;
100
+
};
101
+
102
+
describe('MST Test Suite', () => {
103
+
const allTestCases = loadTestCases();
104
+
105
+
// Run all test cases
106
+
const testCases = allTestCases;
107
+
108
+
it(`should have loaded test cases (${testCases.length} total)`, () => {
109
+
expect(testCases.length).toBeGreaterThan(1000); // Should have 16k+ tests
110
+
});
111
+
112
+
describe.each(testCases)('$testCase.description', ({ testCase }) => {
113
+
it('should compute correct mstDiff', async () => {
114
+
// Load both CARs
115
+
const { store: storeA, root: rootA } = loadCar(testCase.inputs.mst_a);
116
+
const { store: storeB, root: rootB } = loadCar(testCase.inputs.mst_b);
117
+
118
+
// Create NodeStores (combine both block stores for access to all blocks)
119
+
// We need an overlay approach since diff needs to read from both trees
120
+
const combinedStore = new MemoryBlockStore();
121
+
for (const [cid, bytes] of storeA.blocks) {
122
+
combinedStore.blocks.set(cid, bytes);
123
+
}
124
+
for (const [cid, bytes] of storeB.blocks) {
125
+
combinedStore.blocks.set(cid, bytes);
126
+
}
127
+
128
+
const nodeStore = new NodeStore(combinedStore);
129
+
130
+
// Run mstDiff
131
+
const [createdNodes, deletedNodes] = await mstDiff(nodeStore, rootA, rootB);
132
+
133
+
// Compare created_nodes (as sets, order doesn't matter)
134
+
const expectedCreated = new Set(testCase.results.created_nodes);
135
+
expect(createdNodes).toEqual(expectedCreated);
136
+
137
+
// Compare deleted_nodes (as sets, order doesn't matter)
138
+
const expectedDeleted = new Set(testCase.results.deleted_nodes);
139
+
expect(deletedNodes).toEqual(expectedDeleted);
140
+
});
141
+
142
+
it('should compute correct recordDiff', async () => {
143
+
// Load both CARs
144
+
const { store: storeA, root: rootA } = loadCar(testCase.inputs.mst_a);
145
+
const { store: storeB, root: rootB } = loadCar(testCase.inputs.mst_b);
146
+
147
+
// Create combined NodeStore
148
+
const combinedStore = new MemoryBlockStore();
149
+
for (const [cid, bytes] of storeA.blocks) {
150
+
combinedStore.blocks.set(cid, bytes);
151
+
}
152
+
for (const [cid, bytes] of storeB.blocks) {
153
+
combinedStore.blocks.set(cid, bytes);
154
+
}
155
+
156
+
const nodeStore = new NodeStore(combinedStore);
157
+
158
+
// Run mstDiff and recordDiff
159
+
const [createdNodes, deletedNodes] = await mstDiff(nodeStore, rootA, rootB);
160
+
161
+
const deltas = [];
162
+
for await (const delta of recordDiff(nodeStore, createdNodes, deletedNodes)) {
163
+
deltas.push(delta);
164
+
}
165
+
166
+
// Sort both actual and expected by rpath for comparison
167
+
const sortedDeltas = deltas.sort((a, b) => a.path.localeCompare(b.path));
168
+
const sortedExpected = [...testCase.results.record_ops].sort((a, b) => a.rpath.localeCompare(b.rpath));
169
+
170
+
expect(sortedDeltas.length).toBe(sortedExpected.length);
171
+
172
+
for (let i = 0; i < sortedDeltas.length; i++) {
173
+
const actual = sortedDeltas[i];
174
+
const expected = sortedExpected[i];
175
+
176
+
expect(actual.path).toBe(expected.rpath);
177
+
expect(actual.priorValue?.$link ?? null).toBe(expected.old_value);
178
+
expect(actual.laterValue?.$link ?? null).toBe(expected.new_value);
179
+
180
+
// Verify delta type is correct
181
+
if (expected.old_value === null) {
182
+
expect(actual.deltaType).toBe(DeltaType.CREATED);
183
+
} else if (expected.new_value === null) {
184
+
expect(actual.deltaType).toBe(DeltaType.DELETED);
185
+
} else {
186
+
expect(actual.deltaType).toBe(DeltaType.UPDATED);
187
+
}
188
+
}
189
+
});
190
+
});
191
+
});
+1
packages/utilities/mst/package.json
+1
packages/utilities/mst/package.json
+8
-5
pnpm-lock.yaml
+8
-5
pnpm-lock.yaml
···
488
488
'@valibot/to-json-schema':
489
489
specifier: ^1.3.0
490
490
version: 1.3.0(valibot@1.1.0(typescript@5.9.2))
491
-
tsx:
492
-
specifier: ^4.19.2
493
-
version: 4.20.6
494
491
495
492
packages/lexicons/lexicon-doc:
496
493
dependencies:
···
754
751
specifier: ^0.4.6
755
752
version: 0.4.6
756
753
devDependencies:
754
+
'@atcute/car':
755
+
specifier: workspace:^
756
+
version: link:../car
757
757
'@vitest/coverage-v8':
758
758
specifier: ^3.2.4
759
759
version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4)
760
760
vitest:
761
761
specifier: ^3.2.4
762
-
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(yaml@2.8.0)
762
+
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(tsx@4.20.6)(yaml@2.8.0)
763
763
764
764
packages/utilities/multibase:
765
765
dependencies:
···
6273
6273
get-tsconfig@4.12.0:
6274
6274
dependencies:
6275
6275
resolve-pkg-maps: 1.0.0
6276
+
optional: true
6276
6277
6277
6278
github-from-package@0.0.0: {}
6278
6279
···
6881
6882
6882
6883
resolve-from@5.0.0: {}
6883
6884
6884
-
resolve-pkg-maps@1.0.0: {}
6885
+
resolve-pkg-maps@1.0.0:
6886
+
optional: true
6885
6887
6886
6888
reusify@1.1.0: {}
6887
6889
···
7188
7190
get-tsconfig: 4.12.0
7189
7191
optionalDependencies:
7190
7192
fsevents: 2.3.3
7193
+
optional: true
7191
7194
7192
7195
tunnel-agent@0.6.0:
7193
7196
dependencies: