a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
feat(mst): initial implementation
+1419
+25
packages/utilities/mst/lib/blockmap.ts
+25
packages/utilities/mst/lib/blockmap.ts
···
1
+
import * as CBOR from '@atcute/cbor';
2
+
import * as CID from '@atcute/cid';
3
+
4
+
type BlockEntry = [cid: string, bytes: Uint8Array<ArrayBuffer>];
5
+
6
+
export type BlockMap = Map<string, Uint8Array<ArrayBuffer>>;
7
+
8
+
export const add = async (map: BlockMap, data: unknown): Promise<void> => {
9
+
const encoded = CBOR.encode(data);
10
+
const cid = await CID.create(0x71, encoded);
11
+
12
+
map.set(CID.toString(cid), encoded);
13
+
};
14
+
15
+
export const setMany = (map: BlockMap, entries: Iterable<Readonly<BlockEntry>>) => {
16
+
for (const [cid, bytes] of entries) {
17
+
map.set(cid, bytes);
18
+
}
19
+
};
20
+
21
+
export const deleteMany = (map: BlockMap, cids: Iterable<string>) => {
22
+
for (const cid of cids) {
23
+
map.delete(cid);
24
+
}
25
+
};
+23
packages/utilities/mst/lib/errors.ts
+23
packages/utilities/mst/lib/errors.ts
···
1
+
export class InvalidMstKeyError extends Error {
2
+
constructor(public key: string) {
3
+
super(`invalid mst key; key=${key}`);
4
+
}
5
+
}
6
+
7
+
export class MissingBlockError extends Error {
8
+
constructor(
9
+
public cid: string,
10
+
public def?: string,
11
+
) {
12
+
super(`missing block in store; cid=${cid}` + (def ? `; type=${def}` : ``));
13
+
}
14
+
}
15
+
16
+
export class UnexpectedObjectError extends Error {
17
+
constructor(
18
+
public cid: string,
19
+
public def: string,
20
+
) {
21
+
super(`unexpected object in store; cid=${cid}; expected=${def}`);
22
+
}
23
+
}
+45
packages/utilities/mst/lib/node-store.ts
+45
packages/utilities/mst/lib/node-store.ts
···
1
+
import { MissingBlockError } from './errors.js';
2
+
import { MSTNode } from './node.js';
3
+
import type { BlockStore } from './stores.js';
4
+
5
+
import LRUCache from './utils/lru.js';
6
+
7
+
export class NodeStore {
8
+
store: BlockStore;
9
+
cache = new LRUCache<string | null, MSTNode>(1024);
10
+
11
+
constructor(store: BlockStore) {
12
+
this.store = store;
13
+
}
14
+
15
+
async get(cid: string | null): Promise<MSTNode> {
16
+
let node = this.cache.get(cid);
17
+
if (node === undefined) {
18
+
if (cid === null) {
19
+
node = MSTNode.empty();
20
+
this.cache.put((await node.cid()).$link, node);
21
+
} else {
22
+
const bytes = await this.store.get(cid);
23
+
if (bytes === null) {
24
+
throw new MissingBlockError(cid, 'MST node');
25
+
}
26
+
27
+
node = await MSTNode.deserialize(bytes);
28
+
node._bytes = bytes;
29
+
}
30
+
31
+
this.cache.put(cid, node);
32
+
}
33
+
34
+
return node;
35
+
}
36
+
37
+
async put(node: MSTNode): Promise<MSTNode> {
38
+
const cid = (await node.cid()).$link;
39
+
40
+
this.cache.put(cid, node);
41
+
await this.store.put(cid, await node.serialize());
42
+
43
+
return node;
44
+
}
45
+
}
+119
packages/utilities/mst/lib/node-walker.test.ts
+119
packages/utilities/mst/lib/node-walker.test.ts
···
1
+
import { describe, expect, it } from 'vitest';
2
+
3
+
import * as CID from '@atcute/cid';
4
+
import { encodeUtf8 } from '@atcute/uint8array';
5
+
6
+
import { NodeStore } from './node-store.js';
7
+
import { NodeWalker } from './node-walker.js';
8
+
import { MSTNode } from './node.js';
9
+
import { MemoryBlockStore } from './stores.js';
10
+
11
+
const createCid = async (data: string) => {
12
+
const bytes = encodeUtf8(data);
13
+
return CID.toCidLink(await CID.create(0x55, bytes));
14
+
};
15
+
16
+
describe('NodeWalker', () => {
17
+
it('should iterate over entries in empty tree', async () => {
18
+
const store = new NodeStore(new MemoryBlockStore());
19
+
const emptyNode = MSTNode.empty();
20
+
await store.put(emptyNode);
21
+
const emptyCid = (await emptyNode.cid()).$link;
22
+
23
+
const walker = await NodeWalker.create(store, emptyCid);
24
+
25
+
const pairs: Array<[string, any]> = [];
26
+
for await (const pair of walker.entries()) {
27
+
pairs.push(pair);
28
+
}
29
+
30
+
expect(pairs).toEqual([]);
31
+
});
32
+
33
+
it('should iterate over entries in single-entry tree', async () => {
34
+
const store = new NodeStore(new MemoryBlockStore());
35
+
36
+
// Create a simple node with one entry
37
+
const testKey = 'test/key';
38
+
const testValue = await createCid('test-value');
39
+
const node = await MSTNode.create([testKey], [testValue], [null, null]);
40
+
41
+
await store.put(node);
42
+
const nodeCid = (await node.cid()).$link;
43
+
44
+
const walker = await NodeWalker.create(store, nodeCid);
45
+
46
+
const pairs: Array<[string, any]> = [];
47
+
for await (const pair of walker.entries()) {
48
+
pairs.push(pair);
49
+
}
50
+
51
+
expect(pairs.length).toBe(1);
52
+
expect(pairs[0][0]).toBe(testKey);
53
+
expect(pairs[0][1]).toEqual(testValue);
54
+
});
55
+
56
+
it('should check done correctly', async () => {
57
+
const store = new NodeStore(new MemoryBlockStore());
58
+
const emptyNode = MSTNode.empty();
59
+
await store.put(emptyNode);
60
+
const emptyCid = (await emptyNode.cid()).$link;
61
+
62
+
const walker = await NodeWalker.create(store, emptyCid);
63
+
64
+
expect(walker.done).toBe(true);
65
+
});
66
+
67
+
it('should get height correctly', async () => {
68
+
const store = new NodeStore(new MemoryBlockStore());
69
+
const emptyNode = MSTNode.empty();
70
+
await store.put(emptyNode);
71
+
const emptyCid = (await emptyNode.cid()).$link;
72
+
73
+
const walker = await NodeWalker.create(store, emptyCid);
74
+
75
+
expect(walker.height).toBe(0);
76
+
});
77
+
78
+
it('should iterate over all nodes', async () => {
79
+
const store = new NodeStore(new MemoryBlockStore());
80
+
81
+
const testKey = 'test/key';
82
+
const testValue = await createCid('test-value');
83
+
const node = await MSTNode.create([testKey], [testValue], [null, null]);
84
+
85
+
await store.put(node);
86
+
const nodeCid = (await node.cid()).$link;
87
+
88
+
const walker = await NodeWalker.create(store, nodeCid);
89
+
90
+
const nodes: MSTNode[] = [];
91
+
for await (const n of walker.nodes()) {
92
+
nodes.push(n);
93
+
}
94
+
95
+
expect(nodes.length).toBe(1);
96
+
expect(nodes[0]).toBe(walker.frame.node);
97
+
});
98
+
99
+
it('should get nodeCids', async () => {
100
+
const store = new NodeStore(new MemoryBlockStore());
101
+
102
+
const testKey = 'test/key';
103
+
const testValue = await createCid('test-value');
104
+
const node = await MSTNode.create([testKey], [testValue], [null, null]);
105
+
106
+
await store.put(node);
107
+
const nodeCid = (await node.cid()).$link;
108
+
109
+
const walker = await NodeWalker.create(store, nodeCid);
110
+
111
+
const cids: any[] = [];
112
+
for await (const cid of walker.nodeCids()) {
113
+
cids.push(cid);
114
+
}
115
+
116
+
expect(cids.length).toBe(1);
117
+
expect(cids[0].$link).toBe(nodeCid);
118
+
});
119
+
});
+329
packages/utilities/mst/lib/node-walker.ts
+329
packages/utilities/mst/lib/node-walker.ts
···
1
+
import type { CidLink } from '@atcute/cid';
2
+
3
+
import { MSTNode, getKeyHeight } from './node.js';
4
+
import { NodeStore } from './node-store.js';
5
+
import Stack from './utils/stack.js';
6
+
7
+
interface StackFrame {
8
+
node: MSTNode;
9
+
lpath: string;
10
+
rpath: string;
11
+
idx: number;
12
+
}
13
+
14
+
/**
15
+
* NodeWalker makes implementing tree diffing and other MST query ops more
16
+
* convenient (but it does not, itself, implement them).
17
+
*
18
+
* A NodeWalker starts off at the root of a tree, and can walk along or recurse
19
+
* down into subtrees.
20
+
*
21
+
* Walking "off the end" of a subtree brings you back up to its next non-empty parent.
22
+
*
23
+
* Recall MSTNode layout:
24
+
*
25
+
* ```
26
+
* keys: (lpath) (0, 1, 2, 3) (rpath)
27
+
* vals: (0, 1, 2, 3)
28
+
* subtrees: (0, 1, 2, 3, 4)
29
+
* ```
30
+
*/
31
+
export class NodeWalker {
32
+
static readonly PATH_MIN = ''; // string that compares less than all legal path strings
33
+
static readonly PATH_MAX = '\xff'; // string that compares greater than all legal path strings
34
+
35
+
private store: NodeStore;
36
+
private stack: Stack<StackFrame>;
37
+
private rootHeight: number;
38
+
private trusted: boolean;
39
+
40
+
private constructor(store: NodeStore, stack: Stack<StackFrame>, rootHeight: number, trusted: boolean) {
41
+
this.store = store;
42
+
this.stack = stack;
43
+
this.rootHeight = rootHeight;
44
+
this.trusted = trusted;
45
+
}
46
+
47
+
/**
48
+
* create a new NodeWalker
49
+
* @param store NodeStore to fetch nodes from
50
+
* @param rootCid CID of the root node to start walking from
51
+
* @param lpath left boundary path (defaults to minimum)
52
+
* @param rpath right boundary path (defaults to maximum)
53
+
* @param trusted skip height validation checks if true (faster but unsafe for untrusted trees)
54
+
* @param rootHeight pre-computed root height (optional optimization)
55
+
* @returns a new NodeWalker instance
56
+
*/
57
+
static async create(
58
+
store: NodeStore,
59
+
rootCid: string | null,
60
+
lpath: string = NodeWalker.PATH_MIN,
61
+
rpath: string = NodeWalker.PATH_MAX,
62
+
trusted: boolean = false,
63
+
rootHeight?: number,
64
+
): Promise<NodeWalker> {
65
+
const node = await store.get(rootCid);
66
+
const height = rootHeight ?? (await node.height());
67
+
68
+
if (height === null) {
69
+
throw new Error(`indeterminate node height; provide rootHeight if known`);
70
+
}
71
+
72
+
const stack = new Stack<StackFrame>();
73
+
stack.push({
74
+
node,
75
+
lpath,
76
+
rpath,
77
+
idx: 0,
78
+
});
79
+
80
+
return new NodeWalker(store, stack, height, trusted);
81
+
}
82
+
83
+
/**
84
+
* create a new walker rooted at the current position's subtree.
85
+
* treats the subtree as an independent tree for traversal.
86
+
* @returns a new NodeWalker instance for the subtree
87
+
*/
88
+
async createSubtreeWalker(): Promise<NodeWalker> {
89
+
return await NodeWalker.create(
90
+
this.store,
91
+
this.subtree?.$link ?? null,
92
+
this.lpath,
93
+
this.rpath,
94
+
this.trusted,
95
+
this.height - 1,
96
+
);
97
+
}
98
+
99
+
/** current stack frame (internal) */
100
+
get frame(): StackFrame {
101
+
const frame = this.stack.peek();
102
+
if (frame === undefined) {
103
+
throw new Error(`stack is empty`);
104
+
}
105
+
106
+
return frame;
107
+
}
108
+
109
+
/** current height in the tree (decreases as you descend) */
110
+
get height(): number {
111
+
return this.rootHeight - (this.stack.size - 1);
112
+
}
113
+
114
+
/** key/path to the left of current cursor position */
115
+
get lpath(): string {
116
+
return this.frame.idx === 0 ? this.frame.lpath : this.frame.node.keys[this.frame.idx - 1];
117
+
}
118
+
119
+
/** value (CID) to the left of current cursor position */
120
+
get lval(): CidLink | null {
121
+
return this.frame.idx === 0 ? null : this.frame.node.values[this.frame.idx - 1];
122
+
}
123
+
124
+
/** subtree CID at current cursor position (null if no subtree) */
125
+
get subtree(): CidLink | null {
126
+
const subtree = this.frame.node.subtrees[this.frame.idx];
127
+
return subtree ?? null;
128
+
}
129
+
130
+
/** key/path to the right of current cursor position */
131
+
get rpath(): string {
132
+
return this.frame.idx === this.frame.node.keys.length
133
+
? this.frame.rpath
134
+
: this.frame.node.keys[this.frame.idx];
135
+
}
136
+
137
+
/** value (CID) to the right of current cursor position */
138
+
get rval(): CidLink | null {
139
+
return this.frame.idx === this.frame.node.values.length ? null : this.frame.node.values[this.frame.idx];
140
+
}
141
+
142
+
/** whether the walker has reached the end of the tree */
143
+
get done(): boolean {
144
+
// is (not this.stack) really necessary here? is that a reachable state?
145
+
const bottom = this.stack.peekBottom();
146
+
return (
147
+
this.stack.size === 0 || (this.subtree === null && bottom !== undefined && this.rpath === bottom.rpath)
148
+
);
149
+
}
150
+
151
+
/** whether the cursor can move right in current node */
152
+
get canGoRight(): boolean {
153
+
return this.frame.idx + 1 < this.frame.node.subtrees.length;
154
+
}
155
+
156
+
/**
157
+
* move cursor right, or up if at end of current node.
158
+
* automatically recurses up through empty intermediates.
159
+
* @throws if attempting to navigate beyond root (check done first)
160
+
*/
161
+
rightOrUp(): void {
162
+
if (!this.canGoRight) {
163
+
// we reached the end of this node, go up a level
164
+
this.stack.pop();
165
+
if (this.stack.size === 0) {
166
+
throw new Error(`cannot navigate beyond root; check .done before calling`);
167
+
}
168
+
return this.rightOrUp(); // we need to recurse, to skip over empty intermediates on the way back up
169
+
}
170
+
this.frame.idx += 1;
171
+
}
172
+
173
+
/**
174
+
* move cursor right within current node.
175
+
* @throws if already at rightmost position (check canGoRight first)
176
+
*/
177
+
right(): void {
178
+
if (!this.canGoRight) {
179
+
throw new Error(`cursor is already at rightmost position in node`);
180
+
}
181
+
this.frame.idx += 1;
182
+
}
183
+
184
+
/**
185
+
* descend into the subtree at current cursor position.
186
+
* @throws if no subtree exists at current position
187
+
*/
188
+
async down(): Promise<void> {
189
+
const subtree = this.frame.node.subtrees[this.frame.idx];
190
+
if (subtree === null) {
191
+
throw new Error(`cannot descend; no subtree at current position`);
192
+
}
193
+
194
+
const subtreeNode = await this.store.get(subtree.$link);
195
+
196
+
if (!this.trusted) {
197
+
// if we "trust" the source we can elide this check
198
+
// the "null" case occurs for empty intermediate nodes
199
+
const subtreeHeight = await subtreeNode.height();
200
+
if (subtreeHeight !== null && subtreeHeight !== this.height - 1) {
201
+
throw new Error(`inconsistent subtree height; got=${subtreeHeight} expected=${this.height - 1}`);
202
+
}
203
+
}
204
+
205
+
this.stack.push({
206
+
node: subtreeNode,
207
+
lpath: this.lpath,
208
+
rpath: this.rpath,
209
+
idx: 0,
210
+
});
211
+
}
212
+
213
+
/**
214
+
* advance to and return the next key-value pair in the tree.
215
+
* descends into all subtrees automatically.
216
+
* @returns Tuple of [key, value CID]
217
+
*/
218
+
async nextEntry(): Promise<[string, CidLink]> {
219
+
while (this.subtree) {
220
+
// recurse down every subtree
221
+
await this.down();
222
+
}
223
+
this.rightOrUp();
224
+
return [this.lpath, this.lval!]; // the kv pair we just jumped over
225
+
}
226
+
227
+
/**
228
+
* iterate over all key-value pairs in the tree in sorted order.
229
+
* @yields Tuples of [key, value CID]
230
+
*/
231
+
async *entries(): AsyncIterableIterator<[string, CidLink]> {
232
+
while (!this.done) {
233
+
yield await this.nextEntry();
234
+
}
235
+
}
236
+
237
+
/**
238
+
* iterate over all MST nodes from current position to the end of tree.
239
+
* @yields MSTNode instances
240
+
*/
241
+
async *nodes(): AsyncIterableIterator<MSTNode> {
242
+
yield this.frame.node;
243
+
244
+
while (!this.done) {
245
+
while (this.subtree) {
246
+
// recurse down every subtree
247
+
await this.down();
248
+
yield this.frame.node;
249
+
}
250
+
251
+
this.rightOrUp();
252
+
}
253
+
}
254
+
255
+
/**
256
+
* iterate over CIDs of all MST nodes from current position to end of tree.
257
+
* @yields CID links to nodes
258
+
*/
259
+
async *nodeCids(): AsyncIterableIterator<CidLink> {
260
+
for await (const node of this.nodes()) {
261
+
yield await node.cid();
262
+
}
263
+
}
264
+
265
+
/**
266
+
* iterate over key-value pairs within a specific key range.
267
+
* @param start start key (inclusive)
268
+
* @param end end key
269
+
* @param endInclusive whether end key is inclusive
270
+
* @yields tuples of [key, value CID] within range
271
+
*/
272
+
async *entriesInRange(
273
+
start: string,
274
+
end: string,
275
+
endInclusive: boolean = false,
276
+
): AsyncIterableIterator<[string, CidLink]> {
277
+
while (true) {
278
+
while (this.rpath < start) {
279
+
this.rightOrUp();
280
+
}
281
+
if (!this.subtree) {
282
+
break;
283
+
}
284
+
await this.down();
285
+
}
286
+
287
+
for await (const [k, v] of this.entries()) {
288
+
if (k > end || (!endInclusive && k === end)) {
289
+
break;
290
+
}
291
+
yield [k, v];
292
+
}
293
+
}
294
+
295
+
/**
296
+
* search for a specific key (rpath) in the tree.
297
+
* @param rpath key to search for
298
+
* @returns value CID if found, null otherwise
299
+
*/
300
+
async findRpath(rpath: string): Promise<CidLink | null> {
301
+
const rpathHeight = await getKeyHeight(rpath);
302
+
while (true) {
303
+
// if the rpath we're looking for is higher than the current cursor,
304
+
// we're never going to find it (i.e. we early-exit)
305
+
if (rpathHeight > this.height) {
306
+
return null;
307
+
}
308
+
309
+
while (this.rpath < rpath) {
310
+
// either look for the rpath, or the right point to go down
311
+
if (!this.canGoRight) {
312
+
return null;
313
+
}
314
+
315
+
this.right();
316
+
}
317
+
318
+
if (this.rpath === rpath) {
319
+
return this.rval; // found it!
320
+
}
321
+
322
+
if (!this.subtree) {
323
+
return null; // need to go down, but we can't
324
+
}
325
+
326
+
await this.down();
327
+
}
328
+
}
329
+
}
+216
packages/utilities/mst/lib/node.ts
+216
packages/utilities/mst/lib/node.ts
···
1
+
import * as CBOR from '@atcute/cbor';
2
+
import type { CidLink } from '@atcute/cid';
3
+
import * as CID from '@atcute/cid';
4
+
import { decodeUtf8From, encodeUtf8, toSha256 } from '@atcute/uint8array';
5
+
6
+
import { isNodeData, type NodeData, type TreeEntry } from './types.js';
7
+
8
+
export class MSTNode {
9
+
/** @internal */
10
+
_height: number | null | undefined;
11
+
/** @internal */
12
+
_cid: CidLink | undefined;
13
+
/** @internal */
14
+
_bytes: Uint8Array<ArrayBuffer> | undefined;
15
+
16
+
protected constructor(
17
+
readonly keys: readonly string[],
18
+
readonly values: readonly CidLink[],
19
+
readonly subtrees: readonly (CidLink | null)[],
20
+
) {}
21
+
22
+
static async create(
23
+
keys: readonly string[],
24
+
values: readonly CidLink[],
25
+
subtrees: readonly (CidLink | null)[],
26
+
): Promise<MSTNode> {
27
+
if (subtrees.length !== keys.length + 1) {
28
+
throw new TypeError(`malformed MST node; invalid subtree count`);
29
+
}
30
+
31
+
if (keys.length !== values.length) {
32
+
throw new TypeError(`malformed MST node; mismatched keys/values lengths`);
33
+
}
34
+
35
+
let expectedHeight: number | undefined;
36
+
for (const key of keys) {
37
+
const height = await getKeyHeight(key);
38
+
expectedHeight ??= height;
39
+
40
+
if (height !== expectedHeight) {
41
+
throw new TypeError(`malformed MST node; inconsistent key heights`);
42
+
}
43
+
}
44
+
45
+
return new MSTNode(keys, values, subtrees);
46
+
}
47
+
48
+
static empty(): MSTNode {
49
+
return new MSTNode([], [], [null]);
50
+
}
51
+
52
+
static async deserialize(bytes: Uint8Array): Promise<MSTNode> {
53
+
const node = CBOR.decode(bytes);
54
+
if (!isNodeData(node)) {
55
+
throw new TypeError(`malformed MST node; invalid structure`);
56
+
}
57
+
58
+
const keys: string[] = [];
59
+
const values: CidLink[] = [];
60
+
const subtrees: (CidLink | null)[] = [node.l];
61
+
62
+
let prevKey = '';
63
+
64
+
for (const entry of node.e) {
65
+
const prefixLen = entry.p;
66
+
if (prefixLen > prevKey.length) {
67
+
throw new TypeError(`malformed MST node; unexpected key prefix length`);
68
+
}
69
+
70
+
const suffix = decodeUtf8From(CBOR.fromBytes(entry.k));
71
+
if (prevKey[prefixLen] === suffix[0]) {
72
+
throw new TypeError(`malformed MST node; suboptimal key prefix length`);
73
+
}
74
+
75
+
const key = prevKey.slice(0, prefixLen) + suffix;
76
+
if (key <= prevKey) {
77
+
throw new TypeError(`malformed MST node; invalid key sort order`);
78
+
}
79
+
80
+
keys.push(key);
81
+
values.push(entry.v);
82
+
subtrees.push(entry.t);
83
+
84
+
prevKey = key;
85
+
}
86
+
87
+
return await MSTNode.create(keys, values, subtrees);
88
+
}
89
+
90
+
async serialize(): Promise<Uint8Array<ArrayBuffer>> {
91
+
let bytes = this._bytes;
92
+
if (bytes === undefined) {
93
+
const keys = this.keys;
94
+
const values = this.values;
95
+
const subtrees = this.subtrees;
96
+
97
+
const e: TreeEntry[] = [];
98
+
99
+
let prevKey = '';
100
+
101
+
for (let idx = 0, len = keys.length; idx < len; idx++) {
102
+
const key = keys[idx];
103
+
const prefixLen = commonPrefixLength(prevKey, key);
104
+
const suffix = key.slice(prefixLen);
105
+
106
+
e.push({
107
+
k: CBOR.toBytes(encodeUtf8(suffix)),
108
+
p: prefixLen,
109
+
t: subtrees[idx + 1],
110
+
v: values[idx],
111
+
});
112
+
}
113
+
114
+
const n: NodeData = {
115
+
l: subtrees[0],
116
+
e: e,
117
+
};
118
+
119
+
this._bytes = bytes = CBOR.encode(n);
120
+
}
121
+
122
+
return bytes;
123
+
}
124
+
125
+
async cid(): Promise<CidLink> {
126
+
let cid = this._cid;
127
+
if (cid === undefined) {
128
+
this._cid = cid = CID.toCidLink(await CID.create(0x71, await this.serialize()));
129
+
}
130
+
131
+
return cid;
132
+
}
133
+
134
+
isEmpty(): boolean {
135
+
return this.subtrees.length === 1 && this.subtrees[0] === null;
136
+
}
137
+
138
+
async height(): Promise<number | null> {
139
+
let height = this._height;
140
+
if (height === undefined) {
141
+
const keys = this.keys;
142
+
143
+
if (this.isEmpty()) {
144
+
height = 0;
145
+
} else if (keys.length > 0) {
146
+
height = await getKeyHeight(keys[0]);
147
+
} else {
148
+
height = null;
149
+
}
150
+
151
+
this._height = height;
152
+
}
153
+
154
+
return height;
155
+
}
156
+
157
+
async requireHeight(): Promise<number> {
158
+
const height = await this.height();
159
+
if (height === null) {
160
+
throw new Error(`indeterminate node height`);
161
+
}
162
+
163
+
return height;
164
+
}
165
+
166
+
lowerBound(key: string): number {
167
+
const keys = this.keys;
168
+
const len = keys.length;
169
+
170
+
for (let idx = 0; idx < len; idx++) {
171
+
if (key <= keys[idx]) {
172
+
return idx;
173
+
}
174
+
}
175
+
176
+
return len;
177
+
}
178
+
}
179
+
180
+
export const getKeyHeight = async (key: string): Promise<number> => {
181
+
const hash = await toSha256(encodeUtf8(key));
182
+
183
+
let lz = 0;
184
+
for (let idx = 0, len = hash.length; idx < len; idx++) {
185
+
const byte = hash[idx];
186
+
187
+
if (byte < 64) {
188
+
lz++;
189
+
}
190
+
if (byte < 16) {
191
+
lz++;
192
+
}
193
+
if (byte < 4) {
194
+
lz++;
195
+
}
196
+
197
+
if (byte === 0) {
198
+
lz++;
199
+
} else {
200
+
break;
201
+
}
202
+
}
203
+
204
+
return lz;
205
+
};
206
+
207
+
const commonPrefixLength = (a: string, b: string): number => {
208
+
let idx = 0;
209
+
for (let len = Math.min(a.length, b.length); idx < len; idx++) {
210
+
if (a[idx] !== b[idx]) {
211
+
break;
212
+
}
213
+
}
214
+
215
+
return idx;
216
+
};
+161
packages/utilities/mst/lib/stores.ts
+161
packages/utilities/mst/lib/stores.ts
···
1
+
import * as CBOR from '@atcute/cbor';
2
+
3
+
import { deleteMany, setMany, type BlockMap } from './blockmap.js';
4
+
import { MissingBlockError, UnexpectedObjectError } from './errors.js';
5
+
6
+
export interface ReadonlyBlockStore {
7
+
get(cid: string): Promise<Uint8Array<ArrayBuffer> | null>;
8
+
getMany(cids: string[]): Promise<{ found: BlockMap; missing: string[] }>;
9
+
10
+
has(cid: string): Promise<boolean>;
11
+
}
12
+
13
+
export interface BlockStore extends ReadonlyBlockStore {
14
+
put(cid: string, bytes: Uint8Array<ArrayBuffer>): Promise<void>;
15
+
putMany(blocks: BlockMap): Promise<void>;
16
+
17
+
delete(cid: string): Promise<void>;
18
+
deleteMany(cids: string[]): Promise<void>;
19
+
}
20
+
21
+
export class ReadonlyMemoryBlockStore implements ReadonlyBlockStore {
22
+
blocks: BlockMap = new Map();
23
+
24
+
constructor(blocks?: BlockMap) {
25
+
if (blocks !== undefined) {
26
+
setMany(this.blocks, blocks);
27
+
}
28
+
}
29
+
30
+
get(cid: string): Promise<Uint8Array<ArrayBuffer> | null> {
31
+
return Promise.resolve(this.blocks.get(cid) ?? null);
32
+
}
33
+
34
+
getMany(cids: string[]): Promise<{ found: BlockMap; missing: string[] }> {
35
+
const found: BlockMap = new Map();
36
+
const missing: string[] = [];
37
+
38
+
for (const cid of cids) {
39
+
const bytes = this.blocks.get(cid);
40
+
if (bytes !== undefined) {
41
+
found.set(cid, bytes);
42
+
} else {
43
+
missing.push(cid);
44
+
}
45
+
}
46
+
47
+
return Promise.resolve({ found, missing });
48
+
}
49
+
50
+
has(cid: string): Promise<boolean> {
51
+
return Promise.resolve(this.blocks.has(cid));
52
+
}
53
+
}
54
+
55
+
export class MemoryBlockStore extends ReadonlyMemoryBlockStore implements BlockStore {
56
+
put(cid: string, bytes: Uint8Array<ArrayBuffer>): Promise<void> {
57
+
this.blocks.set(cid, bytes);
58
+
return Promise.resolve();
59
+
}
60
+
61
+
putMany(blocks: BlockMap): Promise<void> {
62
+
setMany(this.blocks, blocks);
63
+
return Promise.resolve();
64
+
}
65
+
66
+
delete(cid: string): Promise<void> {
67
+
this.blocks.delete(cid);
68
+
return Promise.resolve();
69
+
}
70
+
71
+
deleteMany(cids: string[]): Promise<void> {
72
+
deleteMany(this.blocks, cids);
73
+
return Promise.resolve();
74
+
}
75
+
}
76
+
77
+
export class OverlayBlockStore implements BlockStore {
78
+
upper: BlockStore;
79
+
lower: ReadonlyBlockStore;
80
+
81
+
constructor(upper: BlockStore, lower: ReadonlyBlockStore) {
82
+
this.upper = upper;
83
+
this.lower = lower;
84
+
}
85
+
86
+
async get(cid: string): Promise<Uint8Array<ArrayBuffer> | null> {
87
+
let bytes = await this.upper.get(cid);
88
+
if (bytes === null) {
89
+
bytes = await this.lower.get(cid);
90
+
}
91
+
92
+
return bytes;
93
+
}
94
+
95
+
async getMany(cids: string[]): Promise<{ found: BlockMap; missing: string[] }> {
96
+
const upper = await this.upper.getMany(cids);
97
+
const lower = await this.lower.getMany(upper.missing);
98
+
99
+
const found = upper.found;
100
+
const missing = lower.missing;
101
+
102
+
setMany(found, lower.found);
103
+
104
+
return { found, missing };
105
+
}
106
+
107
+
async has(cid: string): Promise<boolean> {
108
+
let exists = await this.upper.has(cid);
109
+
if (!exists) {
110
+
exists = await this.lower.has(cid);
111
+
}
112
+
113
+
return exists;
114
+
}
115
+
116
+
async put(cid: string, bytes: Uint8Array<ArrayBuffer>): Promise<void> {
117
+
await this.upper.put(cid, bytes);
118
+
}
119
+
120
+
async putMany(blocks: BlockMap): Promise<void> {
121
+
await this.upper.putMany(blocks);
122
+
}
123
+
124
+
async delete(cid: string): Promise<void> {
125
+
await this.upper.delete(cid);
126
+
}
127
+
128
+
async deleteMany(cids: string[]): Promise<void> {
129
+
return await this.upper.deleteMany(cids);
130
+
}
131
+
}
132
+
133
+
export const readObject = async <T>(store: ReadonlyBlockStore, cid: string, def: CheckDef<T>): Promise<T> => {
134
+
const bytes = await store.get(cid);
135
+
if (bytes === null) {
136
+
throw new MissingBlockError(cid, def.name);
137
+
}
138
+
139
+
const decoded = CBOR.decode(bytes);
140
+
if (!def.check(decoded)) {
141
+
throw new UnexpectedObjectError(cid, def.name);
142
+
}
143
+
144
+
return decoded;
145
+
};
146
+
147
+
export const readRecord = async (store: ReadonlyBlockStore, cid: string): Promise<unknown> => {
148
+
const bytes = await store.get(cid);
149
+
if (bytes === null) {
150
+
throw new MissingBlockError(cid, undefined);
151
+
}
152
+
153
+
const decoded = CBOR.decode(bytes);
154
+
155
+
return decoded;
156
+
};
157
+
158
+
interface CheckDef<T> {
159
+
name: string;
160
+
check: (value: unknown) => value is T;
161
+
}
+42
packages/utilities/mst/lib/types.ts
+42
packages/utilities/mst/lib/types.ts
···
1
+
import { isBytes, type Bytes } from '@atcute/cbor';
2
+
import { isCidLink, type CidLink } from '@atcute/cid';
3
+
4
+
export interface TreeEntry {
5
+
/** count of bytes shared with previous TreeEntry in this Node (if any) */
6
+
p: number;
7
+
/** remainder of key for this TreeEntry, after "prefixlen" have been removed */
8
+
k: Bytes;
9
+
/** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */
10
+
v: CidLink;
11
+
/** next subtree (to the right of leaf) */
12
+
t: CidLink | null;
13
+
}
14
+
15
+
export const isTreeEntry = (value: unknown): value is TreeEntry => {
16
+
if (value === null || typeof value !== 'object') {
17
+
return false;
18
+
}
19
+
20
+
const obj = value as Record<string, unknown>;
21
+
22
+
return (
23
+
typeof obj.p === 'number' && isBytes(obj.k) && isCidLink(obj.v) && (obj.t === null || isCidLink(obj.t))
24
+
);
25
+
};
26
+
27
+
export interface NodeData {
28
+
/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
29
+
l: CidLink | null;
30
+
/** ordered list of TreeEntry objects */
31
+
e: TreeEntry[];
32
+
}
33
+
34
+
export const isNodeData = (value: unknown): value is NodeData => {
35
+
if (value === null || typeof value !== 'object') {
36
+
return false;
37
+
}
38
+
39
+
const obj = value as Record<string, unknown>;
40
+
41
+
return (obj.l === null || isCidLink(obj.l)) && Array.isArray(obj.e) && obj.e.every(isTreeEntry);
42
+
};
+232
packages/utilities/mst/lib/utils/lru.ts
+232
packages/utilities/mst/lib/utils/lru.ts
···
1
+
interface LRUNode<K, V> {
2
+
key: K;
3
+
value: V;
4
+
prev: LRUNode<K, V> | null;
5
+
next: LRUNode<K, V> | null;
6
+
}
7
+
8
+
/**
9
+
* a least recently used (LRU) cache with fixed capacity
10
+
* evicts the least recently used items when capacity is exceeded
11
+
*/
12
+
class LRUCache<K, V> {
13
+
readonly #size: number;
14
+
#count = 0;
15
+
16
+
#map = new Map<K, LRUNode<K, V>>();
17
+
#head: LRUNode<K, V> | null = null;
18
+
#tail: LRUNode<K, V> | null = null;
19
+
20
+
/**
21
+
* creates a new LRU cache with the specified capacity
22
+
* @param size the maximum number of items the cache can hold
23
+
*/
24
+
constructor(size: number) {
25
+
this.#size = size;
26
+
}
27
+
28
+
/** the maximum capacity of the cache */
29
+
get size(): number {
30
+
return this.#size;
31
+
}
32
+
33
+
/**
34
+
* gets a value without affecting its position in the cache
35
+
* @param key the key to look up
36
+
* @returns the value associated with the key, or undefined if not found
37
+
*/
38
+
peek(key: K): V | undefined {
39
+
const node = this.#map.get(key);
40
+
if (node === undefined) {
41
+
return undefined;
42
+
}
43
+
44
+
return node.value;
45
+
}
46
+
47
+
/**
48
+
* gets a value and marks it as most recently used
49
+
* @param key the key to look up
50
+
* @returns the value associated with the key, or undefined if not found
51
+
*/
52
+
get(key: K): V | undefined {
53
+
const node = this.#map.get(key);
54
+
if (node === undefined) {
55
+
return undefined;
56
+
}
57
+
58
+
this.#moveToFront(node);
59
+
return node.value;
60
+
}
61
+
62
+
/**
63
+
* stores a value for the given key, marking it as most recently used
64
+
* evicts the least recently used item if the cache is at capacity
65
+
* @param key the key to store
66
+
* @param value the value to associate with the key
67
+
*/
68
+
put(key: K, value: V): void {
69
+
{
70
+
const existing = this.#map.get(key);
71
+
72
+
if (existing !== undefined) {
73
+
existing.value = value;
74
+
this.#moveToFront(existing);
75
+
return;
76
+
}
77
+
}
78
+
79
+
{
80
+
const node: LRUNode<K, V> = { key, value, prev: null, next: null };
81
+
this.#map.set(key, node);
82
+
this.#addToFront(node);
83
+
84
+
this.#count++;
85
+
}
86
+
87
+
this.#evict();
88
+
}
89
+
90
+
/**
91
+
* removes a key from the cache
92
+
* @param key the key to remove
93
+
* @returns true if the key was found and removed, false otherwise
94
+
*/
95
+
delete(key: K): boolean {
96
+
const node = this.#map.get(key);
97
+
if (node === undefined) {
98
+
return false;
99
+
}
100
+
101
+
this.#map.delete(key);
102
+
this.#removeNode(node);
103
+
this.#count--;
104
+
return true;
105
+
}
106
+
107
+
/**
108
+
* removes all items from the cache
109
+
*/
110
+
clear(): void {
111
+
this.#map.clear();
112
+
this.#head = null;
113
+
this.#tail = null;
114
+
this.#count = 0;
115
+
}
116
+
117
+
/**
118
+
* checks if a key exists in the cache
119
+
* @param key the key to check
120
+
* @returns true if the key exists, false otherwise
121
+
*/
122
+
has(key: K): boolean {
123
+
return this.#map.has(key);
124
+
}
125
+
126
+
/**
127
+
* iterates over the keys in LRU order (most to least recently used)
128
+
* @returns iterator of keys
129
+
*/
130
+
*keys(): IterableIterator<K> {
131
+
let current = this.#head;
132
+
while (current !== null) {
133
+
yield current.key;
134
+
current = current.next;
135
+
}
136
+
}
137
+
138
+
/**
139
+
* iterates over the values in LRU order (most to least recently used)
140
+
* @returns iterator of values
141
+
*/
142
+
*values(): IterableIterator<V> {
143
+
let current = this.#head;
144
+
while (current !== null) {
145
+
yield current.value;
146
+
current = current.next;
147
+
}
148
+
}
149
+
150
+
/**
151
+
* iterates over the key-value pairs in LRU order (most to least recently used)
152
+
* @returns iterator of [key, value] tuples
153
+
*/
154
+
*entries(): IterableIterator<[K, V]> {
155
+
let current = this.#head;
156
+
while (current !== null) {
157
+
yield [current.key, current.value];
158
+
current = current.next;
159
+
}
160
+
}
161
+
162
+
#moveToFront(node: LRUNode<K, V>): void {
163
+
if (this.#head === node) {
164
+
return;
165
+
}
166
+
167
+
if (node.prev !== null) {
168
+
node.prev.next = node.next;
169
+
}
170
+
171
+
if (node.next !== null) {
172
+
node.next.prev = node.prev;
173
+
} else {
174
+
this.#tail = node.prev;
175
+
}
176
+
177
+
node.prev = null;
178
+
node.next = this.#head;
179
+
180
+
// Safe because this method is only called when head exists
181
+
this.#head!.prev = node;
182
+
this.#head = node;
183
+
}
184
+
185
+
#addToFront(node: LRUNode<K, V>): void {
186
+
node.next = this.#head;
187
+
node.prev = null;
188
+
189
+
if (this.#head !== null) {
190
+
this.#head.prev = node;
191
+
} else {
192
+
this.#tail = node;
193
+
}
194
+
195
+
this.#head = node;
196
+
}
197
+
198
+
#removeNode(node: LRUNode<K, V>): void {
199
+
if (node.prev !== null) {
200
+
node.prev.next = node.next;
201
+
} else {
202
+
this.#head = node.next;
203
+
}
204
+
205
+
if (node.next !== null) {
206
+
node.next.prev = node.prev;
207
+
} else {
208
+
this.#tail = node.prev;
209
+
}
210
+
}
211
+
212
+
#evict(): void {
213
+
const excess = this.#count - this.#size;
214
+
if (excess <= 0) {
215
+
return;
216
+
}
217
+
218
+
let current: LRUNode<K, V> = this.#tail!;
219
+
220
+
for (let i = 0; i < excess; i++) {
221
+
this.#map.delete(current.key);
222
+
current = current.prev!;
223
+
}
224
+
225
+
current.next = null;
226
+
this.#tail = current;
227
+
228
+
this.#count -= excess;
229
+
}
230
+
}
231
+
232
+
export default LRUCache;
+129
packages/utilities/mst/lib/utils/stack.ts
+129
packages/utilities/mst/lib/utils/stack.ts
···
1
+
interface Node<T> {
2
+
value: T;
3
+
next: Node<T> | undefined;
4
+
}
5
+
6
+
/** a stack data structure (lifo) */
7
+
class Stack<T> implements Iterable<T> {
8
+
#head: Node<T> | undefined;
9
+
#tail: Node<T> | undefined;
10
+
#size: number = 0;
11
+
12
+
/** size of the stack */
13
+
get size(): number {
14
+
return this.#size;
15
+
}
16
+
17
+
/**
18
+
* clear the stack
19
+
*/
20
+
clear(): void {
21
+
this.#head = undefined;
22
+
this.#tail = undefined;
23
+
this.#size = 0;
24
+
}
25
+
26
+
/**
27
+
* adds a value to the top of the stack
28
+
* @param value value to add
29
+
* @returns the stack instance
30
+
*/
31
+
push(value: T): this {
32
+
const node: Node<T> = { value, next: this.#head };
33
+
34
+
if (this.#head === undefined) {
35
+
this.#tail = node;
36
+
}
37
+
38
+
this.#head = node;
39
+
this.#size++;
40
+
return this;
41
+
}
42
+
43
+
/**
44
+
* removes the top value from the stack
45
+
* @returns last added value, or undefined if empty
46
+
*/
47
+
pop(): T | undefined {
48
+
const head = this.#head;
49
+
if (head === undefined) {
50
+
return;
51
+
}
52
+
53
+
this.#head = head.next;
54
+
this.#size--;
55
+
56
+
if (this.#head === undefined) {
57
+
this.#tail = undefined;
58
+
}
59
+
60
+
return head.value;
61
+
}
62
+
63
+
/**
64
+
* get the top value without removing from stack
65
+
* @returns last added value, or undefined if empty
66
+
*/
67
+
peek(): T | undefined {
68
+
return this.#head?.value;
69
+
}
70
+
71
+
/**
72
+
* get the bottom value without removing from stack
73
+
* @returns first added value, or undefined if empty
74
+
*/
75
+
peekBottom(): T | undefined {
76
+
return this.#tail?.value;
77
+
}
78
+
79
+
/**
80
+
* returns an iterator that drains all the values from stack
81
+
*/
82
+
drain(): IterableIterator<T, undefined, undefined> {
83
+
// deno-lint-ignore no-this-alias
84
+
const self = this;
85
+
86
+
return {
87
+
next() {
88
+
const head = self.#head;
89
+
if (head === undefined) {
90
+
return { done: true, value: undefined };
91
+
}
92
+
93
+
self.#head = head.next;
94
+
self.#size--;
95
+
96
+
if (self.#head === undefined) {
97
+
self.#tail = undefined;
98
+
}
99
+
100
+
return { done: false, value: head.value };
101
+
},
102
+
[Symbol.iterator]() {
103
+
return this;
104
+
},
105
+
};
106
+
}
107
+
108
+
/**
109
+
* iterates over the stack without draining
110
+
*/
111
+
[Symbol.iterator](): Iterator<T, undefined, undefined> {
112
+
let current = this.#head;
113
+
114
+
return {
115
+
next() {
116
+
if (current === undefined) {
117
+
return { done: true, value: undefined };
118
+
}
119
+
120
+
const value = current.value;
121
+
current = current.next;
122
+
123
+
return { done: false, value: value };
124
+
},
125
+
};
126
+
}
127
+
}
128
+
129
+
export default Stack;
+44
packages/utilities/mst/package.json
+44
packages/utilities/mst/package.json
···
1
+
{
2
+
"type": "module",
3
+
"name": "@atcute/mst",
4
+
"version": "0.1.0",
5
+
"description": "atproto mst manipulation utilities",
6
+
"keywords": [
7
+
"atproto",
8
+
"repo",
9
+
"mst",
10
+
"dasl",
11
+
"car"
12
+
],
13
+
"license": "0BSD",
14
+
"repository": {
15
+
"url": "https://github.com/mary-ext/atcute",
16
+
"directory": "packages/utilities/mst"
17
+
},
18
+
"files": [
19
+
"dist/",
20
+
"lib/",
21
+
"!lib/**/*.bench.ts",
22
+
"!lib/**/*.test.ts"
23
+
],
24
+
"exports": {
25
+
".": "./dist/index.js"
26
+
},
27
+
"sideEffects": false,
28
+
"scripts": {
29
+
"build": "tsc --project tsconfig.build.json",
30
+
"test": "vitest run",
31
+
"prepublish": "rm -rf dist; pnpm run build"
32
+
},
33
+
"devDependencies": {
34
+
"@vitest/coverage-v8": "^3.2.4",
35
+
"vitest": "^3.2.4"
36
+
},
37
+
"dependencies": {
38
+
"@atcute/cbor": "workspace:^",
39
+
"@atcute/cid": "workspace:^",
40
+
"@atcute/uint8array": "workspace:^",
41
+
"@atcute/varint": "workspace:^",
42
+
"@badrap/valita": "^0.4.6"
43
+
}
44
+
}
+4
packages/utilities/mst/tsconfig.build.json
+4
packages/utilities/mst/tsconfig.build.json
+25
packages/utilities/mst/tsconfig.json
+25
packages/utilities/mst/tsconfig.json
···
1
+
{
2
+
"compilerOptions": {
3
+
"types": [],
4
+
"outDir": "dist/",
5
+
"esModuleInterop": true,
6
+
"skipLibCheck": true,
7
+
"target": "ESNext",
8
+
"allowJs": true,
9
+
"resolveJsonModule": true,
10
+
"moduleDetection": "force",
11
+
"isolatedModules": true,
12
+
"verbatimModuleSyntax": true,
13
+
"strict": true,
14
+
"noImplicitOverride": true,
15
+
"noUnusedLocals": true,
16
+
"noUnusedParameters": true,
17
+
"noFallthroughCasesInSwitch": true,
18
+
"module": "NodeNext",
19
+
"sourceMap": true,
20
+
"declaration": true,
21
+
"declarationMap": true,
22
+
"stripInternal": true
23
+
},
24
+
"include": ["lib"]
25
+
}
+25
pnpm-lock.yaml
+25
pnpm-lock.yaml
···
736
736
specifier: ^3.2.4
737
737
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(tsx@4.20.6)(yaml@2.8.0)
738
738
739
+
packages/utilities/mst:
740
+
dependencies:
741
+
'@atcute/cbor':
742
+
specifier: workspace:^
743
+
version: link:../cbor
744
+
'@atcute/cid':
745
+
specifier: workspace:^
746
+
version: link:../cid
747
+
'@atcute/uint8array':
748
+
specifier: workspace:^
749
+
version: link:../uint8array
750
+
'@atcute/varint':
751
+
specifier: workspace:^
752
+
version: link:../varint
753
+
'@badrap/valita':
754
+
specifier: ^0.4.6
755
+
version: 0.4.6
756
+
devDependencies:
757
+
'@vitest/coverage-v8':
758
+
specifier: ^3.2.4
759
+
version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4)
760
+
vitest:
761
+
specifier: ^3.2.4
762
+
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(yaml@2.8.0)
763
+
739
764
packages/utilities/multibase:
740
765
dependencies:
741
766
'@atcute/uint8array':