a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
refactor(mst): clean it up
+275
-38
+16
packages/utilities/mst/lib/blockmap.ts
+16
packages/utilities/mst/lib/blockmap.ts
···
3
3
4
4
type BlockEntry = [cid: string, bytes: Uint8Array<ArrayBuffer>];
5
5
6
+
/** a map from CID strings to their encoded block data */
6
7
export type BlockMap = Map<string, Uint8Array<ArrayBuffer>>;
7
8
9
+
/**
10
+
* encodes data as CBOR, computes its CID, and adds it to the map
11
+
* @param map the block map to add to
12
+
* @param data the data to encode and add
13
+
*/
8
14
export const add = async (map: BlockMap, data: unknown): Promise<void> => {
9
15
const encoded = CBOR.encode(data);
10
16
const cid = await CID.create(0x71, encoded);
···
12
18
map.set(CID.toString(cid), encoded);
13
19
};
14
20
21
+
/**
22
+
* copies multiple blocks from an iterable into the map
23
+
* @param map the block map to add to
24
+
* @param entries the block entries to add
25
+
*/
15
26
export const setMany = (map: BlockMap, entries: Iterable<Readonly<BlockEntry>>) => {
16
27
for (const [cid, bytes] of entries) {
17
28
map.set(cid, bytes);
18
29
}
19
30
};
20
31
32
+
/**
33
+
* removes multiple blocks from the map by their CIDs
34
+
* @param map the block map to remove from
35
+
* @param cids the CID strings to remove
36
+
*/
21
37
export const deleteMany = (map: BlockMap, cids: Iterable<string>) => {
22
38
for (const cid of cids) {
23
39
map.delete(cid);
+9
packages/utilities/mst/lib/errors.ts
+9
packages/utilities/mst/lib/errors.ts
···
1
+
/**
2
+
* thrown when an MST key is invalid or malformed
3
+
*/
1
4
export class InvalidMstKeyError extends Error {
2
5
constructor(public key: string) {
3
6
super(`invalid mst key; key=${key}`);
4
7
}
5
8
}
6
9
10
+
/**
11
+
* thrown when a referenced block cannot be found in the store
12
+
*/
7
13
export class MissingBlockError extends Error {
8
14
constructor(
9
15
public cid: string,
···
13
19
}
14
20
}
15
21
22
+
/**
23
+
* thrown when a block's decoded object doesn't match the expected type
24
+
*/
16
25
export class UnexpectedObjectError extends Error {
17
26
constructor(
18
27
public cid: string,
+16
packages/utilities/mst/lib/node-store.ts
+16
packages/utilities/mst/lib/node-store.ts
···
4
4
5
5
import LRUCache from './utils/lru.js';
6
6
7
+
/**
8
+
* manages caching and storage of MST nodes with LRU eviction
9
+
*/
7
10
export class NodeStore {
11
+
/** underlying block store for persistent storage */
8
12
store: BlockStore;
13
+
/** LRU cache for recently accessed nodes */
9
14
cache = new LRUCache<string | null, MSTNode>(1024);
10
15
11
16
constructor(store: BlockStore) {
12
17
this.store = store;
13
18
}
14
19
20
+
/**
21
+
* retrieves an MST node by its CID, using cache when available
22
+
* @param cid the CID of the node to retrieve, or null for empty node
23
+
* @returns the MST node
24
+
* @throws {MissingBlockError} if the node cannot be found in the store
25
+
*/
15
26
async get(cid: string | null): Promise<MSTNode> {
16
27
let node = this.cache.get(cid);
17
28
if (node === undefined) {
···
34
45
return node;
35
46
}
36
47
48
+
/**
49
+
* stores an MST node in both the cache and the underlying block store
50
+
* @param node the node to store
51
+
* @returns the same node that was passed in
52
+
*/
37
53
async put(node: MSTNode): Promise<MSTNode> {
38
54
const cid = (await node.cid()).$link;
39
55
+52
-28
packages/utilities/mst/lib/node-walker.ts
+52
-28
packages/utilities/mst/lib/node-walker.ts
···
1
1
import type { CidLink } from '@atcute/cid';
2
2
3
-
import { MSTNode, getKeyHeight } from './node.js';
4
3
import { NodeStore } from './node-store.js';
4
+
import { MSTNode, getKeyHeight } from './node.js';
5
5
import Stack from './utils/stack.js';
6
6
7
-
interface StackFrame {
7
+
/**
8
+
* represents a single frame in the NodeWalker traversal stack
9
+
* tracks position within a node and the current search boundaries
10
+
*/
11
+
export interface StackFrame {
12
+
/** current MST node */
8
13
node: MSTNode;
14
+
/** left boundary path for this frame */
9
15
lpath: string;
16
+
/** right boundary path for this frame */
10
17
rpath: string;
18
+
/** current cursor index within the node */
11
19
idx: number;
12
20
}
13
21
14
22
/**
15
-
* NodeWalker makes implementing tree diffing and other MST query ops more
16
-
* convenient (but it does not, itself, implement them).
23
+
* provides a cursor-based interface for traversing MST nodes
24
+
* supports tree diffing and various MST query operations
17
25
*
18
-
* A NodeWalker starts off at the root of a tree, and can walk along or recurse
19
-
* down into subtrees.
26
+
* a NodeWalker starts at the root of a tree and can walk along or recurse
27
+
* down into subtrees
20
28
*
21
-
* Walking "off the end" of a subtree brings you back up to its next non-empty parent.
29
+
* walking "off the end" of a subtree brings you back up to its next non-empty parent
22
30
*
23
-
* Recall MSTNode layout:
31
+
* recall MSTNode layout:
24
32
*
25
33
* ```
26
34
* keys: (lpath) (0, 1, 2, 3) (rpath)
···
32
40
static readonly PATH_MIN = ''; // string that compares less than all legal path strings
33
41
static readonly PATH_MAX = '\xff'; // string that compares greater than all legal path strings
34
42
35
-
private store: NodeStore;
36
-
private stack: Stack<StackFrame>;
37
-
private rootHeight: number;
38
-
private trusted: boolean;
43
+
/**
44
+
* node store for fetching nodes
45
+
* @internal
46
+
*/
47
+
_store: NodeStore;
48
+
/**
49
+
* stack of frames representing the traversal path
50
+
* @internal
51
+
*/
52
+
_stack: Stack<StackFrame>;
53
+
/**
54
+
* height of the root node
55
+
* @internal
56
+
*/
57
+
_rootHeight: number;
58
+
/**
59
+
* whether to skip height validation (for trusted trees)
60
+
* @internal
61
+
*/
62
+
_trusted: boolean;
39
63
40
64
private constructor(store: NodeStore, stack: Stack<StackFrame>, rootHeight: number, trusted: boolean) {
41
-
this.store = store;
42
-
this.stack = stack;
43
-
this.rootHeight = rootHeight;
44
-
this.trusted = trusted;
65
+
this._store = store;
66
+
this._stack = stack;
67
+
this._rootHeight = rootHeight;
68
+
this._trusted = trusted;
45
69
}
46
70
47
71
/**
···
87
111
*/
88
112
async createSubtreeWalker(): Promise<NodeWalker> {
89
113
return await NodeWalker.create(
90
-
this.store,
114
+
this._store,
91
115
this.subtree?.$link ?? null,
92
116
this.lpath,
93
117
this.rpath,
94
-
this.trusted,
118
+
this._trusted,
95
119
this.height - 1,
96
120
);
97
121
}
98
122
99
-
/** current stack frame (internal) */
123
+
/** current stack frame */
100
124
get frame(): StackFrame {
101
-
const frame = this.stack.peek();
125
+
const frame = this._stack.peek();
102
126
if (frame === undefined) {
103
127
throw new Error(`stack is empty`);
104
128
}
···
108
132
109
133
/** current height in the tree (decreases as you descend) */
110
134
get height(): number {
111
-
return this.rootHeight - (this.stack.size - 1);
135
+
return this._rootHeight - (this._stack.size - 1);
112
136
}
113
137
114
138
/** key/path to the left of current cursor position */
···
142
166
/** whether the walker has reached the end of the tree */
143
167
get done(): boolean {
144
168
// is (not this.stack) really necessary here? is that a reachable state?
145
-
const bottom = this.stack.peekBottom();
169
+
const bottom = this._stack.peekBottom();
146
170
return (
147
-
this.stack.size === 0 || (this.subtree === null && bottom !== undefined && this.rpath === bottom.rpath)
171
+
this._stack.size === 0 || (this.subtree === null && bottom !== undefined && this.rpath === bottom.rpath)
148
172
);
149
173
}
150
174
···
161
185
rightOrUp(): void {
162
186
if (!this.canGoRight) {
163
187
// we reached the end of this node, go up a level
164
-
this.stack.pop();
165
-
if (this.stack.size === 0) {
188
+
this._stack.pop();
189
+
if (this._stack.size === 0) {
166
190
throw new Error(`cannot navigate beyond root; check .done before calling`);
167
191
}
168
192
return this.rightOrUp(); // we need to recurse, to skip over empty intermediates on the way back up
···
191
215
throw new Error(`cannot descend; no subtree at current position`);
192
216
}
193
217
194
-
const subtreeNode = await this.store.get(subtree.$link);
218
+
const subtreeNode = await this._store.get(subtree.$link);
195
219
196
-
if (!this.trusted) {
220
+
if (!this._trusted) {
197
221
// if we "trust" the source we can elide this check
198
222
// the "null" case occurs for empty intermediate nodes
199
223
const subtreeHeight = await subtreeNode.height();
···
202
226
}
203
227
}
204
228
205
-
this.stack.push({
229
+
this._stack.push({
206
230
node: subtreeNode,
207
231
lpath: this.lpath,
208
232
rpath: this.rpath,
+78
-8
packages/utilities/mst/lib/node.ts
+78
-8
packages/utilities/mst/lib/node.ts
···
5
5
6
6
import { isNodeData, type NodeData, type TreeEntry } from './types.js';
7
7
8
+
/**
9
+
* represents a node in a Merkle Search Tree (MST)
10
+
* stores sorted keys, their associated values (CIDs), and subtree pointers
11
+
*/
8
12
export class MSTNode {
9
-
/** @internal */
13
+
/**
14
+
* cached height of this node in the tree
15
+
* @internal
16
+
*/
10
17
_height: number | null | undefined;
11
-
/** @internal */
18
+
/**
19
+
* cached CID for this node
20
+
* @internal
21
+
*/
12
22
_cid: CidLink | undefined;
13
-
/** @internal */
23
+
/**
24
+
* cached serialized bytes for this node
25
+
* @internal
26
+
*/
14
27
_bytes: Uint8Array<ArrayBuffer> | undefined;
15
28
16
29
protected constructor(
30
+
/** sorted array of keys stored in this node */
17
31
readonly keys: readonly string[],
32
+
/** array of value CIDs corresponding to each key */
18
33
readonly values: readonly CidLink[],
34
+
/** array of subtree CIDs (length is keys.length + 1) */
19
35
readonly subtrees: readonly (CidLink | null)[],
20
36
) {}
21
37
38
+
/**
39
+
* creates a new MST node with validation
40
+
* @param keys sorted array of keys
41
+
* @param values array of value CIDs corresponding to keys
42
+
* @param subtrees array of subtree CIDs (length must be keys.length + 1)
43
+
* @returns a new validated MST node
44
+
* @throws {TypeError} if node structure is invalid or keys have inconsistent heights
45
+
*/
22
46
static async create(
23
47
keys: readonly string[],
24
48
values: readonly CidLink[],
···
45
69
return new MSTNode(keys, values, subtrees);
46
70
}
47
71
72
+
/**
73
+
* creates an empty MST node
74
+
* @returns a new empty node
75
+
*/
48
76
static empty(): MSTNode {
49
77
return new MSTNode([], [], [null]);
50
78
}
51
79
80
+
/**
81
+
* deserializes an MST node from CBOR-encoded bytes
82
+
* @param bytes the CBOR-encoded node data
83
+
* @returns the deserialized MST node
84
+
* @throws {TypeError} if the bytes don't represent a valid MST node
85
+
*/
52
86
static async deserialize(bytes: Uint8Array): Promise<MSTNode> {
53
87
const node = CBOR.decode(bytes);
54
88
if (!isNodeData(node)) {
···
87
121
return await MSTNode.create(keys, values, subtrees);
88
122
}
89
123
124
+
/**
125
+
* serializes the node to CBOR-encoded bytes with prefix compression
126
+
* @returns the CBOR-encoded node data
127
+
*/
90
128
async serialize(): Promise<Uint8Array<ArrayBuffer>> {
91
129
let bytes = this._bytes;
92
130
if (bytes === undefined) {
···
122
160
return bytes;
123
161
}
124
162
163
+
/**
164
+
* whether the node is empty (no keys or values)
165
+
*/
166
+
get isEmpty(): boolean {
167
+
return this.subtrees.length === 1 && this.subtrees[0] === null;
168
+
}
169
+
170
+
/**
171
+
* computes the CID for this node
172
+
* @returns the CID link for this node
173
+
*/
125
174
async cid(): Promise<CidLink> {
126
175
let cid = this._cid;
127
176
if (cid === undefined) {
···
131
180
return cid;
132
181
}
133
182
134
-
isEmpty(): boolean {
135
-
return this.subtrees.length === 1 && this.subtrees[0] === null;
136
-
}
137
-
183
+
/**
184
+
* computes the height of this node in the MST
185
+
* @returns the height, or null if indeterminate (empty intermediate node)
186
+
*/
138
187
async height(): Promise<number | null> {
139
188
let height = this._height;
140
189
if (height === undefined) {
141
190
const keys = this.keys;
142
191
143
-
if (this.isEmpty()) {
192
+
if (this.isEmpty) {
144
193
height = 0;
145
194
} else if (keys.length > 0) {
146
195
height = await getKeyHeight(keys[0]);
···
154
203
return height;
155
204
}
156
205
206
+
/**
207
+
* gets the node height, throwing if indeterminate
208
+
* @returns the height
209
+
* @throws {Error} if height cannot be determined
210
+
*/
157
211
async requireHeight(): Promise<number> {
158
212
const height = await this.height();
159
213
if (height === null) {
···
163
217
return height;
164
218
}
165
219
220
+
/**
221
+
* finds the index of the first key >= the given key
222
+
* @param key the key to search for
223
+
* @returns the index of the lower bound
224
+
*/
166
225
lowerBound(key: string): number {
167
226
const keys = this.keys;
168
227
const len = keys.length;
···
177
236
}
178
237
}
179
238
239
+
/**
240
+
* computes the MST height for a given key by counting leading zeros in its hash
241
+
* @param key the key to compute height for
242
+
* @returns the height (number of leading zero bits in 2-bit chunks)
243
+
*/
180
244
export const getKeyHeight = async (key: string): Promise<number> => {
181
245
const hash = await toSha256(encodeUtf8(key));
182
246
···
204
268
return lz;
205
269
};
206
270
271
+
/**
272
+
* computes the length of the common prefix between two strings
273
+
* @param a first string
274
+
* @param b second string
275
+
* @returns length of common prefix
276
+
*/
207
277
const commonPrefixLength = (a: string, b: string): number => {
208
278
let idx = 0;
209
279
for (let len = Math.min(a.length, b.length); idx < len; idx++) {
+87
-1
packages/utilities/mst/lib/stores.ts
+87
-1
packages/utilities/mst/lib/stores.ts
···
3
3
import { deleteMany, setMany, type BlockMap } from './blockmap.js';
4
4
import { MissingBlockError, UnexpectedObjectError } from './errors.js';
5
5
6
+
/**
7
+
* a read-only interface for retrieving blocks by their CID
8
+
*/
6
9
export interface ReadonlyBlockStore {
10
+
/**
11
+
* retrieves a single block by its CID
12
+
* @param cid the CID of the block to retrieve
13
+
* @returns the block data, or null if not found
14
+
*/
7
15
get(cid: string): Promise<Uint8Array<ArrayBuffer> | null>;
16
+
17
+
/**
18
+
* retrieves multiple blocks by their CIDs
19
+
* @param cids array of CIDs to retrieve
20
+
* @returns object containing found blocks and missing CIDs
21
+
*/
8
22
getMany(cids: string[]): Promise<{ found: BlockMap; missing: string[] }>;
9
23
24
+
/**
25
+
* checks if a block exists in the store
26
+
* @param cid the CID to check
27
+
* @returns true if the block exists, false otherwise
28
+
*/
10
29
has(cid: string): Promise<boolean>;
11
30
}
12
31
32
+
/**
33
+
* a writable block store supporting both read and write operations
34
+
*/
13
35
export interface BlockStore extends ReadonlyBlockStore {
36
+
/**
37
+
* stores a single block
38
+
* @param cid the CID of the block
39
+
* @param bytes the block data to store
40
+
*/
14
41
put(cid: string, bytes: Uint8Array<ArrayBuffer>): Promise<void>;
42
+
43
+
/**
44
+
* stores multiple blocks at once
45
+
* @param blocks map of CIDs to block data
46
+
*/
15
47
putMany(blocks: BlockMap): Promise<void>;
16
48
49
+
/**
50
+
* removes a single block from the store
51
+
* @param cid the CID of the block to remove
52
+
*/
17
53
delete(cid: string): Promise<void>;
54
+
55
+
/**
56
+
* removes multiple blocks from the store
57
+
* @param cids array of CIDs to remove
58
+
*/
18
59
deleteMany(cids: string[]): Promise<void>;
19
60
}
20
61
62
+
/**
63
+
* an in-memory read-only block store using a Map
64
+
*/
21
65
export class ReadonlyMemoryBlockStore implements ReadonlyBlockStore {
66
+
/** underlying map storing CID to block data */
22
67
blocks: BlockMap = new Map();
23
68
69
+
/**
70
+
* creates a new read-only memory block store
71
+
* @param blocks optional initial blocks to populate the store with
72
+
*/
24
73
constructor(blocks?: BlockMap) {
25
74
if (blocks !== undefined) {
26
75
setMany(this.blocks, blocks);
···
52
101
}
53
102
}
54
103
104
+
/**
105
+
* an in-memory writable block store using a Map
106
+
*/
55
107
export class MemoryBlockStore extends ReadonlyMemoryBlockStore implements BlockStore {
56
108
put(cid: string, bytes: Uint8Array<ArrayBuffer>): Promise<void> {
57
109
this.blocks.set(cid, bytes);
···
74
126
}
75
127
}
76
128
129
+
/**
130
+
* a block store that overlays one store on top of another
131
+
* reads check upper first, then fall back to lower
132
+
* all writes go to the upper store only
133
+
*/
77
134
export class OverlayBlockStore implements BlockStore {
135
+
/** the writable upper layer store */
78
136
upper: BlockStore;
137
+
/** the read-only lower layer store */
79
138
lower: ReadonlyBlockStore;
80
139
140
+
/**
141
+
* creates a new overlay block store
142
+
* @param upper the writable upper layer store
143
+
* @param lower the read-only lower layer store
144
+
*/
81
145
constructor(upper: BlockStore, lower: ReadonlyBlockStore) {
82
146
this.upper = upper;
83
147
this.lower = lower;
···
130
194
}
131
195
}
132
196
197
+
/**
198
+
* reads and decodes a block, validating it matches the expected type
199
+
* @param store block store to read from
200
+
* @param cid CID of the block to read
201
+
* @param def schema definition with name and validation function
202
+
* @returns the decoded and validated object
203
+
* @throws {MissingBlockError} if block is not found
204
+
* @throws {UnexpectedObjectError} if block doesn't match expected type
205
+
*/
133
206
export const readObject = async <T>(store: ReadonlyBlockStore, cid: string, def: CheckDef<T>): Promise<T> => {
134
207
const bytes = await store.get(cid);
135
208
if (bytes === null) {
···
144
217
return decoded;
145
218
};
146
219
220
+
/**
221
+
* reads and decodes a block without type validation
222
+
* @param store block store to read from
223
+
* @param cid CID of the block to read
224
+
* @returns the decoded object
225
+
* @throws {MissingBlockError} if block is not found
226
+
*/
147
227
export const readRecord = async (store: ReadonlyBlockStore, cid: string): Promise<unknown> => {
148
228
const bytes = await store.get(cid);
149
229
if (bytes === null) {
···
155
235
return decoded;
156
236
};
157
237
158
-
interface CheckDef<T> {
238
+
/**
239
+
* defines a type validator for use with readObject
240
+
* combines a human-readable type name with a type guard function
241
+
*/
242
+
export interface CheckDef<T> {
243
+
/** human-readable name of the expected type */
159
244
name: string;
245
+
/** type guard function to validate the decoded value */
160
246
check: (value: unknown) => value is T;
161
247
}
+16
packages/utilities/mst/lib/types.ts
+16
packages/utilities/mst/lib/types.ts
···
1
1
import { isBytes, type Bytes } from '@atcute/cbor';
2
2
import { isCidLink, type CidLink } from '@atcute/cid';
3
3
4
+
/**
5
+
* represents a single entry in an MST node
6
+
*/
4
7
export interface TreeEntry {
5
8
/** count of bytes shared with previous TreeEntry in this Node (if any) */
6
9
p: number;
···
12
15
t: CidLink | null;
13
16
}
14
17
18
+
/**
19
+
* validates that an unknown value is a valid TreeEntry
20
+
* @param value the value to check
21
+
* @returns true if value is a TreeEntry, false otherwise
22
+
*/
15
23
export const isTreeEntry = (value: unknown): value is TreeEntry => {
16
24
if (value === null || typeof value !== 'object') {
17
25
return false;
···
24
32
);
25
33
};
26
34
35
+
/**
36
+
* represents the serialized data structure of an MST node
37
+
*/
27
38
export interface NodeData {
28
39
/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
29
40
l: CidLink | null;
···
31
42
e: TreeEntry[];
32
43
}
33
44
45
+
/**
46
+
* validates that an unknown value is valid NodeData
47
+
* @param value the value to check
48
+
* @returns true if value is NodeData, false otherwise
49
+
*/
34
50
export const isNodeData = (value: unknown): value is NodeData => {
35
51
if (value === null || typeof value !== 'object') {
36
52
return false;