a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
refactor: biig changes
+748
-494
+5
.changeset/calm-mice-fetch.md
+5
.changeset/calm-mice-fetch.md
+5
.changeset/calm-moons-glow.md
+5
.changeset/calm-moons-glow.md
+13
.changeset/ninety-beers-lie.md
+13
.changeset/ninety-beers-lie.md
···
1
+
---
2
+
'@atcute/car': major
3
+
---
4
+
5
+
remove AT Protocol repository reader
6
+
7
+
the addition of `@atcute/mst` into the atcute family of packages has put `@atcute/car` in a weird
8
+
spot, we can't have `@atcute/car` depend on `@atcute/mst` because we've already done the reverse to
9
+
test `@atcute/mst`'s functionalities.
10
+
11
+
if you need this functionality back, use `@atcute/repo`.
12
+
13
+
this does mean that we are skipping the stable release of v4, and straight into v5.
+1
-1
README.md
+1
-1
README.md
···
59
59
| [`identity-resolver`](./packages/identity/identity-resolver): handle and DID document resolution |
60
60
| [`identity-resolver-node`](./packages/identity/identity-resolver-node): additional identity resolvers for Node.js |
61
61
| **Utility packages** |
62
-
| [`car`](./packages/utilities/car): DASL CAR and atproto repository decoder |
62
+
| [`car`](./packages/utilities/car): DASL CAR codec |
63
63
| [`cbor`](./packages/utilities/cbor): DASL dCBOR42 codec |
64
64
| [`cid`](./packages/utilities/cid): DASL CID codec |
65
65
| [`crypto`](./packages/utilities/crypto): cryptographic utilities |
+43
-44
packages/lexicons/lexicon-resolver/lib/schemas/verify.ts
packages/utilities/repo/lib/verify.ts
+43
-44
packages/lexicons/lexicon-resolver/lib/schemas/verify.ts
packages/utilities/repo/lib/verify.ts
···
1
-
import { CarReader } from '@atcute/car';
2
-
import { type BlockMap, type Commit, isMstNode, type MstNode } from '@atcute/car/repo-reader';
1
+
import * as CAR from '@atcute/car';
3
2
import * as CBOR from '@atcute/cbor';
4
3
import * as CID from '@atcute/cid';
5
-
import { type FoundPublicKey, getPublicKeyFromDidController, verifySig } from '@atcute/crypto';
6
-
import { type DidDocument, getAtprotoVerificationMaterial } from '@atcute/identity';
7
-
import { type AtprotoDid } from '@atcute/lexicons/syntax';
8
-
import { toSha256 } from '@atcute/uint8array';
4
+
import type { PublicKey } from '@atcute/crypto';
5
+
import type { AtprotoDid } from '@atcute/lexicons/syntax';
6
+
import { isNodeData, type NodeData } from '@atcute/mst';
7
+
import { decodeUtf8From, encodeUtf8, toSha256 } from '@atcute/uint8array';
8
+
9
+
import { isCommit, type Commit } from './types.js';
10
+
11
+
type BlockMap = Map<string, Uint8Array>;
9
12
10
13
export interface VerifiedRecord {
11
-
/** AT-URI of the record */
12
-
uri: string;
13
14
/** CID of the record */
14
15
cid: string;
15
16
/** Record data */
···
17
18
}
18
19
19
20
export interface VerifyRecordOptions {
20
-
did: AtprotoDid;
21
+
did?: AtprotoDid;
21
22
collection: string;
22
23
rkey: string;
23
-
didDocument: DidDocument;
24
+
publicKey?: PublicKey;
24
25
carBytes: Uint8Array;
25
26
}
26
27
···
28
29
did,
29
30
collection,
30
31
rkey,
31
-
didDocument,
32
+
publicKey,
32
33
carBytes,
33
34
}: VerifyRecordOptions): Promise<VerifiedRecord> => {
34
-
// grab public key from did document
35
-
let publicKey: FoundPublicKey;
36
-
{
37
-
const controller = getAtprotoVerificationMaterial(didDocument);
38
-
if (!controller) {
39
-
throw new Error(`did document does not contain verification material`);
40
-
}
41
-
42
-
publicKey = getPublicKeyFromDidController(controller);
43
-
}
44
-
45
35
// read the car
46
36
let blockmap: BlockMap;
47
37
let commit: Commit;
48
38
{
49
-
const reader = CarReader.fromUint8Array(carBytes);
39
+
const reader = CAR.fromUint8Array(carBytes);
50
40
if (reader.header.data.roots.length !== 1) {
51
41
throw new Error(`car must have exactly one root`);
52
42
}
···
61
51
throw new Error(`cid does not match bytes`);
62
52
}
63
53
64
-
blockmap.set(cidString, entry);
54
+
blockmap.set(cidString, entry.bytes);
65
55
}
66
56
67
57
if (blockmap.size === 0) {
68
58
throw new Error(`car must have at least one block`);
69
59
}
70
60
71
-
commit = CAR.readBlock(blockmap, reader.header.data.roots[0], CAR.isCommit);
61
+
commit = readBlock(blockmap, reader.header.data.roots[0].$link, isCommit);
72
62
}
73
63
74
64
// verify did in commit matches the did
75
-
if (commit.did !== did) {
65
+
if (did !== undefined && commit.did !== did) {
76
66
throw new Error(`did in commit does not match expected did`);
77
67
}
78
68
79
-
// verify signature contained in commit is valid
80
-
{
69
+
// verify signature contained in commit is valid (if publicKey provided)
70
+
if (publicKey) {
81
71
const { sig, ...unsigned } = commit;
82
72
83
73
const data = CBOR.encode(unsigned);
84
-
const valid = await verifySig(
85
-
publicKey,
74
+
const valid = await publicKey.verify(
86
75
CBOR.fromBytes(sig) as Uint8Array<ArrayBuffer>,
87
76
data as Uint8Array<ArrayBuffer>,
88
77
);
···
100
89
}
101
90
102
91
return {
103
-
uri: `at://${did}/${collection}/${rkey}`,
104
92
cid: found.cid,
105
93
record: found.record,
106
94
};
107
95
};
108
96
97
+
const readBlock = <T>(blockmap: BlockMap, cid: string, validate: (value: unknown) => value is T): T => {
98
+
const bytes = blockmap.get(cid);
99
+
if (!bytes) {
100
+
throw new Error(`cid not found in blockmap; cid=${cid}`);
101
+
}
102
+
103
+
const decoded = CBOR.decode(bytes);
104
+
if (!validate(decoded)) {
105
+
throw new Error(`validation failed for cid=${cid}`);
106
+
}
107
+
108
+
return decoded;
109
+
};
110
+
109
111
interface DfsResult {
110
112
found: false | { cid: string; record: unknown };
111
113
min?: string;
···
113
115
depth?: number;
114
116
}
115
117
116
-
const encoder = new TextEncoder();
117
-
const decoder = new TextDecoder();
118
-
119
118
const dfs = async (
120
119
blockmap: BlockMap,
121
120
from: string | undefined,
···
137
136
}
138
137
139
138
// Get the block data
140
-
let node: MstNode;
139
+
let node: NodeData;
141
140
{
142
-
const entry = blockmap.get(from);
143
-
if (!entry) {
141
+
const bytes = blockmap.get(from);
142
+
if (!bytes) {
144
143
return { found: false };
145
144
}
146
145
147
-
const decoded = CBOR.decode(entry.bytes);
148
-
if (!isMstNode(decoded)) {
146
+
const decoded = CBOR.decode(bytes);
147
+
if (!isNodeData(decoded)) {
149
148
throw new Error(`invalid mst node; cid=${from}`);
150
149
}
151
150
···
164
163
// Process all entries in this node
165
164
for (const entry of node.e) {
166
165
// Construct the key by truncating and appending
167
-
key = key.substring(0, entry.p) + decoder.decode(CBOR.fromBytes(entry.k));
166
+
key = key.substring(0, entry.p) + decodeUtf8From(CBOR.fromBytes(entry.k));
168
167
169
168
// Check if this is our target key
170
169
if (key === targetKey) {
171
-
const recordBlock = blockmap.get(entry.v.$link);
172
-
if (recordBlock) {
173
-
const record = CBOR.decode(recordBlock.bytes);
170
+
const recordBytes = blockmap.get(entry.v.$link);
171
+
if (recordBytes) {
172
+
const record = CBOR.decode(recordBytes);
174
173
found = { cid: entry.v.$link, record };
175
174
}
176
175
}
177
176
178
177
// Calculate depth based on leading zeros in the hash
179
-
const keyDigest = await toSha256(encoder.encode(key));
178
+
const keyDigest = await toSha256(encodeUtf8(key));
180
179
let zeroCount = 0;
181
180
182
181
outerLoop: for (const byte of keyDigest) {
+30
-4
packages/lexicons/lexicon-resolver/lib/schemas/xrpc.ts
+30
-4
packages/lexicons/lexicon-resolver/lib/schemas/xrpc.ts
···
1
-
import { getPdsEndpoint } from '@atcute/identity';
1
+
import {
2
+
getPublicKeyFromDidController,
3
+
P256PublicKey,
4
+
Secp256k1PublicKey,
5
+
type PublicKey,
6
+
} from '@atcute/crypto';
7
+
import { getAtprotoVerificationMaterial, getPdsEndpoint } from '@atcute/identity';
2
8
import type { DidDocumentResolver } from '@atcute/identity-resolver';
3
9
import { lexiconDoc, type LexiconDoc } from '@atcute/lexicon-doc';
4
10
import type { AtprotoDid, Nsid } from '@atcute/lexicons/syntax';
11
+
import { verifyRecord, type VerifiedRecord } from '@atcute/repo';
5
12
6
13
import { FailedResponseError } from '@atcute/util-fetch';
7
14
8
15
import { LEXICON_SCHEMA_COLLECTION } from '../constants.js';
9
16
import * as err from '../errors.js';
10
17
import type { ResolvedSchema, ResolveLexiconRecordOptions } from '../types.js';
11
-
import { verifyRecord, type VerifiedRecord } from './verify.js';
12
18
13
19
export interface LexiconSchemaResolverOptions {
14
20
didDocumentResolver: DidDocumentResolver;
···
69
75
// Step 3: Verify record and extract data
70
76
let verifiedRecord: VerifiedRecord;
71
77
try {
78
+
// Extract public key from DID document for signature verification
79
+
const controller = getAtprotoVerificationMaterial(didDocument);
80
+
if (!controller) {
81
+
throw new Error(`did document does not contain verification material`);
82
+
}
83
+
84
+
const found = getPublicKeyFromDidController(controller);
85
+
86
+
let publicKey: PublicKey;
87
+
switch (found.type) {
88
+
case 'p256': {
89
+
publicKey = await P256PublicKey.importRaw(found.publicKeyBytes);
90
+
break;
91
+
}
92
+
case 'secp256k1': {
93
+
publicKey = await Secp256k1PublicKey.importRaw(found.publicKeyBytes);
94
+
break;
95
+
}
96
+
}
97
+
72
98
verifiedRecord = await verifyRecord({
73
99
did: authority,
74
100
collection: LEXICON_SCHEMA_COLLECTION,
75
101
rkey: nsid,
76
-
didDocument,
102
+
publicKey,
77
103
carBytes,
78
104
});
79
105
} catch (cause) {
···
99
125
}
100
126
101
127
return {
102
-
uri: verifiedRecord.uri,
128
+
uri: `at://${authority}/${LEXICON_SCHEMA_COLLECTION}/${nsid}`,
103
129
cid: verifiedRecord.cid,
104
130
rawSchema: verifiedRecord.record,
105
131
schema: schema,
+1
packages/lexicons/lexicon-resolver/package.json
+1
packages/lexicons/lexicon-resolver/package.json
+1
packages/utilities/car/.gitignore
+1
packages/utilities/car/.gitignore
···
1
+
/coverage/
+16
-48
packages/utilities/car/README.md
+16
-48
packages/utilities/car/README.md
···
1
1
# @atcute/car
2
2
3
-
lightweight [DASL CAR (content-addressable archives)][dasl-car] and atproto repository decoder
4
-
library for AT Protocol.
3
+
lightweight [DASL CAR (content-addressable archives)][dasl-car] codec library for AT Protocol.
5
4
6
5
[dasl-car]: https://dasl.ing/car.html
7
6
···
10
9
### streaming usage
11
10
12
11
```ts
13
-
import { CarReader, RepoReader } from '@atcute/car';
12
+
import { fromStream } from '@atcute/car';
14
13
15
14
const stream = new ReadableStream({
16
15
/* ... */
17
16
});
18
17
19
-
// read AT Protocol repository exports
20
-
{
21
-
await using repo = RepoReader.fromStream(stream);
22
-
23
-
for await (const entry of repo) {
24
-
entry;
25
-
// ^? RepoEntry { collection: 'app.bsky.feed.post', rkey: '3lprcc55bb222', ... }
26
-
}
27
-
28
-
repo.missingBlocks;
29
-
// ^? []
30
-
}
31
-
32
-
// read generic CAR archives
33
-
{
34
-
await using car = CarReader.fromStream(stream);
18
+
await using car = fromStream(stream);
35
19
36
-
const roots = await car.roots();
20
+
const roots = await car.roots();
37
21
38
-
for await (const entry of car) {
39
-
entry;
40
-
// ^? CarEntry { cid: CidLink {}, bytes: Uint8Array {}, ... }
41
-
}
22
+
for await (const entry of car) {
23
+
entry;
24
+
// ^? CarEntry { cid: CidLink {}, bytes: Uint8Array {}, ... }
42
25
}
43
26
```
44
27
45
28
### streaming usage (for runtimes without `await using` yet)
46
29
47
30
```ts
48
-
const repo = RepoReader.fromStream(stream);
31
+
const car = fromStream(stream);
49
32
50
33
try {
51
-
for await (const entry of repo) {
34
+
for await (const entry of car) {
52
35
entry;
53
-
// ^? RepoEntry
36
+
// ^? CarEntry { ... }
54
37
}
55
38
} finally {
56
-
await repo.dispose();
39
+
await car.dispose();
57
40
}
58
41
```
59
42
···
64
47
/* ... */
65
48
]);
66
49
67
-
// read AT Protocol repository exports
68
-
{
69
-
const repo = RepoReader.fromUint8Array(buffer);
70
-
71
-
for (const entry of repo) {
72
-
entry;
73
-
// ^? RepoEntry { collection: 'app.bsky.feed.post', rkey: '3lprcc55bb222', ... }
74
-
}
75
-
76
-
repo.missingBlocks;
77
-
// ^? []
78
-
}
79
-
80
50
// read generic CAR archives
81
-
{
82
-
const car = CarReader.fromUint8Array(buffer);
51
+
const car = fromUint8Array(buffer);
83
52
84
-
const roots = car.roots;
53
+
const roots = car.roots;
85
54
86
-
for (const entry of car) {
87
-
entry;
88
-
// ^? CarEntry { cid: CidLink {}, bytes: Uint8Array {}, ... }
89
-
}
55
+
for (const entry of car) {
56
+
entry;
57
+
// ^? CarEntry { cid: CidLink {}, bytes: Uint8Array {}, ... }
90
58
}
91
59
```
+3
-3
packages/utilities/car/lib/car-reader/index.test.ts
packages/utilities/car/lib/reader.test.ts
+3
-3
packages/utilities/car/lib/car-reader/index.test.ts
packages/utilities/car/lib/reader.test.ts
···
1
-
import { describe, expect, it } from 'bun:test';
1
+
import { describe, expect, it } from 'vitest';
2
2
3
3
import { fromString, toCidLink } from '@atcute/cid';
4
4
import { fromBase64 } from '@atcute/multibase';
5
5
6
-
import { carEntryTransform, fromStream } from './stream-car-reader.js';
7
-
import { fromUint8Array } from './sync-car-reader.js';
6
+
import { fromUint8Array } from './reader.js';
7
+
import { carEntryTransform, fromStream } from './streamed-reader.js';
8
8
9
9
describe('fromUint8Array', () => {
10
10
it('reads car files', () => {
-4
packages/utilities/car/lib/car-reader/index.ts
-4
packages/utilities/car/lib/car-reader/index.ts
+5
-6
packages/utilities/car/lib/car-reader/stream-car-reader.ts
packages/utilities/car/lib/streamed-reader.ts
+5
-6
packages/utilities/car/lib/car-reader/stream-car-reader.ts
packages/utilities/car/lib/streamed-reader.ts
···
1
1
import * as CBOR from '@atcute/cbor';
2
+
import type { Cid, CidLink } from '@atcute/cid';
2
3
import * as CID from '@atcute/cid';
3
4
import { concat } from '@atcute/uint8array';
4
5
···
6
7
7
8
export interface StreamedCarReader {
8
9
header(): Promise<CarHeader>;
9
-
roots(): Promise<CBOR.CidLink[]>;
10
+
roots(): Promise<CidLink[]>;
10
11
11
12
dispose(): Promise<void>;
12
13
···
110
111
return buffer;
111
112
};
112
113
113
-
const readCid = async (): Promise<CID.Cid> => {
114
+
const readCid = async (): Promise<Cid> => {
114
115
const head = await readExact(4);
115
116
116
117
const version = head[0];
···
138
139
const bytes = concat([head, await readExact(digestSize)]);
139
140
const digest = bytes.subarray(4, 4 + digestSize);
140
141
141
-
const cid: CID.Cid = {
142
+
const cid: Cid = {
142
143
version: version,
143
144
codec: codec,
144
145
digest: {
···
146
147
contents: digest,
147
148
},
148
149
bytes: bytes,
149
-
// @ts-expect-error
150
-
_str: undefined,
151
150
};
152
151
153
152
return cid;
···
188
187
return (_header = { data, headerStart, headerEnd, dataStart, dataEnd });
189
188
},
190
189
191
-
async roots(): Promise<CBOR.CidLink[]> {
190
+
async roots(): Promise<CidLink[]> {
192
191
const header = await this.header();
193
192
194
193
return header.data.roots;
+2
-3
packages/utilities/car/lib/car-reader/sync-car-reader.ts
packages/utilities/car/lib/reader.ts
+2
-3
packages/utilities/car/lib/car-reader/sync-car-reader.ts
packages/utilities/car/lib/reader.ts
···
1
1
import * as CBOR from '@atcute/cbor';
2
+
import type { CidLink } from '@atcute/cid';
2
3
import * as CID from '@atcute/cid';
3
4
import * as varint from '@atcute/varint';
4
5
···
13
14
14
15
export interface SyncCarReader {
15
16
readonly header: CarHeader;
16
-
readonly roots: CBOR.CidLink[];
17
+
readonly roots: CidLink[];
17
18
18
19
/** @deprecated do for..of on the reader directly */
19
20
iterate(): Generator<CarEntry>;
···
165
166
contents: digest,
166
167
},
167
168
bytes: bytes,
168
-
// @ts-expect-error
169
-
_str: undefined,
170
169
};
171
170
172
171
return cid;
+14
-5
packages/utilities/car/lib/car-reader/types.ts
packages/utilities/car/lib/types.ts
+14
-5
packages/utilities/car/lib/car-reader/types.ts
packages/utilities/car/lib/types.ts
···
1
-
import * as CBOR from '@atcute/cbor';
2
-
import * as CID from '@atcute/cid';
1
+
import { CidLinkWrapper, type Cid, type CidLink } from '@atcute/cid';
3
2
4
3
export interface CarV1Header {
5
4
version: 1;
6
-
roots: CID.CidLink[];
5
+
roots: CidLink[];
7
6
}
8
7
9
8
export const isCarV1Header = (value: unknown): value is CarV1Header => {
···
12
11
}
13
12
14
13
const { version, roots } = value as CarV1Header;
15
-
return version === 1 && Array.isArray(roots) && roots.every((root) => root instanceof CBOR.CidLinkWrapper);
14
+
return version === 1 && Array.isArray(roots) && roots.every((root) => root instanceof CidLinkWrapper);
16
15
};
17
16
18
17
export interface CarHeader {
···
28
27
entryStart: number;
29
28
entryEnd: number;
30
29
31
-
cid: CID.Cid;
30
+
cid: Cid;
32
31
cidStart: number;
33
32
cidEnd: number;
34
33
···
36
35
bytesStart: number;
37
36
bytesEnd: number;
38
37
}
38
+
39
+
/**
40
+
* represents a block to be written to a CAR file
41
+
*/
42
+
export interface CarBlock {
43
+
/** the CID of the block (as bytes) */
44
+
cid: Uint8Array;
45
+
/** the block data */
46
+
data: Uint8Array;
47
+
}
+6
-2
packages/utilities/car/lib/index.ts
+6
-2
packages/utilities/car/lib/index.ts
+2
-3
packages/utilities/car/lib/repo-reader/index.test.ts
packages/utilities/repo/lib/index.test.ts
+2
-3
packages/utilities/car/lib/repo-reader/index.test.ts
packages/utilities/repo/lib/index.test.ts
···
1
-
import { describe, expect, it } from 'bun:test';
1
+
import { describe, expect, it } from 'vitest';
2
2
3
3
import { fromCidLink, toString } from '@atcute/cid';
4
4
import { fromBase64 } from '@atcute/multibase';
5
5
6
-
import { fromStream, repoEntryTransform } from './stream-repo-reader.js';
7
-
import { fromUint8Array } from './sync-repo-reader.js';
6
+
import { fromStream, fromUint8Array, repoEntryTransform } from './index.js';
8
7
9
8
describe('fromUint8Array', () => {
10
9
it('decodes atproto car files', () => {
-6
packages/utilities/car/lib/repo-reader/index.ts
-6
packages/utilities/car/lib/repo-reader/index.ts
-110
packages/utilities/car/lib/repo-reader/mst.ts
-110
packages/utilities/car/lib/repo-reader/mst.ts
···
1
-
import * as CBOR from '@atcute/cbor';
2
-
import * as CID from '@atcute/cid';
3
-
4
-
const isCidLink = (value: unknown): value is CID.CidLink => {
5
-
if (value instanceof CID.CidLinkWrapper) {
6
-
return true;
7
-
}
8
-
9
-
if (value === null || typeof value !== 'object') {
10
-
return false;
11
-
}
12
-
13
-
return '$link' in value && typeof value.$link === 'string';
14
-
};
15
-
16
-
const isBytes = (value: unknown): value is CBOR.Bytes => {
17
-
if (value instanceof CBOR.BytesWrapper) {
18
-
return true;
19
-
}
20
-
21
-
if (value === null || typeof value !== 'object') {
22
-
return false;
23
-
}
24
-
25
-
return '$bytes' in value && typeof value.$bytes === 'string';
26
-
};
27
-
28
-
/** commit object */
29
-
export interface Commit {
30
-
version: 3;
31
-
did: string;
32
-
data: CID.CidLink;
33
-
rev: string;
34
-
prev: CID.CidLink | null;
35
-
sig: CBOR.Bytes;
36
-
}
37
-
38
-
/**
39
-
* checks if a value is a valid commit object
40
-
* @param value the value to check
41
-
* @returns true if the value is a valid commit object, false otherwise
42
-
*/
43
-
export const isCommit = (value: unknown): value is Commit => {
44
-
if (value === null || typeof value !== 'object') {
45
-
return false;
46
-
}
47
-
48
-
const obj = value as Record<string, unknown>;
49
-
50
-
return (
51
-
obj.version === 3 &&
52
-
typeof obj.did === 'string' &&
53
-
isCidLink(obj.data) &&
54
-
typeof obj.rev === 'string' &&
55
-
(obj.prev === null || isCidLink(obj.prev)) &&
56
-
isBytes(obj.sig)
57
-
);
58
-
};
59
-
60
-
/** mst tree entry object */
61
-
export interface TreeEntry {
62
-
/** count of bytes shared with previous TreeEntry in this Node (if any) */
63
-
p: number;
64
-
/** remainder of key for this TreeEntry, after "prefixlen" have been removed */
65
-
k: CBOR.Bytes;
66
-
/** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */
67
-
v: CID.CidLink;
68
-
/** next subtree (to the right of leaf) */
69
-
t: CID.CidLink | null;
70
-
}
71
-
72
-
/**
73
-
* checks if a value is a valid mst tree entry object
74
-
* @param value the value to check
75
-
* @returns true if the value is a valid mst tree entry object, false otherwise
76
-
*/
77
-
export const isTreeEntry = (value: unknown): value is TreeEntry => {
78
-
if (value === null || typeof value !== 'object') {
79
-
return false;
80
-
}
81
-
82
-
const obj = value as Record<string, unknown>;
83
-
84
-
return (
85
-
typeof obj.p === 'number' && isBytes(obj.k) && isCidLink(obj.v) && (obj.t === null || isCidLink(obj.t))
86
-
);
87
-
};
88
-
89
-
/** mst node object */
90
-
export interface MstNode {
91
-
/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
92
-
l: CID.CidLink | null;
93
-
/** ordered list of TreeEntry objects */
94
-
e: TreeEntry[];
95
-
}
96
-
97
-
/**
98
-
* checks if a value is a valid mst node object
99
-
* @param value the value to check
100
-
* @returns true if the value is a valid mst node object, false otherwise
101
-
*/
102
-
export const isMstNode = (value: unknown): value is MstNode => {
103
-
if (value === null || typeof value !== 'object') {
104
-
return false;
105
-
}
106
-
107
-
const obj = value as Record<string, unknown>;
108
-
109
-
return (obj.l === null || isCidLink(obj.l)) && Array.isArray(obj.e) && obj.e.every(isTreeEntry);
110
-
};
+10
-11
packages/utilities/car/lib/repo-reader/stream-repo-reader.ts
packages/utilities/repo/lib/streamed-reader.ts
+10
-11
packages/utilities/car/lib/repo-reader/stream-repo-reader.ts
packages/utilities/repo/lib/streamed-reader.ts
···
1
-
import Queue from 'yocto-queue';
2
-
1
+
import type { CarEntry } from '@atcute/car';
2
+
import * as CAR from '@atcute/car';
3
3
import * as CBOR from '@atcute/cbor';
4
4
import * as CID from '@atcute/cid';
5
+
import { isNodeData } from '@atcute/mst';
5
6
import { decodeUtf8From } from '@atcute/uint8array';
6
7
7
-
import * as CarReader from '../car-reader/index.js';
8
-
import { assert } from '../utils.js';
9
-
10
-
import { isCommit, isMstNode } from './mst.js';
11
-
import { RepoEntry } from './types.js';
8
+
import { isCommit, RepoEntry } from './types.js';
9
+
import { assert } from './utils.js';
10
+
import Queue from './utils/queue.js';
12
11
13
12
type EntryMeta = { t: 0 } | { t: 1 } | { t: 2; k: string };
14
13
15
14
type Task = {
16
15
c: string;
17
-
e: CarReader.CarEntry;
16
+
e: CarEntry;
18
17
m: EntryMeta;
19
18
};
20
19
···
94
93
},
95
94
async *[Symbol.asyncIterator]() {
96
95
// await using car = CarReader.fromStream(stream);
97
-
const car = CarReader.fromStream(stream);
96
+
const car = CAR.fromStream(stream);
98
97
99
98
try {
100
99
const pending = new Map<string, EntryMeta>();
101
-
const strays = new Map<string, CarReader.CarEntry>();
100
+
const strays = new Map<string, CarEntry>();
102
101
103
102
const queue = new Queue<Task>();
104
103
···
149
148
}
150
149
case 1: {
151
150
const node = CBOR.decode(entry.bytes);
152
-
assert(isMstNode(node), `expected mst node block; cid=${cid}`);
151
+
assert(isNodeData(node), `expected mst node block; cid=${cid}`);
153
152
154
153
const entries = node.e;
155
154
const left = node.l;
-85
packages/utilities/car/lib/repo-reader/sync-blockmap.ts
-85
packages/utilities/car/lib/repo-reader/sync-blockmap.ts
···
1
-
import * as CBOR from '@atcute/cbor';
2
-
import * as CID from '@atcute/cid';
3
-
import { decodeUtf8From } from '@atcute/uint8array';
4
-
5
-
import * as CarReader from '../car-reader/index.js';
6
-
import { assert } from '../utils.js';
7
-
8
-
import { isMstNode } from './mst.js';
9
-
10
-
export type BlockMap = Map<string, CarReader.CarEntry>;
11
-
12
-
/**
13
-
* collects entries from a CAR archive into a mapping of CID string -> actual bytes
14
-
* @param iterator a generator that yields objects with a `cid` and `bytes` property
15
-
* @returns a mapping of CID string -> actual bytes
16
-
*/
17
-
export const collectBlock = (iterator: Iterable<CarReader.CarEntry>): BlockMap => {
18
-
const blockmap: BlockMap = new Map();
19
-
for (const entry of iterator) {
20
-
blockmap.set(CID.toString(entry.cid), entry);
21
-
}
22
-
23
-
return blockmap;
24
-
};
25
-
26
-
/**
27
-
* reads a block from the blockmap and validates it against the provided validation function
28
-
* @param map a mapping of CID string -> actual bytes
29
-
* @param link a CID link to read
30
-
* @param validate a validation function to validate the decoded data
31
-
* @returns the decoded and validated data
32
-
*/
33
-
export const readBlock = <T>(
34
-
map: BlockMap,
35
-
link: CID.CidLink,
36
-
validate: (value: unknown) => value is T,
37
-
): T => {
38
-
const cid = link.$link;
39
-
40
-
const entry = map.get(cid);
41
-
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
42
-
43
-
const data = CBOR.decode(entry.bytes);
44
-
assert(validate(data), `validation failed for cid=${cid}`);
45
-
46
-
return data;
47
-
};
48
-
49
-
/** node entry object */
50
-
export interface NodeEntry {
51
-
key: string;
52
-
cid: CID.CidLink;
53
-
}
54
-
55
-
/**
56
-
* walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
57
-
* @param map a mapping of CID string -> actual bytes
58
-
* @param pointer a CID link to the root of the MST
59
-
* @returns a generator that yields the entries of the MST
60
-
*/
61
-
export function* walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry> {
62
-
const data = readBlock(map, pointer, isMstNode);
63
-
const entries = data.e;
64
-
65
-
let lastKey = '';
66
-
67
-
if (data.l !== null) {
68
-
yield* walkMstEntries(map, data.l);
69
-
}
70
-
71
-
for (let i = 0, il = entries.length; i < il; i++) {
72
-
const entry = entries[i];
73
-
74
-
const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
75
-
const key = lastKey.slice(0, entry.p) + key_str;
76
-
77
-
lastKey = key;
78
-
79
-
yield { key: key, cid: entry.v };
80
-
81
-
if (entry.t !== null) {
82
-
yield* walkMstEntries(map, entry.t);
83
-
}
84
-
}
85
-
}
-27
packages/utilities/car/lib/repo-reader/sync-repo-reader.ts
-27
packages/utilities/car/lib/repo-reader/sync-repo-reader.ts
···
1
-
import * as CarReader from '../car-reader/index.js';
2
-
import { assert } from '../utils.js';
3
-
4
-
import { isCommit } from './mst.js';
5
-
import { collectBlock, readBlock, walkMstEntries } from './sync-blockmap.js';
6
-
import { RepoEntry } from './types.js';
7
-
8
-
export function* fromUint8Array(buf: Uint8Array): Generator<RepoEntry> {
9
-
const car = CarReader.fromUint8Array(buf);
10
-
const roots = car.roots;
11
-
12
-
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
13
-
14
-
const blockmap = collectBlock(car);
15
-
assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
16
-
17
-
const commit = readBlock(blockmap, roots[0], isCommit);
18
-
19
-
for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
20
-
const [collection, rkey] = key.split('/');
21
-
22
-
const carEntry = blockmap.get(cid.$link);
23
-
assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
24
-
25
-
yield new RepoEntry(collection, rkey, cid, carEntry);
26
-
}
27
-
}
-32
packages/utilities/car/lib/repo-reader/types.ts
-32
packages/utilities/car/lib/repo-reader/types.ts
···
1
-
import * as CBOR from '@atcute/cbor';
2
-
import * as CID from '@atcute/cid';
3
-
4
-
import { type CarEntry } from '../car-reader/index.js';
5
-
6
-
export class RepoEntry {
7
-
/** @internal */
8
-
constructor(
9
-
/** the collection this record belongs to */
10
-
public readonly collection: string,
11
-
/** record key */
12
-
public readonly rkey: string,
13
-
/** CID of this record */
14
-
public readonly cid: CID.CidLink,
15
-
/** the associated CarEntry for this record */
16
-
public readonly carEntry: CarEntry,
17
-
) {}
18
-
19
-
/**
20
-
* raw contents of this record
21
-
*/
22
-
get bytes(): Uint8Array {
23
-
return this.carEntry.bytes;
24
-
}
25
-
26
-
/**
27
-
* decoded contents of this record
28
-
*/
29
-
get record(): unknown {
30
-
return CBOR.decode(this.bytes);
31
-
}
32
-
}
+7
-12
packages/utilities/car/package.json
+7
-12
packages/utilities/car/package.json
···
1
1
{
2
2
"type": "module",
3
3
"name": "@atcute/car",
4
-
"version": "3.1.3",
5
-
"description": "lightweight DASL CAR and atproto repository decoder for AT Protocol.",
4
+
"version": "4.0.0",
5
+
"description": "lightweight DASL CAR (content-addressable archives) codec for AT Protocol.",
6
6
"keywords": [
7
7
"atproto",
8
8
"dasl",
···
20
20
"!lib/**/*.test.ts"
21
21
],
22
22
"exports": {
23
-
".": "./dist/index.js",
24
-
"./car-reader": "./dist/car-reader/index.js",
25
-
"./repo-reader": "./dist/repo-reader/index.js",
26
-
"./v4": "./dist/index.js",
27
-
"./v4/car-reader": "./dist/car-reader/index.js",
28
-
"./v4/repo-reader": "./dist/repo-reader/index.js"
23
+
".": "./dist/index.js"
29
24
},
30
25
"sideEffects": false,
31
26
"scripts": {
32
27
"build": "tsc --project tsconfig.build.json",
33
-
"test": "bun test --coverage",
28
+
"test": "vitest --coverage",
34
29
"prepublish": "rm -rf dist; pnpm run build"
35
30
},
36
31
"devDependencies": {
37
32
"@atcute/multibase": "workspace:^",
38
-
"@types/bun": "^1.2.21"
33
+
"@vitest/coverage-v8": "^3.2.4",
34
+
"vitest": "^3.2.4"
39
35
},
40
36
"dependencies": {
41
37
"@atcute/cbor": "workspace:^",
42
38
"@atcute/cid": "workspace:^",
43
39
"@atcute/uint8array": "workspace:^",
44
-
"@atcute/varint": "workspace:^",
45
-
"yocto-queue": "^1.2.1"
40
+
"@atcute/varint": "workspace:^"
46
41
}
47
42
}
+1
-1
packages/utilities/car/tsconfig.json
+1
-1
packages/utilities/car/tsconfig.json
+2
packages/utilities/mst/.gitignore
+2
packages/utilities/mst/.gitignore
-39
packages/utilities/mst/lib/blockmap.ts
-39
packages/utilities/mst/lib/blockmap.ts
···
1
-
import * as CBOR from '@atcute/cbor';
2
-
import * as CID from '@atcute/cid';
3
-
4
-
type BlockEntry = [cid: string, bytes: Uint8Array<ArrayBuffer>];
5
-
6
1
/** a map from CID strings to their encoded block data */
7
2
export type BlockMap = Map<string, Uint8Array<ArrayBuffer>>;
8
-
9
-
/**
10
-
* encodes data as CBOR, computes its CID, and adds it to the map
11
-
* @param map the block map to add to
12
-
* @param data the data to encode and add
13
-
*/
14
-
export const add = async (map: BlockMap, data: unknown): Promise<void> => {
15
-
const encoded = CBOR.encode(data);
16
-
const cid = await CID.create(0x71, encoded);
17
-
18
-
map.set(CID.toString(cid), encoded);
19
-
};
20
-
21
-
/**
22
-
* copies multiple blocks from an iterable into the map
23
-
* @param map the block map to add to
24
-
* @param entries the block entries to add
25
-
*/
26
-
export const setMany = (map: BlockMap, entries: Iterable<Readonly<BlockEntry>>) => {
27
-
for (const [cid, bytes] of entries) {
28
-
map.set(cid, bytes);
29
-
}
30
-
};
31
-
32
-
/**
33
-
* removes multiple blocks from the map by their CIDs
34
-
* @param map the block map to remove from
35
-
* @param cids the CID strings to remove
36
-
*/
37
-
export const deleteMany = (map: BlockMap, cids: Iterable<string>) => {
38
-
for (const cid of cids) {
39
-
map.delete(cid);
40
-
}
41
-
};
+7
-6
packages/utilities/mst/lib/car-writer.test.ts
packages/utilities/car/lib/writer.test.ts
+7
-6
packages/utilities/mst/lib/car-writer.test.ts
packages/utilities/car/lib/writer.test.ts
···
3
3
import * as CID from '@atcute/cid';
4
4
import { concat, encodeUtf8 } from '@atcute/uint8array';
5
5
6
-
import { type CarBlock, createCarStream, serializeCarEntry, serializeCarHeader } from './car-writer.js';
6
+
import type { CarBlock } from './types.js';
7
+
import { serializeCarEntry, serializeCarHeader, writeCarStream } from './writer.js';
7
8
8
9
describe('serializeCarHeader', () => {
9
10
it('should serialize a header with one root', async () => {
···
42
43
});
43
44
});
44
45
45
-
describe('createCarStream', () => {
46
+
describe('writeCarStream', () => {
46
47
it('should stream a CAR with a single block', async () => {
47
48
const rootCid = CID.toCidLink(await CID.create(0x55, encodeUtf8('root')));
48
49
const blockCid = await CID.create(0x55, encodeUtf8('block1'));
···
53
54
};
54
55
55
56
const chunks: Uint8Array[] = [];
56
-
for await (const chunk of createCarStream(rootCid, blocks())) {
57
+
for await (const chunk of writeCarStream([rootCid], blocks())) {
57
58
chunks.push(chunk);
58
59
}
59
60
···
75
76
}
76
77
};
77
78
78
-
const chunks = await Array.fromAsync(createCarStream(rootCid, blocks()));
79
+
const chunks = await Array.fromAsync(writeCarStream([rootCid], blocks()));
79
80
const car = concat(chunks);
80
81
81
82
expect(chunks).toHaveLength(6); // header + 5 blocks
···
90
91
const blocks: CarBlock[] = [{ cid: blockCid.bytes, data: blockData }];
91
92
92
93
const chunks1: Uint8Array[] = [];
93
-
for await (const chunk of createCarStream(rootCid, blocks)) {
94
+
for await (const chunk of writeCarStream([rootCid], blocks)) {
94
95
chunks1.push(chunk);
95
96
}
96
97
97
98
const chunks2: Uint8Array[] = [];
98
-
for await (const chunk of createCarStream(rootCid, blocks)) {
99
+
for await (const chunk of writeCarStream([rootCid], blocks)) {
99
100
chunks2.push(chunk);
100
101
}
101
102
+14
-20
packages/utilities/mst/lib/car-writer.ts
packages/utilities/car/lib/writer.ts
+14
-20
packages/utilities/mst/lib/car-writer.ts
packages/utilities/car/lib/writer.ts
···
3
3
import { allocUnsafe } from '@atcute/uint8array';
4
4
import * as varint from '@atcute/varint';
5
5
6
+
import type { CarBlock } from './types.js';
7
+
6
8
/**
7
-
* Encodes a number as an unsigned varint (variable-length integer)
9
+
* encodes a number as an unsigned varint (variable-length integer)
8
10
* @param n the number to encode
9
11
* @returns the varint-encoded bytes
10
12
*/
···
16
18
};
17
19
18
20
/**
19
-
* Serializes a CAR v1 header
21
+
* serializes a CAR v1 header
22
+
* @internal
20
23
* @param roots array of root CIDs (typically just one)
21
24
* @returns the serialized header bytes
22
25
*/
···
36
39
};
37
40
38
41
/**
39
-
* Serializes a single CAR entry (block)
42
+
* serializes a single CAR entry (block)
43
+
* @internal
40
44
* @param cid the CID of the block (as bytes)
41
45
* @param data the block data
42
46
* @returns the serialized entry bytes
···
53
57
};
54
58
55
59
/**
56
-
* Represents a block to be written to a CAR file
57
-
*/
58
-
export interface CarBlock {
59
-
/** the CID of the block (as bytes) */
60
-
cid: Uint8Array;
61
-
/** the block data */
62
-
data: Uint8Array;
63
-
}
64
-
65
-
/**
66
-
* Creates an async generator that yields CAR file chunks
67
-
* @param root the root CID for the CAR file
60
+
* creates an async generator that yields CAR file chunks
61
+
* @param root root CIDs for the CAR file
68
62
* @param blocks async iterable of blocks to write
69
63
* @yields Uint8Array chunks of the CAR file (header, then entries)
70
64
*
···
76
70
* };
77
71
*
78
72
* // Stream chunks
79
-
* for await (const chunk of createCarStream(rootCid, blocks())) {
73
+
* for await (const chunk of writeCarStream([rootCid], blocks())) {
80
74
* stream.write(chunk);
81
75
* }
82
76
*
83
77
* // Or collect into array (requires Array.fromAsync or polyfill)
84
-
* const chunks = await Array.fromAsync(createCarStream(rootCid, blocks()));
78
+
* const chunks = await Array.fromAsync(writeCarStream([rootCid], blocks()));
85
79
* ```
86
80
*/
87
-
export async function* createCarStream(
88
-
root: CidLink,
81
+
export async function* writeCarStream(
82
+
roots: CidLink[],
89
83
blocks: AsyncIterable<CarBlock> | Iterable<CarBlock>,
90
84
): AsyncGenerator<Uint8Array<ArrayBuffer>> {
91
85
// Emit header first
92
-
yield serializeCarHeader([root]);
86
+
yield serializeCarHeader(roots);
93
87
94
88
// Then emit each block entry
95
89
for await (const block of blocks) {
+6
-5
packages/utilities/mst/lib/diff.ts
+6
-5
packages/utilities/mst/lib/diff.ts
···
1
1
import type { CidLink } from '@atcute/cid';
2
2
3
-
import { MSTNode } from './node.js';
4
3
import type { NodeStore } from './node-store.js';
5
4
import { NodeWalker } from './node-walker.js';
5
+
import { MSTNode } from './node.js';
6
6
7
7
/**
8
8
* Type of change to a record
···
164
164
* @param rootB CID of second MST root
165
165
* @returns tuple of [created nodes, deleted nodes]
166
166
*/
167
-
export const mstDiff = async (ns: NodeStore, rootA: string, rootB: string): Promise<[Set<string>, Set<string>]> => {
167
+
export const mstDiff = async (
168
+
ns: NodeStore,
169
+
rootA: string,
170
+
rootB: string,
171
+
): Promise<[Set<string>, Set<string>]> => {
168
172
const created = new Set<string>(); // nodes in B but not in A
169
173
const deleted = new Set<string>(); // nodes in A but not in B
170
174
···
265
269
}
266
270
267
271
// The rpaths now match, but the subtrees below us might not
268
-
const aSubtree = a.subtree;
269
-
const bSubtree = b.subtree;
270
-
271
272
// Recursively diff the subtrees
272
273
const aSubWalker = await a.createSubtreeWalker();
273
274
const bSubWalker = await b.createSubtreeWalker();
+10
packages/utilities/mst/lib/index.ts
+10
packages/utilities/mst/lib/index.ts
···
1
+
export * from './types.js';
2
+
export * from './node.js';
3
+
export * from './node-store.js';
4
+
export * from './node-wrangler.js';
5
+
export * from './node-walker.js';
6
+
export * from './diff.js';
7
+
export * from './proof.js';
8
+
export * from './errors.js';
9
+
export * from './blockmap.js';
10
+
export * from './stores.js';
-1
packages/utilities/mst/lib/node-store.ts
-1
packages/utilities/mst/lib/node-store.ts
+1
-1
packages/utilities/mst/lib/node-wrangler.ts
+1
-1
packages/utilities/mst/lib/node-wrangler.ts
+2
-1
packages/utilities/mst/lib/stores.ts
+2
-1
packages/utilities/mst/lib/stores.ts
+3
-3
packages/utilities/mst/lib/test-suite.test.ts
+3
-3
packages/utilities/mst/lib/test-suite.test.ts
···
4
4
import * as fs from 'node:fs/promises';
5
5
import * as path from 'node:path';
6
6
7
-
import { CarReader } from '@atcute/car';
7
+
import * as CAR from '@atcute/car';
8
8
import * as CID from '@atcute/cid';
9
9
10
-
import { setMany } from './blockmap.js';
11
10
import { DeltaType, mstDiff, recordDiff } from './diff.js';
12
11
import { NodeStore } from './node-store.js';
13
12
import { NodeWrangler } from './node-wrangler.js';
···
18
17
OverlayBlockStore,
19
18
ReadonlyMemoryBlockStore,
20
19
} from './stores.js';
20
+
import { setMany } from './utils/blockmap.js';
21
21
22
22
const mstDiffTestCaseSchema = v.object({
23
23
$type: v.literal('mst-diff'),
···
52
52
const filename = path.join(testSuiteRoot, relname);
53
53
const bytes = await fs.readFile(filename);
54
54
55
-
const car = CarReader.fromUint8Array(bytes);
55
+
const car = CAR.fromUint8Array(bytes);
56
56
const store = new MemoryBlockStore();
57
57
58
58
for (const entry of car) {
+40
packages/utilities/mst/lib/utils/blockmap.ts
+40
packages/utilities/mst/lib/utils/blockmap.ts
···
1
+
import * as CBOR from '@atcute/cbor';
2
+
import * as CID from '@atcute/cid';
3
+
4
+
import type { BlockMap } from '../blockmap.js';
5
+
6
+
type BlockEntry = [cid: string, bytes: Uint8Array<ArrayBuffer>];
7
+
8
+
/**
9
+
* encodes data as CBOR, computes its CID, and adds it to the map
10
+
* @param map the block map to add to
11
+
* @param data the data to encode and add
12
+
*/
13
+
export const add = async (map: BlockMap, data: unknown): Promise<void> => {
14
+
const encoded = CBOR.encode(data);
15
+
const cid = await CID.create(0x71, encoded);
16
+
17
+
map.set(CID.toString(cid), encoded);
18
+
};
19
+
20
+
/**
21
+
* copies multiple blocks from an iterable into the map
22
+
* @param map the block map to add to
23
+
* @param entries the block entries to add
24
+
*/
25
+
export const setMany = (map: BlockMap, entries: Iterable<Readonly<BlockEntry>>) => {
26
+
for (const [cid, bytes] of entries) {
27
+
map.set(cid, bytes);
28
+
}
29
+
};
30
+
31
+
/**
32
+
* removes multiple blocks from the map by their CIDs
33
+
* @param map the block map to remove from
34
+
* @param cids the CID strings to remove
35
+
*/
36
+
export const deleteMany = (map: BlockMap, cids: Iterable<string>) => {
37
+
for (const cid of cids) {
38
+
map.delete(cid);
39
+
}
40
+
};
+1
-2
packages/utilities/mst/package.json
+1
-2
packages/utilities/mst/package.json
+1
packages/utilities/repo/.gitignore
+1
packages/utilities/repo/.gitignore
···
1
+
/coverage/
+58
packages/utilities/repo/README.md
+58
packages/utilities/repo/README.md
···
1
+
# @atcute/repo
2
+
3
+
read AT Protocol repository exports
4
+
5
+
## usage
6
+
7
+
### streaming usage
8
+
9
+
```ts
10
+
import { fromStream } from '@atcute/repo';
11
+
12
+
const stream = new ReadableStream({
13
+
/* ... */
14
+
});
15
+
16
+
await using repo = fromStream(stream);
17
+
18
+
for await (const entry of repo) {
19
+
entry;
20
+
// ^? RepoEntry { collection: 'app.bsky.feed.post', rkey: '3lprcc55bb222', ... }
21
+
}
22
+
23
+
repo.missingBlocks;
24
+
// ^? []
25
+
```
26
+
27
+
### streaming usage (for runtimes without `await using` yet)
28
+
29
+
```ts
30
+
const repo = fromStream(stream);
31
+
32
+
try {
33
+
for await (const entry of repo) {
34
+
entry;
35
+
// ^? RepoEntry
36
+
}
37
+
} finally {
38
+
await repo.dispose();
39
+
}
40
+
```
41
+
42
+
### sync usage
43
+
44
+
```ts
45
+
const buffer = Uint8Array.from([
46
+
/* ... */
47
+
]);
48
+
49
+
const repo = fromUint8Array(buffer);
50
+
51
+
for (const entry of repo) {
52
+
entry;
53
+
// ^? RepoEntry { collection: 'app.bsky.feed.post', rkey: '3lprcc55bb222', ... }
54
+
}
55
+
56
+
repo.missingBlocks;
57
+
// ^? []
58
+
```
+5
packages/utilities/repo/lib/index.ts
+5
packages/utilities/repo/lib/index.ts
+98
packages/utilities/repo/lib/reader.ts
+98
packages/utilities/repo/lib/reader.ts
···
1
+
import type { CarEntry } from '@atcute/car';
2
+
import * as CAR from '@atcute/car';
3
+
import * as CBOR from '@atcute/cbor';
4
+
import type { CidLink } from '@atcute/cid';
5
+
import * as CID from '@atcute/cid';
6
+
import { isNodeData } from '@atcute/mst';
7
+
import { decodeUtf8From } from '@atcute/uint8array';
8
+
9
+
import { isCommit, RepoEntry } from './types.js';
10
+
import { assert } from './utils.js';
11
+
12
+
/** @internal */
13
+
type EntryMap = Map<string, CarEntry>;
14
+
15
+
/** node entry object */
16
+
interface NodeEntry {
17
+
key: string;
18
+
cid: CidLink;
19
+
}
20
+
21
+
export function* fromUint8Array(buf: Uint8Array): Generator<RepoEntry> {
22
+
const car = CAR.fromUint8Array(buf);
23
+
const roots = car.roots;
24
+
25
+
assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
26
+
27
+
const map: EntryMap = new Map();
28
+
for (const entry of car) {
29
+
map.set(CID.toString(entry.cid), entry);
30
+
}
31
+
32
+
// [commit, mst node, record?]
33
+
assert(map.size >= 2, `expected at least 2 blocks in the archive; got=${map.size}`);
34
+
35
+
const commit = readEntry(map, roots[0], isCommit);
36
+
37
+
for (const { key, cid } of walkMstEntries(map, commit.data)) {
38
+
const [collection, rkey] = key.split('/');
39
+
40
+
const carEntry = map.get(cid.$link);
41
+
assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
42
+
43
+
yield new RepoEntry(collection, rkey, cid, carEntry);
44
+
}
45
+
}
46
+
47
+
/**
48
+
* reads a block from the blockmap and validates it against the provided validation function
49
+
* @internal
50
+
* @param map a mapping of CID string -> actual bytes
51
+
* @param link a CID link to read
52
+
* @param validate a validation function to validate the decoded data
53
+
* @returns the decoded and validated data
54
+
*/
55
+
export const readEntry = <T>(map: EntryMap, link: CidLink, validate: (value: unknown) => value is T): T => {
56
+
const cid = link.$link;
57
+
58
+
const entry = map.get(cid);
59
+
assert(entry != null, `cid not found in blockmap; cid=${cid}`);
60
+
61
+
const data = CBOR.decode(entry.bytes);
62
+
assert(validate(data), `validation failed for cid=${cid}`);
63
+
64
+
return data;
65
+
};
66
+
67
+
/**
68
+
* walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
69
+
* @internal
70
+
* @param map a mapping of CID string -> actual bytes
71
+
* @param pointer a CID link to the root of the MST
72
+
* @returns a generator that yields the entries of the MST
73
+
*/
74
+
export function* walkMstEntries(map: EntryMap, pointer: CidLink): Generator<NodeEntry> {
75
+
const data = readEntry(map, pointer, isNodeData);
76
+
const entries = data.e;
77
+
78
+
let lastKey = '';
79
+
80
+
if (data.l !== null) {
81
+
yield* walkMstEntries(map, data.l);
82
+
}
83
+
84
+
for (let i = 0, il = entries.length; i < il; i++) {
85
+
const entry = entries[i];
86
+
87
+
const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
88
+
const key = lastKey.slice(0, entry.p) + key_str;
89
+
90
+
lastKey = key;
91
+
92
+
yield { key: key, cid: entry.v };
93
+
94
+
if (entry.t !== null) {
95
+
yield* walkMstEntries(map, entry.t);
96
+
}
97
+
}
98
+
}
+64
packages/utilities/repo/lib/types.ts
+64
packages/utilities/repo/lib/types.ts
···
1
+
import type { CarEntry } from '@atcute/car';
2
+
import * as CBOR from '@atcute/cbor';
3
+
import { isBytes, type Bytes } from '@atcute/cbor';
4
+
import { isCidLink, type CidLink } from '@atcute/cid';
5
+
6
+
export class RepoEntry {
7
+
/** @internal */
8
+
constructor(
9
+
/** the collection this record belongs to */
10
+
public readonly collection: string,
11
+
/** record key */
12
+
public readonly rkey: string,
13
+
/** CID of this record */
14
+
public readonly cid: CidLink,
15
+
/** the associated CarEntry for this record */
16
+
public readonly carEntry: CarEntry,
17
+
) {}
18
+
19
+
/**
20
+
* raw contents of this record
21
+
*/
22
+
get bytes(): Uint8Array {
23
+
return this.carEntry.bytes;
24
+
}
25
+
26
+
/**
27
+
* decoded contents of this record
28
+
*/
29
+
get record(): unknown {
30
+
return CBOR.decode(this.bytes);
31
+
}
32
+
}
33
+
34
+
/** commit object */
35
+
export interface Commit {
36
+
version: 3;
37
+
did: string;
38
+
data: CidLink;
39
+
rev: string;
40
+
prev: CidLink | null;
41
+
sig: Bytes;
42
+
}
43
+
44
+
/**
45
+
* checks if value is a valid commit object
46
+
* @param value value to check
47
+
* @returns true if the value is a valid commit object, false otherwise
48
+
*/
49
+
export const isCommit = (value: unknown): value is Commit => {
50
+
if (value === null || typeof value !== 'object') {
51
+
return false;
52
+
}
53
+
54
+
const obj = value as Record<string, unknown>;
55
+
56
+
return (
57
+
obj.version === 3 &&
58
+
typeof obj.did === 'string' &&
59
+
isCidLink(obj.data) &&
60
+
typeof obj.rev === 'string' &&
61
+
(obj.prev === null || isCidLink(obj.prev)) &&
62
+
isBytes(obj.sig)
63
+
);
64
+
};
+7
packages/utilities/repo/lib/utils.ts
+7
packages/utilities/repo/lib/utils.ts
+148
packages/utilities/repo/lib/utils/queue.ts
+148
packages/utilities/repo/lib/utils/queue.ts
···
1
+
interface Node<T> {
2
+
value: T;
3
+
next: Node<T> | undefined;
4
+
}
5
+
6
+
const createNode = <T>(value: T, next: Node<T> | undefined): Node<T> => {
7
+
return { value, next };
8
+
};
9
+
10
+
/** a queue data structure (fifo) */
11
+
class Queue<T> implements Iterable<T> {
12
+
#head: Node<T> | undefined;
13
+
#tail: Node<T> | undefined;
14
+
#size: number = 0;
15
+
16
+
/** size of the queue */
17
+
get size(): number {
18
+
return this.#size;
19
+
}
20
+
21
+
/**
22
+
* clear the queue
23
+
*/
24
+
clear(): void {
25
+
this.#head = undefined;
26
+
this.#tail = undefined;
27
+
this.#size = 0;
28
+
}
29
+
30
+
/**
31
+
* adds a value to the end of the queue
32
+
* @param value value to add
33
+
* @returns the queue instance
34
+
*/
35
+
enqueue(value: T): this {
36
+
const tail = this.#tail;
37
+
const node = createNode(value, undefined);
38
+
39
+
if (tail !== undefined) {
40
+
tail.next = node;
41
+
} else {
42
+
this.#head = node;
43
+
}
44
+
45
+
this.#tail = node;
46
+
this.#size++;
47
+
return this;
48
+
}
49
+
50
+
/**
51
+
* adds a value to the front of the queue
52
+
* @param value value to add
53
+
* @returns the queue instance
54
+
*/
55
+
enqueueFront(value: T): this {
56
+
const head = this.#head;
57
+
const node = createNode(value, head);
58
+
59
+
if (head === undefined) {
60
+
this.#tail = node;
61
+
}
62
+
63
+
this.#head = node;
64
+
this.#size++;
65
+
return this;
66
+
}
67
+
68
+
/**
69
+
* removes the first value from the queue
70
+
* @returns first queued value, or undefined if empty
71
+
*/
72
+
dequeue(): T | undefined {
73
+
const head = this.#head;
74
+
if (!head) {
75
+
return;
76
+
}
77
+
78
+
const next = head.next;
79
+
80
+
this.#head = next;
81
+
if (next === undefined) {
82
+
this.#tail = undefined;
83
+
}
84
+
85
+
this.#size--;
86
+
return head.value;
87
+
}
88
+
89
+
/**
90
+
* get the first value without removing from queue
91
+
* @returns first queued value, or undefined if empty
92
+
*/
93
+
peek(): T | undefined {
94
+
return this.#head?.value;
95
+
}
96
+
97
+
/**
98
+
* returns an iterator that drains all values from the queue
99
+
*/
100
+
drain(): IterableIterator<T, undefined, undefined> {
101
+
// deno-lint-ignore no-this-alias
102
+
const self = this;
103
+
104
+
return {
105
+
next() {
106
+
const head = self.#head;
107
+
if (!head) {
108
+
return { done: true, value: undefined };
109
+
}
110
+
111
+
const next = head.next;
112
+
113
+
self.#head = next;
114
+
if (next === undefined) {
115
+
self.#tail = undefined;
116
+
}
117
+
118
+
self.#size--;
119
+
return { done: false, value: head.value };
120
+
},
121
+
[Symbol.iterator]() {
122
+
return this;
123
+
},
124
+
};
125
+
}
126
+
127
+
/**
128
+
* iterates over the queue without draining
129
+
*/
130
+
[Symbol.iterator](): Iterator<T, undefined, undefined> {
131
+
let current = this.#head;
132
+
133
+
return {
134
+
next() {
135
+
if (current === undefined) {
136
+
return { done: true, value: undefined };
137
+
}
138
+
139
+
const value = current.value;
140
+
current = current.next;
141
+
142
+
return { done: false, value: value };
143
+
},
144
+
};
145
+
}
146
+
}
147
+
148
+
export default Queue;
+44
packages/utilities/repo/package.json
+44
packages/utilities/repo/package.json
···
1
+
{
2
+
"type": "module",
3
+
"name": "@atcute/repo",
4
+
"version": "0.1.0",
5
+
"description": "AT Protocol repository decoder for AT Protocol.",
6
+
"keywords": [
7
+
"atproto",
8
+
"repo"
9
+
],
10
+
"license": "0BSD",
11
+
"repository": {
12
+
"url": "https://github.com/mary-ext/atcute",
13
+
"directory": "packages/utilities/repo"
14
+
},
15
+
"files": [
16
+
"dist/",
17
+
"lib/",
18
+
"!lib/**/*.bench.ts",
19
+
"!lib/**/*.test.ts"
20
+
],
21
+
"exports": {
22
+
".": "./dist/index.js"
23
+
},
24
+
"sideEffects": false,
25
+
"scripts": {
26
+
"build": "tsc --project tsconfig.build.json",
27
+
"test": "vitest run --coverage",
28
+
"prepublish": "rm -rf dist; pnpm run build"
29
+
},
30
+
"devDependencies": {
31
+
"@atcute/multibase": "workspace:^",
32
+
"@vitest/coverage-v8": "^3.2.4",
33
+
"vitest": "^3.2.4"
34
+
},
35
+
"dependencies": {
36
+
"@atcute/car": "workspace:^",
37
+
"@atcute/cbor": "workspace:^",
38
+
"@atcute/cid": "workspace:^",
39
+
"@atcute/crypto": "workspace:^",
40
+
"@atcute/lexicons": "workspace:^",
41
+
"@atcute/mst": "workspace:^",
42
+
"@atcute/uint8array": "workspace:^"
43
+
}
44
+
}
+4
packages/utilities/repo/tsconfig.build.json
+4
packages/utilities/repo/tsconfig.build.json
+25
packages/utilities/repo/tsconfig.json
+25
packages/utilities/repo/tsconfig.json
···
1
+
{
2
+
"compilerOptions": {
3
+
"types": [],
4
+
"outDir": "dist/",
5
+
"esModuleInterop": true,
6
+
"skipLibCheck": true,
7
+
"target": "ESNext",
8
+
"allowJs": true,
9
+
"resolveJsonModule": true,
10
+
"moduleDetection": "force",
11
+
"isolatedModules": true,
12
+
"verbatimModuleSyntax": true,
13
+
"strict": true,
14
+
"noImplicitOverride": true,
15
+
"noUnusedLocals": true,
16
+
"noUnusedParameters": true,
17
+
"noFallthroughCasesInSwitch": true,
18
+
"module": "NodeNext",
19
+
"sourceMap": true,
20
+
"declaration": true,
21
+
"declarationMap": true,
22
+
"stripInternal": true
23
+
},
24
+
"include": ["lib"]
25
+
}
+43
-9
pnpm-lock.yaml
+43
-9
pnpm-lock.yaml
···
522
522
'@atcute/lexicons':
523
523
specifier: workspace:^
524
524
version: link:../lexicons
525
+
'@atcute/repo':
526
+
specifier: workspace:^
527
+
version: link:../../utilities/repo
525
528
'@atcute/uint8array':
526
529
specifier: workspace:^
527
530
version: link:../../utilities/uint8array
···
656
659
'@atcute/varint':
657
660
specifier: workspace:^
658
661
version: link:../varint
659
-
yocto-queue:
660
-
specifier: ^1.2.1
661
-
version: 1.2.1
662
662
devDependencies:
663
663
'@atcute/multibase':
664
664
specifier: workspace:^
665
665
version: link:../multibase
666
-
'@types/bun':
667
-
specifier: ^1.2.21
668
-
version: 1.2.21(@types/react@19.1.8)
666
+
'@vitest/coverage-v8':
667
+
specifier: ^3.2.4
668
+
version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4)
669
+
vitest:
670
+
specifier: ^3.2.4
671
+
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(tsx@4.20.6)(yaml@2.8.0)
669
672
670
673
packages/utilities/cbor:
671
674
dependencies:
···
744
747
'@atcute/uint8array':
745
748
specifier: workspace:^
746
749
version: link:../uint8array
747
-
'@atcute/varint':
748
-
specifier: workspace:^
749
-
version: link:../varint
750
750
devDependencies:
751
751
'@atcute/car':
752
752
specifier: workspace:^
···
770
770
'@types/bun':
771
771
specifier: ^1.2.21
772
772
version: 1.2.21(@types/react@19.1.8)
773
+
774
+
packages/utilities/repo:
775
+
dependencies:
776
+
'@atcute/car':
777
+
specifier: workspace:^
778
+
version: link:../car
779
+
'@atcute/cbor':
780
+
specifier: workspace:^
781
+
version: link:../cbor
782
+
'@atcute/cid':
783
+
specifier: workspace:^
784
+
version: link:../cid
785
+
'@atcute/crypto':
786
+
specifier: workspace:^
787
+
version: link:../crypto
788
+
'@atcute/lexicons':
789
+
specifier: workspace:^
790
+
version: link:../../lexicons/lexicons
791
+
'@atcute/mst':
792
+
specifier: workspace:^
793
+
version: link:../mst
794
+
'@atcute/uint8array':
795
+
specifier: workspace:^
796
+
version: link:../uint8array
797
+
devDependencies:
798
+
'@atcute/multibase':
799
+
specifier: workspace:^
800
+
version: link:../multibase
801
+
'@vitest/coverage-v8':
802
+
specifier: ^3.2.4
803
+
version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4)
804
+
vitest:
805
+
specifier: ^3.2.4
806
+
version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(tsx@4.20.6)(yaml@2.8.0)
773
807
774
808
packages/utilities/tid:
775
809
devDependencies: