Suite of AT Protocol TypeScript libraries built on web standards
feat: @atp/repo lexification
+609
-340
+1
-2
deno.lock
+1
-2
deno.lock
+31
-23
repo/block-map.ts
+31
-23
repo/block-map.ts
···
1
-
import { CID } from "multiformats/cid";
1
+
import { type Cid, parseCid } from "@atp/lex/data";
2
2
import { equals } from "@atp/bytes";
3
-
import { dataToCborBlock } from "@atp/common";
4
-
import { lexToIpld, type LexValue } from "@atp/lexicon";
3
+
import {
4
+
cidForCbor,
5
+
encode as encodeLexCbor,
6
+
type LexValue as EncodableLexValue,
7
+
} from "@atp/lex/cbor";
8
+
import type { RepoInputValue } from "./types.ts";
9
+
import { lexToCborValue } from "./util.ts";
5
10
6
-
export class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
11
+
export class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> {
7
12
private map: Map<string, Uint8Array> = new Map();
8
13
9
-
constructor(entries?: Iterable<readonly [cid: CID, bytes: Uint8Array]>) {
14
+
constructor(entries?: Iterable<readonly [cid: Cid, bytes: Uint8Array]>) {
10
15
if (entries) {
11
16
for (const [cid, bytes] of entries) {
12
17
this.set(cid, bytes);
···
14
19
}
15
20
}
16
21
17
-
async add(value: LexValue): Promise<CID> {
18
-
const block = await dataToCborBlock(lexToIpld(value));
19
-
this.set(block.cid, block.bytes);
20
-
return block.cid;
22
+
async add(value: RepoInputValue): Promise<Cid> {
23
+
const bytes = encodeLexCbor(
24
+
lexToCborValue(value) as EncodableLexValue,
25
+
);
26
+
const cid = await cidForCbor(bytes);
27
+
this.set(cid, bytes);
28
+
return cid;
21
29
}
22
30
23
-
set(cid: CID, bytes: Uint8Array): BlockMap {
31
+
set(cid: Cid, bytes: Uint8Array): BlockMap {
24
32
this.map.set(cid.toString(), bytes);
25
33
return this;
26
34
}
27
35
28
-
get(cid: CID): Uint8Array | undefined {
36
+
get(cid: Cid): Uint8Array | undefined {
29
37
return this.map.get(cid.toString());
30
38
}
31
39
32
-
delete(cid: CID): BlockMap {
40
+
delete(cid: Cid): BlockMap {
33
41
this.map.delete(cid.toString());
34
42
return this;
35
43
}
36
44
37
-
getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
38
-
const missing: CID[] = [];
45
+
getMany(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {
46
+
const missing: Cid[] = [];
39
47
const blocks = new BlockMap();
40
48
for (const cid of cids) {
41
49
const got = this.map.get(cid.toString());
···
48
56
return { blocks, missing };
49
57
}
50
58
51
-
has(cid: CID): boolean {
59
+
has(cid: Cid): boolean {
52
60
return this.map.has(cid.toString());
53
61
}
54
62
···
56
64
this.map.clear();
57
65
}
58
66
59
-
forEach(cb: (bytes: Uint8Array, cid: CID) => void): void {
67
+
forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void {
60
68
for (const [cid, bytes] of this) cb(bytes, cid);
61
69
}
62
70
···
64
72
return Array.from(this, toEntry);
65
73
}
66
74
67
-
cids(): CID[] {
75
+
cids(): Cid[] {
68
76
return Array.from(this.keys());
69
77
}
70
78
···
97
105
return true;
98
106
}
99
107
100
-
*keys(): Generator<CID, void, unknown> {
108
+
*keys(): Generator<Cid, void, unknown> {
101
109
for (const key of this.map.keys()) {
102
-
yield CID.parse(key);
110
+
yield parseCid(key);
103
111
}
104
112
}
105
113
···
107
115
yield* this.map.values();
108
116
}
109
117
110
-
*[Symbol.iterator](): Generator<[CID, Uint8Array], void, unknown> {
118
+
*[Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown> {
111
119
for (const [key, value] of this.map) {
112
-
yield [CID.parse(key), value];
120
+
yield [parseCid(key), value];
113
121
}
114
122
}
115
123
}
116
124
117
-
function toEntry([cid, bytes]: readonly [CID, Uint8Array]): Entry {
125
+
function toEntry([cid, bytes]: readonly [Cid, Uint8Array]): Entry {
118
126
return { cid, bytes };
119
127
}
120
128
121
129
type Entry = {
122
-
cid: CID;
130
+
cid: Cid;
123
131
bytes: Uint8Array;
124
132
};
125
133
+10
-16
repo/car.ts
+10
-16
repo/car.ts
···
1
1
import * as cbor from "@ipld/dag-cbor";
2
-
import type { CID } from "multiformats/cid";
2
+
import type { Cid } from "@atp/lex/data";
3
+
import { parseCidFromBytes, verifyCidForBytes } from "@atp/lex/cbor";
3
4
import * as ui8 from "@atp/bytes";
4
5
import { encodeVarint } from "@std/encoding/varint";
5
-
import {
6
-
type CarHeader,
7
-
check,
8
-
def,
9
-
parseCidFromBytes,
10
-
streamToBuffer,
11
-
verifyCidForBytes,
12
-
} from "@atp/common";
6
+
import { type CarHeader, check, def, streamToBuffer } from "@atp/common";
13
7
import { BlockMap } from "./block-map.ts";
14
8
import type { CarBlock } from "./types.ts";
15
9
···
34
28
};
35
29
36
30
export async function* writeCarStream(
37
-
root: CID | null,
31
+
root: Cid | null,
38
32
blocks: AsyncIterable<CarBlock>,
39
33
): AsyncIterable<Uint8Array> {
40
34
const headerObj = {
···
56
50
}
57
51
58
52
export const blocksToCarFile = (
59
-
root: CID | null,
53
+
root: Cid | null,
60
54
blocks: BlockMap,
61
55
): Promise<Uint8Array> => {
62
56
const carStream = blocksToCarStream(root, blocks);
···
64
58
};
65
59
66
60
export const blocksToCarStream = (
67
-
root: CID | null,
61
+
root: Cid | null,
68
62
blocks: BlockMap,
69
63
): AsyncIterable<Uint8Array> => {
70
64
return writeCarStream(root, iterateBlocks(blocks));
···
86
80
export const readCar = async (
87
81
bytes: Uint8Array,
88
82
opts?: ReadCarOptions,
89
-
): Promise<{ roots: CID[]; blocks: BlockMap }> => {
83
+
): Promise<{ roots: Cid[]; blocks: BlockMap }> => {
90
84
const { roots, blocks } = await readCarStream([bytes], opts);
91
85
const blockMap = new BlockMap();
92
86
for await (const block of blocks) {
···
98
92
export const readCarWithRoot = async (
99
93
bytes: Uint8Array,
100
94
opts?: ReadCarOptions,
101
-
): Promise<{ root: CID; blocks: BlockMap }> => {
95
+
): Promise<{ root: Cid; blocks: BlockMap }> => {
102
96
const { roots, blocks } = await readCar(bytes, opts);
103
97
if (roots.length !== 1) {
104
98
throw new Error(`Expected one root, got ${roots.length}`);
···
117
111
car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
118
112
opts?: ReadCarOptions,
119
113
): Promise<{
120
-
roots: CID[];
114
+
roots: Cid[];
121
115
blocks: CarBlockIterable;
122
116
}> => {
123
117
const reader = new BufferedReader(car);
···
130
124
const headerData = cbor.decode(headerBytes);
131
125
const header = check.assure(def.carHeader.schema, headerData) as CarHeader;
132
126
return {
133
-
roots: header.roots as CID[],
127
+
roots: header.roots as Cid[],
134
128
blocks: readCarBlocksIter(reader, opts),
135
129
};
136
130
} catch (err) {
+7
-7
repo/cid-set.ts
+7
-7
repo/cid-set.ts
···
1
-
import { CID } from "multiformats";
1
+
import { type Cid, parseCid } from "@atp/lex/data";
2
2
3
3
export class CidSet {
4
4
private set: Set<string>;
5
5
6
-
constructor(arr: CID[] = []) {
6
+
constructor(arr: Cid[] = []) {
7
7
const strArr = arr.map((c) => c.toString());
8
8
this.set = new Set(strArr);
9
9
}
10
10
11
-
add(cid: CID): CidSet {
11
+
add(cid: Cid): CidSet {
12
12
this.set.add(cid.toString());
13
13
return this;
14
14
}
···
23
23
return this;
24
24
}
25
25
26
-
delete(cid: CID): CidSet {
26
+
delete(cid: Cid): CidSet {
27
27
this.set.delete(cid.toString());
28
28
return this;
29
29
}
30
30
31
-
has(cid: CID): boolean {
31
+
has(cid: Cid): boolean {
32
32
return this.set.has(cid.toString());
33
33
}
34
34
···
41
41
return this;
42
42
}
43
43
44
-
toList(): CID[] {
45
-
return [...this.set].map((c) => CID.parse(c));
44
+
toList(): Cid[] {
45
+
return [...this.set].map((c) => parseCid(c));
46
46
}
47
47
}
48
48
+10
-10
repo/data-diff.ts
+10
-10
repo/data-diff.ts
···
1
-
import type { CID } from "multiformats";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import { BlockMap } from "./block-map.ts";
3
3
import { CidSet } from "./cid-set.ts";
4
4
import { type MST, mstDiff, type NodeEntry } from "./mst/index.ts";
···
37
37
}
38
38
}
39
39
40
-
leafAdd(key: string, cid: CID) {
40
+
leafAdd(key: string, cid: Cid) {
41
41
this.adds[key] = { key, cid };
42
42
if (this.removedCids.has(cid)) {
43
43
this.removedCids.delete(cid);
···
46
46
}
47
47
}
48
48
49
-
leafUpdate(key: string, prev: CID, cid: CID) {
49
+
leafUpdate(key: string, prev: Cid, cid: Cid) {
50
50
if (prev.equals(cid)) return;
51
51
this.updates[key] = { key, prev, cid };
52
52
this.removedCids.add(prev);
53
53
this.newLeafCids.add(cid);
54
54
}
55
55
56
-
leafDelete(key: string, cid: CID) {
56
+
leafDelete(key: string, cid: Cid) {
57
57
this.deletes[key] = { key, cid };
58
58
if (this.newLeafCids.has(cid)) {
59
59
this.newLeafCids.delete(cid);
···
62
62
}
63
63
}
64
64
65
-
treeAdd(cid: CID, bytes: Uint8Array) {
65
+
treeAdd(cid: Cid, bytes: Uint8Array) {
66
66
if (this.removedCids.has(cid)) {
67
67
this.removedCids.delete(cid);
68
68
} else {
···
70
70
}
71
71
}
72
72
73
-
treeDelete(cid: CID) {
73
+
treeDelete(cid: Cid) {
74
74
if (this.newMstBlocks.has(cid)) {
75
75
this.newMstBlocks.delete(cid);
76
76
} else {
···
102
102
103
103
export type DataAdd = {
104
104
key: string;
105
-
cid: CID;
105
+
cid: Cid;
106
106
};
107
107
108
108
export type DataUpdate = {
109
109
key: string;
110
-
prev: CID;
111
-
cid: CID;
110
+
prev: Cid;
111
+
cid: Cid;
112
112
};
113
113
114
114
export type DataDelete = {
115
115
key: string;
116
-
cid: CID;
116
+
cid: Cid;
117
117
};
+1
-2
repo/deno.json
+1
-2
repo/deno.json
···
1
1
{
2
2
"name": "@atp/repo",
3
-
"version": "0.1.0-alpha.5",
3
+
"version": "0.1.0-alpha.6",
4
4
"exports": "./mod.ts",
5
5
"license": "MIT",
6
6
"imports": {
7
7
"@ipld/dag-cbor": "npm:@ipld/dag-cbor@^9.2.5",
8
8
"@std/encoding": "jsr:@std/encoding@^1.0.10",
9
-
"multiformats": "npm:multiformats@^13.4.1",
10
9
"zod": "jsr:@zod/zod@^4.1.13"
11
10
},
12
11
"test": {
+6
-6
repo/error.ts
+6
-6
repo/error.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
3
3
export class MissingBlockError extends Error {
4
4
constructor(
5
-
public cid: CID,
5
+
public cid: Cid,
6
6
def?: string,
7
7
) {
8
8
let msg = `block not found: ${cid.toString()}`;
···
16
16
export class MissingBlocksError extends Error {
17
17
constructor(
18
18
public context: string,
19
-
public cids: CID[],
19
+
public cids: Cid[],
20
20
) {
21
21
const cidStr = cids.map((c) => c.toString());
22
22
super(`missing ${context} blocks: ${cidStr}`);
···
25
25
26
26
export class MissingCommitBlocksError extends Error {
27
27
constructor(
28
-
public commit: CID,
29
-
public cids: CID[],
28
+
public commit: Cid,
29
+
public cids: Cid[],
30
30
) {
31
31
const cidStr = cids.map((c) => c.toString());
32
32
super(`missing blocks for commit ${commit.toString()}: ${cidStr}`);
···
35
35
36
36
export class UnexpectedObjectError extends Error {
37
37
constructor(
38
-
public cid: CID,
38
+
public cid: Cid,
39
39
public def: string,
40
40
) {
41
41
super(`unexpected object at ${cid.toString()}, expected: ${def}`);
+49
-37
repo/mst/mst.ts
+49
-37
repo/mst/mst.ts
···
1
-
import type { CID } from "multiformats";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import { z } from "zod";
3
-
import { cidForCbor, dataToCborBlock, schema as common } from "@atp/common";
3
+
import {
4
+
cidForCbor,
5
+
encode as encodeLexCbor,
6
+
type LexValue as EncodableLexValue,
7
+
} from "@atp/lex/cbor";
4
8
import { BlockMap } from "../block-map.ts";
5
9
import { CidSet } from "../cid-set.ts";
6
10
import { MissingBlockError, MissingBlocksError } from "../error.ts";
7
11
import * as parse from "../parse.ts";
8
12
import type { ReadableBlockstore } from "../storage/index.ts";
9
-
import type { CarBlock } from "../types.ts";
13
+
import { type CarBlock, schema } from "../types.ts";
10
14
import * as util from "./util.ts";
11
15
12
16
/**
···
42
46
* Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`,
43
47
* and the second will be described as `prefix: 16, key: 'hi'.`
44
48
*/
45
-
const subTreePointer: SubTreePointerType = z.nullable(common.cid);
46
-
type SubTreePointerType = z.ZodNullable<typeof common.cid>;
47
-
const treeEntry: TreeEntryType = z.object({
49
+
const subTreePointer: z.ZodNullable<
50
+
z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>
51
+
> = z.nullable(schema.cid);
52
+
const treeEntry: z.ZodObject<{
53
+
p: z.ZodNumber;
54
+
k: z.ZodCustom<Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>>;
55
+
v: z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>;
56
+
t: z.ZodNullable<z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>>;
57
+
}, z.core.$strip> = z.object({
48
58
p: z.number(), // prefix count of ascii chars that this key shares with the prev key
49
-
k: common.bytes, // the rest of the key outside the shared prefix
50
-
v: common.cid, // value
59
+
k: schema.bytes, // the rest of the key outside the shared prefix
60
+
v: schema.cid, // value
51
61
t: subTreePointer, // next subtree (to the right of leaf)
52
62
});
53
-
type TreeEntryType = z.ZodObject<{
54
-
p: z.ZodNumber;
55
-
k: typeof common.bytes;
56
-
v: typeof common.cid;
57
-
t: SubTreePointerType;
58
-
}, z.core.$strip>;
59
-
const nodeData: NodeDataType = z.object({
63
+
const nodeData: z.ZodObject<{
64
+
l: z.ZodNullable<z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>>;
65
+
e: z.ZodArray<
66
+
z.ZodObject<{
67
+
p: z.ZodNumber;
68
+
k: z.ZodCustom<Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>>;
69
+
v: z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>;
70
+
t: z.ZodNullable<z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>>;
71
+
}, z.core.$strip>
72
+
>;
73
+
}, z.core.$strip> = z.object({
60
74
l: subTreePointer, // left-most subtree
61
75
e: z.array(treeEntry), //entries
62
76
});
63
-
type NodeDataType = z.ZodObject<{
64
-
l: SubTreePointerType;
65
-
e: z.ZodArray<TreeEntryType>;
66
-
}, z.core.$strip>;
67
-
export type NodeData = z.infer<NodeDataType>;
77
+
export type NodeData = z.infer<typeof nodeData>;
68
78
69
79
export const nodeDataDef = {
70
80
name: "mst node",
···
81
91
storage: ReadableBlockstore;
82
92
entries: NodeEntry[] | null;
83
93
layer: number | null;
84
-
pointer: CID;
94
+
pointer: Cid;
85
95
outdatedPointer = false;
86
96
87
97
constructor(
88
98
storage: ReadableBlockstore,
89
-
pointer: CID,
99
+
pointer: Cid,
90
100
entries: NodeEntry[] | null,
91
101
layer: number | null,
92
102
) {
···
113
123
): Promise<MST> {
114
124
const { layer = null } = opts || {};
115
125
const entries = util.deserializeNodeData(storage, data, opts);
116
-
const pointer = await cidForCbor(data);
126
+
const bytes = encodeLexCbor(data as EncodableLexValue);
127
+
const pointer = await cidForCbor(bytes);
117
128
return new MST(storage, pointer, entries, layer);
118
129
}
119
130
120
131
// this is really a *lazy* load, doesn't actually touch storage
121
132
static load(
122
133
storage: ReadableBlockstore,
123
-
cid: CID,
134
+
cid: Cid,
124
135
opts?: Partial<MstOpts>,
125
136
): MST {
126
137
const { layer = null } = opts || {};
···
147
158
const data = this.storage.readObj(this.pointer, nodeDataDef);
148
159
const firstLeaf = data.e[0];
149
160
const layer = firstLeaf !== undefined
150
-
? util.leadingZerosOnHash(firstLeaf.k as Uint8Array)
161
+
? util.leadingZerosOnHash(firstLeaf.k)
151
162
: undefined;
152
163
this.entries = util.deserializeNodeData(this.storage, data, {
153
164
layer,
···
160
171
161
172
// We don't hash the node on every mutation for performance reasons
162
173
// Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
163
-
async getPointer(): Promise<CID> {
174
+
async getPointer(): Promise<Cid> {
164
175
if (!this.outdatedPointer) return this.pointer;
165
176
const { cid } = await this.serialize();
166
177
this.pointer = cid;
···
168
179
return this.pointer;
169
180
}
170
181
171
-
async serialize(): Promise<{ cid: CID; bytes: Uint8Array }> {
182
+
async serialize(): Promise<{ cid: Cid; bytes: Uint8Array }> {
172
183
let entries = this.getEntries();
173
184
const outdated = entries.filter(
174
185
(e) => e.isTree() && e.outdatedPointer,
···
178
189
entries = this.getEntries();
179
190
}
180
191
const data = util.serializeNodeData(entries);
181
-
const block = await dataToCborBlock(data);
192
+
const bytes = encodeLexCbor(data as EncodableLexValue);
193
+
const cid = await cidForCbor(bytes);
182
194
return {
183
-
cid: block.cid,
184
-
bytes: block.bytes,
195
+
cid,
196
+
bytes,
185
197
};
186
198
}
187
199
···
218
230
// -------------------
219
231
220
232
// Return the necessary blocks to persist the MST to repo storage
221
-
async getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }> {
233
+
async getUnstoredBlocks(): Promise<{ root: Cid; blocks: BlockMap }> {
222
234
const blocks = new BlockMap();
223
235
const pointer = await this.getPointer();
224
236
const alreadyHas = this.storage.has(pointer);
···
237
249
238
250
// Adds a new leaf for the given key/value pair
239
251
// Throws if a leaf with that key already exists
240
-
async add(key: string, value: CID, knownZeros?: number): Promise<MST> {
252
+
async add(key: string, value: Cid, knownZeros?: number): Promise<MST> {
241
253
util.ensureValidMstKey(key);
242
254
const keyZeros = knownZeros ?? (util.leadingZerosOnHash(key));
243
255
const layer = await this.getLayer();
···
307
319
}
308
320
309
321
// Gets the value at the given key
310
-
get(key: string): CID | null {
322
+
get(key: string): Cid | null {
311
323
const index = this.findGtOrEqualLeafIndex(key);
312
324
const found = this.atIndex(index);
313
325
if (found && found.isLeaf() && found.key === key) {
···
322
334
323
335
// Edits the value at the given key
324
336
// Throws if the given key does not exist
325
-
async update(key: string, value: CID): Promise<MST> {
337
+
async update(key: string, value: Cid): Promise<MST> {
326
338
util.ensureValidMstKey(key);
327
339
const index = this.findGtOrEqualLeafIndex(key);
328
340
const found = this.atIndex(index);
···
775
787
}
776
788
}
777
789
778
-
async cidsForPath(key: string): Promise<CID[]> {
779
-
const cids: CID[] = [await this.getPointer()];
790
+
async cidsForPath(key: string): Promise<Cid[]> {
791
+
const cids: Cid[] = [await this.getPointer()];
780
792
const index = this.findGtOrEqualLeafIndex(key);
781
793
const found = this.atIndex(index);
782
794
if (found && found.isLeaf() && found.key === key) {
···
882
894
export class Leaf {
883
895
constructor(
884
896
public key: string,
885
-
public value: CID,
897
+
public value: Cid,
886
898
) {}
887
899
888
900
isTree(): this is MST {
+8
-8
repo/mst/util.ts
+8
-8
repo/mst/util.ts
···
1
-
import type { CID } from "multiformats";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import * as bytes from "@atp/bytes";
3
-
import { cidForCbor } from "@atp/common";
3
+
import { cidForLex, type LexValue as EncodableLexValue } from "@atp/lex/cbor";
4
4
import { sha256 } from "@atp/crypto";
5
5
import type { ReadableBlockstore } from "../storage/index.ts";
6
6
import {
···
45
45
const entries: NodeEntry[] = [];
46
46
if (data.l !== null) {
47
47
entries.push(
48
-
MST.load(storage, data.l as CID, {
48
+
MST.load(storage, data.l as Cid, {
49
49
layer: layer ? layer - 1 : undefined,
50
50
}),
51
51
);
···
55
55
const keyStr = bytes.toString(entry.k as Uint8Array, "ascii");
56
56
const key = lastKey.slice(0, entry.p) + keyStr;
57
57
ensureValidMstKey(key);
58
-
entries.push(new Leaf(key, entry.v as CID));
58
+
entries.push(new Leaf(key, entry.v as Cid));
59
59
lastKey = key;
60
60
if (entry.t !== null) {
61
61
entries.push(
62
-
MST.load(storage, entry.t as CID, {
62
+
MST.load(storage, entry.t as Cid, {
63
63
layer: layer ? layer - 1 : undefined,
64
64
}),
65
65
);
···
86
86
throw new Error("Not a valid node: two subtrees next to each other");
87
87
}
88
88
i++;
89
-
let subtree: CID | null = null;
89
+
let subtree: Cid | null = null;
90
90
if (next?.isTree()) {
91
91
subtree = next.pointer;
92
92
i++;
···
115
115
return i;
116
116
};
117
117
118
-
export const cidForEntries = (entries: NodeEntry[]): Promise<CID> => {
118
+
export const cidForEntries = (entries: NodeEntry[]): Promise<Cid> => {
119
119
const data = serializeNodeData(entries);
120
-
return cidForCbor(data);
120
+
return cidForLex(data as EncodableLexValue);
121
121
};
122
122
123
123
export const isValidMstKey = (str: string): boolean => {
+8
-7
repo/parse.ts
+8
-7
repo/parse.ts
···
1
-
import type { CID } from "multiformats/cid";
2
-
import { cborDecode, type check } from "@atp/common";
3
-
import type { RepoRecord } from "@atp/lexicon";
1
+
import type { Cid } from "@atp/lex/data";
2
+
import { decode as decodeLexCbor } from "@atp/lex/cbor";
3
+
import type { check } from "@atp/common";
4
4
import type { BlockMap } from "./block-map.ts";
5
5
import { MissingBlockError, UnexpectedObjectError } from "./error.ts";
6
+
import type { RepoRecord } from "./types.ts";
6
7
import { cborToLexRecord } from "./util.ts";
7
8
8
9
export const getAndParseRecord = (
9
10
blocks: BlockMap,
10
-
cid: CID,
11
+
cid: Cid,
11
12
): { record: RepoRecord; bytes: Uint8Array } => {
12
13
const bytes = blocks.get(cid);
13
14
if (!bytes) {
···
19
20
20
21
export const getAndParseByDef = <T>(
21
22
blocks: BlockMap,
22
-
cid: CID,
23
+
cid: Cid,
23
24
def: check.Def<T>,
24
25
): { obj: T; bytes: Uint8Array } => {
25
26
const bytes = blocks.get(cid);
···
31
32
32
33
export const parseObjByDef = <T>(
33
34
bytes: Uint8Array,
34
-
cid: CID,
35
+
cid: Cid,
35
36
def: check.Def<T>,
36
37
): { obj: T; bytes: Uint8Array } => {
37
-
const obj = cborDecode(bytes);
38
+
const obj = decodeLexCbor(bytes);
38
39
const res = def.schema.safeParse(obj);
39
40
if (res.success) {
40
41
return { obj: res.data, bytes };
+18
-11
repo/readable-repo.ts
+18
-11
repo/readable-repo.ts
···
1
-
import type { CID } from "multiformats/cid";
2
-
import type { RepoRecord } from "@atp/lexicon";
1
+
import type { Cid } from "@atp/lex/data";
3
2
import { MissingBlocksError } from "./error.ts";
4
3
import log from "./logger.ts";
5
4
import { MST } from "./mst/index.ts";
6
5
import * as parse from "./parse.ts";
7
6
import type { ReadableBlockstore } from "./storage/index.ts";
8
-
import { type Commit, def, type RepoContents } from "./types.ts";
7
+
import {
8
+
type Commit,
9
+
def,
10
+
type RepoContents,
11
+
type RepoRecord,
12
+
} from "./types.ts";
9
13
import * as util from "./util.ts";
10
14
11
15
type Params = {
12
16
storage: ReadableBlockstore;
13
17
data: MST;
14
18
commit: Commit;
15
-
cid: CID;
19
+
cid: Cid;
16
20
};
17
21
18
22
export class ReadableRepo {
19
23
storage: ReadableBlockstore;
20
24
data: MST;
21
25
commit: Commit;
22
-
cid: CID;
26
+
cid: Cid;
23
27
24
28
constructor(params: Params) {
25
29
this.storage = params.storage;
···
28
32
this.cid = params.cid;
29
33
}
30
34
31
-
static load(storage: ReadableBlockstore, commitCid: CID): ReadableRepo {
35
+
static load(storage: ReadableBlockstore, commitCid: Cid): ReadableRepo {
32
36
const commit = storage.readObj(commitCid, def.versionedCommit);
33
-
const data = MST.load(storage, (commit as { data: CID }).data);
37
+
const data = MST.load(storage, (commit as { data: Cid }).data);
34
38
log.info("loaded repo for", { did: commit.did });
35
39
return new ReadableRepo({
36
40
storage,
···
51
55
async *walkRecords(from?: string): AsyncIterable<{
52
56
collection: string;
53
57
rkey: string;
54
-
cid: CID;
58
+
cid: Cid;
55
59
record: RepoRecord;
56
60
}> {
57
61
for await (const leaf of this.data.walkLeavesFrom(from ?? "")) {
···
61
65
}
62
66
}
63
67
64
-
async getRecord(collection: string, rkey: string): Promise<unknown | null> {
68
+
async getRecord(
69
+
collection: string,
70
+
rkey: string,
71
+
): Promise<RepoRecord | null> {
65
72
const dataKey = collection + "/" + rkey;
66
73
const cid = await this.data.get(dataKey);
67
74
if (!cid) return null;
68
-
return this.storage.readObj(cid, def.unknown);
75
+
return this.storage.readRecord(cid);
69
76
}
70
77
71
78
async getContents(): Promise<RepoContents> {
72
79
const entries = await this.data.list();
73
-
const cids = entries.map((e: { key: string; value: CID }) => e.value);
80
+
const cids = entries.map((e: { key: string; value: Cid }) => e.value);
74
81
const { blocks, missing } = await this.storage.getBlocks(cids);
75
82
if (missing.length > 0) {
76
83
throw new MissingBlocksError("getContents record", missing);
+19
-13
repo/repo.ts
+19
-13
repo/repo.ts
···
1
-
import type { CID } from "multiformats/cid";
2
-
import { dataToCborBlock, TID } from "@atp/common";
1
+
import type { Cid } from "@atp/lex/data";
2
+
import {
3
+
cidForCbor,
4
+
encode as encodeLexCbor,
5
+
type LexValue as EncodableLexValue,
6
+
} from "@atp/lex/cbor";
7
+
import { TID } from "@atp/common";
3
8
import type * as crypto from "@atp/crypto";
4
-
import { lexToIpld } from "@atp/lexicon";
5
9
import { BlockMap } from "./block-map.ts";
6
10
import { CidSet } from "./cid-set.ts";
7
11
import { DataDiff } from "./data-diff.ts";
···
18
22
WriteOpAction,
19
23
} from "./types.ts";
20
24
import * as util from "./util.ts";
21
-
import type { Version } from "multiformats/link/interface";
22
25
23
26
type Params = {
24
27
storage: RepoStorage;
25
28
data: MST;
26
29
commit: Commit;
27
-
cid: CID;
30
+
cid: Cid;
28
31
};
29
32
30
33
export class Repo extends ReadableRepo {
···
100
103
return Repo.createFromCommit(storage, commit);
101
104
}
102
105
103
-
static override load(storage: RepoStorage, cid?: CID): Repo {
106
+
static override load(storage: RepoStorage, cid?: Cid): Repo {
104
107
const commitCid = cid || (storage.getRoot());
105
108
if (!commitCid) {
106
109
throw new Error("No cid provided and none in storage");
107
110
}
108
111
const commit = storage.readObj(commitCid, def.versionedCommit);
109
-
const data = MST.load(storage, (commit as { data: CID }).data);
112
+
const data = MST.load(storage, (commit as { data: Cid }).data);
110
113
log.info("loaded repo for", { did: commit.did });
111
114
return new Repo({
112
115
storage,
···
170
173
},
171
174
keypair,
172
175
);
173
-
const commitBlock = await dataToCborBlock(lexToIpld(commit));
174
-
if (!commitBlock.cid.equals(this.cid)) {
175
-
newBlocks.set(commitBlock.cid, commitBlock.bytes);
176
-
relevantBlocks.set(commitBlock.cid, commitBlock.bytes);
176
+
const commitBytes = encodeLexCbor(
177
+
util.lexToCborValue(commit) as EncodableLexValue,
178
+
);
179
+
const commitCid = await cidForCbor(commitBytes);
180
+
if (!commitCid.equals(this.cid)) {
181
+
newBlocks.set(commitCid, commitBytes);
182
+
relevantBlocks.set(commitCid, commitBytes);
177
183
removedCids.add(this.cid);
178
184
}
179
185
180
186
return {
181
-
cid: commitBlock.cid,
187
+
cid: commitCid,
182
188
rev,
183
189
since: this.commit.rev,
184
190
prev: this.cid,
···
202
208
}
203
209
204
210
async formatResignCommit(rev: string, keypair: crypto.Keypair): Promise<{
205
-
cid: CID<unknown, number, number, Version>;
211
+
cid: Cid;
206
212
rev: string;
207
213
since: null;
208
214
prev: null;
+8
-8
repo/storage/memory-blockstore.ts
+8
-8
repo/storage/memory-blockstore.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import { BlockMap } from "../block-map.ts";
3
3
import type { CommitData } from "../types.ts";
4
4
import { ReadableBlockstore } from "./readable-blockstore.ts";
···
7
7
export class MemoryBlockstore extends ReadableBlockstore
8
8
implements RepoStorage {
9
9
blocks: BlockMap;
10
-
root: CID | null = null;
10
+
root: Cid | null = null;
11
11
rev: string | null = null;
12
12
13
13
constructor(blocks?: BlockMap) {
···
18
18
}
19
19
}
20
20
21
-
getRoot(): CID | null {
21
+
getRoot(): Cid | null {
22
22
return this.root;
23
23
}
24
24
25
-
getBytes(cid: CID): Uint8Array | null {
25
+
getBytes(cid: Cid): Uint8Array | null {
26
26
return this.blocks.get(cid) || null;
27
27
}
28
28
29
-
has(cid: CID): boolean {
29
+
has(cid: Cid): boolean {
30
30
return this.blocks.has(cid);
31
31
}
32
32
33
-
getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
33
+
getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {
34
34
return this.blocks.getMany(cids);
35
35
}
36
36
37
-
putBlock(cid: CID, block: Uint8Array): void {
37
+
putBlock(cid: Cid, block: Uint8Array): void {
38
38
this.blocks.set(cid, block);
39
39
}
40
40
···
42
42
this.blocks.addMap(blocks);
43
43
}
44
44
45
-
updateRoot(cid: CID, rev: string): void {
45
+
updateRoot(cid: Cid, rev: string): void {
46
46
this.root = cid;
47
47
this.rev = rev;
48
48
}
+11
-11
repo/storage/readable-blockstore.ts
+11
-11
repo/storage/readable-blockstore.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { check } from "@atp/common";
3
-
import type { RepoRecord } from "@atp/lexicon";
4
3
import type { BlockMap } from "../block-map.ts";
5
4
import { MissingBlockError } from "../error.ts";
6
5
import * as parse from "../parse.ts";
6
+
import type { RepoRecord } from "../types.ts";
7
7
import { cborToLexRecord } from "../util.ts";
8
8
9
9
export abstract class ReadableBlockstore {
10
-
abstract getBytes(cid: CID): Uint8Array | null;
11
-
abstract has(cid: CID): boolean;
10
+
abstract getBytes(cid: Cid): Uint8Array | null;
11
+
abstract has(cid: Cid): boolean;
12
12
abstract getBlocks(
13
-
cids: CID[],
14
-
): { blocks: BlockMap; missing: CID[] };
13
+
cids: Cid[],
14
+
): { blocks: BlockMap; missing: Cid[] };
15
15
16
16
attemptRead<T>(
17
-
cid: CID,
17
+
cid: Cid,
18
18
def: check.Def<T>,
19
19
): { obj: T; bytes: Uint8Array } | null {
20
20
const bytes = this.getBytes(cid);
···
23
23
}
24
24
25
25
readObjAndBytes<T>(
26
-
cid: CID,
26
+
cid: Cid,
27
27
def: check.Def<T>,
28
28
): { obj: T; bytes: Uint8Array } {
29
29
const read = this.attemptRead(cid, def);
···
33
33
return read;
34
34
}
35
35
36
-
readObj<T>(cid: CID, def: check.Def<T>): T {
36
+
readObj<T>(cid: Cid, def: check.Def<T>): T {
37
37
const obj = this.readObjAndBytes(cid, def);
38
38
return obj.obj;
39
39
}
40
40
41
-
attemptReadRecord(cid: CID): RepoRecord | null {
41
+
attemptReadRecord(cid: Cid): RepoRecord | null {
42
42
try {
43
43
return this.readRecord(cid);
44
44
} catch {
···
46
46
}
47
47
}
48
48
49
-
readRecord(cid: CID): RepoRecord {
49
+
readRecord(cid: Cid): RepoRecord {
50
50
const bytes = this.getBytes(cid);
51
51
if (!bytes) {
52
52
throw new MissingBlockError(cid);
+4
-4
repo/storage/sync-storage.ts
+4
-4
repo/storage/sync-storage.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { BlockMap } from "../block-map.ts";
3
3
import { ReadableBlockstore } from "./readable-blockstore.ts";
4
4
···
10
10
super();
11
11
}
12
12
13
-
getBytes(cid: CID): Uint8Array | null {
13
+
getBytes(cid: Cid): Uint8Array | null {
14
14
const got = this.staged.getBytes(cid);
15
15
if (got) return got;
16
16
return this.saved.getBytes(cid);
17
17
}
18
18
19
-
getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
19
+
getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {
20
20
const fromStaged = this.staged.getBlocks(cids);
21
21
const fromSaved = this.saved.getBlocks(fromStaged.missing);
22
22
const blocks = fromStaged.blocks;
···
27
27
};
28
28
}
29
29
30
-
has(cid: CID): boolean {
30
+
has(cid: Cid): boolean {
31
31
return (this.staged.has(cid)) || (this.saved.has(cid));
32
32
}
33
33
}
+22
-23
repo/storage/types.ts
+22
-23
repo/storage/types.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { check } from "@atp/common";
3
-
import type { RepoRecord } from "@atp/lexicon";
4
3
import type { BlockMap } from "../block-map.ts";
5
-
import type { CommitData } from "../types.ts";
4
+
import type { CommitData, RepoRecord } from "../types.ts";
6
5
7
6
export interface RepoStorage {
8
7
// Writable
9
-
getRoot(): CID | null;
10
-
putBlock(cid: CID, block: Uint8Array, rev: string): void;
8
+
getRoot(): Cid | null;
9
+
putBlock(cid: Cid, block: Uint8Array, rev: string): void;
11
10
putMany(blocks: BlockMap, rev: string): void;
12
-
updateRoot(cid: CID, rev: string): void;
11
+
updateRoot(cid: Cid, rev: string): void;
13
12
applyCommit(commit: CommitData): void;
14
13
15
14
// Readable
16
-
getBytes(cid: CID): Uint8Array | null;
17
-
has(cid: CID): boolean;
18
-
getBlocks(cids: CID[]): { blocks: BlockMap; missing: CID[] };
15
+
getBytes(cid: Cid): Uint8Array | null;
16
+
has(cid: Cid): boolean;
17
+
getBlocks(cids: Cid[]): { blocks: BlockMap; missing: Cid[] };
19
18
attemptRead<T>(
20
-
cid: CID,
19
+
cid: Cid,
21
20
def: check.Def<T>,
22
21
): { obj: T; bytes: Uint8Array } | null;
23
22
readObjAndBytes<T>(
24
-
cid: CID,
23
+
cid: Cid,
25
24
def: check.Def<T>,
26
25
): { obj: T; bytes: Uint8Array };
27
-
readObj<T>(cid: CID, def: check.Def<T>): T;
28
-
attemptReadRecord(cid: CID): RepoRecord | null;
29
-
readRecord(cid: CID): RepoRecord;
26
+
readObj<T>(cid: Cid, def: check.Def<T>): T;
27
+
attemptReadRecord(cid: Cid): RepoRecord | null;
28
+
readRecord(cid: Cid): RepoRecord;
30
29
}
31
30
32
31
export interface BlobStore {
33
32
putTemp(bytes: Uint8Array | ReadableStream): Promise<string>;
34
-
makePermanent(key: string, cid: CID): Promise<void>;
35
-
putPermanent(cid: CID, bytes: Uint8Array | ReadableStream): Promise<void>;
36
-
quarantine(cid: CID): Promise<void>;
37
-
unquarantine(cid: CID): Promise<void>;
38
-
getBytes(cid: CID): Uint8Array;
39
-
getStream(cid: CID): Promise<ReadableStream>;
33
+
makePermanent(key: string, cid: Cid): Promise<void>;
34
+
putPermanent(cid: Cid, bytes: Uint8Array | ReadableStream): Promise<void>;
35
+
quarantine(cid: Cid): Promise<void>;
36
+
unquarantine(cid: Cid): Promise<void>;
37
+
getBytes(cid: Cid): Uint8Array;
38
+
getStream(cid: Cid): Promise<ReadableStream>;
40
39
hasTemp(key: string): Promise<boolean>;
41
-
hasStored(cid: CID): Promise<boolean>;
42
-
delete(cid: CID): Promise<void>;
43
-
deleteMany(cid: CID[]): Promise<void>;
40
+
hasStored(cid: Cid): Promise<boolean>;
41
+
delete(cid: Cid): Promise<void>;
42
+
deleteMany(cid: Cid[]): Promise<void>;
44
43
}
45
44
46
45
export class BlobNotFoundError extends Error {}
+6
-6
repo/sync/consumer.ts
+6
-6
repo/sync/consumer.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { BlockMap } from "../block-map.ts";
3
3
import { readCarWithRoot } from "../car.ts";
4
4
import { DataDiff } from "../data-diff.ts";
···
29
29
30
30
export const verifyRepo = async (
31
31
blocks: BlockMap,
32
-
head: CID,
32
+
head: Cid,
33
33
did?: string,
34
34
signingKey?: string,
35
35
opts?: { ensureLeaves?: boolean },
···
56
56
export const verifyDiff = async (
57
57
repo: ReadableRepo | null,
58
58
updateBlocks: BlockMap,
59
-
updateRoot: CID,
59
+
updateRoot: Cid,
60
60
did?: string,
61
61
signingKey?: string,
62
62
opts?: { ensureLeaves?: boolean },
···
107
107
// @NOTE only verifies the root, not the repo contents
108
108
const verifyRepoRoot = (
109
109
storage: ReadableBlockstore,
110
-
head: CID,
110
+
head: Cid,
111
111
did?: string,
112
112
signingKey?: string,
113
113
): ReadableRepo => {
···
144
144
`Invalid signature on commit: ${car.root.toString()}`,
145
145
);
146
146
}
147
-
const mst = MST.load(blockstore, (commit as { data: CID }).data);
147
+
const mst = MST.load(blockstore, (commit as { data: Cid }).data);
148
148
const verified: RecordCidClaim[] = [];
149
149
const unverified: RecordCidClaim[] = [];
150
150
for (const claim of claims) {
···
186
186
`Invalid signature on commit: ${car.root.toString()}`,
187
187
);
188
188
}
189
-
const mst = MST.load(blockstore, (commit as { data: CID }).data);
189
+
const mst = MST.load(blockstore, (commit as { data: Cid }).data);
190
190
191
191
const records: RecordClaim[] = [];
192
192
const leaves = await mst.reachableLeaves();
+7
-7
repo/sync/provider.ts
+7
-7
repo/sync/provider.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import { writeCarStream } from "../car.ts";
3
3
import { CidSet } from "../cid-set.ts";
4
4
import { MissingBlocksError } from "../error.ts";
···
12
12
13
13
export const getFullRepo = (
14
14
storage: RepoStorage,
15
-
commitCid: CID,
15
+
commitCid: Cid,
16
16
): AsyncIterable<Uint8Array> => {
17
17
return writeCarStream(commitCid, iterateFullRepo(storage, commitCid));
18
18
};
19
19
20
-
async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
20
+
async function* iterateFullRepo(storage: RepoStorage, commitCid: Cid) {
21
21
const commit = storage.readObjAndBytes(commitCid, def.commit);
22
22
yield { cid: commitCid, bytes: commit.bytes };
23
-
const mst = MST.load(storage, commit.obj.data as CID);
23
+
const mst = MST.load(storage, commit.obj.data as Cid);
24
24
for await (const block of mst.carBlockStream()) {
25
25
yield block;
26
26
}
···
31
31
32
32
export const getRecords = (
33
33
storage: ReadableBlockstore,
34
-
commitCid: CID,
34
+
commitCid: Cid,
35
35
paths: RecordPath[],
36
36
): AsyncIterable<Uint8Array> => {
37
37
return writeCarStream(
···
42
42
43
43
async function* iterateRecordBlocks(
44
44
storage: ReadableBlockstore,
45
-
commitCid: CID,
45
+
commitCid: Cid,
46
46
paths: RecordPath[],
47
47
) {
48
48
const commit = storage.readObjAndBytes(commitCid, def.commit);
49
49
yield { cid: commitCid, bytes: commit.bytes };
50
-
const mst = MST.load(storage, commit.obj.data as CID);
50
+
const mst = MST.load(storage, commit.obj.data as Cid);
51
51
const cidsForPaths = await Promise.all(
52
52
paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))),
53
53
);
+7
-7
repo/tests/_util.ts
+7
-7
repo/tests/_util.ts
···
1
1
import fs from "node:fs";
2
-
import { CID } from "multiformats";
2
+
import { type Cid, parseCid } from "@atp/lex/data";
3
3
import { dataToCborBlock, TID } from "@atp/common";
4
4
import type * as crypto from "@atp/crypto";
5
5
import { type Keypair, randomBytes } from "@atp/crypto";
···
18
18
import { Repo } from "../repo.ts";
19
19
import type { RepoStorage } from "../storage/index.ts";
20
20
21
-
type IdMapping = Record<string, CID>;
21
+
type IdMapping = Record<string, Cid>;
22
22
23
-
export const randomCid = async (storage?: RepoStorage): Promise<CID> => {
23
+
export const randomCid = async (storage?: RepoStorage): Promise<Cid> => {
24
24
const block = await dataToCborBlock({ test: randomStr(50) });
25
25
if (storage) {
26
26
// @ts-expect-error FIXME remove this comment (and fix the TS error)
···
173
173
export const pathsForOps = (ops: RecordWriteOp[]): RecordPath[] =>
174
174
ops.map((op) => ({ collection: op.collection, rkey: op.rkey }));
175
175
176
-
export const saveMst = async (storage: RepoStorage, mst: MST): Promise<CID> => {
176
+
export const saveMst = async (storage: RepoStorage, mst: MST): Promise<Cid> => {
177
177
const diff = await mst.getUnstoredBlocks();
178
178
// @ts-expect-error FIXME remove this comment (and fix the TS error)
179
179
await storage.putMany(diff.blocks);
···
239
239
fs.writeFileSync(filename, log);
240
240
};
241
241
242
-
export const saveMstEntries = (filename: string, entries: [string, CID][]) => {
242
+
export const saveMstEntries = (filename: string, entries: [string, Cid][]) => {
243
243
const writable = entries.map(([key, val]) => [key, val.toString()]);
244
244
fs.writeFileSync(filename, JSON.stringify(writable));
245
245
};
246
246
247
-
export const loadMstEntries = (filename: string): [string, CID][] => {
247
+
export const loadMstEntries = (filename: string): [string, Cid][] => {
248
248
const contents = fs.readFileSync(filename);
249
249
const parsed = JSON.parse(contents.toString());
250
250
return parsed.map((
251
251
[key, value]: [string, string],
252
-
) => [key, CID.parse(value)]);
252
+
) => [key, parseCid(value)]);
253
253
};
+5
-5
repo/tests/car_test.ts
+5
-5
repo/tests/car_test.ts
···
1
-
import { CID } from "multiformats/cid";
1
+
import { parseCid } from "@atp/lex/data";
2
2
import * as ui8 from "@atp/bytes";
3
3
import { dataToCborBlock, streamToBytes, wait } from "@atp/common";
4
4
import { type CarBlock, readCarStream, writeCarStream } from "../mod.ts";
···
7
7
8
8
for (const fixture of fixtures) {
9
9
Deno.test("correctly writes car files", async () => {
10
-
const root = CID.parse(fixture.root);
10
+
const root = parseCid(fixture.root);
11
11
async function* blockIter() {
12
12
for (const block of fixture.blocks) {
13
-
const cid = CID.parse(block.cid);
13
+
const cid = parseCid(block.cid);
14
14
const bytes = ui8.fromString(block.bytes, "base64");
15
15
yield { cid, bytes };
16
16
}
···
75
75
}
76
76
};
77
77
const badCar = await readCarStream(writeCarStream(block0.cid, blockIter()));
78
-
await assertRejects(() => flush(badCar.blocks), "Not a valid CID for bytes");
78
+
await assertRejects(() => flush(badCar.blocks), "Not a valid Cid for bytes");
79
79
});
80
80
81
-
Deno.test("skips CID verification", async () => {
81
+
Deno.test("skips Cid verification", async () => {
82
82
const block0 = await dataToCborBlock({ block: 0 });
83
83
const block1 = await dataToCborBlock({ block: 1 });
84
84
const block2 = await dataToCborBlock({ block: 2 });
+3
-3
repo/tests/commit-proofs_test.ts
+3
-3
repo/tests/commit-proofs_test.ts
···
1
-
import { CID } from "multiformats";
1
+
import { parseCid } from "@atp/lex/data";
2
2
import { MST } from "../mst/index.ts";
3
3
import { BlockMap, MemoryBlockstore } from "../mod.ts";
4
4
import fixtures from "./commit-proof-fixtures.json" with { type: "json" };
···
7
7
for (const fixture of fixtures) {
8
8
Deno.test(fixture.comment, async () => {
9
9
const { leafValue, keys, adds, dels } = fixture;
10
-
const leaf = CID.parse(leafValue);
10
+
const leaf = parseCid(leafValue);
11
11
12
12
const storage = new MemoryBlockstore();
13
13
let mst = await MST.create(storage);
···
33
33
(acc, cur) => acc.addMap(cur),
34
34
new BlockMap(),
35
35
);
36
-
const blocksInProof = fixture.blocksInProof.map((cid) => CID.parse(cid));
36
+
const blocksInProof = fixture.blocksInProof.map((cid) => parseCid(cid));
37
37
for (const cid of blocksInProof) {
38
38
assert(proof.has(cid));
39
39
}
+6
-6
repo/tests/covering-proofs_test.ts
+6
-6
repo/tests/covering-proofs_test.ts
···
1
-
import { CID } from "multiformats";
1
+
import { parseCid } from "@atp/lex/data";
2
2
import { BlockMap } from "../mod.ts";
3
3
import { MST } from "../mst/index.ts";
4
4
import { MemoryBlockstore } from "../storage/index.ts";
···
21
21
*/
22
22
Deno.test("two deep split ", async () => {
23
23
const storage = new MemoryBlockstore();
24
-
const cid = CID.parse(
24
+
const cid = parseCid(
25
25
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
26
26
);
27
27
···
57
57
*/
58
58
Deno.test("two deep leafless splits ", async () => {
59
59
const storage = new MemoryBlockstore();
60
-
const cid = CID.parse(
60
+
const cid = parseCid(
61
61
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
62
62
);
63
63
···
89
89
*/
90
90
Deno.test("add on edge with neighbor two layers down", async () => {
91
91
const storage = new MemoryBlockstore();
92
-
const cid = CID.parse(
92
+
const cid = parseCid(
93
93
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
94
94
);
95
95
···
120
120
*/
121
121
Deno.test("merge and split in multi op commit", async () => {
122
122
const storage = new MemoryBlockstore();
123
-
const cid = CID.parse(
123
+
const cid = parseCid(
124
124
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
125
125
);
126
126
···
182
182
*/
183
183
Deno.test("complex multi-op commit", async () => {
184
184
const storage = new MemoryBlockstore();
185
-
const cid = CID.parse(
185
+
const cid = parseCid(
186
186
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
187
187
);
188
188
+23
-23
repo/tests/mst_test.ts
+23
-23
repo/tests/mst_test.ts
···
1
-
import { CID } from "multiformats";
1
+
import { type Cid, parseCid } from "@atp/lex/data";
2
2
import { assertEquals, assertRejects } from "@std/assert";
3
3
import {
4
4
type DataAdd,
···
13
13
14
14
let blockstore: MemoryBlockstore;
15
15
let mst: MST;
16
-
let mapping: Record<string, CID>;
17
-
let shuffled: [string, CID][];
16
+
let mapping: Record<string, Cid>;
17
+
let shuffled: [string, Cid][];
18
18
19
19
// Setup for main MST tests
20
20
Deno.test("MST setup", async () => {
···
41
41
let editedMst = mst;
42
42
const toEdit = shuffled.slice(0, 100);
43
43
44
-
const edited: [string, CID][] = [];
44
+
const edited: [string, Cid][] = [];
45
45
for (const entry of toEdit) {
46
46
const newCid = await util.randomCid();
47
47
editedMst = await editedMst.update(entry[0], newCid);
···
148
148
149
149
// ensure we correctly report all added CIDs
150
150
for await (const entry of toDiff.walk()) {
151
-
let cid: CID;
151
+
let cid: Cid;
152
152
if (entry.isTree()) {
153
153
cid = await entry.getPointer();
154
154
} else {
···
181
181
Deno.test("MST Allowable Keys rejects the empty key", async () => {
182
182
const blockstore = new MemoryBlockstore();
183
183
const mst = await MST.create(blockstore);
184
-
const cid1 = CID.parse(
184
+
const cid1 = parseCid(
185
185
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
186
186
);
187
187
···
194
194
Deno.test("MST Allowable Keys rejects a key with no collection", async () => {
195
195
const blockstore = new MemoryBlockstore();
196
196
const mst = await MST.create(blockstore);
197
-
const cid1 = CID.parse(
197
+
const cid1 = parseCid(
198
198
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
199
199
);
200
200
···
207
207
Deno.test("MST Allowable Keys rejects a key with a nested collection", async () => {
208
208
const blockstore = new MemoryBlockstore();
209
209
const mst = await MST.create(blockstore);
210
-
const cid1 = CID.parse(
210
+
const cid1 = parseCid(
211
211
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
212
212
);
213
213
···
220
220
Deno.test("MST Allowable Keys rejects on empty coll or rkey", async () => {
221
221
const blockstore = new MemoryBlockstore();
222
222
const mst = await MST.create(blockstore);
223
-
const cid1 = CID.parse(
223
+
const cid1 = parseCid(
224
224
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
225
225
);
226
226
···
237
237
Deno.test("MST Allowable Keys rejects non-ascii chars", async () => {
238
238
const blockstore = new MemoryBlockstore();
239
239
const mst = await MST.create(blockstore);
240
-
const cid1 = CID.parse(
240
+
const cid1 = parseCid(
241
241
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
242
242
);
243
243
···
258
258
Deno.test("MST Allowable Keys rejects ascii that we dont support", async () => {
259
259
const blockstore = new MemoryBlockstore();
260
260
const mst = await MST.create(blockstore);
261
-
const cid1 = CID.parse(
261
+
const cid1 = parseCid(
262
262
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
263
263
);
264
264
···
290
290
Deno.test("MST Allowable Keys rejects keys over 1024 chars", async () => {
291
291
const blockstore = new MemoryBlockstore();
292
292
const mst = await MST.create(blockstore);
293
-
const cid1 = CID.parse(
293
+
const cid1 = parseCid(
294
294
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
295
295
);
296
296
···
306
306
Deno.test("MST Allowable Keys allows valid keys", async () => {
307
307
const blockstore = new MemoryBlockstore();
308
308
let mst = await MST.create(blockstore);
309
-
const cid1 = CID.parse(
309
+
const cid1 = parseCid(
310
310
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
311
311
);
312
312
···
327
327
});
328
328
329
329
// MST Interop Known Maps tests
330
-
Deno.test('MST Known Maps computes "empty" tree root CID', async () => {
330
+
Deno.test('MST Known Maps computes "empty" tree root Cid', async () => {
331
331
const blockstore = new MemoryBlockstore();
332
332
const mst = await MST.create(blockstore);
333
333
···
338
338
);
339
339
});
340
340
341
-
Deno.test('MST Known Maps computes "trivial" tree root CID', async () => {
341
+
Deno.test('MST Known Maps computes "trivial" tree root Cid', async () => {
342
342
const blockstore = new MemoryBlockstore();
343
343
let mst = await MST.create(blockstore);
344
-
const cid1 = CID.parse(
344
+
const cid1 = parseCid(
345
345
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
346
346
);
347
347
···
353
353
);
354
354
});
355
355
356
-
Deno.test('MST Known Maps computes "singlelayer2" tree root CID', async () => {
356
+
Deno.test('MST Known Maps computes "singlelayer2" tree root Cid', async () => {
357
357
const blockstore = new MemoryBlockstore();
358
358
let mst = await MST.create(blockstore);
359
-
const cid1 = CID.parse(
359
+
const cid1 = parseCid(
360
360
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
361
361
);
362
362
···
369
369
);
370
370
});
371
371
372
-
Deno.test('MST Known Maps computes "simple" tree root CID', async () => {
372
+
Deno.test('MST Known Maps computes "simple" tree root Cid', async () => {
373
373
const blockstore = new MemoryBlockstore();
374
374
let mst = await MST.create(blockstore);
375
-
const cid1 = CID.parse(
375
+
const cid1 = parseCid(
376
376
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
377
377
);
378
378
···
392
392
Deno.test("MST Edge Cases trims top of tree on delete", async () => {
393
393
const blockstore = new MemoryBlockstore();
394
394
let mst = await MST.create(blockstore);
395
-
const cid1 = CID.parse(
395
+
const cid1 = parseCid(
396
396
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
397
397
);
398
398
···
419
419
Deno.test("MST Edge Cases handles insertion that splits two layers down", async () => {
420
420
const blockstore = new MemoryBlockstore();
421
421
let mst = await MST.create(blockstore);
422
-
const cid1 = CID.parse(
422
+
const cid1 = parseCid(
423
423
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
424
424
);
425
425
···
459
459
Deno.test("MST Edge Cases handles new layers that are two higher than existing", async () => {
460
460
const blockstore = new MemoryBlockstore();
461
461
let mst = await MST.create(blockstore);
462
-
const cid1 = CID.parse(
462
+
const cid1 = parseCid(
463
463
"bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
464
464
);
465
465
+4
-3
repo/tests/proofs_test.ts
+4
-3
repo/tests/proofs_test.ts
···
1
-
import { cidForCbor, streamToBuffer, TID } from "@atp/common";
1
+
import { streamToBuffer, TID } from "@atp/common";
2
2
import * as crypto from "@atp/crypto";
3
3
import {
4
+
cidForRecord,
4
5
type RecordCidClaim,
5
6
type RecordPath,
6
7
Repo,
···
44
45
claims.push({
45
46
collection: coll,
46
47
rkey: rkey,
47
-
cid: await cidForCbor(contents[coll][rkey]),
48
+
cid: await cidForRecord(contents[coll][rkey]),
48
49
});
49
50
}
50
51
}
···
142
143
}
143
144
assertEquals(
144
145
foundClaim.cid,
145
-
await cidForCbor(repoData[record.collection][record.rkey]),
146
+
await cidForRecord(repoData[record.collection][record.rkey]),
146
147
);
147
148
}
148
149
});
+113
-1
repo/tests/repo_test.ts
+113
-1
repo/tests/repo_test.ts
···
1
1
import { TID } from "@atp/common";
2
+
import { l } from "@atp/lex";
3
+
import { BlobRef as LegacyBlobRef } from "@atp/lexicon";
2
4
import { assertEquals } from "@std/assert";
3
5
import type * as crypto from "@atp/crypto";
4
6
import { Secp256k1Keypair } from "@atp/crypto";
5
-
import { type RepoContents, verifyCommitSig, WriteOpAction } from "../mod.ts";
7
+
import { type Cid, parseCid } from "@atp/lex/data";
8
+
import {
9
+
BlockMap,
10
+
cidForRecord,
11
+
type RepoContents,
12
+
type RepoInputRecord,
13
+
verifyCommitSig,
14
+
WriteOpAction,
15
+
} from "../mod.ts";
6
16
import { Repo } from "../repo.ts";
7
17
import { MemoryBlockstore } from "../storage/index.ts";
8
18
import * as util from "./_util.ts";
9
19
10
20
const collName = "com.example.posts";
21
+
const lexRecordSchema = l.record(
22
+
"tid",
23
+
"com.example.lexRecord",
24
+
l.object({
25
+
text: l.string(),
26
+
note: l.optional(l.string()),
27
+
ref: l.cidLink(),
28
+
bytes: l.bytes(),
29
+
blob: l.optional(l.blob()),
30
+
}),
31
+
);
11
32
12
33
let storage: MemoryBlockstore;
13
34
let keypair: crypto.Keypair;
···
103
124
assertEquals(repo.did, keypair.did());
104
125
assertEquals(repo.version, 3);
105
126
});
127
+
128
+
Deno.test("repo accepts records inferred from @atp/lex", async () => {
129
+
const storage = new MemoryBlockstore();
130
+
const keypair = Secp256k1Keypair.create();
131
+
const collection = "com.example.lexRecord";
132
+
const rkey = TID.nextStr();
133
+
const ref = parseCid(
134
+
"bafyreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku",
135
+
);
136
+
const record = lexRecordSchema.build({
137
+
text: "hello",
138
+
ref,
139
+
bytes: new Uint8Array([1, 2, 3]),
140
+
blob: {
141
+
$type: "blob",
142
+
mimeType: "image/png",
143
+
ref,
144
+
size: 3,
145
+
},
146
+
});
147
+
148
+
const cid = await cidForRecord(record);
149
+
const blocks = new BlockMap();
150
+
151
+
assertEquals((await blocks.add(record)).toString(), cid.toString());
152
+
153
+
const repo = await Repo.create(storage, keypair.did(), keypair, [{
154
+
action: WriteOpAction.Create,
155
+
collection,
156
+
rkey,
157
+
record,
158
+
}]);
159
+
const stored = await repo.getRecord(collection, rkey) as typeof record;
160
+
161
+
assertEquals(stored.$type, record.$type);
162
+
assertEquals(stored.text, record.text);
163
+
assertEquals(stored.ref.toString(), record.ref.toString());
164
+
assertEquals(stored.bytes, record.bytes);
165
+
assertEquals(stored.blob?.ref.toString(), record.blob?.ref.toString());
166
+
assertEquals(stored.blob?.mimeType, record.blob?.mimeType);
167
+
assertEquals(stored.blob?.size, record.blob?.size);
168
+
assertEquals("note" in stored, false);
169
+
});
170
+
171
+
Deno.test("repo accepts legacy lexicon blob refs in our compatibility layer", async () => {
172
+
const storage = new MemoryBlockstore();
173
+
const keypair = Secp256k1Keypair.create();
174
+
const collection = "com.example.legacyBlob";
175
+
const rkey = TID.nextStr();
176
+
const ref = parseCid(
177
+
"bafyreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku",
178
+
);
179
+
const record: RepoInputRecord = {
180
+
$type: collection,
181
+
text: "legacy",
182
+
blob: new LegacyBlobRef(
183
+
ref as unknown as ConstructorParameters<typeof LegacyBlobRef>[0],
184
+
"image/png",
185
+
7,
186
+
),
187
+
};
188
+
189
+
const cid = await cidForRecord(record);
190
+
const blocks = new BlockMap();
191
+
192
+
assertEquals((await blocks.add(record)).toString(), cid.toString());
193
+
194
+
const repo = await Repo.create(storage, keypair.did(), keypair, [{
195
+
action: WriteOpAction.Create,
196
+
collection,
197
+
rkey,
198
+
record,
199
+
}]);
200
+
const stored = await repo.getRecord(collection, rkey) as {
201
+
$type: string;
202
+
text: string;
203
+
blob: {
204
+
$type: string;
205
+
ref: Cid;
206
+
mimeType: string;
207
+
size: number;
208
+
};
209
+
};
210
+
211
+
assertEquals(stored.$type, collection);
212
+
assertEquals(stored.text, "legacy");
213
+
assertEquals(stored.blob.$type, "blob");
214
+
assertEquals(stored.blob.ref.toString(), ref.toString());
215
+
assertEquals(stored.blob.mimeType, "image/png");
216
+
assertEquals(stored.blob.size, 7);
217
+
});
+160
-60
repo/types.ts
+160
-60
repo/types.ts
···
1
-
import type { CID } from "multiformats";
1
+
import { asCid, type Cid } from "@atp/lex/data";
2
2
import { z } from "zod";
3
-
import { schema as common } from "@atp/common";
4
3
import { def as commonDef } from "@atp/common";
5
-
import type { RepoRecord } from "@atp/lexicon";
4
+
import type { BlobRef, LegacyBlobRef } from "@atp/lex";
5
+
import type { BlobRef as LexiconBlobRef } from "@atp/lexicon";
6
6
import type { BlockMap } from "./block-map.ts";
7
7
import type { CidSet } from "./cid-set.ts";
8
8
9
-
// Repo nodes
10
-
// ---------------
9
+
export type LexScalar =
10
+
| number
11
+
| string
12
+
| boolean
13
+
| null
14
+
| Cid
15
+
| Uint8Array
16
+
| BlobRef
17
+
| LegacyBlobRef;
18
+
19
+
export type LexValue =
20
+
| LexScalar
21
+
| LexValue[]
22
+
| { [key: string]: LexValue | undefined };
23
+
24
+
export interface RepoRecord {
25
+
[key: string]: LexValue | undefined;
26
+
}
27
+
28
+
export type RepoInputScalar = LexScalar | LexiconBlobRef;
29
+
30
+
export type RepoInputValue =
31
+
| RepoInputScalar
32
+
| RepoInputValue[]
33
+
| { [key: string]: RepoInputValue | undefined };
11
34
12
-
type UnsignedCommitType = z.ZodObject<{
35
+
export interface RepoInputRecord {
36
+
[key: string]: RepoInputValue | undefined;
37
+
}
38
+
39
+
type CidSchema = z.ZodPipe<z.ZodUnknown, z.ZodTransform<Cid, unknown>>;
40
+
type BytesSchema = z.ZodCustom<
41
+
Uint8Array<ArrayBufferLike>,
42
+
Uint8Array<ArrayBufferLike>
43
+
>;
44
+
type NullableCidSchema = z.ZodNullable<CidSchema>;
45
+
type CommitShape<
46
+
Version extends 2 | 3,
47
+
Rev extends z.ZodString | z.ZodOptional<z.ZodString>,
48
+
> = {
13
49
did: z.ZodString;
14
-
version: z.ZodLiteral<3>;
15
-
data: typeof common.cid;
16
-
rev: z.ZodString;
17
-
prev: z.ZodNullable<typeof common.cid>;
18
-
}, z.core.$strip>;
19
-
export type UnsignedCommit = z.infer<UnsignedCommitType> & { sig?: never };
50
+
version: z.ZodLiteral<Version>;
51
+
data: CidSchema;
52
+
rev: Rev;
53
+
prev: NullableCidSchema;
54
+
};
55
+
type UnsignedCommitSchema = z.ZodObject<
56
+
CommitShape<3, z.ZodString>,
57
+
z.core.$strip
58
+
>;
59
+
type CommitSchema = z.ZodObject<
60
+
CommitShape<3, z.ZodString> & { sig: BytesSchema },
61
+
z.core.$strip
62
+
>;
63
+
type LegacyV2CommitSchema = z.ZodObject<
64
+
CommitShape<2, z.ZodOptional<z.ZodString>> & { sig: BytesSchema },
65
+
z.core.$strip
66
+
>;
67
+
type VersionedCommitSchema = z.ZodDiscriminatedUnion<
68
+
readonly [CommitSchema, LegacyV2CommitSchema],
69
+
"version"
70
+
>;
20
71
21
-
const commit: CommitType = z.object({
72
+
const cidSchema: CidSchema = z
73
+
.unknown().transform((obj, ctx): Cid => {
74
+
const cid = asCid(obj);
75
+
76
+
if (cid == null) {
77
+
ctx.addIssue({
78
+
code: "custom",
79
+
message: "Not a valid CID",
80
+
});
81
+
return z.NEVER;
82
+
}
83
+
84
+
return cid;
85
+
});
86
+
87
+
const bytesSchema: BytesSchema = z.custom<Uint8Array>((value) =>
88
+
value instanceof Uint8Array
89
+
);
90
+
const stringSchema: z.ZodString = z.string();
91
+
const arraySchema: z.ZodArray<z.ZodUnknown> = z.array(z.unknown());
92
+
const mapSchema: z.ZodRecord<z.ZodString, z.ZodUnknown> = z.record(
93
+
z.string(),
94
+
z.unknown(),
95
+
);
96
+
const unknownSchema: z.ZodUnknown = z.unknown();
97
+
98
+
const unsignedCommit: UnsignedCommitSchema = z.object({
22
99
did: z.string(),
23
100
version: z.literal(3),
24
-
data: common.cid,
101
+
data: cidSchema,
25
102
rev: z.string(),
26
-
prev: z.nullable(common.cid),
27
-
sig: common.bytes,
103
+
prev: z.nullable(cidSchema),
28
104
});
29
-
type CommitType = z.ZodObject<{
30
-
did: z.ZodString;
31
-
version: z.ZodLiteral<3>;
32
-
data: typeof common.cid;
33
-
rev: z.ZodString;
34
-
prev: z.ZodNullable<typeof common.cid>;
35
-
sig: typeof common.bytes;
36
-
}, z.core.$strip>;
37
-
export type Commit = z.infer<CommitType>;
105
+
export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never };
38
106
39
-
const legacyV2Commit: LegacyV2CommitType = z.object({
107
+
const commit: CommitSchema = z.object({
108
+
did: z.string(),
109
+
version: z.literal(3),
110
+
data: cidSchema,
111
+
rev: z.string(),
112
+
prev: z.nullable(cidSchema),
113
+
sig: bytesSchema,
114
+
});
115
+
export type Commit = z.infer<typeof commit>;
116
+
117
+
export type LegacyV2Commit = {
118
+
did: string;
119
+
version: 2;
120
+
data: Cid;
121
+
rev?: string | undefined;
122
+
prev: Cid | null;
123
+
sig: Uint8Array;
124
+
};
125
+
126
+
const legacyV2Commit: LegacyV2CommitSchema = z.object({
40
127
did: z.string(),
41
128
version: z.literal(2),
42
-
data: common.cid,
129
+
data: cidSchema,
43
130
rev: z.string().optional(),
44
-
prev: z.nullable(common.cid),
45
-
sig: common.bytes,
131
+
prev: z.nullable(cidSchema),
132
+
sig: bytesSchema,
46
133
});
47
-
type LegacyV2CommitType = z.ZodObject<{
48
-
did: z.ZodString;
49
-
version: z.ZodLiteral<2>;
50
-
data: typeof common.cid;
51
-
rev: z.ZodOptional<z.ZodString>;
52
-
prev: z.ZodNullable<typeof common.cid>;
53
-
sig: typeof common.bytes;
54
-
}, z.core.$strip>;
55
-
export type LegacyV2Commit = z.infer<LegacyV2CommitType>;
56
134
57
-
const versionedCommit: VersionedCommitType = z.discriminatedUnion("version", [
58
-
commit,
59
-
legacyV2Commit,
60
-
]);
61
-
type VersionedCommitType = z.ZodDiscriminatedUnion<
62
-
[CommitType, LegacyV2CommitType],
63
-
"version"
64
-
>;
65
-
export type VersionedCommit = z.infer<VersionedCommitType>;
135
+
export type VersionedCommit = Commit | LegacyV2Commit;
136
+
137
+
const versionedCommit: VersionedCommitSchema = z.discriminatedUnion(
138
+
"version",
139
+
[commit, legacyV2Commit],
140
+
);
66
141
67
142
export const schema = {
68
-
...common,
143
+
cid: cidSchema,
144
+
bytes: bytesSchema,
145
+
string: stringSchema,
146
+
array: arraySchema,
147
+
map: mapSchema,
148
+
unknown: unknownSchema,
69
149
commit,
70
150
legacyV2Commit,
71
151
versionedCommit,
···
73
153
74
154
export const def = {
75
155
...commonDef,
156
+
cid: {
157
+
name: "cid",
158
+
schema: schema.cid,
159
+
},
160
+
bytes: {
161
+
name: "bytes",
162
+
schema: schema.bytes,
163
+
},
164
+
string: {
165
+
name: "string",
166
+
schema: schema.string,
167
+
},
168
+
map: {
169
+
name: "map",
170
+
schema: schema.map,
171
+
},
172
+
unknown: {
173
+
name: "unknown",
174
+
schema: schema.unknown,
175
+
},
76
176
commit: {
77
177
name: "commit",
78
178
schema: schema.commit,
···
96
196
action: WriteOpAction.Create;
97
197
collection: string;
98
198
rkey: string;
99
-
record: RepoRecord;
199
+
record: RepoInputRecord;
100
200
};
101
201
102
202
export type RecordUpdateOp = {
103
203
action: WriteOpAction.Update;
104
204
collection: string;
105
205
rkey: string;
106
-
record: RepoRecord;
206
+
record: RepoInputRecord;
107
207
};
108
208
109
209
export type RecordDeleteOp = {
···
118
218
action: WriteOpAction.Create;
119
219
collection: string;
120
220
rkey: string;
121
-
cid: CID;
221
+
cid: Cid;
122
222
};
123
223
124
224
export type RecordUpdateDescript = {
125
225
action: WriteOpAction.Update;
126
226
collection: string;
127
227
rkey: string;
128
-
prev: CID;
129
-
cid: CID;
228
+
prev: Cid;
229
+
cid: Cid;
130
230
};
131
231
132
232
export type RecordDeleteDescript = {
133
233
action: WriteOpAction.Delete;
134
234
collection: string;
135
235
rkey: string;
136
-
cid: CID;
236
+
cid: Cid;
137
237
};
138
238
139
239
export type RecordWriteDescript =
···
147
247
// ---------------
148
248
149
249
export type CommitData = {
150
-
cid: CID;
250
+
cid: Cid;
151
251
rev: string;
152
252
since: string | null;
153
-
prev: CID | null;
253
+
prev: Cid | null;
154
254
newBlocks: BlockMap;
155
255
relevantBlocks: BlockMap;
156
256
removedCids: CidSet;
···
163
263
export type CollectionContents = Record<string, RepoRecord>;
164
264
export type RepoContents = Record<string, CollectionContents>;
165
265
166
-
export type RepoRecordWithCid = { cid: CID; value: RepoRecord };
266
+
export type RepoRecordWithCid = { cid: Cid; value: RepoRecord };
167
267
export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>;
168
268
export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>;
169
269
170
-
export type DatastoreContents = Record<string, CID>;
270
+
export type DatastoreContents = Record<string, Cid>;
171
271
172
272
export type RecordPath = {
173
273
collection: string;
···
177
277
export type RecordCidClaim = {
178
278
collection: string;
179
279
rkey: string;
180
-
cid: CID | null;
280
+
cid: Cid | null;
181
281
};
182
282
183
283
export type RecordClaim = {
···
200
300
};
201
301
202
302
export type CarBlock = {
203
-
cid: CID;
303
+
cid: Cid;
204
304
bytes: Uint8Array;
205
305
};
+46
-14
repo/util.ts
+46
-14
repo/util.ts
···
1
-
import * as cbor from "@ipld/dag-cbor";
2
-
import { cborDecode, check, cidForCbor, schema, TID } from "@atp/common";
1
+
import {
2
+
cidForLex,
3
+
decode as decodeLexCbor,
4
+
encode as encodeLexCbor,
5
+
type LexValue as EncodableLexValue,
6
+
} from "@atp/lex/cbor";
7
+
import { asCid, type Cid } from "@atp/lex/data";
8
+
import { check, schema, TID } from "@atp/common";
3
9
import * as crypto from "@atp/crypto";
4
10
import type { Keypair } from "@atp/crypto";
5
-
import {
6
-
ipldToLex,
7
-
lexToIpld,
8
-
type LexValue,
9
-
type RepoRecord,
10
-
} from "@atp/lexicon";
11
+
import { BlobRef as LexiconBlobRef } from "@atp/lexicon";
11
12
import type { DataDiff } from "./data-diff.ts";
12
13
import {
13
14
type Commit,
14
15
type LegacyV2Commit,
16
+
type LexValue,
15
17
type RecordCreateDescript,
16
18
type RecordDeleteDescript,
17
19
type RecordPath,
18
20
type RecordUpdateDescript,
19
21
type RecordWriteDescript,
22
+
type RepoInputRecord,
23
+
type RepoInputValue,
24
+
type RepoRecord,
20
25
type UnsignedCommit,
21
26
WriteOpAction,
22
27
} from "./types.ts";
23
-
import type { CID } from "multiformats/basics";
24
28
25
29
/**
26
30
* Converts a DataDiff of a repo three arrays of RecordWriteDescripts,
···
100
104
unsigned: UnsignedCommit,
101
105
keypair: Keypair,
102
106
): Commit => {
103
-
const encoded = cbor.encode(unsigned);
107
+
const encoded = encodeLexCbor(
108
+
lexToCborValue(unsigned) as EncodableLexValue,
109
+
);
104
110
const sig = keypair.sign(encoded);
105
111
return {
106
112
...unsigned,
···
117
123
didKey: string,
118
124
): boolean => {
119
125
const { sig, ...rest } = commit;
120
-
const encoded = cbor.encode(rest);
126
+
const encoded = encodeLexCbor(
127
+
lexToCborValue(rest) as EncodableLexValue,
128
+
);
121
129
return crypto.verifySignature(didKey, encoded, sig as Uint8Array);
122
130
};
123
131
132
+
export const lexToCborValue = (value: RepoInputValue): unknown => {
133
+
if (Array.isArray(value)) {
134
+
return value.map((item) => lexToCborValue(item));
135
+
}
136
+
if (value && typeof value === "object") {
137
+
if (value instanceof LexiconBlobRef) {
138
+
return value.original;
139
+
}
140
+
if (asCid(value) || value instanceof Uint8Array) {
141
+
return value;
142
+
}
143
+
const mapped: Record<string, unknown> = {};
144
+
for (const [key, item] of Object.entries(value)) {
145
+
if (item !== undefined) {
146
+
mapped[key] = lexToCborValue(item);
147
+
}
148
+
}
149
+
return mapped;
150
+
}
151
+
return value;
152
+
};
153
+
124
154
/**
125
155
* Converts CBOR-encoded bytes to a LexValue using {@linkcode ipldToLex}.
126
156
*/
127
157
export const cborToLex = (val: Uint8Array): LexValue => {
128
-
return ipldToLex(cborDecode(val));
158
+
return decodeLexCbor(val) as LexValue;
129
159
};
130
160
131
161
/**
···
140
170
return parsed as RepoRecord;
141
171
};
142
172
143
-
export const cidForRecord = async (val: LexValue): Promise<CID> => {
144
-
return await cidForCbor(lexToIpld(val));
173
+
export const cidForRecord = async (val: RepoInputRecord): Promise<Cid> => {
174
+
return await cidForLex(
175
+
lexToCborValue(val) as EncodableLexValue,
176
+
);
145
177
};
146
178
147
179
export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
+6
-7
sync/events.ts
+6
-7
sync/events.ts
···
1
-
import type { CID } from "multiformats/cid";
2
1
import type { DidDocument } from "@atp/identity";
3
-
import type { RepoRecord } from "@atp/lexicon";
4
-
import type { BlockMap } from "@atp/repo";
2
+
import type { Cid } from "@atp/lex/data";
3
+
import type { BlockMap, RepoRecord } from "@atp/repo";
5
4
import type { AtUri } from "@atp/syntax";
6
5
7
6
/** Broad sync event type for all sync events */
···
24
23
export type CommitMeta = {
25
24
seq: number;
26
25
time: string;
27
-
commit: CID;
26
+
commit: Cid;
28
27
blocks: BlockMap;
29
28
rev: string;
30
29
uri: AtUri;
···
40
39
export type Create = CommitMeta & {
41
40
event: "create";
42
41
record: RepoRecord;
43
-
cid: CID;
42
+
cid: Cid;
44
43
};
45
44
46
45
/** {@link CommitEvt} for record updates/edits */
47
46
export type Update = CommitMeta & {
48
47
event: "update";
49
48
record: RepoRecord;
50
-
cid: CID;
49
+
cid: Cid;
51
50
};
52
51
53
52
/** {@link CommitEvt} for record deletions */
···
72
71
time: string;
73
72
event: "sync";
74
73
did: string;
75
-
cid: CID;
74
+
cid: Cid;
76
75
rev: string;
77
76
blocks: BlockMap;
78
77
};
+2
-2
sync/firehose/index.ts
+2
-2
sync/firehose/index.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { WebSocketOptions } from "@atp/xrpc-server";
3
3
import { createDeferrable, type Deferrable, wait } from "@atp/common";
4
4
import {
···
405
405
};
406
406
});
407
407
const key = await idResolver.did.resolveAtprotoKey(did, forceKeyRefresh);
408
-
const verifiedCids: Record<string, CID | null> = {};
408
+
const verifiedCids: Record<string, Cid | null> = {};
409
409
try {
410
410
const results = await verifyProofs(evt.blocks, claims, did, key);
411
411
results.verified.forEach((op) => {
+5
-5
sync/firehose/lexicons.ts
+5
-5
sync/firehose/lexicons.ts
···
1
1
import type { IncomingMessage } from "node:http";
2
-
import type { CID } from "multiformats/cid";
2
+
import type { Cid } from "@atp/lex/data";
3
3
import { type LexiconDoc, Lexicons } from "@atp/lexicon";
4
4
import type { Auth, ErrorFrame } from "@atp/xrpc-server";
5
5
···
51
51
/** The repo this event comes from. */
52
52
repo: string;
53
53
/** Repo commit object CID. */
54
-
commit: CID;
54
+
commit: Cid;
55
55
/** DEPRECATED -- unused. WARNING -- nullable and optional; stick with optional to ensure golang interoperability. */
56
-
prev?: CID | null;
56
+
prev?: Cid | null;
57
57
/** The rev of the emitted commit. Note that this information is also in the commit object included in blocks, unless this is a tooBig event. */
58
58
rev: string;
59
59
/** The rev of the last emitted commit from this repo (if any). */
···
61
61
/** CAR file containing relevant blocks, as a diff since the previous repo state. */
62
62
blocks: Uint8Array;
63
63
ops: RepoOp[];
64
-
blobs: CID[];
64
+
blobs: Cid[];
65
65
/** Timestamp of when this message was originally broadcast. */
66
66
time: string;
67
67
[k: string]: unknown;
···
155
155
action: "create" | "update" | "delete" | string;
156
156
path: string;
157
157
/** For creates and updates, the new record CID. For deletions, null. */
158
-
cid: CID | null;
158
+
cid: Cid | null;
159
159
[k: string]: unknown;
160
160
}
161
161
+3
-3
sync/tests/mock-relay.ts
+3
-3
sync/tests/mock-relay.ts
···
1
-
import type { CID } from "multiformats/cid";
1
+
import type { Cid } from "@atp/lex/data";
2
2
import type { RepoEvent } from "../firehose/lexicons.ts";
3
3
4
4
export interface MockFirehoseServerOptions {
···
167
167
seq,
168
168
time: new Date().toISOString(),
169
169
repo,
170
-
commit: mockCID as unknown as CID,
170
+
commit: mockCID as unknown as Cid,
171
171
rev: `rev-${seq}`,
172
172
ops: [{
173
173
action,
174
174
path: `${collection}/${rkey}`,
175
-
cid: action === "delete" ? null : mockCID as unknown as CID,
175
+
cid: action === "delete" ? null : mockCID as unknown as Cid,
176
176
}],
177
177
blocks: new Uint8Array(
178
178
JSON.stringify(record).split("").map((c) => c.charCodeAt(0)),