Suite of AT Protocol TypeScript libraries built on web standards
feat: lex resolver and installer
+1833
-46
+2
deno.lock
+2
deno.lock
+4
-4
lex-gen/builder/def-builder.ts
lex/build/def-builder.ts
+4
-4
lex-gen/builder/def-builder.ts
lex/build/def-builder.ts
···
25
25
LexiconUnknown,
26
26
MainLexiconDefinition,
27
27
NamedLexiconDefinition,
28
-
} from "@atp/lex/document";
28
+
} from "../document/mod.ts";
29
29
30
30
import {
31
31
getPublicIdentifiers,
···
753
753
754
754
if (def.knownValues?.length) {
755
755
return (
756
-
def.knownValues.map((v) => JSON.stringify(v)).join(" | ") +
756
+
def.knownValues.map((v: string) => JSON.stringify(v)).join(" | ") +
757
757
" | l.UnknownString"
758
758
);
759
759
}
···
813
813
}
814
814
815
815
const refs = await Promise.all(
816
-
def.refs.map(async (ref) => {
816
+
def.refs.map(async (ref: string) => {
817
817
const { varName, typeName } = await this.refResolver.resolve(ref);
818
818
return this.pure(
819
819
`l.typedRef<${typeName}>(() => ${varName})`,
···
828
828
829
829
private async compileRefUnionType(def: LexiconRefUnion): Promise<string> {
830
830
const types = await Promise.all(
831
-
def.refs.map(async (ref) => {
831
+
def.refs.map(async (ref: string) => {
832
832
const { typeName } = await this.refResolver.resolve(ref);
833
833
return `l.TypedRef<${typeName}>`;
834
834
}),
+1
-1
lex-gen/builder/directory-indexer.ts
lex/build/directory-indexer.ts
+1
-1
lex-gen/builder/directory-indexer.ts
lex/build/directory-indexer.ts
lex-gen/builder/filter.ts
lex/build/filter.ts
lex-gen/builder/filter.ts
lex/build/filter.ts
+2
-2
lex-gen/builder/filtered-indexer.ts
lex/build/filtered-indexer.ts
+2
-2
lex-gen/builder/filtered-indexer.ts
lex/build/filtered-indexer.ts
···
1
-
import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document";
1
+
import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts";
2
2
import type { Filter } from "./filter.ts";
3
3
4
4
export class FilteredIndexer implements LexiconIndexer, AsyncDisposable {
5
-
protected readonly returned = new Set<string>();
5
+
protected readonly returned: Set<string> = new Set<string>();
6
6
7
7
constructor(
8
8
readonly indexer: LexiconIndexer & AsyncIterable<LexiconDocument>,
+1
-1
lex-gen/builder/lex-builder.ts
lex/build/lex-builder.ts
+1
-1
lex-gen/builder/lex-builder.ts
lex/build/lex-builder.ts
···
1
1
import { mkdir, rm, stat, writeFile } from "node:fs/promises";
2
2
import { dirname, join, resolve } from "node:path";
3
3
import { IndentationText, Project } from "ts-morph";
4
-
import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document";
4
+
import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts";
5
5
import { buildFilter, type BuildFilterOptions } from "./filter.ts";
6
6
import { FilteredIndexer } from "./filtered-indexer.ts";
7
7
import { LexDefBuilder, type LexDefBuilderOptions } from "./def-builder.ts";
+1
-22
lex-gen/builder/mod.ts
+1
-22
lex-gen/builder/mod.ts
···
1
-
export * from "./filter.ts";
2
-
export * from "./directory-indexer.ts";
3
-
export * from "./filtered-indexer.ts";
4
-
export * from "./lex-builder.ts";
5
-
6
-
export type {
7
-
LexBuilderLoadOptions,
8
-
LexBuilderOptions,
9
-
LexBuilderSaveOptions,
10
-
} from "./lex-builder.ts";
11
-
12
-
export async function build(
13
-
options:
14
-
& import("./lex-builder.ts").LexBuilderOptions
15
-
& import("./lex-builder.ts").LexBuilderLoadOptions
16
-
& import("./lex-builder.ts").LexBuilderSaveOptions,
17
-
): Promise<void> {
18
-
const { LexBuilder } = await import("./lex-builder.ts");
19
-
const builder = new LexBuilder(options);
20
-
await builder.load(options);
21
-
await builder.save(options);
22
-
}
1
+
export * from "@atp/lex/build";
+1
-1
lex-gen/builder/ref-resolver.ts
lex/build/ref-resolver.ts
+1
-1
lex-gen/builder/ref-resolver.ts
lex/build/ref-resolver.ts
···
1
1
import assert from "node:assert";
2
2
import { join } from "node:path";
3
3
import type { SourceFile } from "ts-morph";
4
-
import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document";
4
+
import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts";
5
5
import { isReservedWord, isSafeIdentifier } from "./ts-lang.ts";
6
6
import {
7
7
asRelativePath,
lex-gen/builder/ts-lang.ts
lex/build/ts-lang.ts
lex-gen/builder/ts-lang.ts
lex/build/ts-lang.ts
lex-gen/builder/util.ts
lex/build/util.ts
lex-gen/builder/util.ts
lex/build/util.ts
+1
-1
lex-gen/cmd/build.ts
lex/cli/build.ts
+1
-1
lex-gen/cmd/build.ts
lex/cli/build.ts
+1
-2
lex-gen/cmd/index.ts
+1
-2
lex-gen/cmd/index.ts
···
1
-
import build from "./build.ts";
2
1
import genMd from "./gen-md.ts";
3
2
import genApi from "./gen-api.ts";
4
3
import genServer from "./gen-server.ts";
5
4
import genTsObj from "./gen-ts-obj.ts";
6
5
7
-
export { build, genApi, genMd, genServer, genTsObj };
6
+
export { genApi, genMd, genServer, genTsObj };
+16
-5
lex-gen/mod.ts
+16
-5
lex-gen/mod.ts
···
11
11
*
12
12
* @example
13
13
* ```bash
14
-
* lex-gen build -i ./lexicons -o ./lex
14
+
* lex-gen api -i ./lexicons -o ./api
15
15
* ```
16
16
*
17
17
* @module
18
18
*/
19
19
import { Command } from "@cliffy/command";
20
-
import { build, genApi, genMd, genServer, genTsObj } from "./cmd/index.ts";
21
20
import { defineLexiconConfig, loadLexiconConfig } from "./config.ts";
22
21
import process from "node:process";
23
22
24
23
export { defineLexiconConfig, loadLexiconConfig };
25
-
export { build as buildCommand } from "./builder/mod.ts";
24
+
export { build } from "./builder/mod.ts";
26
25
export type {
27
26
LexBuilderLoadOptions,
28
27
LexBuilderOptions,
···
38
37
} from "./types.ts";
39
38
40
39
const isDeno = typeof Deno !== "undefined";
40
+
const args = isDeno ? Deno.args : process.argv.slice(2);
41
+
42
+
const [
43
+
{ default: genApi },
44
+
{ default: genMd },
45
+
{ default: genServer },
46
+
{ default: genTsObj },
47
+
] = await Promise.all([
48
+
import("./cmd/gen-api.ts"),
49
+
import("./cmd/gen-md.ts"),
50
+
import("./cmd/gen-server.ts"),
51
+
import("./cmd/gen-ts-obj.ts"),
52
+
]);
41
53
42
54
await new Command()
43
55
.name("lex-gen")
···
46
58
.command("md", genMd)
47
59
.command("server", genServer)
48
60
.command("ts-obj", genTsObj)
49
-
.command("build", build)
50
-
.parse(isDeno ? Deno.args : process.argv.slice(2));
61
+
.parse(args);
+1
-1
lex-gen/tests/lex-builder_test.ts
lex/tests/lex-builder_test.ts
+1
-1
lex-gen/tests/lex-builder_test.ts
lex/tests/lex-builder_test.ts
···
1
1
import { assertRejects, assertStringIncludes } from "@std/assert";
2
2
import { join } from "node:path";
3
-
import { LexBuilder } from "../builder/lex-builder.ts";
3
+
import { LexBuilder } from "../build/lex-builder.ts";
4
4
5
5
Deno.test({
6
6
name: "save writes files under output directory and rejects existing files",
+1
-1
lex-gen/tests/method-generation_test.ts
lex/tests/method-generation_test.ts
+1
-1
lex-gen/tests/method-generation_test.ts
lex/tests/method-generation_test.ts
···
5
5
lexiconDocumentSchema,
6
6
type LexiconIndexer,
7
7
} from "@atp/lex/document";
8
-
import { LexDefBuilder } from "../builder/def-builder.ts";
8
+
import { LexDefBuilder } from "../build/def-builder.ts";
9
9
10
10
class DummyIndexer implements LexiconIndexer, AsyncIterable<LexiconDocument> {
11
11
readonly #docs: Map<string, LexiconDocument>;
+22
lex/build/mod.ts
+22
lex/build/mod.ts
···
1
+
export * from "./filter.ts";
2
+
export * from "./directory-indexer.ts";
3
+
export * from "./filtered-indexer.ts";
4
+
export * from "./lex-builder.ts";
5
+
6
+
export type {
7
+
LexBuilderLoadOptions,
8
+
LexBuilderOptions,
9
+
LexBuilderSaveOptions,
10
+
} from "./lex-builder.ts";
11
+
12
+
export async function build(
13
+
options:
14
+
& import("./lex-builder.ts").LexBuilderOptions
15
+
& import("./lex-builder.ts").LexBuilderLoadOptions
16
+
& import("./lex-builder.ts").LexBuilderSaveOptions,
17
+
): Promise<void> {
18
+
const { LexBuilder } = await import("./lex-builder.ts");
19
+
const builder = new LexBuilder(options);
20
+
await builder.load(options);
21
+
await builder.save(options);
22
+
}
+15
lex/cli.ts
+15
lex/cli.ts
···
1
+
import { Command } from "@cliffy/command";
2
+
import buildCommand from "./cli/build.ts";
3
+
import installCommand from "./cli/install.ts";
4
+
5
+
const command = new Command()
6
+
.name("lex")
7
+
.description("AT Protocol Lex tools")
8
+
.command("build", buildCommand)
9
+
.command("install", installCommand);
10
+
11
+
if (import.meta.main) {
12
+
await command.parse(Deno.args);
13
+
}
14
+
15
+
export default command;
+43
lex/cli/install.ts
+43
lex/cli/install.ts
···
1
+
import { Command } from "@cliffy/command";
2
+
3
+
const command = new Command()
4
+
.description("Fetch and install lexicon documents")
5
+
.arguments("[additions...:string]")
6
+
.option(
7
+
"-i, --lexicons <lexicons>",
8
+
"directory containing lexicon JSON files",
9
+
{ default: "./lexicons" },
10
+
)
11
+
.option(
12
+
"-m, --manifest <manifest>",
13
+
"path to lexicons manifest file",
14
+
{ default: "./lexicons.json" },
15
+
)
16
+
.option(
17
+
"--save [save:boolean]",
18
+
"write the updated lexicons manifest to disk",
19
+
{ default: true },
20
+
)
21
+
.option(
22
+
"--update",
23
+
"re-resolve and re-install existing lexicons instead of reusing local files",
24
+
{ default: false },
25
+
)
26
+
.option(
27
+
"--ci",
28
+
"error if the current install would change the manifest",
29
+
{ default: false },
30
+
)
31
+
.action(async (opts, ...additions: string[]) => {
32
+
const { install } = await import("../installer/mod.ts");
33
+
await install({
34
+
add: additions,
35
+
lexicons: opts.lexicons,
36
+
manifest: opts.manifest,
37
+
save: opts.save,
38
+
update: opts.update,
39
+
ci: opts.ci,
40
+
});
41
+
});
42
+
43
+
export default command;
+15
-3
lex/deno.json
+15
-3
lex/deno.json
···
4
4
"exports": {
5
5
".": "./mod.ts",
6
6
"./cbor": "./cbor/mod.ts",
7
-
"./document": "./document/mod.ts"
7
+
"./document": "./document/mod.ts",
8
+
"./build": "./build/mod.ts",
9
+
"./installer": "./installer/mod.ts",
10
+
"./resolver": "./resolver/mod.ts"
8
11
},
9
12
"license": "MIT",
10
13
"imports": {
14
+
"@cliffy/command": "jsr:@cliffy/command@^1.0.0-rc.8",
11
15
"cborg": "npm:cborg@^4.2.15",
12
16
"multiformats/cid": "npm:multiformats@^13.4.1/cid",
13
17
"multiformats/hashes/digest": "npm:multiformats@^13.4.1/hashes/digest",
14
-
"multiformats/hashes/sha2": "npm:multiformats@^13.4.1/hashes/sha2"
18
+
"multiformats/hashes/sha2": "npm:multiformats@^13.4.1/hashes/sha2",
19
+
"ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0"
20
+
},
21
+
"test": {
22
+
"permissions": {
23
+
"env": true,
24
+
"read": true,
25
+
"write": true
26
+
}
15
27
},
16
28
"lint": {
17
29
"rules": {
18
-
"exclude": ["no-explicit-any", "no-slow-types", "require-await"]
30
+
"exclude": ["no-explicit-any"]
19
31
}
20
32
}
21
33
}
+6
-2
lex/document/lexicon.ts
+6
-2
lex/document/lexicon.ts
···
1
-
import { l } from "../mod.ts";
1
+
import * as l from "../external.ts";
2
2
3
3
const bool: l.BooleanSchema = l.boolean();
4
4
const int: l.IntegerSchema = l.integer();
···
183
183
required?: string[];
184
184
properties: Record<string, unknown>;
185
185
}> = {
186
-
check: (v) => !v.required || v.required.every((k) => k in v.properties),
186
+
check: (
187
+
value: { required?: string[]; properties: Record<string, unknown> },
188
+
) =>
189
+
!value.required ||
190
+
value.required.every((key: string) => key in value.properties),
187
191
message: "All required parameters must be defined in properties",
188
192
path: "required",
189
193
};
+24
lex/installer/fs.ts
+24
lex/installer/fs.ts
···
1
+
import { dirname } from "node:path";
2
+
3
+
export async function readJsonFile(path: string): Promise<unknown> {
4
+
const contents = await Deno.readTextFile(path);
5
+
return JSON.parse(contents);
6
+
}
7
+
8
+
export async function writeJsonFile(
9
+
path: string,
10
+
data: unknown,
11
+
): Promise<void> {
12
+
await Deno.mkdir(dirname(path), { recursive: true });
13
+
await Deno.writeTextFile(path, JSON.stringify(data, null, 2));
14
+
}
15
+
16
+
export function isEnoentError(err: unknown): boolean {
17
+
return err instanceof Deno.errors.NotFound ||
18
+
(
19
+
err instanceof Error &&
20
+
"code" in err &&
21
+
typeof err.code === "string" &&
22
+
err.code === "ENOENT"
23
+
);
24
+
}
+45
lex/installer/install.ts
+45
lex/installer/install.ts
···
1
+
import { LexInstaller } from "./lex-installer.ts";
2
+
import { isEnoentError, readJsonFile } from "./fs.ts";
3
+
import { LexInstallerError } from "./lex-installer-error.ts";
4
+
import {
5
+
type LexiconsManifest,
6
+
lexiconsManifestSchema,
7
+
} from "./lexicons-manifest.ts";
8
+
import type { LexInstallerOptions } from "./lex-installer.ts";
9
+
10
+
export interface LexInstallOptions extends LexInstallerOptions {
11
+
add?: string[];
12
+
save?: boolean;
13
+
ci?: boolean;
14
+
}
15
+
16
+
export async function install(options: LexInstallOptions): Promise<void> {
17
+
const manifest = await readJsonFile(options.manifest).then(
18
+
(json) => lexiconsManifestSchema.parse(json) as LexiconsManifest,
19
+
(cause: unknown) => {
20
+
if (isEnoentError(cause)) return undefined;
21
+
throw new LexInstallerError("Failed to read lexicons manifest", {
22
+
cause,
23
+
});
24
+
},
25
+
);
26
+
27
+
const installer = new LexInstaller(options);
28
+
try {
29
+
await installer.install({
30
+
additions: new Set(options.add ?? []),
31
+
manifest,
32
+
write: !options.ci,
33
+
});
34
+
35
+
if (options.ci) {
36
+
if (!manifest || !installer.equals(manifest)) {
37
+
throw new LexInstallerError("Lexicons manifest is out of date");
38
+
}
39
+
} else if (options.save) {
40
+
await installer.save();
41
+
}
42
+
} finally {
43
+
await installer[Symbol.asyncDispose]();
44
+
}
45
+
}
+10
lex/installer/lex-installer-error.ts
+10
lex/installer/lex-installer-error.ts
+369
lex/installer/lex-installer.ts
+369
lex/installer/lex-installer.ts
···
1
+
import { join } from "node:path";
2
+
import { type Cid, cidForLex } from "../cbor/mod.ts";
3
+
import { parseCid } from "../data/cid.ts";
4
+
import type { LexValue } from "../data/lex.ts";
5
+
import type {
6
+
LexiconDocument,
7
+
LexiconParameters,
8
+
LexiconPermission,
9
+
LexiconRef,
10
+
LexiconRefUnion,
11
+
LexiconUnknown,
12
+
MainLexiconDefinition,
13
+
NamedLexiconDefinition,
14
+
} from "../document/mod.ts";
15
+
import { LexiconDirectoryIndexer } from "../build/mod.ts";
16
+
import {
17
+
LexResolver,
18
+
type LexResolverFetchOptions,
19
+
type LexResolverOptions,
20
+
type LexResolverResult,
21
+
} from "../resolver/mod.ts";
22
+
import { AtUri, ensureValidDid, NSID } from "@atp/syntax";
23
+
import { isEnoentError, writeJsonFile } from "./fs.ts";
24
+
import { LexInstallerError } from "./lex-installer-error.ts";
25
+
import {
26
+
type LexiconsManifest,
27
+
normalizeLexiconsManifest,
28
+
} from "./lexicons-manifest.ts";
29
+
import { NsidMap } from "./nsid-map.ts";
30
+
import { NsidSet } from "./nsid-set.ts";
31
+
32
+
const LEXICON_COLLECTION = "com.atproto.lexicon.schema";
33
+
34
+
export interface LexInstallerResolver {
35
+
resolve(nsidStr: NSID | string): Promise<AtUri>;
36
+
fetch(
37
+
uriStr: AtUri | string,
38
+
options?: LexResolverFetchOptions,
39
+
): Promise<LexResolverResult>;
40
+
}
41
+
42
+
export interface LexInstallerOptions extends LexResolverOptions {
43
+
lexicons: string;
44
+
manifest: string;
45
+
update?: boolean;
46
+
resolver?: LexInstallerResolver;
47
+
}
48
+
49
+
export interface LexInstallerFetchResult {
50
+
lexicon: LexiconDocument;
51
+
cid: Cid;
52
+
}
53
+
54
+
export class LexInstaller implements AsyncDisposable {
55
+
protected readonly lexiconResolver: LexInstallerResolver;
56
+
protected readonly indexer: LexiconDirectoryIndexer;
57
+
protected readonly documents: NsidMap<LexiconDocument> = new NsidMap<
58
+
LexiconDocument
59
+
>();
60
+
protected readonly manifest: LexiconsManifest = {
61
+
version: 1,
62
+
lexicons: [],
63
+
resolutions: {},
64
+
};
65
+
66
+
constructor(protected readonly options: LexInstallerOptions) {
67
+
this.lexiconResolver = options.resolver ?? new LexResolver(options);
68
+
this.indexer = new LexiconDirectoryIndexer({
69
+
lexicons: options.lexicons,
70
+
});
71
+
}
72
+
73
+
async [Symbol.asyncDispose](): Promise<void> {
74
+
await this.indexer[Symbol.asyncDispose]();
75
+
}
76
+
77
+
equals(manifest: LexiconsManifest): boolean {
78
+
return JSON.stringify(normalizeLexiconsManifest(manifest)) ===
79
+
JSON.stringify(normalizeLexiconsManifest(this.manifest));
80
+
}
81
+
82
+
async install(
83
+
{
84
+
additions,
85
+
manifest,
86
+
write = true,
87
+
}: {
88
+
additions?: Iterable<string>;
89
+
manifest?: LexiconsManifest;
90
+
write?: boolean;
91
+
} = {},
92
+
): Promise<void> {
93
+
const roots = new NsidMap<AtUri | null>();
94
+
95
+
for (const addition of new Set(additions ?? [])) {
96
+
const [nsid, uri] = addition.startsWith("at://")
97
+
? ((parsedUri) => [NSID.from(parsedUri.rkey), parsedUri] as const)(
98
+
new AtUri(addition),
99
+
)
100
+
: [NSID.from(addition), null] as const;
101
+
102
+
if (roots.has(nsid)) {
103
+
throw new LexInstallerError(
104
+
`Duplicate lexicon addition: ${nsid} (${
105
+
roots.get(nsid) ?? addition
106
+
})`,
107
+
);
108
+
}
109
+
110
+
roots.set(nsid, uri);
111
+
}
112
+
113
+
if (manifest) {
114
+
for (const lexiconId of manifest.lexicons) {
115
+
const nsid = NSID.from(lexiconId);
116
+
if (roots.has(nsid)) continue;
117
+
118
+
const resolution = manifest.resolutions[lexiconId];
119
+
roots.set(nsid, resolution ? new AtUri(resolution.uri) : null);
120
+
}
121
+
}
122
+
123
+
await Promise.all(
124
+
Array.from(roots, async ([nsid, uri]) => {
125
+
const { lexicon } = uri
126
+
? await this.installFromUri(uri, { write })
127
+
: await this.installFromNsid(nsid, { write });
128
+
this.manifest.lexicons.push(lexicon.id);
129
+
}),
130
+
);
131
+
132
+
let installedCount = 0;
133
+
do {
134
+
const missing = Array.from(this.getMissingIds());
135
+
installedCount = missing.length;
136
+
137
+
await Promise.all(
138
+
missing.map(async (nsid) => {
139
+
const resolution = manifest?.resolutions[nsid.toString()];
140
+
if (resolution?.uri) {
141
+
await this.installFromUri(new AtUri(resolution.uri), { write });
142
+
} else {
143
+
await this.installFromNsid(nsid, { write });
144
+
}
145
+
}),
146
+
);
147
+
} while (installedCount > 0);
148
+
}
149
+
150
+
async save(): Promise<void> {
151
+
await writeJsonFile(
152
+
this.options.manifest,
153
+
normalizeLexiconsManifest(this.manifest),
154
+
);
155
+
}
156
+
157
+
async fetch(
158
+
uri: AtUri,
159
+
{ write = true }: { write?: boolean } = {},
160
+
): Promise<LexInstallerFetchResult> {
161
+
const { lexicon, cid } = await this.lexiconResolver.fetch(uri, {
162
+
noCache: this.options.update,
163
+
});
164
+
const normalizedCid = parseCid(cid.toString());
165
+
if (write) {
166
+
const filePath = join(this.options.lexicons, ...lexicon.id.split(".")) +
167
+
".json";
168
+
await writeJsonFile(filePath, lexicon);
169
+
}
170
+
return { lexicon, cid: normalizedCid };
171
+
}
172
+
173
+
protected getMissingIds(): NsidSet {
174
+
const missing = new NsidSet();
175
+
176
+
for (const document of this.documents.values()) {
177
+
for (const nsid of listDocumentNsidRefs(document)) {
178
+
if (!this.documents.has(nsid)) {
179
+
missing.add(nsid);
180
+
}
181
+
}
182
+
}
183
+
184
+
return missing;
185
+
}
186
+
187
+
protected async installFromNsid(
188
+
nsid: NSID,
189
+
options?: { write?: boolean },
190
+
): Promise<{ lexicon: LexiconDocument; uri: AtUri }> {
191
+
const uri = await this.lexiconResolver.resolve(nsid);
192
+
return this.installFromUri(uri, options);
193
+
}
194
+
195
+
protected async installFromUri(
196
+
uri: AtUri,
197
+
{ write = true }: { write?: boolean } = {},
198
+
): Promise<{ lexicon: LexiconDocument; uri: AtUri }> {
199
+
assertLexiconUri(uri);
200
+
201
+
const { lexicon, cid } = this.options.update
202
+
? await this.fetch(uri, { write })
203
+
: await this.indexer.get(uri.rkey).then(
204
+
async (existingLexicon) => ({
205
+
lexicon: existingLexicon,
206
+
cid: await cidForLexicon(existingLexicon),
207
+
}),
208
+
async (cause) => {
209
+
if (isEnoentError(cause)) return await this.fetch(uri, { write });
210
+
throw cause;
211
+
},
212
+
);
213
+
214
+
this.documents.set(NSID.from(lexicon.id), lexicon);
215
+
this.manifest.resolutions[lexicon.id] = {
216
+
cid: cid.toString(),
217
+
uri: uri.toString(),
218
+
};
219
+
220
+
return { lexicon, uri };
221
+
}
222
+
}
223
+
224
+
function assertLexiconUri(uri: AtUri): void {
225
+
if (uri.collection !== LEXICON_COLLECTION) {
226
+
throw new LexInstallerError(
227
+
`Invalid lexicon URI collection for ${uri}: expected ${LEXICON_COLLECTION}`,
228
+
);
229
+
}
230
+
231
+
try {
232
+
ensureValidDid(uri.host);
233
+
} catch (cause) {
234
+
throw new LexInstallerError(
235
+
`Invalid lexicon URI authority for ${uri}: expected DID authority`,
236
+
{ cause },
237
+
);
238
+
}
239
+
}
240
+
241
+
function cidForLexicon(lexicon: LexiconDocument): Promise<Cid> {
242
+
return cidForLex(lexicon as unknown as LexValue);
243
+
}
244
+
245
+
function* listDocumentNsidRefs(doc: LexiconDocument): Iterable<NSID> {
246
+
try {
247
+
for (const def of Object.values(doc.defs)) {
248
+
if (!def) continue;
249
+
for (const ref of defRefs(def)) {
250
+
const [nsid] = ref.split("#", 1);
251
+
if (nsid) {
252
+
yield NSID.from(nsid);
253
+
}
254
+
}
255
+
}
256
+
} catch (cause) {
257
+
throw new LexInstallerError(
258
+
`Failed to extract refs from lexicon ${doc.id}`,
259
+
{ cause },
260
+
);
261
+
}
262
+
}
263
+
264
+
function* defRefs(
265
+
def:
266
+
| MainLexiconDefinition
267
+
| NamedLexiconDefinition
268
+
| LexiconPermission
269
+
| LexiconParameters
270
+
| LexiconRef
271
+
| LexiconRefUnion
272
+
| LexiconUnknown,
273
+
): Iterable<string> {
274
+
switch (def.type) {
275
+
case "string":
276
+
for (const value of def.knownValues ?? []) {
277
+
const [nsid, hash, extra] = value.split("#");
278
+
if (!nsid || !hash || extra) continue;
279
+
try {
280
+
NSID.from(nsid);
281
+
yield value;
282
+
} catch {
283
+
continue;
284
+
}
285
+
}
286
+
return;
287
+
case "array":
288
+
yield* defRefs(def.items);
289
+
return;
290
+
case "params":
291
+
case "object":
292
+
for (const property of Object.values(def.properties)) {
293
+
yield* defRefs(property);
294
+
}
295
+
return;
296
+
case "union":
297
+
yield* def.refs;
298
+
return;
299
+
case "ref":
300
+
yield def.ref;
301
+
return;
302
+
case "record":
303
+
yield* defRefs(def.record);
304
+
return;
305
+
case "procedure":
306
+
if (def.input?.schema) {
307
+
yield* defRefs(def.input.schema);
308
+
}
309
+
if (def.output?.schema) {
310
+
yield* defRefs(def.output.schema);
311
+
}
312
+
if (def.parameters) {
313
+
yield* defRefs(def.parameters);
314
+
}
315
+
return;
316
+
case "query":
317
+
if (def.output?.schema) {
318
+
yield* defRefs(def.output.schema);
319
+
}
320
+
if (def.parameters) {
321
+
yield* defRefs(def.parameters);
322
+
}
323
+
return;
324
+
case "subscription":
325
+
if (def.parameters) {
326
+
yield* defRefs(def.parameters);
327
+
}
328
+
if (def.message?.schema) {
329
+
yield* defRefs(def.message.schema);
330
+
}
331
+
return;
332
+
case "permission-set":
333
+
for (const permission of def.permissions) {
334
+
yield* defRefs(permission);
335
+
}
336
+
return;
337
+
case "permission":
338
+
if (def.resource === "rpc" && Array.isArray(def.lxm)) {
339
+
for (const lxm of def.lxm) {
340
+
if (typeof lxm === "string") {
341
+
yield lxm;
342
+
}
343
+
}
344
+
}
345
+
if (def.resource === "repo" && Array.isArray(def.collection)) {
346
+
for (const collection of def.collection) {
347
+
if (typeof collection === "string") {
348
+
yield collection;
349
+
}
350
+
}
351
+
}
352
+
return;
353
+
case "boolean":
354
+
case "blob":
355
+
case "bytes":
356
+
case "cid-link":
357
+
case "integer":
358
+
case "token":
359
+
case "unknown":
360
+
return;
361
+
default: {
362
+
throw new LexInstallerError(
363
+
`Unknown lexicon def type: ${
364
+
(def as { type?: string }).type ?? "unknown"
365
+
}`,
366
+
);
367
+
}
368
+
}
369
+
}
+57
lex/installer/lexicons-manifest.ts
+57
lex/installer/lexicons-manifest.ts
···
1
+
import * as l from "../external.ts";
2
+
3
+
export const lexiconsManifestSchema: l.ObjectSchema<{
4
+
version: l.LiteralSchema<1>;
5
+
lexicons: l.ArraySchema<l.StringSchema<{ format: "nsid" }>>;
6
+
resolutions: l.DictSchema<
7
+
l.StringSchema<{ format: "nsid" }>,
8
+
l.ObjectSchema<{
9
+
uri: l.StringSchema<{ format: "at-uri" }>;
10
+
cid: l.StringSchema<{ format: "cid" }>;
11
+
}>
12
+
>;
13
+
}> = l.object({
14
+
version: l.literal(1),
15
+
lexicons: l.array(l.string({ format: "nsid" })),
16
+
resolutions: l.dict(
17
+
l.string({ format: "nsid" }),
18
+
l.object({
19
+
uri: l.string({ format: "at-uri" }),
20
+
cid: l.string({ format: "cid" }),
21
+
}),
22
+
),
23
+
});
24
+
25
+
export interface LexiconsManifestResolution {
26
+
uri: string;
27
+
cid: string;
28
+
}
29
+
30
+
export interface LexiconsManifest {
31
+
version: 1;
32
+
lexicons: string[];
33
+
resolutions: Record<string, LexiconsManifestResolution>;
34
+
}
35
+
36
+
export function normalizeLexiconsManifest(
37
+
manifest: LexiconsManifest,
38
+
): LexiconsManifest {
39
+
return lexiconsManifestSchema.parse({
40
+
version: manifest.version,
41
+
lexicons: [...manifest.lexicons].sort(),
42
+
resolutions: Object.fromEntries(
43
+
Object.entries(manifest.resolutions)
44
+
.sort(compareObjectEntries)
45
+
.map(([key, value]) => [key, { uri: value.uri, cid: value.cid }]),
46
+
),
47
+
}) as LexiconsManifest;
48
+
}
49
+
50
+
function compareObjectEntries(
51
+
a: [string, unknown],
52
+
b: [string, unknown],
53
+
): number {
54
+
if (a[0] > b[0]) return 1;
55
+
if (a[0] < b[0]) return -1;
56
+
return 0;
57
+
}
+7
lex/installer/mod.ts
+7
lex/installer/mod.ts
+75
lex/installer/nsid-map.ts
+75
lex/installer/nsid-map.ts
···
1
+
import { NSID } from "@atp/syntax";
2
+
3
+
class MappedMap<K, V, I> {
4
+
readonly #map = new Map<I, V>();
5
+
6
+
constructor(
7
+
private readonly encodeKey: (key: K) => I,
8
+
private readonly decodeKey: (enc: I) => K,
9
+
) {}
10
+
11
+
get size(): number {
12
+
return this.#map.size;
13
+
}
14
+
15
+
clear(): void {
16
+
this.#map.clear();
17
+
}
18
+
19
+
set(key: K, value: V): this {
20
+
this.#map.set(this.encodeKey(key), value);
21
+
return this;
22
+
}
23
+
24
+
get(key: K): V | undefined {
25
+
return this.#map.get(this.encodeKey(key));
26
+
}
27
+
28
+
has(key: K): boolean {
29
+
return this.#map.has(this.encodeKey(key));
30
+
}
31
+
32
+
delete(key: K): boolean {
33
+
return this.#map.delete(this.encodeKey(key));
34
+
}
35
+
36
+
values(): IterableIterator<V> {
37
+
return this.#map.values();
38
+
}
39
+
40
+
*keys(): IterableIterator<K> {
41
+
for (const key of this.#map.keys()) {
42
+
yield this.decodeKey(key);
43
+
}
44
+
}
45
+
46
+
*entries(): IterableIterator<[K, V]> {
47
+
for (const [key, value] of this.#map.entries()) {
48
+
yield [this.decodeKey(key), value];
49
+
}
50
+
}
51
+
52
+
forEach(
53
+
callbackfn: (value: V, key: K, map: MappedMap<K, V, I>) => void,
54
+
thisArg?: unknown,
55
+
): void {
56
+
for (const [key, value] of this) {
57
+
callbackfn.call(thisArg, value, key, this);
58
+
}
59
+
}
60
+
61
+
[Symbol.iterator](): IterableIterator<[K, V]> {
62
+
return this.entries();
63
+
}
64
+
65
+
readonly [Symbol.toStringTag] = "MappedMap";
66
+
}
67
+
68
+
export class NsidMap<T> extends MappedMap<NSID, T, string> {
69
+
constructor() {
70
+
super(
71
+
(key) => key.toString(),
72
+
(enc) => NSID.from(enc),
73
+
);
74
+
}
75
+
}
+71
lex/installer/nsid-set.ts
+71
lex/installer/nsid-set.ts
···
1
+
import { NSID } from "@atp/syntax";
2
+
3
+
class MappedSet<K, I> {
4
+
readonly #set = new Set<I>();
5
+
6
+
constructor(
7
+
private readonly encodeValue: (val: K) => I,
8
+
private readonly decodeValue: (enc: I) => K,
9
+
) {}
10
+
11
+
get size(): number {
12
+
return this.#set.size;
13
+
}
14
+
15
+
clear(): void {
16
+
this.#set.clear();
17
+
}
18
+
19
+
add(val: K): this {
20
+
this.#set.add(this.encodeValue(val));
21
+
return this;
22
+
}
23
+
24
+
has(val: K): boolean {
25
+
return this.#set.has(this.encodeValue(val));
26
+
}
27
+
28
+
delete(val: K): boolean {
29
+
return this.#set.delete(this.encodeValue(val));
30
+
}
31
+
32
+
*values(): IterableIterator<K> {
33
+
for (const val of this.#set.values()) {
34
+
yield this.decodeValue(val);
35
+
}
36
+
}
37
+
38
+
keys(): IterableIterator<K> {
39
+
return this.values();
40
+
}
41
+
42
+
*entries(): IterableIterator<[K, K]> {
43
+
for (const value of this) {
44
+
yield [value, value];
45
+
}
46
+
}
47
+
48
+
forEach(
49
+
callbackfn: (value: K, value2: K, set: MappedSet<K, I>) => void,
50
+
thisArg?: unknown,
51
+
): void {
52
+
for (const value of this) {
53
+
callbackfn.call(thisArg, value, value, this);
54
+
}
55
+
}
56
+
57
+
[Symbol.iterator](): IterableIterator<K> {
58
+
return this.values();
59
+
}
60
+
61
+
readonly [Symbol.toStringTag] = "MappedSet";
62
+
}
63
+
64
+
export class NsidSet extends MappedSet<NSID, string> {
65
+
constructor() {
66
+
super(
67
+
(val) => val.toString(),
68
+
(enc) => NSID.from(enc),
69
+
);
70
+
}
71
+
}
+5
lex/mod.ts
+5
lex/mod.ts
+20
lex/resolver/lex-resolver-error.ts
+20
lex/resolver/lex-resolver-error.ts
···
1
+
import { NSID } from "@atp/syntax";
2
+
3
+
export class LexResolverError extends Error {
4
+
override name = "LexResolverError";
5
+
6
+
constructor(
7
+
public readonly nsid: NSID,
8
+
public readonly description = "Could not resolve Lexicon for NSID",
9
+
options?: ErrorOptions,
10
+
) {
11
+
super(`${description} (${nsid})`, options);
12
+
}
13
+
14
+
static from(nsid: NSID | string, description?: string): LexResolverError {
15
+
return new LexResolverError(
16
+
typeof nsid === "string" ? NSID.from(nsid) : nsid,
17
+
description,
18
+
);
19
+
}
20
+
}
+386
lex/resolver/lex-resolver.ts
+386
lex/resolver/lex-resolver.ts
···
1
+
import type { CID } from "multiformats/cid";
2
+
import { resolveTxt as resolveTxtWithNode } from "node:dns/promises";
3
+
import { type AtprotoData, type DidCache, DidResolver } from "@atp/identity";
4
+
import {
5
+
assertDid,
6
+
assertRecordKey,
7
+
type DidString,
8
+
type RecordKeyString,
9
+
} from "../external.ts";
10
+
import * as l from "../external.ts";
11
+
import {
12
+
type LexiconDocument,
13
+
lexiconDocumentSchema,
14
+
} from "../document/mod.ts";
15
+
import {
16
+
def as repoDef,
17
+
MemoryBlockstore,
18
+
MST,
19
+
readCarWithRoot,
20
+
verifyCommitSig,
21
+
} from "@atp/repo";
22
+
import { AtUri, ensureValidDid, NSID } from "@atp/syntax";
23
+
import { XrpcClient } from "@atp/xrpc";
24
+
import { LexResolverError } from "./lex-resolver-error.ts";
25
+
26
+
const LEXICON_COLLECTION = "com.atproto.lexicon.schema";
27
+
28
+
const getRecordQuery = l.query(
29
+
"com.atproto.sync.getRecord",
30
+
l.params({
31
+
did: l.string({ format: "did" }),
32
+
collection: l.string({ format: "nsid" }),
33
+
rkey: l.string({ format: "record-key" }),
34
+
}),
35
+
l.payload("application/vnd.ipld.car"),
36
+
);
37
+
38
+
type MaybePromise<T> = Promise<T> | T;
39
+
40
+
export type LexResolverResult = {
41
+
uri: AtUri;
42
+
cid: CID;
43
+
lexicon: LexiconDocument;
44
+
};
45
+
46
+
export type LexResolverFetchResult = {
47
+
cid: CID;
48
+
lexicon: LexiconDocument;
49
+
};
50
+
51
+
export type LexResolverHooks = {
52
+
onResolveAuthority?(data: { nsid: NSID }): MaybePromise<string | void>;
53
+
onResolveAuthorityResult?(
54
+
data: { nsid: NSID; did: string },
55
+
): MaybePromise<void>;
56
+
onResolveAuthorityError?(
57
+
data: { nsid: NSID; err: unknown },
58
+
): MaybePromise<void>;
59
+
onFetch?(data: { uri: AtUri }): MaybePromise<LexResolverFetchResult | void>;
60
+
onFetchResult?(data: {
61
+
uri: AtUri;
62
+
cid: CID;
63
+
lexicon: LexiconDocument;
64
+
}): MaybePromise<void>;
65
+
onFetchError?(data: { uri: AtUri; err: unknown }): MaybePromise<void>;
66
+
};
67
+
68
+
export type TxtResolver = (domain: string) => Promise<string[][]>;
69
+
70
+
type DenoResolveTxt = (
71
+
domain: string,
72
+
recordType: "TXT",
73
+
) => Promise<string[][]>;
74
+
75
+
export type LexResolverDidResolver = {
76
+
resolveAtprotoData(
77
+
did: string,
78
+
forceRefresh?: boolean,
79
+
): Promise<AtprotoData>;
80
+
};
81
+
82
+
export type LexResolverOptions = {
83
+
timeout?: number;
84
+
plcUrl?: string;
85
+
didCache?: DidCache;
86
+
fetch?: typeof globalThis.fetch;
87
+
hooks?: LexResolverHooks;
88
+
didResolver?: LexResolverDidResolver;
89
+
resolveTxt?: TxtResolver;
90
+
};
91
+
92
+
export type LexResolverFetchOptions = {
93
+
signal?: AbortSignal;
94
+
forceRefresh?: boolean;
95
+
noCache?: boolean;
96
+
};
97
+
98
+
export type DefaultTxtResolverOptions = {
99
+
denoResolveDns?: DenoResolveTxt | null;
100
+
nodeResolveTxt?: TxtResolver;
101
+
};
102
+
103
+
export { AtUri, NSID };
104
+
export type { CID, LexiconDocument };
105
+
106
+
export class LexResolver {
107
+
protected readonly didResolver: LexResolverDidResolver;
108
+
protected readonly resolveTxt: TxtResolver;
109
+
110
+
constructor(protected readonly options: LexResolverOptions = {}) {
111
+
const { timeout = 3000, plcUrl, didCache } = options;
112
+
this.didResolver = options.didResolver ??
113
+
new DidResolver({ timeout, plcUrl, didCache });
114
+
this.resolveTxt = options.resolveTxt ?? defaultResolveTxt;
115
+
}
116
+
117
+
async get(
118
+
nsidStr: NSID | string,
119
+
options?: LexResolverFetchOptions,
120
+
): Promise<LexResolverResult> {
121
+
const uri = await this.resolve(nsidStr);
122
+
return this.fetch(uri, options);
123
+
}
124
+
125
+
async resolve(nsidStr: NSID | string): Promise<AtUri> {
126
+
const nsid = NSID.from(nsidStr);
127
+
128
+
const hookedDid = await this.options.hooks?.onResolveAuthority?.({ nsid });
129
+
if (hookedDid !== undefined) {
130
+
ensureValidDid(hookedDid);
131
+
return AtUri.make(hookedDid, LEXICON_COLLECTION, nsid.toString());
132
+
}
133
+
134
+
const did = await this.resolveLexiconAuthority(nsid).then(
135
+
async (resolvedDid) => {
136
+
await this.options.hooks?.onResolveAuthorityResult?.({
137
+
nsid,
138
+
did: resolvedDid,
139
+
});
140
+
return resolvedDid;
141
+
},
142
+
async (err) => {
143
+
await this.options.hooks?.onResolveAuthorityError?.({ nsid, err });
144
+
throw err;
145
+
},
146
+
);
147
+
148
+
return AtUri.make(did, LEXICON_COLLECTION, nsid.toString());
149
+
}
150
+
151
+
async fetch(
152
+
uriStr: AtUri | string,
153
+
options?: LexResolverFetchOptions,
154
+
): Promise<LexResolverResult> {
155
+
const uri = typeof uriStr === "string" ? new AtUri(uriStr) : uriStr;
156
+
157
+
const hookedResult = await this.options.hooks?.onFetch?.({ uri });
158
+
if (hookedResult !== undefined) {
159
+
return { uri, ...validateLexiconResult(uri, hookedResult) };
160
+
}
161
+
162
+
const fetched = await this.fetchLexiconUri(uri, options).then(
163
+
async (result) => {
164
+
const validated = validateLexiconResult(uri, result);
165
+
await this.options.hooks?.onFetchResult?.({ uri, ...validated });
166
+
return validated;
167
+
},
168
+
async (err) => {
169
+
await this.options.hooks?.onFetchError?.({ uri, err });
170
+
throw err;
171
+
},
172
+
);
173
+
174
+
return { uri, ...fetched };
175
+
}
176
+
177
+
protected async resolveLexiconAuthority(nsid: NSID): Promise<string> {
178
+
try {
179
+
const did = parseDomainTxtDid(
180
+
await this.resolveTxt(`_lexicon.${nsid.authority}`),
181
+
);
182
+
ensureValidDid(did);
183
+
return did;
184
+
} catch (cause) {
185
+
throw new LexResolverError(
186
+
nsid,
187
+
`Failed to resolve lexicon DID authority for ${nsid}`,
188
+
{ cause },
189
+
);
190
+
}
191
+
}
192
+
193
+
protected async fetchLexiconUri(
194
+
uri: AtUri,
195
+
options?: LexResolverFetchOptions,
196
+
): Promise<LexResolverFetchResult> {
197
+
const { did, nsid } = parseLexiconUri(uri);
198
+
199
+
const atprotoData = await this.didResolver.resolveAtprotoData(
200
+
did,
201
+
options?.forceRefresh,
202
+
).catch((cause) => {
203
+
throw new LexResolverError(
204
+
nsid,
205
+
`Failed to resolve DID document for ${did}`,
206
+
{ cause },
207
+
);
208
+
});
209
+
210
+
if (!atprotoData.signingKey || !atprotoData.pds) {
211
+
throw new LexResolverError(
212
+
nsid,
213
+
`No atproto PDS service endpoint or signing key found in ${did} DID document`,
214
+
);
215
+
}
216
+
217
+
const client = new XrpcClient({
218
+
service: atprotoData.pds,
219
+
fetch: this.options.fetch,
220
+
});
221
+
const didParam = did;
222
+
const rkey = nsid.toString();
223
+
assertDid(didParam);
224
+
assertRecordKey(rkey);
225
+
226
+
const response = await client.call(getRecordQuery, {
227
+
params: {
228
+
did: didParam as DidString,
229
+
collection: LEXICON_COLLECTION,
230
+
rkey: rkey as RecordKeyString,
231
+
},
232
+
headers: options?.noCache ? { "cache-control": "no-cache" } : undefined,
233
+
signal: options?.signal,
234
+
validateRequest: true,
235
+
validateResponse: false,
236
+
}).catch((cause) => {
237
+
throw new LexResolverError(nsid, `Failed to fetch Record ${uri}`, {
238
+
cause,
239
+
});
240
+
});
241
+
242
+
if (!(response.data instanceof Uint8Array)) {
243
+
throw new LexResolverError(
244
+
nsid,
245
+
`Invalid record response at ${uri}`,
246
+
{ cause: new TypeError("Expected CAR bytes") },
247
+
);
248
+
}
249
+
250
+
return verifyRecordProof(
251
+
response.data,
252
+
did,
253
+
atprotoData.signingKey,
254
+
LEXICON_COLLECTION,
255
+
nsid.toString(),
256
+
).catch((cause) => {
257
+
throw new LexResolverError(
258
+
nsid,
259
+
`Failed to verify Lexicon record proof at ${uri}`,
260
+
{ cause },
261
+
);
262
+
});
263
+
}
264
+
}
265
+
266
+
export function createDefaultResolveTxt(
267
+
options: DefaultTxtResolverOptions = {},
268
+
): TxtResolver {
269
+
const denoResolveDns = options.denoResolveDns === undefined
270
+
? getDenoResolveDns()
271
+
: options.denoResolveDns;
272
+
273
+
if (denoResolveDns) {
274
+
return (domain) => denoResolveDns(domain, "TXT");
275
+
}
276
+
277
+
const nodeResolveTxt = options.nodeResolveTxt ?? resolveTxtWithNode;
278
+
return (domain) => nodeResolveTxt(domain);
279
+
}
280
+
281
+
const defaultResolveTxt = createDefaultResolveTxt();
282
+
283
+
function getDenoResolveDns(): DenoResolveTxt | undefined {
284
+
if (typeof Deno === "undefined") {
285
+
return undefined;
286
+
}
287
+
288
+
return (domain, recordType) => Deno.resolveDns(domain, recordType);
289
+
}
290
+
291
+
function parseDomainTxtDid(records: string[][]): string {
292
+
const didLines = records
293
+
.map((chunks) => chunks.join(""))
294
+
.filter((value) => value.startsWith("did="));
295
+
296
+
if (didLines.length === 1) {
297
+
return didLines[0].slice(4);
298
+
}
299
+
300
+
throw didLines.length > 1
301
+
? new Error("Multiple DIDs found in DNS TXT records")
302
+
: new Error("No DID found in DNS TXT records");
303
+
}
304
+
305
+
function parseLexiconUri(uri: AtUri): { did: string; nsid: NSID } {
306
+
const nsid = NSID.from(uri.rkey);
307
+
308
+
if (uri.collection !== LEXICON_COLLECTION) {
309
+
throw new LexResolverError(
310
+
nsid,
311
+
`URI collection is not ${LEXICON_COLLECTION}: ${uri}`,
312
+
);
313
+
}
314
+
315
+
try {
316
+
ensureValidDid(uri.host);
317
+
return { did: uri.host, nsid };
318
+
} catch (cause) {
319
+
throw new LexResolverError(nsid, `URI host is not a DID ${uri}`, { cause });
320
+
}
321
+
}
322
+
323
+
function validateLexiconResult(
324
+
uri: AtUri,
325
+
result: LexResolverFetchResult,
326
+
): LexResolverFetchResult {
327
+
const nsid = NSID.from(uri.rkey);
328
+
const validation = lexiconDocumentSchema.safeParse(result.lexicon);
329
+
330
+
if (!validation.success) {
331
+
throw new LexResolverError(nsid, `Invalid Lexicon document at ${uri}`, {
332
+
cause: validation.error,
333
+
});
334
+
}
335
+
336
+
if (validation.value.id !== uri.rkey) {
337
+
throw new LexResolverError(
338
+
nsid,
339
+
`Invalid document id "${validation.value.id}" at ${uri}`,
340
+
);
341
+
}
342
+
343
+
return {
344
+
cid: result.cid,
345
+
lexicon: validation.value,
346
+
};
347
+
}
348
+
349
+
async function verifyRecordProof(
350
+
car: Uint8Array,
351
+
did: string,
352
+
signingKey: string,
353
+
collection: string,
354
+
rkey: string,
355
+
): Promise<LexResolverFetchResult> {
356
+
const { root, blocks } = await readCarWithRoot(car);
357
+
const blockstore = new MemoryBlockstore(blocks);
358
+
359
+
const commit = blockstore.readObj(root, repoDef.commit);
360
+
if (commit.did !== did) {
361
+
throw new Error(`Invalid repo did: ${commit.did}`);
362
+
}
363
+
364
+
const validSig = verifyCommitSig(commit, signingKey);
365
+
if (!validSig) {
366
+
throw new Error(`Invalid signature on commit: ${root.toString()}`);
367
+
}
368
+
369
+
const mst = MST.load(blockstore, (commit as { data: CID }).data);
370
+
const cid = await mst.get(`${collection}/${rkey}`);
371
+
if (!cid) {
372
+
throw new Error("Record not found in proof");
373
+
}
374
+
375
+
const record = blockstore.readRecord(cid);
376
+
if (record.$type !== collection) {
377
+
throw new Error(
378
+
`Invalid record type: expected ${collection}, got ${record.$type}`,
379
+
);
380
+
}
381
+
382
+
return {
383
+
cid,
384
+
lexicon: record as LexiconDocument,
385
+
};
386
+
}
+2
lex/resolver/mod.ts
+2
lex/resolver/mod.ts
+308
lex/tests/lex-installer_test.ts
+308
lex/tests/lex-installer_test.ts
···
1
+
import { CID } from "multiformats/cid";
2
+
import { assertEquals, assertRejects } from "@std/assert";
3
+
import { join } from "node:path";
4
+
import { AtUri, NSID } from "@atp/syntax";
5
+
import { cidForLex } from "../cbor/mod.ts";
6
+
import {
7
+
install,
8
+
type LexiconsManifest,
9
+
LexInstaller,
10
+
LexInstallerError,
11
+
type LexInstallerResolver,
12
+
normalizeLexiconsManifest,
13
+
} from "../installer/mod.ts";
14
+
import {
15
+
type LexiconDocument,
16
+
lexiconDocumentSchema,
17
+
} from "../document/mod.ts";
18
+
import type { LexValue } from "../data/lex.ts";
19
+
20
+
const COLLECTION = "com.atproto.lexicon.schema";
21
+
22
+
class StubResolver implements LexInstallerResolver {
23
+
readonly resolved: string[] = [];
24
+
readonly fetched: string[] = [];
25
+
26
+
constructor(
27
+
private readonly documents: Record<string, LexiconDocument>,
28
+
private readonly authority = "did:plc:test",
29
+
) {}
30
+
31
+
resolve(nsidStr: NSID | string): Promise<AtUri> {
32
+
const nsid = NSID.from(nsidStr).toString();
33
+
this.resolved.push(nsid);
34
+
return Promise.resolve(AtUri.make(this.authority, COLLECTION, nsid));
35
+
}
36
+
37
+
async fetch(uriStr: AtUri | string) {
38
+
const uri = typeof uriStr === "string" ? new AtUri(uriStr) : uriStr;
39
+
const lexicon = this.documents[uri.rkey];
40
+
if (!lexicon) {
41
+
throw new Error(`Unknown lexicon ${uri.rkey}`);
42
+
}
43
+
this.fetched.push(uri.toString());
44
+
const cid = CID.parse((await cidForDocument(lexicon)).toString());
45
+
return { uri, cid, lexicon };
46
+
}
47
+
}
48
+
49
+
function createLexicon(
50
+
id: string,
51
+
dependency?: string,
52
+
): LexiconDocument {
53
+
return lexiconDocumentSchema.parse({
54
+
lexicon: 1,
55
+
id,
56
+
defs: {
57
+
main: dependency
58
+
? {
59
+
type: "query",
60
+
output: {
61
+
encoding: "application/json",
62
+
schema: {
63
+
type: "ref" as const,
64
+
ref: dependency,
65
+
},
66
+
},
67
+
}
68
+
: {
69
+
type: "object" as const,
70
+
properties: {
71
+
ok: { type: "boolean" },
72
+
},
73
+
},
74
+
},
75
+
});
76
+
}
77
+
78
+
function cidForDocument(lexicon: LexiconDocument) {
79
+
return cidForLex(lexicon as unknown as LexValue);
80
+
}
81
+
82
+
Deno.test("installer reuses local lexicons and fetches missing dependencies", async () => {
83
+
const root = await Deno.makeTempDir({ prefix: "lex-installer-" });
84
+
85
+
try {
86
+
const lexicons = join(root, "lexicons");
87
+
const manifestPath = join(root, "lexicons.json");
88
+
const rootLexicon = createLexicon("com.example.root", "com.example.dep");
89
+
const depLexicon = createLexicon("com.example.dep");
90
+
const resolver = new StubResolver({
91
+
[rootLexicon.id]: rootLexicon,
92
+
[depLexicon.id]: depLexicon,
93
+
});
94
+
95
+
await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true });
96
+
await Deno.writeTextFile(
97
+
join(lexicons, "com", "example", "root.json"),
98
+
JSON.stringify(rootLexicon),
99
+
);
100
+
101
+
const installer = new LexInstaller({
102
+
lexicons,
103
+
manifest: manifestPath,
104
+
resolver,
105
+
});
106
+
107
+
try {
108
+
await installer.install({
109
+
additions: [rootLexicon.id],
110
+
});
111
+
await installer.save();
112
+
} finally {
113
+
await installer[Symbol.asyncDispose]();
114
+
}
115
+
116
+
assertEquals(resolver.resolved, [rootLexicon.id, depLexicon.id]);
117
+
assertEquals(resolver.fetched, [
118
+
`at://did:plc:test/${COLLECTION}/${depLexicon.id}`,
119
+
]);
120
+
121
+
const savedDep = JSON.parse(
122
+
await Deno.readTextFile(
123
+
join(lexicons, "com", "example", "dep.json"),
124
+
),
125
+
);
126
+
assertEquals(savedDep.id, depLexicon.id);
127
+
128
+
const manifest = JSON.parse(
129
+
await Deno.readTextFile(manifestPath),
130
+
) as LexiconsManifest;
131
+
assertEquals(
132
+
manifest,
133
+
normalizeLexiconsManifest({
134
+
version: 1,
135
+
lexicons: [rootLexicon.id],
136
+
resolutions: {
137
+
[rootLexicon.id]: {
138
+
uri: `at://did:plc:test/${COLLECTION}/${rootLexicon.id}`,
139
+
cid: (await cidForDocument(rootLexicon)).toString(),
140
+
},
141
+
[depLexicon.id]: {
142
+
uri: `at://did:plc:test/${COLLECTION}/${depLexicon.id}`,
143
+
cid: (await cidForDocument(depLexicon)).toString(),
144
+
},
145
+
},
146
+
}),
147
+
);
148
+
} finally {
149
+
await Deno.remove(root, { recursive: true });
150
+
}
151
+
});
152
+
153
+
Deno.test("install supports explicit at:// additions without resolving", async () => {
154
+
const root = await Deno.makeTempDir({ prefix: "lex-installer-uri-" });
155
+
156
+
try {
157
+
const lexicons = join(root, "lexicons");
158
+
const manifestPath = join(root, "lexicons.json");
159
+
const lexicon = createLexicon("com.example.uri");
160
+
const resolver = new StubResolver({
161
+
[lexicon.id]: lexicon,
162
+
}, "did:plc:uri");
163
+
164
+
await install({
165
+
lexicons,
166
+
manifest: manifestPath,
167
+
resolver,
168
+
add: [`at://did:plc:uri/${COLLECTION}/${lexicon.id}`],
169
+
save: true,
170
+
});
171
+
172
+
assertEquals(resolver.resolved, []);
173
+
assertEquals(resolver.fetched, [
174
+
`at://did:plc:uri/${COLLECTION}/${lexicon.id}`,
175
+
]);
176
+
} finally {
177
+
await Deno.remove(root, { recursive: true });
178
+
}
179
+
});
180
+
181
+
Deno.test("install fails in ci mode when manifest is stale", async () => {
182
+
const root = await Deno.makeTempDir({ prefix: "lex-installer-ci-" });
183
+
184
+
try {
185
+
const lexicons = join(root, "lexicons");
186
+
const manifestPath = join(root, "lexicons.json");
187
+
const lexicon = createLexicon("com.example.ci");
188
+
const resolver = new StubResolver({
189
+
[lexicon.id]: lexicon,
190
+
});
191
+
const staleManifest = {
192
+
version: 1 as const,
193
+
lexicons: [],
194
+
resolutions: {},
195
+
};
196
+
197
+
await Deno.writeTextFile(
198
+
manifestPath,
199
+
JSON.stringify(staleManifest),
200
+
);
201
+
202
+
const error = await assertRejects(
203
+
() =>
204
+
install({
205
+
lexicons,
206
+
manifest: manifestPath,
207
+
resolver,
208
+
add: [lexicon.id],
209
+
ci: true,
210
+
save: false,
211
+
}),
212
+
LexInstallerError,
213
+
"Lexicons manifest is out of date",
214
+
);
215
+
216
+
assertEquals(error.name, "LexInstallerError");
217
+
await assertRejects(
218
+
() => Deno.readTextFile(join(lexicons, "com", "example", "ci.json")),
219
+
Deno.errors.NotFound,
220
+
);
221
+
assertEquals(
222
+
JSON.parse(await Deno.readTextFile(manifestPath)),
223
+
staleManifest,
224
+
);
225
+
} finally {
226
+
await Deno.remove(root, { recursive: true });
227
+
}
228
+
});
229
+
230
+
Deno.test("install rejects explicit at:// additions outside the lexicon collection", async () => {
231
+
const root = await Deno.makeTempDir({ prefix: "lex-installer-invalid-uri-" });
232
+
233
+
try {
234
+
const lexicons = join(root, "lexicons");
235
+
const manifestPath = join(root, "lexicons.json");
236
+
const lexicon = createLexicon("com.example.root");
237
+
const resolver = new StubResolver({
238
+
[lexicon.id]: lexicon,
239
+
}, "did:plc:uri");
240
+
241
+
await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true });
242
+
await Deno.writeTextFile(
243
+
join(lexicons, "com", "example", "root.json"),
244
+
JSON.stringify(lexicon),
245
+
);
246
+
247
+
const error = await assertRejects(
248
+
() =>
249
+
install({
250
+
lexicons,
251
+
manifest: manifestPath,
252
+
resolver,
253
+
add: [`at://did:plc:uri/app.bsky.feed.post/${lexicon.id}`],
254
+
save: true,
255
+
}),
256
+
LexInstallerError,
257
+
"Invalid lexicon URI collection",
258
+
);
259
+
260
+
assertEquals(error.name, "LexInstallerError");
261
+
assertEquals(resolver.resolved, []);
262
+
assertEquals(resolver.fetched, []);
263
+
await assertRejects(() => Deno.stat(manifestPath), Deno.errors.NotFound);
264
+
} finally {
265
+
await Deno.remove(root, { recursive: true });
266
+
}
267
+
});
268
+
269
+
Deno.test("install rejects explicit at:// additions with handle authorities", async () => {
270
+
const root = await Deno.makeTempDir({
271
+
prefix: "lex-installer-handle-uri-",
272
+
});
273
+
274
+
try {
275
+
const lexicons = join(root, "lexicons");
276
+
const manifestPath = join(root, "lexicons.json");
277
+
const lexicon = createLexicon("com.example.root");
278
+
const resolver = new StubResolver({
279
+
[lexicon.id]: lexicon,
280
+
}, "did:plc:uri");
281
+
282
+
await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true });
283
+
await Deno.writeTextFile(
284
+
join(lexicons, "com", "example", "root.json"),
285
+
JSON.stringify(lexicon),
286
+
);
287
+
288
+
const error = await assertRejects(
289
+
() =>
290
+
install({
291
+
lexicons,
292
+
manifest: manifestPath,
293
+
resolver,
294
+
add: [`at://example.com/${COLLECTION}/${lexicon.id}`],
295
+
save: true,
296
+
}),
297
+
LexInstallerError,
298
+
"Invalid lexicon URI authority",
299
+
);
300
+
301
+
assertEquals(error.name, "LexInstallerError");
302
+
assertEquals(resolver.resolved, []);
303
+
assertEquals(resolver.fetched, []);
304
+
await assertRejects(() => Deno.stat(manifestPath), Deno.errors.NotFound);
305
+
} finally {
306
+
await Deno.remove(root, { recursive: true });
307
+
}
308
+
});
+321
lex/tests/lex-resolver_test.ts
+321
lex/tests/lex-resolver_test.ts
···
1
+
import { cidForCbor, streamToBuffer } from "@atp/common";
2
+
import * as crypto from "@atp/crypto";
3
+
import { getRecords, MemoryBlockstore, Repo, WriteOpAction } from "@atp/repo";
4
+
import { AtUri } from "@atp/syntax";
5
+
import { assertEquals, assertInstanceOf, assertRejects } from "@std/assert";
6
+
import { LexResolver, LexResolverError } from "../resolver/mod.ts";
7
+
import { createDefaultResolveTxt } from "../resolver/lex-resolver.ts";
8
+
9
+
const collection = "com.atproto.lexicon.schema";
10
+
const nsid = "app.bsky.feed.post";
11
+
12
+
type ProofFixture = {
13
+
car: Uint8Array;
14
+
cid: string;
15
+
did: string;
16
+
pds: string;
17
+
signingKey: string;
18
+
lexicon: Record<string, unknown>;
19
+
};
20
+
21
+
const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer => {
22
+
const copy = new Uint8Array(bytes.byteLength);
23
+
copy.set(bytes);
24
+
return copy.buffer;
25
+
};
26
+
27
+
const createLexiconRecord = (
28
+
id: string,
29
+
): Record<string, unknown> => ({
30
+
$type: collection,
31
+
lexicon: 1,
32
+
id,
33
+
defs: {
34
+
main: {
35
+
type: "query",
36
+
},
37
+
},
38
+
});
39
+
40
+
const createProofFixture = async (
41
+
record = createLexiconRecord(nsid),
42
+
): Promise<ProofFixture> => {
43
+
const storage = new MemoryBlockstore();
44
+
const keypair = crypto.Secp256k1Keypair.create();
45
+
const did = "did:plc:resolvertest";
46
+
const repo = await Repo.create(storage, did, keypair, [{
47
+
action: WriteOpAction.Create,
48
+
collection,
49
+
rkey: nsid,
50
+
record,
51
+
}]);
52
+
const car = await streamToBuffer(getRecords(storage, repo.cid, [{
53
+
collection,
54
+
rkey: nsid,
55
+
}]));
56
+
const fetched = await repo.data.get(`${collection}/${nsid}`);
57
+
if (!fetched) {
58
+
throw new Error("expected cid");
59
+
}
60
+
return {
61
+
car,
62
+
cid: fetched.toString(),
63
+
did,
64
+
pds: "https://pds.test",
65
+
signingKey: keypair.did(),
66
+
lexicon: record,
67
+
};
68
+
};
69
+
70
+
Deno.test("get resolves and fetches lexicons through xrpc", async () => {
71
+
const fixture = await createProofFixture();
72
+
const calls: { forceRefresh?: boolean; url?: string } = {};
73
+
const events: string[] = [];
74
+
75
+
const resolver = new LexResolver({
76
+
didResolver: {
77
+
resolveAtprotoData(did, forceRefresh) {
78
+
assertEquals(did, fixture.did);
79
+
calls.forceRefresh = forceRefresh;
80
+
return Promise.resolve({
81
+
did,
82
+
handle: "resolver.test",
83
+
pds: fixture.pds,
84
+
signingKey: fixture.signingKey,
85
+
});
86
+
},
87
+
},
88
+
resolveTxt(domain) {
89
+
assertEquals(domain, "_lexicon.feed.bsky.app");
90
+
return Promise.resolve([[`did=${fixture.did}`]]);
91
+
},
92
+
fetch: ((input, init) => {
93
+
const url = input instanceof URL ? input : new URL(String(input));
94
+
const headers = new Headers(init?.headers);
95
+
calls.url = url.toString();
96
+
assertEquals(init?.method, "get");
97
+
assertEquals(headers.get("accept"), "application/vnd.ipld.car");
98
+
return Promise.resolve(
99
+
new Response(toArrayBuffer(fixture.car), {
100
+
headers: { "content-type": "application/vnd.ipld.car" },
101
+
}),
102
+
);
103
+
}) as typeof fetch,
104
+
hooks: {
105
+
onResolveAuthorityResult() {
106
+
events.push("resolve");
107
+
},
108
+
onFetchResult() {
109
+
events.push("fetch");
110
+
},
111
+
},
112
+
});
113
+
114
+
const result = await resolver.get(nsid, {
115
+
forceRefresh: true,
116
+
noCache: true,
117
+
});
118
+
119
+
assertEquals(
120
+
calls.url,
121
+
"https://pds.test/xrpc/com.atproto.sync.getRecord?" +
122
+
"did=did%3Aplc%3Aresolvertest&collection=com.atproto.lexicon.schema" +
123
+
"&rkey=app.bsky.feed.post",
124
+
);
125
+
assertEquals(calls.forceRefresh, true);
126
+
assertEquals(
127
+
result.uri.toString(),
128
+
`at://${fixture.did}/${collection}/${nsid}`,
129
+
);
130
+
assertEquals(result.cid.toString(), fixture.cid);
131
+
assertEquals(result.lexicon.id, nsid);
132
+
assertEquals(events, ["resolve", "fetch"]);
133
+
});
134
+
135
+
Deno.test("resolve and fetch hooks can short-circuit the network path", async () => {
136
+
const uri = AtUri.make("did:plc:hooked", collection, nsid);
137
+
const cid = await cidForCbor({ ok: true });
138
+
const lexicon = createLexiconRecord(nsid);
139
+
140
+
const resolver = new LexResolver({
141
+
didResolver: {
142
+
resolveAtprotoData() {
143
+
throw new Error("did resolver should not be called");
144
+
},
145
+
},
146
+
resolveTxt() {
147
+
throw new Error("dns should not be called");
148
+
},
149
+
hooks: {
150
+
onResolveAuthority({ nsid }) {
151
+
assertEquals(nsid.toString(), "app.bsky.feed.post");
152
+
return "did:plc:hooked";
153
+
},
154
+
onFetch({ uri }) {
155
+
assertEquals(
156
+
uri.toString(),
157
+
`at://did:plc:hooked/${collection}/${nsid}`,
158
+
);
159
+
return {
160
+
cid,
161
+
lexicon: lexicon as never,
162
+
};
163
+
},
164
+
},
165
+
});
166
+
167
+
const resolved = await resolver.resolve(nsid);
168
+
const fetched = await resolver.fetch(uri);
169
+
170
+
assertEquals(resolved.toString(), uri.toString());
171
+
assertEquals(fetched.cid.toString(), cid.toString());
172
+
assertEquals(fetched.lexicon.id, nsid);
173
+
});
174
+
175
+
Deno.test("resolve wraps dns failures in LexResolverError", async () => {
176
+
const seen: unknown[] = [];
177
+
178
+
const resolver = new LexResolver({
179
+
resolveTxt() {
180
+
return Promise.resolve([["v=spf1 -all"]]);
181
+
},
182
+
hooks: {
183
+
onResolveAuthorityError({ err }) {
184
+
seen.push(err);
185
+
},
186
+
},
187
+
});
188
+
189
+
const error = await assertRejects(
190
+
() => resolver.resolve(nsid),
191
+
LexResolverError,
192
+
);
193
+
194
+
assertEquals(error.nsid.toString(), nsid);
195
+
assertEquals(seen.length, 1);
196
+
});
197
+
198
+
Deno.test("createDefaultResolveTxt prefers Deno DNS when available", async () => {
199
+
const calls: Array<[string, "TXT"]> = [];
200
+
const resolveTxt = createDefaultResolveTxt({
201
+
denoResolveDns(domain, recordType) {
202
+
calls.push([domain, recordType]);
203
+
return Promise.resolve([["did=did:plc:deno"]]);
204
+
},
205
+
nodeResolveTxt() {
206
+
throw new Error("node dns should not be called");
207
+
},
208
+
});
209
+
210
+
const records = await resolveTxt("_lexicon.feed.bsky.app");
211
+
212
+
assertEquals(calls, [["_lexicon.feed.bsky.app", "TXT"]]);
213
+
assertEquals(records, [["did=did:plc:deno"]]);
214
+
});
215
+
216
+
Deno.test("createDefaultResolveTxt falls back to Node DNS when Deno DNS is unavailable", async () => {
217
+
const calls: string[] = [];
218
+
const resolveTxt = createDefaultResolveTxt({
219
+
denoResolveDns: null,
220
+
nodeResolveTxt(domain) {
221
+
calls.push(domain);
222
+
return Promise.resolve([["did=did:plc:node"]]);
223
+
},
224
+
});
225
+
226
+
const records = await resolveTxt("_lexicon.feed.bsky.app");
227
+
228
+
assertEquals(calls, ["_lexicon.feed.bsky.app"]);
229
+
assertEquals(records, [["did=did:plc:node"]]);
230
+
});
231
+
232
+
Deno.test("fetch wraps proof verification failures in LexResolverError", async () => {
233
+
const fixture = await createProofFixture();
234
+
const seen: unknown[] = [];
235
+
236
+
const resolver = new LexResolver({
237
+
didResolver: {
238
+
resolveAtprotoData(did) {
239
+
return Promise.resolve({
240
+
did,
241
+
handle: "resolver.test",
242
+
pds: fixture.pds,
243
+
signingKey: "did:key:zWrongKey",
244
+
});
245
+
},
246
+
},
247
+
fetch: (() => {
248
+
return Promise.resolve(
249
+
new Response(toArrayBuffer(fixture.car), {
250
+
headers: { "content-type": "application/vnd.ipld.car" },
251
+
}),
252
+
);
253
+
}) as typeof fetch,
254
+
hooks: {
255
+
onFetchError({ err }) {
256
+
seen.push(err);
257
+
},
258
+
},
259
+
});
260
+
261
+
const error = await assertRejects(
262
+
() => resolver.fetch(`at://${fixture.did}/${collection}/${nsid}`),
263
+
LexResolverError,
264
+
);
265
+
266
+
assertEquals(error.nsid.toString(), nsid);
267
+
assertEquals(seen.length, 1);
268
+
});
269
+
270
+
Deno.test("fetch rejects lexicons with mismatched ids", async () => {
271
+
const fixture = await createProofFixture(
272
+
createLexiconRecord("app.bsky.feed.like"),
273
+
);
274
+
275
+
const resolver = new LexResolver({
276
+
didResolver: {
277
+
resolveAtprotoData(did) {
278
+
return Promise.resolve({
279
+
did,
280
+
handle: "resolver.test",
281
+
pds: fixture.pds,
282
+
signingKey: fixture.signingKey,
283
+
});
284
+
},
285
+
},
286
+
fetch: (() => {
287
+
return Promise.resolve(
288
+
new Response(toArrayBuffer(fixture.car), {
289
+
headers: { "content-type": "application/vnd.ipld.car" },
290
+
}),
291
+
);
292
+
}) as typeof fetch,
293
+
});
294
+
295
+
const error = await assertRejects(
296
+
() => resolver.fetch(`at://${fixture.did}/${collection}/${nsid}`),
297
+
LexResolverError,
298
+
);
299
+
300
+
assertEquals(error.nsid.toString(), nsid);
301
+
});
302
+
303
+
Deno.test("fetch rejects non-lexicon collections", async () => {
304
+
const resolver = new LexResolver({
305
+
didResolver: {
306
+
resolveAtprotoData() {
307
+
throw new Error("did resolver should not be called");
308
+
},
309
+
},
310
+
});
311
+
312
+
const error = await assertRejects(
313
+
() =>
314
+
resolver.fetch(
315
+
"at://did:plc:resolvertest/app.bsky.feed.post/app.bsky.feed.like",
316
+
),
317
+
LexResolverError,
318
+
);
319
+
320
+
assertInstanceOf(error, LexResolverError);
321
+
});