An experimental TypeSpec syntax for Lexicon
more
+253
-116
+74
-32
typelex-emitter/src/emitter.ts
+74
-32
typelex-emitter/src/emitter.ts
···
32
32
) {}
33
33
34
34
async emit() {
35
-
// Walk through all models in the program
36
-
for (const [_, type] of this.program.getGlobalNamespaceType().models) {
37
-
this.visitModel(type);
38
-
}
39
-
40
-
// Recursively process all namespaces
41
-
this.processNamespace(this.program.getGlobalNamespaceType());
35
+
const globalNs = this.program.getGlobalNamespaceType();
36
+
37
+
// Process all namespaces to find models and operations
38
+
this.processNamespace(globalNs);
42
39
43
40
// Write all lexicon files
44
41
for (const [id, lexicon] of this.lexicons) {
···
48
45
}
49
46
50
47
private processNamespace(ns: any) {
51
-
for (const [_, model] of ns.models) {
52
-
this.visitModel(model);
48
+
const fullName = getNamespaceFullName(ns);
49
+
50
+
// Skip built-in TypeSpec namespaces but still process their children
51
+
if (fullName && !fullName.startsWith("TypeSpec")) {
52
+
// Check if this namespace should be a lexicon file
53
+
const hasModels = ns.models.size > 0;
54
+
const hasOperations = ns.operations?.size > 0;
55
+
const hasChildNamespaces = ns.namespaces.size > 0;
56
+
57
+
// Heuristic: if namespace has models but no operations and no child namespaces,
58
+
// it's likely a defs file
59
+
const isLikelyDefsFile = hasModels && !hasOperations && !hasChildNamespaces;
60
+
61
+
if (isLikelyDefsFile) {
62
+
// Create a single lexicon for all models in this namespace
63
+
const lexiconId = fullName + ".defs";
64
+
const lexicon: LexiconDocument = {
65
+
lexicon: 1,
66
+
id: lexiconId,
67
+
defs: {},
68
+
};
69
+
70
+
// Add all models as definitions
71
+
for (const [_, model] of ns.models) {
72
+
const defName = model.name.charAt(0).toLowerCase() + model.name.slice(1);
73
+
const modelDef = this.modelToLexiconObject(model);
74
+
lexicon.defs[defName] = modelDef;
75
+
}
76
+
77
+
this.lexicons.set(lexiconId, lexicon);
78
+
} else if (hasModels && !hasOperations) {
79
+
// Process models individually for non-defs files
80
+
for (const [_, model] of ns.models) {
81
+
this.visitModel(model);
82
+
}
83
+
} else if (hasOperations) {
84
+
// TODO: Process operations for queries and procedures
85
+
for (const [_, operation] of ns.operations) {
86
+
// this.visitOperation(operation);
87
+
}
88
+
}
53
89
}
54
90
91
+
// Always recursively process child namespaces
55
92
for (const [_, childNs] of ns.namespaces) {
56
-
// Skip built-in TypeSpec namespaces
57
-
const fullName = getNamespaceFullName(childNs);
58
-
if (fullName?.startsWith("TypeSpec")) {
59
-
continue;
60
-
}
61
93
this.processNamespace(childNs);
62
94
}
63
95
}
···
94
126
const defName = model.name.charAt(0).toLowerCase() + model.name.slice(1);
95
127
const modelDef = this.modelToLexiconObject(model);
96
128
97
-
// For now, assume the first model in a lexicon is the main one
98
-
if (Object.keys(lexicon.defs).length === 0) {
99
-
// Check if this is the lexicon schema special case
100
-
const key = lexiconId === "com.atproto.lexicon.schema" ? "nsid" : "tid";
101
-
102
-
const recordDef: any = {
103
-
type: "record",
104
-
key: key,
105
-
record: modelDef,
106
-
};
107
-
108
-
// Move description from record object to record def
129
+
// Check if this is a defs file (ends with .defs)
130
+
if (lexiconId.endsWith(".defs")) {
131
+
// For defs files, all models go directly into defs object
109
132
const description = getDoc(this.program, model);
110
-
if (description) {
111
-
recordDef.description = description;
112
-
delete modelDef.description;
133
+
if (description && !modelDef.description) {
134
+
modelDef.description = description;
113
135
}
114
-
115
-
lexicon.defs.main = recordDef;
116
-
} else {
117
136
lexicon.defs[defName] = modelDef;
137
+
} else {
138
+
// For non-defs files, treat the first model as the main record
139
+
if (Object.keys(lexicon.defs).length === 0) {
140
+
// Check if this is the lexicon schema special case
141
+
const key = lexiconId === "com.atproto.lexicon.schema" ? "nsid" : "tid";
142
+
143
+
const recordDef: any = {
144
+
type: "record",
145
+
key: key,
146
+
record: modelDef,
147
+
};
148
+
149
+
// Move description from record object to record def
150
+
const description = getDoc(this.program, model);
151
+
if (description) {
152
+
recordDef.description = description;
153
+
delete modelDef.description;
154
+
}
155
+
156
+
lexicon.defs.main = recordDef;
157
+
} else {
158
+
lexicon.defs[defName] = modelDef;
159
+
}
118
160
}
119
161
}
120
162
+97
-84
typelex-emitter/test/fixtures.test.ts
+97
-84
typelex-emitter/test/fixtures.test.ts
···
1
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
2
-
import { exec } from "child_process";
3
-
import { promisify } from "util";
4
-
import { readFile, rm, mkdir, writeFile, readdir, stat } from "fs/promises";
1
+
import { describe, it, expect } from "vitest";
2
+
import { readFile, readdir, stat } from "fs/promises";
5
3
import { join, relative, dirname } from "path";
6
4
import { fileURLToPath } from "url";
7
-
import { randomBytes } from "crypto";
8
-
import { tmpdir } from "os";
5
+
import { compile, NodeHost, resolvePath } from "@typespec/compiler";
6
+
import { TypeLexEmitter } from "../src/emitter.js";
9
7
10
-
const execAsync = promisify(exec);
11
8
const __filename = fileURLToPath(import.meta.url);
12
9
const __dirname = dirname(__filename);
13
10
11
+
const fixturesDir = join(__dirname, "fixtures");
12
+
const inputDir = join(fixturesDir, "input");
13
+
const outputDir = join(fixturesDir, "output");
14
+
15
+
async function findAllTspFiles(dir: string): Promise<string[]> {
16
+
const files: string[] = [];
17
+
18
+
async function walk(currentDir: string) {
19
+
const entries = await readdir(currentDir);
20
+
21
+
for (const entry of entries) {
22
+
const fullPath = join(currentDir, entry);
23
+
const stats = await stat(fullPath);
24
+
25
+
if (stats.isDirectory()) {
26
+
await walk(fullPath);
27
+
} else if (entry.endsWith('.tsp')) {
28
+
files.push(fullPath);
29
+
}
30
+
}
31
+
}
32
+
33
+
await walk(dir);
34
+
return files;
35
+
}
36
+
14
37
describe("Fixtures Tests", { timeout: 30000 }, () => {
15
-
const fixturesDir = join(__dirname, "fixtures");
16
-
const inputDir = join(fixturesDir, "input");
17
-
const outputDir = join(fixturesDir, "output");
18
-
19
-
let tempDir: string;
38
+
it("should compile all TypeSpec fixtures correctly", async () => {
39
+
const tspFiles = await findAllTspFiles(inputDir);
40
+
41
+
for (const tspFile of tspFiles) {
42
+
// Get relative path for test name
43
+
const relativePath = relative(inputDir, tspFile);
44
+
const testName = relativePath.replace(/\.tsp$/, '');
45
+
46
+
// Corresponding JSON file
47
+
const expectedJsonFile = join(outputDir, relativePath.replace(/\.tsp$/, '.json'));
48
+
49
+
// Skip if expected file doesn't exist
50
+
try {
51
+
await stat(expectedJsonFile);
52
+
} catch {
53
+
console.log(`Skipping ${testName} - no expected output file`);
54
+
continue;
55
+
}
20
56
21
-
beforeEach(async () => {
22
-
// Create a unique temp directory for each test
23
-
tempDir = join(tmpdir(), `typelex-test-${randomBytes(8).toString('hex')}`);
24
-
await mkdir(tempDir, { recursive: true });
25
-
});
57
+
console.log(`Testing ${testName}...`);
26
58
27
-
afterEach(async () => {
28
-
// Always clean up temp directory
29
-
if (tempDir) {
30
-
await rm(tempDir, { recursive: true, force: true }).catch(() => {
31
-
// Ignore errors during cleanup
59
+
// Compile in-memory using TypeSpec programmatic API
60
+
const host = NodeHost;
61
+
const program = await compile(host, tspFile, {
62
+
noEmit: false,
63
+
emitters: {},
32
64
});
33
-
}
34
-
});
35
65
36
-
async function findAllTspFiles(dir: string): Promise<string[]> {
37
-
const files: string[] = [];
38
-
39
-
async function walk(currentDir: string) {
40
-
const entries = await readdir(currentDir);
41
-
42
-
for (const entry of entries) {
43
-
const fullPath = join(currentDir, entry);
44
-
const stats = await stat(fullPath);
45
-
46
-
if (stats.isDirectory()) {
47
-
await walk(fullPath);
48
-
} else if (entry.endsWith('.tsp')) {
49
-
files.push(fullPath);
66
+
// Check for compilation errors
67
+
if (program.diagnostics.length > 0) {
68
+
const errors = program.diagnostics.filter(d => d.severity === "error");
69
+
if (errors.length > 0) {
70
+
throw new Error(`Compilation failed with errors for ${testName}: ${JSON.stringify(errors)}`);
50
71
}
51
72
}
52
-
}
53
-
54
-
await walk(dir);
55
-
return files;
56
-
}
73
+
74
+
// Create an in-memory output collector
75
+
const outputs = new Map<string, string>();
57
76
58
-
it("should compile com.atproto.lexicon.schema correctly", async () => {
59
-
const tspFile = join(inputDir, "com/atproto/lexicon/schema.tsp");
60
-
const expectedJsonFile = join(outputDir, "com/atproto/lexicon/schema.json");
61
-
62
-
// Write config to temp directory
63
-
const configPath = join(tempDir, "tspconfig.yaml");
64
-
const config = `
65
-
emit:
66
-
- "@typelex/emitter"
67
-
options:
68
-
"@typelex/emitter":
69
-
output-dir: "./output"
70
-
`;
71
-
await writeFile(configPath, config);
72
-
73
-
// Compile the TypeSpec file
74
-
try {
75
-
const { stdout, stderr } = await execAsync(
76
-
`npx -p @typespec/compiler tsp compile ${tspFile} --config ${configPath}`,
77
-
{ cwd: tempDir, timeout: 20000 }
78
-
);
79
-
80
-
console.log("Compilation stdout:", stdout);
81
-
if (stderr) console.log("Compilation stderr:", stderr);
82
-
83
-
// Only fail on actual errors, not warnings
84
-
if (stdout.includes("Found") && stdout.includes("error")) {
85
-
throw new Error(`Compilation failed with errors: ${stdout}\n${stderr}`);
77
+
// Create emitter with in-memory file writer
78
+
const emitter = new TypeLexEmitter(program, {
79
+
outputDir: "output",
80
+
});
81
+
82
+
// Override writeFile to capture output in memory
83
+
(emitter as any).writeFile = async (filePath: string, content: string) => {
84
+
// Extract the relative path from the full path
85
+
const relativePath = filePath.replace(/^output[\/\\]/, '');
86
+
outputs.set(relativePath, content);
87
+
};
88
+
89
+
await emitter.emit();
90
+
91
+
// Get the expected output path
92
+
const expectedRelativePath = relativePath.replace(/\.tsp$/, '.json');
93
+
const generatedJson = outputs.get(expectedRelativePath);
94
+
95
+
if (!generatedJson) {
96
+
throw new Error(`Expected output file not generated: ${expectedRelativePath}`);
86
97
}
87
-
} catch (error: any) {
88
-
console.error("Compilation error:", error);
89
-
throw error;
98
+
99
+
// Read the expected JSON
100
+
const expectedJson = JSON.parse(await readFile(expectedJsonFile, "utf-8"));
101
+
const parsedGenerated = JSON.parse(generatedJson);
102
+
103
+
// Compare with better error messages
104
+
try {
105
+
expect(parsedGenerated).toEqual(expectedJson);
106
+
console.log(`✅ ${testName} passed`);
107
+
} catch (error) {
108
+
console.error(`❌ ${testName} failed`);
109
+
console.error("Generated:", JSON.stringify(parsedGenerated, null, 2));
110
+
console.error("Expected:", JSON.stringify(expectedJson, null, 2));
111
+
throw error;
112
+
}
90
113
}
91
-
92
-
// Read the generated JSON
93
-
const generatedPath = join(tempDir, "output/com/atproto/lexicon/schema.json");
94
-
const generatedJson = JSON.parse(await readFile(generatedPath, "utf-8"));
95
-
96
-
// Read the expected JSON
97
-
const expectedJson = JSON.parse(await readFile(expectedJsonFile, "utf-8"));
98
-
99
-
// Compare
100
-
expect(generatedJson).toEqual(expectedJson);
101
114
});
102
115
});
+40
typelex-emitter/test/fixtures/input/app/bsky/feed/defs.tsp
+40
typelex-emitter/test/fixtures/input/app/bsky/feed/defs.tsp
···
1
+
namespace app.bsky.feed;
2
+
3
+
model PostView {
4
+
uri: string;
5
+
cid: string;
6
+
author: unknown; // ref to app.bsky.actor.defs#profileViewBasic
7
+
record: unknown;
8
+
embed?: unknown; // union type
9
+
bookmarkCount?: int32;
10
+
replyCount?: int32;
11
+
repostCount?: int32;
12
+
likeCount?: int32;
13
+
quoteCount?: int32;
14
+
indexedAt: utcDateTime;
15
+
viewer?: ViewerState;
16
+
labels?: unknown[]; // ref to com.atproto.label.defs#label
17
+
threadgate?: unknown; // ref to #threadgateView
18
+
}
19
+
20
+
@doc("Metadata about the requesting account's relationship with the subject content. Only has meaningful content for authed requests.")
21
+
model ViewerState {
22
+
repost?: string;
23
+
like?: string;
24
+
bookmarked?: boolean;
25
+
threadMuted?: boolean;
26
+
replyDisabled?: boolean;
27
+
embeddingDisabled?: boolean;
28
+
pinned?: boolean;
29
+
}
30
+
31
+
@doc("Metadata about this post within the context of the thread it is in.")
32
+
model ThreadContext {
33
+
rootAuthorLike?: string;
34
+
}
35
+
36
+
model FeedViewPost {
37
+
post: PostView;
38
+
reason?: unknown; // union type
39
+
feedContext?: string;
40
+
}
+9
typelex-emitter/test/fixtures/input/com/atproto/identity/resolveHandle.tsp
+9
typelex-emitter/test/fixtures/input/com/atproto/identity/resolveHandle.tsp
+27
typelex-emitter/test/fixtures/input/com/atproto/repo/createRecord.tsp
+27
typelex-emitter/test/fixtures/input/com/atproto/repo/createRecord.tsp
···
1
+
namespace com.atproto.repo;
2
+
3
+
@doc("Create a single new repository record. Requires auth, implemented by PDS.")
4
+
op createRecord(
5
+
@doc("The handle or DID of the repo (aka, current account).")
6
+
repo: string,
7
+
8
+
@doc("The NSID of the record collection.")
9
+
collection: string,
10
+
11
+
@doc("The Record Key.")
12
+
rkey?: string,
13
+
14
+
@doc("Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons.")
15
+
validate?: boolean,
16
+
17
+
@doc("The record itself. Must contain a $type field.")
18
+
record: unknown,
19
+
20
+
@doc("Compare and swap with the previous commit by CID.")
21
+
swapCommit?: string,
22
+
): {
23
+
uri: string;
24
+
cid: string;
25
+
commit?: unknown; // Should be ref to commitMeta
26
+
validationStatus?: string;
27
+
};