An experimental TypeSpec syntax for Lexicon
replacing bundling with externals in playground
+210
-86
+1
DOCS.md
+1
DOCS.md
+66
-3
packages/playground/samples/build.js
+66
-3
packages/playground/samples/build.js
···
1
1
// @ts-check
2
-
import { writeFileSync, mkdirSync } from "fs";
2
+
import { writeFileSync, mkdirSync, readFileSync } from "fs";
3
3
import { dirname, resolve, join } from "path";
4
4
import { fileURLToPath } from "url";
5
+
import { deepStrictEqual } from "assert";
5
6
import { lexicons, bundleLexicon } from "./index.js";
7
+
import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing";
6
8
7
9
const __dirname = dirname(fileURLToPath(import.meta.url));
8
10
const outputDir = resolve(__dirname, "dist");
11
+
const pkgRoot = await findTestPackageRoot(import.meta.url);
12
+
13
+
// TypeSpec library setup for testing
14
+
const TypelexTestLibrary = {
15
+
name: "@typelex/emitter",
16
+
packageRoot: pkgRoot.replace("/playground", "/emitter"),
17
+
files: [
18
+
{ realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" },
19
+
{ realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" },
20
+
{ realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" },
21
+
],
22
+
};
9
23
10
24
// Create output directory
11
25
mkdirSync(outputDir, { recursive: true });
12
26
13
-
// Write each bundled lexicon to disk
27
+
// Write each bundled lexicon to disk and verify it compiles correctly
14
28
const samplesList = {};
15
29
16
30
for (const [namespace, lexicon] of lexicons) {
···
20
34
21
35
writeFileSync(filepath, bundled);
22
36
37
+
const host = await createTestHost({ libraries: [TypelexTestLibrary] });
38
+
host.addTypeSpecFile("main.tsp", bundled);
39
+
40
+
const baseOutputPath = resolveVirtualPath("test-output/");
41
+
const [, diagnostics] = await host.compileAndDiagnose("main.tsp", {
42
+
outputDir: baseOutputPath,
43
+
noEmit: false,
44
+
emit: ["@typelex/emitter"],
45
+
});
46
+
47
+
if (diagnostics.length > 0) {
48
+
console.error(`❌ ${namespace}: Compilation errors`);
49
+
diagnostics.forEach(d => console.error(` ${d.message}`));
50
+
process.exit(1);
51
+
}
52
+
53
+
// Get emitted JSON
54
+
const outputFiles = [...host.fs.entries()]
55
+
.filter(([name]) => name.startsWith(baseOutputPath))
56
+
.map(([name, value]) => {
57
+
let relativePath = name.replace(baseOutputPath, "");
58
+
if (relativePath.startsWith("@typelex/emitter/")) {
59
+
relativePath = relativePath.replace("@typelex/emitter/", "");
60
+
}
61
+
return [relativePath, value];
62
+
});
63
+
64
+
const expectedJsonPath = namespace.replace(/\./g, "/") + ".json";
65
+
const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath);
66
+
67
+
if (!emittedJson) {
68
+
console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`);
69
+
process.exit(1);
70
+
}
71
+
72
+
// Compare with expected JSON
73
+
const expectedJsonFile = join(
74
+
pkgRoot.replace("/playground", "/emitter"),
75
+
"test/integration",
76
+
lexicon.suite,
77
+
"output",
78
+
lexicon.file.replace(".tsp", ".json")
79
+
);
80
+
81
+
const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8"));
82
+
const actualJson = JSON.parse(emittedJson[1]);
83
+
84
+
deepStrictEqual(actualJson, expectedJson);
85
+
23
86
samplesList[namespace] = {
24
87
filename: `samples/dist/${filename}`,
25
88
preferredEmitter: "@typelex/emitter",
···
30
93
const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`;
31
94
writeFileSync(join(outputDir, "samples.js"), samplesIndex);
32
95
33
-
console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`);
96
+
console.log(`\n✅ ${lexicons.size} samples verified successfully`);
+143
-83
packages/playground/samples/index.js
+143
-83
packages/playground/samples/index.js
···
5
5
6
6
const __dirname = dirname(fileURLToPath(import.meta.url));
7
7
8
-
// Get all tsp files
9
-
function getAllTspFiles(dir, baseDir = dir) {
8
+
// Get all tsp and json files
9
+
function getAllFiles(dir, baseDir = dir) {
10
10
const files = [];
11
11
const entries = readdirSync(dir);
12
12
···
15
15
const stat = statSync(fullPath);
16
16
17
17
if (stat.isDirectory()) {
18
-
files.push(...getAllTspFiles(fullPath, baseDir));
19
-
} else if (entry.endsWith(".tsp")) {
18
+
files.push(...getAllFiles(fullPath, baseDir));
19
+
} else if (entry.endsWith(".tsp") || entry.endsWith(".json")) {
20
20
files.push(relative(baseDir, fullPath));
21
21
}
22
22
}
···
24
24
return files.sort();
25
25
}
26
26
27
-
// Extract dependencies from a file
28
-
function extractDependencies(content) {
29
-
const deps = new Set();
30
-
// Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main"
31
-
// Pattern: word.word.word... followed by dot and identifier starting with capital letter
32
-
const pattern =
33
-
/\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g;
34
-
const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, "");
27
+
// Extract all refs from JSON (recursively search for strings with #)
28
+
function extractRefsFromJson(obj, refs = new Map()) {
29
+
if (typeof obj === "string") {
30
+
// Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref)
31
+
const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/);
32
+
if (match) {
33
+
const ns = match[1];
34
+
const def = match[2];
35
+
const modelName = def.charAt(0).toUpperCase() + def.slice(1);
36
+
if (!refs.has(ns)) {
37
+
refs.set(ns, new Set());
38
+
}
39
+
refs.get(ns).add(modelName);
40
+
} else {
41
+
// Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots)
42
+
const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/);
43
+
if (nsMatch) {
44
+
const ns = nsMatch[1];
45
+
if (!refs.has(ns)) {
46
+
refs.set(ns, new Set());
47
+
}
48
+
refs.get(ns).add("Main");
49
+
}
50
+
}
51
+
} else if (Array.isArray(obj)) {
52
+
for (const item of obj) {
53
+
extractRefsFromJson(item, refs);
54
+
}
55
+
} else if (obj && typeof obj === "object") {
56
+
for (const value of Object.values(obj)) {
57
+
extractRefsFromJson(value, refs);
58
+
}
59
+
}
60
+
return refs;
61
+
}
35
62
36
-
const matches = withoutDeclaration.matchAll(pattern);
37
-
for (const match of matches) {
38
-
deps.add(match[1]);
39
-
}
63
+
const integrationDir = join(__dirname, "../../emitter/test/integration");
64
+
65
+
// Get all test suite directories
66
+
const testSuites = readdirSync(integrationDir).filter((name) => {
67
+
const fullPath = join(integrationDir, name);
68
+
return statSync(fullPath).isDirectory() && !name.startsWith(".");
69
+
});
40
70
41
-
return Array.from(deps);
42
-
}
71
+
// Build lexicons with refs extracted from JSON
72
+
const lexicons = new Map(); // namespace -> { file, content, refs, suite }
43
73
44
-
const atprotoInputDir = join(
45
-
__dirname,
46
-
"../../emitter/test/integration/atproto/input",
47
-
);
48
-
const lexiconExamplesDir = join(
49
-
__dirname,
50
-
"../../emitter/test/integration/lexicon-examples/input",
51
-
);
74
+
// Process all test suites
75
+
for (const suite of testSuites) {
76
+
const inputDir = join(integrationDir, suite, "input");
77
+
const outputDir = join(integrationDir, suite, "output");
52
78
53
-
const atprotoFiles = getAllTspFiles(atprotoInputDir);
54
-
const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir);
79
+
const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp"));
55
80
56
-
// Build dependency graph
57
-
const lexicons = new Map(); // namespace -> { file, content, deps }
81
+
for (const file of inputFiles) {
82
+
const fullPath = join(inputDir, file);
83
+
const content = readFileSync(fullPath, "utf-8");
84
+
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
58
85
59
-
// Process atproto files
60
-
for (const file of atprotoFiles) {
61
-
const fullPath = join(atprotoInputDir, file);
62
-
const content = readFileSync(fullPath, "utf-8");
63
-
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
64
-
const deps = extractDependencies(content);
86
+
// Find corresponding JSON output
87
+
const jsonFile = file.replace(/\.tsp$/, ".json");
88
+
const jsonPath = join(outputDir, jsonFile);
89
+
const jsonContent = readFileSync(jsonPath, "utf-8");
90
+
const jsonData = JSON.parse(jsonContent);
91
+
const refs = extractRefsFromJson(jsonData);
65
92
66
-
lexicons.set(namespace, { file: `atproto/${file}`, content, deps });
93
+
lexicons.set(namespace, { file, content, refs, suite });
94
+
}
67
95
}
68
96
69
-
// Process lexicon-examples files
70
-
for (const file of lexiconExampleFiles) {
71
-
const fullPath = join(lexiconExamplesDir, file);
72
-
const content = readFileSync(fullPath, "utf-8");
73
-
const namespace = file.replace(/\.tsp$/, "").replace(/\//g, ".");
74
-
const deps = extractDependencies(content);
97
+
// TypeSpec reserved keywords that need escaping
98
+
const TYPESPEC_KEYWORDS = new Set([
99
+
"record",
100
+
"pub",
101
+
"interface",
102
+
"model",
103
+
"namespace",
104
+
"op",
105
+
"import",
106
+
"export",
107
+
"using",
108
+
"alias",
109
+
"enum",
110
+
"union",
111
+
"scalar",
112
+
"extends",
113
+
]);
75
114
76
-
lexicons.set(namespace, { file: `examples/${file}`, content, deps });
115
+
// Escape a namespace part if it's a reserved keyword
116
+
function escapeNamespacePart(part) {
117
+
return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part;
77
118
}
78
119
79
-
// Recursively collect all dependencies (topological sort)
80
-
function collectDependencies(
81
-
namespace,
82
-
collected = new Set(),
83
-
visiting = new Set(),
84
-
) {
85
-
if (collected.has(namespace)) return;
86
-
if (visiting.has(namespace)) return; // circular dependency
120
+
// Escape a full namespace path
121
+
function escapeNamespace(namespace) {
122
+
return namespace.split(".").map(escapeNamespacePart).join(".");
123
+
}
87
124
125
+
// Get the JSON for a lexicon to check its definitions
126
+
function getLexiconJson(namespace) {
88
127
const lexicon = lexicons.get(namespace);
89
-
if (!lexicon) return;
128
+
if (!lexicon) return null;
90
129
91
-
visiting.add(namespace);
130
+
const jsonPath = join(
131
+
integrationDir,
132
+
lexicon.suite,
133
+
"output",
134
+
lexicon.file.replace(".tsp", ".json"),
135
+
);
92
136
93
-
// First collect all dependencies
94
-
for (const dep of lexicon.deps) {
95
-
collectDependencies(dep, collected, visiting);
137
+
try {
138
+
return JSON.parse(readFileSync(jsonPath, "utf-8"));
139
+
} catch {
140
+
return null;
96
141
}
142
+
}
97
143
98
-
visiting.delete(namespace);
99
-
collected.add(namespace);
144
+
// Check if a definition in JSON is a token
145
+
function isToken(lexiconJson, defName) {
146
+
if (!lexiconJson || !lexiconJson.defs) return false;
147
+
const def = lexiconJson.defs[defName];
148
+
return def && def.type === "token";
100
149
}
101
150
102
-
// Bundle a lexicon with all its dependencies
151
+
// Bundle a lexicon with stubs for referenced types (from JSON)
103
152
function bundleLexicon(namespace) {
104
-
const collected = new Set();
105
-
collectDependencies(namespace, collected);
153
+
const mainLexicon = lexicons.get(namespace);
154
+
if (!mainLexicon) return "";
106
155
107
-
// Put the main lexicon FIRST, then its dependencies
108
-
const mainLexicon = lexicons.get(namespace);
109
-
const deps = Array.from(collected).filter((ns) => ns !== namespace);
156
+
let bundled = mainLexicon.content;
110
157
111
-
let bundled = 'import "@typelex/emitter";\n\n';
158
+
// Add stubs from refs extracted from JSON output (excluding self-references)
159
+
if (mainLexicon.refs.size > 0) {
160
+
let hasExternalRefs = false;
161
+
for (const [ns] of mainLexicon.refs) {
162
+
if (ns !== namespace) {
163
+
hasExternalRefs = true;
164
+
break;
165
+
}
166
+
}
112
167
113
-
// Main lexicon first (so it shows in the playground)
114
-
if (mainLexicon) {
115
-
const contentWithoutImport = mainLexicon.content.replace(
116
-
/^import "@typelex\/emitter";\s*\n/,
117
-
"",
118
-
);
119
-
bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`;
120
-
}
168
+
if (hasExternalRefs) {
169
+
bundled += "\n// --- Externals ---\n";
170
+
}
121
171
122
-
// Then dependencies
123
-
for (const ns of deps) {
124
-
const lexicon = lexicons.get(ns);
125
-
if (!lexicon) continue;
172
+
for (const [ns, models] of mainLexicon.refs) {
173
+
// Skip if this is the current namespace
174
+
if (ns === namespace) continue;
126
175
127
-
const contentWithoutImport = lexicon.content.replace(
128
-
/^import "@typelex\/emitter";\s*\n/,
129
-
"",
130
-
);
131
-
bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`;
176
+
// Get the JSON for this referenced namespace to check for tokens
177
+
const refJson = getLexiconJson(ns);
178
+
179
+
const escapedNs = escapeNamespace(ns);
180
+
bundled += `\n@external\nnamespace ${escapedNs} {\n`;
181
+
for (const model of models) {
182
+
// Check if this definition exists in the JSON and is a token
183
+
const defName = model.charAt(0).toLowerCase() + model.slice(1);
184
+
if (refJson && isToken(refJson, defName)) {
185
+
bundled += ` @token model ${model} { }\n`;
186
+
} else {
187
+
bundled += ` model ${model} { }\n`;
188
+
}
189
+
}
190
+
bundled += `}\n`;
191
+
}
132
192
}
133
193
134
194
return bundled;