a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
feat(lex-cli): external formatter support
+216
-74
+5
.changeset/warm-tires-double.md
+5
.changeset/warm-tires-double.md
+1
packages/definitions/atproto/lex.config.js
+1
packages/definitions/atproto/lex.config.js
+1
packages/definitions/bluemoji/lex.config.js
+1
packages/definitions/bluemoji/lex.config.js
+1
packages/definitions/bluesky/lex.config.js
+1
packages/definitions/bluesky/lex.config.js
+1
packages/definitions/frontpage/lex.config.js
+1
packages/definitions/frontpage/lex.config.js
+1
packages/definitions/leaflet/lex.config.js
+1
packages/definitions/leaflet/lex.config.js
+1
packages/definitions/lexicon-community/lex.config.js
+1
packages/definitions/lexicon-community/lex.config.js
+1
packages/definitions/microcosm/lex.config.js
+1
packages/definitions/microcosm/lex.config.js
+1
packages/definitions/ozone/lex.config.js
+1
packages/definitions/ozone/lex.config.js
+1
packages/definitions/pckt/lex.config.js
+1
packages/definitions/pckt/lex.config.js
+1
packages/definitions/standard-site/lex.config.js
+1
packages/definitions/standard-site/lex.config.js
+1
packages/definitions/tangled/lex.config.js
+1
packages/definitions/tangled/lex.config.js
+1
packages/definitions/whitewind/lex.config.js
+1
packages/definitions/whitewind/lex.config.js
+6
-27
packages/lexicons/lex-cli/src/codegen.ts
+6
-27
packages/lexicons/lex-cli/src/codegen.ts
···
16
16
} from '@atcute/lexicon-doc';
17
17
import { formatLexiconRef, parseLexiconRef, type ParsedLexiconRef } from '@atcute/lexicon-doc';
18
18
19
-
import * as prettier from 'prettier';
20
-
21
19
export interface SourceFile {
22
20
filename: string;
23
21
code: string;
···
34
32
modules: {
35
33
importSuffix: string;
36
34
};
37
-
prettier: {
38
-
cwd: string;
39
-
};
40
-
}
41
-
42
-
export interface LexiconApiResult {
43
-
files: SourceFile[];
44
35
}
45
36
46
37
type DocumentMap = Map<string, LexiconDoc>;
···
68
59
69
60
const PURE = `/*#__PURE__*/`;
70
61
71
-
export const generateLexiconApi = async (opts: LexiconApiOptions): Promise<LexiconApiResult> => {
62
+
export function* generateLexiconApi(opts: LexiconApiOptions): Generator<SourceFile> {
72
63
const importExt = opts.modules?.importSuffix;
73
64
74
65
const documents = opts.documents.toSorted((a, b) => {
···
83
74
});
84
75
85
76
const map: DocumentMap = new Map(documents.map((doc) => [doc.id, doc]));
86
-
const files: SourceFile[] = [];
87
77
const generatedIds = new Set<string>();
88
78
89
79
for (const doc of documents) {
···
336
326
if (file.exports) {
337
327
generatedIds.add(doc.id);
338
328
339
-
files.push({
329
+
yield {
340
330
filename: filename,
341
331
code:
342
332
file.imports +
···
354
344
file.sinterfaces +
355
345
`\n\n` +
356
346
file.ambients,
357
-
});
347
+
};
358
348
}
359
349
}
360
350
···
369
359
code += `export * as ${toTitleCase(doc.id)} from ${lit(`./types/${doc.id.replaceAll('.', '/')}${importExt}`)};\n`;
370
360
}
371
361
372
-
files.push({
362
+
yield {
373
363
filename: 'index.ts',
374
364
code: code,
375
-
});
365
+
};
376
366
}
377
-
378
-
if (opts.prettier) {
379
-
const config = await prettier.resolveConfig(opts.prettier.cwd, { editorconfig: true });
380
-
381
-
for (const file of files) {
382
-
const formatted = await prettier.format(file.code, { ...config, parser: 'typescript' });
383
-
file.code = formatted;
384
-
}
385
-
}
386
-
387
-
return { files };
388
-
};
367
+
}
389
368
390
369
const generateXrpcQuery = (imports: ImportSet, path: ParsedLexiconRef, spec: LexXrpcQuery): string => {
391
370
const params = generateXrpcParameters(imports, path, spec.parameters);
+5
-17
packages/lexicons/lex-cli/src/commands/export.ts
+5
-17
packages/lexicons/lex-cli/src/commands/export.ts
···
8
8
import { type InferValue } from '@optique/core/parser';
9
9
import { command, constant } from '@optique/core/primitives';
10
10
import pc from 'picocolors';
11
-
import prettier from 'prettier';
12
11
13
12
import { loadConfig, type ExportConfig, type NormalizedConfig } from '../config.ts';
13
+
import { createFormatter, type Formatter } from '../formatter.ts';
14
14
import { loadLexicons } from '../lexicon-loader.ts';
15
15
import { sharedOptions } from '../shared-options.ts';
16
16
···
44
44
return config.export;
45
45
};
46
46
47
-
/**
48
-
* writes a lexicon document to disk as formatted JSON
49
-
* @param outdir output directory
50
-
* @param nsid the NSID of the lexicon
51
-
* @param doc the lexicon document
52
-
* @param prettierConfig prettier configuration
53
-
*/
54
47
const writeLexicon = async (
55
48
outdir: string,
56
49
nsid: string,
57
50
doc: LexiconDoc,
58
-
prettierConfig: prettier.Options | null,
51
+
formatter: Formatter,
59
52
): Promise<void> => {
60
53
const nsidPath = nsid.replaceAll('.', '/');
61
54
const target = path.join(outdir, `${nsidPath}.json`);
62
55
const dirname = path.dirname(target);
63
56
64
-
const code = await prettier.format(JSON.stringify(doc, null, 2), {
65
-
...prettierConfig,
66
-
parser: 'json',
67
-
});
57
+
const code = await formatter.format(JSON.stringify(doc, null, 2), target);
68
58
69
59
await fs.mkdir(dirname, { recursive: true });
70
60
await fs.writeFile(target, code);
···
81
71
// use export.files if specified, otherwise fall back to root files config
82
72
const files = exportConfig.files ?? config.files;
83
73
const outdir = path.resolve(config.root, exportConfig.outdir);
84
-
const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
74
+
const formatter = await createFormatter(config.formatter, config.root);
85
75
86
76
// load lexicons from files
87
77
const loaded = await loadLexicons(files, config.root);
···
99
89
await fs.mkdir(outdir, { recursive: true });
100
90
101
91
// write each lexicon as JSON
102
-
for (const { nsid, doc } of loaded) {
103
-
await writeLexicon(outdir, nsid, doc, prettierConfig);
104
-
}
92
+
await Promise.all(loaded.map(({ nsid, doc }) => writeLexicon(outdir, nsid, doc, formatter)));
105
93
106
94
console.log(pc.green(`exported ${loaded.length} lexicon(s) to ${outdir}`));
107
95
};
+16
-12
packages/lexicons/lex-cli/src/commands/generate.ts
+16
-12
packages/lexicons/lex-cli/src/commands/generate.ts
···
9
9
10
10
import { generateLexiconApi, type ImportMapping } from '../codegen.ts';
11
11
import { loadConfig } from '../config.ts';
12
+
import { createFormatter } from '../formatter.ts';
12
13
import { loadLexicons } from '../lexicon-loader.ts';
13
14
import { packageJsonSchema } from '../lexicon-metadata.ts';
14
15
import { sharedOptions } from '../shared-options.ts';
···
147
148
const loaded = await loadLexicons(config.files, config.root);
148
149
const documents = loaded.map((l) => l.doc);
149
150
150
-
const generationResult = await generateLexiconApi({
151
+
const outdir = path.join(config.root, config.outdir);
152
+
const formatter = await createFormatter(config.formatter, config.root);
153
+
const pending: Promise<void>[] = [];
154
+
155
+
for (const file of generateLexiconApi({
151
156
documents: documents,
152
157
mappings: allMappings,
153
158
modules: {
154
159
importSuffix: config.modules?.importSuffix ?? '.js',
155
160
},
156
-
prettier: {
157
-
cwd: process.cwd(),
158
-
},
159
-
});
160
-
161
-
const outdir = path.join(config.root, config.outdir);
162
-
163
-
for (const file of generationResult.files) {
161
+
})) {
164
162
const filename = path.join(outdir, file.filename);
165
-
const dirname = path.dirname(filename);
166
163
167
-
await fs.mkdir(dirname, { recursive: true });
168
-
await fs.writeFile(filename, file.code);
164
+
pending.push(
165
+
(async () => {
166
+
const formatted = await formatter.format(file.code, filename);
167
+
await fs.mkdir(path.dirname(filename), { recursive: true });
168
+
await fs.writeFile(filename, formatted);
169
+
})(),
170
+
);
169
171
}
172
+
173
+
await Promise.all(pending);
170
174
};
+12
-18
packages/lexicons/lex-cli/src/commands/pull.ts
+12
-18
packages/lexicons/lex-cli/src/commands/pull.ts
···
8
8
import { type InferValue } from '@optique/core/parser';
9
9
import { command, constant } from '@optique/core/primitives';
10
10
import pc from 'picocolors';
11
-
import prettier from 'prettier';
12
11
13
12
import { loadConfig, type NormalizedConfig, type PullConfig, type SourceConfig } from '../config.ts';
13
+
import { createFormatter, type Formatter } from '../formatter.ts';
14
14
import { pullAtprotoSource } from '../pull-sources/atproto.ts';
15
15
import { pullGitSource } from '../pull-sources/git.ts';
16
16
import type { PullResult, PulledLexicon, SourceLocation } from '../pull-sources/types.ts';
···
110
110
outdir: string,
111
111
nsid: string,
112
112
doc: LexiconDoc,
113
-
prettierConfig: prettier.Options | null,
113
+
formatter: Formatter,
114
114
): Promise<void> => {
115
115
const nsidPath = nsid.replaceAll('.', '/');
116
116
const target = path.join(outdir, `${nsidPath}.json`);
117
117
const dirname = path.dirname(target);
118
118
119
-
const code = await prettier.format(JSON.stringify(doc, null, 2), {
120
-
...prettierConfig,
121
-
parser: 'json',
122
-
});
119
+
const code = await formatter.format(JSON.stringify(doc, null, 2), target);
123
120
124
121
await fs.mkdir(dirname, { recursive: true });
125
122
await fs.writeFile(target, code);
···
139
136
const writeSourceReadme = async (
140
137
outdir: string,
141
138
revisions: SourceRevision[],
142
-
prettierConfig: prettier.Options | null,
139
+
formatter: Formatter,
143
140
): Promise<void> => {
144
141
const lines = [
145
142
'# lexicon sources',
···
173
170
lines.push('');
174
171
175
172
const content = lines.join('\n');
176
-
const formatted = await prettier.format(content, {
177
-
...prettierConfig,
178
-
parser: 'markdown',
179
-
});
173
+
const target = path.join(outdir, 'README.md');
174
+
const formatted = await formatter.format(content, target);
180
175
181
-
await fs.writeFile(path.join(outdir, 'README.md'), formatted);
176
+
await fs.writeFile(target, formatted);
182
177
};
183
178
184
179
/**
···
190
185
const pullConfig = ensurePullConfig(config);
191
186
192
187
const outdir = path.resolve(config.root, pullConfig.outdir);
193
-
const prettierConfig = await prettier.resolveConfig(config.root, { editorconfig: true });
188
+
const formatter = await createFormatter(config.formatter, config.root);
194
189
195
190
const seen = new Map<string, SourceLocation>();
196
191
const collected: PulledLexicon[] = [];
···
224
219
225
220
await fs.mkdir(outdir, { recursive: true });
226
221
227
-
for (const entry of collected) {
228
-
await writeLexicon(outdir, entry.nsid, entry.doc, prettierConfig);
229
-
}
230
-
231
-
await writeSourceReadme(outdir, sourceRevisions, prettierConfig);
222
+
await Promise.all([
223
+
...collected.map((entry) => writeLexicon(outdir, entry.nsid, entry.doc, formatter)),
224
+
writeSourceReadme(outdir, sourceRevisions, formatter),
225
+
]);
232
226
};
+15
packages/lexicons/lex-cli/src/config.ts
+15
packages/lexicons/lex-cli/src/config.ts
···
1
1
import * as fs from 'node:fs/promises';
2
+
import { availableParallelism } from 'node:os';
2
3
import * as path from 'node:path';
3
4
import * as url from 'node:url';
4
5
···
63
64
clean: v.boolean().optional(),
64
65
});
65
66
67
+
const formatterConfigSchema = v.union(
68
+
v.object({ type: v.literal('prettier') }),
69
+
v.object({
70
+
type: v.literal('command'),
71
+
command: v.string().assert((value) => value.length > 0, `must not be empty`),
72
+
concurrency: v
73
+
.number()
74
+
.assert((value) => Number.isInteger(value) && value > 0, `must be a positive integer`)
75
+
.optional(() => availableParallelism()),
76
+
}),
77
+
);
78
+
66
79
export type GitSourceConfig = v.Infer<typeof gitSourceConfigSchema>;
67
80
export type AtprotoNsidsSourceConfig = v.Infer<typeof atprotoNsidsSourceConfigSchema>;
68
81
export type AtprotoAuthoritySourceConfig = v.Infer<typeof atprotoAuthoritySourceConfigSchema>;
···
70
83
export type SourceConfig = v.Infer<typeof sourceConfigSchema>;
71
84
export type PullConfig = v.Infer<typeof pullConfigSchema>;
72
85
export type ExportConfig = v.Infer<typeof exportConfigSchema>;
86
+
export type FormatterConfig = v.Infer<typeof formatterConfigSchema>;
73
87
74
88
const isValidLexiconPattern = (pattern: string): boolean => {
75
89
if (pattern.endsWith('.*')) {
···
126
140
})
127
141
.partial()
128
142
.optional(),
143
+
formatter: formatterConfigSchema.optional((): FormatterConfig => ({ type: 'prettier' })),
129
144
pull: pullConfigSchema.optional(),
130
145
export: exportConfigSchema.optional(),
131
146
});
+145
packages/lexicons/lex-cli/src/formatter.ts
+145
packages/lexicons/lex-cli/src/formatter.ts
···
1
+
import { spawn } from 'node:child_process';
2
+
import { availableParallelism } from 'node:os';
3
+
4
+
import type { FormatterConfig } from './config.ts';
5
+
6
+
/** formats source code */
7
+
export interface Formatter {
8
+
/**
9
+
* formats the given code
10
+
* @param code source code to format
11
+
* @param filepath filepath hint for language detection and config resolution
12
+
* @returns formatted code
13
+
*/
14
+
format(code: string, filepath: string): Promise<string>;
15
+
}
16
+
17
+
const inferPrettierParser = (filepath: string): string => {
18
+
if (filepath.endsWith('.ts') || filepath.endsWith('.tsx')) {
19
+
return 'typescript';
20
+
}
21
+
if (filepath.endsWith('.json')) {
22
+
return 'json';
23
+
}
24
+
if (filepath.endsWith('.md') || filepath.endsWith('.markdown')) {
25
+
return 'markdown';
26
+
}
27
+
return 'typescript';
28
+
};
29
+
30
+
// #region semaphore
31
+
32
+
interface Lock {
33
+
release(): void;
34
+
}
35
+
36
+
class Semaphore {
37
+
private waiting: (() => void)[] = [];
38
+
private active = 0;
39
+
private max: number;
40
+
41
+
constructor(max: number) {
42
+
this.max = max;
43
+
}
44
+
45
+
acquire(): Promise<Lock> {
46
+
const lock: Lock = {
47
+
release: () => {
48
+
this.active--;
49
+
const next = this.waiting.shift();
50
+
if (next) {
51
+
this.active++;
52
+
next();
53
+
}
54
+
},
55
+
};
56
+
57
+
if (this.active < this.max) {
58
+
this.active++;
59
+
return Promise.resolve(lock);
60
+
}
61
+
62
+
const { promise, resolve } = Promise.withResolvers<Lock>();
63
+
this.waiting.push(() => resolve(lock));
64
+
return promise;
65
+
}
66
+
}
67
+
68
+
// #endregion
69
+
70
+
/**
71
+
* creates a formatter from the given configuration
72
+
* @param config formatter configuration
73
+
* @param root project root for config resolution
74
+
* @returns a formatter instance
75
+
*/
76
+
export const createFormatter = async (config: FormatterConfig, root: string): Promise<Formatter> => {
77
+
let inner: Formatter;
78
+
let concurrency: number;
79
+
80
+
if (config.type === 'prettier') {
81
+
const prettier = await import('prettier');
82
+
const prettierConfig = await prettier.resolveConfig(root, { editorconfig: true });
83
+
84
+
// prettier is in-process and CPU-bound, so concurrency only helps
85
+
// avoid buffering all files in memory at once
86
+
concurrency = availableParallelism();
87
+
inner = {
88
+
async format(code, filepath) {
89
+
return prettier.format(code, { ...prettierConfig, parser: inferPrettierParser(filepath) });
90
+
},
91
+
};
92
+
} else {
93
+
// the template uses {filepath} as a placeholder, which is passed as a
94
+
// positional argument to sh to avoid shell injection via filenames
95
+
const shellCmd = config.command.replaceAll('{filepath}', '"$1"');
96
+
97
+
concurrency = config.concurrency;
98
+
inner = {
99
+
format(code, filepath) {
100
+
return new Promise<string>((resolve, reject) => {
101
+
const child = spawn('sh', ['-c', shellCmd, 'sh', filepath], {
102
+
stdio: ['pipe', 'pipe', 'pipe'],
103
+
});
104
+
105
+
const stdoutChunks: Buffer[] = [];
106
+
const stderrChunks: Buffer[] = [];
107
+
108
+
child.stdout.on('data', (chunk: Buffer) => {
109
+
stdoutChunks.push(chunk);
110
+
});
111
+
112
+
child.stderr.on('data', (chunk: Buffer) => {
113
+
stderrChunks.push(chunk);
114
+
});
115
+
116
+
child.on('error', reject);
117
+
118
+
child.on('close', (exitCode: number | null) => {
119
+
if (exitCode !== 0) {
120
+
const stderr = Buffer.concat(stderrChunks).toString();
121
+
reject(new Error(`formatter exited with code ${exitCode}:\n${stderr}`));
122
+
} else {
123
+
resolve(Buffer.concat(stdoutChunks).toString());
124
+
}
125
+
});
126
+
127
+
child.stdin.end(code);
128
+
});
129
+
},
130
+
};
131
+
}
132
+
133
+
const semaphore = new Semaphore(concurrency);
134
+
135
+
return {
136
+
async format(code, filepath) {
137
+
const lock = await semaphore.acquire();
138
+
try {
139
+
return await inner.format(code, filepath);
140
+
} finally {
141
+
lock.release();
142
+
}
143
+
},
144
+
};
145
+
};