The Trans Directory
chore(build): separate markdown and html handling into two separate stages (#1675)
authored by
Anton Bulakh
and committed by
GitHub
c90dbaca
b7a945e0
+100
-34
+3
-2
quartz/bootstrap-worker.mjs
+3
-2
quartz/bootstrap-worker.mjs
···
1
1
#!/usr/bin/env node
2
2
import workerpool from "workerpool"
3
3
const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
4
-
const { parseFiles } = await import(cacheFile)
4
+
const { parseMarkdown, processHtml } = await import(cacheFile)
5
5
workerpool.worker({
6
-
parseFiles,
6
+
parseMarkdown,
7
+
processHtml,
7
8
})
+5
-3
quartz/plugins/vfile.ts
+5
-3
quartz/plugins/vfile.ts
···
1
-
import { Node, Parent } from "hast"
1
+
import { Root as HtmlRoot } from "hast"
2
+
import { Root as MdRoot } from "mdast"
2
3
import { Data, VFile } from "vfile"
3
4
4
5
export type QuartzPluginData = Data
5
-
export type ProcessedContent = [Node, VFile]
6
+
export type MarkdownContent = [MdRoot, VFile]
7
+
export type ProcessedContent = [HtmlRoot, VFile]
6
8
7
9
export function defaultProcessedContent(vfileData: Partial<QuartzPluginData>): ProcessedContent {
8
-
const root: Parent = { type: "root", children: [] }
10
+
const root: HtmlRoot = { type: "root", children: [] }
9
11
const vfile = new VFile("")
10
12
vfile.data = vfileData
11
13
return [root, vfile]
+63
-23
quartz/processors/parse.ts
+63
-23
quartz/processors/parse.ts
···
4
4
import { Processor, unified } from "unified"
5
5
import { Root as MDRoot } from "remark-parse/lib"
6
6
import { Root as HTMLRoot } from "hast"
7
-
import { ProcessedContent } from "../plugins/vfile"
7
+
import { MarkdownContent, ProcessedContent } from "../plugins/vfile"
8
8
import { PerfTimer } from "../util/perf"
9
9
import { read } from "to-vfile"
10
-
import { FilePath, QUARTZ, slugifyFilePath } from "../util/path"
10
+
import { FilePath, FullSlug, QUARTZ, slugifyFilePath } from "../util/path"
11
11
import path from "path"
12
12
import workerpool, { Promise as WorkerPromise } from "workerpool"
13
13
import { QuartzLogger } from "../util/log"
14
14
import { trace } from "../util/trace"
15
15
import { BuildCtx } from "../util/ctx"
16
16
17
-
export type QuartzProcessor = Processor<MDRoot, MDRoot, HTMLRoot>
18
-
export function createProcessor(ctx: BuildCtx): QuartzProcessor {
17
+
export type QuartzMdProcessor = Processor<MDRoot, MDRoot, MDRoot>
18
+
export type QuartzHtmlProcessor = Processor<undefined, MDRoot, HTMLRoot>
19
+
20
+
export function createMdProcessor(ctx: BuildCtx): QuartzMdProcessor {
19
21
const transformers = ctx.cfg.plugins.transformers
20
22
21
23
return (
···
24
26
.use(remarkParse)
25
27
// MD AST -> MD AST transforms
26
28
.use(
27
-
transformers
28
-
.filter((p) => p.markdownPlugins)
29
-
.flatMap((plugin) => plugin.markdownPlugins!(ctx)),
30
-
)
29
+
transformers.flatMap((plugin) => plugin.markdownPlugins?.(ctx) ?? []),
30
+
) as unknown as QuartzMdProcessor
31
+
// ^ sadly the typing of `use` is not smart enough to infer the correct type from our plugin list
32
+
)
33
+
}
34
+
35
+
export function createHtmlProcessor(ctx: BuildCtx): QuartzHtmlProcessor {
36
+
const transformers = ctx.cfg.plugins.transformers
37
+
return (
38
+
unified()
31
39
// MD AST -> HTML AST
32
40
.use(remarkRehype, { allowDangerousHtml: true })
33
41
// HTML AST -> HTML AST transforms
34
-
.use(transformers.filter((p) => p.htmlPlugins).flatMap((plugin) => plugin.htmlPlugins!(ctx)))
42
+
.use(transformers.flatMap((plugin) => plugin.htmlPlugins?.(ctx) ?? []))
35
43
)
36
44
}
37
45
···
75
83
76
84
export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
77
85
const { argv, cfg } = ctx
78
-
return async (processor: QuartzProcessor) => {
79
-
const res: ProcessedContent[] = []
86
+
return async (processor: QuartzMdProcessor) => {
87
+
const res: MarkdownContent[] = []
80
88
for (const fp of fps) {
81
89
try {
82
90
const perf = new PerfTimer()
···
100
108
res.push([newAst, file])
101
109
102
110
if (argv.verbose) {
103
-
console.log(`[process] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
111
+
console.log(`[markdown] ${fp} -> ${file.data.slug} (${perf.timeSince()})`)
104
112
}
105
113
} catch (err) {
106
-
trace(`\nFailed to process \`${fp}\``, err as Error)
114
+
trace(`\nFailed to process markdown \`${fp}\``, err as Error)
115
+
}
116
+
}
117
+
118
+
return res
119
+
}
120
+
}
121
+
122
+
export function createMarkdownParser(ctx: BuildCtx, mdContent: MarkdownContent[]) {
123
+
return async (processor: QuartzHtmlProcessor) => {
124
+
const res: ProcessedContent[] = []
125
+
for (const [ast, file] of mdContent) {
126
+
try {
127
+
const perf = new PerfTimer()
128
+
129
+
const newAst = await processor.run(ast as MDRoot, file)
130
+
res.push([newAst, file])
131
+
132
+
if (ctx.argv.verbose) {
133
+
console.log(`[html] ${file.data.slug} (${perf.timeSince()})`)
134
+
}
135
+
} catch (err) {
136
+
trace(`\nFailed to process html \`${file.data.filePath}\``, err as Error)
107
137
}
108
138
}
109
139
···
113
143
114
144
const clamp = (num: number, min: number, max: number) =>
115
145
Math.min(Math.max(Math.round(num), min), max)
146
+
116
147
export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
117
148
const { argv } = ctx
118
149
const perf = new PerfTimer()
···
126
157
log.start(`Parsing input files using ${concurrency} threads`)
127
158
if (concurrency === 1) {
128
159
try {
129
-
const processor = createProcessor(ctx)
130
-
const parse = createFileParser(ctx, fps)
131
-
res = await parse(processor)
160
+
const mdRes = await createFileParser(ctx, fps)(createMdProcessor(ctx))
161
+
res = await createMarkdownParser(ctx, mdRes)(createHtmlProcessor(ctx))
132
162
} catch (error) {
133
163
log.end()
134
164
throw error
···
140
170
maxWorkers: concurrency,
141
171
workerType: "thread",
142
172
})
173
+
const errorHandler = (err: any) => {
174
+
console.error(`${err}`.replace(/^error:\s*/i, ""))
175
+
process.exit(1)
176
+
}
177
+
178
+
const mdPromises: WorkerPromise<[MarkdownContent[], FullSlug[]]>[] = []
179
+
for (const chunk of chunks(fps, CHUNK_SIZE)) {
180
+
mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk]))
181
+
}
182
+
const mdResults: [MarkdownContent[], FullSlug[]][] =
183
+
await WorkerPromise.all(mdPromises).catch(errorHandler)
143
184
144
185
const childPromises: WorkerPromise<ProcessedContent[]>[] = []
145
-
for (const chunk of chunks(fps, CHUNK_SIZE)) {
146
-
childPromises.push(pool.exec("parseFiles", [ctx.buildId, argv, chunk, ctx.allSlugs]))
186
+
for (const [_, extraSlugs] of mdResults) {
187
+
ctx.allSlugs.push(...extraSlugs)
188
+
}
189
+
for (const [mdChunk, _] of mdResults) {
190
+
childPromises.push(pool.exec("processHtml", [ctx.buildId, argv, mdChunk, ctx.allSlugs]))
147
191
}
192
+
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler)
148
193
149
-
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch((err) => {
150
-
const errString = err.toString().slice("Error:".length)
151
-
console.error(errString)
152
-
process.exit(1)
153
-
})
154
194
res = results.flat()
155
195
await pool.terminate()
156
196
}
+29
-6
quartz/worker.ts
+29
-6
quartz/worker.ts
···
3
3
import cfg from "../quartz.config"
4
4
import { Argv, BuildCtx } from "./util/ctx"
5
5
import { FilePath, FullSlug } from "./util/path"
6
-
import { createFileParser, createProcessor } from "./processors/parse"
6
+
import {
7
+
createFileParser,
8
+
createHtmlProcessor,
9
+
createMarkdownParser,
10
+
createMdProcessor,
11
+
} from "./processors/parse"
7
12
import { options } from "./util/sourcemap"
13
+
import { MarkdownContent, ProcessedContent } from "./plugins/vfile"
8
14
9
15
// only called from worker thread
10
-
export async function parseFiles(
16
+
export async function parseMarkdown(
11
17
buildId: string,
12
18
argv: Argv,
13
19
fps: FilePath[],
20
+
): Promise<[MarkdownContent[], FullSlug[]]> {
21
+
// this is a hack
22
+
// we assume markdown parsers can add to `allSlugs`,
23
+
// but don't actually use them
24
+
const allSlugs: FullSlug[] = []
25
+
const ctx: BuildCtx = {
26
+
buildId,
27
+
cfg,
28
+
argv,
29
+
allSlugs,
30
+
}
31
+
return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs]
32
+
}
33
+
34
+
// only called from worker thread
35
+
export function processHtml(
36
+
buildId: string,
37
+
argv: Argv,
38
+
mds: MarkdownContent[],
14
39
allSlugs: FullSlug[],
15
-
) {
40
+
): Promise<ProcessedContent[]> {
16
41
const ctx: BuildCtx = {
17
42
buildId,
18
43
cfg,
19
44
argv,
20
45
allSlugs,
21
46
}
22
-
const processor = createProcessor(ctx)
23
-
const parse = createFileParser(ctx, fps)
24
-
return parse(processor)
47
+
return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx))
25
48
}