forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Unified build+compile+link pipeline in one DAG executor pass
Build phase integrated into the doc generation DAG. When --with-doc,
a single Dag_executor.execute call handles building packages, compiling
docs, and linking docs — all with parallel execution and priority
scheduling (link > compile > build).
The build_one callback is passed from cmd_batch into Generate.run,
so the doc library doesn't need to know about patches, repo mounts,
or other build-specific config.
When --with-doc is not given, falls back to a build-only executor pass.
Tested: fresh cache, 19 targets, 550 layers, 8069 HTML files, 0 failures.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+35
-58
+23
-50
day11/bin/cmd_batch.ml
+23
-50
day11/bin/cmd_batch.ml
···
281
281
"/home/opam/.opam/repo/default" in
282
282
(* Bless *)
283
283
let blessing_maps = Day11_batch.Blessing.compute_blessings solutions in
284
-
(* Build *)
285
-
let succeeded = Atomic.make 0 in
286
-
let failed = Atomic.make 0 in
287
-
Day11_build.Dag_executor.execute env ~np
288
-
~on_complete:(fun ~total ~completed ~failed:f node success ->
289
-
if success then begin
290
-
Atomic.incr succeeded;
291
-
if completed mod 100 = 0 then
292
-
Printf.printf "[%d/%d, %d failed] %s\n%!"
293
-
completed total f (OpamPackage.to_string node.pkg)
294
-
end else begin
295
-
Atomic.incr failed;
296
-
Printf.printf "[%d/%d] FAIL: %s\n%!"
297
-
completed total (OpamPackage.to_string node.pkg)
298
-
end)
299
-
~on_cascade:(fun ~failed ~failed_dep ->
300
-
(* Write a cascade marker so we can distinguish root failures
301
-
from dependency-caused failures *)
302
-
let layer_dir = Day11_layer.Layer_type.build_dir ~os_dir failed in
303
-
let layer_json = Fpath.(layer_dir / "layer.json") in
304
-
if not (Bos.OS.File.exists layer_json |> Result.get_ok) then begin
305
-
ignore (Bos.OS.Dir.create ~path:true layer_dir);
306
-
let meta : Day11_layer.Layer_meta.build_meta = {
307
-
package = OpamPackage.to_string failed.pkg;
308
-
exit_status = -1;
309
-
deps = List.map (fun (d : Day11_layer.Layer_type.build) ->
310
-
OpamPackage.to_string d.pkg) failed.deps;
311
-
hashes = List.map (fun (d : Day11_layer.Layer_type.build) ->
312
-
d.hash) failed.deps;
313
-
uid = 0; gid = 0; base_hash = "";
314
-
installed_libs = []; installed_docs = []; patches = [];
315
-
failed_dep = Some (OpamPackage.to_string failed_dep.pkg);
316
-
disk_usage = 0;
317
-
created_at = Day11_layer.Layer_meta.now_iso8601 ();
318
-
} in
319
-
ignore (Day11_layer.Layer_meta.save_build layer_json meta)
320
-
end)
321
-
nodes
322
-
(fun node ->
323
-
match Day11_build.Build_layer.build env benv ?patches
324
-
~mounts:[repo_mount] node () with
325
-
| Day11_build.Types.Success _ -> true
326
-
| _ -> false);
327
-
let n_succeeded = Atomic.get succeeded in
328
-
let n_failed = Atomic.get failed in
329
-
Printf.printf "\n=== Build: %d succeeded, %d failed ===\n%!"
330
-
n_succeeded n_failed;
284
+
(* Build function for the unified DAG *)
285
+
let build_one node =
286
+
match Day11_build.Build_layer.build env benv ?patches
287
+
~mounts:[repo_mount] node () with
288
+
| Day11_build.Types.Success _ -> true
289
+
| _ -> false
290
+
in
331
291
(* Write run summary — maps each layer hash to its package + status *)
332
292
let runs_dir = Fpath.(cache_dir / "runs") in
333
293
Bos.OS.Dir.create ~path:true runs_dir |> ignore;
···
357
317
] in
358
318
let run_file = Fpath.(runs_dir / (timestamp ^ ".json")) in
359
319
ignore (Bos.OS.File.write run_file (Yojson.Safe.pretty_to_string run_json));
360
-
(* Docs *)
320
+
(* Build + Docs (unified pipeline when --with-doc) *)
361
321
if with_doc then
362
322
Day11_doc.Generate.build_tools_and_run env benv ~np ~os_dir
363
323
~packages:git_packages ~opam_env ~mounts:[repo_mount]
364
-
~driver_compiler ~odoc_repo
365
-
~nodes ~solutions ~blessing_maps;
324
+
~driver_compiler ~odoc_repo ~build_one
325
+
~nodes ~solutions ~blessing_maps
326
+
else begin
327
+
(* Build only — no docs *)
328
+
Day11_build.Dag_executor.execute env ~np
329
+
~on_complete:(fun ~total ~completed ~failed node success ->
330
+
if not success then
331
+
Printf.printf "[%d/%d, %d failed] FAIL: %s\n%!"
332
+
completed total failed (OpamPackage.to_string node.pkg)
333
+
else if completed mod 100 = 0 then
334
+
Printf.printf "[%d/%d, %d failed] %s\n%!"
335
+
completed total failed (OpamPackage.to_string node.pkg))
336
+
~on_cascade:(fun ~failed:_ ~failed_dep:_ -> ())
337
+
nodes build_one
338
+
end;
366
339
(* JTW *)
367
340
(match jtw_repo with
368
341
| Some dir ->
+5
-4
day11/doc/generate.ml
+5
-4
day11/doc/generate.ml
···
161
161
result
162
162
163
163
let run env benv ~np ~os_dir ~driver_tool ~odoc_tools
164
-
~nodes ~solutions ~blessing_maps =
164
+
~build_one ~nodes ~solutions ~blessing_maps =
165
165
(* Map package -> compiler for odoc tool selection *)
166
166
let pkg_compiler = Hashtbl.create 64 in
167
167
List.iter (fun (_target, solution) ->
···
305
305
true
306
306
| None -> true
307
307
end else
308
-
true (* build node — already built, skip *));
308
+
(* Build node — delegate to the build callback *)
309
+
build_one node);
309
310
(Atomic.get doc_count, Atomic.get doc_html)
310
311
311
312
let unique_compilers solutions =
···
319
320
) solutions
320
321
321
322
let build_tools_and_run env benv ~np ~os_dir ~packages ~opam_env
322
-
~mounts ~driver_compiler ~odoc_repo
323
+
~mounts ~driver_compiler ~odoc_repo ~build_one
323
324
~nodes ~solutions ~blessing_maps =
324
325
Printf.printf "\nBuilding doc tools...\n%!";
325
326
let all_pins, all_source_dirs = match odoc_repo with
···
369
370
(* 3. Generate docs *)
370
371
Printf.printf "Generating docs...\n%!";
371
372
let doc_count, doc_html =
372
-
run env benv ~np ~os_dir ~driver_tool ~odoc_tools
373
+
run env benv ~np ~os_dir ~driver_tool ~odoc_tools ~build_one
373
374
~nodes ~solutions ~blessing_maps in
374
375
Printf.printf "\n=== Docs: %d packages, %d HTML files ===\n%!"
375
376
doc_count doc_html
+7
-4
day11/doc/generate.mli
+7
-4
day11/doc/generate.mli
···
34
34
os_dir:Fpath.t ->
35
35
driver_tool:Day11_layer.Layer_type.tool ->
36
36
odoc_tools:(OpamPackage.t * Day11_layer.Layer_type.tool) list ->
37
+
build_one:(Day11_layer.Layer_type.build -> bool) ->
37
38
nodes:Day11_layer.Layer_type.build list ->
38
39
solutions:(OpamPackage.t * Day11_graph.Graph.solution) list ->
39
40
blessing_maps:(OpamPackage.t * bool OpamPackage.Map.t) list ->
40
41
int * int
41
-
(** [run env benv ~np ~os_dir ~driver_tool ~odoc_tools ~nodes ~solutions
42
-
~blessing_maps] generates documentation for all packages in
43
-
[nodes]. Compile phase runs in dependency order with [np] workers.
44
-
Link phase runs fully parallel. Returns [(doc_count, html_count)]. *)
42
+
(** [run env benv ~np ~os_dir ~driver_tool ~odoc_tools ~build_one
43
+
~nodes ~solutions ~blessing_maps] builds and documents all packages
44
+
in [nodes] using a single parallel DAG. Build nodes use [build_one],
45
+
compile nodes follow build dependency order, link nodes depend on
46
+
all compiles in the solution. Returns [(doc_count, html_count)]. *)
45
47
46
48
val build_tools_and_run :
47
49
Eio_unix.Stdenv.base ->
···
53
55
mounts:Day11_container.Mount.t list ->
54
56
driver_compiler:OpamPackage.t ->
55
57
odoc_repo:string option ->
58
+
build_one:(Day11_layer.Layer_type.build -> bool) ->
56
59
nodes:Day11_layer.Layer_type.build list ->
57
60
solutions:(OpamPackage.t * Day11_graph.Graph.solution) list ->
58
61
blessing_maps:(OpamPackage.t * bool OpamPackage.Map.t) list ->