forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Extract business logic from cmd_batch: 800 -> 428 lines
- Target resolution (small_universe, resolve, pick_latest_version)
moved to day11_batch.Targets
- Doc tool building + generation wrapped in
Day11_doc.Generate.build_tools_and_run
- JTW tool building moved to Day11_jtw.Build_tools.build_per_compiler
cmd_batch.ml is now orchestration: parse args, solve, build DAG,
build packages, write summary, call doc/jtw libraries.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+224
-183
+71
day11/batch/targets.ml
+71
day11/batch/targets.ml
···
1
+
let small_universe = [
2
+
"astring"; "fmt"; "fpath"; "rresult"; "bos"; "logs"; "cmdliner";
3
+
"ptime"; "uutf"; "mtime"; "yojson"; "eio"; "lwt"; "ppxlib";
4
+
"odoc"; "odoc-parser"; "re"; "cstruct"; "bigstringaf";
5
+
]
6
+
7
+
let find_latest_versions git_packages =
8
+
let all_names = Day11_solver.Git_packages.all_names git_packages in
9
+
let compiler_names = Day11_layer.Opamh.compiler_packages in
10
+
let all_names = List.filter (fun name ->
11
+
not (List.mem name compiler_names)
12
+
) all_names in
13
+
List.filter_map (fun name ->
14
+
let versions = Day11_solver.Git_packages.get_versions git_packages name in
15
+
let non_avoided =
16
+
OpamPackage.Version.Map.filter (fun _v opam ->
17
+
not (OpamFile.OPAM.has_flag Pkgflag_AvoidVersion opam)
18
+
) versions
19
+
in
20
+
let versions = if OpamPackage.Version.Map.is_empty non_avoided
21
+
then versions else non_avoided in
22
+
match OpamPackage.Version.Map.max_binding_opt versions with
23
+
| Some (v, _) -> Some (OpamPackage.create name v)
24
+
| None -> None
25
+
) all_names
26
+
27
+
let load_package_list filename =
28
+
let json = Yojson.Safe.from_file filename in
29
+
let open Yojson.Safe.Util in
30
+
match json |> member "packages" with
31
+
| `List l -> List.filter_map (fun j ->
32
+
try Some (OpamPackage.of_string (to_string j))
33
+
with _ -> None) l
34
+
| _ ->
35
+
json |> to_list |> List.filter_map (fun j ->
36
+
try Some (OpamPackage.of_string (to_string j))
37
+
with _ -> None)
38
+
39
+
let pick_latest_version git_packages name =
40
+
let n = OpamPackage.Name.of_string name in
41
+
let versions = Day11_solver.Git_packages.get_versions git_packages n in
42
+
let non_avoided =
43
+
OpamPackage.Version.Map.filter (fun _v opam ->
44
+
not (OpamFile.OPAM.has_flag Pkgflag_AvoidVersion opam)
45
+
) versions in
46
+
let versions = if OpamPackage.Version.Map.is_empty non_avoided
47
+
then versions else non_avoided in
48
+
let candidates = OpamPackage.Version.Map.bindings versions
49
+
|> List.rev in
50
+
List.map (fun (v, _) -> OpamPackage.create n v) candidates
51
+
52
+
let resolve ?(small = false) git_packages target =
53
+
match target with
54
+
| None when small ->
55
+
Printf.printf "Using small universe (%d packages)...\n%!"
56
+
(List.length small_universe);
57
+
List.filter_map (fun name ->
58
+
match pick_latest_version git_packages name with
59
+
| v :: _ -> Some v
60
+
| [] -> None
61
+
) small_universe
62
+
| None ->
63
+
Printf.printf "Finding all latest package versions...\n%!";
64
+
find_latest_versions git_packages
65
+
| Some t when String.length t > 0 && t.[0] = '@' ->
66
+
let filename = String.sub t 1 (String.length t - 1) in
67
+
Printf.printf "Loading package list from %s...\n%!" filename;
68
+
load_package_list filename
69
+
| Some t ->
70
+
Printf.printf "Target: %s\n%!" t;
71
+
[ OpamPackage.of_string t ]
+23
day11/batch/targets.mli
+23
day11/batch/targets.mli
···
1
+
(** Target package resolution for batch builds.
2
+
3
+
Determines which packages to solve and build, from CLI arguments:
4
+
- No target + [--small-universe]: a curated list of ~20 packages
5
+
- No target: all latest versions from the package index
6
+
- [@filename]: load from a JSON file
7
+
- [pkg.version]: a single explicit target *)
8
+
9
+
val small_universe : string list
10
+
(** Package names for the small universe. *)
11
+
12
+
val pick_latest_version :
13
+
Day11_solver.Git_packages.t -> string -> OpamPackage.t list
14
+
(** [pick_latest_version packages name] returns all non-avoid versions
15
+
of [name] from newest to oldest. Used for retry on solve failure. *)
16
+
17
+
val resolve :
18
+
?small:bool ->
19
+
Day11_solver.Git_packages.t ->
20
+
string option ->
21
+
OpamPackage.t list
22
+
(** [resolve ?small packages target] resolves the target specification
23
+
to a list of packages. *)
+10
-172
day11/bin/cmd_batch.ml
+10
-172
day11/bin/cmd_batch.ml
···
3
3
open Cmdliner
4
4
open Day11_layer.Layer_type
5
5
6
-
let find_latest_versions git_packages =
7
-
let all_names = Day11_solver.Git_packages.all_names git_packages in
8
-
(* Filter out compiler packages — they're built as part of the
9
-
base image, not as regular targets *)
10
-
let compiler_names = Day11_layer.Opamh.compiler_packages in
11
-
let all_names = List.filter (fun name ->
12
-
not (List.mem name compiler_names)
13
-
) all_names in
14
-
List.filter_map (fun name ->
15
-
let versions = Day11_solver.Git_packages.get_versions git_packages name in
16
-
let non_avoided =
17
-
OpamPackage.Version.Map.filter (fun _v opam ->
18
-
not (OpamFile.OPAM.has_flag Pkgflag_AvoidVersion opam)
19
-
) versions
20
-
in
21
-
let versions = if OpamPackage.Version.Map.is_empty non_avoided
22
-
then versions else non_avoided in
23
-
match OpamPackage.Version.Map.max_binding_opt versions with
24
-
| Some (v, _) -> Some (OpamPackage.create name v)
25
-
| None -> None
26
-
) all_names
27
-
28
-
let load_package_list filename =
29
-
let json = Yojson.Safe.from_file filename in
30
-
let open Yojson.Safe.Util in
31
-
match json |> member "packages" with
32
-
| `List l -> List.filter_map (fun j ->
33
-
try Some (OpamPackage.of_string (to_string j))
34
-
with _ -> None) l
35
-
| _ ->
36
-
(* Try as plain list *)
37
-
json |> to_list |> List.filter_map (fun j ->
38
-
try Some (OpamPackage.of_string (to_string j))
39
-
with _ -> None)
40
-
41
-
let small_universe = [
42
-
"astring"; "fmt"; "fpath"; "rresult"; "bos"; "logs"; "cmdliner";
43
-
"ptime"; "uutf"; "mtime"; "yojson"; "eio"; "lwt"; "ppxlib";
44
-
"odoc"; "odoc-parser"; "re"; "cstruct"; "bigstringaf";
45
-
]
46
-
47
-
let pick_latest_version git_packages name =
48
-
let n = OpamPackage.Name.of_string name in
49
-
let versions = Day11_solver.Git_packages.get_versions git_packages n in
50
-
let non_avoided =
51
-
OpamPackage.Version.Map.filter (fun _v opam ->
52
-
not (OpamFile.OPAM.has_flag Pkgflag_AvoidVersion opam)
53
-
) versions in
54
-
let versions = if OpamPackage.Version.Map.is_empty non_avoided
55
-
then versions else non_avoided in
56
-
(* Return ALL versions from newest to oldest so the caller can try
57
-
multiple if the first fails *)
58
-
let candidates = OpamPackage.Version.Map.bindings versions
59
-
|> List.rev in
60
-
List.map (fun (v, _) -> OpamPackage.create n v) candidates
61
-
62
-
let resolve_targets ?(small = false) git_packages target =
63
-
match target with
64
-
| None when small ->
65
-
Printf.printf "Using small universe (%d packages)...\n%!"
66
-
(List.length small_universe);
67
-
(* For small universe, take just the latest version — if the solve
68
-
fails for any, we'll retry with older versions *)
69
-
List.filter_map (fun name ->
70
-
match pick_latest_version git_packages name with
71
-
| v :: _ -> Some v
72
-
| [] -> None
73
-
) small_universe
74
-
| None ->
75
-
Printf.printf "Finding all latest package versions...\n%!";
76
-
find_latest_versions git_packages
77
-
| Some t when String.length t > 0 && t.[0] = '@' ->
78
-
let filename = String.sub t 1 (String.length t - 1) in
79
-
Printf.printf "Loading package list from %s...\n%!" filename;
80
-
load_package_list filename
81
-
| Some t ->
82
-
Printf.printf "Target: %s\n%!" t;
83
-
[ OpamPackage.of_string t ]
84
6
85
7
86
8
let cleanup_stale_mounts () =
···
114
36
let driver_compiler = OpamPackage.of_string driver_compiler_str in
115
37
let git_packages, repos_with_shas, opam_env =
116
38
Common.setup_solver opam_repositories in
117
-
let targets = resolve_targets ~small:small_universe git_packages target in
39
+
let targets = Day11_batch.Targets.resolve ~small:small_universe git_packages target in
118
40
Printf.printf "Targets: %d packages\n%!" (List.length targets);
119
41
(match ocaml_version with
120
42
| Some v -> Printf.printf "Compiler: %s\n%!" (OpamPackage.to_string v)
···
178
100
| Ok _ -> ()
179
101
| Error _ ->
180
102
let name = OpamPackage.Name.to_string (OpamPackage.name target) in
181
-
let candidates = pick_latest_version git_packages name in
103
+
let candidates = Day11_batch.Targets.pick_latest_version git_packages name in
182
104
let older = List.filter (fun pkg ->
183
105
OpamPackage.Version.compare (OpamPackage.version pkg)
184
106
(OpamPackage.version target) < 0
···
436
358
let run_file = Fpath.(runs_dir / (timestamp ^ ".json")) in
437
359
ignore (Bos.OS.File.write run_file (Yojson.Safe.pretty_to_string run_json));
438
360
(* Docs *)
439
-
if with_doc then begin
440
-
Printf.printf "\nBuilding doc tools...\n%!";
441
-
let all_pins, all_source_dirs = match odoc_repo with
442
-
| Some dir ->
443
-
Printf.printf "Using local odoc from %s\n%!" dir;
444
-
let pins = Day11_build.Tools.read_pins_from_dir dir in
445
-
let source_dirs = OpamPackage.Name.Map.fold (fun name _ acc ->
446
-
OpamPackage.Name.Map.add name dir acc
447
-
) pins OpamPackage.Name.Map.empty in
448
-
(pins, source_dirs)
449
-
| None ->
450
-
(OpamPackage.Name.Map.empty, OpamPackage.Name.Map.empty)
451
-
in
452
-
(* 1. Build driver with fixed compiler.
453
-
Always use the released version from the base repo — the driver
454
-
just needs working executables, not bleeding-edge source.
455
-
No pins, no source dirs, no overlay repos. *)
456
-
let driver_pkg = OpamPackage.of_string "odoc-driver.3.1.0" in
457
-
Printf.printf "Building doc driver (%s)...\n%!"
458
-
(OpamPackage.to_string driver_compiler);
459
-
let driver_result = Day11_build.Tools.build_tool env benv ~np
460
-
~packages:git_packages ~env:opam_env ~doc:false
461
-
~mounts:[repo_mount] ~ocaml_version:driver_compiler driver_pkg in
462
-
match driver_result with
463
-
| Error (`Msg e) ->
464
-
Printf.printf "Doc driver build failed: %s\n%!" e
465
-
| Ok driver_tool ->
466
-
(* 2. Build odoc per unique compiler in batch *)
467
-
let compiler_versions =
468
-
let seen = Hashtbl.create 4 in
469
-
List.filter_map (fun (_target, solution) ->
470
-
match Day11_doc.Generate.find_compiler solution with
471
-
| Some c when not (Hashtbl.mem seen (OpamPackage.to_string c)) ->
472
-
Hashtbl.replace seen (OpamPackage.to_string c) ();
473
-
Some c
474
-
| _ -> None
475
-
) solutions
476
-
in
477
-
if compiler_versions = [] then
478
-
Printf.printf "No compiler versions found in solutions, skipping docs\n%!"
479
-
else begin
480
-
let odoc_pkg = match odoc_repo with
481
-
| Some _ -> OpamPackage.of_string "odoc.dev"
482
-
| None -> OpamPackage.of_string "odoc.3.1.0"
483
-
in
484
-
let odoc_tools = List.filter_map (fun compiler_v ->
485
-
Printf.printf "Building odoc for %s...\n%!"
486
-
(OpamPackage.to_string compiler_v);
487
-
match Day11_build.Tools.build_tool env benv ~np
488
-
~packages:git_packages ~env:opam_env ~pins:all_pins
489
-
~source_dirs:all_source_dirs ~doc:false
490
-
~mounts:[repo_mount] ~ocaml_version:compiler_v odoc_pkg with
491
-
| Error (`Msg e) ->
492
-
Printf.printf "Odoc build for %s failed: %s\n%!"
493
-
(OpamPackage.to_string compiler_v) e;
494
-
None
495
-
| Ok tool -> Some (compiler_v, tool)
496
-
) compiler_versions in
497
-
(* 3. Generate docs *)
498
-
Printf.printf "Generating docs...\n%!";
499
-
let doc_count, doc_html =
500
-
Day11_doc.Generate.run env benv ~os_dir ~driver_tool ~odoc_tools
501
-
~nodes ~solutions ~blessing_maps in
502
-
Printf.printf "\n=== Docs: %d packages, %d HTML files ===\n%!"
503
-
doc_count doc_html
504
-
end
505
-
end;
361
+
if with_doc then
362
+
Day11_doc.Generate.build_tools_and_run env benv ~np ~os_dir
363
+
~packages:git_packages ~opam_env ~mounts:[repo_mount]
364
+
~driver_compiler ~odoc_repo
365
+
~nodes ~solutions ~blessing_maps;
506
366
(* JTW *)
507
367
(match jtw_repo with
508
368
| Some dir ->
509
-
Printf.printf "\nBuilding JTW tools from %s...\n%!" dir;
510
-
let compiler_versions =
511
-
let seen = Hashtbl.create 4 in
512
-
List.filter_map (fun (_target, solution) ->
513
-
match Day11_doc.Generate.find_compiler solution with
514
-
| Some c when not (Hashtbl.mem seen (OpamPackage.to_string c)) ->
515
-
Hashtbl.replace seen (OpamPackage.to_string c) ();
516
-
Some c
517
-
| _ -> None
518
-
) solutions
519
-
in
520
-
List.iter (fun compiler_v ->
521
-
Printf.printf "Building JTW for %s...\n%!"
522
-
(OpamPackage.to_string compiler_v);
523
-
match Day11_build.Tools.build_tool_from_repo env benv ~np
524
-
~packages:git_packages ~env:opam_env ~ocaml_version:compiler_v
525
-
~mounts:[repo_mount] ~repo_dir:dir
526
-
~target_name:Day11_jtw.Tool_layer.tool_target () with
527
-
| Ok _tool ->
528
-
Printf.printf "JTW tools for %s: OK\n%!"
529
-
(OpamPackage.to_string compiler_v)
530
-
| Error (`Msg e) ->
531
-
Printf.printf "JTW tools for %s failed: %s\n%!"
532
-
(OpamPackage.to_string compiler_v) e
533
-
) compiler_versions
369
+
Day11_jtw.Build_tools.build_per_compiler env benv ~np
370
+
~packages:git_packages ~opam_env ~mounts:[repo_mount]
371
+
~repo_dir:dir ~solutions
534
372
| None -> ());
535
373
0
536
374
end
+67
day11/doc/generate.ml
+67
day11/doc/generate.ml
···
212
212
| None -> ()
213
213
) nodes;
214
214
(!doc_count, !doc_html)
215
+
216
+
let unique_compilers solutions =
217
+
let seen = Hashtbl.create 4 in
218
+
List.filter_map (fun (_target, solution) ->
219
+
match find_compiler solution with
220
+
| Some c when not (Hashtbl.mem seen (OpamPackage.to_string c)) ->
221
+
Hashtbl.replace seen (OpamPackage.to_string c) ();
222
+
Some c
223
+
| _ -> None
224
+
) solutions
225
+
226
+
let build_tools_and_run env benv ~np ~os_dir ~packages ~opam_env
227
+
~mounts ~driver_compiler ~odoc_repo
228
+
~nodes ~solutions ~blessing_maps =
229
+
Printf.printf "\nBuilding doc tools...\n%!";
230
+
let all_pins, all_source_dirs = match odoc_repo with
231
+
| Some dir ->
232
+
Printf.printf "Using local odoc from %s\n%!" dir;
233
+
let pins = Day11_build.Tools.read_pins_from_dir dir in
234
+
let source_dirs = OpamPackage.Name.Map.fold (fun name _ acc ->
235
+
OpamPackage.Name.Map.add name dir acc
236
+
) pins OpamPackage.Name.Map.empty in
237
+
(pins, source_dirs)
238
+
| None ->
239
+
(OpamPackage.Name.Map.empty, OpamPackage.Name.Map.empty)
240
+
in
241
+
(* 1. Build driver with fixed compiler *)
242
+
let driver_pkg = OpamPackage.of_string "odoc-driver.3.1.0" in
243
+
Printf.printf "Building doc driver (%s)...\n%!"
244
+
(OpamPackage.to_string driver_compiler);
245
+
let driver_result = Day11_build.Tools.build_tool env benv ~np
246
+
~packages ~env:opam_env ~doc:false
247
+
~mounts ~ocaml_version:driver_compiler driver_pkg in
248
+
match driver_result with
249
+
| Error (`Msg e) ->
250
+
Printf.printf "Doc driver build failed: %s\n%!" e
251
+
| Ok driver_tool ->
252
+
(* 2. Build odoc per unique compiler in batch *)
253
+
let compiler_versions = unique_compilers solutions in
254
+
if compiler_versions = [] then
255
+
Printf.printf "No compiler versions found in solutions, skipping docs\n%!"
256
+
else begin
257
+
let odoc_pkg = match odoc_repo with
258
+
| Some _ -> OpamPackage.of_string "odoc.dev"
259
+
| None -> OpamPackage.of_string "odoc.3.1.0"
260
+
in
261
+
let odoc_tools = List.filter_map (fun compiler_v ->
262
+
Printf.printf "Building odoc for %s...\n%!"
263
+
(OpamPackage.to_string compiler_v);
264
+
match Day11_build.Tools.build_tool env benv ~np
265
+
~packages ~env:opam_env ~pins:all_pins
266
+
~source_dirs:all_source_dirs ~doc:false
267
+
~mounts ~ocaml_version:compiler_v odoc_pkg with
268
+
| Error (`Msg e) ->
269
+
Printf.printf "Odoc build for %s failed: %s\n%!"
270
+
(OpamPackage.to_string compiler_v) e;
271
+
None
272
+
| Ok tool -> Some (compiler_v, tool)
273
+
) compiler_versions in
274
+
(* 3. Generate docs *)
275
+
Printf.printf "Generating docs...\n%!";
276
+
let doc_count, doc_html =
277
+
run env benv ~os_dir ~driver_tool ~odoc_tools
278
+
~nodes ~solutions ~blessing_maps in
279
+
Printf.printf "\n=== Docs: %d packages, %d HTML files ===\n%!"
280
+
doc_count doc_html
281
+
end
+17
-10
day11/doc/generate.mli
+17
-10
day11/doc/generate.mli
···
22
22
([ocaml-base-compiler], [ocaml-variants], or [ocaml-system])
23
23
from [solution], or [None] if none is found. *)
24
24
25
-
val run :
25
+
val unique_compilers :
26
+
(OpamPackage.t * Day11_graph.Graph.solution) list ->
27
+
OpamPackage.t list
28
+
(** Extract unique concrete compiler packages from solutions. *)
29
+
30
+
val build_tools_and_run :
26
31
Eio_unix.Stdenv.base ->
27
32
Day11_build.Types.build_env ->
33
+
np:int ->
28
34
os_dir:Fpath.t ->
29
-
driver_tool:Day11_layer.Layer_type.tool ->
30
-
odoc_tools:(OpamPackage.t * Day11_layer.Layer_type.tool) list ->
35
+
packages:Day11_solver.Git_packages.t ->
36
+
opam_env:(string -> OpamVariable.variable_contents option) ->
37
+
mounts:Day11_container.Mount.t list ->
38
+
driver_compiler:OpamPackage.t ->
39
+
odoc_repo:string option ->
31
40
nodes:Day11_layer.Layer_type.build list ->
32
41
solutions:(OpamPackage.t * Day11_graph.Graph.solution) list ->
33
42
blessing_maps:(OpamPackage.t * bool OpamPackage.Map.t) list ->
34
-
int * int
35
-
(** [run env benv ~os_dir ~driver_tool ~odoc_tools ~nodes ~solutions
36
-
~blessing_maps] generates documentation for all packages in
37
-
[nodes]. [driver_tool] provides odoc_driver_voodoo, odoc-md, and
38
-
sherlodoc binaries (built with a fixed compiler). [odoc_tools] is
39
-
a list of [(compiler, tool)] pairs providing per-compiler odoc
40
-
binaries. Returns [(doc_count, html_count)]. *)
43
+
unit
44
+
(** Build doc tools (driver + per-compiler odoc) and generate
45
+
documentation. The driver is built with [driver_compiler]; odoc is
46
+
built once per unique compiler in [solutions]. When [odoc_repo] is
47
+
given, odoc packages are pinned from that local checkout. *)
+19
day11/jtw/build_tools.ml
+19
day11/jtw/build_tools.ml
···
1
+
let build_per_compiler env benv ~np ~packages ~opam_env ~mounts
2
+
~repo_dir ~solutions =
3
+
Printf.printf "\nBuilding JTW tools from %s...\n%!" repo_dir;
4
+
let compiler_versions =
5
+
Day11_doc.Generate.unique_compilers solutions in
6
+
List.iter (fun compiler_v ->
7
+
Printf.printf "Building JTW for %s...\n%!"
8
+
(OpamPackage.to_string compiler_v);
9
+
match Day11_build.Tools.build_tool_from_repo env benv ~np
10
+
~packages ~env:opam_env ~ocaml_version:compiler_v
11
+
~mounts ~repo_dir
12
+
~target_name:Tool_layer.tool_target () with
13
+
| Ok _tool ->
14
+
Printf.printf "JTW tools for %s: OK\n%!"
15
+
(OpamPackage.to_string compiler_v)
16
+
| Error (`Msg e) ->
17
+
Printf.printf "JTW tools for %s failed: %s\n%!"
18
+
(OpamPackage.to_string compiler_v) e
19
+
) compiler_versions
+15
day11/jtw/build_tools.mli
+15
day11/jtw/build_tools.mli
···
1
+
(** Build JTW tools from a local source checkout.
2
+
3
+
Builds [js_top_worker-bin] for each unique compiler version in
4
+
the batch solutions, using {!Day11_build.Tools.build_tool_from_repo}. *)
5
+
6
+
val build_per_compiler :
7
+
Eio_unix.Stdenv.base ->
8
+
Day11_build.Types.build_env ->
9
+
np:int ->
10
+
packages:Day11_solver.Git_packages.t ->
11
+
opam_env:(string -> OpamVariable.variable_contents option) ->
12
+
mounts:Day11_container.Mount.t list ->
13
+
repo_dir:string ->
14
+
solutions:(OpamPackage.t * Day11_graph.Graph.solution) list ->
15
+
unit