forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
feat(day10): thread compiler_layers through container interface
Layer hashes for doc, jtw, and solution workers now depend on the
compiler stack, ensuring cache invalidation when the compiler layers
change. Adds ~compiler_layers parameter to doc_layer_hash, generate_docs,
jtw_layer_hash, generate_jtw, and build_solution_worker across all
platform implementations.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+226
-143
+41
-36
day10/bin/doc_tools.ml
+41
-36
day10/bin/doc_tools.ml
···
21
21
| None -> None
22
22
23
23
(** Compute hash for the shared driver layer.
24
-
Only depends on repo/branch since it's always built with a fixed OCaml version. *)
25
-
let driver_layer_hash ~(config : Config.t) =
24
+
Depends on repo/branch and compiler layers (when stacking on pre-built compiler). *)
25
+
let driver_layer_hash ~(config : Config.t) ~(compiler_layers : string list) =
26
26
let source_component = match Local_repo.find_for_packages ~local_repos:config.local_repos driver_packages with
27
27
| Some (path, _) -> Local_repo.repo_hash path
28
28
| None -> config.doc_tools_repo ^ "|" ^ config.doc_tools_branch
29
29
in
30
-
let components = [ "driver"; source_component ] in
30
+
let components = [ "driver"; source_component ] @ compiler_layers in
31
31
String.concat "|" components |> Digest.string |> Digest.to_hex
32
32
33
33
(** Directory name for the driver layer *)
34
-
let driver_layer_name ~(config : Config.t) =
35
-
"doc-driver-" ^ driver_layer_hash ~config
34
+
let driver_layer_name ~(config : Config.t) ~(compiler_layers : string list) =
35
+
"doc-driver-" ^ driver_layer_hash ~config ~compiler_layers
36
36
37
37
(** Full path to the driver layer *)
38
-
let driver_layer_path ~(config : Config.t) =
38
+
let driver_layer_path ~(config : Config.t) ~(compiler_layers : string list) =
39
39
let os_key = Config.os_key ~config in
40
-
Path.(config.dir / os_key / driver_layer_name ~config)
40
+
Path.(config.dir / os_key / driver_layer_name ~config ~compiler_layers)
41
41
42
42
(** Generate build script for the shared driver layer.
43
-
Builds odoc_driver_voodoo, sherlodoc, and odoc-md with OCaml 5.x. *)
44
-
let driver_build_script ~(config : Config.t) =
43
+
Builds odoc_driver_voodoo, sherlodoc, and odoc-md.
44
+
When [needs_compiler] is true, installs ocaml-base-compiler.5.2.1 first.
45
+
When false (compiler comes from lower layers), skips compiler installation. *)
46
+
let driver_build_script ~(config : Config.t) ~needs_compiler =
45
47
let repo = config.doc_tools_repo in
46
48
let branch = config.doc_tools_branch in
47
49
let pin_cmds = match Local_repo.find_for_packages ~local_repos:config.local_repos driver_packages with
···
60
62
Printf.sprintf "opam pin add -yn %s %s#%s" pkg repo branch
61
63
) driver_packages
62
64
in
65
+
let compiler_cmds = if needs_compiler then [ "opam install -y ocaml-base-compiler.5.2.1" ] else [] in
63
66
String.concat " && "
64
-
([ "opam install -y ocaml-base-compiler.5.2.1" ]
67
+
(compiler_cmds
65
68
@ pin_cmds
66
69
@ [ "opam install -y odoc-driver odoc-md sherlodoc";
67
70
"eval $(opam env) && sherlodoc js > /home/opam/sherlodoc.js";
68
71
"which odoc_driver_voodoo && which sherlodoc" ])
69
72
70
73
(** Check if driver layer exists *)
71
-
let driver_exists ~(config : Config.t) : bool =
72
-
Sys.file_exists (driver_layer_path ~config)
74
+
let driver_exists ~(config : Config.t) ~(compiler_layers : string list) : bool =
75
+
Sys.file_exists (driver_layer_path ~config ~compiler_layers)
73
76
74
77
(** Get the hash/name for the driver layer *)
75
-
let get_driver_hash ~(config : Config.t) : string =
76
-
driver_layer_name ~config
78
+
let get_driver_hash ~(config : Config.t) ~(compiler_layers : string list) : string =
79
+
driver_layer_name ~config ~compiler_layers
77
80
78
81
(** Check if odoc_driver_voodoo is available in the driver layer *)
79
-
let has_odoc_driver_voodoo ~(config : Config.t) : bool =
80
-
let voodoo_path = Path.(driver_layer_path ~config / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc_driver_voodoo") in
82
+
let has_odoc_driver_voodoo ~(config : Config.t) ~(compiler_layers : string list) : bool =
83
+
let voodoo_path = Path.(driver_layer_path ~config ~compiler_layers / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc_driver_voodoo") in
81
84
Sys.file_exists voodoo_path
82
85
83
86
(** Path to sherlodoc.js within the driver layer *)
84
-
let sherlodoc_js_path ~(config : Config.t) =
85
-
Path.(driver_layer_path ~config / "fs" / "home" / "opam" / "sherlodoc.js")
87
+
let sherlodoc_js_path ~(config : Config.t) ~(compiler_layers : string list) =
88
+
Path.(driver_layer_path ~config ~compiler_layers / "fs" / "home" / "opam" / "sherlodoc.js")
86
89
87
90
(* --- Per-version odoc layer --- *)
88
91
89
92
(** Compute hash for the per-version odoc layer.
90
-
Depends on OCaml version and repo/branch for odoc 3.1. *)
91
-
let odoc_layer_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
92
-
let version = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in
93
+
Depends on OCaml version, repo/branch, and compiler layers. *)
94
+
let odoc_layer_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
95
+
let compiler_pkg = OpamPackage.to_string ocaml_version in
93
96
let source_component = match Local_repo.find_for_packages ~local_repos:config.local_repos odoc_packages with
94
97
| Some (path, _) -> Local_repo.repo_hash path
95
98
| None -> config.doc_tools_repo ^ "|" ^ config.doc_tools_branch
96
99
in
97
-
let components = [ "odoc"; version; source_component ] in
100
+
let components = [ "odoc"; compiler_pkg; source_component ] @ compiler_layers in
98
101
String.concat "|" components |> Digest.string |> Digest.to_hex
99
102
100
103
(** Directory name for the odoc layer *)
101
-
let odoc_layer_name ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
102
-
"doc-odoc-" ^ odoc_layer_hash ~config ~ocaml_version
104
+
let odoc_layer_name ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
105
+
"doc-odoc-" ^ odoc_layer_hash ~config ~ocaml_version ~compiler_layers
103
106
104
107
(** Full path to the odoc layer *)
105
-
let odoc_layer_path ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
108
+
let odoc_layer_path ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
106
109
let os_key = Config.os_key ~config in
107
-
Path.(config.dir / os_key / odoc_layer_name ~config ~ocaml_version)
110
+
Path.(config.dir / os_key / odoc_layer_name ~config ~ocaml_version ~compiler_layers)
108
111
109
112
(** Generate build script for the per-version odoc layer.
110
-
Builds odoc with the specified OCaml version, pinned to 3.1 from repo/branch. *)
111
-
let odoc_build_script ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
113
+
Builds odoc with the specified OCaml version, pinned to 3.1 from repo/branch.
114
+
When [needs_compiler] is false, skips compiler installation (comes from lower layers). *)
115
+
let odoc_build_script ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~needs_compiler =
112
116
let repo = config.doc_tools_repo in
113
117
let branch = config.doc_tools_branch in
114
-
let version = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in
118
+
let compiler_pkg = OpamPackage.to_string ocaml_version in
115
119
let pin_cmds = match Local_repo.find_for_packages ~local_repos:config.local_repos odoc_packages with
116
120
| Some (_, matched) ->
117
121
let local_mount = "/home/opam/local/odoc" in
···
128
132
Printf.sprintf "opam pin add -yn %s %s#%s" pkg repo branch
129
133
) odoc_packages
130
134
in
135
+
let compiler_cmds = if needs_compiler then [ Printf.sprintf "opam install -y %s" compiler_pkg ] else [] in
131
136
String.concat " && "
132
-
([ Printf.sprintf "opam install -y ocaml-base-compiler.%s" version ]
137
+
(compiler_cmds
133
138
@ pin_cmds
134
139
@ [ "opam install -y odoc";
135
140
"eval $(opam env) && which odoc && odoc --version" ])
136
141
137
142
(** Check if odoc layer exists for this OCaml version *)
138
-
let odoc_exists ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =
139
-
Sys.file_exists (odoc_layer_path ~config ~ocaml_version)
143
+
let odoc_exists ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : bool =
144
+
Sys.file_exists (odoc_layer_path ~config ~ocaml_version ~compiler_layers)
140
145
141
146
(** Get the hash/name for the odoc layer *)
142
-
let get_odoc_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : string =
143
-
odoc_layer_name ~config ~ocaml_version
147
+
let get_odoc_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : string =
148
+
odoc_layer_name ~config ~ocaml_version ~compiler_layers
144
149
145
150
(** Check if odoc is available in the odoc layer *)
146
-
let has_odoc ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =
147
-
let odoc_path = Path.(odoc_layer_path ~config ~ocaml_version / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc") in
151
+
let has_odoc ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : bool =
152
+
let odoc_path = Path.(odoc_layer_path ~config ~ocaml_version ~compiler_layers / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc") in
148
153
Sys.file_exists odoc_path
+5
-5
day10/bin/dummy.ml
+5
-5
day10/bin/dummy.ml
···
29
29
let _rootfs = Path.(temp_dir / "fs") in
30
30
0
31
31
32
-
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ = ""
32
+
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ ~compiler_layers:_ = ""
33
33
34
34
(* Documentation generation not supported in dummy container *)
35
-
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ = None
35
+
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ ~compiler_layers:_ = None
36
36
37
-
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ = ""
37
+
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ ~compiler_layers:_ = ""
38
38
39
39
(* JTW generation not supported in dummy container *)
40
-
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
40
+
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ ~compiler_layers:_ = None
41
41
42
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")
42
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ ~compiler_layers:_ = (1, "")
+5
-5
day10/bin/freebsd.ml
+5
-5
day10/bin/freebsd.ml
···
248
248
let _ = Os.sudo [ "sh"; "-c"; ("rm -f " ^ Path.(upperdir / "home" / "opam" / ".opam" / "repo" / "state-*.cache")) ] in
249
249
result
250
250
251
-
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ = ""
251
+
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ ~compiler_layers:_ = ""
252
252
253
253
(* Documentation generation not supported on FreeBSD *)
254
-
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ = None
254
+
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ ~compiler_layers:_ = None
255
255
256
-
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ = ""
256
+
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ ~compiler_layers:_ = ""
257
257
258
258
(* JTW generation not supported on FreeBSD *)
259
-
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
259
+
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ ~compiler_layers:_ = None
260
260
261
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")
261
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ ~compiler_layers:_ = (1, "")
+22
-20
day10/bin/jtw_tools.ml
+22
-20
day10/bin/jtw_tools.ml
···
9
9
"js_top_worker-web"; "js_top_worker_rpc_def" ]
10
10
11
11
(** Compute hash for the jtw-tools layer.
12
-
Depends on OCaml version, repo, and branch. *)
13
-
let layer_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
14
-
let version = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in
12
+
Depends on OCaml version, repo, branch, and compiler layers. *)
13
+
let layer_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
14
+
let compiler_pkg = OpamPackage.to_string ocaml_version in
15
15
let source_component = match Local_repo.find_for_packages ~local_repos:config.local_repos jtw_packages with
16
16
| Some (path, _) -> Local_repo.repo_hash path
17
17
| None -> config.jtw_tools_repo ^ "|" ^ config.jtw_tools_branch
18
18
in
19
-
let components = [ "jtw-tools"; version; source_component ] in
19
+
let components = [ "jtw-tools"; compiler_pkg; source_component ] @ compiler_layers in
20
20
String.concat "|" components |> Digest.string |> Digest.to_hex
21
21
22
22
(** Directory name for the jtw-tools layer *)
23
-
let layer_name ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
24
-
"jtw-tools-" ^ layer_hash ~config ~ocaml_version
23
+
let layer_name ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
24
+
"jtw-tools-" ^ layer_hash ~config ~ocaml_version ~compiler_layers
25
25
26
26
(** Full path to the jtw-tools layer *)
27
-
let layer_path ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
27
+
let layer_path ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) =
28
28
let os_key = Config.os_key ~config in
29
-
Path.(config.dir / os_key / layer_name ~config ~ocaml_version)
29
+
Path.(config.dir / os_key / layer_name ~config ~ocaml_version ~compiler_layers)
30
30
31
31
(** Generate build script for the jtw-tools layer.
32
32
Pins js_top_worker packages from the configured repo/branch,
33
33
installs js_of_ocaml and js_top_worker-bin, then builds worker.js
34
-
and extracts stdlib cmis. *)
35
-
let build_script ~(config : Config.t) ~(ocaml_version : OpamPackage.t) =
36
-
let version = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in
34
+
and extracts stdlib cmis.
35
+
When [needs_compiler] is false, skips compiler installation (comes from lower layers). *)
36
+
let build_script ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~needs_compiler =
37
+
let compiler_pkg = OpamPackage.to_string ocaml_version in
37
38
let pin_cmds = match Local_repo.find_for_packages ~local_repos:config.local_repos jtw_packages with
38
39
| Some (_, matched) ->
39
40
let local_mount = "/home/opam/local/js_top_worker" in
···
50
51
Printf.sprintf "opam pin add -yn %s git+%s#%s" pkg config.jtw_tools_repo config.jtw_tools_branch
51
52
) jtw_packages
52
53
in
54
+
let compiler_cmds = if needs_compiler then [ Printf.sprintf "opam install -y %s" compiler_pkg ] else [] in
53
55
String.concat " && "
54
-
([ Printf.sprintf "opam install -y ocaml-base-compiler.%s" version ]
56
+
(compiler_cmds
55
57
@ pin_cmds
56
58
@ [ "opam install -y js_of_ocaml js_top_worker-bin js_top_worker-web";
57
59
"eval $(opam env) && which js_of_ocaml && which jtw";
58
60
"eval $(opam env) && jtw opam --no-worker -o /home/opam/jtw-tools-output stdlib" ])
59
61
60
62
(** Check if jtw-tools layer exists for this OCaml version *)
61
-
let exists ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =
62
-
Sys.file_exists (layer_path ~config ~ocaml_version)
63
+
let exists ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : bool =
64
+
Sys.file_exists (layer_path ~config ~ocaml_version ~compiler_layers)
63
65
64
66
(** Get the hash/name for the jtw-tools layer *)
65
-
let get_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : string =
66
-
layer_name ~config ~ocaml_version
67
+
let get_hash ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : string =
68
+
layer_name ~config ~ocaml_version ~compiler_layers
67
69
68
70
(** Check if js_of_ocaml is available in the jtw-tools layer *)
69
-
let has_jsoo ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =
70
-
let jsoo_path = Path.(layer_path ~config ~ocaml_version / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "js_of_ocaml") in
71
+
let has_jsoo ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : bool =
72
+
let jsoo_path = Path.(layer_path ~config ~ocaml_version ~compiler_layers / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "js_of_ocaml") in
71
73
Sys.file_exists jsoo_path
72
74
73
75
(** Check if worker.js was built in the jtw-tools layer *)
74
-
let has_worker_js ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =
75
-
let worker_path = Path.(layer_path ~config ~ocaml_version / "fs" / "home" / "opam" / "jtw-tools-output" / "worker.js") in
76
+
let has_worker_js ~(config : Config.t) ~(ocaml_version : OpamPackage.t) ~(compiler_layers : string list) : bool =
77
+
let worker_path = Path.(layer_path ~config ~ocaml_version ~compiler_layers / "fs" / "home" / "opam" / "jtw-tools-output" / "worker.js") in
76
78
Sys.file_exists worker_path
77
79
78
80
(** Return the local repo path to bind-mount for jtw, if any. *)
+110
-58
day10/bin/linux.ml
+110
-58
day10/bin/linux.ml
···
161
161
in
162
162
String.concat " " hashes |> Digest.string |> Digest.to_hex
163
163
164
-
let doc_layer_hash ~t ~build_hash ~dep_doc_hashes ~ocaml_version ~blessed =
164
+
let doc_layer_hash ~t ~build_hash ~dep_doc_hashes ~ocaml_version ~blessed ~compiler_layers =
165
165
let config = t.config in
166
-
let driver_hash = Doc_tools.get_driver_hash ~config in
167
-
let odoc_hash = Doc_tools.get_odoc_hash ~config ~ocaml_version in
166
+
let driver_hash = Doc_tools.get_driver_hash ~config ~compiler_layers in
167
+
let odoc_hash = Doc_tools.get_odoc_hash ~config ~ocaml_version ~compiler_layers in
168
168
let blessed_str = if blessed then "blessed" else "universe" in
169
169
let components = build_hash :: dep_doc_hashes @ [ driver_hash; odoc_hash; blessed_str ] in
170
170
String.concat " " components |> Digest.string |> Digest.to_hex
···
267
267
let _ = Os.sudo [ "sh"; "-c"; ("rm -f " ^ Path.(upperdir / "home" / "opam" / ".opam" / "repo" / "state-*.cache")) ] in
268
268
result
269
269
270
+
(** Create a merged opam repository from multiple repos.
271
+
When there's only one repo, symlinks directly to it.
272
+
When there are multiple repos, creates a merged directory with
273
+
actual copies so it works inside runc containers where symlinks
274
+
to host paths would be broken by mount namespace isolation. *)
275
+
let setup_merged_opam_repo ~repos ~target =
276
+
match repos with
277
+
| [single_repo] ->
278
+
Unix.symlink single_repo target
279
+
| first :: rest ->
280
+
(* Start with a copy of the first repo *)
281
+
ignore (Sys.command (Printf.sprintf "cp -a %s %s"
282
+
(Filename.quote first) (Filename.quote target)));
283
+
(* Merge in packages from remaining repos (no clobber) *)
284
+
List.iter (fun repo ->
285
+
ignore (Sys.command (Printf.sprintf "cp -an %s/packages/* %s/packages/ 2>/dev/null; true"
286
+
(Filename.quote repo) (Filename.quote target)))
287
+
) rest
288
+
| [] -> failwith "No opam repositories configured"
289
+
270
290
(** Build a doc tools layer using runc.
271
-
This is built directly on the base layer without any compiler layers.
272
-
Takes a build_script parameter to support both driver and odoc layers.
291
+
When [lower_layers] is empty, builds directly on the base layer.
292
+
When non-empty, stacks the given layers (compiler build layers) into a
293
+
lowerdir so tool packages can reuse the pre-built compiler.
294
+
Takes a build_script parameter to support driver, odoc, and jtw layers.
273
295
Returns the exit status of the build. *)
274
-
let build_doc_tools_layer ~t ~temp_dir ~build_script ?(extra_mounts=[]) build_log =
296
+
let build_doc_tools_layer ~t ~temp_dir ~build_script ?(extra_mounts=[]) ?(lower_layers=[]) build_log =
275
297
let config = t.config in
276
298
let os_key = Config.os_key ~config in
299
+
let lowerdir = Path.(temp_dir / "lower") in
277
300
let upperdir = Path.(temp_dir / "fs") in
278
301
let workdir = Path.(temp_dir / "work") in
279
302
let rootfsdir = Path.(temp_dir / "rootfs") in
280
303
let () = List.iter Os.mkdir [ upperdir; workdir; rootfsdir ] in
304
+
(* When lower_layers are provided, cp --link their fs/ dirs into lowerdir
305
+
and regenerate opam switch-state, same technique as the build function *)
306
+
let () =
307
+
if lower_layers <> [] then begin
308
+
Os.mkdir lowerdir;
309
+
List.iter (fun hash ->
310
+
let src = Path.(config.dir / os_key / hash / "fs") in
311
+
ignore (Os.sudo ~stderr:"/dev/null"
312
+
[ "cp"; "-n"; "--archive"; "--no-dereference"; "--recursive";
313
+
"--link"; "--no-target-directory"; src; lowerdir ])
314
+
) lower_layers;
315
+
let packages_dir = Path.(lowerdir / "home" / "opam" / ".opam" / "default" / ".opam-switch" / "packages") in
316
+
let state_file = Path.(upperdir / "home" / "opam" / ".opam" / "default" / ".opam-switch" / "switch-state") in
317
+
if Sys.file_exists packages_dir then
318
+
Opamh.dump_state packages_dir state_file;
319
+
(* Chown /home so overlay permissions are correct *)
320
+
let home_dir = Path.(upperdir / "home") in
321
+
if Sys.file_exists home_dir then
322
+
ignore (Os.sudo [ "chown"; "-R"; string_of_int t.uid ^ ":" ^ string_of_int t.gid; home_dir ])
323
+
end
324
+
in
281
325
let argv =
282
326
[ "/usr/bin/env"; "bash"; "-c"; build_script ]
283
327
in
284
328
let etc_hosts = Path.(temp_dir / "hosts") in
285
329
let () = Os.write_to_file etc_hosts ("127.0.0.1 localhost " ^ hostname) in
286
-
(* Build directly on base layer - no compiler layers needed *)
287
-
let ld = "lowerdir=" ^ Path.(config.dir / os_key / "base" / "fs") in
330
+
(* Build on base + optional lower layers *)
331
+
let ld =
332
+
if lower_layers <> [] then
333
+
"lowerdir=" ^ String.concat ":" [ lowerdir; Path.(config.dir / os_key / "base" / "fs") ]
334
+
else
335
+
"lowerdir=" ^ Path.(config.dir / os_key / "base" / "fs")
336
+
in
288
337
let ud = "upperdir=" ^ upperdir in
289
338
let wd = "workdir=" ^ workdir in
290
339
let _ =
···
319
368
let _ = Os.sudo ~stderr:"/dev/null" [ "umount"; rootfsdir ] in
320
369
let _ =
321
370
Os.sudo
322
-
[
371
+
([
323
372
"rm";
324
373
"-rf";
325
374
workdir;
···
328
377
Path.(upperdir / "home" / "opam" / ".opam" / "default" / ".opam-switch" / "sources");
329
378
Path.(upperdir / "home" / "opam" / ".opam" / "default" / ".opam-switch" / "build");
330
379
Path.(upperdir / "home" / "opam" / ".opam" / "default" / ".opam-switch" / "packages" / "cache");
331
-
]
380
+
] @ if lower_layers <> [] then [ lowerdir ] else [])
332
381
in
333
382
let _ =
334
383
Os.sudo
···
337
386
result
338
387
339
388
(** Ensure the shared driver layer exists and is built.
340
-
Contains odoc_driver_voodoo, sherlodoc, odoc-md - built once with OCaml 5.x.
389
+
Contains odoc_driver_voodoo, sherlodoc, odoc-md.
390
+
When [compiler_layers] is non-empty, stacks on the pre-built compiler
391
+
instead of installing ocaml-base-compiler from scratch.
341
392
Returns the layer directory path if successful, None if build failed. *)
342
-
let ensure_driver_layer ~t : string option =
393
+
let ensure_driver_layer ~t ~(compiler_layers : string list) : string option =
343
394
let config = t.config in
344
-
let layer_dir = Doc_tools.driver_layer_path ~config in
345
-
let driver_layer_name = Doc_tools.driver_layer_name ~config in
395
+
let layer_dir = Doc_tools.driver_layer_path ~config ~compiler_layers in
396
+
let driver_layer_name = Doc_tools.driver_layer_name ~config ~compiler_layers in
346
397
let layer_json = Path.(layer_dir / "layer.json") in
347
398
let write_layer ~set_temp_log_path target_dir =
348
399
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-doc-driver-" "" in
349
400
let build_log = Path.(temp_dir / "build.log") in
350
401
set_temp_log_path build_log;
351
-
let opam_repo_src = List.hd config.opam_repositories in
352
402
let opam_repo = Path.(temp_dir / "opam-repository") in
353
-
Unix.symlink opam_repo_src opam_repo;
354
-
let build_script = Doc_tools.driver_build_script ~config in
403
+
setup_merged_opam_repo ~repos:config.opam_repositories ~target:opam_repo;
404
+
let needs_compiler = compiler_layers = [] in
405
+
let build_script = Doc_tools.driver_build_script ~config ~needs_compiler in
355
406
let extra_mounts = match Doc_tools.local_repo_mount ~config with
356
407
| Some (src, dst) ->
357
408
[ { Mount.ty = "bind"; src; dst; options = [ "ro"; "rbind"; "rprivate" ] } ]
358
409
| None -> []
359
410
in
360
-
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts build_log in
411
+
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts ~lower_layers:compiler_layers build_log in
361
412
let () = Os.safe_rename_dir ~marker_file:layer_json temp_dir target_dir in
362
413
let dummy_pkg = OpamPackage.of_string "doc-driver.0" in
363
414
Util.save_layer_info layer_json dummy_pkg [] [] r
···
371
422
372
423
(** Ensure the per-version odoc layer exists and is built.
373
424
Contains odoc built with the specified OCaml version.
425
+
When [compiler_layers] is non-empty, stacks on the pre-built compiler.
374
426
Returns the layer directory path if successful, None if build failed. *)
375
-
let ensure_odoc_layer ~t ~ocaml_version : string option =
427
+
let ensure_odoc_layer ~t ~ocaml_version ~(compiler_layers : string list) : string option =
376
428
let config = t.config in
377
-
let layer_dir = Doc_tools.odoc_layer_path ~config ~ocaml_version in
378
-
let odoc_layer_name = Doc_tools.odoc_layer_name ~config ~ocaml_version in
429
+
let layer_dir = Doc_tools.odoc_layer_path ~config ~ocaml_version ~compiler_layers in
430
+
let odoc_layer_name = Doc_tools.odoc_layer_name ~config ~ocaml_version ~compiler_layers in
379
431
let layer_json = Path.(layer_dir / "layer.json") in
380
432
let write_layer ~set_temp_log_path target_dir =
381
433
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-doc-odoc-" "" in
382
434
let build_log = Path.(temp_dir / "build.log") in
383
435
set_temp_log_path build_log;
384
-
let opam_repo_src = List.hd config.opam_repositories in
385
436
let opam_repo = Path.(temp_dir / "opam-repository") in
386
-
Unix.symlink opam_repo_src opam_repo;
387
-
let build_script = Doc_tools.odoc_build_script ~config ~ocaml_version in
437
+
setup_merged_opam_repo ~repos:config.opam_repositories ~target:opam_repo;
438
+
let needs_compiler = compiler_layers = [] in
439
+
let build_script = Doc_tools.odoc_build_script ~config ~ocaml_version ~needs_compiler in
388
440
let extra_mounts = match Doc_tools.local_repo_mount ~config with
389
441
| Some (src, dst) ->
390
442
[ { Mount.ty = "bind"; src; dst; options = [ "ro"; "rbind"; "rprivate" ] } ]
391
443
| None -> []
392
444
in
393
-
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts build_log in
445
+
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts ~lower_layers:compiler_layers build_log in
394
446
let () = Os.safe_rename_dir ~marker_file:layer_json temp_dir target_dir in
395
447
let dummy_pkg = OpamPackage.of_string "doc-odoc.0" in
396
448
Util.save_layer_info layer_json dummy_pkg [] [] r
···
415
467
Since the fs is an overlay of all dep layers, odoc_driver can find
416
468
dependencies' .odoc files. New .odoc files are captured by the overlay's
417
469
upperdir and end up in layer/fs/home/opam/compile/. *)
418
-
let run_odoc_driver_voodoo ~t ~temp_dir ~build_log ~build_layer_dir ~doc_layer_dir ~pkg ~universe ~blessed ~dep_doc_hashes ~actions ~html_output ~ocaml_version =
470
+
let run_odoc_driver_voodoo ~t ~temp_dir ~build_log ~build_layer_dir ~doc_layer_dir ~pkg ~universe ~blessed ~dep_doc_hashes ~actions ~html_output ~ocaml_version ~compiler_layers =
419
471
let config = t.config in
420
472
let os_key = Config.os_key ~config in
421
473
let lowerdir = Path.(temp_dir / "lower") in
···
439
491
let doc_tools_bin_host = Path.(lowerdir / "home" / "opam" / "doc-tools" / "bin") in
440
492
Os.mkdir ~parents:true doc_tools_bin_host;
441
493
(* Paths to doc tool binaries in their respective layers *)
442
-
let odoc_layer = Doc_tools.odoc_layer_path ~config ~ocaml_version in
443
-
let driver_layer = Doc_tools.driver_layer_path ~config in
494
+
let odoc_layer = Doc_tools.odoc_layer_path ~config ~ocaml_version ~compiler_layers in
495
+
let driver_layer = Doc_tools.driver_layer_path ~config ~compiler_layers in
444
496
let odoc_src = Path.(odoc_layer / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc") in
445
497
let odoc_md_src = Path.(driver_layer / "fs" / "home" / "opam" / ".opam" / "default" / "bin" / "odoc-md") in
446
498
(* Container paths for the copied binaries *)
···
505
557
end
506
558
in
507
559
(* Copy dependency doc layers' fs/ (these contain compile/ output with .odoc files) *)
508
-
let doc_tool_hashes = [ Doc_tools.get_odoc_hash ~config ~ocaml_version; Doc_tools.get_driver_hash ~config ] in
560
+
let doc_tool_hashes = [ Doc_tools.get_odoc_hash ~config ~ocaml_version ~compiler_layers; Doc_tools.get_driver_hash ~config ~compiler_layers ] in
509
561
let () =
510
562
List.iter
511
563
(fun hash ->
···
628
680
ignore (Os.sudo [ "chown"; "-R"; uid_gid; pkg_html_dir ]);
629
681
result
630
682
631
-
let jtw_layer_hash ~t ~build_hash ~ocaml_version =
683
+
let jtw_layer_hash ~t ~build_hash ~ocaml_version ~compiler_layers =
632
684
let config = t.config in
633
-
let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version in
685
+
let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version ~compiler_layers in
634
686
Jtw_gen.compute_jtw_layer_hash ~build_hash ~jtw_tools_hash
635
687
636
688
(** Ensure the jtw-tools layer exists and is built.
637
689
Contains js_of_ocaml and js_top_worker built with the specified OCaml version.
690
+
When [compiler_layers] is non-empty, stacks on the pre-built compiler.
638
691
Returns the layer directory path if successful, None if build failed. *)
639
-
let ensure_jtw_tools_layer ~t ~ocaml_version : string option =
692
+
let ensure_jtw_tools_layer ~t ~ocaml_version ~(compiler_layers : string list) : string option =
640
693
let config = t.config in
641
-
let layer_dir = Jtw_tools.layer_path ~config ~ocaml_version in
642
-
let jtw_tools_layer_name = Jtw_tools.layer_name ~config ~ocaml_version in
694
+
let layer_dir = Jtw_tools.layer_path ~config ~ocaml_version ~compiler_layers in
695
+
let jtw_tools_layer_name = Jtw_tools.layer_name ~config ~ocaml_version ~compiler_layers in
643
696
let layer_json = Path.(layer_dir / "layer.json") in
644
697
let write_layer ~set_temp_log_path target_dir =
645
698
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-tools-" "" in
646
699
let build_log = Path.(temp_dir / "build.log") in
647
700
set_temp_log_path build_log;
648
-
let opam_repo_src = List.hd config.opam_repositories in
649
701
let opam_repo = Path.(temp_dir / "opam-repository") in
650
-
Unix.symlink opam_repo_src opam_repo;
651
-
let build_script = Jtw_tools.build_script ~config ~ocaml_version in
702
+
setup_merged_opam_repo ~repos:config.opam_repositories ~target:opam_repo;
703
+
let needs_compiler = compiler_layers = [] in
704
+
let build_script = Jtw_tools.build_script ~config ~ocaml_version ~needs_compiler in
652
705
let extra_mounts = match Jtw_tools.local_repo_mount ~config with
653
706
| Some (src, dst) ->
654
707
[ { Mount.ty = "bind"; src; dst; options = [ "ro"; "rbind"; "rprivate" ] } ]
655
708
| None -> []
656
709
in
657
-
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts build_log in
710
+
let r = build_doc_tools_layer ~t ~temp_dir ~build_script ~extra_mounts ~lower_layers:compiler_layers build_log in
658
711
let () = Os.safe_rename_dir ~marker_file:layer_json temp_dir target_dir in
659
712
let dummy_pkg = OpamPackage.of_string "jtw-tools.0" in
660
713
Util.save_layer_info layer_json dummy_pkg [] [] r
···
668
721
if exit_status = 0 then Some layer_dir else None
669
722
670
723
(** Run jtw generation in a container using jtw opam *)
671
-
let run_jtw_in_container ~t ~temp_dir ~build_log ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version =
724
+
let run_jtw_in_container ~t ~temp_dir ~build_log ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version ~compiler_layers =
672
725
let config = t.config in
673
726
let os_key = Config.os_key ~config in
674
727
let lowerdir = Path.(temp_dir / "lower") in
···
694
747
["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; layer_fs; lowerdir])
695
748
) dep_build_hashes;
696
749
(* Copy jtw-tools layer *)
697
-
let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version in
750
+
let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version ~compiler_layers in
698
751
let jtw_tools_fs = Path.(config.dir / os_key / jtw_tools_hash / "fs") in
699
752
if Sys.file_exists jtw_tools_fs then
700
753
ignore (Os.sudo ~stderr:"/dev/null"
···
748
801
result
749
802
end
750
803
751
-
let generate_jtw ~t ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version =
804
+
let generate_jtw ~t ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version ~compiler_layers =
752
805
let config = t.config in
753
806
if not config.with_jtw then None
754
807
else
755
-
match ensure_jtw_tools_layer ~t ~ocaml_version with
808
+
match ensure_jtw_tools_layer ~t ~ocaml_version ~compiler_layers with
756
809
| Some _tools_dir ->
757
-
if not (Jtw_tools.has_jsoo ~config ~ocaml_version) then
810
+
if not (Jtw_tools.has_jsoo ~config ~ocaml_version ~compiler_layers) then
758
811
Some (Jtw_gen.jtw_result_to_yojson (Jtw_gen.Jtw_failure "js_of_ocaml not installed in jtw-tools layer"))
759
812
else begin
760
813
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-" "" in
761
814
let build_log = Path.(temp_dir / "jtw.log") in
762
815
let status =
763
816
try
764
-
run_jtw_in_container ~t ~temp_dir ~build_log ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version
817
+
run_jtw_in_container ~t ~temp_dir ~build_log ~build_layer_dir ~jtw_layer_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version ~compiler_layers
765
818
with _ -> 1
766
819
in
767
820
let layer_log = Path.(jtw_layer_dir / "jtw.log") in
···
785
838
786
839
Returns the worker output directory path on success (containing worker.js
787
840
and lib/), or "" on failure. *)
788
-
let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages =
841
+
let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages ~compiler_layers =
789
842
let config = t.config in
790
843
let os_key = Config.os_key ~config in
791
844
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-worker-" "" in
···
813
866
let jtw_output_host = Path.(temp_dir / "jtw-worker-output") in
814
867
Os.mkdir ~parents:true jtw_output_host;
815
868
ignore (Os.sudo [ "chown"; uid_gid; jtw_output_host ]);
816
-
(* Create opam repository symlink for the container *)
817
-
let opam_repo_src = List.hd config.opam_repositories in
869
+
(* Create opam repository from all configured repos *)
818
870
let opam_repo = Path.(temp_dir / "opam-repository") in
819
-
Unix.symlink opam_repo_src opam_repo;
871
+
setup_merged_opam_repo ~repos:config.opam_repositories ~target:opam_repo;
820
872
let etc_hosts = Path.(temp_dir / "hosts") in
821
873
let () = Os.write_to_file etc_hosts ("127.0.0.1 localhost " ^ hostname) in
822
874
let ld = "lowerdir=" ^ String.concat ":" [ lowerdir; Path.(config.dir / os_key / "base" / "fs") ] in
···
852
904
let _ = Os.sudo ~stderr:"/dev/null" [ "umount"; rootfsdir ] in
853
905
(* Copy worker output to a persistent location before cleaning up temp.
854
906
Hash based on dep_build_hashes + jtw source identity. *)
855
-
let jtw_source = Jtw_tools.layer_hash ~config ~ocaml_version in
907
+
let jtw_source = Jtw_tools.layer_hash ~config ~ocaml_version ~compiler_layers in
856
908
let hash_input = String.concat " " dep_build_hashes ^ " " ^ jtw_source in
857
909
let worker_dir_name = "jtw-worker-" ^ (Digest.to_hex (Digest.string hash_input)) in
858
910
let worker_output_dir = Path.(config.dir / os_key / worker_dir_name) in
···
873
925
(result, worker_output_dir)
874
926
end
875
927
876
-
let build_solution_worker ~t ~dep_build_hashes ~ocaml_version ~solution_packages =
928
+
let build_solution_worker ~t ~dep_build_hashes ~ocaml_version ~solution_packages ~compiler_layers =
877
929
let config = t.config in
878
930
if not config.with_jtw then (1, "")
879
931
else
880
-
match ensure_jtw_tools_layer ~t ~ocaml_version with
932
+
match ensure_jtw_tools_layer ~t ~ocaml_version ~compiler_layers with
881
933
| Some _tools_dir ->
882
-
if not (Jtw_tools.has_jsoo ~config ~ocaml_version) then (1, "")
883
-
else run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages
934
+
if not (Jtw_tools.has_jsoo ~config ~ocaml_version ~compiler_layers) then (1, "")
935
+
else run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages ~compiler_layers
884
936
| None -> (1, "")
885
937
886
-
let generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version =
938
+
let generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version ~compiler_layers =
887
939
let config = t.config in
888
940
if not config.with_doc then None
889
941
else
890
942
(* Ensure both doc tool layers exist: shared driver layer + per-version odoc layer *)
891
-
match ensure_driver_layer ~t, ensure_odoc_layer ~t ~ocaml_version with
943
+
match ensure_driver_layer ~t ~compiler_layers, ensure_odoc_layer ~t ~ocaml_version ~compiler_layers with
892
944
| Some _driver_dir, Some _odoc_dir ->
893
945
(* Check if odoc_driver_voodoo is available in driver layer *)
894
-
if not (Doc_tools.has_odoc_driver_voodoo ~config) then
946
+
if not (Doc_tools.has_odoc_driver_voodoo ~config ~compiler_layers) then
895
947
Some (Odoc_gen.doc_result_to_yojson (Odoc_gen.Doc_failure "odoc_driver_voodoo not installed in driver layer"))
896
948
(* Check if odoc is available in odoc layer *)
897
-
else if not (Doc_tools.has_odoc ~config ~ocaml_version) then
949
+
else if not (Doc_tools.has_odoc ~config ~ocaml_version ~compiler_layers) then
898
950
Some (Odoc_gen.doc_result_to_yojson (Odoc_gen.Doc_failure "odoc not installed in odoc layer"))
899
951
else begin
900
952
(* Compute universe hash from dependency doc hashes.
···
957
1009
let status =
958
1010
try
959
1011
(* Write to staging directory instead of final *)
960
-
run_odoc_driver_voodoo ~t ~temp_dir:voodoo_temp_dir ~build_log:voodoo_log ~build_layer_dir ~doc_layer_dir ~pkg ~universe ~blessed ~dep_doc_hashes ~actions ~html_output:staging_html_dir ~ocaml_version
1012
+
run_odoc_driver_voodoo ~t ~temp_dir:voodoo_temp_dir ~build_log:voodoo_log ~build_layer_dir ~doc_layer_dir ~pkg ~universe ~blessed ~dep_doc_hashes ~actions ~html_output:staging_html_dir ~ocaml_version ~compiler_layers
961
1013
with
962
1014
| _ -> 1
963
1015
in
+26
-12
day10/bin/main.ml
+26
-12
day10/bin/main.ml
···
359
359
Reads installed files from the build layer, runs doc generation,
360
360
and saves doc layer info.
361
361
Returns [Some doc_layer_name] on success, [None] on failure. *)
362
-
let doc_layer t pkg build_layer_name dep_doc_hashes ~ocaml_version =
362
+
let doc_layer t pkg build_layer_name dep_doc_hashes ~ocaml_version ~compiler_layers =
363
363
match ocaml_version with
364
364
| None -> None (* No OCaml version means no docs (e.g., conf-* packages) *)
365
365
| Some ocaml_version ->
···
373
373
| Some map -> Blessing.is_blessed map pkg
374
374
| None -> false
375
375
in
376
-
let doc_hash = Container.doc_layer_hash ~t ~build_hash:build_layer_name ~dep_doc_hashes ~ocaml_version ~blessed in
376
+
let doc_hash = Container.doc_layer_hash ~t ~build_hash:build_layer_name ~dep_doc_hashes ~ocaml_version ~blessed ~compiler_layers in
377
377
let doc_layer_name = "doc-" ^ doc_hash in
378
378
let doc_layer_dir = Path.(config.dir / os_key / doc_layer_name) in
379
379
let build_layer_dir = Path.(config.dir / os_key / build_layer_name) in
···
397
397
else S.Doc_compile_only
398
398
in
399
399
let doc_result =
400
-
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir:target_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version
400
+
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir:target_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version ~compiler_layers
401
401
in
402
402
Util.save_doc_layer_info ?doc_result (Path.(target_dir / "layer.json")) pkg ~build_hash:build_layer_name ~dep_doc_hashes
403
403
in
···
449
449
(** Build a jtw layer for a package.
450
450
Compiles .cma to .cma.js, copies .cmi and META, generates dynamic_cmis.json.
451
451
Returns [Some jtw_layer_name] on success, [None] on failure. *)
452
-
let jtw_layer t pkg build_layer_name dep_build_hashes ~ocaml_version =
452
+
let jtw_layer t pkg build_layer_name dep_build_hashes ~ocaml_version ~compiler_layers =
453
453
match ocaml_version with
454
454
| None -> None (* No OCaml version means no jtw *)
455
455
| Some ocaml_version ->
···
459
459
Os.log "jtw_layer: starting %s (build=%s, ocaml=%s)" pkg_str build_layer_name (OpamPackage.to_string ocaml_version);
460
460
let config = Container.config ~t in
461
461
let os_key = Config.os_key ~config in
462
-
let jtw_hash = Container.jtw_layer_hash ~t ~build_hash:build_layer_name ~ocaml_version in
462
+
let jtw_hash = Container.jtw_layer_hash ~t ~build_hash:build_layer_name ~ocaml_version ~compiler_layers in
463
463
let jtw_layer_name = "jtw-" ^ jtw_hash in
464
464
let jtw_layer_dir = Path.(config.dir / os_key / jtw_layer_name) in
465
465
let build_layer_dir = Path.(config.dir / os_key / build_layer_name) in
···
470
470
let build_layer_json = Path.(build_layer_dir / "layer.json") in
471
471
let installed_libs = Util.load_layer_info_installed_libs build_layer_json in
472
472
let jtw_result =
473
-
Container.generate_jtw ~t ~build_layer_dir ~jtw_layer_dir:target_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version
473
+
Container.generate_jtw ~t ~build_layer_dir ~jtw_layer_dir:target_dir ~dep_build_hashes ~pkg ~installed_libs ~ocaml_version ~compiler_layers
474
474
in
475
475
Jtw_gen.save_jtw_layer_info ?jtw_result (Path.(target_dir / "layer.json")) pkg ~build_hash:build_layer_name
476
476
in
···
546
546
(* Track packages that need deferred doc linking (have post deps) *)
547
547
let deferred_doc_link = ref [] in
548
548
549
+
(* Track compiler build layers - captured after compiler package builds successfully.
550
+
Tool layers (doc-driver, doc-odoc, jtw-tools) stack on these to avoid
551
+
recompiling the OCaml compiler from scratch. *)
552
+
let compiler_layers = ref [] in
553
+
549
554
(* Three accumulators: results, build_map (pkg -> build_result), doc_map (pkg -> doc_layer_name) *)
550
555
let results, _, doc_map =
551
556
List.fold_left
···
565
570
let build_layer_name = "build-" ^ hash in
566
571
let do_build () =
567
572
let r = build_layer t pkg build_layer_name ordered_deps ordered_build_hashes in
573
+
(* After compiler package builds successfully, capture the layer stack.
574
+
This allows tool layers to reuse the pre-built compiler. *)
575
+
(match r with
576
+
| Success _ ->
577
+
(match ocaml_version with
578
+
| Some ov when OpamPackage.equal pkg ov ->
579
+
compiler_layers := ordered_build_hashes @ [ build_layer_name ]
580
+
| _ -> ())
581
+
| _ -> ());
568
582
(* If build succeeded and with_doc, create doc layer *)
569
583
let r, dm =
570
584
if config.with_doc then
571
585
match r with
572
586
| Success _ ->
573
587
let dep_doc_hashes = List.filter_map (fun p -> OpamPackage.Map.find_opt p dm) ordered_deps in
574
-
(match doc_layer t pkg build_layer_name dep_doc_hashes ~ocaml_version with
588
+
(match doc_layer t pkg build_layer_name dep_doc_hashes ~ocaml_version ~compiler_layers:!compiler_layers with
575
589
| Some doc_name ->
576
590
(* Track packages with extra link deps (post deps + x-extra-doc-deps) for deferred doc linking *)
577
591
let opamfile = Util.opam_file config.opam_repositories pkg in
···
591
605
if config.with_jtw then
592
606
match r with
593
607
| Success _ ->
594
-
ignore (jtw_layer t pkg build_layer_name ordered_build_hashes ~ocaml_version)
608
+
ignore (jtw_layer t pkg build_layer_name ordered_build_hashes ~ocaml_version ~compiler_layers:!compiler_layers)
595
609
| _ -> ()
596
610
in
597
611
(r, dm)
···
658
672
let installed_docs = Util.load_layer_info_installed_docs build_layer_json in
659
673
Option.iter (fun ocaml_version ->
660
674
let _doc_result =
661
-
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase:S.Doc_link_only ~ocaml_version
675
+
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase:S.Doc_link_only ~ocaml_version ~compiler_layers:!compiler_layers
662
676
in
663
677
()) ocaml_version
664
678
) !deferred_doc_link
···
1001
1015
Os.log "global_deferred_doc_link: Running link-only for %s with %d dep hashes"
1002
1016
(OpamPackage.to_string pkg) (List.length dep_doc_hashes);
1003
1017
let _doc_result =
1004
-
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase:S.Doc_link_only ~ocaml_version
1018
+
Container.generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase:S.Doc_link_only ~ocaml_version ~compiler_layers:[]
1005
1019
in
1006
1020
()
1007
1021
) packages_to_relink
···
1366
1380
Printf.printf " Building worker.js for %s (%d build layers)...\n%!"
1367
1381
(OpamPackage.to_string target) (List.length unique_hashes);
1368
1382
let status, worker_output_dir =
1369
-
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages
1383
+
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages ~compiler_layers:[]
1370
1384
in
1371
1385
if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then
1372
1386
Some (target, solution, ocaml_version, worker_output_dir)
···
1444
1458
Printf.printf " Building worker.js for %s (%d build layers)...\n%!"
1445
1459
(OpamPackage.to_string target) (List.length unique_hashes);
1446
1460
let status, worker_output_dir =
1447
-
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages
1461
+
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages ~compiler_layers:[]
1448
1462
in
1449
1463
if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then
1450
1464
Some (target, solution, ocaml_version, worker_output_dir)
+12
-2
day10/bin/s.ml
+12
-2
day10/bin/s.ml
···
16
16
17
17
(** Compute hash for a doc layer.
18
18
The doc hash depends on the build hash, dependency doc hashes,
19
-
driver layer hash, odoc layer hash, and blessing status. *)
19
+
driver layer hash, odoc layer hash, and blessing status.
20
+
[compiler_layers] are the build layer hashes for the compiler stack. *)
20
21
val doc_layer_hash :
21
22
t:t ->
22
23
build_hash:string ->
23
24
dep_doc_hashes:string list ->
24
25
ocaml_version:OpamPackage.t ->
25
26
blessed:bool ->
27
+
compiler_layers:string list ->
26
28
string
27
29
28
30
(** Documentation generation support.
···
35
37
[doc_layer_dir] is the doc layer (for compile output and prep structure).
36
38
[dep_doc_hashes] are the doc layer hashes of dependencies.
37
39
[ocaml_version] is the OCaml compiler package from the solution.
40
+
[compiler_layers] are the build layer hashes for the compiler stack.
38
41
Returns None if doc generation is not supported on this platform. *)
39
42
val generate_docs :
40
43
t:t ->
···
46
49
installed_docs:string list ->
47
50
phase:doc_phase ->
48
51
ocaml_version:OpamPackage.t ->
52
+
compiler_layers:string list ->
49
53
Yojson.Safe.t option
50
54
51
55
(** Compute hash for a jtw layer.
52
-
Depends on the build hash and jtw-tools layer hash. *)
56
+
Depends on the build hash and jtw-tools layer hash.
57
+
[compiler_layers] are the build layer hashes for the compiler stack. *)
53
58
val jtw_layer_hash :
54
59
t:t ->
55
60
build_hash:string ->
56
61
ocaml_version:OpamPackage.t ->
62
+
compiler_layers:string list ->
57
63
string
58
64
59
65
(** JTW generation: compile .cma to .cma.js, extract .cmi, META, generate dynamic_cmis.json.
···
61
67
[jtw_layer_dir] is the output jtw layer.
62
68
[dep_build_hashes] are the build layer hashes of dependencies.
63
69
[installed_libs] are files installed by this package.
70
+
[compiler_layers] are the build layer hashes for the compiler stack.
64
71
Returns Some json on success/failure, None if not supported. *)
65
72
val generate_jtw :
66
73
t:t ->
···
70
77
pkg:OpamPackage.t ->
71
78
installed_libs:string list ->
72
79
ocaml_version:OpamPackage.t ->
80
+
compiler_layers:string list ->
73
81
Yojson.Safe.t option
74
82
75
83
(** Build worker.js + stdlib for a solution using the solution's dependency overlay.
76
84
[solution_packages] is the list of package strings in the solution (e.g., ["yojson.2.2.2"])
77
85
used to constrain opam install to match the solution's dependency versions.
86
+
[compiler_layers] are the build layer hashes for the compiler stack.
78
87
Returns (exit_status, worker_output_dir). *)
79
88
val build_solution_worker :
80
89
t:t ->
81
90
dep_build_hashes:string list ->
82
91
ocaml_version:OpamPackage.t ->
83
92
solution_packages:string list ->
93
+
compiler_layers:string list ->
84
94
int * string
85
95
end
+5
-5
day10/bin/windows.ml
+5
-5
day10/bin/windows.ml
···
172
172
let () = List.iter (fun hash -> Os.clense_tree ~source:Path.(config.dir / os_key / hash / "fs") ~target) sources in
173
173
result
174
174
175
-
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ = ""
175
+
let doc_layer_hash ~t:_ ~build_hash:_ ~dep_doc_hashes:_ ~ocaml_version:_ ~blessed:_ ~compiler_layers:_ = ""
176
176
177
177
(* Documentation generation not supported on Windows *)
178
-
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ = None
178
+
let generate_docs ~t:_ ~build_layer_dir:_ ~doc_layer_dir:_ ~dep_doc_hashes:_ ~pkg:_ ~installed_libs:_ ~installed_docs:_ ~phase:_ ~ocaml_version:_ ~compiler_layers:_ = None
179
179
180
-
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ = ""
180
+
let jtw_layer_hash ~t:_ ~build_hash:_ ~ocaml_version:_ ~compiler_layers:_ = ""
181
181
182
182
(* JTW generation not supported on Windows *)
183
-
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
183
+
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ ~compiler_layers:_ = None
184
184
185
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")
185
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ ~compiler_layers:_ = (1, "")