forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
day10: install jtw tools via opam inside worker container with version constraints
Instead of overlaying the pre-built jtw-tools layer (which was compiled
against potentially different dependency versions), install js_of_ocaml
and js_top_worker via opam inside the worker container. Pass
solution_packages version constraints to opam install so shared
dependencies (e.g., yojson) match the solution's exact versions,
eliminating CRC conflicts between universes.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+78
-30
+1
-1
day10/bin/dummy.ml
+1
-1
day10/bin/dummy.ml
···
39
39
(* JTW generation not supported in dummy container *)
40
40
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
41
41
42
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")
42
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")
+1
-1
day10/bin/freebsd.ml
+1
-1
day10/bin/freebsd.ml
···
258
258
(* JTW generation not supported on FreeBSD *)
259
259
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
260
260
261
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")
261
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")
+47
-10
day10/bin/jtw_gen.ml
+47
-10
day10/bin/jtw_gen.ml
···
133
133
|> List.sort_uniq String.compare
134
134
135
135
(** Build the shell script for building worker.js + stdlib inside a container.
136
-
Runs `jtw opam` WITHOUT --no-worker to produce worker.js linked against
137
-
the packages in the current opam switch. This ensures the worker is
138
-
compiled against the exact dependency versions in the solution's overlay. *)
139
-
let jtw_worker_container_script =
140
-
String.concat " && " [
141
-
"eval $(opam env)";
142
-
"echo 'JTW: Building worker.js + stdlib'";
143
-
"jtw opam -o /home/opam/jtw-worker-output stdlib";
144
-
"echo 'JTW: Worker build done'";
145
-
]
136
+
The container starts with the solution's build layers (which have the correct
137
+
dependency versions) and installs jtw tools on top, ensuring js_of_ocaml is
138
+
compiled against the exact same dependency versions.
139
+
[solution_packages] is the list of packages in the solution (e.g., ["yojson.2.2.2"]),
140
+
used to constrain opam install so it doesn't upgrade shared dependencies. *)
141
+
let jtw_worker_container_script ~(config : Config.t) ~solution_packages =
142
+
let pin_cmds = match Local_repo.find_for_packages ~local_repos:config.local_repos Jtw_tools.jtw_packages with
143
+
| Some (_, matched) ->
144
+
let local_mount = "/home/opam/local/js_top_worker" in
145
+
let from_local = List.map (fun pkg ->
146
+
Printf.sprintf "opam pin add -yn %s %s" pkg local_mount
147
+
) matched in
148
+
let from_git = List.filter_map (fun pkg ->
149
+
if List.mem pkg matched then None
150
+
else Some (Printf.sprintf "opam pin add -yn %s git+%s#%s" pkg config.jtw_tools_repo config.jtw_tools_branch)
151
+
) Jtw_tools.jtw_packages in
152
+
from_local @ from_git
153
+
| None ->
154
+
List.map (fun pkg ->
155
+
Printf.sprintf "opam pin add -yn %s git+%s#%s" pkg config.jtw_tools_repo config.jtw_tools_branch
156
+
) Jtw_tools.jtw_packages
157
+
in
158
+
(* Version constraints from the solution to prevent opam from upgrading
159
+
shared dependencies (e.g., yojson). Format: 'name=version' *)
160
+
let constraints = List.filter_map (fun pkg_str ->
161
+
let pkg = OpamPackage.of_string pkg_str in
162
+
let name = OpamPackage.name_to_string pkg in
163
+
let version = OpamPackage.version_to_string pkg in
164
+
(* Skip virtual/base packages and compiler packages *)
165
+
if version = "base" || String.length name >= 5 && String.sub name 0 5 = "ocaml"
166
+
|| name = "dune" || name = "base-unix" || name = "base-threads" then
167
+
None
168
+
else
169
+
Some (Printf.sprintf "'%s=%s'" name version)
170
+
) solution_packages in
171
+
let install_cmd = "opam install -y js_of_ocaml js_top_worker-bin js_top_worker-web"
172
+
^ (if constraints <> [] then " " ^ String.concat " " constraints else "") in
173
+
String.concat " && " (
174
+
[ "eval $(opam env)";
175
+
"echo 'JTW: Installing jtw tools...'" ]
176
+
@ pin_cmds
177
+
@ [ install_cmd;
178
+
"eval $(opam env)";
179
+
"echo 'JTW: Building worker.js + stdlib'";
180
+
"jtw opam -o /home/opam/jtw-worker-output stdlib";
181
+
"echo 'JTW: Worker build done'" ]
182
+
)
146
183
147
184
(** Build the shell script to run inside the container for jtw generation.
148
185
Calls `jtw opam` to handle all per-package artifact generation:
+21
-15
day10/bin/linux.ml
+21
-15
day10/bin/linux.ml
···
785
785
786
786
Returns the worker output directory path on success (containing worker.js
787
787
and lib/), or "" on failure. *)
788
-
let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version =
788
+
let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages =
789
789
let config = t.config in
790
790
let os_key = Config.os_key ~config in
791
791
let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-worker-" "" in
···
797
797
let () = List.iter Os.mkdir [ lowerdir; upperdir; workdir; rootfsdir ] in
798
798
let uid_gid = Printf.sprintf "%d:%d" t.uid t.gid in
799
799
let () = ignore (Os.sudo [ "chown"; uid_gid; upperdir; workdir ]) in
800
-
let script = Jtw_gen.jtw_worker_container_script in
800
+
let script = Jtw_gen.jtw_worker_container_script ~config ~solution_packages in
801
801
let argv = [ "/usr/bin/env"; "bash"; "-c"; script ] in
802
802
(* Build lower directory from all dependency build layers.
803
-
cp -n means first layer wins for conflicts. *)
803
+
cp -n means first layer wins for conflicts.
804
+
Note: jtw-tools are NOT overlaid here — they are installed via opam inside
805
+
the container to ensure they compile against the solution's dependency versions. *)
804
806
List.iter (fun hash ->
805
807
let layer_fs = Path.(config.dir / os_key / hash / "fs") in
806
808
if Sys.file_exists layer_fs then
807
809
ignore (Os.sudo ~stderr:"/dev/null"
808
810
["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; layer_fs; lowerdir])
809
811
) dep_build_hashes;
810
-
(* Copy jtw-tools layer (last, so build layers win for conflicts) *)
811
-
let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version in
812
-
let jtw_tools_fs = Path.(config.dir / os_key / jtw_tools_hash / "fs") in
813
-
if Sys.file_exists jtw_tools_fs then
814
-
ignore (Os.sudo ~stderr:"/dev/null"
815
-
["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; jtw_tools_fs; lowerdir]);
816
812
(* Create output directory in container *)
817
813
let jtw_output_host = Path.(temp_dir / "jtw-worker-output") in
818
814
Os.mkdir ~parents:true jtw_output_host;
819
815
ignore (Os.sudo [ "chown"; uid_gid; jtw_output_host ]);
816
+
(* Create opam repository symlink for the container *)
817
+
let opam_repo_src = List.hd config.opam_repositories in
818
+
let opam_repo = Path.(temp_dir / "opam-repository") in
819
+
Unix.symlink opam_repo_src opam_repo;
820
820
let etc_hosts = Path.(temp_dir / "hosts") in
821
821
let () = Os.write_to_file etc_hosts ("127.0.0.1 localhost " ^ hostname) in
822
822
let ld = "lowerdir=" ^ String.concat ":" [ lowerdir; Path.(config.dir / os_key / "base" / "fs") ] in
···
836
836
in
837
837
let mounts = [
838
838
{ Mount.ty = "bind"; src = jtw_output_host; dst = "/home/opam/jtw-worker-output"; options = [ "rw"; "rbind"; "rprivate" ] };
839
+
{ ty = "bind"; src = opam_repo; dst = "/home/opam/.opam/repo/default"; options = [ "rbind"; "rprivate" ] };
839
840
{ ty = "bind"; src = etc_hosts; dst = "/etc/hosts"; options = [ "ro"; "rbind"; "rprivate" ] };
840
841
] @ local_mounts in
841
842
let jtw_env = List.map (fun (k, v) ->
842
843
if k = "PATH" then (k, "/home/opam/.opam/default/bin:" ^ v) else (k, v)
843
844
) env in
844
-
let config_runc = make ~root:rootfsdir ~cwd:"/home/opam" ~argv ~hostname ~uid:t.uid ~gid:t.gid ~env:jtw_env ~mounts ~network:false in
845
+
(* Need network for opam install of jtw tools inside container *)
846
+
let config_runc = make ~root:rootfsdir ~cwd:"/home/opam" ~argv ~hostname ~uid:t.uid ~gid:t.gid ~env:jtw_env ~mounts ~network:true in
845
847
let () = Os.write_to_file Path.(temp_dir / "config.json") (Yojson.Safe.pretty_to_string config_runc) in
846
848
let container_id = "jtw-worker-" ^ Filename.basename temp_dir in
847
849
let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in
848
850
let result = Os.sudo ~stdout:build_log ~stderr:build_log [ "runc"; "run"; "-b"; temp_dir; container_id ] in
849
851
let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in
850
852
let _ = Os.sudo ~stderr:"/dev/null" [ "umount"; rootfsdir ] in
851
-
(* Copy worker output to a persistent location before cleaning up temp *)
852
-
let hash_input = String.concat " " dep_build_hashes ^ " " ^ jtw_tools_hash in
853
+
(* Copy worker output to a persistent location before cleaning up temp.
854
+
Hash based on dep_build_hashes + jtw source identity. *)
855
+
let jtw_source = Jtw_tools.layer_hash ~config ~ocaml_version in
856
+
let hash_input = String.concat " " dep_build_hashes ^ " " ^ jtw_source in
853
857
let worker_dir_name = "jtw-worker-" ^ (Digest.to_hex (Digest.string hash_input)) in
854
858
let worker_output_dir = Path.(config.dir / os_key / worker_dir_name) in
855
859
if result = 0 && Sys.file_exists jtw_output_host then begin
···
863
867
let _ = Os.sudo [ "rm"; "-rf"; lowerdir; workdir; rootfsdir; upperdir; jtw_output_host ] in
864
868
(* Copy build log to worker output dir for debugging *)
865
869
(try if Sys.file_exists worker_output_dir then Os.cp build_log Path.(worker_output_dir / "jtw-worker.log") with _ -> ());
866
-
(try Os.rm ~recursive:true temp_dir with _ -> ());
870
+
(* On failure, keep temp_dir for debugging *)
871
+
if result = 0 then
872
+
(try Os.rm ~recursive:true temp_dir with _ -> ());
867
873
(result, worker_output_dir)
868
874
end
869
875
870
-
let build_solution_worker ~t ~dep_build_hashes ~ocaml_version =
876
+
let build_solution_worker ~t ~dep_build_hashes ~ocaml_version ~solution_packages =
871
877
let config = t.config in
872
878
if not config.with_jtw then (1, "")
873
879
else
874
880
match ensure_jtw_tools_layer ~t ~ocaml_version with
875
881
| Some _tools_dir ->
876
882
if not (Jtw_tools.has_jsoo ~config ~ocaml_version) then (1, "")
877
-
else run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version
883
+
else run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version ~solution_packages
878
884
| None -> (1, "")
879
885
880
886
let generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version =
+4
-2
day10/bin/main.ml
+4
-2
day10/bin/main.ml
···
1362
1362
else None
1363
1363
) ordered in
1364
1364
let unique_hashes = List.sort_uniq String.compare dep_build_hashes in
1365
+
let solution_packages = List.map OpamPackage.to_string ordered in
1365
1366
Printf.printf " Building worker.js for %s (%d build layers)...\n%!"
1366
1367
(OpamPackage.to_string target) (List.length unique_hashes);
1367
1368
let status, worker_output_dir =
1368
-
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version
1369
+
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages
1369
1370
in
1370
1371
if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then
1371
1372
Some (target, solution, ocaml_version, worker_output_dir)
···
1439
1440
else None
1440
1441
) ordered in
1441
1442
let unique_hashes = List.sort_uniq String.compare dep_build_hashes in
1443
+
let solution_packages = List.map OpamPackage.to_string ordered in
1442
1444
Printf.printf " Building worker.js for %s (%d build layers)...\n%!"
1443
1445
(OpamPackage.to_string target) (List.length unique_hashes);
1444
1446
let status, worker_output_dir =
1445
-
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version
1447
+
Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version ~solution_packages
1446
1448
in
1447
1449
if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then
1448
1450
Some (target, solution, ocaml_version, worker_output_dir)
+3
day10/bin/s.ml
+3
day10/bin/s.ml
···
73
73
Yojson.Safe.t option
74
74
75
75
(** Build worker.js + stdlib for a solution using the solution's dependency overlay.
76
+
[solution_packages] is the list of package strings in the solution (e.g., ["yojson.2.2.2"])
77
+
used to constrain opam install to match the solution's dependency versions.
76
78
Returns (exit_status, worker_output_dir). *)
77
79
val build_solution_worker :
78
80
t:t ->
79
81
dep_build_hashes:string list ->
80
82
ocaml_version:OpamPackage.t ->
83
+
solution_packages:string list ->
81
84
int * string
82
85
end
+1
-1
day10/bin/windows.ml
+1
-1
day10/bin/windows.ml
···
182
182
(* JTW generation not supported on Windows *)
183
183
let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None
184
184
185
-
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")
185
+
let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ ~solution_packages:_ = (1, "")