day10: build per-solution worker.js to eliminate CRC conflicts · jon.recoil.org/monopam-myspace@aa4ae1c

+2

day10/bin/dummy.ml

··· 38 38 39 39 (* JTW generation not supported in dummy container *) 40 40 let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None 41 + 42 + let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")

+2

day10/bin/freebsd.ml

··· 257 257 258 258 (* JTW generation not supported on FreeBSD *) 259 259 let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None 260 + 261 + let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")

+58 -29

day10/bin/jtw_gen.ml

··· 132 132 ) installed_libs 133 133 |> List.sort_uniq String.compare 134 134 135 + (** Build the shell script for building worker.js + stdlib inside a container. 136 + Runs `jtw opam` WITHOUT --no-worker to produce worker.js linked against 137 + the packages in the current opam switch. This ensures the worker is 138 + compiled against the exact dependency versions in the solution's overlay. *) 139 + let jtw_worker_container_script = 140 + String.concat " && " [ 141 + "eval $(opam env)"; 142 + "echo 'JTW: Building worker.js + stdlib'"; 143 + "jtw opam -o /home/opam/jtw-worker-output stdlib"; 144 + "echo 'JTW: Worker build done'"; 145 + ] 146 + 135 147 (** Build the shell script to run inside the container for jtw generation. 136 148 Calls `jtw opam` to handle all per-package artifact generation: 137 149 .cmi copying, .cma -> .cma.js compilation, dynamic_cmis.json, findlib_index.json. *) ··· 167 179 findlib_index.json (JSON: compiler info + META paths to ../../p/...) 168 180 v} 169 181 182 + Each solution carries its own OCaml version and worker output directory, 183 + so different solutions can have different worker.js files compiled against 184 + their specific dependency versions. 185 + 186 + [solutions] is a list of (target_pkg, solution_map, ocaml_version, worker_output_dir) 187 + where worker_output_dir contains worker.js and lib/ produced by jtw opam 188 + in the solution's dependency overlay. 189 + 170 190 The findlib_index is the single entry point for clients. It contains: 171 191 - compiler.version and compiler.content_hash (for constructing worker URL) 172 - - metas: list of relative META file paths (pointing into p/) *) 173 - let assemble_jtw_output ~config ~jtw_output ~ocaml_version ~solutions ~blessed_maps:_ = 192 + - meta_files: list of relative META file paths (pointing into p/) *) 193 + let assemble_jtw_output ~config ~jtw_output ~solutions ~blessed_maps:_ = 174 194 let os_key = Config.os_key ~config in 175 - let ocaml_ver = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in 195 + 196 + (* Step 1: Set up compiler dirs for each unique (ocaml_version, worker_output_dir). 197 + Different solutions may have different worker.js files even with the same 198 + OCaml version, because the worker's transitive dependencies differ. *) 199 + let compiler_info = Hashtbl.create 4 in 176 200 177 - (* Step 1: Compute compiler content hash, copy to compiler/<ver>/<whash>/ *) 178 - let jtw_tools_dir = Jtw_tools.layer_path ~config ~ocaml_version in 179 - let tools_output = Path.(jtw_tools_dir / "fs" / "home" / "opam" / "jtw-tools-output") in 180 - let compiler_hash = compute_compiler_content_hash tools_output in 181 - let compiler_dir = Path.(jtw_output / "compiler" / ocaml_ver / compiler_hash) in 182 - Os.mkdir ~parents:true compiler_dir; 183 - let worker_src = Path.(tools_output / "worker.js") in 184 - if Sys.file_exists worker_src then 185 - Os.cp worker_src Path.(compiler_dir / "worker.js"); 186 - (* Copy stdlib lib directory from jtw-tools output *) 187 - let stdlib_src = Path.(tools_output / "lib") in 188 - if Sys.file_exists stdlib_src then begin 189 - let stdlib_dst = Path.(compiler_dir / "lib") in 190 - Os.mkdir ~parents:true stdlib_dst; 191 - ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; stdlib_src; stdlib_dst]) 192 - end; 201 + List.iter (fun (_target_pkg, _solution, ocaml_version, worker_output_dir) -> 202 + if not (Hashtbl.mem compiler_info worker_output_dir) then begin 203 + let ocaml_ver = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in 204 + let compiler_hash = compute_compiler_content_hash worker_output_dir in 205 + let compiler_dir = Path.(jtw_output / "compiler" / ocaml_ver / compiler_hash) in 206 + if not (Sys.file_exists compiler_dir) then begin 207 + Os.mkdir ~parents:true compiler_dir; 208 + let worker_src = Path.(worker_output_dir / "worker.js") in 209 + if Sys.file_exists worker_src then 210 + Os.cp worker_src Path.(compiler_dir / "worker.js"); 211 + let stdlib_src = Path.(worker_output_dir / "lib") in 212 + if Sys.file_exists stdlib_src then begin 213 + let stdlib_dst = Path.(compiler_dir / "lib") in 214 + Os.mkdir ~parents:true stdlib_dst; 215 + ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; stdlib_src; stdlib_dst]) 216 + end 217 + end; 218 + Hashtbl.add compiler_info worker_output_dir (ocaml_ver, compiler_hash) 219 + end 220 + ) solutions; 193 221 194 222 (* Step 2: For each solution, assemble universe directories *) 195 - List.iter (fun (_target_pkg, solution) -> 223 + List.iter (fun (_target_pkg, solution, _ocaml_version, worker_output_dir) -> 224 + let ocaml_ver, compiler_hash = 225 + Hashtbl.find compiler_info worker_output_dir 226 + in 196 227 let ordered = List.map fst (OpamPackage.Map.bindings solution) in 197 228 (* Compute universe hash from build hashes of all packages in solution *) 198 229 let build_hashes = List.filter_map (fun pkg -> ··· 232 263 let jtw_layer_dir = Path.(config.dir / os_key / jtw_name) in 233 264 let jtw_lib_src = Path.(jtw_layer_dir / "lib") in 234 265 if Sys.file_exists jtw_lib_src then begin 235 - (* Compute content hash from payload files in the jtw layer *) 236 266 let content_hash = compute_content_hash jtw_lib_src in 237 - 238 - (* Copy to content-hashed path: p/<pkg>/<ver>/<hash>/lib/ *) 239 267 let p_pkg_dir = Path.(jtw_output / "p" / pkg_name / pkg_version / content_hash) in 240 268 let p_lib_dst = Path.(p_pkg_dir / "lib") in 241 269 if not (Sys.file_exists p_lib_dst) then begin ··· 243 271 ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; jtw_lib_src; p_lib_dst]) 244 272 end; 245 273 246 - (* Rewrite dynamic_cmis.json files with dcs_url relative to compiler/<ver>/<whash>/ *) 247 - (* The worker resolves dcs_url relative to its base URL (compiler/<ver>/<whash>/). 248 - We need ../../../p/<pkg>/<ver>/<chash>/lib/<rel> to navigate there. 249 - 3 levels up: <whash> -> <ver> -> compiler -> root, then into p/... *) 250 274 let rec rewrite_dcs_urls base rel = 251 275 let full = if rel = "" then base else Path.(base / rel) in 252 276 if Sys.file_exists full && Sys.is_directory full then begin ··· 270 294 in 271 295 rewrite_dcs_urls p_lib_dst ""; 272 296 273 - (* Collect META paths pointing to content-hashed p/ paths *) 274 297 (try 275 298 let rec find_metas base rel = 276 299 let full = Path.(base / rel) in ··· 281 304 find_metas base (if rel = "" then name else rel ^ "/" ^ name) 282 305 ) entries 283 306 end else if Filename.basename rel = "META" then 284 - (* Path from u/<universe>/ to p/<pkg>/<ver>/<hash>/lib/<fl>/META *) 285 307 meta_paths := 286 308 ("../../p/" ^ pkg_name ^ "/" ^ pkg_version ^ "/" ^ content_hash ^ 287 309 "/lib/" ^ rel) :: !meta_paths ··· 291 313 292 314 end 293 315 ) ordered; 316 + 317 + (* Include stdlib META from the compiler directory *) 318 + let compiler_dir = Path.(jtw_output / "compiler" / ocaml_ver / compiler_hash) in 319 + let stdlib_meta = Path.(compiler_dir / "lib" / "ocaml" / "stdlib" / "META") in 320 + if Sys.file_exists stdlib_meta then 321 + meta_paths := ("../../compiler/" ^ ocaml_ver ^ "/" ^ compiler_hash ^ 322 + "/lib/ocaml/stdlib/META") :: !meta_paths; 294 323 295 324 (* Write findlib_index for this universe *) 296 325 let sorted_metas = List.sort String.compare !meta_paths in

+1 -1

day10/bin/jtw_tools.ml

··· 55 55 @ pin_cmds 56 56 @ [ "opam install -y js_of_ocaml js_top_worker-bin js_top_worker-web"; 57 57 "eval $(opam env) && which js_of_ocaml && which jtw"; 58 - "eval $(opam env) && jtw opam -o /home/opam/jtw-tools-output stdlib" ]) 58 + "eval $(opam env) && jtw opam --no-worker -o /home/opam/jtw-tools-output stdlib" ]) 59 59 60 60 (** Check if jtw-tools layer exists for this OCaml version *) 61 61 let exists ~(config : Config.t) ~(ocaml_version : OpamPackage.t) : bool =

+102

day10/bin/linux.ml

··· 775 775 | None -> 776 776 Some (Jtw_gen.jtw_result_to_yojson Jtw_gen.Jtw_skipped) 777 777 778 + (** Build worker.js + stdlib for a solution by running jtw opam in a container 779 + with the solution's full dependency set overlaid. 780 + 781 + The container overlay is built from all dep_build_hashes (the build layers 782 + of every package in the solution) plus the jtw-tools layer (which provides 783 + the jtw binary and js_of_ocaml). The worker.js produced is linked against 784 + the exact dependency versions in this overlay. 785 + 786 + Returns the worker output directory path on success (containing worker.js 787 + and lib/), or "" on failure. *) 788 + let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version = 789 + let config = t.config in 790 + let os_key = Config.os_key ~config in 791 + let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-worker-" "" in 792 + let build_log = Path.(temp_dir / "jtw-worker.log") in 793 + let lowerdir = Path.(temp_dir / "lower") in 794 + let upperdir = Path.(temp_dir / "upper") in 795 + let workdir = Path.(temp_dir / "work") in 796 + let rootfsdir = Path.(temp_dir / "rootfs") in 797 + let () = List.iter Os.mkdir [ lowerdir; upperdir; workdir; rootfsdir ] in 798 + let uid_gid = Printf.sprintf "%d:%d" t.uid t.gid in 799 + let () = ignore (Os.sudo [ "chown"; uid_gid; upperdir; workdir ]) in 800 + let script = Jtw_gen.jtw_worker_container_script in 801 + let argv = [ "/usr/bin/env"; "bash"; "-c"; script ] in 802 + (* Build lower directory from all dependency build layers. 803 + cp -n means first layer wins for conflicts. *) 804 + List.iter (fun hash -> 805 + let layer_fs = Path.(config.dir / os_key / hash / "fs") in 806 + if Sys.file_exists layer_fs then 807 + ignore (Os.sudo ~stderr:"/dev/null" 808 + ["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; layer_fs; lowerdir]) 809 + ) dep_build_hashes; 810 + (* Copy jtw-tools layer (last, so build layers win for conflicts) *) 811 + let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version in 812 + let jtw_tools_fs = Path.(config.dir / os_key / jtw_tools_hash / "fs") in 813 + if Sys.file_exists jtw_tools_fs then 814 + ignore (Os.sudo ~stderr:"/dev/null" 815 + ["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; jtw_tools_fs; lowerdir]); 816 + (* Create output directory in container *) 817 + let jtw_output_host = Path.(temp_dir / "jtw-worker-output") in 818 + Os.mkdir ~parents:true jtw_output_host; 819 + ignore (Os.sudo [ "chown"; uid_gid; jtw_output_host ]); 820 + let etc_hosts = Path.(temp_dir / "hosts") in 821 + let () = Os.write_to_file etc_hosts ("127.0.0.1 localhost " ^ hostname) in 822 + let ld = "lowerdir=" ^ String.concat ":" [ lowerdir; Path.(config.dir / os_key / "base" / "fs") ] in 823 + let ud = "upperdir=" ^ upperdir in 824 + let wd = "workdir=" ^ workdir in 825 + let mount_result = Os.sudo ~stderr:"/dev/null" 826 + [ "mount"; "-t"; "overlay"; "overlay"; rootfsdir; "-o"; String.concat "," [ ld; ud; wd ] ] in 827 + if mount_result <> 0 then begin 828 + let _ = Os.sudo [ "rm"; "-rf"; temp_dir ] in 829 + (1, "") 830 + end 831 + else begin 832 + let local_mounts = match Jtw_tools.local_repo_mount ~config with 833 + | Some (src, dst) -> 834 + [ { Mount.ty = "bind"; src; dst; options = [ "ro"; "rbind"; "rprivate" ] } ] 835 + | None -> [] 836 + in 837 + let mounts = [ 838 + { Mount.ty = "bind"; src = jtw_output_host; dst = "/home/opam/jtw-worker-output"; options = [ "rw"; "rbind"; "rprivate" ] }; 839 + { ty = "bind"; src = etc_hosts; dst = "/etc/hosts"; options = [ "ro"; "rbind"; "rprivate" ] }; 840 + ] @ local_mounts in 841 + let jtw_env = List.map (fun (k, v) -> 842 + if k = "PATH" then (k, "/home/opam/.opam/default/bin:" ^ v) else (k, v) 843 + ) env in 844 + let config_runc = make ~root:rootfsdir ~cwd:"/home/opam" ~argv ~hostname ~uid:t.uid ~gid:t.gid ~env:jtw_env ~mounts ~network:false in 845 + let () = Os.write_to_file Path.(temp_dir / "config.json") (Yojson.Safe.pretty_to_string config_runc) in 846 + let container_id = "jtw-worker-" ^ Filename.basename temp_dir in 847 + let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in 848 + let result = Os.sudo ~stdout:build_log ~stderr:build_log [ "runc"; "run"; "-b"; temp_dir; container_id ] in 849 + let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in 850 + let _ = Os.sudo ~stderr:"/dev/null" [ "umount"; rootfsdir ] in 851 + (* Copy worker output to a persistent location before cleaning up temp *) 852 + let hash_input = String.concat " " dep_build_hashes ^ " " ^ jtw_tools_hash in 853 + let worker_dir_name = "jtw-worker-" ^ (Digest.to_hex (Digest.string hash_input)) in 854 + let worker_output_dir = Path.(config.dir / os_key / worker_dir_name) in 855 + if result = 0 && Sys.file_exists jtw_output_host then begin 856 + if not (Sys.file_exists worker_output_dir) then begin 857 + Os.mkdir ~parents:true worker_output_dir; 858 + ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; jtw_output_host; worker_output_dir]); 859 + let uid_gid_self = Printf.sprintf "%d:%d" (Unix.getuid ()) (Unix.getgid ()) in 860 + ignore (Os.sudo [ "chown"; "-R"; uid_gid_self; worker_output_dir ]) 861 + end 862 + end; 863 + let _ = Os.sudo [ "rm"; "-rf"; lowerdir; workdir; rootfsdir; upperdir; jtw_output_host ] in 864 + (* Copy build log to worker output dir for debugging *) 865 + (try if Sys.file_exists worker_output_dir then Os.cp build_log Path.(worker_output_dir / "jtw-worker.log") with _ -> ()); 866 + (try Os.rm ~recursive:true temp_dir with _ -> ()); 867 + (result, worker_output_dir) 868 + end 869 + 870 + let build_solution_worker ~t ~dep_build_hashes ~ocaml_version = 871 + let config = t.config in 872 + if not config.with_jtw then (1, "") 873 + else 874 + match ensure_jtw_tools_layer ~t ~ocaml_version with 875 + | Some _tools_dir -> 876 + if not (Jtw_tools.has_jsoo ~config ~ocaml_version) then (1, "") 877 + else run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version 878 + | None -> (1, "") 879 + 778 880 let generate_docs ~t ~build_layer_dir ~doc_layer_dir ~dep_doc_hashes ~pkg ~installed_libs ~installed_docs ~phase ~ocaml_version = 779 881 let config = t.config in 780 882 if not config.with_doc then None

+79 -13

day10/bin/main.ml

··· 1339 1339 (* Assemble JTW output if enabled *) 1340 1340 (match config.with_jtw, config.jtw_output with 1341 1341 | true, Some jtw_output -> 1342 - Printf.printf "Phase 4: Assembling JTW output...\n%!"; 1343 - (* Find OCaml version from any solution *) 1344 - let ocaml_version = List.find_map (fun (_target, solution) -> extract_ocaml_version solution) solutions in 1345 - (match ocaml_version with 1346 - | Some ocaml_version -> 1347 - Jtw_gen.assemble_jtw_output ~config ~jtw_output ~ocaml_version ~solutions ~blessed_maps:blessing_maps 1348 - | None -> Printf.printf " Warning: no OCaml version found in solutions, skipping JTW assembly\n%!") 1342 + Printf.printf "Phase 4: Building per-solution workers and assembling JTW output...\n%!"; 1343 + let t = Container.init ~config in 1344 + let os_key = Config.os_key ~config in 1345 + let jtw_solutions = List.filter_map (fun (target, solution) -> 1346 + match extract_ocaml_version solution with 1347 + | None -> 1348 + Printf.printf " Warning: no OCaml version for %s, skipping\n%!" 1349 + (OpamPackage.to_string target); 1350 + None 1351 + | Some ocaml_version -> 1352 + (* Collect all build layer hashes for packages in this solution *) 1353 + let ordered = List.map fst (OpamPackage.Map.bindings solution) in 1354 + let dep_build_hashes = List.filter_map (fun pkg -> 1355 + let pkg_str = OpamPackage.to_string pkg in 1356 + let pkg_dir = Path.(config.dir / os_key / "packages" / pkg_str) in 1357 + if Sys.file_exists pkg_dir then 1358 + try 1359 + Sys.readdir pkg_dir |> Array.to_list 1360 + |> List.find_opt (fun name -> String.length name > 6 && String.sub name 0 6 = "build-") 1361 + with _ -> None 1362 + else None 1363 + ) ordered in 1364 + let unique_hashes = List.sort_uniq String.compare dep_build_hashes in 1365 + Printf.printf " Building worker.js for %s (%d build layers)...\n%!" 1366 + (OpamPackage.to_string target) (List.length unique_hashes); 1367 + let status, worker_output_dir = 1368 + Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version 1369 + in 1370 + if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then 1371 + Some (target, solution, ocaml_version, worker_output_dir) 1372 + else begin 1373 + Printf.printf " Warning: worker build failed for %s (status=%d), skipping\n%!" 1374 + (OpamPackage.to_string target) status; 1375 + None 1376 + end 1377 + ) solutions in 1378 + if jtw_solutions <> [] then 1379 + Jtw_gen.assemble_jtw_output ~config ~jtw_output ~solutions:jtw_solutions ~blessed_maps:blessing_maps 1380 + else 1381 + Printf.printf " Warning: no solutions with working worker.js, skipping JTW assembly\n%!" 1349 1382 | _ -> ()); 1350 1383 (* Update progress: entering GC phase *) 1351 1384 progress_ref := Day10_lib.Progress.set_phase !progress_ref Day10_lib.Progress.Gc; ··· 1384 1417 (* Assemble JTW output if enabled *) 1385 1418 (match config.with_jtw, config.jtw_output with 1386 1419 | true, Some jtw_output -> 1387 - Printf.printf "Phase 4: Assembling JTW output...\n%!"; 1388 - let ocaml_version = List.find_map (fun (_target, solution) -> extract_ocaml_version solution) solutions in 1389 - (match ocaml_version with 1390 - | Some ocaml_version -> 1391 - Jtw_gen.assemble_jtw_output ~config ~jtw_output ~ocaml_version ~solutions ~blessed_maps:blessing_maps 1392 - | None -> Printf.printf " Warning: no OCaml version found in solutions, skipping JTW assembly\n%!") 1420 + Printf.printf "Phase 4: Building per-solution workers and assembling JTW output...\n%!"; 1421 + let t = Container.init ~config in 1422 + let os_key = Config.os_key ~config in 1423 + let jtw_solutions = List.filter_map (fun (target, solution) -> 1424 + match extract_ocaml_version solution with 1425 + | None -> 1426 + Printf.printf " Warning: no OCaml version for %s, skipping\n%!" 1427 + (OpamPackage.to_string target); 1428 + None 1429 + | Some ocaml_version -> 1430 + let ordered = List.map fst (OpamPackage.Map.bindings solution) in 1431 + let dep_build_hashes = List.filter_map (fun pkg -> 1432 + let pkg_str = OpamPackage.to_string pkg in 1433 + let pkg_dir = Path.(config.dir / os_key / "packages" / pkg_str) in 1434 + if Sys.file_exists pkg_dir then 1435 + try 1436 + Sys.readdir pkg_dir |> Array.to_list 1437 + |> List.find_opt (fun name -> String.length name > 6 && String.sub name 0 6 = "build-") 1438 + with _ -> None 1439 + else None 1440 + ) ordered in 1441 + let unique_hashes = List.sort_uniq String.compare dep_build_hashes in 1442 + Printf.printf " Building worker.js for %s (%d build layers)...\n%!" 1443 + (OpamPackage.to_string target) (List.length unique_hashes); 1444 + let status, worker_output_dir = 1445 + Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version 1446 + in 1447 + if status = 0 && worker_output_dir <> "" && Sys.file_exists Path.(worker_output_dir / "worker.js") then 1448 + Some (target, solution, ocaml_version, worker_output_dir) 1449 + else begin 1450 + Printf.printf " Warning: worker build failed for %s (status=%d), skipping\n%!" 1451 + (OpamPackage.to_string target) status; 1452 + None 1453 + end 1454 + ) solutions in 1455 + if jtw_solutions <> [] then 1456 + Jtw_gen.assemble_jtw_output ~config ~jtw_output ~solutions:jtw_solutions ~blessed_maps:blessing_maps 1457 + else 1458 + Printf.printf " Warning: no solutions with working worker.js, skipping JTW assembly\n%!" 1393 1459 | _ -> ()); 1394 1460 (* Update progress: entering GC phase *) 1395 1461 progress_ref := Day10_lib.Progress.set_phase !progress_ref Day10_lib.Progress.Gc;

+8

day10/bin/s.ml

··· 71 71 installed_libs:string list -> 72 72 ocaml_version:OpamPackage.t -> 73 73 Yojson.Safe.t option 74 + 75 + (** Build worker.js + stdlib for a solution using the solution's dependency overlay. 76 + Returns (exit_status, worker_output_dir). *) 77 + val build_solution_worker : 78 + t:t -> 79 + dep_build_hashes:string list -> 80 + ocaml_version:OpamPackage.t -> 81 + int * string 74 82 end

+2

day10/bin/windows.ml

··· 181 181 182 182 (* JTW generation not supported on Windows *) 183 183 let generate_jtw ~t:_ ~build_layer_dir:_ ~jtw_layer_dir:_ ~dep_build_hashes:_ ~pkg:_ ~installed_libs:_ ~ocaml_version:_ = None 184 + 185 + let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "")

+578

day10/docs/plans/2026-02-22-per-solution-worker-js.md

··· 1 + # Per-Solution worker.js Implementation Plan 2 + 3 + > **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. 4 + 5 + **Goal:** Build a separate worker.js per solution so that each universe's worker is compiled against the exact dependency versions in that solution, eliminating CRC mismatches when packages like yojson have different versions across universes. 6 + 7 + **Architecture:** Currently there is one jtw-tools layer per OCaml version that builds a single worker.js. We add a dedicated per-solution container run that builds worker.js + stdlib inside the solution's dependency overlay. The jtw-tools layer continues to provide the tools (js_of_ocaml, jtw binary) but no longer builds worker.js itself. `assemble_jtw_output` is refactored to accept per-solution OCaml versions and worker output paths. 8 + 9 + **Tech Stack:** OCaml, dune, runc containers, overlayfs, `jtw opam` CLI 10 + 11 + **Key insight:** `jtw opam -o <dir> stdlib` (without `--no-worker`) builds worker.js from whatever packages are installed in the current opam switch. The per-package container overlay already has the correct dependency versions. We run this command in the overlay to get a solution-specific worker.js. 12 + 13 + **Separation of concerns:** day10 coordinates which containers to run and assembles the final output structure. jtw does all compilation and metadata generation. No jtw logic leaks into day10. 14 + 15 + --- 16 + 17 + ### Task 1: Add `jtw_worker_container_script` to jtw_gen.ml 18 + 19 + This generates the shell script for building worker.js + stdlib inside a container. 20 + 21 + **Files:** 22 + - Modify: `day10/bin/jtw_gen.ml:135-151` 23 + 24 + **Step 1: Add the new function** 25 + 26 + Add this function before `jtw_container_script` (before line 135): 27 + 28 + ```ocaml 29 + (** Build the shell script for building worker.js + stdlib inside a container. 30 + Runs `jtw opam` WITHOUT --no-worker to produce worker.js linked against 31 + the packages in the current opam switch. This ensures the worker is 32 + compiled against the exact dependency versions in the solution's overlay. *) 33 + let jtw_worker_container_script = 34 + String.concat " && " [ 35 + "eval $(opam env)"; 36 + "echo 'JTW: Building worker.js + stdlib'"; 37 + "jtw opam -o /home/opam/jtw-worker-output stdlib"; 38 + "echo 'JTW: Worker build done'"; 39 + ] 40 + ``` 41 + 42 + **Step 2: Verify it compiles** 43 + 44 + Run: `cd /home/jons-agent/workspace/mono && dune build day10/bin/main.exe 2>&1 | head -20` 45 + Expected: Compiles successfully (new function is unused for now, but no errors) 46 + 47 + **Step 3: Commit** 48 + 49 + ```bash 50 + git add day10/bin/jtw_gen.ml 51 + git commit -m "day10: add jtw_worker_container_script for per-solution worker builds" 52 + ``` 53 + 54 + --- 55 + 56 + ### Task 2: Add `run_jtw_worker_in_container` to linux.ml 57 + 58 + This runs the worker build script in a container with a solution's dependency overlay. It's modelled on `run_jtw_in_container` but simpler: no per-package output, just worker.js + stdlib. 59 + 60 + **Files:** 61 + - Modify: `day10/bin/linux.ml` (after `run_jtw_in_container`, before `generate_jtw`) 62 + - Modify: `day10/bin/s.ml` (add signature) 63 + - Modify: `day10/bin/dummy.ml`, `day10/bin/freebsd.ml`, `day10/bin/windows.ml` (stub implementations) 64 + 65 + **Step 1: Add the function to linux.ml** 66 + 67 + Insert after `run_jtw_in_container` (after line 747), before `generate_jtw`: 68 + 69 + ```ocaml 70 + (** Build worker.js + stdlib for a solution by running jtw opam in a container 71 + with the solution's full dependency set overlaid. 72 + 73 + The container overlay is built from all dep_build_hashes (the build layers 74 + of every package in the solution) plus the jtw-tools layer (which provides 75 + the jtw binary and js_of_ocaml). The worker.js produced is linked against 76 + the exact dependency versions in this overlay. 77 + 78 + Returns (exit_status, worker_output_dir) where worker_output_dir contains 79 + worker.js and lib/ on success. *) 80 + let run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version = 81 + let config = t.config in 82 + let os_key = Config.os_key ~config in 83 + let temp_dir = Os.temp_dir ~perms:0o755 ~parent_dir:config.dir "temp-jtw-worker-" "" in 84 + let build_log = Path.(temp_dir / "jtw-worker.log") in 85 + let lowerdir = Path.(temp_dir / "lower") in 86 + let upperdir = Path.(temp_dir / "upper") in 87 + let workdir = Path.(temp_dir / "work") in 88 + let rootfsdir = Path.(temp_dir / "rootfs") in 89 + let () = List.iter Os.mkdir [ lowerdir; upperdir; workdir; rootfsdir ] in 90 + let uid_gid = Printf.sprintf "%d:%d" t.uid t.gid in 91 + let () = ignore (Os.sudo [ "chown"; uid_gid; upperdir; workdir ]) in 92 + let script = Jtw_gen.jtw_worker_container_script in 93 + let argv = [ "/usr/bin/env"; "bash"; "-c"; script ] in 94 + (* Build lower directory from all dependency build layers + jtw-tools layer. 95 + cp -n means first layer wins for conflicts. We iterate dep_build_hashes 96 + which are all build layers in the solution. *) 97 + List.iter (fun hash -> 98 + let layer_fs = Path.(config.dir / os_key / hash / "fs") in 99 + if Sys.file_exists layer_fs then 100 + ignore (Os.sudo ~stderr:"/dev/null" 101 + ["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; layer_fs; lowerdir]) 102 + ) dep_build_hashes; 103 + (* Copy jtw-tools layer (last, so build layers win for conflicts) *) 104 + let jtw_tools_hash = Jtw_tools.get_hash ~config ~ocaml_version in 105 + let jtw_tools_fs = Path.(config.dir / os_key / jtw_tools_hash / "fs") in 106 + if Sys.file_exists jtw_tools_fs then 107 + ignore (Os.sudo ~stderr:"/dev/null" 108 + ["cp"; "-n"; "--archive"; "--no-dereference"; "--recursive"; "--link"; "--no-target-directory"; jtw_tools_fs; lowerdir]); 109 + (* Create output directory in container *) 110 + let jtw_output_host = Path.(temp_dir / "jtw-worker-output") in 111 + Os.mkdir ~parents:true jtw_output_host; 112 + ignore (Os.sudo [ "chown"; uid_gid; jtw_output_host ]); 113 + let etc_hosts = Path.(temp_dir / "hosts") in 114 + let () = Os.write_to_file etc_hosts ("127.0.0.1 localhost " ^ hostname) in 115 + let ld = "lowerdir=" ^ String.concat ":" [ lowerdir; Path.(config.dir / os_key / "base" / "fs") ] in 116 + let ud = "upperdir=" ^ upperdir in 117 + let wd = "workdir=" ^ workdir in 118 + let mount_result = Os.sudo ~stderr:"/dev/null" 119 + [ "mount"; "-t"; "overlay"; "overlay"; rootfsdir; "-o"; String.concat "," [ ld; ud; wd ] ] in 120 + if mount_result <> 0 then begin 121 + let _ = Os.sudo [ "rm"; "-rf"; temp_dir ] in 122 + (1, "") 123 + end 124 + else begin 125 + let local_mounts = match Jtw_tools.local_repo_mount ~config with 126 + | Some (src, dst) -> 127 + [ { Mount.ty = "bind"; src; dst; options = [ "ro"; "rbind"; "rprivate" ] } ] 128 + | None -> [] 129 + in 130 + let mounts = [ 131 + { Mount.ty = "bind"; src = jtw_output_host; dst = "/home/opam/jtw-worker-output"; options = [ "rw"; "rbind"; "rprivate" ] }; 132 + { ty = "bind"; src = etc_hosts; dst = "/etc/hosts"; options = [ "ro"; "rbind"; "rprivate" ] }; 133 + ] @ local_mounts in 134 + let jtw_env = List.map (fun (k, v) -> 135 + if k = "PATH" then (k, "/home/opam/.opam/default/bin:" ^ v) else (k, v) 136 + ) env in 137 + let config_runc = make ~root:rootfsdir ~cwd:"/home/opam" ~argv ~hostname ~uid:t.uid ~gid:t.gid ~env:jtw_env ~mounts ~network:false in 138 + let () = Os.write_to_file Path.(temp_dir / "config.json") (Yojson.Safe.pretty_to_string config_runc) in 139 + let container_id = "jtw-worker-" ^ Filename.basename temp_dir in 140 + let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in 141 + let result = Os.sudo ~stdout:build_log ~stderr:build_log [ "runc"; "run"; "-b"; temp_dir; container_id ] in 142 + let _ = Os.sudo ~stdout:"/dev/null" ~stderr:"/dev/null" [ "runc"; "delete"; "-f"; container_id ] in 143 + let _ = Os.sudo ~stderr:"/dev/null" [ "umount"; rootfsdir ] in 144 + (* Copy worker output to a persistent location before cleaning up temp *) 145 + let worker_output_dir = Path.(config.dir / os_key / "jtw-worker-" ^ (Digest.to_hex (Digest.string (String.concat " " dep_build_hashes)))) in 146 + if result = 0 && Sys.file_exists jtw_output_host then begin 147 + if not (Sys.file_exists worker_output_dir) then begin 148 + Os.mkdir ~parents:true worker_output_dir; 149 + ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; jtw_output_host; worker_output_dir]); 150 + let uid_gid_self = Printf.sprintf "%d:%d" (Unix.getuid ()) (Unix.getgid ()) in 151 + ignore (Os.sudo [ "chown"; "-R"; uid_gid_self; worker_output_dir ]) 152 + end 153 + end; 154 + let _ = Os.sudo [ "rm"; "-rf"; lowerdir; workdir; rootfsdir; upperdir; jtw_output_host ] in 155 + (result, worker_output_dir) 156 + end 157 + ``` 158 + 159 + **Step 2: Add signature to s.ml** 160 + 161 + Add after `generate_jtw` in the CONTAINER signature (after line 71): 162 + 163 + ```ocaml 164 + (** Build worker.js + stdlib for a solution using the solution's dependency overlay. 165 + Returns (exit_status, worker_output_dir). *) 166 + val build_solution_worker : 167 + t:t -> 168 + dep_build_hashes:string list -> 169 + ocaml_version:OpamPackage.t -> 170 + int * string 171 + ``` 172 + 173 + **Step 3: Add the public wrapper in linux.ml** 174 + 175 + After `generate_jtw` (after line 774): 176 + 177 + ```ocaml 178 + let build_solution_worker ~t ~dep_build_hashes ~ocaml_version = 179 + let config = t.config in 180 + if not config.with_jtw then (1, "") 181 + else 182 + match ensure_jtw_tools_layer ~t ~ocaml_version with 183 + | Some _tools_dir -> 184 + run_jtw_worker_in_container ~t ~dep_build_hashes ~ocaml_version 185 + | None -> (1, "") 186 + ``` 187 + 188 + **Step 4: Add stubs in dummy.ml, freebsd.ml, windows.ml** 189 + 190 + Each needs: 191 + ```ocaml 192 + let build_solution_worker ~t:_ ~dep_build_hashes:_ ~ocaml_version:_ = (1, "") 193 + ``` 194 + 195 + **Step 5: Verify it compiles** 196 + 197 + Run: `cd /home/jons-agent/workspace/mono && dune build day10/bin/main.exe 2>&1 | head -20` 198 + Expected: Compiles successfully 199 + 200 + **Step 6: Commit** 201 + 202 + ```bash 203 + git add day10/bin/linux.ml day10/bin/s.ml day10/bin/dummy.ml day10/bin/freebsd.ml day10/bin/windows.ml 204 + git commit -m "day10: add build_solution_worker for per-solution worker.js builds" 205 + ``` 206 + 207 + --- 208 + 209 + ### Task 3: Refactor `assemble_jtw_output` for per-solution compiler directories 210 + 211 + Port the per-solution architecture from commit `9fd4beb` in `/home/jons-agent/workspace/src/day10`, updated for the current codebase. The key change: each solution carries its own `ocaml_version` and `worker_output_dir` (path to the per-solution worker.js + stdlib output). 212 + 213 + **Files:** 214 + - Modify: `day10/bin/jtw_gen.ml:173-315` (replace `assemble_jtw_output`) 215 + 216 + **Step 1: Change the function signature and Step 1 (compiler setup)** 217 + 218 + Replace the entire `assemble_jtw_output` function (lines 173-315) with: 219 + 220 + ```ocaml 221 + (** Assemble the jtw output directory structure from completed jtw layers. 222 + 223 + Output structure (content-hashed paths for immutable caching): 224 + {v 225 + <jtw_output>/ 226 + compiler/<ocaml-version>/<compiler-hash>/ 227 + worker.js 228 + lib/ocaml/ 229 + dynamic_cmis.json 230 + stdlib.cmi, ... 231 + p/<package>/<version>/<content-hash>/ 232 + lib/<findlib-name>/ 233 + META, dynamic_cmis.json, *.cmi, *.cma.js 234 + u/<universe-hash>/ 235 + findlib_index.json (JSON: compiler info + META paths to ../../p/...) 236 + v} 237 + 238 + Each solution carries its own OCaml version and worker output directory, 239 + so different solutions can have different worker.js files compiled against 240 + their specific dependency versions. 241 + 242 + [solutions] is a list of (target_pkg, solution_map, ocaml_version, worker_output_dir) 243 + where worker_output_dir contains worker.js and lib/ produced by jtw opam 244 + in the solution's dependency overlay. 245 + 246 + The findlib_index is the single entry point for clients. It contains: 247 + - compiler.version and compiler.content_hash (for constructing worker URL) 248 + - meta_files: list of relative META file paths (pointing into p/) *) 249 + let assemble_jtw_output ~config ~jtw_output ~solutions ~blessed_maps:_ = 250 + let os_key = Config.os_key ~config in 251 + 252 + (* Step 1: Set up compiler dirs for each unique (ocaml_version, worker_output_dir). 253 + Different solutions may have different worker.js files even with the same 254 + OCaml version, because the worker's transitive dependencies differ. *) 255 + let compiler_info = Hashtbl.create 4 in 256 + 257 + List.iter (fun (_target_pkg, _solution, ocaml_version, worker_output_dir) -> 258 + if not (Hashtbl.mem compiler_info worker_output_dir) then begin 259 + let ocaml_ver = OpamPackage.Version.to_string (OpamPackage.version ocaml_version) in 260 + let compiler_hash = compute_compiler_content_hash worker_output_dir in 261 + let compiler_dir = Path.(jtw_output / "compiler" / ocaml_ver / compiler_hash) in 262 + if not (Sys.file_exists compiler_dir) then begin 263 + Os.mkdir ~parents:true compiler_dir; 264 + let worker_src = Path.(worker_output_dir / "worker.js") in 265 + if Sys.file_exists worker_src then 266 + Os.cp worker_src Path.(compiler_dir / "worker.js"); 267 + let stdlib_src = Path.(worker_output_dir / "lib") in 268 + if Sys.file_exists stdlib_src then begin 269 + let stdlib_dst = Path.(compiler_dir / "lib") in 270 + Os.mkdir ~parents:true stdlib_dst; 271 + ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; stdlib_src; stdlib_dst]) 272 + end 273 + end; 274 + Hashtbl.add compiler_info worker_output_dir (ocaml_ver, compiler_hash) 275 + end 276 + ) solutions; 277 + 278 + (* Step 2: For each solution, assemble universe directories *) 279 + List.iter (fun (_target_pkg, solution, ocaml_version, worker_output_dir) -> 280 + let ocaml_ver, compiler_hash = 281 + Hashtbl.find compiler_info worker_output_dir 282 + in 283 + let _ = ocaml_version in (* used only for compiler_info lookup *) 284 + let ordered = List.map fst (OpamPackage.Map.bindings solution) in 285 + (* Compute universe hash from build hashes of all packages in solution *) 286 + let build_hashes = List.filter_map (fun pkg -> 287 + let pkg_str = OpamPackage.to_string pkg in 288 + let pkg_dir = Path.(config.dir / os_key / "packages" / pkg_str) in 289 + if Sys.file_exists pkg_dir then begin 290 + try 291 + Sys.readdir pkg_dir |> Array.to_list 292 + |> List.find_opt (fun name -> String.length name > 6 && String.sub name 0 6 = "build-") 293 + with _ -> None 294 + end else None 295 + ) ordered in 296 + let universe = Odoc_gen.compute_universe_hash build_hashes in 297 + 298 + (* Collect META paths for findlib_index *) 299 + let meta_paths = ref [] in 300 + 301 + List.iter (fun pkg -> 302 + let pkg_name = OpamPackage.name_to_string pkg in 303 + let pkg_version = OpamPackage.version_to_string pkg in 304 + let pkg_str = OpamPackage.to_string pkg in 305 + 306 + (* Find jtw layer for this package *) 307 + let pkg_layers_dir = Path.(config.dir / os_key / "packages" / pkg_str) in 308 + let jtw_layer_name = 309 + if Sys.file_exists pkg_layers_dir then 310 + try 311 + Sys.readdir pkg_layers_dir |> Array.to_list 312 + |> List.find_opt (fun name -> String.length name > 4 && String.sub name 0 4 = "jtw-") 313 + with _ -> None 314 + else None 315 + in 316 + 317 + match jtw_layer_name with 318 + | None -> () 319 + | Some jtw_name -> 320 + let jtw_layer_dir = Path.(config.dir / os_key / jtw_name) in 321 + let jtw_lib_src = Path.(jtw_layer_dir / "lib") in 322 + if Sys.file_exists jtw_lib_src then begin 323 + let content_hash = compute_content_hash jtw_lib_src in 324 + let p_pkg_dir = Path.(jtw_output / "p" / pkg_name / pkg_version / content_hash) in 325 + let p_lib_dst = Path.(p_pkg_dir / "lib") in 326 + if not (Sys.file_exists p_lib_dst) then begin 327 + Os.mkdir ~parents:true p_lib_dst; 328 + ignore (Os.sudo ["cp"; "-a"; "--no-target-directory"; jtw_lib_src; p_lib_dst]) 329 + end; 330 + 331 + let rec rewrite_dcs_urls base rel = 332 + let full = if rel = "" then base else Path.(base / rel) in 333 + if Sys.file_exists full && Sys.is_directory full then begin 334 + let entries = try Sys.readdir full |> Array.to_list with _ -> [] in 335 + let entries = List.sort String.compare entries in 336 + let cmi_files = List.filter (fun f -> Filename.check_suffix f ".cmi") entries in 337 + if cmi_files <> [] then begin 338 + let cmi_files = List.sort String.compare cmi_files in 339 + let new_dcs_url = Printf.sprintf "../../../p/%s/%s/%s/lib/%s" 340 + pkg_name pkg_version content_hash (if rel = "" then "" else rel) in 341 + let dcs_json = generate_dynamic_cmis_json ~dcs_url:new_dcs_url cmi_files in 342 + Os.write_to_file Path.(full / "dynamic_cmis.json") dcs_json 343 + end; 344 + List.iter (fun name -> 345 + let sub = if rel = "" then name else rel ^ "/" ^ name in 346 + let sub_full = Path.(base / sub) in 347 + if Sys.file_exists sub_full && Sys.is_directory sub_full then 348 + rewrite_dcs_urls base sub 349 + ) entries 350 + end 351 + in 352 + rewrite_dcs_urls p_lib_dst ""; 353 + 354 + (try 355 + let rec find_metas base rel = 356 + let full = Path.(base / rel) in 357 + if Sys.is_directory full then begin 358 + let entries = Sys.readdir full |> Array.to_list 359 + |> List.sort String.compare in 360 + List.iter (fun name -> 361 + find_metas base (if rel = "" then name else rel ^ "/" ^ name) 362 + ) entries 363 + end else if Filename.basename rel = "META" then 364 + meta_paths := 365 + ("../../p/" ^ pkg_name ^ "/" ^ pkg_version ^ "/" ^ content_hash ^ 366 + "/lib/" ^ rel) :: !meta_paths 367 + in 368 + find_metas jtw_lib_src "" 369 + with _ -> ()); 370 + 371 + end 372 + ) ordered; 373 + 374 + (* Include stdlib META from the compiler directory *) 375 + let compiler_dir = Path.(jtw_output / "compiler" / ocaml_ver / compiler_hash) in 376 + let stdlib_meta = Path.(compiler_dir / "lib" / "ocaml" / "stdlib" / "META") in 377 + if Sys.file_exists stdlib_meta then 378 + meta_paths := ("../../compiler/" ^ ocaml_ver ^ "/" ^ compiler_hash ^ 379 + "/lib/ocaml/stdlib/META") :: !meta_paths; 380 + 381 + (* Write findlib_index for this universe *) 382 + let sorted_metas = List.sort String.compare !meta_paths in 383 + if sorted_metas <> [] then begin 384 + let u_dir = Path.(jtw_output / "u" / universe) in 385 + Os.mkdir ~parents:true u_dir; 386 + let compiler_json = `Assoc [ 387 + ("version", `String ocaml_ver); 388 + ("content_hash", `String compiler_hash); 389 + ] in 390 + let findlib_index = generate_findlib_index ~compiler:compiler_json sorted_metas in 391 + Os.write_to_file Path.(u_dir / "findlib_index.json") findlib_index 392 + end 393 + ) solutions 394 + ``` 395 + 396 + **Step 2: Verify it compiles** 397 + 398 + It will NOT compile yet because main.ml still calls the old signature. That's expected. Just verify the file parses: 399 + 400 + Run: `cd /home/jons-agent/workspace/mono && ocamlfind ocamlc -package opam-format,yojson,str -c day10/bin/jtw_gen.ml 2>&1 | head -10` 401 + 402 + This may not work standalone due to module deps. Instead just check syntax: 403 + Run: `cd /home/jons-agent/workspace/mono && ocaml -stdin <<< 'let () = ignore (Sys.file_exists "test")'` (just verifying OCaml works) 404 + 405 + Skip compilation check for now — Task 4 will update main.ml to match. 406 + 407 + **Step 3: Commit** 408 + 409 + ```bash 410 + git add day10/bin/jtw_gen.ml 411 + git commit -m "day10: refactor assemble_jtw_output for per-solution worker.js" 412 + ``` 413 + 414 + --- 415 + 416 + ### Task 4: Update main.ml to build per-solution workers and pass them to assembly 417 + 418 + This is the coordinator change: for each solution, build worker.js in the solution's overlay, then pass the results to `assemble_jtw_output`. 419 + 420 + **Files:** 421 + - Modify: `day10/bin/main.ml` (lines 1340-1350 and 1385-1394) 422 + 423 + **Step 1: Replace both Phase 4 blocks** 424 + 425 + Replace the serial mode block (lines 1340-1350): 426 + 427 + ```ocaml 428 + (* Assemble JTW output if enabled *) 429 + (match config.with_jtw, config.jtw_output with 430 + | true, Some jtw_output -> 431 + Printf.printf "Phase 4: Assembling JTW output...\n%!"; 432 + (* Build worker.js per solution in a container with the solution's deps *) 433 + let jtw_solutions = List.filter_map (fun (target, solution) -> 434 + match extract_ocaml_version solution with 435 + | None -> 436 + Printf.printf " Warning: no OCaml version for %s, skipping\n%!" 437 + (OpamPackage.to_string target); 438 + None 439 + | Some ocaml_version -> 440 + (* Collect all build layer hashes for this solution *) 441 + let ordered = topological_sort solution in 442 + let dependencies = pkg_deps solution ordered in 443 + let dep_build_hashes = List.filter_map (fun pkg -> 444 + let ordered_deps = extract_dag dependencies pkg |> topological_sort |> List.rev in 445 + let hash = Container.layer_hash ~t (pkg :: (List.tl ordered_deps)) in 446 + let build_name = "build-" ^ hash in 447 + let os_key = Config.os_key ~config in 448 + if Sys.file_exists Path.(config.dir / os_key / build_name / "layer.json") 449 + then Some build_name 450 + else None 451 + ) ordered in 452 + let unique_hashes = List.sort_uniq String.compare dep_build_hashes in 453 + Printf.printf " Building worker.js for %s (%d build layers)...\n%!" 454 + (OpamPackage.to_string target) (List.length unique_hashes); 455 + let status, worker_output_dir = 456 + Container.build_solution_worker ~t ~dep_build_hashes:unique_hashes ~ocaml_version 457 + in 458 + if status = 0 && Sys.file_exists Path.(worker_output_dir / "worker.js") then 459 + Some (target, solution, ocaml_version, worker_output_dir) 460 + else begin 461 + Printf.printf " Warning: worker build failed for %s (status=%d), skipping\n%!" 462 + (OpamPackage.to_string target) status; 463 + None 464 + end 465 + ) solutions in 466 + if jtw_solutions <> [] then 467 + Jtw_gen.assemble_jtw_output ~config ~jtw_output ~solutions:jtw_solutions ~blessed_maps:blessing_maps 468 + else 469 + Printf.printf " Warning: no solutions with working worker.js, skipping JTW assembly\n%!" 470 + | _ -> ()); 471 + ``` 472 + 473 + Apply the same replacement to the parallel mode block (lines 1385-1394). 474 + 475 + **Step 2: Verify it compiles** 476 + 477 + Run: `cd /home/jons-agent/workspace/mono && dune build day10/bin/main.exe 2>&1 | head -20` 478 + Expected: Compiles successfully 479 + 480 + **Step 3: Commit** 481 + 482 + ```bash 483 + git add day10/bin/main.ml 484 + git commit -m "day10: build per-solution worker.js in Phase 4 assembly" 485 + ``` 486 + 487 + --- 488 + 489 + ### Task 5: Stop building worker.js in the jtw-tools layer 490 + 491 + The jtw-tools layer should only provide tools (js_of_ocaml, jtw binary), not build worker.js. Change `jtw opam -o ... stdlib` to `jtw opam --no-worker -o ... stdlib`. 492 + 493 + **Files:** 494 + - Modify: `day10/bin/jtw_tools.ml:53-58` 495 + 496 + **Step 1: Add --no-worker to jtw-tools build script** 497 + 498 + In `build_script`, change line 58 from: 499 + 500 + ```ocaml 501 + "eval $(opam env) && jtw opam -o /home/opam/jtw-tools-output stdlib" ]) 502 + ``` 503 + 504 + to: 505 + 506 + ```ocaml 507 + "eval $(opam env) && jtw opam --no-worker -o /home/opam/jtw-tools-output stdlib" ]) 508 + ``` 509 + 510 + **Step 2: Update `has_worker_js` — this function checks for worker.js in jtw-tools but is no longer relevant there** 511 + 512 + Either remove it or update it. Since it may be referenced: 513 + 514 + Run: `grep -rn "has_worker_js" day10/bin/` 515 + 516 + If unused, remove it. If used, leave it but it will now return false (which is fine — the worker comes from per-solution builds). 517 + 518 + **Step 3: Verify it compiles** 519 + 520 + Run: `cd /home/jons-agent/workspace/mono && dune build day10/bin/main.exe 2>&1 | head -20` 521 + Expected: Compiles successfully 522 + 523 + **Step 4: Commit** 524 + 525 + ```bash 526 + git add day10/bin/jtw_tools.ml 527 + git commit -m "day10: stop building worker.js in jtw-tools layer (now per-solution)" 528 + ``` 529 + 530 + **Important note:** After this change, new jtw-tools layers will not have worker.js. Existing cached layers still have it but won't be used by the new assembly code. You may need to delete old jtw-tools layers to force a rebuild if you want a clean state. 531 + 532 + --- 533 + 534 + ### Task 6: End-to-end test with day10 batch 535 + 536 + Test the full pipeline by running a day10 batch build and verifying the output. 537 + 538 + **Step 1: Clean the jtw-tools cache to force rebuild without worker.js** 539 + 540 + ```bash 541 + rm -rf /home/jons-agent/day10-cache/ubuntu-25.04-x86_64/jtw-tools-* 542 + ``` 543 + 544 + **Step 2: Run day10 batch** 545 + 546 + ```bash 547 + day10 batch --with-jtw --jtw-output /home/jons-agent/day10-jtw-output-v2 \ 548 + --cache-dir /home/jons-agent/day10-cache \ 549 + --opam-repository /home/jons-agent/.opam/repo/default \ 550 + --local-repo /home/jons-agent/workspace/mono/js_top_worker \ 551 + yojson.3.0.0 552 + ``` 553 + 554 + **Step 3: Verify output structure** 555 + 556 + ```bash 557 + # Check that compiler directory has worker.js 558 + find /home/jons-agent/day10-jtw-output-v2/compiler -name worker.js 559 + 560 + # Check universe findlib_index.json references the compiler hash 561 + cat /home/jons-agent/day10-jtw-output-v2/u/*/findlib_index.json | python3 -m json.tool 562 + ``` 563 + 564 + **Step 4: Run the Playwright test** 565 + 566 + Update `/home/jons-agent/day10-demo/demo_day10.html` with new universe/compiler hashes from the output, then: 567 + 568 + ```bash 569 + cd /home/jons-agent/day10-demo/test && node run_tests.js 570 + ``` 571 + 572 + Expected: 4/4 tests pass 573 + 574 + **Step 5: Commit any test fixture updates** 575 + 576 + ```bash 577 + git add -A && git commit -m "day10: verify per-solution worker.js with end-to-end test" 578 + ```

Configure Feed

Configure Feed