forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
day11/doc: deterministic doc layer hashes computed bottom-up
Compile/doc-all/link hashes are now computed deterministically before
execution using a memoized bottom-up walk of the dep graph. Each
compile hash includes:
- build layer hash
- tool hashes (driver + odoc)
- blessed status
- dep compile hashes (recursive, sorted)
This ensures DAG construction hashes match execution hashes exactly,
fixing the cache miss bug where compile layers were never reused
because the DAG and execution computed different hashes.
The execution functions now receive dag_hash from the DAG node
instead of recomputing their own hash.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+95
-85
+95
-85
day11/doc/generate.ml
+95
-85
day11/doc/generate.ml
···
115
115
| Error _ -> None)
116
116
117
117
let compile_package env benv ~os_dir ~driver_tool ~odoc_tools
118
-
~build_hash_blessed ~find_odoc_tool ~compile_results (node : build) =
118
+
~build_hash_blessed ~find_odoc_tool ~compile_results
119
+
~dag_hash (node : build) =
119
120
match prepare_package ~os_dir ~driver_tool ~odoc_tools
120
121
~build_hash_blessed ~find_odoc_tool node with
121
122
| None -> None
122
-
| Some (composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
123
+
| Some (_composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
123
124
(* Collect dep compile layers for overlayfs stacking *)
124
125
let seen = Hashtbl.create 16 in
125
126
List.iter (collect_transitive_deps seen) node.deps;
126
-
let dep_compile_layers = Hashtbl.fold (fun bh () acc ->
127
+
let dep_compile_dirs = Hashtbl.fold (fun bh () acc ->
127
128
match Hashtbl.find_opt compile_results bh with
128
-
| Some bl -> bl :: acc
129
+
| Some bl -> Build.dir ~os_dir bl :: acc
129
130
| None -> acc
130
131
) seen [] in
131
-
let dep_compile_dirs = List.map (Build.dir ~os_dir) dep_compile_layers in
132
-
let dep_compile_hashes = List.map (fun (bl : build) -> bl.hash)
133
-
dep_compile_layers in
134
132
let cmd =
135
133
"export PATH=/home/opam/doc-tools/bin:$PATH && eval $(opam env) && " ^
136
134
Command.odoc_driver_voodoo ~pkg:node.pkg ~universe
137
135
~blessed ~actions:"compile-only" ~odoc_bin ~odoc_md_bin in
138
-
let hash = Day11_layer.Hash.of_strings
139
-
([ "compile"; node.hash; composite_tool_hash;
140
-
(if blessed then "blessed" else "unblessed") ]
141
-
@ dep_compile_hashes) in
142
136
let compile_node : build =
143
-
{ hash; pkg = node.pkg; deps = [ node ]; universe = Day11_solution.Universe.dummy } in
137
+
{ hash = dag_hash; pkg = node.pkg; deps = [ node ];
138
+
universe = Day11_solution.Universe.dummy } in
144
139
let on_extract ~layer_dir ~success:_ =
145
140
let dm : Doc_meta.t = {
146
141
package = OpamPackage.to_string node.pkg;
···
170
165
let link_package env benv ~os_dir ~driver_tool ~odoc_tools
171
166
~build_hash_blessed ~find_odoc_tool ~compile_results
172
167
~doc_dep_hashes
173
-
~build_hash
168
+
~build_hash ~dag_hash
174
169
(node : build) =
175
170
match Hashtbl.find_opt compile_results build_hash with
176
171
| None -> None
···
178
173
match prepare_package ~os_dir ~driver_tool ~odoc_tools
179
174
~build_hash_blessed ~find_odoc_tool node with
180
175
| None -> None
181
-
| Some (composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
182
-
(* Collect compile layers from doc_deps (wider set, includes {post}
183
-
and x-extra-doc-deps for cross-referencing) *)
176
+
| Some (_composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
177
+
(* Collect compile layers from doc_deps (wider set) *)
184
178
let doc_dep_bhs = match Hashtbl.find_opt doc_dep_hashes build_hash with
185
179
| Some bhs -> bhs | None -> [] in
186
-
let dep_compile_layers = List.filter_map (fun bh ->
187
-
Hashtbl.find_opt compile_results bh
180
+
let dep_compile_dirs = List.filter_map (fun bh ->
181
+
match Hashtbl.find_opt compile_results bh with
182
+
| Some bl -> Some (Build.dir ~os_dir bl)
183
+
| None -> None
188
184
) doc_dep_bhs in
189
-
let dep_compile_dirs = List.map (Build.dir ~os_dir) dep_compile_layers in
190
-
let dep_hashes = List.map (fun (bl : build) -> bl.hash)
191
-
dep_compile_layers in
192
185
(* Also include own compile layer *)
193
186
let own_compile_dir = match Hashtbl.find_opt compile_results build_hash with
194
187
| Some bl -> [ Build.dir ~os_dir bl ]
···
203
196
"export PATH=/home/opam/doc-tools/bin:$PATH && eval $(opam env) && " ^
204
197
Command.odoc_driver_voodoo ~pkg:node.pkg ~universe
205
198
~blessed ~actions:"link-and-gen" ~odoc_bin ~odoc_md_bin in
206
-
let hash = Day11_layer.Hash.of_strings
207
-
([ "link"; compile_bl.hash; universe; composite_tool_hash;
208
-
(if blessed then "blessed" else "unblessed") ]
209
-
@ dep_hashes) in
210
199
let link_node : build =
211
-
{ hash; pkg = node.pkg;
212
-
deps = [ node; compile_bl ] @ dep_compile_layers; universe = Day11_solution.Universe.dummy } in
200
+
{ hash = dag_hash; pkg = node.pkg;
201
+
deps = [ node; compile_bl ];
202
+
universe = Day11_solution.Universe.dummy } in
213
203
let on_extract ~layer_dir ~success:_ =
214
204
let dm : Doc_meta.t = {
215
205
package = OpamPackage.to_string node.pkg;
···
239
229
240
230
let doc_all_package env benv ~os_dir ~driver_tool ~odoc_tools
241
231
~build_hash_blessed ~find_odoc_tool ~compile_results
242
-
~build_hash
232
+
~build_hash ~dag_hash
243
233
(node : build) =
244
234
match prepare_package ~os_dir ~driver_tool ~odoc_tools
245
235
~build_hash_blessed ~find_odoc_tool node with
246
236
| None -> None
247
-
| Some (composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
248
-
(* Collect dep compile layers for overlayfs stacking + hash *)
237
+
| Some (_composite_tool_hash, universe, blessed, _pkg_loc, mounts, prep_dir) ->
238
+
(* Collect dep compile layers for overlayfs stacking *)
249
239
let seen = Hashtbl.create 16 in
250
240
List.iter (collect_transitive_deps seen) node.deps;
251
-
let dep_compile_layers = Hashtbl.fold (fun bh () acc ->
241
+
let dep_compile_dirs = Hashtbl.fold (fun bh () acc ->
252
242
match Hashtbl.find_opt compile_results bh with
253
-
| Some bl -> bl :: acc
243
+
| Some bl -> Build.dir ~os_dir bl :: acc
254
244
| None -> acc
255
245
) seen [] in
256
-
let dep_compile_dirs = List.map (Build.dir ~os_dir) dep_compile_layers in
257
-
let dep_compile_hashes = List.map (fun (bl : build) -> bl.hash)
258
-
dep_compile_layers in
259
-
(* HTML RW mount — mount entire html dir so global assets (css, js)
260
-
and per-package output both land in the shared dir *)
246
+
(* HTML RW mount *)
261
247
let html_base = Fpath.(os_dir / "html") in
262
248
let html_mount = Day11_container.Mount.bind_rw
263
249
~src:(Fpath.to_string html_base)
···
266
252
"export PATH=/home/opam/doc-tools/bin:$PATH && eval $(opam env) && " ^
267
253
Command.odoc_driver_voodoo ~pkg:node.pkg ~universe
268
254
~blessed ~actions:"all" ~odoc_bin ~odoc_md_bin in
269
-
let hash = Day11_layer.Hash.of_strings
270
-
([ "doc-all"; node.hash; universe; composite_tool_hash;
271
-
(if blessed then "blessed" else "unblessed") ]
272
-
@ dep_compile_hashes) in
273
255
let doc_node : build =
274
-
{ hash; pkg = node.pkg;
275
-
deps = [ node ] @ dep_compile_layers; universe = Day11_solution.Universe.dummy } in
256
+
{ hash = dag_hash; pkg = node.pkg;
257
+
deps = [ node ]; universe = Day11_solution.Universe.dummy } in
276
258
let on_extract ~layer_dir ~success:_ =
277
259
let dm : Doc_meta.t = {
278
260
package = OpamPackage.to_string node.pkg;
···
474
456
Hashtbl.replace build_hash_blessed node.hash true
475
457
| _ -> ()
476
458
) nodes;
477
-
(* Build doc DAG nodes.
478
-
For packages that DON'T need separate link: "doc-all" node
479
-
depends on build(A), tool finals, and compile/doc-all of deps
480
-
For packages that DO need separate link: compile + link nodes
481
-
(compile produces .odoc files, link consumes them from other packages) *)
459
+
(* Build doc DAG nodes with deterministic hashes.
460
+
Hashes are computed bottom-up: each compile/doc-all hash includes
461
+
the dep compile hashes, blessed status, build hash, and tool hash.
462
+
This ensures DAG hashes match execution hashes exactly. *)
482
463
let compile_nodes : (string, build) Hashtbl.t = Hashtbl.create 64 in
483
464
let doc_all_nodes : (string, build) Hashtbl.t = Hashtbl.create 64 in
484
465
let link_nodes_list = ref [] in
485
-
(* First pass: create compile or doc-all nodes *)
466
+
(* Memoized compile hash computation — walks build dep DAG bottom-up *)
467
+
let compile_hash_cache : (string, string) Hashtbl.t = Hashtbl.create 64 in
468
+
let rec compute_compile_hash (node : build) =
469
+
match Hashtbl.find_opt compile_hash_cache node.hash with
470
+
| Some h -> h
471
+
| None ->
472
+
let blessed = match Hashtbl.find_opt build_hash_blessed node.hash with
473
+
| Some true -> true | _ -> false in
474
+
let composite_tool_hash = match find_odoc_tool_for_hash node.hash with
475
+
| Some odoc_tool ->
476
+
Day11_layer.Hash.of_strings [ driver_tool.hash; odoc_tool.hash ]
477
+
| None -> "" in
478
+
(* Collect dep compile hashes (recursive, memoized) *)
479
+
let seen = Hashtbl.create 16 in
480
+
List.iter (collect_transitive_deps seen) node.deps;
481
+
let dep_compile_hashes = Hashtbl.fold (fun dep_bh () acc ->
482
+
match Hashtbl.find_opt build_by_hash dep_bh with
483
+
| Some dep_node -> compute_compile_hash dep_node :: acc
484
+
| None -> acc
485
+
) seen [] in
486
+
let phase = if Hashtbl.mem needs_split_bh node.hash
487
+
then "compile" else "doc-all" in
488
+
let universe = Command.compute_universe_hash [ node.hash ] in
489
+
let hash = Day11_layer.Hash.of_strings
490
+
([ phase; node.hash; universe; composite_tool_hash;
491
+
(if blessed then "blessed" else "unblessed") ]
492
+
@ List.sort String.compare dep_compile_hashes) in
493
+
Hashtbl.replace compile_hash_cache node.hash hash;
494
+
hash
495
+
in
496
+
(* First pass: create compile or doc-all nodes with correct hashes *)
486
497
List.iter (fun (node : build) ->
487
498
match find_odoc_tool_for_hash node.hash,
488
499
find_odoc_final_for_hash node.hash with
489
500
| None, _ | _, None -> ()
490
-
| Some odoc_tool, Some odoc_final ->
491
-
let composite_tool_hash = Day11_layer.Hash.of_strings
492
-
[ driver_tool.hash; odoc_tool.hash ] in
493
-
if Hashtbl.mem needs_split_bh node.hash then begin
494
-
(* Needs separate compile phase *)
495
-
let compile_hash = Day11_layer.Hash.of_strings
496
-
[ "compile"; node.hash; composite_tool_hash ] in
497
-
let cn : build = { hash = compile_hash; pkg = node.pkg;
498
-
deps = [ node; driver_final; odoc_final ]; universe = Day11_solution.Universe.dummy } in
499
-
Hashtbl.replace compile_nodes node.hash cn
500
-
end else begin
501
-
(* Single doc-all phase — deps computed after universe is known *)
502
-
let universe = Command.compute_universe_hash [ node.hash ] in
503
-
let doc_all_hash = Day11_layer.Hash.of_strings
504
-
[ "doc-all"; node.hash; universe; composite_tool_hash ] in
505
-
let dn : build = { hash = doc_all_hash; pkg = node.pkg;
506
-
deps = [ node; driver_final; odoc_final ]; universe = Day11_solution.Universe.dummy } in
507
-
Hashtbl.replace doc_all_nodes node.hash dn
508
-
end
501
+
| Some _odoc_tool, Some odoc_final ->
502
+
let hash = compute_compile_hash node in
503
+
(* Collect dep doc nodes for DAG edges *)
504
+
let seen = Hashtbl.create 16 in
505
+
List.iter (collect_transitive_deps seen) node.deps;
506
+
let dep_docs = Hashtbl.fold (fun dep_bh () acc ->
507
+
(* Will be filled in after all nodes created — use placeholder *)
508
+
match Hashtbl.find_opt build_by_hash dep_bh with
509
+
| Some _ -> dep_bh :: acc
510
+
| None -> acc
511
+
) seen [] in
512
+
let dn : build = { hash; pkg = node.pkg;
513
+
deps = [ node; driver_final; odoc_final ];
514
+
universe = Day11_solution.Universe.dummy } in
515
+
if Hashtbl.mem needs_split_bh node.hash then
516
+
Hashtbl.replace compile_nodes node.hash dn
517
+
else
518
+
Hashtbl.replace doc_all_nodes node.hash dn;
519
+
ignore dep_docs
509
520
) nodes;
510
-
(* Patch deps: compile/doc-all nodes depend on their transitive
511
-
build deps' compile/doc-all nodes *)
521
+
(* Patch deps: add transitive build deps' compile/doc-all nodes *)
512
522
let patch_doc_deps build_hash (dn : build) =
513
523
let build_node = Hashtbl.find build_by_hash build_hash in
514
524
let seen = Hashtbl.create 16 in
···
517
527
match Hashtbl.find_opt compile_nodes dep_bh with
518
528
| Some cn -> cn :: acc
519
529
| None ->
520
-
match Hashtbl.find_opt doc_all_nodes dep_bh with
530
+
(match Hashtbl.find_opt doc_all_nodes dep_bh with
521
531
| Some dn -> dn :: acc
522
-
| None -> acc
532
+
| None -> acc)
523
533
) seen [] in
524
534
{ dn with deps = dn.deps @ dep_docs; universe = Day11_solution.Universe.dummy }
525
535
in
···
531
541
let doc_all_snapshot = Hashtbl.fold (fun k v acc -> (k, v) :: acc)
532
542
doc_all_nodes [] in
533
543
List.iter (fun (bh, dn) ->
534
-
(* Doc-all depends on transitive build deps' doc nodes. This follows
535
-
the package dependency DAG (acyclic), so no circular deps.
536
-
Ensures deps' odoc files are in the store before we link. *)
537
-
let patched = patch_doc_deps bh dn in
538
-
Hashtbl.replace doc_all_nodes bh patched
544
+
Hashtbl.replace doc_all_nodes bh (patch_doc_deps bh dn)
539
545
) doc_all_snapshot;
540
546
(* Create link nodes for packages that need separate link *)
541
547
Hashtbl.iter (fun build_hash _cn ->
···
550
556
) dep_bhs
551
557
in
552
558
let own_compile = Hashtbl.find compile_nodes build_hash in
559
+
let blessed = match Hashtbl.find_opt build_hash_blessed build_hash with
560
+
| Some true -> true | _ -> false in
553
561
let odoc_tool = Option.get (find_odoc_tool_for_hash build_hash) in
554
562
let composite_tool_hash = Day11_layer.Hash.of_strings
555
563
[ driver_tool.hash; odoc_tool.hash ] in
556
-
let universe = Command.compute_universe_hash [ build_node.hash ] in
557
-
let dep_hashes = List.map (fun (bl : build) -> bl.hash)
558
-
dep_compile_layers in
564
+
let dep_hashes = List.sort String.compare
565
+
(List.map (fun (bl : build) -> bl.hash) dep_compile_layers) in
559
566
let link_hash = Day11_layer.Hash.of_strings
560
-
([ "link"; own_compile.hash; universe; composite_tool_hash ]
567
+
([ "link"; own_compile.hash; composite_tool_hash;
568
+
(if blessed then "blessed" else "unblessed") ]
561
569
@ dep_hashes) in
562
570
let ln : build = { hash = link_hash; pkg = build_node.pkg;
563
-
deps = [ build_node; own_compile ] @ dep_compile_layers; universe = Day11_solution.Universe.dummy } in
571
+
deps = [ build_node; own_compile ] @ dep_compile_layers;
572
+
universe = Day11_solution.Universe.dummy } in
564
573
link_nodes_list := ln :: !link_nodes_list
565
574
) compile_nodes;
566
575
let compile_list = Hashtbl.fold (fun _ cn acc -> cn :: acc) compile_nodes [] in
···
698
707
let build_node = Hashtbl.find build_by_hash build_hash in
699
708
current_build_hash := build_hash;
700
709
(match compile_package env benv ~os_dir ~driver_tool ~odoc_tools
701
-
~build_hash_blessed ~find_odoc_tool ~compile_results build_node with
710
+
~build_hash_blessed ~find_odoc_tool ~compile_results
711
+
~dag_hash:node.hash build_node with
702
712
| Some _bl -> true
703
713
| None -> true)
704
714
end else if Hashtbl.mem doc_all_set node.hash then begin
···
710
720
current_build_hash := build_hash;
711
721
(match doc_all_package env benv ~os_dir ~driver_tool ~odoc_tools
712
722
~build_hash_blessed ~find_odoc_tool ~compile_results
713
-
~build_hash
723
+
~build_hash ~dag_hash:node.hash
714
724
build_node with
715
725
| Some n ->
716
726
Atomic.incr doc_count;
···
727
737
(match link_package env benv ~os_dir ~driver_tool ~odoc_tools
728
738
~build_hash_blessed ~find_odoc_tool ~compile_results
729
739
~doc_dep_hashes
730
-
~build_hash
740
+
~build_hash ~dag_hash:node.hash
731
741
build_node with
732
742
| Some n ->
733
743
Atomic.incr doc_count;