forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Fix DAG executor fiber explosion for large dep lists
Link nodes with thousands of deps (all compiles in a universe) were
spawning a fiber per dep via Fiber.List.map, causing over 1M fiber
spawns. Now checks if dep promises are already resolved before
spawning, avoiding fibers for completed deps.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+14
-3
+14
-3
day11/build/dag_executor.ml
+14
-3
day11/build/dag_executor.ml
···
29
29
| None ->
30
30
let p, r = Promise.create () in
31
31
Hashtbl.add promises node.hash p;
32
-
(* Await all dependencies *)
33
-
let dep_outcomes =
34
-
Fiber.List.map (fun dep -> run_node dep) node.deps
32
+
(* Await all dependencies. Check resolved promises first to avoid
33
+
spawning fibers for deps that are already done (critical for link
34
+
nodes which can have thousands of deps). *)
35
+
let resolved, unresolved = List.partition (fun (dep : build) ->
36
+
match Hashtbl.find_opt promises dep.hash with
37
+
| Some dp -> Promise.is_resolved dp
38
+
| None -> false
39
+
) node.deps in
40
+
let resolved_outcomes = List.map (fun (dep : build) ->
41
+
Promise.await (Hashtbl.find promises dep.hash)
42
+
) resolved in
43
+
let unresolved_outcomes =
44
+
Fiber.List.map (fun dep -> run_node dep) unresolved
35
45
in
46
+
let dep_outcomes = resolved_outcomes @ unresolved_outcomes in
36
47
let any_dep_failed =
37
48
List.exists (fun o -> match o with Ok -> false | _ -> true) dep_outcomes
38
49
in