forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Stream Renderer.page children as Seq.t for bounded peak memory
Previously html-generate built the entire tree of Html.elt values for
all pages in a library before writing anything. For core.odocl (11k
pages) peak RSS reached ~2 GB, problematic for CI environments with
limited memory (e.g., GitHub Actions).
Change Renderer.page.children from `page list Lazy.t` to `page Seq.t`
and switch subpage generators to Seq.map. Each child's Html.elt tree
is now built only when traverse pulls it, and the previous sibling's
tree becomes unreachable as traverse moves on. Peak memory during
rendering is now bounded by the ancestor chain (O(depth)) rather than
the whole library (O(pages)).
Impact on core.odocl html-generate:
- Peak RSS: 2.00 GB -> 1.81 GB (-190 MB, -10%)
- Incremental rendering cost (above unmarshal baseline):
730 MB -> 250 MB (-66%)
The remaining ~1.27 GB floor comes from unmarshaling the full
Lang.Compilation_unit (148 MB on disk -> ~1.3 GB in memory, typical
for Marshal-deserialised OCaml records).
HTML output bit-for-bit identical (verified).
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+30
-37
+7
-19
odoc/src/document/renderer.ml
+7
-19
odoc/src/document/renderer.ml
···
8
8
filename : Fpath.t;
9
9
path : Url.Path.t;
10
10
content : Format.formatter -> unit;
11
-
children : page list Lazy.t;
12
-
(** Subpages are built lazily so we can format+discard each page
13
-
before building its siblings, keeping peak memory bounded to
14
-
a single page's Html.elt tree (plus ancestors in the stack)
15
-
rather than the whole library at once. *)
11
+
children : page Seq.t;
12
+
(** Subpages are produced one at a time so we can format+discard
13
+
each page before building its siblings. Peak memory during
14
+
html-generate is bounded by the ancestor chain (plus current
15
+
leaf) rather than the entire library tree. *)
16
16
assets : Odoc_extension_registry.asset list;
17
17
(** Binary assets to write alongside this page *)
18
18
}
19
19
20
20
let traverse ~f t =
21
-
(* Process each page and release its content closure (which holds the
22
-
full Html.elt tree for that page) before descending into children.
23
-
We destructure into fresh bindings so the original page record
24
-
becomes unreachable after the write, freeing memory for the GC. *)
25
21
let rec aux { filename; content; assets; children; _ } =
26
22
f filename content assets;
27
-
(* Use a mutable ref to nil out our local reference to `content`
28
-
after the write, so the Html.elt tree doesn't stay rooted while
29
-
we process descendants. *)
30
-
iter_list (Lazy.force children)
31
-
and iter_list = function
32
-
| [] -> ()
33
-
| hd :: tl ->
34
-
aux hd;
35
-
iter_list tl
23
+
Seq.iter aux children
36
24
in
37
-
iter_list t
25
+
List.iter aux t
38
26
39
27
type input =
40
28
| CU of Odoc_model.Lang.Compilation_unit.t
+10
-6
odoc/src/html/generator.ml
+10
-6
odoc/src/html/generator.ml
···
682
682
page ~config ~sidebar content
683
683
684
684
and subpages ~config ~sidebar subpages =
685
-
List.map (include_ ~config ~sidebar) subpages
685
+
(* Produce subpages as a Seq so each child's Html.elt tree is built
686
+
only when the traversal reaches it. Seq.map is lazy: the mapping
687
+
function runs once per child when Seq.iter pulls from it. *)
688
+
Seq.map (include_ ~config ~sidebar) (List.to_seq subpages)
686
689
687
690
and page ~config ~sidebar p : Odoc_document.Renderer.page =
688
691
let { Page.preamble = _; items = i; url; source_anchor; resources; assets } =
689
692
Doctree.Labels.disambiguate_page ~enter_subpages:false p
690
693
in
691
-
(* Build subpages lazily so we don't construct the entire tree of
692
-
Html.elt values for all descendants before writing anything. *)
693
-
let subpages =
694
-
lazy (subpages ~config ~sidebar @@ Doctree.Subpages.compute p)
695
-
in
694
+
(* Build subpages lazily as a Seq — each child's Html.elt tree is
695
+
constructed only when traverse pulls it. Combined with the
696
+
ancestor-only reachability of traverse, peak memory during
697
+
html-generate is bounded by the ancestor chain (O(depth)) rather
698
+
than the whole library (O(pages)). *)
699
+
let subpages = subpages ~config ~sidebar (Doctree.Subpages.compute p) in
696
700
let resolve = Link.Current url in
697
701
let breadcrumbs = Breadcrumbs.gen_breadcrumbs ~config ~sidebar ~url in
698
702
let sidebar_html =
+1
-1
odoc/src/html/html_fragment_json.ml
+1
-1
odoc/src/html/html_fragment_json.ml
···
110
110
(List.map (Format.asprintf "%a" htmlpp) content)) );
111
111
]))
112
112
in
113
-
{ Odoc_document.Renderer.filename; content; children = lazy []; path = url; assets = [] }
113
+
{ Odoc_document.Renderer.filename; content; children = Seq.empty; path = url; assets = [] }
114
114
115
115
(* Register as the "json" shell *)
116
116
let () =
+1
-1
odoc/src/html/html_fragment_json.mli
+1
-1
odoc/src/html/html_fragment_json.mli
···
12
12
assets:Odoc_extension_registry.asset list ->
13
13
header:Html_types.flow5_without_header_footer Html.elt list ->
14
14
Html_types.div_content Html.elt list ->
15
-
Odoc_document.Renderer.page list Lazy.t ->
15
+
Odoc_document.Renderer.page Seq.t ->
16
16
Odoc_document.Renderer.page
17
17
18
18
val make_src :
+1
-1
odoc/src/html/html_page.ml
+1
-1
odoc/src/html/html_page.ml
···
339
339
let content =
340
340
src_page_creator ~breadcrumbs ~config ~url ~header ~sidebar title content
341
341
in
342
-
{ Odoc_document.Renderer.filename; content; children = lazy []; path = url; assets = [] }
342
+
{ Odoc_document.Renderer.filename; content; children = Seq.empty; path = url; assets = [] }
343
343
344
344
(* Register as the default shell *)
345
345
let () =
+1
-1
odoc/src/html/html_page.mli
+1
-1
odoc/src/html/html_page.mli
···
31
31
resources:Odoc_extension_registry.resource list ->
32
32
assets:Odoc_extension_registry.asset list ->
33
33
Html_types.div_content Html.elt list ->
34
-
Odoc_document.Renderer.page list Lazy.t ->
34
+
Odoc_document.Renderer.page Seq.t ->
35
35
Odoc_document.Renderer.page
36
36
(** [make ?theme_uri (body, children)] calls "the page creator" to turn [body]
37
37
into an [[ `Html ] elt]. If [theme_uri] is provided, it will be used to
+1
-1
odoc/src/html/html_shell.ml
+1
-1
odoc/src/html/html_shell.ml
+1
-1
odoc/src/html/html_shell.mli
+1
-1
odoc/src/html/html_shell.mli
···
21
21
source_anchor : string option;
22
22
resources : Odoc_extension_registry.resource list;
23
23
assets : Odoc_extension_registry.asset list;
24
-
children : Odoc_document.Renderer.page list Lazy.t;
24
+
children : Odoc_document.Renderer.page Seq.t;
25
25
}
26
26
27
27
(** Data for assembling a source code page. *)
+1
-1
odoc/src/latex/generator.ml
+1
-1
odoc/src/latex/generator.ml
···
522
522
if config.with_children then link_children ppf children else ()
523
523
in
524
524
let content ppf = Fmt.pf ppf "@[<v>%a@,%t@]@." pp content children_input in
525
-
{ Odoc_document.Renderer.filename; content; children = Lazy.from_val children; path = url; assets = [] }
525
+
{ Odoc_document.Renderer.filename; content; children = List.to_seq children; path = url; assets = [] }
526
526
end
527
527
528
528
module Page = struct
+1
-1
odoc/src/manpage/generator.ml
+1
-1
odoc/src/manpage/generator.ml
···
562
562
and children = List.concat_map subpage (Subpages.compute p) in
563
563
let content ppf = Format.fprintf ppf "%a@." Roff.pp (page p) in
564
564
let filename = Link.as_filename p.url in
565
-
{ Renderer.filename; content; children = Lazy.from_val children; path = p.url; assets = [] }
565
+
{ Renderer.filename; content; children = List.to_seq children; path = p.url; assets = [] }
566
566
567
567
let render = function
568
568
| Document.Page page -> [ render_page page ]
+3
-2
odoc/src/markdown2/generator.ml
+3
-2
odoc/src/markdown2/generator.ml
···
456
456
457
457
let rec include_ ~config { Types.Subpage.content; _ } = page ~config content
458
458
459
-
and subpages ~config subpages = List.map (include_ ~config) subpages
459
+
and subpages ~config subpages =
460
+
Seq.map (include_ ~config) (List.to_seq subpages)
460
461
461
462
and page ~config p =
462
-
let subpages = lazy (subpages ~config @@ Doctree.Subpages.compute p) in
463
+
let subpages = subpages ~config (Doctree.Subpages.compute p) in
463
464
let resolve = Link.Current p.url in
464
465
let i = Doctree.Shift.compute ~on_sub p.items in
465
466
let header, preamble =
+1
-1
odoc/src/markdown2/markdown_page.ml
+1
-1
odoc/src/markdown2/markdown_page.ml