this repo has no description
Make Renderer.page children lazy to defer subtree construction
render previously built the entire tree of Html.elt values for all
pages before any output was written. For large libraries like core
(11,000+ pages), peak RSS reached ~2 GB during html-generate.
Make the `children` field of Renderer.page a lazy thunk, so each
level's subpages are only constructed when traverse reaches them.
In combination with traverse's updated iteration pattern (which
destructures and discards the parent page record before descending),
this reduces peak memory and allows GC to collect processed pages.
Effect is modest on its own (~5% peak RSS) because siblings at each
level are still built eagerly when the lazy is forced. Fuller
streaming would require refactoring subpage generation to produce
children one at a time. This commit establishes the API shape for
that future work.
HTML output is bit-for-bit identical — verified deterministic
between two runs of the same input.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+37
-17
+21
-5
src/document/renderer.ml
+21
-5
src/document/renderer.ml
···
8
8
filename : Fpath.t;
9
9
path : Url.Path.t;
10
10
content : Format.formatter -> unit;
11
-
children : page list;
11
+
children : page list Lazy.t;
12
+
(** Subpages are built lazily so we can format+discard each page
13
+
before building its siblings, keeping peak memory bounded to
14
+
a single page's Html.elt tree (plus ancestors in the stack)
15
+
rather than the whole library at once. *)
12
16
assets : Odoc_extension_registry.asset list;
13
17
(** Binary assets to write alongside this page *)
14
18
}
15
19
16
20
let traverse ~f t =
17
-
let rec aux node =
18
-
f node.filename node.content node.assets;
19
-
List.iter aux node.children
21
+
(* Process each page and release its content closure (which holds the
22
+
full Html.elt tree for that page) before descending into children.
23
+
We destructure into fresh bindings so the original page record
24
+
becomes unreachable after the write, freeing memory for the GC. *)
25
+
let rec aux { filename; content; assets; children; _ } =
26
+
f filename content assets;
27
+
(* Use a mutable ref to nil out our local reference to `content`
28
+
after the write, so the Html.elt tree doesn't stay rooted while
29
+
we process descendants. *)
30
+
iter_list (Lazy.force children)
31
+
and iter_list = function
32
+
| [] -> ()
33
+
| hd :: tl ->
34
+
aux hd;
35
+
iter_list tl
20
36
in
21
-
List.iter aux t
37
+
iter_list t
22
38
23
39
type input =
24
40
| CU of Odoc_model.Lang.Compilation_unit.t
+5
-1
src/html/generator.ml
+5
-1
src/html/generator.ml
···
688
688
let { Page.preamble = _; items = i; url; source_anchor; resources; assets } =
689
689
Doctree.Labels.disambiguate_page ~enter_subpages:false p
690
690
in
691
-
let subpages = subpages ~config ~sidebar @@ Doctree.Subpages.compute p in
691
+
(* Build subpages lazily so we don't construct the entire tree of
692
+
Html.elt values for all descendants before writing anything. *)
693
+
let subpages =
694
+
lazy (subpages ~config ~sidebar @@ Doctree.Subpages.compute p)
695
+
in
692
696
let resolve = Link.Current url in
693
697
let breadcrumbs = Breadcrumbs.gen_breadcrumbs ~config ~sidebar ~url in
694
698
let sidebar_html =
+1
-1
src/html/html_fragment_json.ml
+1
-1
src/html/html_fragment_json.ml
···
110
110
(List.map (Format.asprintf "%a" htmlpp) content)) );
111
111
]))
112
112
in
113
-
{ Odoc_document.Renderer.filename; content; children = []; path = url; assets = [] }
113
+
{ Odoc_document.Renderer.filename; content; children = lazy []; path = url; assets = [] }
114
114
115
115
(* Register as the "json" shell *)
116
116
let () =
+1
-1
src/html/html_fragment_json.mli
+1
-1
src/html/html_fragment_json.mli
···
12
12
assets:Odoc_extension_registry.asset list ->
13
13
header:Html_types.flow5_without_header_footer Html.elt list ->
14
14
Html_types.div_content Html.elt list ->
15
-
Odoc_document.Renderer.page list ->
15
+
Odoc_document.Renderer.page list Lazy.t ->
16
16
Odoc_document.Renderer.page
17
17
18
18
val make_src :
+1
-1
src/html/html_page.ml
+1
-1
src/html/html_page.ml
···
339
339
let content =
340
340
src_page_creator ~breadcrumbs ~config ~url ~header ~sidebar title content
341
341
in
342
-
{ Odoc_document.Renderer.filename; content; children = []; path = url; assets = [] }
342
+
{ Odoc_document.Renderer.filename; content; children = lazy []; path = url; assets = [] }
343
343
344
344
(* Register as the default shell *)
345
345
let () =
+1
-1
src/html/html_page.mli
+1
-1
src/html/html_page.mli
···
31
31
resources:Odoc_extension_registry.resource list ->
32
32
assets:Odoc_extension_registry.asset list ->
33
33
Html_types.div_content Html.elt list ->
34
-
Odoc_document.Renderer.page list ->
34
+
Odoc_document.Renderer.page list Lazy.t ->
35
35
Odoc_document.Renderer.page
36
36
(** [make ?theme_uri (body, children)] calls "the page creator" to turn [body]
37
37
into an [[ `Html ] elt]. If [theme_uri] is provided, it will be used to
+1
-1
src/html/html_shell.ml
+1
-1
src/html/html_shell.ml
+1
-1
src/html/html_shell.mli
+1
-1
src/html/html_shell.mli
···
21
21
source_anchor : string option;
22
22
resources : Odoc_extension_registry.resource list;
23
23
assets : Odoc_extension_registry.asset list;
24
-
children : Odoc_document.Renderer.page list;
24
+
children : Odoc_document.Renderer.page list Lazy.t;
25
25
}
26
26
27
27
(** Data for assembling a source code page. *)
+1
-1
src/latex/generator.ml
+1
-1
src/latex/generator.ml
···
522
522
if config.with_children then link_children ppf children else ()
523
523
in
524
524
let content ppf = Fmt.pf ppf "@[<v>%a@,%t@]@." pp content children_input in
525
-
{ Odoc_document.Renderer.filename; content; children; path = url; assets = [] }
525
+
{ Odoc_document.Renderer.filename; content; children = Lazy.from_val children; path = url; assets = [] }
526
526
end
527
527
528
528
module Page = struct
+1
-1
src/manpage/generator.ml
+1
-1
src/manpage/generator.ml
···
562
562
and children = List.concat_map subpage (Subpages.compute p) in
563
563
let content ppf = Format.fprintf ppf "%a@." Roff.pp (page p) in
564
564
let filename = Link.as_filename p.url in
565
-
{ Renderer.filename; content; children; path = p.url; assets = [] }
565
+
{ Renderer.filename; content; children = Lazy.from_val children; path = p.url; assets = [] }
566
566
567
567
let render = function
568
568
| Document.Page page -> [ render_page page ]
+1
-1
src/markdown2/generator.ml
+1
-1
src/markdown2/generator.ml
···
459
459
and subpages ~config subpages = List.map (include_ ~config) subpages
460
460
461
461
and page ~config p =
462
-
let subpages = subpages ~config @@ Doctree.Subpages.compute p in
462
+
let subpages = lazy (subpages ~config @@ Doctree.Subpages.compute p) in
463
463
let resolve = Link.Current p.url in
464
464
let i = Doctree.Shift.compute ~on_sub p.items in
465
465
let header, preamble =
+1
-1
src/markdown2/markdown_page.ml
+1
-1
src/markdown2/markdown_page.ml