forked from
jonmsterling.com/ocaml-forester
ocaml
Add documentation
Move graphs to core
authored by
Kento Okura
and committed by
Jon Sterling
f1ce3ed0
3a2f6043
+211
-58
+1
docs/dune
+1
docs/dune
···
1
+
(documentation)
+1
docs/test.mld
+1
docs/test.mld
···
1
+
{0 Test}
+4
dune
+4
dune
+35
index.mld
+35
index.mld
···
1
+
{0 Forester developer docs}
2
+
3
+
Forester is a compiled markup language for authoring scientific hypertext. This documentation is intended to give an overview of forester internals for people interested in contributing to the project.
4
+
5
+
{1 Questions}
6
+
7
+
If you have questions, feel free to write an email to [https://lists.sr.ht/~jonsterling/forester-discuss]. Please employ {{: https://useplaintext.email/#etiquette} plaintext email etiquette}, including line wrapping.
8
+
9
+
{1 Getting started}
10
+
11
+
Forester is implementend in the {{: https://ocaml.org/}OCaml programming language}, so you will need to have the OCaml toolchain installed. You can find instructions {{: https://ocaml.org/install#linux_mac_bsd}here}. Alternatively, you can get access to the toolchain using the {{: https://nixos.org/}Nix package manager}.
12
+
13
+
{2 Building the project}
14
+
15
+
[dune build]
16
+
17
+
{2 Testing}
18
+
19
+
[dune runtest]
20
+
21
+
{2 Sending a patch}
22
+
23
+
Once your changes are ready, follow {{: https://lists.sr.ht/~jonsterling/forester-devel}these instructions} to submit them for review.
24
+
25
+
{1 Codebase overview}
26
+
27
+
The code is organized into several libraries:
28
+
29
+
{!modules:
30
+
Forester_core
31
+
Forester_compiler
32
+
Forester_frontend
33
+
Forester_lsp}
34
+
35
+
{{!test} Test}
+62
lib/compiler/Code.mli
+62
lib/compiler/Code.mli
···
1
+
type node =
2
+
Text of string
3
+
| Verbatim of string
4
+
| Group of Forester_core.delim * t
5
+
| Math of Forester_core.math_mode * t
6
+
| Ident of Forester_core.Trie.path
7
+
| Hash_ident of string
8
+
| Xml_ident of string option * string
9
+
| Subtree of string option * t
10
+
| Let of Forester_core.Trie.path *
11
+
Forester_core.Trie.path Forester_core.binding list * t
12
+
| Open of Forester_core.Trie.path
13
+
| Scope of t
14
+
| Put of Forester_core.Trie.path * t
15
+
| Default of Forester_core.Trie.path * t
16
+
| Get of Forester_core.Trie.path
17
+
| Fun of Forester_core.Trie.path Forester_core.binding list * t
18
+
| Object of t _object
19
+
| Patch of t patch
20
+
| Call of t * string
21
+
| Import of Forester_core.visibility * string
22
+
| Def of Forester_core.Trie.path *
23
+
Forester_core.Trie.path Forester_core.binding list * t
24
+
| Decl_xmlns of string * string
25
+
| Alloc of Forester_core.Trie.path
26
+
| Namespace of Forester_core.Trie.path * t
27
+
| Dx_sequent of t * t list
28
+
| Dx_query of string * t list * t list
29
+
| Dx_prop of t * t list
30
+
| Dx_var of string
31
+
| Dx_const_content of t
32
+
| Dx_const_iri of t
33
+
| Comment of string
34
+
| Error of string
35
+
36
+
and t = node Forester_core.Range.located list
37
+
38
+
and 'a _object = {
39
+
self : Forester_core.Trie.path option;
40
+
methods : (string * 'a) list;
41
+
}
42
+
43
+
and 'a patch = {
44
+
obj : 'a;
45
+
self : Forester_core.Trie.path option;
46
+
methods : (string * 'a) list;
47
+
}
48
+
49
+
val t : t Repr.t
50
+
val pp : Format.formatter -> t -> unit
51
+
52
+
type tree = { source_path : string option; addr : string option; code : t; }
53
+
54
+
val parens : t -> node
55
+
val squares : t -> node
56
+
val braces : t -> node
57
+
58
+
val import_private : string -> node
59
+
val import_public : string -> node
60
+
val inline_math : t -> node
61
+
val display_math : t -> node
62
+
+4
-4
lib/compiler/Forest.mli
+4
-4
lib/compiler/Forest.mli
···
18
18
(**/**)
19
19
20
20
val analyse_resource : (module Forest_graphs.S) -> T.content T.resource -> unit
21
-
(** [analyse_resource graphs resource] traverses {{!Forester_core.Types.resource}[resource]}, recording facts about it in {{!Forest_graphs.S}[graphs]}.
21
+
(** [analyse_resource graphs resource] traverses {{!Forester_core.Types.resource}[resource]}, recording facts about it in {{!Forester_core.Forest_graphs.S}[graphs]}.
22
22
23
-
- When encountering a {{!Forester_core.Types.Link}[Link]}, it adds an edge to the {{!Forester_core.Builtin_relation.links_to}[links_to]} relation in {{!Forest_graphs.S}[graphs]}.
23
+
- When encountering a {{!Forester_core.Types.Link}[Link]}, it adds an edge to the {{!Forester_core.Builtin_relation.links_to}[links_to]} relation in {{!Forester_core.Forest_graphs.S}[graphs]}.
24
24
25
-
- When encountering a {{!Forester_core.Types.Transclude}[Transclude]}, it adds an edge to the {{!Forester_core.Builtin_relation.transcludes}[transcludes]} relation in {{!Forest_graphs.S}[graphs]}.
25
+
- When encountering a {{!Forester_core.Types.Transclude}[Transclude]}, it adds an edge to the {{!Forester_core.Builtin_relation.transcludes}[transcludes]} relation in {{!Forester_core.Forest_graphs.S}[graphs]}.
26
26
27
-
- When encountering a {{!Forester_core.Types.Datalog_script}[Datalog_script script]}, it runs the script and records the results in {{!Forest_graphs.S}[graphs]}.
27
+
- When encountering a {{!Forester_core.Types.Datalog_script}[Datalog_script script]}, it runs the script and records the results in {{!Forester_core.Forest_graphs.S}[graphs]}.
28
28
*)
29
29
30
30
val iri_for_resource : 'a T.resource -> Forester_core__.Base.iri option
-2
lib/compiler/Forest_graph.ml
lib/core/Forest_graph.ml
-2
lib/compiler/Forest_graph.ml
lib/core/Forest_graph.ml
-2
lib/compiler/Forest_graph.mli
lib/core/Forest_graph.mli
-2
lib/compiler/Forest_graph.mli
lib/core/Forest_graph.mli
-1
lib/compiler/Forest_graphs.ml
lib/core/Forest_graphs.ml
-1
lib/compiler/Forest_graphs.ml
lib/core/Forest_graphs.ml
+1
-3
lib/compiler/Forest_graphs.mli
lib/core/Forest_graphs.mli
+1
-3
lib/compiler/Forest_graphs.mli
lib/core/Forest_graphs.mli
···
4
4
* SPDX-License-Identifier: GPL-3.0-or-later
5
5
*)
6
6
7
-
open Forester_core
7
+
(** A simple graph database. Used to record the {{!Forester_core.Builtin_relation}relations} that exist between trees.*)
8
8
9
9
module type S = sig
10
-
(** A simple graph database. Used to record the {{!Forester_core.Builtin_relation}relations} that exist between trees.*)
11
-
12
10
val dl_db : Datalog_engine.db
13
11
val register_iri : Iri.t -> unit
14
12
val get_all_vertices : unit -> Vertex_set.t
+38
-17
lib/compiler/Forester_compiler.ml
+38
-17
lib/compiler/Forester_compiler.ml
···
4
4
* SPDX-License-Identifier: GPL-3.0-or-later
5
5
*)
6
6
7
-
(** This library implements the compiler for Forester.
8
-
*)
7
+
(** This library implements the compiler for Forester. *)
8
+
9
+
(** {1 Base types }*)
9
10
10
11
module Code = Code
11
12
(** Abstract syntax *)
···
14
15
15
16
module Symbol = Symbol
16
17
17
-
(***)
18
+
module Xml_forester = Xml_forester
19
+
(** Definition of the forester XML schema. This is the compilation target.*)
20
+
21
+
(** {1 Compilation phases}*)
22
+
23
+
(** {2 Parsing}
24
+
25
+
The lexer and parser are implemented with {{: https://ocaml.org/manual/5.3/lexyacc.html} ocamllex} and {{: https://gallium.inria.fr/~fpottier/menhir/} menhir}*)
26
+
27
+
module Lexer = Lexer
28
+
29
+
module Parse = Parse
18
30
19
31
module Resolver = Resolver
32
+
33
+
module Imports = Imports
34
+
(** Create {{!Forester_core.Forest_graph.t}import and dependency graphs}.
35
+
*)
36
+
20
37
module Expand = Expand
38
+
(** Transform {!Code.tree}s into {!Syn.tree}s by {{!Forester_core.Forest_graph.topo_fold}folding} over the {{!Forester_core.Forest_graph}import graph.}*)
39
+
21
40
module Eval = Eval
22
-
module Parse = Parse
23
-
module Lexer = Lexer
41
+
(** Transform {!Syn.tree}s into {{!Forester_core.Types.article}[articles]}.*)
42
+
43
+
module Legacy_query_engine = Legacy_query_engine
44
+
(* [@@ocaml.deprecated "Use Datalog_query_engine instead"] *)
24
45
25
-
module Xml_forester = Xml_forester
26
-
(** Definition of the forester XML schema*)
46
+
(** {1 High-level architecture}
27
47
28
-
module Forest_graph = Forest_graph
29
-
(** The graph type used by the datalog database *)
48
+
The compiler needs to support both batch-style and incremental compilation. To this end, we define a {{!State.t}state type} and {{!Phases}transition functions} that act on this state.
30
49
31
-
module Forest_graphs = Forest_graphs
50
+
In the future, we want to record more knowledge in {{!field:State.graphs}[graphs]} of the {{!State.t}state} and derive the information we need for the language server via the query system.
32
51
33
-
module Imports = Imports
34
-
(** Analysis of the abstract syntax to create import and dependency graphs *)
52
+
*)
35
53
36
54
module Config = Config
37
55
(** Configuration*)
38
56
39
57
module Forest = Forest
40
-
(**)
58
+
(** Augmented hash table used throughout compilation phases.*)
41
59
42
60
module State = State
43
61
module Phases = Phases
44
62
module Asset_router = Asset_router
45
-
module Legacy_query_engine = Legacy_query_engine
46
-
(* [@@ocaml.deprecated "Use Datalog_query_engine instead"] *)
47
63
48
64
module Iri_resolver = Iri_resolver
65
+
(** Maps URIs and IRIs to paths, documents or code*)
66
+
49
67
module Iri_util = Iri_util
50
68
51
69
module Diagnostic_store = Diagnostic_store
52
70
53
-
module Job = Job
54
-
(** Definition of LaTeX jobs*)
71
+
(** {1 IO}*)
72
+
73
+
(** {2 LaTeX pipeline}*)
55
74
56
75
module LaTeX_pipeline = LaTeX_pipeline
57
76
module LaTeX_template = LaTeX_template
77
+
module Job = Job
78
+
(** Definition of LaTeX jobs*)
58
79
59
80
module Export_for_test = Export_for_test
+5
-4
lib/compiler/Phases.ml
+5
-4
lib/compiler/Phases.ml
···
8
8
open Forester_prelude
9
9
open Forester_core
10
10
11
+
(**/**)
11
12
module T = Types
12
-
module EP = Eio.Path
13
13
14
14
type state = State.t
15
+
(**/**)
15
16
16
17
type transition = state -> state
17
18
···
40
41
}
41
42
42
43
let load
43
-
: Eio.Fs.dir_ty EP.t list -> transition
44
+
: Eio.Fs.dir_ty Eio.Path.t list -> transition
44
45
= fun tree_dirs forest ->
45
46
Logs.debug (fun m -> m "loading trees from file system");
46
47
tree_dirs
···
51
52
(* WARNING: using realpath. This is necessary because later on,
52
53
the lsp client will send notifications using the absolute path of
53
54
a file.*)
54
-
let source_path = EP.native_exn path |> Unix.realpath in
55
-
let content = EP.load path in
55
+
let source_path = Eio.Path.native_exn path |> Unix.realpath in
56
+
let content = Eio.Path.load path in
56
57
let uri = source_path |> Lsp.Uri.of_path in
57
58
let tree =
58
59
Lsp.Text_document.make
+20
lib/compiler/Phases.mli
+20
lib/compiler/Phases.mli
···
1
+
(*
2
+
* SPDX-FileCopyrightText: 2024 The Forester Project Contributors
3
+
*
4
+
* SPDX-License-Identifier: GPL-3.0-or-later
5
+
*
6
+
*)
7
+
8
+
type transition = State.t -> State.t
9
+
10
+
val init : env: Eio_unix.Stdenv.base -> config: Config.t -> dev: bool -> State.t
11
+
val load : Eio.Fs.dir_ty Eio.Path.t list -> transition
12
+
val load_configured_dirs : transition
13
+
val parse : quit_on_error: bool -> transition
14
+
val reparse : Lsp.Text_document.t -> transition
15
+
val build_import_graph : transition
16
+
val build_import_graph_for : addr: Forester_core.iri -> transition
17
+
val expand : quit_on_error: bool -> transition
18
+
val expand_only : Forester_core.iri -> transition
19
+
val eval : dev: bool -> transition
20
+
val eval_only : Forester_core.iri -> transition
+34
-6
lib/core/Forester_core.ml
+34
-6
lib/core/Forester_core.ml
···
4
4
* SPDX-License-Identifier: GPL-3.0-or-later
5
5
*)
6
6
7
+
(** Core types
8
+
*)
9
+
7
10
include Base
8
11
9
12
module Iri_scheme = Iri_scheme
10
13
14
+
(** {1 Vertices}
15
+
16
+
The type of vertices used by the {{!Forester_core.Forest_graphs}graph database}
17
+
18
+
*)
19
+
11
20
module Vertex = Vertex
12
21
module Vertex_set = Vertex_set
22
+
23
+
(** {1 Error handling}*)
24
+
25
+
(** Definition of compiler diagnostics and other errors.*)
26
+
module Reporter = Reporter
27
+
28
+
(** Source locations. Among other things these are used by compiler diagnostics and the language server.*)
29
+
module Range = Range
30
+
31
+
(** {1 Query system}
32
+
*)
13
33
14
34
module Query = Query
35
+
36
+
(** {2 Builtins}*)
15
37
module Builtin_relation = Builtin_relation
16
38
module Builtin_queries = Builtin_queries
17
39
18
-
module Reporter = Reporter
19
-
module Range = Range
40
+
module Forest_graph = Forest_graph
41
+
(** The graph type used by the datalog database. Used in particular to track imports, links and transclusions. *)
42
+
43
+
module Forest_graphs = Forest_graphs
44
+
(** The graph database*)
45
+
46
+
(** {2 Datalog bindings}*)
47
+
48
+
module Datalog_engine = Datalog_engine
49
+
module Datalog_expr = Datalog_expr
50
+
module Datalog_eval = Datalog_eval
51
+
20
52
module Prim = Prim
21
53
22
54
module TeX_cs = TeX_cs
···
24
56
module Types = Types
25
57
26
58
module Trie = Trie
27
-
28
-
module Datalog_engine = Datalog_engine
29
-
module Datalog_expr = Datalog_expr
30
-
module Datalog_eval = Datalog_eval
+2
lib/core/Iri_scheme.mli
+2
lib/core/Iri_scheme.mli
···
6
6
7
7
open Base
8
8
9
+
(** Forester uses an IRI scheme that incorporates a host that demarcates the current forest. A typical tree has have an address like this: [forest://host/xxx-NNNN]. Resources such as PDFs or images will be referred to by their hash.*)
10
+
9
11
val scheme : string
10
12
11
13
val base_iri :
+1
-3
lib/frontend/Forester_frontend.ml
+1
-3
lib/frontend/Forester_frontend.ml
···
4
4
* SPDX-License-Identifier: GPL-3.0-or-later
5
5
*)
6
6
7
-
(* module Eio_util = Eio_util *)
7
+
(** Bits and pieces*)
8
8
9
-
(* module Compiler = Compiler *)
10
9
module Config_parser = Config_parser
11
10
module Forester = Forester
12
11
module Render = Render
···
16
15
module Export_for_test = Export_for_test
17
16
module State_machine = State_machine
18
17
module Http_server = Http_server
19
-
(* module Compilation_phases = Compilation_phases *)
+3
-3
lib/language_server/Forester_lsp.mli
+3
-3
lib/language_server/Forester_lsp.mli
···
1
1
(*
2
-
* SPDX-FileCopyrightText: 2024 The Forester Project Contributors AND The RedPRL Development Team
2
+
* SPDX-FileCopyrightText: 2024 The Forester Project Contributors
3
3
*
4
4
* SPDX-License-Identifier: GPL-3.0-or-later OR Apache-2.0 WITH LLVM-exception
5
5
*
···
7
7
8
8
open Forester_compiler
9
9
10
-
(** An implementation of the {{: https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/} Microsoft Language Server Protocol } for forester. See {!Server.Handlers} for an overview of which methods are currently supported.*)
10
+
(** An implementation of the {{: https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/} Microsoft Language Server Protocol } for forester. *)
11
11
12
-
(** The compiler infrastructure used by the language server*)
12
+
(** Analysis of the {{!type:Forester_compiler.Code.t}concrete syntax}*)
13
13
module Analysis = Analysis
14
14
15
15
(** The event loop implementation and request handlers*)
-13
static.patch
-13
static.patch
···
1
-
diff --git a/bin/forester/dune b/bin/forester/dune
2
-
index 8267c08..32fb7fd 100644
3
-
--- a/bin/forester/dune
4
-
+++ b/bin/forester/dune
5
-
@@ -6,7 +6,7 @@
6
-
(public_name forester)
7
-
(name main)
8
-
(flags (-g))
9
-
- (link_flags (-g))
10
-
+ (link_flags (-g -ccopt -static))
11
-
(preprocess
12
-
(pps ppx_deriving.show))
13
-
(libraries