forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
feat: migrate content from jon.recoil.org-src to mono/site
Copy all .mld content (blog, notebooks, drafts, reference) and static
assets from the old site repository, applying format transformations:
- @ocamltop -> @ocaml (with autorun -> run-on=load, skip preserved)
- @libs (space-separated) -> @x-ocaml.requires (comma-separated)
- @switch and @notanotebook lines removed
- @published and @children_order tags preserved as-is
Includes the migration script at scripts/migrate_content.sh which also
flags files needing manual review for {x@ocaml[ deferred blocks and
# prompt prefixes inside code blocks.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+9290
+286
scripts/migrate_content.sh
+286
scripts/migrate_content.sh
···
1
+
#!/usr/bin/env bash
2
+
#
3
+
# migrate_content.sh
4
+
#
5
+
# Migrates content from the old jon.recoil.org-src site to the mono repo's
6
+
# site/ directory. Copies .mld files, static assets, and applies format
7
+
# transformations for the new odoc-based build system.
8
+
#
9
+
# Usage: ./scripts/migrate_content.sh
10
+
# Run from the mono repo root.
11
+
12
+
set -euo pipefail
13
+
14
+
# Resolve paths
15
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16
+
MONO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
17
+
OLD_SITE="/home/jons-agent/workspace/jon.recoil.org-src"
18
+
NEW_SITE="$MONO_ROOT/site"
19
+
20
+
if [ ! -d "$OLD_SITE" ]; then
21
+
echo "ERROR: Old site directory not found at $OLD_SITE"
22
+
exit 1
23
+
fi
24
+
25
+
echo "=== Content Migration Script ==="
26
+
echo " Source: $OLD_SITE"
27
+
echo " Destination: $NEW_SITE"
28
+
echo ""
29
+
30
+
# Track files needing manual review
31
+
REVIEW_FILES_OCAMLTOP=()
32
+
REVIEW_FILES_DEFERRED=()
33
+
REVIEW_FILES_PROMPT=()
34
+
35
+
# ============================================================
36
+
# Step 1: Copy content directories
37
+
# ============================================================
38
+
39
+
echo "--- Step 1: Copying content ---"
40
+
41
+
# 1a. Copy blog .mld files, preserving directory structure
42
+
echo " Copying blog/ ..."
43
+
if [ -d "$OLD_SITE/blog" ]; then
44
+
find "$OLD_SITE/blog" -name "*.mld" -type f | while read -r src; do
45
+
rel="${src#$OLD_SITE/}"
46
+
dest="$NEW_SITE/$rel"
47
+
mkdir -p "$(dirname "$dest")"
48
+
cp "$src" "$dest"
49
+
done
50
+
blog_count=$(find "$NEW_SITE/blog" -name "*.mld" -type f | wc -l)
51
+
echo " Copied $blog_count .mld files"
52
+
else
53
+
echo " WARNING: $OLD_SITE/blog does not exist"
54
+
fi
55
+
56
+
# 1b. Copy notebooks .mld files, preserving directory structure
57
+
echo " Copying notebooks/ ..."
58
+
if [ -d "$OLD_SITE/notebooks" ]; then
59
+
find "$OLD_SITE/notebooks" -name "*.mld" -type f | while read -r src; do
60
+
rel="${src#$OLD_SITE/}"
61
+
dest="$NEW_SITE/$rel"
62
+
mkdir -p "$(dirname "$dest")"
63
+
cp "$src" "$dest"
64
+
done
65
+
nb_count=$(find "$NEW_SITE/notebooks" -name "*.mld" -type f | wc -l)
66
+
echo " Copied $nb_count .mld files"
67
+
else
68
+
echo " WARNING: $OLD_SITE/notebooks does not exist"
69
+
fi
70
+
71
+
# 1c. Copy static assets
72
+
echo " Copying static/ ..."
73
+
if [ -d "$OLD_SITE/static" ]; then
74
+
cp -r "$OLD_SITE/static/." "$NEW_SITE/static/"
75
+
static_count=$(find "$NEW_SITE/static" -type f | wc -l)
76
+
echo " Copied $static_count static files"
77
+
else
78
+
echo " WARNING: $OLD_SITE/static does not exist"
79
+
fi
80
+
81
+
# 1d. Copy notes/ if it exists
82
+
echo " Copying notes/ ..."
83
+
if [ -d "$OLD_SITE/notes" ] && [ -n "$(ls -A "$OLD_SITE/notes" 2>/dev/null)" ]; then
84
+
mkdir -p "$NEW_SITE/notes"
85
+
find "$OLD_SITE/notes" -name "*.mld" -type f | while read -r src; do
86
+
rel="${src#$OLD_SITE/}"
87
+
dest="$NEW_SITE/$rel"
88
+
mkdir -p "$(dirname "$dest")"
89
+
cp "$src" "$dest"
90
+
done
91
+
notes_count=$(find "$NEW_SITE/notes" -name "*.mld" -type f 2>/dev/null | wc -l)
92
+
echo " Copied $notes_count .mld files"
93
+
else
94
+
echo " Skipped (directory empty or does not exist)"
95
+
fi
96
+
97
+
# 1e. Copy drafts/ if it exists
98
+
echo " Copying drafts/ ..."
99
+
if [ -d "$OLD_SITE/drafts" ] && [ -n "$(ls -A "$OLD_SITE/drafts" 2>/dev/null)" ]; then
100
+
mkdir -p "$NEW_SITE/drafts"
101
+
find "$OLD_SITE/drafts" -name "*.mld" -type f | while read -r src; do
102
+
rel="${src#$OLD_SITE/}"
103
+
dest="$NEW_SITE/$rel"
104
+
mkdir -p "$(dirname "$dest")"
105
+
cp "$src" "$dest"
106
+
done
107
+
drafts_count=$(find "$NEW_SITE/drafts" -name "*.mld" -type f 2>/dev/null | wc -l)
108
+
echo " Copied $drafts_count .mld files"
109
+
else
110
+
echo " Skipped (directory empty or does not exist)"
111
+
fi
112
+
113
+
# 1f. Copy reference/ .mld files only (not build output)
114
+
echo " Copying reference/ ..."
115
+
if [ -d "$OLD_SITE/reference" ]; then
116
+
mkdir -p "$NEW_SITE/reference"
117
+
find "$OLD_SITE/reference" -name "*.mld" -type f | while read -r src; do
118
+
rel="${src#$OLD_SITE/}"
119
+
dest="$NEW_SITE/$rel"
120
+
mkdir -p "$(dirname "$dest")"
121
+
cp "$src" "$dest"
122
+
done
123
+
ref_count=$(find "$NEW_SITE/reference" -name "*.mld" -type f 2>/dev/null | wc -l)
124
+
echo " Copied $ref_count .mld files"
125
+
else
126
+
echo " Skipped (directory does not exist)"
127
+
fi
128
+
129
+
echo ""
130
+
131
+
# ============================================================
132
+
# Step 2: Apply transformations to all .mld files
133
+
# ============================================================
134
+
135
+
echo "--- Step 2: Applying transformations ---"
136
+
137
+
transform_count=0
138
+
libs_count=0
139
+
switch_count=0
140
+
notanotebook_count=0
141
+
autorun_count=0
142
+
ocamltop_plain_count=0
143
+
144
+
find "$NEW_SITE" -name "*.mld" -type f | sort | while read -r file; do
145
+
modified=false
146
+
147
+
# 2a. Replace {@ocamltop autorun[ with {@ocaml run-on=load[
148
+
# This handles variants like: {@ocamltop autorun hidden[
149
+
if grep -q '{@ocamltop autorun' "$file" 2>/dev/null; then
150
+
# Replace {@ocamltop autorun[ with {@ocaml run-on=load[
151
+
# Also handle {@ocamltop autorun hidden[ -> {@ocaml run-on=load hidden[
152
+
sed -i 's/{@ocamltop autorun\b/{@ocaml run-on=load/g' "$file"
153
+
modified=true
154
+
autorun_count=$((autorun_count + 1))
155
+
fi
156
+
157
+
# Also handle {x@ocamltop autorun[ -> {x@ocaml run-on=load[
158
+
if grep -q '{x@ocamltop autorun' "$file" 2>/dev/null; then
159
+
sed -i 's/{x@ocamltop autorun\b/{x@ocaml run-on=load/g' "$file"
160
+
modified=true
161
+
fi
162
+
163
+
# Also handle {@ocaml autorun -> {@ocaml run-on=load (non-ocamltop variant)
164
+
if grep -q '{@ocaml autorun' "$file" 2>/dev/null; then
165
+
sed -i 's/{@ocaml autorun\b/{@ocaml run-on=load/g' "$file"
166
+
modified=true
167
+
fi
168
+
169
+
# 2b. Replace {@ocamltop skip[ with {@ocaml skip[
170
+
if grep -q '{@ocamltop skip' "$file" 2>/dev/null; then
171
+
sed -i 's/{@ocamltop skip\b/{@ocaml skip/g' "$file"
172
+
modified=true
173
+
fi
174
+
175
+
# 2c. Replace remaining {@ocamltop[ with {@ocaml[
176
+
if grep -q '{@ocamltop\[' "$file" 2>/dev/null; then
177
+
sed -i 's/{@ocamltop\[/{@ocaml[/g' "$file"
178
+
modified=true
179
+
ocamltop_plain_count=$((ocamltop_plain_count + 1))
180
+
fi
181
+
182
+
# Also handle {x@ocamltop[ -> {x@ocaml[
183
+
if grep -q '{x@ocamltop\[' "$file" 2>/dev/null; then
184
+
sed -i 's/{x@ocamltop\[/{x@ocaml[/g' "$file"
185
+
modified=true
186
+
fi
187
+
188
+
# 2d. Transform @libs lines: space-separated to comma-separated @x-ocaml.requires
189
+
# e.g. @libs pkg1 pkg2 pkg3 -> @x-ocaml.requires pkg1,pkg2,pkg3
190
+
if grep -q '^@libs ' "$file" 2>/dev/null; then
191
+
sed -i '/^@libs /{s/^@libs //; s/ /,/g; s/^/@x-ocaml.requires /}' "$file"
192
+
modified=true
193
+
libs_count=$((libs_count + 1))
194
+
fi
195
+
196
+
# 2e. Remove @switch oxcaml lines entirely
197
+
if grep -q '^@switch ' "$file" 2>/dev/null; then
198
+
sed -i '/^@switch /d' "$file"
199
+
modified=true
200
+
switch_count=$((switch_count + 1))
201
+
fi
202
+
203
+
# 2f. Remove @notanotebook lines entirely
204
+
if grep -q '^@notanotebook' "$file" 2>/dev/null; then
205
+
sed -i '/^@notanotebook$/d' "$file"
206
+
modified=true
207
+
notanotebook_count=$((notanotebook_count + 1))
208
+
fi
209
+
210
+
if [ "$modified" = true ]; then
211
+
transform_count=$((transform_count + 1))
212
+
fi
213
+
done
214
+
215
+
echo " Files transformed: see summary below"
216
+
echo ""
217
+
218
+
# ============================================================
219
+
# Step 3: Flag files for manual review
220
+
# ============================================================
221
+
222
+
echo "--- Step 3: Checking for files needing manual review ---"
223
+
echo ""
224
+
225
+
# 3a. Check for remaining ocamltop (incomplete transformation)
226
+
echo " [CHECK] Files still containing 'ocamltop' (incomplete transformation):"
227
+
remaining_ocamltop=0
228
+
while IFS= read -r file; do
229
+
rel="${file#$NEW_SITE/}"
230
+
echo " - $rel"
231
+
remaining_ocamltop=$((remaining_ocamltop + 1))
232
+
done < <(grep -rl 'ocamltop' "$NEW_SITE" --include="*.mld" 2>/dev/null || true)
233
+
if [ "$remaining_ocamltop" -eq 0 ]; then
234
+
echo " (none -- all ocamltop references transformed)"
235
+
fi
236
+
echo ""
237
+
238
+
# 3b. Check for {x@ocaml[ (deferred blocks needing manual handling)
239
+
echo " [CHECK] Files containing '{x@ocaml[' (deferred blocks, need manual review):"
240
+
deferred_count=0
241
+
while IFS= read -r file; do
242
+
rel="${file#$NEW_SITE/}"
243
+
echo " - $rel"
244
+
deferred_count=$((deferred_count + 1))
245
+
done < <(grep -rl '{x@ocaml\[' "$NEW_SITE" --include="*.mld" 2>/dev/null || true)
246
+
if [ "$deferred_count" -eq 0 ]; then
247
+
echo " (none)"
248
+
fi
249
+
echo ""
250
+
251
+
# 3c. Check for lines starting with # inside code blocks (old prompt prefixes)
252
+
echo " [CHECK] Files with '# ' prompt prefixes inside code blocks (need manual review):"
253
+
prompt_count=0
254
+
while IFS= read -r file; do
255
+
rel="${file#$NEW_SITE/}"
256
+
echo " - $rel"
257
+
prompt_count=$((prompt_count + 1))
258
+
done < <(grep -rl '^# ' "$NEW_SITE" --include="*.mld" 2>/dev/null || true)
259
+
if [ "$prompt_count" -eq 0 ]; then
260
+
echo " (none)"
261
+
fi
262
+
echo ""
263
+
264
+
# ============================================================
265
+
# Summary
266
+
# ============================================================
267
+
268
+
echo "=== Migration Summary ==="
269
+
total_mld=$(find "$NEW_SITE" -name "*.mld" -type f | wc -l)
270
+
total_static=$(find "$NEW_SITE/static" -type f 2>/dev/null | wc -l)
271
+
echo " Total .mld files migrated: $total_mld"
272
+
echo " Total static files copied: $total_static"
273
+
echo ""
274
+
echo " Transformations applied:"
275
+
echo " @ocamltop autorun -> @ocaml run-on=load"
276
+
echo " @ocamltop[ -> @ocaml["
277
+
echo " @libs -> @x-ocaml.requires (comma-separated)"
278
+
echo " @switch lines removed"
279
+
echo " @notanotebook lines removed"
280
+
echo ""
281
+
echo " Files needing manual review: $((remaining_ocamltop + deferred_count + prompt_count))"
282
+
echo " - Remaining ocamltop: $remaining_ocamltop"
283
+
echo " - Deferred blocks ({x@ocaml[): $deferred_count"
284
+
echo " - Prompt prefixes (# ): $prompt_count"
285
+
echo ""
286
+
echo "=== Done ==="
+29
site/blog/2025/03/code-block-metadata.mld
+29
site/blog/2025/03/code-block-metadata.mld
···
1
+
{0 Code block metadata}
2
+
3
+
@published 2025-03-07
4
+
5
+
Back in 2021 {{:https://github.com/julow}julow} introduced some {{:https://github.com/ocaml-doc/odoc-parser/pull/2}new syntax} to odoc’s code blocks to allow us to attach arbitrary metadata to the blocks. We imposed no structure on this; it was simply a block of text in between the language tag and the start of the code block. Now odoc needs to use it itself, we need to be a bit more precise about how it’s defined.
6
+
7
+
The original concept looked like this:
8
+
9
+
{v
10
+
{@ocaml metadata goes here in an unstructured way[
11
+
... code ...
12
+
]}
13
+
v}
14
+
15
+
where everything in between the language (“ocaml” in this case) and the opening square bracket would be captured and put into the AST verbatim. Odoc itself has had no particular use for this, but it has been used in {{:https://github.com/realworldocaml/mdx}mdx} to control how it handles the code blocks, for example to skip processing of the block, to synchronise the block with another file, to disable testing the block on particular OSs and so on.
16
+
17
+
As part of the Odoc 3 release we decided to address one of our {{:https://github.com/ocaml/odoc/pull/303}oldest open issues}, that of extracting code blocks from mli/mld files for inclusion into other files. This is similar to the file-sync facility in mdx but it works in the other direction: the canonical source is in the mld/mli file. In order to do this, we now need to use the metadata so we can select which code blocks to extract, and so we needed a more concrete specification of how the metadata should be parsed.
18
+
19
+
We looked at what {{:https://github.com/realworldocaml/mdx/blob/main/lib/label.ml#L195-L210}mdx does}, but the way it works is rather ad-hoc, using very simple String.splits to chop up the metadata. This is OK for mdx as it’s fully in charge of what things the user might want to put into the metadata, but for a general parsing library like odoc.parser we need to be a bit more careful. Daniel Bünzli {{:https://github.com/ocaml/odoc/pull/1326#issuecomment-2702260053}suggested} a simple strategy of atoms and bindings inspired by s-expressions. The idea is that we can have something like this:
20
+
21
+
{v
22
+
{@ocaml atom1 "atom two" key1=value1 "key 2"="value with spaces"[
23
+
... code content ...
24
+
]}
25
+
v}
26
+
27
+
Daniel suggested a very minimal escaping rule, whereby a string could contain a literal " by prefixing with a backslash - something like; "value with a \" and spaces", but we discussed it during the {{:https://ocaml.org/governance/platform}odoc developer meeting} and felt that we might want something a little more familiar. So we took a look at the lexer in {{:https://github.com/janestreet/sexplib/blob/master/src/lexer.mll}sexplib} and found that it follows the {{:https://github.com/janestreet/sexplib/blob/d7c5e3adc16fcf0435220c3cd44bb695775020c1/README.org#lexical-conventions-of-s-expression}lexical conventions} of OCaml’s strings, and decided that would be a reasonable approach for us to follow too.
28
+
29
+
The resulting code, including the extraction logic, was implemented in {{:https://github.com/ocaml/odoc/pull/1326/}PR 1326} mainly by {{:https://github.com/panglesd}panglesd} with a little help from me on the lexer.
+4
site/blog/2025/03/index.mld
+4
site/blog/2025/03/index.mld
+134
site/blog/2025/03/module-type-of.mld
+134
site/blog/2025/03/module-type-of.mld
···
1
+
{0 The Road to Odoc 3: Module Type Of}
2
+
3
+
@published 2025-03-08
4
+
5
+
There are {{:https://discuss.ocaml.org/t/ann-odoc-3-beta-release/16043}many new and improved features}
6
+
that Odoc 3 brings, but there are also a large number of bugfixes. I thought I'd write about one in
7
+
particular here, an {{:https://github.com/ocaml/odoc/pull/1081}overhaul of "module type of"} that
8
+
landed in May 2024.
9
+
10
+
{1 Module Type Of}
11
+
module type of is a language feature of OCaml allowing one to recover the signature of an existing
12
+
module. For example, if I had a module [X]:
13
+
14
+
{@ocaml[
15
+
# module X = struct
16
+
type t = Foo | Bar
17
+
end;;
18
+
module X : sig type t = Foo | Bar end
19
+
]}
20
+
21
+
then I can get back the signature of [X] using [module type of]:
22
+
23
+
{@ocaml[
24
+
# module type Xsig = module type of X;;
25
+
module type Xsig = sig type t = Foo | Bar end
26
+
]}
27
+
28
+
which can be very useful if you’re trying to
29
+
{{:https://discuss.ocaml.org/t/extend-existing-module/1389}extend existing modules} amongst other things.
30
+
31
+
OCaml and Odoc treat module type of in somewhat different ways. OCaml internally expands the expression
32
+
immediately it sees it, and effectively replaces it with the signature - ie, in the above example Xsig
33
+
is now a signature, not a module type of expression.
34
+
35
+
In contrast, Odoc would like to keep track of the fact that this signature came from a [module type of]
36
+
expression, as it’s very useful to know. If you’re extending a module, your signature might look like:
37
+
38
+
{@ocaml[
39
+
# module type UnitExtended = sig
40
+
include module type of Unit
41
+
val extra_unit_function : unit -> unit
42
+
end;;
43
+
module type UnitExtended =
44
+
sig
45
+
type t = unit = ()
46
+
val equal : t -> t -> bool
47
+
val compare : t -> t -> int
48
+
val to_string : t -> string
49
+
val extra_unit_function : unit -> unit
50
+
end
51
+
]}
52
+
53
+
The documentation we produce will expand the contents of the [include] statement, but keep track of the
54
+
fact that it came from a [module type of] expression so the reader can see where these signature items came
55
+
from. In practice, you'd probably want to use [module type of struct include Unit end], which is a bit
56
+
different from simply [module type of Unit], and I'll talk about this at some point in a future post.
57
+
58
+
{1 The problem}
59
+
We run into difficulties as soon as we introduce another language feature that operates on signatures:
60
+
with. Let’s start with a module type [S]:
61
+
62
+
{@ocaml[
63
+
# module type S = sig
64
+
module X : sig
65
+
type t = int
66
+
end
67
+
68
+
module type Y =
69
+
module type of X
70
+
end;;
71
+
module type S =
72
+
sig
73
+
module X : sig type t = int end
74
+
module type Y = sig type t = int end
75
+
end
76
+
]}
77
+
78
+
We’ll now define a new module [X2] that we intend to use as a replacement for [X]:
79
+
80
+
{@ocaml[
81
+
# module X2 = struct
82
+
type t = int
83
+
type u = float
84
+
end;;
85
+
module X2 : sig type t = int type u = float end
86
+
]}
87
+
88
+
Now we’ll define a new module type [T] which is [S] but with [X] replaced:
89
+
90
+
{@ocaml[
91
+
# module type T = S with module X := X2;;
92
+
module type T = sig module type Y = sig type t = int end end
93
+
]}
94
+
95
+
Here you can see that OCaml has expanded the [module type of] expressions and told us the computed signature.
96
+
The interesting thing here is that in module type [T], module type [Y] only has a type [t] in it, not a
97
+
type [u]. As above, Odoc wants to keep the [module type of] expression so the reader can tell where module
98
+
type [Y] came from. However, the substitution would do a different thing in this case - we would have
99
+
the following:
100
+
101
+
{@ocaml[
102
+
# module type T = sig
103
+
module type Y = module type of X2
104
+
end;;
105
+
module type T = sig module type Y = sig type t = int type u = float end end
106
+
]}
107
+
108
+
and the expansion of this would then clearly have both types [t] and [u] in it.
109
+
110
+
So now Odoc has two problems: We need to compute the correct signature, and we need to be able to
111
+
describe how we computed it.
112
+
113
+
{1 The solution}
114
+
The previous solution to this was to have a ‘phase 0’ of odoc which would compute the expansions
115
+
of all module type of expressions before doing any other work. This was necessary because of a
116
+
‘simplfying’ assumption in how we handled the typing environment. The new, simpler approach was
117
+
to calculate the expansion during the normal flow of work, and never to attempt to recalculate
118
+
it, but simply operate on the signature. This was a nice big simplification and optimisation that
119
+
removed a few corner cases in the previous code (including an
120
+
{{:https://github.com/ocaml/odoc/blob/v2.4/src/xref2/type_of.ml#L167-L174}infinite loop} that we
121
+
{e hoped} always terminated…!)
122
+
123
+
The second issue was how to describe it. We still want it clear that this signature was derived
124
+
from another, but it’s clear we can’t honestly say that in the above example that it’s [module type
125
+
of X2]. The answer is that we have applied a transparent ascription to the signature. Essentially,
126
+
the signature is [X2] but constrained to only have the fields of [X].
127
+
128
+
This is not a current feature of OCaml, though Jane Street has
129
+
{{:https://blog.janestreet.com/plans-for-ocaml-408/}done some work} on this, including declaring
130
+
the syntax: [X2 <: X]. However, there’s another interesting wrinkle here. [X] is a module defined in
131
+
the module type [S], so it’s not possible to write a valid OCaml path that points to it – [S.X] has no
132
+
meaning. In addition, the right-hand side of the [<:] operator should be a module type, so we’d
133
+
actually need to write [X2 <: module type of S.X] . We’re still figuring out the right thing to do
134
+
here, so for now Odoc 3 will still pretend that it’s simply [module type of X2].
+4
site/blog/2025/04/index.mld
+4
site/blog/2025/04/index.mld
+42
site/blog/2025/04/meeting-the-team.mld
+42
site/blog/2025/04/meeting-the-team.mld
···
1
+
{0 Meeting the Team}
2
+
3
+
@published 2025-04-08
4
+
5
+
It's tremendously exciting to be back in the {{:https://www.cst.cam.ac.uk/}Computer Laboratory},
6
+
as the last time I worked here was just before the pandemic. I'm now a member of the {{:https://www.cst.cam.ac.uk/research/eeg}Energy and Environment Group}
7
+
whose goal is "to have a measurable impact on tools and techniques for de-risking the future".
8
+
9
+
{1 What's going on?}
10
+
With such a broad goal, it's hard to know where to start and how I'll fit in, so my first few weeks have been spent
11
+
getting to know the other members of the group and what they're up to. It's an incredibly inspiring group of
12
+
individuals who are all doing amazing work, and it's really humbling and daunting to be a part of it.
13
+
14
+
There's some really interesting work going on in our group on LLMs, principally led by the fantastic
15
+
{{:https://toao.com/}Sadiq Jaffer}. We had a chat a few weeks ago and have started to explore some ideas around seeing how well LLMs
16
+
can program in OCaml already before we start to do some RL training on them. Having not done any LLM stuff
17
+
before, it's a steep learning curve for me, but we're already seeing some interesting results. We should have
18
+
some more to say about this in the coming weeks.
19
+
20
+
Last week I met with {{:https://digitalflapjack.com/}Michael Dales}, and he talked about the project
21
+
{{:https://github.com/quantifyearth/shark}shark} that he and {{:patrick.sirref.org}Patrick Ferris} have been working on.
22
+
It's kind of a mix between a shell and a jupyter-style notebook, with a strong focus on reproducibility.
23
+
The traditional pain of notebooks is, of course, the execution model, whereby cells might be executed in any
24
+
order you like. This means that the state you find the notebook in might not be even reachable again, let alone
25
+
consistently reproducible. Shark is trying to address this by using file-system snapshots and clever analysis
26
+
of the inputs and outputs of each cell to both ensure reproducibility, but also to allow a fast editing cycle,
27
+
rerunning of only the bits that need to be rerun, even in the presence of slow data processing steps. It's a
28
+
fascinating project, and I can't wait to see it in action when Michael gives us a demo!
29
+
30
+
I also met up with {{:https://ryan.freumh.org}Ryan Gibb} with {{:https://www.dra27.uk/blog/}David Allsopp}
31
+
and we had a good chat about his project {{:https://github.com/RyanGibb/babel}Babel}, which is using the
32
+
{{:https://nex3.medium.com/pubgrub-2fb6470504f}PubGrub} algorithm to do package resolution for multiple
33
+
package domains at once. We've got a number of avenues to explore here, from building a PubGrub implementation
34
+
in OxCaml, to using Babel to construct Docker images for opam packages entirely from scratch, without using
35
+
a base image.
36
+
37
+
With my other hat on as a member of the CTO office at {{:https://tarides.com/}Tarides}, I'm very much looking
38
+
forward to using OCaml and OxCaml to solve some real-world problems that are in an entirely different domain than I've
39
+
been used to over the last few years.
40
+
41
+
42
+
+77
site/blog/2025/04/ocaml-docs-ci-and-odoc-3.mld
+77
site/blog/2025/04/ocaml-docs-ci-and-odoc-3.mld
···
1
+
{0 OCaml-Docs-CI and Odoc 3}
2
+
3
+
@published 2025-04-29
4
+
5
+
The release of Odoc 3 means that we need to update the {{!https://docs.ci.ocaml.org}docs-ci}
6
+
project so that the documentation that appears on {{!https://ocaml.org/p/}ocaml.org}
7
+
is using the latest, greatest Odoc. With this major release of Odoc, it's also
8
+
time to give the CI pipeline a bit of an overhaul too, and fix some of the
9
+
irritations that it causes.
10
+
11
+
{1 The challenge of documenting OCaml}
12
+
13
+
As I wrote about {{!/site/blog/2025/04/page-"semantic-versioning-is-hard"}recently}, the
14
+
APIs of OCaml libraries are dependent not only on the version of its package, but possibly
15
+
also on the versions of any of its dependencies. Due to this fact, to produce the docs for
16
+
ocaml.org means that sometimes we need to build the docs for a particular version of a
17
+
particular package multiple times with different versions of its dependencies.
18
+
19
+
It's clearly impractical to try to build every possible combination, so what we do is to
20
+
run the opam solver once for each version of each package. This gives us a set of packages
21
+
at particular versions. We then take that, and for each package in the set, we pluck out {i
22
+
its} dependencies from the set, producing a "universe" of dependencies for every package
23
+
in the set. Let's look at a very simple example; the package [cry] from the
24
+
{{:https://www.liquidsoap.info}LiquidSoap} project.
25
+
26
+
The oldest version of [cry] from before the {{:https://discuss.ocaml.org/t/opam-repository-archival-phase-1-unavailable-packages/15797/6}Great Purge} was
27
+
0.2.2, which when solved produced the following dependencies:
28
+
29
+
{v
30
+
cry.0.2.2
31
+
ocaml.4.05.0
32
+
ocaml-base-compiler.4.05.0
33
+
ocaml-config.1
34
+
ocamlfind.1.9.6
35
+
v}
36
+
37
+
and the oldest version of [cry] after the purge is 0.6.0 which produces the following
38
+
solution:
39
+
40
+
{v
41
+
cry.0.6.0
42
+
ocaml.5.2.1
43
+
ocaml-base-compiler.5.2.1
44
+
ocaml-config.3
45
+
ocamlfind.1.9.6
46
+
v}
47
+
48
+
so we we can see from these two solutions that we'll need to build [ocamlfind.1.9.6] twice,
49
+
once with [ocaml.4.05.0] and once with [ocaml.5.2.1].
50
+
51
+
Once we've got, for every version of every package, a set of dependency universes, we choose
52
+
one of these to be the one presented to the user under the [ocaml.org/p/] hierarchy. For all
53
+
of the other universes, we build the package againt them, and put the docs under the [ocaml.org/u/]
54
+
hierarchy.
55
+
56
+
{1 Performing the builds}
57
+
Once we've got a complete set of solutions and builds to do, the current CI pipeline batches
58
+
the builds up to try and build as many packages as possible in as few builds as possible.
59
+
While this works well enough, it does mean that we build a lot packages more than once -
60
+
dune, for example, is built thousands of times during this process, producing exactly the same
61
+
binaries each time.
62
+
63
+
In the new pipeline, I wrote a {{:https://github.com/jonludlam/opamh}little tool} that allows
64
+
opam packages to be archived and restored, which happens to work nicely because we're always
65
+
building the packages in the same container in the same location. This means there are no
66
+
worries about relocatability, although that is something that is {{:https://www.dra27.uk/blog/platform/2025/04/22/branching-out.html}nearly here!}
67
+
68
+
The downside to this is that our storage requirements are quite a bit larger, as we're storing
69
+
the entire package rather than just the bits that odoc needs. However, we were always going to
70
+
use more storage than before simply because the new [odoc] and [odoc_driver] pair are
71
+
more capable, and the new features like {{:https://github.com/ocaml/odoc/pull/909}source code rendering}
72
+
and {{:https://github.com/ocaml/odoc/pull/1121/files#diff-10c8829023814c0bcc3316f95f643623404c000b13c68849ef3d61097a6e03a6R1-R415}classify}
73
+
require more files from the original package.
74
+
75
+
The upshot is that I'll be working with {{:https://www.tunbury.org/}Mark Elvers} to move the
76
+
docs CI from its current machine to a shiny new {{:https://www.tunbury.org/blade-reallocation/}blade server}.
77
+
+103
site/blog/2025/04/odoc-3.mld
+103
site/blog/2025/04/odoc-3.mld
···
1
+
{0 Odoc 3: So what?}
2
+
3
+
@published 2025-04-25
4
+
5
+
Odoc 3 was {{:https://discuss.ocaml.org/t/ann-odoc-3-0-released/16339}released last month}
6
+
and although we did write a list of the new features, I don't think we've made
7
+
it clear enough why anyone should care.
8
+
9
+
It's {b manuals}, the theme of Odoc 3 is {b manuals}. It's got a load of
10
+
features to make it much better for writing [mld] pages (files written using
11
+
odoc's markup) to document your packages and their relationship to the
12
+
surrounding ecosystem. Previous versions of Odoc were very library-centric,
13
+
in that while we did have mld-file support, most of the effort went into
14
+
making sure that we were generating correct per-module pages, which show the
15
+
shape of your API even if you've not put in any doc comments at all. We've
16
+
still got that, obviously, but we've added many features to make write [mld]
17
+
pages far more useful, and we're really hoping that these will draw people in
18
+
to make documenting packages a much more enjoyable experience.
19
+
20
+
{1 Odoc's special skill: links!}
21
+
22
+
But why you might want to use Odoc at all for your package's manuals, rather
23
+
than, say, markdown, asciidoc, rst or any other similar language? The biggest
24
+
thing that Odoc brings, and has always brought, is {b reliable linking}. Just
25
+
write [{!Module.func}] and Odoc will check that the target exists and ensure
26
+
that the link goes to the correct place, no matter how complex the definition
27
+
of [Module] is or what the layout of the docs. We can link to almost all
28
+
elements of an OCaml library, from modules and types through to fields of
29
+
records, exceptions and extensions, and we have facilities for disambiguating,
30
+
so if you happen to have both a module [S] and a module type [S] you can easily
31
+
link to whichever you please.
32
+
33
+
In Odoc 2 though, these links were pretty limited - the only ones possible
34
+
were only those to docs and API elements (modules, types, values, etc) in your
35
+
own package, or to API elements in any libraries that your package depends on.
36
+
When writing API docs, which tend to be at the level of types and functions,
37
+
this wasn't a huge problem, but when considering manuals this turned out to be
38
+
a really limiting constraint. For example, in Odoc's own docs, we really want
39
+
to have a link to [odoc-driver], but since [odoc-driver] is a separate package
40
+
and depends upon [odoc], the only way to do that in Odoc 2.x would be to use an
41
+
HTML link. With Odoc 3, this constraint is gone, so you can {b link to any
42
+
other package or library}. The link to [odoc-driver] would look like
43
+
[{!/odoc-driver/page-index}], as can be seen in
44
+
{{: https://github.com/ocaml/odoc/blob/master/doc/driver.mld#L10} odoc's source}.
45
+
The only requirement is that you must be able to simultaneously install all of
46
+
the packages you'd like to link to, so you can't easily link to, for example,
47
+
different versions of the same package.
48
+
49
+
This will be particularly useful for any projects that's grouped into multiple
50
+
packages. For example, the {{:https://mirage.io}Mirage project}. The main
51
+
package there -- [mirage] -- is actually right at the bottom of the dependency
52
+
hierarchy, but it's the perfect place to have docs that link to all of the
53
+
other Mirage packages. On a smaller scale, the
54
+
{{:https://github.com/ocaml-multicore/picos}Picos project} consists of multiple
55
+
packages all from a single git repository, and this would allow the docs pages
56
+
from the [picos] package to link to any of the other packages.
57
+
58
+
Of course there are also a lot of other new features in this release, which are
59
+
called out in the {{: https://discuss.ocaml.org/t/ann-odoc-3-beta-release/16043} annoucement post on discuss},
60
+
some of which I may post about in the future.
61
+
62
+
{1 Can I use it now?}
63
+
64
+
Of course! These new features can be used right now, so long as you're happy to
65
+
self-host the docs. All that's needed is to create a switch containing all the
66
+
packages you're interested in together, and use [odoc_driver] to generate the
67
+
HTML and push them to your web server. At time of writing though, ocaml.org is
68
+
still using Odoc 2.4, so any packages that are published to opam that choose
69
+
to use these new features will be missing the new features. Furthermore, it's
70
+
actually quite a challenge to do this, since we'll have to extend the
71
+
package-universe solutions to include all relevant packages, for which we need
72
+
extra fields in the opam metadata.
73
+
74
+
{1 What's next?}
75
+
76
+
We're actively working on getting Odoc 3 into the pipeline generating the docs
77
+
found in https://ocaml.org/p/. This will bring with it some of the developments
78
+
that landed in Odoc 2, but didn't make it onto ocaml.org - for example, the
79
+
rendering of source pages. Not only are there challenges related to the
80
+
package-universe solutions as mentioned above, but the storage requirements are
81
+
considerably larger, so I'll be working with
82
+
{{: https://tunbury.org/} Mark Elvers} to complete this project.
83
+
84
+
We've also got work to do to update the build rules in dune to take advantage
85
+
of these features. While [odoc_driver] works very well as part of the process
86
+
of deploying a docs site, it's quite impractical as a tool to help while you're
87
+
actually writing the docs. For that, we'll need to make sure [dune] understands
88
+
how to use these new features. Fortunately we've had some experience with those
89
+
rules in the past, and part of the work that's gone into Odoc 3 was to ensure
90
+
that incremental build rules should be far more straightforward to write than
91
+
for Odoc 2. In addition, some of the logic that previously only existed in
92
+
{{:https://github.com/ocaml-doc/voodoo}Voodoo} - the old driver that builds
93
+
docs for ocaml.org - has been integrated into [odoc] itself, meaning one again
94
+
that getting dune to produce correct docs for non-dune packages (e.g. the
95
+
standard library!) should again be simpler.
96
+
97
+
After we've done these, there are plans afoot to make more improvements to the
98
+
manual writing experience. {{: https://choum.net/} \@panglesd} has been
99
+
investigating how to add admonitions to the language, I've been thinking about
100
+
custom tag support, we're looking at the
101
+
{{:https://discuss.ocaml.org/t/ann-oxidizing-ocaml-an-update/15237}modes} work
102
+
coming from Jane Street to see how to support that. There's plenty more to do,
103
+
so if you'd like to lend a hand, reach out and join in!
+185
site/blog/2025/04/semantic-versioning-is-hard.mld
+185
site/blog/2025/04/semantic-versioning-is-hard.mld
···
1
+
{0 Semantic Versioning in OCaml is Hard}
2
+
3
+
@published 2025-04-20
4
+
5
+
{{: https://semver.org} Semantic versioning} is a lovely and simple idea that,
6
+
if it were reliably implemented everywhere, would make life a lot simpler. So,
7
+
is it possible to make our OCaml libraries stick to this scheme? There are some
8
+
projects that are trying to do this, including a recent {{:https://www.outreachy.org}Outreachy} project
9
+
by {{:https://github.com/azzsal/}Abdulaziz Alkurd} mentored by {{:https://choum.net}panglesd} and
10
+
{{:https://github.com/nathanreb}Nathan Reb}.
11
+
While this is a great start, there are some subtleties of the OCaml module system
12
+
that make it a good deal more complex than in other languages.
13
+
14
+
{1 opam-format.2.3.0 ≠ opam-format.2.3.0?}
15
+
Let's take the case that hit me this morning. I've been working on
16
+
{{: https://github.com/ocurrent/ocaml-docs-ci} ocaml-docs-ci} in order to bring
17
+
the exciting new {{: https://ocaml.github.io/odoc} odoc 3} features to
18
+
{{: https://ocaml.org/} ocaml.org} for everyone to enjoy. I have it checked out
19
+
and building locally, but to deploy it to the infrastructure managed by
20
+
{{: https://tunbury.org/} Mark Elvers} it needs to be packaged up into a Docker
21
+
image. So I issued the usual [docker build .] and after it churned through the
22
+
setup stages and got on to building the project, it hit an error:
23
+
24
+
{v
25
+
File "src/solver/solver.ml", line 58, characters 75-98:
26
+
let deps = List.map (fun pkg -> OpamPackage.Map.find pkg simple_deps) (OpamPackage.Set.to_list pkgs) in
27
+
Error: Unbound value OpamPackage.Set.to_list
28
+
Hint: Did you mean of_list?
29
+
v}
30
+
31
+
Now [OpamPackage] is a module in the [opam-format] library, which is easily
32
+
discovered using the excellent
33
+
{{: https://doc.sherlocode.com/?q=OpamPackage} Sherlodoc} tool, so I checked
34
+
what version I had locally, and what version I had in the Docker container,
35
+
and it turned out I was using exactly the same version -- 2.3.0 -- both
36
+
locally and in the container. So what's going on?
37
+
38
+
The problem is that the Dockerfile I was using was using OCaml version 4.14,
39
+
whereas locally I was using OCaml 5.3. "But how on earth can this cause the API
40
+
of [opam-format] to change?" I hear you wail! Well, this is actually one of the
41
+
simpler outcomes of the way the OCaml module system works. Let's look at
42
+
{{: https://github.com/ocaml/opam/blob/2.3.0/src/format/opamPackage.mli} the code}.
43
+
44
+
The first thing we note is the absence of any definition of [Set] or [Map] here
45
+
- where do they come from? It turns out they come from
46
+
{{: https://github.com/ocaml/opam/blob/2.3.0/src/format/opamPackage.mli#L49} this line here}:
47
+
48
+
{@ocaml[
49
+
include OpamStd.ABSTRACT with type t := t
50
+
]}
51
+
52
+
So let's take a look over in [opamStd.mli] to see what that signature looks like:
53
+
54
+
{@ocaml[
55
+
(** A signature for handling abstract keys and collections thereof *)
56
+
module type ABSTRACT = sig
57
+
58
+
type t
59
+
60
+
val compare: t -> t -> int
61
+
val equal: t -> t -> bool
62
+
val of_string: string -> t
63
+
val to_string: t -> string
64
+
val to_json: t OpamJson.encoder
65
+
val of_json: t OpamJson.decoder
66
+
67
+
module Set: SET with type elt = t
68
+
module Map: MAP with type key = t
69
+
end
70
+
]}
71
+
72
+
OK, so we've found the definitions of [Set] and [Map] - they refer to signatures
73
+
[SET] and [MAP] which are defined just above in
74
+
{{: https://github.com/ocaml/opam/blob/2.3.0/src/core/opamStd.mli#L17-L98} opamStd.mli}.
75
+
Let's just look at [Set] since that's where the problem was:
76
+
77
+
{@ocaml[
78
+
module type SET = sig
79
+
80
+
include Set.S
81
+
82
+
val map: (elt -> elt) -> t -> t
83
+
84
+
val is_singleton: t -> bool
85
+
86
+
(** Returns one element, assuming the set is a singleton. Raises [Not_found]
87
+
on an empty set, [Failure] on a non-singleton. *)
88
+
val choose_one : t -> elt
89
+
90
+
val choose_opt: t -> elt option
91
+
92
+
val of_list: elt list -> t
93
+
val to_list_map: (elt -> 'b) -> t -> 'b list
94
+
val to_string: t -> string
95
+
val to_json: t OpamJson.encoder
96
+
val of_json: t OpamJson.decoder
97
+
val find: (elt -> bool) -> t -> elt
98
+
val find_opt: (elt -> bool) -> t -> elt option
99
+
100
+
(** Raises Failure in case the element is already present *)
101
+
val safe_add: elt -> t -> t
102
+
103
+
(** Accumulates the resulting sets of a function of elements until a fixpoint
104
+
is reached *)
105
+
val fixpoint: (elt -> t) -> t -> t
106
+
107
+
(** [map_reduce f op t] applies [f] to every element of [t] and combines the
108
+
results using associative operator [op]. Raises [Invalid_argument] on an
109
+
empty set, or returns [default] if it is defined. *)
110
+
val map_reduce: ?default:'a -> (elt -> 'a) -> ('a -> 'a -> 'a) -> t -> 'a
111
+
112
+
module Op : sig
113
+
val (++): t -> t -> t (** Infix set union *)
114
+
115
+
val (--): t -> t -> t (** Infix set difference *)
116
+
117
+
val (%%): t -> t -> t (** Infix set intersection *)
118
+
end
119
+
120
+
end
121
+
]}
122
+
123
+
Sure enough, there's no [to_list] function defined in there. Once again though,
124
+
there's an [include Set.S] in there. It turns out that that refers to the [Set]
125
+
module in the OCaml standard library. We can again
126
+
{{: https://github.com/ocaml/ocaml/blob/5.3.0/stdlib/set.mli} look at the source}:
127
+
128
+
{@ocaml[
129
+
val to_list : t -> elt list
130
+
(** [to_list s] is {!elements}[ s].
131
+
@since 5.1 *)
132
+
]}
133
+
134
+
And there it is. The [to_list] function has only been in the [Set] module since
135
+
version 5.1.
136
+
137
+
{1 Using ocaml.org docs}
138
+
It was pretty difficult to figure that out from the source, but happily there's
139
+
a better way. We can browse the docs on https://ocaml.org/ - We can look at the
140
+
docs for the {{: https://ocaml.org/p/opam-format/2.3.0/doc/OpamPackage/Set/index.html} OpamPackage.Set module}
141
+
which, as of today, does not contain any [to_list] function. The [include Set.S]
142
+
is there with the expansion showing all of the types and values coming from it,
143
+
so we can click on the [Set.S] link on the include line which takes us to the
144
+
documentation for the stdlib from OCaml 4.11.2. Changing the version from the
145
+
dropdown at the top to something more recent takes us to a page containing the
146
+
[to_list] function with the helpful [since 5.1] annotation.
147
+
148
+
This is, in fact, a relatively simple example of the sorts of issues that can
149
+
occur that make semantic versioning a headache. In this example, it was a
150
+
change due to a difference in the compiler version used, but there's nothing
151
+
particularly special about that - a package may expose signatures derived from
152
+
any of its dependencies! So is there anything we can do about this? Obviously,
153
+
yes!
154
+
155
+
{1 Towards a solution}
156
+
157
+
Step 1 of any approach to solving this is to be able to identify which bits of a libraries API come from which
158
+
packages, and therefore which versions of those packages. It turns out there may well be a nice way to piggy-back on a recent
159
+
feature from Odoc, which was originally intended to help with suppressing
160
+
suprious warnings.
161
+
162
+
The problem we were tackling was that if your library ends
163
+
up exporting a module whose signature is defined in someone else's package,
164
+
then any warnings that come from it are unfixable. To fix this we added a tag
165
+
to each signature of a module that indicates which package it originally came
166
+
from. Odoc is then very careful to keep track of this as it performs its
167
+
signature manipulations, resulting in an accurate way to know which signature
168
+
elements came from which package. This fixed the problem of the spurious
169
+
warnings nicely.
170
+
171
+
Quite separately, we've got the docs CI that is attempting to build docs for
172
+
every version of every package. Obviously given the above, in order to
173
+
exhaustively show all the possible APIs of every library, we should build
174
+
all possible combinations of every version of every package. Clearly we can't
175
+
possibly do this, so the docs CI focuses on the goal of building at least one
176
+
solution for every version of every package.
177
+
178
+
Now if you combine these two ideas, we can use the builds of the packages
179
+
with the tracking of the package of the originating signatures to be able to
180
+
precisely track the differences in API between different versions of a
181
+
package. This would allow us to build a database of those changes, and with
182
+
this in hand we can look at what APIs are used in any other package and be
183
+
able to suggest upper and lower bounds on the versions of its dependencies.
184
+
185
+
Now wouldn't that be cool?
+130
site/blog/2025/04/this-site.mld
+130
site/blog/2025/04/this-site.mld
···
1
+
{0 This site}
2
+
3
+
@x-ocaml.requires mime_printer
4
+
@published 2025-04-07
5
+
6
+
I've spent a {e lot} of time over the past few years working on Odoc, the OCaml documentation
7
+
generator, so when it came time to (re)start my own website and blog, I found it hard to resist
8
+
thinking about how I might use odoc as part of it. We've spent a lot of time recently trying to
9
+
make odoc more able to generate structured documentation sites, so I've gone all in and am
10
+
trialling using it as a tool to generate my entire site. This is a bit of an experiment, and I
11
+
don't know how well it will work out, but let's see how it goes.
12
+
13
+
Additionally, I've recently been working on a project currently called [odoc_notebook], which is a
14
+
set of tools to allow odoc [mld] files to be used as a sort of Jupyter-style notebook. The idea is
15
+
that you can write both text and code in the same file, and then run the code in the notebook
16
+
interactively. Since I've only got a webserver, all the execution of code has to be done client
17
+
side, so I'm making extensive use of the phenomenal {{:https://github.com/ocsigen/js_of_ocaml}Js_of_ocaml}
18
+
project to get an OCaml engine running in the browser.
19
+
20
+
My focus has initially been on getting 'toplevel-style' code execution working. As an example, let's
21
+
write a little demo.
22
+
23
+
{1 Demo}
24
+
Let's start with a little demo:
25
+
26
+
{@ocaml[
27
+
# let x = 1 + 2;;
28
+
val x : int = 3
29
+
]}
30
+
31
+
It's intended to look like an OCaml toplevel session, so each new expression starts with a [#] and is
32
+
terminated with a double semicolon. The response from the toplevel is then below that indented with
33
+
2 spaces. Right now, there's not much in the way of error checking so you can make it all very
34
+
confused by deleting the hash, removing the [;;] and so on. Avoiding this, however, you can edit the
35
+
numbers here and hit 'run' (maybe twice!) to see the results being updated.
36
+
37
+
There is also a little integration to allow the code to produce output more interesting than just
38
+
text. The following cell creates an SVG image and 'pushes' it to [Mime_printer], which receives the
39
+
mime value and renders it in the browser below the code block.
40
+
41
+
{x@ocaml[
42
+
# let svg = [
43
+
{|<svg height="210" width="500" xmlns="http://www.w3.org/2000/svg">|};
44
+
{|<polygon points="100,10 40,198 190,78 10,78 160,198" |};
45
+
{|style="fill:lime;stroke:purple;stroke-width:5;"/></svg>|}];;
46
+
val svg : string list =
47
+
["<svg height=\"210\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\">";
48
+
"<polygon points=\"100,10 40,198 190,78 10,78 160,198\" ";
49
+
"style=\"fill:lime;stroke:purple;stroke-width:5;\"/></svg>"]
50
+
# Mime_printer.push "image/svg" (String.concat "\n" svg);;
51
+
- : unit = ()
52
+
]x[
53
+
{%html: <svg height="210" width="500" xmlns="http://www.w3.org/2000/svg">
54
+
<polygon points="100,10 40,198 190,78 10,78 160,198"
55
+
style="fill:lime;stroke:purple;stroke-width:5;"/></svg> %}
56
+
]}
57
+
58
+
{1 Things to come}
59
+
60
+
{2 Merlin support}
61
+
There are a bunch of things I want to add to this, for example, Merlin support.
62
+
In fact, {{:https://github.com/voodoos/merlin-js}merlin-js} already exists and works,
63
+
thanks to the fantastic work of {{:https://github.com/voodoos}Ulysse}, but the
64
+
problem is that it's not
65
+
really designed for toplevel work, and it doesn't work when the code is broken
66
+
up into chunks like I do here. So either I need to concatenate all the cells
67
+
together before I give it to Merlin, or I need to make each cell it's own little
68
+
module and 'open' every previous cell's module.
69
+
70
+
Within a single cell, it does already work. You can see that Merlin is correctly
71
+
underlining the error in the following cell. You should also be able to hover over
72
+
the variables and see their types.
73
+
74
+
{@ocaml[
75
+
type t = { foo : int; bar : string };;
76
+
77
+
let x = { foo = 1; bar = "hello" };;
78
+
79
+
let this_line_has_an_error = { foo = 1; bar = None };;
80
+
]}
81
+
82
+
But across cells, I've broken Merlin, though the code is executes correctly. You can see the
83
+
problem in the following cell, which re-pushes the SVG image using the variable [svg] defined
84
+
in the cell above. Merlin highlights the use of the varible [svg] is, because it's not aware
85
+
of the varible, but the code gets executed correctly and the image is rendered below the cell.
86
+
87
+
{x@ocaml[
88
+
Mime_printer.push "image/svg" (String.concat "\n" svg);;
89
+
]x[
90
+
{%html: <svg height="210" width="500" xmlns="http://www.w3.org/2000/svg">
91
+
<polygon points="100,10 40,198 190,78 10,78 160,198"
92
+
style="fill:lime;stroke:purple;stroke-width:5;"/></svg> %}
93
+
]}
94
+
95
+
Edit 2025-05-20: I have now got merlin working across cells, though I'm not convinced the current
96
+
solution is the right long-term solution. S
97
+
98
+
{2 Dynamic libraries}
99
+
Currently the use of libraries it quite limited - they are defined more-or-less statically. I've had
100
+
dynamic libraries working in the past, but I need to re-implement them. The plan is to have the
101
+
[cma] files converted to [js] files and then load them on-demand when the notebook specifies them. The
102
+
tricky thing here is that we need to be able to use them both in the browser and in bytecode executables
103
+
so that the 'test-promote' workflow still works. This will probably require specifying the libraries
104
+
by name, and having to re-implement the work that {{:https://projects.camlcity.org/projects/findlib.html}findlib}
105
+
does to find the libraries and load them and their dependencies in the right order, though this time
106
+
entirely over HTTP.
107
+
108
+
{2 Other things}
109
+
There are loads of other things I'm interested in doing, including:
110
+
- Investigating how to do 'exercises' to allow readers to try things out in a guided way
111
+
- 'Test cells' to see if implementations are correct
112
+
- Persistence of the notebook state - both using local and remote storage
113
+
- Integration of docs
114
+
- Exploration of the execution model - how to run the code in the right order and ensure reproducibility
115
+
- Use of remote execution engines rather than just in the browser
116
+
- Other languages?
117
+
118
+
Right now though, my focus is on the functionality required for this blog, with a secondary goal of
119
+
looking at how we might use this sort of technology on the docs site on ocaml.org. Wouldn't it be
120
+
cool to be able to drop into a live OCaml toplevel for any library in opam?
121
+
122
+
{1 Example notebooks}
123
+
124
+
As a more extended example of odoc notebooks, I have converted to this format the course that I help teach
125
+
at the University of Cambridge; {{:https://www.cl.cam.ac.uk/teaching/2425/FoundsCS/}Foundations of Computer Science}.
126
+
{{!/site/notebooks/foundations/page-index}Try them out for yourself!}.
127
+
128
+
129
+
130
+
+101
site/blog/2025/05/ai-for-climate-and-nature-day.mld
+101
site/blog/2025/05/ai-for-climate-and-nature-day.mld
···
1
+
{0 AI for Climate & Nature Community Day}
2
+
3
+
@published 2025-05-01
4
+
5
+
{{image!melissa.jpg}Melissa Leach}
6
+
7
+
{i Melissa Leach introducing the day}
8
+
Today was the "AI for Climate & Nature Community Day" at the {{:https://map.cam.ac.uk/?maplon=0.12032&maplat=52.20354&mapzoom=18&maplayers=Building+Labels%2CExternal+Sites%2CColleges%2CUniversity+Sites%2CBuildings%2CTransport&mapfeature=mfid257%2CBuildings}David Attenborough Building}. A whole bunch of the EEG were either presenting or contributing in some way so I thought I'd come along
9
+
to see what's going on.
10
+
11
+
12
+
{1 Keynote and main talks}
13
+
14
+
Following the intro talks from Professors {{:https://www.cambridgeconservation.org/about/people/prof-melissa-leach/}Melissa Leach} and {{:https://www.zoo.cam.ac.uk/directory/bill-sutherland}Bill Sutherland}, the day started with the keynote talk from {{:https://www.biology.ox.ac.uk/people/amy-hinsley}Amy Hinsley}, who, using the specific case of animial trafficking, talked about the need to make AI in conservation equitable, explainable and useful.
15
+
16
+
{{image!amy.jpg}Amy Hinsley}
17
+
18
+
{i Amy Hinsley delivering the keynote talk}
19
+
20
+
We then moved into the first session with {{:https://www.geog.cam.ac.uk/people/lines/}Emily Lines} from the Geography Department who talked about the challenges processing sensor data in the context of forests. Her group has a variety of data collected from forests across Europe, collected from using many different methods, from drones taking pictures of the canopies to ground-based laser scanners producing 3d point clouds. The challenge is then not only to identify individual trees, which is pretty tricky, but also to then distinguish between the leaves of the trees and the wood.
21
+
22
+
After Emily came {{:https://ai.cam.ac.uk/people/robert-rouse.html}Robert Rouse} from the {{:https://iccs.cam.ac.uk}ICCS}, who's using a small neural net and genetic algorithms to extend a study from
23
+
the RSPB on figuring out an optimal way to do some land use adjustments to cut carbon and improve outcomes for birds, whilst not significantly impacting the ability to produce food.
24
+
25
+
We then had {{:https://www.zoo.cam.ac.uk/directory/dr-sam-reynolds}Sam Reynolds} and {{:https://toao.com}Sadiq Jaffer} who talked about their project; using AI to sift through millions of papers looking for those relevant to a specified conservation topic. They're able to directly compare their results with results obtained by manually doing this process, a project that's been going on over the last 20 or so years summing to something like 75 man years of effort. In the end they only missed a few papers that the manual process had found, but actually found many relevant papers that had been missed - and all in only a few days of compute.
26
+
27
+
{{image!sadiq.jpg}Sam Reynolds and Sadiq Jaffer}
28
+
29
+
{i Sam Reynolds and Sadiq Jaffer sorting millions of papers}
30
+
31
+
{1 Lightning talks}
32
+
We then had a number of 'lightning talks', with each presenter having only three minutes to talk about their work.
33
+
34
+
- {{:https://www.maths.cam.ac.uk/person/ss3299}Sebastian Schemm} presented his work on creating a foundational model for the climate
35
+
- {{:https://www.eng.cam.ac.uk/profiles/ac685}Alice Cicirello} talked about the prospects of applying machine learning to {{:https://en.wikipedia.org/wiki/Marine_cloud_brightening}Marine Cloud Brightening}
36
+
- {{:https://www.maths.cam.ac.uk/person/sdat2}Simon Thomas} has been looking at analysing the heights of tropical cyclone storm surges
37
+
- {{:https://github.com/niccolozanotti}Niccolò Zanotti} gave us an introduction to {{:https://github.com/cambridge-ICCS/FTorch}FTorch}, a library to integrate the worlds of PyTorch and Fortran
38
+
- {{:https://www.nceo.ac.uk/contact-us/people/dr-simon-driscoll/}Simon Driscoll} then talked about melt ponds on arctic sea ice, a poorly understood but important component of the climate in the Arctic.
39
+
- {{:https://www.zoo.cam.ac.uk/directory/emilio-luz-ricca}Emilio Luz-Ricca} talked about his project to apply machine learning to predict hunting pressure
40
+
- {{:https://orlando-code.github.io}Orlando Timmerman} gave us some insights into how he's been using machine learning to predict the future of coral reefs, and how we might use this to help with their conservation.
41
+
- {{:https://www.zoo.cam.ac.uk/directory/ruari-marshall-hawkes}Ruari Marshall-Hawkes} showed us how to listen very carefully to figure out population numbers,
42
+
- {{:https://www.linkedin.com/in/harriet-branson-a93a8313b/}Hattie Branson} from {{:https://www.fauna-flora.org}Fauna & Flora} talked about habitat detection in South Sudan,
43
+
- {{:https://www.linkedin.com/in/martakoch/}Marta Koch} showed us an analysis of how well ChatGPT, Claude and the like would perform at setting the agendas for SDPs,
44
+
- {{:https://www.linkedin.com/in/zhengpeng-feng-2410a132a/}Frank Feng} finished the session with a talk on the {{:https://www.cst.cam.ac.uk/seminars/list/227335}Barlow Twins Earth Foundation Model}.
45
+
46
+
{%html:
47
+
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px;">
48
+
<figure style="margin:0; width: 100%;">
49
+
<img src="sebastian.jpg" alt="Sebastian Schemm" style="max-width: 100%; height: auto;">
50
+
<figcaption>Sebastian Schemm</figcaption>
51
+
</figure>
52
+
<figure style="margin:0; width: 100%;">
53
+
<img src="alice.jpg" alt="Alice Cicirello" style="max-width: 100%; height: auto;">
54
+
<figcaption>Alice Cicirello</figcaption>
55
+
</figure>
56
+
<figure style="margin:0; width: 100%;">
57
+
<img src="simon.jpg" alt="Simon Thomas" style="max-width: 100%; height: auto;">
58
+
<figcaption>Simon Thomas</figcaption>
59
+
</figure>
60
+
<figure style="margin:0; width: 100%;">
61
+
<img src="simond.jpg" alt="Simon Driscoll" style="max-width: 100%; height: auto;">
62
+
<figcaption>Simon Driscoll</figcaption>
63
+
</figure>
64
+
<figure style="margin:0; width: 100%;">
65
+
<img src="emilio.jpg" alt="Emilio Luz-Ricca" style="max-width: 100%; height: auto;">
66
+
<figcaption>Emilio Luz-Ricca</figcaption>
67
+
</figure>
68
+
<figure style="margin:0; width: 100%;">
69
+
<img src="orlando.jpg" alt="Orlando Timmerman" style="max-width: 100%; height: auto;">
70
+
<figcaption>Orlando Timmerman</figcaption>
71
+
</figure>
72
+
<figure style="margin:0; width: 100%;">
73
+
<img src="ruari.jpg" alt="Ruari Marshall-Hawkes" style="max-width: 100%; height: auto;">
74
+
<figcaption>Ruari Marshall-Hawkes</figcaption>
75
+
</figure>
76
+
<figure style="margin:0; width: 100%;">
77
+
<img src="hattie.jpg" alt="Hattie Branson" style="max-width: 100%; height: auto;">
78
+
<figcaption>Hattie Branson</figcaption>
79
+
</figure>
80
+
<figure style="margin:0; width: 100%;">
81
+
<img src="frank.jpg" alt="Frank Feng" style="max-width: 100%; height: auto;">
82
+
<figcaption>Frank Feng</figcaption>
83
+
</figure>
84
+
85
+
</div>
86
+
%}
87
+
88
+
{1 Discussions}
89
+
We then split up into three discussion groups; one on the future of this work, one on how to continue building this community of researchers, and the last on applying AI to real-world problems. As a newcomer to the field I was interested in the direction it's
90
+
heading in, so I joined in {{:https://dorchard.github.io}Dominic Orchard}'s led session on the future of AI.
91
+
92
+
We had a fascinating discussion on both the
93
+
immediate things we can do and longer term worries. We were imagining a world where AI becomes 'just a tool' that we don't need to be experts in to apply it, but right now we're in a much more tightly coupled collaborative world where we need experts in AI to
94
+
complement the experts in the application field to make progress. This comes with challenges - applying for funding for multidisciplinary work is not the norm, so we spent some time discussing this too.
95
+
96
+
One of our group spoke about statistics now being 'just a tool', but it's been one that we've worked with for a long time now and
97
+
we know where the sharp corners are. We have protocols for applying statistical tools and we have diagnostic plots to tell us whether the results are trustworthy, but not only do we not have these for AI models, but it's not yet clear whether such a thing will be even possible.
98
+
99
+
Overall it was a fascinating day, and I'm very much looking forward to following the work of these outstanding researchers, and maybe even contributing to their work in some way in the future.
100
+
101
+
{i Thanks to {{:https://anil.recoil.org}Anil Madhavapeddy} for the photos of the day.}
+246
site/blog/2025/05/docs-progress.mld
+246
site/blog/2025/05/docs-progress.mld
···
1
+
{0 Progress in OCaml docs}
2
+
3
+
@published 2025-05-29
4
+
5
+
The docs build is progress well, and we've {i just about} hit 20,000 packages (20,038 to be precise).
6
+
So at this point I thought it'd be useful to take a look through the various failures to see if there are
7
+
any insights to be gained.
8
+
9
+
Odoc requires a built package in order to generate the docs, there are two steps that have to be done before
10
+
we can begin building the docs. Step one is to figure out the exact set of packages to build - ie, doing an opam solve,
11
+
and step two is to actually build the packages.
12
+
These two steps are, to some extent, out of docs-ci's control, and rely on the state of opam repository.
13
+
While there are efforts to keep this in as good a state as possible, it's still the case that these steps fail
14
+
much more often than the actual docs build itself. Let's take a look at some of the failures we see in each
15
+
of these steps.
16
+
17
+
{1 Step 1: opam solve}
18
+
There are 2,074 solver failures. A good chunk of these are due to the way docs ci works itself, that it starts
19
+
with a specific version of OCaml. In order to do this, the solution must have a specific version of OCaml in
20
+
it, and this is not always the case, for example, all of the [conf-*] packages fail in this way. This
21
+
particular class of "failures" is not at all important, as mostly they don't contain useful documentation,
22
+
but even if they do, if they're actually being used then they will be built as part of another solution. For
23
+
example, while [conf-faad] fails with this error, the solution of the [faad] package pulls it in anyway, so
24
+
we can build any docs that it includes. Roughly two thirds (685) of the reported failures are due to this, and
25
+
by checking the resulting HTML docs we can see that we do get docs for 278 of these, so they must be pulled in
26
+
by other solutions.
27
+
28
+
The remaining failures are "real" in the sense that we never currently get docs for these packages. In turn, these can
29
+
be subcategorised. One class of failures happen with platform-specific packages, for example [camlkit] which provides bindings
30
+
to Cocoa frameworks, and is only available on macOS, or [eio_windows] which clearly requires Windows.
31
+
The current docs-ci setup only builds on Linux, and extending this to other platforms will require a little
32
+
more work, and is not currently scheduled. These are "fixable" failures.
33
+
34
+
The third class of failures are those that will "just never work". For example, some early versions of [domainslib]
35
+
were released before the OCaml 5.0 APIs were finalised, and so they can only work with alpha versions of OCaml 5.0.
36
+
We won't be documenting these.
37
+
38
+
Finally there are some more 'unexplained' failures, such as [docteur.0.0.2]. This one's particularly interesting
39
+
as the solve actually succeeds when using the stand-alone tool opam-0install, whereas it's failing in docs-ci,
40
+
which uses opam-0install as a library! I'm currently suspicious about the 'deprecated' flag, as the failure log
41
+
contains:
42
+
43
+
{v
44
+
- git-cohttp-unix -> (problem)
45
+
No usable implementations:
46
+
git-cohttp-unix.3.6.0: Availability condition not satisfied
47
+
git-cohttp-unix.3.5.0: Availability condition not satisfied
48
+
git-cohttp-unix.3.4.0: Availability condition not satisfied
49
+
git-cohttp-unix.3.3.3: Availability condition not satisfied
50
+
git-cohttp-unix.3.3.2: Availability condition not satisfied
51
+
...
52
+
v}
53
+
54
+
and that flag is the only thing I can immediately see that stands out in [git-cohttp-unix]. In contrast,
55
+
the solution given by opam-0install contains [git-cohttp-unix.3.6.0] as a dependency. I suspect fixing
56
+
this will cause quite a few more packages to succeed.
57
+
58
+
{1 Step 2: building packages}
59
+
60
+
The next step, once we've got the solutions, is to build the packages. This is using the new method
61
+
I {{!/site/blog/2025/04/page-"ocaml-docs-ci-and-odoc-3"}previously wrote about}. There are about
62
+
1,000 packages that fail to build, and once again we can take a look and categorise some of these
63
+
failures. There are a wider variety of failures here, and it's quite useful to cross-check with
64
+
{{!https://check.ci.ocaml.org/}opam health check} to see if it's known to be broken. Unfortunately
65
+
OHC only builds the latest versions of everything, so we can't check in some cases. The interesting
66
+
issues are where we're failing to build something that seems to work in OHC.
67
+
68
+
{2 llvm.18}
69
+
70
+
This is an interesting type of error, where the build fails because of a missing external dependency.
71
+
The [llvm] package depends upon [conf-llvm-static.18], which should be able to install the depext.
72
+
Looking at the package, it does indeed have a depext for Debian:
73
+
74
+
{v
75
+
depexts: [
76
+
["llvm@18" "zstd"] {os-distribution = "homebrew" & os = "macos"}
77
+
["llvm-18"] {os-distribution = "macports" & os = "macos"}
78
+
["llvm-18-dev" "zlib1g-dev" "libzstd-dev"] {os-family = "debian"}
79
+
["llvm18-dev"] {os-distribution = "alpine"}
80
+
["llvm18"] {os-family = "arch"}
81
+
["llvm18-devel"] {os-family = "suse" | os-family = "opensuse"}
82
+
["llvm18-devel"] {os-distribution = "fedora" & os-version >= "41"}
83
+
["llvm-devel"] {os-distribution = "fedora" & os-version = "40"}
84
+
["llvm18-devel" "epel-release"] {os-distribution = "centos"}
85
+
["devel/llvm18"] {os = "freebsd"}
86
+
]
87
+
v}
88
+
89
+
However, in Debian 12, they've already updated to [llvm-19], so the depext is
90
+
not available.
91
+
92
+
{2 camlimages.5.0.5}
93
+
94
+
This one fails due to a linking error. Oddly enough it does seem to work in OHC.
95
+
96
+
{v
97
+
(cd _build/default && /home/opam/.opam/4.14/bin/ocamlmklib.opt -g -o freetype/camlimages_freetype_stubs freetype/ftintf.o -ldopt -lfreetype)
98
+
# /usr/bin/ld: freetype/ftintf.o: warning: relocation against `Caml_state' in read-only section `.text'
99
+
# /usr/bin/ld: freetype/ftintf.o: relocation R_X86_64_PC32 against undefined symbol `Caml_state' can not be used when making a shared object; recompile with -fPIC
100
+
# /usr/bin/ld: final link failed: bad value
101
+
v}
102
+
103
+
{2 ahrocksdb.0.2.2}
104
+
105
+
This one fails in OHC too, but it looks like it's a build failure with more recent gccs, fixed upstream: https://github.com/ahrefs/ocaml-ahrocksdb/commit/e52316b3d30fededac023141bf8b47da79cabfed
106
+
107
+
{v
108
+
# run: gcc -O2 -fno-strict-aliasing -fwrapv -fPIC -pthread -I/usr/include/rocksdb -I /home/opam/.opam/5.3/lib/ocaml -o /tmp/build_02b340_dune/ocaml-configuratordc5e55/c-test-2/test.exe /tmp/build_02b340_dune/ocaml-configuratordc5e55/c-test-2/test.c -lm -lpthread -lrocksdb
109
+
# -> process exited with code 1
110
+
# -> stdout:
111
+
# -> stderr:
112
+
# | In file included from /tmp/build_02b340_dune/ocaml-configuratordc5e55/c-test-2/test.c:4:
113
+
# | /usr/include/rocksdb/version.h:7:10: fatal error: string: No such file or directory
114
+
# | 7 | #include <string>
115
+
# | | ^~~~~~~~
116
+
# | compilation terminated.
117
+
# Error: discover error
118
+
v}
119
+
120
+
{2 alt-ergo.2.2.0}
121
+
122
+
Looks like it's trying to write outside the sandbox. The failure only occurs on alt-ergo 1.3.0 - 2.2.0.
123
+
124
+
{v
125
+
# mkdir -p /home/opam/.opam/4.14/man/man1
126
+
# cp -f doc/alt-ergo.1 /home/opam/.opam/4.14/man/man1
127
+
# mkdir -p /usr/local/lib/alt-ergo/preludes
128
+
# mkdir: cannot create directory '/usr/local/lib/alt-ergo': Permission denied
129
+
# make: *** [Makefile.users:243: install-preludes] Error 1
130
+
v}
131
+
132
+
{2 ctypes-foreign.0.18.0}
133
+
134
+
This one is a much more interesting failure. The logs show:
135
+
136
+
{v
137
+
[ERROR] No solution for ctypes-foreign.0.18.0: * Missing dependency:
138
+
- ctypes-foreign -> ctypes
139
+
unknown package
140
+
v}
141
+
142
+
which is happening because of the optimisation I {{!/site/blog/2025/04/page-"ocaml-docs-ci-and-odoc-3"}mentioned before} where we
143
+
build a new [opam-repository] with only the packages we're going to need. In this case, we've somehow
144
+
missed out the [ctypes] package. Looking at the opam file for [ctypes-foreign], it has a [post] dependency
145
+
on [ctypes]. The [post] keyword indicates that [ctypes] should be installed with [ctypes-foreign], but
146
+
that having it as a "normal" dependency would introduce a dependency cycle. Since we require a DAG
147
+
of dependencies, we explicitly remove any [post] dependencies from the set of packages to build, but
148
+
it seems that [opam] would like to know about it anyway!
149
+
150
+
{2 others}
151
+
152
+
There are many more. An automatic cross-check with OHC would be really useful, mainly to distinguish
153
+
between the packages that are broken due to [ocaml-docs-ci] issues (like [ctypes-foreign]) and those
154
+
that are broken for other reasons (like [ahrocksdb]).
155
+
156
+
{1 Step 3: building docs}
157
+
158
+
Finally, we have the actual docs build. This is where we run [odoc] and [odoc_driver] to produce the
159
+
HTML docs. All the errors here are ones that we should be able to fix!
160
+
161
+
Firstly, there are the internal errors:
162
+
163
+
{v
164
+
Uncaught exception: Failure("\"rm\" \"-rf\" \"/var/cache/obuilder/merged/582e973685d380d4c91eadc2611eee02c82c5fe4f8bd732e0080fb22bc4404cd\" \"/var/cache/obuilder/work/582e973685d380d4c91eadc2611eee02c82c5fe4f8bd732e0080fb22bc4404cd\" failed with exit status 1")
165
+
2025-05-22 09:30.18: Job failed: Failed: Internal error
166
+
v}
167
+
168
+
These are some [obuilder] error that needs fixing. Currently we're just rerunning the job to fix these.
169
+
170
+
{2 odoc.2.0.0}
171
+
172
+
Oops, we can't build our own docs! At least it's an old version :-)
173
+
174
+
{v
175
+
odoc: internal error, uncaught exception:
176
+
File "src/html/link.ml", line 101, characters 16-22: Assertion failed
177
+
Raised at Odoc_html__Link.href in file "src/html/link.ml", line 101, characters 16-57
178
+
Called from Odoc_html__Generator.internallink in file "src/html/generator.ml", line 108, characters 19-49
179
+
...
180
+
v}
181
+
182
+
The failure points {{:https://github.com/ocaml/odoc/blob/42190737339d9be4510eeeb0e3c47e84badf4d73/src/html/link.ml#L101}here},
183
+
an assertion about the common ancestor of two paths. {{:https://github.com/ocaml/odoc/issues/1345}Issue filed}.
184
+
185
+
{2 ocaml-base-compiler.4.07.0}
186
+
187
+
This one happens because of our "optimisation" to use a base image with OCaml pre-installed.
188
+
What we {i actually} do is find the major/minor version of OCaml and use the corresponding
189
+
docker image - so in this case we'll use ocaml/opam:debian-12-ocaml-4.07. Now this image
190
+
actually contains OCaml 4.07.1, and the format of [cmt] and [cmti] files changed between
191
+
these releases, so we get a failure.
192
+
193
+
We'll fix this by getting rid of the optimisation and building from an empty switch.
194
+
195
+
{2 lascar.0.7.0}
196
+
This one is quite interesting. It's another assertion failure in odoc:
197
+
198
+
{v
199
+
odoc: internal error, uncaught exception:
200
+
File "src/xref2/cpath.ml", line 364, characters 37-43: Assertion failed
201
+
Raised at Odoc_xref2__Cpath.unresolve_resolved_parent_path in file "src/xref2/cpath.ml", line 364, characters 37-49
202
+
Called from Odoc_xref2__Cpath.unresolve_module_path in file "src/xref2/cpath.ml", line 349, characters 28-60
203
+
Called from Odoc_xref2__Tools.fragmap.map_module_decl in file "src/xref2/tools.ml", line 1792, characters 48-80
204
+
v}
205
+
206
+
It's happening when we 'unresolve' a previously resolved path. We end up having to do this when
207
+
something about the path has changed, in this case while we're handling a [S with module Foo = Bar] or similar.
208
+
Issue {{:https://github.com/ocaml/odoc/issues/1346}filed}.
209
+
210
+
{2 camlp5}
211
+
212
+
This one actually occurs in [odoc_driver] rather than in [odoc] itself.
213
+
214
+
{v
215
+
odoc_driver_voodoo: [DEBUG] Found cmi_only_lib in dir: /home/opam/.opam/4.08/lib/camlp5
216
+
odoc_driver_voodoo: internal error, uncaught exception:
217
+
Invalid_argument("\"/home/opam/.opam/4.08/lib/camlp5\": invalid segment")
218
+
219
+
v}
220
+
221
+
Here we're trying to add a segment to a path, but rather than a single path segment we've
222
+
got an entire fully qualified path. Issue {{:https://github.com/ocaml/odoc/issues/1347}filed}.
223
+
224
+
{1 Conclusion}
225
+
226
+
It's pretty good that we've only got 4 types of error happening at the doc-generation phase.
227
+
However, as a whole, any error that occurs earlier in the pipeline ends up with a missing
228
+
documentation tab on the website, and we need to do a bit more so that the actual problem can
229
+
be tracked down and fixed. This is obviously a more general problem than just the docs, and
230
+
one that {{:https://check.ci.ocaml.org}opam health check} seeks to highlight. However, the current incarnation of OHC
231
+
is significantly less efficient than docs-ci, so generalising the approach we've taken with
232
+
{{:https://github.com/jonludlam/opamh}opamh} should really help with making this more responsive.
233
+
234
+
In addition, a number of the issues seen here could be addressed with a tool my colleague {{:https://ryan.freumh.org/}Ryan} is working on:
235
+
{{:https://ryan.freumh.org/enki.html}Enki}. This tool would allow us to run a solve that
236
+
actually determines not only the set of packages we wish to install, but the platform to install
237
+
onto - e.g. for [eio_windows] the solution would be to install on Windows, and for [llvm.18-static]
238
+
the solution might be Fedora 40.
239
+
240
+
241
+
242
+
243
+
244
+
245
+
246
+
+4
site/blog/2025/05/index.mld
+4
site/blog/2025/05/index.mld
+142
site/blog/2025/05/lots-of-things.mld
+142
site/blog/2025/05/lots-of-things.mld
···
1
+
{0 Lots of things have been happening}
2
+
3
+
@published 2025-05-20
4
+
5
+
I've been working on a whole lot of thing recently in many different areas,
6
+
making what's felt like only a bit of progress in each. Consequently I've not
7
+
felt like I had anything substantial to say, so I haven't written up anything
8
+
for a while.
9
+
10
+
Time for a little summary of things then!
11
+
12
+
{1 Ocaml-docs-ci}
13
+
14
+
I've been working with {{:https://tunbury.org/}Mark Elvers} on getting the docs
15
+
CI running using Odoc 3.0. There are quite a few changes involved, both in how
16
+
we're {{!/site/blog/2025/04/page-"ocaml-docs-ci-and-odoc-3"}building the packages}
17
+
but also how we're running odoc - it's building using [odoc_driver] rather than [voodoo] now,
18
+
and while it's looking promising now we had hit a few hurdles along the way.
19
+
20
+
We set the CI going last weekend but discovered that it was having some issues building
21
+
packages using OCaml 5.3.0. The way the builds work is that we first do a "solve" step
22
+
for each version of every package so we've got exact versions of all of the packages
23
+
required to build them. We then look through that solution to figure out the version of
24
+
OCaml required, and the (to avoid a little bit of work) we start from one of the {{:https://hub.docker.com/r/ocaml/opam}opam
25
+
docker images} for that version of OCaml.
26
+
27
+
When installing a package using opam it does a few operations that scale with the size
28
+
of the opam repository, which ends up adding around ten of seconds to the build time. When
29
+
we're building 50,000 packages, this adds up to quite a lot of time, so we short-cut this
30
+
process with the simple expedient of creating an opam-repository that only contains the
31
+
packages we need for the build. However, since we've already got a few packages installed
32
+
in the image, we need to make sure our repository contains these packages too, otherwise
33
+
opam gets thoroughly confused. My mistake was that we were missing out the `ocaml-compiler`
34
+
package, which is new in OCaml 5.3.0, which led to the builds failing. Adding this in
35
+
and kicking off the build again it's now got a lot further - at time of writing it has built
36
+
14,000 packages, there are 6,000 still building, and 1000 that have failed. If it continues
37
+
in a similar fashion, this will compare quite favourably with the docs CI that's currently
38
+
powering ocaml.org, where it has successfully built 17,000 packages, and 4,500 have failed.
39
+
40
+
Mark has been working on a different approach to the build process, which is to come up with
41
+
a new binary that doesn't do any of the [O(n)] operations and just builds the package! This
42
+
is definitely a promising direction, and I'm hoping to take a look at {{:https://github.com/mtelvers/ohc}his prototype}
43
+
soon.
44
+
45
+
Meanwhile, {{:https://choum.net}panglesd} is working on integrating this into the ocaml.org
46
+
site, and is making good progress. He spotted last week that we were overwriting the `status.json`
47
+
file that comes out of `odoc_driver` which we will use to power the redirections on ocaml.org.
48
+
One of the changes of odoc 3.0 is that we carefully put modules into a directory structure
49
+
that represents the library in which they are found. It's long been a pain that OCaml libraries
50
+
(what Ocamlfind unhelpfully calls 'packages') are not always the same name as the opam package in which they're
51
+
found. For example, the package [ocamlfind] contains the library [findlib]. So to help the user
52
+
figure out where to find the module, we're putting it into the URL of the docs, and therefore into
53
+
the breadcrumbs. The downside is that the modules are now in a different place on the website to
54
+
where they were before, so the [status.json] file is there to help with the redirections.
55
+
56
+
{1 Notebooks}
57
+
58
+
I've been working on Merlin integration with the notebooks, which has been a fun little project.
59
+
The bits that needed improving most were that merlin didn't work with toplevel-style code, and
60
+
that each cell was a separate typing context, so while you could define a function in one cell and
61
+
execute it in another, Merlin would tell you the function was undefined.
62
+
63
+
For the toplevel-style code, what I've ended up doing is to essentially strip out all of the toplevel
64
+
bits and pieces, and replace them with whitespace. So where I have a cell that looks like:
65
+
66
+
{v
67
+
# let x = 1 + 2;;
68
+
- val x : int = 3
69
+
# let y = x + 1;;
70
+
- val y : int = 4
71
+
v}
72
+
73
+
I tell Merlin that the contents are:
74
+
75
+
{v
76
+
let x = 1 + 2;;
77
+
78
+
let y = x + 1;;
79
+
80
+
v}
81
+
82
+
where I'm careful to maintain the position of the original code. This bit is working quite nicely, but only
83
+
when the code is syntactically correct, as I'm using the standard toplevel parser to figure out where the
84
+
expression ends. I think I'm going to end up needing to write a custom parser for this, so something that
85
+
will end on a [;;] but ignore them in string constants, comments and so on.
86
+
87
+
The approach I've taken for the second problem is to treat each cell as a separate module. I then write out
88
+
a [cmi] file into the virtual filesystem as [cell__id_0.cmi] and [open] all the previous
89
+
modules in 'line 0' of every cell. I then remap all of the reported locations by removing 'line 0'.
90
+
91
+
There are a number of issues with the current approaches:
92
+
1. The stripping of the toplevel bits is a little fragile, and currently only
93
+
works when the toplevel is syntactically correct. This is fairly fixable.
94
+
2. When the contents of the cells change we need to flush any caches merlin and
95
+
the compiler have.
96
+
3. An [open] statement in once cell does _not_ cause the module to be
97
+
available in the next cell.
98
+
4. A lot of cells leads to a lot of opens!
99
+
100
+
I suspect that this the 'cells as modules' approach might end up being a bit of a dead-end, so I'll have a chat with
101
+
{{:https://github.com/voodoos}Ulysse} to figure out the next experiment.
102
+
103
+
{1 Oxcaml}
104
+
105
+
I've been working on trying out oxcaml too, which has been a bit challenging. Firstly, although
106
+
Jane Street provide a version of [js_of_ocaml], the toplevel didn't work. Fortunately, my amazing
107
+
colleagues {{:https://patrick.sirref.org/}Patrick O'Ferris} and {{:https://github.com/art-w}Arthur
108
+
Wendling} spent a good chunk of time fixing this and provided an {{:https://github.com/patricoferris/opam-repository-js#with-extensions}opam repository}
109
+
with the relevant changes, without which I would have not been able to make any progress. Thanks,
110
+
guys! So my goal of making my notebooks work with it looked doable, but I almost immediately hit
111
+
more dependency issues that make it problematic to port the whole site over - including odoc and
112
+
various PPXes that I use.
113
+
114
+
I've therefore decided that I would bring forward a feature that I'd had in mind for a while -
115
+
that we could have different "backends" for the notebooks. So I'd still build the frontend using
116
+
"normal" OCaml, but the web-worker serving as the toplevel would be an oxcaml one.
117
+
118
+
Of course, it didn't work first time! After a bit of head-scratching, it turned out that the interface between the worker and the main
119
+
thread, although I'd {i almost} got it ocaml-agnostic, wasn't quite right. The way it works is
120
+
that it uses the jsonrpc protocol to communicate, and while it had marshalled the requests into
121
+
a string, it hadn't turned that final OCaml string into a Javascript string, so it was sending the
122
+
js_of_ocaml representation of a string as an object, rather than a simple string. When the frontend and
123
+
workers were built with different compilers, this ended up just failing with an obscure error, which
124
+
took a good deal of time to track down. Once that was fixed, it was just a case of making sure I
125
+
could have 2 independent 'switches' on my site - one for oxcaml and one for standard OCaml.
126
+
127
+
The upshot of all this is that I now have a semi-working version of the notebooks using oxcaml. As
128
+
an initial demonstration I ported one of my colleague {{:https://github.com/cuihtlauac}Cuihtlauac}'s
129
+
oxcaml tutorial docs to the notebook format, and it {{!/site/notebooks/oxcaml/page-"local"}works quite nicely}.
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
+25
site/blog/2025/05/oxcaml-gets-closer.mld
+25
site/blog/2025/05/oxcaml-gets-closer.mld
···
1
+
{0 OxCaml is getting closer...}
2
+
3
+
@published 2025-05-02
4
+
5
+
I joined the OxCaml weekly meeting representing Tarides for the first time this week, as
6
+
Jane Street gear up to an official release of their OxCaml compiler.
7
+
8
+
It seems that mainly what needs to be done before the release can be made is to ensure
9
+
there is some reasonable documentation for the new features, and that a reasonable number
10
+
of packages are working, so people are furiously writing and bugfixing to try and get this
11
+
ready.
12
+
13
+
As well as this though, there are some challenges of a more organisational level
14
+
that will need to be addressed to ensure the success of the project. Jane Street have long
15
+
had a public branch of their compiler, but
16
+
while they've had patches internally to ensure the tooling and other libraries work, these patches haven't
17
+
previously been made public in a usable way. In order for OxCaml to be useful, it will clearly need these
18
+
patches not only to be available, but also to be maintained and to easily allow contributions from the community --
19
+
in short, they need to be properly Open Source!
20
+
21
+
Personally, I'm looking forward to seeing their branch of {{:https://ocaml.github.io/odoc/}odoc} and
22
+
having a look to see how the modes will fit into the documentation. I'm also keen to see whether the
23
+
{{!/site/blog/2025/04/page-"this-site"}notebook features} I've been working on can be ported over to run on OxCaml!
24
+
25
+
+36
site/blog/2025/05/ticks-solved-by-ai.mld
+36
site/blog/2025/05/ticks-solved-by-ai.mld
···
1
+
{0 Solving First-year OCaml exercises with AI}
2
+
3
+
@published 2025-05-07
4
+
5
+
My colleague {{:https://toao.com}Sadiq Jaffer} and I have been working on a little
6
+
project to see how well small AI models can solve the OCaml exercises we give to
7
+
our first-year students at the University of Cambridge. Sadiq has done an excellent
8
+
{{:https://toao.com/blog/ocaml-local-code-models}write up} of our initial results,
9
+
which you should all go and read! The tl;dr though, as Sadiq writes, is that even
10
+
some of the smaller models would score top marks on these exercises!
11
+
12
+
One interesting aspect we discovered quite quickly is that we had to make the
13
+
testing feedback a little more generous than just "exception raised"! The problems
14
+
are presented as a Jupyter notebook using {{:https://github.com/akabe}akabe's} excellent OCaml kernel, with {{:https://nbgrader.readthedocs.io/en/stable/}nbgrader} to do the assessment.
15
+
Our students can see the tests that are run, and if they fail they're able to copy the
16
+
test cell out and play with their code to figure out exactly what went wrong. The AI models, however, have a
17
+
far less interactive experience, and get just 3 chances to write code that passes the
18
+
tests. We found that the performance of the models increased hugely when we adjusted
19
+
the test cells such that they clearly indicated which test failed, the results that
20
+
were expected, and the results the code actually produced.
21
+
22
+
Of course, we {{:https://anil.recoil.org/notes/claude-copilot-sandbox}already knew}
23
+
that AI models can code OCaml very well,
24
+
and we (along with the rest of the teaching world) are still ruminating on the implications of this from a pedagogical perspective.
25
+
Our plan, though, is to try and make the 'problem' worse by training these models on
26
+
more OCaml code, and see just how well we can get them to perform! It's pretty amazing,
27
+
and a little startling to know that a model that'll run pretty comfortably on my
28
+
laptop can solve these problems so well even without extra training, though given how hot it gets, I'd rather
29
+
not have the laptop on my actual lap while it's doing so!
30
+
31
+
32
+
33
+
34
+
35
+
36
+
+170
site/blog/2025/06/week23.mld
+170
site/blog/2025/06/week23.mld
···
1
+
{0 Week 23}
2
+
3
+
@x-ocaml.requires opam-format,fpath,rresult,bos
4
+
@merlinonly
5
+
@published 2025-06-09
6
+
7
+
Some brief notes on last week.
8
+
9
+
{1 Docs CI and Sherlodoc}
10
+
11
+
Anil has been working on an {{:https://tangled.sh/@anil.recoil.org/odoc-mcp}MCP server} that
12
+
searches through the output of the docs CI to find relevant packages and API information for
13
+
opam packages. For expediency, this works by scraping the HTML output, but a potentially
14
+
better solution would be to integrate properly with {{:https://doc.sherlocode.com}Sherlodoc},
15
+
{{:https://github.com/art-w/}Arthur's} code search engine.
16
+
17
+
{2 Building the index}
18
+
To make this work with the new docs CI, first we need to build a sherlodoc search database.
19
+
This involves grabbing all of the [.odocl] files that odoc produces for the latest version
20
+
of each library, copying them locally and running [sherlodoc index] on the output. Getting
21
+
{e all} of the odocl files is simple, but filtering so we only have the latest version is
22
+
slightly more complex, as we need to use [opam]'s library to make sure we're looking at the
23
+
latest versions.
24
+
25
+
The simple way to get the odocl files ends up unpacking them into the filesystem in a
26
+
directory hierarchy that matches the URL on ocaml.org, so we see files like:
27
+
28
+
{v
29
+
p/odoc/3.0.0/doc/odoc.document/odoc_document.odocl
30
+
v}
31
+
32
+
So finding the version number is a matter of listing the directories, for which I
33
+
took {{:https://github.com/ocurrent/ocaml-docs-ci/blob/4dfe7e6265610da4e0ce2a386cfbf0b8eac3d9bd/src/lib/track.ml#L58-L76}some code}
34
+
from docs CI:
35
+
36
+
{@ocaml[
37
+
type p = {
38
+
opam : OpamPackage.t;
39
+
path : Fpath.t;
40
+
}
41
+
42
+
let rec take n l =
43
+
match n, l with
44
+
| n, x::xs when n > 0 ->
45
+
x :: take (n - 1) xs
46
+
| _, _ -> []
47
+
48
+
let get_versions ~limit path =
49
+
let open Rresult in
50
+
let package = Fpath.basename path in
51
+
let mk_pkg v =
52
+
Printf.sprintf "%s.%s" package v
53
+
in
54
+
Bos.OS.Dir.contents path
55
+
>>| (fun versions ->
56
+
versions
57
+
|> List.map (fun path ->
58
+
{ opam = Fpath.basename path |> mk_pkg |> OpamPackage.of_string;
59
+
path = path })
60
+
)
61
+
|> Result.get_ok
62
+
|> (fun v ->
63
+
v
64
+
|> List.sort (fun a b ->
65
+
-OpamPackage.compare a.opam b.opam)
66
+
|> take limit)
67
+
]}
68
+
69
+
This gives us a sorted list of the versions for the package, and we can pick the first one
70
+
to get the latest version. With the output from this we can then run [sherlodoc index]
71
+
and we get a nice big (1.7 gig!) index file.
72
+
73
+
{2 Serving the index}
74
+
The next step is to serve this index file so that the MCP server can access it. The file
75
+
format is a marshalled OCaml value, so we need to have an OCaml program to read it in
76
+
and perform the search, and it'll have to be a server since the whole index needs to be
77
+
unmarshalled into memory before any search can be performed, and it would be dumb to do
78
+
this for every query.
79
+
80
+
Sherlodoc got partially integrated into odoc's code base before the 3.0 release with the
81
+
exception of the server, which was left out to avoid pulling a load of new dependencies
82
+
to odoc. Unfortunately, we didn't expose the sherlodoc libraries publicly, so we'll need
83
+
to do that in order to make anything useful with sherlodoc. In addition, odoc embeds the
84
+
version of odoc used into the odocl files, and since right now the docs CI is building
85
+
with a version of odoc that {e doesn't} expose the libraries, we might have to hack around
86
+
that in order to use those odocl files. Obviously the longer term solution is just to make
87
+
a new release of odoc with this change and update the docs CI to use that.
88
+
89
+
{1 Package to Library map}
90
+
91
+
A related quest was to assemble a map of opam package to ocamlfind library names. It's
92
+
a quirk of history that the library names that an opam package provides are not necessarily
93
+
related to the name of the package. That means that tools like [dune] have a hard time
94
+
linting projects to check that the libraries they're using are mentioned in the opam files.
95
+
Dune, of course, has resolved this be ensuring that it's an error to build a package using
96
+
dune where the library names don't start with the package name, but as dune is just one of
97
+
many OCaml build systems, the problem remains.
98
+
99
+
Since docs CI has built every version of every package, and because the Odoc 3 package
100
+
layout includes the library names in the paths to the documentation, we should be able to
101
+
produce a fairly definitive list of the libraries that each package provides, which tools
102
+
like dune can then use for this sort of lint check. We can just tweak the code above
103
+
slightly to get the library names and output a big JSON file with the mapping - or perhaps
104
+
with the exceptions to dune's rule.
105
+
106
+
I thought this would be a neat first project to try Claude Code on - a 'starter for ten' -
107
+
as it were, so I signed up to use Claude code and gave it a shot.
108
+
109
+
It handily produced a working program that did exactly what I wanted, including creating
110
+
a test directory that it used to verify the code worked. One fascinating thing I noted
111
+
as it scrolled past was that it tried to use [yojson] to write the output, but failed to
112
+
get it to work and reverted back to [printf] output. I suspect this will be due to it
113
+
finding it troublesome to figure out the various steps that need to be taken to use a
114
+
new library in a dune project, so this is something to have a play with later.
115
+
116
+
After a couple of iterations with different heuristics to disambiguate between library
117
+
names and other directories, I got a working program producing a JSON file with only
118
+
the exceptions to dune's rule. I took a look through and almost immediately found
119
+
[camlidl] suggesting it produces a library called [com]. This didn't look right at all
120
+
so I installed it and found that the library is actually named [camlidl]. The [cma] file,
121
+
though, is named [com.cma], so it looks like [odoc_driver] has a bug. Interestingly,
122
+
running [odoc_driver] locally gets the library name correct, so it's only an issue when
123
+
running it in the docs CI. {{:https://github.com/ocaml/odoc/issues/1351}Issue filed}.
124
+
125
+
{1 Further claude code experiments}
126
+
127
+
To see how well Claude Code could handle more complex tasks, I thought I'd give it a whirl
128
+
on something more like its home territory, and somewhere where I was less familiar.
129
+
I decided to ask it to write some code to make a nicer editor experience for the notebooks
130
+
project. Since the {{:https://github.com/jonludlam/jsoo-code-mirror}bindings to codemirror}
131
+
I'm using are very minimal, any new features I want to use end up with needing to write
132
+
a bunch of bindings first, and only then seeing if the feature works as I'd like. So instead
133
+
I thought I'd get claude to write the editor code for me in javascript, and then I could
134
+
make sure it works as I want and only then convert it to OCaml. This worked pretty nicely,
135
+
and I've now got a neat {{:https://jon.ludl.am/experiments/notebook-editor/notebook-editable.html}demonstration editor}
136
+
that I can use to guide the OCaml implementation.
137
+
138
+
{1 More notebook work}
139
+
The oxcaml project will be launching this week hopefully. I've been looking at Luke's
140
+
{{:https://github.com/ocaml-flambda/flambda-backend/pull/3886}Parallelism tutorial}
141
+
and have been thinking about how this will work with the online notebooks. The parallel
142
+
library works via effects, and the oxcaml branch of js_of_ocaml doesn't support effects
143
+
yet, and it might be a while before it does. However, the blog post is mainly talking
144
+
about the intricacies of the type system work that's been done to ensure the parallel
145
+
library is safe, and as such perhaps we can get a lot out of doing this online with just
146
+
Merlin.
147
+
148
+
Some early experimentation showed that the parallel library breaks the worker on load,
149
+
so we need to do something a bit more sophisticated than just 'not call exec', so I
150
+
did some work to have a mode of worker that doesn't load the cmas, just the cmis for
151
+
Merlin. This is almost there.
152
+
153
+
{1 Odoc work}
154
+
Ocaml 5.4 is just around the corner, and there's some odoc work to be done before the
155
+
release. One of the main new features that will impact odoc is the new {{:https://tyconmismatch.com/papers/ml2024_labeled_tuples.pdf}labelled tuples}
156
+
feature. Fortunately {{:https://github.com/lukemaurer}Luke Maurer} has already done a lot of work to plumb this into
157
+
odoc, so this will save us a lot of work - thanks, Luke! There's likely to be a few other
158
+
bits and pieces to do, but hopefully not too much.
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
+4
site/blog/2025/07/index.mld
+4
site/blog/2025/07/index.mld
+58
site/blog/2025/07/odoc-3-live-on-ocaml-org.mld
+58
site/blog/2025/07/odoc-3-live-on-ocaml-org.mld
···
1
+
{0 Odoc 3 is live on OCaml.org!}
2
+
3
+
@published 2025-07-14
4
+
5
+
As of today, Odoc 3 is now live on OCaml.org! This is a major update to odoc, and has
6
+
brought a whole host of new features and improvements to the documentation pages.
7
+
8
+
Some of the highlights include:
9
+
- Source code rendering
10
+
- Hierarchical manual pages
11
+
- Image, video and audio support
12
+
- Separation of API docs by library
13
+
14
+
A huge amount of work went into the {{:https://discuss.ocaml.org/t/ann-odoc-3-0-released/16339}Odoc 3.0 release}, and I'd like to thank my colleagues at
15
+
Tarides, in particular {{:https://github.com/panglesd}Paul-Elliot} and {{:https://github.com/julow/}Jules} for the work they put into this.
16
+
17
+
But the odoc release happened months ago, so why is it only going live now? So, the doc tool
18
+
itself is only one small part of getting the docs onto ocaml.org. Odoc works on the
19
+
{{:https://discuss.ocaml.org/t/cmt-cmti-question/5308}cmt and cmti} files that are produced
20
+
during the build process, and so part of the process of building docs is to build the packages,
21
+
so we have to, at minimum, attempt to build all 17,000 or so distinct versions of the packages
22
+
in opam-repository. The {{:https://github.com/ocurrent}ocurrent} tool {{:https://github.com/ocurrent/ocaml-docs-ci}ocaml-docs-ci},
23
+
which I've previously {{!/site/blog/2025/05/page-"docs-progress"}written} {{!/site/blog/2025/04/page-"ocaml-docs-ci-and-odoc-3"}about},
24
+
is responsible for these builds and in this new release has demonstrated a new approach to this task,
25
+
where we attempt to do the build in as efficient a way as possible by effectively building
26
+
binary packages once for each required package in a specific 'universe' of dependencies. For
27
+
example, many packages require e.g. {{:https://erratique.ch/software/cmdliner}cmdliner.1.3.0} to
28
+
build, and some require a specific version of OCaml too. So we'll build cmdliner.1.3.0 once
29
+
against each version of OCaml required -- but {i only once}, which is in contrast to how some
30
+
of the other tools in the ocurrent suite work, e.g. {{:https://github.com/ocurrent/opam-repo-ci}opam-repo-ci}.
31
+
Once the packages are built, we then run the new tool {{:https://ocaml.github.io/odoc/odoc-driver/index.html}odoc_driver}
32
+
to actually build the HTML docs. In addition to this, a new feature of Odoc 3 is to be able
33
+
to link to packages that are your direct dependencies - so for example, the docs of odoc
34
+
contain links to the docs of odoc_driver, even though odoc_driver depends upon odoc. This,
35
+
whilst sounding easy enough, required some radical changes in the docs ci, which I promise I will
36
+
write about later!
37
+
38
+
The builds and the generation of the docs is all done on a
39
+
single blade server, called {{:https://sage.caelum.ci.dev}sage} with 40 threads, 2 8TiB spinning drives and a 1.8TiB SSD cache, and it
40
+
produces about 1 TiB of data over the course of a couple of days. The changes required
41
+
to this part of the process since odoc 2.x were primarily myself and {{:https://tunbury.org}Mark Elvers}
42
+
43
+
Once the docs are built, how do they get onto ocaml.org? Odoc itself knows nothing about the
44
+
layout and styling of ocaml.org, so the HTML it produces isn't suitable to be just rendered
45
+
when a user requests particular docs. What happens is that odoc produces, as well as a
46
+
self-contained HTML file, a json file with the body of the page, the sidebars, the breadcrumbs
47
+
and so on as structured data, one per HTML page, which are then served from sage over
48
+
HTTP. When a user requests a particular docs page, the ocaml.org server will request that json
49
+
file from sage, then render this with the ocaml.org styling, then send it back to the user.
50
+
51
+
As odoc 3 moved a fair bit of logic from ocaml.org into odoc itself, there were quite a few
52
+
changes that needed to be made to the ocaml.org server to integrate this into the site.
53
+
This work was mostly done by {{:https://github.com/panglesd}Paul-Elliot} and myself, with a
54
+
lot of help from the {{:https://github.com/ocaml/ocaml.org?tab=readme-ov-file#maintainers}ocaml.org team},
55
+
in particular {{:}Sabine Schmaltz} and {{:https://github.com/cuihtlauac}Cuihtlauac Alvarado}.
56
+
57
+
So, quite a lot of integration and infrastructure work was required to get the new docs
58
+
site up and running, and I'm very happy to see this particular task concluded!
+194
site/blog/2025/07/retrospective.mld
+194
site/blog/2025/07/retrospective.mld
···
1
+
{0 4 months in, a retrospective}
2
+
3
+
@published 2025-07-18
4
+
5
+
Astonishingly, it's already been {i four whole months} since starting back at the university,
6
+
which I find incredibly hard to believe. I'm utterly convinced that it was only a couple of
7
+
weeks ago that I walked back into the Computer Laboratory as an SRA for the first time since
8
+
2021, but here we are, at the end of term already. Time to do a bit of
9
+
a retrospective and forward-looking plan for the next 3-4 months!
10
+
11
+
{1 What's happened?}
12
+
On wednesday this week, I had a chance to sit down with Anil, supposedly to talk about
13
+
the upcoming lecturing of 1A Foundations of Computer Science, but we ended up talking
14
+
about what I've been doing for the past few months, and where it fits into the broader
15
+
picture of the group as a whole. It was a really useful conversation, and I thought it
16
+
would be good to outline it here while it's fresh in my mind.
17
+
18
+
So then, to start, what have I been doing? What have I achieved? What have I learnt?
19
+
It's been a bit of a daunting experience, landing in a team that are already working
20
+
one hundred miles an hour on things well out of my comfort zone. I've been going to
21
+
group meetings and having lots of interesting conversations, but I've found it
22
+
difficult to make the next steps happen. One area where I've had some success is in
23
+
working with Sadiq on LLMs - in particular, getting local LLMs to solve programming
24
+
exercises that we both {{:https://toao.com/blog/ocaml-local-code-models}wrote}
25
+
{{!/site/blog/2025/05/page-"ticks-solved-by-ai"}up}. I've also been working with him
26
+
on taking the output from the docs CI and {{:https://github.com/sadiqj/odoc-llm}summarising it with LLMs} in order to
27
+
create an MCP server that would help tools like {{:https://anthropic.com/}Claude Code}
28
+
to choose OCaml packages to solve users' problems.
29
+
30
+
It's been somewhat easier, partly due to inertia, to carry on with projects that had been in flight at the time
31
+
I started. Things like getting the Odoc 3 generated docs onto ocaml.org, which is
32
+
finally complete only {{!/site/blog/2025/07/page-"odoc-3-live-on-ocaml-org"}as of this week!}.
33
+
This has taken a whole lot of time, but I'm really pleased with the end results. There's
34
+
still an awful lot of improvements that I'd like to see made, which, after drawing breath for
35
+
a couple of weeks, I'll be writing down.
36
+
37
+
An itch I'd been wanting to scratch for a long time has been to look at client-side ocaml
38
+
notebooks. I decided to make this an integral {{!/site/blog/2025/04/page-"this-site"}feature of this blog},
39
+
and I've learnt an awful
40
+
lot doing it. An important feature of this that I've been keeping in mind is the idea that
41
+
we could use the ocaml-docs-ci tool to build the libraries, which would allow us to host
42
+
a toplevel for every single package in opam-repository - allowing at best
43
+
{{:https://discuss.ocaml.org/t/an-example-for-every-ocaml-package/16953/10}interactive examples}, and at bare minimum merlin for
44
+
live type-checking and autocompletion. The important principles to keep in mind for this
45
+
are that:
46
+
47
+
- We have one 'toplevel' javascript file, and libraries and cmis are dynamically loaded
48
+
- The interface between the frontend and the worker must not rely on a matched pair, e.g.
49
+
an OCaml-5.3-compiled frontend might be talking to an OCaml-4.08-compiled worker thread -
50
+
or even an oxcaml one!
51
+
52
+
I have this all working on my blog, where I have both an oxcaml worker and a standard ocaml
53
+
worker and they both dynamically load in libraries and cmis as specified on the page.
54
+
55
+
I've also supervised a 1A course for the first time - {{:https://www.cl.cam.ac.uk/teaching/2425/IntroProb/}Introduction to Probability},
56
+
and I've done some marking for the 1A Foundations of Computer Science.
57
+
58
+
Something that I'd been expecting to do a lot on was work with oxcaml, but as the release
59
+
happened later than anticipated and it coinciding with the marking and supervising, I've
60
+
not done quite as much of this as I had thought I would. In addition, I had anticipated
61
+
working on Odoc to start implementing the new features of oxcaml, but to avoid
62
+
duplicating effort I've been waiting for the patches that have already been written at Jane Street to at least
63
+
get odoc to compile, which have taken longer than I had hoped to get to me.
64
+
65
+
{1 What's next?}
66
+
With that in mind, Anil and I then talked about the bigger picture, as those of you who
67
+
know Anil will be entirely unsurprised to hear! In particular, how will we be weaving
68
+
the various threads of these activites - the teaching of OCaml, the large-scale (for OCaml)
69
+
CI work, the LLMs and Oxcaml work together to form a coherent whole? How do I find a
70
+
balance between them and ensure that we find {{:https://arxiv.org/abs/1106.0848}synergies}
71
+
as opposed to pulling in different directions? How do make sure what we're doing helps us
72
+
navigate the upending of the nature of development that agentic coding is bringing?
73
+
74
+
{2 Efficient and reusable CI}
75
+
A clear and obvious area where we'll be able to see real progress is to extract from docs CI the logic that I've been
76
+
using to do efficient builds of packages. As I previously {{!/site/blog/2025/07/page-"odoc-3-live-on-ocaml-org"}wrote about},
77
+
the new CI system is far more efficient than some of the other ocurrent-based pipelines,
78
+
and it would save a huge amount of compute time if we were to take this tech and apply it
79
+
elsewhere.
80
+
81
+
So, how might we take what we've got and produce something useful to everyone? We need to
82
+
take a hammer to the fracture points of the docs CI service and split it into individually
83
+
useful parts. Here are some next steps as I see them now. Let's take the solver out of docs
84
+
CI, and have a service whose sole job is to create a repository of
85
+
up-to-date solutions for all versions of all packages in opam-repository. These are the data
86
+
that allow us to build the tree of package builds.
87
+
88
+
Next, turn these solutions into one giant build. Perhaps a script? Maybe a giant buildkit dockerfile? This is
89
+
very similar to Mark Elvers' {{:https://github.com/mtelvers/ohc}day10} project. We can
90
+
get this running on a big machine and just see how fast we can build everything. The key thing
91
+
here is that it should be {e trivial} to run this on a linux box. A raspberry pi or a 768-core
92
+
behemoth with 3TiB of ram. Just how fast {e can} we get it going? It's already building in a
93
+
couple of days using {{!/site/blog/2025/07/page-"odoc-3-live-on-ocaml-org"}sage}, but that's
94
+
using ocurrent/obuilder, which isn't quite the right tool for the job, and on a relatively
95
+
puny machine. Can we do it in an hour? 10 minutes? Certainly the incrememntal builds ought
96
+
to be done in seconds. What's the limit?
97
+
98
+
These tools can then be used as the foundation for other CI systems. For opam-repo-ci, where we
99
+
should be able to do the builds for a new package incredibly quickly. For opam-health-check, where
100
+
we currently build foundational packages like dune and findlib {i thousands of times} per run.
101
+
102
+
Once we've got the packages built, docs CI is simply a pass over the top of the built artifacts.
103
+
ocaml-docs-ci already demonstrates this - it only takes a few hours to rebuild all the docs when
104
+
a new version of odoc is released, but in a way that only benefits docs! All the CI systems should
105
+
be able to use this.
106
+
107
+
We should also then be able to run js_of_ocaml on the libraries to build to infrastructure needed
108
+
for the per-package toplevels for ocaml.org that I mentioned above. Each of these steps should be
109
+
separate stages in a pipeline - one where each step produces artifacts for the next to consume.
110
+
111
+
When we mix in some of the projects that other people in the team are working on, like David's
112
+
work on {{:https://www.dra27.uk/blog/}relocatable OCaml}, we've got something that might be able
113
+
to form a basis for a binary cache for Dune Package Management, particularly when we involve
114
+
Ryan's {{:https://ryan.freumh.org/papers.html#2025-arxiv-hyperres}Hyperres} paper so we might
115
+
check that dependencies from outside of the OCaml universe are correct. Can we use {{:https://github.com/quantifyearth/shark}Patrick and Michael's shark}
116
+
to do the build steps? Can we use these images to serve up toplevels for ocaml.org that are
117
+
{e real toplevels} rather than javascript toplevels? Can we use these build environments to
118
+
do help with reinforcement learning to train LLMs on OCaml code? There are a lot of interesting directions
119
+
to take this work.
120
+
121
+
{2 Other projects}
122
+
There are, of course, other responsibilities that I have. Some of these I'll be able to fit
123
+
in with the theme above, and some - well - maybe I'll have to figure out how to delegate them,
124
+
a skill that I am not particularly good at, but one that I feel I should learn!
125
+
126
+
{3 Teaching}
127
+
A looming, terrifying, but tremendously exciting opportunity is teaching of 1A Foundations
128
+
of Computer Science. This is amongst the first courses we teach our incoming undergraduates,
129
+
currently lectured by {{:https://www.cl.cam.ac.uk/teaching/2425/FoundsCS/}Anil}. As he's on
130
+
sabbatical this year, he
131
+
has asked me to step up and lecture it. This is definitely not one for delegation!
132
+
133
+
The immediate question, partly raised by my work with Sadiq, is:
134
+
what do we do about LLMs? How should we adjust our teaching to take into account the existence
135
+
of these tools? We had a very interesting chat earlier in the term with Professor {{:https://eecs.iisc.ac.in/people/prof-viraj-kumar/}Viraj Kumar}
136
+
from {{:https://eecs.iisc.ac.in/}IISc} who was visiting Cambridge earlier this year. He's been
137
+
{{:https://dl.acm.org/doi/10.1145/3724363.3729100}working on this question} for a while now, and I hope to have some more conversations with him
138
+
over the summer.
139
+
140
+
{3 Odoc paper}
141
+
An area where I've really made a shockingly small amount of progress is to write up all the
142
+
work that's gone into Odoc over the past 6 (!!!) years.
143
+
144
+
{3 Odoc notebooks}
145
+
This needs to be tidied up and a v0.1 released. In particular, the work on js_top_worker
146
+
might well be shared with Arthur's {{:https://github.com/art-w/x-ocaml}x-ocaml} for a unified
147
+
toplevel experience.
148
+
149
+
{3 AI work}
150
+
I'd like to carry on the work I've started with Sadiq on the interaction of LLMs with OCaml.
151
+
Getting the package search to work sensibly for an MCP server is first on the list, but also
152
+
doing some reinforcement learning to improve specifically the perfomance on OCaml is very
153
+
interesting, but not something I've managed to carve out the time for yet. Something along
154
+
the lines of {{:https://arxiv.org/abs/2504.21798}swesmith} but adapted for OCaml.
155
+
156
+
{3 Oxcaml Odoc}
157
+
Odoc needs to have some work done on it to support the new work that's gone into oxcaml,
158
+
for example documenting of the modes. This is something I do expect to be working on soon.
159
+
160
+
{3 Dune and odoc}
161
+
Work needs to be done on the dune rules for odoc, which currently only support the feature-set
162
+
in odoc 2.x. Paul-Elliot has {{:https://github.com/ocaml/dune/pull/11716}done some work on this},
163
+
but much more needs to be done.
164
+
165
+
{3 Further general odoc work}
166
+
- Better source rendering
167
+
- Syntax for linking to source
168
+
- Custom tags (used in odoc_notebook)
169
+
- Web-native rendering, for embedding odoc in a website
170
+
- Unifying paths and cpaths (https://github.com/jonludlam/odoc/tree/parameterised-paths)
171
+
172
+
{1 What to {i actually} do?}
173
+
There are a lot of things in the above list. I'm not sure yet how I manage to figure out
174
+
what I actually end up doing, and how that helps me to help Tarides, to fit in as a
175
+
useful member of the EEG group, and to make sure I'm doing what's right for my own
176
+
future. I feel the core project of the CI work will help everyone, but slotting the other
177
+
work into the bigger picture will require some careful thought.
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
+132
site/blog/2025/07/week27.mld
+132
site/blog/2025/07/week27.mld
···
1
+
{0 Weeks 24-27}
2
+
3
+
@published 2025-07-07
4
+
5
+
It's been a busy few weeks. There's been exam marking for the 1A Foundations of Computer
6
+
Science course, an Odoc release to plan, and some really interesting new work on using
7
+
LLMs to summarise OCaml documentation. This post is about anaspect of that last one that
8
+
I found particularly interesting.
9
+
10
+
{1 odoc-llm}
11
+
Sadiq and I have been {{:https://toao.com/blog/ocaml-local-code-models}looking at using LLMs} to summarise the documentation produced by Odoc.
12
+
The idea is to get a concise summary of the purpose of each module, so that we can quickly identify
13
+
which modules are relevant to a particular task.
14
+
15
+
For testing this, we need to see how it works on different types of libraries. The first axis
16
+
I wanted to test on goes between 'well documented' and 'poorly documented', and so I need at
17
+
least two libraries on opposite ends of the spectrum.
18
+
19
+
For the 'well documented' case, I chose [cmdliner]. It's a library that I almost always have
20
+
to look at the docs for when I'm using it, as, despite using it many many times, the interface
21
+
doesn't seem to stick in my head.
22
+
23
+
For the 'poorly documented' case, I chose [odoc] itself, somewhat ironically. In defence of
24
+
the odoc authors (me included!), the libraries it provides are simply there for code
25
+
organisation and aren't meant to be consumed outside of the tool itself!
26
+
27
+
{2 The approach}
28
+
The output from Odoc is a set of HTML files, one per module/module type/class/etc., containing
29
+
the documentation for that item. Our first take on this was to parse the HTML files and
30
+
extract the text content, which we then fed to an LLM to summarise. However, this was pretty
31
+
awkward, so we decided to try a PR that {{:https://github.com/ocaml/odoc/pull/1341}davesnx recently made to Odoc} to output
32
+
markdown instead of HTML.
33
+
34
+
We look for leaf modules that don't contain any submodules, and start by summarising
35
+
those, then move onto the parent modules, splicing in the summaries of the children, and so
36
+
on, up to the top-level modules. We then move on to summarising the whole library, which
37
+
usually is just a single namespace module, but occasionally is a group of top-level
38
+
modules.
39
+
40
+
One of the early prompts for the module [Cmdliner.Term.Syntax] looked roughly as follows:
41
+
42
+
{v
43
+
You are an expert OCaml developer. Write a 2-3 sentence description focusing on:
44
+
- The specific operations and functions this module provides
45
+
- What data types or structures it works with
46
+
- Concrete use cases (avoid generic terms like "utility functions" or "common operations")
47
+
48
+
Do NOT:
49
+
- Repeat the module name in the description
50
+
- Mention "functional programming patterns" or "code clarity"
51
+
- Use filler phrases like "provides functionality for" or "collection of functions"
52
+
- Describe how it works with other modules
53
+
54
+
Module: Syntax
55
+
Module Documentation: let operators.
56
+
( let+ ) is map.
57
+
( and* ) is product.
58
+
- val (let+) : 'a t -> ('a -> 'b) -> 'b t (* ( let+ ) is map. *)
59
+
- val (and+) : 'a t ->
60
+
'b t ->
61
+
('a * 'b) t (* ( and* ) is product. *)
62
+
v}
63
+
64
+
and the output using a small model (qwen3-30b-a3b) was:
65
+
66
+
{v
67
+
"The module provides (let+) for applying functions to values within a context and (and+) for combining two contexts into a product. It operates on applicative
68
+
structures like option, list, or custom types that support these operations. For example, it enables sequential transformation of values in a context or
69
+
pairing elements from two separate contexts."
70
+
v}
71
+
72
+
There are quite a few issues with the input here. Firstly, we've only given the
73
+
module name, not the full path. Secondly, there's nothing to let the model know
74
+
what [t] might be, so it has decided it's a completely generic type. It also has
75
+
no idea about the context in which this module was found, so it has no idea
76
+
that it's part of a command-line processing library. By fixing these issues, we
77
+
end up with the prompt:
78
+
79
+
{v
80
+
You are an expert OCaml developer. Write a 2-3 sentence description focusing on:
81
+
- The specific operations and functions this module provides
82
+
- What data types or structures it works with
83
+
- Concrete use cases (avoid generic terms like "utility functions" or "common operations")
84
+
85
+
Do NOT:
86
+
- Repeat the module name in the description
87
+
- Mention "functional programming patterns" or "code clarity"
88
+
- Use filler phrases like "provides functionality for" or "collection of functions"
89
+
- Describe how it works with other modules
90
+
91
+
Module: Cmdliner.Term.Syntax
92
+
93
+
Ancestor Module Context:
94
+
- Cmdliner: Declarative definition of command line interfaces.
95
+
Consult the tutorial, details about the supported command line syntax and examples of use.
96
+
Open the module to use it, it defines only three modules in your scope.
97
+
- Cmdliner.Term: Terms.
98
+
A term is evaluated by a program to produce a result, which can be turned into an exit status. A term made of terms referring to command line arguments implicitly defines a command line syntax.
99
+
100
+
Module Documentation: let operators.
101
+
- val (let+) : 'a Cmdliner.Term.t -> ('a -> 'b) -> 'b Cmdliner.Term.t (* ( let+ ) is map. *)
102
+
- val (and+) : 'a Cmdliner.Term.t ->
103
+
'b Cmdliner.Term.t ->
104
+
('a * 'b) Cmdliner.Term.t (* ( and* ) is product. *)
105
+
v}
106
+
107
+
The output of this improved prompt is much better:
108
+
109
+
{v
110
+
The module provides operators to map and combine terms, which represent command line argument parsers and their results. `let+` transforms a parsed argument
111
+
into a new value, while `and+` merges two independent arguments into a tuple. These enable building structured command line interfaces, such as parsing a
112
+
filename and a flag simultaneously, then combining them into a configuration record.
113
+
v}
114
+
115
+
It still occasionally incorrectly decides that this module provides monadic combinators
116
+
rather than applicative, but this is where we get better results from using a larger model.
117
+
118
+
There are quite a few other issues that we've fixed - for example, treating module types
119
+
differently than modules, and a bug where infix operators were being omitted from the
120
+
documentation. In one case, it uncovered a bug in the markdown generator where includes
121
+
weren't getting expanded, which I got {{:https://github.com/jonludlam/odoc/commit/926cca100c307818e57281c3d40e98f1975f0f95}Claude to fix}.
122
+
My {i modus operandi} has essentially been to look at the output for the
123
+
test packages, find a summary that looks bonkers, and then look back at the prompt to find
124
+
that, indeed, the input was missing some crucial information.
125
+
126
+
One thing I'd quite like to try is to re-open a {{:https://github.com/ocaml/odoc/pull/655}PR that Drup made} as
127
+
an April Fool's joke back in 2001, which ended up outputting OCaml formatted code.
128
+
This is actually pretty close to what we might want to give to the LLM - though we'd
129
+
probably format the comments as markdown, and we'd be replacing types with fully-qualified
130
+
types as above. Funny how things turn out!
131
+
132
+
+87
site/blog/2025/07/week28.mld
+87
site/blog/2025/07/week28.mld
···
1
+
{0 Week 28}
2
+
3
+
@published 2025-07-14
4
+
@x-ocaml.requires caqti.platform,mariadb
5
+
6
+
{1 OCaml MCP server}
7
+
Last week I got the summarisation to the point where it felt useful to run it across all the modules in opam.
8
+
With this completed we then got to try out the MCP server to see how useful it would be in practice.
9
+
10
+
One of the first queries {{:https://anil.recoil.org/}Anil} tried was to ask it which libraries would be most
11
+
useful for "date time parsing and formatting". We were surprised to see that the first two libraries it returned were [caqti] and [mariadb],
12
+
specifically mentioning the module {!Caqti_platform.Conv} and {!Mariadb.S.Time}. While these do indeed provide
13
+
the required functionality, they're probably not the right libraries to provide this. It's going to be tricky
14
+
to decide this in the MCP server, so we should probably be leaving it up to the LLM to decide amongst them on
15
+
the client. However, for very general queries we might end up with a large number of matching libraries, so
16
+
we'll need to have a limit on the number of packages returned, which implies some form of ranking.
17
+
18
+
One way we can do this is by using the occurrences code in odoc. The idea is that we examine module implementation files
19
+
(ie, ml rather than mli files), and counts the number of times the code uses values, types and other identifiers from
20
+
other libraries. We can then aggregate these counts over all packages in opam repository and use it as an effective
21
+
marker of popularity, which allows us to rank the results by popularity and only return the top N results.
22
+
23
+
We're not currently using the occurrences for anything, so I wasn't especially surprised to find that it's not
24
+
working as intended. There were a number of issues:
25
+
26
+
- The occurrences output file was being written at a path not within the package dir, so it wasn't being persisted.
27
+
- The CLI interface for generating occurrences works by providing a directory containing the odocl files, but we
28
+
were only providing the top-level directory and it wasn't recursively searching.
29
+
- Once the occurrences were captured, the aggregation step used the full identifier of the value being aggregated,
30
+
meaning that, for example, {!List.length} in OCaml 5.3 was counted separately from {!List.length} in OCaml 4.14.
31
+
32
+
All of these issues are with code in the odoc repository, which, as it happens, also needs a release soon to ensure
33
+
that it works with the imminent launch of OCaml 5.4. During the week, before I discovered the problems above, I had
34
+
attempted to make a release of Odoc 3.1, but there was a license kerfuffle that, when combined with the issues in
35
+
the occurrences code, gave me enough cause to pull the release.
36
+
37
+
Before I try to make the release again, this time I'll be running the release candidate with docs-ci, and checking
38
+
that the occurrences make sense. I set this running on Friday afternoon, and it had completed by Friday evening,
39
+
so it's actually pretty quick to rerun odoc on the 15,000 or so packages required for ocaml.org.
40
+
41
+
{1 Trouble with this blog}
42
+
In other news, in trying to post my blog at the beginning of the week, I was stymied a little by the changes in
43
+
oxcaml. I had been using a custom opam-repository forked from the official oxcaml one, because I needed a patched
44
+
js_of_ocaml in order to fix the toplevel code. I had hoped this would mean that I could update it on my schedule,
45
+
rather than being at the mercy of upstream changes. Unfortunately though, the download URL for ocaml-flambda wasn't
46
+
pointing at an immutable commit, so when I tried it I got a checksum error. So I ended up trying to rebase the
47
+
changes onto the latest oxcaml opam-repository, which didn't go well at all. The version numbers had all changed,
48
+
which in opam means that files are in different directories, so git got thoroughly confused. On top of that, because
49
+
the js_of_ocaml repository has multiple packages in it, whereas opam repository has a directory per-package, we end
50
+
up having multiple copies of the patches. So in the end I've just committed all the patches to a git repo on github,
51
+
and pinned it in the Dockerfile that builds this site.
52
+
53
+
What would be handy is a way to apply the patches in a package in opam repository to and from a git repository,
54
+
similar to quilt/guilt. We don't quite have all of the pieces to do this, as although we have a download URL and
55
+
often a dev-repo, I don't believe we currently have a way to get the base commit of that repository.
56
+
57
+
{1 Oxcaml continues}
58
+
We had a meeting on Thursday with Jane Street on the next steps for oxcaml. There are a number of areas in which
59
+
JS are keen for us to help out with.
60
+
- Playgrounds - both javascript and docker-based. The playground on the oxcaml website right now uses github
61
+
codespaces, which works nicely but currently takes an absolute age to start up. We can almost certainly improve
62
+
this by building docker images and pushing them to the docker hub, rather than building oxcaml from scratch
63
+
when starting the codespace. There's also interest in the javascript playgrounds, which can serve a slightly
64
+
different purpose than the docker-based one, more limited in how it can be used, but without requiring someone
65
+
to spin up a full docker container.
66
+
- Documentation - Odoc has had some patches to run on oxcaml, but there's no support for documenting many of the
67
+
new features yet, including modes. We've got to do some experiments here to see what the best way is to show
68
+
the new type-system features in the generated docs. There were some suggestions of using javascript to show/hide
69
+
the modes, for example.
70
+
- Improvements in Merlin - again this is an area ripe for investigation. In particular, how do we best expose the
71
+
new features of the type system for users? What's needed here is user feedback from people who are actually using
72
+
oxcaml to build real projects.
73
+
- Better error messages - OCaml has been getting improved error messages with each release, but there's still
74
+
room for improvement, and the new features of the type system in particular have many different failure modes.
75
+
Again, we need user feedback to understand the pain points and improve the error messages accordingly.
76
+
77
+
{1 Next week}
78
+
Next week, the plan is to:
79
+
- Check the occurrences from docs-ci, and integrate into the MCP server
80
+
- Talk to {{:https://tunbury.org/}Mark} about building the docker image for the oxcaml playground
81
+
- Tidy up the {!Js_top_worker} code so it can be used in the javascript oxcaml playground
82
+
- Release Odoc 3.1
83
+
84
+
85
+
86
+
87
+
+4
site/blog/2025/08/index.mld
+4
site/blog/2025/08/index.mld
+27
site/blog/2025/08/ocaml-lsp-mcp.mld
+27
site/blog/2025/08/ocaml-lsp-mcp.mld
···
1
+
{0 Using ocaml-lsp-server via an MCP server}
2
+
3
+
@published 2025-08-27
4
+
5
+
Here's a quick post on how to get the OCaml Language Server (ocaml-lsp-server) working with an MCP server.
6
+
7
+
We're going to use {{:https://github.com/isaacphi}issacphi}'s adapter for LSP servers, which is written in go. So install go, and then:
8
+
9
+
{@bash[
10
+
# go install github.com/isaacphi/mcp-language-server@latest
11
+
]}
12
+
13
+
Once that's done, make sure you've got `ocaml-lsp-server` installed in your switch:
14
+
15
+
{@bash[
16
+
# opam install ocaml-lsp-server
17
+
]}
18
+
19
+
Then add the MCP config for claude where you want to run it:
20
+
21
+
{@bash[
22
+
# claude mcp add ocamllsp -s local -t stdio -- /Users/jon/go/bin/mcp-language-server -workspace . -lsp ocamllsp
23
+
]}
24
+
25
+
It'd be nice to get this working `globally` - that is, with `-s user` - but I haven't been able to get that to work yet.
26
+
27
+
+544
site/blog/2025/08/ocaml-mcp-server.mld
+544
site/blog/2025/08/ocaml-mcp-server.mld
···
1
+
{0 An OCaml MCP server}
2
+
3
+
@published 2025-08-20
4
+
5
+
6
+
LLMs are proving themselves superbly capable of a variety of coding
7
+
tasks, having been trained against the enormous amount of code,
8
+
tutorials and manuals available online. However, with smaller
9
+
languages like OCaml there simply isn't enough training material out
10
+
there, particularly when it comes to new language features like {{:
11
+
https://ocaml.org/manual/5.3/effects.html} effects} or new packages
12
+
that haven't had time to be widely used. With my colleagues {{:
13
+
https://anil.recoil.org/} Anil}, {{: https://ryan.freumh.org/} Ryan}
14
+
and {{: https://toao.com/} Sadiq} we've been exploring ways to {{:
15
+
https://anil.recoil.org/notes/cresting-the-ocaml-ai-hump} improve this
16
+
situation}. One way we can mitigate these challenges is to provide a
17
+
Model Context Protocol ({{: https://modelcontextprotocol.io} MCP})
18
+
server that's capable of providing up-to-date info on the current
19
+
state of the OCaml world.
20
+
21
+
The {{: https://docs.anthropic.com/en/docs/mcp} MCP specification} was
22
+
released by Anthropic at the end of last year. Since then it has
23
+
become an astonishingly popular mechanism for extending the
24
+
capabilities of LLMs, allowing them to become incredibly powerful
25
+
agents capable of much more than simply chatting. There are now a huge
26
+
variety of MCP servers, from one that provides {{:
27
+
https://github.com/r-huijts/firstcycling-mcp} professional cycling
28
+
data} to one that can {{:
29
+
https://github.com/GongRzhe/Gmail-MCP-Server} do your email}. The {{:
30
+
https://github.com/punkpeye/awesome-mcp-servers} awesome mcp server
31
+
list} already lists hundreds, and these are just the {e awesome} ones!
32
+
33
+
I've been working with {{: https://toao.com/} Sadiq} to make an {{:
34
+
https://github.com/sadiqj/odoc-llm/} MCP server for OCaml}, with an
35
+
initial focus on building it such that it can be hosted for everyone
36
+
rather than something that is run locally. Our plan is to start with a
37
+
service that can help with choosing OCaml libraries, by taking
38
+
advantage of the work done by {{:
39
+
https://github.com/ocurrent/ocaml-docs-ci/} ocaml-docs-ci} which is
40
+
the tool used to generate the documentation for all packages in {{:
41
+
https://github.com/ocaml/opam-repository} opam-repository} and is
42
+
served by {{: https://ocaml.org/} ocaml.org}. As well as producing
43
+
HTML docs, we can also extract a number of other formats from the
44
+
pipeline, including a newly created {{:
45
+
https://github.com/ocaml/odoc/pull/1341} markdown backend}. Using
46
+
this, we can get markdown-formatted documentation for the every
47
+
version of every package in the OCaml ecosystem.
48
+
49
+
{1 Semantic searching}
50
+
51
+
The first thing we focused on was being able to do a {e semantic
52
+
search} over the whole OCaml ecosystem. To do this, we're using {{:
53
+
https://huggingface.co/spaces/hesamation/primer-llm-embedding} LLM
54
+
embeddings}, for which we need some natural-language description to
55
+
seach through.
56
+
57
+
The documentation produced by [ocaml-docs-ci] is generated per library
58
+
module using {{: https://github.com/ocaml/odoc} odoc}, relying on the
59
+
package author to provide documentation comments for each element in
60
+
the signature. However, even if the package authors {e hasn't}
61
+
provided any documentation, we can still see the types, values,
62
+
modules and so on that the library exposes, and this is often enough
63
+
to get a good idea of what the module does. We then take these
64
+
documentation pages, which are formatted in markdown, and summarise
65
+
them via an LLM at the module level. This is done hierarchically, so
66
+
we start with the 'deepest' modules, and then insert their summaries
67
+
into the text of their parent module, then summarise those and so on.
68
+
We found it useful to include the names and {{:
69
+
https://ocaml.github.io/odoc/odoc/odoc_for_authors.html#preamble}
70
+
preambles} of the ancestor modules when doing the summarisation to
71
+
give additional context to the LLM. For example, here is the prompt
72
+
generated for a submodule of the {{:
73
+
https://erratique.ch/software/astring} astring} library:
74
+
75
+
{@markdown[
76
+
Module: Astring.String.Ascii
77
+
78
+
Ancestor Module Context:
79
+
- Astring: Alternative `Char` and `String` modules. Open the module to
80
+
use it. This defines one value in your scope, redefines the `(^)`
81
+
operator, the `Char` module and the `String` module. Consult the
82
+
differences with the OCaml `String` module, the porting guide and a
83
+
few examples.
84
+
- Astring.String: Strings, `substrings`, string sets and maps. A
85
+
string `s` of length `l` is a zero-based indexed sequence of `l`
86
+
bytes. An index `i` of `s` is an integer in the range [`0`;`l-1`], it
87
+
represents the `i`th byte of `s` which can be accessed using the
88
+
string indexing operator `s.[i]`.
89
+
Important. OCaml's `string`s became immutable since 4.02. Whenever
90
+
possible compile your code with the `-safe-string` option. This module
91
+
does not expose any mutable operation on strings and assumes strings
92
+
are immutable. See the porting guide.
93
+
94
+
Module Documentation: US-ASCII string support.
95
+
References.
96
+
97
+
## Predicates
98
+
- val is_valid : string -> bool (* `is_valid s` is `true` iff only for
99
+
all indices `i` of `s`, `s.[i]` is an US-ASCII character, i.e. a
100
+
byte in the range [`0x00`;`0x7F`]. *)
101
+
102
+
## Casing transforms
103
+
The following functions act only on US-ASCII code points that is on
104
+
bytes in range [`0x00`;`0x7F`], leaving any other byte intact. The
105
+
functions can be safely used on UTF-8 encoded strings; they will of
106
+
course only deal with US-ASCII casings.
107
+
108
+
- val uppercase : string -> string (* `uppercase s` is `s` with
109
+
US-ASCII characters `'a'` to `'z'` mapped to `'A'` to `'Z'`. *)
110
+
- val lowercase : string -> string (* `lowercase s` is `s` with
111
+
US-ASCII characters `'A'` to `'Z'` mapped to `'a'` to `'z'`. *)
112
+
- val capitalize : string -> string (* `capitalize s` is like
113
+
`uppercase` but performs the map only on `s.[0]`. *)
114
+
- val uncapitalize : string -> string (* `uncapitalize s` is like
115
+
`lowercase` but performs the map only on `s.[0]`. *)
116
+
117
+
## Escaping to printable US-ASCII
118
+
- val escape : string -> string (* `escape s` is `s` with: *)
119
+
- val unescape : string -> string option (* `unescape s` unescapes
120
+
what `escape` did. The letters of hex escapes can be upper, lower or
121
+
mixed case, and any two letter hex escape is decoded to its
122
+
corresponding byte. Any other escape not defined by `escape` or
123
+
truncated escape makes the function return `None`. *)
124
+
- val escape_string : string -> string (* `escape_string s` is like
125
+
`escape` except it escapes `s` according to OCaml's lexical
126
+
conventions for strings with: *)
127
+
- val unescape_string : string -> string option (* `unescape_string`
128
+
is to `escape_string` what `unescape` is to `escape` and also
129
+
additionally unescapes the sequence `"\\'"` (`0x5C,0x27`) to `"'"`
130
+
(`0x27`). *)
131
+
]}
132
+
133
+
where clearly the package author has provided excellent documentation
134
+
comments. This is then passed to an LLM which generated the following
135
+
description:
136
+
137
+
{[
138
+
This module provides functions to check if a string contains only
139
+
US-ASCII characters, convert case for ASCII letters, and escape or
140
+
unescape strings using ASCII conventions. It operates on standard
141
+
OCaml strings, treating them as sequences of bytes, and ensures
142
+
compatibility with UTF-8 encoded strings when transforming case. Use
143
+
cases include sanitizing input for ASCII-only protocols, preparing
144
+
strings for environments requiring strict ASCII formatting, and
145
+
handling escaped string representations in configuration or
146
+
serialization contexts.
147
+
]}
148
+
149
+
Once we have these natural language descriptions, we can generate
150
+
embeddings for them to allow for semantic search amongst all modules
151
+
in opam.
152
+
153
+
In addition to the module descriptions, we also generate similar
154
+
natural-language descriptions of the {e package} as a whole, by taking
155
+
the README from the package and summarising it similarly. Where there
156
+
is no README, we summarise the summaries of the modules of the
157
+
libraries, so we're always able to generate some text description of
158
+
the entire package.
159
+
160
+
To help with the ranking, we're also using a measure of popularity for
161
+
both modules and packages. For packages, we're using the number of
162
+
reverse dependencies in opam as a proxy for popularity, and for
163
+
modules, we're using the "occurrences" generated as part of the docs
164
+
build. These \[occurrences\] are a count of how often modules are used
165
+
in other modules, and are calculated by looking at the compiled
166
+
\[cmt\] files and resolving references to external modules using
167
+
odoc's internal logic and counting them.
168
+
169
+
Once we have both the module and package summaries, we generate an
170
+
embedding of the descriptions to allow for a semantic search to be
171
+
performed efficiently. We're using this in two ways - to search for
172
+
packages for broad queries of functionality, which just uses the
173
+
package summaries, and for more specific queries to search for modules
174
+
within packages.
175
+
176
+
For the module search, if the packages to search in haven't been
177
+
specified, we search for both modules and packages and then combine
178
+
the results. This is particularly helpful when the search is for
179
+
generic functionality that might be found in more specific packages.
180
+
For example, a module-only search for the term "time and date
181
+
manipulation functions" returns the strongest match with a {{:
182
+
https://ocaml.org/p/caqti/2.2.4/doc/caqti.platform/Caqti_platform/Conv/index.html}
183
+
module from caqti}, which, as caqti is a library for talking to
184
+
relational databases, might not be what the user is looking for.
185
+
186
+
We then put these search tools into an MCP server, along with a little
187
+
more functionality. The server currently provides these five
188
+
functions: {ol {+ Search for OCaml packages }{+ Search for OCaml
189
+
modules (optionally within packages) }{+ Get the summary description
190
+
of a package }{+ Get the raw Markdown docs for a module (optionally
191
+
within a package) }{+ Search using sherldoc }}
192
+
193
+
The first 2 use the LLM-generated summaries as described above, and
194
+
the last is using {{: https://github.com/art-w/} Arthur's} {{:
195
+
https://github.com/art-w/sherlodoc} sherlodoc tool} which can do
196
+
various searches, including type-based search, across the output of
197
+
the {{: https://github.com/ocurrent/ocaml-docs-ci} ocaml-docs-ci}.
198
+
199
+
{1 Example searches}
200
+
201
+
The following are the results from some example package searches: {ul
202
+
{- "HTTP client" }}
203
+
204
+
{@nolang[
205
+
#1 - http (v6.1.1)
206
+
Similarity: 0.7593
207
+
Reverse Dependencies: 407
208
+
Combined Score: 0.6588
209
+
Description: This package provides a comprehensive OCaml library for
210
+
building HTTP clients and servers with support for multiple
211
+
asynchronous programming model s. It enables developers to implement
212
+
efficient, portable HTTP services using different backends such as
213
+
Lwt, Async, Eio, and JavaScript, making it suitable for both Unix and
214
+
browser environments. The library emphasizes performance, modularity,
215
+
and interoperability, allowing custom backend implementations and
216
+
seamless in tegration with other OCaml libraries. It is commonly used
217
+
in web services, API clients, standalone microkernels, and
218
+
OCaml-to-JavaScript compilations for web app lications.
219
+
220
+
#2 - cohttp (v6.1.1)
221
+
Similarity: 0.7377
222
+
Reverse Dependencies: 403
223
+
Combined Score: 0.6435
224
+
Description: This package provides a comprehensive library for
225
+
building HTTP clients and servers in OCaml. It supports multiple
226
+
asynchronous programming models and backends, enabling flexible
227
+
development across different runtime environments. The library offers
228
+
efficient handling of HTTP/1.1 and HTTPS, with portable pa rsing and
229
+
modular architecture. It is widely used for web services, API clients,
230
+
and standalone network applications.
231
+
232
+
#3 - cohttp-lwt-unix (v6.1.1)
233
+
Similarity: 0.7089
234
+
Reverse Dependencies: 338
235
+
Combined Score: 0.6212
236
+
Description: This package provides an implementation of the Cohttp
237
+
library using the Lwt asynchronous programming framework with Unix
238
+
bindings. It enables buil ding efficient HTTP clients and servers in
239
+
OCaml, supporting both synchronous and asynchronous network
240
+
operations. The package handles core HTTP functionality, i ncluding
241
+
request and response parsing, connection management, and HTTPS support
242
+
via OCaml-TLS. It is suitable for applications requiring
243
+
high-performance web ser vices, microservices, or networked
244
+
applications in the OCaml ecosystem.
245
+
246
+
#4 - cohttp-lwt (v6.1.1)
247
+
Similarity: 0.7067
248
+
Reverse Dependencies: 367
249
+
Combined Score: 0.6207
250
+
Description: This package provides a comprehensive library for
251
+
building HTTP clients and servers in OCaml, supporting multiple
252
+
asynchronous programming models. It enables developers to implement
253
+
efficient, portable HTTP services with support for both synchronous
254
+
and asynchronous I/O, including secure HTTPS communicatio n. The
255
+
package includes backends for Lwt, Async, Mirage, JavaScript, and
256
+
Eio, making it versatile for use in different runtime environments,
257
+
from Unix servers to web browsers. It is well-suited for applications
258
+
requiring high-performance networking, such as web services, API
259
+
clients, and embedded networked systems.
260
+
261
+
#5 - quests (v0.1.3)
262
+
Similarity: 0.7960
263
+
Reverse Dependencies: 1
264
+
Combined Score: 0.6180
265
+
Description: This package provides a high-level HTTP client library
266
+
for making web requests in OCaml. It simplifies interacting with HTTP
267
+
servers by offering a n intuitive API for common methods like GET and
268
+
POST, supporting features such as query parameters, form and JSON
269
+
data submission, and automatic handling of gzip compression and
270
+
redirects. It also includes authentication mechanisms like basic and
271
+
bearer tokens, with partial support for sessions. Typical use cases
272
+
include consuming REST APIs, scraping web content, or integrating
273
+
with web services securely and efficiently.
274
+
275
+
#6 - ezcurl (v0.2.4)
276
+
Similarity: 0.7395
277
+
Reverse Dependencies: 6
278
+
Combined Score: 0.5979
279
+
Description: This package provides a simplified interface for making
280
+
HTTP requests in OCaml, built on top of the OCurl library. It
281
+
addresses the need for an ea sy-to-use, reliable, and stable API for
282
+
handling common web interaction tasks, such as fetching URLs and
283
+
processing responses. The package supports both synchron ous and
284
+
asynchronous operations, enabling efficient handling of parallel
285
+
requests and non-blocking I/O. Practical use cases include web
286
+
scraping, API client deve lopment, and integrating HTTP-based services
287
+
into OCaml applications.
288
+
289
+
]}
290
+
{ul {- "Cryptographic hash"
291
+
}}
292
+
293
+
{@nolang[
294
+
#1 - digestif (v1.3.0)
295
+
Similarity: 0.8165
296
+
Reverse Dependencies: 621
297
+
Combined Score: 0.7041
298
+
Description: This package provides a comprehensive implementation of
299
+
cryptographic hash functions, supporting algorithms such as MD5,
300
+
SHA1, SHA2, SHA3, WHIRLPOOL, BLAKE2, and RIPEMD160. It allows users
301
+
to choose between C and OCaml backends at link time, offering
302
+
flexibility in performance and deployment scenarios. The library is
303
+
designed for applications requiring secure hashing, such as data
304
+
integrity verification, digital signatures, and cryptographic
305
+
protocols. It is well-suited for systems programming and
306
+
security-related applications in the OCaml ecosystem.
307
+
308
+
#2 - ppx_hash (vv0.17.0)
309
+
Similarity: 0.7284
310
+
Reverse Dependencies: 3337
311
+
Combined Score: 0.6833
312
+
Description: This package generates efficient hash functions for
313
+
OCaml types based on their structure, enabling precise control over
314
+
hashing behavior. It addresses the limitations of OCaml's built-in
315
+
polymorphic hashing by allowing users to define custom hash
316
+
functions during type derivation. Key features include selective
317
+
field ignoring, support for folding-style hash accumulation, and
318
+
compatibility with comparison and serialization systems. It is
319
+
suitable for use with hash tables, persistent data structures, and
320
+
any application requiring deterministic, type-driven hashing.
321
+
322
+
#3 - ez_hash (v0.5.3)
323
+
Similarity: 0.8366
324
+
Reverse Dependencies: 3
325
+
Combined Score: 0.6583
326
+
Description: This package provides a straightforward interface to
327
+
common cryptographic hash functions, simplifying their use in OCaml
328
+
applications. It wraps secure, widely-used algorithms like SHA-256
329
+
and Blake2b, offering consistent and safe APIs for hashing data. The
330
+
library is designed for clarity and ease of integration, making it
331
+
ideal for developers needing reliable cryptographic operations
332
+
without deep expertise in security. Practical uses include data
333
+
integrity verification, digital signatures, and secure data storage.
334
+
335
+
#4 - murmur3 (v0.3)
336
+
Similarity: 0.7805
337
+
Reverse Dependencies: 1
338
+
Combined Score: 0.6072
339
+
Description: This package provides OCaml bindings for MurmurHash, a
340
+
fast and widely used non-cryptographic hash function. It enables
341
+
efficient hash value compu tation for arbitrary data, making it
342
+
suitable for applications like hash tables, checksums, and data
343
+
fingerprinting. The bindings offer consistent hashing across platforms
344
+
and integrate seamlessly into OCaml projects requiring
345
+
high-performance hashing. Use cases include caching, distributed
346
+
systems, and data integrity ve rification where cryptographic security
347
+
is not required.
348
+
349
+
#5 - kdf (v1.0.0)
350
+
Similarity: 0.6775
351
+
Reverse Dependencies: 473
352
+
Combined Score: 0.6033
353
+
Description: This package implements standard key derivation
354
+
functions (KDFs) for cryptographic applications in OCaml. It supports
355
+
scrypt, PBKDF1, PBKDF2, and HKDF, enabling secure generation of
356
+
cryptographic keys from passwords or shared secrets. These functions
357
+
help mitigate brute-force attacks and ensure keys are de rived in a
358
+
reproducible, secure manner. Use cases include password-based
359
+
encryption, secure token generation, and key material expansion in
360
+
cryptographic protocols.
361
+
]}
362
+
363
+
and a module-level search for "time and date manipulation functions"
364
+
365
+
{@nolang[
366
+
#1 - timmy-jsoo: Timmy_jsoo
367
+
Similarity: 0.5460
368
+
Original Similarity: 0.7800
369
+
Popularity Score: 0.0000
370
+
Description: This module provides precise date and time arithmetic,
371
+
conversion, and comparison operations across multiple representations,
372
+
including OCaml-nati ve, JavaScript, and string formats. It works with
373
+
structured types like `Date.t`, `Time.t`, and ISO weeks, supporting
374
+
timezone-aware transformations and RFC3339 formatting. Concrete use
375
+
cases include cross-runtime timestamp synchronization, calendar-aware
376
+
scheduling, and robust temporal data validation in distributed
377
+
systems.
378
+
379
+
#2 - calendar: CalendarLib
380
+
Similarity: 0.5331
381
+
Original Similarity: 0.7616
382
+
Popularity Score: 0.3448
383
+
Description: This module provides precise date and time manipulation
384
+
with support for calendar operations, time zones, periods, and
385
+
formatted input/output. It works with types like `Calendar.t`,
386
+
`Date.t`, `Time.t`, and `Period.t` to handle tasks such as event
387
+
scheduling, timestamp conversion, and historical date calculations.
388
+
Concrete use cases include scheduling systems, log timestamping,
389
+
holiday calculations, and cross-timezone time normalization.
390
+
391
+
#3 - calendar: CalendarLib.Fcalendar
392
+
Similarity: 0.5191
393
+
Original Similarity: 0.6820
394
+
Popularity Score: 0.1390
395
+
Description: This module provides float-based calendar operations
396
+
for date creation, conversion, and manipulation, including time zone
397
+
adjustments, component extraction (year/month/day/hour/second), and
398
+
arithmetic with periods. It works with a `t` type representing time
399
+
as float seconds, alongside `day`, `month`, `year`, and Unix time
400
+
structures, prioritizing Unix time precision over sub-second
401
+
accuracy. It suits applications tolerating minor imprecision in date
402
+
comparisons or arithmetic, such as logging systems or coarse-grained
403
+
scheduling, where exact floating-point equality isn't critical.
404
+
405
+
#4 - calendar: CalendarLib.Calendar_builder.Make
406
+
Similarity: 0.5112
407
+
Original Similarity: 0.7302
408
+
Popularity Score: 0.0785
409
+
Description: This module combines date and time functionality to
410
+
construct and manipulate calendar values with float-based precision,
411
+
offering operations like timezone conversion, component extraction
412
+
(day, month, year, etc.), and arithmetic using `Period.t`. It works
413
+
with a calendar type `t` that integrates date and time components,
414
+
alongside conversions to Unix timestamps, Julian day numbers, and
415
+
structured representations like `Unix.tm`. Designed for scenarios
416
+
requiring precise temporal calculations (e.g., calendar arithmetic,
417
+
Gregorian date validation, or leap day checks), it balances
418
+
flexibility with known precision limitations inherent to float-based
419
+
time representations.
420
+
421
+
#5 - timmy-unix: Clock
422
+
Similarity: 0.5080
423
+
Original Similarity: 0.7257
424
+
Popularity Score: 0.0000
425
+
Description: This module provides functions to retrieve the current
426
+
POSIX time, the local timezone, and the current date in the local
427
+
timezone. It works with time and date types from the Timmy library,
428
+
specifically `Timmy.Time.t` and `Timmy.Date.t`. Use this module to
429
+
obtain precise time and date information for logging, scheduling, or
430
+
time-based computations.
431
+
]}
432
+
433
+
and for "Balanced Tree":
434
+
435
+
{@nolang[
436
+
#1 - grenier: Mbt
437
+
Similarity: 0.5274
438
+
Original Similarity: 0.7534
439
+
Popularity Score: 0.0495
440
+
Description: This module implements a balanced binary tree structure
441
+
with efficient concatenation and size-based operations. It supports
442
+
tree construction through leaf and node functions, automatically
443
+
balancing nodes and annotating them with values from a provided
444
+
measure module. It is useful for applications requiring fast access,
445
+
dynamic sequence management, and efficient merging of tree-based
446
+
data structures.
447
+
448
+
#2 - camomile: CamomileLib.AvlTree
449
+
Similarity: 0.5008
450
+
Original Similarity: 0.7155
451
+
Popularity Score: 0.0495
452
+
Description: This module implements balanced binary trees (AVL
453
+
trees) with operations for constructing, deconstructing, and
454
+
traversing trees. It supports key operations like inserting nodes,
455
+
extracting leftmost/rightmost elements, concatenating trees, and
456
+
folding or iterating over elements. It is useful for maintaining
457
+
ordered data with efficient lookup, insertion, and deletion, such as
458
+
in symbol tables or priority queues.
459
+
460
+
#3 - batteries: BatAvlTree
461
+
Similarity: 0.5003
462
+
Original Similarity: 0.7147
463
+
Popularity Score: 0.1485
464
+
Description: This module implements balanced binary trees (AVL
465
+
trees) with operations for creating, modifying, and traversing
466
+
trees. It supports tree construction with optional rebalancing,
467
+
splitting, and concatenation, and provides root, left, and right
468
+
accessors with failure handling. Concrete use cases include
469
+
efficient ordered key-value storage, set-like structures, and
470
+
maintaining sorted data with logarithmic-time insertions and
471
+
lookups.
472
+
473
+
#4 - grenier: Bt2
474
+
Similarity: 0.4927
475
+
Original Similarity: 0.7039
476
+
Popularity Score: 0.2634
477
+
Description: This module implements a balanced binary tree structure
478
+
with efficient concatenation and rank-based access. It supports
479
+
creating empty trees, constructing balanced nodes, and joining two
480
+
trees with logarithmic cost relative to the smaller tree's size. Use
481
+
cases include maintaining ordered collections with frequent splits
482
+
and joins, and efficiently accessing elements by position.
483
+
484
+
#5 - grenier: Mbt.Make
485
+
Similarity: 0.4913
486
+
Original Similarity: 0.7019
487
+
Popularity Score: 0.0495
488
+
Description: This module implements a balanced tree structure with
489
+
efficient concatenation and size-based operations. It supports
490
+
construction of trees using leaf and node functions, where nodes are
491
+
automatically balanced and annotated with measurable values from
492
+
module M. The module enables efficient rank queries and joining of
493
+
trees, with applications in managing dynamic sequences where fast
494
+
access and concatenation are critical.
495
+
]}
496
+
497
+
{1 Limitations and future work}
498
+
499
+
We're aware that there are currently a number of limitations with
500
+
what's been done so far, and there's a lot of exciting things that
501
+
could quite easily be added!
502
+
503
+
We haven't done much prompt optimisation either for the tools
504
+
themselves, nor their descriptions in the MCP server. We also haven't
505
+
done much optimisation of the information retrieval - and it's clear
506
+
from some of the results shown above that there are improvements to be
507
+
made in the ranking algorithms. Some obvious next steps would be to do
508
+
some {{: https://arxiv.org/html/2406.12433v2} re-ranking} or some form
509
+
of hybrid search.
510
+
511
+
A particular challenge is that since this is based entirely off of the
512
+
[ocaml-docs-ci] build, it won't necessarily reflect the actual API
513
+
your local build, as for OCaml, this {{:
514
+
https://jon.recoil.org/blog/2025/04/semantic-versioning-is-hard.html}
515
+
can't be done}. Thibaut Mattio is working on a {{:
516
+
https://github.com/tmattio/ocaml-mcp} local MCP server} that would be
517
+
perfectly positioned to do some of what we're doing, although we'd
518
+
need to have a good local docs build implemented in dune for this to
519
+
work well.
520
+
521
+
Also, there's plenty more data that we've collected during the docs
522
+
builds! We can show the implementations of functions, we can expose
523
+
code samples, select different versions of packages and much more.
524
+
While we've concentrated on the search aspects, there's still a lot of
525
+
low-hanging fruit that can be worked on.
526
+
527
+
If you're interested in helping us out on this, the project lives {{:
528
+
https://github.com/sadiqj/odoc-llm} on github} - come along and join
529
+
us!
530
+
531
+
{1 Using the server}
532
+
533
+
If you'd like to try it, we've got a demo server running right now.
534
+
It's hosted on dill.caelum.ci.dev here at the Computer Laboratory in
535
+
the University of Cambridge. To enable it with Claude, try this:
536
+
537
+
{@bash[
538
+
# claude mcp add -t sse ocaml http://dill.caelum.ci.dev:8000/sse
539
+
]}
540
+
541
+
Obviously this is pre-alpha quality software, and we might take it
542
+
down with no notice, and it might not work as expected, and all of the
543
+
other usual caveats. Let us know if it works, or doesn't, or if you've
544
+
got some suggestions for improvements!
+105
site/blog/2025/08/week33.mld
+105
site/blog/2025/08/week33.mld
···
1
+
{0 Week 33}
2
+
3
+
@published 2025-08-19
4
+
@x-ocaml.requires cohttp,yojson,jsonm
5
+
6
+
More work this week on the OCaml MCP server. Sadiq and I met before I went away on holiday and discussed
7
+
the next steps to 'park' the work on the MCP server. The final steps are:
8
+
9
+
- Write a README
10
+
- Write and run a small script to fix a problem with module-type names
11
+
- Write up and publish a blog post
12
+
13
+
Not much, right? As always though, writing things up lead to a whole load more work.
14
+
15
+
The first problem occurred when writing up how it parsed the input docs. It turned out
16
+
that when converting the repo so that it took markdown formatted files (using a
17
+
{{:https://github.com/jonludlam/odoc/tree/odoc-llm-markdown}slightly
18
+
tweaked} version of {{:https://github.com/ocaml/odoc/pull/1341}davesnx's PR}), Claude had decided
19
+
that the way to do this was to first convert the markdown into HTML, and then use the HTML parser
20
+
it had already built. Whilst tidying this up, Claude was remarkably keen to just use regexps
21
+
to parse the markdown rather than using a pre-existing markdown library, so it took a little
22
+
persuasion to get it into a state I was happy with.
23
+
24
+
The second issue was that the script that form the bulk of the repo had been written at
25
+
different times, and therefore Claude didn't really take into account any of the decisions
26
+
it had made in one script when building the next. So most of the command-line arguments were
27
+
slightly different, which made writing up a mini 'howto' in the README quite a jarring
28
+
experience.
29
+
30
+
Thirdly, and most importantly, we had decided that we needed a few example searches to show
31
+
how the system worked. We'd already had a {{!/site/blog/2025/07/page-week28}useful experience}
32
+
with this when Anil had tried to search for a 'time and date parsing and formatting' library,
33
+
so it shouldn't really have been a surprise that trying a few more examples showed some more
34
+
interesting behaviour. Specifically, the searches I wanted to do were for an "HTTP client", "JSON parser",
35
+
"Cryptographic Hash" and Anil's time-and-date query, and in actually trying these searches
36
+
and critically examining the results, I had to go back and figure out why they weren't giving
37
+
me the results I had expected.
38
+
39
+
The first of these searches I had anticipated would be quite interesting, as this is a query that should show the OCaml ecosystem
40
+
{{:https://discuss.ocaml.org/t/simple-modern-http-client-library/11239}missing an obvious HTTP client}.
41
+
However, even with this in mind one of the top results was one of Cohttp's module types,
42
+
{!Cohttp.Generic.Client.S}. This, of course, isn't much use if you're looking for an HTTP client,
43
+
as module-types aren't going to give you an implementation to actually use. So I decided that
44
+
we'd exclude module-types from the results. This turned out to be slightly more tricky than I
45
+
anticipated as we'd lost the distinction between modules and module types further back in the
46
+
pipeline, so Claude had to do some plumbing to ensure we had this information at the point we
47
+
were doing the search.
48
+
49
+
The cryptographic hash search gave some plausible looking results, so I moved on to the JSON
50
+
search. I was expecting to see {!Yojson} somewhere near the top of the list as that's a very
51
+
popular library. I was also expecting to see {!Jsonm} somewhere near the top - or at least
52
+
I'd like to be able to find it by searching for a 'streaming parser' as that's one of its
53
+
key strengths. However, searching for "JSON parser" yielded some less than brilliant answers.
54
+
The top 5 results were for modules in the packages [yojson-five], [decoders-yojson],
55
+
[decoders-jsonaf], [ocplib-json-typed-browser] and [ppx_protocol_conv_jsonm]. While all of
56
+
these are clearly in the same realm as I was after, having [jsonm] show up literally
57
+
99th in the list, and [yojson] itself not in the top 100 wasn't a great result.
58
+
59
+
Some investigation showed that yojson had a particularly bad showing because the description
60
+
of the module {!Yojson.Basic} was the empty string! This turned out to be because of some
61
+
bad error-handling logic in the summariser script, which ended up turning some errors into
62
+
a blank description. Since running the summariser costs actual money, I didn't want to just
63
+
rerun the whole thing, so I asked Claude for a script to find these problems and rerun them.
64
+
The problem is not totally trivial as the summaries of child modules are used when generating
65
+
the summary for parents, so when one is regenerated we should regenerate the summaries of all
66
+
ancestors too. Given my recent experiences with Claude I'd like to look this over quite
67
+
carefully before letting it loose on my data, so I've run it on yojson, which seemed to do
68
+
the right thing, but not yet on the rest of the packages.
69
+
70
+
Having fixed this, I still found that [jsonm] was making a very poor showing. This turned out
71
+
to be because the description it gives itself is a "Non-blocking streaming JSON codec for OCaml"
72
+
which had a fairly low similarity with "JSON parser". I was using a fairly small embedding
73
+
model for the queries - Qwen/Qwen3-Embedding-0.6B, so I thought I might address this by using a
74
+
larger one, and opted for Qwen/Qwen3-Embedding-8B. The machine I had been using for the
75
+
MCP server has no GPU and had taken a while to do the embeddings using the 0.6B model, so I
76
+
switched to generating them on my M4 macbook. This went {i much} faster, though since I have
77
+
about 70Mb of module summaries it still took quite a while. This improved the situation somewhat,
78
+
but it was still not high in the list.
79
+
80
+
So I took a step back and had a think about the problem some more. Searching for a JSON parser
81
+
is really quite a high-level search, and when evaluating the results I realised I was really
82
+
thinking in terms of packages rather than modules. So I thought we could split the search in
83
+
two - a package search and a module search. The package search would be used for the broad queries
84
+
where you're interested in pulling in whole chunks of functionality, and the module search is
85
+
for more low-level queries. In fact, the 'time and dating formatting' query is somewhere in
86
+
between, so I might need to have some more example queries for the module search functions.
87
+
In addition, the module search could be restricted to the set of packages you're using, which
88
+
might make it even more useful.
89
+
90
+
Part of the split meant that I needed a different source of 'popularity' for the packages
91
+
than the occurrences data that came out of docs ci, as that was per-module and I needed something
92
+
per-package. The obvious thing is to look at reverse dependencies in opam. I have this
93
+
kind-of working, but it's currently not particularly smart, so this will need a little more
94
+
attention. For example, it currently thinks that {{:https://melange.re/v5.0.0/}melange} has
95
+
over 3000 reverse dependencies.
96
+
97
+
With these changes in place, a package search for 'JSON parser' now returns [yojson] as
98
+
number one, followed by [ppx_deriving_yojson], [ezjsonm], [ocplib-json-typed] and
99
+
[jsonaf]. Unfortunately [jsonm] is still languishing in 27th place, so there's still some
100
+
tweaking to do.
101
+
102
+
103
+
104
+
105
+
+44
site/blog/2025/09/build-ids-for-day10.mld
+44
site/blog/2025/09/build-ids-for-day10.mld
···
1
+
{0 Build IDs for Day10}
2
+
3
+
@published 2025-09-08
4
+
5
+
{{:https://tunbury.org}mtelvers}, {{:https://www.dra27.uk/blog/}dra27} and I have been working
6
+
on a system to build opam packages similar to the way that the docs-ci system does - effectively
7
+
building a per-package binary cache to do very fast builds of the entire opam repository. It
8
+
supports building even mutually-incompatible packages by dynamically creating the build environment
9
+
for each package, and thus allows us to generate something akin to {{:}opam health check} but much faster.
10
+
11
+
Currently the cache of a package is a key-value store where the key is a hash of the package name
12
+
and version and all of its dependencies and their name and version, alongside some information
13
+
about the OS. This is great when this info can uniquely identify the output, but this isn't always
14
+
the case. In particular, the oxcaml opam-repository has several packages where the version number
15
+
is the upstream version number with `-ox` appended, as they have patches to make them compatible
16
+
with oxcaml. If these patches change without bumping the suffix the currently caching mechanism
17
+
would lead to trouble. When we discussed this David pointed out the idea of the
18
+
{{:https://github.com/ocaml/opam/blob/c36dd1ce40a715ef27122184715bbf3e9aa7f0c9/src/state/opamPackageVar.ml#L178-L211}build-id}
19
+
in opam, which would perfectly satisfy our needs. Unfortunately this code is quite deep within
20
+
the opam codebase and at the point we need it we don't have an installed opam switch, so we need
21
+
to pull the code out and insert it into our project.
22
+
23
+
One of the first challenges was that day10 currently includes the OS details in the hash so that we
24
+
can test across different distros. This is at odds with the opam build-id which doesn't include
25
+
that, so in order to try to get as close as possible to the opam hash I split the cache into 2
26
+
layers - a per-OS cache directory containing hashes based on pure opam metadata. The idea is that
27
+
these should be identical to the build-ids of opam. With that fixed, the new cache layout looks like:
28
+
29
+
{[
30
+
debian-12-x86_64/123...abc/{build.log,config,...}
31
+
]}
32
+
33
+
where the [123...abc] should be the same as the build-id you would get with all the packages
34
+
contained installed.
35
+
36
+
Now my actual use case for this is to track the state of the oxcaml world day by day, so for this
37
+
I need to track both the opam-repository for OCaml and also the opam repository for OxCaml. The
38
+
project currently uses a Makefile for coordinating the builds, but I thought it was time we moved
39
+
on to a dedicated batch execution process. So I asked Claude to knock me up one of those, using
40
+
odoc_driver for inspiration. It's very basic right now, simply iterating through the latest
41
+
versions of every package, but I have got it to check on cache hits and misses, so I should be able
42
+
to run it tomorrow to see how quickly we can test PRs to oxcaml/opam-repository
43
+
44
+
+181
site/blog/2025/09/caching-opam-solutions.mld
+181
site/blog/2025/09/caching-opam-solutions.mld
···
1
+
{0 Caching opam solutions}
2
+
3
+
@published 2025-09-09
4
+
5
+
The {{:https://github.com/ocurrent/ocaml-docs-ci}ocaml-docs-ci} system works by watching
6
+
opam-repository for changes, and then when it notices a new package it performs an opam
7
+
solve and builds the package, a prerequisite for building the documentation. In order
8
+
to give the docs some stability, as the docs may well {{!/site/blog/2025/04/page-"semantic-versioning-is-hard"}depend upon your dependencies}, we currently cache the solve results so that a package
9
+
will always be built with the same set of dependencies, even if a new version of one
10
+
of those dependencies has been released.
11
+
12
+
The downside to this is that as time goes on, the number of distinct universes that we
13
+
build increases, and docs get more and more out of date. So it's not necessarily the best
14
+
thing to do, though it does mean we minimise the amount of time spent solving.
15
+
16
+
The alternative approach is that on every commit to opam-repository we could resolve for
17
+
all packages and use the latest, greatest solution to build the docs. Using this approach
18
+
we would maximise the sharing of builds and keep the total amount of required storage
19
+
steadier. Of course, this would mean solving for every package on every commit to opam-repository,
20
+
even if we didn't end up rebuilding all of them due to the way that the cache works.
21
+
22
+
One possibility that might be worth investigating is to cache the solutions - but
23
+
then Leon Bambrick {{:https://twitter.com/secretGeek/status/7269997868}advises us}:
24
+
25
+
{@quote[
26
+
There are 2 hard problems in computer science: cache invalidation,
27
+
naming things, and off-by-1 errors.
28
+
]}
29
+
30
+
and indeed it's not obvious what the best approach to cache invalidation is here. A
31
+
sledgehammer approach would be to hook into the solver and note what questions it
32
+
asks of opam-repository and record the responses. If any of these change, then it's
33
+
safe to say that we need to recalculate. I had a quick look at this and checked what
34
+
packages were involved in the solution of [ocaml] as this would
35
+
represent a minimum set of packages that would affect virtually all packages. The
36
+
list was big, but not {i too} big:
37
+
38
+
{v
39
+
winpthreads, system-msvc, system-mingw, ocaml-variants, ocaml-system,
40
+
ocaml-options-vanilla, ocaml-option-tsan, ocaml-option-static,
41
+
ocaml-option-spacetime, ocaml-option-no-flat-float-array,
42
+
ocaml-option-no-compression, ocaml-option-nnpchecker,
43
+
ocaml-option-nnp, ocaml-option-musl, ocaml-option-mingw,
44
+
ocaml-option-leak-sanitizer, ocaml-option-fp, ocaml-option-flambda,
45
+
ocaml-option-default-unsafe-string, ocaml-option-bytecode-only,
46
+
ocaml-option-afl, ocaml-option-address-sanitizer,ocaml-option-32bit,
47
+
ocaml-config, ocaml-compiler, ocaml-beta, ocaml-base-compiler, ocaml,
48
+
dkml-base-compiler, conf-unwind, conf-pkg-config, base-unix,
49
+
base-threads, base-ocamlbuild, base-nnp, base-metaocaml-ocamlfind,
50
+
base-implicits, base-effects, base-domains, base-bigarray
51
+
v}
52
+
53
+
I tried the same thing whilst using the oxcaml opam-repository, and this time,
54
+
the list became much {i much} larger:
55
+
56
+
{v
57
+
zed, zarith-xen, zarith-freestanding, zarith, yojson, xenstore, xdg,
58
+
x509, webbrowser, wasm_of_ocaml-compiler, variantslib, uutf, uuseg,
59
+
uunf, uucp, uTop, uri-sexp, uri, uopt, univ_map, uchar, tyxml,
60
+
typerex, typerep, trie, topkg, tls-lwt, tls, timezone, time_now,
61
+
textutils_kernel, textutils, tcpip, system-msvc, system-mingw,
62
+
swhid_core, stringext, string_dict, stdune, stdlib-shims, stdio, ssl,
63
+
splittable_random, spdx_licenses, spawn, shell,
64
+
shared-memory-ring-lwt, shared-memory-ring, sha, sexplib0, sexplib,
65
+
sexp_pretty, seq, sedlex, rresult, result, regex_parser_intf,
66
+
record_builder, react, re2, re, randomconv, publish, ptime, psq,
67
+
protocol_version_header, ppxlib_jane, ppxlib_ast, ppxlib, ppxfind,
68
+
ppx_yojson_conv_lib, ppx_yojson_conv, ppx_variants_conv, ppx_var_name,
69
+
ppx_typerep_conv, ppx_typed_fields, ppx_tydi, ppx_tools_versioned,
70
+
ppx_tools, ppx_template, ppx_string_conv, ppx_string,
71
+
ppx_stable_witness, ppx_stable, ppx_shorthand, ppx_sexp_value,
72
+
ppx_sexp_message, ppx_sexp_conv, ppx_pipebang, ppx_optional,
73
+
ppx_optcomp, ppx_module_timer, ppx_log, ppx_let, ppx_js_style,
74
+
ppx_jane, ppx_inline_test, ppx_ignore_instrumentation, ppx_here,
75
+
ppx_helpers, ppx_hash, ppx_globalize, ppx_fixed_literal,
76
+
ppx_fields_conv, ppx_fail, ppx_expect, ppx_enumerate,
77
+
ppx_disable_unused_warnings, ppx_diff, ppx_deriving, ppx_derivers,
78
+
ppx_custom_printf, ppx_cstruct, ppx_compare, ppx_cold, ppx_bin_prot,
79
+
ppx_bench, ppx_base, ppx_assert, pp, portable, pipe_with_writer_error,
80
+
pcre, pbkdf, patch, parsexp, ounit2, ordering, optint, opam-state,
81
+
opam-repository, opam-publish, opam-lib, opam-format,
82
+
opam-file-format, opam-core, ojs, ohex, odoc-parser, odoc, octavius,
83
+
ocplib-endian, ocp-indent, ocp-build, ocb-stubblr, ocamlnet,
84
+
ocamlgraph, ocamlformat-rpc-lib, ocamlformat-lib, ocamlformat,
85
+
ocamlfind-secondary, ocamlfind, ocamlc-loc, ocamlbuild,
86
+
ocaml_intrinsics_kernel, ocaml_intrinsics, ocaml-version,
87
+
ocaml-variants, ocaml-system, ocaml-syntax-shims,
88
+
ocaml-secondary-compiler, ocaml-options-vanilla, ocaml-option-tsan,
89
+
ocaml-option-static, ocaml-option-spacetime,
90
+
ocaml-option-no-flat-float-array, ocaml-option-no-compression,
91
+
ocaml-option-nnpchecker, ocaml-option-nnp, ocaml-option-musl,
92
+
ocaml-option-mingw, ocaml-option-leak-sanitizer, ocaml-option-fp,
93
+
ocaml-option-flambda, ocaml-option-default-unsafe-string,
94
+
ocaml-option-bytecode-only, ocaml-option-afl,
95
+
ocaml-option-address-sanitizer, ocaml-option-32bit,
96
+
ocaml-migrate-parsetree, ocaml-lsp-server, ocaml-index,
97
+
ocaml-freestanding, ocaml-config, ocaml-compiler-libs,
98
+
ocaml-base-compiler, ocaml, obuild, num, nocrypto, mtime, mmap,
99
+
mirage-xen-posix, mirage-xen, mirage-types, mirage-time, mirage-stack,
100
+
mirage-solo5, mirage-sleep, mirage-runtime, mirage-random,
101
+
mirage-ptime, mirage-protocols, mirage-profile, mirage-no-xen,
102
+
mirage-no-solo5, mirage-net-xen, mirage-net, mirage-mtime,
103
+
mirage-kv-mem, mirage-kv-lwt, mirage-kv, mirage-flow, mirage-entropy,
104
+
mirage-device, mirage-crypto-rng-mirage, mirage-crypto-rng-lwt,
105
+
mirage-crypto-rng, mirage-crypto-pk, mirage-crypto-ec, mirage-crypto,
106
+
mirage-clock-unix, mirage-clock-lwt, mirage-clock, mew_vi, mew,
107
+
metrics-lwt, metrics, merlin-lib, merlin, menhirSdk, menhirLib,
108
+
menhirCST, menhir, mdx, magic-mime, macaddr-cstruct, macaddr, lwt_ssl,
109
+
lwt_react, lwt_ppx, lwt_log, lwt-dllist, lwt, lsp, lru, logs,
110
+
lambda-term, kdf, jst-config, jsonrpc, jsonm, js_of_ocaml-toplevel,
111
+
js_of_ocaml-ppx, js_of_ocaml-lwt, js_of_ocaml-compiler, js_of_ocaml,
112
+
jbuilder, jane_rope, jane-street-headers, ipaddr-sexp, ipaddr-cstruct,
113
+
ipaddr, io-page, int_repr, http, hkdf, hex, hacl_x25519, gmap,
114
+
github-unix, github-data, github, gen_js_api, gen, gel,
115
+
functoria-runtime, fpath, fmt, fix, fieldslib, fiber, fiat-p256,
116
+
ezjsonm, extlib-compat, extlib, expectree, expect_test_helpers_core,
117
+
ethernet, eqaf, either, easy-format, dyn, duration, dune-site,
118
+
dune-rpc, dune-release, dune-private-libs, dune-configurator,
119
+
dune-compiledb, dune-build-info, dune, dot-merlin-reader, domain-name,
120
+
dkml-base-compiler, digestif, curly, cstruct-sexp, cstruct-lwt,
121
+
cstruct, csexp, crunch, cpuid, cppo, core_unix, core_kernel,
122
+
core_extended, core, configurator, conf-which, conf-unwind,
123
+
conf-pkg-config, conf-ninja, conf-m4, conf-libssl, conf-libpcre,
124
+
conf-gmp-powm-sec, conf-gmp, conf-g++, conf-cmake, conf-c++,
125
+
conf-bash, conf-autoconf, conduit-lwt-unix, conduit-lwt, conduit,
126
+
cohttp-lwt-unix, cohttp-lwt-jsoo, cohttp-lwt, cohttp, cmdliner,
127
+
cmarkit, chrome-trace, charInfo_width, capitalization, camomile,
128
+
camlp4, camlp-streams, ca-certs, bos, biniou, binaryen-bin, bin_prot,
129
+
bigstringaf, bigarray-compat, bheap, basement, base_quickcheck,
130
+
base_bigstring, base64, base-unix, base-threads, base-ocamlbuild,
131
+
base-num, base-nnp, base-effects, base-domains, base-bytes,
132
+
base-bigarray, base, backoff, atdgen-runtime, atdgen, atd, async_unix,
133
+
async_rpc_kernel, async_log, async_kernel, async_extra, async,
134
+
astring, asn1-combinators, arp, angstrom, alcotest
135
+
v}
136
+
137
+
This enormous list is because the opam file for oxcaml -
138
+
[ocaml-variants.5.2.0+ox] - lists a bunch of conflicts to ensure that
139
+
various incompatible packages are never selected:
140
+
141
+
{v
142
+
conflicts: [
143
+
"base" {< "v0.18~"}
144
+
"alcotest" {!= "1.9.0+ox"}
145
+
"backoff" {!= "0.1.1+ox"}
146
+
"dot-merlin-reader" {!= "5.2.1-502+ox"}
147
+
"gen_js_api" {!= "1.1.2+ox"}
148
+
"js_of_ocaml" {!= "6.0.1+ox"}
149
+
"js_of_ocaml-compiler" {!= "6.0.1+ox"}
150
+
"js_of_ocaml-ppx" {!= "6.0.1+ox"}
151
+
"js_of_ocaml-toplevel" {!= "6.0.1+ox"}
152
+
"jsonrpc" {!= "1.19.0+ox"}
153
+
"lsp" {!= "1.19.0+ox"}
154
+
"lwt_ppx" {!= "5.9.1+ox"}
155
+
"mdx" {!= "2.5.0+ox"}
156
+
"merlin" {!= "5.2.1-502+ox"}
157
+
"merlin-lib" {!= "5.2.1-502+ox"}
158
+
"ocaml-compiler-libs" {!= "v0.17.0+ox"}
159
+
"ocaml-index" {!= "1.1+ox"}
160
+
"ocaml-lsp-server" {!= "1.19.0+ox"}
161
+
"ocamlbuild" {!= "0.15.0+ox"}
162
+
"ocamlformat" {!= "0.26.2+ox"}
163
+
"ocamlformat-lib" {!= "0.26.2+ox"}
164
+
"ojs" {!= "1.1.2+ox"}
165
+
"ppxlib" {!= "0.33.0+ox"}
166
+
"ppxlib_ast" {!= "0.33.0+ox"}
167
+
"sedlex" {!= "3.3+ox"}
168
+
"topkg" {!= "1.0.8+ox"}
169
+
"uTop" {!= "2.15.0+ox"}
170
+
"uutf" {!= "1.0.3+ox"}
171
+
"wasm_of_ocaml-compiler" {!= "6.0.1+ox"}
172
+
"zarith" {!= "1.12+ox"}
173
+
]
174
+
v}
175
+
176
+
and it seems that the solver is looking not just at these packages, but also at all
177
+
of their dependencies too. So this is a much larger set of packages that we need
178
+
to track changes for, probably making the caching an awful lot less effective. It's
179
+
not clear to me that this is the best way for the solver
180
+
to handle conflicts, but I don't know enough about how it works yet to say for sure.
181
+
+73
site/blog/2025/09/caching-opam-solutions2.mld
+73
site/blog/2025/09/caching-opam-solutions2.mld
···
1
+
{0 Caching opam solutions - part 2}
2
+
3
+
@published 2025-09-23
4
+
5
+
Some results from the {{!/site/blog/2025/09/page-"caching-opam-solutions"}previous post}.
6
+
This time I've run day10 on 144 or so commits from opam-repository to see how well the
7
+
cache performs. The results are quite interesting.
8
+
9
+
First let's talk about the "examination map". This is a map from package name to a list
10
+
of other packages whose solutions should be recalculated if the package in question is
11
+
altered. It's built by first looking at the packages that the solver asks about during the
12
+
solution for a package, and then taking {e all} of the solutions, and 'inverting' the map,
13
+
so for example, if both packages 'a' and 'b' ask about package 'c' during their solutions,
14
+
then altering 'c' means that the solutions for both 'a' and 'b' need to be recalculated.
15
+
The examination map entry for 'c' would then be ['a'; 'b']. We can plot the histogram of
16
+
the sizes of each entry in the examination map:
17
+
18
+
{{image!examination_map_histogram.svg}Package Examiner Distribution Histogram}
19
+
20
+
Some interesting features from these data:
21
+
22
+
- The most common number of observers is 1, meaning that the
23
+
package is not involved in the solution of any other package.
24
+
There are approximately 2000 such packages.
25
+
- Most (~80%) of packages have fewer than 100 observers. This
26
+
means that if we alter one of these packages, we only need to
27
+
recalculate the solutions for fewer than 100 other packages.
28
+
- A {e very} small number of packages are observed in all
29
+
4,400 solutions. This is actually a bit artificial, as
30
+
the solver adds the ocaml-compiler package as an input to
31
+
all solves to ensure we get the correct compiler version.
32
+
There's another way to do this which would avoid this
33
+
particular problem.
34
+
- A small number of packages have a very large number of observers,
35
+
around 3800.
36
+
This mostly corresponds with [dune] and its dependencies and
37
+
associated packages. There are around 350 such packages, and
38
+
any change to these means we need to recalcuate most of the
39
+
solutions.
40
+
41
+
This last point doesn't mean that we actually {e recompile}
42
+
3,800 packages, just that we need to recalcualte the solution,
43
+
which might then lead to a cache hit of the layer and no actual
44
+
compilation. However, recalculating the solutions of all of the
45
+
packages takes (on my computer) around 10,000 seconds, or
46
+
roughly 5 minutes of wall-clock time as I've got 32 threads.
47
+
48
+
However, if the package that's changes {i isn't} one of those
49
+
350 packages, then the number of solutions that need to be
50
+
recalculated is dramatically reduced. I ran the logic over
51
+
the last few weeks of commits to opam-repository, from commit [109398e2fd61803126becd398df0f1eabc9f3ca2]
52
+
of the 10th September up until commit [3f21ebe342ce440d9c9142ffe1185d8e5a326085]
53
+
from the 22nd. In this time there were 144 commits (counting
54
+
only those from [git log --first-parent]). Of these, only
55
+
4 resulted in a full resolve - the first commit, since obviously
56
+
we have no cache at that point, the {{:https://github.com/ocaml/opam-repository/commit/40283204789e7116e1c99466de902cd565d121cf}release of OCaml 5.4.0 beta2}
57
+
by {{:https://perso.quaesituri.org/florian.angeletti/}Florian Angeletti},
58
+
a fix of {{:https://github.com/ocaml/opam-repository/commit/6ef6813522b6ea29933f6451236a1639bdbaec61}ocaml-base-compiler for MSVC} by {{:https://www.dra27.uk/blog/}David}
59
+
and a fix for {{:https://github.com/ocaml/opam-repository/commit/d141887ab0b4fc0836ad0787f1f806585a260bc8}BER-OCaml} by
60
+
{{:https://www.cl.cam.ac.uk/~jdy22/}Jeremy Yallop}. Then 25 commits
61
+
resulted in recalculating solutions for 3800 packages as they hit dune-adjacent packages,
62
+
5 commits resulted in recalculating between 100 and 300 packages and the remaining
63
+
110 commits resulted in recalculating fewer than 100 packages, the majority of
64
+
which resulted in recalculating fewer than 5 packages.
65
+
66
+
Overall, at a rough estimate, this means that over this period, using this
67
+
caching strategy gave us a 5x speedup in the solver!
68
+
69
+
70
+
71
+
72
+
73
+
+33
site/blog/2025/09/giving-hub-cl-an-upgrade.mld
+33
site/blog/2025/09/giving-hub-cl-an-upgrade.mld
···
1
+
{0 Giving hub.cl an upgrade}
2
+
3
+
@published 2025-09-07
4
+
5
+
For a few years now we've been running [hub.cl.cam.ac.uk], a Jupyterhub instance, for the
6
+
first year course "Foundations of Computer Science". It serves as a hosting site for the
7
+
lecture notes, which come in the form of Jupyter notebooks, and as a playground where students
8
+
can try OCaml, and it also is used to run the assessed exercises that are a mandatory
9
+
part of the course.
10
+
11
+
Since I spent some time setting it up back in 2018 or so, its aggregated some cruft over
12
+
the years, and has also fallen somewhat behind the bleeding edge of the Jupyter software
13
+
stack. So I thought this year, as I'm actually lecturing the course, I'd give it a bit of
14
+
loving care and attention.
15
+
16
+
We were still on Jupyterhub 1.5.3 whereas the current release is 5.3.0 - so there was quite
17
+
a bit of work to do. I brief play with putting things on the latest version seemed to break
18
+
quite a lot of things, so I thought it might be better to go back to the drawing board and
19
+
start the config again from scratch. So with some help from Claude, I've now managed to hugely
20
+
simplify the whole config of Jupyterhub, and even given it a makeover to try to match the
21
+
style of www.cst.cam.ac.uk as well. The improvements include:
22
+
23
+
- Using caddy as a reverse proxy for TLS termination, meaning I don't have to manually renew the letsencrypt cert every 3 months
24
+
- Unifying the configuration of the two container images used for students and instructors
25
+
- Upgrading to much newer jupyterhub, notebook and nbgrader images
26
+
- Simplifying the configuration required to make it work on a new server - persistent user directories are now docker volumes rather than bindmounts on the local filesystem
27
+
- Updating the authentication method to use Raven via OAuth2 rather than the unmaintained {{:https://github.com/pyCav/jupyterhub-raven-auth}jupyterhub-raven-auth} which I'd had to maintain {{:https://github.com/jonludlam/jupyterhub-raven-auth/commit/36eaf16b410e7ac3cfc532269e0ae5f1de34f231}a patch}.
28
+
- Rebasing {{:https://github.com/jonludlam/nbgrader/commit/c83a6cbb7b530ce87b0b157accddcdc832bcba38}my patch} to nbgrader to verify all of the output of the cells when grading answers
29
+
30
+
As ever, this took longer than I'd anticipated, but I'm mostly there now. There are a few more steps to try:
31
+
32
+
- trial the {{:https://github.com/akabe/ocaml-jupyter/pull/210}new patch} for using ocaml-jupyter with OCaml 5.x
33
+
- see how to upgrade to notebook v7, as I've stuck with v6 in order to keep the extensions we're using going.
+4
site/blog/2025/09/index.mld
+4
site/blog/2025/09/index.mld
+271
site/blog/2025/09/odoc-bugs.mld
+271
site/blog/2025/09/odoc-bugs.mld
···
1
+
{0 Odoc bugs}
2
+
3
+
@published 2025-09-22
4
+
@x-ocaml.requires odoc.model
5
+
6
+
This post is a brief write-up of a couple of bugs in odoc that I've been working
7
+
on over the past 2 weeks. I was convinced at the start of this that I was
8
+
actually fixing one bug, but although they both had the same backtrace and
9
+
similar immediate causes, they're actually quite different. They both involve
10
+
{e expansion}, which is the process that odoc uses to work out the contents of
11
+
a module from its expression - what allows you to see the contents of a module
12
+
such as [module M = Map.Make(String)].
13
+
14
+
{2 Bug 930: inline destructive substitutions}
15
+
16
+
Bug #930 in odoc is about a substitution problem:
17
+
18
+
{@ocaml norun [
19
+
module type S1 = sig
20
+
type t0
21
+
type 'a t := unit
22
+
23
+
val x : t0 t
24
+
end
25
+
26
+
module type S2 = sig
27
+
type t (* must be the same name as [S1.t] *)
28
+
29
+
include S1 with type t0 := t
30
+
end
31
+
32
+
module type S3 = sig
33
+
type t1
34
+
35
+
include S2 with type t := t1
36
+
end
37
+
]}
38
+
39
+
which when processed by odoc 2.4 throws an exception:
40
+
41
+
{v
42
+
odoc: internal error, uncaught exception:
43
+
Invalid_argument("List.fold_left2")
44
+
Raised at Stdlib.invalid_arg in file "stdlib.ml", line 33, characters 20-45
45
+
Called from Odoc_xref2__Subst.type_expr in file "subst.ml", line 598, characters 21-59
46
+
Called from Odoc_xref2__Subst.value in file "subst.ml" (inlined), line 842, characters 19-38
47
+
Called from Odoc_xref2__Subst.apply_sig_map.inner.(fun) in file "subst.ml", line 1089, characters 19-52
48
+
Called from Odoc_xref2__Component.Delayed.get in file "component.ml" (inlined), line 55, characters 16-22
49
+
Called from Odoc_xref2__Lang_of.signature_items.inner in file "lang_of.ml", line 438, characters 16-39
50
+
Called from Odoc_xref2__Lang_of.signature in file "lang_of.ml" (inlined), line 466, characters 12-43
51
+
Called from Odoc_xref2__Lang_of.include_ in file "lang_of.ml", line 641, characters 18-69
52
+
v}
53
+
54
+
The key thing here is that definition of ['a t] in [S1] - a destructive substituion.
55
+
If you type this code into an OCaml toplevel, you will see that the signature of
56
+
[S1] is:
57
+
58
+
{@ocaml run-on=load [
59
+
# module type S1 = sig
60
+
type t0
61
+
type 'a t := unit
62
+
63
+
val x : t0 t
64
+
end;;
65
+
]}
66
+
67
+
where the substitution has clearly taken place. In contrast, odoc takes the position
68
+
that the use of these inline destructive substitutions is
69
+
to make the code easier to understand, and so it tries to keep them in the signature
70
+
rather than simply apply them and present the resulting signature. So when rendering
71
+
[S1] we end up with:
72
+
73
+
{%html:
74
+
<div class="inset" style="border: 1px solid var(--pre-border-color); padding: 10px; border-radius: 5px">
75
+
<a id="module-type-S1" class="anchor"></a><h2>Module type <code><span>S1</span></code></h2>
76
+
<div class="odoc-spec"><div class="spec type anchored" id="type-t0"><a href="#type-t0" class="anchor"></a><code><span><span class="keyword">type</span> t0</span></code></div></div><div class="odoc-spec"><div class="spec type subst anchored" id="type-t"><a href="#type-t" class="anchor"></a><code><span><span class="keyword">type</span> <span>'a t</span></span><span> := unit</span></code></div></div><div class="odoc-spec"><div class="spec value anchored" id="val-x"><a href="#val-x" class="anchor"></a><code><span><span class="keyword">val</span> x : <span><a href="#type-t0">t0</a> <a href="#type-t">t</a></span></span></code></div></div>
77
+
</div>
78
+
%}
79
+
80
+
The reported problem is a failure with a stack trace while processing [S3], but upon looking
81
+
closely the real problem has happened when expanding [S2]. What happens is that we have
82
+
a type [t] defined in [S2] and a type [t] that will later be substituted away that comes
83
+
from the inclusion of [S1]. The rendered signature of [S2] is:
84
+
85
+
{%html:
86
+
<div class="inset" style="border: 1px solid var(--pre-border-color); padding: 10px; padding-right:30px; border-radius: 5px">
87
+
<a id="module-type-S2" class="anchor"></a><h2>Module type <code><span>S2</span></code></h2>
88
+
<div class="odoc-spec"><div class="spec type anchored" id="type-s2-t"><a href="#type-s2-t" class="anchor"></a><code><span><span class="keyword">type</span> t</span></code></div></div><div class="odoc-include"><details open="open"><summary class="spec include"><code><span><span class="keyword">include</span> <a href="#module-type-S1">S1</a> <span class="keyword">with</span> <span><span class="keyword">type</span> <a href="#type-t0">t0</a> := <a href="#type-s2-t">t</a></span></span></code></summary><div class="odoc-spec"><div class="spec type subst anchored" id="type-s2-t"><a href="#type-s2-t" class="anchor"></a><code><span><span class="keyword">type</span> <span>'a t</span></span><span> := unit</span></code></div></div><div class="odoc-spec"><div class="spec value anchored" id="val-x"><a href="#val-x" class="anchor"></a><code><span><span class="keyword">val</span> x : <span><a href="#type-s2-t">t</a> <a href="#type-s2-t">t</a></span></span></code></div></div></details></div>
89
+
</div>
90
+
%}
91
+
92
+
where the type of [x] is now [t t], which is clearly incorrect.
93
+
The problem is that odoc assumes that type names are unique within a signature (modulo
94
+
shadowing, which isn't quite what's going on here), but in this signature there are
95
+
two definitions of [type t], one of which is parameterised and one is not. At this
96
+
point nothing fatal has happened, but when we try to process [S3] the substitution code
97
+
gets very confused by these different arities and [List.fold_left2] throws the above
98
+
exception.
99
+
100
+
The fix I'm trialling for this is that when we're including a signature that contains an
101
+
inline destructive substitution, we will perform that substitution when the expansion of the
102
+
include is done. This means that the rendered signature of [S1] will be just the same as
103
+
before, but the rendered signature of [S2] will now be:
104
+
105
+
{%html:
106
+
<div class="inset" style="border: 1px solid var(--pre-border-color); padding: 10px; padding-right:30px; border-radius: 5px">
107
+
<a id="module-type-newS2" class="anchor"></a><h2>Module type <code><span>S2</span></code></h2>
108
+
<div class="odoc-spec"><div class="spec type anchored" id="type-s2new-t"><a href="#type-s2new-t" class="anchor"></a><code><span><span class="keyword">type</span> t</span></code></div></div><div class="odoc-include"><details open="open"><summary class="spec include"><code><span><span class="keyword">include</span> <a href="#module-type-S1">S1</a> <span class="keyword">with</span> <span><span class="keyword">type</span> <a href="#type-t0">t0</a> := <a href="#type-s2new-t">t</a></span></span></code></summary><div class="odoc-spec"><div class="spec value anchored" id="val-x"><a href="#val-x" class="anchor"></a><code><span><span class="keyword">val</span> x : unit</span></code></div></div></details></div>
109
+
</div>
110
+
%}
111
+
112
+
where the type of [x] is now simply [unit], which is what OCaml itself thinks, happily!
113
+
I think this strikes the balance between keeping the substitutions visible for clarity
114
+
where they are originally defined, but when including them elsewhere we simply see the
115
+
resulting signature.
116
+
117
+
{2 Bug #1385: Exception raised during compilation}
118
+
119
+
The second bug has the identical backtrace, indicating a problem with arities. However,
120
+
the repro case for this one does not involve any inline destructive substitution, though
121
+
it does involve destructive substitution at the module expression level:
122
+
123
+
{@ocaml norun[
124
+
module type Creators_base = sig
125
+
type ('a, _, _) t
126
+
type (_, _, _) concat
127
+
128
+
include sig
129
+
type ('a, 'b, 'c) t
130
+
131
+
val concat : (('a, 'p1, 'p2) t, 'p1, 'p2) concat -> ('a, 'p1, 'p2) t
132
+
end
133
+
with type ('a, 'b, 'c) t := ('a, 'b, 'c) t
134
+
end
135
+
136
+
module type S0_with_creators_base = sig
137
+
type t
138
+
139
+
include Creators_base with type ('a, _, _) t := t and type ('a, _, _) concat := t
140
+
end
141
+
]}
142
+
143
+
There's quite a lot of type parameters flying around here, so the first step was to
144
+
try to simplify this as much as possible while still getting the exception. I got
145
+
it down to:
146
+
147
+
{@ocaml norun[
148
+
module type Creators_base = sig
149
+
type 'a t
150
+
type _ concat
151
+
152
+
include sig
153
+
type 'a t
154
+
155
+
val concat : 'a concat -> 'a t
156
+
end
157
+
with type 'a t := 'a t
158
+
end
159
+
160
+
module type S0_with_creators_base = sig
161
+
type t
162
+
163
+
include Creators_base with type _ t := t with type _ concat := t
164
+
end
165
+
]}
166
+
167
+
which still throws the same exception. So, what's going on here? Fundamentally,
168
+
it's a similar issue to the first bug, just caused in a different way, in that
169
+
once again we'll end up with a signature that has two definitions of [type t]
170
+
with different arities. In this case, the problem occurs during the
171
+
expansion of [S0_with_creators_base].
172
+
173
+
This is the intermediate expansion of [Creators_base] that odoc calculates:
174
+
175
+
{@ocaml norun[
176
+
module type S0_with_creators_base = sig
177
+
type t
178
+
179
+
include Creators_base with type _ t := t with type _ concat := t (*
180
+
181
+
The expansion as calculated by odoc is:
182
+
183
+
include sig
184
+
type 'a t
185
+
val concat : t -> 'a t
186
+
end
187
+
*)
188
+
end
189
+
]}
190
+
191
+
What's happened here is during the calculation of the body of the include, odoc has taken
192
+
the signature of [Creators_base] and has its two type definitions
193
+
both replaced with [type t] (with no parameters). However, since the [type t] in the body
194
+
of the include is defined in that signature, that one wasn't replaced. So we end up with
195
+
the type of [concat] being [t -> 'a t], which looks very odd! At this point though,
196
+
odoc knows very well that they're different types. However, when odoc converts this signature
197
+
back into the datatype that represents the expansions, it loses that information and
198
+
we end up with the two types mixed up. We then go on to process this signature,
199
+
the mixup of the arities causes the failure.
200
+
201
+
There are several independent fixes that we can make here. Firstly we can make sure that
202
+
we don't mix up the types. This we can do because we can distinguish between items
203
+
that are declared within the signature of the include's declaration and those that come
204
+
from the outer context. We don't have to do this for the expansion of the include as
205
+
OCaml's type system means that there can't be two types of the same name in the
206
+
resulting signature. We never actually render any signature that occurs within the
207
+
body of an include, so this doesn't actually make any difference to the output.
208
+
209
+
The second fix is to make sure that we only calculate the expansion of the include
210
+
once. Currently the bug happens because we try to re-calculate the expansion of the
211
+
[include sig ... end] expression, even though we calculated it during the processing
212
+
of [S0_with_creators_base]. What we should do instead is apply the substitutions to
213
+
the expansion of that calculated include, which would end up with the same result.
214
+
This isn't a perfect solution though, as there are occasions when we have to
215
+
recalculate the signature anyway.
216
+
217
+
The third fix is - and this takes a little care to parse - to ensure that we never
218
+
actually try to process the items within a signature within a "with" expression
219
+
within a module-type expression. Before diving into the 'why' of this, let's first
220
+
explain how Odoc represents module-type expressions.
221
+
222
+
Internally, we have {{!Odoc_model.Lang.ModuleType.expr}a datatype} that represents module expressions, which looks
223
+
like this:
224
+
225
+
{@ocaml nomerlin norun[
226
+
type expr =
227
+
| Path of path_t
228
+
| Signature of Signature.t
229
+
| Functor of FunctorParameter.t * expr
230
+
| With of with_t
231
+
| TypeOf of typeof_t
232
+
]}
233
+
234
+
Now, each of the arguments to these constructors might contain an expansion of the expression that Odoc will
235
+
calculate. For example, the definition of {{!Odoc_model.Lang.ModuleType.path_t}path_t} is:
236
+
237
+
{@ocaml nomerlin norun[
238
+
type path_t = {
239
+
p_expansion : simple_expansion option;
240
+
p_path : Paths.Path.ModuleType.t;
241
+
}
242
+
]}
243
+
244
+
and this expansion is initially [None] and then filled in by Odoc in order to render the expansion in the
245
+
HTML. In the case of a [With] expression, the {{!Odoc_model.Lang.ModuleType.with_t}with_t} type is:
246
+
247
+
{@ocaml nomerlin norun[
248
+
type with_t = {
249
+
w_substitutions : substitution list;
250
+
w_expansion : simple_expansion option;
251
+
w_expr : U.expr;
252
+
}
253
+
]}
254
+
255
+
here you can see that the [With] expression contains another module expression, as a [with] expression operates on
256
+
another module type. Early during Odoc's development, this simply was another `ModuleType.expr`, but we had a couple
257
+
of bugs where we ended up calculating expansions for these inner expressions, which was all very wasteful as we
258
+
only ever rendered the "outer" expansion. So we changed this to be a {{!Odoc_model.Lang.ModuleType.U.expr}U.expr},
259
+
which is an "unexpanded" module type expression, and is very similar to the main expression above, but without
260
+
the expansions and also with the functor case, as we can't have functors inside a "with" expression.
261
+
262
+
These "unexpanded" expressions still contain signatures though, so aren't {e completely} unexpanded, and it's
263
+
{e these} signatures that we should avoid processing.
264
+
265
+
So, what I expected to be just one bug when I started looking at this turned out to be two related issues,
266
+
and a total of four different fixes!
267
+
268
+
269
+
270
+
271
+
+145
site/blog/2025/11/foundations-of-computer-science.mld
+145
site/blog/2025/11/foundations-of-computer-science.mld
···
1
+
{0 Foundations of Computer Science}
2
+
3
+
@published 2025-11-14
4
+
5
+
I recently completed lecturing the course
6
+
{{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/}"Foundations of Computer
7
+
Science"} to our newly arrived first-year computer scientists here at
8
+
{{:https://www.cam.ac.uk}Cambridge}. This is the first time I've lectured this
9
+
course, taking over from {{:https://anil.recoil.org/}Anil} while he's on
10
+
sabbatical. Although I was very nervous indeed about it, I ended up really
11
+
enjoying the experience - and I hope the students did too! This post is a little
12
+
brain dump of my thoughts on how it went and how we might improve it for next
13
+
year.
14
+
15
+
{1 Course Overview}
16
+
17
+
The course is 12 lectures long and has been lectured in a similar way since I
18
+
myself was an undergraduate here, way back in 1996. There have been a few
19
+
changes, not least of which is that back then it was in Standard ML rather than
20
+
OCaml, but the core material has remained largely the same: lists, recursive
21
+
functions, trees, higher-order functions, search and finally mutability. There
22
+
are no prerequisites for the course, although all students have at least a maths
23
+
A-level (or equivalent), and almost all of them have done some programming
24
+
before, though the experience varies widely. Very few have done any functional
25
+
programming, and even fewer have written any OCaml before.
26
+
27
+
The notes for the course are distributed both in hard copy and also as an
28
+
{{:https://github.com/ocamllabs/focs-notebooks/blob/main/1A%20Foundations%20of%20Computer%20Science.ipynb}interactive
29
+
Jupyter Notebook}, which we host on our {{:https://hub.cl.cam.ac.uk}JupyterHub
30
+
server} that I maintain. The idea is that the students can read through the
31
+
notes and then play around with the code examples directly in the notebook. I
32
+
don't encourage them or give them time to do much {i during} the lectures - not
33
+
that I think this is a terrible idea, but it's a struggle to fit all the
34
+
material in otherwise! The notes are pretty closely coupled to the lectures,
35
+
organised into 11 chapters that correspond to the first 11 lectures, with
36
+
exercises at the end of each chapter that are intended to be covered in the
37
+
supervisions. We also have some assessed exercises - "Ticks" - that the students
38
+
complete in their own time using the JupyterHub server using {{:https://github.com/jupyter/nbgrader}nbgrader}. They
39
+
are automatically assessed in a very transparent way; each "tick" is a Jupyter
40
+
notebook with editable answer cells and read-only test cells. Overall we're
41
+
aiming for the students not to {i have} to install OCaml locally at all, though
42
+
I hope many of them will choose to do so anyway.
43
+
44
+
While I didn't want them playing around with the notebook during the lectures, I
45
+
do, however, try to get them to interact by getting them to answer questions.
46
+
It's pretty intimidating to stick your head above the parapet like this, so as
47
+
an incentive I rewarded those that answered (rightly or wrongly) with some of
48
+
the excellent stickers that Tarides has printed over the years. Everybody loves
49
+
stickers!
50
+
51
+
The questions I asked varied quite a lot in their difficulty, and many were in
52
+
the first few minutes of each lecture, where I had a short 'warm-up' where we
53
+
recapped the contents of the previous lecture. These warm-ups were strongly
54
+
suggested by Anil, and as well as reminding everyone of where we left off, they
55
+
also gave me a bit of feedback on the things that the students found
56
+
challenging.
57
+
58
+
One entertaining aspect is that during the first lecture I do actually encourage
59
+
them to at least log on to the JupyterHub server, mostly to get them used to the
60
+
idea of trying it. The entertaining part is that our server isn't particularly
61
+
big and beefy, and so with 130 students all trying to log on at once, it
62
+
invariably caves in under the load. At this point in the lecture I ssh to the
63
+
server and run btop/htop and we watch it die in real time!
64
+
65
+
{1 What changed this year}
66
+
67
+
During the lectures themselves, rather than use Keynote or PowerPoint for the
68
+
slides, I decided to try using
69
+
{{:https://slipshow.readthedocs.io/en/stable/}Slipshow}, augmented with
70
+
{{:https://github.com/art-w/x-ocaml}x-ocaml} to embed executable OCaml code
71
+
snippets. I'm very happy with how this worked out. I was able to prepare both
72
+
working and broken snippets, modify them live during the lecture, and things
73
+
like type-on-hover was very useful. In a few lectures where we were discussing
74
+
big-O notation, I was able to run code on different input sizes and really
75
+
demonstrate the big difference in run-time of certain algorithms. After the
76
+
lectures, I posted the slides onto the course website so that students can refer
77
+
back to them, and they can also try out the live code snippets directly in the
78
+
slides.
79
+
80
+
Both Slipshow and x-ocaml are still quite young projects, so it was inevitable
81
+
that there were a few rough edges, and in fact the interaction of the two
82
+
revealed the biggest problem: that when you use the 'speaker-view' mode of
83
+
Slipshow, where you have a separate window with notes and the current slide, the
84
+
x-ocaml widgets are effectively independent in the two windows, so updating in one doesn't
85
+
update in the other. {{:https://choum.net/panglesd/}Paul-Elliot}, the author of
86
+
Slipshow, had already got a potential fix for this in the works when I spoke to
87
+
him about it, so hopefully next time I use this I'll be able to have speaker
88
+
notes on screen, instead of hand-written index cards! The x-ocaml project is a
89
+
lot smaller than Slipshow, so I was able to use Claude to help me add
90
+
functionality I needed, such as being able to programmatically highlight
91
+
sections of the code.
92
+
93
+
Another new thing I tried this year was to go over 'tracing' of execution to
94
+
help the students understand how programs run. We've always taught reduction
95
+
steps in the course, which works well as it's only the last lecture where we
96
+
introduce mutability, but it can quickly become unwieldy, and it can be
97
+
challenging to do this all by hand. Tracing a function tells the runtime to log
98
+
when function calls and returns happen, so you just need to call the function on
99
+
your desired input, and you get a fully automatic trace of the execution. As
100
+
it's only function calls and returns, it doesn't tell the full story, but
101
+
alongside the handwritten reduction, it can help reassure students that they're
102
+
on the right track. I ended up writing up a trace of a particularly complicated
103
+
lazy-list evaluation using Slipshow and x-ocaml, which I posted
104
+
{{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/interleave_explanation.html}
105
+
here}.
106
+
107
+
{1 Thoughts for next year}
108
+
109
+
Overall I'm very happy with how the course went this year, though in some ways
110
+
it did feel a little bit like the course finished just when it had started to
111
+
get to the good stuff! There's a Tripos review process going on at the moment,
112
+
so maybe we'll get to expand this course a bit in future years.
113
+
114
+
While the Slipshow+x-ocaml combination worked well, the fact that we ended up
115
+
with two separate systems for executing OCaml wasn't ideal. I think it'd be a
116
+
really nice project to investigate just how far we can push x-ocaml / Slipshow /
117
+
some other web technology to have a true "serverless" experience so we can ditch
118
+
the JupyterHub server entirely. By caching the x-ocaml 'execution' web worker in
119
+
the browser, we could have a system that works fully offline, removing an
120
+
annoyingly failure-prone single point of failure. Of course, we'd still need
121
+
some way to do the assessed exercises, but that's a small point in a much larger
122
+
problem: we really can't continue to ignore how LLMs are impacting the way that
123
+
students are approaching these exercises in both positive and negative ways.
124
+
To answer this properly, we need to
125
+
think hard about what the purpose of these exercises is and look around to see
126
+
what our {{:https://eecs.iisc.ac.in/people/prof-viraj-kumar/}colleagues} are
127
+
doing {{:https://dl.acm.org/doi/10.1145/3724363.3729100}in this space}.
128
+
129
+
The slide decks themselves are fully open and available on the {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/}course website}:
130
+
131
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture1/lecture1.html}Introduction}
132
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture2/lecture2.html}Recursion and Complexity}
133
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture3/lecture3.html}Lists and Polymorphism}
134
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture4/lecture4.html}More Lists and Making Change}
135
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture5/lecture5.html}Sorting}
136
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture6/lecture6.html}Datatypes and Trees}
137
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture7/lecture7.html}Dictionaries and Functional Arrays}
138
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture8/lecture8.html}Currying}
139
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture9/lecture9.html}Sequences, or Lazy Lists}
140
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture10/lecture10.html}Search}
141
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture11/lecture11.html}Procedural Programming}
142
+
+ {{:https://www.cl.cam.ac.uk/teaching/2526/FoundsCS/lecture12/lecture12.html}Recap and Real World Use!}
143
+
144
+
145
+
+4
site/blog/2025/11/index.mld
+4
site/blog/2025/11/index.mld
+65
site/blog/2025/12/an-svg-is-all-you-need.mld
+65
site/blog/2025/12/an-svg-is-all-you-need.mld
···
1
+
{0 An SVG is all you need}
2
+
3
+
@published 2025-12-09
4
+
5
+
SVGs are pretty cool - vector graphics in a simple XML format. They are supported on just
6
+
about every device and platform, are crisp on every display, and can have embedded scripts
7
+
in to make them interactive. They're {{:https://www.youtube.com/watch?v=4laPOtTRteI}way more capable} than many people realise,
8
+
and I think we can capitalise on some of that unrealised potential.
9
+
10
+
Anil's recent post {{:https://anil.recoil.org/notes/principles-for-collective-knowledge}Four Ps for Building Massive Collective Knowledge Systems}
11
+
got me thinking about the permanence of the experimentation that underlies our scientific
12
+
papers. In my idealistic vision of how scientific publishing should work, each paper would
13
+
be accompanied by a fully interactive environment where the reader could explore the data,
14
+
rerun the experiments, tweak the parameters, and see how the results changed. Obviously
15
+
we can't do this in the general case - some experiments are just too expensive or time-consuming
16
+
to rerun on demand. But for many papers, especially in computer science, this is entirely
17
+
feasible.
18
+
19
+
That line of thought reminded me of a project I tackled about 20 years ago as a post-doc in
20
+
the Department of Plant Sciences here in Cambridge. I was writing a paper on
21
+
{{:https://royalsocietypublishing.org/rsif/article/9/70/949/173/Applications-of-percolation-theory-to-fungal}synergy
22
+
in fungal networks} and built a tiny SVG
23
+
visualisation tool that let readers wander through the raw data captured from a real fungal network growing in a petri dish. I dug it up recently and was surprised
24
+
(and delighted) to see that it still works perfectly in modern browsers - even though the original “cover page” suggested Firefox 1.5 or the Adobe SVG plug-in (!). Give
25
+
it a spin; click the 'forward', 'back' and other buttons below the petri dish!
26
+
27
+
{image!fungus.svg}
28
+
29
+
And that, dear reader, is literally all you need. A completely self-contained SVG file can either fetch data from a versioned repository or embed the data directly, as the example
30
+
does. It can process that data, generate visualisations, and render knobs and sliders for interactive exploration. No server-side magic required - everything runs
31
+
client-side in the browser, served by a plain static web server, and very easily to share.
32
+
33
+
How does it fit in with Anil's four Ps?
34
+
35
+
- Permanence: SVGs can be assigned DOIs just like papers, blog posts, or datasets. The fact that the above SVG still works after two decades is a testament to
36
+
the durability of the format.
37
+
38
+
- Provenance: Because SVG is plain text, it plays nicely with version control systems such as Git. When an SVG pulls in external data, the same
39
+
provenance-tracking strategies Anil describes for datasets apply here as well.
40
+
41
+
- Permission: Once again, with the separation between the processing in the SVG and that
42
+
data that it works on, the same permissioning models apply as for data in general.
43
+
44
+
- Placement: SVGs are {i inherently} spatial; it's very easy, for example, to make beautiful
45
+
{{:https://stephanwagner.me/coding/blog/create-world-map-charts-with-svgmap#svgMapDemoGDP}world maps} with
46
+
SVG.
47
+
48
+
The SVG above is only a visualisation tool for data; it doesn't really do any processing,
49
+
but it certainly {i could}. The biggest change that's happened over the 20 years since I
50
+
wrote this is the {i massive} increase in the computation power available in the browser.
51
+
If would be entirely feasible to implement the entire data analysis pipeline for that
52
+
paper in an SVG today, probably without even spinning up the fans on my laptop!
53
+
54
+
So this is yet another tool in our ongoing effort to be able to effortlessly share and
55
+
remix our work - added to the pile of Jupyter notebooks, {{:https://digitalflapjack.com/blog/marimo/}Marimo botebooks},
56
+
the {{:https://slipshow.readthedocs.io/en/stable/}slipshow}/{{:https://github.com/art-w/x-ocaml/}x-ocaml}
57
+
{{!/site/blog/2025/11/page-"foundations-of-computer-science"}combination},
58
+
{{:https://patrick.sirref.org/weekly-2025-w45/index.xml}Patrick's take} on Jon Sterling's
59
+
{{:https://sr.ht/~jonsterling/forester/}Forester}, my own {{!/site/notebooks/page-index}notebooks},
60
+
and many others - and this is a subset of what we're using just in our own group!
61
+
62
+
63
+
64
+
65
+
+201
site/blog/2025/12/claude-and-dune.mld
+201
site/blog/2025/12/claude-and-dune.mld
···
1
+
{0 Claude and Dune}
2
+
3
+
@published 2025-12-18
4
+
5
+
Back in March of this year we released {{:https://ocaml.github.io/odoc/odoc/index.html}odoc 3.0.0}, a
6
+
major new version of the OCaml documentation generator. It had a whole load of {{:https://discuss.ocaml.org/t/ann-odoc-3-beta-release/16043}new features},
7
+
many of which came with new demands on the build system driving it. We decided when
8
+
working on it to build a new driver for odoc so that we could adjust it as we were
9
+
building the new features, and this driver is now used to {{!/site/blog/2025/07/page-"odoc-3-live-on-ocaml-org"}build the documentation} that appears on
10
+
{{:https://ocaml.org/p/base/latest/doc/index.html}ocaml.org}. However, it was
11
+
always the plan to integrate the new features into {{:https://dune.build}Dune} so
12
+
that everyone could just run [dune build @doc] and be able to use all of the new
13
+
odoc 3 features.
14
+
15
+
So over the last few weeks I have been wrestling with getting Claude to update
16
+
the odoc rules in Dune to support {i some} of the new features of odoc v3.
17
+
What began as a background experiment during a lecture series has turned into a
18
+
multi-week effort to turn mostly-working code into a clean, reviewable patch.
19
+
AI-developed software is clearly going to be a big part of our future, and
20
+
Anil is showing us all the way with his {{:https://anil.recoil.org/notes/aoah-2025-1}Advent of Agentic Humps}
21
+
by building {i new} software, but upstreaming AI-generated changes to an
22
+
existing, well established code base
23
+
{{:https://github.com/ocaml/ocaml/pull/14369}hasn't got off to a good start}
24
+
in the OCaml community, so I wanted to be extra careful to get this right.
25
+
26
+
{2 Claude as a protyping tool}
27
+
The initial progress was pretty amazing, despite my initial worries that the dune
28
+
code-base would be {{:https://github.com/ocaml/dune/pull/12529}too large and subtle} for an LLM to be able to make workable
29
+
changes. In order to get going, first I had it look at several bits of example code:
30
+
31
+
1. {{:https://github.com/ocaml/dune/blob/3.20.2/src/dune_rules/odoc.ml}dune_rules/odoc.ml} -
32
+
this is the current home of the odoc rules in dune. It's
33
+
local-only, meaning it only builds the docs for the current package in
34
+
isolation, so no resolution of links to stdlib, other packages or libraries.
35
+
36
+
2. {{:https://github.com/ocaml/dune/blob/3.20.2/src/dune_rules/odoc_new.ml}dune_rules/odoc_new.ml} - these are the rules for odoc v2, which allow you to
37
+
build the docs for your package plus all of the dependencies. I wrote this
38
+
mostly myself some time ago. It does a pretty poor job of caching, error
39
+
reporting, and has none of the odoc v3 features like assets, source rendering,
40
+
hierarchical docs, better errors and so on.
41
+
42
+
3. {{:https://github.com/ocaml/odoc/tree/d8460cdaa2b91a03434a9a045d673703b7fabfb2/src/driver}odoc_driver} - this is the driver we wrote when building odoc v3. It's fully
43
+
featured, but not at all incremental, and actually external to the dune codebase.
44
+
It's the reference implementation that's used to build the docs that appear on {{:https://ocaml.org/p/base/latest/doc/index.html}ocaml.org}.
45
+
46
+
Armed with these three code-bases, I asked Claude to synthesise a new incremental
47
+
version of the odoc rules for dune that has some of the features of [odoc_driver].
48
+
49
+
{2 The working prototype}
50
+
Claude quickly produced a prototype that actually compiled and generated
51
+
documentation. At that stage I was not interested in the quality of the
52
+
generated source; I only needed to know whether Claude could navigate Dune's
53
+
codebase and produce something that {b works}. I let the prototype evolve
54
+
incrementally, adding in new features one at a time, for example, fixing
55
+
the error reporting so that you only get warned about documentation errors
56
+
that you can actually fix.
57
+
58
+
When the lectures finished, it turned out I had something that was pretty useful
59
+
to me, and had a good chance to be useful to others too. So I opened up my
60
+
editor and had a look through what had been produced, at this point hoping that
61
+
a little bit of polishing should be enough - after all, it {i was} working!
62
+
63
+
It was dreadful.
64
+
65
+
There were long, rambling functions, code duplication, bad comments, it was
66
+
unstructured, with repeated-but-slightly-different chunks all over the place. It
67
+
wasn't just bad on one length scale - it was bad from the large-scale
68
+
organisation of the code down to small scale baffling weirdnesses on one line.
69
+
The more I looked, the more bonkers it appeared. But it did {i work}! So I
70
+
thought I'd get Claude to clean up its own messes.
71
+
72
+
{2 The clean-up}
73
+
I resolved that I would continue to let Claude do {i all} of the editing, and not do
74
+
{i any} myself, and so thus began the more frustrating part of this adventure! I
75
+
ended up giving a mix of very specific instructions: "move this code here",
76
+
"factorize out this functionality", "rename this function", and sometimes more
77
+
general ones: "Remove any comments that don't add anything of value", or "Think
78
+
of a better way to do this". The constant was that I needed to be looking over
79
+
each change that it did, because while most of them were pretty good, there were
80
+
still a few, even with the very explicit instructions, where it messed up. From
81
+
the very broad, where at one point it told me "I'll remove this code to create
82
+
odoc files for external dependencies, as they're installed by opam", which isn't
83
+
true, down to the very small - for example, it produced the following:
84
+
85
+
{@ocaml[
86
+
let lib_names = deps.Odoc_config.libraries in
87
+
if List.is_empty lib_names
88
+
then Memo.return []
89
+
else Memo.List.filter_map lib_names ~f:(fun lib_name -> Lib.DB.find lib_db lib_name)
90
+
]}
91
+
92
+
where it has come up with a totally redundant check for the empty list.
93
+
94
+
It was at this point where it became frustrating, because although it's almost
95
+
magical that Claude can do what it does in the time it does, this fact of having
96
+
to keep a constant eye on it meant the the tens-of-seconds to minutes delay in
97
+
between it doing something meant I ended up either twiddling my thumbs for long
98
+
periods of time, or getting started on some other task and forgetting to come
99
+
back to Claude, sometimes for hours!
100
+
101
+
{2 OCaml is {b not} the problem}
102
+
One part that particularly impressed, and also quite surprised me, was with its
103
+
knowledge of OCaml. In particular, I had at one point two different types
104
+
representing the 'target' - either a library or a package - and a 'kind' -
105
+
either a module or a page. Now pages can only be associated with package
106
+
targets, and modules can only be associated with libraries, but these two values
107
+
were distinct, so there was a fair bit of code pattern matching invalid
108
+
combinations and either throwing exceptions or picking some random value,
109
+
depending on the whims of Claude's context. I bravely suggested it think of a
110
+
better way to represent this, maybe using GADTs, and it did indeed come up with
111
+
a pretty nice refactoring of the types:
112
+
113
+
Before:
114
+
115
+
{@ocaml[
116
+
type target =
117
+
| Lib of Package.Name.t * Lib.t
118
+
| Pkg of Package.Name.t
119
+
120
+
type artifact_kind =
121
+
| Module of
122
+
{ visible : bool
123
+
; module_name : Module_name.t
124
+
; archive : string (* Which archive the module belongs to *)
125
+
}
126
+
| Page of
127
+
{ name : string
128
+
; pkg_libs : Lib.t list
129
+
}
130
+
]}
131
+
132
+
After:
133
+
134
+
{@ocaml[
135
+
(* Artifact data types *)
136
+
type page = { name : string; pkg_libs : Lib.t list }
137
+
138
+
type mod_ =
139
+
{ visible : bool
140
+
; module_name : Module_name.t
141
+
; archive : string (* Which archive the module belongs to *)
142
+
}
143
+
144
+
type _ target =
145
+
| Lib : Package.Name.t * Lib.t -> mod_ target
146
+
| Pkg : Package.Name.t -> page target
147
+
148
+
type artifact_kind =
149
+
| Module : mod_ * mod_ target -> artifact_kind
150
+
| Page : page * page target -> artifact_kind
151
+
]}
152
+
153
+
This refactoring immediately removed a whole swathe of invalid combinations,
154
+
making the code both safer and clearer. It's quite clear that Claude had
155
+
no trouble understanding how GADTs work in OCaml, quite happily also using
156
+
some existentials to pack them into lists and so on.
157
+
158
+
{2 Odd behaviours}
159
+
Sometimes Claude just went a little bit bananas. One annoyance that {i repeatedly}
160
+
occurred was that it would forget how to build and test the dune executable,
161
+
despite clear instructions in [Claude.md]. Most of the time when
162
+
it went wrong it would build dune, execute [dune clean], then try to run
163
+
the dune binary that it had just removed with the [clean]. Sometimes it would
164
+
decide to use the bootstrap binary instead, which isn't rebuilt on every
165
+
change, sometimes it would run the switch-installed dune binary, and on one
166
+
occasion it tried to run [./configure && make]!
167
+
168
+
It would usually figure out eventually what the right thing to do was, but
169
+
when you're waiting for it to complete so you can check what it's done these
170
+
sorts of delays got a bit frustrating.
171
+
172
+
{2 Reflections}
173
+
At one point, I ran out of Claude credits (despite paying $100 a month or so),
174
+
at about 6:20pm one evening, and it told me that I needed to wait until 7pm to
175
+
carry on. I'd just got to the point when I needed to write a short bit of code
176
+
rather than refactoring what was already there, and I realised that while it
177
+
would take me maybe 10 mins, it would take Claude maybe 10 seconds. Now, it
178
+
could just be that it was the end of a long day and I was running out of steam,
179
+
but I was content to switch focus elsewhere for a bit to wait for my credits to
180
+
reset before carrying on! The point being that for the small implementation that
181
+
I was after, it would be possible for me to get Claude to do it, and to eyeball
182
+
the result to make sure it was OK in less time than I would have been able to do
183
+
it myself. But I absolutely wouldn't have trusted Claude to do it in an upstreamable way
184
+
{b without} looking at the result.
185
+
186
+
Overall, It's clear that Claude will be an incredibly useful tool for working with
187
+
software. It's unbelievably good at jumping into a new code-base and figuring things
188
+
out quickly, but less good at producing high-quality code that can be directly
189
+
submitted upstream (yet?) - at least, not that {b I} would be comfortable submitting
190
+
anyway. However, I think it's still a bit of an open question as to what the quality bar
191
+
{e should} be. If it builds correctly, passes the tests, looks {i broadly} sensible and
192
+
isn't on the critical path for performance, how much should we care about the line-to-line
193
+
quality? {b I} certainly care, but am I being old fashioned?
194
+
195
+
I've submitted a {{:https://github.com/ocaml/dune/pull/12995}PR with these changes} for review, and we'll see what happens there.
196
+
I ended up squashing all of the commits into one, as the intermediate steps are very
197
+
likely not useful. However, for historical interest, the branch on which I did most of
198
+
the work is {{:https://github.com/ocaml/dune/compare/main...jonludlam:dune:odoc3-global-sidebar}here}.
199
+
200
+
201
+
+4
site/blog/2025/12/index.mld
+4
site/blog/2025/12/index.mld
+32
site/blog/2025/index.mld
+32
site/blog/2025/index.mld
···
1
+
{0 2025}
2
+
3
+
@children_order 12/ 11/ 09/ 08/ 07/ 06/ 05/ 04/ 03/
4
+
5
+
- {{!/site/blog/2025/12/page-"claude-and-dune"}Claude and Dune}
6
+
- {{!/site/blog/2025/12/page-"an-svg-is-all-you-need"}An SVG is all you need}
7
+
- {{!/site/blog/2025/11/page-"foundations-of-computer-science"}Foundations of Computer Science}
8
+
- {{!/site/blog/2025/09/page-"caching-opam-solutions2"}Caching opam solutions - part 2}
9
+
- {{!/site/blog/2025/09/page-"odoc-bugs"}Odoc bugs}
10
+
- {{!/site/blog/2025/09/page-"caching-opam-solutions"}Caching opam solutions}
11
+
- {{!/site/blog/2025/09/page-"build-ids-for-day10"}Build IDs for Day10}
12
+
- {{!/site/blog/2025/09/page-"giving-hub-cl-an-upgrade"}Giving hub.cl an upgrade}
13
+
- {{!/site/blog/2025/08/page-"ocaml-lsp-mcp"}Using ocaml-lsp-server via an MCP server}
14
+
- {{!/site/blog/2025/08/page-"ocaml-mcp-server"}An OCaml MCP server}
15
+
- {{!/site/blog/2025/08/page-week33}Week 33}
16
+
- {{!/site/blog/2025/07/page-retrospective}4 months in, a retrospective}
17
+
- {{!/site/blog/2025/07/page-"odoc-3-live-on-ocaml-org"}Odoc 3 is live on OCaml.org!}
18
+
- {{!/site/blog/2025/07/page-week28}Week 28}
19
+
- {{!/site/blog/2025/07/page-week27}Weeks 24-27}
20
+
- {{!/site/blog/2025/06/page-week23}Week 23}
21
+
- {{!/site/blog/2025/05/page-"docs-progress"}Progress in OCaml docs}
22
+
- {{!/site/blog/2025/05/page-"lots-of-things"}Lots of things have been happening}
23
+
- {{!/site/blog/2025/05/page-"ticks-solved-by-ai"}Solving First-year OCaml exercises with AI}
24
+
- {{!/site/blog/2025/05/page-"oxcaml-gets-closer"}OxCaml is getting closer...}
25
+
- {{!/site/blog/2025/05/page-"ai-for-climate-and-nature-day"}AI for Climate & Nature Community Day}
26
+
- {{!/site/blog/2025/04/page-"ocaml-docs-ci-and-odoc-3"}OCaml-Docs-CI and Odoc 3}
27
+
- {{!/site/blog/2025/04/page-"odoc-3"}Odoc 3: So what?}
28
+
- {{!/site/blog/2025/04/page-"semantic-versioning-is-hard"}Semantic Versioning in OCaml is Hard}
29
+
- {{!/site/blog/2025/04/page-"meeting-the-team"}Meeting the Team}
30
+
- {{!/site/blog/2025/04/page-"this-site"}This site}
31
+
- {{!/site/blog/2025/03/page-"module-type-of"}The Road to Odoc 3: Module Type Of}
32
+
- {{!/site/blog/2025/03/page-"code-block-metadata"}Code block metadata}
+4
site/blog/2026/01/index.mld
+4
site/blog/2026/01/index.mld
+114
site/blog/2026/01/weeknotes-2026-03.mld
+114
site/blog/2026/01/weeknotes-2026-03.mld
···
1
+
{0 Weeknotes for week 3}
2
+
3
+
@published 2026-01-19
4
+
5
+
First week back of 2026! Let's write some terse weeknotes.
6
+
7
+
{1 Projects}
8
+
{2 Dune odoc rules}
9
+
10
+
Last thing I did last year was to push the new rules for odoc 3. This week, Anil handed me an excellent opportunity to test the
11
+
rules on the monorepo containing his {{:https://anil.recoil.org/notes/aoah-2025}AOAH} projects. Claude tends to actually write ocamldoc-formatted comments, so this
12
+
is really useful to test the rules. I've {{:https://github.com/jonludlam/dune/tree/odoc-v3-rules-3.21}rebased the commits} on
13
+
the just-released Dune 3.21 and we've been trying them out. There were a few things to fix:
14
+
15
+
- More careful {{:https://github.com/jonludlam/dune/commit/25158eabf0c3cac2826e16ce590b4bd4d7c09818}dependency tracking} during the compile phase - this particularly affected the [@doc] target, which was pulling
16
+
in unnecessary dependencies. Most of these dependencies were compiling just fine, but one - Anstrom - is slightly odd in that
17
+
the opam install of Angstrom installs a META file that references libraries that aren't in the dependencies of its opam package.
18
+
This is a backward-compatibility hack that was implemented when the Anstrom package was split into several in order to manage
19
+
the dependencies better.
20
+
- A similar issue happens with eio, where the documentation of the package depends upon [bigstring], which isn't in eio's dependencies.
21
+
This is entirely intentional - the extra doc dependencies is stated in the opam file with a [x-extra-doc-deps] field. However,
22
+
[opam install] totally ignores this field (quite reasonably), and so a simple install gives you an opam repo whose docs can't
23
+
be built. Once again, this broke [dune build @doc] unnecessarily, but the fix was {{:https://github.com/jonludlam/dune/commit/2afe046cf4290d7a83b5f2c5646e3391ca94b630}relatively simple}. The {i real} fix here is to not use [x-extra-doc-deps], but switch to using a {i real} dependency, but marked with [with-doc]
24
+
and [post] if it would otherwise introduce a circular dependency. That way, an [opam install --with-doc] {i would} install the
25
+
extra dependency.
26
+
- Over the Christmas break, {{:https://discuss.ocaml.org/u/tbrk}tbrk} posted {{:https://discuss.ocaml.org/t/odoc-index-for-multiple-packages-inter-package-links-and-local-global-sidebar/17652}on discuss} a question
27
+
about building docs, for which my dune branch was a partial answer. One feature he was requesting though was the ability to use
28
+
a custom top-level index. It's a useful feature that's implemented in [odoc_driver] so I've {{:https://github.com/jonludlam/dune/commit/efecdee1b36b7e47906e7c64b7496a1fc7954a2d}added it}.
29
+
- More sensible {{:https://github.com/jonludlam/dune/commit/039eb3d2a3e9d28f8b195905f43839daf5ce8c21}default link scope}. By default, documentation references in the [mli] files of a library can link to any other
30
+
library in the package. However, by default it wasn't possible to link to the dependencies of another library, unless it happened
31
+
to be a dependency of your own library. Similarly, the package-wide mld files could only reference the modules in the package's
32
+
libraries, not to the dependencies. This seems overly cautious, as we can be sure that if we've managed to build the libraries
33
+
then their dependencies are installed, and if there are any module name conflicts, we can resolve them via the [/<lib>/Module]
34
+
syntax.
35
+
- Lastly, implementations of virtual libraries {{:https://github.com/jonludlam/dune/commit/12f9ecbd4888444c2d359049a914ffb4827912f9}need to be skipped} as they've all got the same docs (as they share mli files), and
36
+
the rules as they were causing Dune to crash with a "Conflicting implementations" error.
37
+
38
+
I've also rebased the PR onto latest [main], but I've not yet put these patches there, which I'll need to do for the PR to be
39
+
mergable. For now, the 3.21 branch is successfully building the docs for the monorepo.
40
+
41
+
{2 OCaml Docs CI}
42
+
43
+
{{:https://github.com/jmid}Jan Midtgaard} noticed over xmas that the Docs CI {{:https://github.com/ocaml/ocaml.org/issues/3437}was broken} and submitted {{:https://github.com/jonludlam/opamh/pull/1}a fix}.
44
+
I've therefore been poking {{:https://github.com/ocurrent/ocaml-docs-ci}ocaml-docs-ci} to get the fix incorporated and into production.
45
+
I almost immediately hit the issue that [odoc_driver] now breaks for the exact same reason. I couldn't
46
+
quite understand how [opam-format] {{:https://github.com/ocaml/opam-repository/pull/28978}had been merged} to [opam-repository] without someone noticing that it had broken [odoc_driver],
47
+
but it turned out that it {i had} been noticed, but on a {{:https://github.com/ocaml/opam-repository/pull/28877}beta release}.
48
+
The fix to docs ci was to install [odoc_driver] from opam rather than {{:https://github.com/ocurrent/ocaml-docs-ci/blob/81ca17c7b7a2f47ca571b1d6bc866720cebef136/src/lib/config.ml#L226}pinning directly} to a github hash,
49
+
especially if that hash happens to be the hash of the released version!
50
+
51
+
While I'm working on docs CI, I thought it's probably also a good idea to move over to the [with-doc & post] suggestion from above, so we're ready for
52
+
when packages start to use that. This is now being tested, and hopefully we'll have the CI back up and running early next week.
53
+
54
+
{2 Better styling for odoc}
55
+
56
+
I've done very little to the styling of odoc since I took maintainership way back in 2019 or so. It's a bit dated, and there
57
+
are some annoying usability issues, so I thought it's a good opportunity to vibe-code a nice new frontend for it. Rather than
58
+
hack directly on the HTML generator of odoc, this seemed to be a good opportunity to test the JSON output from the new Dune rules,
59
+
so I asked Claude to make me a static site generator that read in the JSON files and spat out some nicely styled HTML. This
60
+
worked like a charm, and the results are {{:https://jon.ludl.am/experiments/vibe-coded-odoc-frontend/}here}. Next steps are to
61
+
see what it would take to get the native odoc output looking more like that.
62
+
63
+
{2 Custom tags in odoc}
64
+
65
+
One of the themes of Anil's {{:}AOAH} coding spree was that many libraries were implementations of RFCs. In many places in
66
+
the docs, there are links to relevant sections of the RFCs. It'd be nice in future to be able to validate that we've covered
67
+
all of the parts of the RFCs, so making the links a little more parsable seemed like a good idea. In fact, it seemed that this
68
+
might be a perfect use for custom tags - a feature that was present in ocamldoc that odoc has yet to implement.
69
+
70
+
{{:https://github.com/art-w}Arthur Wendling} then pointed me at dune's {{:https://dune.readthedocs.io/en/stable/reference/dune/plugin.html}plugin system}, which seemed just the ticket as a way
71
+
to implement this. It's really nice, taking all of the hard work out of creating OCaml plugins, so I've now got
72
+
{{:https://github.com/jonludlam/odoc/tree/extension-plugins}an extension-plugins branch} that implements this. It allows
73
+
you to add support to odoc for tags like [@rfc] which generate custom HTML, markdown or any other backend, can include
74
+
links in their bodies, and can add custom headers to the web page, and custom files to be output by [odoc support-files].
75
+
It looks like this should "just work" and no further changes to the dune rules are needed - though I need to actually
76
+
test this out.
77
+
78
+
{2 Day10 and docs}
79
+
I've {{!/site/blog/2025/09/page-"build-ids-for-day10"}written about} {{:https://tunbury.org/}Mark's} day10 project
80
+
before. It's a tool to very rapidly build odoc packages mainly in order to test that they build correctly. An obvious
81
+
extension would be to use this to then build the docs for those packages, as the way we do this requires the packages
82
+
to be built first. This would be a replacement for the Docs CI that I talked about above, though there's considerable
83
+
work to do before it's fully-featured enough to be a viable alternative. It seemed like a good time to experiment with
84
+
this though, so I set up one of Anil's {{:https://anil.recoil.org/notes/ocaml-claude-dev}devcontainers}, gave Claude
85
+
some instructions on what to do, took the safety belt off, and let him hack away! Previously most of my interactions
86
+
with Claude had been via the vscode plugin, so using the terminal interface was a bit of a different experience. I'm
87
+
fairly certain though that I'm going to switch everything over to working this way, as letting Claude just get on with
88
+
things without having to OK every step is a far more efficient way to work - especially when you're not that concerned
89
+
with the actual code being produced. This has been mostly a good experience, though Claude does sometimes go off in
90
+
rather odd directions. At one point there was a network error with a dependency while trying to build odoc_driver,
91
+
so it decided that it should have a fallback mechanism that executed odoc directly. I told it {i NEVER} to replace
92
+
functionality in odoc_driver, so it rolled this back, but a few hours later in then did exactly the same thing again.
93
+
94
+
{2 Misc other stuff}
95
+
A few other things too - {{:https://github.com/jonludlam/odoc/commit/59037341cd53d8734a5874f7af2b728b5be70035}improving the [--warn-error] logic in odoc},
96
+
and one of its {{:https://github.com/jonludlam/odoc/commit/9d18feff5eda543652c6749062750de6e5bb4d6e}error messages},
97
+
improving the build of this website so I can iterate on it more quickly, fixing up some of my self-hosted services
98
+
like my tangled knot, and other bits and bobs.
99
+
100
+
{1 Reflections}
101
+
102
+
I think the most important thing this week has been the slightly eye-opening benefits of using Claude
103
+
outside of the context of VSCode. I suspect I'll be doing much more of my work this way in future. There's
104
+
also a good chance I'll have to upgrade my subscription from the $100-per-month to the $200 one...
105
+
106
+
{1 Next week}
107
+
108
+
- Start of term tutorial meetings
109
+
- Sherldoc in monopam-myspace
110
+
- Get ocaml-docs-ci deployed and working
111
+
- Update the Dune PR
112
+
- Integrate the custom-tags and website generator into monopam-myspace
113
+
- Unleash Claude on my js-top-worker repo
114
+
+159
site/blog/2026/01/weeknotes-2026-04-05.mld
+159
site/blog/2026/01/weeknotes-2026-04-05.mld
···
1
+
{0 Weeknotes for weeks 4-5}
2
+
3
+
@published 2026-01-30
4
+
@x-ocaml.requires odoc.extension_api
5
+
@packages odoc-admonition-extension odoc-rfc-extension odoc-msc-extension odoc-mermaid-extension odoc-dot-extension
6
+
7
+
I've been battling the seasonal illnesses this week, so I've combined two weeknotes into one. Fortunately the 'flu
8
+
doesn't hold Claude back!
9
+
10
+
Probably the most interesting part of this is the {!retrospective}, so make sure to read that bit.
11
+
12
+
{1 The Last Two Weeks}
13
+
14
+
As is becoming more and more apparent, the {e breadth} of what I'm working on is ever expanding, powered by
15
+
agentic AI. It's become so much more (cognitively) cheaper to have an idea and set an agent off investigating it
16
+
that I've been finding that I'm working in parallel on far more things in a single week than I would have
17
+
even six months ago. Here are some of the bigger headings though.
18
+
19
+
{2 Monorepo excitement}
20
+
21
+
We're currently experimenting with a new tool - {{:https://tangled.org/anil.recoil.org/monopam}monopam} to help develop across multiple OCaml libraries
22
+
by using git subtrees to create a monorepo with all of the packages in. We then extract patches to the individual
23
+
repos to push upstream. I've been moving my development workflow from in-vscode-claude with careful permissions checking
24
+
to running claude with `--dangerously-skip-permissions` in a container with the monorepo checked out. This has been
25
+
a bit of a bumpy ride, with the tool evolving daily, but I'm very much seeing the benefits of letting Claude just get
26
+
on with things, given a strict enough early design and testing strategy, and using Anil's method of creating the interfaces
27
+
first.
28
+
29
+
{3 Odoc}
30
+
31
+
I also did quite a bit related to odoc these 2 weeks, split over improving functionality and bugfixing.
32
+
33
+
{4 Plugins}
34
+
35
+
Getting Claude to run with all of the monorepo libraries implicitly requires that they're well documented, as looking at the
36
+
source to figure out how to use them exhausts the context window pretty rapidly. Odoc's main focus has been on getting
37
+
the expansions and referencing correct, and while we've made progress on the actual content markup, introducing
38
+
{{:https://ocaml.github.io/odoc/odoc/odoc_for_authors.html#media}media tags} for example, there's still a good distance to go.
39
+
40
+
Using the plugins mechanism I {{!/site/blog/2026/01/page-"weeknotes-2026-03"}wrote about last week},
41
+
I've made a plugin interface for odoc and implemented a few plugins. Initially I was just going to support 'custom tags'
42
+
but it occurred to me that rendering code blocks could also be done in this way. So I've made a few. Two custom tag plugins:
43
+
44
+
- {{!/odoc-admonition-extension/page-index}odoc-admonition-extension} - styled callout blocks for notes, warnings, tips. Note that
45
+
we are intending to make this more first-class - there's a {{:https://hackmd.io/ETSOAmetTI-E3vrDk3Bfrw}design out there}. This
46
+
was just a convenient way to test the feature!
47
+
- {{!/odoc-rfc-extension/page-index}odoc-rfc-extension} - links to IETF RFC documents
48
+
49
+
and 3 code block plugins:
50
+
51
+
- {{!/odoc-msc-extension/page-index}odoc-msc-extension} - Message Sequence Charts
52
+
- {{!/odoc-mermaid-extension/page-index}odoc-mermaid-extension} - Mermaid diagrams (flowcharts, sequence diagrams, etc.)
53
+
- {{!/odoc-dot-extension/page-index}odoc-dot-extension} - Graphviz/DOT diagrams
54
+
55
+
The module signatures relevant to the plugins are documented in {!/odoc.extension_api/Odoc_extension_api} and the
56
+
plugins each have to implement an interface described in {!Odoc_extension_api.Code_Block_Extension} or {!Odoc_extension_api.Extension} for
57
+
custom tags.
58
+
59
+
{4 Bugfixing}
60
+
61
+
{{:https://github.com/lukemaurer}Luke Maurer} at Jane Street pointed out that they're still suffering from
62
+
yet another repro of {{:https://github.com/ocaml/odoc/issues/930}issue 930} at Jane Street. I'd worked on this
63
+
{{!/site/blog/2025/09/page-"odoc-bugs"}back in September} but turns out I hadn't actually made a PR, so I tidied
64
+
up the branch and {{:https://github.com/ocaml/odoc/pull/1400}made a PR}.
65
+
66
+
{2 Docs CI}
67
+
Docs CI has been fixed and is even now rebuilding all of the docs for
68
+
ocaml.org. I've added in the {{:https://github.com/ocurrent/ocaml-docs-ci/commit/c6231fa383820b4c700aaa1e72107536b1872112}handling of `post & with-doc`} in place of
69
+
x-extra-doc-deps, so we should be able to use either mechanism now. The
70
+
idea is to deprecate x-extra-doc-deps soon though. Somehow despite
71
+
an explicit button to press to update the epoch symlinks, it got updated
72
+
anyway and broke most of the docs on ocaml.org. Fortunately {{:https://discuss.ocaml.org/t/is-caqti-doc-missing/17741/5}someone noticed}
73
+
and posted on discuss and so I switched it back.
74
+
75
+
Unfortunately, it seemed to be taking a long time to build the docs - at time of writing it's now Friday, and the CI jobs have been running since Tuesday.
76
+
In that time, it's only managed to build about 6500 packages, a long way short of the 16,000 or so that I expect a full build will produce.
77
+
Looking through the logs, it seems that some change to opam is causing it to sometime rebuild the entire opam universe when it should only be building 1 package.
78
+
For example, in a job that should be building just `tezos-protocol-004-Pt24m4xi`, it installs all of the prebuilt dependencies, then
79
+
runs `opamh` to try to convince opam that everything is all set up to just run the build step for the package we want. Unfortunately
80
+
the logs show the following:
81
+
82
+
{[
83
+
The following actions will be performed:
84
+
=== recompile 178 packages
85
+
- recompile aches 1.1.0 [uses ocaml]
86
+
- recompile aches-lwt 1.1.0 [uses ocaml]
87
+
...
88
+
- recompile mtime 2.1.0 [uses ocaml]
89
+
- recompile ocaml 4.14.2 [upstream or system changes]
90
+
- recompile ocaml-compiler-libs v0.12.4 [uses ocaml]
91
+
...
92
+
]}
93
+
94
+
where it seems opam has decided that something has changed enough for it to want to recompile the `ocaml` package, and therefore
95
+
{i everything} in the entire opam switch! So this job took 12 minutes instead of 21 seconds, which was the time required to finally
96
+
build the `tezos-protocol` package.
97
+
98
+
{2 Day10 and docs}
99
+
In closely related news, {{:https://tunbury.org/}mtelver's} day10 project looked precisely the right shape for building docs -
100
+
in fact it shares its architecture and some components with the docs CI. So I asked Claude to take a look and see what it would
101
+
take, and discovered that it doesn't take very much! We have a Really Big Machine here at the CL that was temporarily underused;
102
+
and by Really Big I mean 768 cores and 3TB of RAM. So, how long could building all of the docs for all of the packages possibly
103
+
take? Well, it takes 5 hours 40 mins. And I was only using roughly a third of the machine. Nice!
104
+
105
+
So should I push on with fixing ocaml-docs-ci and figure out why it's rebuilding everything all the time? Or should I forge ahead
106
+
with day10 and turn it into a proper CI system as opposed to a slightly flakey bespoke thing I have to handhold through a build?
107
+
This is next week's problem.
108
+
109
+
{2 JS toplevels}
110
+
111
+
Something I keep coming back to is javascript toplevels. I'd really like to be able to be able to host JS toplevels on ocaml.org
112
+
for each different version of each different package. This is something I've worked on on-and-off for a long time now, and
113
+
several fixes to help have been merged to various projects along the way. The tricky thing is to not put a massive load onto
114
+
ocaml.org with this, so we need to be efficient. That means firstly having a single toplevel js file with all of the logic in
115
+
but none of the libraries, and then dynamically loading libraries as we need them. Also we can save some bandwidth by not
116
+
immediately sending all of the cmi files, as these can be faulted in as necessary too. So once again I've got Claude on the task,
117
+
and things are honestly looking pretty hopeful now. I've got 2 demos:
118
+
119
+
- {{:https://jon.ludl.am/experiments/findlibish/}Dynamic library loading}
120
+
- {{:https://jon.ludl.am/experiments/multi-universe-demo/}Multi-version support}
121
+
122
+
In both cases, make sure you take a look at the network tab to see it dynamically loading only what it needs.
123
+
124
+
{1 Retrospective}
125
+
126
+
{2 Autonomous Claude}
127
+
The power of sending Claude off to do some work can be immense. However, it does mean investing time up front telling it
128
+
precisely what problem you're trying to solve, what approach to take, finer details on how you want it done, and
129
+
how you can tell if it's working when it finishes. A 'failure mode' I've been experiencing is when I end up in a long, drawn out
130
+
real time interaction, especially if that's happening with 2 projects simultaneously - and by 'failure' I really mean just
131
+
'slow'. Ideally what would be going on is for all of my agents to be getting on with whatever task they've been allocated
132
+
without bothering me for more details. For Claude to have to ask me a question has much more latency involved than it just
133
+
getting on with things, especially if I don't notice it immediately.
134
+
135
+
{2 When to Stop}
136
+
The 'finishing criteria' are important - many times this week I've had Claude tell me it's finished something, having
137
+
verified that it's passing all the tests, only for me to take a look to find that it's very obviously broken. As quite a
138
+
few things recently have involved the web, I've
139
+
put Playwright into all of my devcontainers, and told Claude to use it to verify things are working. This has been working
140
+
pretty well, so I'll be adding it to my prompts. It's not too dissimilar to what we used to call 'pre-flight checks'
141
+
back in the Citrix days.
142
+
143
+
{2 Containers vs accounts}
144
+
I've been running everything with `--dangerously-ignore-permissions` in containers, and while the outcome is amazing,
145
+
the containers bit has been a bit of a headache. Next week I'll be trialling the idea of just giving the agents their
146
+
own account (non-admin!) on my servers, their own github account, tangled account and so on, and just treating them
147
+
more like I would if I had a real colleague. It's always slightly alarming to see my own name on the output of the
148
+
bots, assigning me (or sometimes someone else (!!)) copyright over code I've never seen. This is, of course, a whole
149
+
other pandora's box that I really don't want to open right now - but I think the point is that I'll feel a lot more
150
+
comfortable if the commits are all by `Jon's Agent <jon+claude@recoil.org>` rather than by me!
151
+
152
+
{2 Deciding next steps}
153
+
The question of whether I should fix up ocaml-docs-ci or improve the day10 solution requires a bit of thought. In
154
+
fact, it requires a bit of a gap analysis between the two. This isn't something I've asked Claude to do before,
155
+
so I'll try that and see how it turns out. I'll be asking it to be "scientific" in its approach, coming up with
156
+
hypotheses and verifying them - for which I think I'll need to give it a platform on which it can perform experiments.
157
+
This is a bit trickier with ocaml-docs-ci than day10 as day10 runs entirely on any given linux computer, whereas
158
+
ocaml-docs-ci needs ocurrent workers and a routable ssh server. I'll report on the outcome of this next week!
159
+
+104
site/blog/2026/02/weeknotes-2026-06.mld
+104
site/blog/2026/02/weeknotes-2026-06.mld
···
1
+
{0 Weeknotes for week 6}
2
+
3
+
@published 2026-02-09
4
+
@x-ocaml.requires odoc.xref2,odoc.loader,odoc.model
5
+
@packages odoc
6
+
7
+
Highlights:
8
+
- {{:https://jon.ludl.am/experiments/day10-jtw/standalone/index.html}day10 / javascript toplevels integration}
9
+
- {{:https://jon.ludl.am/experiments/scrollycoder/}Scrollycode experiments}
10
+
11
+
{1 Oxmono}
12
+
I spent some time on Anil's oxmono repo getting odoc to work correctly. It turned out that the bug
13
+
I was working on last week was critically important for this - and that the bugfix was incomplete.
14
+
One of the issues was to do with identifiers needing to be unique. For example, consider the following
15
+
code:
16
+
17
+
{@ocaml[
18
+
module type S = sig
19
+
type t
20
+
21
+
include sig
22
+
type t
23
+
24
+
val f : t -> t
25
+
end with type t := t
26
+
end
27
+
]}
28
+
29
+
The problem here is that both definitions of `type t` have the same identifier, which causes problems
30
+
when we move to and from the 'Component' types. The solution was to introduce a 'dummy' parent for the
31
+
type defined within the include. This works because we never actually render the body of the include
32
+
into HTML - we render the {i expansion}, which {i doesn't} have [type t] in it, as it has been substituted
33
+
out.
34
+
35
+
The fix I made last week fixed the {{!Odoc_loader}loader}, which reads in the [cmt]/[cmti] files produced
36
+
by the compiler. There's one more place where we create these in the code - when we translate from the
37
+
{{!Odoc_xref2.Component}Component} types back into {{!Odoc_model.Lang}Lang} types. I was a little curious
38
+
about whether it was possible to make this happen, so I thought I'd ask Claude to see if it could come up
39
+
with a scenario where we'd end up in this situation. This was a complete failure, which was a real
40
+
disappointment to me, as doing this sort of thing is a quite tedious and annoying part of working on odoc.
41
+
42
+
Meanwhile, I was running odoc on Anil's {{:https://github.com/avsm/oxmono}oxmono} repo, which was using
43
+
{{:https://github.com/art-w}art-w}'s {{:https://github.com/ocaml/odoc/pull/1399}PR to upstream oxcaml support}.
44
+
It was failing with an exception that was very familiar, so I pulled in the fix I'd been working on, and
45
+
that enabled it to get much further. However, it did subsequently fail with another slightly different
46
+
exception. I had my suspicions at this point that it might be due to the other place, but I thought this
47
+
again was a good opportunity to test Claude's debugging skills. However, this again was a complete failure.
48
+
I spend quite a long time prodding it - at least 4 separate sessions - and it really didn't get anywhere
49
+
close to a solution, despite knowing precisely that the commit we'd made that had fixed the first problem.
50
+
Two of the four times it ended up telling me that the oxcaml compiler was broken and suggesting that we
51
+
create an issue!
52
+
53
+
I'm only very mildly disappointed in this - it's all quite subtle, and something I still end up scratching
54
+
my head over sometimes, but it would have been wonderful to be able to offload this sort of work!
55
+
56
+
In any case, the docs now all build on {{:https://github.com/jonludlam/oxmono/commit/2a53f6857d5b8849a73f5bb3e5244b9ac0f36708}my fork of oxmono}.
57
+
58
+
{1 Docs CI}
59
+
The fix I deployed last week for ocaml-docs-ci was taking forever to complete, so I ended up spending some time
60
+
investigating this. The problem was happening during the 'prep' phase, which is the first part of the pipeline
61
+
where we simply build the package to be documented. This is supposed to work by building a graph of all
62
+
inter-package dependencies across all of the solved packages, so we maximise sharing of built artefacts. Each
63
+
'prep' job builds precisely one package by coping in the dependencies from previous prep jobs, then running
64
+
{{:https://github.com/jonludlam/opamh}opamh} to fix up the metadata so that opam believes it has installed
65
+
everything itself, then running opam to build the one package required. It was this last step that was going
66
+
wrong, where it would decide that there had been upstream changes to the compiler itself, and rebuild {i everything},
67
+
so rather than a prep job taking a few seconds, it would take a few minutes.
68
+
69
+
I was totally unable to repro this locally - everything build very quickly and just how it should have done.
70
+
After much head-scratching I finally realised that the problem was somewhere in the caching. I think what's
71
+
going on is that we dynamically build an opam repository to make the `opam install` command faster, and that repo
72
+
contains only the packages that are required to build whatever it is we're building. Those opam files are
73
+
cached by the docs CI server and passed to the build script as a base64-encoded gzipped tarball inline in the
74
+
obuilder file (!). This should all be totally consistent as we're also caching all the builds - except for the
75
+
compiler itself, which comes from the base docker image. This, of course, is the problem. The ocaml compiler
76
+
opam files had been updated, and then when we reconstructed the opam repo with our cached opam files, opam
77
+
noticed they had changed (gone {i backwards} in time!) and decided it needed to rebuild the compiler, and
78
+
therefore {i everything} else. Clearing out the opam-files cache and restarting the builds fixed this entirely,
79
+
and the full rebuild job completed after about 2 days. I flipped the switch on Saturday night and the docs
80
+
are now fully up to date again. Phew!
81
+
82
+
{1 day10 work}
83
+
This was a fun week of large-scale building! I integrated day10 and odoc_driver and js_top_worker and x-ocaml and have now successfully
84
+
got a docs-ci-like system that's able to build docs and toplevels that can coexist in the one HTML tree.
85
+
I've not got a full integrated demo yet, but you can see the test cases for this {{:https://jon.ludl.am/experiments/day10-jtw/standalone/index.html}here}.
86
+
Be sure to take a look at the 'network' tab in the browser dev tools to see what it's doing!
87
+
88
+
{1 Scrollycode experiments}
89
+
I've long been a fan of {{:https://pomb.us/}Rodrigo Pombo's} work on "building tools for better code reading comprehension",
90
+
ever since first seeing his post "{{:https://pomb.us/build-your-own-react/}Build your own React}".
91
+
Claude is {i fantastically good} at doing this sort of thing, so I asked it to go and build me some simple
92
+
OCaml-focused versions. We came up with 5 variations in the end - and they're all pretty neat!
93
+
{{:https://jon.ludl.am/experiments/scrollycoder/}take a look!}. The best part of this was that it took me less than
94
+
half-an-hour to get Claude to do all this.
95
+
96
+
{1 Dune PR}
97
+
I attended the bi-weekly dune dev meeting to talk about the first part of the dune PR - the bit that Paul Elliot did
98
+
almost a year ago.
99
+
100
+
{1 Coming week}
101
+
So the clock is ticking on writing the exam questions for FoCS, so I'll need to be spending time this week on
102
+
that.
103
+
104
+
+7
site/blog/2026/index.mld
+7
site/blog/2026/index.mld
+5
site/blog/index.mld
+5
site/blog/index.mld
+1
site/drafts/index.mld
+1
site/drafts/index.mld
···
1
+
{0 Drafts}
+437
site/notebooks/foundations/foundations1.mld
+437
site/notebooks/foundations/foundations1.mld
···
1
+
{0 Lecture 1: Introduction to Programming}
2
+
3
+
{1 Basic Concepts in Computer Science}
4
+
{ul {- Computers: a child can use them; {b nobody} can fully understand them!
5
+
}{- We can master complexity through levels of abstraction.
6
+
}{- Focus on 2 or 3 levels at most!
7
+
}}
8
+
9
+
{b Recurring issues:}
10
+
{ul {- {e what services} to provide at each level
11
+
}{- {e how to implement} them using lower-level services
12
+
}{- {e the interface} that defines how the two levels should communicate
13
+
}}
14
+
15
+
A basic concept in computer science is that large systems can only be
16
+
understood in levels, with each level further subdivided into functions or
17
+
services of some sort. The interface to the higher level should supply the
18
+
advertised services. Just as important, it should block access to the means by
19
+
which those services are implemented. This {e abstraction barrier} allows one
20
+
level to be changed without affecting levels above. For example, when a
21
+
manufacturer designs a faster version of a processor, it is essential that
22
+
existing programs continue to run on it. Any differences between the old and
23
+
new processors should be invisible to the program.
24
+
25
+
Modern processors have elaborate specifications, which still sometimes leave
26
+
out important details. In the old days, you then had to consult the circuit
27
+
diagrams.
28
+
29
+
{2 Example 1: Dates}
30
+
{ul {- Abstract level: dates over a certain interval
31
+
}{- Concrete level: typically 6 characters: [YYMMDD] (where each character is represented by 8 bits)
32
+
}{- Date crises caused by {b inadequate} internal formats:
33
+
{ul {- Digital’s PDP-10: using 12-bit dates (good for at most 11 years)
34
+
}{- 2000 crisis: 48 bits could be good for lifetime of universe!
35
+
}}
36
+
}}
37
+
38
+
Digital Equipment Corporation’s date crisis occurred in 1975. The
39
+
PDP-10 was a 36-bit mainframe computer. It represented dates using a 12-bit
40
+
format designed for the tiny PDP-8. With 12 bits, one can distinguish
41
+
{m 2^{12} = 4096} days or 11 years.
42
+
43
+
The most common industry format for dates uses six characters: two for the
44
+
year, two for the month and two for the day. The most common “solution” to the
45
+
year 2000 crisis is to add two further characters, thereby altering file sizes.
46
+
Others have noticed that the existing six characters consist of 48 bits,
47
+
already sufficient to represent all dates over the projected lifetime of the
48
+
{m 2^{48}}
49
+
universe: {m 2^{48} = 2.8\times 10^{14}\textrm{days} } = {m 7.7\times 10^{11} \textrm{years!} }
50
+
51
+
Mathematicians think in terms of unbounded ranges, but the representation we
52
+
choose for the computer usually imposes hard limits. A good programming
53
+
language like OCaml lets one easily change the representation used in the
54
+
program. But if files in the old representation exist all over the place,
55
+
there will still be conversion problems. The need for compatibility with older
56
+
systems causes problems across the computer industry.
57
+
58
+
{2 Example II: Floating Point Numbers}
59
+
60
+
Computers have integers like [1066] and floats like {m 1.066\times 10^3 }.
61
+
A floating-point number is represented by two integers.
62
+
The concept of {e data type} involves:
63
+
{ul {- how a value is represented inside the computer
64
+
}{- the suite of operations given to programmers
65
+
}{- valid and invalid (or exceptional) results, such as “infinity”
66
+
}}
67
+
68
+
Computer arithmetic can yield {e incorrect answers!}
69
+
70
+
In science, numbers written with finite precision and a decimal exponent are
71
+
said to be in {e standard form}. The computational equivalent is the {e floating
72
+
point number}. These are familiar to anybody who has used a scientific
73
+
calculator. Internally, a float consists of two integers.
74
+
75
+
Because of its finite precision, floating-point computations are potentially
76
+
inaccurate. To see an example, use your nearest electronic calculator to
77
+
compute {m (2^{1/10000})^{10000} }. I get {m 1.99999959}! With certain computations,
78
+
the errors spiral out of control. Many programming languages fail to check
79
+
whether even integer computations fall within the allowed range: you can add
80
+
two positive integers and get a negative one!
81
+
82
+
Most computers give us a choice of precisions. In 32-bit precision, integers
83
+
typically range from {m 2^{31}-1 } (namely 2 147 483 647) to {m -2^{31} }; floats
84
+
are accurate to about six decimal places and can get as large as {m 10^{35} } or so.
85
+
For floats, 64-bit precision is often preferred. Early languages like Fortran
86
+
required variables to be declared as [INTEGER], [REAL] or [COMPLEX] and barred
87
+
programmers from mixing numbers in a computation. Nowadays, programs handle
88
+
many different kinds of data, including text and symbols. The concept of a
89
+
{e data type} can ensure that different types of data are not combined in a
90
+
senseless way.
91
+
92
+
Inside the computer, all data are stored as bits. In most programming
93
+
languages, the compiler uses types to generate correct machine code, and types
94
+
are not stored during program execution. In this course, we focus almost
95
+
entirely on programming in a high-level language: OCaml.
96
+
97
+
{1 Goals of Programming}
98
+
{ul {- to describe a computation so that it can be done {b mechanically}:
99
+
{ul {- Expressions compute values.
100
+
}{- Commands cause effects.
101
+
}}
102
+
}{- to do so efficiently and {b correctly}, giving the right answers quickly
103
+
}{- to allow easy modification as needs change
104
+
{ul {- Through an orderly {b structure} based on abstraction principles
105
+
}{- Such as modules or classes
106
+
}}
107
+
}}
108
+
109
+
Programming {e in-the-small} concerns the writing of code to do simple, clearly
110
+
defined tasks. Programs provide expressions for describing mathematical
111
+
formulae and so forth. This was the original contribution of FORTRAN, the
112
+
FORmula TRANslator. Commands describe how control should flow from one part of
113
+
the program to the next.
114
+
115
+
As we code layer upon layer, we eventually find ourselves programming
116
+
{e in the large} : joining large modules to solve some messy task. Programming
117
+
languages have used various mechanisms to allow one part of the program to
118
+
provide interfaces to other parts. Modules encapsulate a body of code, allowing
119
+
outside access only through a programmer-defined interface. {e Abstract Data
120
+
Types} are a simpler version of this concept, which implement a single concept
121
+
such as dates or floating-point numbers.
122
+
123
+
{e Object-oriented programming} is the most complicated approach to modularity.
124
+
{e Classes} define concepts, and they can be built upon other classes. Operations
125
+
can be defined that work in appropriately specialised ways on a family of
126
+
related classes. {e Objects} are instances of classes and hold the data that is
127
+
being manipulated.
128
+
129
+
This course does not cover OCaml’s sophisticated module system, which can do
130
+
many of the same things as classes. You will learn all about objects when you
131
+
study Java. OCaml includes a powerful object system, although this is not used
132
+
as much as its module system.
133
+
134
+
{1 Why Program in OCaml?}
135
+
136
+
Why program in OCaml at all?
137
+
{ul {- It is interactive.
138
+
}{- It has a flexible notion of {e data type}.
139
+
}{- It hides the underlying hardware: {e no crashes}.
140
+
}{- Programs can easily be understood mathematically.
141
+
}{- It distinguishes naming something from {e updating memory}.
142
+
}{- It manages storage for us.
143
+
}}
144
+
145
+
Programming languages matter. They affect the reliability, security, and
146
+
efficiency of the code you write, as well as how easy it is to read, refactor,
147
+
and extend. The languages you know can also change how you think, influencing
148
+
the way you design software even when you’re not using them.
149
+
150
+
What makes OCaml special is that it occupies a sweet spot in the space of
151
+
programming language designs. It provides a combination of efficiency,
152
+
expressiveness and practicality that is difficult to find matched by any other language.
153
+
“ML” was originally the meta language of the LCF (Logic for Computable Functions)
154
+
proof assistant released by Robin Milner in 1972 (at Stanford, and later at Cambridge).
155
+
ML was turned into a compiler in order to make it easier to use LCF on different machines,
156
+
and it was gradually turned into a full-fledged system of its own by the 1980s.
157
+
158
+
The modern OCaml emerged in 1996, and the past twenty five years have seen OCaml
159
+
attract a significant user base with language improvements being steadily
160
+
added to support the growing commercial and academic codebases.
161
+
OCaml is therefore the outcome of years of research into programming languages,
162
+
and a good base to begin our journey into learning the foundations of computer
163
+
science.
164
+
165
+
Because of its connection to mathematics, OCaml programs can be designed and
166
+
understood without thinking in detail about how the computer will run them.
167
+
Although a program can abort, it cannot crash: it remains under the control of
168
+
the OCaml system. It still achieves respectable efficiency and provides
169
+
lower-level primitives for those who need them. Most other languages allow
170
+
direct access to the underlying machine and even try to execute illegal
171
+
operations, causing crashes.
172
+
173
+
The only way to learn programming is by writing and running programs. This web
174
+
notebook provides an interactive environment where you can modify the example
175
+
fragments and see the results for yourself. You should also consider
176
+
installing OCaml on your own computer so that you try more advanced programs
177
+
locally.
178
+
179
+
{1 A first session with OCaml}
180
+
181
+
{@ocaml run-on=load[
182
+
# let pi = 3.14159265358979;;
183
+
val pi : float = 3.14159265358979
184
+
]}
185
+
186
+
The first line of this simple session is a {e value declaration}. It makes the
187
+
name [pi] stand for the floating point number [3.14159]. (Such names are called
188
+
{e identifiers}.) OCaml echoes the name ([pi]) and type ([float]) of the
189
+
declared identifier.
190
+
191
+
{@ocaml run-on=load[
192
+
# pi *. 1.5 *. 1.5;;
193
+
- : float = 7.06858347057702829
194
+
]}
195
+
196
+
The second line computes the area of the circle with radius [1.5] using the
197
+
formula {m A = \pi r^2 }. We use [pi] as an abbreviation for [3.14159].
198
+
Multiplication is expressed using [*.], which is called an {e infix operator}
199
+
because it is written between its two operands.
200
+
201
+
OCaml replies with the computed value (about [7.07]) and its type (again [float]).
202
+
203
+
{@ocaml run-on=load[
204
+
# let area r = pi *. r *. r;;
205
+
val area : float -> float = <fun>
206
+
]}
207
+
208
+
To work abstractly, we should provide the service “compute the area of a
209
+
circle,” so that we no longer need to remember the formula. This sort of
210
+
encapsulated computation is called a {e function}. The third line declares the
211
+
function [area]. Given any floating point number [r], it returns another
212
+
floating point number computed using the [area] formula; note that the function
213
+
has type [float -> float].
214
+
215
+
{@ocaml run-on=load[
216
+
# area 2.0;;
217
+
- : float = 12.56637061435916
218
+
]}
219
+
220
+
The fourth line calls the function [area] supplying [2.0] as the argument. A
221
+
circle of radius [2] has an area of about [12.6]. Note that brackets around a
222
+
function argument are not necessary.
223
+
224
+
The function uses [pi] to stand for [3.14159]. Unlike what you may have seen in
225
+
other programming languages, [pi] cannot be "assigned to" or otherwise updated.
226
+
Its meaning within [area] will persist even if we issue a new [let] declaration
227
+
for [pi] afterwards.
228
+
229
+
{1 Raising a Number to a Power}
230
+
231
+
{@ocaml run-on=load[
232
+
# let rec npower x n =
233
+
if n = 0 then 1.0
234
+
else x *. npower x (n - 1);;
235
+
val npower : float -> int -> float = <fun>
236
+
]}
237
+
238
+
Our new [npower] definition can now take additional arguments, reflected in the arrows
239
+
present in the type of [npower]; these represent {e parameters} that can be passed to the
240
+
new value being defined, with the final segment being the resulting type. Thus our [npower]
241
+
type can be read as "pass in a float and integer to return a float".
242
+
243
+
{e Mathematical Justification} (for {m x\not=0 }):
244
+
{math
245
+
\begin{aligned}
246
+
x^0 & = 1 \
247
+
x^{n+1} & = x\times x^n.
248
+
\end{aligned}
249
+
}
250
+
251
+
The function [npower] raises its float argument [x] to the power [n], a
252
+
non-negative integer. The function is {b recursive}: it calls itself. You
253
+
can spot a recursive function due to the [rec] keyword in the definition:
254
+
this indicates that any invocation of the function name within the function body should call itself.
255
+
This concept should be familiar from mathematics, since exponentiation is defined by the
256
+
rules shown above. You may also have seen recursion in the product rule for
257
+
differentiation: {m (u\cdot v)' = u\cdot v' + u'\cdot v }. In finding the derivative of {m u\cdot v },
258
+
we recursively find the derivatives of {m u } and {m v }, combining them to obtain the desired result.
259
+
The recursion is meaningful because it terminates: we reduce the problem to two smaller
260
+
problems, and this cannot go on forever. The OCaml programmer uses recursion
261
+
heavily. For {m n\geq0 }, the equation {m x^{n+1} = x\times x^n } yields an obvious
262
+
computation:
263
+
264
+
{math x^3 = x\times x^2 = x\times x\times x^1 = x\times x\times x\times x^0 = x\times x\times x }
265
+
266
+
The equation clearly holds even for negative {m n }. However, the corresponding
267
+
computation runs forever:
268
+
269
+
{math x^{-1} = x\times x^{-2} = x\times x\times x^{-3}=\cdots }
270
+
271
+
Note that the function [npower] contains both an integer constant ([0]) and a
272
+
floating point constant ([1.0]). The decimal point makes all the difference.
273
+
OCaml will notice and ascribe different meaning to each type of constant.
274
+
275
+
{@ocaml run-on=load[
276
+
# let square x = x *. x;;
277
+
val square : float -> float = <fun>
278
+
]}
279
+
280
+
Now for a tiresome but necessary aside. In most languages, the types of
281
+
arguments and results must always be specified. OCaml is unusual that it normally
282
+
infers the types itself. However, sometimes it is useful to supply a hint to
283
+
help you debug and develop your program. OCaml will still infer the types even if you don’t specify them, but in some cases
284
+
it will use a more inefficient function than a specialised one. Some languages
285
+
have just one type of number, converting automatically between different
286
+
formats; this is slow and could lead to unexpected rounding errors. Type
287
+
constraints are allowed almost anywhere. We can put one on any occurrence of x
288
+
in the function.
289
+
290
+
{@ocaml run-on=load[
291
+
# let square (x : float) = x *. x;;
292
+
val square : float -> float = <fun>
293
+
]}
294
+
295
+
Or we can constrain the type of the function’s result:
296
+
297
+
{@ocaml run-on=load[
298
+
# let square x : float = x *. x;;
299
+
val square : float -> float = <fun>
300
+
]}
301
+
302
+
OCaml treats the equality and comparison test specially. Expressions like [if x = y then] …
303
+
are allowed provided [x] and [y] have the same type and equality testing is
304
+
possible for that type. (We discuss equality further in a later lecture.)
305
+
Note that [x <> y] is OCaml for {m x\not=y }.
306
+
307
+
A characteristic feature of the computer is its ability to test for conditions
308
+
and act accordingly. In the early days, a program might jump to a given
309
+
address depending on the sign of some number. Later, John McCarthy defined
310
+
the {e conditional expression} to satisfy [if true then x else y = x] and
311
+
[if false then x else y = y].
312
+
313
+
OCaml evaluates the expression [if] {m B } [then] {m E_1 } [else] {m E_2 } by first evaluating {m B }.
314
+
If the result is [true] then OCaml evaluates {m E_1 } and otherwise {m E_2 }. Only one
315
+
of the two expressions {m E_1 } and {m E_2 } is evaluated! If both were evaluated,
316
+
then recursive functions like [npower] above would run forever.
317
+
318
+
The [if]-expression is governed by an expression of type [bool], whose two
319
+
values are [true] and [false]. In modern programming languages, tests are not
320
+
built into “conditional branch” constructs but can just be part of normal expressions.
321
+
Tests, or {e Boolean expressions,} can be expressed using relational operators
322
+
such as [<] and [=]. They can be combined using the Boolean operators for
323
+
negation ([not]), conjunction (written as [&&]) and disjunction (written as [||]). New
324
+
properties can be declared as functions: here, to test whether an integer is
325
+
even, for example:
326
+
327
+
{@ocaml run-on=load[
328
+
# let even n = n mod 2 = 0;;
329
+
val even : int -> bool = <fun>
330
+
]}
331
+
332
+
{1 {e Efficiently} Raising a Number to a Power}
333
+
334
+
{@ocaml run-on=load[
335
+
# let rec power x n =
336
+
if n = 1 then x
337
+
else if even n then
338
+
power (x *. x) (n / 2)
339
+
else
340
+
x *. power (x *. x) (n / 2);;
341
+
val power : float -> int -> float = <fun>
342
+
]}
343
+
344
+
{e Mathematical Justification}
345
+
{math \begin{aligned}
346
+
x^1 & = x \
347
+
x^{2n} & = (x^2)^n \
348
+
x^{2n+1} & = x\times(x^2)^n.
349
+
\end{aligned} }
350
+
351
+
For large [n], computing powers using {m x^{n+1} = x\times x^n } is too slow to
352
+
be practical. The equations above are much faster. Example:
353
+
354
+
{math 2^{12} = 4^6 = 16^3 = 16\times 256^1 = 16\times 256 = 4096. }
355
+
356
+
Instead of [n] multiplications, we need at most {m 2\lg n } multiplications,
357
+
where {m \lg n } is the logarithm of {m n } to the base {m 2 }.
358
+
359
+
We use the function [even], declared previously, to test whether the
360
+
exponent is even. Integer division ([/]) truncates its result to an
361
+
integer: dividing {m 2n+1 } by 2 yields {m n }.
362
+
363
+
A recurrence is a useful computation rule only if it is bound to terminate.
364
+
If {m n>0 } then {m n } is smaller than both {m 2n } and {m 2n+1 }. After enough
365
+
recursive calls, the exponent will be reduced to {m 1 }. The equations also hold
366
+
if {m n\leq0 }, but the corresponding computation runs forever.
367
+
368
+
Our reasoning assumes arithmetic to be {e exact}. Fortunately, the calculation is
369
+
well-behaved using floating-point.
370
+
371
+
Computer numbers have a finite range, which if exceeded results in the
372
+
integer wrapping around. You will understand this behaviour more as you
373
+
learn about computer architecture and how modern systems represent
374
+
numbers in memory.
375
+
376
+
If integers and floats must be combined in a calculation, OCaml provides functions
377
+
to convert between them:
378
+
379
+
{@ocaml run-on=load[
380
+
# int_of_float 3.14159;;
381
+
- : int = 3
382
+
# float_of_int 3;;
383
+
- : float = 3.
384
+
]}
385
+
386
+
OCaml’s libraries are organised using “modules”, so we may use compound
387
+
identifiers such as {!Float.of_int} to refer to library functions. There
388
+
are many thousands of library functions available in the OCaml ecosystem,
389
+
including text-processing and operating systems functions in addition to the
390
+
usual numerical ones.
391
+
392
+
{1 Exercises}
393
+
394
+
{2 Exercise 1.1}
395
+
396
+
One solution to the year 2000 bug involves storing years as two digits, but interpreting them such
397
+
that 50 means 1950 and 49 means 2049. Comment on the merits and demerits of this approach.
398
+
399
+
{2 Exercise 1.2}
400
+
401
+
Using the date representation of the previous exercise, code OCaml functions to (a) compare two
402
+
years (b) add/subtract some given number of years from another year.
403
+
404
+
{2 Exercise 1.3}
405
+
406
+
Why would no experienced programmer write an expression of the form [if] … [then true else false]?
407
+
What about expressions of the form [if] … [then false else true]?
408
+
409
+
{2 Exercise 1.4}
410
+
411
+
Functions [npower] and [power] both return a [float]. The definition of [npower] returns the float
412
+
value [1.0] in its base case. The definition of [power] does not, so how does the OCaml type checker
413
+
know that [power] returns a [float]?
414
+
415
+
{2 Exercise 1.5}
416
+
417
+
Because computer arithmetic is based on binary numbers, simple decimals such as 0.1 often cannot be
418
+
represented exactly. Write a function [mul] that performs the computation
419
+
{math \underbrace{x+x+\cdots+x}_{n} }
420
+
where {m x } has type [float]. (It is essential to use repeated addition rather than multiplication!)
421
+
422
+
The value computed with [n = 10000] and [x = 0.1] may print as
423
+
[1000.0], which looks exact. If that happens, then evaluate the
424
+
expression [mul 0.1 10000 - 1000.0]
425
+
426
+
An error of this type has been blamed for the failure of an American Patriot Missile battery to
427
+
intercept an incoming Iraqi missile during the {{: https://en.wikipedia.org/wiki/MIM-104_Patriot#Failure_at_Dhahran} first Gulf War}.
428
+
The missile hit an American Army barracks, killing 28.
429
+
430
+
{2 Exercise 1.6}
431
+
432
+
Another example of the inaccuracy of floating-point arithmetic takes the golden ratio
433
+
{m \phi\approx1.618\ldots } as its starting point:
434
+
{math \gamma_0 = \frac{1+\sqrt5}{2} \quad\text{and}\quad\gamma_{n+1} = \frac{1}{\gamma_n-1}. }
435
+
In theory, it is easy to prove that {m \gamma_n=\cdots = \gamma_1 = \gamma_0 } for all {m n>0 }. Code this
436
+
computation in OCaml and report the value of {m \gamma_{50} }. {e Hint:} in OCaml, {m \sqrt5 } is expressed
437
+
as [sqrt 5.0].
+321
site/notebooks/foundations/foundations10.mld
+321
site/notebooks/foundations/foundations10.mld
···
1
+
{0 Lecture 10: Queues and Search Strategies}
2
+
3
+
{@ocaml hidden[
4
+
type 'a tree = Lf | Br of 'a * 'a tree * 'a tree;;
5
+
]}
6
+
7
+
8
+
9
+
{1 Breadth-First v Depth-First Tree Traversal}
10
+
{ul {- binary trees as {e decision trees}
11
+
}{- look for {e solution nodes}
12
+
{ul {- Depth-first: search one subtree in full before moving on
13
+
}{- Breadth-first: search all nodes at level {m k} before moving to {m k+1}
14
+
}}
15
+
}{- finds {e all} solutions --- nearest first!
16
+
}}
17
+
18
+
Preorder, inorder and postorder tree traversals all have something in common:
19
+
they are depth-first. At each node, the left subtree is entirely
20
+
traversed before the right subtree. Depth-first traversals are easy to code
21
+
and can be efficient, but they are ill-suited for some problems.
22
+
23
+
Suppose the tree represents the possible moves in a puzzle, and the purpose
24
+
of the traversal is to search for a node containing a solution. Then a
25
+
depth-first traversal may find one solution node deep in the left subtree,
26
+
when another solution is at the very top of the right subtree. Often we
27
+
want the shortest path to a solution.
28
+
29
+
Suppose the tree is {e infinite} or simply extremely large. Depth-first search
30
+
is almost useless with such trees, for if the left subtree is infinite then the
31
+
search will never reach the right subtree. OCaml can represent infinite trees by
32
+
the means discussed in the lecture on laziness. Another tree representation (suitable
33
+
for solving solitaire, for example) is by a function [next : pos -> pos list],
34
+
which maps a board position to a list of the positions possible after
35
+
the next move. For simplicity, the examples below use the OCaml datatype
36
+
[tree], which has only finite trees.
37
+
38
+
A {e breadth-first} traversal explores the nodes horizontally rather than
39
+
vertically. When visiting a node, it does not traverse the subtrees until
40
+
it has visited all other nodes at the current depth. This is easily
41
+
implemented by keeping a list of trees to visit. Initially, this list
42
+
consists of one element: the entire tree. Each iteration removes a tree
43
+
from the head of the list and adds its subtrees after the end of the
44
+
list.
45
+
46
+
{1 Breadth-First Tree Traversal --- Using Append}
47
+
48
+
{@ocaml[
49
+
# let rec nbreadth = function
50
+
| [] -> []
51
+
| Lf :: ts -> nbreadth ts
52
+
| Br (v, t, u) :: ts ->
53
+
v :: nbreadth (ts @ [t; u]);;
54
+
val nbreadth : 'a tree list -> 'a list = <fun>
55
+
]}
56
+
57
+
Keeps an {e enormous queue} of nodes of search, and is a wasteful use of [append].
58
+
59
+
Breadth-first search can be inefficient, this naive implementation especially
60
+
so. When the search is at depth {m d} of the tree, the list contains all the
61
+
remaining trees at depth {m d}, followed by the subtrees (all at depth {m d+1}) of
62
+
the trees that have already been visited. At depth 10, the list could already
63
+
contain 1024 elements. It requires a lot of space, and aggravates this with a
64
+
gross misuse of append. Evaluating [ts@[t, u]] copies the long list
65
+
[ts] just to insert two elements.
66
+
67
+
{1 An Abstract Data Type: Queues}
68
+
{ul {- [qempty] is the {e empty queue}
69
+
}{- [qnull] {e tests} whether a queue is empty
70
+
}{- [qhd] {e returns} the element at the {e head} of a queue
71
+
}{- [deq] {e discards} the element at the {e head} of a queue
72
+
}{- [enq] {e adds} an element at the {e end} of a queue
73
+
}}
74
+
75
+
Breadth-first search becomes much faster if we replace the lists by
76
+
{e queues}. A queue represents a sequence, allowing elements to be taken
77
+
from the head and added to the tail. This is a First-In-First-Out (FIFO)
78
+
discipline: the item next to be removed is the one that has been in the queue
79
+
for the longest time. Lists can implement queues, but append is a poor means
80
+
of adding elements to the tail.
81
+
82
+
Our functional arrays are suitable, provided we
83
+
augment them with a function to delete the first array element. (See {e ML
84
+
for the Working Programmer} page 156.) Each operation would take {m O(\log n)}
85
+
time for a queue of length {m n}.
86
+
87
+
We shall describe a representation of queues that is purely functional, based
88
+
upon lists, and efficient. Operations take {m O(1)} time when “amortized”:
89
+
averaged over the lifetime of a queue.
90
+
91
+
A conventional programming technique is to represent a queue by an array. Two
92
+
indices point to the front and back of the queue, which may wrap around the
93
+
end of the array. The coding is somewhat tricky. Worse, the length of the
94
+
queue must be given a fixed upper bound.
95
+
96
+
{1 Efficient Functional Queues: Idea}
97
+
{ul {- Represent the queue {m x_1; x_2; \ldots; x_m; y_n; \ldots; y_1} by any {e pair of lists}
98
+
{math ([x_1,x_2,\ldots,x_m], [y_1,y_2,\ldots,y_n])}
99
+
}{- Add new items to the {e rear list}
100
+
}{- Remove items from {e front list} and if empty move {e rear} to {e front}
101
+
}{- {e Amortized} time per operation is {m O(1)}
102
+
}}
103
+
104
+
Queues require efficient access at both ends: at the front, for removal, and
105
+
at the back, for insertion. Ideally, access should take constant time,
106
+
{m O(1)}. It may appear that lists cannot provide such access. If
107
+
[enq(q, x)] performs [q@[x]], then this operation will be {m O(n)}. We
108
+
could represent queues by reversed lists, implementing [enq(q, x)] by
109
+
[x::q], but then the [deq] and [qhd] operations would be
110
+
{m O(n)}. Linear time is intolerable: a series of {m n} queue operations
111
+
could then require {m O(n^2)} time.
112
+
113
+
The solution is to represent a queue by a pair of lists, where
114
+
{math ([x_1,x_2,\ldots,x_m], [y_1,y_2,\ldots,y_n]) }
115
+
represents the queue {m x_1 x_2 \ldots x_m y_n \ldots y_1}.
116
+
117
+
The front part of the queue is stored in order, and the rear part is stored in
118
+
reverse order. The [enq] operation adds elements to the rear part
119
+
using cons, since this list is reversed; thus, [enq] takes constant
120
+
time. The [deq] and [qhd] operations look at the front part,
121
+
which normally takes constant time, since this list is stored in order. But
122
+
sometimes [deq] removes the last element from the front part; when this
123
+
happens, it reverses the rear part, which becomes the new front part.
124
+
125
+
{e Amortized} time refers to the cost per operation averaged over the
126
+
lifetime of any complete execution. Even for the worst possible execution,
127
+
the average cost per operation turns out to be constant; see the analysis
128
+
below.
129
+
130
+
{1 Efficient Functional Queues: Code}
131
+
132
+
{@ocaml[
133
+
# type 'a queue =
134
+
| Q of 'a list * 'a list;;
135
+
type 'a queue = Q of 'a list * 'a list
136
+
# let norm = function
137
+
| Q ([], tls) -> Q (List.rev tls, [])
138
+
| q -> q;;
139
+
val norm : 'a queue -> 'a queue = <fun>
140
+
# let qnull = function
141
+
| Q ([], []) -> true
142
+
| _ -> false;;
143
+
val qnull : 'a queue -> bool = <fun>
144
+
# let enq (Q (hds, tls)) x = norm (Q (hds, x::tls));;
145
+
val enq : 'a queue -> 'a -> 'a queue = <fun>
146
+
# exception Empty;;
147
+
exception Empty
148
+
# let deq = function
149
+
| Q (x::hds, tls) -> norm (Q (hds, tls))
150
+
| _ -> raise Empty;;
151
+
val deq : 'a queue -> 'a queue = <fun>
152
+
# let qempty = Q ([], []);;
153
+
val qempty : 'a queue = Q ([], [])
154
+
# let qhd = function
155
+
| Q (x::_, _) -> x
156
+
| _ -> raise Empty;;
157
+
val qhd : 'a queue -> 'a = <fun>
158
+
]}
159
+
160
+
The datatype of queues prevents confusion with other pairs of lists. The empty
161
+
queue has both parts empty.
162
+
163
+
The function [norm] puts a queue into normal form, ensuring that the front part
164
+
is never empty unless the entire queue is empty. Functions [deq] and [enq]
165
+
call [norm] to normalise their result.
166
+
167
+
Because queues are in normal form, their head is certain to be in their
168
+
front part, so [qhd] looks there.
169
+
170
+
Let us analyse the cost of an execution comprising (in any possible order) {m n}
171
+
[enq] operations and {m n} [deq] operations, starting with an
172
+
empty queue. Each [enq] operation will perform one cons, adding an
173
+
element to the rear part. Since the final queue must be empty, each element
174
+
of the rear part gets transferred to the front part. The corresponding
175
+
reversals perform one cons per element. Thus, the total cost of the series of
176
+
queue operations is {m 2n} cons operations, an average of 2 per operation. The
177
+
amortized time is {m O(1)}.
178
+
179
+
There is a catch. The conses need not be distributed evenly; reversing a long
180
+
list could take up to {m n-1} of them. Unpredictable delays make the approach
181
+
unsuitable for {e real-time programming} where deadlines must be met.
182
+
183
+
{1 Breadth-First Tree Traversal --- Using Queues}
184
+
185
+
{@ocaml[
186
+
# let rec breadth q =
187
+
if qnull q then []
188
+
else
189
+
match qhd q with
190
+
| Lf -> breadth (deq q)
191
+
| Br (v, t, u) -> v :: breadth (enq (enq (deq q) t) u);;
192
+
val breadth : 'a tree queue -> 'a list = <fun>
193
+
]}
194
+
195
+
This function implements the same algorithm as [nbreadth] but uses a different
196
+
data structure. It represents queues using type [queue] instead of type
197
+
[list].
198
+
199
+
To compare their efficiency, I applied both functions to the full binary tree
200
+
of depth 12, which contains 4095 labels. The function [nbreadth] took 30
201
+
seconds while [breadth] took only 0.15 seconds: faster by a factor of 200.
202
+
203
+
For larger trees, the speedup would be greater. Choosing the right data
204
+
structure pays handsomely.
205
+
206
+
{1 Iterative deepening: Another Exhaustive Search}
207
+
{ul {- Breadth-first search examines {m O(b^d)} nodes:
208
+
{math 1 + b + \cdots + b^d = {b^{d+1}-1 \over b-1}
209
+
\qquad \begin{array}[c]{r@{}l}
210
+
b & {} = \hbox{branching factor}\\
211
+
d & {} = \hbox{depth}
212
+
\end{array}
213
+
}
214
+
}{- Recompute nodes at depth {m d} instead of storing them
215
+
}{- Time factor is {m b/(b-1)} if {m b>1}; complexity is still {m O(b^d)}
216
+
}{- Space required at depth {m d} drops from {m b^d} to {m d}
217
+
}}
218
+
219
+
Breadth-first search is not practical for infinite trees: it uses too much
220
+
space. Large parts of the tree have to be stored.
221
+
Consider the slightly more general problem of searching trees whose
222
+
branching factor is {m b} (for binary trees, {m b=2}). Then breadth-first search
223
+
to depth {m d} examines {m (b^{d+1}-1)/(b-1)} nodes, which is {m O(b^d)}, ignoring
224
+
the constant factor of {m b/(b-1)}. Since all nodes that are examined are also
225
+
stored, the space and time requirements are both {m O(b^d)}.
226
+
227
+
{e Depth-first iterative deepening} combines the space efficiency of
228
+
depth-first with the “nearest-first” property of breadth-first search. It
229
+
performs repeated depth-first searches with increasing depth bounds, each time
230
+
discarding the result of the previous search. Thus it searches to depth 1,
231
+
then to depth 2, and so on until it finds a solution. We can afford to
232
+
discard previous results because the number of nodes is growing exponentially.
233
+
There are {m b^{d+1}} nodes at level {m d+1}; if {m b\geq2}, this number actually
234
+
exceeds the total number of nodes of all previous levels put together, namely
235
+
{m (b^{d+1}-1) / (b-1)}.
236
+
237
+
{{: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.91.288} Korf shows} that the time needed for iterative deepening to reach
238
+
depth {m d} is only {m b/(b-1)} times that for breadth-first search, if {m b>1}.
239
+
This is a constant factor; both algorithms have the same time complexity,
240
+
{m O(b^d)}. In typical applications where {m b\geq2} the extra factor of
241
+
{m b/(b-1)} is quite tolerable. The reduction in the space requirement is
242
+
exponential, from {m O(b^d)} for breadth-first to {m O(d)} for iterative
243
+
deepening. Of course, this assumes that the tree itself is not stored in memory.
244
+
245
+
{1 Another Abstract Data Type: Stacks}
246
+
{ul {- [empty] is the {e empty stack}
247
+
}{- [null] tests whether a stack is empty
248
+
}{- [top] returns the element at the {e top} of a stack
249
+
}{- [pop] discards the element at the {e top} of a stack
250
+
}{- [push] adds an element at the {e top} of a stack
251
+
}}
252
+
253
+
A {e stack} is a sequence such that items can be added or removed from the head
254
+
only. A stack obeys a Last-In-First-Out (LIFO) discipline: the item next to be
255
+
removed is the one that has been in the queue for the {e shortest} time. Lists
256
+
can easily implement stacks because both [cons] and [hd] affect the head. But
257
+
unlike lists, stacks are often regarded as an imperative data structure: the
258
+
effect of [push] or [pop] is to change an existing stack, not return a new one.
259
+
260
+
In conventional programming languages, a stack is often implemented by storing
261
+
the elements in an array, using a variable (the “stack pointer”) to count them.
262
+
Most language processors keep track of recursive function calls using an
263
+
internal stack.
264
+
265
+
{1 A Survey of Search Methods}
266
+
{ul {- Depth-first: use a {e stack} (efficient but incomplete)
267
+
}{- Breadth-first: use a {e queue} (uses too much space!)
268
+
}{- Iterative deepening: use depth-first to get benefits of breadth-first (trades time for space)
269
+
}{- Best-first: use a {e priority queue} (heuristic search)
270
+
}}
271
+
272
+
The data structure determines the search!
273
+
274
+
Search procedures can be classified by the data structure used to store
275
+
pending subtrees. Depth-first search stores them on a stack, which is
276
+
implicit in functions like [inorder], but can be made explicit.
277
+
Breadth-first search stores such nodes in a queue.
278
+
279
+
An important variation is to store the nodes in a priority queue, which
280
+
is an ordered sequence. The priority queue applies some sort of ranking
281
+
function to the nodes, placing higher-ranked nodes before lower-ranked ones.
282
+
The ranking function typically estimates the distance from the node to a
283
+
solution. If the estimate is good, the solution is located swiftly. This
284
+
method is called best-first search.
285
+
286
+
The priority queue can be kept as a sorted list, although this is slow.
287
+
Binary search trees would be much better on average, and fancier data
288
+
structures improve matters further.
289
+
290
+
{2 Exercise 10.1}
291
+
292
+
Suppose that we have an implementation of queues, based on binary trees, such that each operation
293
+
takes logarithmic time in the worst case. Outline the advantages and drawbacks of such an
294
+
implementation compared with one presented above.
295
+
296
+
{2 Exercise 10.2}
297
+
298
+
The traditional way to implement queues uses a fixed-length array. Two indices into the array
299
+
indicate the start and end of the queue, which wraps around from the end of the array to the start.
300
+
How appropriate is such a data structure for implementing breadth-first search?
301
+
302
+
{2 Exercise 10.3}
303
+
304
+
Write a version of the function [breadth] using a nested [let] construction rather than
305
+
[match].
306
+
307
+
{2 Exercise 10.4}
308
+
309
+
Iterative deepening is inappropriate if {m b\approx1}, where {m b} is the branching factor. What search
310
+
strategy is appropriate in this case?
311
+
312
+
{2 Exercise 10.5}
313
+
314
+
Consider the following OCaml function.
315
+
316
+
{@ocaml[
317
+
let next n = [2 * n; 2 * n + 1]
318
+
]}
319
+
320
+
If we regard it as representing a tree, where the subtrees are computed from the current label, what
321
+
tree does [next 1] represent?
+445
site/notebooks/foundations/foundations11.mld
+445
site/notebooks/foundations/foundations11.mld
···
1
+
{0 Lecture 11: Elements of Procedural Programming}
2
+
3
+
{1 Procedural Programming}
4
+
{ul {- Procedural programs can change the machine state.
5
+
}{- They can interact with its environment
6
+
}{- They use control structures like branching, iteration and procedures.
7
+
}}
8
+
9
+
They use data abstractions of the computer’s memory:
10
+
{ul {- {e references} to memory cells
11
+
}{- {e arrays} that are blocks of memory cells
12
+
}{- {e linked structures} such as {e linked lists}
13
+
}}
14
+
15
+
Procedural programming is programming in the traditional sense of the word. A
16
+
program {e state} is repeatedly transformed by the execution of
17
+
{e commands} or {e statements}. A state change might be local to the
18
+
machine and consist of updating a variable or array. A state change might
19
+
consist of sending data to the outside world. Even reading data counts as a
20
+
state change, since this act normally removes the data from the environment.
21
+
22
+
Procedural programming languages provide primitive commands and control
23
+
structures for combining them. The primitive commands include
24
+
{e assignment} for updating variables, and various {e input/output}
25
+
commands for communication. Control structures include [if] and
26
+
[match] constructs for conditional execution, and repetitive constructs
27
+
such as [while]. Programmers can package up their own commands as
28
+
{e procedures} taking arguments. The need for such “subroutines” was
29
+
evident from the earliest days; they represent one of the first
30
+
examples of abstraction in programming languages.
31
+
32
+
OCaml makes no distinction between commands and expressions. OCaml provides built-in
33
+
‘functions’ to perform assignment and communication, and these can be used
34
+
in the traditional (procedural) style. OCaml programmers often follow a
35
+
functional style for most internal computations and use imperative features
36
+
mainly for communication with the outside world.
37
+
38
+
{1 OCaml Primitives for References}
39
+
40
+
{table
41
+
{tr {th Syntax}{th Effect}}
42
+
{tr {td [ref E]}{td {e create} a reference with {e initial contents} = value of [E]}}
43
+
{tr {td [!P]}{td return {e current contents} of reference [P]}}
44
+
{tr {td [P := E]}{td {e update} contents of [P] to value of [E]}}
45
+
}
46
+
47
+
The above text presents the OCaml primitives, but most languages have analogues of
48
+
them, often heavily disguised. We need a means of creating references (or
49
+
allocating storage), getting at the current contents of a reference cell, and
50
+
updating that cell.
51
+
52
+
The function [ref] creates references (also called
53
+
“locations”). Calling [ref] allocates a new location in memory.
54
+
Initially, this location holds the value given by
55
+
expression {m E}.
56
+
57
+
The function [!], when applied to a reference, returns its contents.
58
+
This operation is called {e dereferencing}. Clearly [!] is not a
59
+
mathematical function; its result depends upon the store.
60
+
61
+
The assignment [P:=E] evaluates expression {m P}, which must return a
62
+
reference {m p}, and {m E}. It stores at address {m p} the value of {m E}.
63
+
Syntactically, [:=] is a function and [P:=E] is an
64
+
expression, even though it updates the store. Like many functions that change
65
+
the state, it returns the value [()] of type [unit].
66
+
67
+
If {m \tau} is some OCaml type, then {m \tau} [ref] is the type of references
68
+
to cells that can hold values of {m \tau}. Please do not confuse the type
69
+
[ref] with the function [ref]. This table of the primitive
70
+
functions and their types might be useful:
71
+
72
+
{table
73
+
{tr {th Syntax}{th OCaml Type}}
74
+
{tr {td [ref]}{td ['a -> 'a ref]}}
75
+
{tr {td [!]}{td ['a ref -> 'a ]}}
76
+
{tr {td [:=]}{td ['a ref -> 'a -> unit]}}
77
+
}
78
+
79
+
{1 Trying Out References}
80
+
81
+
{@ocaml[
82
+
# let p = ref 5 (* create a reference *);;
83
+
val p : int ref = {contents = 5}
84
+
# p := !p + 1 (* p now holds value 6 *);;
85
+
- : unit = ()
86
+
# let ps = [ ref 77; p ];;
87
+
val ps : int ref list = [{contents = 77}; {contents = 6}]
88
+
# List.hd ps := 3;;
89
+
- : unit = ()
90
+
# ps;;
91
+
- : int ref list = [{contents = 3}; {contents = 6}]
92
+
]}
93
+
94
+
The first line declares [p] to hold a reference to an integer,
95
+
initially 5. Its type is [int ref], not just [int], so it
96
+
admits assignment. Assignment never changes [let] bindings: they are
97
+
{e immutable}. The identifier [p] will always denote the reference
98
+
mentioned in its declaration unless superseded by a new usage of [p].
99
+
Only the {e contents} of the reference is mutable.
100
+
101
+
OCaml displays a reference value as [{contents=v}], where value {m v} is the
102
+
contents. This notation is readable but gives us no way of telling whether
103
+
two references holding the same value are actually the same reference. To
104
+
display a reference as a machine address has obvious drawbacks!
105
+
106
+
In the first assignment, the expression [!p] yields the reference’s
107
+
current contents, namely 5. The assignment changes the contents of [p]
108
+
to 6. Most languages do not have an explicit dereferencing operator
109
+
(like [!]) because of its inconvenience. Instead, by convention,
110
+
occurrences of the reference on the {e left-hand} side of the [:=]
111
+
denote locations and those on the {e right-hand} side denote the contents.
112
+
A special ‘address of’ operator may be available to override the convention
113
+
and make a reference on the right-hand side to denote a location. Logically
114
+
this is a mess, but it makes programs shorter.
115
+
116
+
The list [ps] is declared to hold a new reference (initially
117
+
containing 77) as well as [p]. Then the new reference is
118
+
updated to hold 3. The assignment to [hd ps] does {e not}
119
+
update [ps], only the contents of a reference in that list.
120
+
121
+
{1 Commands: Expressions with Effects}
122
+
{ul {- Basic commands update references, write to files, etc.
123
+
}{- {m C_1 ; \ldots ; C_n} causes a series of expressions to be evaluated and returns the value of {m C_n}.
124
+
}{- A typical command returns the empty tuple: [()]
125
+
}{- [if] {m B} [then] {m C_1} [else] {m C_2} behaves like the traditional control structure if {m C_1} and {m C_2} have effects.
126
+
}{- Other OCaml constructs behave naturally with commands, including [match] expressions and recursive functions.
127
+
}}
128
+
129
+
We use the term {e command} informally to refer to an expression that has an
130
+
effect on the state. All expressions denote some value, but they can return
131
+
[()], which conveys no actual information.
132
+
133
+
We need a way to execute one command after another.
134
+
The construct {m C_1 ; \ldots ; C_n} evaluates the expressions {m C_1}
135
+
to {m C_n} in the order given and returns the value of {m C_n}. The values
136
+
of the other expressions are discarded; their only purpose is to change the
137
+
state.
138
+
139
+
Commands may be used with [if] and [match] much as in conventional languages.
140
+
OCaml functions play the role of procedures.
141
+
142
+
Other languages that combine the functional and imperative programming
143
+
paradigms include Lisp (and its dialect Scheme), Scala, and even a
144
+
systems programming language, BLISS (now long extinct).
145
+
146
+
{1 Iteration: the [while] command}
147
+
148
+
{@ocaml[
149
+
# let tlopt = function
150
+
| [] -> None
151
+
| _::xs -> Some xs;;
152
+
val tlopt : 'a list -> 'a list option = <fun>
153
+
# let length xs =
154
+
let lp = ref xs in (* list of uncounted elements *)
155
+
let np = ref 0 in (* accumulated count *)
156
+
let fin = ref false in
157
+
while not !fin do
158
+
match tlopt !lp with
159
+
| None -> fin := true
160
+
| Some xs ->
161
+
lp := xs;
162
+
np := 1 + !np
163
+
done;
164
+
!np (* the final count is returned *);;
165
+
val length : 'a list -> int = <fun>
166
+
]}
167
+
168
+
Once we can change the state, we need to do so repeatedly. Recursion can
169
+
serve this purpose, but having to declare a procedure for every loop is
170
+
clumsy, and compilers for conventional languages seldom exploit
171
+
tail-recursion.
172
+
173
+
Early programming languages provided little support for repetition. The
174
+
programmer had to set up loops using goto commands, exiting the loop using
175
+
another goto controlled by an [if]. Modern languages provide a
176
+
confusing jumble of looping constructs, the most fundamental of which is
177
+
[while B do C]. The boolean expression {m B} is evaluated,
178
+
and if true, command {m C} is executed and the command repeats. If {m B}
179
+
evaluates to false then the [while] command terminates, perhaps without
180
+
executing {m C} even once.
181
+
182
+
OCaml’s main looping construct is [while], which returns the value [()]. The
183
+
function [length] declares references to hold the list under
184
+
examination ([lp]) and number of elements counted so far ([np]) as well
185
+
as whether the end of the list has been reached (the boolean reference [fin]).
186
+
While the list is non-empty, we skip over one more element (by setting it to
187
+
its tail) and count that element.
188
+
189
+
The body of the [while] loop first checks to see if the end of the list has
190
+
been reached, in which case it sets the [fin] variable to true. If there is a
191
+
tail value, then two assignments are executed in sequence. The [lp] reference
192
+
is set to the tail of the list, and the [np] reference integer is incremented
193
+
by one. When the while loop terminates due to the [fin] variable being set to
194
+
true, the expression [!np] returns the computed length as the function’s
195
+
result.
196
+
197
+
{1 Private, Persistent References}
198
+
199
+
{@ocaml[
200
+
# exception TooMuch of int;;
201
+
exception TooMuch of int
202
+
# let makeAccount initBalance =
203
+
let balance = ref initBalance in
204
+
let withdraw amt =
205
+
if amt > !balance then
206
+
raise (TooMuch (amt - !balance))
207
+
else begin
208
+
balance := !balance - amt;
209
+
!balance
210
+
end
211
+
in
212
+
withdraw;;
213
+
val makeAccount : int -> int -> int = <fun>
214
+
]}
215
+
216
+
As you may have noticed, OCaml’s programming style looks clumsy compared with
217
+
that of languages like C. OCaml omits the defaults and abbreviations they
218
+
provide to shorten programs. However, OCaml’s explicitness makes it ideal for
219
+
teaching the fine points of references and arrays. OCaml’s references are more
220
+
flexible than those found in other languages.
221
+
222
+
The function [makeAccount] models a bank. Calling the function with a
223
+
specified initial balance creates a new reference [balance]) to
224
+
maintain the account balance and returns a function ([withdraw]) having
225
+
sole access to that reference. Calling [withdraw] reduces the balance
226
+
by the specified amount and returns the new balance. You can pay money in by
227
+
withdrawing a negative amount. The [if]-construct prevents the account
228
+
from going overdrawn, raising an exception.
229
+
230
+
Look at the {m \tt (E_1; E_2)} construct in the {e else} part above.
231
+
The first expression updates the account balance and returns the trivial
232
+
value (). The second expression, [!balance], returns the current
233
+
balance but does not return the reference itself: that would allow
234
+
unauthorised updates.
235
+
236
+
This example is based on one by Dr A C Norman.
237
+
238
+
{1 Two Bank Accounts}
239
+
240
+
{@ocaml[
241
+
# let student = makeAccount 500;;
242
+
val student : int -> int = <fun>
243
+
# let director = makeAccount 4000000;;
244
+
val director : int -> int = <fun>
245
+
# student 5 (* coach fare *);;
246
+
- : int = 495
247
+
# director 150000 (* Tesla *);;
248
+
- : int = 3850000
249
+
# student 500 (* oh oh *);;
250
+
Exception: TooMuch 5.
251
+
Called from Topeval.load_lambda in file "toplevel/byte/topeval.ml", line 89, characters 4-14
252
+
]}
253
+
254
+
Each call to [makeAccount] returns a copy of [withdraw] holding
255
+
a {e fresh} instance of the reference [balance]. As with a real bank
256
+
pass-book, there is no access to the account balance except via the
257
+
corresponding [withdraw] function. If that function is discarded, the
258
+
reference cell becomes unreachable; the computer will eventually reclaim it,
259
+
just as banks close down dormant accounts.
260
+
261
+
Here we see two people managing their accounts. For better or worse, neither
262
+
can take money from the other.
263
+
264
+
We could generalise [makeAccount] to return several functions that
265
+
jointly manage information held in shared references. The functions might be
266
+
packaged using OCaml records, which are not discussed in this course.
267
+
Most procedural languages do not properly support the concept of private
268
+
references, although {e object-oriented} languages take them as a basic theme.
269
+
270
+
{1 OCaml Primitives for Arrays}
271
+
272
+
{@ocaml[
273
+
# [|"a"; "b"; "c"|] (* allocate a fresh string array *);;
274
+
- : string array = [|"a"; "b"; "c"|]
275
+
# Array.make 3 'a' (* array[3] with cell containing 'a' *);;
276
+
- : char array = [|'a'; 'a'; 'a'|]
277
+
# let aa = Array.init 5 (fun i -> i * 10) (* array[5] initialised to (fun i) *);;
278
+
val aa : int array = [|0; 10; 20; 30; 40|]
279
+
# Array.get aa 3 (* retrieve the 4th cell in the array *);;
280
+
- : int = 30
281
+
# Array.set aa 3 42 (* set the 4th cell's value to 42 *);;
282
+
- : unit = ()
283
+
]}
284
+
285
+
There are many other array operations in the [Array] module in the OCaml standard
286
+
library.
287
+
288
+
{@ocaml[
289
+
# Array.make;;
290
+
- : int -> 'a -> 'a array = <fun>
291
+
# Array.init;;
292
+
- : int -> (int -> 'a) -> 'a array = <fun>
293
+
# Array.get;;
294
+
- : 'a array -> int -> 'a = <fun>
295
+
# Array.set;;
296
+
- : 'a array -> int -> 'a -> unit = <fun>
297
+
]}
298
+
299
+
OCaml arrays are like references that hold several elements instead of one. The
300
+
elements of an {m n}-element array are designated by the integers from 0
301
+
to {m n-1}. The {m i}th array element is usually written {m A.(i)}.
302
+
If {m \tau} is a type then {m \tau} [array] is the
303
+
type of arrays (of any size) with elements from {m \tau}.
304
+
305
+
Calling [Array.init n f] creates an array of the size specified in {m n}
306
+
by function {m f}. Initially, element {m A.(i)} holds the value of {m f(i)} for {m i=0, \ldots, ~n-1}.
307
+
Like [ref], it allocates mutable storage to hold the specified values.
308
+
309
+
Calling [Array.get A i] returns the contents of {m A.(i)}.
310
+
311
+
Calling [Array.set A i E] modifies the array {m A} by storing the
312
+
value of {m E} as the new contents of {m A\[i\]}; it returns [()] as its value.
313
+
314
+
OCaml’s arrays are much safer than C’s. In C, an array is nothing more than an
315
+
address indicating the start of a storage area. Nothing indicates the size of
316
+
the area. Therefore C programs are vulnerable to {e buffer overrun attacks:} an
317
+
attacker sends more data than the receiving program expects, overrunning the
318
+
area of storage set aside to hold it. The attack eventually overwrites the
319
+
program itself, replacing it with code controlled by the attacker.
320
+
321
+
{1 Array Examples}
322
+
323
+
In the following session, the identifier [ar] is bound to an array of 20 elements, which
324
+
are initially set to the squares of their subscripts. The array’s third
325
+
element (which actually has subscript 2) is inspected and found to be four. The
326
+
second call to [Array.get] supplies a subscript that is out of range, so OCaml
327
+
rejects it.
328
+
329
+
{@ocaml[
330
+
# let ar = Array.init 20 (fun i -> i * i);;
331
+
val ar : int array =
332
+
[|0; 1; 4; 9; 16; 25; 36; 49; 64; 81; 100; 121; 144; 169; 196; 225; 256;
333
+
289; 324; 361|]
334
+
# Array.get ar 2;;
335
+
- : int = 4
336
+
# Array.get ar 20;;
337
+
Exception: Invalid_argument "index out of bounds".
338
+
Raised by primitive operation at unknown location
339
+
Called from Topeval.load_lambda in file "toplevel/byte/topeval.ml", line 89, characters 4-14
340
+
# Array.set ar 2 33; ar;;
341
+
- : int array =
342
+
[|0; 1; 33; 9; 16; 25; 36; 49; 64; 81; 100; 121; 144; 169; 196; 225; 256;
343
+
289; 324; 361|]
344
+
]}
345
+
346
+
By calling [Array.set], we then modify the element with subscript 2. Note
347
+
however that we cannot modify the array’s length. If we outgrow the array, we
348
+
have to create a new one, copy the data into it, and then forget the old array.
349
+
Typically the new array would be double the size of the old one, so that the
350
+
cost of copying is insignificant.
351
+
352
+
OCaml provides numerous operators for modifying, computing over and searching in
353
+
arrays. Many are analogous to functions on lists. For example,
354
+
[Array.exists] takes a boolean-valued function and returns [true] if an
355
+
array element satisfies it.
356
+
357
+
{@ocaml[
358
+
# Array.exists (fun i -> i > 200) ar;;
359
+
- : bool = true
360
+
# Array.exists (fun i -> i < 0) ar;;
361
+
- : bool = false
362
+
]}
363
+
364
+
{1 References: OCaml {e vs} conventional languages}
365
+
{ul {- We must write [!p] to get the {e contents} of [p]
366
+
}{- We write just [p] for the {e address} of [p]
367
+
}{- We can store private reference cells (like [balance]) in functions---analogous to elements of {e object-oriented programming}
368
+
}{- OCaml’s assignment syntax is {m \tt V , := , E} instead of {m V} = {m E}
369
+
}{- OCaml has few control structures: [while], [match], [if] and [for] (the latter is not covered in this course)
370
+
}{- OCaml has syntax for updating an array via the [a.(i) <- v] syntax which is the same as [Array.set a i v].
371
+
}}
372
+
373
+
Conventional syntax for variables and assignments has hardly changed since
374
+
Fortran, the first high-level language. In conventional languages,
375
+
virtually all variables can be updated. We declare something like
376
+
[p: int], mentioning no reference type even if the language provides
377
+
them. If we do not specify an initial value, we may get whatever bits were
378
+
previously at that address. Illegal values arising from uninitialised
379
+
variables can cause errors that are almost impossible to diagnose.
380
+
381
+
Dereferencing operators (like OCaml’s [!]) are especially unpopular, because
382
+
they clutter the program text. Virtually all programming languages make dereferencing
383
+
implicit (that is, automatic).
384
+
385
+
It is generally accepted these days that a two-dimensional array {m A} is nothing
386
+
but an array of arrays. An assignment to such an array is typically written
387
+
something like {m A\[i,j\] \{:=\} x}; in C, the syntax is [A[i][j] = x]. Higher
388
+
dimensions are treated analogously. The corresponding OCaml code can either
389
+
declare an array of arrays, or use the [A.(i)] syntax to calculate the linear
390
+
offset into a single array.
391
+
392
+
You can use the constructs we have learnt to easily create linked (mutable) lists as
393
+
an alternative to arrays.
394
+
395
+
{@ocaml[
396
+
# type 'a mlist =
397
+
| Nil
398
+
| Cons of 'a * 'a mlist ref;;
399
+
type 'a mlist = Nil | Cons of 'a * 'a mlist ref
400
+
]}
401
+
402
+
It is worth mentioning that OCaml’s references fully suffice for coding the sort of linked data structures
403
+
taught in algorithms courses, and is illustrated in the figure above. The
404
+
programming style is a little different from the usual, but the principles are
405
+
the same. OCaml also provides comprehensive input/output primitives for various
406
+
types of file and operating system.
407
+
408
+
OCaml’s system of modules include {e structures,} which can be seen as encapsulated
409
+
groups of declarations, and {e signatures,} which are specifications of
410
+
structures listing the name and type of each component. Finally, there are
411
+
{e functors,} which are analogous to functions that combine a number of argument
412
+
structures, and which can be used to plug program components together. These
413
+
primitives are useful for managing large programming projects.
414
+
415
+
{2 Exercise 11.1}
416
+
417
+
Comment, with examples, on the differences between an [int ref list] and an [int list ref].
418
+
419
+
{2 Exercise 11.2}
420
+
421
+
Write a version of function [power] (Lecture 1) using [while] instead of recursion.
422
+
423
+
{2 Exercise 11.3}
424
+
425
+
What is the effect of {m \tt while , C_1; ; B , do , C_2 , done} ?
426
+
427
+
{2 Exercise 11.4}
428
+
429
+
Write a function to exchange the values of two references, [xr] and [yr].
430
+
431
+
{2 Exercise 11.5}
432
+
433
+
Arrays of multiple dimensions are represented in OCaml by arrays of arrays. Write functions to
434
+
(a) create an {m n\times n} identity matrix, given {m n}, and
435
+
(b) to transpose an {m m\times n} matrix. Identity matrices have the following form:
436
+
{math
437
+
\left( {
438
+
\begin{array}{cccc}
439
+
1 & 0 & \cdots & 0 \\
440
+
0 & 1 & \cdots & 0 \\
441
+
\vdots & \vdots & \ddots & \vdots \\
442
+
0 & 0 & \cdots & 1 \\
443
+
\end{array}
444
+
} \right)
445
+
}
+408
site/notebooks/foundations/foundations2.mld
+408
site/notebooks/foundations/foundations2.mld
···
1
+
{0 Lecture 2: Recursion and Efficiency}
2
+
3
+
{1 Expression Evaluation}
4
+
5
+
Expression evaluation concerns expressions and the values they return. This
6
+
view of computation may seem to be too narrow. It is certainly far removed from
7
+
computer hardware, but that can be seen as an advantage. For the traditional
8
+
concept of computing solutions to problems, expression evaluation is entirely
9
+
adequate.
10
+
11
+
Starting with {m E_0 }, the expression {m E_i } is reduced to {m E_{i+1} } until this
12
+
process concludes with a value {m v }. A {e value} is something like a number
13
+
that cannot be further reduced.
14
+
15
+
We write {m E \rightarrow E' } to say that {m E } is {e reduced} to {m E' }.
16
+
Mathematically, they are equal: {m E=E' }, but the computation goes from {m E } to
17
+
{m E' } and never the other way around.
18
+
19
+
Computers also interact with the outside world. For a start, they need some
20
+
means of accepting problems and delivering solutions. Many computer systems
21
+
monitor and control industrial processes. This role of computers is familiar
22
+
now, but was never envisaged in the early days. Computer pioneers focused on
23
+
mathematical calculations. Modelling interaction and control requires a notion
24
+
of {e states} that can be observed and changed. Then we can consider
25
+
updating the state by assigning to variables or performing input/output,
26
+
finally arriving at conventional programs as coded in C, for instance.
27
+
28
+
For now, we remain at the level of expressions, which is usually termed
29
+
{e functional programming}.
30
+
31
+
{1 Summing the first {e n} integers}
32
+
33
+
{@ocaml run-on=load[
34
+
# let rec nsum n =
35
+
if n = 0 then
36
+
0
37
+
else
38
+
n + nsum (n - 1);;
39
+
val nsum : int -> int = <fun>
40
+
]}
41
+
42
+
The function call [nsum n] computes the sum [1 +] … [+ nz] rather naively, hence the
43
+
initial [n] in its name:
44
+
45
+
{math
46
+
\begin{aligned}
47
+
\text{nsum } 3 \Rightarrow & 3 + (\text{nsum } 2) \\
48
+
\Rightarrow & 3 + (2 + (\text{nsum } 1) \\
49
+
\Rightarrow & 3 + (2 + (1 + \text{nsum } 0)) \\
50
+
\Rightarrow & 3 + (2 + (1 + 0))
51
+
\end{aligned}
52
+
}
53
+
54
+
The nesting of parentheses is not just an artifact of
55
+
our notation; it indicates a real problem. The function gathers up a
56
+
collection of numbers, but none of the additions can be performed until [nsum 0] is reached. Meanwhile, the computer must store the numbers in an internal
57
+
data structure, typically the {e stack}. For large [n], say [nsum 10000], the
58
+
computation might fail due to stack overflow.
59
+
60
+
We all know that the additions can be performed as we go along. How do we
61
+
make the computer do that?
62
+
63
+
{1 Iteratively summing the first [n] integers}
64
+
65
+
{@ocaml run-on=load[
66
+
# let rec summing n total =
67
+
if n = 0 then
68
+
total
69
+
else
70
+
summing (n - 1) (n + total);;
71
+
val summing : int -> int -> int = <fun>
72
+
]}
73
+
74
+
Function [summing] takes an additional argument: a running total. If
75
+
[n] is zero then it returns the running total; otherwise, [summing]
76
+
adds to it and continues. The recursive calls do not nest; the additions are
77
+
done immediately.
78
+
79
+
A recursive function whose computation does not nest is called
80
+
{e iterative} or {e tail-recursive}. Many functions can be made iterative by
81
+
introducing an argument analogous to [total], which is often called an
82
+
{e accumulator}.
83
+
84
+
The gain in efficiency is sometimes worthwhile and sometimes not. The function
85
+
[power] is not iterative because nesting occurs whenever the exponent is odd.
86
+
Adding a third argument makes it iterative, but the change complicates the
87
+
function and the gain in efficiency is minute; for 32-bit integers, the maximum
88
+
possible nesting is 30 for the exponent {m 2^{31}-1 }.
89
+
90
+
{1 Recursion {e vs} Iteration}
91
+
{ul {- “Iterative” normally refers to a loop, coded using [while] for example (see the final lecture)
92
+
}{- Tail-recursion is only efficient if the compiler detects it
93
+
}{- Mainly it saves space (memory), though iterative code can also run faster
94
+
}{- Do not make programs iterative unless the gain is worth it
95
+
}}
96
+
97
+
A {{: https://en.wikipedia.org/wiki/Structure_and_Interpretation_of_Computer_Programs} classic book}
98
+
by Abelson and Sussman, which describes the Lisp dialect known as Scheme,
99
+
used {e iterative} to mean {e tail-recursive}. Iterative functions produce computations
100
+
resembling those that can be done using while-loops in conventional languages.
101
+
102
+
Many algorithms can be expressed naturally using recursion, but only awkwardly
103
+
using iteration. There is a story that Dijkstra sneaked recursion into Algol-60
104
+
by inserting the words “any other occurrence of the procedure name denotes
105
+
execution of the procedure.” By not using the word “recursion”, he managed to
106
+
slip this amendment past sceptical colleagues.
107
+
108
+
Obsession with tail recursion leads to a coding style in which functions
109
+
have many more arguments than necessary. Write straightforward code first,
110
+
avoiding only gross inefficiency. If the program turns out to be too slow,
111
+
tools are available for pinpointing the cause. Always remember KISS (Keep
112
+
It Simple, Stupid).
113
+
114
+
I hope you have all noticed by now that the summation can be done even more
115
+
efficiently using the arithmetic progression formula:
116
+
117
+
{math 1+\cdots+n = n(n+1)/2 }
118
+
119
+
{1 Silly Summing the First {e n} Integers}
120
+
121
+
{@ocaml run-on=load[
122
+
# let rec sillySum n =
123
+
if n = 0 then
124
+
0
125
+
else
126
+
n + (sillySum (n - 1) + sillySum (n - 1)) / 2;;
127
+
val sillySum : int -> int = <fun>
128
+
]}
129
+
130
+
The function calls itself {m 2^n } times! Bigger inputs mean higher costs---but
131
+
what’s the growth rate?
132
+
133
+
Now let us consider how to estimate various costs associated with a program.
134
+
{e Asymptotic complexity} refers to how costs---usually time or space---grow with
135
+
increasing inputs. Space complexity can never exceed time complexity, for it
136
+
takes time to do anything with the space. Time complexity often greatly
137
+
exceeds space complexity.
138
+
139
+
The function [sillySum] calls itself twice in each recursive step. This
140
+
function is contrived, but many mathematical formulas refer to a particular
141
+
quantity more than once. In OCaml, we can create a local binding to a computed
142
+
value using the {e local declaration} syntax. In the following expression, [y] is
143
+
computed once and used twice:
144
+
145
+
{@ocaml run-on=load[
146
+
# let x = 2.0 in
147
+
let y = Float.pow x 20.0 in
148
+
y *. (x /. y);;
149
+
- : float = 2.
150
+
]}
151
+
152
+
You can read [let x = e1 in e2] as assigning (or "binding") the name [x] with
153
+
the value of [e1] into [e2]. Any use of [x] within [e2] will have the value of [e1],
154
+
and [x] will only be visible in subexpressions into which it has been bound.
155
+
156
+
Why do we need let bindings? Fast hardware does not make good algorithms unnecessary.
157
+
On the contrary, faster hardware magnifies the superiority of better algorithms.
158
+
Typically, we want to handle the largest inputs possible. If we double our processing power,
159
+
what do we gain? How much can we increase {m n }, the input to our function?
160
+
161
+
With [sillySum], we can only go from {m n } to {m n+1 }. We are limited to this
162
+
modest increase because the function’s running time is proportional to {m 2^n }.
163
+
With the function [npower] defined in the previous section, we can go from {m n }
164
+
to {m 2n }: we can handle problems twice as big. With [power] we can do much
165
+
better still, going from {m n } to {m n^2 }.
166
+
167
+
The following table (excerpted from {{: https://archive.org/details/designanalysisof00ahoarich} a 50-year-old book}!)
168
+
illustrates the effect of various time complexities. The left-hand column (dubbed "complexity")
169
+
is defind as how many milliseconds are required to process an input of size {m n }.
170
+
The other entries show the maximum size of {m n } that can be processed in the given time (one
171
+
second, minute or hour).
172
+
173
+
{table
174
+
{tr {th complexity}{th 1 second}{th 1 minute}{th 1 hour}{th gain}}
175
+
{tr {td {m n }}{td 1000}{td 60 000}{td 3 600 000}{td {m \times 60 }}}
176
+
{tr {td {m n \log n }}{td 140}{td 4 895}{td 204 095}{td {m \times 41 }}}
177
+
{tr {td {m n^{2} }}{td 31}{td 244}{td 1 897}{td {m \times 8 }}}
178
+
{tr {td {m n^{3} }}{td 10}{td 39}{td 153}{td {m \times 4 }}}
179
+
{tr {td {m 2^{n} }}{td 9}{td 15}{td 21}{td {m +6 }}}
180
+
}
181
+
182
+
The table illustrates how large an input can be processed as a function
183
+
of time. As we increase the computer time per input from one second to one
184
+
minute and then to one hour, the size of the input increases accordingly.
185
+
186
+
The top two rows (complexities {m n } and {m n \lg n }) increase rapidly: for {m n }, by
187
+
a factor of 60 per column. The bottom two start out close together, but {m n^3 } (which
188
+
grows by a factor of 3.9) pulls well away from {m 2^n } (whose growth is only
189
+
additive). If an algorithm’s complexity is exponential then it can never
190
+
handle large inputs, even if it is given huge resources. On the other hand,
191
+
suppose the complexity has the form {m n^c }, where {m c } is a constant. (We say
192
+
the complexity is {e polynomial}.) Doubling the argument then increases the
193
+
cost by a constant factor. That is much better, though if {m c>3 } the algorithm
194
+
may not be considered practical.
195
+
196
+
{1 Comparing Algorithms: O Notation}
197
+
{ul {- Formally, define {m f(n) = O(g(n)) } provided {m |f(n)| \leq c|g(n)| } as {m n\to\infty }
198
+
}{- {m |f(n)| } is bounded for some constant {m c } and all {e sufficiently large} {m n }.
199
+
}{- Intuitively, look at the {e most significant} term.
200
+
}{- Ignore {e constant factors} as they seldom dominate and are often transitory
201
+
}}
202
+
203
+
For example: consider {m n^2 } instead of {m 3n^2+34n+433 }.
204
+
205
+
The cost of a program is usually a complicated formula. Often we should
206
+
consider only the most significant term. If the cost is {m n^2 + 99n + 900 }
207
+
for an input of size {m n }, then the {m n^2 } term will eventually dominate,
208
+
even though {m 99n } is bigger for {m n<99 }.
209
+
The constant term {m 900 } may look big, but it is soon dominated by {m n^2 }.
210
+
211
+
Constant factors in costs can be ignored unless they are large. For one thing,
212
+
they seldom make a difference: {m 100n^2 } will be better than {m n^3 } in the long
213
+
run: or {e asymptotically} to use the jargon. Moreover, constant factors are
214
+
seldom stable. They depend upon details such as which hardware, operating
215
+
system or programming language is being used. By ignoring constant factors, we
216
+
can make comparisons between algorithms that remain valid in a broad range of
217
+
circumstances.
218
+
219
+
The “Big O” notation is commonly used to describe efficiency---to be precise,
220
+
{e asymptotic complexity}. It concerns the limit of a function as its
221
+
argument tends to infinity. It is an abstraction that meets the informal
222
+
criteria that we have just discussed.
223
+
In the definition, {e sufficiently large} means there is some constant {m n_0 }
224
+
such that {m |f(n)|\leq c|g(n)| } for all {m n } greater than {m n_0 }. The
225
+
role of {m n_0 } is to ignore finitely many exceptions to the bound, such as the
226
+
cases when {m 99n } exceeds {m n^2 }.
227
+
228
+
{1 Simple Facts About O Notation}
229
+
230
+
{math
231
+
\begin{aligned}
232
+
O(2g(n)) & \text{ is the same as } O(g(n)) \\
233
+
O(\log_{10}n) & \text{ is the same as } O(\ln n) \\
234
+
O(n^2+50n+36) & \text{ is the same as } O(n^2) \\
235
+
O(n^2) & \text{ is contained in } O(n^3) \\
236
+
O(2^n) & \text{ is contained in } O(3^n) \\
237
+
O(\log n) & \text{ is contained in } O(\sqrt n)
238
+
\end{aligned}
239
+
}
240
+
241
+
{m O } notation lets us reason about the costs of algorithms easily.
242
+
{ul {- Constant factors such as the {m 2 } in {m O(2g(n)) } drop out: we can use {m O(g(n)) } with twice the value of {m c } in the definition.
243
+
}{- Because constant factors drop out, the base of logarithms is irrelevant.
244
+
}{- Insignificant terms drop out. To see that {m O(n^2+50n+36) } is the same as {m O(n^2) }, consider that {m n^2+50n+36/n^2 } converges to 1 for increasing {m n }. In fact, {m n^2+50n+36 \le 2n^2 } for {m n\ge 51 }, so can double the constant factor
245
+
}}
246
+
247
+
If {m c } and {m d } are constants (that is, they are independent of {m n }) with {m 0 < c < d } then
248
+
{ul {- {m O(n^c) } is contained in {m O(n^d) }
249
+
}{- {m O(c^n) } is contained in {m O(d^n) }
250
+
}{- {m O(\log n) } is contained {m in O(n^c) }
251
+
}}
252
+
253
+
To say that {m O(c^n) } {e is contained in} {m O(d^n) } means that the former gives
254
+
a tighter bound than the latter. For example, if {m f(n)=O(2^n) } then
255
+
{m f(n)=O(3^n) } trivially, but the converse does not hold.
256
+
257
+
{1 Common Complexity Classes}
258
+
{ul {- {m O(1) } is {e constant}
259
+
}{- {m O(\log n) } is {e logarithmic}
260
+
}{- {m O(n) } is {e linear}
261
+
}{- {m O(n\log n) } is {e quasi-linear}
262
+
}{- {m O(n^2) } is {e quadratic}
263
+
}{- {m O(n^3) } is {e cubic}
264
+
}{- {m O(a^n) } is {e exponential} (for fixed {m a })
265
+
}}
266
+
267
+
Logarithms grow very slowly, so {m O(\log n) } complexity is excellent. Because
268
+
{m O } notation ignores constant factors, the base of the logarithm is
269
+
irrelevant!
270
+
271
+
Under linear we might mention {m O(n\log n) }, which occasionally is called
272
+
{e quasilinear} and which scales up well for large {m n }.
273
+
274
+
An example of quadratic complexity is matrix addition: forming the sum of two
275
+
{m n\times n } matrices obviously takes {m n^2 } additions. Matrix
276
+
multiplication is of cubic complexity, which limits the size of matrices that
277
+
we can multiply in reasonable time. An {m O(n^{2.81}) } algorithm exists, but it
278
+
is too complicated to be of much use, even though it is theoretically better.
279
+
280
+
An exponential growth rate such as {m 2^n } restricts us to small values of {m n }.
281
+
Already with {m n=20 } the cost exceeds one million. However, the worst case
282
+
might not arise in normal practice. OCaml type-checking is exponential in the
283
+
worst case, but not for ordinary programs.
284
+
285
+
{1 Sample costs in O notation}
286
+
287
+
Recall that [npower] computes {m x^n }
288
+
by repeated multiplication while [nsum] naively computes the sum
289
+
{m 1+\cdots+n }. Each obviously performs {m O(n) } arithmetic operations. Because
290
+
they are not tail recursive, their use of space is also {m O(n) }. The function
291
+
[summing] is a version of [nsum] with an accumulating argument;
292
+
its iterative behaviour lets it work in constant space. {m O } notation spares
293
+
us from having to specify the units used to measure space.
294
+
295
+
{table
296
+
{tr {th Function}{th Time}{th Space}}
297
+
{tr {td npower, nsum}{td O({m n })}{td O({m n })}}
298
+
{tr {td summing}{td O({m n })}{td O({m 1 })}}
299
+
{tr {td {m n(n+1)/2 }}{td O({m 1 })}{td O({m 1 })}}
300
+
{tr {td power}{td O({m \log~n })}{td O({m \log~n })}}
301
+
{tr {td sillySum}{td O({m 2^n })}{td O({m n })}}
302
+
}
303
+
304
+
Even ignoring constant factors, the units chosen can influence the result.
305
+
Multiplication may be regarded as a single unit of cost. However, the cost of
306
+
multiplying two {m n }-digit numbers for large {m n } is itself an important
307
+
question, especially now that public-key cryptography uses numbers hundreds of
308
+
digits long.
309
+
310
+
Few things can {e really} be done in constant time or stored in constant
311
+
space. Merely to store the number {m n } requires {m O(\log n) } bits. If a
312
+
program cost is {m O(1) }, then we have probably assumed that certain operations
313
+
it performs are also {m O(1) }---typically because we expect never to exceed the
314
+
capacity of the standard hardware arithmetic.
315
+
316
+
With [power], the precise number of operations depends upon {m n } in a
317
+
complicated way, depending on how many odd numbers arise, so it is convenient
318
+
that we can just write {m O(\log n) }. An accumulating argument could reduce its
319
+
space cost to {m O(1) }.
320
+
321
+
{1 Some Simple Recurrence Relations}
322
+
323
+
Consider a function {m T(n) } that has a cost we want to bound using {m O } notation.
324
+
A typical {e base case} is {m T(1)=1 }. Some {e recurrences} are:
325
+
326
+
{table
327
+
{tr {th Equation}{th Complexity}}
328
+
{tr {td {m T(n+1) = T(n)+1 }}{td {m O(n) }}}
329
+
{tr {td {m T(n+1) = T(n)+n }}{td {m O(n^2) }}}
330
+
{tr {td {m T(n) = T(n/2)+1 }}{td {m O(\log n) }}}
331
+
{tr {td {m T(n) = 2T(n/2)+n }}{td {m O(n\log n) }}}
332
+
}
333
+
334
+
To analyse a function, inspect its OCaml declaration. Recurrence equations for
335
+
the cost function {m T(n) } can usually be read off. Since we ignore constant
336
+
factors, we can give the base case a cost of one unit. Constant work done in
337
+
the recursive step can also be given unit cost; since we only need an upper
338
+
bound, this unit represents the larger of the two actual costs. We could use
339
+
other constants if it simplifies the algebra.
340
+
341
+
For example, recall our function [nsum]:
342
+
343
+
{@ocaml run-on=load[
344
+
# let rec nsum n =
345
+
if n = 0 then
346
+
0
347
+
else
348
+
n + nsum (n - 1);;
349
+
val nsum : int -> int = <fun>
350
+
]}
351
+
352
+
Given {m n+1 }, it performs a constant amount of work (an addition and
353
+
subtraction) and calls itself recursively with argument {m n }. We get the
354
+
recurrence equations {m T(0)=1 } and {m T(n+1) = T(n)+1 }. The closed form is
355
+
clearly {m T(n)=n+1 }, as we can easily verify by substitution. The cost is
356
+
{e linear}.
357
+
358
+
This function, given {m n+1 }, calls [nsum], performing {m O(n) } work.
359
+
Again ignoring constant factors, we can say that this call takes exactly {m n }
360
+
units.
361
+
362
+
{@ocaml run-on=load[
363
+
# let rec nsumsum n =
364
+
if n = 0 then
365
+
0
366
+
else
367
+
nsum n + nsumsum (n - 1);;
368
+
val nsumsum : int -> int = <fun>
369
+
]}
370
+
371
+
We get the recurrence equations {m T(0)=1 } and {m T(n+1) = T(n)+n }. It is easy to
372
+
see that {m T(n)=(n-1)+\cdots+1=n(n-1)/2=O(n^2) }. The cost is
373
+
{e quadratic}.
374
+
375
+
The function [power] divides its input {m n } into two, with
376
+
the recurrence equation {m T(n) = T(n/2)+1 }. Clearly {m T(2^n)=n+1 }, so
377
+
{m T(n)=O(\log n) }.
378
+
379
+
{1 Exercises}
380
+
381
+
{2 Exercise 2.1}
382
+
383
+
Code an iterative version of the function [power].
384
+
385
+
{2 Exercise 2.2}
386
+
387
+
Add a column to the table of complexities from {e The Design and Analysis of Computer Algorithms} with the heading {e 60 hours:}
388
+
389
+
{table
390
+
{tr {th complexity}{th 1 second}{th 1 minute}{th 1 hour}{th 60 hours}}
391
+
{tr {td {m n }}{td 1000}{td 60 000}{td 3 600 000}{td }}
392
+
{tr {td {m n \log n }}{td 140}{td 4 895}{td 204 095}{td }}
393
+
{tr {td {m n^{2} }}{td 31}{td 244}{td 1 897}{td }}
394
+
{tr {td {m n^{3} }}{td 10}{td 39}{td 153}{td }}
395
+
{tr {td {m 2^{n} }}{td 9}{td 15}{td 21}{td }}
396
+
}
397
+
398
+
{2 Exercise 2.3}
399
+
400
+
Let {m g_1 }, …, {m g_k } be functions such that {m g_i(n)\ge0 } for {m i=1 }, …, {m k } and all sufficiently
401
+
large {m n }.
402
+
403
+
Show that if {m f(n) = O(a_1 g_1(n)+\cdots+a_k g_k(n)) } then {m f(n) = O(g_1(n)+\cdots+g_k(n)) }.
404
+
405
+
{2 Exercise 2.4}
406
+
407
+
Find an upper bound for the recurrence given by {m T(1) = 1 } and {m T(n) = 2T(n/2)+1 }. You should be
408
+
able to find a tighter bound than {m O(n\log n) }.
+452
site/notebooks/foundations/foundations3.mld
+452
site/notebooks/foundations/foundations3.mld
···
1
+
{0 Lecture 3: Lists}
2
+
3
+
{1 Introduction}
4
+
5
+
{@ocaml run-on=load[
6
+
# let x = [3; 5; 9];;
7
+
val x : int list = [3; 5; 9]
8
+
# let y = [(1, "one"); (2, "two")];;
9
+
val y : (int * string) list = [(1, "one"); (2, "two")]
10
+
]}
11
+
12
+
A {e list} is an ordered series of elements; repetitions are significant.
13
+
So [[3; 5; 9]] differs from [[5; 3; 9]] and from [[3; 3; 5; 9]]. Elements in the
14
+
list are separated with [;] when constructed, as opposed to the [,] syntax
15
+
used for fixed-length tuples.
16
+
17
+
All elements of a list must have the same type. Above we see a list of
18
+
integers and a list of [(integer, string)] pairs. One can also have lists of
19
+
lists, such as [[[3]; []; [5; 6]]], which has type [int list list].
20
+
21
+
In the general case, if {m x_1; \ldots; x_n } all have the same type (say
22
+
{m \tau }) then the list {m [x_1;\ldots;x_n] } has type {m (\tau)\texttt{list} }.
23
+
24
+
Lists are the simplest data structure that can be used to process collections
25
+
of items. Conventional languages use {e arrays} whose elements are
26
+
accessed using subscripting: for example, {m A[i] } yields the {m i }th element of
27
+
the array {m A }. Subscripting errors are a known cause of programmer grief,
28
+
however, so arrays should be replaced by higher-level data structures whenever
29
+
possible.
30
+
31
+
{@ocaml run-on=load[
32
+
# x @ [2; 10];;
33
+
- : int list = [3; 5; 9; 2; 10]
34
+
# List.rev [(1, "one"); (2, "two")];;
35
+
- : (int * string) list = [(2, "two"); (1, "one")]
36
+
]}
37
+
38
+
The infix operator [@] (also called [List.append]) concatenates two lists.
39
+
Also built-in is [List.rev], which reverses a list. These are demonstrated
40
+
in the session above.
41
+
42
+
{1 The List Primitives}
43
+
44
+
There are two kinds of lists:
45
+
{ul {- [[]] represents the empty list
46
+
}{- [x :: l] is the list with head {m x } and tail {m l }
47
+
}}
48
+
49
+
{@ocaml run-on=load[
50
+
# let nil = [];;
51
+
val nil : 'a list = []
52
+
# 1 :: nil;;
53
+
- : int list = [1]
54
+
# 1 :: 2 :: nil;;
55
+
- : int list = [1; 2]
56
+
]}
57
+
58
+
The operator [::] (also called [List.cons] for “construct”), puts a new element on
59
+
to the head of an existing list. While we should not be too preoccupied with
60
+
implementation details, it is essential to know that [::] is an {m O(1) }
61
+
operation. It uses constant time and space, regardless of the length of the
62
+
resulting list. Lists are represented internally with a linked structure;
63
+
adding a new element to a list merely hooks the new element to the front of
64
+
the existing structure. Moreover, that structure continues to denote the same
65
+
list as it did before; to see the new list, one must look at the new [::] node
66
+
(or “cons cell”) just created. We will explain the ['a] notation in the next
67
+
section.
68
+
69
+
Here we see the element [1] being consed to the front of the list [[3; 5; 9]]:
70
+
71
+
{math
72
+
\begin{array}{ccccccccccc}
73
+
:: & \to & \cdots & :: & \to & :: & \to & :: & \to & [] \\
74
+
\downarrow & & & \downarrow & & \downarrow & & \downarrow \\
75
+
1 & & & 3 & & 5 & & 9
76
+
\end{array}
77
+
}
78
+
79
+
Given a list, taking its first element (its “head”) or its list of
80
+
remaining elements (its “tail”) also takes constant time. Each
81
+
operation just follows a link. In the diagram above, the first down arrow
82
+
leads to the head and the leftmost right arrow leads to the tail. Once we
83
+
have the tail, its head is the second element of the original list, etc.
84
+
85
+
The tail is {e not} the last element; it is the {e list} of all elements
86
+
other than the head!
87
+
88
+
{1 Getting at the Head and Tail}
89
+
90
+
{@ocaml run-on=load[
91
+
# let null = function
92
+
| [] -> true
93
+
| x :: l -> false;;
94
+
val null : 'a list -> bool = <fun>
95
+
# null [];;
96
+
- : bool = true
97
+
# null [1; 2; 3];;
98
+
- : bool = false
99
+
# let hd (x::l) = x;;
100
+
Line 1, characters 7-17:
101
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
102
+
Here is an example of a case that is not matched:
103
+
[]
104
+
val hd : 'a list -> 'a = <fun>
105
+
# hd [1; 2; 3];;
106
+
- : int = 1
107
+
# let tl (x::l) = l;;
108
+
Line 1, characters 7-17:
109
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
110
+
Here is an example of a case that is not matched:
111
+
[]
112
+
val tl : 'a list -> 'a list = <fun>
113
+
# tl [7; 6; 5];;
114
+
- : int list = [6; 5]
115
+
]}
116
+
117
+
The empty list has neither head nor tail. Applying [hd] or [tl] to [[]]
118
+
is an error---strictly speaking, an “exception”. The function [null] can
119
+
be used to check for the empty list beforehand. Taking a list apart using
120
+
combinations of [hd] and [tl] is hard to get right. Fortunately, it is seldom
121
+
necessary because of {e pattern-matching}.
122
+
123
+
The declaration of [null] introduces a new concept known as "pattern matching",
124
+
which we will explore more in subsequent lectures. For now, it is sufficient
125
+
to observe that [let null = function] allows for matching on the two possible values
126
+
that might be passed in as argument to [null] here: one for the empty list (for which it returns [true])
127
+
and one for non-empty lists (for which it returns [false]).
128
+
129
+
The declaration of [hd] above has only one clause, for non-empty lists. They
130
+
have the form [x::l] and the function returns [x], which is the head. If you
131
+
compile this program, OCaml also prints a warning to tell us that calling
132
+
the function could raise an exception because not all possible inputs are handled,
133
+
including a counter-example (in this case, the empty list [[]]). The declaration of [tl] is similar to [hd].
134
+
135
+
These three primitive functions are {e polymorphic} and allow flexibility in the
136
+
types of their arguments and results. Note their types!
137
+
138
+
{@ocaml[
139
+
# null;;
140
+
- : 'a list -> bool = <fun>
141
+
# hd;;
142
+
- : 'a list -> 'a = <fun>
143
+
# tl;;
144
+
- : 'a list -> 'a list = <fun>
145
+
]}
146
+
147
+
Symbols ['a] and ['b] are called {e type variables} and stand for any types. Code
148
+
written using these functions is checked for type correctness at compile time.
149
+
And this guarantees strong properties at run time, for example that the
150
+
elements of any list all have the same type. They are usually read as their
151
+
corresponding greek characters; ['a] is "alpha", ['b] is "beta", and so on.
152
+
153
+
{1 Computing the Length of a List}
154
+
155
+
{@ocaml[
156
+
# let rec nlength = function
157
+
| [] -> 0
158
+
| x :: xs -> 1 + nlength xs;;
159
+
val nlength : 'a list -> int = <fun>
160
+
# nlength [];;
161
+
- : int = 0
162
+
# nlength [5; 6; 7];;
163
+
- : int = 3
164
+
]}
165
+
166
+
{math
167
+
\begin{aligned}
168
+
\text{nlength }[a; b; c] \Rightarrow &; 1 + \text{nlength }[b; c[ \\
169
+
\Rightarrow &; 1 + (1 + \text{nlength }[c[) \\
170
+
\Rightarrow &; 1 + (1 + (1 + \text{nlength }[[)) \\
171
+
\Rightarrow &; 1 + (1 + (1 + 0)) \\
172
+
\Rightarrow &; \ldots ;; 3
173
+
\end{aligned}
174
+
}
175
+
176
+
Most list processing involves recursion. This is a simple example; patterns
177
+
can be more complex. Observe the use of a vertical bar [|] to separate the function’s
178
+
clauses. We have {e one} function declaration that handles two cases.
179
+
To understand its role, consider the following faulty code:
180
+
181
+
{@ocaml run-on=load[
182
+
# let rec nlength [] = 0;;
183
+
Line 1, characters 16-22:
184
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
185
+
Here is an example of a case that is not matched:
186
+
_::_
187
+
val nlength : 'a list -> int = <fun>
188
+
# let rec nlength (x::xs) = 1 + nlength xs;;
189
+
Line 1, characters 16-40:
190
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
191
+
Here is an example of a case that is not matched:
192
+
[]
193
+
val nlength : 'a list -> int = <fun>
194
+
]}
195
+
196
+
These are two declarations, not one. First we declare [nlength] to be a
197
+
function that handles only empty lists. Then we redeclare it to be a function
198
+
that handles only non-empty lists; it can never deliver a result. We see that
199
+
a second [let] declaration replaces any previous one rather than extending it
200
+
to cover new cases.
201
+
202
+
Now, let us return to our original declaration of [nlength]. The length function
203
+
is {e polymorphic} and applies to {e all} lists regardless of element
204
+
type! Most programming languages lack such flexibility.
205
+
206
+
Unfortunately, this length computation is naive and wasteful. Like
207
+
[nsum] earlier, it is not tail-recursive. It
208
+
uses {m O(n) } space, where {m n } is the length of its input. As usual, the
209
+
solution is to add an accumulating argument.
210
+
211
+
{1 Efficiently Computing the Length of a List}
212
+
213
+
{@ocaml run-on=load[
214
+
# let rec addlen n = function
215
+
| [] -> n
216
+
| x::xs -> addlen (n + 1) xs;;
217
+
val addlen : int -> 'a list -> int = <fun>
218
+
# addlen 0 [5; 6; 7];;
219
+
- : int = 3
220
+
]}
221
+
222
+
Recall that the use of [function] introduces an extra (unnamed) argument
223
+
that is pattern matched in the subsequent clauses; in this case, to break
224
+
open the list.
225
+
226
+
{math
227
+
\begin{aligned}
228
+
\text{addlen }0 [a; b; c] \Rightarrow & \text{addlen 1 } [b; c] \\
229
+
\Rightarrow & \text{addlen 2 } [c] \\
230
+
\Rightarrow & \text{addlen 3 } [] \\
231
+
\Rightarrow & 3
232
+
\end{aligned}
233
+
}
234
+
235
+
Function [addlen] is again polymorphic. Its type mentions the integer
236
+
accumulator.
237
+
238
+
Now we may declare an efficient length function. It is simply a wrapper for
239
+
[addlen], supplying zero as the initial value of {m n }.
240
+
241
+
{@ocaml run-on=load[
242
+
# let length xs = addlen 0 xs;;
243
+
val length : 'a list -> int = <fun>
244
+
# length [5; 6; 7; 8];;
245
+
- : int = 4
246
+
]}
247
+
248
+
The recursive calls do not nest: this version is iterative. It takes {m O(1) }
249
+
space. Obviously its time requirement is {m O(n) } because it takes at least {m n }
250
+
steps to find the length of an {m n }-element list.
251
+
252
+
{1 Append: List Concatenation}
253
+
254
+
{@ocaml run-on=load[
255
+
# let rec append xs ys =
256
+
match xs, ys with
257
+
| [], ys -> ys
258
+
| x::xs, ys -> x :: append xs ys;;
259
+
val append : 'a list -> 'a list -> 'a list = <fun>
260
+
# append [1; 2; 3] [4];;
261
+
- : int list = [1; 2; 3; 4]
262
+
# let (@) = append;;
263
+
val ( @ ) : 'a list -> 'a list -> 'a list = <fun>
264
+
# [1; 2; 3] @ [4];;
265
+
- : int list = [1; 2; 3; 4]
266
+
]}
267
+
268
+
Patterns can be as complicated as we like. Here, the two patterns are
269
+
[[], ys] and [x::xs, ys].
270
+
271
+
{math
272
+
\begin{aligned}
273
+
\text{append }[1; 2; 3] [4] \Rightarrow & 1 :: \text{append }[2; 3] [4] \\
274
+
\Rightarrow & 1 :: (2 :: \text{append }[3] [4]) \\
275
+
\Rightarrow & 1 :: (2 :: (3 :: \text{append }[] [4])) \\
276
+
\Rightarrow & 1 :: (2 :: (3 :: [4])) \\
277
+
\Rightarrow & [1; 2; 3; 4]
278
+
\end{aligned}
279
+
}
280
+
281
+
Here is how append might be declared, also noting that we have defined [@] as
282
+
an infix operator that is a more convenient way to call [append] on two lists.
283
+
However, this function is also not iterative. It scans its first
284
+
argument, sets up a string of [cons] operations ([::]) and finally does them.
285
+
286
+
It uses {m O(n) } space and time, where {m n } is the length of its first argument.
287
+
{e Its costs are independent of its second argument.}
288
+
289
+
An accumulating argument could make it iterative, but with considerable
290
+
complication. The iterative version would still require {m O(n) } space and time
291
+
because concatenation requires copying all the elements of the first list.
292
+
Therefore, we cannot hope for asymptotic gains; at best we can decrease the
293
+
constant factor involved in {m O(n) }, but complicating the code is likely to
294
+
increase that factor. Never add an accumulator merely out of habit.
295
+
296
+
Note append’s polymorphic type. It tells us that two lists can be joined if
297
+
their element types agree.
298
+
299
+
{1 Reversing a List in {m O(n^2) }}
300
+
301
+
Let us consider one way to reverse a list.
302
+
303
+
{@ocaml run-on=load[
304
+
# let rec nrev = function
305
+
| [] -> []
306
+
| x::xs -> (nrev xs) @ [x];;
307
+
val nrev : 'a list -> 'a list = <fun>
308
+
]}
309
+
310
+
{@ocaml run-on=load[
311
+
# nrev [1; 2; 3];;
312
+
- : int list = [3; 2; 1]
313
+
]}
314
+
315
+
{math
316
+
\begin{aligned}
317
+
\text{nrev }[a; b; c] \Rightarrow &; \text{nrev }[b; c];@;[a] \\
318
+
\Rightarrow &; (\text{nrev }[c];@;[b]);@;[a] \\
319
+
\Rightarrow &; ((\text{nrev }[];@;[c]);@;[b]);@;[a] \\
320
+
\Rightarrow &; (([];@;[c]);@;[b]);@;[a] \ \ldots \ [c; b; a]\
321
+
\end{aligned}
322
+
}
323
+
324
+
This reverse function is grossly inefficient due to poor usage of [append], which
325
+
copies its first argument. If [nrev] is given a list of length {m n>0 }, then
326
+
append makes {m n-1 } conses to copy the reversed tail. Constructing the list
327
+
[[x]] calls [cons] again, for a total of {m n } calls. Reversing the tail
328
+
requires {m n-1 } more conses, and so forth. The total number of conses is:
329
+
330
+
{math 0 + 1 + 2 + \cdots + n = {n(n+1)/2} }
331
+
332
+
The time complexity is therefore {m O(n^2) }. Space complexity is only {m O(n) }
333
+
because the copies don’t all exist at the same time.
334
+
335
+
{1 Reversing a List in {m O(n) }}
336
+
337
+
{@ocaml run-on=load[
338
+
# let rec rev_app xs ys =
339
+
match xs, ys with
340
+
| [], ys -> ys
341
+
| x::xs, ys -> rev_app xs (x::ys);;
342
+
val rev_app : 'a list -> 'a list -> 'a list = <fun>
343
+
]}
344
+
345
+
{math
346
+
\begin{aligned}
347
+
\text{rev\_app }[a; b; c] [] \Rightarrow & \text{rev\_app }[b; c] [a] \\
348
+
\Rightarrow & \text{rev\_app }[c] [b; a] \\
349
+
\Rightarrow & \text{rev\_app }[] [c; b; a] \\
350
+
\Rightarrow & [c; b; a]
351
+
\end{aligned}
352
+
}
353
+
354
+
Calling [rev_app xs ys] reverses the elements of [xs] and
355
+
prepends them to [ys]. Now we may declare
356
+
357
+
{@ocaml run-on=load[
358
+
# let rev xs = rev_app xs [];;
359
+
val rev : 'a list -> 'a list = <fun>
360
+
# rev [1; 2; 3];;
361
+
- : int list = [3; 2; 1]
362
+
]}
363
+
364
+
It is easy to see that this reverse function performs just {m n } conses, given
365
+
an {m n }-element list. For both reverse functions, we could count the number of
366
+
conses precisely---not just up to a constant factor. {m O } notation is still
367
+
useful to describe the overall running time: the time taken by a cons
368
+
varies from one system to another.
369
+
370
+
The accumulator {m y } makes the function iterative. But the gain in complexity
371
+
arises from the removal of [append]. Replacing an expensive operation (append)
372
+
by a series of cheap operations (cons) is called {e reduction in strength}
373
+
and is a common technique in computer science. It originated when many
374
+
computers did not have a hardware multiply instruction; the series of products
375
+
{m i\times r } for {m i=0 }, {m \ldots, n } could more efficiently be computed by
376
+
repeated addition. Reduction in strength can be done in various ways; we
377
+
shall see many instances of removing append.
378
+
379
+
Consing to an accumulator produces the result in reverse. If
380
+
that forces the use of an extra list reversal then the iterative function
381
+
may be much slower than the recursive one.
382
+
383
+
{1 Lists, Strings and Characters}
384
+
385
+
Strings are provided in most programming languages to allow text processing.
386
+
Strings are essential for communication with users. Even a purely numerical
387
+
program formats its results ultimately as strings.
388
+
389
+
{@ocaml run-on=load[
390
+
# (* a character constant *) 'a';;
391
+
- : char = 'a'
392
+
# (* a string constant of length 1 *) "a";;
393
+
- : string = "a"
394
+
# (* a string constant of length 3 *) "abc";;
395
+
- : string = "abc"
396
+
# String.length "abc";;
397
+
- : int = 3
398
+
# (* concatenate two strings *) "abc" ^ "def";;
399
+
- : string = "abcdef"
400
+
]}
401
+
402
+
In a few programming languages, strings simply are lists of characters. In
403
+
OCaml they are a separate type, unrelated to lists, reflecting the fact that
404
+
strings are an abstract concept in themselves.
405
+
406
+
Similarly, characters are not strings of size one, but are a primitive concept.
407
+
Character constants in OCaml have the form ['c'], where {m c } is any character.
408
+
For example, the comma character is [','].
409
+
410
+
Special characters are coded in strings using {e escape sequences} involving the
411
+
backslash character; among many others, a double quote is written ["\\"] and
412
+
the newline character is written ["\n"]. For example, the string
413
+
["I\nLIKE\nCHEESE\n"] represents three text lines.
414
+
415
+
In addition to the operators described above, the relations [<], [<=], [>], and
416
+
[>=] work for strings and yield alphabetic order (more precisely, lexicographic
417
+
order with respect to ASCII character codes).
418
+
419
+
{1 Exercises}
420
+
421
+
{2 Exercise 3.1}
422
+
423
+
Code a recursive function to compute the sum of a list’s elements. Then code an iterative version
424
+
and comment on the improvement in efficiency.
425
+
426
+
{2 Exercise 3.2}
427
+
428
+
Code a function to return the last element of a non-empty list. How efficiently can this be done?
429
+
430
+
{2 Exercise 3.3}
431
+
432
+
Code a function to return the list consisting of the even-numbered elements of the list given as its
433
+
argument. For example, given [[a; b; c; d]] it should return [[b; d]].
434
+
435
+
{2 Exercise 3.4}
436
+
437
+
Consider the polymorphic types in these two function declarations:
438
+
439
+
{@ocaml[
440
+
# let id x = x;;
441
+
val id : 'a -> 'a = <fun>
442
+
# let rec loop x = loop x;;
443
+
val loop : 'a -> 'b = <fun>
444
+
]}
445
+
446
+
Explain why these types make logical sense, preventing run time type errors, even for expressions
447
+
like [id [id [id 0]]] or [loop true / loop 3]. ([/] is the integer division operator in OCaml)
448
+
449
+
{2 Exercise 3.5}
450
+
451
+
Code a function [tails] to return the list of the tails of its argument. For example, given
452
+
[[1; 2; 3]] it should return [[[1; 2; 3]; [2; 3]; [3]; []]].
+411
site/notebooks/foundations/foundations4.mld
+411
site/notebooks/foundations/foundations4.mld
···
1
+
{0 Lecture 4: More on Lists}
2
+
3
+
{1 List Utilities: take and drop}
4
+
5
+
This lecture examines more list utilities, illustrating more patterns of
6
+
recursion, and concludes with a small program for making change.
7
+
8
+
The functions [take] and [drop] divide a list
9
+
into parts, returning or discarding the first {m i} elements.
10
+
11
+
{math
12
+
xs = [\underbrace{x_0,\ldots,x_{i-1}}_{\text{take i xs}},
13
+
\underbrace{x_i,\ldots,x_{n-1}}_{\text{drop i xs}} ]
14
+
}
15
+
16
+
They can be implemented in OCaml as follows:
17
+
18
+
{@ocaml[
19
+
# let rec take i = function
20
+
| [] -> []
21
+
| x::xs ->
22
+
if i > 0 then x :: take (i - 1) xs
23
+
else [];;
24
+
val take : int -> 'a list -> 'a list = <fun>
25
+
# let rec drop i = function
26
+
| [] -> []
27
+
| x::xs ->
28
+
if i > 0 then drop (i-1) xs
29
+
else x::xs;;
30
+
val drop : int -> 'a list -> 'a list = <fun>
31
+
]}
32
+
33
+
Applications of [take] and [drop] will appear in future lectures. Typically,
34
+
they divide a collection of items into equal parts for recursive processing.
35
+
36
+
The [take] function is not iterative, but making it so would not improve
37
+
its efficiency. The task requires copying up to {m i} list elements, which must
38
+
take {m O(i)} space and time.
39
+
40
+
Function [drop] simply skips over {m i} list elements. This requires
41
+
{m O(i)} time but only constant space. It is iterative and much faster than
42
+
[take]. Both functions use {m O(i)} time, but skipping elements is faster
43
+
than copying them: [drop]’s constant factor is smaller.
44
+
45
+
Both functions take an integer and a list, returning a list of the same type.
46
+
So their type is [int -> 'a list -> 'a list].
47
+
48
+
{1 Linear Search}
49
+
{ul {- find {m x} in list {m [x_1,\ldots,x_n]} by comparing with each element
50
+
}{- obviously {m O(n)} time
51
+
}{- simple & general
52
+
}{- ordered searching needs only {m O(\log n)}
53
+
}{- indexed lookup needs only {m O(1)}
54
+
}}
55
+
56
+
{e Linear search} is the obvious way to find a desired item in a
57
+
collection: simply look through all the items, one at a time. If {m x} is in
58
+
the list, then it will be found in {m n/2} steps on average, and even the worst
59
+
case is obviously {m O(n)}.
60
+
61
+
Large collections of data are usually ordered or indexed so that items can be
62
+
found in {m O(\log n)} time, which is exponentially better than {m O(n)}. Even
63
+
{m O(1)} is achievable (using a hash table), though subject to the usual
64
+
proviso that machine limits are not exceeded.
65
+
66
+
Efficient indexing methods are of prime importance: consider Web
67
+
search engines. Nevertheless, linear search is often used to search small
68
+
collections because it is so simple and general, and it is the starting point
69
+
for better algorithms.
70
+
71
+
{1 Equality Tests}
72
+
73
+
{@ocaml[
74
+
# let rec member x = function
75
+
| [] -> false
76
+
| y::l ->
77
+
if x = y then true
78
+
else member x l;;
79
+
val member : 'a -> 'a list -> bool = <fun>
80
+
]}
81
+
82
+
All the list functions we have encountered up to now have been “polymorphic”,
83
+
working for lists of any type. Function [member] uses linear search to report
84
+
whether or not [x] occurs in [l].
85
+
86
+
To do this generically, it uses a special feature of OCaml known as
87
+
“polymorphic equality”, which manifests itself via the [=], [>=], [<=], [>] and
88
+
[<] operators. These operators inspect the {e structure} of the values using a
89
+
consistent order. Types you can legitimately compare this way include integers,
90
+
strings, booleans, and tuples or lists of primitive types.
91
+
92
+
More complex types can be compared this way within careful limits: recursive
93
+
structures or function values will not work (we will cover function values in
94
+
the Currying lecture later). For now, it is sufficient to use these magic
95
+
polymorphic equality operators. As you get more familiar with OCaml and the
96
+
use of higher order functions (also covered in a later lecture), you will
97
+
encounter the use of explicit [compare] functions that are used to provide more
98
+
complex equality tests.
99
+
100
+
The presence of polymorphic equality is a contentious feature in OCaml. While
101
+
it provides a great ease of use in smaller codebases, it starts to become more
102
+
dangerous when building larger OCaml-based systems. Most large-scale users of
103
+
OCaml tend towards not using it in important code, but it is just fine for our
104
+
purposes while learning the beginning steps of computer science.
105
+
106
+
{1 Building a List of Pairs}
107
+
108
+
{@ocaml[
109
+
# let rec zip xs ys =
110
+
match xs, ys with
111
+
| (x::xs, y::ys) -> (x, y) :: zip xs ys
112
+
| _ -> [];;
113
+
val zip : 'a list -> 'b list -> ('a * 'b) list = <fun>
114
+
]}
115
+
116
+
{math \left.[x_1,\ldots,x_n]\atop
117
+
[y_1,\ldots,y_n]\right\}\;\longmapsto\;[(x_1,y_1),\ldots,(x_n,y_n)]
118
+
}
119
+
120
+
The {e wildcard} pattern [_] matches {e anything}. We could have written a
121
+
variable such as [p] instead, but the wildcard reminds us that the
122
+
relevant clause ignores this argument.
123
+
124
+
The patterns are also tested in order of their definitions: first
125
+
[(x::xs, y::ys)], then [_].
126
+
127
+
A list of pairs of the form {m [(x_1,y_1),\ldots,(x_n,y_n)]} associates each
128
+
{m x_i} with {m y_i}. Conceptually, a telephone directory could be regarded as
129
+
such a list, where {m x_i} ranges over names and {m y_i} over the corresponding
130
+
telephone number. Linear search in such a list can find the {m y_i} associated
131
+
with a given {m x_i}, or vice versa---very slowly.
132
+
133
+
In other cases, the {m (x_i,y_i)} pairs might have been generated by applying a
134
+
function to the elements of another list {m [z_1,\ldots,z_n] }.
135
+
136
+
{@ocaml[
137
+
# let rec unzip = function
138
+
| [] -> ([], [])
139
+
| (x, y)::pairs ->
140
+
let xs, ys = unzip pairs in
141
+
(x::xs, y::ys);;
142
+
val unzip : ('a * 'b) list -> 'a list * 'b list = <fun>
143
+
]}
144
+
145
+
Given a list of pairs, [unzip] has to build {e two} lists of
146
+
results, which is awkward using recursion. The version shown above uses the
147
+
{e local binding} [let p = ]{m ;E_1;}[in]{m ;E_2},
148
+
where the value of {m E_1} is bound to the pattern [P] within {m E_2}. The
149
+
let-construct counts as an expression and can be used (perhaps wrapped
150
+
within parentheses) wherever an expression is expected.
151
+
152
+
Note especially the phrase [let xs, ys = unzip pairs]
153
+
which binds [xs] and [ys] to the results of the recursive call.
154
+
In general, the phrase [let P = E] matches the
155
+
pattern {m P} against the value of expression {m E}. It binds all the variables
156
+
in {m P} to the corresponding values.
157
+
158
+
The functions [zip] and [unzip] build and take apart lists of
159
+
pairs: [zip] pairs up corresponding list elements and [unzip]
160
+
inverts this operation. Their types reflect what they do:
161
+
162
+
{@ocaml[
163
+
# zip;;
164
+
- : 'a list -> 'b list -> ('a * 'b) list = <fun>
165
+
# unzip;;
166
+
- : ('a * 'b) list -> 'a list * 'b list = <fun>
167
+
]}
168
+
169
+
If the lists are of unequal length, [zip] discards surplus items at the
170
+
end of the longer list. Its first pattern only matches a pair of non-empty
171
+
lists. The second pattern is just a wildcard and could match anything. OCaml
172
+
tries the clauses in the order given, so the first pattern is tried first.
173
+
The second only gets arguments where at least one of the lists is empty.
174
+
175
+
{1 Building a Pair of Results}
176
+
177
+
Here is a version of [unzip] that replaces the local declaration by a
178
+
function [conspair] for taking apart the pair of lists in the
179
+
recursive call. It defines the same
180
+
computation as the previous version of [unzip] and is possibly clearer,
181
+
but not every local binding can be eliminated as easily.
182
+
183
+
{@ocaml[
184
+
# let conspair ((x, y), (xs, ys)) = (x::xs, y::ys);;
185
+
val conspair : ('a * 'b) * ('a list * 'b list) -> 'a list * 'b list = <fun>
186
+
# let rec unzip = function
187
+
| [] -> ([], [])
188
+
| xy :: pairs -> conspair (xy, unzip pairs);;
189
+
val unzip : ('a * 'b) list -> 'a list * 'b list = <fun>
190
+
]}
191
+
192
+
Making the function iterative yields [revUnzip] below, which is
193
+
very simple. Iteration can construct many results at once in different
194
+
argument positions. Both output lists are built in reverse order, which can
195
+
be corrected by reversing the input to [revUnzip]. The total costs
196
+
will probably exceed those of [unzip] despite the advantages of
197
+
iteration.
198
+
199
+
{@ocaml[
200
+
# let rec revUnzip = function
201
+
| ([], xs, ys) -> (xs, ys)
202
+
| ((x, y)::pairs, xs, ys) ->
203
+
revUnzip (pairs, x::xs, y::ys);;
204
+
val revUnzip : ('a * 'b) list * 'a list * 'b list -> 'a list * 'b list =
205
+
<fun>
206
+
]}
207
+
208
+
{1 An Application: Making Change}
209
+
210
+
Consider a till that has unlimited supplies of coins. The largest coins should be tried
211
+
first, to avoid giving change all in pennies. The list of legal coin values,
212
+
called [till], is given in descending order, such as 50, 20, 10, 5,
213
+
2 and 1. (Recall that the head of a list is the element most easily reached.)
214
+
The code for [change] is based on simple observations:
215
+
{ul {- Change for zero consists of no coins at all. (Note the pattern of [0] in the first clause.)
216
+
}{- For a nonzero amount, try the largest available coin. If it is small enough, use it and decrease the amount accordingly.
217
+
}{- Exclude from consideration any coins that are too large.
218
+
}}
219
+
220
+
{@ocaml[
221
+
# let rec change till amt =
222
+
match till, amt with
223
+
| _, 0 -> []
224
+
| [], _ -> raise (Failure "no more coins!")
225
+
| c::till, amt -> if amt < c then change till amt
226
+
else c :: change (c::till) (amt - c);;
227
+
val change : int list -> int -> int list = <fun>
228
+
]}
229
+
230
+
Although nobody considers making change for zero, this is the simplest way to
231
+
make the algorithm terminate. Most iterative procedures become simplest if,
232
+
in their base case, they do nothing. A base case of one instead of zero is
233
+
often a sign of a novice programmer.
234
+
{ul {- The recursion {e terminates} when [amt = 0].
235
+
}{- Tries the {e largest coin first} to use large coins.
236
+
}{- The algorithm is {e greedy} and can fail!
237
+
}}
238
+
239
+
The function can terminate either with success or failure. It fails by
240
+
raising exception [Failure] namely if [till] becomes empty while [amt] is still nonzero.
241
+
(Exceptions will be discussed later.)
242
+
243
+
Unfortunately, failure can occur even when change can be made. The greedy
244
+
"largest coin first" approach is to blame. Suppose we have coins of values 5
245
+
and 2, and must make change for 6; the only way is {m 6=2+2+2}, ignoring the 5.
246
+
{e Greedy algorithms} are often effective, but not here.
247
+
248
+
{1 All Ways of Making Change}
249
+
250
+
Now we generalise the problem to return the list of {e all possible ways} of making change,
251
+
and write a new [change] function.
252
+
253
+
{@ocaml[
254
+
# let rec change till amt =
255
+
match till, amt with
256
+
| _ , 0 -> [ [] ]
257
+
| [] , _ -> []
258
+
| c::till , amt -> if amt < c then change till amt
259
+
else let rec allc = function
260
+
| [] -> []
261
+
| cs :: css -> (c::cs) :: allc css
262
+
in
263
+
allc (change (c::till) (amt - c)) @
264
+
change till amt;;
265
+
val change : int list -> int -> int list list = <fun>
266
+
]}
267
+
268
+
Look at the type: the result is now a list of lists.
269
+
The code will also never raise exceptions. It expresses failure by returning an
270
+
empty list of solutions: it returns [[]] if the till is empty and the
271
+
amount is nonzero.
272
+
273
+
If the amount is zero, then there is only one way of making change;
274
+
the result should be [[[]]]. This is success in the base case.
275
+
276
+
In nontrivial cases, there are two sources of solutions: to use a coin (if
277
+
possible) and decrease the amount accordingly, or to remove the current coin
278
+
value from consideration.
279
+
280
+
The function [allc] is declared locally in order to make use
281
+
of [c], the current coin. It adds an extra [c] to all the
282
+
solutions returned by the recursive call to make change for [amt - c].
283
+
284
+
Observe the naming convention: [cs] is a list of coins, while
285
+
[css] is a list of such lists. The trailing `s' is suggestive of a
286
+
plural.
287
+
288
+
This complicated program, and the even trickier one on the next slide, are
289
+
included as challenges. Are you enthusiastic enough to work them out? We
290
+
shall revisit the “making change” task later to illustrate exception-handling.
291
+
292
+
{1 All Ways of Making Change --- Faster!}
293
+
294
+
{@ocaml[
295
+
# let rec change till amt chg chgs =
296
+
match till, amt with
297
+
| _ , 0 -> chg::chgs
298
+
| [] , _ -> chgs
299
+
| c::till , amt -> if amt < 0 then chgs
300
+
else change (c::till) (amt - c) (c::chg)
301
+
(change till amt chg chgs);;
302
+
val change : int list -> int -> int list -> int list list -> int list list =
303
+
<fun>
304
+
]}
305
+
306
+
We’ve added {e another} accumulating parameter! Repeatedly improving simple code
307
+
is called {e stepwise refinement}.
308
+
309
+
Two extra arguments eliminate many [::] and append operations from the previous
310
+
slide’s [change] function. The first, [chg], accumulates the coins chosen so
311
+
far; one evaluation of c::chg\} replaces many evaluations of [allc]. The
312
+
second, [chgs], accumulates the list of solutions so far; it avoids the need
313
+
for append. This version runs several times faster than the previous one.
314
+
315
+
Making change is still extremely slow for an obvious reason: the number of
316
+
solutions grows rapidly in the amount being changed. Using 50, 20, 10, 5,
317
+
2 and 1, there are 4366 ways of expressing 99.
318
+
319
+
Our three change functions illustrate a basic technique: program development
320
+
by stepwise refinement. Begin by writing a very simple program and add
321
+
requirements individually. Add efficiency refinements last of all.
322
+
Even if the simpler program cannot be included in the next version and has
323
+
to be discarded, one has learned about the task by writing it.
324
+
325
+
{2 Exercise 4.1}
326
+
327
+
Sets can be represented in OCaml using lists containing no duplicated items
328
+
(i.e. where no item is equal to another using polymorphic comparison).
329
+
330
+
Using the [member] function defined above, code a function to implement set
331
+
union. It should avoid introducing repetitions, for example the union of the
332
+
lists [[4; 7; 1]] and [[6; 4; 7]] should be [[1; 6; 4; 7]] (though the order
333
+
does not matter).
334
+
335
+
{2 Exercise 4.2}
336
+
337
+
Code a function that takes a list of integers and returns two lists, the first consisting of all
338
+
non-negative numbers found in the input and the second consisting of all the negative numbers.
339
+
340
+
{2 Exercise 4.3}
341
+
342
+
How does this version of [zip] differ from the one above?
343
+
344
+
{@ocaml skip[
345
+
# let rec zip xs ys =
346
+
match xs, ys with
347
+
| (x::xs, y::ys) -> (x, y) :: zip xs ys
348
+
| ([], []) -> [];;
349
+
Lines 2-4, characters 2-20:
350
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
351
+
Here is an example of a case that is not matched:
352
+
(x::xs, [])
353
+
val zip : 'a list -> 'b list -> ('a * 'b) list = <fun>
354
+
]}
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+
386
+
387
+
388
+
389
+
390
+
Lines 2-4, characters 5-23:
391
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
392
+
Here is an example of a case that is not matched:
393
+
(_::_, [])
394
+
]}
395
+
396
+
{2 Exercise 4.4}
397
+
398
+
What assumptions do the ‘making change’ functions make about the variables [till] and [amt]?
399
+
Describe what could happen if these assumptions were violated.
400
+
401
+
{2 Exercise 4.5}
402
+
403
+
Show that the number of ways of making change for {m n} (ignoring order) is {m O(n)} if there are two
404
+
legal coin values. What if there are three, four, … coin values?
405
+
406
+
{2 Exercise 4.6}
407
+
408
+
We know nothing about the functions [f] and [g] other than their polymorphic types:
409
+
[val f : 'a * 'b -> 'b * 'a] and [val g : 'a -> 'a list].
410
+
Suppose that [f (1, true)] and [g 0] are evaluated and return their results. State, with reasons,
411
+
what you think the resulting values will be.
+383
site/notebooks/foundations/foundations5.mld
+383
site/notebooks/foundations/foundations5.mld
···
1
+
{0 Lecture 5: Sorting}
2
+
3
+
A few applications for sorting and arranging items into order are:
4
+
{ul {- search
5
+
}{- merging
6
+
}{- duplicates
7
+
}{- inverting tables
8
+
}{- graphics algorithms
9
+
}}
10
+
11
+
Sorting is perhaps the most deeply studied aspect of algorithm design.
12
+
Knuth’s series {e The Art of Computer Programming} devotes an entire
13
+
volume to sorting and searching! {{: https://algs4.cs.princeton.edu/home/} Sedgewick}
14
+
also covers sorting. Sorting has countless applications.
15
+
16
+
Sorting a collection allows items to be found quickly. Recall that linear
17
+
search requires {m O(n)} steps to search among {m n} items. A sorted collection
18
+
admits {e binary search} which requires only {m O(\log n)} time. The idea
19
+
of binary search is to compare the item being sought with the middle item (in
20
+
position {m n/2}) and then to discard either the left half or the right,
21
+
depending on the result of the comparison. Binary search needs arrays or
22
+
trees, not lists; we shall come to binary search trees later.
23
+
24
+
Two sorted files can quickly be {e merged} to form a larger sorted file. Other
25
+
applications include finding {e duplicates} that, after sorting, are adjacent.
26
+
27
+
A telephone directory is sorted alphabetically by name. The same information
28
+
can instead be sorted by telephone number (useful to the police) or by street
29
+
address (useful to junk-mail firms). Sorting information in different ways
30
+
gives it different applications.
31
+
32
+
Common sorting algorithms include insertion sort, quicksort,
33
+
mergesort and heapsort. We shall consider the first three of
34
+
these. Each algorithm has its advantages.
35
+
36
+
As a concrete basis for comparison, runtimes are quoted for DECstation
37
+
computers. (These were based on the MIPS chip, an early RISC design.)
38
+
39
+
{1 How Fast Can We Sort?}
40
+
{ul {- typically count {e comparisons} {m C(n)}
41
+
}{- there are {m n!} permutations of {m n} elements
42
+
}{- each comparison eliminates {e half} of the permutations {m 2^\{C(n)\}\geq n!}
43
+
}{- therefore {m C(n)\geq \log(n!)\approx n\log n-1.44n}
44
+
}}
45
+
46
+
The usual measure of efficiency for sorting algorithms is the number of
47
+
comparison operations required. Mergesort requires only {m O(n\log n)}
48
+
comparisons to sort an input of {m n} items. It is straightforward to prove
49
+
that this complexity is the best possible. There
50
+
are {m n!} permutations of {m n} elements and each comparison distinguishes two
51
+
permutations. The lower bound on the number of comparisons, {m C(n)}, is
52
+
obtained by solving {m 2^\{C(n)\}\geq n!}; therefore {m C(n)\geq \log(n!)\approx n\log n-1.44n}.
53
+
54
+
In order to compare the sorting algorithms, we use the {{: http://www.firstpr.com.au/dsp/rand31/p1192-park.pdf} following source} of
55
+
pseudo-random numbers. Never mind how this works: generating
56
+
statistically good random numbers is hard. Much effort has gone into those few
57
+
lines of code.
58
+
59
+
{@ocaml[
60
+
# let nextrandom seed =
61
+
let a = 16807.0 in
62
+
let m = 2147483647.0 in
63
+
let t = a *. seed in
64
+
t -. m *. (floor (t /. m));;
65
+
val nextrandom : float -> float = <fun>
66
+
# let rec randlist (seed, seeds) = function
67
+
| 0 -> (seed, seeds)
68
+
| n -> randlist (nextrandom seed, seed::seeds) (n-1);;
69
+
val randlist : float * float list -> int -> float * float list = <fun>
70
+
]}
71
+
72
+
We can now bind the identifier [rs] to a list of 10,000 random numbers.
73
+
74
+
{@ocaml[
75
+
# let seed, rs = randlist (1.0, []) 10000;;
76
+
val seed : float = 1043618065.
77
+
val rs : float list =
78
+
[1484786315.; 925166085.; 1614852353.; 721631166.; 173942219.; 1229443779.;
79
+
789328014.; 570809709.; 1760109362.; 270600523.; 2108528931.; 16480421.;
80
+
519782231.; 162430624.; 372212905.; 1954184989.; 898872741.; 1651521688.;
81
+
1114791388.; 1325968501.; 1469981427.; 465437343.; 1732504088.;
82
+
280054095.; 1924919450.; 1244369648.; 1524535715.; 706293012.;
83
+
1372325856.; 1302473561.; 941382430.; 2137445578.; 1937168414.;
84
+
1852570660.; 495231255.; 1092873378.; 140232191.; 328129841.; 632752255.;
85
+
227857208.; 1616471915.; 719842438.; 1402481130.; 745001020.; 791471334.;
86
+
2131048000.; 312659966.; 1389551813.; 443838892.; 854190041.; 741774068.;
87
+
267473377.; 1372555293.; 1539748349.; 697860888.; 1261546017.; 734770781.;
88
+
1512111397.; 813238415.; 1034499961.; 602256496.; 462191385.; 250718457.;
89
+
246489360.; 295426232.; 468306241.; 877829533.; 1130589227.; 1914364883.;
90
+
1479854970.; 878528585.; 1268712064.; 115837978.; 1803525169.; 689954646.;
91
+
1174020926.; 651968560.; 391152461.; 1776325865.; 2015344107.; 246977673.;
92
+
1381242649.; 1115030853.; 190703911.; 316761032.; 464218769.; 1537522160.;
93
+
1958981931.; 390463588.; 224009597.; 235243732.; 620352731.; 1374109567.;
94
+
832140633.; 675075162.; 1296171190.; 2009054653.; 1534419747.; 145880482.;
95
+
1649432515.; 403989126.; 1112417244.; 1290575192.; 896661113.; 218545469.;
96
+
1002393512.; 2131316096.; 551979127.; 932010335.; 665881436.; 1975412808.;
97
+
639877791.; 1781707137.; 894518191.; 568004958.; 1331430214.; 629489848.;
98
+
183264178.; 162027282.; 464592882.; 93302056.; 1178713033.; 1401486247.;
99
+
1846150129.; 1646978216.; 1104441491.; 111995009.; 66193165.; 2038880392.;
100
+
79340676.; 871801051.; 967550305.; 2067810758.; 1600354198.; 1746626663.;
101
+
1516388116.; 1308870791.; 173082747.; 189881227.; 478010722.; 739707315.;
102
+
255334803.; 164203714.; 1893097038.; 1587694259.; 292950569.; 918323194.;
103
+
41453146.; 1217297445.; 256768724.; 586494122.; 586258194.; 660494391.;
104
+
507554325.; 699716071.; 672895139.; 76065072.; 1594869218.; 1439459639.;
105
+
641123634.; 1650611940.; 177447368.; 301427463.; 525804524.; 553672425.;
106
+
926899509.; 794676486.; 690277940.; 2115070333.; 1062048650.; 1653192448.;
107
+
1808855340.; 126475289.; 1028198214.; 1739565096.; 1515748830.;
108
+
427491435.; 319330584.; 666483848.; 854842154.; 1853528448.; 1975611245.;
109
+
1905343266.; 1229802342.; 1416055428.; 2091603253.; 1068308139.;
110
+
198239748.; 982076370.; 1094563396.; 44402415.; 889814989.; 290736902.;
111
+
417580014.; 1935788352.; 595665917.; 367638848.; 894945148.; 1868608068.;
112
+
317883051.; 941451621.; 1595942893.; 789094274.; 1150772108.; 422742112.;
113
+
1444245279.; 1273601104.; 256005435.; 1742330161.; 1514599036.;
114
+
956344512.; 2113041793.; 293237373.; 1386995194.; 1509339194.; 891946522.;
115
+
1020832915.; 592544922.; 1746311153.; 1471539715.; 143832370.;
116
+
2041568248.; 1039556199.; 1608726047.; 1205124472.; 2123533995.;
117
+
1560620058.; 1837598795.; 1028172251.; 98318742.; 1405510706.;
118
+
1047695837.; 59221314.; 1822176683.; 1096018886.; 1528104537.;
119
+
1270922857.; 812074106.; 291115596.; 795788616.; 638657646.; 2034314619.;
120
+
1527649272.; 156357479.; 1010056202.; 1139413443.; 1110927723.;
121
+
1216083346.; 846825145.; 2100385733.; 315213605.; 1629637749.;
122
+
1139833627.; 895118866.; 296359237.; 1361440746.; 1188627020.;
123
+
1964199872.; 166733080.; 54185744.; 575493576.; 1810324496.; 1765549585.;
124
+
53514233.; 747348448.; 61758907.; 1710119765.; 188311628.; 8827553.;
125
+
67975851.; 1808633248.; 1290488843.; 1264775607.; 1711469075.;
126
+
1537468597.; 706677101.; 518290019.; 190285086.; 157683412.; 985907152.;
127
+
1571668636.; 632570698.; 791081325.; 1773794197.; 1787141077.;
128
+
1727982894.; 794213057.; 633163306.; 682601940.; 1573439414.; 1041956036.;
129
+
1169697582.; 758914445.; 2096291761.; 1502226099.; 1665995955.;
130
+
948048264.; 1596326605.; 1816773893.; ...]
131
+
]}
132
+
133
+
{1 Insertion Sort}
134
+
135
+
An insert operation does {m n/2} comparisons on average.
136
+
137
+
{@ocaml[
138
+
# let rec ins x = function
139
+
| [] -> [x]
140
+
| y::ys -> if x <= y then x :: y :: ys
141
+
else y :: ins x ys;;
142
+
val ins : 'a -> 'a list -> 'a list = <fun>
143
+
]}
144
+
145
+
{e Insertion sort} takes {m O(n^2)} comparisons on average:
146
+
147
+
{@ocaml[
148
+
# let rec insort = function
149
+
| [] -> []
150
+
| x::xs -> ins x (insort xs);;
151
+
val insort : 'a list -> 'a list = <fun>
152
+
]}
153
+
154
+
Items from the input are copied one at a time to the output. Each new item is
155
+
inserted into the right place so that the output is always in order.
156
+
157
+
We could easily write iterative versions of these functions, but to no purpose.
158
+
Insertion sort is slow because it does {m O(n^2)} comparisons (and a lot of list
159
+
copying), not because it is recursive. Its quadratic runtime makes it nearly
160
+
useless: it takes 174 seconds for our example while the next-worst figure is
161
+
1.4 seconds.
162
+
163
+
Insertion sort is worth considering because it is easy to code and illustrates
164
+
the concepts. Two efficient sorting algorithms, mergesort and heapsort, can be
165
+
regarded as refinements of insertion sort.
166
+
167
+
{1 Quicksort: The Idea}
168
+
169
+
The Quicksort algorithm has the following flow:
170
+
{ul {- Choose a {e pivot} element, {m a}
171
+
}{- Divide to partition the input into two sublists:
172
+
{ul {- those {e at most} {m a} in value
173
+
}{- those {e exceeding} {m a}
174
+
}}
175
+
}{- Conquer using recursive calls to sort the sublists
176
+
}{- Combine the sorted lists by appending one to the other
177
+
}}
178
+
179
+
Quicksort was invented by Sir Anthony Hoare, who works at Microsoft Research,
180
+
Cambridge. Quicksort works by {e divide and conquer,} a basic algorithm design
181
+
principle. Quicksort chooses from the input some value {m a}, called the
182
+
{e pivot}. It partitions the remaining items into two parts: those {m \leq a}, and
183
+
those {m >a}. It sorts each part recursively, then puts the smaller part before
184
+
the greater.
185
+
186
+
The cleverest feature of Hoare’s algorithm was that the partition could be done
187
+
{e in place} by exchanging array elements. Quicksort was invented before
188
+
recursion was well known, and people found it extremely hard to understand. As
189
+
usual, we shall consider a list version based on functional programming.
190
+
191
+
{1 Quicksort: The Code}
192
+
193
+
{@ocaml[
194
+
# let rec quick = function
195
+
| [] -> []
196
+
| [x] -> [x]
197
+
| a::bs ->
198
+
let rec part l r = function
199
+
| [] -> (quick l) @ (a :: quick r)
200
+
| x::xs ->
201
+
if (x <= a) then
202
+
part (x::l) r xs
203
+
else
204
+
part l (x::r) xs
205
+
in
206
+
part [] [] bs;;
207
+
val quick : 'a list -> 'a list = <fun>
208
+
]}
209
+
210
+
Our OCaml quicksort copies the items. It is still pretty fast, and it is much
211
+
easier to understand. It takes roughly 0.74 seconds to sort our list of random
212
+
numbers.
213
+
214
+
The function declaration consists of three clauses. The first handles the
215
+
empty list; the second handles singleton lists (those of the form [[x]]); the
216
+
third handles lists of two or more elements. Often, lists of length up to five
217
+
or so are treated as special cases to boost speed.
218
+
219
+
The locally declared function [part] partitions the input using [a] as the
220
+
pivot. The arguments [l] and [r] accumulate items for the left ({m \leq a}) and
221
+
right ({m >a}) parts of the input, respectively.
222
+
223
+
It is not hard to prove that quicksort does {m n\log n} comparisons, {e in the average case}
224
+
(see {{: https://archive.org/details/designanalysisof00ahoarich} page 94 of Aho}). With random data, the pivot
225
+
usually has an average value that divides the input in two approximately equal
226
+
parts. We have the recurrence {m T(1) = 1} and {m T(n) = 2T(n/2)+n}, which is
227
+
{m O(n\log n)}. In our example, it is about 235 times faster than insertion
228
+
sort.
229
+
230
+
In the worst case, quicksort’s running time is quadratic! An example is when
231
+
its input is almost sorted or reverse sorted. Nearly all of the items end up
232
+
in one partition; work is not divided evenly. We have the recurrence
233
+
{m T(1) = 1} and {m T(n+1) = T(n)+n}, which is {m O(n^2)}. Randomising the input
234
+
makes the worst case highly unlikely.
235
+
236
+
{1 Append-Free Quicksort}
237
+
238
+
{@ocaml[
239
+
# let rec quik = function
240
+
| ([], sorted) -> sorted
241
+
| ([x], sorted) -> x::sorted
242
+
| a::bs, sorted ->
243
+
let rec part = function
244
+
| l, r, [] -> quik (l, a :: quik (r, sorted))
245
+
| l, r, x::xs ->
246
+
if x <= a then
247
+
part (x::l, r, xs)
248
+
else
249
+
part (l, x::r, xs)
250
+
in
251
+
part ([], [], bs);;
252
+
val quik : 'a list * 'a list -> 'a list = <fun>
253
+
]}
254
+
255
+
The list [sorted] accumulates the result in the {e combine} stage of
256
+
the quicksort algorithm. We have again used the standard technique for
257
+
eliminating append. Calling [quik(xs, sorted)] reverses the elements of
258
+
[xs] and prepends them to the list [sorted].
259
+
260
+
Looking closely at [part], observe that [quik(r, sorted)] is
261
+
performed first. Then [a] is consed to this sorted list. Finally,
262
+
[quik] is called again to sort the elements of [l].
263
+
264
+
The speedup is significant. An imperative quicksort coded in Pascal (taken
265
+
from {{: https://algs4.cs.princeton.edu/20sorting/} Sedgewick}) is just slightly faster than function
266
+
[quik]. The near-agreement is surprising because the computational overheads
267
+
of lists exceed those of arrays. In realistic applications, comparisons are
268
+
the dominant cost and the overheads matter even less.
269
+
270
+
{1 Merging Two Lists}
271
+
272
+
Merge joins two sorted lists.
273
+
274
+
{@ocaml[
275
+
# let rec merge = function
276
+
| [], ys -> ys
277
+
| xs, [] -> xs
278
+
| x::xs, y::ys ->
279
+
if x <= y then
280
+
x :: merge (xs, y::ys)
281
+
else
282
+
y :: merge (x::xs, ys);;
283
+
val merge : 'a list * 'a list -> 'a list = <fun>
284
+
]}
285
+
286
+
Generalises insert to two lists, and does at most {m m+n-1} comparisons.
287
+
288
+
{e Merging} means combining two sorted lists to form a larger sorted list.
289
+
It does at most {m m+n} comparisons, where {m m} and {m n} are the lengths of the
290
+
input lists. If {m m} and {m n} are roughly equal then we have a fast way of
291
+
constructing sorted lists; if {m n=1} then merging degenerates to insertion,
292
+
doing much work for little gain.
293
+
294
+
Merging is the basis of several sorting algorithms; we look at a
295
+
divide-and-conquer one. Mergesort is seldom found in conventional programming
296
+
because it is hard to code for arrays; it works nicely with lists. It divides
297
+
the input (if non-trivial) into two roughly equal parts, sorts them
298
+
recursively, then merges them.
299
+
300
+
Function [merge] is not iterative; the recursion is deep. An iterative
301
+
version is of little benefit for the same reasons that apply to
302
+
[append] in the earlier lecture on Lists.
303
+
304
+
{1 Top-down Merge sort}
305
+
306
+
{@ocaml[
307
+
# let rec tmergesort = function
308
+
| [] -> []
309
+
| [x] -> [x]
310
+
| xs ->
311
+
let k = List.length xs / 2 in
312
+
let l = tmergesort (take k xs) in
313
+
let r = tmergesort (drop k xs) in
314
+
merge (l, r);;
315
+
Line 6, characters 28-32:
316
+
Error: Unbound value take
317
+
]}
318
+
319
+
{m O(n\log n)} comparisons in worst case
320
+
321
+
Mergesort’s {e divide} stage divides the input not by choosing a pivot (as
322
+
in quicksort) but by simply counting out half of the elements. The
323
+
{e conquer} stage again involves recursive calls, and the {e combine}
324
+
stage involves merging. Function [tmergesort] takes roughly 1.4
325
+
seconds to sort the list [rs].
326
+
327
+
In the worst case, mergesort does {m O(n\log n)} comparisons, with the same
328
+
recurrence equation as in quicksort’s average case. Because [take] and
329
+
[drop] divide the input in two equal parts (they differ at most by
330
+
one element), we always have {m T(n) = 2T(n/2)+n}.
331
+
332
+
Quicksort is nearly 3 times as fast in the example. But it risks a
333
+
quadratic worst case! Merge sort is safe but slow. So which algorithm is
334
+
best?
335
+
336
+
We have seen a {e top-down} mergesort. {e Bottom-up} algorithms also
337
+
exist. They start with a list of one-element lists and repeatedly merge
338
+
adjacent lists until only one is left. A refinement, which exploits any
339
+
initial order among the input, is to start with a list of increasing or
340
+
decreasing runs of input items.
341
+
342
+
{1 Summary of Sorting Algorithms}
343
+
{ul {- Optimal is {m O(n\log n)} comparisons
344
+
}{- Insertion sort: simple to code; too slow (quadratic) \[174 secs\]
345
+
}{- Quicksort: fast on average; quadratic in worst case \[0.53 secs\]
346
+
}{- Mergesort: optimal in theory; often slower than quicksort \[1.4 secs\]
347
+
}{- {e Match the algorithm to the application}
348
+
}}
349
+
350
+
Quicksort’s worst case cannot be ignored. For large {m n}, a complexity of
351
+
{m O(n^2)} is catastrophic. Mergesort has an {m O(n\log n)} worst case running
352
+
time, which is optimal, but it is typically slower than quicksort for random
353
+
data.
354
+
355
+
Non-comparison sorting deserves mentioning. We can sort a large number of
356
+
small integers using their radix representation in {m O(n)} time. This result
357
+
does not contradict the comparison-counting argument because comparisons are
358
+
not used at all. Linear time is achievable only if the greatest integer is
359
+
fixed in advance; as {m n} goes to infinity, increasingly many of the items
360
+
are the same. It is a simple special case.
361
+
362
+
Many other sorting algorithms exist. A few are outlined in the exercises.
363
+
364
+
{2 Exercise 5.1}
365
+
366
+
Another sorting algorithm (selection sort) consists of looking at the elements to be sorted,
367
+
identifying and removing a minimal element, which is placed at the head of the result. The tail is
368
+
obtained by recursively sorting the remaining elements. State, with justification, the time
369
+
complexity of this approach.
370
+
371
+
{2 Exercise 5.2}
372
+
373
+
Implement selection sort (see previous exercise) using OCaml.
374
+
375
+
{2 Exercise 5.3}
376
+
377
+
Another sorting algorithm (bubble sort) consists of looking at adjacent pairs of elements,
378
+
exchanging them if they are out of order and repeating this process until no more exchanges are
379
+
possible. State, with justification, the time complexity of this approach.
380
+
381
+
{2 Exercise 5.4}
382
+
383
+
Implement bubble sort (see previous exercise) using OCaml.
+649
site/notebooks/foundations/foundations6.mld
+649
site/notebooks/foundations/foundations6.mld
···
1
+
{0 Lecture 6: Datatypes and Trees}
2
+
3
+
@x-ocaml.requires tyxml,mime_printer,tyxml.functor
4
+
5
+
{@ocaml run-on=load hidden[
6
+
module TreePrinter = struct
7
+
let output_svg my_svg =
8
+
let s = Format.asprintf "%a" (Tyxml.Svg.pp ()) my_svg in
9
+
Mime_printer.push "image/svg" s
10
+
11
+
let rect (x, y) (width, height) =
12
+
let open Tyxml.Svg in
13
+
let rect =
14
+
rect
15
+
~a:
16
+
[
17
+
a_width (float_of_int width, Some `Px);
18
+
a_height (float_of_int height, Some `Px);
19
+
a_x (0.5 +. float_of_int x, None);
20
+
a_y (0.5 +. float_of_int y, None);
21
+
a_rx (5.0, None);
22
+
a_fill (`Color ("#8888ff", None));
23
+
a_stroke (`Color ("black", None));
24
+
]
25
+
[]
26
+
in
27
+
rect
28
+
29
+
let svg (width, height) content =
30
+
let open Tyxml.Svg in
31
+
let bg =
32
+
rect
33
+
~a:
34
+
[
35
+
a_width (100., Some `Percent);
36
+
a_height (100., Some `Percent);
37
+
a_x (0.0, None);
38
+
a_y (0.0, None);
39
+
a_fill `None;
40
+
]
41
+
[]
42
+
in
43
+
svg
44
+
~a:
45
+
[
46
+
a_width (float_of_int width, None);
47
+
a_height (float_of_int height, None);
48
+
]
49
+
(bg :: content)
50
+
51
+
let curve (x1, y1) (x2, y2) =
52
+
let s =
53
+
Printf.sprintf "M%d %d C %d %d, %d %d, %d %d" x1 y1 x1 y2 x2 y1 x2 y2
54
+
in
55
+
let open Tyxml.Svg in
56
+
path ~a:[ a_d s; a_stroke (`Color ("black", None)); a_fill `None ] []
57
+
58
+
let text (x, y) s =
59
+
let open Tyxml.Svg in
60
+
text
61
+
~a:
62
+
[
63
+
a_x_list [ (float_of_int x, None) ];
64
+
a_y_list [ (float_of_int y, None) ];
65
+
a_text_anchor `Middle;
66
+
a_font_size "10pt";
67
+
]
68
+
[ txt s ]
69
+
70
+
type 'a tree = Lf | Br of 'a * 'a tree list
71
+
72
+
type params = {
73
+
node_width : int;
74
+
node_height : int;
75
+
inter_node_width : int;
76
+
inter_node_height : int;
77
+
}
78
+
79
+
let rec width params : 'a tree -> int = function
80
+
| Lf -> params.node_width
81
+
| Br (_, []) -> params.node_width
82
+
| Br (_, ts) ->
83
+
let n = List.length ts in
84
+
List.fold_left
85
+
(fun acc t -> acc + width params t)
86
+
((n - 1) * params.inter_node_width)
87
+
ts
88
+
89
+
let rec height params : 'a tree -> int = function
90
+
| Lf -> params.node_height
91
+
| Br (_, []) -> params.node_height
92
+
| Br (_, ts) ->
93
+
let h = List.fold_left (fun acc t -> max acc (height params t)) 0 ts in
94
+
params.node_height + params.inter_node_height + h
95
+
96
+
type tree_result = {
97
+
rects : Svg_types.rect Tyxml_svg.elt list;
98
+
lines : Svg_types.path Tyxml_svg.elt list;
99
+
text : Svg_types.text Tyxml_svg.elt list;
100
+
width : int;
101
+
height : int;
102
+
}
103
+
104
+
let pp fmt tree_result =
105
+
Format.fprintf fmt "width: %d, height: %d" tree_result.width
106
+
tree_result.height
107
+
108
+
let rec combine_horizontal params acc = function
109
+
| [] -> acc
110
+
| { rects; width; height; lines; text } :: ts ->
111
+
let acc =
112
+
{
113
+
rects = acc.rects @ rects;
114
+
lines = acc.lines @ lines;
115
+
text = acc.text @ text;
116
+
width = acc.width + width + params.inter_node_width;
117
+
height = max acc.height height;
118
+
}
119
+
in
120
+
combine_horizontal params acc ts
121
+
122
+
let rec draw_tree params to_txt (x, y) tree : tree_result =
123
+
let txt = to_txt tree in
124
+
match tree with
125
+
| Lf | Br (_, []) ->
126
+
let svg = rect (x, y) (params.node_width, params.node_height) in
127
+
let text =
128
+
text (x + (params.node_width / 2), y + (params.node_height / 2)) txt
129
+
in
130
+
{
131
+
rects = [ svg ];
132
+
width = params.node_width;
133
+
height = params.node_height;
134
+
lines = [];
135
+
text = [ text ];
136
+
}
137
+
| Br (_, nodes) ->
138
+
let new_y = y + params.node_height + params.inter_node_height in
139
+
let rec inner (acc, lines) x = function
140
+
| [] -> (List.rev acc, lines)
141
+
| t :: ts ->
142
+
let res = draw_tree params to_txt (x, new_y) t in
143
+
let line_dest = (x + (res.width / 2), new_y) in
144
+
let new_x = x + params.inter_node_width + res.width in
145
+
inner (res :: acc, line_dest :: lines) new_x ts
146
+
in
147
+
let (r, rs), lines =
148
+
match inner ([], []) x nodes with
149
+
| [], _ -> failwith "impossible"
150
+
| r :: rs, lines -> ((r, rs), lines)
151
+
in
152
+
let combined = combine_horizontal params r rs in
153
+
let box_x = x + ((combined.width - params.node_width) / 2) in
154
+
let center = x + (combined.width / 2) in
155
+
let text = text (center, y + (params.node_height / 2)) txt in
156
+
let lines =
157
+
List.map
158
+
(fun dest -> curve (center, y + params.node_height) dest)
159
+
lines
160
+
in
161
+
let new_box = rect (box_x, y) (params.node_width, params.node_height) in
162
+
{
163
+
rects = new_box :: combined.rects;
164
+
width = combined.width;
165
+
height =
166
+
combined.height + params.inter_node_height + params.node_height;
167
+
lines = lines @ combined.lines;
168
+
text = text :: combined.text;
169
+
}
170
+
171
+
let svg_of_tree params to_txt tree =
172
+
let result = draw_tree params to_txt (0, 0) tree in
173
+
svg
174
+
(result.width + 2, result.height + 2)
175
+
((result.rects :> Svg_types.svg_content Tyxml_svg.elt list)
176
+
@ (result.lines :> Svg_types.svg_content Tyxml_svg.elt list)
177
+
@ (result.text :> Svg_types.svg_content Tyxml_svg.elt list))
178
+
179
+
let test_tree1 = Lf
180
+
let test_tree2 = Br (1, [ Lf ])
181
+
let test_tree3 = Br (1, [ Lf; Lf ])
182
+
let test_tree4 = Br (1, [ Lf; Br (2, [ Lf; Lf ]) ])
183
+
let test_tree5 = Br (1, [ Lf; Lf; Lf ])
184
+
185
+
let test_tree6 =
186
+
Br (1, [ test_tree1; test_tree2; test_tree3; test_tree4; test_tree5 ])
187
+
188
+
let default_params =
189
+
{
190
+
node_width = 50;
191
+
node_height = 40;
192
+
inter_node_width = 30;
193
+
inter_node_height = 30;
194
+
}
195
+
end;;
196
+
]}
197
+
198
+
199
+
{1 An Enumeration Type}
200
+
201
+
We will now learn how to define more expressive types than the
202
+
basic ones supplied with the core OCaml language.
203
+
204
+
{@ocaml run-on=load[
205
+
# type vehicle = Bike
206
+
| Motorbike
207
+
| Car
208
+
| Lorry;;
209
+
type vehicle = Bike | Motorbike | Car | Lorry
210
+
]}
211
+
212
+
{ul {- We have declared a {e new type} named [vehicle].
213
+
}{- {m \ldots} along with four new constants.
214
+
}{- They are the {e constructors} of the datatype.
215
+
}}
216
+
217
+
The [type] declaration adds a new type to our OCaml session. Type
218
+
[vehicle] is as good as any built-in type and even admits
219
+
pattern-matching (as we used with the built-in list types earlier).
220
+
The four new identifiers of type [vehicle] are called {e constructors}.
221
+
222
+
We could represent the various vehicles by the numbers 0--3. However, the code would be
223
+
hard to read and even harder to maintain. Consider adding [Tricycle]
224
+
as a new vehicle. If we wanted to add it before [Bike], then all the
225
+
numbers would have to be changed. Using [type], such additions are
226
+
trivial and the compiler can (at least sometimes) warn us when it encounters a
227
+
function declaration that doesn’t yet have a case for [Tricycle].
228
+
229
+
Representing vehicles by strings like ["Bike"], ["Car"], etc.,
230
+
is also bad. Comparing string values is slow and the compiler
231
+
can’t warn us of misspellings like ["MOtorbike"]: they will make our
232
+
code fail.
233
+
234
+
Most programming languages allow the declaration of types like
235
+
[vehicle]. Because they consist of a series of identifiers, they are
236
+
called {e enumeration types}. Other common examples are days of the week
237
+
or colours. The compiler chooses the integers for us; type-checking prevents
238
+
us from confusing [Bike] with [Red] or [Sunday].
239
+
240
+
{1 Declaring a Function on Vehicles}
241
+
242
+
{@ocaml run-on=load[
243
+
# let wheels = function
244
+
| Bike -> 2
245
+
| Motorbike -> 2
246
+
| Car -> 4
247
+
| Lorry -> 18;;
248
+
val wheels : vehicle -> int = <fun>
249
+
]}
250
+
251
+
{ul {- Datatype constructors can be used in patterns.
252
+
}{- Pattern-matching is fast, even complicated nested patterns.
253
+
}{- Notice the type of the argument is [vehicle], which we defined earlier.
254
+
}}
255
+
256
+
The beauty of datatype declarations is that the new types behave as if they
257
+
were built into OCaml. Type-checking catches common errors, such as mixing up
258
+
different datatypes in a function like [wheels], as well as missing
259
+
and redundant patterns.
260
+
261
+
{1 A Datatype whose Constructors have Arguments}
262
+
263
+
{@ocaml run-on=load[
264
+
# type vehicle = Bike
265
+
| Motorbike of int
266
+
| Car of bool
267
+
| Lorry of int;;
268
+
type vehicle = Bike | Motorbike of int | Car of bool | Lorry of int
269
+
]}
270
+
{ul {- Constructors with arguments (like [Lorry]) are {e distinct values}. (So [Car true] is distinct from [Car false]).
271
+
}{- Different kinds of [vehicle] can belong to one list: [[Bike, Car true, Motorbike 450]]
272
+
}}
273
+
274
+
OCaml generalises the notion of enumeration type to allow data to be associated
275
+
with each constructor. The constructor [Bike] is a vehicle all by itself, but
276
+
the other three constructors create vehicles from arguments.
277
+
278
+
Since we might find it hard to remember what the various [int] and
279
+
[bool] components are for, it is wise to include {e comments} in
280
+
complex declarations. In OCaml, comments are enclosed in the brackets
281
+
[(*] and [*)]. Programmers should comment their code to explain
282
+
design decisions and key features of the algorithms (sometimes by citing a
283
+
reference work).
284
+
285
+
{@ocaml run-on=load[
286
+
# type vehicle = Bike
287
+
| Motorbike of int (* engine size in CCs *)
288
+
| Car of bool (* true if a Reliant Robin *)
289
+
| Lorry of int;;
290
+
type vehicle = Bike | Motorbike of int | Car of bool | Lorry of int
291
+
]}
292
+
293
+
The list shown on the slide represents a bicycle, a Reliant Robin and a large
294
+
motorbike. It can be almost seen as a mixed-type list containing integers and
295
+
booleans. It is actually a list of vehicles; datatypes lessen the impact of
296
+
the restriction that all list elements must have the same type.
297
+
298
+
{1 A Finer Wheel Computation}
299
+
300
+
We now define a [wheels] function to calculate the number of wheels
301
+
in any [vehicle]. This requires pattern matching to retrieve the
302
+
constructors and their associated arguments, around which we build the
303
+
logic:
304
+
305
+
{@ocaml[
306
+
# let wheels = function
307
+
| Bike -> 2
308
+
| Motorbike _ -> 2
309
+
| Car robin -> if robin then 3 else 4
310
+
| Lorry w -> w;;
311
+
val wheels : vehicle -> int = <fun>
312
+
]}
313
+
314
+
This function consists of four clauses:
315
+
{ul {- A Bike has two wheels. This is a constant result.
316
+
}{- A Motorbike has two wheels. The [_] signifies a "wildcard" pattern match that we discard, since the engine size of the bike is not relevant to the number of wheels.
317
+
}{- A Reliant Robin has three wheels; all other cars have four. We bind [robin] to the [bool] argument and then use it in the right hand side of the pattern match, much like a [let] binding in normal code.
318
+
}{- A Lorry has the number of wheels stored with its constructor, and we simply return that.
319
+
}}
320
+
321
+
There is no overlap between the [Motorbike] and [Lorry] cases. Although
322
+
[Motorbike] and [Lorry] both hold an integer, OCaml takes the
323
+
constructor into account and keeps any [Motorbike] distinct from any [Lorry].
324
+
325
+
Vehicles are one example of a concept consisting of several varieties with
326
+
distinct features. Most programming languages can represent such concepts
327
+
using something analogous to datatypes. (They are sometimes called
328
+
{e union types} or {e variant records} whose {e tag fields} play the
329
+
role of the constructors.)
330
+
331
+
A pattern may be built from the constructors of several datatypes, including
332
+
lists. A pattern may also contain integer and string constants. There is no
333
+
limit to the size of patterns or the number of clauses in a function
334
+
declaration. OCaml performs pattern-matching {{: https://dl.acm.org/citation.cfm?id=507641} efficiently}
335
+
(you do not need to understand the details of how it optimises pattern-matching at this stage).
336
+
337
+
{1 Error Handling: Exceptions}
338
+
339
+
During a computation, what happens if something goes {e wrong?}
340
+
{ul {- Division by zero
341
+
}{- Pattern matching failure
342
+
}}
343
+
344
+
{e Exception-handling} lets us recover gracefully.
345
+
{ul {- Raising an exception abandons the current computation.
346
+
}{- Handling the exception attempts an alternative computation.
347
+
}{- The raising and handling can be far apart in the code.
348
+
}{- Errors of {e different sorts} can be handled separately.
349
+
}}
350
+
351
+
Exceptions are necessary because it is not always possible to tell in advance
352
+
whether or not a search will lead to a dead end or whether a numerical
353
+
calculation will encounter errors such as overflow or divide by zero. Rather
354
+
than just crashing, programs should check whether things have gone wrong, and
355
+
perhaps attempt an alternative computation (perhaps using a different algorithm
356
+
or higher precision). A number of modern languages provide exception handling.
357
+
358
+
{1 Exceptions in OCaml}
359
+
360
+
{@ocaml[
361
+
# exception Failure;;
362
+
exception Failure
363
+
# exception NoChange of int;;
364
+
exception NoChange of int
365
+
# raise Failure;;
366
+
Exception: Failure.
367
+
Called from Topeval.load_lambda in file "toplevel/byte/topeval.ml", line 89, characters 4-14
368
+
]}
369
+
370
+
Each [exception] declaration introduces a distinct sort of exception, which can
371
+
be handled separately from others. If {m E} raises an exception, then its
372
+
evaluation has failed; {e handling} an exception means evaluating another
373
+
expression and returning its value instead. One exception handler can specify
374
+
separate expressions for different sorts of exceptions.
375
+
376
+
Exception names are {e constructors} of the special datatype [exn]. This is a
377
+
peculiarity of OCaml that lets exception-handlers use pattern-matching. Note that
378
+
exception [Failure] is just an error indication, while [NoChange n] carries
379
+
further information: the integer {m n}.
380
+
381
+
{@ocaml[
382
+
# try
383
+
print_endline "pre exception";
384
+
raise (NoChange 1);
385
+
print_endline "post exception";
386
+
with
387
+
| NoChange _ ->
388
+
print_endline "handled a NoChange exception";;
389
+
pre exception
390
+
handled a NoChange exception
391
+
Line 3, characters 4-22:
392
+
Warning 21 [nonreturning-statement]: this statement never returns (or has an unsound type.)
393
+
- : unit = ()
394
+
]}
395
+
396
+
The effect of [raise <expr>] is to jump to the most recently-encountered
397
+
handler that matches [<expr>]. The matching handler can only be found
398
+
{e dynamically} (during execution); contrast with how OCaml associates occurrences
399
+
of identifiers with their matching declarations, which does not require running
400
+
the program. A handler is introduced via the [try] keyword, which executes
401
+
the subexpression and dispatches any exceptions encountered to the corresponding
402
+
pattern match for exceptions defined in the [with] block.
403
+
404
+
This is also the first time that we have encountered the [unit] type. This
405
+
represents a type that has no values, and is used to indicate that a block
406
+
has no meaningful return value. We will come back to this when learning more
407
+
about {e imperative} programming later on. For now, it is sufficient to understand
408
+
that [print_endline] will print out the argument to the console output, and return
409
+
a [unit] type. The semicolon at the end of the expression is a convenient way to
410
+
execute sequential statements that return the [unit] type.
411
+
412
+
One criticism of OCaml’s exceptions is that---unlike the Java language---nothing
413
+
in a function declaration indicates which exceptions it might raise. One
414
+
alternative to exceptions is to instead return a value of datatype [option].
415
+
416
+
{@ocaml[
417
+
# let x = Some 1;;
418
+
val x : int option = Some 1
419
+
]}
420
+
421
+
[None] signifies an error, while [Some x] returns the solution {m x}. This
422
+
approach looks clean, but the drawback is that many places in the code would
423
+
have to check for [None]. Despite this, there is a builtin [option] type
424
+
in OCaml as it is so useful. We will see in later lectures how to define our
425
+
own version of [option] using polymorphic datatype definitions.
426
+
427
+
{1 Making Change with Exceptions}
428
+
429
+
{@ocaml[
430
+
# exception Change;;
431
+
exception Change
432
+
# let rec change till amt =
433
+
match till, amt with
434
+
| _, 0 -> []
435
+
| [], _ -> raise Change
436
+
| c::till, amt -> if amt < 0 then raise Change
437
+
else try c :: change (c::till) (amt - c)
438
+
with Change -> change till amt;;
439
+
val change : int list -> int -> int list = <fun>
440
+
]}
441
+
442
+
In the Lists lectures, we considered the problem of making change. The greedy
443
+
algorithm presented there could not express “6 using 5 and 2” because it always
444
+
took the largest coin. Returning the list of all possible solutions avoids
445
+
that problem rather expensively: we only need one solution.
446
+
447
+
Using exceptions, we can code a {e backtracking} algorithm: one that can undo
448
+
past decisions if it comes to a dead end. The exception [Change] is raised if
449
+
we run out of coins (with a non-zero amount) or if the amount goes negative.
450
+
We always try the largest coin, but enclose the recursive call in an exception
451
+
handler, which undoes the choice if it goes wrong.
452
+
453
+
Carefully observe how exceptions interact with recursion. The exception
454
+
handler always undoes the {e most recent} choice, leaving others possibly to
455
+
be undone later. If making change really is impossible, then eventually
456
+
[exception Change] will be raised with no handler to catch it, and it
457
+
will be reported at top level.
458
+
459
+
\newpage
460
+
461
+
{1 Making Change: A Trace}
462
+
463
+
Here is the full execution. Observe how the exception handlers nest and how
464
+
they drop away once the given expression has returned a value.
465
+
466
+
{math
467
+
\begin{aligned}
468
+
\text{change [5; 2] 6}
469
+
\Rightarrow &\; \text{try 5::change [5; 2] 1}\\
470
+
&\; \text{with Change -> change [2] 6}\\
471
+
\Rightarrow &\; \text{try 5::(try 5::change [5; 2] (-4)}\\
472
+
&\; \text{with Change -> change [2] 1)}\\
473
+
&\; \text{with Change -> change [2] 6}\\
474
+
\Rightarrow &\; \text{5::(change [2] 1)}\\
475
+
&\; \text{with Change -> change [2] 6}\\
476
+
\Rightarrow &\; \text{try 5::(try 2::change [2] (-1)}\\
477
+
&\; \text{with Change -> change [] 1)}\\
478
+
&\; \text{with Change -> change [2] 6} \\
479
+
\Rightarrow &\; \text{try 5::(change [] 1)}\\
480
+
&\; \text{with Change -> change [2] 6} \\
481
+
\Rightarrow &\; \text{change [2] 6} \\
482
+
\Rightarrow &\; \text{try 2::change [2] 4}\\
483
+
&\; \text{with Change -> change [] 6} \\
484
+
\Rightarrow &\; \text{try 2::(try 2::change [2] 2}\\
485
+
&\; \text{with Change -> change [] 4)}\\
486
+
&\; \text{with Change -> change [] 6} \\
487
+
\Rightarrow &\; \text{try 2::(try 2::(try 2::change [2] 0 }\\
488
+
&\; \text{with Change -> change [] 2)}\\
489
+
&\; \text{with Change -> change [] 4)}\\
490
+
&\; \text{with Change -> change [] 6} \\
491
+
\Rightarrow &\; \text{try 2::(try 2::[2]}\\
492
+
&\; \text{with Change -> change [] 4)}\\
493
+
&\; \text{with Change -> change [] 6} \\
494
+
\Rightarrow &\; \text{try 2::[2; 2]}\\
495
+
&\; \text{with Change -> change [] 6} \\
496
+
\Rightarrow &\; \text{[2; 2; 2]}
497
+
\end{aligned}
498
+
}
499
+
500
+
{1 Binary Trees, a Recursive Datatype}
501
+
502
+
{@ocaml run-on=load[
503
+
# type 'a tree =
504
+
Lf
505
+
| Br of 'a * 'a tree * 'a tree;;
506
+
type 'a tree = Lf | Br of 'a * 'a tree * 'a tree
507
+
]}
508
+
509
+
{@ocaml run-on=load hidden[
510
+
# let tree_printer to_txt t =
511
+
let rec tree = function
512
+
| Lf -> TreePrinter.Lf
513
+
| Br (n, l, r) -> TreePrinter.Br (n, [tree l; tree r])
514
+
in
515
+
let to_txt = function
516
+
| TreePrinter.Lf -> "Lf"
517
+
| TreePrinter.Br (n, _) -> to_txt n
518
+
in
519
+
let svg = TreePrinter.svg_of_tree TreePrinter.default_params to_txt (tree t) in
520
+
Mime_printer.push "image/svg" (Format.asprintf "%a" (Tyxml.Svg.pp ()) svg);;
521
+
val tree_printer :
522
+
('a -> string Tyxml.Svg.wrap Tyxml.Svg.wrap) -> 'a tree -> unit = <fun>
523
+
]}
524
+
525
+
A data structure with multiple branching is called a “tree”. Trees can
526
+
represent mathematical expressions, logical formulae, computer programs, the
527
+
phrase structure of English sentences, etc.
528
+
529
+
{x@ocaml run-on=load[
530
+
# let tree = Br(1, Br(2, Br(4, Lf, Lf),
531
+
Br(5, Lf, Lf)),
532
+
Br(3, Lf, Lf));;
533
+
val tree : int tree =
534
+
Br (1, Br (2, Br (4, Lf, Lf), Br (5, Lf, Lf)), Br (3, Lf, Lf))
535
+
# tree_printer string_of_int tree;;
536
+
- : unit = ()
537
+
]x[
538
+
{%html: <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
539
+
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
540
+
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="452" height="252"><rect width="100%" height="100%" x="0" y="0" fill="none"></rect><rect width="50px" height="40px" x="200.5" y="0.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="120.5" y="70.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="40.5" y="140.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="0.5" y="210.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="80.5" y="210.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="200.5" y="140.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="160.5" y="210.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="240.5" y="210.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="360.5" y="70.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="320.5" y="140.5" rx="5" fill="#8888ff" stroke="black"></rect><rect width="50px" height="40px" x="400.5" y="140.5" rx="5" fill="#8888ff" stroke="black"></rect><path d="M225 40 C 225 70, 385 40, 385 70" stroke="black" fill="none"></path><path d="M225 40 C 225 70, 145 40, 145 70" stroke="black" fill="none"></path><path d="M145 110 C 145 140, 225 110, 225 140" stroke="black" fill="none"></path><path d="M145 110 C 145 140, 65 110, 65 140" stroke="black" fill="none"></path><path d="M65 180 C 65 210, 105 180, 105 210" stroke="black" fill="none"></path><path d="M65 180 C 65 210, 25 180, 25 210" stroke="black" fill="none"></path><path d="M225 180 C 225 210, 265 180, 265 210" stroke="black" fill="none"></path><path d="M225 180 C 225 210, 185 180, 185 210" stroke="black" fill="none"></path><path d="M385 110 C 385 140, 425 110, 425 140" stroke="black" fill="none"></path><path d="M385 110 C 385 140, 345 110, 345 140" stroke="black" fill="none"></path><text x="225" y="20" text-anchor="middle" font-size="10pt">1</text><text x="145" y="90" text-anchor="middle" font-size="10pt">2</text><text x="65" y="160" text-anchor="middle" font-size="10pt">4</text><text x="25" y="230" text-anchor="middle" font-size="10pt">Lf</text><text x="105" y="230" text-anchor="middle" font-size="10pt">Lf</text><text x="225" y="160" text-anchor="middle" font-size="10pt">5</text><text x="185" y="230" text-anchor="middle" font-size="10pt">Lf</text><text x="265" y="230" text-anchor="middle" font-size="10pt">Lf</text><text x="385" y="90" text-anchor="middle" font-size="10pt">3</text><text x="345" y="160" text-anchor="middle" font-size="10pt">Lf</text><text x="425" y="160" text-anchor="middle" font-size="10pt">Lf</text></svg> %}
541
+
]}
542
+
543
+
544
+
{e Binary trees} are nearly as fundamental as lists. They can provide
545
+
efficient storage and retrieval of information. In a binary tree, each node
546
+
is empty ({m Lf}), or is a branch ({m Br}) with a label and two subtrees.
547
+
548
+
OCaml lists are a datatype and could be declared as follows:
549
+
550
+
{@ocaml run-on=load[
551
+
# type 'a mylist =
552
+
| Nil
553
+
| Cons of 'a * 'a mylist;;
554
+
type 'a mylist = Nil | Cons of 'a * 'a mylist
555
+
]}
556
+
557
+
We could even declare [::] as an infix constructor. The only
558
+
thing we could not define is the [[...]] notation, which is
559
+
part of the OCaml grammar (although there does exist a mechanism
560
+
to use a {e similar} syntax for custom indexed datatypes).
561
+
562
+
A recursive type does not have to be polymorphic.
563
+
For example, here is a simple datatype of tree shapes with no attached data
564
+
that is recursive but not polymorphic.
565
+
566
+
{@ocaml run-on=load[
567
+
# type shape =
568
+
| Null
569
+
| Join of shape * shape;;
570
+
type shape = Null | Join of shape * shape
571
+
]}
572
+
573
+
The datatype ['a option] (mentioned above) is the opposite -- it is
574
+
polymorphic, but not recursive.
575
+
576
+
{1 Basic Properties of Binary Trees}
577
+
578
+
{@ocaml run-on=load[
579
+
# let rec count = function
580
+
| Lf -> 0 (* number of branch nodes *)
581
+
| Br (v, t1, t2) -> 1 + count t1 + count t2;;
582
+
val count : 'a tree -> int = <fun>
583
+
# let rec depth = function
584
+
| Lf -> 0 (* length of longest path *)
585
+
| Br (v, t1, t2) -> 1 + max (depth t1) (depth t2);;
586
+
val depth : 'a tree -> int = <fun>
587
+
]}
588
+
589
+
The invariant {m \texttt{count}(t)\le 2^{\texttt{depth}(t)} - 1} holds in the functions above.
590
+
591
+
Functions on trees are expressed recursively using pattern-matching. Both
592
+
functions above are analogous to \texttt\{length\} on lists. Here is a third
593
+
measure of a tree’s size:
594
+
595
+
{@ocaml run-on=load[
596
+
# let rec leaves = function
597
+
| Lf -> 1
598
+
| Br (v, t1, t2) -> leaves t1 + leaves t2;;
599
+
val leaves : 'a tree -> int = <fun>
600
+
]}
601
+
602
+
This function is redundant because of a basic fact about trees, which can be
603
+
proved by induction: for every tree {m t}, we have {m \texttt{leaves}(t) = \texttt{count}(t)+1}.
604
+
The inequality shown on the slide also has an elementary
605
+
proof by induction.
606
+
607
+
A tree of depth 20 can store {m 2^{20}-1} or approximately one million elements.
608
+
The access paths to these elements are short, particularly when compared with
609
+
a million-element list!
610
+
611
+
{1 Exercises}
612
+
613
+
{2 Exercise 6.1}
614
+
615
+
Give the declaration of an OCaml type for the days of the week. Comment on the practicality of such
616
+
a type in a calendar application.
617
+
618
+
{2 Exercise 6.2}
619
+
620
+
Write an OCaml function taking a binary tree labelled with integers and returning their sum.
621
+
622
+
{2 Exercise 6.3}
623
+
624
+
Using the definition of ['a tree] from before:
625
+
626
+
{@ocaml[
627
+
# type 'a tree = Lf | Br of 'a * 'a tree * 'a tree;;
628
+
type 'a tree = Lf | Br of 'a * 'a tree * 'a tree
629
+
]}
630
+
631
+
Examine the following function declaration. What does [ftree (1, n)] accomplish?
632
+
633
+
{@ocaml[
634
+
# let rec ftree k n =
635
+
if n = 0 then Lf
636
+
else Br (k, ftree (2 * k) (n - 1), ftree (2 * k + 1) (n - 1));;
637
+
val ftree : int -> int -> int tree = <fun>
638
+
]}
639
+
640
+
{2 Exercise 6.4}
641
+
642
+
Give the declaration of an OCaml type for arithmetic expressions that have the following
643
+
possibilities: floating-point numbers, variables (represented by strings), or expressions of the form {m -E},
644
+
{m E+E}, {m E\times E}.
645
+
646
+
{2 Exercise 6.5}
647
+
648
+
Continuing the previous exercise, write a function that evaluates an expression. If the expression
649
+
contains any variables, your function should raise an exception indicating the variable name.
+411
site/notebooks/foundations/foundations7.mld
+411
site/notebooks/foundations/foundations7.mld
···
1
+
{0 Lecture 7: Dictionaries and Functional Arrays}
2
+
3
+
{1 Dictionaries}
4
+
{ul {- lookup: find an item in the dictionary
5
+
}{- update (insert): replace (store) an item in the dictionary
6
+
}{- delete: remove an item from the dictionary
7
+
}{- empty: the null dictionary
8
+
}{- Missing: exception for errors in [lookup] and [delete]
9
+
}}
10
+
11
+
Ideally, an {e abstract type} should provide these operations and hide the internal data structures.
12
+
13
+
A dictionary attaches values to identifiers, called “keys”. Before choosing
14
+
the internal representation for a data structure, you need to specify the full
15
+
set of operations. In fact, here we only consider [update] (associating a
16
+
value with an identifier) and [lookup] (retrieving such a value). Deletion
17
+
is more difficult and would limit our choices. Some applications may need
18
+
additional operations, such as [merge] (combining two dictionaries). We
19
+
shall see that update can be done efficiently in a functional style, without
20
+
excessive copying.
21
+
22
+
An {e abstract type} provides specified operations while hiding low-level
23
+
details, such as the data structure used to represent dictionaries. Abstract
24
+
types can be declared in any modern programming language. Java’s {e objects}
25
+
serve this role, as do OCaml’s modules. This course does not cover modules, and we
26
+
simply declare the dictionary operations individually.
27
+
28
+
An {e association list} (a list of pairs) is the simplest dictionary representation.
29
+
Lookup is by linear search, and therefore slow: {m O(n)}. Association lists are
30
+
only usable if there are few keys in use. However, they are general in that the
31
+
keys do not need a concept of ordering, only equality.
32
+
33
+
{@ocaml[
34
+
# exception Missing
35
+
exception Missing
36
+
# let rec lookup a = function
37
+
| [] -> raise Missing
38
+
| (x, y) :: pairs ->
39
+
if a = x then y
40
+
else lookup a pairs
41
+
val lookup : 'a -> ('a * 'b) list -> 'b = <fun>
42
+
# let update (l, b, y) = (b, y) :: l
43
+
val update : ('a * 'b) list * 'a * 'b -> ('a * 'b) list = <fun>
44
+
]}
45
+
46
+
To enter a new [(key, value)] pair, simply “cons” it to the list with [update].
47
+
This takes constant time, which is the best we could hope for. But the space
48
+
requirement is huge: linear in the number of updates, not in the number of
49
+
distinct keys. Obsolete entries are never deleted: that would require first
50
+
finding them, increasing the update time from {m O(1)} to {m O(n)}.
51
+
52
+
{1 Binary Search Trees}
53
+
54
+
A {e dictionary} associates {e values} (here, numbers) with {e keys}.
55
+
56
+
{%html: <img src="binsearch.png" alt=""/>%}
57
+
58
+
Binary search trees are an important application of binary trees. They work
59
+
for keys that have a total ordering, such as strings. Each branch of the tree
60
+
carries a {m (key, value)} pair; its left subtree holds smaller keys; the right
61
+
subtree holds greater keys. If the tree remains reasonably balanced, then
62
+
update and lookup both take {m O(\log n)} for a tree of size {m n}. These times
63
+
hold in the average case; given random data, the tree is likely to remain
64
+
balanced.
65
+
66
+
At a given node, all keys in the left subtree are smaller (or equal) while all
67
+
trees in the right subtree are greater.
68
+
69
+
An unbalanced tree has a linear access time in the worst case. Examples
70
+
include building a tree by repeated insertions of elements in increasing or
71
+
decreasing order; there is a close resemblance to quicksort. Building a binary
72
+
search tree, then converting it to inorder, yields a sorting algorithm called
73
+
{e treesort}.
74
+
75
+
Self-balancing trees, such as Red-Black trees, attain {m O(\log n)} in the worst
76
+
case. They are complicated to implement.
77
+
78
+
{1 Lookup: Seeks Left or Right}
79
+
80
+
{@ocaml[
81
+
# exception Missing of string
82
+
exception Missing of string
83
+
# let rec lookup b = function
84
+
| Br ((a, x), t1, t2) ->
85
+
if b < a then
86
+
lookup b t1
87
+
else if a < b then
88
+
lookup b t2
89
+
else
90
+
x
91
+
| Lf -> raise (Missing b)
92
+
val lookup : string -> (string * 'a) tree -> 'a = <fun>
93
+
]}
94
+
95
+
This has guaranteed {m O(\log n)} access time {e if} the tree is balanced!
96
+
97
+
Lookup in the binary search tree goes to the left subtree if the desired
98
+
key is smaller than the current one and to the right if it is greater.
99
+
It raises [Missing] if it encounters an empty tree.
100
+
101
+
Since an ordering is involved, we have to declare the functions for a specific
102
+
type, here [string]. Now exception [Missing] mentions that type: if lookup
103
+
fails, the exception returns the missing key. The exception could be
104
+
eliminated using type [option] of our earlier Datatypes lecture, using the
105
+
constructor [None] for failure.
106
+
107
+
{1 Update}
108
+
109
+
{@ocaml[
110
+
# let rec update k v = function
111
+
| Lf -> Br ((k, v), Lf, Lf)
112
+
| Br ((a, x), t1, t2) ->
113
+
if k < a then
114
+
Br ((a, x), update k v t1, t2)
115
+
else if a < k then
116
+
Br ((a, x), t1, update k v t2)
117
+
else (* a = k *)
118
+
Br ((a, v), t1, t2)
119
+
val update : 'a -> 'b -> ('a * 'b) tree -> ('a * 'b) tree = <fun>
120
+
]}
121
+
122
+
This is also {m O(\log n)} as it copies the path only, and {e not whole subtrees!}
123
+
124
+
If you are familiar with the usual update operation for this sort of tree, you
125
+
may wonder whether it can be implemented in OCaml, where there is no direct way to
126
+
replace part of a data structure by something else.
127
+
128
+
The update operation is a nice piece of functional programming. It searches
129
+
in the same manner as [lookup], but the recursive calls reconstruct a
130
+
new tree around the result of the update. One subtree is updated and the
131
+
other left unchanged. The internal representation of trees ensures that
132
+
unchanged parts of the tree are not copied, but {e shared}.
133
+
Therefore, update copies only the path from the root to the new
134
+
node. Its time and space requirements, for a reasonably balanced tree, are
135
+
both {m O(\log n)}.
136
+
137
+
The comparison between {m b} and {m a} allows three cases:
138
+
{ul {- smaller: update the left subtree; share the right
139
+
}{- greater: update the right subtree; share the left
140
+
}{- equal: update the label and share both subtrees
141
+
}}
142
+
143
+
Note: in the function definition, [(* a = b*)] is a comment. Comments
144
+
in OCaml are enclosed in the brackets [(*] and [*)].
145
+
146
+
{1 Aside: Traversing Trees (3 Methods)}
147
+
148
+
{@ocaml[
149
+
# let rec preorder = function
150
+
| Lf -> []
151
+
| Br (v, t1, t2) ->
152
+
[v] @ preorder t1 @ preorder t2
153
+
val preorder : 'a tree -> 'a list = <fun>
154
+
# let rec inorder = function
155
+
| Lf -> []
156
+
| Br (v, t1, t2) ->
157
+
inorder t1 @ [v] @ inorder t2
158
+
val inorder : 'a tree -> 'a list = <fun>
159
+
# let rec postorder = function
160
+
| Lf -> []
161
+
| Br (v, t1, t2) ->
162
+
postorder t1 @ postorder t2 @ [v]
163
+
val postorder : 'a tree -> 'a list = <fun>
164
+
]}
165
+
166
+
{e Tree traversal} means examining each node of a tree in some order. {{: https://en.wikipedia.org/wiki/The_Art_of_Computer_Programming} D. E.
167
+
Knuth} has
168
+
identified three forms of tree traversal: preorder, inorder and
169
+
postorder. We can code these “visiting orders” as functions
170
+
that convert trees into lists of labels. Algorithms based on these notions
171
+
typically perform some action at each node; the functions above simply copy
172
+
the nodes into lists. Consider the tree:
173
+
174
+
{%html: <img src="bintree2.png" alt=""/>%}
175
+
{ul {- [preorder] visits the label first (“Polish notation”), yielding [ABDECFG]
176
+
}{- [inorder] visits the label midway, yielding [DBEAFCG]
177
+
}{- [postorder] visits the label last (“Reverse Polish”), yielding [DEBFGCA]. You might be familiar with this concept if you own an ancient RPN calculator!
178
+
}}
179
+
180
+
What is the use of [inorder]? Consider applying it to a binary search tree: the
181
+
result is a sorted list of pairs. We could use this, for example, to merge two
182
+
binary search trees. It is not difficult to transform a sorted list of pairs
183
+
into a binary search tree.
184
+
185
+
{1 Efficiently Traversing Trees}
186
+
187
+
Unfortunately, the functions shown on the previous slide are quadratic in the
188
+
worst case: the appends in the recursive calls are inefficient. To correct
189
+
that problem, we (as usual) add an accumulating argument. Observe how
190
+
each function constructs its result list and compare with how appends were
191
+
eliminated from [quicksort] in the Sorting lecture.
192
+
193
+
{@ocaml[
194
+
# let rec preord = function
195
+
| Lf, vs -> vs
196
+
| Br (v, t1, t2), vs ->
197
+
v :: preord (t1, preord (t2, vs))
198
+
val preord : 'a tree * 'a list -> 'a list = <fun>
199
+
# let rec inord = function
200
+
| Lf, vs -> vs
201
+
| Br (v, t1, t2), vs ->
202
+
inord (t1, v::inord (t2, vs))
203
+
val inord : 'a tree * 'a list -> 'a list = <fun>
204
+
# let rec postord = function
205
+
| Lf, vs -> vs
206
+
| Br (v, t1, t2), vs ->
207
+
postord (t1, postord (t2, v::vs))
208
+
val postord : 'a tree * 'a list -> 'a list = <fun>
209
+
]}
210
+
211
+
One can prove equations relating each of these functions to its counterpart on
212
+
the previous section. For example:
213
+
214
+
{math \texttt{inord}(t, vs) = \texttt{inorder}(t) @ vs }
215
+
216
+
These three types of tree traversal are related in that all are depth-first.
217
+
They each traverse the left subtree in full before traversing the right
218
+
subtree. Breadth-first search (from the Queues lecture) is another
219
+
possibility. That involves going through the levels of a tree one at a time.
220
+
221
+
{1 Arrays}
222
+
{ul {- A conventional array is an indexed storage area.
223
+
{ul {- It is updated {e in place} by the command [a.(k) <- x]
224
+
}{- The concept is inherently {e imperative}.
225
+
}}
226
+
}{- A {e functional array} is a finite map from integers to data.
227
+
{ul {- Updating implies {e copying} to return [update(A, k, x)]
228
+
}{- The new array equals [A] except that [A.(k) = x].
229
+
}}
230
+
}{- Can we do updates efficiently?
231
+
}}
232
+
233
+
The elements of a list can only be reached by counting from the front.
234
+
Elements of a tree are reached by following a path from the root. An
235
+
{e array} hides such structural matters; its elements are uniformly
236
+
designated by number. Immediate access to arbitrary parts of a data structure
237
+
is called {e random access}.
238
+
239
+
Arrays are the dominant data structure in conventional programming languages.
240
+
The ingenious use of arrays is the key to many of the great classical
241
+
algorithms, such as Hoare’s original quicksort (the partition step) and
242
+
Warshall’s transitive-closure algorithm.
243
+
244
+
The drawback is that subscripting is a chief cause of programmer error. That
245
+
is why arrays play little role in this introductory course.
246
+
247
+
Functional arrays are described below in order to illustrate another way of
248
+
using trees to organise data. Here is a summary of basic dictionary data
249
+
structures in order of decreasing generality and increasing efficiency:
250
+
{ul {- Linear search: Most general, needing only equality on keys, but inefficient: linear time.
251
+
}{- Binary search: Needs an ordering on keys. Logarithmic access time in the average case, but our binary search trees are linear in the worst case.
252
+
}{- Array subscripting: Least general, requiring keys to be integers, but even worst-case time is logarithmic.
253
+
}}
254
+
255
+
{1 Functional Arrays as Binary Trees}
256
+
257
+
The path to element {m i} follows the {e binary code} for {m i} (its “subscript”).
258
+
259
+
{%html: <img src="array1.png" alt=""/>%}
260
+
261
+
This simple representation (credited to W. Braun) ensures that the tree is
262
+
balanced. Complexity of access is always {m O(\log n)}, which is optimal. For
263
+
actual running time, access to conventional arrays is much faster: it requires
264
+
only a few hardware instructions. Array access is often taken to be {m O(1)},
265
+
which (as always) presumes that hardware limits are never exceeded.
266
+
267
+
The lower bound for array subscripts (or ``indices'') is one. The upper bound starts at zero
268
+
(which signifies the empty array) and can grow without limit. Inspection of
269
+
the diagram above should make it clear that these trees are always balanced:
270
+
the left subtree can have at most one node more than the right subtree,
271
+
recursively all the way down. (This assumes that the array is defined for
272
+
subscripts {m 1\ldots n} with no gaps; an array defined only for odd numbers, for
273
+
example, would obviously be unbalanced.)
274
+
275
+
The numbers in the diagram above are not the labels of branch nodes, but
276
+
indicate the positions of array elements. For example, the label corresponding
277
+
to {m A\[2\]} is at the position shown. The nodes of a functional array are
278
+
labelled with the data we want to store, not with these integers.
279
+
280
+
{1 The Lookup Function}
281
+
282
+
{@ocaml[
283
+
# exception Subscript
284
+
let rec sub = function
285
+
| Lf, _ -> raise Subscript (* Not found *)
286
+
| Br (v, t1, t2), k ->
287
+
if k = 1 then v
288
+
else if k mod 2 = 0 then
289
+
sub (t1, k / 2)
290
+
else
291
+
sub (t2, k / 2)
292
+
exception Subscript
293
+
val sub : 'a tree * int -> 'a = <fun>
294
+
# let rec sub = function (* Alternative implementation *)
295
+
| Lf, _ -> raise Subscript
296
+
| Br (v, t1, t2), 1 -> v
297
+
| Br (v, t1, t2), k when k mod 2 = 0 -> sub (t1, k / 2)
298
+
| Br (v, t1, t2), k -> sub (t2, k / 2)
299
+
val sub : 'a tree * int -> 'a = <fun>
300
+
]}
301
+
302
+
Notice that we have used a new keyword [when] above, which changes
303
+
pattern clauses to be only matched if the expression evalutes to true.
304
+
This can be equivalently expressed by moving the corresponding
305
+
checks into an [if] clause on the right hand side of the pattern match,
306
+
but is often more readable using [when] (as above).
307
+
308
+
The lookup function [sub], divides the subscript by 2 until 1 is
309
+
reached. If the remainder is 0 then the function follows the left subtree,
310
+
otherwise the right. If it reaches a leaf, it signals error by raising
311
+
exception [Subscript].
312
+
313
+
Array access can also be understood in terms of the subscript’s binary code.
314
+
Because the subscript must be a positive integer, in binary it has a leading
315
+
one. Discard this one and reverse the remaining bits. Interpreting zero
316
+
as {e left} and one as {e right} yields the path from the root to the
317
+
subscript.
318
+
319
+
Popular literature often explains the importance of binary as being led by
320
+
hardware: because a circuit is either on or off. The truth is almost the
321
+
opposite. Designers of digital electronics go to a lot of trouble to suppress
322
+
the continuous behaviour that would naturally arise. The real reason why
323
+
binary is important is its role in algorithms: an [if-then-else] decision leads
324
+
to binary branching.
325
+
326
+
Data structures, such as trees, and algorithms, such as mergesort, use binary
327
+
branching in order to reduce a cost from {m O(n)} to {m O(\log n)}. Two is the
328
+
smallest integer divisor that achieves this reduction. (Larger divisors are
329
+
only occasionally helpful, as in the case of B-trees, where they reduce the
330
+
constant factor.) The simplicity of binary arithmetic compared with decimal
331
+
arithmetic is just another instance of the simplicity of algorithms based on
332
+
binary choices.
333
+
334
+
{1 The Update Function}
335
+
336
+
{@ocaml[
337
+
# let rec update = function
338
+
| Lf, k, w ->
339
+
if k = 1 then
340
+
Br (w, Lf, Lf)
341
+
else
342
+
raise Subscript (* Gap in tree *)
343
+
| Br (v, t1, t2), k, w ->
344
+
if k = 1 then
345
+
Br (w, t1, t2)
346
+
else if k mod 2 = 0 then
347
+
Br (v, update (t1, k / 2, w), t2)
348
+
else
349
+
Br (v, t1, update (t2, k / 2, w))
350
+
val update : 'a tree * int * 'a -> 'a tree = <fun>
351
+
]}
352
+
353
+
The [update] function also divides the subscript repeatedly by two. When it
354
+
reaches a value of one, it has identified the element position. Then it
355
+
replaces the branch node by another branch with the new label.
356
+
357
+
A leaf may be replaced by a branch, extending the array, provided no
358
+
intervening nodes have to be generated. This suffices for arrays without gaps
359
+
in their subscripting. (The data structure can be modified to allow {e sparse}
360
+
arrays, where most subscript positions are undefined.) Exception [Subscript]
361
+
indicates that the subscript position does not exist and cannot be created.
362
+
This use of exceptions is not easily replaced by [None] and [Some].
363
+
364
+
Note that there are two tests involving {m k=1}. If we have reached a leaf,
365
+
it returns a branch, extending the array by one. If we are still at a branch
366
+
node, then the effect is to update an existing array element.
367
+
368
+
A similar function can {e shrink} an array by one.
369
+
370
+
{2 Exercise 7.1}
371
+
372
+
Draw the binary search tree that arises from successively inserting the following pairs into the
373
+
empty tree: [("Alice", 6)], [("Tobias", 2)], [("Gerald", 8)], [("Lucy", 9)]. Then repeat this
374
+
task using the order [("Gerald", 8)], [("Alice", 6)], [("Lucy", 9)], [("Tobias", 2)]. Why are
375
+
results different?
376
+
377
+
{2 Exercise 7.2}
378
+
379
+
Code an insertion function for binary search trees. It should resemble the existing [update]
380
+
function except that it should raise the exception [Collision] if the item to be inserted is already
381
+
present.
382
+
383
+
{2 Exercise 7.3}
384
+
385
+
Continuing the previous exercise, it would be natural for exceptional [Collision] to return the
386
+
value previously stored in the dictionary. Why is that goal difficult to achieve?
387
+
388
+
{2 Exercise 7.4}
389
+
390
+
Describe an algorithm for deleting an entry from a binary search tree. Comment on the suitability of
391
+
your approach.
392
+
393
+
{2 Exercise 7.5}
394
+
395
+
Code the delete function outlined in the previous exercise.
396
+
397
+
{2 Exercise 7.6}
398
+
399
+
Show that the functions [preorder], [inorder] and [postorder] all require {m O(n^2)} time in the worst
400
+
case, where {m n} is the size of the tree.
401
+
402
+
{2 Exercise 7.7}
403
+
404
+
Show that the functions [preord], [inord] and [postord] all take linear time in the size of the
405
+
tree.
406
+
407
+
{2 Exercise 7.8}
408
+
409
+
Write a function to remove the first element from a functional array. All the other elements are to
410
+
have their subscripts reduced by one. The cost of this operation should be linear in the size of the
411
+
array.
+525
site/notebooks/foundations/foundations8.mld
+525
site/notebooks/foundations/foundations8.mld
···
1
+
{0 Lecture 8: Functions as Values}
2
+
3
+
In OCaml, functions can be
4
+
{ul {- passed as arguments to other functions,
5
+
}{- returned as results,
6
+
}{- put into lists, trees, etc.,
7
+
}{- but {e not} tested for equality.
8
+
}}
9
+
10
+
{@ocaml[
11
+
# [(fun n -> n * 2);
12
+
(fun n -> n * 3);
13
+
(fun n -> n + 1)];;
14
+
- : (int -> int) list = [<fun>; <fun>; <fun>]
15
+
]}
16
+
17
+
Progress in programming languages can be measured by what abstractions they
18
+
admit. Conditional expressions (descended from conditional jumps based
19
+
on the sign of some numeric variable) and parametric types such as
20
+
{m \alpha,\texttt{list}} are examples. The idea that functions could be used
21
+
as values in a computation arose early, but it took some time before the idea
22
+
was fully realised. Many programming languages let functions be passed as
23
+
arguments to other functions, but few take the trouble needed to allow
24
+
functions to be returned as results.
25
+
26
+
In mathematics, a {e functional} or {e higher-order function} is a
27
+
function that operates on other functions. Many functionals are familiar from
28
+
mathematics: for example, the differential operator maps functions to their
29
+
derivatives, which are also functions. To a mathematician, a function is
30
+
typically an infinite, uncomputable object. We use OCaml functions to represent
31
+
algorithms. Sometimes they represent infinite collections of data given by
32
+
computation rules.
33
+
34
+
Functions cannot be compared for equality. We could compare the machine
35
+
addresses of the compiled code, but that would merely be a test of identity: it
36
+
would regard any two separate functions as unequal even if they were compiled
37
+
from identical pieces of source code. Such a low-level feature has no place in
38
+
a principled language.
39
+
40
+
{1 Functions Without Names}
41
+
42
+
If functions are to be regarded as computational values, then we need a
43
+
notation for them. The [fun] notation expresses a non-recursive function
44
+
value without giving the function a name.
45
+
46
+
{m \tt fun\;x\;\rightarrow E} is the function {m f} such that {m f(x)=E}.
47
+
The function [fun n -> n*2] is a {e doubling function}.
48
+
49
+
{@ocaml[
50
+
# fun n -> n * 2;;
51
+
- : int -> int = <fun>
52
+
]}
53
+
54
+
The main purpose of [fun]-notation is to package up small expressions that are to be
55
+
applied repeatedly using some other function.
56
+
The expression [fun n -> n*2] has the same value as the identifier
57
+
[double], declared as follows:
58
+
59
+
{@ocaml[
60
+
# let double n = n * 2;;
61
+
val double : int -> int = <fun>
62
+
]}
63
+
64
+
The [fun] notation can also do pattern matching, and the [function] keyword
65
+
adds an anonymous variable name to pattern match against. The following functions
66
+
are all equivalent, with the latter definitions bound to the [is_zero] value and the earlier ones anonymous:
67
+
68
+
{@ocaml[
69
+
# fun x -> match x with 0 -> true | _ -> false;;
70
+
- : int -> bool = <fun>
71
+
]}
72
+
73
+
{1 Curried Functions}
74
+
75
+
A {e curried function} returns another function as its result. We use
76
+
the string concetenation operator [(^)] to illustrate how this works.
77
+
78
+
{@ocaml[
79
+
# (^);;
80
+
- : string -> string -> string = <fun>
81
+
]}
82
+
83
+
A short form for the definition of [prefix] is simply to pass multiple
84
+
arguments to the function definition. The following two definitions
85
+
are equivalent in OCaml:
86
+
87
+
{@ocaml[
88
+
# let prefix = fun a -> fun b -> a ^ b;;
89
+
val prefix : string -> string -> string = <fun>
90
+
]}
91
+
92
+
Currying is the technique of expressing a function taking multiple arguments as nested functions, each taking a single argument.
93
+
The [fun]-notation lets us package [n*2] as the function
94
+
[fun n -> n * 2], but what if there are several variables, as in
95
+
[fun n -> n * 2 + k]? A function of two arguments could be coded using
96
+
pattern-matching on pairs, writing [fun (n, k) -> n * 2 + k].
97
+
98
+
Currying is an alternative, where we {e nest} the [fun]-notation:
99
+
100
+
{@ocaml[
101
+
# fun k -> fun n -> n * 2 + k;;
102
+
- : int -> int -> int = <fun>
103
+
]}
104
+
105
+
Applying this curried function to the argument 1 yields another function, in which [k] has been replaced by 1:
106
+
107
+
{@ocaml[
108
+
# let fn = fun k -> fun n -> n * 2 + k;;
109
+
val fn : int -> int -> int = <fun>
110
+
]}
111
+
112
+
And this function, when applied to 3, yields the result 7. The two arguments are supplied one after another.
113
+
114
+
The example on the slide is similar but refers to the expression [a^b],
115
+
where [^] is the infix operator for string concatenation. Function [promote] binds the first argument of [prefix] to
116
+
["Professor"]; the resulting function prefixes that title
117
+
to any string to which it is applied.
118
+
119
+
{1 Shorthand for Curried Functions}
120
+
121
+
A function-returning function is just a function of two arguments.
122
+
123
+
This curried function syntax is nicer than nested [fun] binders:
124
+
125
+
{@ocaml[
126
+
# let prefix a b = a ^ b;;
127
+
val prefix : string -> string -> string = <fun>
128
+
]}
129
+
130
+
Curried functions allows {e partial application} (to the first argument).
131
+
132
+
In OCaml, an {m n}-argument curried function [f] can be declared using the syntax:
133
+
134
+
{math \tt let \; f \; x_1 \: \ldots \: x_{n} \: = \: E}
135
+
136
+
and applied using the syntax:
137
+
138
+
{math \tt \; E_1 \; \ldots \; E_n }
139
+
140
+
If [f] is not recursive, then it is equivalent to the function expressed via nesting as follows:
141
+
142
+
{math \tt fun \; x_1 \; \rightarrow \cdots \rightarrow fun \; x_{n} \rightarrow E }
143
+
144
+
We now have two ways of expressing functions of multiple arguments: either by
145
+
passing a pair of arguments or by currying. Currying allows {e partial application}
146
+
which is useful when fixing the first argument yields a function
147
+
that is interesting in its own right. An example from mathematics is the
148
+
function {m x^y}, where fixing {m y=2} yields a function in {m x} alone, namely
149
+
squaring. Similarly, {m y=3} yields cubing, while {m y=1} yields the identity
150
+
function.
151
+
152
+
Though the function [hd] (which returns the head of a list) is not
153
+
curried, it may be used with the curried application syntax in some
154
+
expressions:
155
+
156
+
{@ocaml[
157
+
# List.hd [dub; promote] "Hamilton";;
158
+
Line 1, characters 9-12:
159
+
Error: Unbound value dub
160
+
]}
161
+
162
+
Here [List.hd] is applied to a list of functions, and the resulting function
163
+
[dub] is then applied to the string ["Hamilton"]. The idea of
164
+
executing code stored in data structures reaches its full development in
165
+
{e object-oriented} programming, like in Java.
166
+
167
+
{1 Partial Application: A Curried Insertion Sort}
168
+
169
+
{@ocaml[
170
+
# let insort lessequal =
171
+
let rec ins x = function
172
+
| [] -> [x]
173
+
| y::ys -> if lessequal x y then x :: y :: ys
174
+
else y :: ins x ys
175
+
in
176
+
let rec sort = function
177
+
| [] -> []
178
+
| x::xs -> ins x (sort xs)
179
+
in
180
+
sort;;
181
+
val insort : ('a -> 'a -> bool) -> 'a list -> 'a list = <fun>
182
+
]}
183
+
184
+
The sorting functions we discussed in earlier lectures are coded to sort floating-point
185
+
numbers. They can be generalised to an arbitrary ordered type by passing the
186
+
ordering predicate [lessequal] as an argument.
187
+
188
+
Functions [ins] and [sort] are declared locally, referring to [lessequal].
189
+
Though it may not be obvious, [insort] is a curried function. Given its first
190
+
argument, a predicate for comparing some particular type of items, it returns
191
+
the function [sort] for sorting lists of that type of items.
192
+
193
+
Some examples of its use:
194
+
195
+
{@ocaml[
196
+
# insort (<=) [5; 3; 9; 8];;
197
+
- : int list = [3; 5; 8; 9]
198
+
]}
199
+
200
+
An obscure point: the syntax [(<=)] denotes the comparison operator as a
201
+
function, which is then given to [insort]. Passing the relation {m \geq} for
202
+
[lessequal] gives a decreasing sort. This is no coding trick; it is justified
203
+
in mathematics, since if {m \leq} is a partial ordering then so is {m \geq}.
204
+
205
+
{1 map: the “Apply to All” Function}
206
+
207
+
{@ocaml[
208
+
# let rec map f = function
209
+
| [] -> []
210
+
| x::xs -> (f x) :: map f xs;;
211
+
val map : ('a -> 'b) -> 'a list -> 'b list = <fun>
212
+
]}
213
+
214
+
The functional [map] applies a function to every element of a list,
215
+
returning a list of the function’s results. “Apply to all” is a fundamental
216
+
operation and we shall see several applications of it below. We
217
+
again see the advantages of [fun]-notation, currying and
218
+
[map]. If we did not have them, the first use of [map] in the above code block
219
+
would require a preliminary function declaration:
220
+
221
+
{@ocaml[
222
+
# let rec sillylist = function
223
+
| [] -> []
224
+
| s::ss -> (s ^ "ppy") :: sillylist ss;;
225
+
val sillylist : string list -> string list = <fun>
226
+
]}
227
+
228
+
An expression containing several applications of functionals---such as our
229
+
second example---can abbreviate a long series of declarations. Sometimes this
230
+
coding style is cryptic, but it can be clear as crystal. Treating functions
231
+
as values lets us capture common program structures once and for all.
232
+
233
+
In the second example, [double] is the obvious integer doubling function we
234
+
defined earlier. Note that \texttt\{map\} is a built-in OCaml function in the
235
+
form of [List.map]. OCaml’s standard library includes, among much else, many
236
+
list functions.
237
+
238
+
{1 Example: Matrix Transpose}
239
+
240
+
{math
241
+
\begin{pmatrix}
242
+
a & b & c \\
243
+
d & e & f
244
+
\end{pmatrix}^T =
245
+
\begin{pmatrix}
246
+
a & d \\
247
+
b & e \\
248
+
c & f
249
+
\end{pmatrix}
250
+
}
251
+
252
+
{@ocaml[
253
+
# let rec transp = function
254
+
| []::_ -> []
255
+
| rows -> (map List.hd rows) ::
256
+
(transp (map List.tl rows));;
257
+
val transp : 'a list list -> 'a list list = <fun>
258
+
]}
259
+
260
+
A matrix can be viewed as a list of rows, each row a list of matrix elements.
261
+
This representation is not especially efficient compared with the conventional
262
+
one (using arrays). Lists of lists turn up often, though, and we can see how
263
+
to deal with them by taking familiar matrix operations as examples.
264
+
{e ML for the Working Programmer} goes as far as Gaussian elimination,
265
+
which presents surprisingly few difficulties.
266
+
267
+
The transpose of the matrix
268
+
{math \left(\begin{smallmatrix} a & b & c \\
269
+
d & e & f\end{smallmatrix}\right)}
270
+
is
271
+
{math \left(\begin{smallmatrix}
272
+
a & d \\
273
+
b & e \\
274
+
c & f
275
+
\end{smallmatrix}\right)}
276
+
which in OCaml corresponds to the following transformation on lists of lists:
277
+
278
+
{math
279
+
\begin{aligned}
280
+
\text{[[a; b; c]; [d; e; f]]} \Rightarrow& \text{ [[a; d]; [b; e]; [c; f]]}
281
+
\end{aligned}
282
+
}
283
+
284
+
The workings of function [transp] are simple. If [rows] is the
285
+
matrix to be transposed, then [map hd] extracts its first column and
286
+
[map tl] extracts its second column:
287
+
288
+
{math
289
+
\begin{aligned}
290
+
\text{map hd rows} \Rightarrow & \text{ [a; d]}\\
291
+
\text{map tl rows} \Rightarrow & \text{ [[b; c]; [e; f]]}
292
+
\end{aligned}
293
+
}
294
+
295
+
A recursive call transposes the latter matrix, which is then given the column
296
+
[[a; d]] as its first row.
297
+
The two functions expressed using [map] would otherwise have to be declared
298
+
separately.
299
+
300
+
{1 Review of Matrix Multiplication}
301
+
302
+
{math
303
+
\begin{pmatrix} A_1 & \cdots & A_k \end{pmatrix} \cdot
304
+
\begin{pmatrix}
305
+
B_1 \\ \vdots \\ B_k
306
+
\end{pmatrix} =
307
+
\begin{pmatrix}
308
+
A_1 B_1 + \cdots + A_k B_k
309
+
\end{pmatrix}
310
+
}
311
+
312
+
The right side is the {e vector dot product} {m \vec\{A\}\cdot \vec\{B\}}.
313
+
Repeat for each {e row} of {m A} and {e column} of {m B}.
314
+
315
+
The {e dot product} of two vectors is
316
+
{math (a_1,\ldots,a_k) \cdot (b_1,\ldots,b_k) = a_1b_1 + \cdots + a_kb_k }
317
+
318
+
A simple case of matrix multiplication is when {m A} consists of a single row
319
+
and {m B} consists of a single column. Provided {m A} and {m B} contain the same
320
+
number {m k} of elements, multiplying them yields a {m 1\times1} matrix whose
321
+
single element is the dot product shown above.
322
+
323
+
If {m A} is an {m m\times k} matrix and {m B} is a {m k\times n} matrix
324
+
then {m A\times B} is an {m m\times n} matrix.
325
+
For each {m i} and {m j}, the {m (i,j)} element of {m A\times B} is the dot
326
+
product of row {m i} of {m A} with column {m j} of {m B}.
327
+
328
+
{math
329
+
\begin{pmatrix}
330
+
2 & 0 \\
331
+
3 &-1 \\
332
+
0 & 1 \\
333
+
1 & 1
334
+
\end{pmatrix}
335
+
\begin{pmatrix}
336
+
1 & 0 & 2 \\
337
+
4 &-1 & 0
338
+
\end{pmatrix} =
339
+
\begin{pmatrix}
340
+
2 & 0 & 4 \\
341
+
-1 & 1 & 6 \\
342
+
4 &-1 & 0 \\
343
+
5 &-1 & 2
344
+
\end{pmatrix}
345
+
}
346
+
347
+
The (1, 1) element above is computed by
348
+
{math (2,0)\cdot(1,4) = 2\times1 + 0\times4 = 2. }
349
+
350
+
Coding matrix multiplication in a conventional programming language usually
351
+
involves three nested loops. It is hard to avoid mistakes in the subscripting,
352
+
which often runs slowly due to redundant internal calculations.
353
+
354
+
{1 Matrix Multiplication in OCaml}
355
+
356
+
{e Dot product} of two vectors---a {e curried function}
357
+
358
+
{@ocaml[
359
+
# let rec dotprod xs ys =
360
+
match xs, ys with
361
+
| [], [] -> 0.0
362
+
| x::xs, y::ys -> (x *. y) +. (dotprod xs ys);;
363
+
Lines 2-4, characters 4-50:
364
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
365
+
Here is an example of a case that is not matched:
366
+
([], _::_)
367
+
val dotprod : float list -> float list -> float = <fun>
368
+
]}
369
+
370
+
{e Matrix product}
371
+
372
+
{@ocaml[
373
+
# let rec matprod arows brows =
374
+
let cols = transp brows in
375
+
map (fun row -> map (dotprod row) cols) arows;;
376
+
val matprod : float list list -> float list list -> float list list = <fun>
377
+
]}
378
+
379
+
The [transp brows] converts {m B} into a list of columns, yielding a
380
+
list whose elements are the columns of {m B}. Each row of {m A\times B} is
381
+
obtained by multiplying a row of {m A} by the columns of {m B}.
382
+
383
+
Because [dotprod] is curried, it can be applied to a row of {m A}. The
384
+
resulting function is applied to all the columns of {m B}. We have another
385
+
example of currying and partial application.
386
+
387
+
The outer [map] applies [dotprod] to each row of {m A}. The inner
388
+
[map], using [fun]-notation, applies [dotprod row] to each
389
+
column of {m B}. Compare with the version in {e ML for the Working
390
+
Programmer} (page 89) which does not use [map] and requires two
391
+
additional function declarations.
392
+
393
+
In the dot product function, the two vectors must have the same length.
394
+
Otherwise, exception [Match_failure] is raised.
395
+
396
+
{1 List Functionals for Predicates}
397
+
398
+
{@ocaml[
399
+
# let rec exists p = function
400
+
| [] -> false
401
+
| x::xs -> (p x) || (exists p xs);;
402
+
val exists : ('a -> bool) -> 'a list -> bool = <fun>
403
+
]}
404
+
405
+
A {e predicate} is a {e boolean-valued} function.
406
+
407
+
The functional [exists] transforms a predicate into a predicate over
408
+
lists. Given a list, [exists p] tests whether or not some list element
409
+
satisfies [p] (making it return [true]). If it finds one, it stops
410
+
searching immediately, thanks to the behaviour of the lazy [||] operator.
411
+
412
+
Dually, we have a functional to test whether all list elements satisfy the
413
+
predicate. If it finds a counterexample then it, too, stops searching.
414
+
415
+
{@ocaml[
416
+
# let rec all p = function
417
+
| [] -> true
418
+
| x::xs -> (p x) && all p xs;;
419
+
val all : ('a -> bool) -> 'a list -> bool = <fun>
420
+
]}
421
+
422
+
The [filter] functional, like [map], transforms lists. It applies a
423
+
predicate to all the list elements, but instead of returning the
424
+
resulting values (which could only be [true] or [false]), it returns
425
+
the list of elements satisfying the predicate.
426
+
427
+
{1 Applications of the Predicate Functionals}
428
+
429
+
{@ocaml[
430
+
# let member y xs =
431
+
exists (fun x -> x=y) xs;;
432
+
val member : 'a -> 'a list -> bool = <fun>
433
+
]}
434
+
435
+
{e Testing whether two lists have no common elements}
436
+
437
+
{@ocaml[
438
+
# let disjoint xs ys =
439
+
all (fun x -> all (fun y -> x<>y) ys) xs;;
440
+
val disjoint : 'a list -> 'a list -> bool = <fun>
441
+
]}
442
+
443
+
The Lists lecture presented the function [member], which tests whether a
444
+
specified value can be found as a list element, and [inter], which returns the
445
+
“intersection” of two lists: the list of elements they have in common.
446
+
447
+
But remember: the purpose of list functionals is not to replace the
448
+
declarations of popular functions, which probably are available already. It is
449
+
to eliminate the need for separate declarations of ad-hoc functions. When they
450
+
are nested, like the calls to \texttt\{all\} in \texttt\{disjoint\} above, the
451
+
inner functions are almost certainly one-offs, not worth declaring separately.
452
+
453
+
Our primitives themselves can be seen as a programming language. Part of the
454
+
task of programming is to extend our programming language with notation for
455
+
solving the problem at hand. The levels of notation that we define should
456
+
correspond to natural levels of abstraction in the problem domain.
457
+
458
+
Historical Note:
459
+
Alonzo Church’s {m \lambda}-calculus gave a simple syntax, {m \lambda}-notation,
460
+
for expressing functions. It is the direct precursor of OCaml’s
461
+
[fun]-notation. It was soon shown that his system was equivalent in
462
+
computational power to Turing machines, and {e Church’s thesis} states that
463
+
this defines precisely the set of functions that can be computed effectively.
464
+
465
+
The {m \lambda}-calculus had a tremendous influence on the design of functional
466
+
programming languages. McCarthy’s Lisp was something of a false start; it
467
+
interpreted variable binding incorrectly, an error that stood for some 20
468
+
years. But in 1966, Peter Landin (of Queen Mary College, University of London)
469
+
sketched out the main features of functional languages.
470
+
471
+
{2 Exercise 8.1}
472
+
473
+
What does the following function do, and what are its uses?
474
+
475
+
{@ocaml[
476
+
# let sw f x y = f y x;;
477
+
val sw : ('a -> 'b -> 'c) -> 'b -> 'a -> 'c = <fun>
478
+
]}
479
+
480
+
{2 Exercise 8.2}
481
+
482
+
There are many ways of combining orderings. The [lexicographic ordering] uses two keys for
483
+
comparisons. It is specified by
484
+
{math (x',y')<(x,y)\iff x'<x \vee (x'=x \wedge y'<y).}
485
+
Write an OCaml function to lexicographically combine two orderings, supplied as functions. Explain
486
+
how it allows function [insort] to sort a list of pairs.
487
+
488
+
{2 Exercise 8.3}
489
+
490
+
Without using [map] write a function [map2] such that [map2 f] is equivalent to [map (map f)]. The
491
+
obvious solution requires declaring two recursive functions. Try to get away with only one by
492
+
exploiting nested pattern-matching.
493
+
494
+
{2 Exercise 8.4}
495
+
496
+
The type ['a option], declared below, can be viewed as a type of lists having at most one element.
497
+
(It is typically used as an alternative to exceptions.) Declare an analogue of the function [map]
498
+
for type ['a option].
499
+
500
+
{@ocaml[
501
+
# type 'a option = None | Some of 'a;;
502
+
type 'a option = None | Some of 'a
503
+
]}
504
+
505
+
{2 Exercise 8.5}
506
+
507
+
Recall the making change function of Lecture 4:
508
+
509
+
{@ocaml[
510
+
# let rec change till amt =
511
+
match till, amt with
512
+
| _ , 0 -> [ [] ]
513
+
| [] , _ -> []
514
+
| c::till , amt -> if amt < c then change till amt
515
+
else let rec allc = function
516
+
| [] -> []
517
+
| cs :: css -> (c::cs) :: allc css
518
+
in
519
+
allc (change (c::till) (amt - c)) @
520
+
change till amt;;
521
+
val change : int list -> int -> int list list = <fun>
522
+
]}
523
+
524
+
Function [allc] applies the function ‘cons a [c]’ to every element of a list. Eliminate it by
525
+
declaring a curried cons function and applying [map].
+364
site/notebooks/foundations/foundations9.mld
+364
site/notebooks/foundations/foundations9.mld
···
1
+
{0 Lecture 9: Sequences, or Lazy Lists}
2
+
3
+
{1 A Pipeline}
4
+
5
+
{math
6
+
\boxed{Producer} \to \boxed{Filter} \to\cdots\to
7
+
\boxed{Filter} \to \boxed{Consumer}
8
+
}
9
+
{ul {- Produce sequence of items
10
+
}{- Filter sequence in stages
11
+
}{- Consume results as needed
12
+
}{- {e Lazy lists} join the stages together
13
+
}}
14
+
15
+
Two types of program can be distinguished. A sequential program
16
+
accepts a problem to solve, processes for a while, and finally terminates
17
+
with its result. A typical example is the huge numerical simulations that are
18
+
run on supercomputers. Most of our OCaml functions also fit this model.
19
+
20
+
At the other extreme are {e reactive} programs, whose job is to interact
21
+
with the environment. They communicate constantly during their operation and
22
+
run for as long as is necessary. A typical example is the software that
23
+
controls many modern aircraft. Reactive programs often consist of
24
+
{e concurrent processes} running at the same time and communicating with
25
+
one another.
26
+
27
+
Concurrency is too difficult to consider in this course, but we can model
28
+
simple pipelines such as that shown above. The {e Producer} represents one
29
+
or more sources of data, which it outputs as a stream. The {e Filter}
30
+
stages convert the input stream to an output stream, perhaps consuming several
31
+
input items to yield a single output item. The {e Consumer} takes as many
32
+
elements as necessary.
33
+
34
+
The Consumer drives the pipeline: nothing is computed except in response to
35
+
its demand for an additional datum. Execution of the Filter stages is
36
+
interleaved as required for the computation to go through. The programmer
37
+
sets up the data dependencies but has no clear idea of what happens when. We
38
+
have the illusion of concurrent computation.
39
+
40
+
The Unix operating system provides similar ideas through its {e pipes} that
41
+
link processes together. In OCaml, we can model pipelines using {e lazy lists}.
42
+
43
+
{1 Lazy Lists (or Streams)}
44
+
{ul {- Lists of possibly {e infinite} length
45
+
}{- Elements {e computed upon demand}
46
+
}{- {e Avoids waste} if there are many solutions
47
+
}{- {e Infinite} values are a useful abstraction
48
+
}}
49
+
50
+
In OCaml, we can implement laziness by {e delaying evaluation} of the tail of
51
+
the list.
52
+
53
+
Lazy lists have practical uses. Some algorithms, like making change, can
54
+
yield many solutions when only a few are required. Sometimes the original
55
+
problem concerns infinite series: with lazy lists, we can pretend they really
56
+
exist!
57
+
58
+
We are now dealing with {e infinite} (or at least unbounded) computations.
59
+
A potentially infinite source of data is processed one element at a time, upon
60
+
demand. Such programs are harder to understand than terminating ones and have
61
+
more ways of going wrong.
62
+
63
+
Some purely functional languages, such as Haskell, use lazy evaluation
64
+
everywhere. Even the if-then-else construct can be a function, and all lists
65
+
are lazy. In OCaml, we can declare a type of lists such that evaluation of the
66
+
tail does not occur until demanded. {e Delayed} evaluation is weaker than
67
+
{e lazy} evaluation, but it is good enough for our purposes and often the
68
+
best compromise for performance and memory usage.
69
+
70
+
The traditional word “stream” is reserved in OCaml parlance for
71
+
input/output channels. Let us call lazy lists {e sequences} instead.
72
+
73
+
{1 Lazy Lists in OCaml}
74
+
{ul {- The empty tuple [()] and its {e type} [unit]
75
+
}{- Delayed version of {m E} is [fun () -> E]
76
+
}}
77
+
78
+
{@ocaml[
79
+
# type 'a seq =
80
+
| Nil
81
+
| Cons of 'a * (unit -> 'a seq);;
82
+
type 'a seq = Nil | Cons of 'a * (unit -> 'a seq)
83
+
# let head (Cons (x, _)) = x;;
84
+
Line 1, characters 9-26:
85
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
86
+
Here is an example of a case that is not matched:
87
+
Nil
88
+
val head : 'a seq -> 'a = <fun>
89
+
# let tail (Cons (_, xf)) = xf ();;
90
+
Line 1, characters 9-31:
91
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
92
+
Here is an example of a case that is not matched:
93
+
Nil
94
+
val tail : 'a seq -> 'a seq = <fun>
95
+
]}
96
+
97
+
{m \tt Cons(x, xf)} has {e head} {m x} and {e tail function} {m xf}
98
+
99
+
The primitive OCaml type [unit] has one element, which is
100
+
written [()]. This element may be regarded as a 0-tuple, and
101
+
[unit] as the nullary Cartesian product. (Think of the connection
102
+
between multiplication and the number 1.)
103
+
104
+
The empty tuple serves as a placeholder in situations where no information is
105
+
required. It may:
106
+
{ul {- appear in a data structure. For example, a [unit]-valued dictionary represents a set of keys.
107
+
}{- be the argument of a function, where its effect is to {e delay evaluation}.
108
+
}{- be the argument or result of a procedure. (see the Procedural Programming section)
109
+
}}
110
+
111
+
The empty tuple, like all tuples, is a constructor and is allowed in patterns;
112
+
for example: [let f () = ...]
113
+
114
+
In particular {m \tt fun , () \rightarrow E} is the function that takes an argument of
115
+
type [unit] and returns the value of {m E} as its result. Expression {m E}
116
+
is not evaluated until the function is called, even though the only possible
117
+
argument is [()]. The function simply delays the evaluation of {m E}.
118
+
119
+
{1 The Infinite Sequence: {m k}, {m k+1}, {m k+2}, …}
120
+
121
+
{@ocaml[
122
+
# let rec from k = Cons (k, fun () -> from (k+1));;
123
+
val from : int -> int seq = <fun>
124
+
# let it = from 1;;
125
+
val it : int seq = Cons (1, <fun>)
126
+
# let it = tail it;;
127
+
val it : int seq = Cons (2, <fun>)
128
+
# let it = tail it;;
129
+
val it : int seq = Cons (3, <fun>)
130
+
]}
131
+
132
+
Function [from] constructs the infinite sequence of integers starting
133
+
from {m k}. Execution terminates because of the [fun] enclosing the
134
+
recursive call. OCaml displays the tail of a sequence as [fun], which
135
+
stands for some function value. Each call to [tail] generates the next
136
+
sequence element. We could do this forever.
137
+
138
+
This example is of little practical value because the cost of computing a
139
+
sequence element will be dominated by that of creating the dummy function.
140
+
Lazy lists tend to have high overheads.
141
+
142
+
{1 Consuming a Sequence}
143
+
144
+
{@ocaml[
145
+
# let rec get n s =
146
+
match n, s with
147
+
| 0, _ -> []
148
+
| n, Nil -> []
149
+
| n, Cons (x, xf) -> x :: get (n-1) (xf ());;
150
+
val get : int -> 'a seq -> 'a list = <fun>
151
+
]}
152
+
153
+
The above code gets the first {m n} elements as a list.
154
+
[xf ()] {e forces} evaluation.
155
+
156
+
The function [get] converts a sequence to a list. It takes the
157
+
first {m n} elements; it takes all of them if {m n<0}, which can terminate only if
158
+
the sequence is finite.
159
+
160
+
In the last line of [get], the expression [xf()] calls the tail
161
+
function, demanding evaluation of the next element. This operation is called
162
+
{e forcing} the sequence.
163
+
164
+
{1 Sample Evaluation}
165
+
166
+
{math
167
+
\begin{aligned}
168
+
\tt get(2,\, from \; 6) \\
169
+
\tt get(2,\, Cons(6, fun \; () \rightarrow from \; (6+1))) \\
170
+
\tt 6 :: get(1,\, from \; (6+1)) \\
171
+
\tt 6 :: get(1,\, Cons \; (7,\, fun \; () \rightarrow from \; (7+1))) \\
172
+
\tt 6 :: 7 :: get(0,\, Cons \; (8,\, fun \; () \rightarrow from \; (8+1))) \\
173
+
\tt 6 :: 7 :: [] \\
174
+
\tt [6; 7]
175
+
\end{aligned}
176
+
}
177
+
178
+
Here we ask for two elements of the infinite sequence. In fact, three
179
+
elements are computed: 6, 7 and 8. Our implementation is slightly too eager.
180
+
A more complicated [type] declaration could avoid this problem.
181
+
Another problem is that if one repeatedly examines some particular list
182
+
element using forcing, that element is repeatedly evaluated. In a lazy
183
+
programming language, the result of the first evaluation would be stored for
184
+
later reference. To get the same effect in OCaml requires the use of
185
+
references.
186
+
187
+
We should be grateful that the potentially infinite computation is kept
188
+
finite. The tail of the original sequence even contains the unevaluated
189
+
expression 6+1.
190
+
191
+
{1 Joining Two Sequences}
192
+
193
+
{@ocaml[
194
+
# let rec appendq xq yq =
195
+
match xq with
196
+
| Nil -> yq
197
+
| Cons (x, xf) -> Cons(x, fun () -> appendq (xf ()) yq);;
198
+
val appendq : 'a seq -> 'a seq -> 'a seq = <fun>
199
+
]}
200
+
201
+
A more fair alternative:
202
+
203
+
{@ocaml[
204
+
# let rec interleave xq yq =
205
+
match xq with
206
+
| Nil -> yq
207
+
| Cons (x, xf) -> Cons (x, fun () -> interleave yq (xf ()));;
208
+
val interleave : 'a seq -> 'a seq -> 'a seq = <fun>
209
+
]}
210
+
211
+
Most list functions and functionals have analogues on sequences, but strange
212
+
things can happen. Can an infinite list be reversed?
213
+
214
+
Function [appendq] is precisely the same idea as [append]
215
+
from the Lists lecture; it concatenates two sequences. If the first
216
+
argument is infinite, then [appendq] never gets to its second argument,
217
+
which is lost. Concatenation of infinite sequences is not terribly
218
+
interesting.
219
+
220
+
The function [interleave] avoids this problem by exchanging the two
221
+
arguments in each recursive call. It combines the two lazy lists, losing no
222
+
elements. Interleaving is the right way to combine two potentially infinite
223
+
information sources into one.
224
+
225
+
In both function declarations, observe that each [xf ()] is enclosed
226
+
within a {m \{\tt fun () \rightarrow \ldots\}}. Each {e force} is enclosed within a
227
+
{e delay}. This practice makes the functions lazy. A force not enclosed
228
+
in a delay, as in [get] above, runs the risk of evaluating the sequence
229
+
in full.
230
+
231
+
{1 Functionals for Lazy Lists}
232
+
233
+
Filtering lazy lists:
234
+
235
+
{@ocaml[
236
+
# let rec filterq p = function
237
+
| Nil -> Nil
238
+
| Cons (x, xf) ->
239
+
if p x then
240
+
Cons (x, fun () -> filterq p (xf ()))
241
+
else
242
+
filterq p (xf ());;
243
+
val filterq : ('a -> bool) -> 'a seq -> 'a seq = <fun>
244
+
]}
245
+
246
+
The infinite sequence {m x}, {m f(x)}, {m f(f(x))}, …
247
+
248
+
{@ocaml[
249
+
# let rec iterates f x =
250
+
Cons (x, fun () -> iterates f (f x));;
251
+
val iterates : ('a -> 'a) -> 'a -> 'a seq = <fun>
252
+
]}
253
+
254
+
The functional [filterq] demands elements of [xq] until it finds
255
+
one satisfying [p]. (Recall [filter], the analogous operation for ordinary lists.) It
256
+
contains a {e force} not protected by a {e delay}. If [xq] is
257
+
infinite and contains no satisfactory element, then [filtering] runs
258
+
forever.
259
+
260
+
The functional [iterates] generalises [from]. It creates the
261
+
next element not by adding one but by calling the function [f].
262
+
263
+
{1 Numerical Computations on Infinite Sequences}
264
+
265
+
{@ocaml[
266
+
# let next a x = (a /. x +. x) /. 2.0;;
267
+
val next : float -> float -> float = <fun>
268
+
]}
269
+
270
+
Close enough?
271
+
272
+
{@ocaml[
273
+
# let rec within eps = function
274
+
| Cons (x, xf) ->
275
+
match xf () with
276
+
| Cons (y, yf) ->
277
+
if abs_float (x -. y) <= eps then y
278
+
else within eps (Cons (y, yf));;
279
+
Lines 3-6, characters 6-40:
280
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
281
+
Here is an example of a case that is not matched:
282
+
Nil
283
+
284
+
Lines 1-6, characters 21-40:
285
+
Warning 8 [partial-match]: this pattern-matching is not exhaustive.
286
+
Here is an example of a case that is not matched:
287
+
Nil
288
+
val within : float -> float seq -> float = <fun>
289
+
]}
290
+
291
+
Square Roots:
292
+
293
+
{@ocaml[
294
+
# let root a = within 1e6 (iterates (next a) 1.0);;
295
+
val root : float -> float = <fun>
296
+
]}
297
+
298
+
The {e Newton-Raphson method} is widely used for computing square roots.
299
+
The infinite series {m x_0, (a/x_0+x_0)/2, \ldots{}} converges rapidly to {m \sqrt{a}}.
300
+
The initial approximation, {m x_0}, is typically retrieved from a table, and is accurate enough
301
+
that only a few iterations of the method are necessary.
302
+
Calling [iterates (next a) x0] generates the {e infinite series} of
303
+
approximations to the square root of {m a} using the Newton-Raphson method.
304
+
To compute {m \sqrt2}, the resulting series begins 1, 1.5, 1.41667, 1.4142157, 1.414213562, …,
305
+
and this last figure is already accurate to 10 significant digits!
306
+
307
+
Function [within] searches down the lazy list for two points whose
308
+
difference is less than [eps]. It tests their absolute difference.
309
+
Relative difference and other “close enough” tests can be coded. Such
310
+
components can be used to implement other numerical functions directly as
311
+
functions over sequences. The point is to build programs from small,
312
+
interchangeable parts.
313
+
314
+
Function [root] uses [within], [iterates] and [next] to
315
+
to apply Newton-Raphson with a tolerance of {m 10^\{-6\}}
316
+
and a (poor) initial approximation of 1.0.
317
+
318
+
This treatment of numerical computation has received some attention in the
319
+
research literature; a recurring example is Richardson extrapolation.
320
+
321
+
{2 Exercise 9.1}
322
+
323
+
Code an analogue of [map] for sequences.
324
+
325
+
{2 Exercise 9.2}
326
+
327
+
Consider the list function [concat], which concatenates a list of lists to form a single list. Can
328
+
it be generalised to concatenate a sequence of sequences? What can go wrong?
329
+
330
+
{@ocaml[
331
+
# let rec concat = function
332
+
| [] -> []
333
+
| l::ls -> l @ concat ls;;
334
+
val concat : 'a list list -> 'a list = <fun>
335
+
]}
336
+
337
+
{2 Exercise 9.3}
338
+
339
+
Code a function to make change using lazy lists, delivering the sequence of all possible ways of
340
+
making change. Using sequences allows us to compute solutions one at a time when there exists an
341
+
astronomical number. Represent lists of coins using ordinary lists. ({e Hint}: to benefit from
342
+
laziness you may need to pass around the sequence of alternative solutions as a function of type
343
+
[unit -> (int list) seq].)
344
+
345
+
{2 Exercise 9.4}
346
+
347
+
A {e lazy binary tree} is either empty or is a branch containing a label and two lazy binary trees,
348
+
possibly to infinite depth. Present an OCaml datatype to represent lazy binary trees, along with a
349
+
function that accepts a lazy binary tree and produces a lazy list that contains all of the tree’s
350
+
labels. (Taken from the exam question 2008 Paper 1 Question 5.)
351
+
352
+
{2 Exercise 9.5}
353
+
354
+
Code the lazy list whose elements are all ordinary lists of zeroes and ones, namely
355
+
[[]; [0]; [1]; [0; 0]; [0; 1]; [1; 0]; [1; 1]; [0; 0; 0]; ]…. (Taken from the exam question
356
+
2003 Paper 1 Question 5.)
357
+
358
+
{2 Exercise 9.6}
359
+
360
+
(Continuing the previous exercise.)
361
+
A {e palindrome} is a list that equals its own reverse. Code the lazy list whose elements are all
362
+
palindromes of 0s and 1s, namely
363
+
[[]; [0]; [1]; [0; 0]; [0; 0; 0]; [0; 1; 0]; [1; 1]; [1; 0; 1]; [1; 1; 1]; [0; 0; 0; 0]; ], …. You
364
+
may take the reversal function [List.rev] as given.
+21
site/notebooks/foundations/index.mld
+21
site/notebooks/foundations/index.mld
···
1
+
{0 Foundations of Computer Science}
2
+
3
+
@children_order foundations1 foundations2 foundations3 foundations4 foundations5 foundations6 foundations7 foundations8 foundations9 foundations10 foundations11
4
+
5
+
{1 Introduction}
6
+
These are a series of odoc notebooks created from the lecture notes of the 1A course
7
+
"Foundations of Computer Science", taught at the University of Cambridge.
8
+
9
+
This course is lectured by Anil Madhavapeddy, with the practical exercises managed by me.
10
+
These notes are translated from Lawrence C. Paulson’s earlier course on Standard ML,
11
+
which had credits to David Allsopp, Stuart Becker, Gavin Bierman, Chloë Brown, Silas Brown,
12
+
Qi Chen, David Cottingham, William Denman, Robert Harle, Daniel Hulme, Frank King, Jack
13
+
Lawrence-Jones, Joseph Lord, Dimitrios Los, Farhan Mannan, James Margetson, David Morgan,
14
+
Alan Mycroft, Sridhar Prabhu, Frank Stajano, Alex Trifanov, Thomas Tuerk, Xincheng Wang,
15
+
Philip Withnall and Assel Zhiyenbayeva for pointing out errors. The current notes were ported to
16
+
OCaml in 2019 by Anil Madhavapeddy, David Allsopp, and me and subsequently edited
17
+
by Jeremy Yallop. We thank Richard Sharp, Srinivasan Keshav, Ambroise Lafont, Vojtěch Tvrdík
18
+
and Jeremy Yallop for further feedback and corrections since 2020.
19
+
20
+
{!/site/notebooks/foundations/page-foundations1}
21
+
+5
site/notebooks/index.mld
+5
site/notebooks/index.mld
+122
site/notebooks/oxcaml/local.mld
+122
site/notebooks/oxcaml/local.mld
···
1
+
{0 Locality Mode OxCaml}
2
+
3
+
@x-ocaml.requires base
4
+
5
+
{@ocaml[
6
+
let f () =
7
+
let u @ local = [6; 2; 8] in (* mode *)
8
+
let len = Base.List.length u in
9
+
len;;
10
+
]}
11
+
12
+
{@ocaml[
13
+
let f () =
14
+
let local_ u = [6; 2; 8] in
15
+
let len = Base.List.length u in
16
+
len;;
17
+
]}
18
+
19
+
{@ocaml[
20
+
let f () =
21
+
let u : int list @@ local = stack_ [6; 2; 8] in (* modality *)
22
+
let len = Base.List.length u in
23
+
len;;
24
+
]}
25
+
26
+
{@ocaml[
27
+
let f () =
28
+
let u = local_ [6; 2; 8] in
29
+
let len = Base.List.length u in
30
+
len;;
31
+
]}
32
+
33
+
Other keywords to spot: [stack_], [global_], [@ global], [exclave_] and [[@local_opt]]
34
+
35
+
{1 {b Type}: what it is -; {b Mode}: how it's used}
36
+
37
+
{t
38
+
| Mode | Lifetime | Allocation |
39
+
|----------------|-----------------|-----------------|
40
+
| [global] or missing | MAY outlive its region | MUST be on the heap |
41
+
| [local] | MUST NOT outlive its region | MAY be on the stack |}
42
+
43
+
- Region: compile-time representation of a stack frame
44
+
+ Function bodies
45
+
+ Loop bodies
46
+
+ Lazy expressions
47
+
+ Module level bindings
48
+
- Inference decides how to allocate, defaults to the stack
49
+
- Regions can nest and are wider than scopes
50
+
51
+
{@ocaml[
52
+
let f () =
53
+
let foo =
54
+
let local_ bar = ("region", "scope") in
55
+
bar in
56
+
fst foo;;
57
+
]}
58
+
59
+
{1 What does [local] mean?}
60
+
61
+
- The value does not escape its region
62
+
+ Neither function result nor exception payload
63
+
+ Not captured in closure, not referred from mutable area
64
+
+ Not reachable from a heap allocated value
65
+
+ Freed at its region's end, without triggering the GC
66
+
- In function types -; {b This is the most important}
67
+
+ Contract between caller and callee
68
+
+ [local] means in the caller's region
69
+
+ Parameter: callee respects caller's locality
70
+
+ Result: callee stores in caller's region
71
+
+ This really defines 4 arrows
72
+
{@ocaml[
73
+
val global_global : s -> t * t (* Legacy *)
74
+
val local_global : local_ s -> t * t
75
+
val global_local : s -> local_ t * t
76
+
val local_local : local_ s -> local_ t * t
77
+
]}
78
+
79
+
{1 What is `local` for?}
80
+
81
+
0. Low-latency code
82
+
More importantly, stack allocations will never trigger a GC, and so they're safe to use in low-latency code that must currently be zero-alloc
83
+
1. Functions passed to higher-order iterators (such as `map`, `fold`, `bind` and others) are allocated on the stack
84
+
2. Safer callbacks
85
+
86
+
{1 Hands-on}
87
+
88
+
{@ocaml[
89
+
# let monday () = let str = "mon" ^ "day" in str;;
90
+
]}
91
+
92
+
{@ocaml[
93
+
# let bye () = let ciao = "sorry" in failwith ciao;;
94
+
]}
95
+
96
+
{@ocaml[
97
+
# let make_counter () =
98
+
let counter = ref (-1) in
99
+
fun () -> incr counter; !counter;;
100
+
]}
101
+
102
+
{@ocaml[
103
+
# let state = ref "";;
104
+
# let set () = state := "disco";;
105
+
]}
106
+
107
+
{@ocaml[
108
+
# let rec map f = function [] -> [] | x :: u -> f x :: map f u;;
109
+
]}
110
+
111
+
{@ocaml[
112
+
# let f1 (local_ u : int list) = [1; 2; 3];;
113
+
]}
114
+
115
+
{@ocaml[
116
+
# let f2 (local_ u : int list) = u;;
117
+
]}
118
+
119
+
{@ocaml[
120
+
# let f3 (local_ u : int list) = 42 :: u;;
121
+
]}
122
+
+4
site/reference/index.mld
+4
site/reference/index.mld
site/static/assets/jon.jpg
site/static/assets/jon.jpg
This is a binary file and will not be displayed.