forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
day11/doc: auto-detect driver compiler from solutions
The driver_compiler for doc tool builds is now optional. When not
specified in the profile, the latest compiler from the package
solutions is used. This allows profiles to work across compiler
releases without hardcoding a specific version.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+217
-18
+1
-1
day11/batch/profile.ml
+1
-1
day11/batch/profile.ml
+5
-3
day11/bin/cmd_batch.ml
+5
-3
day11/bin/cmd_batch.ml
···
38
38
let cache_dir = paths.cache_dir in
39
39
let os_dir = paths.os_dir in
40
40
let ocaml_version = Common.parse_ocaml_version profile.compiler in
41
-
let driver_compiler = OpamPackage.of_string profile.driver_compiler in
41
+
let driver_compiler = if profile.driver_compiler = ""
42
+
then None
43
+
else Some (OpamPackage.of_string profile.driver_compiler) in
42
44
let opam_repositories = profile.opam_repositories in
43
45
let with_doc = profile.with_doc in
44
46
let os_distribution = profile.os_distribution in
···
377
379
Day11_doc.Generate.build_tools_and_run env benv ~np ~os_dir
378
380
~packages:git_packages ~repos:repos_with_shas ~opam_env
379
381
~mounts:base_mounts
380
-
~driver_compiler ~odoc_repo ~build_one
382
+
?driver_compiler ~odoc_repo ~build_one
381
383
~opam_repositories ~cache ~run_log
382
-
~nodes ~solutions ~blessing_maps
384
+
~nodes ~solutions ~blessing_maps ()
383
385
end
384
386
else begin
385
387
(* Build only — no docs *)
+5
-3
day11/bin/cmd_build.ml
+5
-3
day11/bin/cmd_build.ml
···
182
182
Printf.printf "\nGenerating docs to %s...\n%!" output_dir;
183
183
let output = Fpath.v output_dir in
184
184
ignore (Bos.OS.Dir.create ~path:true output);
185
-
let driver_compiler = OpamPackage.of_string profile.driver_compiler in
185
+
let driver_compiler = if profile.driver_compiler = ""
186
+
then None
187
+
else Some (OpamPackage.of_string profile.driver_compiler) in
186
188
let odoc_repo = profile.odoc_repo in
187
189
let solutions = [ (target, solve_result) ] in
188
190
let blessing_maps =
···
193
195
~os_dir
194
196
~packages:git_packages ~repos:repos_with_shas
195
197
~opam_env:_opam_env ~mounts:base_mounts
196
-
~driver_compiler ~odoc_repo
198
+
?driver_compiler ~odoc_repo
197
199
~build_one:(fun node ->
198
200
match Day11_opam_build.Build_layer.build env benv ?patches
199
201
~mounts:base_mounts node () with
200
202
| Day11_opam_build.Types.Success _ -> true
201
203
| _ -> false)
202
204
~opam_repositories ~cache ~run_log
203
-
~nodes ~solutions ~blessing_maps;
205
+
~nodes ~solutions ~blessing_maps ();
204
206
(* Copy HTML for packages in this solution to the output dir *)
205
207
let html_root = Fpath.(os_dir / "html" / "p") in
206
208
let n_copied = ref 0 in
+2
-2
day11/bin/cmd_profile.ml
+2
-2
day11/bin/cmd_profile.ml
···
62
62
info [ "os-version" ] ~docv:"VER" ~doc)
63
63
64
64
let driver_compiler_term =
65
-
let doc = "Compiler for doc driver tools" in
66
-
Arg.(value & opt string "ocaml-base-compiler.5.4.1" &
65
+
let doc = "Compiler for doc driver tools (default: auto-detect from solutions)" in
66
+
Arg.(value & opt string "" &
67
67
info [ "driver-compiler" ] ~docv:"PKG" ~doc)
68
68
69
69
let run_create profile_dir name opam_repositories odoc_repo opam_build_repo
+14
-4
day11/doc/generate.ml
+14
-4
day11/doc/generate.ml
···
798
798
| _ -> None
799
799
) solutions
800
800
801
-
let build_tools_and_run env benv ~np ~os_dir ~packages ~repos ~opam_env:_
802
-
~mounts ~driver_compiler ~odoc_repo ~build_one
801
+
let build_tools_and_run env benv ~np ~os_dir ?driver_compiler ~packages ~repos ~opam_env:_
802
+
~mounts ~odoc_repo ~build_one
803
803
~opam_repositories:_
804
804
~cache ~run_log
805
-
~nodes ~solutions ~blessing_maps =
805
+
~nodes ~solutions ~blessing_maps () =
806
806
Printf.printf "\nPlanning doc tools...\n%!";
807
807
let all_pin_dirs, all_source_dirs = match odoc_repo with
808
808
| Some dir ->
···
815
815
| None ->
816
816
([], OpamPackage.Name.Map.empty)
817
817
in
818
-
(* 1. Plan driver with fixed compiler *)
818
+
(* 1. Plan driver — use specified compiler or auto-detect from solutions *)
819
+
let driver_compiler = match driver_compiler with
820
+
| Some c -> c
821
+
| None ->
822
+
let compilers = unique_compilers solutions in
823
+
(match List.sort (fun a b ->
824
+
OpamPackage.Version.compare
825
+
(OpamPackage.version b) (OpamPackage.version a)) compilers with
826
+
| c :: _ -> c
827
+
| [] -> OpamPackage.of_string "ocaml-base-compiler.5.4.1")
828
+
in
819
829
let driver_pkg = OpamPackage.of_string "odoc-driver.3.1.0" in
820
830
Printf.printf "Planning doc driver (%s)...\n%!"
821
831
(OpamPackage.to_string driver_compiler);
+7
-5
day11/doc/generate.mli
+7
-5
day11/doc/generate.mli
···
51
51
Day11_opam_build.Types.build_env ->
52
52
np:int ->
53
53
os_dir:Fpath.t ->
54
+
?driver_compiler:OpamPackage.t ->
54
55
packages:Day11_opam.Git_packages.t ->
55
56
repos:(string * string) list ->
56
57
opam_env:(string -> OpamVariable.variable_contents option) ->
57
58
mounts:Day11_container.Mount.t list ->
58
-
driver_compiler:OpamPackage.t ->
59
59
odoc_repo:string option ->
60
60
build_one:(Day11_opam_layer.Build.t -> bool) ->
61
61
opam_repositories:string list ->
···
64
64
nodes:Day11_opam_layer.Build.t list ->
65
65
solutions:(OpamPackage.t * Day11_solution.Solve_result.t) list ->
66
66
blessing_maps:(OpamPackage.t * bool OpamPackage.Map.t) list ->
67
+
unit ->
67
68
unit
68
69
(** Plan doc tools (driver + per-compiler odoc) and run a unified DAG
69
-
that builds packages, tools, and docs in parallel. The driver is
70
-
solved with [driver_compiler]; odoc is solved once per unique
71
-
compiler in [solutions]. When [odoc_repo] is given, odoc packages
72
-
are pinned from that local checkout. *)
70
+
that builds packages, tools, and docs in parallel. When
71
+
[driver_compiler] is omitted, the latest compiler from the
72
+
solutions is used. Odoc is solved once per unique compiler in
73
+
[solutions]. When [odoc_repo] is given, odoc packages are pinned
74
+
from that local checkout. *)
+183
day11/test/test_snapshot_service.sh
+183
day11/test/test_snapshot_service.sh
···
1
+
#!/bin/bash
2
+
# Test snapshot service flow: simulate running day11 as a service
3
+
# across 6 opam-repository commits, using the small universe profile.
4
+
#
5
+
# Each step checks out a specific commit, runs batch, and verifies
6
+
# that snapshots, caching, and docs work correctly.
7
+
#
8
+
# Usage: ./test_snapshot_service.sh [day11-binary]
9
+
10
+
set -e
11
+
12
+
DAY11="${1:-$(dirname "$0")/../../_build/default/day11/bin/main.exe}"
13
+
OPAM_REPO="$HOME/ocaml/opam-repository"
14
+
PROFILE_DIR="/tmp/day11-snapshot-test"
15
+
PROFILE_NAME="snapshot-test"
16
+
17
+
# Commits representing meaningful transitions for the small universe
18
+
COMMITS=(
19
+
"267ac40078" # eio 1.3 released
20
+
"0d494a45eb" # fmt 0.11.0
21
+
"6092b78314" # ppxlib 0.37.0
22
+
"7524e5c122" # cmdliner 2.1.0 + logs 0.10.0
23
+
"b04e914780" # odoc 3.1.0 (tool change)
24
+
"55776d6a15" # HEAD — no small-universe changes
25
+
)
26
+
27
+
LABELS=(
28
+
"eio 1.3"
29
+
"fmt 0.11.0"
30
+
"ppxlib 0.37.0"
31
+
"cmdliner 2.1.0 + logs 0.10.0"
32
+
"odoc 3.1.0"
33
+
"HEAD (no-op)"
34
+
)
35
+
36
+
# Save current HEAD so we can restore it
37
+
ORIG_HEAD=$(git -C "$OPAM_REPO" rev-parse HEAD)
38
+
39
+
cleanup() {
40
+
echo ""
41
+
echo "Restoring opam-repository to $ORIG_HEAD..."
42
+
git -C "$OPAM_REPO" checkout -q "$ORIG_HEAD"
43
+
echo "Cleaning up profile..."
44
+
"$DAY11" profile delete --name "$PROFILE_NAME" --profile-dir "$PROFILE_DIR" 2>/dev/null || true
45
+
sudo rm -rf "$PROFILE_DIR"
46
+
}
47
+
trap cleanup EXIT
48
+
49
+
# Ensure fork helper is linked
50
+
BINDIR=$(dirname "$DAY11")
51
+
if [ ! -f "$BINDIR/day11-fork-helper" ]; then
52
+
HELPER=$(find "$(dirname "$DAY11")/../.." -name fork_helper.exe -type f 2>/dev/null | head -1)
53
+
if [ -n "$HELPER" ]; then
54
+
ln -sf "$HELPER" "$BINDIR/day11-fork-helper"
55
+
fi
56
+
fi
57
+
58
+
echo "=========================================="
59
+
echo " Snapshot Service Test"
60
+
echo "=========================================="
61
+
echo "Binary: $DAY11"
62
+
echo "Repo: $OPAM_REPO"
63
+
echo "Profile dir: $PROFILE_DIR"
64
+
echo ""
65
+
66
+
# Clean start
67
+
sudo rm -rf "$PROFILE_DIR"
68
+
mkdir -p "$PROFILE_DIR"
69
+
70
+
# Create profile
71
+
echo "Creating profile '$PROFILE_NAME'..."
72
+
"$DAY11" profile create \
73
+
--name "$PROFILE_NAME" \
74
+
--profile-dir "$PROFILE_DIR" \
75
+
--opam-repository "$OPAM_REPO" \
76
+
--small-universe \
77
+
--with-doc \
78
+
2>&1 | sed 's/^/ /'
79
+
echo ""
80
+
81
+
PREV_SNAPSHOT=""
82
+
TOTAL_OK=0
83
+
TOTAL_FAIL=0
84
+
85
+
for i in "${!COMMITS[@]}"; do
86
+
COMMIT="${COMMITS[$i]}"
87
+
LABEL="${LABELS[$i]}"
88
+
STEP=$((i + 1))
89
+
90
+
echo "=========================================="
91
+
echo " Step $STEP/6: $LABEL"
92
+
echo " Commit: $COMMIT"
93
+
echo "=========================================="
94
+
95
+
# Checkout the commit
96
+
git -C "$OPAM_REPO" checkout -q "$COMMIT"
97
+
98
+
# Run batch
99
+
echo "Running batch..."
100
+
OUTPUT=$("$DAY11" batch \
101
+
--profile "$PROFILE_NAME" \
102
+
--profile-dir "$PROFILE_DIR" \
103
+
-j 4 \
104
+
2>&1)
105
+
106
+
# Extract key metrics
107
+
SNAPSHOT=$(echo "$OUTPUT" | grep "^Snapshot:" | awk '{print $2}')
108
+
TARGETS=$(echo "$OUTPUT" | grep "^Targets:" | awk '{print $2}')
109
+
CACHED=$(echo "$OUTPUT" | grep "^Layers:" | grep -oP '\d+ cached' | awk '{print $1}')
110
+
NEED_BUILD=$(echo "$OUTPUT" | grep "^Layers:" | grep -oP '\d+ need building' | awk '{print $1}')
111
+
SOLVE_CACHED=$(echo "$OUTPUT" | grep "^Solving:" | grep -oP '\d+ cached' | awk '{print $1}')
112
+
DOCS=$(echo "$OUTPUT" | grep "^=== Docs:" | grep -oP '\d+ packages' | awk '{print $1}')
113
+
HTML=$(echo "$OUTPUT" | grep "^=== Docs:" | grep -oP '\d+ HTML' | awk '{print $1}')
114
+
BUILD_OK=$(echo "$OUTPUT" | grep "^Build:" | grep -oP '\d+ success' | awk '{print $1}')
115
+
BUILD_FAIL=$(echo "$OUTPUT" | grep "^Build:" | grep -oP '\d+ failed' | awk '{print $1}')
116
+
117
+
# Check snapshot changed (except step 6 which should reuse)
118
+
SNAPSHOT_CHANGED="yes"
119
+
if [ "$SNAPSHOT" = "$PREV_SNAPSHOT" ]; then
120
+
SNAPSHOT_CHANGED="no (reused)"
121
+
fi
122
+
123
+
echo " Snapshot: $SNAPSHOT ($SNAPSHOT_CHANGED)"
124
+
echo " Targets: $TARGETS"
125
+
echo " Solve: $SOLVE_CACHED cached"
126
+
echo " Layers: $CACHED cached, $NEED_BUILD to build"
127
+
echo " Build: $BUILD_OK ok, $BUILD_FAIL failed"
128
+
echo " Docs: $DOCS packages, $HTML HTML files"
129
+
130
+
# Verify snapshot dir exists
131
+
SNAP_DIR="$PROFILE_DIR/snapshots/$PROFILE_NAME/$SNAPSHOT"
132
+
if [ -d "$SNAP_DIR" ]; then
133
+
echo " Snapshot dir: exists"
134
+
SOLUTIONS=$(ls "$SNAP_DIR/solutions/" 2>/dev/null | wc -l)
135
+
echo " Solutions: $SOLUTIONS cached"
136
+
else
137
+
echo " Snapshot dir: MISSING!"
138
+
fi
139
+
140
+
# Check status command works
141
+
STATUS=$("$DAY11" status \
142
+
--profile "$PROFILE_NAME" \
143
+
--profile-dir "$PROFILE_DIR" 2>&1 | head -1)
144
+
echo " Status: $STATUS"
145
+
146
+
# Accumulate
147
+
TOTAL_OK=$((TOTAL_OK + ${BUILD_OK:-0}))
148
+
TOTAL_FAIL=$((TOTAL_FAIL + ${BUILD_FAIL:-0}))
149
+
PREV_SNAPSHOT="$SNAPSHOT"
150
+
151
+
echo ""
152
+
done
153
+
154
+
echo "=========================================="
155
+
echo " Summary"
156
+
echo "=========================================="
157
+
echo "Total builds: $TOTAL_OK ok, $TOTAL_FAIL failed"
158
+
159
+
# Count snapshots
160
+
N_SNAPSHOTS=$(ls -d "$PROFILE_DIR/snapshots/$PROFILE_NAME/"*/ 2>/dev/null | wc -l)
161
+
echo "Snapshots created: $N_SNAPSHOTS"
162
+
163
+
# Count layers in shared cache
164
+
CACHE_DIR="$PROFILE_DIR/cache"
165
+
if [ -d "$CACHE_DIR" ]; then
166
+
N_LAYERS=$(ls -d "$CACHE_DIR"/debian-bookworm-x86_64/*/ 2>/dev/null | wc -l)
167
+
echo "Layers in shared cache: $N_LAYERS"
168
+
HTML_COUNT=$(find "$CACHE_DIR"/debian-bookworm-x86_64/html -name '*.html' 2>/dev/null | wc -l)
169
+
echo "HTML files: $HTML_COUNT"
170
+
fi
171
+
172
+
# Verify step 6 was a no-op (same snapshot as step 5)
173
+
echo ""
174
+
if [ "$N_SNAPSHOTS" -eq 5 ]; then
175
+
echo "PASS: Step 6 reused step 5's snapshot (no-op)"
176
+
elif [ "$N_SNAPSHOTS" -eq 6 ]; then
177
+
echo "NOTE: Step 6 created a new snapshot (repo HEAD may differ)"
178
+
else
179
+
echo "UNEXPECTED: $N_SNAPSHOTS snapshots"
180
+
fi
181
+
182
+
echo ""
183
+
echo "Done."