forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Wire failure classification and history recording into build pipeline
Add record_build_result helper and classify_build_failure for pattern-matching
build logs against transient/depext failure categories. Record history entries
in print_batch_summary for build successes, classified build failures, and
blessed doc failures. Record solver failures after the solve phase in run_batch.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
+81
-4
+1
-1
day10/bin/dune
+1
-1
day10/bin/dune
···
3
3
(name main)
4
4
(enabled_if (>= %{ocaml_version} 5.3.0))
5
5
(package day10)
6
-
(libraries opam-0install yojson ppx_deriving_yojson.runtime cmdliner dockerfile day10_lib)
6
+
(libraries opam-0install yojson ppx_deriving_yojson.runtime cmdliner dockerfile day10_lib str)
7
7
(preprocess
8
8
(pps ppx_deriving_yojson)))
+80
-3
day10/bin/main.ml
+80
-3
day10/bin/main.ml
···
265
265
| Failure _ -> "failure"
266
266
| Success _ -> "success"
267
267
268
+
let record_build_result ~packages_dir ~run_id ~pkg_str ~build_hash
269
+
~compiler ~blessed ~status ~category ?error ?failed_dep ?failed_dep_hash () =
270
+
let entry : Day10_lib.History.entry = {
271
+
ts = Day10_lib.Run_log.format_time (Unix.gettimeofday ());
272
+
run = run_id;
273
+
build_hash;
274
+
status;
275
+
category;
276
+
compiler;
277
+
blessed;
278
+
error;
279
+
failed_dep;
280
+
failed_dep_hash;
281
+
} in
282
+
Day10_lib.History.append ~packages_dir ~pkg_str entry
283
+
284
+
(** Classify a build failure by scanning the build log for known patterns. *)
285
+
let classify_build_failure build_log_path =
286
+
let log_content =
287
+
try Os.read_from_file build_log_path
288
+
with _ -> ""
289
+
in
290
+
let transient_patterns = [
291
+
"No space left on device";
292
+
"Connection timed out";
293
+
"Could not resolve host";
294
+
"Temporary failure in name resolution";
295
+
"Network is unreachable";
296
+
] in
297
+
let depext_patterns = [
298
+
"Unable to locate package";
299
+
"Package .* is not available";
300
+
"unmet dependencies";
301
+
"dpkg: dependency problems";
302
+
] in
303
+
if List.exists (fun pat -> try ignore (Str.search_forward (Str.regexp_case_fold pat) log_content 0); true with Not_found -> false) transient_patterns then
304
+
("failure", "transient_failure", Some "Transient infrastructure failure detected in build log")
305
+
else if List.exists (fun pat -> try ignore (Str.search_forward (Str.regexp_case_fold pat) log_content 0); true with Not_found -> false) depext_patterns then
306
+
("failure", "depext_unavailable", Some "Missing system dependency detected in build log")
307
+
else
308
+
("failure", "build_failure", None)
309
+
268
310
let print_build_result = function
269
311
| Solution _ -> ()
270
312
| No_solution _ -> ()
···
1181
1223
let newly_cached = new_cached_count - cached_count in
1182
1224
Printf.printf " %d solutions (%d newly solved), %d failed\n%!" (List.length solutions) newly_cached total_failed;
1183
1225
1226
+
(* Record solver failures in history *)
1227
+
let os_key = Config.os_key ~config in
1228
+
let packages_dir = Path.(config.dir / os_key / "packages") in
1229
+
let run_id = Day10_lib.Run_log.get_id run_info in
1230
+
List.iter (fun (pkg_name, result) ->
1231
+
match result with
1232
+
| None ->
1233
+
record_build_result ~packages_dir ~run_id ~pkg_str:pkg_name
1234
+
~build_hash:"none" ~compiler:"unknown" ~blessed:false
1235
+
~status:"failure" ~category:"solver_failure" ()
1236
+
| Some _ -> ()
1237
+
) results;
1238
+
1184
1239
(* Write initial progress after Phase 1 *)
1185
1240
let progress = Day10_lib.Progress.create
1186
1241
~run_id:(Day10_lib.Run_log.get_id run_info)
···
1251
1306
(* Count actual results by scanning the filesystem *)
1252
1307
let os_key = Config.os_key ~config in
1253
1308
let layer_dir = Path.(config.dir / os_key) in
1309
+
let packages_dir = Path.(config.dir / os_key / "packages") in
1310
+
let run_id = Day10_lib.Run_log.get_id run_info in
1254
1311
let build_success = ref 0 in
1255
1312
let build_fail = ref 0 in
1256
1313
let doc_success = ref 0 in
···
1268
1325
(* Build layer *)
1269
1326
let pkg_name = json |> member "package" |> to_string in
1270
1327
let exit_status = json |> member "exit_status" |> to_int_option |> Option.value ~default:(-1) in
1328
+
(* Check if this build is blessed *)
1329
+
let blessed_build_link = Path.(packages_dir / pkg_name / "blessed-build") in
1330
+
let is_blessed = try
1331
+
let target = Unix.readlink blessed_build_link in
1332
+
Filename.basename target = name
1333
+
with _ -> false in
1271
1334
if exit_status = 0 then begin
1272
1335
incr build_success;
1273
1336
(* Add build log to run *)
1274
1337
let build_log = Path.(layer_dir / name / "build.log") in
1275
-
Day10_lib.Run_log.add_build_log run_info ~package:pkg_name ~source_log:build_log
1338
+
Day10_lib.Run_log.add_build_log run_info ~package:pkg_name ~source_log:build_log;
1339
+
(* Record success in history *)
1340
+
record_build_result ~packages_dir ~run_id ~pkg_str:pkg_name
1341
+
~build_hash:name ~compiler:"" ~blessed:is_blessed
1342
+
~status:"success" ~category:"success" ()
1276
1343
end else begin
1277
1344
incr build_fail;
1278
1345
failures := (pkg_name, Printf.sprintf "build exit code %d" exit_status) :: !failures;
1279
1346
let build_log = Path.(layer_dir / name / "build.log") in
1280
-
Day10_lib.Run_log.add_build_log run_info ~package:pkg_name ~source_log:build_log
1347
+
Day10_lib.Run_log.add_build_log run_info ~package:pkg_name ~source_log:build_log;
1348
+
(* Classify and record build failure in history *)
1349
+
let (status, category, error) = classify_build_failure build_log in
1350
+
record_build_result ~packages_dir ~run_id ~pkg_str:pkg_name
1351
+
~build_hash:name ~compiler:"" ~blessed:is_blessed
1352
+
~status ~category ?error ()
1281
1353
end
1282
1354
end else if String.length name > 4 && String.sub name 0 4 = "doc-" then begin
1283
1355
(* Doc layer - count blessed ones, but log all *)
···
1297
1369
else begin
1298
1370
incr doc_fail;
1299
1371
let error_msg = doc |> member "error" |> to_string_option |> Option.value ~default:"unknown error" in
1300
-
failures := (pkg_name, Printf.sprintf "doc: %s" error_msg) :: !failures
1372
+
failures := (pkg_name, Printf.sprintf "doc: %s" error_msg) :: !failures;
1373
+
(* Record blessed doc failure in history *)
1374
+
record_build_result ~packages_dir ~run_id ~pkg_str:pkg_name
1375
+
~build_hash:name ~compiler:"" ~blessed
1376
+
~status:"failure" ~category:"doc_compile_failure"
1377
+
~error:error_msg ()
1301
1378
end
1302
1379
end
1303
1380
end