Monorepo management for opam overlays
monopam, merlint: migrate to ocaml-opam + ocaml-meta
Replaces the opam-file-format and findlib library dependencies with
the in-tree streaming codecs:
- opam-file-format -> opam + opam.bytesrw
- Fl_metascanner + Fl_split -> meta + meta.bytesrw
All file reads go through Bytesrw.Bytes.Reader (via
Bytesrw_eio.bytes_reader_of_flow for Eio paths, or of_in_channel for
plain channels) so the parser sees slices directly — no upfront
slurp-the-whole-file that would defeat the streaming design.
Sites touched:
- monopam/lib/opam_repo.ml: load_package, scan_opam_files_for_deps
- monopam/lib/forks.ml: scan_verse_opam_repo dev-repo lookup
- monopam/lib/lint.ml: opam_depends_of_reader, scan_opam_files,
load_meta / scan_meta_dir, check_package,
reexports_of_pkg. in_words replaces
Fl_split.in_words for value tokenization.
- merlint/lib/rules/e915.ml: read_tags uses Opam_bytesrw.field_reader
for true streaming single-field lookup.
dune-project and *.opam regenerated: drops opam-file-format, adds opam,
meta, bytesrw-eio as appropriate.
All 406 monopam tests + merlint suite pass; merlint reports 0
regressions on the migrated files.
+188
-161
+3
-1
dune-project
+3
-1
dune-project
+6
-2
lib/dune
+6
-2
lib/dune
+4
-3
lib/forks.ml
+4
-3
lib/forks.ml
···
455
455
let opam_path = Fpath.(pkg_dir / version / "opam") in
456
456
let eio_opam = Eio.Path.(fs / Fpath.to_string opam_path) in
457
457
try
458
-
let content = Eio.Path.load eio_opam in
459
458
let opamfile =
460
-
OpamParser.FullPos.string content (Fpath.to_string opam_path)
459
+
Eio.Path.with_open_in eio_opam (fun flow ->
460
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
461
+
Opam_bytesrw.of_reader ~file:(Fpath.to_string opam_path) r)
461
462
in
462
-
match Opam_repo.dev_repo opamfile.file_contents with
463
+
match Opam_repo.dev_repo opamfile.contents with
463
464
| None -> None
464
465
| Some url_str ->
465
466
if Opam_repo.is_git_url url_str then
+139
-116
lib/lint.ml
+139
-116
lib/lint.ml
···
42
42
43
43
(* ---- META parsing ---- *)
44
44
45
-
let parse_meta content =
46
-
try Ok (Fl_metascanner.parse_lexing (Lexing.from_string content))
47
-
with Fl_metascanner.Error msg -> Error msg
45
+
let load_meta ~fs path =
46
+
let eio_path = Eio.Path.(fs / Fpath.to_string path) in
47
+
try
48
+
Eio.Path.with_open_in eio_path (fun flow ->
49
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
50
+
try Ok (Meta_bytesrw.of_reader r)
51
+
with Meta.Error e -> Error (Meta.Error.to_string e))
52
+
with Eio.Io _ -> Error "I/O error"
48
53
49
-
(** Collect values of [field] from a parsed META, recursively through
50
-
sub-packages, using findlib's own field splitter [Fl_split.in_words]. *)
51
-
let rec collect_field field (pkg : Fl_metascanner.pkg_expr) =
52
-
let own =
53
-
List.concat_map
54
-
(fun (def : Fl_metascanner.pkg_definition) ->
55
-
if def.def_var = field then Fl_split.in_words def.def_value else [])
56
-
pkg.pkg_defs
57
-
in
58
-
let children =
59
-
List.concat_map
60
-
(fun (_, child) -> collect_field field child)
61
-
pkg.pkg_children
54
+
(** Split a META value string into words on whitespace and commas — matches
55
+
findlib's [Fl_split.in_words]. *)
56
+
let in_words s =
57
+
let buf = Buffer.create 16 in
58
+
let words = ref [] in
59
+
let flush () =
60
+
if Buffer.length buf > 0 then begin
61
+
words := Buffer.contents buf :: !words;
62
+
Buffer.clear buf
63
+
end
62
64
in
63
-
own @ children
65
+
String.iter
66
+
(fun c ->
67
+
match c with
68
+
| ' ' | '\t' | '\n' | '\r' | ',' -> flush ()
69
+
| _ -> Buffer.add_char buf c)
70
+
s;
71
+
flush ();
72
+
List.rev !words
64
73
65
-
let collect_requires = collect_field "requires"
66
-
let collect_exports = collect_field "exports"
74
+
(** Collect values of [field] from the items of a META file or subpackage,
75
+
recursively through sub-packages. *)
76
+
let rec collect_field field items =
77
+
List.concat_map
78
+
(function
79
+
| Meta.Value.Variable v when v.Meta.Value.name = field -> in_words v.value
80
+
| Meta.Value.Subpackage sub -> collect_field field sub.items
81
+
| _ -> [])
82
+
items
67
83
68
-
(** Index all library names from a parsed META into [lib_name -> opam_pkg]. *)
69
-
let rec index_meta ~opam_pkg ~prefix (pkg : Fl_metascanner.pkg_expr) index =
84
+
let collect_requires (f : Meta.Value.file) = collect_field "requires" f.items
85
+
let collect_exports (f : Meta.Value.file) = collect_field "exports" f.items
86
+
87
+
(** Index all library names reachable from a META into [lib_name -> opam_pkg].
88
+
*)
89
+
let rec index_items ~opam_pkg ~prefix items index =
70
90
Hashtbl.replace index prefix opam_pkg;
71
91
List.iter
72
-
(fun (name, child) ->
73
-
index_meta ~opam_pkg ~prefix:(prefix ^ "." ^ name) child index)
74
-
pkg.pkg_children
92
+
(function
93
+
| Meta.Value.Subpackage sub ->
94
+
index_items ~opam_pkg
95
+
~prefix:(prefix ^ "." ^ sub.Meta.Value.name)
96
+
sub.items index
97
+
| _ -> ())
98
+
items
99
+
100
+
let index_meta ~opam_pkg ~prefix (f : Meta.Value.file) index =
101
+
index_items ~opam_pkg ~prefix f.items index
75
102
76
103
(* ---- Library index ---- *)
77
104
···
85
112
List.iter
86
113
(fun pkg ->
87
114
let meta = Eio.Path.(dir / pkg / "META") in
88
-
match Eio.Path.load meta with
89
-
| content -> (
90
-
match parse_meta content with
91
-
| Ok expr -> index_meta ~opam_pkg:pkg ~prefix:pkg expr index
92
-
| Error msg ->
93
-
Log.debug (fun m -> m "Failed to parse %s/META: %s" pkg msg))
94
-
| exception Eio.Io _ -> ())
115
+
try
116
+
Eio.Path.with_open_in meta (fun flow ->
117
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
118
+
match Meta_bytesrw.of_reader r with
119
+
| expr -> index_meta ~opam_pkg:pkg ~prefix:pkg expr index
120
+
| exception Meta.Error e ->
121
+
Log.debug (fun m ->
122
+
m "Failed to parse %s/META: %s" pkg (Meta.Error.to_string e)))
123
+
with Eio.Io _ -> ())
95
124
entries
96
125
97
126
(** Build the full library index from build install dir and opam lib. *)
···
121
150
122
151
(* ---- Opam file deps parsing ---- *)
123
152
124
-
module OP = OpamParserTypes.FullPos
125
-
126
-
let rec extract_dep_name (v : OP.value) =
127
-
match v.pelem with
128
-
| OP.String s -> Some s
129
-
| OP.Option (inner, _) -> extract_dep_name inner
153
+
let rec extract_dep_name (v : Opam.Value.t) =
154
+
match v with
155
+
| Opam.Value.String s -> Some s
156
+
| Opam.Value.Option (inner, _) -> extract_dep_name inner
130
157
| _ -> None
131
158
132
159
(** Check if a dep entry has [with-test], [with-doc], or [build] scope. *)
133
-
let rec is_scoped (v : OP.value) =
134
-
match v.pelem with
135
-
| OP.Option (_, constraints) -> has_scope constraints.pelem
160
+
let rec is_scoped (v : Opam.Value.t) =
161
+
match v with
162
+
| Opam.Value.Option (_, constraints) -> has_scope constraints
136
163
| _ -> false
137
164
138
165
and has_scope items =
139
166
List.exists
140
-
(fun (item : OP.value) ->
141
-
match item.pelem with
142
-
| OP.Ident s -> s = "with-test" || s = "with-doc" || s = "build"
143
-
| OP.Logop (_, l, r) -> has_scope [ l ] || has_scope [ r ]
144
-
| OP.Pfxop (_, inner) -> has_scope [ inner ]
167
+
(fun (item : Opam.Value.t) ->
168
+
match item with
169
+
| Opam.Value.Ident s -> s = "with-test" || s = "with-doc" || s = "build"
170
+
| Opam.Value.Logop (_, l, r) -> has_scope [ l ] || has_scope [ r ]
171
+
| Opam.Value.Pfxop (_, inner) -> has_scope [ inner ]
145
172
| _ -> false)
146
173
items
147
174
···
149
176
150
177
Runtime deps exclude those annotated with [:with-test], [:with-doc], or
151
178
[build]. *)
152
-
let opam_depends content =
179
+
let opam_depends_of_reader ~file r =
153
180
try
154
-
let opam = OpamParser.FullPos.string content "opam" in
181
+
let opam = Opam_bytesrw.of_reader ~file r in
155
182
List.fold_left
156
-
(fun (runtime, all) (item : OP.opamfile_item) ->
157
-
match item.pelem with
158
-
| OP.Variable (name, value) when name.pelem = "depends" ->
183
+
(fun (runtime, all) (item : Opam.Value.item) ->
184
+
match item with
185
+
| Opam.Value.Variable ("depends", value) ->
159
186
let deps =
160
-
match value.pelem with
161
-
| OP.List { pelem = items; _ } -> items
162
-
| _ -> []
187
+
match value with Opam.Value.List items -> items | _ -> []
163
188
in
164
189
List.fold_left
165
190
(fun (rt, al) dep ->
···
171
196
(runtime, all) deps
172
197
| _ -> (runtime, all))
173
198
(String_set.empty, String_set.empty)
174
-
opam.file_contents
199
+
opam.contents
175
200
with exn ->
176
201
Log.debug (fun m -> m "opam parse failed: %s" (Printexc.to_string exn));
177
202
(String_set.empty, String_set.empty)
···
186
211
(fun name ->
187
212
if Filename.check_suffix name ".opam" then
188
213
let pkg_name = Filename.chop_suffix name ".opam" in
189
-
match load_file fs Fpath.(subtree_path / name) with
190
-
| Some content ->
191
-
let runtime, all = opam_depends content in
192
-
Some (pkg_name, runtime, all)
193
-
| None -> None
214
+
let opam_path = Fpath.(subtree_path / name) in
215
+
let file = Fpath.to_string opam_path in
216
+
let eio_opam = Eio.Path.(fs / file) in
217
+
try
218
+
let runtime, all =
219
+
Eio.Path.with_open_in eio_opam (fun flow ->
220
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
221
+
opam_depends_of_reader ~file r)
222
+
in
223
+
Some (pkg_name, runtime, all)
224
+
with Eio.Io _ -> None
194
225
else None)
195
226
entries
196
227
···
291
322
let check_package ~index ~build_lib ~dune_pkgs ~own_set ~all_deps ~subtree
292
323
(pkg_name, runtime_deps, _all_deps) ~fs =
293
324
let meta_path = Fpath.(build_lib / pkg_name / "META") in
294
-
match load_file fs meta_path with
295
-
| None ->
325
+
match load_meta ~fs meta_path with
326
+
| Error msg when msg = "I/O error" ->
296
327
Log.debug (fun m -> m "%s/%s: no META file (not built?)" subtree pkg_name);
297
328
[]
298
-
| Some content -> (
299
-
match parse_meta content with
300
-
| Error msg ->
301
-
Log.warn (fun m ->
302
-
m "%s/%s: META parse error: %s" subtree pkg_name msg);
303
-
[]
304
-
| Ok expr ->
305
-
let required_libs =
306
-
collect_requires expr |> List.sort_uniq String.compare
307
-
in
308
-
let meta_pkgs =
309
-
List.fold_left
310
-
(fun acc lib ->
311
-
if is_builtin lib then acc
312
-
else
313
-
let pkg = lib_to_package index lib in
314
-
if String_set.mem pkg own_set then acc
315
-
else String_set.add pkg acc)
316
-
String_set.empty required_libs
317
-
in
318
-
let missing =
319
-
String_set.fold
320
-
(fun pkg acc ->
321
-
if not (String_set.mem pkg all_deps) then
322
-
{ subtree; kind = Missing; package = pkg } :: acc
323
-
else acc)
324
-
meta_pkgs []
325
-
in
326
-
let needed =
327
-
String_set.union meta_pkgs
328
-
(String_set.union dune_pkgs
329
-
(String_set.union own_set implicit_deps))
330
-
in
331
-
let unused =
332
-
String_set.diff runtime_deps needed
333
-
|> String_set.filter (fun p -> not (is_conf_pkg p))
334
-
in
335
-
let unused_issues =
336
-
String_set.fold
337
-
(fun pkg acc -> { subtree; kind = Unused; package = pkg } :: acc)
338
-
unused []
339
-
in
340
-
missing @ unused_issues)
329
+
| Error msg ->
330
+
Log.warn (fun m -> m "%s/%s: META parse error: %s" subtree pkg_name msg);
331
+
[]
332
+
| Ok expr ->
333
+
let required_libs =
334
+
collect_requires expr |> List.sort_uniq String.compare
335
+
in
336
+
let meta_pkgs =
337
+
List.fold_left
338
+
(fun acc lib ->
339
+
if is_builtin lib then acc
340
+
else
341
+
let pkg = lib_to_package index lib in
342
+
if String_set.mem pkg own_set then acc else String_set.add pkg acc)
343
+
String_set.empty required_libs
344
+
in
345
+
let missing =
346
+
String_set.fold
347
+
(fun pkg acc ->
348
+
if not (String_set.mem pkg all_deps) then
349
+
{ subtree; kind = Missing; package = pkg } :: acc
350
+
else acc)
351
+
meta_pkgs []
352
+
in
353
+
let needed =
354
+
String_set.union meta_pkgs
355
+
(String_set.union dune_pkgs (String_set.union own_set implicit_deps))
356
+
in
357
+
let unused =
358
+
String_set.diff runtime_deps needed
359
+
|> String_set.filter (fun p -> not (is_conf_pkg p))
360
+
in
361
+
let unused_issues =
362
+
String_set.fold
363
+
(fun pkg acc -> { subtree; kind = Unused; package = pkg } :: acc)
364
+
unused []
365
+
in
366
+
missing @ unused_issues
341
367
342
368
let sort_issues issues =
343
369
List.sort
···
360
386
else
361
387
let seen = String_set.add pkg seen in
362
388
let meta_path = Fpath.(build_lib / pkg / "META") in
363
-
match load_file fs meta_path with
364
-
| None -> seen
365
-
| Some content -> (
366
-
match parse_meta content with
367
-
| Error _ -> seen
368
-
| Ok expr ->
369
-
let exported =
370
-
collect_exports expr
371
-
|> List.map (lib_to_package index)
372
-
|> List.filter (fun p -> p <> pkg)
373
-
in
374
-
List.fold_left walk seen exported)
389
+
match load_meta ~fs meta_path with
390
+
| Error _ -> seen
391
+
| Ok expr ->
392
+
let exported =
393
+
collect_exports expr
394
+
|> List.map (lib_to_package index)
395
+
|> List.filter (fun p -> p <> pkg)
396
+
in
397
+
List.fold_left walk seen exported
375
398
in
376
399
String_set.remove pkg (walk String_set.empty pkg)
377
400
+32
-37
lib/opam_repo.ml
+32
-37
lib/opam_repo.ml
···
33
33
in
34
34
Uri.of_string url
35
35
36
-
module OP = OpamParserTypes.FullPos
36
+
let extract_string_value (v : Opam.Value.t) : string option =
37
+
match v with Opam.Value.String s -> Some s | _ -> None
37
38
38
-
let extract_string_value (v : OP.value) : string option =
39
-
match v.pelem with OP.String s -> Some s | _ -> None
40
-
41
-
let dev_repo (items : OP.opamfile_item list) : string option =
39
+
let dev_repo (items : Opam.Value.item list) : string option =
42
40
List.find_map
43
-
(fun (item : OP.opamfile_item) ->
44
-
match item.pelem with
45
-
| OP.Variable (name, value) when name.pelem = "dev-repo" ->
46
-
extract_string_value value
41
+
(function
42
+
| Opam.Value.Variable ("dev-repo", value) -> extract_string_value value
47
43
| _ -> None)
48
44
items
49
45
···
51
47
- "pkgname"
52
48
- "pkgname" [>= "1.0"]
53
49
- "pkgname" [with-test] Returns the package name if found. *)
54
-
let rec extract_dep_name (v : OP.value) : string option =
55
-
match v.pelem with
56
-
| OP.String s -> Some s
57
-
| OP.Option (inner, _) -> extract_dep_name inner
50
+
let rec extract_dep_name (v : Opam.Value.t) : string option =
51
+
match v with
52
+
| Opam.Value.String s -> Some s
53
+
| Opam.Value.Option (inner, _) -> extract_dep_name inner
58
54
| _ -> None
59
55
60
56
(** Extract all dependency package names from a depends value. The depends field
61
57
is a list of package formulas. *)
62
-
let extract_depends_list (v : OP.value) : string list =
63
-
match v.pelem with
64
-
| OP.List { pelem = items; _ } -> List.filter_map extract_dep_name items
58
+
let extract_depends_list (v : Opam.Value.t) : string list =
59
+
match v with
60
+
| Opam.Value.List items -> List.filter_map extract_dep_name items
65
61
| _ -> ( match extract_dep_name v with Some s -> [ s ] | None -> [])
66
62
67
-
let depends (items : OP.opamfile_item list) : string list =
63
+
let depends (items : Opam.Value.item list) : string list =
68
64
List.find_map
69
-
(fun (item : OP.opamfile_item) ->
70
-
match item.pelem with
71
-
| OP.Variable (name, value) when name.pelem = "depends" ->
65
+
(function
66
+
| Opam.Value.Variable ("depends", value) ->
72
67
Some (extract_depends_list value)
73
68
| _ -> None)
74
69
items
75
70
|> Option.value ~default:[]
76
71
77
-
let synopsis (items : OP.opamfile_item list) : string option =
72
+
let synopsis (items : Opam.Value.item list) : string option =
78
73
List.find_map
79
-
(fun (item : OP.opamfile_item) ->
80
-
match item.pelem with
81
-
| OP.Variable (name, value) when name.pelem = "synopsis" ->
82
-
extract_string_value value
74
+
(function
75
+
| Opam.Value.Variable ("synopsis", value) -> extract_string_value value
83
76
| _ -> None)
84
77
items
85
78
···
104
97
| Some (name, version) -> (
105
98
try
106
99
let eio_path = Eio.Path.(fs / path_str) in
107
-
let content = Eio.Path.load eio_path in
108
-
let opamfile = OpamParser.FullPos.string content path_str in
109
-
match dev_repo opamfile.file_contents with
100
+
let opamfile =
101
+
Eio.Path.with_open_in eio_path (fun flow ->
102
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
103
+
Opam_bytesrw.of_reader ~file:path_str r)
104
+
in
105
+
match dev_repo opamfile.contents with
110
106
| None -> Error (No_dev_repo name)
111
107
| Some url ->
112
108
if not (is_git_url url) then Error (Not_git_remote (name, url))
113
109
else
114
110
let dev_repo = normalize_git_url url in
115
-
let depends = depends opamfile.file_contents in
116
-
let synopsis = synopsis opamfile.file_contents in
111
+
let depends = depends opamfile.contents in
112
+
let synopsis = synopsis opamfile.contents in
117
113
Ok (Package.v ~name ~version ~dev_repo ~depends ?synopsis ())
118
114
with
119
115
| Eio.Io _ as e -> Error (Io_error (Printexc.to_string e))
···
172
168
List.concat_map
173
169
(fun opam_file ->
174
170
let opam_path = Eio.Path.(eio_path / opam_file) in
171
+
let file = Fpath.to_string dir_path ^ "/" ^ opam_file in
175
172
try
176
-
let content = Eio.Path.load opam_path in
177
-
let opamfile =
178
-
OpamParser.FullPos.string content
179
-
(Fpath.to_string dir_path ^ "/" ^ opam_file)
180
-
in
181
-
depends opamfile.file_contents
182
-
with Eio.Io _ | Parsing.Parse_error -> [])
173
+
Eio.Path.with_open_in opam_path (fun flow ->
174
+
let r = Bytesrw_eio.bytes_reader_of_flow flow in
175
+
let opamfile = Opam_bytesrw.of_reader ~file r in
176
+
depends opamfile.contents)
177
+
with Eio.Io _ | Opam.Error _ -> [])
183
178
opam_files
184
179
with Eio.Io _ -> []
185
180
+1
-1
lib/opam_repo.mli
+1
-1
lib/opam_repo.mli
···
89
89
90
90
(** {1 Low-level Opam File Parsing} *)
91
91
92
-
val dev_repo : OpamParserTypes.FullPos.opamfile_item list -> string option
92
+
val dev_repo : Opam.Value.item list -> string option
93
93
(** [dev_repo items] extracts the dev-repo field from parsed opam file items. *)
94
94
95
95
(** {1 Writing Packages} *)