forked from
anil.recoil.org/monopam
Monorepo management for opam overlays
Fix dune-project dependency calculation to scan all opam files in subtrees
Previously, collect_external_deps only looked at dependencies from packages
registered in the opam overlay. This missed dependencies from .opam files
that exist in monorepo subtree directories but aren't in the overlay.
Now scans each subtree directory for all .opam files and takes the union
of their dependencies for the root dune-project.
Also adds a warning to `monopam status` when local opam files aren't
registered in the overlay, helping identify packages that should be added.
Fixes String.Map usage in verse.ml (changed to Hashtbl).
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+155
-11
+12
bin/main.ml
+12
bin/main.ml
···
70
70
match Monopam.status ~proc ~fs ~config () with
71
71
| Ok statuses ->
72
72
Fmt.pr "%a@." Monopam.Status.pp_summary statuses;
73
+
(* Check for unregistered opam files *)
74
+
(match Monopam.discover_packages ~fs ~config () with
75
+
| Ok pkgs ->
76
+
let unregistered = Monopam.find_unregistered_opam_files ~fs ~config pkgs in
77
+
if unregistered <> [] then begin
78
+
Fmt.pr "@.";
79
+
Fmt.pr "@[<v>Warning: Found opam files not in overlay:@,";
80
+
List.iter (fun (repo, pkg) ->
81
+
Fmt.pr " %s/%s.opam@," repo pkg) unregistered;
82
+
Fmt.pr "Consider adding these packages to the opam overlay.@]@."
83
+
end
84
+
| Error _ -> ());
73
85
`Ok ()
74
86
| Error e ->
75
87
Fmt.epr "Error: %a@." Monopam.pp_error e;
+90
-8
lib/monopam.ml
+90
-8
lib/monopam.ml
···
77
77
discover_packages ~fs:(fs :> _ Eio.Path.t) ~config ()
78
78
|> Result.map (Status.compute_all ~proc ~fs ~config)
79
79
80
+
(** Find opam files in monorepo subtrees that aren't registered in the overlay.
81
+
Returns a list of (subtree_name, unregistered_package_name) pairs. *)
82
+
let find_unregistered_opam_files ~fs ~config pkgs =
83
+
let fs = fs_typed fs in
84
+
let monorepo = Config.Paths.monorepo config in
85
+
(* Group registered packages by repo name *)
86
+
let registered_by_repo = Hashtbl.create 16 in
87
+
List.iter
88
+
(fun pkg ->
89
+
let repo = Package.repo_name pkg in
90
+
let name = Package.name pkg in
91
+
let existing = try Hashtbl.find registered_by_repo repo with Not_found -> [] in
92
+
Hashtbl.replace registered_by_repo repo (name :: existing))
93
+
pkgs;
94
+
(* Get unique subtree directories *)
95
+
let seen_repos = Hashtbl.create 16 in
96
+
let repos =
97
+
List.filter
98
+
(fun pkg ->
99
+
let repo = Package.repo_name pkg in
100
+
if Hashtbl.mem seen_repos repo then false
101
+
else begin
102
+
Hashtbl.add seen_repos repo ();
103
+
true
104
+
end)
105
+
pkgs
106
+
in
107
+
(* For each subtree, find opam files not in the registry *)
108
+
List.concat_map
109
+
(fun pkg ->
110
+
let repo = Package.repo_name pkg in
111
+
let subtree_dir = Fpath.(monorepo / Package.subtree_prefix pkg) in
112
+
let eio_path = Eio.Path.(fs / Fpath.to_string subtree_dir) in
113
+
let registered = try Hashtbl.find registered_by_repo repo with Not_found -> [] in
114
+
try
115
+
Eio.Path.read_dir eio_path
116
+
|> List.filter_map (fun name ->
117
+
if Filename.check_suffix name ".opam" then
118
+
let pkg_name = Filename.chop_suffix name ".opam" in
119
+
if List.mem pkg_name registered then None
120
+
else Some (repo, pkg_name)
121
+
else None)
122
+
with Eio.Io _ -> [])
123
+
repos
124
+
80
125
let get_branch ~config pkg =
81
126
let default = Config.default_branch config in
82
127
match Package.branch pkg with
···
230
275
root.opam
231
276
|}
232
277
233
-
(** Collect all external dependencies from packages.
278
+
(** Collect all external dependencies by scanning monorepo subtree directories.
279
+
This scans all .opam files in each subtree directory to find dependencies,
280
+
ensuring we get dependencies from all packages in a directory, not just
281
+
those registered in the opam overlay.
234
282
Returns a sorted, deduplicated list of package names that are dependencies
235
283
but not packages in the repo itself. *)
236
-
let collect_external_deps pkgs =
237
-
let pkg_names = List.map Package.name pkgs in
284
+
let collect_external_deps ~fs ~config pkgs =
285
+
let monorepo = Config.Paths.monorepo config in
286
+
(* Get unique repos to avoid scanning the same directory multiple times *)
287
+
let seen = Hashtbl.create 16 in
288
+
let repos =
289
+
List.filter
290
+
(fun pkg ->
291
+
let repo = Package.repo_name pkg in
292
+
if Hashtbl.mem seen repo then false
293
+
else begin
294
+
Hashtbl.add seen repo ();
295
+
true
296
+
end)
297
+
pkgs
298
+
in
299
+
(* Scan each subtree directory for .opam files and collect dependencies *)
238
300
let all_deps =
239
-
List.concat_map Package.depends pkgs
301
+
List.concat_map
302
+
(fun pkg ->
303
+
let subtree_dir = Fpath.(monorepo / Package.subtree_prefix pkg) in
304
+
Opam_repo.scan_opam_files_for_deps ~fs subtree_dir)
305
+
repos
306
+
|> List.sort_uniq String.compare
307
+
in
308
+
(* Get all package names from all .opam files in monorepo *)
309
+
let pkg_names =
310
+
List.concat_map
311
+
(fun pkg ->
312
+
let subtree_dir = Fpath.(monorepo / Package.subtree_prefix pkg) in
313
+
let eio_path = Eio.Path.(fs / Fpath.to_string subtree_dir) in
314
+
try
315
+
Eio.Path.read_dir eio_path
316
+
|> List.filter_map (fun name ->
317
+
if Filename.check_suffix name ".opam" then
318
+
Some (Filename.chop_suffix name ".opam")
319
+
else None)
320
+
with Eio.Io _ -> [])
321
+
repos
240
322
|> List.sort_uniq String.compare
241
323
in
242
324
(* Filter out packages that are in the repo *)
···
244
326
245
327
(** Generate dune-project content for the monorepo root.
246
328
Lists all external dependencies as a virtual package. *)
247
-
let generate_dune_project pkgs =
248
-
let external_deps = collect_external_deps pkgs in
329
+
let generate_dune_project ~fs ~config pkgs =
330
+
let external_deps = collect_external_deps ~fs ~config pkgs in
249
331
let buf = Buffer.create 1024 in
250
332
Buffer.add_string buf "(lang dune 3.20)\n";
251
333
Buffer.add_string buf "(name root)\n";
···
269
351
let monorepo = Config.Paths.monorepo config in
270
352
let monorepo_eio = Eio.Path.(fs / Fpath.to_string monorepo) in
271
353
let dune_project_path = Eio.Path.(monorepo_eio / "dune-project") in
272
-
let content = generate_dune_project pkgs in
354
+
let content = generate_dune_project ~fs ~config pkgs in
273
355
(* Check if dune-project already exists with same content *)
274
356
let needs_update =
275
357
match Eio.Path.load dune_project_path with
···
294
376
ignore (Eio.Process.await child));
295
377
Log.app (fun m ->
296
378
m "Updated dune-project with %d external dependencies"
297
-
(List.length (collect_external_deps pkgs)))
379
+
(List.length (collect_external_deps ~fs ~config pkgs)))
298
380
end
299
381
300
382
let ensure_monorepo_initialized ~proc ~fs ~config =
+16
lib/monopam.mli
+16
lib/monopam.mli
···
169
169
@param config Monopam configuration
170
170
@param name Package name to find *)
171
171
172
+
val find_unregistered_opam_files :
173
+
fs:Eio.Fs.dir_ty Eio.Path.t ->
174
+
config:Config.t ->
175
+
Package.t list ->
176
+
(string * string) list
177
+
(** [find_unregistered_opam_files ~fs ~config pkgs] finds opam files in monorepo
178
+
subtree directories that aren't registered in the opam overlay.
179
+
180
+
Returns a list of [(repo_name, package_name)] pairs for each unregistered
181
+
.opam file found. This helps identify packages that exist in the source
182
+
repositories but aren't being tracked by the overlay.
183
+
184
+
@param fs Eio filesystem
185
+
@param config Monopam configuration
186
+
@param pkgs List of packages discovered from the opam overlay *)
187
+
172
188
(** {1 Changelog Generation} *)
173
189
174
190
val changes :
+23
lib/opam_repo.ml
+23
lib/opam_repo.ml
···
162
162
let validate_repo ~fs repo_path =
163
163
let _, errors = scan_all ~fs repo_path in
164
164
errors
165
+
166
+
(** Scan a directory for .opam files and extract all dependencies.
167
+
This is used to find dependencies from monorepo subtree directories,
168
+
where multiple .opam files may exist that aren't in the opam overlay. *)
169
+
let scan_opam_files_for_deps ~fs dir_path =
170
+
let eio_path = Eio.Path.(fs / Fpath.to_string dir_path) in
171
+
try
172
+
let files = Eio.Path.read_dir eio_path in
173
+
let opam_files =
174
+
List.filter (fun name -> Filename.check_suffix name ".opam") files
175
+
in
176
+
List.concat_map
177
+
(fun opam_file ->
178
+
let opam_path = Eio.Path.(eio_path / opam_file) in
179
+
try
180
+
let content = Eio.Path.load opam_path in
181
+
let opamfile =
182
+
OpamParser.FullPos.string content (Fpath.to_string dir_path ^ "/" ^ opam_file)
183
+
in
184
+
find_depends opamfile.file_contents
185
+
with _ -> [])
186
+
opam_files
187
+
with Eio.Io _ -> []
+11
lib/opam_repo.mli
+11
lib/opam_repo.mli
···
75
75
76
76
For example, "git+https://example.com/repo.git" becomes
77
77
"https://example.com/repo.git". *)
78
+
79
+
val scan_opam_files_for_deps : fs:_ Eio.Path.t -> Fpath.t -> string list
80
+
(** [scan_opam_files_for_deps ~fs dir_path] scans a directory for .opam files
81
+
and extracts all dependencies from them.
82
+
83
+
This is used to find dependencies from monorepo subtree directories,
84
+
where multiple .opam files may exist that aren't in the opam overlay.
85
+
86
+
@param fs Eio filesystem capability
87
+
@param dir_path Path to the directory to scan
88
+
@return List of dependency package names *)
+3
-3
lib/verse.ml
+3
-3
lib/verse.ml
···
318
318
| Ok registry ->
319
319
(* Get already tracked handles to skip them *)
320
320
let tracked = get_tracked_handles ~fs config in
321
-
let tracked_set = List.fold_left (fun acc h ->
322
-
String.Map.add h () acc) String.Map.empty tracked in
321
+
let tracked_set = Hashtbl.create (List.length tracked) in
322
+
List.iter (fun h -> Hashtbl.add tracked_set h ()) tracked;
323
323
(* Ensure verse directory exists *)
324
324
let verse_dir = Verse_config.verse_path config in
325
325
ensure_dir ~fs verse_dir;
···
329
329
List.filter_map
330
330
(fun (member : Verse_registry.member) ->
331
331
let handle = member.handle in
332
-
if String.Map.mem handle tracked_set then begin
332
+
if Hashtbl.mem tracked_set handle then begin
333
333
Logs.info (fun m -> m "Skipping %s (already tracked)" handle);
334
334
None
335
335
end