Monorepo management for opam overlays
fix(monopam): use public HTTPS URLs in opam files, commit before push
- pkg.ml: derive_dev_repo now constructs opam dev-repo/url from the
origin base URL (HTTPS) instead of the sources.toml entry source
(often SSH/SCP). Falls back to entry source when no origin is set.
This fixes opam failing to fetch from SCP-style URLs like
git@host:path which it mis-parses.
- push.ml: add commit_pending to stage+commit pending changes in
workspace repos (e.g. opam-repo) before pushing, so generated
opam files are not left uncommitted.
- opam_sync.ml: delegate derive_urls to Pkg.derive_dev_repo to
remove duplicate URL derivation logic. Add ensure_repo_file for
opam-repo metadata.
- test_pkg.ml: update tests to expect HTTPS origin URLs, add
test for no-origin fallback to entry source (129 tests).
- merlint e617: add multi-line suite definition cram test.
+103
-26
+15
lib/opam_sync.ml
+15
lib/opam_sync.ml
···
33
33
34
34
let read_file_opt path = try Some (Eio.Path.load path) with Eio.Io _ -> None
35
35
36
+
let ensure_repo_file ~fs opam_repo =
37
+
let repo_content = "opam-version: \"2.0\"\n" in
38
+
let repo_path = Eio.Path.(fs / Fpath.to_string opam_repo / "repo") in
39
+
match read_file_opt repo_path with
40
+
| Some c when c = repo_content -> ()
41
+
| _ -> (
42
+
Eio.Path.save ~create:(`Or_truncate 0o644) repo_path repo_content;
43
+
let repo = Git.Repository.open_repo ~fs opam_repo in
44
+
match Git.Repository.add_to_index repo [ "repo" ] with
45
+
| Ok () -> ()
46
+
| Error (`Msg e) ->
47
+
Log.warn (fun m -> m "Failed to add repo file to index: %s" e))
48
+
36
49
let list_opam_repo_packages ~fs ~config =
37
50
let opam_repo = Config.Paths.opam_repo config in
38
51
let packages_dir = Eio.Path.(fs / Fpath.to_string opam_repo / "packages") in
···
200
213
| Ok all_pkgs ->
201
214
let pkgs = filter_packages ~packages all_pkgs in
202
215
let opam_repo = Config.Paths.opam_repo config in
216
+
ensure_repo_file ~fs opam_repo;
203
217
let sync_results =
204
218
sync_packages_with_progress ~fs ~opam_repo ~label:"Split" pkgs
205
219
in
···
417
431
else []
418
432
419
433
let export_packages ~fs ~target ~packages ~dry_run ~no_commit pkgs =
434
+
if not dry_run then ensure_repo_file ~fs target;
420
435
let total = List.length pkgs in
421
436
let progress = Tty.Progress.create ~total "Export" in
422
437
let sync_results =
+20
-9
lib/pkg.ml
+20
-9
lib/pkg.ml
···
35
35
| Ok dev_repo, Ok url_src -> Some (dev_repo, url_src)
36
36
| _ -> None
37
37
in
38
-
let derive_from_default_base () =
39
-
match Sources_registry.derive_url sources ~subtree with
40
-
| Some dev_repo ->
41
-
Log.debug (fun m ->
42
-
m "Using default_url_base for %s: %s" subtree dev_repo);
43
-
Some (dev_repo, dev_repo ^ "#main")
38
+
let derive_from_origin () =
39
+
match Sources_registry.origin sources with
40
+
| Some base ->
41
+
let base =
42
+
if String.ends_with ~suffix:"/" base then
43
+
String.sub base 0 (String.length base - 1)
44
+
else base
45
+
in
46
+
Some (base ^ "/" ^ subtree)
44
47
| None -> None
45
48
in
46
49
match sources_override with
47
50
| Some entry ->
48
-
let dev_repo = entry.Sources_registry.source in
51
+
(* Use the public HTTPS URL (from origin) for opam,
52
+
falling back to the entry source if no origin is configured. *)
53
+
let dev_repo =
54
+
match derive_from_origin () with
55
+
| Some url -> url
56
+
| None -> entry.Sources_registry.source
57
+
in
49
58
let branch =
50
59
match entry.Sources_registry.branch with
51
60
| Some b -> b
···
61
70
match derive_from_dune () with
62
71
| Some result -> Some result
63
72
| None -> (
64
-
match derive_from_default_base () with
65
-
| Some result -> Some result
73
+
match derive_from_origin () with
74
+
| Some (dev_repo as url) ->
75
+
Log.debug (fun m -> m "Using origin for %s: %s" subtree dev_repo);
76
+
Some (url, url ^ "#main")
66
77
| None ->
67
78
Log.warn (fun m ->
68
79
m
+23
lib/push.ml
+23
lib/push.ml
···
132
132
| None -> true
133
133
| Some remote_head -> not (Git.Hash.equal local_head remote_head))
134
134
135
+
let commit_pending ~fs path name =
136
+
let repo = Git.Repository.open_repo ~fs path in
137
+
match Git.Repository.add_all repo with
138
+
| Error (`Msg e) ->
139
+
Log.warn (fun m -> m "Failed to stage changes in %s: %s" name e)
140
+
| Ok () -> (
141
+
match Git_cli.global_git_user ~fs () with
142
+
| None ->
143
+
Log.warn (fun m -> m "No git user config, skipping commit in %s" name)
144
+
| Some user -> (
145
+
let msg = "Sync from monorepo" in
146
+
match
147
+
Git.Repository.commit_index repo ~author:user ~committer:user
148
+
~message:msg ()
149
+
with
150
+
| Ok _ -> Log.info (fun m -> m "Committed pending changes in %s" name)
151
+
| Error (`Msg msg)
152
+
when String.starts_with ~prefix:"nothing to commit" msg ->
153
+
()
154
+
| Error (`Msg e) ->
155
+
Log.warn (fun m -> m "Failed to commit in %s: %s" name e)))
156
+
135
157
let workspace_repos ~proc ~fs ~config ~force ~push_mono =
136
158
let push_repo name path =
137
159
if Git.Repository.is_repo ~fs path then begin
···
139
161
match Git.Repository.remote_url repo "origin" with
140
162
| None -> Log.debug (fun m -> m "%s has no origin remote, skipping" name)
141
163
| Some _ -> (
164
+
commit_pending ~fs path name;
142
165
let branch =
143
166
Git.Repository.current_branch repo |> Option.value ~default:"main"
144
167
in
+45
-17
test/test_pkg.ml
+45
-17
test/test_pkg.ml
···
42
42
(* {1 derive_dev_repo: sources.toml override (highest priority)} *)
43
43
44
44
let test_derive_sources_override () =
45
+
(* With origin set, the HTTPS origin URL is used instead of the SSH entry *)
45
46
let sources =
46
-
SR.add SR.empty ~subtree:"memtrace"
47
+
SR.with_origin SR.empty "git+https://tangled.org/gazagnaire.org" |> fun s ->
48
+
SR.add s ~subtree:"memtrace"
47
49
(sr_entry "git@git.recoil.org:gazagnaire.org/memtrace")
48
50
in
49
51
let dp =
50
52
dune_proj ~source:(Github { user = "janestreet"; repo = "memtrace" }) ()
51
53
in
52
-
check_dev_repo "sources.toml wins over dune-project"
54
+
check_dev_repo "sources.toml entry + origin → HTTPS URL"
53
55
(Some
54
-
( "git@git.recoil.org:gazagnaire.org/memtrace",
55
-
"git@git.recoil.org:gazagnaire.org/memtrace#main" ))
56
+
( "git+https://tangled.org/gazagnaire.org/memtrace",
57
+
"git+https://tangled.org/gazagnaire.org/memtrace#main" ))
56
58
(Pkg.derive_dev_repo ~sources ~subtree:"memtrace" dp)
57
59
58
60
let test_derive_sources_override_with_branch () =
61
+
(* With origin set, HTTPS URL is used; branch comes from entry *)
59
62
let sources =
60
-
SR.add SR.empty ~subtree:"memtrace"
63
+
SR.with_origin SR.empty "git+https://tangled.org/gazagnaire.org" |> fun s ->
64
+
SR.add s ~subtree:"memtrace"
61
65
(sr_entry ~branch:(Some "master")
62
66
"git@git.recoil.org:gazagnaire.org/memtrace")
63
67
in
64
68
let dp = dune_proj () in
65
-
check_dev_repo "sources.toml branch used"
69
+
check_dev_repo "sources.toml branch used with HTTPS URL"
70
+
(Some
71
+
( "git+https://tangled.org/gazagnaire.org/memtrace",
72
+
"git+https://tangled.org/gazagnaire.org/memtrace#master" ))
73
+
(Pkg.derive_dev_repo ~sources ~subtree:"memtrace" dp)
74
+
75
+
let test_derive_sources_override_no_origin () =
76
+
(* Without origin, the entry SSH source is used as fallback *)
77
+
let sources =
78
+
SR.add SR.empty ~subtree:"memtrace"
79
+
(sr_entry "git@git.recoil.org:gazagnaire.org/memtrace")
80
+
in
81
+
let dp = dune_proj () in
82
+
check_dev_repo "no origin → falls back to entry source"
66
83
(Some
67
84
( "git@git.recoil.org:gazagnaire.org/memtrace",
68
-
"git@git.recoil.org:gazagnaire.org/memtrace#master" ))
85
+
"git@git.recoil.org:gazagnaire.org/memtrace#main" ))
69
86
(Pkg.derive_dev_repo ~sources ~subtree:"memtrace" dp)
70
87
71
88
let test_derive_sources_override_branch_from_dune () =
···
174
191
(* {1 derive_dev_repo: priority ordering} *)
175
192
176
193
let test_derive_sources_wins_over_dune () =
194
+
(* With origin, HTTPS URL from origin is used even when dune-project has github *)
177
195
let sources =
178
-
SR.add SR.empty ~subtree:"pkg" (sr_entry "git@git.recoil.org:user/pkg")
196
+
SR.with_origin SR.empty "git+https://tangled.org/user" |> fun s ->
197
+
SR.add s ~subtree:"pkg" (sr_entry "git@git.recoil.org:user/pkg")
179
198
in
180
199
let dp = dune_proj ~source:(Github { user = "upstream"; repo = "pkg" }) () in
181
200
check_dev_repo "sources.toml takes priority over github dune-project"
182
-
(Some ("git@git.recoil.org:user/pkg", "git@git.recoil.org:user/pkg#main"))
201
+
(Some
202
+
( "git+https://tangled.org/user/pkg",
203
+
"git+https://tangled.org/user/pkg#main" ))
183
204
(Pkg.derive_dev_repo ~sources ~subtree:"pkg" dp)
184
205
185
206
let test_derive_dune_wins_over_origin () =
···
192
213
(Pkg.derive_dev_repo ~sources ~subtree:"eio" dp)
193
214
194
215
let test_derive_sources_wins_over_origin () =
216
+
(* When both entry and origin exist, origin HTTPS URL is used for opam *)
195
217
let sources =
196
218
SR.with_origin SR.empty "git+https://tangled.org/user" |> fun s ->
197
219
SR.add s ~subtree:"pkg" (sr_entry "git+https://custom.org/pkg")
198
220
in
199
221
let dp = dune_proj () in
200
-
check_dev_repo "sources.toml entry wins over origin derivation"
201
-
(Some ("git+https://custom.org/pkg", "git+https://custom.org/pkg#main"))
222
+
check_dev_repo "origin HTTPS URL used even when entry has different URL"
223
+
(Some
224
+
( "git+https://tangled.org/user/pkg",
225
+
"git+https://tangled.org/user/pkg#main" ))
202
226
(Pkg.derive_dev_repo ~sources ~subtree:"pkg" dp)
203
227
204
228
(* {1 derive_dev_repo: real-world scenarios} *)
205
229
206
230
let test_derive_memtrace_scenario () =
207
-
(* The actual memtrace case: sources.toml has tangled URL,
208
-
dune-project has github, branch is master *)
231
+
(* The actual memtrace case: sources.toml has SSH URL, origin has HTTPS,
232
+
dune-project has github, branch is master.
233
+
The opam file should get the HTTPS URL from origin, not the SSH URL. *)
209
234
let sources =
210
-
SR.add SR.empty ~subtree:"memtrace"
235
+
SR.with_origin SR.empty "git+https://tangled.org/gazagnaire.org" |> fun s ->
236
+
SR.add s ~subtree:"memtrace"
211
237
(sr_entry ~branch:(Some "master")
212
238
~upstream:(Some "git+https://github.com/janestreet/memtrace")
213
239
"git@git.recoil.org:gazagnaire.org/memtrace")
···
215
241
let dp =
216
242
dune_proj ~source:(Github { user = "janestreet"; repo = "memtrace" }) ()
217
243
in
218
-
check_dev_repo "memtrace: sources.toml tangled URL with master branch"
244
+
check_dev_repo "memtrace: HTTPS origin URL used for opam, not SSH entry"
219
245
(Some
220
-
( "git@git.recoil.org:gazagnaire.org/memtrace",
221
-
"git@git.recoil.org:gazagnaire.org/memtrace#master" ))
246
+
( "git+https://tangled.org/gazagnaire.org/memtrace",
247
+
"git+https://tangled.org/gazagnaire.org/memtrace#master" ))
222
248
(Pkg.derive_dev_repo ~sources ~subtree:"memtrace" dp)
223
249
224
250
let test_derive_no_entry_uses_dune_project () =
···
247
273
test_derive_sources_override;
248
274
Alcotest.test_case "derive: sources.toml with branch" `Quick
249
275
test_derive_sources_override_with_branch;
276
+
Alcotest.test_case "derive: sources.toml no origin" `Quick
277
+
test_derive_sources_override_no_origin;
250
278
Alcotest.test_case "derive: sources.toml branch from dune" `Quick
251
279
test_derive_sources_override_branch_from_dune;
252
280
Alcotest.test_case "derive: sources.toml no branch" `Quick