this repo has no description
trim
+1
-1019
+1
-1
stack/jmap/lib/dune
+1
-1
stack/jmap/lib/dune
-5
stack/jmap/lib/jmap.ml
-5
stack/jmap/lib/jmap.ml
-5
stack/jmap/lib/jmap.mli
-5
stack/jmap/lib/jmap.mli
-10
stack/mltessera/bin/dune
-10
stack/mltessera/bin/dune
-247
stack/mltessera/bin/mltessera_cli.ml
-247
stack/mltessera/bin/mltessera_cli.ml
···
1
-
(** MLTessera CLI - Command-line interface for tessera tile downloads *)
2
-
3
-
open Cmdliner
4
-
open Mltessera
5
-
6
-
(* Setup logging *)
7
-
let setup_logging _style_renderer level =
8
-
Fmt.set_style_renderer Fmt.stderr `Ansi_tty;
9
-
Logs.set_level level;
10
-
Logs.set_reporter (Logs_fmt.reporter ())
11
-
12
-
(* Common arguments *)
13
-
let bbox_arg =
14
-
let doc = "Bounding box: Use format 'min_lon,min_lat,max_lon,max_lat' (e.g., '-0.2,51.4,0.0,51.6'). \
15
-
For negative coordinates, quote the entire string or use = syntax (--bbox=-0.2,51.4,0.0,51.6)" in
16
-
Arg.(value & pos 0 (some string) None & info [] ~docv:"BBOX" ~doc)
17
-
18
-
let year_arg =
19
-
let doc = "Year of data to download (e.g., 2024)" in
20
-
Arg.(value & opt string "2024" & info ["y"; "year"] ~docv:"YEAR" ~doc)
21
-
22
-
let data_type_arg =
23
-
let doc = "Type of data to download (embeddings or landmasks)" in
24
-
let data_type = Arg.enum [
25
-
("embeddings", `Embeddings);
26
-
("landmasks", `Landmasks)
27
-
] in
28
-
Arg.(value & opt data_type `Embeddings & info ["t"; "type"] ~docv:"TYPE" ~doc)
29
-
30
-
let concurrency_arg =
31
-
let doc = "Number of concurrent downloads" in
32
-
Arg.(value & opt int 4 & info ["j"; "jobs"] ~docv:"N" ~doc)
33
-
34
-
let output_dir_arg =
35
-
let doc = "Export downloaded tiles to this directory" in
36
-
Arg.(value & opt (some string) None & info ["o"; "output"] ~docv:"DIR" ~doc)
37
-
38
-
let base_url_arg =
39
-
let doc = "Base URL for tile downloads (default: https://dl.geotessera.org/)" in
40
-
Arg.(value & opt (some string) None & info ["u"; "url"] ~docv:"URL" ~doc)
41
-
42
-
let cache_dir_arg =
43
-
let doc = "Directory for caching tiles (default: XDG cache dir)" in
44
-
Arg.(value & opt (some string) None & info ["c"; "cache"] ~docv:"DIR" ~doc)
45
-
46
-
let verbose_arg =
47
-
let doc = "Verbose output" in
48
-
Arg.(value & flag & info ["v"; "verbose"] ~doc)
49
-
50
-
(* Download command *)
51
-
let download_cmd =
52
-
let download bbox_str year data_type concurrency output_dir base_url cache_dir verbose =
53
-
(* Setup logging *)
54
-
let log_level = if verbose then Some Logs.Debug else Some Logs.Info in
55
-
setup_logging None log_level;
56
-
57
-
(* Parse bounding box *)
58
-
let bbox =
59
-
match bbox_str with
60
-
| None ->
61
-
Printf.eprintf "Error: Bounding box required\n";
62
-
Printf.eprintf "\nExamples:\n";
63
-
Printf.eprintf " mltessera download '0,50,1,51' # Simple positive coordinates\n";
64
-
Printf.eprintf " mltessera download -- '-0.2,51.4,0.0,51.6' # Negative coordinates with --\n";
65
-
Printf.eprintf " mltessera download '-0.2,51.4,0.0,51.6' # Quoted negative coordinates\n";
66
-
exit 1
67
-
| Some str ->
68
-
match Coordinates.parse_bbox str with
69
-
| Ok bbox -> bbox
70
-
| Error _msg ->
71
-
(* Try using LonLat's parser directly for more flexibility *)
72
-
try
73
-
Lonlat.Bbox.of_string str
74
-
with Lonlat.Error.Coordinate_error e ->
75
-
Printf.eprintf "Error parsing bounding box: %s\n" (Lonlat.Error.to_string e);
76
-
Printf.eprintf "\nFormat: min_lon,min_lat,max_lon,max_lat\n";
77
-
Printf.eprintf "Examples:\n";
78
-
Printf.eprintf " '0,50,1,51' # Simple coordinates\n";
79
-
Printf.eprintf " '-0.2,51.4,0.0,51.6' # Negative coordinates (quote the string)\n";
80
-
Printf.eprintf " '-180,-90,180,90' # Global extent\n";
81
-
exit 1
82
-
in
83
-
84
-
(* Run with Eio *)
85
-
Eio_main.run @@ fun env ->
86
-
Eio.Switch.run @@ fun sw ->
87
-
88
-
(* Initialize MLTessera *)
89
-
let mltessera = Mltessera.create ~sw ~env ?base_url ~cache_dir () in
90
-
91
-
(* Show what we're doing *)
92
-
Printf.printf "📍 Downloading %s tiles for year %s\n"
93
-
(match data_type with `Embeddings -> "embeddings" | `Landmasks -> "landmasks")
94
-
year;
95
-
Printf.printf "📦 Bounding box: %s\n" (Coordinates.format_bbox bbox);
96
-
Printf.printf "🔢 Max tiles: %d\n" (Coordinates.count_tiles_in_bbox bbox);
97
-
Printf.printf "⚡ Concurrency: %d\n\n" concurrency;
98
-
99
-
(* Download tiles *)
100
-
let results = Mltessera.download_bbox mltessera
101
-
~data_type ~year ~bbox ~concurrency () in
102
-
103
-
(* Export if requested *)
104
-
(match output_dir with
105
-
| Some dir ->
106
-
Printf.printf "\n📤 Exporting tiles to %s...\n" dir;
107
-
let exported = Mltessera.export_tiles mltessera ~dest_dir:dir results in
108
-
Printf.printf "✅ Exported %d tiles\n" exported
109
-
| None -> ());
110
-
111
-
(* Show summary *)
112
-
let successes = List.filter (fun (_, r) ->
113
-
match r with Ok _ -> true | Error _ -> false
114
-
) results in
115
-
Printf.printf "\n✨ Download complete: %d/%d tiles successful\n"
116
-
(List.length successes) (List.length results);
117
-
118
-
0
119
-
in
120
-
121
-
let doc = "Download tessera tiles within a bounding box" in
122
-
let info = Cmd.info "download" ~doc ~man:[
123
-
`S "DESCRIPTION";
124
-
`P "Download tessera tiles (embeddings or landmasks) for a geographic bounding box.";
125
-
`S "EXAMPLES";
126
-
`P "Download embeddings for London area:";
127
-
`Pre " mltessera download '-0.2,51.4,0.0,51.6' --year 2024";
128
-
`P "Download with explicit parameters:";
129
-
`Pre " mltessera download -- '-0.2,51.4,0.0,51.6' --type embeddings --jobs 8";
130
-
`P "Export tiles to a directory:";
131
-
`Pre " mltessera download '10.0,45.0,11.0,46.0' --output ./tiles/";
132
-
] in
133
-
Cmd.v info Term.(const download $ bbox_arg $ year_arg $ data_type_arg $
134
-
concurrency_arg $ output_dir_arg $ base_url_arg $
135
-
cache_dir_arg $ verbose_arg)
136
-
137
-
(* List command - show available years *)
138
-
let list_cmd =
139
-
let list verbose =
140
-
setup_logging None (if verbose then Some Logs.Debug else Some Logs.Info);
141
-
142
-
Eio_main.run @@ fun env ->
143
-
Eio.Switch.run @@ fun sw ->
144
-
145
-
let mltessera = Mltessera.create ~sw ~env () in
146
-
let years = Mltessera.get_available_years mltessera in
147
-
148
-
Printf.printf "📅 Available years:\n";
149
-
List.iter (Printf.printf " • %s\n") years;
150
-
151
-
0
152
-
in
153
-
154
-
let doc = "List available data years" in
155
-
let info = Cmd.info "list" ~doc in
156
-
Cmd.v info Term.(const list $ verbose_arg)
157
-
158
-
(* Update command - update tessera-manifests *)
159
-
let update_cmd =
160
-
let update verbose =
161
-
setup_logging None (if verbose then Some Logs.Debug else Some Logs.Info);
162
-
163
-
Eio_main.run @@ fun env ->
164
-
Eio.Switch.run @@ fun sw ->
165
-
166
-
let mltessera = Mltessera.create ~sw ~env () in
167
-
168
-
Printf.printf "🔄 Updating tessera-manifests repository...\n";
169
-
Mltessera.update_manifests mltessera;
170
-
Printf.printf "✅ Repository updated\n";
171
-
172
-
0
173
-
in
174
-
175
-
let doc = "Update the tessera-manifests repository" in
176
-
let info = Cmd.info "update" ~doc in
177
-
Cmd.v info Term.(const update $ verbose_arg)
178
-
179
-
(* Cache command - manage tile cache *)
180
-
let cache_cmd =
181
-
let cache_info verbose =
182
-
setup_logging None (if verbose then Some Logs.Debug else Some Logs.Info);
183
-
184
-
Eio_main.run @@ fun env ->
185
-
Eio.Switch.run @@ fun sw ->
186
-
187
-
let mltessera = Mltessera.create ~sw ~env () in
188
-
let (size, count) = Mltessera.get_cache_stats mltessera in
189
-
190
-
Printf.printf "💾 Cache statistics:\n";
191
-
Printf.printf " Files: %d\n" count;
192
-
Printf.printf " Size: %.2f MB\n" (Int64.to_float size /. 1024.0 /. 1024.0);
193
-
194
-
0
195
-
in
196
-
197
-
let clear verbose =
198
-
setup_logging None (if verbose then Some Logs.Debug else Some Logs.Info);
199
-
200
-
Eio_main.run @@ fun env ->
201
-
Eio.Switch.run @@ fun sw ->
202
-
203
-
let mltessera = Mltessera.create ~sw ~env () in
204
-
205
-
Printf.printf "🗑️ Clearing tile cache...\n";
206
-
Mltessera.clear_cache mltessera;
207
-
Printf.printf "✅ Cache cleared\n";
208
-
209
-
0
210
-
in
211
-
212
-
let doc = "Manage tile cache" in
213
-
let info_cmd =
214
-
let doc = "Show cache statistics" in
215
-
let info = Cmd.info "info" ~doc in
216
-
Cmd.v info Term.(const cache_info $ verbose_arg)
217
-
in
218
-
let clear_cmd =
219
-
let doc = "Clear tile cache" in
220
-
let info = Cmd.info "clear" ~doc in
221
-
Cmd.v info Term.(const clear $ verbose_arg)
222
-
in
223
-
224
-
let info = Cmd.info "cache" ~doc in
225
-
Cmd.group info [info_cmd; clear_cmd]
226
-
227
-
(* Main command *)
228
-
let main_cmd =
229
-
let doc = "MLTessera - Download tessera tiles from geotessera.org" in
230
-
let info = Cmd.info "mltessera" ~version:"0.1.0" ~doc ~man:[
231
-
`S "DESCRIPTION";
232
-
`P "MLTessera downloads geospatial tiles from the tessera repository. \
233
-
It manages the tessera-manifests registry and caches downloaded tiles.";
234
-
`S "HANDLING NEGATIVE COORDINATES";
235
-
`P "When using negative coordinates in bounding boxes, you have several options:";
236
-
`Noblank;
237
-
`P "1. Quote the entire bbox string: mltessera download '-0.2,51.4,0.0,51.6'";
238
-
`Noblank;
239
-
`P "2. Use -- before the bbox: mltessera download -- -0.2,51.4,0.0,51.6";
240
-
`Noblank;
241
-
`P "3. Use the = syntax: mltessera download --bbox=-0.2,51.4,0.0,51.6";
242
-
`S "SEE ALSO";
243
-
`P "Repository: https://github.com/ucam-eo/tessera-manifests";
244
-
] in
245
-
Cmd.group info [download_cmd; list_cmd; update_cmd; cache_cmd]
246
-
247
-
let () = exit (Cmd.eval' main_cmd)
-26
stack/mltessera/dune-project
-26
stack/mltessera/dune-project
···
1
-
(lang dune 3.0)
2
-
(name mltessera)
3
-
4
-
(generate_opam_files true)
5
-
6
-
(package
7
-
(name mltessera)
8
-
(synopsis "OCaml library for downloading tessera tiles from geotessera.org")
9
-
(description "MLTessera is an OCaml equivalent of the Python geotessera library. \
10
-
It manages tessera-manifests registry files and downloads geospatial \
11
-
tiles within bounding boxes using the Toru library for HTTP operations.")
12
-
(depends
13
-
ocaml
14
-
dune
15
-
toru
16
-
lonlat
17
-
(cmdliner (>= 2.0.0))
18
-
eio
19
-
eio_main
20
-
yojson
21
-
logs
22
-
fmt))
23
-
24
-
(authors "MLTessera Authors")
25
-
(maintainers "maintainer@example.com")
26
-
(license MIT)
-125
stack/mltessera/lib/coordinates.ml
-125
stack/mltessera/lib/coordinates.ml
···
1
-
(** Coordinate system for tessera tiles using LonLat library
2
-
3
-
This module handles the three-level coordinate hierarchy used by tessera:
4
-
1. BLOCKS (5×5 degrees): Registry files are organized into blocks
5
-
2. TILES (0.1×0.1 degrees): Individual data files containing embeddings/landmasks
6
-
3. WORLD: Arbitrary decimal degree coordinates
7
-
*)
8
-
9
-
open Lonlat
10
-
11
-
let block_size = 5 (* 5x5 degree blocks *)
12
-
let tile_size = 0.1 (* 0.1x0.1 degree tiles *)
13
-
14
-
type world_coord = Coord.t
15
-
16
-
type block_coord = {
17
-
block_lon : int;
18
-
block_lat : int;
19
-
}
20
-
21
-
type tile_coord = {
22
-
tile_lon : float;
23
-
tile_lat : float;
24
-
}
25
-
26
-
type bounding_box = Bbox.t
27
-
28
-
(** Convert world coordinates to containing registry block coordinates *)
29
-
let block_from_world coord =
30
-
let lat = Coord.lat_float coord in
31
-
let lon = Coord.lon_float coord in
32
-
let block_lon = int_of_float (floor (lon /. float_of_int block_size)) * block_size in
33
-
let block_lat = int_of_float (floor (lat /. float_of_int block_size)) * block_size in
34
-
{ block_lon; block_lat }
35
-
36
-
(** Get all blocks that intersect with a bounding box *)
37
-
let blocks_from_bbox bbox =
38
-
let min_lat = Bbox.min_lat bbox in
39
-
let min_lon = Bbox.min_lon bbox in
40
-
let max_lat = Bbox.max_lat bbox in
41
-
let max_lon = Bbox.max_lon bbox in
42
-
43
-
let min_coord = Coord.create ~lat:min_lat ~lon:min_lon in
44
-
let max_coord = Coord.create ~lat:max_lat ~lon:max_lon in
45
-
46
-
let min_block = block_from_world min_coord in
47
-
let max_block = block_from_world max_coord in
48
-
49
-
let blocks = ref [] in
50
-
let lon = ref min_block.block_lon in
51
-
while !lon <= max_block.block_lon do
52
-
let lat = ref min_block.block_lat in
53
-
while !lat <= max_block.block_lat do
54
-
blocks := { block_lon = !lon; block_lat = !lat } :: !blocks;
55
-
lat := !lat + block_size
56
-
done;
57
-
lon := !lon + block_size
58
-
done;
59
-
List.rev !blocks
60
-
61
-
(** Convert world coordinates to tile coordinates (centered at 0.05 offset) *)
62
-
let tile_from_world coord =
63
-
let lat = Coord.lat_float coord in
64
-
let lon = Coord.lon_float coord in
65
-
(* Tiles are centered at 0.05 degree offsets *)
66
-
let tile_lon = floor (lon /. tile_size) *. tile_size +. 0.05 in
67
-
let tile_lat = floor (lat /. tile_size) *. tile_size +. 0.05 in
68
-
{ tile_lon; tile_lat }
69
-
70
-
(** Get all tiles within a bounding box *)
71
-
let tiles_from_bbox bbox =
72
-
let min_lat = Bbox.min_lat bbox in
73
-
let min_lon = Bbox.min_lon bbox in
74
-
let max_lat = Bbox.max_lat bbox in
75
-
let max_lon = Bbox.max_lon bbox in
76
-
77
-
let min_coord = Coord.create ~lat:min_lat ~lon:min_lon in
78
-
let max_coord = Coord.create ~lat:max_lat ~lon:max_lon in
79
-
80
-
let min_tile = tile_from_world min_coord in
81
-
let max_tile = tile_from_world max_coord in
82
-
83
-
let tiles = ref [] in
84
-
let lon = ref min_tile.tile_lon in
85
-
while !lon <= max_tile.tile_lon do
86
-
let lat = ref min_tile.tile_lat in
87
-
while !lat <= max_tile.tile_lat do
88
-
tiles := { tile_lon = !lon; tile_lat = !lat } :: !tiles;
89
-
lat := !lat +. tile_size
90
-
done;
91
-
lon := !lon +. tile_size
92
-
done;
93
-
List.rev !tiles
94
-
95
-
(** Generate embeddings registry filename for a block *)
96
-
let embeddings_registry_filename ~year block =
97
-
Printf.sprintf "embeddings_%s_lon%d_lat%d.txt" year block.block_lon block.block_lat
98
-
99
-
(** Generate landmasks registry filename for a block *)
100
-
let landmasks_registry_filename block =
101
-
Printf.sprintf "landmasks_lon%d_lat%d.txt" block.block_lon block.block_lat
102
-
103
-
(** Generate tile filename for embeddings *)
104
-
let embeddings_tile_filename ~year tile =
105
-
Printf.sprintf "%s/grid_%.2f_%.2f/grid_%.2f_%.2f.npy"
106
-
year tile.tile_lon tile.tile_lat tile.tile_lon tile.tile_lat
107
-
108
-
(** Generate tile filename for landmasks *)
109
-
let landmasks_tile_filename tile =
110
-
Printf.sprintf "grid_%.2f_%.2f.tiff" tile.tile_lon tile.tile_lat
111
-
112
-
(** Parse a bounding box from string format "min_lon,min_lat,max_lon,max_lat" *)
113
-
let parse_bbox str =
114
-
try
115
-
Ok (Bbox.of_string str)
116
-
with Lonlat.Error.Coordinate_error e ->
117
-
Error (Lonlat.Error.to_string e)
118
-
119
-
(** Format a bounding box for display *)
120
-
let format_bbox bbox = Bbox.to_string bbox
121
-
122
-
(** Calculate the number of tiles in a bounding box *)
123
-
let count_tiles_in_bbox bbox =
124
-
let tiles = tiles_from_bbox bbox in
125
-
List.length tiles
-15
stack/mltessera/lib/dune
-15
stack/mltessera/lib/dune
-127
stack/mltessera/lib/mltessera.ml
-127
stack/mltessera/lib/mltessera.ml
···
1
-
(** MLTessera - Main library interface
2
-
3
-
This module provides the high-level interface for downloading
4
-
tessera tiles within bounding boxes.
5
-
*)
6
-
7
-
open Eio
8
-
9
-
let src = Logs.Src.create "mltessera" ~doc:"MLTessera main"
10
-
module Log = (val Logs.src_log src : Logs.LOG)
11
-
12
-
module Coordinates = Coordinates
13
-
module Registry_manager = Registry_manager
14
-
module Tile_downloader = Tile_downloader
15
-
16
-
type t = {
17
-
sw : Switch.t;
18
-
env : Eio_unix.Stdenv.base;
19
-
registry_manager : Registry_manager.t;
20
-
downloader : Tile_downloader.t;
21
-
}
22
-
23
-
type data_type = [`Embeddings | `Landmasks]
24
-
25
-
(** Initialize MLTessera *)
26
-
let create ~sw ~env ?base_url ?cache_dir () =
27
-
Log.info (fun m -> m "Initializing MLTessera...");
28
-
29
-
(* Create registry manager *)
30
-
let registry_manager = Registry_manager.create ~sw ~env in
31
-
32
-
(* Create tile downloader - base_url is deprecated, using default URLs *)
33
-
let downloader = match base_url with
34
-
| Some _url ->
35
-
Log.warn (fun m -> m "base_url parameter is deprecated, using default URLs");
36
-
Tile_downloader.create ~sw ~env ?cache_dir ()
37
-
| None ->
38
-
Tile_downloader.create ~sw ~env ?cache_dir ()
39
-
in
40
-
41
-
{ sw; env; registry_manager; downloader }
42
-
43
-
(** Download tiles within a bounding box *)
44
-
let download_bbox t ~data_type ~year ~bbox ?(concurrency = 4) () =
45
-
Log.info (fun m -> m "Downloading %s tiles for %s in %s"
46
-
(match data_type with `Embeddings -> "embeddings" | `Landmasks -> "landmasks")
47
-
year (Coordinates.format_bbox bbox));
48
-
49
-
(* Calculate number of tiles *)
50
-
let num_tiles = Coordinates.count_tiles_in_bbox bbox in
51
-
Log.info (fun m -> m "Bounding box contains up to %d tiles" num_tiles);
52
-
53
-
(* Download tiles *)
54
-
let results = Tile_downloader.download_bbox t.downloader
55
-
~registry_manager:t.registry_manager
56
-
~data_type ~year bbox ~concurrency () in
57
-
58
-
(* Count successes *)
59
-
let successes = List.filter (fun (_, r) ->
60
-
match r with Ok _ -> true | Error _ -> false
61
-
) results in
62
-
63
-
Log.info (fun m -> m "Downloaded %d/%d tiles successfully"
64
-
(List.length successes) (List.length results));
65
-
66
-
results
67
-
68
-
(** Download tiles for a geographic region *)
69
-
let download_region t ~data_type ~year ~region ?(concurrency = 4) () =
70
-
match region with
71
-
| `Bbox bbox -> download_bbox t ~data_type ~year ~bbox ~concurrency ()
72
-
| `Country _country ->
73
-
(* TODO: Implement country bbox lookup *)
74
-
failwith "Country regions not yet implemented"
75
-
76
-
(** Get available years *)
77
-
let get_available_years t =
78
-
Registry_manager.get_available_years t.registry_manager
79
-
80
-
(** Update the tessera-manifests repository *)
81
-
let update_manifests t =
82
-
Log.info (fun m -> m "Updating tessera-manifests repository...");
83
-
Registry_manager.update t.registry_manager;
84
-
Log.info (fun m -> m "Repository updated")
85
-
86
-
(** Get cache statistics *)
87
-
let get_cache_stats t =
88
-
let (size, count) = Tile_downloader.cache_stats t.downloader in
89
-
Log.info (fun m -> m "Cache contains %d tiles using %Ld bytes" count size);
90
-
(size, count)
91
-
92
-
(** Clear the tile cache *)
93
-
let clear_cache t =
94
-
Log.info (fun m -> m "Clearing tile cache...");
95
-
Tile_downloader.clear_cache t.downloader
96
-
97
-
(** Export downloaded tiles to a directory *)
98
-
let export_tiles _t ~dest_dir results =
99
-
Log.info (fun m -> m "Exporting %d tiles to %s"
100
-
(List.length results) dest_dir);
101
-
102
-
(* Ensure destination directory exists *)
103
-
(try Unix.mkdir dest_dir 0o755 with Unix.Unix_error _ -> ());
104
-
105
-
(* Copy successful downloads *)
106
-
let exported = ref 0 in
107
-
List.iter (fun (entry, result) ->
108
-
match result with
109
-
| Ok src_path ->
110
-
let filename = Toru.Registry.filename entry in
111
-
let dest_path = Filename.concat dest_dir (Filename.basename filename) in
112
-
113
-
(* Copy file *)
114
-
let src = Path.native_exn src_path in
115
-
let cmd = Printf.sprintf "cp %s %s"
116
-
(Filename.quote src) (Filename.quote dest_path) in
117
-
118
-
if Sys.command cmd = 0 then begin
119
-
incr exported;
120
-
Log.debug (fun m -> m "Exported: %s" dest_path)
121
-
end else
122
-
Log.warn (fun m -> m "Failed to export: %s" filename)
123
-
| Error _ -> ()
124
-
) results;
125
-
126
-
Log.info (fun m -> m "Exported %d tiles" !exported);
127
-
!exported
-208
stack/mltessera/lib/registry_manager.ml
-208
stack/mltessera/lib/registry_manager.ml
···
1
-
(** Registry manager for tessera-manifests
2
-
3
-
This module manages the tessera-manifests git repository and
4
-
provides access to block-based registry files.
5
-
*)
6
-
7
-
open Eio
8
-
9
-
let src = Logs.Src.create "mltessera.registry" ~doc:"MLTessera registry manager"
10
-
module Log = (val Logs.src_log src : Logs.LOG)
11
-
12
-
type t = {
13
-
sw : Switch.t;
14
-
env : Eio_unix.Stdenv.base;
15
-
manifests_dir : Fs.dir_ty Path.t;
16
-
cache_dir : Fs.dir_ty Path.t;
17
-
registries : (string, Toru.Registry.t) Hashtbl.t;
18
-
}
19
-
20
-
let tessera_manifests_url = "https://github.com/ucam-eo/tessera-manifests.git"
21
-
22
-
(** Get XDG-compliant directory for tessera manifests *)
23
-
let get_manifests_dir ~sw:_ ~env =
24
-
(* Use XDG_CACHE_HOME if set, otherwise default to ~/.cache *)
25
-
let cache_home = match Sys.getenv_opt "XDG_CACHE_HOME" with
26
-
| Some dir -> dir
27
-
| None ->
28
-
match Sys.getenv_opt "HOME" with
29
-
| Some home -> Filename.concat home ".cache"
30
-
| None -> "/tmp"
31
-
in
32
-
33
-
let mltessera_cache = Filename.concat cache_home "mltessera" in
34
-
let manifests_path = Filename.concat mltessera_cache "tessera-manifests" in
35
-
36
-
(* Ensure directories exist *)
37
-
(try Unix.mkdir cache_home 0o755 with Unix.Unix_error _ -> ());
38
-
(try Unix.mkdir mltessera_cache 0o755 with Unix.Unix_error _ -> ());
39
-
40
-
(* Return as Path.t *)
41
-
Path.(Eio.Stdenv.fs env / manifests_path)
42
-
43
-
(** Clone or update the tessera-manifests repository *)
44
-
let ensure_manifests_repo ~sw:_ ~env manifests_dir =
45
-
let manifests_path = Path.native_exn manifests_dir in
46
-
47
-
if Sys.file_exists (Filename.concat manifests_path ".git") then begin
48
-
Log.info (fun m -> m "Updating tessera-manifests repository...");
49
-
50
-
(* Pull latest changes *)
51
-
let proc_mgr = env#process_mgr in
52
-
try
53
-
Process.run ~cwd:manifests_dir proc_mgr
54
-
["git"; "pull"; "--quiet"];
55
-
Log.info (fun m -> m "Repository updated successfully")
56
-
with exn ->
57
-
Log.warn (fun m -> m "Failed to update repository: %s" (Printexc.to_string exn))
58
-
end else begin
59
-
Log.info (fun m -> m "Cloning tessera-manifests repository to %s..." manifests_path);
60
-
61
-
(* Ensure parent directory exists *)
62
-
let manifests_parent = Filename.dirname manifests_path in
63
-
(try Unix.mkdir manifests_parent 0o755 with Unix.Unix_error _ -> ());
64
-
65
-
(* Clone the repository *)
66
-
let proc_mgr = env#process_mgr in
67
-
let parent_path = Path.(Eio.Stdenv.fs env / manifests_parent) in
68
-
try
69
-
Process.run ~cwd:parent_path proc_mgr
70
-
["git"; "clone"; "--depth"; "1"; tessera_manifests_url];
71
-
Log.info (fun m -> m "Repository cloned successfully")
72
-
with exn ->
73
-
failwith (Printf.sprintf "Failed to clone repository: %s" (Printexc.to_string exn))
74
-
end
75
-
76
-
(** Create a registry manager *)
77
-
let create ~sw ~env =
78
-
let manifests_dir = get_manifests_dir ~sw ~env in
79
-
let cache_dir = Path.(manifests_dir / ".cache") in
80
-
81
-
(* Ensure manifests repo is available *)
82
-
ensure_manifests_repo ~sw ~env manifests_dir;
83
-
84
-
(* Create cache directory *)
85
-
(try Path.mkdir ~perm:0o755 cache_dir with
86
-
| Eio.Io _ -> ()
87
-
| _ -> ());
88
-
89
-
{
90
-
sw;
91
-
env;
92
-
manifests_dir;
93
-
cache_dir;
94
-
registries = Hashtbl.create 32;
95
-
}
96
-
97
-
(** Load a registry file from the manifests repository *)
98
-
let load_registry t ~subdir ~filename =
99
-
let cache_key = Printf.sprintf "%s/%s" subdir filename in
100
-
101
-
(* Check cache first *)
102
-
match Hashtbl.find_opt t.registries cache_key with
103
-
| Some registry -> registry
104
-
| None ->
105
-
let registry_path = Path.(t.manifests_dir / "registry" / subdir / filename) in
106
-
107
-
if not (Path.is_file registry_path) then
108
-
failwith (Printf.sprintf "Registry file not found: %s" (Path.native_exn registry_path));
109
-
110
-
Log.info (fun m -> m "Loading registry: %s/%s" subdir filename);
111
-
112
-
(* Load the registry *)
113
-
let registry = Toru.Registry.load registry_path in
114
-
115
-
(* Debug: Show number of entries loaded *)
116
-
let entries = Toru.Registry.entries registry in
117
-
Log.info (fun m -> m "Loaded %d entries from %s/%s" (List.length entries) subdir filename);
118
-
119
-
(* Debug: Show first few entries if any *)
120
-
(match entries with
121
-
| e1 :: e2 :: e3 :: _ ->
122
-
Log.debug (fun m -> m " Sample entries:");
123
-
Log.debug (fun m -> m " %s" (Toru.Registry.filename e1));
124
-
Log.debug (fun m -> m " %s" (Toru.Registry.filename e2));
125
-
Log.debug (fun m -> m " %s" (Toru.Registry.filename e3))
126
-
| _ -> ());
127
-
128
-
(* Cache it *)
129
-
Hashtbl.add t.registries cache_key registry;
130
-
registry
131
-
132
-
(** Load embeddings registry for a specific block and year *)
133
-
let load_embeddings_registry t ~year block =
134
-
let filename = Coordinates.embeddings_registry_filename ~year block in
135
-
load_registry t ~subdir:"embeddings" ~filename
136
-
137
-
(** Load landmasks registry for a specific block *)
138
-
let load_landmasks_registry t block =
139
-
let filename = Coordinates.landmasks_registry_filename block in
140
-
load_registry t ~subdir:"landmasks" ~filename
141
-
142
-
(** Get all available years for embeddings *)
143
-
let get_available_years t =
144
-
let embeddings_dir = Path.(t.manifests_dir / "registry" / "embeddings") in
145
-
146
-
if not (Path.is_directory embeddings_dir) then []
147
-
else
148
-
let years = ref [] in
149
-
let files = Path.read_dir embeddings_dir in
150
-
List.iter (fun name ->
151
-
(* Extract year from filename like "embeddings_2024_lon0_lat0.txt" *)
152
-
if String.starts_with ~prefix:"embeddings_" name then
153
-
match String.split_on_char '_' name with
154
-
| _ :: year :: _ ->
155
-
if not (List.mem year !years) then
156
-
years := year :: !years
157
-
| _ -> ()
158
-
) files;
159
-
List.sort String.compare !years
160
-
161
-
(** Load all registries for blocks intersecting a bounding box *)
162
-
let load_bbox_registries t ~data_type ~year bbox =
163
-
let blocks = Coordinates.blocks_from_bbox bbox in
164
-
165
-
Log.info (fun m -> m "Loading %d registry blocks for bbox %s"
166
-
(List.length blocks) (Coordinates.format_bbox bbox));
167
-
168
-
List.map (fun block ->
169
-
let registry = match data_type with
170
-
| `Embeddings -> load_embeddings_registry t ~year block
171
-
| `Landmasks -> load_landmasks_registry t block
172
-
in
173
-
(block, registry)
174
-
) blocks
175
-
176
-
(** Find a tile entry across multiple registries *)
177
-
let find_tile_entry registries filename =
178
-
List.find_map (fun (_block, registry) ->
179
-
Toru.Registry.find filename registry
180
-
) registries
181
-
182
-
(** Get all tile entries within a bounding box *)
183
-
let get_bbox_tiles t ~data_type ~year bbox =
184
-
(* Load relevant registries *)
185
-
let registries = load_bbox_registries t ~data_type ~year bbox in
186
-
187
-
(* Get all tiles in bbox *)
188
-
let tiles = Coordinates.tiles_from_bbox bbox in
189
-
190
-
(* Map tiles to registry entries *)
191
-
List.filter_map (fun tile ->
192
-
let filename = match data_type with
193
-
| `Embeddings -> Coordinates.embeddings_tile_filename ~year tile
194
-
| `Landmasks -> Coordinates.landmasks_tile_filename tile
195
-
in
196
-
197
-
match find_tile_entry registries filename with
198
-
| Some entry -> Some (tile, entry)
199
-
| None ->
200
-
Log.warn (fun m -> m "File not found in registry: %s" filename);
201
-
None
202
-
) tiles
203
-
204
-
(** Update the manifests repository *)
205
-
let update t =
206
-
ensure_manifests_repo ~sw:t.sw ~env:t.env t.manifests_dir;
207
-
(* Clear cache after update *)
208
-
Hashtbl.clear t.registries
-163
stack/mltessera/lib/tile_downloader.ml
-163
stack/mltessera/lib/tile_downloader.ml
···
1
-
(** Tile downloader using Toru
2
-
3
-
This module downloads tessera tiles from dl.geotessera.org using
4
-
the Toru library for HTTP downloading and caching.
5
-
*)
6
-
7
-
open Eio
8
-
9
-
let src = Logs.Src.create "mltessera.downloader" ~doc:"MLTessera tile downloader"
10
-
module Log = (val Logs.src_log src : Logs.LOG)
11
-
12
-
type t = {
13
-
toru : Toru.t;
14
-
embeddings_base_url : string;
15
-
landmasks_base_url : string;
16
-
}
17
-
18
-
let default_embeddings_base_url = "https://dl.geotessera.org/v1/global_0.1_degree_representation/"
19
-
let default_landmasks_base_url = "https://dl.geotessera.org/v1/global_0.1_degree_tiff_all/"
20
-
21
-
(** Create a tile downloader *)
22
-
let create ~sw ~env ?(embeddings_base_url = default_embeddings_base_url)
23
-
?(landmasks_base_url = default_landmasks_base_url) ?(cache_dir = None) () =
24
-
(* Get cache directory *)
25
-
let cache_path = match cache_dir with
26
-
| Some dir -> dir
27
-
| None ->
28
-
(* Use XDG_CACHE_HOME if set, otherwise default to ~/.cache *)
29
-
let cache_home = match Sys.getenv_opt "XDG_CACHE_HOME" with
30
-
| Some dir -> dir
31
-
| None ->
32
-
match Sys.getenv_opt "HOME" with
33
-
| Some home -> Filename.concat home ".cache"
34
-
| None -> "/tmp"
35
-
in
36
-
let mltessera_cache = Filename.concat cache_home "mltessera" in
37
-
let tiles_cache = Filename.concat mltessera_cache "tiles" in
38
-
39
-
(* Ensure directories exist *)
40
-
(try Unix.mkdir cache_home 0o755 with Unix.Unix_error _ -> ());
41
-
(try Unix.mkdir mltessera_cache 0o755 with Unix.Unix_error _ -> ());
42
-
(try Unix.mkdir tiles_cache 0o755 with Unix.Unix_error _ -> ());
43
-
44
-
tiles_cache
45
-
in
46
-
47
-
(* Create Toru instance with embeddings base URL as default *)
48
-
let toru = Toru.create ~sw ~env
49
-
~base_url:embeddings_base_url
50
-
~cache_path
51
-
()
52
-
in
53
-
54
-
{ toru; embeddings_base_url; landmasks_base_url }
55
-
56
-
(** Download a single tile *)
57
-
let download_tile t ~data_type entry =
58
-
let filename = Toru.Registry.filename entry in
59
-
60
-
Log.info (fun m -> m "Downloading tile: %s" filename);
61
-
62
-
(* Update Toru with the correct base URL for this data type *)
63
-
let base_url = match data_type with
64
-
| `Embeddings -> t.embeddings_base_url
65
-
| `Landmasks -> t.landmasks_base_url
66
-
in
67
-
let toru_with_base = Toru.update_base_url t.toru base_url in
68
-
69
-
(* Add the entry to Toru's registry so it can be fetched *)
70
-
let toru_with_entry = Toru.add_registry_entry toru_with_base entry in
71
-
72
-
match Toru.fetch toru_with_entry ~filename () with
73
-
| Ok path ->
74
-
Log.info (fun m -> m "Downloaded: %s -> %s" filename (Path.native_exn path));
75
-
Ok path
76
-
| Error msg ->
77
-
Log.err (fun m -> m "Failed to download %s: %s" filename msg);
78
-
Error msg
79
-
80
-
(** Download multiple tiles concurrently *)
81
-
let download_tiles t ~data_type entries ?(concurrency = 4) () =
82
-
let total = List.length entries in
83
-
let completed = ref 0 in
84
-
let failed = ref [] in
85
-
86
-
Log.info (fun m -> m "Downloading %d tiles with concurrency %d" total concurrency);
87
-
88
-
(* Create a pool for concurrent downloads *)
89
-
let pool = Eio.Pool.create concurrency (fun () -> ()) in
90
-
91
-
(* Download each tile *)
92
-
let results = List.map (fun entry ->
93
-
Pool.use pool @@ fun () ->
94
-
let result = download_tile t ~data_type entry in
95
-
96
-
(* Update progress *)
97
-
incr completed;
98
-
let pct = float_of_int !completed /. float_of_int total *. 100.0 in
99
-
Log.info (fun m -> m "Progress: %d/%d (%.1f%%)" !completed total pct);
100
-
101
-
match result with
102
-
| Ok path -> (entry, Ok path)
103
-
| Error msg ->
104
-
failed := Toru.Registry.filename entry :: !failed;
105
-
(entry, Error msg)
106
-
) entries in
107
-
108
-
(* Report summary *)
109
-
let successful = total - List.length !failed in
110
-
Log.info (fun m -> m "Download complete: %d successful, %d failed"
111
-
successful (List.length !failed));
112
-
113
-
if !failed <> [] then
114
-
Log.warn (fun m -> m "Failed tiles: %s" (String.concat ", " !failed));
115
-
116
-
results
117
-
118
-
(** Download all tiles within a bounding box *)
119
-
let download_bbox t ~registry_manager ~data_type ~year bbox ?(concurrency = 4) () =
120
-
Log.info (fun m -> m "Downloading %s tiles for year %s in bbox %s"
121
-
(match data_type with `Embeddings -> "embeddings" | `Landmasks -> "landmasks")
122
-
year (Coordinates.format_bbox bbox));
123
-
124
-
(* Get tiles from registry *)
125
-
let tile_entries = Registry_manager.get_bbox_tiles
126
-
registry_manager ~data_type ~year bbox in
127
-
128
-
if tile_entries = [] then begin
129
-
Log.warn (fun m -> m "No tiles found in bbox %s" (Coordinates.format_bbox bbox));
130
-
[]
131
-
end else begin
132
-
Log.info (fun m -> m "Found %d tiles to download" (List.length tile_entries));
133
-
134
-
(* Extract just the entries *)
135
-
let entries = List.map snd tile_entries in
136
-
137
-
(* Download them *)
138
-
download_tiles t ~data_type entries ~concurrency ()
139
-
end
140
-
141
-
(** Get the local path for a tile if it exists in cache *)
142
-
let get_cached_tile t entry =
143
-
let filename = Toru.Registry.filename entry in
144
-
let cache = Toru.cache t.toru in
145
-
let path = Toru.Cache.file_path cache filename in
146
-
147
-
if Toru.Cache.exists cache filename then
148
-
Some path
149
-
else
150
-
None
151
-
152
-
(** Clear the tile cache *)
153
-
let clear_cache t =
154
-
let cache = Toru.cache t.toru in
155
-
Toru.Cache.clear cache;
156
-
Log.info (fun m -> m "Tile cache cleared")
157
-
158
-
(** Get cache statistics *)
159
-
let cache_stats t =
160
-
let cache = Toru.cache t.toru in
161
-
let size = Toru.Cache.size_bytes cache in
162
-
let files = Toru.Cache.list_files cache in
163
-
(size, List.length files)
-35
stack/mltessera/mltessera.opam
-35
stack/mltessera/mltessera.opam
···
1
-
# This file is generated by dune, edit dune-project instead
2
-
opam-version: "2.0"
3
-
synopsis: "OCaml library for downloading tessera tiles from geotessera.org"
4
-
description:
5
-
"MLTessera is an OCaml equivalent of the Python geotessera library. It manages tessera-manifests registry files and downloads geospatial tiles within bounding boxes using the Toru library for HTTP operations."
6
-
maintainer: ["maintainer@example.com"]
7
-
authors: ["MLTessera Authors"]
8
-
license: "MIT"
9
-
depends: [
10
-
"ocaml"
11
-
"dune" {>= "3.0"}
12
-
"toru"
13
-
"lonlat"
14
-
"cmdliner" {>= "2.0.0"}
15
-
"eio"
16
-
"eio_main"
17
-
"yojson"
18
-
"logs"
19
-
"fmt"
20
-
"odoc" {with-doc}
21
-
]
22
-
build: [
23
-
["dune" "subst"] {dev}
24
-
[
25
-
"dune"
26
-
"build"
27
-
"-p"
28
-
name
29
-
"-j"
30
-
jobs
31
-
"@install"
32
-
"@runtest" {with-test}
33
-
"@doc" {with-doc}
34
-
]
35
-
]
-18
stack/mltessera/test/dune
-18
stack/mltessera/test/dune
···
1
-
(executable
2
-
(public_name test_coordinates)
3
-
(name test_coordinates)
4
-
(libraries mltessera lonlat))
5
-
6
-
(executable
7
-
(public_name test_tile_download)
8
-
(name test_tile_download)
9
-
(libraries
10
-
mltessera
11
-
lonlat
12
-
toru
13
-
requests
14
-
eio
15
-
eio_main
16
-
logs
17
-
logs.fmt
18
-
digestif))
-19
stack/mltessera/test/test_coordinates.ml
-19
stack/mltessera/test/test_coordinates.ml
···
1
-
open Mltessera
2
-
open Lonlat
3
-
4
-
let () =
5
-
(* Test tile generation for London coordinates *)
6
-
let coord = Coord.create ~lat:51.45 ~lon:(-0.15) in
7
-
let tile = Coordinates.tile_from_world coord in
8
-
let filename = Coordinates.embeddings_tile_filename ~year:"2024" tile in
9
-
Printf.printf "Generated filename: %s\n" filename;
10
-
Printf.printf "Tile lon: %.2f, lat: %.2f\n" tile.tile_lon tile.tile_lat;
11
-
12
-
(* Test a few more tiles in the bbox *)
13
-
let bbox = Bbox.create ~min_lat:51.4 ~min_lon:(-0.2) ~max_lat:51.6 ~max_lon:0.0 in
14
-
let tiles = Coordinates.tiles_from_bbox bbox in
15
-
Printf.printf "\nAll tiles in bbox:\n";
16
-
List.iter (fun t ->
17
-
let fname = Coordinates.embeddings_tile_filename ~year:"2024" t in
18
-
Printf.printf " %.2f,%.2f -> %s\n" t.tile_lon t.tile_lat fname
19
-
) tiles
-11
stack/mltessera/test_tiles.ml
-11
stack/mltessera/test_tiles.ml
···
1
-
open Mltessera
2
-
open Lonlat
3
-
4
-
let () =
5
-
let coord = Coord.create ~lat:51.45 ~lon:(-0.15) in
6
-
let tile = Coordinates.tile_from_world coord in
7
-
let filename = Coordinates.embeddings_tile_filename ~year:"2024" tile in
8
-
Printf.printf "Generated filename: %s\n" filename;
9
-
10
-
(* Also show the tile coordinates *)
11
-
Printf.printf "Tile lon: %.2f, lat: %.2f\n" tile.tile_lon tile.tile_lat
-4
stack/requests/lib/requests.ml
-4
stack/requests/lib/requests.ml
···
27
27
sw : Eio.Switch.t;
28
28
clock : 'clock;
29
29
net : 'net;
30
-
31
-
(* Connection pools owned by this session *)
32
30
http_pool : ('clock, 'net) Conpool.t;
33
31
https_pool : ('clock, 'net) Conpool.t;
34
-
35
-
(* Session state - immutable *)
36
32
cookie_jar : Cookeio.jar;
37
33
cookie_mutex : Eio.Mutex.t;
38
34
default_headers : Headers.t;