forked from
anil.recoil.org/monopam-myspace
My aggregated monorepo of OCaml code, automaintained
Support GeoTessera Zarr v2 store with year as dimension
The v2 store uses a single store.zarr with 4D embeddings [time, band, y, x]
instead of per-year stores with 3D [y, x, band]. Auto-detects version from
array dimensionality, keeping full v1 backward compatibility via base_url.
Adds year selector (2017-2025) to the interactive map notebook.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+154
-36
+47
-9
site/notebooks/interactive_map_zarr.mld
+47
-9
site/notebooks/interactive_map_zarr.mld
···
4
4
{{:https://geotessera.org}GeoTessera} foundation model
5
5
directly in the browser. This version uses the GeoTessera
6
6
{{:https://anil.recoil.org/notes/tessera-zarr-v3-layout}Zarr v3 store}
7
-
and a reactive (FRP) architecture — drawing a bounding box
8
-
automatically triggers fetching, PCA visualisation, and overlay
9
-
updates.
7
+
(v2 format — single store with year as a dimension) and a reactive
8
+
(FRP) architecture — drawing a bounding box automatically triggers
9
+
fetching, PCA visualisation, and overlay updates.
10
10
11
11
{@ocaml kind=setup[
12
12
#require "tessera-zarr-jsoo";;
···
18
18
(* Load fzstd (Zstd decompressor) and TensorFlow.js *)
19
19
let () =
20
20
let open Js_of_ocaml in
21
-
let import url = Js.Unsafe.fun_call
21
+
let import url : unit = Js.Unsafe.fun_call
22
22
(Js.Unsafe.get Js.Unsafe.global (Js.string "importScripts"))
23
23
[| Js.Unsafe.inject (Js.string url) |] in
24
24
import "https://cdn.jsdelivr.net/npm/fzstd@0.1.1/umd/index.js";
···
49
49
50
50
let bbox_signal, set_bbox =
51
51
Note.S.create ~eq:(fun a b -> a = b) (None : Geotessera.bbox option)
52
+
53
+
let year_signal, set_year = Note.S.create ~eq:Int.equal 2024
52
54
53
55
let current_class_signal, set_current_class = Note.S.create ~eq:Int.equal 0
54
56
let class_names_signal, set_class_names =
···
118
120
done;
119
121
(out, h', w')
120
122
123
+
let bbox_year_signal =
124
+
Note.S.l2 (fun b y -> Option.map (fun b -> (b, y)) b) bbox_signal year_signal
125
+
121
126
let mosaic_signal : mosaic_data Tessera_zarr_jsoo.Frp_async.state Note.signal =
122
-
Tessera_zarr_jsoo.Frp_async.async_bind bbox_signal
123
-
(fun bbox progress ->
124
-
progress "Opening Zarr store...";
127
+
Tessera_zarr_jsoo.Frp_async.async_bind bbox_year_signal
128
+
(fun (bbox, year) progress ->
129
+
progress (Printf.sprintf "Opening Zarr v2 store (year %d)..." year);
125
130
let open Lwt.Syntax in
126
-
let* store = Tessera_zarr_jsoo.open_store () in
131
+
let* store = Tessera_zarr_jsoo.open_store ~year () in
127
132
let* (mat_full, h_full, w_full, geo_bounds) =
128
-
Tessera_zarr.fetch_region ~progress ~store bbox in
133
+
Tessera_zarr.fetch_region ~progress ~year ~store bbox in
129
134
progress (Printf.sprintf "Fetched %d×%d. Downsampling..." h_full w_full);
130
135
let (mat, h, w) = downsample mat_full ~h:h_full ~w:w_full ~max_pixels:200_000 in
131
136
let bounds = Leaflet_map.{
···
172
177
Widget.update ~id:"status" (status_view text)
173
178
)) (Note.S.obs mosaic_signal))
174
179
in Note.Logr.hold logr
180
+
]}
181
+
182
+
{1 Year selection}
183
+
184
+
Select the year for the GeoTessera embeddings (2017–2025).
185
+
Changing the year re-fetches the current bounding box.
186
+
187
+
{@ocaml x[
188
+
let make_year_buttons () =
189
+
let open Widget.View in
190
+
let current = Note.S.value year_signal in
191
+
let buttons = List.map (fun y ->
192
+
let border = if y = current then "3px solid black" else "1px solid #ccc" in
193
+
Element { tag = "button";
194
+
attrs = [
195
+
Handler ("click", "year_" ^ string_of_int y);
196
+
Style ("margin", "2px"); Style ("padding", "6px 12px");
197
+
Style ("border", border); Style ("border-radius", "4px");
198
+
Style ("cursor", "pointer");
199
+
];
200
+
children = [Text (string_of_int y)] }
201
+
) [2017; 2018; 2019; 2020; 2021; 2022; 2023; 2024; 2025] in
202
+
Element { tag = "div";
203
+
attrs = [Style ("padding", "8px")];
204
+
children = Element { tag = "b"; attrs = []; children = [Text "Year: "] } :: buttons }
205
+
206
+
let () =
207
+
let handlers = List.map (fun y ->
208
+
"year_" ^ string_of_int y, (fun (_v : string option) ->
209
+
set_year y;
210
+
Widget.update ~id:"year-buttons" (make_year_buttons ()))
211
+
) [2017; 2018; 2019; 2020; 2021; 2022; 2023; 2024; 2025] in
212
+
Widget.display ~id:"year-buttons" ~handlers (make_year_buttons ())
175
213
]}
176
214
177
215
{1 Label training points}
+19
-7
tessera-zarr-jsoo/lib/tessera_zarr_jsoo.ml
+19
-7
tessera-zarr-jsoo/lib/tessera_zarr_jsoo.ml
···
73
73
| "zstd" -> Some zstd_decompress
74
74
| _ -> None
75
75
76
-
let open_store ?(base_url = "https://dl2.geotessera.org/zarr/v1")
76
+
(* Build the store URL from a base URL and year.
77
+
v1: base_url is e.g. ".../zarr/v1", store is "{base_url}/{year}.zarr"
78
+
v2: base_url is e.g. ".../zarr/v2/store.zarr", store is just the base_url *)
79
+
let store_url base_url year =
80
+
if String.length base_url > 5 &&
81
+
String.sub base_url (String.length base_url - 5) 5 = ".zarr" then
82
+
(* v2-style: URL already points to the store *)
83
+
base_url
84
+
else
85
+
(* v1-style: append {year}.zarr *)
86
+
Printf.sprintf "%s/%d.zarr" base_url year
87
+
88
+
let open_store ?(base_url = "https://dl2.geotessera.org/zarr/v2/store.zarr")
77
89
?(year = 2024) () =
78
-
let url = Printf.sprintf "%s/%d.zarr" base_url year in
90
+
let url = store_url base_url year in
79
91
Zarr_v3.Store.open_store ~fetch ~codecs url
80
92
81
-
let fetch_region ?progress ?(base_url = "https://dl2.geotessera.org/zarr/v1")
93
+
let fetch_region ?progress ?(base_url = "https://dl2.geotessera.org/zarr/v2/store.zarr")
82
94
?(year = 2024) bbox =
83
95
let open Lwt.Syntax in
84
96
let* store = open_store ~base_url ~year () in
85
-
Tessera_zarr.fetch_region ?progress ~store bbox
97
+
Tessera_zarr.fetch_region ?progress ~year ~store bbox
86
98
87
99
(* Synchronous version for notebook cells.
88
100
Uses synchronous XHR so all Lwt promises resolve immediately. *)
89
-
let fetch_region_sync ?(base_url = "https://dl2.geotessera.org/zarr/v1")
101
+
let fetch_region_sync ?(base_url = "https://dl2.geotessera.org/zarr/v2/store.zarr")
90
102
?(year = 2024) bbox =
91
-
let url = Printf.sprintf "%s/%d.zarr" base_url year in
103
+
let url = store_url base_url year in
92
104
let store_lwt = Zarr_v3.Store.open_store ~fetch:fetch_sync ~codecs url in
93
105
let store = match Lwt.poll store_lwt with
94
106
| Some s -> s
95
107
| None -> failwith "Tessera_zarr_jsoo: unexpected async in store open"
96
108
in
97
-
let result_lwt = Tessera_zarr.fetch_region ~store bbox in
109
+
let result_lwt = Tessera_zarr.fetch_region ~year ~store bbox in
98
110
match Lwt.poll result_lwt with
99
111
| Some v -> v
100
112
| None -> failwith "Tessera_zarr_jsoo: unexpected async in fetch_region"
+10
-3
tessera-zarr-jsoo/lib/tessera_zarr_jsoo.mli
+10
-3
tessera-zarr-jsoo/lib/tessera_zarr_jsoo.mli
···
2
2
3
3
Provides async and sync HTTP fetch with byte-range support,
4
4
convenience wrappers for opening the GeoTessera Zarr store,
5
-
and an FRP bridge for reactive notebooks. *)
5
+
and an FRP bridge for reactive notebooks.
6
+
7
+
Defaults to the v2 store ([store.zarr]) which is a single store
8
+
with year as a dimension. Pass [~base_url] ending in [".zarr"]
9
+
for v2, or a directory URL (e.g., [".../zarr/v1"]) for v1. *)
6
10
7
11
val fetch : Zarr_v3.Store.fetch
8
12
(** Async HTTP fetch via [XMLHttpRequest] with [Range] header support. *)
···
14
18
(** Codec registry. Currently returns [None] for all codecs. *)
15
19
16
20
val open_store : ?base_url:string -> ?year:int -> unit -> Zarr_v3.Store.store Lwt.t
17
-
(** Open the GeoTessera Zarr store (async fetch). *)
21
+
(** Open the GeoTessera Zarr store (async fetch).
22
+
For v2 (default), [year] is used to select the time dimension.
23
+
For v1, [year] selects which per-year store to open. *)
18
24
19
25
val fetch_region : ?progress:(string -> unit) -> ?base_url:string -> ?year:int ->
20
26
Geotessera.bbox -> (Linalg.mat * int * int * Geotessera.bbox) Lwt.t
21
27
(** Async: opens the store and fetches embeddings for a bbox.
22
-
[progress] receives status messages during fetching. *)
28
+
[progress] receives status messages during fetching.
29
+
Defaults to v2 store; pass a v1 [base_url] for backward compat. *)
23
30
24
31
val fetch_region_sync : ?base_url:string -> ?year:int ->
25
32
Geotessera.bbox -> Linalg.mat * int * int * Geotessera.bbox
+43
-10
tessera-zarr/lib/tessera_zarr.ml
+43
-10
tessera-zarr/lib/tessera_zarr.ml
···
31
31
(Int32.shift_left (Int32.of_int (Char.code s.[off+3])) 24))) in
32
32
Int32.float_of_bits bits
33
33
34
-
let fetch_region ?(progress = fun (_:string) -> ()) ~store bbox =
34
+
let fetch_region ?(progress = fun (_:string) -> ()) ?(year = 2024) ~store bbox =
35
35
let open Lwt.Syntax in
36
36
let open Geotessera in
37
37
(* 1. Determine UTM zone from bbox centre *)
···
73
73
let tile_w = col_stop - col_start in
74
74
let n_features = 128 in
75
75
76
-
(* 4. Fetch embeddings and scales in parallel *)
76
+
(* 4. Fetch embeddings and scales in parallel.
77
+
Detect v1 vs v2 by the number of dimensions in the embeddings array:
78
+
v1: [y, x, band] — 3D, one store per year
79
+
v2: [time, band, y, x] — 4D, single store with year as dimension *)
77
80
let* emb_arr = Zarr_v3.Store.open_array store (zone_name ^ "/embeddings") in
78
81
let* scales_arr = Zarr_v3.Store.open_array store (zone_name ^ "/scales") in
79
82
83
+
let emb_meta = Zarr_v3.Store.array_meta emb_arr in
84
+
let is_v2 = Array.length emb_meta.shape = 4 in
85
+
80
86
let on_emb_shard i n =
81
87
progress (Printf.sprintf "Fetching embeddings: shard %d/%d" i n) in
82
88
let on_scales_shard i n =
83
89
progress (Printf.sprintf "Fetching scales: shard %d/%d" i n) in
84
90
85
-
let emb_fetch = Zarr_v3.Store.read ~on_shard:on_emb_shard emb_arr
86
-
~start:[| row_start; col_start; 0 |]
87
-
~shape:[| tile_h; tile_w; n_features |] in
88
-
let scales_fetch = Zarr_v3.Store.read ~on_shard:on_scales_shard scales_arr
89
-
~start:[| row_start; col_start |]
90
-
~shape:[| tile_h; tile_w |] in
91
+
let emb_fetch, scales_fetch =
92
+
if is_v2 then begin
93
+
(* v2: embeddings[time, band, y, x], scales[time, y, x] *)
94
+
let time_idx = year - 2017 in
95
+
let n_times = emb_meta.shape.(0) in
96
+
if time_idx < 0 || time_idx >= n_times then
97
+
failwith (Printf.sprintf "Year %d out of range (2017-%d)" year
98
+
(2016 + n_times));
99
+
( Zarr_v3.Store.read ~on_shard:on_emb_shard emb_arr
100
+
~start:[| time_idx; 0; row_start; col_start |]
101
+
~shape:[| 1; n_features; tile_h; tile_w |],
102
+
Zarr_v3.Store.read ~on_shard:on_scales_shard scales_arr
103
+
~start:[| time_idx; row_start; col_start |]
104
+
~shape:[| 1; tile_h; tile_w |] )
105
+
end else
106
+
(* v1: embeddings[y, x, band], scales[y, x] *)
107
+
( Zarr_v3.Store.read ~on_shard:on_emb_shard emb_arr
108
+
~start:[| row_start; col_start; 0 |]
109
+
~shape:[| tile_h; tile_w; n_features |],
110
+
Zarr_v3.Store.read ~on_shard:on_scales_shard scales_arr
111
+
~start:[| row_start; col_start |]
112
+
~shape:[| tile_h; tile_w |] )
113
+
in
91
114
92
115
let* emb_data = emb_fetch
93
116
and* scales_data = scales_fetch in
94
117
95
118
progress "Dequantizing...";
96
-
(* 5. Dequantize: float32 = int8 × scale *)
119
+
(* 5. Dequantize: float32 = int8 × scale.
120
+
v1 data is C-order [y, x, band] → offset = (row*W + col)*128 + f
121
+
v2 data is C-order [1, band, y, x] → offset = (f*H*W + row*W + col) *)
97
122
let mat = Linalg.create_mat ~rows:(tile_h * tile_w) ~cols:n_features in
98
123
for i = 0 to tile_h - 1 do
99
124
for j = 0 to tile_w - 1 do
100
125
let pixel = i * tile_w + j in
126
+
(* Scales offset is pixel*4 for both v1 [H,W] and v2 [1,H,W] *)
101
127
let scale = read_f32_le scales_data (pixel * 4) in
102
128
for f = 0 to n_features - 1 do
103
-
let e_off = (pixel * n_features + f) in
129
+
let e_off =
130
+
if is_v2 then
131
+
(* v2: C-order [1, 128, H, W] → offset = f*H*W + pixel *)
132
+
f * tile_h * tile_w + pixel
133
+
else
134
+
(* v1: C-order [H, W, 128] → offset = pixel*128 + f *)
135
+
pixel * n_features + f
136
+
in
104
137
let emb_val = Char.code emb_data.[e_off] in
105
138
let emb_signed = if emb_val >= 128 then emb_val - 256 else emb_val in
106
139
Linalg.mat_set mat pixel f (Float.of_int emb_signed *. scale)
+35
-7
tessera-zarr/lib/tessera_zarr.mli
+35
-7
tessera-zarr/lib/tessera_zarr.mli
···
5
5
int8 embeddings using float32 scales and reprojects from the
6
6
native UTM grid to a regular WGS84 grid.
7
7
8
+
Supports both v1 (per-year stores) and v2 (single store with
9
+
year as dimension). The version is detected automatically from
10
+
the array dimensionality.
11
+
8
12
{2 Example}
9
13
10
14
{[
15
+
(* v1: one store per year *)
11
16
let store = Zarr_v3.Store.open_store ~fetch ~codecs
12
17
"https://dl2.geotessera.org/zarr/v1/2024.zarr" in
13
-
let bbox = Geotessera.{ min_lon = 0.1; min_lat = 52.1;
14
-
max_lon = 0.2; max_lat = 52.2 } in
15
18
let (mat, h, w, bounds) = Tessera_zarr.fetch_region ~store bbox in
16
-
(* mat has h*w rows and 128 columns of float32 dequantized embeddings *)
19
+
20
+
(* v2: single store, year as dimension *)
21
+
let store = Zarr_v3.Store.open_store ~fetch ~codecs
22
+
"https://dl2.geotessera.org/zarr/v2/store.zarr" in
23
+
let (mat, h, w, bounds) = Tessera_zarr.fetch_region ~year:2024 ~store bbox in
17
24
]}
18
25
19
-
{2 Store layout}
26
+
{2 Store layouts}
20
27
21
-
The GeoTessera Zarr store is organised by UTM zone:
28
+
{b v1} — one store per year:
22
29
{[
23
30
2024.zarr/
24
31
├── zarr.json (consolidated metadata for all zones)
···
30
37
└── ...
31
38
]}
32
39
40
+
{b v2} — single store, year as first dimension:
41
+
{[
42
+
store.zarr/
43
+
├── zarr.json (consolidated metadata for all zones)
44
+
├── utm30/
45
+
│ ├── embeddings (int8, T×128×H×W, sharded 1×128×4096×4096)
46
+
│ ├── scales (float32, T×H×W, sharded 1×4096×4096)
47
+
│ └── ...
48
+
└── utm31/
49
+
└── ...
50
+
]}
51
+
33
52
Each zone group carries [spatial:transform] (6-element affine)
34
53
and [proj:code] (e.g., ["EPSG:32631"]) attributes. *)
35
54
···
52
71
53
72
val fetch_region :
54
73
?progress:(string -> unit) ->
74
+
?year:int ->
55
75
store:Zarr_v3.Store.store ->
56
76
Geotessera.bbox ->
57
77
(Linalg.mat * int * int * Geotessera.bbox) Lwt.t
58
-
(** [fetch_region ~store bbox] fetches dequantized embeddings for
78
+
(** [fetch_region ?year ~store bbox] fetches dequantized embeddings for
59
79
a WGS84 bounding box.
60
80
81
+
Automatically detects the store version from array dimensionality:
82
+
- {b v1} stores have 3D embeddings [[y, x, band]] — one store per year,
83
+
[year] is ignored.
84
+
- {b v2} stores have 4D embeddings [[time, band, y, x]] — single store
85
+
with year as dimension. [year] selects the time slice (default 2024,
86
+
range 2017–2025).
87
+
61
88
Steps:
62
89
+ Determines the UTM zone from the bbox centre
63
90
+ Converts bbox corners to UTM pixel coordinates
···
69
96
tuple shape as {!Geotessera.fetch_mosaic_sync} for compatibility
70
97
with existing notebook code.
71
98
72
-
@raise Failure if the UTM zone is not in the store. *)
99
+
@raise Failure if the UTM zone is not in the store or [year] is
100
+
out of range for v2 stores. *)