this repo has no description
more vibe
+982
+340
toru/bin/toru_main.ml
+340
toru/bin/toru_main.ml
···
1
+
(** Main Toru CLI tool for registry inspection and manipulation *)
2
+
3
+
open Cmdliner
4
+
open Toru
5
+
6
+
(** Global options shared across commands *)
7
+
type global_opts = {
8
+
app_name : string;
9
+
cache_dir : string option;
10
+
version : string option;
11
+
verbose_level : int;
12
+
env : Eio_unix.Stdenv.base;
13
+
}
14
+
15
+
(** Resolve cache directory using XDG if none specified *)
16
+
let resolve_cache_dir app_name cache_dir =
17
+
match cache_dir with
18
+
| Some dir -> dir
19
+
| None ->
20
+
(* Try application-specific environment variable first *)
21
+
let app_env_var = String.uppercase_ascii app_name ^ "_CACHE_DIR" in
22
+
(match Sys.getenv_opt app_env_var with
23
+
| Some dir -> dir
24
+
| None ->
25
+
(* Use XDG cache directory as default *)
26
+
(match Sys.getenv_opt "XDG_CACHE_HOME" with
27
+
| Some dir -> Filename.concat dir app_name
28
+
| None ->
29
+
(match Sys.getenv_opt "HOME" with
30
+
| Some home -> Filename.concat (Filename.concat home ".cache") app_name
31
+
| None -> Filename.concat "/tmp" ("cache-" ^ app_name))))
32
+
33
+
let create_global_opts app_name cache_dir version verbose_level env =
34
+
(* Setup logging based on verbosity level *)
35
+
Toru.Logging.setup_logging verbose_level;
36
+
(* Resolve cache directory using XDG *)
37
+
let resolved_cache_dir = resolve_cache_dir app_name cache_dir in
38
+
{ app_name; cache_dir = Some resolved_cache_dir; version; verbose_level; env }
39
+
40
+
let global_opts_t env =
41
+
let app_name =
42
+
let doc = "Application name for cache and config directories" in
43
+
Arg.(value & opt string "toru" & info ["app-name"; "a"] ~doc)
44
+
in
45
+
let cache_dir =
46
+
let doc = "Override default cache directory path (respects XDG_CACHE_HOME)" in
47
+
Arg.(value & opt (some string) None & info ["cache-dir"; "c"] ~doc)
48
+
in
49
+
let version =
50
+
let doc = "Version string for cache subdirectory organization" in
51
+
Arg.(value & opt (some string) None & info ["data-version"] ~doc)
52
+
in
53
+
let verbose_count =
54
+
let doc = "Verbose output (repeat for more verbosity: -v=info, -vv=debug)" in
55
+
Arg.(value & flag_all & info ["verbose"; "v"] ~doc)
56
+
in
57
+
Term.(const (fun app_name cache_dir version verbose_flags ->
58
+
create_global_opts app_name cache_dir version (List.length verbose_flags) env
59
+
) $ app_name $ cache_dir $ version $ verbose_count)
60
+
61
+
(** Registry inspect command *)
62
+
let inspect_cmd env =
63
+
let registry_source =
64
+
let doc = "Registry file path or URL to inspect" in
65
+
Arg.(required & pos 0 (some string) None & info [] ~docv:"REGISTRY" ~doc)
66
+
in
67
+
68
+
let show_stats =
69
+
let doc = "Show detailed statistics about the registry" in
70
+
Arg.(value & flag & info ["stats"; "s"] ~doc)
71
+
in
72
+
73
+
let list_files =
74
+
let doc = "List all files in the registry" in
75
+
Arg.(value & flag & info ["list"; "l"] ~doc)
76
+
in
77
+
78
+
let search_pattern =
79
+
let doc = "Search for files matching this pattern" in
80
+
Arg.(value & opt (some string) None & info ["search"] ~docv:"PATTERN" ~doc)
81
+
in
82
+
83
+
let inspect global_opts registry_source show_stats list_files search_pattern =
84
+
try
85
+
Toru.Logging.Cli.debug (fun m -> m "Starting inspect function");
86
+
87
+
Toru.Logging.Cli.info (fun m -> m "Loading registry from: %s" registry_source);
88
+
89
+
Toru.Logging.Cli.debug (fun m -> m "Determining if source is URL or file path");
90
+
91
+
(* Load the registry *)
92
+
let registry =
93
+
if (String.starts_with ~prefix:"http://" registry_source ||
94
+
String.starts_with ~prefix:"https://" registry_source) then (
95
+
(* Definitely a URL *)
96
+
Toru.Logging.Cli.debug (fun m -> m "Detected URL, loading from network");
97
+
Registry.load_from_url registry_source
98
+
) else (
99
+
(* Treat as file path *)
100
+
Toru.Logging.Cli.debug (fun m -> m "Detected file path, loading with Eio");
101
+
(* Use Eio for file loading - this is the correct approach *)
102
+
Registry.load (Eio.Path.(global_opts.env#fs / registry_source))
103
+
)
104
+
in
105
+
106
+
Toru.Logging.Cli.debug (fun m -> m "Registry loaded successfully");
107
+
108
+
let total_entries = Registry.size registry in
109
+
Printf.printf "Registry contains %d entries\n" total_entries;
110
+
111
+
if show_stats then (
112
+
Printf.printf "\nRegistry Statistics:\n";
113
+
Printf.printf "===================\n";
114
+
let entries = Registry.entries registry in
115
+
116
+
(* Count by hash algorithm *)
117
+
let hash_counts = Hashtbl.create 8 in
118
+
List.iter (fun entry ->
119
+
let hash = Registry.hash entry in
120
+
let algo = Hash.algorithm hash in
121
+
let algo_str = Hash.algorithm_to_string algo in
122
+
let count = match Hashtbl.find_opt hash_counts algo_str with
123
+
| Some n -> n + 1
124
+
| None -> 1
125
+
in
126
+
Hashtbl.replace hash_counts algo_str count
127
+
) entries;
128
+
129
+
Printf.printf "Hash algorithms:\n";
130
+
Hashtbl.iter (fun algo count ->
131
+
Printf.printf " %s: %d entries\n" algo count
132
+
) hash_counts;
133
+
134
+
(* File extension analysis *)
135
+
let ext_counts = Hashtbl.create 16 in
136
+
List.iter (fun entry ->
137
+
let filename = Registry.filename entry in
138
+
let ext =
139
+
match String.rindex_opt filename '.' with
140
+
| Some idx -> String.sub filename (idx + 1) (String.length filename - idx - 1)
141
+
| None -> "(no extension)"
142
+
in
143
+
let count = match Hashtbl.find_opt ext_counts ext with
144
+
| Some n -> n + 1
145
+
| None -> 1
146
+
in
147
+
Hashtbl.replace ext_counts ext count
148
+
) entries;
149
+
150
+
Printf.printf "\nFile extensions:\n";
151
+
Hashtbl.iter (fun ext count ->
152
+
Printf.printf " .%s: %d files\n" ext count
153
+
) ext_counts
154
+
);
155
+
156
+
if list_files then (
157
+
Printf.printf "\nFiles in registry:\n";
158
+
Printf.printf "==================\n";
159
+
let entries = Registry.entries registry in
160
+
List.iter (fun entry ->
161
+
let filename = Registry.filename entry in
162
+
let hash = Registry.hash entry in
163
+
Printf.printf "%s %s\n" (Hash.value hash) filename
164
+
) entries
165
+
);
166
+
167
+
(match search_pattern with
168
+
| Some pattern ->
169
+
Printf.printf "\nFiles matching '%s':\n" pattern;
170
+
Printf.printf "=======================\n";
171
+
let entries = Registry.entries registry in
172
+
let matches = List.filter (fun entry ->
173
+
let filename = Registry.filename entry in
174
+
String.contains filename (String.get pattern 0) ||
175
+
String.length filename >= String.length pattern &&
176
+
String.sub filename 0 (String.length pattern) = pattern
177
+
) entries in
178
+
List.iter (fun entry ->
179
+
let filename = Registry.filename entry in
180
+
let hash = Registry.hash entry in
181
+
Printf.printf "%s %s\n" (Hash.value hash) filename
182
+
) matches
183
+
| None -> ());
184
+
185
+
`Ok ()
186
+
with
187
+
| exn -> `Error (false, "Failed to inspect registry: " ^ (Printexc.to_string exn))
188
+
in
189
+
190
+
let term env = Term.(ret (const inspect $ global_opts_t env $ registry_source $ show_stats $ list_files $ search_pattern)) in
191
+
let info = Cmd.info "inspect" ~doc:"Inspect a registry file or URL" in
192
+
Cmd.v info (term env)
193
+
194
+
(** Registry validate command *)
195
+
let validate_cmd env =
196
+
let registry_source =
197
+
let doc = "Registry file path or URL to validate" in
198
+
Arg.(required & pos 0 (some string) None & info [] ~docv:"REGISTRY" ~doc)
199
+
in
200
+
201
+
let check_hashes =
202
+
let doc = "Check if all hash formats are valid" in
203
+
Arg.(value & flag & info ["check-hashes"] ~doc)
204
+
in
205
+
206
+
let validate global_opts registry_source check_hashes =
207
+
try
208
+
Toru.Logging.Cli.info (fun m -> m "Validating registry: %s" registry_source);
209
+
210
+
let registry =
211
+
if (String.starts_with ~prefix:"http://" registry_source ||
212
+
String.starts_with ~prefix:"https://" registry_source) then (
213
+
Toru.Logging.Cli.debug (fun m -> m "Detected URL, using load_from_url");
214
+
Registry.load_from_url registry_source
215
+
) else (
216
+
Toru.Logging.Cli.debug (fun m -> m "Detected file path, using Eio");
217
+
(* Use Eio for file loading *)
218
+
Registry.load (Eio.Path.(global_opts.env#fs / registry_source))
219
+
)
220
+
in
221
+
222
+
let entries = Registry.entries registry in
223
+
let total = List.length entries in
224
+
Printf.printf "✓ Registry loaded successfully with %d entries\n" total;
225
+
226
+
if check_hashes then (
227
+
Printf.printf "Validating hash formats...\n";
228
+
let valid_count = ref 0 in
229
+
let invalid_count = ref 0 in
230
+
List.iter (fun entry ->
231
+
let filename = Registry.filename entry in
232
+
let hash = Registry.hash entry in
233
+
let hash_str = Hash.value hash in
234
+
let algo = Hash.algorithm hash in
235
+
let expected_len = match algo with
236
+
| SHA256 -> 64
237
+
| SHA1 -> 40
238
+
| MD5 -> 32
239
+
in
240
+
if String.length hash_str = expected_len then
241
+
incr valid_count
242
+
else (
243
+
Printf.printf "✗ Invalid hash length for %s: expected %d chars, got %d\n"
244
+
filename expected_len (String.length hash_str);
245
+
incr invalid_count
246
+
)
247
+
) entries;
248
+
249
+
if !invalid_count = 0 then
250
+
Printf.printf "✓ All %d hash formats are valid\n" !valid_count
251
+
else
252
+
Printf.printf "✗ Found %d invalid hashes out of %d total\n" !invalid_count total
253
+
);
254
+
255
+
`Ok ()
256
+
with
257
+
| exn -> `Error (false, "Registry validation failed: " ^ (Printexc.to_string exn))
258
+
in
259
+
260
+
let term env = Term.(ret (const validate $ global_opts_t env $ registry_source $ check_hashes)) in
261
+
let info = Cmd.info "validate" ~doc:"Validate a registry file format and integrity" in
262
+
Cmd.v info (term env)
263
+
264
+
(** Registry convert command *)
265
+
let convert_cmd env =
266
+
let input_registry =
267
+
let doc = "Input registry file path or URL" in
268
+
Arg.(required & pos 0 (some string) None & info [] ~docv:"INPUT" ~doc)
269
+
in
270
+
271
+
let output_file =
272
+
let doc = "Output registry file path" in
273
+
Arg.(required & pos 1 (some string) None & info [] ~docv:"OUTPUT" ~doc)
274
+
in
275
+
276
+
let convert global_opts input_registry output_file =
277
+
try
278
+
Toru.Logging.Cli.info (fun m -> m "Converting %s -> %s" input_registry output_file);
279
+
280
+
(* Load input registry *)
281
+
let registry =
282
+
if (String.starts_with ~prefix:"http://" input_registry ||
283
+
String.starts_with ~prefix:"https://" input_registry) then (
284
+
Toru.Logging.Cli.debug (fun m -> m "Detected URL, using load_from_url");
285
+
Registry.load_from_url input_registry
286
+
) else (
287
+
Toru.Logging.Cli.debug (fun m -> m "Detected file path, using Eio");
288
+
(* Use Eio for file loading *)
289
+
Registry.load (Eio.Path.(global_opts.env#fs / input_registry))
290
+
)
291
+
in
292
+
293
+
(* Save to output file using Eio *)
294
+
Registry.save (Eio.Path.(global_opts.env#fs / output_file)) registry;
295
+
296
+
let count = Registry.size registry in
297
+
Printf.printf "✓ Converted %d entries from %s to %s\n" count input_registry output_file;
298
+
299
+
`Ok ()
300
+
with
301
+
| exn -> `Error (false, "Conversion failed: " ^ (Printexc.to_string exn))
302
+
in
303
+
304
+
let term env = Term.(ret (const convert $ global_opts_t env $ input_registry $ output_file)) in
305
+
let info = Cmd.info "convert" ~doc:"Convert registry between different formats or sources" in
306
+
Cmd.v info (term env)
307
+
308
+
(** Main command *)
309
+
let main_cmd env =
310
+
let doc = "Toru data repository management tool" in
311
+
let man = [
312
+
`S Cmdliner.Manpage.s_description;
313
+
`P "Toru is an OCaml data repository manager compatible with Python Pooch registry files.";
314
+
`P "It provides automatic downloading, caching, and hash verification of data files from remote repositories.";
315
+
`S "ENVIRONMENT VARIABLES";
316
+
`P "Toru respects the XDG Base Directory Specification:";
317
+
`P "XDG_CACHE_HOME - Override default cache directory location";
318
+
`P "TORU_CACHE_DIR - Application-specific cache directory override";
319
+
`S Cmdliner.Manpage.s_examples;
320
+
`P "Inspect a registry file:";
321
+
`P "$(b,toru inspect registry.txt)";
322
+
`P "Show detailed statistics:";
323
+
`P "$(b,toru inspect --stats registry.txt)";
324
+
`P "Validate registry with hash checking:";
325
+
`P "$(b,toru validate --check-hashes registry.txt)";
326
+
] in
327
+
let info = Cmd.info "toru" ~version:"0.1.0" ~doc ~man in
328
+
let default_term = Term.(ret (const (`Help (`Pager, None)))) in
329
+
Cmd.group info ~default:default_term [
330
+
inspect_cmd env;
331
+
validate_cmd env;
332
+
convert_cmd env;
333
+
]
334
+
335
+
let () =
336
+
(* Run the entire CLI inside Eio_main.run *)
337
+
print_endline "initialising eio";
338
+
Eio_main.run @@ fun env ->
339
+
print_endline "starting";
340
+
exit (Cmd.eval (main_cmd env))
+79
toru/lib/toru/logging.ml
+79
toru/lib/toru/logging.ml
···
1
+
(** Logging module for Toru *)
2
+
3
+
(** Log sources for different parts of the system *)
4
+
let registry_src = Logs.Src.create "toru.registry" ~doc:"Registry operations"
5
+
let cache_src = Logs.Src.create "toru.cache" ~doc:"Cache operations"
6
+
let downloader_src = Logs.Src.create "toru.downloader" ~doc:"Download operations"
7
+
let hash_src = Logs.Src.create "toru.hash" ~doc:"Hash operations"
8
+
let main_src = Logs.Src.create "toru.main" ~doc:"Main Toru operations"
9
+
let cli_src = Logs.Src.create "toru.cli" ~doc:"CLI operations"
10
+
11
+
(** Logging functions for each component *)
12
+
module Registry = struct
13
+
let debug msgf = Logs.debug ~src:registry_src msgf
14
+
let info msgf = Logs.info ~src:registry_src msgf
15
+
let warn msgf = Logs.warn ~src:registry_src msgf
16
+
let err msgf = Logs.err ~src:registry_src msgf
17
+
end
18
+
19
+
module Cache = struct
20
+
let debug msgf = Logs.debug ~src:cache_src msgf
21
+
let info msgf = Logs.info ~src:cache_src msgf
22
+
let warn msgf = Logs.warn ~src:cache_src msgf
23
+
let err msgf = Logs.err ~src:cache_src msgf
24
+
end
25
+
26
+
module Downloader = struct
27
+
let debug msgf = Logs.debug ~src:downloader_src msgf
28
+
let info msgf = Logs.info ~src:downloader_src msgf
29
+
let warn msgf = Logs.warn ~src:downloader_src msgf
30
+
let err msgf = Logs.err ~src:downloader_src msgf
31
+
end
32
+
33
+
module Hash = struct
34
+
let debug msgf = Logs.debug ~src:hash_src msgf
35
+
let info msgf = Logs.info ~src:hash_src msgf
36
+
let warn msgf = Logs.warn ~src:hash_src msgf
37
+
let err msgf = Logs.err ~src:hash_src msgf
38
+
end
39
+
40
+
module Main = struct
41
+
let debug msgf = Logs.debug ~src:main_src msgf
42
+
let info msgf = Logs.info ~src:main_src msgf
43
+
let warn msgf = Logs.warn ~src:main_src msgf
44
+
let err msgf = Logs.err ~src:main_src msgf
45
+
end
46
+
47
+
module Cli = struct
48
+
let debug msgf = Logs.debug ~src:cli_src msgf
49
+
let info msgf = Logs.info ~src:cli_src msgf
50
+
let warn msgf = Logs.warn ~src:cli_src msgf
51
+
let err msgf = Logs.err ~src:cli_src msgf
52
+
end
53
+
54
+
(** Setup logging based on verbosity level *)
55
+
let setup_logging verbose_level =
56
+
let level = match verbose_level with
57
+
| 0 -> Some Logs.Warning (* Only warnings and errors *)
58
+
| 1 -> Some Logs.Info (* Info, warnings, and errors *)
59
+
| _ -> Some Logs.Debug (* Everything including debug *)
60
+
in
61
+
Logs.set_level level;
62
+
(* Use a simple format for CLI tools *)
63
+
Logs.set_reporter (Logs_fmt.reporter ())
64
+
65
+
(** Measure and log execution time *)
66
+
let time_operation ~src ~operation_name f =
67
+
let start_time = Unix.gettimeofday () in
68
+
Logs.debug ~src (fun m -> m "Starting %s" operation_name);
69
+
let result = f () in
70
+
let end_time = Unix.gettimeofday () in
71
+
let duration = end_time -. start_time in
72
+
Logs.info ~src (fun m -> m "Completed %s in %.3fs" operation_name duration);
73
+
result
74
+
75
+
(** Log progress for long operations *)
76
+
let log_progress ~src ~total ~current ~item_name =
77
+
if current mod 100 = 0 || current = total then
78
+
Logs.info ~src (fun m -> m "Progress: %d/%d %s processed (%.1f%%)"
79
+
current total item_name (100. *. float current /. float total))
+66
toru/lib/toru/logging.mli
+66
toru/lib/toru/logging.mli
···
1
+
(** Logging module for Toru *)
2
+
3
+
(** Log sources for different components *)
4
+
val registry_src : Logs.Src.t
5
+
val cache_src : Logs.Src.t
6
+
val downloader_src : Logs.Src.t
7
+
val hash_src : Logs.Src.t
8
+
val main_src : Logs.Src.t
9
+
val cli_src : Logs.Src.t
10
+
11
+
(** Logging functions for Registry operations *)
12
+
module Registry : sig
13
+
val debug : ('a, unit) Logs.msgf -> unit
14
+
val info : ('a, unit) Logs.msgf -> unit
15
+
val warn : ('a, unit) Logs.msgf -> unit
16
+
val err : ('a, unit) Logs.msgf -> unit
17
+
end
18
+
19
+
(** Logging functions for Cache operations *)
20
+
module Cache : sig
21
+
val debug : ('a, unit) Logs.msgf -> unit
22
+
val info : ('a, unit) Logs.msgf -> unit
23
+
val warn : ('a, unit) Logs.msgf -> unit
24
+
val err : ('a, unit) Logs.msgf -> unit
25
+
end
26
+
27
+
(** Logging functions for Downloader operations *)
28
+
module Downloader : sig
29
+
val debug : ('a, unit) Logs.msgf -> unit
30
+
val info : ('a, unit) Logs.msgf -> unit
31
+
val warn : ('a, unit) Logs.msgf -> unit
32
+
val err : ('a, unit) Logs.msgf -> unit
33
+
end
34
+
35
+
(** Logging functions for Hash operations *)
36
+
module Hash : sig
37
+
val debug : ('a, unit) Logs.msgf -> unit
38
+
val info : ('a, unit) Logs.msgf -> unit
39
+
val warn : ('a, unit) Logs.msgf -> unit
40
+
val err : ('a, unit) Logs.msgf -> unit
41
+
end
42
+
43
+
(** Logging functions for Main operations *)
44
+
module Main : sig
45
+
val debug : ('a, unit) Logs.msgf -> unit
46
+
val info : ('a, unit) Logs.msgf -> unit
47
+
val warn : ('a, unit) Logs.msgf -> unit
48
+
val err : ('a, unit) Logs.msgf -> unit
49
+
end
50
+
51
+
(** Logging functions for CLI operations *)
52
+
module Cli : sig
53
+
val debug : ('a, unit) Logs.msgf -> unit
54
+
val info : ('a, unit) Logs.msgf -> unit
55
+
val warn : ('a, unit) Logs.msgf -> unit
56
+
val err : ('a, unit) Logs.msgf -> unit
57
+
end
58
+
59
+
(** Setup logging based on verbosity level (0=warn, 1=info, 2+=debug) *)
60
+
val setup_logging : int -> unit
61
+
62
+
(** Time an operation and log the duration *)
63
+
val time_operation : src:Logs.Src.t -> operation_name:string -> (unit -> 'a) -> 'a
64
+
65
+
(** Log progress for long operations *)
66
+
val log_progress : src:Logs.Src.t -> total:int -> current:int -> item_name:string -> unit
+143
toru/test/basic.t
+143
toru/test/basic.t
···
1
+
Basic Toru CLI Tests
2
+
===================
3
+
4
+
Setup test environment and use actual test registry:
5
+
$ cd $TESTDIR/..
6
+
$ export TORU_CACHE_DIR=/tmp/toru-test-cache
7
+
$ mkdir -p /tmp/toru-test-cache
8
+
9
+
Test 1: Basic inspect functionality
10
+
===================================
11
+
12
+
Test basic inspect without options:
13
+
$ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt
14
+
Registry contains 8 entries
15
+
16
+
Test inspect with verbose logging:
17
+
$ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose
18
+
toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt
19
+
Registry contains 8 entries
20
+
21
+
Test inspect with debug logging:
22
+
$ dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose --verbose 2>&1 | head -6
23
+
toru_main.exe: [DEBUG] Starting inspect function
24
+
toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt
25
+
toru_main.exe: [DEBUG] Determining if source is URL or file path
26
+
toru_main.exe: [DEBUG] Detected file path, loading directly without Eio
27
+
toru_main.exe: [DEBUG] Registry loaded successfully
28
+
Registry contains 8 entries
29
+
30
+
Test inspect with statistics:
31
+
$ dune exec bin/toru_main.exe -- inspect --stats test/python/test_registry_sha256.txt | head -10
32
+
Registry contains 8 entries
33
+
34
+
Registry Statistics:
35
+
===================
36
+
Hash algorithms:
37
+
sha256: 8 entries
38
+
39
+
File extensions:
40
+
.txt: 5 files
41
+
.json: 1 files
42
+
43
+
Test inspect with file listing:
44
+
$ dune exec bin/toru_main.exe -- inspect --list test/python/test_registry_sha256.txt | head -10
45
+
Registry contains 8 entries
46
+
47
+
Files in registry:
48
+
==================
49
+
9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25 .hidden/secret.txt
50
+
dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94 config.json
51
+
d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef data/numbers.csv
52
+
dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f data/simple.txt
53
+
dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310 data/unicode.txt
54
+
55
+
Test inspect with search:
56
+
$ dune exec bin/toru_main.exe -- inspect --search .txt test/python/test_registry_sha256.txt | head -10
57
+
Registry contains 8 entries
58
+
59
+
Files matching '.txt':
60
+
=======================
61
+
9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25 .hidden/secret.txt
62
+
dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f data/simple.txt
63
+
dcccba292708e4d9b8f1e2af482b09b01131ea6e22c3b10173e0d75c8ac0c310 data/unicode.txt
64
+
23c8cf9515ed231a55e63e1d89399a1eac4a529c2b4ac5af29a61af03e3afdd4 docs/readme.md
65
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 empty.txt
66
+
67
+
Test 2: Validate functionality
68
+
==============================
69
+
70
+
Test basic validate:
71
+
$ dune exec bin/toru_main.exe -- validate test/python/test_registry_sha256.txt
72
+
✓ Registry loaded successfully with 8 entries
73
+
74
+
Test validate with hash checking:
75
+
$ dune exec bin/toru_main.exe -- validate --check-hashes test/python/test_registry_sha256.txt
76
+
✓ Registry loaded successfully with 8 entries
77
+
Validating hash formats...
78
+
✓ All 8 hash formats are valid
79
+
80
+
Test 3: Convert functionality
81
+
============================
82
+
83
+
Test convert to new file:
84
+
$ dune exec bin/toru_main.exe -- convert test/python/test_registry_sha256.txt /tmp/converted.txt
85
+
✓ Converted 8 entries from test/python/test_registry_sha256.txt to /tmp/converted.txt
86
+
87
+
Verify converted file has correct content:
88
+
$ head -5 /tmp/converted.txt
89
+
# Pooch registry generated on *-*-*T*:*:*.* (glob)
90
+
# Algorithm: sha256
91
+
.hidden/secret.txt 9ead1fad59f50f905f6a76154e2d0bc1a31c46f223e2ec81e278aa2ee6a10e25
92
+
config.json dcfcfeb20e6334be05d4ed2e39da77ffb84b80bf835dc20ac9963f5e95820b94
93
+
data/numbers.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef
94
+
95
+
Test 4: XDG cache directory handling
96
+
===================================
97
+
98
+
Test with XDG_CACHE_HOME:
99
+
$ XDG_CACHE_HOME=/tmp/xdg-test dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose
100
+
toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt
101
+
Registry contains 8 entries
102
+
103
+
Test with application-specific cache dir:
104
+
$ TORU_CACHE_DIR=/tmp/app-test dune exec bin/toru_main.exe -- inspect test/python/test_registry_sha256.txt --verbose
105
+
toru_main.exe: [INFO] Loading registry from: test/python/test_registry_sha256.txt
106
+
Registry contains 8 entries
107
+
108
+
Test 5: Help system
109
+
===================
110
+
111
+
Test main help shows XDG info:
112
+
$ dune exec bin/toru_main.exe -- --help | grep -A 3 "ENVIRONMENT VARIABLES"
113
+
ENVIRONMENT VARIABLES
114
+
Toru respects the XDG Base Directory Specification:
115
+
116
+
XDG_CACHE_HOME - Override default cache directory location
117
+
118
+
Test 6: Error handling
119
+
======================
120
+
121
+
Test non-existent file:
122
+
$ dune exec bin/toru_main.exe -- inspect nonexistent.txt 2>&1
123
+
[1]
124
+
125
+
Test malformed file doesn't crash:
126
+
$ echo "not a registry" > /tmp/bad.txt
127
+
$ dune exec bin/toru_main.exe -- inspect /tmp/bad.txt
128
+
Registry contains 0 entries
129
+
130
+
Test 7: App name and version parameters
131
+
======================================
132
+
133
+
Test with custom app name:
134
+
$ dune exec bin/toru_main.exe -- --app-name testapp inspect test/python/test_registry_sha256.txt
135
+
Registry contains 8 entries
136
+
137
+
Test with data version:
138
+
$ dune exec bin/toru_main.exe -- --data-version v1.0 inspect test/python/test_registry_sha256.txt
139
+
Registry contains 8 entries
140
+
141
+
Cleanup:
142
+
$ rm -f /tmp/converted.txt /tmp/bad.txt
143
+
$ rm -rf /tmp/toru-test-cache /tmp/xdg-test /tmp/app-test
+354
toru/test/cli.t
+354
toru/test/cli.t
···
1
+
Toru CLI Comprehensive Test Suite
2
+
==================================
3
+
4
+
Setup test environment:
5
+
$ export HOME=/tmp/toru_test_home
6
+
$ mkdir -p $HOME
7
+
$ unset XDG_CACHE_HOME TORU_CACHE_DIR
8
+
$ cd /tmp/toru_test_home
9
+
10
+
Create test registry file:
11
+
$ cat > test_registry.txt << 'EOF'
12
+
> # Test registry for toru CLI
13
+
> simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
14
+
> data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef
15
+
> binary.bin 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9
16
+
> EOF
17
+
18
+
Test 1: Basic inspect command functionality
19
+
===========================================
20
+
21
+
Test basic inspect without options:
22
+
$ toru inspect test_registry.txt
23
+
Registry contains 3 entries
24
+
25
+
Test inspect with stats flag:
26
+
$ toru inspect --stats test_registry.txt
27
+
Registry contains 3 entries
28
+
29
+
Registry Statistics:
30
+
===================
31
+
Hash algorithms:
32
+
sha256: 3 entries
33
+
34
+
File extensions:
35
+
.txt: 1 files
36
+
.csv: 1 files
37
+
.bin: 1 files
38
+
39
+
Test inspect with list flag:
40
+
$ toru inspect --list test_registry.txt
41
+
Registry contains 3 entries
42
+
43
+
Files in registry:
44
+
==================
45
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 simple.txt
46
+
d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef data.csv
47
+
785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9 binary.bin
48
+
49
+
Test inspect with search pattern:
50
+
$ toru inspect --search txt test_registry.txt
51
+
Registry contains 3 entries
52
+
53
+
Files matching 'txt':
54
+
=======================
55
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 simple.txt
56
+
57
+
Test inspect with all flags combined:
58
+
$ toru inspect --stats --list --search csv test_registry.txt
59
+
Registry contains 3 entries
60
+
61
+
Registry Statistics:
62
+
===================
63
+
Hash algorithms:
64
+
sha256: 3 entries
65
+
66
+
File extensions:
67
+
.txt: 1 files
68
+
.csv: 1 files
69
+
.bin: 1 files
70
+
71
+
Files in registry:
72
+
==================
73
+
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 simple.txt
74
+
d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef data.csv
75
+
785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9 binary.bin
76
+
77
+
Files matching 'csv':
78
+
=======================
79
+
d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef data.csv
80
+
81
+
Test 2: Validate command functionality
82
+
======================================
83
+
84
+
Test basic validate:
85
+
$ toru validate test_registry.txt
86
+
✓ Registry loaded successfully with 3 entries
87
+
88
+
Test validate with hash checking:
89
+
$ toru validate --check-hashes test_registry.txt
90
+
✓ Registry loaded successfully with 3 entries
91
+
Validating hash formats...
92
+
✓ All 3 hash formats are valid
93
+
94
+
Test 3: Convert command functionality
95
+
====================================
96
+
97
+
Test convert to same format:
98
+
$ toru convert test_registry.txt converted_registry.txt
99
+
✓ Converted 3 entries from test_registry.txt to converted_registry.txt
100
+
101
+
$ cat converted_registry.txt
102
+
# Pooch registry generated on *-*-*T*:*:*.* (glob)
103
+
# Algorithm: sha256
104
+
simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
105
+
data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef
106
+
binary.bin 785b0751fc2c53dc14a4ce3d800e69ef9ce1009eb327ccf458afe09c242c26c9
107
+
108
+
Test 4: Verbose logging levels
109
+
==============================
110
+
111
+
Test with single verbose flag (info level):
112
+
$ toru inspect test_registry.txt --verbose
113
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
114
+
Registry contains 3 entries
115
+
116
+
Test with double verbose flags (debug level):
117
+
$ toru inspect test_registry.txt --verbose --verbose
118
+
toru_main.exe: [DEBUG] Starting inspect function
119
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
120
+
toru_main.exe: [DEBUG] Determining if source is URL or file path
121
+
toru_main.exe: [DEBUG] Detected file path, loading directly without Eio
122
+
toru_main.exe: [DEBUG] Registry loaded successfully
123
+
Registry contains 3 entries
124
+
125
+
Test 5: XDG Base Directory compliance
126
+
====================================
127
+
128
+
Test default cache directory behavior:
129
+
$ unset XDG_CACHE_HOME TORU_CACHE_DIR
130
+
$ toru inspect test_registry.txt --verbose
131
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
132
+
Registry contains 3 entries
133
+
134
+
Test XDG_CACHE_HOME environment variable:
135
+
$ XDG_CACHE_HOME=/tmp/xdg-cache toru inspect test_registry.txt --verbose
136
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
137
+
Registry contains 3 entries
138
+
139
+
Test application-specific cache directory override:
140
+
$ TORU_CACHE_DIR=/tmp/toru-cache toru inspect test_registry.txt --verbose
141
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
142
+
Registry contains 3 entries
143
+
144
+
Test command-line cache directory override:
145
+
$ toru inspect test_registry.txt --cache-dir /tmp/cli-cache --verbose
146
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
147
+
Registry contains 3 entries
148
+
149
+
Test precedence: CLI overrides environment variables:
150
+
$ XDG_CACHE_HOME=/tmp/xdg TORU_CACHE_DIR=/tmp/app toru inspect test_registry.txt --cache-dir /tmp/cli --verbose
151
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
152
+
Registry contains 3 entries
153
+
154
+
Test 6: Mixed hash algorithms support
155
+
====================================
156
+
157
+
Create registry with mixed hash types:
158
+
$ cat > mixed_hashes.txt << 'EOF'
159
+
> # Registry with different hash algorithms
160
+
> # SHA256 hash
161
+
> file1.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
162
+
> # SHA1 hash
163
+
> file2.txt sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709
164
+
> # MD5 hash
165
+
> file3.txt md5:d41d8cd98f00b204e9800998ecf8427e
166
+
> EOF
167
+
168
+
Test mixed hash validation:
169
+
$ toru validate --check-hashes mixed_hashes.txt
170
+
✓ Registry loaded successfully with 3 entries
171
+
Validating hash formats...
172
+
✓ All 3 hash formats are valid
173
+
174
+
Test 7: URL source handling
175
+
===========================
176
+
177
+
Test detection of HTTP URL (mock):
178
+
$ toru inspect https://example.com/registry.txt 2>&1 | head -3
179
+
toru_main.exe: [ERROR] * (glob)
180
+
181
+
Test detection of HTTPS URL (mock):
182
+
$ toru inspect https://example.com/registry.txt 2>&1 | head -3
183
+
toru_main.exe: [ERROR] * (glob)
184
+
185
+
Test 8: Error handling
186
+
======================
187
+
188
+
Test non-existent file:
189
+
$ toru inspect nonexistent.txt
190
+
[1]
191
+
192
+
Test malformed registry:
193
+
$ echo "invalid registry content" > malformed.txt
194
+
$ toru inspect malformed.txt
195
+
Registry contains 0 entries
196
+
197
+
Test empty registry:
198
+
$ touch empty.txt
199
+
$ toru inspect empty.txt
200
+
Registry contains 0 entries
201
+
202
+
Test 9: App name parameter
203
+
=========================
204
+
205
+
Test custom app name:
206
+
$ toru --app-name myapp inspect test_registry.txt --verbose
207
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
208
+
Registry contains 3 entries
209
+
210
+
Test 10: Data version parameter
211
+
==============================
212
+
213
+
Test with version string:
214
+
$ toru --data-version v1.0 inspect test_registry.txt --verbose
215
+
toru_main.exe: [INFO] Loading registry from: test_registry.txt
216
+
Registry contains 3 entries
217
+
218
+
Test 11: Help system
219
+
===================
220
+
221
+
Test main help:
222
+
$ toru --help | head -10
223
+
TORU(1) * Toru Manual * TORU(1) (glob)
224
+
225
+
226
+
227
+
NAME
228
+
toru - Toru data repository management tool
229
+
230
+
SYNOPSIS
231
+
toru [COMMAND]…
232
+
233
+
234
+
Test inspect subcommand help:
235
+
$ toru inspect --help | head -10
236
+
TORU-INSPECT(1) * Toru Manual * TORU-INSPECT(1) (glob)
237
+
238
+
239
+
240
+
NAME
241
+
toru-inspect - Inspect a registry file or URL
242
+
243
+
SYNOPSIS
244
+
toru-inspect [OPTION]… REGISTRY
245
+
246
+
247
+
Test validate subcommand help:
248
+
$ toru validate --help | head -10
249
+
TORU-VALIDATE(1) * Toru Manual * TORU-VALIDATE(1) (glob)
250
+
251
+
252
+
253
+
NAME
254
+
toru-validate - Validate a registry file format and integrity
255
+
256
+
SYNOPSIS
257
+
toru-validate [OPTION]… REGISTRY
258
+
259
+
260
+
Test convert subcommand help:
261
+
$ toru convert --help | head -10
262
+
TORU-CONVERT(1) * Toru Manual * TORU-CONVERT(1) (glob)
263
+
264
+
265
+
266
+
NAME
267
+
toru-convert - Convert registry between different formats or sources
268
+
269
+
SYNOPSIS
270
+
toru-convert [OPTION]… INPUT OUTPUT
271
+
272
+
273
+
Test 12: Environment variable documentation
274
+
==========================================
275
+
276
+
Test that help includes XDG information:
277
+
$ toru --help | grep -A 5 "ENVIRONMENT VARIABLES"
278
+
ENVIRONMENT VARIABLES
279
+
Toru respects the XDG Base Directory Specification:
280
+
281
+
XDG_CACHE_HOME - Override default cache directory location
282
+
283
+
TORU_CACHE_DIR - Application-specific cache directory override
284
+
285
+
Test 13: Edge cases and boundary conditions
286
+
==========================================
287
+
288
+
Test very long registry file:
289
+
$ python3 -c "
290
+
> for i in range(1000):
291
+
> print(f'file{i:04d}.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
292
+
> " > large_registry.txt
293
+
$ toru inspect large_registry.txt
294
+
Registry contains 1000 entries
295
+
296
+
Test registry with comments and blank lines:
297
+
$ cat > commented_registry.txt << 'EOF'
298
+
> # This is a test registry
299
+
> # With comments and blank lines
300
+
>
301
+
> simple.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
302
+
>
303
+
> # Another comment
304
+
> data.csv d613c3cca7a0aafd47e7cd81c7ee0268504b13e8c24b2668dcde8a86386c5cef
305
+
>
306
+
> EOF
307
+
308
+
$ toru inspect commented_registry.txt
309
+
Registry contains 2 entries
310
+
311
+
Test registry with unusual file names:
312
+
$ cat > unusual_names.txt << 'EOF'
313
+
> file with spaces.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
314
+
> file-with-dashes.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
315
+
> file_with_underscores.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
316
+
> file.with.dots.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
317
+
> file123numbers.txt e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
318
+
> EOF
319
+
320
+
$ toru inspect unusual_names.txt
321
+
Registry contains 5 entries
322
+
323
+
Test 14: Performance and scalability
324
+
===================================
325
+
326
+
Test registry with many file extensions:
327
+
$ python3 -c "
328
+
> import string
329
+
> extensions = [f'.{c}{c}{c}' for c in string.ascii_lowercase[:10]]
330
+
> for i, ext in enumerate(extensions):
331
+
> print(f'file{i:03d}{ext} e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
332
+
> " > many_extensions.txt
333
+
$ toru inspect --stats many_extensions.txt
334
+
Registry contains 10 entries
335
+
336
+
Registry Statistics:
337
+
===================
338
+
Hash algorithms:
339
+
sha256: 10 entries
340
+
341
+
File extensions:
342
+
.aaa: 1 files
343
+
.bbb: 1 files
344
+
.ccc: 1 files
345
+
.ddd: 1 files
346
+
.eee: 1 files
347
+
.fff: 1 files
348
+
.ggg: 1 files
349
+
.hhh: 1 files
350
+
.iii: 1 files
351
+
.jjj: 1 files
352
+
353
+
Cleanup:
354
+
$ rm -rf /tmp/toru_test_home