this repo has no description
own slop
+330
+1
rec-converter/.gitignore
+1
rec-converter/.gitignore
···
1
+
_build
+4
rec-converter/bin/dune
+4
rec-converter/bin/dune
+106
rec-converter/bin/main.ml
+106
rec-converter/bin/main.ml
···
1
+
open Cmdliner
2
+
open Owntracks_to_clickhouse
3
+
4
+
let process_file input_file output_file =
5
+
try
6
+
let records = Owntracks_parser.parse_file input_file in
7
+
match output_file with
8
+
| Some out_path ->
9
+
Clickhouse_formatter.write_jsonl_file out_path records;
10
+
Printf.printf "Converted %d records from %s to %s\n"
11
+
(List.length records) input_file out_path
12
+
| None ->
13
+
Clickhouse_formatter.print_jsonl records
14
+
with
15
+
| Sys_error msg ->
16
+
Printf.eprintf "Error: %s\n" msg;
17
+
exit 1
18
+
| e ->
19
+
Printf.eprintf "Unexpected error: %s\n" (Printexc.to_string e);
20
+
exit 1
21
+
22
+
let process_directory dir_path output_file recursive =
23
+
let rec find_rec_files dir =
24
+
let entries = Sys.readdir dir in
25
+
Array.fold_left (fun acc entry ->
26
+
let full_path = Filename.concat dir entry in
27
+
if Sys.is_directory full_path then
28
+
if recursive then
29
+
acc @ find_rec_files full_path
30
+
else
31
+
acc
32
+
else if Filename.check_suffix entry ".rec" then
33
+
full_path :: acc
34
+
else
35
+
acc
36
+
) [] entries
37
+
in
38
+
39
+
let rec_files = find_rec_files dir_path in
40
+
let all_records = List.fold_left (fun acc file ->
41
+
try
42
+
let records = Owntracks_parser.parse_file file in
43
+
Printf.printf "Processed %s: %d records\n" file (List.length records);
44
+
acc @ records
45
+
with e ->
46
+
Printf.eprintf "Warning: Failed to process %s: %s\n"
47
+
file (Printexc.to_string e);
48
+
acc
49
+
) [] rec_files in
50
+
51
+
match output_file with
52
+
| Some out_path ->
53
+
Clickhouse_formatter.write_jsonl_file out_path all_records;
54
+
Printf.printf "Total: Converted %d records from %d files to %s\n"
55
+
(List.length all_records) (List.length rec_files) out_path
56
+
| None ->
57
+
Clickhouse_formatter.print_jsonl all_records
58
+
59
+
let main input output recursive =
60
+
if Sys.is_directory input then
61
+
process_directory input output recursive
62
+
else
63
+
process_file input output
64
+
65
+
let input_arg =
66
+
let doc = "Input OwnTracks .rec file or directory containing .rec files" in
67
+
Arg.(required & pos 0 (some string) None & info [] ~docv:"INPUT" ~doc)
68
+
69
+
let output_arg =
70
+
let doc = "Output JSON lines file (if not specified, outputs to stdout)" in
71
+
Arg.(value & opt (some string) None & info ["o"; "output"] ~docv:"OUTPUT" ~doc)
72
+
73
+
let recursive_arg =
74
+
let doc = "Recursively process directories for .rec files" in
75
+
Arg.(value & flag & info ["r"; "recursive"] ~doc)
76
+
77
+
let main_term =
78
+
Term.(const main $ input_arg $ output_arg $ recursive_arg)
79
+
80
+
let info =
81
+
let doc = "Convert OwnTracks .rec files to ClickHouse JSON lines" in
82
+
let man = [
83
+
`S Manpage.s_description;
84
+
`P "$(tname) converts OwnTracks recorder files (.rec) to JSON lines format suitable for importing into ClickHouse with geo data types.";
85
+
`P "Each location record is converted to a JSON object with the following fields:";
86
+
`P "- timestamp: ISO 8601 formatted timestamp";
87
+
`P "- timestamp_epoch: Unix timestamp";
88
+
`P "- point: [longitude, latitude] array for ClickHouse Point type";
89
+
`P "- latitude, longitude: Individual coordinate fields";
90
+
`P "- altitude, accuracy, battery: Optional fields from OwnTracks";
91
+
`P "- tracker_id: Device identifier from OwnTracks";
92
+
`S Manpage.s_examples;
93
+
`P "Convert a single file to stdout:";
94
+
`Pre " $(tname) path/to/file.rec";
95
+
`P "Convert a single file to output file:";
96
+
`Pre " $(tname) path/to/file.rec -o output.jsonl";
97
+
`P "Process all .rec files in a directory:";
98
+
`Pre " $(tname) path/to/directory -o output.jsonl";
99
+
`P "Process directory recursively:";
100
+
`Pre " $(tname) path/to/directory -r -o output.jsonl";
101
+
] in
102
+
Cmd.info "owntracks2clickhouse" ~version:"1.0.0" ~doc ~man
103
+
104
+
let cmd = Cmd.v info main_term
105
+
106
+
let () = exit (Cmd.eval cmd)
+14
rec-converter/dune-project
+14
rec-converter/dune-project
···
1
+
(lang dune 3.0)
2
+
(name owntracks_to_clickhouse)
3
+
4
+
(generate_opam_files true)
5
+
6
+
(package
7
+
(name owntracks_to_clickhouse)
8
+
(synopsis "Convert OwnTracks .rec files to ClickHouse JSON lines")
9
+
(description "A library and CLI tool to convert OwnTracks recorder files to JSON lines suitable for ClickHouse import with geo data types")
10
+
(depends
11
+
ocaml
12
+
dune
13
+
ezjsonm
14
+
cmdliner))
+54
rec-converter/example.sh
+54
rec-converter/example.sh
···
1
+
#!/bin/bash
2
+
3
+
# Build the project
4
+
dune build
5
+
6
+
# Example 1: Convert a single .rec file to stdout
7
+
echo "Converting single file to stdout:"
8
+
dune exec owntracks2clickhouse avsm/avsm-ip15/2025-08.rec | head -5
9
+
10
+
# Example 2: Convert a single .rec file to output file
11
+
echo -e "\nConverting single file to output.jsonl:"
12
+
dune exec owntracks2clickhouse avsm/avsm-ip15/2025-08.rec -o single_output.jsonl
13
+
echo "Created single_output.jsonl"
14
+
15
+
# Example 3: Process all .rec files in a directory recursively
16
+
echo -e "\nProcessing all .rec files recursively:"
17
+
dune exec owntracks2clickhouse avsm -r -o all_records.jsonl
18
+
19
+
# Example 4: Create ClickHouse table and import data
20
+
cat << 'EOF'
21
+
22
+
To import into ClickHouse, create a table like this:
23
+
24
+
CREATE TABLE owntracks_locations (
25
+
timestamp DateTime64(3),
26
+
timestamp_epoch UInt32,
27
+
point Point,
28
+
latitude Float64,
29
+
longitude Float64,
30
+
altitude Nullable(Float64),
31
+
accuracy Nullable(Float64),
32
+
battery Nullable(UInt8),
33
+
tracker_id Nullable(String)
34
+
) ENGINE = MergeTree()
35
+
ORDER BY (tracker_id, timestamp);
36
+
37
+
Then import the JSON lines file:
38
+
39
+
clickhouse-client --query="INSERT INTO owntracks_locations FORMAT JSONEachRow" < all_records.jsonl
40
+
41
+
Or using clickhouse-local for testing:
42
+
43
+
clickhouse-local --query="
44
+
SELECT
45
+
tracker_id,
46
+
toDate(timestamp) as date,
47
+
count() as points,
48
+
round(avg(battery), 2) as avg_battery
49
+
FROM file('all_records.jsonl', 'JSONEachRow')
50
+
GROUP BY tracker_id, date
51
+
ORDER BY date DESC
52
+
LIMIT 10
53
+
"
54
+
EOF
+43
rec-converter/lib/clickhouse_formatter.ml
+43
rec-converter/lib/clickhouse_formatter.ml
···
1
+
open Owntracks_parser
2
+
3
+
let option_to_json = function
4
+
| Some v -> v
5
+
| None -> `Null
6
+
7
+
let location_to_clickhouse_json record =
8
+
let point = `A [`Float record.lon; `Float record.lat] in
9
+
let timestamp_epoch = record.tst in
10
+
let timestamp_iso = record.timestamp in
11
+
12
+
`O [
13
+
("timestamp", `String timestamp_iso);
14
+
("timestamp_epoch", `Float (float_of_int timestamp_epoch));
15
+
("point", point);
16
+
("latitude", `Float record.lat);
17
+
("longitude", `Float record.lon);
18
+
("altitude", option_to_json (Option.map (fun x -> `Float x) record.alt));
19
+
("accuracy", option_to_json (Option.map (fun x -> `Float x) record.acc));
20
+
("battery", option_to_json (Option.map (fun x -> `Float (float_of_int x)) record.batt));
21
+
("tracker_id", option_to_json (Option.map (fun x -> `String x) record.tid));
22
+
]
23
+
24
+
let to_jsonl records =
25
+
List.map (fun record ->
26
+
let json = location_to_clickhouse_json record in
27
+
Ezjsonm.to_string json
28
+
) records
29
+
30
+
let write_jsonl_file path records =
31
+
let oc = open_out path in
32
+
List.iter (fun record ->
33
+
let json = location_to_clickhouse_json record in
34
+
output_string oc (Ezjsonm.to_string json);
35
+
output_char oc '\n'
36
+
) records;
37
+
close_out oc
38
+
39
+
let print_jsonl records =
40
+
List.iter (fun record ->
41
+
let json = location_to_clickhouse_json record in
42
+
print_endline (Ezjsonm.to_string json)
43
+
) records
+4
rec-converter/lib/dune
+4
rec-converter/lib/dune
+78
rec-converter/lib/owntracks_parser.ml
+78
rec-converter/lib/owntracks_parser.ml
···
1
+
type location_record = {
2
+
timestamp: string;
3
+
lat: float;
4
+
lon: float;
5
+
alt: float option;
6
+
acc: float option;
7
+
batt: int option;
8
+
tid: string option;
9
+
tst: int;
10
+
}
11
+
12
+
let parse_timestamp line =
13
+
match String.split_on_char '\t' line with
14
+
| timestamp :: _ -> Some timestamp
15
+
| [] -> None
16
+
17
+
let parse_json_payload line =
18
+
match String.split_on_char '\t' line with
19
+
| _ :: _ :: json :: _ ->
20
+
(try Some (Ezjsonm.from_string json) with _ -> None)
21
+
| _ -> None
22
+
23
+
let get_string json key =
24
+
try Some (Ezjsonm.get_string (Ezjsonm.find json [key]))
25
+
with _ -> None
26
+
27
+
let get_float json key =
28
+
try Some (Ezjsonm.get_float (Ezjsonm.find json [key]))
29
+
with _ -> None
30
+
31
+
let get_int json key =
32
+
try Some (Ezjsonm.get_int (Ezjsonm.find json [key]))
33
+
with _ -> None
34
+
35
+
let extract_location_from_json json =
36
+
try
37
+
let dict = Ezjsonm.get_dict json in
38
+
match List.assoc_opt "_type" dict with
39
+
| Some (`String "location") ->
40
+
let lat = Ezjsonm.get_float (List.assoc "lat" dict) in
41
+
let lon = Ezjsonm.get_float (List.assoc "lon" dict) in
42
+
let tst = Ezjsonm.get_int (List.assoc "tst" dict) in
43
+
Some {
44
+
timestamp = "";
45
+
lat;
46
+
lon;
47
+
alt = get_float json "alt";
48
+
acc = get_float json "acc";
49
+
batt = get_int json "batt";
50
+
tid = get_string json "tid";
51
+
tst;
52
+
}
53
+
| _ -> None
54
+
with _ -> None
55
+
56
+
let parse_line line =
57
+
match parse_timestamp line, parse_json_payload line with
58
+
| Some timestamp, Some json ->
59
+
(match extract_location_from_json json with
60
+
| Some record -> Some { record with timestamp }
61
+
| None -> None)
62
+
| _ -> None
63
+
64
+
let parse_file path =
65
+
let ic = open_in path in
66
+
let rec read_lines acc =
67
+
try
68
+
let line = input_line ic in
69
+
let acc' = match parse_line line with
70
+
| Some record -> record :: acc
71
+
| None -> acc
72
+
in
73
+
read_lines acc'
74
+
with End_of_file ->
75
+
close_in ic;
76
+
List.rev acc
77
+
in
78
+
read_lines []
+26
rec-converter/owntracks_to_clickhouse.opam
+26
rec-converter/owntracks_to_clickhouse.opam
···
1
+
# This file is generated by dune, edit dune-project instead
2
+
opam-version: "2.0"
3
+
synopsis: "Convert OwnTracks .rec files to ClickHouse JSON lines"
4
+
description:
5
+
"A library and CLI tool to convert OwnTracks recorder files to JSON lines suitable for ClickHouse import with geo data types"
6
+
depends: [
7
+
"ocaml"
8
+
"dune" {>= "3.0"}
9
+
"ezjsonm"
10
+
"cmdliner"
11
+
"odoc" {with-doc}
12
+
]
13
+
build: [
14
+
["dune" "subst"] {dev}
15
+
[
16
+
"dune"
17
+
"build"
18
+
"-p"
19
+
name
20
+
"-j"
21
+
jobs
22
+
"@install"
23
+
"@runtest" {with-test}
24
+
"@doc" {with-doc}
25
+
]
26
+
]