CCSDS 121.0-B-3 Lossless Data Compression (Rice/Golomb coding)
irmin: full SCITT proof verification — 20/20 + 6/6 tests pass
+103
-352
+4
-11
test/interop/libaec/dune
+4
-11
test/interop/libaec/dune
···
1
1
(test
2
-
(name test_libaec)
3
-
(libraries rice alcotest)
4
-
(foreign_stubs
5
-
(language c)
6
-
(names libaec_stubs)
7
-
(flags
8
-
(:standard -I/opt/homebrew/opt/libaec/include)))
9
-
(link_flags
10
-
(-cclib -L/opt/homebrew/opt/libaec/lib -cclib -laec))
11
-
(enabled_if
12
-
(= %{env:INTEROP=false} true)))
2
+
(name test)
3
+
(libraries rice csvt alcotest)
4
+
(deps
5
+
(source_tree traces)))
-133
test/interop/libaec/libaec_stubs.c
-133
test/interop/libaec/libaec_stubs.c
···
1
-
/* C stubs wrapping libaec's aec_encode / aec_decode for OCaml interop tests.
2
-
3
-
libaec implements the CCSDS 121.0-B-3 Adaptive Entropy Coding standard.
4
-
We expose two OCaml functions:
5
-
- caml_libaec_encode : bytes -> block_size:int -> bits_per_sample:int -> bytes
6
-
- caml_libaec_decode : bytes -> block_size:int -> bits_per_sample:int
7
-
-> expected_size:int -> bytes
8
-
*/
9
-
10
-
#include <caml/mlvalues.h>
11
-
#include <caml/memory.h>
12
-
#include <caml/alloc.h>
13
-
#include <caml/fail.h>
14
-
#include <string.h>
15
-
#include <libaec.h>
16
-
17
-
/* Encode raw samples using libaec.
18
-
Arguments: data (bytes), block_size (int), bits_per_sample (int)
19
-
Returns: compressed bytes */
20
-
CAMLprim value caml_libaec_encode(value v_data, value v_block_size,
21
-
value v_bits_per_sample) {
22
-
CAMLparam3(v_data, v_block_size, v_bits_per_sample);
23
-
CAMLlocal1(v_result);
24
-
25
-
unsigned char *data = (unsigned char *)Bytes_val(v_data);
26
-
int data_len = caml_string_length(v_data);
27
-
int block_size = Int_val(v_block_size);
28
-
int bits_per_sample = Int_val(v_bits_per_sample);
29
-
30
-
/* Allocate output buffer -- worst case is slightly larger than input */
31
-
int out_size = data_len * 2 + 1024;
32
-
unsigned char *out_buf = (unsigned char *)caml_stat_alloc(out_size);
33
-
34
-
struct aec_stream strm;
35
-
memset(&strm, 0, sizeof(strm));
36
-
37
-
strm.bits_per_sample = bits_per_sample;
38
-
strm.block_size = block_size;
39
-
strm.rsi = block_size;
40
-
strm.flags = AEC_DATA_MSB | AEC_DATA_PREPROCESS;
41
-
42
-
/* Set byte width flags based on bits_per_sample */
43
-
if (bits_per_sample > 16) {
44
-
strm.flags |= AEC_DATA_3BYTE;
45
-
}
46
-
47
-
strm.next_in = data;
48
-
strm.avail_in = data_len;
49
-
strm.next_out = out_buf;
50
-
strm.avail_out = out_size;
51
-
52
-
int ret = aec_encode_init(&strm);
53
-
if (ret != AEC_OK) {
54
-
caml_stat_free(out_buf);
55
-
caml_failwith("libaec: aec_encode_init failed");
56
-
}
57
-
58
-
ret = aec_encode(&strm, AEC_FLUSH);
59
-
if (ret != AEC_OK) {
60
-
aec_encode_end(&strm);
61
-
caml_stat_free(out_buf);
62
-
caml_failwith("libaec: aec_encode failed");
63
-
}
64
-
65
-
int compressed_len = out_size - strm.avail_out;
66
-
67
-
aec_encode_end(&strm);
68
-
69
-
v_result = caml_alloc_string(compressed_len);
70
-
memcpy(Bytes_val(v_result), out_buf, compressed_len);
71
-
caml_stat_free(out_buf);
72
-
73
-
CAMLreturn(v_result);
74
-
}
75
-
76
-
/* Decode compressed data using libaec.
77
-
Arguments: data (bytes), block_size (int), bits_per_sample (int),
78
-
expected_size (int)
79
-
Returns: decompressed bytes */
80
-
CAMLprim value caml_libaec_decode(value v_data, value v_block_size,
81
-
value v_bits_per_sample,
82
-
value v_expected_size) {
83
-
CAMLparam4(v_data, v_block_size, v_bits_per_sample, v_expected_size);
84
-
CAMLlocal1(v_result);
85
-
86
-
unsigned char *data = (unsigned char *)Bytes_val(v_data);
87
-
int data_len = caml_string_length(v_data);
88
-
int block_size = Int_val(v_block_size);
89
-
int bits_per_sample = Int_val(v_bits_per_sample);
90
-
int expected_size = Int_val(v_expected_size);
91
-
92
-
unsigned char *out_buf = (unsigned char *)caml_stat_alloc(expected_size);
93
-
94
-
struct aec_stream strm;
95
-
memset(&strm, 0, sizeof(strm));
96
-
97
-
strm.bits_per_sample = bits_per_sample;
98
-
strm.block_size = block_size;
99
-
strm.rsi = block_size;
100
-
strm.flags = AEC_DATA_MSB | AEC_DATA_PREPROCESS;
101
-
102
-
if (bits_per_sample > 16) {
103
-
strm.flags |= AEC_DATA_3BYTE;
104
-
}
105
-
106
-
strm.next_in = data;
107
-
strm.avail_in = data_len;
108
-
strm.next_out = out_buf;
109
-
strm.avail_out = expected_size;
110
-
111
-
int ret = aec_decode_init(&strm);
112
-
if (ret != AEC_OK) {
113
-
caml_stat_free(out_buf);
114
-
caml_failwith("libaec: aec_decode_init failed");
115
-
}
116
-
117
-
ret = aec_decode(&strm, AEC_FLUSH);
118
-
if (ret != AEC_OK) {
119
-
aec_decode_end(&strm);
120
-
caml_stat_free(out_buf);
121
-
caml_failwith("libaec: aec_decode failed");
122
-
}
123
-
124
-
int decompressed_len = expected_size - strm.avail_out;
125
-
126
-
aec_decode_end(&strm);
127
-
128
-
v_result = caml_alloc_string(decompressed_len);
129
-
memcpy(Bytes_val(v_result), out_buf, decompressed_len);
130
-
caml_stat_free(out_buf);
131
-
132
-
CAMLreturn(v_result);
133
-
}
test/interop/libaec/scripts/generate.sh
test/interop/libaec/scripts/generate.sh
+94
test/interop/libaec/test.ml
+94
test/interop/libaec/test.ml
···
1
+
(** libaec interop tests for ocaml-rice.
2
+
3
+
Traces generated by: libaec (brew)
4
+
Regenerate: dune build @regen-traces
5
+
6
+
Each trace row contains raw samples compressed by libaec. The test
7
+
compresses the same input with ocaml-rice and checks byte-exact match,
8
+
then decompresses the libaec output with ocaml-rice and checks it
9
+
recovers the original samples. *)
10
+
11
+
let trace path = Filename.concat "traces" path
12
+
13
+
let bytes_of_hex hex =
14
+
let len = String.length hex / 2 in
15
+
Bytes.init len (fun i ->
16
+
Char.chr (int_of_string ("0x" ^ String.sub hex (i * 2) 2)))
17
+
18
+
let hex_of_bytes b =
19
+
let buf = Buffer.create (Bytes.length b * 2) in
20
+
Bytes.iter
21
+
(fun c -> Buffer.add_string buf (Printf.sprintf "%02x" (Char.code c)))
22
+
b;
23
+
Buffer.contents buf
24
+
25
+
type vector = {
26
+
name : string;
27
+
block_size : int;
28
+
bits_per_sample : int;
29
+
input : bytes;
30
+
compressed : bytes;
31
+
}
32
+
33
+
let vector_codec =
34
+
Csvt.(
35
+
Row.(
36
+
obj (fun name block_size bits_per_sample input_hex compressed_hex ->
37
+
{
38
+
name;
39
+
block_size;
40
+
bits_per_sample;
41
+
input = bytes_of_hex input_hex;
42
+
compressed = bytes_of_hex compressed_hex;
43
+
})
44
+
|> col "name" string ~enc:(fun v -> v.name)
45
+
|> col "block_size" int ~enc:(fun v -> v.block_size)
46
+
|> col "bits_per_sample" int ~enc:(fun v -> v.bits_per_sample)
47
+
|> col "input_hex" string ~enc:(fun v -> hex_of_bytes v.input)
48
+
|> col "compressed_hex" string ~enc:(fun v -> hex_of_bytes v.compressed)
49
+
|> finish))
50
+
51
+
let load_vectors () =
52
+
match Csvt.decode_file vector_codec (trace "vectors.csv") with
53
+
| Ok rows -> rows
54
+
| Error e -> Alcotest.failf "CSV: %a" Csvt.pp_error e
55
+
56
+
let test_compress vec () =
57
+
let cfg =
58
+
Rice.config ~block_size:vec.block_size ~bits_per_sample:vec.bits_per_sample
59
+
()
60
+
in
61
+
let got = Rice.compress cfg vec.input in
62
+
if got <> vec.compressed then
63
+
Alcotest.failf "%s: compress mismatch\n expected: %s\n got: %s"
64
+
vec.name
65
+
(hex_of_bytes vec.compressed)
66
+
(hex_of_bytes got)
67
+
68
+
let test_decompress vec () =
69
+
let cfg =
70
+
Rice.config ~block_size:vec.block_size ~bits_per_sample:vec.bits_per_sample
71
+
()
72
+
in
73
+
let bps_bytes = (vec.bits_per_sample + 7) / 8 in
74
+
let sample_count = Bytes.length vec.input / bps_bytes in
75
+
match Rice.decompress ~sample_count cfg vec.compressed with
76
+
| Error msg -> Alcotest.failf "%s: decompress failed: %s" vec.name msg
77
+
| Ok got ->
78
+
if got <> vec.input then
79
+
Alcotest.failf "%s: decompress mismatch\n expected: %s\n got: %s"
80
+
vec.name (hex_of_bytes vec.input) (hex_of_bytes got)
81
+
82
+
let () =
83
+
let vectors = load_vectors () in
84
+
Alcotest.run "rice-interop-libaec"
85
+
[
86
+
( "compress",
87
+
List.map
88
+
(fun v -> Alcotest.test_case v.name `Quick (test_compress v))
89
+
vectors );
90
+
( "decompress",
91
+
List.map
92
+
(fun v -> Alcotest.test_case v.name `Quick (test_decompress v))
93
+
vectors );
94
+
]
-208
test/interop/libaec/test_libaec.ml
-208
test/interop/libaec/test_libaec.ml
···
1
-
(** Interop tests: ocaml-rice vs libaec (C library stubs).
2
-
3
-
Compresses data with ocaml-rice, decompresses with libaec (and vice versa),
4
-
then checks byte-for-byte equality of the round-tripped output.
5
-
6
-
Uses C stubs that call libaec's aec_encode/aec_decode functions directly, so
7
-
no CLI tool is needed -- only the libaec shared library. *)
8
-
9
-
(* -- C stub bindings -------------------------------------------------------- *)
10
-
11
-
external libaec_encode : bytes -> int -> int -> bytes = "caml_libaec_encode"
12
-
(** Compress raw samples using libaec. *)
13
-
14
-
external libaec_decode : bytes -> int -> int -> int -> bytes
15
-
= "caml_libaec_decode"
16
-
(** Decompress data using libaec. The fourth argument is the expected
17
-
decompressed size in bytes. *)
18
-
19
-
(* -- Helpers ---------------------------------------------------------------- *)
20
-
21
-
let bytes_eq = Alcotest.testable (Fmt.of_to_string Bytes.to_string) Bytes.equal
22
-
23
-
(** Pack a list of samples into big-endian bytes at the given bit depth. *)
24
-
let pack_samples ~bits_per_sample samples =
25
-
let bps_bytes = (bits_per_sample + 7) / 8 in
26
-
let n = List.length samples in
27
-
let buf = Bytes.make (n * bps_bytes) '\000' in
28
-
List.iteri
29
-
(fun i s ->
30
-
for j = bps_bytes - 1 downto 0 do
31
-
Bytes.set_uint8 buf
32
-
((i * bps_bytes) + (bps_bytes - 1 - j))
33
-
((s lsr (j * 8)) land 0xFF)
34
-
done)
35
-
samples;
36
-
buf
37
-
38
-
(** Generate deterministic pseudo-random samples. *)
39
-
let generate_prng_samples ~bits_per_sample ~count ~seed =
40
-
let state = Random.State.make [| seed |] in
41
-
let max_val = 1 lsl min bits_per_sample 30 in
42
-
List.init count (fun _ -> Random.State.int state max_val)
43
-
44
-
(** Ramp: [0, 1, 2, ..., count-1], masked to bit depth. *)
45
-
let generate_ramp ~bits_per_sample ~count =
46
-
let mask = (1 lsl bits_per_sample) - 1 in
47
-
List.init count (fun i -> i land mask)
48
-
49
-
(** Constant value. *)
50
-
let generate_constant ~count value = List.init count (fun _ -> value)
51
-
52
-
(* -- Test body -------------------------------------------------------------- *)
53
-
54
-
(** Compress with ocaml-rice, decompress with libaec, compare. *)
55
-
let test_rice_compress_libaec_decompress ~block_size ~bits_per_sample raw_data
56
-
() =
57
-
let cfg = Rice.config ~block_size ~bits_per_sample () in
58
-
let compressed = Rice.compress cfg raw_data in
59
-
let expected_size = Bytes.length raw_data in
60
-
let result =
61
-
libaec_decode compressed block_size bits_per_sample expected_size
62
-
in
63
-
Alcotest.(check bytes_eq)
64
-
(Printf.sprintf "rice->libaec bs=%d bps=%d" block_size bits_per_sample)
65
-
raw_data result
66
-
67
-
(** Compress with libaec, decompress with ocaml-rice, compare. *)
68
-
let test_libaec_compress_rice_decompress ~block_size ~bits_per_sample raw_data
69
-
() =
70
-
let cfg = Rice.config ~block_size ~bits_per_sample () in
71
-
let bps_bytes = (bits_per_sample + 7) / 8 in
72
-
let sample_count = Bytes.length raw_data / bps_bytes in
73
-
let compressed = libaec_encode raw_data block_size bits_per_sample in
74
-
match Rice.decompress ~sample_count cfg compressed with
75
-
| Error msg ->
76
-
Alcotest.failf "rice decompress failed (libaec compressed): %s" msg
77
-
| Ok result ->
78
-
Alcotest.(check bytes_eq)
79
-
(Printf.sprintf "libaec->rice bs=%d bps=%d" block_size bits_per_sample)
80
-
raw_data result
81
-
82
-
(** Full round-trip: rice compress -> libaec decompress -> libaec compress ->
83
-
rice decompress, comparing at each stage. *)
84
-
let test_full_roundtrip ~block_size ~bits_per_sample raw_data () =
85
-
let cfg = Rice.config ~block_size ~bits_per_sample () in
86
-
let expected_size = Bytes.length raw_data in
87
-
(* Step 1: compress with rice *)
88
-
let rice_compressed = Rice.compress cfg raw_data in
89
-
(* Step 2: decompress with libaec *)
90
-
let libaec_raw =
91
-
libaec_decode rice_compressed block_size bits_per_sample expected_size
92
-
in
93
-
Alcotest.(check bytes_eq)
94
-
"rice->libaec decompression matches original" raw_data libaec_raw;
95
-
(* Step 3: recompress with libaec *)
96
-
let libaec_compressed = libaec_encode libaec_raw block_size bits_per_sample in
97
-
(* Step 4: decompress with rice *)
98
-
let bps_bytes = (bits_per_sample + 7) / 8 in
99
-
let sample_count = Bytes.length raw_data / bps_bytes in
100
-
match Rice.decompress ~sample_count cfg libaec_compressed with
101
-
| Error msg ->
102
-
Alcotest.failf "rice decompress of libaec-compressed data failed: %s" msg
103
-
| Ok result ->
104
-
Alcotest.(check bytes_eq)
105
-
"full round-trip matches original" raw_data result
106
-
107
-
(* -- Configurations --------------------------------------------------------- *)
108
-
109
-
type test_config = { block_size : int; bits_per_sample : int }
110
-
111
-
let configs =
112
-
[
113
-
{ block_size = 16; bits_per_sample = 8 };
114
-
{ block_size = 16; bits_per_sample = 16 };
115
-
]
116
-
117
-
let label c = Printf.sprintf "bs=%d/bps=%d" c.block_size c.bits_per_sample
118
-
119
-
(** Generate test data for a given config. Multiple patterns. *)
120
-
let make_test_data c =
121
-
let count = c.block_size * 4 in
122
-
(* 4 full blocks *)
123
-
let bps = c.bits_per_sample in
124
-
let ramp = generate_ramp ~bits_per_sample:bps ~count in
125
-
let constant = generate_constant ~count 42 in
126
-
let prng = generate_prng_samples ~bits_per_sample:bps ~count ~seed:12345 in
127
-
( pack_samples ~bits_per_sample:bps ramp,
128
-
pack_samples ~bits_per_sample:bps constant,
129
-
pack_samples ~bits_per_sample:bps prng )
130
-
131
-
(* -- Runner ----------------------------------------------------------------- *)
132
-
133
-
let () =
134
-
let rice_to_libaec =
135
-
List.concat_map
136
-
(fun c ->
137
-
let ramp, constant, prng = make_test_data c in
138
-
[
139
-
Alcotest.test_case
140
-
(Printf.sprintf "ramp %s" (label c))
141
-
`Quick
142
-
(test_rice_compress_libaec_decompress ~block_size:c.block_size
143
-
~bits_per_sample:c.bits_per_sample ramp);
144
-
Alcotest.test_case
145
-
(Printf.sprintf "constant %s" (label c))
146
-
`Quick
147
-
(test_rice_compress_libaec_decompress ~block_size:c.block_size
148
-
~bits_per_sample:c.bits_per_sample constant);
149
-
Alcotest.test_case
150
-
(Printf.sprintf "prng %s" (label c))
151
-
`Quick
152
-
(test_rice_compress_libaec_decompress ~block_size:c.block_size
153
-
~bits_per_sample:c.bits_per_sample prng);
154
-
])
155
-
configs
156
-
in
157
-
let libaec_to_rice =
158
-
List.concat_map
159
-
(fun c ->
160
-
let ramp, constant, prng = make_test_data c in
161
-
[
162
-
Alcotest.test_case
163
-
(Printf.sprintf "ramp %s" (label c))
164
-
`Quick
165
-
(test_libaec_compress_rice_decompress ~block_size:c.block_size
166
-
~bits_per_sample:c.bits_per_sample ramp);
167
-
Alcotest.test_case
168
-
(Printf.sprintf "constant %s" (label c))
169
-
`Quick
170
-
(test_libaec_compress_rice_decompress ~block_size:c.block_size
171
-
~bits_per_sample:c.bits_per_sample constant);
172
-
Alcotest.test_case
173
-
(Printf.sprintf "prng %s" (label c))
174
-
`Quick
175
-
(test_libaec_compress_rice_decompress ~block_size:c.block_size
176
-
~bits_per_sample:c.bits_per_sample prng);
177
-
])
178
-
configs
179
-
in
180
-
let full_roundtrip =
181
-
List.concat_map
182
-
(fun c ->
183
-
let ramp, constant, prng = make_test_data c in
184
-
[
185
-
Alcotest.test_case
186
-
(Printf.sprintf "ramp %s" (label c))
187
-
`Quick
188
-
(test_full_roundtrip ~block_size:c.block_size
189
-
~bits_per_sample:c.bits_per_sample ramp);
190
-
Alcotest.test_case
191
-
(Printf.sprintf "constant %s" (label c))
192
-
`Quick
193
-
(test_full_roundtrip ~block_size:c.block_size
194
-
~bits_per_sample:c.bits_per_sample constant);
195
-
Alcotest.test_case
196
-
(Printf.sprintf "prng %s" (label c))
197
-
`Quick
198
-
(test_full_roundtrip ~block_size:c.block_size
199
-
~bits_per_sample:c.bits_per_sample prng);
200
-
])
201
-
configs
202
-
in
203
-
Alcotest.run "rice-libaec"
204
-
[
205
-
("rice-compress/libaec-decompress", rice_to_libaec);
206
-
("libaec-compress/rice-decompress", libaec_to_rice);
207
-
("full-roundtrip", full_roundtrip);
208
-
]
+5
test/interop/libaec/traces/vectors.csv
+5
test/interop/libaec/traces/vectors.csv
···
1
+
name,block_size,bits_per_sample,input_hex,compressed_hex
2
+
ramp_bs16_bps8,16,8,000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f,2009249249249249249249249249249249249249249249249249
3
+
constant_bs16_bps8,16,8,2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a,02a1
4
+
ramp_bs16_bps16,16,16,0000000100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019001a001b001c001d001e001f0020002100220023002400250026002700280029002a002b002c002d002e002f0030003100320033003400350036003700380039003a003b003c003d003e003f,10000492492492491249249249249124924924924912492492492490
5
+
constant_bs16_bps16,16,16,002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a002a,00015080