this repo has no description
1#!/usr/bin/env bash
2set -euo pipefail
3
4# Test functionality of cmprss by comparing it with the official tools
5
6CACHE_DIR="${PRJ_ROOT}/.cache"
7
8# Create a tmp directory and cd into it
9tmpdir() {
10 mkdir -p "$CACHE_DIR"
11 local dir
12 dir=$(mktemp --directory --tmpdir="$CACHE_DIR")
13 cd "$dir"
14}
15
16# Compare the two files/directories and exit with an error if they are different
17compare() {
18 local file1="$1"
19 local file2="$2"
20 if ! diff -qr "$file1" "$file2" >/dev/null; then
21 echo "Diff Detected: $file1 $file2"
22 exit 1
23 fi
24}
25
26# Compare the two archive sizes to check if they are similar
27# TODO: This doesn't work anywhere in the file because we're compressing random data
28# So the algos will always fail to compress things very well
29compare_size() {
30 local file1="$1"
31 local file2="$2"
32 # Use $3 as the max difference or 100 bytes
33 local max_diff=${3:-100}
34 local size1
35 local size2
36 size1=$(stat -c %s "$file1")
37 size2=$(stat -c %s "$file2")
38 if [ $((size1 - size2)) -gt "$max_diff" ]; then
39 echo "Size difference too large: $file1:$size1 $file2:$size2"
40 exit 1
41 fi
42}
43
44# Create a random file with the given size
45random_file() {
46 local size="$1"
47 local file="$2"
48 dd if=/dev/urandom of="$file" bs=1 count="$size" 2>/dev/null
49}
50
51# Create a random directory with the given size
52random_dir() {
53 local size="$1"
54 local dir="$2"
55 mkdir -p "$dir"
56 for i in $(seq 1 "$size"); do
57 random_file 128 "$dir/$i"
58 done
59}
60
61# Run cmprss using cargo to test the current version
62cmprss() {
63 cargo run --release --quiet -- "$1" --ignore-pipes "${@:2}"
64}
65
66# Test gzip using the provided compression level
67test_gzip_level() {
68 tmpdir
69 echo "Testing gzip level $1 in $PWD"
70 echo "Creating random data"
71 random_file 1000000 file
72 echo "Compressing with gzip and cmprss"
73 gzip -"$1" -c file >gzip_file.gz
74 cmprss gzip --level "$1" file cmprss_file.gz --progress=off
75 # Compare the two archives
76 # The archives may have slight variations (versioning or whatever) so we
77 # only compare the sizes to make sure they are similar
78 compare_size gzip_file.gz cmprss_file.gz
79 # Decompress the 4 variations
80 echo "Decompressing"
81 gzip -c -d gzip_file.gz >gzip_gzip
82 gzip -c -d cmprss_file.gz >cmprss_gzip
83 cmprss gzip --extract cmprss_file.gz cmprss_cmprss --progress=off
84 cmprss gzip --extract gzip_file.gz gzip_cmprss --progress=off
85 echo "Comparing the decompressed files"
86 compare file gzip_gzip
87 compare file gzip_cmprss
88 compare file cmprss_cmprss
89 compare file cmprss_gzip
90 echo "No errors detected"
91}
92
93test_gzip() {
94 test_gzip_level 1
95 test_gzip_level 6 # Default
96 test_gzip_level 9
97}
98
99# Test xz using the provided compression level
100test_xz_level() {
101 tmpdir
102 echo "Testing xz level $1 in $PWD"
103 echo "Creating random data"
104 random_file 1000000 file
105 echo "Compressing with xz and cmprss"
106 xz -"$1" --stdout file >xz_file.xz
107 cmprss xz --level "$1" file cmprss_file.xz --progress=off
108 # Compare the two archives
109 # The archives may have slight variations (versioning or whatever) so we
110 # only compare the sizes to make sure they are similar
111 compare_size xz_file.xz cmprss_file.xz
112 # Decompress the 4 variations
113 echo "Decompressing"
114 xz --stdout --decompress xz_file.xz >xz_xz
115 xz --stdout --decompress cmprss_file.xz >xz_cmprss
116 cmprss xz --extract cmprss_file.xz cmprss_cmprss --progress=off
117 cmprss xz --extract xz_file.xz cmprss_xz --progress=off
118 echo "Comparing the decompressed files"
119 compare file xz_xz
120 compare file xz_cmprss
121 compare file cmprss_cmprss
122 compare file cmprss_xz
123 echo "No errors detected"
124}
125
126test_xz() {
127 test_xz_level 0 # No compression
128 test_xz_level 1
129 test_xz_level 6 # Default
130 test_xz_level 9
131}
132
133# Test bzip2 using the provided compression level
134test_bzip2_level() {
135 tmpdir
136 echo "Testing bzip2 level $1 in $PWD"
137 echo "Creating random data"
138 random_file 1000000 file
139 echo "Compressing with bzip2 and cmprss"
140 bzip2 -"$1" --stdout file >bzip2_file.bz2
141 cmprss bzip2 --level "$1" file cmprss_file.bz2 --progress=off
142 # Compare the two archives
143 # The archives may have slight variations (versioning or whatever) so we
144 # only compare the sizes to make sure they are similar
145 compare_size bzip2_file.bz2 cmprss_file.bz2
146 # Decompress the 4 variations
147 echo "Decompressing"
148 bzip2 --stdout --decompress bzip2_file.bz2 >bzip2_bzip2
149 bzip2 --stdout --decompress cmprss_file.bz2 >cmprss_bzip2
150 cmprss bzip2 --extract cmprss_file.bz2 cmprss_cmprss --progress=off
151 cmprss bzip2 --extract bzip2_file.bz2 bzip2_cmprss --progress=off
152 echo "Comparing the decompressed files"
153 compare file bzip2_bzip2
154 compare file bzip2_cmprss
155 compare file cmprss_cmprss
156 compare file cmprss_bzip2
157 echo "No errors detected"
158}
159
160test_bzip2() {
161 test_bzip2_level 1
162 test_bzip2_level 6
163 test_bzip2_level 9 # Default
164}
165
166# Test zstd using the provided compression level
167test_zstd_level() {
168 tmpdir
169 echo "Testing zstd level $1 in $PWD"
170 echo "Creating random data"
171 random_file 1000000 file
172 echo "Compressing with zstd and cmprss"
173 zstd -"$1" -c file >zstd_file.zst
174 cmprss zstd --level "$1" file cmprss_file.zst --progress=off
175 # Compare the two archives
176 # The archives may have slight variations (versioning or whatever) so we
177 # only compare the sizes to make sure they are similar
178 compare_size zstd_file.zst cmprss_file.zst
179 # Decompress the 4 variations
180 echo "Decompressing"
181 zstd -d -c zstd_file.zst >zstd_zstd
182 zstd -d -c cmprss_file.zst >cmprss_zstd
183 cmprss zstd --extract cmprss_file.zst cmprss_cmprss --progress=off
184 cmprss zstd --extract zstd_file.zst zstd_cmprss --progress=off
185 echo "Comparing the decompressed files"
186 compare file zstd_zstd
187 compare file zstd_cmprss
188 compare file cmprss_cmprss
189 compare file cmprss_zstd
190 echo "No errors detected"
191}
192
193test_zstd() {
194 test_zstd_level 1 # Fast compression
195 test_zstd_level 3
196 test_zstd_level 6 # Default
197 test_zstd_level 9 # High compression
198}
199
200# Test lz4 compression
201test_lz4() {
202 tmpdir
203 echo "Testing lz4 in $PWD"
204 echo "Creating random data"
205 random_file 1000000 file
206 echo "Compressing with lz4 and cmprss"
207 lz4 -c file >lz4_file.lz4
208 cmprss lz4 file cmprss_file.lz4 --progress=off
209 # Compare the two archives
210 # The archives may have slight variations (versioning or whatever) so we
211 # only compare the sizes to make sure they are similar
212 compare_size lz4_file.lz4 cmprss_file.lz4
213 # Decompress the 4 variations
214 echo "Decompressing"
215 lz4 -d -c lz4_file.lz4 >lz4_lz4
216 lz4 -d -c cmprss_file.lz4 >cmprss_lz4
217 cmprss lz4 --extract cmprss_file.lz4 cmprss_cmprss --progress=off
218 cmprss lz4 --extract lz4_file.lz4 lz4_cmprss --progress=off
219 echo "Comparing the decompressed files"
220 compare file lz4_lz4
221 compare file lz4_cmprss
222 compare file cmprss_cmprss
223 compare file cmprss_lz4
224 echo "No errors detected"
225}
226
227# Test lzma (legacy LZMA1) at the given level against the lzma CLI from xz-utils
228test_lzma_level() {
229 tmpdir
230 echo "Testing lzma level $1 in $PWD"
231 echo "Creating random data"
232 random_file 1000000 file
233 echo "Compressing with lzma and cmprss"
234 lzma --stdout -"$1" file >lzma_file.lzma
235 cmprss lzma --level "$1" file cmprss_file.lzma --progress=off
236 compare_size lzma_file.lzma cmprss_file.lzma
237 echo "Decompressing"
238 lzma --stdout --decompress lzma_file.lzma >lzma_lzma
239 lzma --stdout --decompress cmprss_file.lzma >cmprss_lzma
240 cmprss lzma --extract cmprss_file.lzma cmprss_cmprss --progress=off
241 cmprss lzma --extract lzma_file.lzma lzma_cmprss --progress=off
242 echo "Comparing the decompressed files"
243 compare file lzma_lzma
244 compare file cmprss_lzma
245 compare file cmprss_cmprss
246 compare file lzma_cmprss
247 echo "No errors detected"
248}
249
250test_lzma() {
251 test_lzma_level 1
252 test_lzma_level 6 # Default
253 test_lzma_level 9
254}
255
256# Test brotli at the given quality against the brotli CLI
257test_brotli_level() {
258 tmpdir
259 echo "Testing brotli quality $1 in $PWD"
260 echo "Creating random data"
261 random_file 1000000 file
262 echo "Compressing with brotli and cmprss"
263 brotli --quality="$1" --stdout file >brotli_file.br
264 cmprss brotli --level "$1" file cmprss_file.br --progress=off
265 compare_size brotli_file.br cmprss_file.br
266 echo "Decompressing"
267 brotli --decompress --stdout brotli_file.br >brotli_brotli
268 brotli --decompress --stdout cmprss_file.br >cmprss_brotli
269 cmprss brotli --extract cmprss_file.br cmprss_cmprss --progress=off
270 cmprss brotli --extract brotli_file.br brotli_cmprss --progress=off
271 echo "Comparing the decompressed files"
272 compare file brotli_brotli
273 compare file cmprss_brotli
274 compare file cmprss_cmprss
275 compare file brotli_cmprss
276 echo "No errors detected"
277}
278
279test_brotli() {
280 test_brotli_level 1
281 test_brotli_level 6 # Default
282 test_brotli_level 11
283}
284
285# Test framed snappy interop against the snzip CLI. snzip's default format is
286# `framing2`, which is the same official framing format the `snap` crate uses
287# (and that cmprss writes with the `.sz` extension). Snappy has no compression
288# level, so no --level is passed.
289test_snappy() {
290 tmpdir
291 echo "Testing snappy in $PWD"
292 echo "Creating random data"
293 random_file 1000000 file
294 echo "Compressing with snzip and cmprss"
295 snzip -c file >snzip_file.sz
296 cmprss snappy file cmprss_file.sz --progress=off
297 compare_size snzip_file.sz cmprss_file.sz
298 echo "Decompressing"
299 snzip -d -c snzip_file.sz >snzip_snzip
300 snzip -d -c cmprss_file.sz >cmprss_snzip
301 cmprss snappy --extract cmprss_file.sz cmprss_cmprss --progress=off
302 cmprss snappy --extract snzip_file.sz snzip_cmprss --progress=off
303 echo "Comparing the decompressed files"
304 compare file snzip_snzip
305 compare file cmprss_snzip
306 compare file cmprss_cmprss
307 compare file snzip_cmprss
308 echo "No errors detected"
309}
310
311# Test tar archive interop with the tar CLI. Tar has no progress bar, so no
312# --progress flag is passed.
313test_tar() {
314 tmpdir
315 echo "Testing tar in $PWD"
316 echo "Creating random data"
317 random_dir 10 indir
318 echo "Creating tar archives with each tool"
319 tar -cf tar_archive.tar indir
320 cmprss tar indir cmprss_archive.tar
321 echo "Extracting each archive with the opposite tool"
322 mkdir -p tar_from_cmprss
323 tar -xf cmprss_archive.tar -C tar_from_cmprss
324 mkdir -p cmprss_from_tar
325 cmprss tar --extract tar_archive.tar cmprss_from_tar
326 echo "Comparing the extracted contents"
327 compare indir tar_from_cmprss/indir
328 compare indir cmprss_from_tar/indir
329 echo "No errors detected"
330}
331
332# Test zip archive interop with the zip/unzip CLIs. Zip has no progress bar,
333# so no --progress flag is passed.
334test_zip() {
335 tmpdir
336 echo "Testing zip in $PWD"
337 echo "Creating random data"
338 random_dir 10 indir
339 echo "Creating zip archives with each tool"
340 (cd "$(dirname indir)" && zip -qr zip_archive.zip "$(basename indir)")
341 cmprss zip indir cmprss_archive.zip
342 echo "Extracting each archive with the opposite tool"
343 mkdir -p zip_from_cmprss
344 (cd zip_from_cmprss && unzip -q ../cmprss_archive.zip)
345 mkdir -p cmprss_from_zip
346 cmprss zip --extract zip_archive.zip cmprss_from_zip
347 echo "Comparing the extracted contents"
348 compare indir zip_from_cmprss/indir
349 compare indir cmprss_from_zip/indir
350 echo "No errors detected"
351}
352
353# Test 7z archive interop with the 7z CLI (from p7zip). 7z has no --progress
354# flag, so it isn't passed here. -bso0/-bsp0 silence 7z's output/progress noise.
355test_7z() {
356 tmpdir
357 echo "Testing 7z in $PWD"
358 echo "Creating random data"
359 random_dir 10 indir
360 echo "Creating 7z archives with each tool"
361 7z a -bso0 -bsp0 sevenz_archive.7z indir
362 cmprss 7z indir cmprss_archive.7z
363 echo "Extracting each archive with the opposite tool"
364 mkdir -p sevenz_from_cmprss
365 7z x -bso0 -bsp0 "-osevenz_from_cmprss" cmprss_archive.7z
366 mkdir -p cmprss_from_sevenz
367 cmprss 7z --extract sevenz_archive.7z cmprss_from_sevenz
368 echo "Comparing the extracted contents"
369 compare indir sevenz_from_cmprss/indir
370 compare indir cmprss_from_sevenz/indir
371 echo "No errors detected"
372}
373
374# Shared helper for tar.<codec> pipeline interop. The first arg is the compound
375# extension; the rest are the tar flags used to compress/extract that codec
376# (e.g. `-z`, `--zstd`, or `-I lzma` for codecs without a short flag).
377# Verifies that cmprss produces archives the tar CLI can read, and vice versa.
378# Pipelines are invoked without a subcommand so --progress isn't accepted; the
379# archive is written to a file, not stdout, which is fine for tests.
380test_tar_pipeline() {
381 local ext="$1"
382 shift
383 local tar_flags=("$@")
384 tmpdir
385 echo "Testing $ext pipeline in $PWD"
386 echo "Creating random data"
387 random_dir 10 indir
388 echo "Creating $ext archives with each tool"
389 tar "${tar_flags[@]}" -cf tar_archive."$ext" indir
390 cmprss indir cmprss_archive."$ext"
391 echo "Extracting each archive with the opposite tool"
392 mkdir -p tar_from_cmprss
393 tar "${tar_flags[@]}" -xf cmprss_archive."$ext" -C tar_from_cmprss
394 mkdir -p cmprss_from_tar
395 cmprss --extract tar_archive."$ext" cmprss_from_tar
396 echo "Comparing the extracted contents"
397 compare indir tar_from_cmprss/indir
398 compare indir cmprss_from_tar/indir
399 echo "No errors detected"
400}
401
402test_tar_gz() { test_tar_pipeline tar.gz -z; }
403test_tar_xz() { test_tar_pipeline tar.xz -J; }
404test_tar_bz2() { test_tar_pipeline tar.bz2 -j; }
405test_tar_zst() { test_tar_pipeline tar.zst --zstd; }
406# tar has no short flag for these codecs; drive the external CLI via -I.
407test_tar_lzma() { test_tar_pipeline tar.lzma -I lzma; }
408test_tar_br() { test_tar_pipeline tar.br -I brotli; }
409test_tar_lz4() { test_tar_pipeline tar.lz4 -I lz4; }
410test_tar_sz() { test_tar_pipeline tar.sz -I snzip; }
411
412# Run all the tests if no arguments are given
413if [ $# -eq 0 ]; then
414 set -- gzip xz bzip2 zstd lz4 lzma brotli snappy tar zip 7z \
415 tar_gz tar_xz tar_bz2 tar_zst tar_lzma tar_br tar_lz4 tar_sz
416fi
417
418# Run the tests given on the command line
419for test in "$@"; do
420 test_"$test"
421done