+26

Cargo.lock

reviewed

··· 256 256 "flate2", 257 257 "indicatif", 258 258 "is-terminal", 259 259 + "lz4_flex", 259 260 "predicates", 260 261 "rand", 261 262 "tar", ··· 592 593 checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" 593 594 594 595 [[package]] 596 596 + name = "lz4_flex" 597 597 + version = "0.11.3" 598 598 + source = "registry+https://github.com/rust-lang/crates.io-index" 599 599 + checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" 600 600 + dependencies = [ 601 601 + "twox-hash", 602 602 + ] 603 603 + 604 604 + [[package]] 595 605 name = "lzma-sys" 596 606 version = "0.1.20" 597 607 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 878 888 ] 879 889 880 890 [[package]] 891 891 + name = "static_assertions" 892 892 + version = "1.1.0" 893 893 + source = "registry+https://github.com/rust-lang/crates.io-index" 894 894 + checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 895 895 + 896 896 + [[package]] 881 897 name = "strsim" 882 898 version = "0.11.1" 883 899 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 947 963 version = "0.1.3" 948 964 source = "registry+https://github.com/rust-lang/crates.io-index" 949 965 checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" 966 966 + 967 967 + [[package]] 968 968 + name = "twox-hash" 969 969 + version = "1.6.3" 970 970 + source = "registry+https://github.com/rust-lang/crates.io-index" 971 971 + checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" 972 972 + dependencies = [ 973 973 + "cfg-if", 974 974 + "static_assertions", 975 975 + ] 950 976 951 977 [[package]] 952 978 name = "typenum"

+1

Cargo.toml

reviewed

··· 21 21 zip = "0.6" 22 22 tempfile = "3.10" 23 23 zstd = "0.13" 24 24 + lz4_flex = "0.11" 24 25 25 26 [dev-dependencies] 26 27 assert_cmd = "2"

+1

README.md

reviewed

··· 18 18 19 19 - bzip2 20 20 - gzip 21 21 + - lz4 21 22 - tar 22 23 - xz 23 24 - zip

+28 -1

bin/test.sh

reviewed

··· 194 194 test_zstd_level 9 # High compression 195 195 } 196 196 197 197 + # Test lz4 compression 198 198 + test_lz4() { 199 199 + tmpdir 200 200 + echo "Testing lz4 in $PWD" 201 201 + echo "Creating random data" 202 202 + random_file 1000000 file 203 203 + echo "Compressing with lz4 and cmprss" 204 204 + lz4 -c file >lz4_file.lz4 205 205 + cmprss lz4 file cmprss_file.lz4 --progress=off 206 206 + # Compare the two archives 207 207 + # The archives may have slight variations (versioning or whatever) so we 208 208 + # only compare the sizes to make sure they are similar 209 209 + compare_size lz4_file.lz4 cmprss_file.lz4 210 210 + # Decompress the 4 variations 211 211 + echo "Decompressing" 212 212 + lz4 -d -c lz4_file.lz4 >lz4_lz4 213 213 + lz4 -d -c cmprss_file.lz4 >cmprss_lz4 214 214 + cmprss lz4 --extract cmprss_file.lz4 cmprss_cmprss --progress=off 215 215 + cmprss lz4 --extract lz4_file.lz4 lz4_cmprss --progress=off 216 216 + echo "Comparing the decompressed files" 217 217 + compare file lz4_lz4 218 218 + compare file lz4_cmprss 219 219 + compare file cmprss_cmprss 220 220 + compare file cmprss_lz4 221 221 + echo "No errors detected" 222 222 + } 223 223 + 197 224 # Run all the tests if no arguments are given 198 225 if [ $# -eq 0 ]; then 199 199 - set -- gzip xz bzip2 zstd 226 226 + set -- gzip xz bzip2 zstd lz4 200 227 fi 201 228 202 229 # Run the tests given on the command line

+1

flake.nix

reviewed

··· 182 182 bzip2 183 183 gnutar 184 184 gzip 185 185 + lz4 185 186 xz 186 187 zstd 187 188 ];

+187

src/backends/lz4.rs

reviewed

··· 1 1 + use crate::progress::{copy_with_progress, ProgressArgs}; 2 2 + use crate::utils::{cmprss_error, CmprssInput, CmprssOutput, CommonArgs, Compressor}; 3 3 + use clap::Args; 4 4 + use lz4_flex::frame::{FrameDecoder, FrameEncoder}; 5 5 + use std::fs::File; 6 6 + use std::io::{self, BufReader, BufWriter, Read, Write}; 7 7 + 8 8 + #[derive(Args, Debug)] 9 9 + pub struct Lz4Args { 10 10 + #[clap(flatten)] 11 11 + pub common_args: CommonArgs, 12 12 + 13 13 + #[clap(flatten)] 14 14 + pub progress_args: ProgressArgs, 15 15 + } 16 16 + 17 17 + #[derive(Default)] 18 18 + pub struct Lz4 { 19 19 + pub progress_args: ProgressArgs, 20 20 + } 21 21 + 22 22 + impl Lz4 { 23 23 + pub fn new(args: &Lz4Args) -> Lz4 { 24 24 + Lz4 { 25 25 + progress_args: args.progress_args, 26 26 + } 27 27 + } 28 28 + } 29 29 + 30 30 + impl Compressor for Lz4 { 31 31 + /// The standard extension for the lz4 format. 32 32 + fn extension(&self) -> &str { 33 33 + "lz4" 34 34 + } 35 35 + 36 36 + /// Full name for lz4. 37 37 + fn name(&self) -> &str { 38 38 + "lz4" 39 39 + } 40 40 + 41 41 + /// Generate a default extracted filename 42 42 + /// lz4 does not support extracting to a directory, so we return a default filename 43 43 + fn default_extracted_filename(&self, in_path: &std::path::Path) -> String { 44 44 + // If the file has no extension, return a default filename 45 45 + if in_path.extension().is_none() { 46 46 + return "archive".to_string(); 47 47 + } 48 48 + // Otherwise, return the filename without the extension 49 49 + in_path.file_stem().unwrap().to_str().unwrap().to_string() 50 50 + } 51 51 + 52 52 + /// Compress an input file or pipe to a lz4 archive 53 53 + fn compress(&self, input: CmprssInput, output: CmprssOutput) -> Result<(), io::Error> { 54 54 + if let CmprssOutput::Path(out_path) = &output { 55 55 + if out_path.is_dir() { 56 56 + return cmprss_error("LZ4 does not support compressing to a directory. Please specify an output file."); 57 57 + } 58 58 + } 59 59 + if let CmprssInput::Path(input_paths) = &input { 60 60 + for x in input_paths { 61 61 + if x.is_dir() { 62 62 + return cmprss_error( 63 63 + "LZ4 does not support compressing a directory. Please specify only files.", 64 64 + ); 65 65 + } 66 66 + } 67 67 + } 68 68 + let mut file_size = None; 69 69 + let mut input_stream: Box<dyn Read + Send> = match input { 70 70 + CmprssInput::Path(paths) => { 71 71 + if paths.len() > 1 { 72 72 + return Err(io::Error::new( 73 73 + io::ErrorKind::InvalidInput, 74 74 + "Multiple input files not supported for lz4", 75 75 + )); 76 76 + } 77 77 + let path = &paths[0]; 78 78 + file_size = Some(std::fs::metadata(path)?.len()); 79 79 + Box::new(BufReader::new(File::open(path)?)) 80 80 + } 81 81 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 82 82 + }; 83 83 + 84 84 + let output_stream: Box<dyn Write + Send> = match &output { 85 85 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 86 86 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 87 87 + }; 88 88 + 89 89 + // Create a lz4 encoder 90 90 + let mut encoder = FrameEncoder::new(output_stream); 91 91 + 92 92 + // Copy the input to the encoder with progress reporting 93 93 + copy_with_progress( 94 94 + &mut input_stream, 95 95 + &mut encoder, 96 96 + self.progress_args.chunk_size.size_in_bytes, 97 97 + file_size, 98 98 + self.progress_args.progress, 99 99 + &output, 100 100 + )?; 101 101 + 102 102 + // Finish the encoder to ensure all data is written 103 103 + encoder.finish()?; 104 104 + 105 105 + Ok(()) 106 106 + } 107 107 + 108 108 + /// Extract a lz4 archive to an output file or pipe 109 109 + fn extract(&self, input: CmprssInput, output: CmprssOutput) -> Result<(), io::Error> { 110 110 + if let CmprssOutput::Path(out_path) = &output { 111 111 + if out_path.is_dir() { 112 112 + return cmprss_error("LZ4 does not support extracting to a directory. Please specify an output file."); 113 113 + } 114 114 + } 115 115 + 116 116 + let input_stream: Box<dyn Read + Send> = match input { 117 117 + CmprssInput::Path(paths) => { 118 118 + if paths.len() > 1 { 119 119 + return Err(io::Error::new( 120 120 + io::ErrorKind::InvalidInput, 121 121 + "Multiple input files not supported for lz4", 122 122 + )); 123 123 + } 124 124 + let path = &paths[0]; 125 125 + Box::new(BufReader::new(File::open(path)?)) 126 126 + } 127 127 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 128 128 + }; 129 129 + 130 130 + // Create a lz4 decoder 131 131 + let mut decoder = FrameDecoder::new(input_stream); 132 132 + 133 133 + let mut output_stream: Box<dyn Write + Send> = match &output { 134 134 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 135 135 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 136 136 + }; 137 137 + 138 138 + // Copy the decoded data to the output with progress reporting 139 139 + copy_with_progress( 140 140 + &mut decoder, 141 141 + &mut output_stream, 142 142 + self.progress_args.chunk_size.size_in_bytes, 143 143 + None, 144 144 + self.progress_args.progress, 145 145 + &output, 146 146 + )?; 147 147 + 148 148 + Ok(()) 149 149 + } 150 150 + } 151 151 + 152 152 + #[cfg(test)] 153 153 + mod tests { 154 154 + use super::*; 155 155 + use tempfile::tempdir; 156 156 + 157 157 + #[test] 158 158 + fn roundtrip() -> Result<(), Box<dyn std::error::Error>> { 159 159 + let dir = tempdir()?; 160 160 + let input_path = dir.path().join("input.txt"); 161 161 + let compressed_path = dir.path().join("input.txt.lz4"); 162 162 + let output_path = dir.path().join("output.txt"); 163 163 + 164 164 + // Create a test file 165 165 + let test_data = b"Hello, world! This is a test file for lz4 compression."; 166 166 + std::fs::write(&input_path, test_data)?; 167 167 + 168 168 + // Compress the file 169 169 + let lz4 = Lz4::default(); 170 170 + lz4.compress( 171 171 + CmprssInput::Path(vec![input_path.clone()]), 172 172 + CmprssOutput::Path(compressed_path.clone()), 173 173 + )?; 174 174 + 175 175 + // Extract the file 176 176 + lz4.extract( 177 177 + CmprssInput::Path(vec![compressed_path]), 178 178 + CmprssOutput::Path(output_path.clone()), 179 179 + )?; 180 180 + 181 181 + // Verify the contents 182 182 + let output_data = std::fs::read(output_path)?; 183 183 + assert_eq!(test_data.to_vec(), output_data); 184 184 + 185 185 + Ok(()) 186 186 + } 187 187 + }

+2

src/backends/mod.rs

reviewed

··· 1 1 mod bzip2; 2 2 mod gzip; 3 3 + mod lz4; 3 4 mod tar; 4 5 mod xz; 5 6 mod zip; ··· 7 8 8 9 pub use bzip2::{Bzip2, Bzip2Args}; 9 10 pub use gzip::{Gzip, GzipArgs}; 11 11 + pub use lz4::{Lz4, Lz4Args}; 10 12 pub use tar::{Tar, TarArgs}; 11 13 pub use xz::{Xz, XzArgs}; 12 14 pub use zip::{Zip, ZipArgs};

+5

src/main.rs

reviewed

··· 43 43 /// zstd compression 44 44 #[clap(visible_alias = "zst")] 45 45 Zstd(ZstdArgs), 46 46 + 47 47 + /// lz4 compression 48 48 + Lz4(Lz4Args), 46 49 } 47 50 48 51 /// Get the input filename or return a default file ··· 88 91 Box::<Bzip2>::default(), 89 92 Box::<Zip>::default(), 90 93 Box::<Zstd>::default(), 94 94 + Box::<Lz4>::default(), 91 95 ]; 92 96 compressors.into_iter().find(|c| c.is_archive(filename)) 93 97 } ··· 499 503 Some(Format::Bzip2(a)) => command(Some(Box::new(Bzip2::new(&a))), &a.common_args), 500 504 Some(Format::Zip(a)) => command(Some(Box::new(Zip::new(&a))), &a.common_args), 501 505 Some(Format::Zstd(a)) => command(Some(Box::new(Zstd::new(&a))), &a.common_args), 506 506 + Some(Format::Lz4(a)) => command(Some(Box::new(Lz4::new(&a))), &a.common_args), 502 507 _ => command(None, &args.base_args), 503 508 } 504 509 .unwrap_or_else(|e| {

+134

tests/cli.rs

reviewed

··· 1249 1249 1250 1250 Ok(()) 1251 1251 } 1252 1252 + 1253 1253 + #[test] 1254 1254 + fn lz4_roundtrip_explicit() -> Result<(), Box<dyn std::error::Error>> { 1255 1255 + let mut cmd = Command::cargo_bin("cmprss")?; 1256 1256 + let dir = assert_fs::TempDir::new()?; 1257 1257 + 1258 1258 + // Create a test file 1259 1259 + let test_file = dir.child("test.txt"); 1260 1260 + test_file.write_str("This is a test file for LZ4 compression.")?; 1261 1261 + 1262 1262 + // Create output paths 1263 1263 + let compressed_file = dir.child("test.txt.lz4"); 1264 1264 + let extracted_file = dir.child("test_extracted.txt"); 1265 1265 + 1266 1266 + // Compress the file 1267 1267 + cmd.arg("lz4") 1268 1268 + .arg("--compress") 1269 1269 + .arg(test_file.path()) 1270 1270 + .arg(compressed_file.path()) 1271 1271 + .arg("--progress=off"); 1272 1272 + 1273 1273 + cmd.assert().success(); 1274 1274 + compressed_file.assert(predicates::path::exists()); 1275 1275 + 1276 1276 + // Extract the file 1277 1277 + let mut cmd = Command::cargo_bin("cmprss")?; 1278 1278 + cmd.arg("lz4") 1279 1279 + .arg("--extract") 1280 1280 + .arg(compressed_file.path()) 1281 1281 + .arg(extracted_file.path()) 1282 1282 + .arg("--progress=off"); 1283 1283 + 1284 1284 + cmd.assert().success(); 1285 1285 + extracted_file.assert(predicates::path::exists()); 1286 1286 + 1287 1287 + // Verify the contents 1288 1288 + let original_content = std::fs::read_to_string(test_file.path())?; 1289 1289 + let extracted_content = std::fs::read_to_string(extracted_file.path())?; 1290 1290 + assert_eq!(original_content, extracted_content); 1291 1291 + 1292 1292 + Ok(()) 1293 1293 + } 1294 1294 + 1295 1295 + #[test] 1296 1296 + fn lz4_roundtrip_stdin() -> Result<(), Box<dyn std::error::Error>> { 1297 1297 + let mut cmd = Command::cargo_bin("cmprss")?; 1298 1298 + let dir = assert_fs::TempDir::new()?; 1299 1299 + 1300 1300 + // Create a test file 1301 1301 + let test_file = dir.child("test.txt"); 1302 1302 + test_file.write_str("This is a test file for LZ4 compression via stdin.")?; 1303 1303 + 1304 1304 + // Create output paths 1305 1305 + let compressed_file = dir.child("test.txt.lz4"); 1306 1306 + let extracted_file = dir.child("test_extracted.txt"); 1307 1307 + 1308 1308 + // Compress the file via stdin 1309 1309 + let test_content = std::fs::read_to_string(test_file.path())?; 1310 1310 + cmd.arg("lz4") 1311 1311 + .arg("--compress") 1312 1312 + .arg("--output") 1313 1313 + .arg(compressed_file.path()) 1314 1314 + .arg("--progress=off") 1315 1315 + .stdin(Stdio::piped()); 1316 1316 + 1317 1317 + let mut child = cmd.spawn()?; 1318 1318 + if let Some(stdin) = child.stdin.as_mut() { 1319 1319 + use std::io::Write; 1320 1320 + stdin.write_all(test_content.as_bytes())?; 1321 1321 + } 1322 1322 + let output = child.wait_with_output()?; 1323 1323 + assert!(output.status.success()); 1324 1324 + compressed_file.assert(predicate::path::exists()); 1325 1325 + 1326 1326 + // Extract the file 1327 1327 + let mut cmd = Command::cargo_bin("cmprss")?; 1328 1328 + cmd.arg("lz4") 1329 1329 + .arg("--extract") 1330 1330 + .arg(compressed_file.path()) 1331 1331 + .arg(extracted_file.path()) 1332 1332 + .arg("--progress=off"); 1333 1333 + 1334 1334 + cmd.assert().success(); 1335 1335 + extracted_file.assert(predicate::path::exists()); 1336 1336 + 1337 1337 + // Verify the contents 1338 1338 + let original_content = std::fs::read_to_string(test_file.path())?; 1339 1339 + let extracted_content = std::fs::read_to_string(extracted_file.path())?; 1340 1340 + assert_eq!(original_content, extracted_content); 1341 1341 + 1342 1342 + Ok(()) 1343 1343 + } 1344 1344 + 1345 1345 + #[test] 1346 1346 + fn lz4_roundtrip_stdout() -> Result<(), Box<dyn std::error::Error>> { 1347 1347 + let mut cmd = Command::cargo_bin("cmprss")?; 1348 1348 + let dir = assert_fs::TempDir::new()?; 1349 1349 + 1350 1350 + // Create a test file 1351 1351 + let test_file = dir.child("test.txt"); 1352 1352 + test_file.write_str("This is a test file for LZ4 compression to stdout.")?; 1353 1353 + 1354 1354 + // Create output paths 1355 1355 + let compressed_file = dir.child("test.txt.lz4"); 1356 1356 + let extracted_file = dir.child("test_extracted.txt"); 1357 1357 + 1358 1358 + // Compress the file 1359 1359 + cmd.arg("lz4") 1360 1360 + .arg("--compress") 1361 1361 + .arg(test_file.path()) 1362 1362 + .arg("--progress=off"); 1363 1363 + 1364 1364 + let output = cmd.output()?; 1365 1365 + assert!(output.status.success()); 1366 1366 + std::fs::write(compressed_file.path(), output.stdout)?; 1367 1367 + 1368 1368 + // Extract the file to stdout 1369 1369 + let mut cmd = Command::cargo_bin("cmprss")?; 1370 1370 + cmd.arg("lz4") 1371 1371 + .arg("--extract") 1372 1372 + .arg(compressed_file.path()) 1373 1373 + .arg("--progress=off"); 1374 1374 + 1375 1375 + let output = cmd.output()?; 1376 1376 + assert!(output.status.success()); 1377 1377 + std::fs::write(extracted_file.path(), output.stdout)?; 1378 1378 + 1379 1379 + // Verify the contents 1380 1380 + let original_content = std::fs::read_to_string(test_file.path())?; 1381 1381 + let extracted_content = std::fs::read_to_string(extracted_file.path())?; 1382 1382 + assert_eq!(original_content, extracted_content); 1383 1383 + 1384 1384 + Ok(()) 1385 1385 + } 1252 1386 }