+230

src/backends/lzma.rs

reviewed

··· 1 1 + use crate::{ 2 2 + progress::{ProgressArgs, copy_with_progress}, 3 3 + utils::*, 4 4 + }; 5 5 + use anyhow::bail; 6 6 + use clap::Args; 7 7 + use std::{ 8 8 + fs::File, 9 9 + io::{self, BufReader, BufWriter, Read, Write}, 10 10 + }; 11 11 + use xz2::read::XzDecoder; 12 12 + use xz2::stream::{LzmaOptions, Stream}; 13 13 + use xz2::write::XzEncoder; 14 14 + 15 15 + /// Memory limit passed to the LZMA decoder. `u64::MAX` disables the limit, 16 16 + /// which matches the behavior of `xz --lzma1 -d` / `unlzma`. 17 17 + const LZMA_DECODER_MEMLIMIT: u64 = u64::MAX; 18 18 + 19 19 + /// Swallows `flush()` calls on the wrapped writer. The legacy LZMA1 20 20 + /// (`lzma_alone`) encoder in liblzma rejects `LZMA_FULL_FLUSH`, which is what 21 21 + /// the inner `XzEncoder::flush` issues, so progress/copy helpers that call 22 22 + /// `flush` mid-stream must see a no-op flush and let `try_finish` (via Drop 23 23 + /// or `finish()`) finalize the stream instead. 24 24 + struct NoFlush<W>(W); 25 25 + 26 26 + impl<W: Write> Write for NoFlush<W> { 27 27 + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { 28 28 + self.0.write(buf) 29 29 + } 30 30 + 31 31 + fn flush(&mut self) -> io::Result<()> { 32 32 + Ok(()) 33 33 + } 34 34 + } 35 35 + 36 36 + #[derive(Args, Debug)] 37 37 + pub struct LzmaArgs { 38 38 + #[clap(flatten)] 39 39 + pub common_args: CommonArgs, 40 40 + 41 41 + #[clap(flatten)] 42 42 + progress_args: ProgressArgs, 43 43 + 44 44 + #[clap(flatten)] 45 45 + pub level_args: LevelArgs, 46 46 + } 47 47 + 48 48 + pub struct Lzma { 49 49 + pub level: i32, 50 50 + pub progress_args: ProgressArgs, 51 51 + } 52 52 + 53 53 + impl Default for Lzma { 54 54 + fn default() -> Self { 55 55 + let validator = DefaultCompressionValidator; 56 56 + Lzma { 57 57 + level: validator.default_level(), 58 58 + progress_args: ProgressArgs::default(), 59 59 + } 60 60 + } 61 61 + } 62 62 + 63 63 + impl Lzma { 64 64 + pub fn new(args: &LzmaArgs) -> Lzma { 65 65 + let validator = DefaultCompressionValidator; 66 66 + let level = validator.validate_and_clamp_level(args.level_args.level.level); 67 67 + 68 68 + Lzma { 69 69 + level, 70 70 + progress_args: args.progress_args, 71 71 + } 72 72 + } 73 73 + 74 74 + /// Build a fresh LZMA1 (`lzma_alone`) encoder stream at the configured level. 75 75 + fn encoder_stream(&self) -> Result<Stream> { 76 76 + let options = LzmaOptions::new_preset(self.level as u32)?; 77 77 + Ok(Stream::new_lzma_encoder(&options)?) 78 78 + } 79 79 + 80 80 + /// Build a fresh LZMA1 (`lzma_alone`) decoder stream. 81 81 + fn decoder_stream() -> Result<Stream> { 82 82 + Ok(Stream::new_lzma_decoder(LZMA_DECODER_MEMLIMIT)?) 83 83 + } 84 84 + } 85 85 + 86 86 + impl Compressor for Lzma { 87 87 + /// The standard extension for legacy LZMA (`.lzma`) files. 88 88 + fn extension(&self) -> &str { 89 89 + "lzma" 90 90 + } 91 91 + 92 92 + /// Full name for lzma. 93 93 + fn name(&self) -> &str { 94 94 + "lzma" 95 95 + } 96 96 + 97 97 + fn compress(&self, input: CmprssInput, output: CmprssOutput) -> Result { 98 98 + if let CmprssOutput::Path(out_path) = &output 99 99 + && out_path.is_dir() 100 100 + { 101 101 + bail!( 102 102 + "LZMA does not support compressing to a directory. Please specify an output file." 103 103 + ); 104 104 + } 105 105 + if let CmprssInput::Path(input_paths) = &input { 106 106 + for x in input_paths { 107 107 + if x.is_dir() { 108 108 + bail!( 109 109 + "LZMA does not support compressing a directory. Please specify only files." 110 110 + ); 111 111 + } 112 112 + } 113 113 + } 114 114 + 115 115 + let mut file_size = None; 116 116 + let mut input_stream: Box<dyn Read + Send> = match input { 117 117 + CmprssInput::Path(paths) => { 118 118 + if paths.len() > 1 { 119 119 + bail!("Multiple input files not supported for lzma"); 120 120 + } 121 121 + let path = &paths[0]; 122 122 + file_size = Some(std::fs::metadata(path)?.len()); 123 123 + Box::new(BufReader::new(File::open(path)?)) 124 124 + } 125 125 + CmprssInput::Pipe(pipe) => Box::new(pipe) as Box<dyn Read + Send>, 126 126 + CmprssInput::Reader(reader) => reader.0, 127 127 + }; 128 128 + 129 129 + if let CmprssOutput::Writer(writer) = output { 130 130 + let mut encoder = XzEncoder::new_stream(writer, self.encoder_stream()?); 131 131 + io::copy(&mut input_stream, &mut encoder)?; 132 132 + encoder.try_finish()?; 133 133 + } else { 134 134 + let output_stream: Box<dyn Write + Send> = match &output { 135 135 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 136 136 + CmprssOutput::Pipe(pipe) => Box::new(pipe) as Box<dyn Write + Send>, 137 137 + CmprssOutput::Writer(_) => unreachable!(), 138 138 + }; 139 139 + let mut encoder = XzEncoder::new_stream(output_stream, self.encoder_stream()?); 140 140 + copy_with_progress( 141 141 + &mut input_stream, 142 142 + NoFlush(&mut encoder), 143 143 + self.progress_args.chunk_size.size_in_bytes, 144 144 + file_size, 145 145 + self.progress_args.progress, 146 146 + &output, 147 147 + )?; 148 148 + encoder.try_finish()?; 149 149 + } 150 150 + 151 151 + Ok(()) 152 152 + } 153 153 + 154 154 + fn extract(&self, input: CmprssInput, output: CmprssOutput) -> Result { 155 155 + let mut file_size = None; 156 156 + let input_stream: Box<dyn Read + Send> = match input { 157 157 + CmprssInput::Path(paths) => { 158 158 + if paths.len() > 1 { 159 159 + bail!("Multiple input files not supported for lzma extraction"); 160 160 + } 161 161 + let path = &paths[0]; 162 162 + file_size = Some(std::fs::metadata(path)?.len()); 163 163 + Box::new(BufReader::new(File::open(path)?)) as Box<dyn Read + Send> 164 164 + } 165 165 + CmprssInput::Pipe(pipe) => Box::new(pipe) as Box<dyn Read + Send>, 166 166 + CmprssInput::Reader(reader) => reader.0, 167 167 + }; 168 168 + let mut decoder = XzDecoder::new_stream(input_stream, Self::decoder_stream()?); 169 169 + 170 170 + if let CmprssOutput::Writer(mut writer) = output { 171 171 + io::copy(&mut decoder, &mut writer)?; 172 172 + } else { 173 173 + let mut output_stream: Box<dyn Write + Send> = match &output { 174 174 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 175 175 + CmprssOutput::Pipe(pipe) => Box::new(pipe) as Box<dyn Write + Send>, 176 176 + CmprssOutput::Writer(_) => unreachable!(), 177 177 + }; 178 178 + copy_with_progress( 179 179 + &mut decoder, 180 180 + &mut *output_stream, 181 181 + self.progress_args.chunk_size.size_in_bytes, 182 182 + file_size, 183 183 + self.progress_args.progress, 184 184 + &output, 185 185 + )?; 186 186 + } 187 187 + 188 188 + Ok(()) 189 189 + } 190 190 + } 191 191 + 192 192 + #[cfg(test)] 193 193 + mod tests { 194 194 + use super::*; 195 195 + use crate::test_utils::*; 196 196 + 197 197 + /// Test the basic interface of the Lzma compressor 198 198 + #[test] 199 199 + fn test_lzma_interface() { 200 200 + let compressor = Lzma::default(); 201 201 + test_compressor_interface(&compressor, "lzma", Some("lzma")); 202 202 + } 203 203 + 204 204 + /// Test the default compression level 205 205 + #[test] 206 206 + fn test_lzma_default_compression() -> Result { 207 207 + let compressor = Lzma::default(); 208 208 + test_compression(&compressor) 209 209 + } 210 210 + 211 211 + /// Test fast compression level 212 212 + #[test] 213 213 + fn test_lzma_fast_compression() -> Result { 214 214 + let fast_compressor = Lzma { 215 215 + level: 1, 216 216 + progress_args: ProgressArgs::default(), 217 217 + }; 218 218 + test_compression(&fast_compressor) 219 219 + } 220 220 + 221 221 + /// Test best compression level 222 222 + #[test] 223 223 + fn test_lzma_best_compression() -> Result { 224 224 + let best_compressor = Lzma { 225 225 + level: 9, 226 226 + progress_args: ProgressArgs::default(), 227 227 + }; 228 228 + test_compression(&best_compressor) 229 229 + } 230 230 + }

+3

src/backends/mod.rs

reviewed

··· 2 2 mod bzip2; 3 3 mod gzip; 4 4 mod lz4; 5 5 + mod lzma; 5 6 mod pipeline; 6 7 mod snappy; 7 8 mod tar; ··· 13 14 pub use bzip2::{Bzip2, Bzip2Args}; 14 15 pub use gzip::{Gzip, GzipArgs}; 15 16 pub use lz4::{Lz4, Lz4Args}; 17 17 + pub use lzma::{Lzma, LzmaArgs}; 16 18 pub use pipeline::Pipeline; 17 19 pub use snappy::{Snappy, SnappyArgs}; 18 20 pub use tar::{Tar, TarArgs}; ··· 35 37 "lz4" => Some(Box::<Lz4>::default()), 36 38 "brotli" | "br" => Some(Box::<Brotli>::default()), 37 39 "snappy" | "sz" => Some(Box::<Snappy>::default()), 40 40 + "lzma" => Some(Box::<Lzma>::default()), 38 41 _ => None, 39 42 } 40 43 }

+5

src/main.rs

reviewed

··· 55 55 /// snappy framed compression 56 56 #[clap(visible_alias = "sz")] 57 57 Snappy(SnappyArgs), 58 58 + 59 59 + /// lzma (legacy LZMA1) compression 60 60 + Lzma(LzmaArgs), 58 61 } 59 62 60 63 /// Get the input filename or return a default file ··· 518 521 Some(Format::Lz4(a)) => command(Some(Box::new(Lz4::new(&a))), &a.common_args), 519 522 Some(Format::Brotli(a)) => command(Some(Box::new(Brotli::new(&a))), &a.common_args), 520 523 Some(Format::Snappy(a)) => command(Some(Box::new(Snappy::new(&a))), &a.common_args), 524 524 + Some(Format::Lzma(a)) => command(Some(Box::new(Lzma::new(&a))), &a.common_args), 521 525 _ => command(None, &args.base_args), 522 526 } 523 527 .unwrap_or_else(|e| { ··· 548 552 assert_eq!(compressor_name("file.lz4"), Some("lz4".into())); 549 553 assert_eq!(compressor_name("file.br"), Some("brotli".into())); 550 554 assert_eq!(compressor_name("file.sz"), Some("snappy".into())); 555 555 + assert_eq!(compressor_name("file.lzma"), Some("lzma".into())); 551 556 assert_eq!(compressor_name("file.tar"), Some("tar".into())); 552 557 assert_eq!(compressor_name("file.zip"), Some("zip".into())); 553 558 }

+181

tests/lzma.rs

reviewed

··· 1 1 + use assert_cmd::prelude::*; 2 2 + use assert_fs::prelude::*; 3 3 + use predicates::prelude::*; 4 4 + use std::{ 5 5 + fs::File, 6 6 + process::{Command, Stdio}, 7 7 + }; 8 8 + 9 9 + mod common; 10 10 + use common::*; 11 11 + 12 12 + mod lzma { 13 13 + use super::*; 14 14 + 15 15 + mod roundtrip { 16 16 + use super::*; 17 17 + 18 18 + /// Lzma roundtrip using explicit filenames 19 19 + /// Compressing: input = test.txt, output = test.txt.lzma 20 20 + /// Extracting: input = test.txt.lzma, output = test.txt 21 21 + /// 22 22 + /// ``` bash 23 23 + /// cmprss lzma test.txt test.txt.lzma 24 24 + /// cmprss lzma --extract --ignore-pipes test.txt.lzma 25 25 + /// ``` 26 26 + #[test] 27 27 + fn explicit() -> Result<(), Box<dyn std::error::Error>> { 28 28 + let file = create_test_file("test.txt", "garbage data for testing")?; 29 29 + let working_dir = create_working_dir()?; 30 30 + let archive = working_dir.child("test.txt.lzma"); 31 31 + archive.assert(predicate::path::missing()); 32 32 + 33 33 + let mut compress = Command::cargo_bin("cmprss")?; 34 34 + compress 35 35 + .current_dir(&working_dir) 36 36 + .arg("lzma") 37 37 + .arg(file.path()) 38 38 + .arg(archive.path()); 39 39 + compress.assert().success(); 40 40 + archive.assert(predicate::path::is_file()); 41 41 + 42 42 + let mut extract = Command::cargo_bin("cmprss")?; 43 43 + extract 44 44 + .current_dir(&working_dir) 45 45 + .arg("lzma") 46 46 + .arg("--ignore-pipes") 47 47 + .arg("--extract") 48 48 + .arg(archive.path()); 49 49 + extract.assert().success(); 50 50 + 51 51 + // Assert the files are identical 52 52 + assert_files_equal(file.path(), &working_dir.child("test.txt")); 53 53 + 54 54 + Ok(()) 55 55 + } 56 56 + 57 57 + /// Lzma roundtrip using stdin 58 58 + /// Compressing: input = stdin, output = test.txt.lzma 59 59 + /// Extracting: input = stdin(test.txt.lzma), output = out.txt 60 60 + /// 61 61 + /// ``` bash 62 62 + /// cat test.txt | cmprss lzma test.txt.lzma 63 63 + /// cat test.txt.lzma | cmprss lzma --extract out.txt 64 64 + /// ``` 65 65 + #[test] 66 66 + fn stdin() -> Result<(), Box<dyn std::error::Error>> { 67 67 + let file = create_test_file("test.txt", "garbage data for testing")?; 68 68 + let working_dir = create_working_dir()?; 69 69 + let archive = working_dir.child("test.txt.lzma"); 70 70 + archive.assert(predicate::path::missing()); 71 71 + 72 72 + let mut compress = Command::cargo_bin("cmprss")?; 73 73 + compress 74 74 + .current_dir(&working_dir) 75 75 + .arg("lzma") 76 76 + .arg("test.txt.lzma") 77 77 + .stdin(Stdio::from(File::open(file.path())?)); 78 78 + compress.assert().success(); 79 79 + archive.assert(predicate::path::is_file()); 80 80 + 81 81 + let mut extract = Command::cargo_bin("cmprss")?; 82 82 + extract 83 83 + .current_dir(&working_dir) 84 84 + .arg("lzma") 85 85 + .stdin(Stdio::from(File::open(archive.path())?)) 86 86 + .arg("--extract") 87 87 + .arg("out.txt"); 88 88 + extract.assert().success(); 89 89 + 90 90 + // Assert the files are identical 91 91 + assert_files_equal(file.path(), &working_dir.child("out.txt")); 92 92 + 93 93 + Ok(()) 94 94 + } 95 95 + 96 96 + /// Lzma roundtrip using stdout 97 97 + /// Compressing: input = test.txt, output = stdout 98 98 + /// Extracting: input = test.txt.lzma, output = stdout 99 99 + /// 100 100 + /// ``` bash 101 101 + /// cmprss lzma test.txt > test.txt.lzma 102 102 + /// cmprss lzma --extract test.txt.lzma > out.txt 103 103 + /// ``` 104 104 + #[test] 105 105 + fn stdout() -> Result<(), Box<dyn std::error::Error>> { 106 106 + let file = create_test_file("test.txt", "garbage data for testing")?; 107 107 + let working_dir = create_working_dir()?; 108 108 + let archive = working_dir.child("test.txt.lzma"); 109 109 + archive.assert(predicate::path::missing()); 110 110 + 111 111 + let mut compress = Command::cargo_bin("cmprss")?; 112 112 + compress 113 113 + .current_dir(&working_dir) 114 114 + .arg("lzma") 115 115 + .arg(file.path()) 116 116 + .stdout(Stdio::from(File::create(archive.path())?)); 117 117 + compress.assert().success(); 118 118 + archive.assert(predicate::path::is_file()); 119 119 + 120 120 + let output = working_dir.child("out.txt"); 121 121 + output.assert(predicate::path::missing()); 122 122 + 123 123 + let mut extract = Command::cargo_bin("cmprss")?; 124 124 + extract 125 125 + .current_dir(&working_dir) 126 126 + .arg("lzma") 127 127 + .arg("--extract") 128 128 + .arg(archive.path()) 129 129 + .stdout(Stdio::from(File::create(output.path())?)); 130 130 + extract.assert().success(); 131 131 + output.assert(predicate::path::is_file()); 132 132 + 133 133 + assert_files_equal(file.path(), output.path()); 134 134 + 135 135 + Ok(()) 136 136 + } 137 137 + 138 138 + /// Lzma roundtrip with compression level 139 139 + /// Compressing: input = test.txt, output = test.txt.lzma, level = 9 140 140 + /// Extracting: input = test.txt.lzma, output = test.txt 141 141 + /// 142 142 + /// ``` bash 143 143 + /// cmprss lzma --level 9 test.txt test.txt.lzma 144 144 + /// cmprss lzma --extract test.txt.lzma test.txt 145 145 + /// ``` 146 146 + #[test] 147 147 + fn with_level() -> Result<(), Box<dyn std::error::Error>> { 148 148 + let file = create_test_file("test.txt", "garbage data for testing")?; 149 149 + let working_dir = create_working_dir()?; 150 150 + let archive = working_dir.child("test.txt.lzma"); 151 151 + archive.assert(predicate::path::missing()); 152 152 + 153 153 + let mut compress = Command::cargo_bin("cmprss")?; 154 154 + compress 155 155 + .current_dir(&working_dir) 156 156 + .arg("lzma") 157 157 + .arg("--level") 158 158 + .arg("9") 159 159 + .arg(file.path()) 160 160 + .arg(archive.path()); 161 161 + compress.assert().success(); 162 162 + archive.assert(predicate::path::is_file()); 163 163 + 164 164 + let output = working_dir.child("test.txt"); 165 165 + 166 166 + let mut extract = Command::cargo_bin("cmprss")?; 167 167 + extract 168 168 + .current_dir(&working_dir) 169 169 + .arg("lzma") 170 170 + .arg("--extract") 171 171 + .arg(archive.path()) 172 172 + .arg(output.path()); 173 173 + extract.assert().success(); 174 174 + 175 175 + // Assert the files are identical 176 176 + assert_files_equal(file.path(), output.path()); 177 177 + 178 178 + Ok(()) 179 179 + } 180 180 + } 181 181 + }