feat(brotli): add brotli compression support · patrick.jackson.dev/cmprss@d7eac96

+44

Cargo.lock

··· 29 29 ] 30 30 31 31 [[package]] 32 + name = "alloc-no-stdlib" 33 + version = "2.0.4" 34 + source = "registry+https://github.com/rust-lang/crates.io-index" 35 + checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" 36 + 37 + [[package]] 38 + name = "alloc-stdlib" 39 + version = "0.2.2" 40 + source = "registry+https://github.com/rust-lang/crates.io-index" 41 + checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" 42 + dependencies = [ 43 + "alloc-no-stdlib", 44 + ] 45 + 46 + [[package]] 32 47 name = "anstream" 33 48 version = "1.0.0" 34 49 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 136 151 ] 137 152 138 153 [[package]] 154 + name = "brotli" 155 + version = "8.0.2" 156 + source = "registry+https://github.com/rust-lang/crates.io-index" 157 + checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" 158 + dependencies = [ 159 + "alloc-no-stdlib", 160 + "alloc-stdlib", 161 + "brotli-decompressor", 162 + ] 163 + 164 + [[package]] 165 + name = "brotli-decompressor" 166 + version = "5.0.0" 167 + source = "registry+https://github.com/rust-lang/crates.io-index" 168 + checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" 169 + dependencies = [ 170 + "alloc-no-stdlib", 171 + "alloc-stdlib", 172 + ] 173 + 174 + [[package]] 139 175 name = "bstr" 140 176 version = "1.12.1" 141 177 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 258 294 "anyhow", 259 295 "assert_cmd", 260 296 "assert_fs", 297 + "brotli", 261 298 "bzip2", 262 299 "clap", 263 300 "flate2", ··· 266 303 "lz4_flex", 267 304 "predicates", 268 305 "rand", 306 + "snap", 269 307 "tar", 270 308 "tempfile", 271 309 "xz2", ··· 1059 1097 version = "0.3.9" 1060 1098 source = "registry+https://github.com/rust-lang/crates.io-index" 1061 1099 checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" 1100 + 1101 + [[package]] 1102 + name = "snap" 1103 + version = "1.1.1" 1104 + source = "registry+https://github.com/rust-lang/crates.io-index" 1105 + checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" 1062 1106 1063 1107 [[package]] 1064 1108 name = "strsim"

+2

Cargo.toml

··· 23 23 tempfile = "3" 24 24 zstd = "0.13" 25 25 lz4_flex = "0.13" 26 + brotli = "8" 27 + snap = "1" 26 28 27 29 [dev-dependencies] 28 30 assert_cmd = "2"

+253

src/backends/brotli.rs

··· 1 + use crate::progress::{ProgressArgs, copy_with_progress}; 2 + use crate::utils::*; 3 + use anyhow::bail; 4 + use brotli::{CompressorWriter, Decompressor}; 5 + use clap::Args; 6 + use std::fs::File; 7 + use std::io::{self, BufReader, BufWriter, Read, Write}; 8 + 9 + /// Brotli buffer size used when constructing the encoder/decoder. 10 + const BROTLI_BUFFER_SIZE: usize = 4096; 11 + 12 + /// Window size (log2) used by the Brotli encoder. 22 is the value used by the 13 + /// reference implementation at quality >= 2 and fits data with no upper bound. 14 + const BROTLI_LGWIN: u32 = 22; 15 + 16 + /// Brotli-specific compression validator. Quality range is 0-11 per RFC 7932, 17 + /// where 0 is fastest and 11 is maximum compression. 18 + #[derive(Debug, Clone, Copy)] 19 + pub struct BrotliCompressionValidator; 20 + 21 + impl CompressionLevelValidator for BrotliCompressionValidator { 22 + fn min_level(&self) -> i32 { 23 + 0 24 + } 25 + fn max_level(&self) -> i32 { 26 + 11 27 + } 28 + fn default_level(&self) -> i32 { 29 + 6 30 + } 31 + 32 + fn name_to_level(&self, name: &str) -> Option<i32> { 33 + match name.to_lowercase().as_str() { 34 + "none" => Some(0), 35 + "fast" => Some(1), 36 + "best" => Some(11), 37 + _ => None, 38 + } 39 + } 40 + } 41 + 42 + #[derive(Args, Debug)] 43 + pub struct BrotliArgs { 44 + #[clap(flatten)] 45 + pub common_args: CommonArgs, 46 + 47 + #[clap(flatten)] 48 + pub level_args: LevelArgs, 49 + 50 + #[clap(flatten)] 51 + pub progress_args: ProgressArgs, 52 + } 53 + 54 + pub struct Brotli { 55 + pub compression_level: i32, 56 + pub progress_args: ProgressArgs, 57 + } 58 + 59 + impl Default for Brotli { 60 + fn default() -> Self { 61 + let validator = BrotliCompressionValidator; 62 + Brotli { 63 + compression_level: validator.default_level(), 64 + progress_args: ProgressArgs::default(), 65 + } 66 + } 67 + } 68 + 69 + impl Brotli { 70 + pub fn new(args: &BrotliArgs) -> Brotli { 71 + let validator = BrotliCompressionValidator; 72 + let level = validator.validate_and_clamp_level(args.level_args.level.level); 73 + 74 + Brotli { 75 + compression_level: level, 76 + progress_args: args.progress_args, 77 + } 78 + } 79 + } 80 + 81 + impl Compressor for Brotli { 82 + /// The standard extension for brotli-compressed files. 83 + fn extension(&self) -> &str { 84 + "br" 85 + } 86 + 87 + /// Full name for brotli. 88 + fn name(&self) -> &str { 89 + "brotli" 90 + } 91 + 92 + /// Compress an input file or pipe to a brotli archive 93 + fn compress(&self, input: CmprssInput, output: CmprssOutput) -> Result { 94 + if let CmprssOutput::Path(out_path) = &output 95 + && out_path.is_dir() 96 + { 97 + bail!( 98 + "Brotli does not support compressing to a directory. Please specify an output file." 99 + ); 100 + } 101 + if let CmprssInput::Path(input_paths) = &input { 102 + for x in input_paths { 103 + if x.is_dir() { 104 + bail!( 105 + "Brotli does not support compressing a directory. Please specify only files." 106 + ); 107 + } 108 + } 109 + } 110 + let mut file_size = None; 111 + let mut input_stream: Box<dyn Read + Send> = match input { 112 + CmprssInput::Path(paths) => { 113 + if paths.len() > 1 { 114 + bail!("Multiple input files not supported for brotli"); 115 + } 116 + let path = &paths[0]; 117 + file_size = Some(std::fs::metadata(path)?.len()); 118 + Box::new(BufReader::new(File::open(path)?)) 119 + } 120 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 121 + CmprssInput::Reader(reader) => reader.0, 122 + }; 123 + 124 + let quality = self.compression_level as u32; 125 + 126 + if let CmprssOutput::Writer(writer) = output { 127 + let mut encoder = 128 + CompressorWriter::new(writer, BROTLI_BUFFER_SIZE, quality, BROTLI_LGWIN); 129 + io::copy(&mut input_stream, &mut encoder)?; 130 + encoder.flush()?; 131 + } else { 132 + let output_stream: Box<dyn Write + Send> = match &output { 133 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 134 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 135 + CmprssOutput::Writer(_) => unreachable!(), 136 + }; 137 + let mut encoder = 138 + CompressorWriter::new(output_stream, BROTLI_BUFFER_SIZE, quality, BROTLI_LGWIN); 139 + copy_with_progress( 140 + &mut input_stream, 141 + &mut encoder, 142 + self.progress_args.chunk_size.size_in_bytes, 143 + file_size, 144 + self.progress_args.progress, 145 + &output, 146 + )?; 147 + encoder.flush()?; 148 + } 149 + 150 + Ok(()) 151 + } 152 + 153 + /// Extract a brotli archive to an output file or pipe 154 + fn extract(&self, input: CmprssInput, output: CmprssOutput) -> Result { 155 + if let CmprssOutput::Path(out_path) = &output 156 + && out_path.is_dir() 157 + { 158 + bail!( 159 + "Brotli does not support extracting to a directory. Please specify an output file." 160 + ); 161 + } 162 + 163 + let mut file_size = None; 164 + let input_stream: Box<dyn Read + Send> = match input { 165 + CmprssInput::Path(paths) => { 166 + if paths.len() > 1 { 167 + bail!("Multiple input files not supported for brotli extraction"); 168 + } 169 + let path = &paths[0]; 170 + file_size = Some(std::fs::metadata(path)?.len()); 171 + Box::new(BufReader::new(File::open(path)?)) 172 + } 173 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 174 + CmprssInput::Reader(reader) => reader.0, 175 + }; 176 + 177 + let mut decoder = Decompressor::new(input_stream, BROTLI_BUFFER_SIZE); 178 + 179 + if let CmprssOutput::Writer(mut writer) = output { 180 + io::copy(&mut decoder, &mut writer)?; 181 + } else { 182 + let mut output_stream: Box<dyn Write + Send> = match &output { 183 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 184 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 185 + CmprssOutput::Writer(_) => unreachable!(), 186 + }; 187 + copy_with_progress( 188 + &mut decoder, 189 + &mut output_stream, 190 + self.progress_args.chunk_size.size_in_bytes, 191 + file_size, 192 + self.progress_args.progress, 193 + &output, 194 + )?; 195 + } 196 + 197 + Ok(()) 198 + } 199 + } 200 + 201 + #[cfg(test)] 202 + mod tests { 203 + use super::*; 204 + use crate::test_utils::*; 205 + 206 + /// Test the basic interface of the Brotli compressor 207 + #[test] 208 + fn test_brotli_interface() { 209 + let compressor = Brotli::default(); 210 + test_compressor_interface(&compressor, "brotli", Some("br")); 211 + } 212 + 213 + /// Test the default compression level 214 + #[test] 215 + fn test_brotli_default_compression() -> Result { 216 + let compressor = Brotli::default(); 217 + test_compression(&compressor) 218 + } 219 + 220 + /// Test fast compression level 221 + #[test] 222 + fn test_brotli_fast_compression() -> Result { 223 + let fast_compressor = Brotli { 224 + compression_level: 1, 225 + progress_args: ProgressArgs::default(), 226 + }; 227 + test_compression(&fast_compressor) 228 + } 229 + 230 + /// Test best compression level 231 + #[test] 232 + fn test_brotli_best_compression() -> Result { 233 + let best_compressor = Brotli { 234 + compression_level: 11, 235 + progress_args: ProgressArgs::default(), 236 + }; 237 + test_compression(&best_compressor) 238 + } 239 + 240 + #[test] 241 + fn test_brotli_compression_validator() { 242 + let validator = BrotliCompressionValidator; 243 + test_compression_validator_helper( 244 + &validator, 245 + 0, // min_level 246 + 11, // max_level 247 + 6, // default_level 248 + Some(1), // fast_name_level 249 + Some(11), // best_name_level 250 + Some(0), // none_name_level 251 + ); 252 + } 253 + }

+3

src/backends/mod.rs

··· 1 + mod brotli; 1 2 mod bzip2; 2 3 mod gzip; 3 4 mod lz4; ··· 7 8 mod zip; 8 9 mod zstd; 9 10 11 + pub use brotli::{Brotli, BrotliArgs}; 10 12 pub use bzip2::{Bzip2, Bzip2Args}; 11 13 pub use gzip::{Gzip, GzipArgs}; 12 14 pub use lz4::{Lz4, Lz4Args}; ··· 29 31 "zip" => Some(Box::<Zip>::default()), 30 32 "zstd" | "zst" => Some(Box::<Zstd>::default()), 31 33 "lz4" => Some(Box::<Lz4>::default()), 34 + "brotli" | "br" => Some(Box::<Brotli>::default()), 32 35 _ => None, 33 36 } 34 37 }

+6

src/main.rs

··· 47 47 48 48 /// lz4 compression 49 49 Lz4(Lz4Args), 50 + 51 + /// brotli compression 52 + #[clap(visible_alias = "br")] 53 + Brotli(BrotliArgs), 50 54 } 51 55 52 56 /// Get the input filename or return a default file ··· 508 512 Some(Format::Zip(a)) => command(Some(Box::new(Zip::new(&a))), &a.common_args), 509 513 Some(Format::Zstd(a)) => command(Some(Box::new(Zstd::new(&a))), &a.common_args), 510 514 Some(Format::Lz4(a)) => command(Some(Box::new(Lz4::new(&a))), &a.common_args), 515 + Some(Format::Brotli(a)) => command(Some(Box::new(Brotli::new(&a))), &a.common_args), 511 516 _ => command(None, &args.base_args), 512 517 } 513 518 .unwrap_or_else(|e| { ··· 536 541 assert_eq!(compressor_name("file.bz2"), Some("bzip2".into())); 537 542 assert_eq!(compressor_name("file.zst"), Some("zstd".into())); 538 543 assert_eq!(compressor_name("file.lz4"), Some("lz4".into())); 544 + assert_eq!(compressor_name("file.br"), Some("brotli".into())); 539 545 assert_eq!(compressor_name("file.tar"), Some("tar".into())); 540 546 assert_eq!(compressor_name("file.zip"), Some("zip".into())); 541 547 }

+222

tests/brotli.rs

··· 1 + use assert_cmd::prelude::*; 2 + use assert_fs::prelude::*; 3 + use predicates::prelude::*; 4 + use std::{ 5 + fs::File, 6 + process::{Command, Stdio}, 7 + }; 8 + 9 + mod common; 10 + use common::*; 11 + 12 + mod brotli { 13 + use super::*; 14 + 15 + mod roundtrip { 16 + use super::*; 17 + 18 + /// Brotli roundtrip using explicit filenames 19 + /// Compressing: input = test.txt, output = test.txt.br 20 + /// Extracting: input = test.txt.br, output = test.txt 21 + /// 22 + /// ``` bash 23 + /// cmprss brotli test.txt test.txt.br 24 + /// cmprss brotli --extract --ignore-pipes test.txt.br 25 + /// ``` 26 + #[test] 27 + fn explicit() -> Result<(), Box<dyn std::error::Error>> { 28 + let file = create_test_file("test.txt", "garbage data for testing")?; 29 + let working_dir = create_working_dir()?; 30 + let archive = working_dir.child("test.txt.br"); 31 + archive.assert(predicate::path::missing()); 32 + 33 + let mut compress = Command::cargo_bin("cmprss")?; 34 + compress 35 + .current_dir(&working_dir) 36 + .arg("brotli") 37 + .arg(file.path()) 38 + .arg(archive.path()); 39 + compress.assert().success(); 40 + archive.assert(predicate::path::is_file()); 41 + 42 + let mut extract = Command::cargo_bin("cmprss")?; 43 + extract 44 + .current_dir(&working_dir) 45 + .arg("brotli") 46 + .arg("--ignore-pipes") 47 + .arg("--extract") 48 + .arg(archive.path()); 49 + extract.assert().success(); 50 + 51 + // Assert the files are identical 52 + assert_files_equal(file.path(), &working_dir.child("test.txt")); 53 + 54 + Ok(()) 55 + } 56 + 57 + /// Brotli roundtrip using the `br` alias 58 + /// Compressing: input = test.txt, output = test.txt.br 59 + /// Extracting: input = test.txt.br, output = out.txt 60 + /// 61 + /// ``` bash 62 + /// cmprss br test.txt test.txt.br 63 + /// cmprss br --extract test.txt.br out.txt 64 + /// ``` 65 + #[test] 66 + fn alias() -> Result<(), Box<dyn std::error::Error>> { 67 + let file = create_test_file("test.txt", "garbage data for the alias test")?; 68 + let working_dir = create_working_dir()?; 69 + let archive = working_dir.child("test.txt.br"); 70 + 71 + let mut compress = Command::cargo_bin("cmprss")?; 72 + compress 73 + .current_dir(&working_dir) 74 + .arg("br") 75 + .arg(file.path()) 76 + .arg(archive.path()); 77 + compress.assert().success(); 78 + archive.assert(predicate::path::is_file()); 79 + 80 + let output = working_dir.child("out.txt"); 81 + let mut extract = Command::cargo_bin("cmprss")?; 82 + extract 83 + .current_dir(&working_dir) 84 + .arg("br") 85 + .arg("--extract") 86 + .arg(archive.path()) 87 + .arg(output.path()); 88 + extract.assert().success(); 89 + 90 + assert_files_equal(file.path(), output.path()); 91 + 92 + Ok(()) 93 + } 94 + 95 + /// Brotli roundtrip using stdin 96 + /// Compressing: input = stdin, output = test.txt.br 97 + /// Extracting: input = stdin(test.txt.br), output = out.txt 98 + /// 99 + /// ``` bash 100 + /// cat test.txt | cmprss brotli test.txt.br 101 + /// cat test.txt.br | cmprss brotli --extract out.txt 102 + /// ``` 103 + #[test] 104 + fn stdin() -> Result<(), Box<dyn std::error::Error>> { 105 + let file = create_test_file("test.txt", "garbage data for testing")?; 106 + let working_dir = create_working_dir()?; 107 + let archive = working_dir.child("test.txt.br"); 108 + archive.assert(predicate::path::missing()); 109 + 110 + // Pipe file to stdin 111 + let mut compress = Command::cargo_bin("cmprss")?; 112 + compress 113 + .current_dir(&working_dir) 114 + .arg("brotli") 115 + .arg("test.txt.br") 116 + .stdin(Stdio::from(File::open(file.path())?)); 117 + compress.assert().success(); 118 + archive.assert(predicate::path::is_file()); 119 + 120 + let mut extract = Command::cargo_bin("cmprss")?; 121 + extract 122 + .current_dir(&working_dir) 123 + .arg("brotli") 124 + .stdin(Stdio::from(File::open(archive.path())?)) 125 + .arg("--extract") 126 + .arg("out.txt"); 127 + extract.assert().success(); 128 + 129 + // Assert the files are identical 130 + assert_files_equal(file.path(), &working_dir.child("out.txt")); 131 + 132 + Ok(()) 133 + } 134 + 135 + /// Brotli roundtrip using stdout 136 + /// Compressing: input = test.txt, output = stdout 137 + /// Extracting: input = test.txt.br, output = stdout 138 + /// 139 + /// ``` bash 140 + /// cmprss brotli test.txt > test.txt.br 141 + /// cmprss brotli --extract test.txt.br > out.txt 142 + /// ``` 143 + #[test] 144 + fn stdout() -> Result<(), Box<dyn std::error::Error>> { 145 + let file = create_test_file("test.txt", "garbage data for testing")?; 146 + let working_dir = create_working_dir()?; 147 + let archive = working_dir.child("test.txt.br"); 148 + archive.assert(predicate::path::missing()); 149 + 150 + // Compress file to stdout 151 + let mut compress = Command::cargo_bin("cmprss")?; 152 + compress 153 + .current_dir(&working_dir) 154 + .arg("brotli") 155 + .arg(file.path()) 156 + .stdout(Stdio::from(File::create(archive.path())?)); 157 + compress.assert().success(); 158 + archive.assert(predicate::path::is_file()); 159 + 160 + let output = working_dir.child("out.txt"); 161 + output.assert(predicate::path::missing()); 162 + 163 + let mut extract = Command::cargo_bin("cmprss")?; 164 + extract 165 + .current_dir(&working_dir) 166 + .arg("brotli") 167 + .arg("--extract") 168 + .arg(archive.path()) 169 + .stdout(Stdio::from(File::create(output.path())?)); 170 + extract.assert().success(); 171 + output.assert(predicate::path::is_file()); 172 + 173 + // Assert the files are identical 174 + assert_files_equal(file.path(), output.path()); 175 + 176 + Ok(()) 177 + } 178 + 179 + /// Brotli roundtrip with compression level 180 + /// Compressing: input = test.txt, output = test.txt.br, level = 11 181 + /// Extracting: input = test.txt.br, output = test.txt 182 + /// 183 + /// ``` bash 184 + /// cmprss brotli --level 11 test.txt test.txt.br 185 + /// cmprss brotli --extract test.txt.br test.txt 186 + /// ``` 187 + #[test] 188 + fn with_level() -> Result<(), Box<dyn std::error::Error>> { 189 + let file = create_test_file("test.txt", "garbage data for testing")?; 190 + let working_dir = create_working_dir()?; 191 + let archive = working_dir.child("test.txt.br"); 192 + archive.assert(predicate::path::missing()); 193 + 194 + let mut compress = Command::cargo_bin("cmprss")?; 195 + compress 196 + .current_dir(&working_dir) 197 + .arg("brotli") 198 + .arg("--level") 199 + .arg("11") 200 + .arg(file.path()) 201 + .arg(archive.path()); 202 + compress.assert().success(); 203 + archive.assert(predicate::path::is_file()); 204 + 205 + let output = working_dir.child("test.txt"); 206 + 207 + let mut extract = Command::cargo_bin("cmprss")?; 208 + extract 209 + .current_dir(&working_dir) 210 + .arg("brotli") 211 + .arg("--extract") 212 + .arg(archive.path()) 213 + .arg(output.path()); 214 + extract.assert().success(); 215 + 216 + // Assert the files are identical 217 + assert_files_equal(file.path(), output.path()); 218 + 219 + Ok(()) 220 + } 221 + } 222 + }

Configure Feed

Configure Feed