+21 -2

Cargo.lock

reviewed

··· 262 262 "tempfile", 263 263 "xz2", 264 264 "zip", 265 265 + "zstd 0.13.3", 265 266 ] 266 267 267 268 [[package]] ··· 1121 1122 "pbkdf2", 1122 1123 "sha1", 1123 1124 "time", 1124 1124 - "zstd", 1125 1125 + "zstd 0.11.2+zstd.1.5.2", 1125 1126 ] 1126 1127 1127 1128 [[package]] ··· 1130 1131 source = "registry+https://github.com/rust-lang/crates.io-index" 1131 1132 checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" 1132 1133 dependencies = [ 1133 1133 - "zstd-safe", 1134 1134 + "zstd-safe 5.0.2+zstd.1.5.2", 1135 1135 + ] 1136 1136 + 1137 1137 + [[package]] 1138 1138 + name = "zstd" 1139 1139 + version = "0.13.3" 1140 1140 + source = "registry+https://github.com/rust-lang/crates.io-index" 1141 1141 + checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" 1142 1142 + dependencies = [ 1143 1143 + "zstd-safe 7.2.3", 1134 1144 ] 1135 1145 1136 1146 [[package]] ··· 1140 1150 checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" 1141 1151 dependencies = [ 1142 1152 "libc", 1153 1153 + "zstd-sys", 1154 1154 + ] 1155 1155 + 1156 1156 + [[package]] 1157 1157 + name = "zstd-safe" 1158 1158 + version = "7.2.3" 1159 1159 + source = "registry+https://github.com/rust-lang/crates.io-index" 1160 1160 + checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" 1161 1161 + dependencies = [ 1143 1162 "zstd-sys", 1144 1163 ] 1145 1164

+1

Cargo.toml

reviewed

··· 20 20 xz2 = "0.1" 21 21 zip = "0.6" 22 22 tempfile = "3.10" 23 23 + zstd = "0.13" 23 24 24 25 [dev-dependencies] 25 26 assert_cmd = "2"

+1

README.md

reviewed

··· 21 21 - tar 22 22 - xz 23 23 - zip 24 24 + - zstd 24 25 25 26 ## Install 26 27

+35 -1

bin/test.sh

reviewed

··· 160 160 test_bzip2_level 9 # Default 161 161 } 162 162 163 163 + # Test zstd using the provided compression level 164 164 + test_zstd_level() { 165 165 + tmpdir 166 166 + echo "Testing zstd level $1 in $PWD" 167 167 + echo "Creating random data" 168 168 + random_file 1000000 file 169 169 + echo "Compressing with zstd and cmprss" 170 170 + zstd -$1 -c file >zstd_file.zst 171 171 + cmprss zstd --level $1 file cmprss_file.zst --progress=off 172 172 + # Compare the two archives 173 173 + # The archives may have slight variations (versioning or whatever) so we 174 174 + # only compare the sizes to make sure they are similar 175 175 + compare_size zstd_file.zst cmprss_file.zst 176 176 + # Decompress the 4 variations 177 177 + echo "Decompressing" 178 178 + zstd -d -c zstd_file.zst >zstd_zstd 179 179 + zstd -d -c cmprss_file.zst >cmprss_zstd 180 180 + cmprss zstd --extract cmprss_file.zst cmprss_cmprss --progress=off 181 181 + cmprss zstd --extract zstd_file.zst zstd_cmprss --progress=off 182 182 + echo "Comparing the decompressed files" 183 183 + compare file zstd_zstd 184 184 + compare file zstd_cmprss 185 185 + compare file cmprss_cmprss 186 186 + compare file cmprss_zstd 187 187 + echo "No errors detected" 188 188 + } 189 189 + 190 190 + test_zstd() { 191 191 + test_zstd_level 1 # Fast compression 192 192 + test_zstd_level 3 193 193 + test_zstd_level 6 # Default 194 194 + test_zstd_level 9 # High compression 195 195 + } 196 196 + 163 197 # Run all the tests if no arguments are given 164 198 if [ $# -eq 0 ]; then 165 165 - set -- gzip xz bzip2 199 199 + set -- gzip xz bzip2 zstd 166 200 fi 167 201 168 202 # Run the tests given on the command line

+1

flake.nix

reviewed

··· 183 183 gnutar 184 184 gzip 185 185 xz 186 186 + zstd 186 187 ]; 187 188 188 189 # Many tools read this to find the sources for rust stdlib

+8

src/main.rs

reviewed

··· 5 5 mod utils; 6 6 mod xz; 7 7 mod zip; 8 8 + mod zstd; 8 9 9 10 use bzip2::{Bzip2, Bzip2Args}; 10 11 use clap::{Parser, Subcommand}; ··· 16 17 use utils::*; 17 18 use xz::{Xz, XzArgs}; 18 19 use zip::{Zip, ZipArgs}; 20 20 + use zstd::{Zstd, ZstdArgs}; 19 21 20 22 /// A compression multi-tool 21 23 #[derive(Parser, Debug)] ··· 47 49 48 50 /// zip archive format 49 51 Zip(ZipArgs), 52 52 + 53 53 + /// zstd compression 54 54 + #[clap(visible_alias = "zst")] 55 55 + Zstd(ZstdArgs), 50 56 } 51 57 52 58 /// Get the input filename or return a default file ··· 91 97 Box::<Xz>::default(), 92 98 Box::<Bzip2>::default(), 93 99 Box::<Zip>::default(), 100 100 + Box::<Zstd>::default(), 94 101 ]; 95 102 compressors.into_iter().find(|c| c.is_archive(filename)) 96 103 } ··· 501 508 Some(Format::Xz(a)) => command(Some(Box::new(Xz::new(&a))), &a.common_args), 502 509 Some(Format::Bzip2(a)) => command(Some(Box::new(Bzip2::new(&a))), &a.common_args), 503 510 Some(Format::Zip(a)) => command(Some(Box::new(Zip::new(&a))), &a.common_args), 511 511 + Some(Format::Zstd(a)) => command(Some(Box::new(Zstd::new(&a))), &a.common_args), 504 512 _ => command(None, &args.base_args), 505 513 } 506 514 .unwrap_or_else(|e| {

+200

src/zstd.rs

reviewed

··· 1 1 + use crate::progress::{copy_with_progress, ProgressArgs}; 2 2 + use crate::utils::*; 3 3 + use clap::Args; 4 4 + use std::fs::File; 5 5 + use std::io::{self, BufReader, BufWriter, Read, Write}; 6 6 + use zstd::stream::{read::Decoder, write::Encoder}; 7 7 + 8 8 + #[derive(Args, Debug)] 9 9 + pub struct ZstdArgs { 10 10 + #[clap(flatten)] 11 11 + pub common_args: CommonArgs, 12 12 + 13 13 + #[clap(flatten)] 14 14 + pub level_args: LevelArgs, 15 15 + 16 16 + #[clap(flatten)] 17 17 + pub progress_args: ProgressArgs, 18 18 + } 19 19 + 20 20 + pub struct Zstd { 21 21 + pub compression_level: u32, 22 22 + pub progress_args: ProgressArgs, 23 23 + } 24 24 + 25 25 + impl Default for Zstd { 26 26 + fn default() -> Self { 27 27 + Zstd { 28 28 + compression_level: 6, 29 29 + progress_args: ProgressArgs::default(), 30 30 + } 31 31 + } 32 32 + } 33 33 + 34 34 + impl Zstd { 35 35 + pub fn new(args: &ZstdArgs) -> Zstd { 36 36 + Zstd { 37 37 + compression_level: args.level_args.level.level, 38 38 + progress_args: args.progress_args, 39 39 + } 40 40 + } 41 41 + } 42 42 + 43 43 + impl Compressor for Zstd { 44 44 + /// The standard extension for the zstd format. 45 45 + fn extension(&self) -> &str { 46 46 + "zst" 47 47 + } 48 48 + 49 49 + /// Full name for zstd. 50 50 + fn name(&self) -> &str { 51 51 + "zstd" 52 52 + } 53 53 + 54 54 + /// Generate a default extracted filename 55 55 + /// zstd does not support extracting to a directory, so we return a default filename 56 56 + fn default_extracted_filename(&self, in_path: &std::path::Path) -> String { 57 57 + // If the file has no extension, return a default filename 58 58 + if in_path.extension().is_none() { 59 59 + return "archive".to_string(); 60 60 + } 61 61 + // Otherwise, return the filename without the extension 62 62 + in_path.file_stem().unwrap().to_str().unwrap().to_string() 63 63 + } 64 64 + 65 65 + /// Compress an input file or pipe to a zstd archive 66 66 + fn compress(&self, input: CmprssInput, output: CmprssOutput) -> Result<(), io::Error> { 67 67 + if let CmprssOutput::Path(out_path) = &output { 68 68 + if out_path.is_dir() { 69 69 + return cmprss_error("Zstd does not support compressing to a directory. Please specify an output file."); 70 70 + } 71 71 + } 72 72 + if let CmprssInput::Path(input_paths) = &input { 73 73 + for x in input_paths { 74 74 + if x.is_dir() { 75 75 + return cmprss_error( 76 76 + "Zstd does not support compressing a directory. Please specify only files.", 77 77 + ); 78 78 + } 79 79 + } 80 80 + } 81 81 + let mut file_size = None; 82 82 + let mut input_stream: Box<dyn Read + Send> = match input { 83 83 + CmprssInput::Path(paths) => { 84 84 + if paths.len() > 1 { 85 85 + return Err(io::Error::new( 86 86 + io::ErrorKind::InvalidInput, 87 87 + "Multiple input files not supported for zstd", 88 88 + )); 89 89 + } 90 90 + let path = &paths[0]; 91 91 + file_size = Some(std::fs::metadata(path)?.len()); 92 92 + Box::new(BufReader::new(File::open(path)?)) 93 93 + } 94 94 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 95 95 + }; 96 96 + 97 97 + let output_stream: Box<dyn Write + Send> = match &output { 98 98 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 99 99 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 100 100 + }; 101 101 + 102 102 + // Create a zstd encoder with the specified compression level 103 103 + let mut encoder = Encoder::new(output_stream, self.compression_level as i32)?; 104 104 + 105 105 + // Copy the input to the encoder with progress reporting 106 106 + copy_with_progress( 107 107 + &mut input_stream, 108 108 + &mut encoder, 109 109 + self.progress_args.chunk_size.size_in_bytes, 110 110 + file_size, 111 111 + self.progress_args.progress, 112 112 + &output, 113 113 + )?; 114 114 + 115 115 + // Finish the encoder to ensure all data is written 116 116 + encoder.finish()?; 117 117 + 118 118 + Ok(()) 119 119 + } 120 120 + 121 121 + /// Extract a zstd archive to an output file or pipe 122 122 + fn extract(&self, input: CmprssInput, output: CmprssOutput) -> Result<(), io::Error> { 123 123 + if let CmprssOutput::Path(out_path) = &output { 124 124 + if out_path.is_dir() { 125 125 + return cmprss_error("Zstd does not support extracting to a directory. Please specify an output file."); 126 126 + } 127 127 + } 128 128 + 129 129 + let input_stream: Box<dyn Read + Send> = match input { 130 130 + CmprssInput::Path(paths) => { 131 131 + if paths.len() > 1 { 132 132 + return Err(io::Error::new( 133 133 + io::ErrorKind::InvalidInput, 134 134 + "Multiple input files not supported for zstd", 135 135 + )); 136 136 + } 137 137 + let path = &paths[0]; 138 138 + Box::new(BufReader::new(File::open(path)?)) 139 139 + } 140 140 + CmprssInput::Pipe(stdin) => Box::new(BufReader::new(stdin)), 141 141 + }; 142 142 + 143 143 + // Create a zstd decoder 144 144 + let mut decoder = Decoder::new(input_stream)?; 145 145 + 146 146 + let mut output_stream: Box<dyn Write + Send> = match &output { 147 147 + CmprssOutput::Path(path) => Box::new(BufWriter::new(File::create(path)?)), 148 148 + CmprssOutput::Pipe(stdout) => Box::new(BufWriter::new(stdout)), 149 149 + }; 150 150 + 151 151 + // Copy the decoded data to the output with progress reporting 152 152 + copy_with_progress( 153 153 + &mut decoder, 154 154 + &mut output_stream, 155 155 + self.progress_args.chunk_size.size_in_bytes, 156 156 + None, 157 157 + self.progress_args.progress, 158 158 + &output, 159 159 + )?; 160 160 + 161 161 + Ok(()) 162 162 + } 163 163 + } 164 164 + 165 165 + #[cfg(test)] 166 166 + mod tests { 167 167 + use super::*; 168 168 + use tempfile::tempdir; 169 169 + 170 170 + #[test] 171 171 + fn roundtrip() -> Result<(), Box<dyn std::error::Error>> { 172 172 + let dir = tempdir()?; 173 173 + let input_path = dir.path().join("input.txt"); 174 174 + let compressed_path = dir.path().join("input.txt.zst"); 175 175 + let output_path = dir.path().join("output.txt"); 176 176 + 177 177 + // Create a test file 178 178 + let test_data = b"Hello, world! This is a test file for zstd compression."; 179 179 + std::fs::write(&input_path, test_data)?; 180 180 + 181 181 + // Compress the file 182 182 + let zstd = Zstd::default(); 183 183 + zstd.compress( 184 184 + CmprssInput::Path(vec![input_path.clone()]), 185 185 + CmprssOutput::Path(compressed_path.clone()), 186 186 + )?; 187 187 + 188 188 + // Extract the file 189 189 + zstd.extract( 190 190 + CmprssInput::Path(vec![compressed_path]), 191 191 + CmprssOutput::Path(output_path.clone()), 192 192 + )?; 193 193 + 194 194 + // Verify the contents 195 195 + let output_data = std::fs::read(output_path)?; 196 196 + assert_eq!(test_data.to_vec(), output_data); 197 197 + 198 198 + Ok(()) 199 199 + } 200 200 + }

+178

tests/cli.rs

reviewed

··· 1071 1071 1072 1072 Ok(()) 1073 1073 } 1074 1074 + 1075 1075 + /// Zstd roundtrip using explicit filenames 1076 1076 + /// Compressing: input = test.txt, output = test.txt.zst 1077 1077 + /// Extracting: input = test.txt.zst, output = test.txt 1078 1078 + /// 1079 1079 + /// ``` bash 1080 1080 + /// cmprss zstd test.txt test.txt.zst 1081 1081 + /// cmprss zstd --extract --ignore-pipes test.txt.zst 1082 1082 + /// ``` 1083 1083 + #[test] 1084 1084 + fn zstd_roundtrip_explicit() -> Result<(), Box<dyn std::error::Error>> { 1085 1085 + let file = assert_fs::NamedTempFile::new("test.txt")?; 1086 1086 + file.write_str("garbage data for testing")?; 1087 1087 + 1088 1088 + let working_dir = assert_fs::TempDir::new()?; 1089 1089 + let archive = working_dir.child("test.txt.zst"); 1090 1090 + archive.assert(predicate::path::missing()); 1091 1091 + 1092 1092 + let mut compress = Command::cargo_bin("cmprss")?; 1093 1093 + compress 1094 1094 + .current_dir(&working_dir) 1095 1095 + .arg("zstd") 1096 1096 + .arg(file.path()) 1097 1097 + .arg(archive.path()); 1098 1098 + compress.assert().success(); 1099 1099 + archive.assert(predicate::path::is_file()); 1100 1100 + 1101 1101 + let mut extract = Command::cargo_bin("cmprss")?; 1102 1102 + extract 1103 1103 + .current_dir(&working_dir) 1104 1104 + .arg("zstd") 1105 1105 + .arg("--ignore-pipes") 1106 1106 + .arg("--extract") 1107 1107 + .arg(archive.path()); 1108 1108 + extract.assert().success(); 1109 1109 + 1110 1110 + // Assert the files are identical 1111 1111 + working_dir 1112 1112 + .child("test.txt") 1113 1113 + .assert(predicate::path::eq_file(file.path())); 1114 1114 + 1115 1115 + Ok(()) 1116 1116 + } 1117 1117 + 1118 1118 + /// Zstd roundtrip using stdin 1119 1119 + /// Compressing: input = stdin, output = test.txt.zst 1120 1120 + /// Extracting: input = stdin(test.txt.zst), output = test.txt 1121 1121 + /// 1122 1122 + /// ``` bash 1123 1123 + /// cat test.txt | cmprss zstd test.txt.zst 1124 1124 + /// cat test.txt.zst | cmprss zstd --extract out.txt 1125 1125 + /// ``` 1126 1126 + #[test] 1127 1127 + fn zstd_roundtrip_stdin() -> Result<(), Box<dyn std::error::Error>> { 1128 1128 + let file = assert_fs::NamedTempFile::new("test.txt")?; 1129 1129 + file.write_str("garbage data for testing")?; 1130 1130 + 1131 1131 + let working_dir = assert_fs::TempDir::new()?; 1132 1132 + let archive = working_dir.child("test.txt.zst"); 1133 1133 + archive.assert(predicate::path::missing()); 1134 1134 + 1135 1135 + // Pipe file to stdin 1136 1136 + let mut compress = Command::cargo_bin("cmprss")?; 1137 1137 + compress 1138 1138 + .current_dir(&working_dir) 1139 1139 + .arg("zstd") 1140 1140 + .arg("test.txt.zst") 1141 1141 + .stdin(Stdio::from(File::open(file.path())?)); 1142 1142 + compress.assert().success(); 1143 1143 + archive.assert(predicate::path::is_file()); 1144 1144 + 1145 1145 + let mut extract = Command::cargo_bin("cmprss")?; 1146 1146 + extract 1147 1147 + .current_dir(&working_dir) 1148 1148 + .arg("zstd") 1149 1149 + .stdin(Stdio::from(File::open(archive.path())?)) 1150 1150 + .arg("--extract") 1151 1151 + .arg("out.txt"); 1152 1152 + extract.assert().success(); 1153 1153 + 1154 1154 + // Assert the files are identical 1155 1155 + working_dir 1156 1156 + .child("out.txt") 1157 1157 + .assert(predicate::path::eq_file(file.path())); 1158 1158 + 1159 1159 + Ok(()) 1160 1160 + } 1161 1161 + 1162 1162 + /// Zstd roundtrip using stdout 1163 1163 + /// Compressing: input = test.txt, output = stdout 1164 1164 + /// Extracting: input = test.txt.zst, output = stdout 1165 1165 + /// 1166 1166 + /// ``` bash 1167 1167 + /// cmprss zstd test.txt > test.txt.zst 1168 1168 + /// cmprss zstd --extract test.txt.zst > out.txt 1169 1169 + /// ``` 1170 1170 + #[test] 1171 1171 + fn zstd_roundtrip_stdout() -> Result<(), Box<dyn std::error::Error>> { 1172 1172 + let file = assert_fs::NamedTempFile::new("test.txt")?; 1173 1173 + file.write_str("garbage data for testing")?; 1174 1174 + 1175 1175 + let working_dir = assert_fs::TempDir::new()?; 1176 1176 + let archive = working_dir.child("test.txt.zst"); 1177 1177 + archive.assert(predicate::path::missing()); 1178 1178 + 1179 1179 + // Redirect stdout to file 1180 1180 + let mut compress = Command::cargo_bin("cmprss")?; 1181 1181 + compress 1182 1182 + .current_dir(&working_dir) 1183 1183 + .arg("zstd") 1184 1184 + .arg(file.path()) 1185 1185 + .stdout(Stdio::from(File::create(archive.path())?)); 1186 1186 + compress.assert().success(); 1187 1187 + archive.assert(predicate::path::is_file()); 1188 1188 + 1189 1189 + let output = working_dir.child("out.txt"); 1190 1190 + output.assert(predicate::path::missing()); 1191 1191 + 1192 1192 + let mut extract = Command::cargo_bin("cmprss")?; 1193 1193 + extract 1194 1194 + .current_dir(&working_dir) 1195 1195 + .arg("zstd") 1196 1196 + .arg("--extract") 1197 1197 + .arg(archive.path()) 1198 1198 + .stdout(Stdio::from(File::create(output.path())?)); 1199 1199 + extract.assert().success(); 1200 1200 + output.assert(predicate::path::is_file()); 1201 1201 + 1202 1202 + // Assert the files are identical 1203 1203 + output.assert(predicate::path::eq_file(file.path())); 1204 1204 + 1205 1205 + Ok(()) 1206 1206 + } 1207 1207 + 1208 1208 + /// Zstd roundtrip with compression level 1209 1209 + /// Compressing: input = test.txt, output = test.txt.zst, level = 9 1210 1210 + /// Extracting: input = test.txt.zst, output = test.txt 1211 1211 + /// 1212 1212 + /// ``` bash 1213 1213 + /// cmprss zstd --level 9 test.txt test.txt.zst 1214 1214 + /// cmprss zstd --extract test.txt.zst test.txt 1215 1215 + /// ``` 1216 1216 + #[test] 1217 1217 + fn zstd_roundtrip_with_level() -> Result<(), Box<dyn std::error::Error>> { 1218 1218 + let file = assert_fs::NamedTempFile::new("test.txt")?; 1219 1219 + file.write_str("garbage data for testing")?; 1220 1220 + 1221 1221 + let working_dir = assert_fs::TempDir::new()?; 1222 1222 + let archive = working_dir.child("test.txt.zst"); 1223 1223 + archive.assert(predicate::path::missing()); 1224 1224 + 1225 1225 + let mut compress = Command::cargo_bin("cmprss")?; 1226 1226 + compress 1227 1227 + .current_dir(&working_dir) 1228 1228 + .arg("zstd") 1229 1229 + .arg("--level") 1230 1230 + .arg("9") 1231 1231 + .arg(file.path()) 1232 1232 + .arg(archive.path()); 1233 1233 + compress.assert().success(); 1234 1234 + archive.assert(predicate::path::is_file()); 1235 1235 + 1236 1236 + let output = working_dir.child("test.txt"); 1237 1237 + 1238 1238 + let mut extract = Command::cargo_bin("cmprss")?; 1239 1239 + extract 1240 1240 + .current_dir(&working_dir) 1241 1241 + .arg("zstd") 1242 1242 + .arg("--extract") 1243 1243 + .arg(archive.path()) 1244 1244 + .arg(output.path()); 1245 1245 + extract.assert().success(); 1246 1246 + 1247 1247 + // Assert the files are identical 1248 1248 + output.assert(predicate::path::eq_file(file.path())); 1249 1249 + 1250 1250 + Ok(()) 1251 1251 + } 1074 1252 }