feat(cli): add --append flag for growing tar and zip archives in place

+9

README.md

··· 174 174 cmprss tar -e archive.tar custom_output_directory 175 175 ``` 176 176 177 + Append new entries to an existing `tar` or `zip` archive: 178 + 179 + ```bash 180 + cmprss tar --append new_file.txt archive.tar 181 + cmprss zip -a new_file.txt extra_dir/ archive.zip 182 + ``` 183 + 184 + `--append` only works for container formats that can grow in place. Stream codecs (gzip, xz, …) and compound pipelines like `tar.gz` have no way to append without rewriting the whole archive, so they error instead of silently doing the wrong thing. 185 + 177 186 `cmprss` will detect if `stdin` or `stdout` is a pipe, and use those for I/O where it makes sense. 178 187 179 188 Create and extract a `tar.gz` archive with pipes:

+14 -1

src/backends/pipeline.rs

··· 1 1 use crate::utils::{ 2 2 CmprssInput, CmprssOutput, Compressor, ExtractedTarget, ReadWrapper, Result, WriteWrapper, 3 3 }; 4 - use anyhow::anyhow; 4 + use anyhow::{anyhow, bail}; 5 5 use std::io::{self, Read, Write}; 6 6 use std::path::Path; 7 7 use std::sync::mpsc::{Receiver, Sender, channel}; ··· 320 320 }; 321 321 last.extract(input, final_output) 322 322 }) 323 + } 324 + 325 + fn append(&self, input: CmprssInput, output: CmprssOutput) -> Result { 326 + debug_assert!(!self.compressors.is_empty(), "pipeline is never empty"); 327 + if self.compressors.len() == 1 { 328 + // Single-stage pipelines are just a wrapper; delegate so tar/zip 329 + // reached via positional-path inference still support --append. 330 + return self.compressors[0].append(input, output); 331 + } 332 + bail!( 333 + "cannot --append to a compound archive ({}); it would require decompressing and recompressing the whole archive", 334 + self.format_chain() 335 + ) 323 336 } 324 337 325 338 fn list(&self, input: CmprssInput) -> Result {

+111 -1

src/backends/tar.rs

··· 3 3 use anyhow::bail; 4 4 use clap::Args; 5 5 use indicatif::ProgressBar; 6 - use std::fs::File; 6 + use std::fs::{File, OpenOptions}; 7 7 use std::io::{self, Read, Seek, SeekFrom, Write}; 8 8 use std::path::Path; 9 9 use tar::{Archive, Builder, EntryType, Header}; ··· 157 157 } 158 158 } 159 159 160 + fn append(&self, input: CmprssInput, output: CmprssOutput) -> Result { 161 + let path = match output { 162 + CmprssOutput::Path(p) => p, 163 + _ => bail!("tar append requires the archive path as the output target"), 164 + }; 165 + if !path.is_file() { 166 + bail!("tar append target must be an existing file: {:?}", path); 167 + } 168 + 169 + // Locate the offset just past the last entry's data (512-byte padded) 170 + // so we can truncate off the trailing zero blocks and resume writing 171 + // entries from there. Using the iterator is cheap: tar entries carry 172 + // their own position, so we walk headers without reading file data. 173 + let end_of_entries = { 174 + let reader = File::open(&path)?; 175 + let mut archive = Archive::new(reader); 176 + let mut end: u64 = 0; 177 + for entry in archive.entries()? { 178 + let entry = entry?; 179 + let file_pos = entry.raw_file_position(); 180 + let size = entry.size(); 181 + // Round up to the next 512-byte block boundary. 182 + let padded = size.div_ceil(512) * 512; 183 + end = file_pos + padded; 184 + } 185 + end 186 + }; 187 + 188 + let mut file = OpenOptions::new().read(true).write(true).open(&path)?; 189 + // Truncate any trailing end-of-archive zero blocks so the new entries 190 + // start at `end_of_entries` and Builder::finish writes fresh ones. 191 + file.set_len(end_of_entries)?; 192 + file.seek(SeekFrom::Start(end_of_entries))?; 193 + 194 + let total = match &input { 195 + CmprssInput::Path(paths) => Some(total_input_bytes(paths)), 196 + _ => None, 197 + }; 198 + let bar = create_progress_bar(total, self.progress_args.progress, OutputTarget::File); 199 + self.compress_internal(input, Builder::new(file), bar.as_ref())?; 200 + if let Some(b) = bar { 201 + b.finish(); 202 + } 203 + Ok(()) 204 + } 205 + 160 206 fn list(&self, input: CmprssInput) -> Result { 161 207 let reader: Box<dyn Read> = match input { 162 208 CmprssInput::Path(paths) => { ··· 300 346 fn test_tar_default_compression() -> Result { 301 347 let compressor = Tar::default(); 302 348 test_compression(&compressor) 349 + } 350 + 351 + /// Append new entries into an existing tar and confirm both old and new 352 + /// entries extract correctly. 353 + #[test] 354 + fn test_append_adds_entries() -> Result { 355 + let compressor = Tar::default(); 356 + let working_dir = assert_fs::TempDir::new()?; 357 + 358 + let original = working_dir.child("original.txt"); 359 + original.write_str("original contents")?; 360 + let extra = working_dir.child("extra.txt"); 361 + extra.write_str("appended contents")?; 362 + 363 + let archive = working_dir.child("archive.tar"); 364 + compressor.compress( 365 + CmprssInput::Path(vec![original.path().to_path_buf()]), 366 + CmprssOutput::Path(archive.path().to_path_buf()), 367 + )?; 368 + let size_before = std::fs::metadata(archive.path())?.len(); 369 + 370 + compressor.append( 371 + CmprssInput::Path(vec![extra.path().to_path_buf()]), 372 + CmprssOutput::Path(archive.path().to_path_buf()), 373 + )?; 374 + let size_after = std::fs::metadata(archive.path())?.len(); 375 + assert!( 376 + size_after > size_before, 377 + "archive did not grow after append: {size_before} -> {size_after}", 378 + ); 379 + 380 + let extract_dir = working_dir.child("extracted"); 381 + std::fs::create_dir_all(extract_dir.path())?; 382 + compressor.extract( 383 + CmprssInput::Path(vec![archive.path().to_path_buf()]), 384 + CmprssOutput::Path(extract_dir.path().to_path_buf()), 385 + )?; 386 + 387 + extract_dir 388 + .child("original.txt") 389 + .assert(predicate::path::eq_file(original.path())); 390 + extract_dir 391 + .child("extra.txt") 392 + .assert(predicate::path::eq_file(extra.path())); 393 + Ok(()) 394 + } 395 + 396 + /// Appending to a missing target must error rather than silently creating 397 + /// a new archive. 398 + #[test] 399 + fn test_append_missing_target_errors() { 400 + let compressor = Tar::default(); 401 + let working_dir = assert_fs::TempDir::new().unwrap(); 402 + let extra = working_dir.child("extra.txt"); 403 + extra.write_str("x").unwrap(); 404 + let missing = working_dir.child("nope.tar"); 405 + 406 + let err = compressor 407 + .append( 408 + CmprssInput::Path(vec![extra.path().to_path_buf()]), 409 + CmprssOutput::Path(missing.path().to_path_buf()), 410 + ) 411 + .expect_err("append to a missing archive should error"); 412 + assert!(err.to_string().contains("must be an existing file")); 303 413 } 304 414 305 415 /// Test tar-specific functionality: directory handling

+87 -7

src/backends/zip.rs

··· 7 7 use anyhow::bail; 8 8 use clap::Args; 9 9 use indicatif::ProgressBar; 10 - use std::fs::File; 10 + use std::fs::{File, OpenOptions}; 11 11 use std::io::{self, Seek, SeekFrom, Write}; 12 12 use std::path::Path; 13 13 use tempfile::tempfile; ··· 80 80 bar: Option<&ProgressBar>, 81 81 ) -> Result { 82 82 let mut zip_writer = ZipWriter::new(writer); 83 - let options = self.file_options(); 83 + self.add_entries(&mut zip_writer, input, bar)?; 84 + zip_writer.finish()?; 85 + Ok(()) 86 + } 84 87 88 + /// Add the given input as entries to an existing `ZipWriter`. Shared by 89 + /// `compress_to_file` and the append path, which respectively initialize 90 + /// the writer via `ZipWriter::new` and `ZipWriter::new_append`. 91 + fn add_entries<W: Write + Seek>( 92 + &self, 93 + zip_writer: &mut ZipWriter<W>, 94 + input: CmprssInput, 95 + bar: Option<&ProgressBar>, 96 + ) -> Result { 97 + let options = self.file_options(); 85 98 match input { 86 99 CmprssInput::Path(paths) => { 87 100 for path in paths { ··· 90 103 zip_writer.start_file(name, options)?; 91 104 let f = File::open(&path)?; 92 105 let mut reader = ProgressReader::new(f, bar.cloned()); 93 - io::copy(&mut reader, &mut zip_writer)?; 106 + io::copy(&mut reader, zip_writer)?; 94 107 } else if path.is_dir() { 95 108 // Use the directory as the base and add its contents 96 109 let base = path.parent().unwrap_or(&path); 97 - add_directory(&mut zip_writer, base, &path, options, bar)?; 110 + add_directory(zip_writer, base, &path, options, bar)?; 98 111 } else { 99 112 bail!("zip does not support this file type"); 100 113 } ··· 103 116 CmprssInput::Pipe(mut pipe) => { 104 117 // For pipe input, we'll create a single file named "archive" 105 118 zip_writer.start_file("archive", options)?; 106 - io::copy(&mut pipe, &mut zip_writer)?; 119 + io::copy(&mut pipe, zip_writer)?; 107 120 } 108 121 CmprssInput::Reader(_) => { 109 122 bail!("zip does not accept an in-memory reader input"); 110 123 } 111 124 } 112 - 113 - zip_writer.finish()?; 114 125 Ok(()) 115 126 } 116 127 } ··· 223 234 } 224 235 } 225 236 237 + fn append(&self, input: CmprssInput, output: CmprssOutput) -> Result { 238 + let path = match output { 239 + CmprssOutput::Path(p) => p, 240 + _ => bail!("zip append requires the archive path as the output target"), 241 + }; 242 + if !path.is_file() { 243 + bail!("zip append target must be an existing file: {:?}", path); 244 + } 245 + 246 + let total = match &input { 247 + CmprssInput::Path(paths) => Some(total_input_bytes(paths)), 248 + _ => None, 249 + }; 250 + let bar = create_progress_bar(total, self.progress_args.progress, OutputTarget::File); 251 + 252 + let file = OpenOptions::new().read(true).write(true).open(&path)?; 253 + let mut zip_writer = ZipWriter::new_append(file)?; 254 + self.add_entries(&mut zip_writer, input, bar.as_ref())?; 255 + zip_writer.finish()?; 256 + if let Some(b) = bar { 257 + b.finish(); 258 + } 259 + Ok(()) 260 + } 261 + 226 262 fn list(&self, input: CmprssInput) -> Result { 227 263 // ZipArchive requires a seekable reader. For non-path inputs we must 228 264 // buffer into a tempfile first. ··· 332 368 progress_args: ProgressArgs::default(), 333 369 }; 334 370 test_compression(&best_compressor) 371 + } 372 + 373 + /// Append new entries into an existing zip and confirm both old and new 374 + /// entries extract correctly. 375 + #[test] 376 + fn test_append_adds_entries() -> Result { 377 + let compressor = Zip::default(); 378 + let working_dir = assert_fs::TempDir::new()?; 379 + 380 + let original = working_dir.child("original.txt"); 381 + original.write_str("original contents")?; 382 + let extra = working_dir.child("extra.txt"); 383 + extra.write_str("appended contents")?; 384 + 385 + let archive = working_dir.child("archive.zip"); 386 + compressor.compress( 387 + CmprssInput::Path(vec![original.path().to_path_buf()]), 388 + CmprssOutput::Path(archive.path().to_path_buf()), 389 + )?; 390 + let size_before = std::fs::metadata(archive.path())?.len(); 391 + 392 + compressor.append( 393 + CmprssInput::Path(vec![extra.path().to_path_buf()]), 394 + CmprssOutput::Path(archive.path().to_path_buf()), 395 + )?; 396 + let size_after = std::fs::metadata(archive.path())?.len(); 397 + assert!( 398 + size_after > size_before, 399 + "archive did not grow after append: {size_before} -> {size_after}", 400 + ); 401 + 402 + let extract_dir = working_dir.child("extracted"); 403 + std::fs::create_dir_all(extract_dir.path())?; 404 + compressor.extract( 405 + CmprssInput::Path(vec![archive.path().to_path_buf()]), 406 + CmprssOutput::Path(extract_dir.path().to_path_buf()), 407 + )?; 408 + extract_dir 409 + .child("original.txt") 410 + .assert(predicate::path::eq_file(original.path())); 411 + extract_dir 412 + .child("extra.txt") 413 + .assert(predicate::path::eq_file(extra.path())); 414 + Ok(()) 335 415 } 336 416 337 417 /// Test zip-specific functionality: directory handling

+33 -3

src/job.rs

··· 14 14 use crate::utils::{CmprssInput, CmprssOutput, CommonArgs, Compressor, Result}; 15 15 16 16 /// Extract an action hint from the CLI flags. Returns `None` when the user 17 - /// hasn't specified `--compress`/`--extract`, in which case the action will 18 - /// be inferred from filenames downstream. 17 + /// hasn't specified `--compress`/`--extract`/`--append`, in which case the 18 + /// action will be inferred from filenames downstream. 19 19 fn action_from_flags(args: &CommonArgs) -> Option<Action> { 20 20 if args.compress { 21 21 Some(Action::Compress) 22 22 } else if args.extract { 23 23 Some(Action::Extract) 24 + } else if args.append { 25 + Some(Action::Append) 24 26 } else { 25 27 None 26 28 } ··· 77 79 // as another input. 78 80 output = Some(path); 79 81 io_list.pop(); 82 + } else if !path.is_dir() && action_hint == Some(Action::Append) { 83 + // --append takes the trailing existing file as the archive to 84 + // grow. Same positional convention as compress: the target is 85 + // last, the new inputs are before it. 86 + output = Some(path); 87 + io_list.pop(); 80 88 } 81 - // TODO: check for scenarios where we want to append to an existing archive. 82 89 } 83 90 84 91 for input in &io_list { ··· 123 130 /// container formats; stream codecs fall through to `Compressor::list`'s 124 131 /// default bail. 125 132 List, 133 + /// Append new entries to an existing archive. Only supported by container 134 + /// formats that can grow in place (tar, zip). 135 + Append, 126 136 } 127 137 128 138 /// Parse the common args and determine the details of the job requested. ··· 189 199 Action::Extract => compressor.default_extracted_filename(get_input_filename(&input)?), 190 200 // List short-circuits above; finalize_without_output never returns it. 191 201 Action::List => unreachable!("List is handled before Branch 3"), 202 + // Append without a target archive path has nothing to append to; 203 + // `finalize_without_output` rejects it before reaching here. 204 + Action::Append => unreachable!("Append requires an existing output path"), 192 205 }; 193 206 Ok(Job { 194 207 compressor, ··· 234 247 .ok_or_else(|| anyhow!("Could not determine compressor to use"))?; 235 248 Ok((c, Action::Extract)) 236 249 } 250 + // Append needs an existing archive path to grow; without one there's 251 + // nothing to append to. 252 + Some(Action::Append) => { 253 + bail!("--append requires an existing archive as the output target") 254 + } 237 255 // List is handled by the short-circuit at the top of get_job and 238 256 // never flows into this helper. 239 257 Some(Action::List) => unreachable!("List is handled before Branch 3"), ··· 273 291 } 274 292 // List is handled by the short-circuit at the top of get_job. 275 293 Some(Action::List) => unreachable!("List is handled before fill_missing_from_io"), 294 + Some(Action::Append) => { 295 + // Append needs an existing archive path to grow; infer the 296 + // compressor from that path's extension when it wasn't given. 297 + match output { 298 + CmprssOutput::Path(path) => { 299 + if compressor.is_none() { 300 + *compressor = get_compressor_from_filename(path); 301 + } 302 + } 303 + _ => bail!("--append requires an archive path, not a pipe, as the target"), 304 + } 305 + } 276 306 Some(Action::Extract) => { 277 307 if let CmprssInput::Path(paths) = input { 278 308 if paths.len() != 1 {

+1

src/main.rs

··· 112 112 Action::Compress => job.compressor.compress(job.input, job.output), 113 113 Action::Extract => job.compressor.extract(job.input, job.output), 114 114 Action::List => job.compressor.list(job.input), 115 + Action::Append => job.compressor.append(job.input, job.output), 115 116 } 116 117 } 117 118

+20

src/utils.rs

··· 34 34 #[arg(short, long, visible_alias = "decompress")] 35 35 pub extract: bool, 36 36 37 + /// Append the input(s) to an existing archive. 38 + /// Only supported by container formats that can grow in place (tar, zip); 39 + /// stream codecs and mixed pipelines like `tar.gz` will error. 40 + #[arg(short, long)] 41 + pub append: bool, 42 + 37 43 /// List of I/O. 38 44 /// This consists of all the inputs followed by the single output, with intelligent fallback to stdin/stdout. 39 45 #[arg()] ··· 245 251 fn compress(&self, input: CmprssInput, output: CmprssOutput) -> Result; 246 252 247 253 fn extract(&self, input: CmprssInput, output: CmprssOutput) -> Result; 254 + 255 + /// Append the input to an existing archive pointed at by `output`. 256 + /// 257 + /// The default implementation bails: only container formats that can grow 258 + /// in place — currently tar and zip — support appending. Stream codecs 259 + /// (gzip, xz, …) have no notion of entries, and compound pipelines like 260 + /// `tar.gz` would require decompress-then-recompress which defeats the 261 + /// point of an in-place append. 262 + fn append(&self, _input: CmprssInput, _output: CmprssOutput) -> Result { 263 + anyhow::bail!( 264 + "{} archives do not support --append; only container formats (tar, zip) can be appended to in place", 265 + self.name() 266 + ) 267 + } 248 268 249 269 /// List the contents of the archive to stdout. 250 270 ///

+157

tests/append.rs

··· 1 + use assert_cmd::prelude::*; 2 + use assert_fs::prelude::*; 3 + use predicates::prelude::*; 4 + use std::process::Command; 5 + 6 + mod common; 7 + use common::*; 8 + 9 + /// `cmprss tar --append` grows an existing tar archive with new entries and 10 + /// the resulting archive extracts both original and appended files. 11 + #[test] 12 + fn tar_append_adds_entry() -> Result<(), Box<dyn std::error::Error>> { 13 + let original = create_test_file("original.txt", "original contents")?; 14 + let extra = create_test_file("extra.txt", "appended contents")?; 15 + let working_dir = create_working_dir()?; 16 + let archive = working_dir.child("archive.tar"); 17 + 18 + Command::cargo_bin("cmprss")? 19 + .arg("tar") 20 + .arg(original.path()) 21 + .arg(archive.path()) 22 + .assert() 23 + .success(); 24 + archive.assert(predicate::path::is_file()); 25 + 26 + Command::cargo_bin("cmprss")? 27 + .arg("tar") 28 + .arg("--append") 29 + .arg(extra.path()) 30 + .arg(archive.path()) 31 + .assert() 32 + .success(); 33 + 34 + Command::cargo_bin("cmprss")? 35 + .arg("tar") 36 + .arg("--extract") 37 + .arg(archive.path()) 38 + .arg(working_dir.path()) 39 + .assert() 40 + .success(); 41 + 42 + assert_files_equal(original.path(), &working_dir.child("original.txt")); 43 + assert_files_equal(extra.path(), &working_dir.child("extra.txt")); 44 + Ok(()) 45 + } 46 + 47 + /// `cmprss zip --append` grows an existing zip archive with new entries. 48 + #[test] 49 + fn zip_append_adds_entry() -> Result<(), Box<dyn std::error::Error>> { 50 + let original = create_test_file("original.txt", "original contents")?; 51 + let extra = create_test_file("extra.txt", "appended contents")?; 52 + let working_dir = create_working_dir()?; 53 + let archive = working_dir.child("archive.zip"); 54 + 55 + Command::cargo_bin("cmprss")? 56 + .arg("zip") 57 + .arg(original.path()) 58 + .arg(archive.path()) 59 + .assert() 60 + .success(); 61 + archive.assert(predicate::path::is_file()); 62 + 63 + Command::cargo_bin("cmprss")? 64 + .arg("zip") 65 + .arg("--append") 66 + .arg(extra.path()) 67 + .arg(archive.path()) 68 + .assert() 69 + .success(); 70 + 71 + Command::cargo_bin("cmprss")? 72 + .arg("zip") 73 + .arg("--extract") 74 + .arg(archive.path()) 75 + .arg(working_dir.path()) 76 + .assert() 77 + .success(); 78 + 79 + assert_files_equal(original.path(), &working_dir.child("original.txt")); 80 + assert_files_equal(extra.path(), &working_dir.child("extra.txt")); 81 + Ok(()) 82 + } 83 + 84 + /// Appending to a stream codec (e.g. gzip) must fail cleanly — there's no 85 + /// notion of "entries" in a stream format. 86 + #[test] 87 + fn gzip_append_fails() -> Result<(), Box<dyn std::error::Error>> { 88 + let working_dir = create_working_dir()?; 89 + let archive = working_dir.child("data.gz"); 90 + let input = create_test_file("data.txt", "hello")?; 91 + 92 + // Create a valid .gz to append to. 93 + Command::cargo_bin("cmprss")? 94 + .arg("gzip") 95 + .arg(input.path()) 96 + .arg(archive.path()) 97 + .assert() 98 + .success(); 99 + 100 + let extra = create_test_file("extra.txt", "more")?; 101 + Command::cargo_bin("cmprss")? 102 + .arg("gzip") 103 + .arg("--append") 104 + .arg(extra.path()) 105 + .arg(archive.path()) 106 + .assert() 107 + .failure() 108 + .stderr(predicate::str::contains("do not support --append")); 109 + Ok(()) 110 + } 111 + 112 + /// Appending to a compound pipeline like `tar.gz` must fail — it would 113 + /// require decompressing and recompressing the whole archive. 114 + #[test] 115 + fn tar_gz_append_fails() -> Result<(), Box<dyn std::error::Error>> { 116 + let working_dir = create_working_dir()?; 117 + let archive = working_dir.child("archive.tar.gz"); 118 + let input = create_test_file("data.txt", "hello")?; 119 + 120 + Command::cargo_bin("cmprss")? 121 + .arg("tar.gz") 122 + .arg(input.path()) 123 + .arg(archive.path()) 124 + .assert() 125 + .success(); 126 + 127 + let extra = create_test_file("extra.txt", "more")?; 128 + Command::cargo_bin("cmprss")? 129 + .arg("tar.gz") 130 + .arg("--append") 131 + .arg(extra.path()) 132 + .arg(archive.path()) 133 + .assert() 134 + .failure() 135 + .stderr(predicate::str::contains("compound archive")); 136 + Ok(()) 137 + } 138 + 139 + /// `--append` with a non-existent target must error rather than create a new 140 + /// archive. 141 + #[test] 142 + fn tar_append_missing_target_errors() -> Result<(), Box<dyn std::error::Error>> { 143 + let working_dir = create_working_dir()?; 144 + let missing = working_dir.child("missing.tar"); 145 + let extra = create_test_file("extra.txt", "x")?; 146 + 147 + // Non-existent trailing path is normally treated as the output to create; 148 + // --append should reject it instead. 149 + Command::cargo_bin("cmprss")? 150 + .arg("tar") 151 + .arg("--append") 152 + .arg(extra.path()) 153 + .arg(missing.path()) 154 + .assert() 155 + .failure(); 156 + Ok(()) 157 + }

Configure Feed

Configure Feed