Expose deduplication ratio to client · awoo.ren/attic@5b42839

+9

attic/src/api/v1/upload_path.rs

··· 61 61 /// The compressed size of the NAR, in bytes. 62 62 #[serde(skip_serializing_if = "Option::is_none")] 63 63 pub file_size: Option<usize>, 64 + 65 + /// The fraction of data that was deduplicated, from 0 to 1. 66 + pub frac_deduplicated: Option<f64>, 64 67 } 65 68 66 69 #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] 67 70 #[non_exhaustive] 68 71 pub enum UploadPathResultKind { 69 72 /// The path was uploaded. 73 + /// 74 + /// This is purely informational and servers may return 75 + /// this variant even when the NAR is deduplicated. 70 76 Uploaded, 71 77 72 78 /// The path was globally deduplicated. 79 + /// 80 + /// The exact semantics of what counts as deduplicated 81 + /// is opaque to the client. 73 82 Deduplicated, 74 83 } 75 84

+31 -29

client/src/command/push.rs

··· 17 17 use crate::cache::{CacheName, CacheRef}; 18 18 use crate::cli::Opts; 19 19 use crate::config::Config; 20 - use attic::api::v1::upload_path::{UploadPathNarInfo, UploadPathResultKind}; 20 + use attic::api::v1::upload_path::{UploadPathNarInfo, UploadPathResult, UploadPathResultKind}; 21 21 use attic::error::AtticResult; 22 22 use attic::nix_store::{NixStore, StorePath, StorePathHash, ValidPathInfo}; 23 23 ··· 132 132 let start = Instant::now(); 133 133 match api.upload_path(upload_info, nar_stream).await { 134 134 Ok(r) => { 135 - if r.is_none() { 136 - mp.suspend(|| { 137 - eprintln!("Warning: Please update your server. Compatibility will be removed in the first stable release."); 138 - }) 139 - } 135 + let r = r.unwrap_or(UploadPathResult { 136 + kind: UploadPathResultKind::Uploaded, 137 + file_size: None, 138 + frac_deduplicated: None, 139 + }); 140 140 141 - let deduplicated = if let Some(r) = r { 142 - r.kind == UploadPathResultKind::Deduplicated 143 - } else { 144 - false 145 - }; 141 + let info_string: String = match r.kind { 142 + UploadPathResultKind::Deduplicated => "deduplicated".to_string(), 143 + _ => { 144 + let elapsed = start.elapsed(); 145 + let seconds = elapsed.as_secs_f64(); 146 + let speed = (path_info.nar_size as f64 / seconds) as u64; 146 147 147 - if deduplicated { 148 - mp.suspend(|| { 149 - eprintln!("✅ {} (deduplicated)", path.as_os_str().to_string_lossy()); 150 - }); 151 - bar.finish_and_clear(); 152 - } else { 153 - let elapsed = start.elapsed(); 154 - let seconds = elapsed.as_secs_f64(); 155 - let speed = (path_info.nar_size as f64 / seconds) as u64; 148 + let mut s = format!("{}/s", HumanBytes(speed)); 149 + 150 + if let Some(frac_deduplicated) = r.frac_deduplicated { 151 + if frac_deduplicated > 0.01f64 { 152 + s += &format!(", {:.1}% deduplicated", frac_deduplicated * 100.0); 153 + } 154 + } 155 + 156 + s 157 + } 158 + }; 156 159 157 - mp.suspend(|| { 158 - eprintln!( 159 - "✅ {} ({}/s)", 160 - path.as_os_str().to_string_lossy(), 161 - HumanBytes(speed) 162 - ); 163 - }); 164 - bar.finish_and_clear(); 165 - } 160 + mp.suspend(|| { 161 + eprintln!( 162 + "✅ {} ({})", 163 + path.as_os_str().to_string_lossy(), 164 + info_string 165 + ); 166 + }); 167 + bar.finish_and_clear(); 166 168 167 169 Ok(()) 168 170 }

+40 -13

server/src/api/v1/upload_path.rs

··· 62 62 Stream(Box<dyn AsyncRead + Send + Unpin + 'static>, Hash, usize), 63 63 } 64 64 65 + /// Result of a chunk upload. 66 + struct UploadChunkResult { 67 + guard: ChunkGuard, 68 + deduplicated: bool, 69 + } 70 + 65 71 /// Applies compression to a stream, computing hashes along the way. 66 72 /// 67 73 /// Our strategy is to stream directly onto a UUID-keyed file on the ··· 247 253 Ok(Json(UploadPathResult { 248 254 kind: UploadPathResultKind::Deduplicated, 249 255 file_size: None, // TODO: Sum the chunks 256 + frac_deduplicated: None, 250 257 })) 251 258 } 252 259 ··· 341 348 } 342 349 343 350 // Wait for all uploads to complete 344 - let chunks: Vec<ChunkGuard> = join_all(futures) 351 + let chunks: Vec<UploadChunkResult> = join_all(futures) 345 352 .await 346 353 .into_iter() 347 354 .map(|join_result| join_result.unwrap()) 348 355 .collect::<ServerResult<Vec<_>>>()?; 349 356 350 - let file_size = chunks 351 - .iter() 352 - .fold(0, |acc, c| acc + c.file_size.unwrap() as usize); 357 + let (file_size, deduplicated_size) = 358 + chunks 359 + .iter() 360 + .fold((0, 0), |(file_size, deduplicated_size), c| { 361 + ( 362 + file_size + c.guard.file_size.unwrap() as usize, 363 + if c.deduplicated { 364 + deduplicated_size + c.guard.chunk_size as usize 365 + } else { 366 + deduplicated_size 367 + }, 368 + ) 369 + }); 353 370 354 371 // Finally... 355 372 let txn = database ··· 385 402 ChunkRef::insert(chunkref::ActiveModel { 386 403 nar_id: Set(nar_id), 387 404 seq: Set(i as i32), 388 - chunk_id: Set(Some(chunk.id)), 389 - chunk_hash: Set(chunk.chunk_hash.clone()), 390 - compression: Set(chunk.compression.clone()), 405 + chunk_id: Set(Some(chunk.guard.id)), 406 + chunk_hash: Set(chunk.guard.chunk_hash.clone()), 407 + compression: Set(chunk.guard.compression.clone()), 391 408 ..Default::default() 392 409 }) 393 410 .exec(&txn) ··· 419 436 Ok(Json(UploadPathResult { 420 437 kind: UploadPathResultKind::Uploaded, 421 438 file_size: Some(file_size), 439 + 440 + // Currently, frac_deduplicated is computed from size before compression 441 + frac_deduplicated: Some(deduplicated_size as f64 / *nar_size as f64), 422 442 })) 423 443 } 424 444 ··· 452 472 state.config.require_proof_of_possession, 453 473 ) 454 474 .await?; 455 - let file_size = chunk.file_size.unwrap() as usize; 475 + let file_size = chunk.guard.file_size.unwrap() as usize; 456 476 457 477 // Finally... 458 478 let txn = database ··· 467 487 compression: Set(compression.to_string()), 468 488 469 489 nar_hash: Set(upload_info.nar_hash.to_typed_base16()), 470 - nar_size: Set(chunk.chunk_size), 490 + nar_size: Set(chunk.guard.chunk_size), 471 491 472 492 num_chunks: Set(1), 473 493 ··· 487 507 ChunkRef::insert(chunkref::ActiveModel { 488 508 nar_id: Set(nar_id), 489 509 seq: Set(0), 490 - chunk_id: Set(Some(chunk.id)), 510 + chunk_id: Set(Some(chunk.guard.id)), 491 511 chunk_hash: Set(upload_info.nar_hash.to_typed_base16()), 492 512 compression: Set(compression.to_string()), 493 513 ..Default::default() ··· 520 540 Ok(Json(UploadPathResult { 521 541 kind: UploadPathResultKind::Uploaded, 522 542 file_size: Some(file_size), 543 + frac_deduplicated: None, 523 544 })) 524 545 } 525 546 ··· 533 554 database: DatabaseConnection, 534 555 state: State, 535 556 require_proof_of_possession: bool, 536 - ) -> ServerResult<ChunkGuard> { 557 + ) -> ServerResult<UploadChunkResult> { 537 558 let compression: Compression = compression_type.into(); 538 559 539 560 let given_chunk_hash = data.hash(); ··· 565 586 } 566 587 } 567 588 568 - return Ok(existing_chunk); 589 + return Ok(UploadChunkResult { 590 + guard: existing_chunk, 591 + deduplicated: true, 592 + }); 569 593 } 570 594 571 595 let key = format!("{}.chunk", Uuid::new_v4()); ··· 680 704 681 705 let guard = ChunkGuard::from_locked(database.clone(), chunk); 682 706 683 - Ok(guard) 707 + Ok(UploadChunkResult { 708 + guard, 709 + deduplicated: false, 710 + }) 684 711 } 685 712 686 713 /// Returns a compressor function that takes some stream as input.

Configure Feed

Configure Feed